lnet/selftest/console.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  */
  30 /*
  31  * This file is part of Lustre, http://www.lustre.org/
  32  * Lustre is a trademark of Sun Microsystems, Inc.
  33  *
  34  * lnet/selftest/conctl.c
  35  *
  36  * Infrastructure of LST console
  37  *
  38  * Author: Liang Zhen <liangzhen@clusterfs.com>
  39  */
  40
  41 #ifdef __KERNEL__
  42
  43 #include <libcfs/libcfs.h>
  44 #include <lnet/lib-lnet.h>
  45 #include "console.h"
  46 #include "conrpc.h"
  47
  48 #define LST_NODE_STATE_COUNTER(nd, p)                   \
  49 do {                                                    \
  50         if ((nd)->nd_state == LST_NODE_ACTIVE)          \
  51                 (p)->nle_nactive ++;                    \
  52         else if ((nd)->nd_state == LST_NODE_BUSY)       \
  53                 (p)->nle_nbusy ++;                      \
  54         else if ((nd)->nd_state == LST_NODE_DOWN)       \
  55                 (p)->nle_ndown ++;                      \
  56         else                                            \
  57                 (p)->nle_nunknown ++;                   \
  58         (p)->nle_nnode ++;                              \
  59 } while (0)
  60
  61 lstcon_session_t        console_session;
  62
  63 void
  64 lstcon_node_get(lstcon_node_t *nd)
  65 {
  66         LASSERT (nd->nd_ref >= 1);
  67
  68         nd->nd_ref++;
  69 }
  70
  71 static int
  72 lstcon_node_find(lnet_process_id_t id, lstcon_node_t **ndpp, int create)
  73 {
  74         lstcon_ndlink_t *ndl;
  75         unsigned int     idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE;
  76
  77         LASSERT (id.nid != LNET_NID_ANY);
  78
  79         cfs_list_for_each_entry_typed(ndl, &console_session.ses_ndl_hash[idx],
  80                                       lstcon_ndlink_t, ndl_hlink) {
  81                 if (ndl->ndl_node->nd_id.nid != id.nid ||
  82                     ndl->ndl_node->nd_id.pid != id.pid)
  83                         continue;
  84
  85                 lstcon_node_get(ndl->ndl_node);
  86                 *ndpp = ndl->ndl_node;
  87                 return 0;
  88         }
  89
  90         if (!create)
  91                 return -ENOENT;
  92
  93         LIBCFS_ALLOC(*ndpp, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
  94         if (*ndpp == NULL)
  95                 return -ENOMEM;
  96
  97         ndl = (lstcon_ndlink_t *)(*ndpp + 1);
  98
  99         ndl->ndl_node = *ndpp;
 100
 101         ndl->ndl_node->nd_ref   = 1;
 102         ndl->ndl_node->nd_id    = id;
 103         ndl->ndl_node->nd_stamp = cfs_time_current();
 104         ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
 105         ndl->ndl_node->nd_timeout = 0;
 106         memset(&ndl->ndl_node->nd_ping, 0, sizeof(lstcon_rpc_t));
 107
 108         /* queued in global hash & list, no refcount is taken by
 109          * global hash & list, if caller release his refcount,
 110          * node will be released */
 111         cfs_list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]);
 112         cfs_list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list);
 113
 114         return 0;
 115 }
 116
 117 void
 118 lstcon_node_put(lstcon_node_t *nd)
 119 {
 120         lstcon_ndlink_t  *ndl;
 121
 122         LASSERT (nd->nd_ref > 0);
 123
 124         if (--nd->nd_ref > 0)
 125                 return;
 126
 127         ndl = (lstcon_ndlink_t *)(nd + 1);
 128
 129         LASSERT (!cfs_list_empty(&ndl->ndl_link));
 130         LASSERT (!cfs_list_empty(&ndl->ndl_hlink));
 131
 132         /* remove from session */
 133         cfs_list_del(&ndl->ndl_link);
 134         cfs_list_del(&ndl->ndl_hlink);
 135
 136         LIBCFS_FREE(nd, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
 137 }
 138
 139 static int
 140 lstcon_ndlink_find(cfs_list_t *hash,
 141                    lnet_process_id_t id, lstcon_ndlink_t **ndlpp, int create)
 142 {
 143         unsigned int     idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
 144         lstcon_ndlink_t *ndl;
 145         lstcon_node_t   *nd;
 146         int              rc;
 147
 148         if (id.nid == LNET_NID_ANY)
 149                 return -EINVAL;
 150
 151         /* search in hash */
 152         cfs_list_for_each_entry_typed(ndl, &hash[idx],
 153                                       lstcon_ndlink_t, ndl_hlink) {
 154                 if (ndl->ndl_node->nd_id.nid != id.nid ||
 155                     ndl->ndl_node->nd_id.pid != id.pid)
 156                         continue;
 157
 158                 *ndlpp = ndl;
 159                 return 0;
 160         }
 161
 162         if (create == 0)
 163                 return -ENOENT;
 164
 165         /* find or create in session hash */
 166         rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0);
 167         if (rc != 0)
 168                 return rc;
 169
 170         LIBCFS_ALLOC(ndl, sizeof(lstcon_ndlink_t));
 171         if (ndl == NULL) {
 172                 lstcon_node_put(nd);
 173                 return -ENOMEM;
 174         }
 175
 176         *ndlpp = ndl;
 177
 178         ndl->ndl_node = nd;
 179         CFS_INIT_LIST_HEAD(&ndl->ndl_link);
 180         cfs_list_add_tail(&ndl->ndl_hlink, &hash[idx]);
 181
 182         return  0;
 183 }
 184
 185 static void
 186 lstcon_ndlink_release(lstcon_ndlink_t *ndl)
 187 {
 188         LASSERT (cfs_list_empty(&ndl->ndl_link));
 189         LASSERT (!cfs_list_empty(&ndl->ndl_hlink));
 190
 191         cfs_list_del(&ndl->ndl_hlink); /* delete from hash */
 192         lstcon_node_put(ndl->ndl_node);
 193
 194         LIBCFS_FREE(ndl, sizeof(*ndl));
 195 }
 196
 197 static int
 198 lstcon_group_alloc(char *name, lstcon_group_t **grpp)
 199 {
 200         lstcon_group_t *grp;
 201         int             i;
 202
 203         LIBCFS_ALLOC(grp, offsetof(lstcon_group_t,
 204                                    grp_ndl_hash[LST_NODE_HASHSIZE]));
 205         if (grp == NULL)
 206                 return -ENOMEM;
 207
 208         memset(grp, 0, offsetof(lstcon_group_t,
 209                                 grp_ndl_hash[LST_NODE_HASHSIZE]));
 210
 211         grp->grp_ref = 1;
 212         if (name != NULL)
 213                 strcpy(grp->grp_name, name);
 214
 215         CFS_INIT_LIST_HEAD(&grp->grp_link);
 216         CFS_INIT_LIST_HEAD(&grp->grp_ndl_list);
 217         CFS_INIT_LIST_HEAD(&grp->grp_trans_list);
 218
 219         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 220                 CFS_INIT_LIST_HEAD(&grp->grp_ndl_hash[i]);
 221
 222         *grpp = grp;
 223
 224         return 0;
 225 }
 226
 227 static void
 228 lstcon_group_addref(lstcon_group_t *grp)
 229 {
 230         grp->grp_ref ++;
 231 }
 232
 233 static void lstcon_group_ndlink_release(lstcon_group_t *, lstcon_ndlink_t *);
 234
 235 static void
 236 lstcon_group_drain(lstcon_group_t *grp, int keep)
 237 {
 238         lstcon_ndlink_t *ndl;
 239         lstcon_ndlink_t *tmp;
 240
 241         cfs_list_for_each_entry_safe_typed(ndl, tmp, &grp->grp_ndl_list,
 242                                            lstcon_ndlink_t, ndl_link) {
 243                 if ((ndl->ndl_node->nd_state & keep) == 0)
 244                         lstcon_group_ndlink_release(grp, ndl);
 245         }
 246 }
 247
 248 static void
 249 lstcon_group_decref(lstcon_group_t *grp)
 250 {
 251         int     i;
 252
 253         if (--grp->grp_ref > 0)
 254                 return;
 255
 256         if (!cfs_list_empty(&grp->grp_link))
 257                 cfs_list_del(&grp->grp_link);
 258
 259         lstcon_group_drain(grp, 0);
 260
 261         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 262                 LASSERT (cfs_list_empty(&grp->grp_ndl_hash[i]));
 263         }
 264
 265         LIBCFS_FREE(grp, offsetof(lstcon_group_t,
 266                                   grp_ndl_hash[LST_NODE_HASHSIZE]));
 267 }
 268
 269 static int
 270 lstcon_group_find(char *name, lstcon_group_t **grpp)
 271 {
 272         lstcon_group_t   *grp;
 273
 274         cfs_list_for_each_entry_typed(grp, &console_session.ses_grp_list,
 275                                       lstcon_group_t, grp_link) {
 276                 if (strncmp(grp->grp_name, name, LST_NAME_SIZE) != 0)
 277                         continue;
 278
 279                 lstcon_group_addref(grp);  /* +1 ref for caller */
 280                 *grpp = grp;
 281                 return 0;
 282         }
 283
 284         return -ENOENT;
 285 }
 286
 287 static void
 288 lstcon_group_put(lstcon_group_t *grp)
 289 {
 290         lstcon_group_decref(grp);
 291 }
 292
 293 static int
 294 lstcon_group_ndlink_find(lstcon_group_t *grp, lnet_process_id_t id,
 295                          lstcon_ndlink_t **ndlpp, int create)
 296 {
 297         int     rc;
 298
 299         rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create);
 300         if (rc != 0)
 301                 return rc;
 302
 303         if (!cfs_list_empty(&(*ndlpp)->ndl_link))
 304                 return 0;
 305
 306         cfs_list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list);
 307         grp->grp_nnode ++;
 308
 309         return 0;
 310 }
 311
 312 static void
 313 lstcon_group_ndlink_release(lstcon_group_t *grp, lstcon_ndlink_t *ndl)
 314 {
 315         cfs_list_del_init(&ndl->ndl_link);
 316         lstcon_ndlink_release(ndl);
 317         grp->grp_nnode --;
 318 }
 319
 320 static void
 321 lstcon_group_ndlink_move(lstcon_group_t *old,
 322                          lstcon_group_t *new, lstcon_ndlink_t *ndl)
 323 {
 324         unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) %
 325                            LST_NODE_HASHSIZE;
 326
 327         cfs_list_del(&ndl->ndl_hlink);
 328         cfs_list_del(&ndl->ndl_link);
 329         old->grp_nnode --;
 330
 331         cfs_list_add_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]);
 332         cfs_list_add_tail(&ndl->ndl_link, &new->grp_ndl_list);
 333         new->grp_nnode ++;
 334
 335         return;
 336 }
 337
 338 static void
 339 lstcon_group_move(lstcon_group_t *old, lstcon_group_t *new)
 340 {
 341         lstcon_ndlink_t *ndl;
 342
 343         while (!cfs_list_empty(&old->grp_ndl_list)) {
 344                 ndl = cfs_list_entry(old->grp_ndl_list.next,
 345                                      lstcon_ndlink_t, ndl_link);
 346                 lstcon_group_ndlink_move(old, new, ndl);
 347         }
 348 }
 349
 350 int
 351 lstcon_sesrpc_condition(int transop, lstcon_node_t *nd, void *arg)
 352 {
 353         lstcon_group_t *grp = (lstcon_group_t *)arg;
 354
 355         switch (transop) {
 356         case LST_TRANS_SESNEW:
 357                 if (nd->nd_state == LST_NODE_ACTIVE)
 358                         return 0;
 359                 break;
 360
 361         case LST_TRANS_SESEND:
 362                 if (nd->nd_state != LST_NODE_ACTIVE)
 363                         return 0;
 364
 365                 if (grp != NULL && nd->nd_ref > 1)
 366                         return 0;
 367                 break;
 368
 369         case LST_TRANS_SESQRY:
 370                 break;
 371
 372         default:
 373                 LBUG();
 374         }
 375
 376         return 1;
 377 }
 378
 379 int
 380 lstcon_sesrpc_readent(int transop, srpc_msg_t *msg,
 381                       lstcon_rpc_ent_t *ent_up)
 382 {
 383         srpc_debug_reply_t *rep;
 384
 385         switch (transop) {
 386         case LST_TRANS_SESNEW:
 387         case LST_TRANS_SESEND:
 388                 return 0;
 389
 390         case LST_TRANS_SESQRY:
 391                 rep = &msg->msg_body.dbg_reply;
 392
 393                 if (cfs_copy_to_user(&ent_up->rpe_priv[0],
 394                                      &rep->dbg_timeout, sizeof(int)) ||
 395                     cfs_copy_to_user(&ent_up->rpe_payload[0],
 396                                      &rep->dbg_name, LST_NAME_SIZE))
 397                         return -EFAULT;
 398
 399                 return 0;
 400
 401         default:
 402                 LBUG();
 403         }
 404
 405         return 0;
 406 }
 407
 408 static int
 409 lstcon_group_nodes_add(lstcon_group_t *grp, int count,
 410                        lnet_process_id_t *ids_up,
 411                        cfs_list_t *result_up)
 412 {
 413         lstcon_rpc_trans_t      *trans;
 414         lstcon_ndlink_t         *ndl;
 415         lstcon_group_t          *tmp;
 416         lnet_process_id_t        id;
 417         int                      i;
 418         int                      rc;
 419
 420         rc = lstcon_group_alloc(NULL, &tmp);
 421         if (rc != 0) {
 422                 CERROR("Out of memory\n");
 423                 return -ENOMEM;
 424         }
 425
 426         for (i = 0 ; i < count; i++) {
 427                 if (cfs_copy_from_user(&id, &ids_up[i], sizeof(id))) {
 428                         rc = -EFAULT;
 429                         break;
 430                 }
 431
 432                 /* skip if it's in this group already */
 433                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 0);
 434                 if (rc == 0)
 435                         continue;
 436
 437                 /* add to tmp group */
 438                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1);
 439                 if (rc != 0) {
 440                         CERROR("Can't create ndlink, out of memory\n");
 441                         break;
 442                 }
 443         }
 444
 445         if (rc != 0) {
 446                 lstcon_group_put(tmp);
 447                 return rc;
 448         }
 449
 450         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 451                                      &tmp->grp_trans_list, LST_TRANS_SESNEW,
 452                                      tmp, lstcon_sesrpc_condition, &trans);
 453         if (rc != 0) {
 454                 CERROR("Can't create transaction: %d\n", rc);
 455                 lstcon_group_put(tmp);
 456                 return rc;
 457         }
 458
 459         /* post all RPCs */
 460         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 461
 462         rc = lstcon_rpc_trans_interpreter(trans, result_up,
 463                                           lstcon_sesrpc_readent);
 464         /* destroy all RPGs */
 465         lstcon_rpc_trans_destroy(trans);
 466
 467         lstcon_group_move(tmp, grp);
 468         lstcon_group_put(tmp);
 469
 470         return rc;
 471 }
 472
 473 static int
 474 lstcon_group_nodes_remove(lstcon_group_t *grp,
 475                           int count, lnet_process_id_t *ids_up,
 476                           cfs_list_t *result_up)
 477 {
 478         lstcon_rpc_trans_t     *trans;
 479         lstcon_ndlink_t        *ndl;
 480         lstcon_group_t         *tmp;
 481         lnet_process_id_t       id;
 482         int                     rc;
 483         int                     i;
 484
 485         /* End session and remove node from the group */
 486
 487         rc = lstcon_group_alloc(NULL, &tmp);
 488         if (rc != 0) {
 489                 CERROR("Out of memory\n");
 490                 return -ENOMEM;
 491         }
 492
 493         for (i = 0; i < count; i++) {
 494                 if (cfs_copy_from_user(&id, &ids_up[i], sizeof(id))) {
 495                         rc = -EFAULT;
 496                         goto error;
 497                 }
 498
 499                 /* move node to tmp group */
 500                 if (lstcon_group_ndlink_find(grp, id, &ndl, 0) == 0)
 501                         lstcon_group_ndlink_move(grp, tmp, ndl);
 502         }
 503
 504         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 505                                      &tmp->grp_trans_list, LST_TRANS_SESEND,
 506                                      tmp, lstcon_sesrpc_condition, &trans);
 507         if (rc != 0) {
 508                 CERROR("Can't create transaction: %d\n", rc);
 509                 goto error;
 510         }
 511
 512         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 513
 514         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 515
 516         lstcon_rpc_trans_destroy(trans);
 517         /* release nodes anyway, because we can't rollback status */
 518         lstcon_group_put(tmp);
 519
 520         return rc;
 521 error:
 522         lstcon_group_move(tmp, grp);
 523         lstcon_group_put(tmp);
 524
 525         return rc;
 526 }
 527
 528 int
 529 lstcon_group_add(char *name)
 530 {
 531         lstcon_group_t *grp;
 532         int             rc;
 533
 534         rc = (lstcon_group_find(name, &grp) == 0)? -EEXIST: 0;
 535         if (rc != 0) {
 536                 /* find a group with same name */
 537                 lstcon_group_put(grp);
 538                 return rc;
 539         }
 540
 541         rc = lstcon_group_alloc(name, &grp);
 542         if (rc != 0) {
 543                 CERROR("Can't allocate descriptor for group %s\n", name);
 544                 return -ENOMEM;
 545         }
 546
 547         cfs_list_add_tail(&grp->grp_link, &console_session.ses_grp_list);
 548
 549         return rc;
 550 }
 551
 552 int
 553 lstcon_nodes_add(char *name, int count,
 554                  lnet_process_id_t *ids_up, cfs_list_t *result_up)
 555 {
 556         lstcon_group_t         *grp;
 557         int                     rc;
 558
 559         LASSERT (count > 0);
 560         LASSERT (ids_up != NULL);
 561
 562         rc = lstcon_group_find(name, &grp);
 563         if (rc != 0) {
 564                 CDEBUG(D_NET, "Can't find group %s\n", name);
 565                 return rc;
 566         }
 567
 568         if (grp->grp_ref > 2) {
 569                 /* referred by other threads or test */
 570                 CDEBUG(D_NET, "Group %s is busy\n", name);
 571                 lstcon_group_put(grp);
 572
 573                 return -EBUSY;
 574         }
 575
 576         rc = lstcon_group_nodes_add(grp, count, ids_up, result_up);
 577
 578         lstcon_group_put(grp);
 579
 580         return rc;
 581 }
 582
 583 int
 584 lstcon_group_del(char *name)
 585 {
 586         lstcon_rpc_trans_t *trans;
 587         lstcon_group_t     *grp;
 588         int                 rc;
 589
 590         rc = lstcon_group_find(name, &grp);
 591         if (rc != 0) {
 592                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 593                 return rc;
 594         }
 595
 596         if (grp->grp_ref > 2) {
 597                 /* referred by others threads or test */
 598                 CDEBUG(D_NET, "Group %s is busy\n", name);
 599                 lstcon_group_put(grp);
 600                 return -EBUSY;
 601         }
 602
 603         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 604                                      &grp->grp_trans_list, LST_TRANS_SESEND,
 605                                      grp, lstcon_sesrpc_condition, &trans);
 606         if (rc != 0) {
 607                 CERROR("Can't create transaction: %d\n", rc);
 608                 lstcon_group_put(grp);
 609                 return rc;
 610         }
 611
 612         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 613
 614         lstcon_rpc_trans_destroy(trans);
 615
 616         lstcon_group_put(grp);
 617         /* -ref for session, it's destroyed,
 618          * status can't be rolled back, destroy group anway */
 619         lstcon_group_put(grp);
 620
 621         return rc;
 622 }
 623
 624 int
 625 lstcon_group_clean(char *name, int args)
 626 {
 627         lstcon_group_t *grp = NULL;
 628         int             rc;
 629
 630         rc = lstcon_group_find(name, &grp);
 631         if (rc != 0) {
 632                 CDEBUG(D_NET, "Can't find group %s\n", name);
 633                 return rc;
 634         }
 635
 636         if (grp->grp_ref > 2) {
 637                 /* referred by test */
 638                 CDEBUG(D_NET, "Group %s is busy\n", name);
 639                 lstcon_group_put(grp);
 640                 return -EBUSY;
 641         }
 642
 643         args = (LST_NODE_ACTIVE | LST_NODE_BUSY |
 644                 LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args;
 645
 646         lstcon_group_drain(grp, args);
 647
 648         lstcon_group_put(grp);
 649         /* release empty group */
 650         if (cfs_list_empty(&grp->grp_ndl_list))
 651                 lstcon_group_put(grp);
 652
 653         return 0;
 654 }
 655
 656 int
 657 lstcon_nodes_remove(char *name, int count,
 658                     lnet_process_id_t *ids_up, cfs_list_t *result_up)
 659 {
 660         lstcon_group_t *grp = NULL;
 661         int             rc;
 662
 663         rc = lstcon_group_find(name, &grp);
 664         if (rc != 0) {
 665                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 666                 return rc;
 667         }
 668
 669         if (grp->grp_ref > 2) {
 670                 /* referred by test */
 671                 CDEBUG(D_NET, "Group %s is busy\n", name);
 672                 lstcon_group_put(grp);
 673                 return -EBUSY;
 674         }
 675
 676         rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up);
 677
 678         lstcon_group_put(grp);
 679         /* release empty group */
 680         if (cfs_list_empty(&grp->grp_ndl_list))
 681                 lstcon_group_put(grp);
 682
 683         return rc;
 684 }
 685
 686 int
 687 lstcon_group_refresh(char *name, cfs_list_t *result_up)
 688 {
 689         lstcon_rpc_trans_t      *trans;
 690         lstcon_group_t          *grp;
 691         int                      rc;
 692
 693         rc = lstcon_group_find(name, &grp);
 694         if (rc != 0) {
 695                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 696                 return rc;
 697         }
 698
 699         if (grp->grp_ref > 2) {
 700                 /* referred by test */
 701                 CDEBUG(D_NET, "Group %s is busy\n", name);
 702                 lstcon_group_put(grp);
 703                 return -EBUSY;
 704         }
 705
 706         /* re-invite all inactive nodes int the group */
 707         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 708                                      &grp->grp_trans_list, LST_TRANS_SESNEW,
 709                                      grp, lstcon_sesrpc_condition, &trans);
 710         if (rc != 0) {
 711                 /* local error, return */
 712                 CDEBUG(D_NET, "Can't create transaction: %d\n", rc);
 713                 lstcon_group_put(grp);
 714                 return rc;
 715         }
 716
 717         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 718
 719         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 720
 721         lstcon_rpc_trans_destroy(trans);
 722         /* -ref for me */
 723         lstcon_group_put(grp);
 724
 725         return rc;
 726 }
 727
 728 int
 729 lstcon_group_list(int index, int len, char *name_up)
 730 {
 731         lstcon_group_t *grp;
 732
 733         LASSERT (index >= 0);
 734         LASSERT (name_up != NULL);
 735
 736         cfs_list_for_each_entry_typed(grp, &console_session.ses_grp_list,
 737                                       lstcon_group_t, grp_link) {
 738                 if (index-- == 0) {
 739                         return cfs_copy_to_user(name_up, grp->grp_name, len) ?
 740                                -EFAULT : 0;
 741                 }
 742         }
 743
 744         return -ENOENT;
 745 }
 746
 747 static int
 748 lstcon_nodes_getent(cfs_list_t *head, int *index_p,
 749                     int *count_p, lstcon_node_ent_t *dents_up)
 750 {
 751         lstcon_ndlink_t  *ndl;
 752         lstcon_node_t    *nd;
 753         int               count = 0;
 754         int               index = 0;
 755
 756         LASSERT (index_p != NULL && count_p != NULL);
 757         LASSERT (dents_up != NULL);
 758         LASSERT (*index_p >= 0);
 759         LASSERT (*count_p > 0);
 760
 761         cfs_list_for_each_entry_typed(ndl, head, lstcon_ndlink_t, ndl_link) {
 762                 if (index++ < *index_p)
 763                         continue;
 764
 765                 if (count >= *count_p)
 766                         break;
 767
 768                 nd = ndl->ndl_node;
 769                 if (cfs_copy_to_user(&dents_up[count].nde_id,
 770                                      &nd->nd_id, sizeof(nd->nd_id)) ||
 771                     cfs_copy_to_user(&dents_up[count].nde_state,
 772                                      &nd->nd_state, sizeof(nd->nd_state)))
 773                         return -EFAULT;
 774
 775                 count ++;
 776         }
 777
 778         if (index <= *index_p)
 779                 return -ENOENT;
 780
 781         *count_p = count;
 782         *index_p = index;
 783
 784         return 0;
 785 }
 786
 787 int
 788 lstcon_group_info(char *name, lstcon_ndlist_ent_t *gents_p,
 789                   int *index_p, int *count_p, lstcon_node_ent_t *dents_up)
 790 {
 791         lstcon_ndlist_ent_t *gentp;
 792         lstcon_group_t      *grp;
 793         lstcon_ndlink_t     *ndl;
 794         int                  rc;
 795
 796         rc = lstcon_group_find(name, &grp);
 797         if (rc != 0) {
 798                 CDEBUG(D_NET, "Can't find group %s\n", name);
 799                 return rc;
 800         }
 801
 802         if (dents_up != 0) {
 803                 /* verbose query */
 804                 rc = lstcon_nodes_getent(&grp->grp_ndl_list,
 805                                          index_p, count_p, dents_up);
 806                 lstcon_group_put(grp);
 807
 808                 return rc;
 809         }
 810
 811         /* non-verbose query */
 812         LIBCFS_ALLOC(gentp, sizeof(lstcon_ndlist_ent_t));
 813         if (gentp == NULL) {
 814                 CERROR("Can't allocate ndlist_ent\n");
 815                 lstcon_group_put(grp);
 816
 817                 return -ENOMEM;
 818         }
 819
 820         memset(gentp, 0, sizeof(lstcon_ndlist_ent_t));
 821
 822         cfs_list_for_each_entry_typed(ndl, &grp->grp_ndl_list,
 823                                       lstcon_ndlink_t, ndl_link)
 824                 LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp);
 825
 826         rc = cfs_copy_to_user(gents_p, gentp,
 827                               sizeof(lstcon_ndlist_ent_t)) ? -EFAULT: 0;
 828
 829         LIBCFS_FREE(gentp, sizeof(lstcon_ndlist_ent_t));
 830
 831         lstcon_group_put(grp);
 832
 833         return 0;
 834 }
 835
 836 int
 837 lstcon_batch_find(char *name, lstcon_batch_t **batpp)
 838 {
 839         lstcon_batch_t   *bat;
 840
 841         cfs_list_for_each_entry_typed(bat, &console_session.ses_bat_list,
 842                                       lstcon_batch_t, bat_link) {
 843                 if (strncmp(bat->bat_name, name, LST_NAME_SIZE) == 0) {
 844                         *batpp = bat;
 845                         return 0;
 846                 }
 847         }
 848
 849         return -ENOENT;
 850 }
 851
 852 int
 853 lstcon_batch_add(char *name)
 854 {
 855         lstcon_batch_t   *bat;
 856         int               i;
 857         int               rc;
 858
 859         rc = (lstcon_batch_find(name, &bat) == 0)? -EEXIST: 0;
 860         if (rc != 0) {
 861                 CDEBUG(D_NET, "Batch %s already exists\n", name);
 862                 return rc;
 863         }
 864
 865         LIBCFS_ALLOC(bat, sizeof(lstcon_batch_t));
 866         if (bat == NULL) {
 867                 CERROR("Can't allocate descriptor for batch %s\n", name);
 868                 return -ENOMEM;
 869         }
 870
 871         LIBCFS_ALLOC(bat->bat_cli_hash,
 872                      sizeof(cfs_list_t) * LST_NODE_HASHSIZE);
 873         if (bat->bat_cli_hash == NULL) {
 874                 CERROR("Can't allocate hash for batch %s\n", name);
 875                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 876
 877                 return -ENOMEM;
 878         }
 879
 880         LIBCFS_ALLOC(bat->bat_srv_hash,
 881                      sizeof(cfs_list_t) * LST_NODE_HASHSIZE);
 882         if (bat->bat_srv_hash == NULL) {
 883                 CERROR("Can't allocate hash for batch %s\n", name);
 884                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 885                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 886
 887                 return -ENOMEM;
 888         }
 889
 890         strcpy(bat->bat_name, name);
 891         bat->bat_hdr.tsb_index = 0;
 892         bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie;
 893
 894         bat->bat_ntest = 0;
 895         bat->bat_state = LST_BATCH_IDLE;
 896
 897         CFS_INIT_LIST_HEAD(&bat->bat_cli_list);
 898         CFS_INIT_LIST_HEAD(&bat->bat_srv_list);
 899         CFS_INIT_LIST_HEAD(&bat->bat_test_list);
 900         CFS_INIT_LIST_HEAD(&bat->bat_trans_list);
 901
 902         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 903                 CFS_INIT_LIST_HEAD(&bat->bat_cli_hash[i]);
 904                 CFS_INIT_LIST_HEAD(&bat->bat_srv_hash[i]);
 905         }
 906
 907         cfs_list_add_tail(&bat->bat_link, &console_session.ses_bat_list);
 908
 909         return rc;
 910 }
 911
 912 int
 913 lstcon_batch_list(int index, int len, char *name_up)
 914 {
 915         lstcon_batch_t    *bat;
 916
 917         LASSERT (name_up != NULL);
 918         LASSERT (index >= 0);
 919
 920         cfs_list_for_each_entry_typed(bat, &console_session.ses_bat_list,
 921                                       lstcon_batch_t, bat_link) {
 922                 if (index-- == 0) {
 923                         return cfs_copy_to_user(name_up,bat->bat_name, len) ?
 924                                -EFAULT: 0;
 925                 }
 926         }
 927
 928         return -ENOENT;
 929 }
 930
 931 int
 932 lstcon_batch_info(char *name, lstcon_test_batch_ent_t *ent_up, int server,
 933                   int testidx, int *index_p, int *ndent_p,
 934                   lstcon_node_ent_t *dents_up)
 935 {
 936         lstcon_test_batch_ent_t *entp;
 937         cfs_list_t              *clilst;
 938         cfs_list_t              *srvlst;
 939         lstcon_test_t           *test = NULL;
 940         lstcon_batch_t          *bat;
 941         lstcon_ndlink_t         *ndl;
 942         int                      rc;
 943
 944         rc = lstcon_batch_find(name, &bat);
 945         if (rc != 0) {
 946                 CDEBUG(D_NET, "Can't find batch %s\n", name);
 947                 return -ENOENT;
 948         }
 949
 950         if (testidx > 0) {
 951                 /* query test, test index start from 1 */
 952                 cfs_list_for_each_entry_typed(test, &bat->bat_test_list,
 953                                               lstcon_test_t, tes_link) {
 954                         if (testidx-- == 1)
 955                                 break;
 956                 }
 957
 958                 if (testidx > 0) {
 959                         CDEBUG(D_NET, "Can't find specified test in batch\n");
 960                         return -ENOENT;
 961                 }
 962         }
 963
 964         clilst = (test == NULL) ? &bat->bat_cli_list :
 965                                   &test->tes_src_grp->grp_ndl_list;
 966         srvlst = (test == NULL) ? &bat->bat_srv_list :
 967                                   &test->tes_dst_grp->grp_ndl_list;
 968
 969         if (dents_up != NULL) {
 970                 rc = lstcon_nodes_getent((server ? srvlst: clilst),
 971                                          index_p, ndent_p, dents_up);
 972                 return rc;
 973         }
 974
 975         /* non-verbose query */
 976         LIBCFS_ALLOC(entp, sizeof(lstcon_test_batch_ent_t));
 977         if (entp == NULL)
 978                 return -ENOMEM;
 979
 980         memset(entp, 0, sizeof(lstcon_test_batch_ent_t));
 981
 982         if (test == NULL) {
 983                 entp->u.tbe_batch.bae_ntest = bat->bat_ntest;
 984                 entp->u.tbe_batch.bae_state = bat->bat_state;
 985
 986         } else {
 987
 988                 entp->u.tbe_test.tse_type   = test->tes_type;
 989                 entp->u.tbe_test.tse_loop   = test->tes_loop;
 990                 entp->u.tbe_test.tse_concur = test->tes_concur;
 991         }
 992
 993         cfs_list_for_each_entry_typed(ndl, clilst, lstcon_ndlink_t, ndl_link)
 994                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle);
 995
 996         cfs_list_for_each_entry_typed(ndl, srvlst, lstcon_ndlink_t, ndl_link)
 997                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle);
 998
 999         rc = cfs_copy_to_user(ent_up, entp,
1000                               sizeof(lstcon_test_batch_ent_t)) ? -EFAULT : 0;
1001
1002         LIBCFS_FREE(entp, sizeof(lstcon_test_batch_ent_t));
1003
1004         return rc;
1005 }
1006
1007 int
1008 lstcon_batrpc_condition(int transop, lstcon_node_t *nd, void *arg)
1009 {
1010         switch (transop) {
1011         case LST_TRANS_TSBRUN:
1012                 if (nd->nd_state != LST_NODE_ACTIVE)
1013                         return -ENETDOWN;
1014                 break;
1015
1016         case LST_TRANS_TSBSTOP:
1017                 if (nd->nd_state != LST_NODE_ACTIVE)
1018                         return 0;
1019                 break;
1020
1021         case LST_TRANS_TSBCLIQRY:
1022         case LST_TRANS_TSBSRVQRY:
1023                 break;
1024         }
1025
1026         return 1;
1027 }
1028
1029 static int
1030 lstcon_batch_op(lstcon_batch_t *bat, int transop,
1031                 cfs_list_t *result_up)
1032 {
1033         lstcon_rpc_trans_t *trans;
1034         int                 rc;
1035
1036         rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list,
1037                                      &bat->bat_trans_list, transop,
1038                                      bat, lstcon_batrpc_condition, &trans);
1039         if (rc != 0) {
1040                 CERROR("Can't create transaction: %d\n", rc);
1041                 return rc;
1042         }
1043
1044         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1045
1046         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1047
1048         lstcon_rpc_trans_destroy(trans);
1049
1050         return rc;
1051 }
1052
1053 int
1054 lstcon_batch_run(char *name, int timeout, cfs_list_t *result_up)
1055 {
1056         lstcon_batch_t *bat;
1057         int             rc;
1058
1059         if (lstcon_batch_find(name, &bat) != 0) {
1060                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1061                 return -ENOENT;
1062         }
1063
1064         bat->bat_arg = timeout;
1065
1066         rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up);
1067
1068         /* mark batch as running if it's started in any node */
1069         if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0) != 0)
1070                 bat->bat_state = LST_BATCH_RUNNING;
1071
1072         return rc;
1073 }
1074
1075 int
1076 lstcon_batch_stop(char *name, int force, cfs_list_t *result_up)
1077 {
1078         lstcon_batch_t *bat;
1079         int             rc;
1080
1081         if (lstcon_batch_find(name, &bat) != 0) {
1082                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1083                 return -ENOENT;
1084         }
1085
1086         bat->bat_arg = force;
1087
1088         rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up);
1089
1090         /* mark batch as stopped if all RPCs finished */
1091         if (lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0) == 0)
1092                 bat->bat_state = LST_BATCH_IDLE;
1093
1094         return rc;
1095 }
1096
1097 static void
1098 lstcon_batch_destroy(lstcon_batch_t *bat)
1099 {
1100         lstcon_ndlink_t    *ndl;
1101         lstcon_test_t      *test;
1102         int                 i;
1103
1104         cfs_list_del(&bat->bat_link);
1105
1106         while (!cfs_list_empty(&bat->bat_test_list)) {
1107                 test = cfs_list_entry(bat->bat_test_list.next,
1108                                       lstcon_test_t, tes_link);
1109                 LASSERT (cfs_list_empty(&test->tes_trans_list));
1110
1111                 cfs_list_del(&test->tes_link);
1112
1113                 lstcon_group_put(test->tes_src_grp);
1114                 lstcon_group_put(test->tes_dst_grp);
1115
1116                 LIBCFS_FREE(test, offsetof(lstcon_test_t,
1117                                            tes_param[test->tes_paramlen]));
1118         }
1119
1120         LASSERT (cfs_list_empty(&bat->bat_trans_list));
1121
1122         while (!cfs_list_empty(&bat->bat_cli_list)) {
1123                 ndl = cfs_list_entry(bat->bat_cli_list.next,
1124                                      lstcon_ndlink_t, ndl_link);
1125                 cfs_list_del_init(&ndl->ndl_link);
1126
1127                 lstcon_ndlink_release(ndl);
1128         }
1129
1130         while (!cfs_list_empty(&bat->bat_srv_list)) {
1131                 ndl = cfs_list_entry(bat->bat_srv_list.next,
1132                                      lstcon_ndlink_t, ndl_link);
1133                 cfs_list_del_init(&ndl->ndl_link);
1134
1135                 lstcon_ndlink_release(ndl);
1136         }
1137
1138         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1139                 LASSERT (cfs_list_empty(&bat->bat_cli_hash[i]));
1140                 LASSERT (cfs_list_empty(&bat->bat_srv_hash[i]));
1141         }
1142
1143         LIBCFS_FREE(bat->bat_cli_hash,
1144                     sizeof(cfs_list_t) * LST_NODE_HASHSIZE);
1145         LIBCFS_FREE(bat->bat_srv_hash,
1146                     sizeof(cfs_list_t) * LST_NODE_HASHSIZE);
1147         LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
1148 }
1149
1150 int
1151 lstcon_testrpc_condition(int transop, lstcon_node_t *nd, void *arg)
1152 {
1153         lstcon_test_t    *test;
1154         lstcon_batch_t   *batch;
1155         lstcon_ndlink_t  *ndl;
1156         cfs_list_t       *hash;
1157         cfs_list_t       *head;
1158
1159         test = (lstcon_test_t *)arg;
1160         LASSERT (test != NULL);
1161
1162         batch = test->tes_batch;
1163         LASSERT (batch != NULL);
1164
1165         if (test->tes_oneside &&
1166             transop == LST_TRANS_TSBSRVADD)
1167                 return 0;
1168
1169         if (nd->nd_state != LST_NODE_ACTIVE)
1170                 return -ENETDOWN;
1171
1172         if (transop == LST_TRANS_TSBCLIADD) {
1173                 hash = batch->bat_cli_hash;
1174                 head = &batch->bat_cli_list;
1175
1176         } else {
1177                 LASSERT (transop == LST_TRANS_TSBSRVADD);
1178
1179                 hash = batch->bat_srv_hash;
1180                 head = &batch->bat_srv_list;
1181         }
1182
1183         LASSERT (nd->nd_id.nid != LNET_NID_ANY);
1184
1185         if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1) != 0)
1186                 return -ENOMEM;
1187
1188         if (cfs_list_empty(&ndl->ndl_link))
1189                 cfs_list_add_tail(&ndl->ndl_link, head);
1190
1191         return 1;
1192 }
1193
1194 static int
1195 lstcon_test_nodes_add(lstcon_test_t *test, cfs_list_t *result_up)
1196 {
1197         lstcon_rpc_trans_t     *trans;
1198         lstcon_group_t         *grp;
1199         int                     transop;
1200         int                     rc;
1201
1202         LASSERT (test->tes_src_grp != NULL);
1203         LASSERT (test->tes_dst_grp != NULL);
1204
1205         transop = LST_TRANS_TSBSRVADD;
1206         grp  = test->tes_dst_grp;
1207 again:
1208         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
1209                                      &test->tes_trans_list, transop,
1210                                      test, lstcon_testrpc_condition, &trans);
1211         if (rc != 0) {
1212                 CERROR("Can't create transaction: %d\n", rc);
1213                 return rc;
1214         }
1215
1216         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1217
1218         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1219             lstcon_trans_stat()->trs_fwk_errno != 0) {
1220                 lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1221
1222                 lstcon_rpc_trans_destroy(trans);
1223                 /* return if any error */
1224                 CDEBUG(D_NET, "Failed to add test %s, "
1225                               "RPC error %d, framework error %d\n",
1226                        transop == LST_TRANS_TSBCLIADD ? "client" : "server",
1227                        lstcon_trans_stat()->trs_rpc_errno,
1228                        lstcon_trans_stat()->trs_fwk_errno);
1229
1230                 return rc;
1231         }
1232
1233         lstcon_rpc_trans_destroy(trans);
1234
1235         if (transop == LST_TRANS_TSBCLIADD)
1236                 return rc;
1237
1238         transop = LST_TRANS_TSBCLIADD;
1239         grp = test->tes_src_grp;
1240         test->tes_cliidx = 0;
1241
1242         /* requests to test clients */
1243         goto again;
1244 }
1245
1246 int
1247 lstcon_test_add(char *name, int type, int loop, int concur,
1248                 int dist, int span, char *src_name, char * dst_name,
1249                 void *param, int paramlen, int *retp,
1250                 cfs_list_t *result_up)
1251 {
1252         lstcon_group_t  *src_grp = NULL;
1253         lstcon_group_t  *dst_grp = NULL;
1254         lstcon_test_t   *test    = NULL;
1255         lstcon_batch_t  *batch;
1256         int              rc;
1257
1258         rc = lstcon_batch_find(name, &batch);
1259         if (rc != 0) {
1260                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1261                 return rc;
1262         }
1263
1264         if (batch->bat_state != LST_BATCH_IDLE) {
1265                 CDEBUG(D_NET, "Can't change running batch %s\n", name);
1266                 return rc;
1267         }
1268
1269         rc = lstcon_group_find(src_name, &src_grp);
1270         if (rc != 0) {
1271                 CDEBUG(D_NET, "Can't find group %s\n", src_name);
1272                 goto out;
1273         }
1274
1275         rc = lstcon_group_find(dst_name, &dst_grp);
1276         if (rc != 0) {
1277                 CDEBUG(D_NET, "Can't find group %s\n", dst_name);
1278                 goto out;
1279         }
1280
1281         if (dst_grp->grp_userland)
1282                 *retp = 1;
1283
1284         LIBCFS_ALLOC(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1285         if (!test) {
1286                 CERROR("Can't allocate test descriptor\n");
1287                 rc = -ENOMEM;
1288
1289                 goto out;
1290         }
1291
1292         memset(test, 0, offsetof(lstcon_test_t, tes_param[paramlen]));
1293         test->tes_hdr.tsb_id    = batch->bat_hdr.tsb_id;
1294         test->tes_batch         = batch;
1295         test->tes_type          = type;
1296         test->tes_oneside       = 0; /* TODO */
1297         test->tes_loop          = loop;
1298         test->tes_concur        = concur;
1299         test->tes_stop_onerr    = 1; /* TODO */
1300         test->tes_span          = span;
1301         test->tes_dist          = dist;
1302         test->tes_cliidx        = 0; /* just used for creating RPC */
1303         test->tes_src_grp       = src_grp;
1304         test->tes_dst_grp       = dst_grp;
1305         CFS_INIT_LIST_HEAD(&test->tes_trans_list);
1306
1307         if (param != NULL) {
1308                 test->tes_paramlen = paramlen;
1309                 memcpy(&test->tes_param[0], param, paramlen);
1310         }
1311
1312         rc = lstcon_test_nodes_add(test, result_up);
1313
1314         if (rc != 0)
1315                 goto out;
1316
1317         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1318             lstcon_trans_stat()->trs_fwk_errno != 0)
1319                 CDEBUG(D_NET, "Failed to add test %d to batch %s\n", type, name);
1320
1321         /* add to test list anyway, so user can check what's going on */
1322         cfs_list_add_tail(&test->tes_link, &batch->bat_test_list);
1323
1324         batch->bat_ntest ++;
1325         test->tes_hdr.tsb_index = batch->bat_ntest;
1326
1327         /*  hold groups so nobody can change them */
1328         return rc;
1329 out:
1330         if (test != NULL)
1331                 LIBCFS_FREE(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1332
1333         if (dst_grp != NULL)
1334                 lstcon_group_put(dst_grp);
1335
1336         if (src_grp != NULL)
1337                 lstcon_group_put(src_grp);
1338
1339         return rc;
1340 }
1341
1342 int
1343 lstcon_test_find(lstcon_batch_t *batch, int idx, lstcon_test_t **testpp)
1344 {
1345         lstcon_test_t *test;
1346
1347         cfs_list_for_each_entry_typed(test, &batch->bat_test_list,
1348                                       lstcon_test_t, tes_link) {
1349                 if (idx == test->tes_hdr.tsb_index) {
1350                         *testpp = test;
1351                         return 0;
1352                 }
1353         }
1354
1355         return -ENOENT;
1356 }
1357
1358 int
1359 lstcon_tsbrpc_readent(int transop, srpc_msg_t *msg,
1360                       lstcon_rpc_ent_t *ent_up)
1361 {
1362         srpc_batch_reply_t *rep = &msg->msg_body.bat_reply;
1363
1364         LASSERT (transop == LST_TRANS_TSBCLIQRY ||
1365                  transop == LST_TRANS_TSBSRVQRY);
1366
1367         /* positive errno, framework error code */
1368         if (cfs_copy_to_user(&ent_up->rpe_priv[0],
1369                              &rep->bar_active, sizeof(rep->bar_active)))
1370                 return -EFAULT;
1371
1372         return 0;
1373 }
1374
1375 int
1376 lstcon_test_batch_query(char *name, int testidx, int client,
1377                         int timeout, cfs_list_t *result_up)
1378 {
1379         lstcon_rpc_trans_t *trans;
1380         cfs_list_t         *translist;
1381         cfs_list_t         *ndlist;
1382         lstcon_tsb_hdr_t   *hdr;
1383         lstcon_batch_t     *batch;
1384         lstcon_test_t      *test = NULL;
1385         int                 transop;
1386         int                 rc;
1387
1388         rc = lstcon_batch_find(name, &batch);
1389         if (rc != 0) {
1390                 CDEBUG(D_NET, "Can't find batch: %s\n", name);
1391                 return rc;
1392         }
1393
1394         if (testidx == 0) {
1395                 translist = &batch->bat_trans_list;
1396                 ndlist    = &batch->bat_cli_list;
1397                 hdr       = &batch->bat_hdr;
1398
1399         } else {
1400                 /* query specified test only */
1401                 rc = lstcon_test_find(batch, testidx, &test);
1402                 if (rc != 0) {
1403                         CDEBUG(D_NET, "Can't find test: %d\n", testidx);
1404                         return rc;
1405                 }
1406
1407                 translist = &test->tes_trans_list;
1408                 ndlist    = &test->tes_src_grp->grp_ndl_list;
1409                 hdr       = &test->tes_hdr;
1410         }
1411
1412         transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY;
1413
1414         rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr,
1415                                      lstcon_batrpc_condition, &trans);
1416         if (rc != 0) {
1417                 CERROR("Can't create transaction: %d\n", rc);
1418                 return rc;
1419         }
1420
1421         lstcon_rpc_trans_postwait(trans, timeout);
1422
1423         if (testidx == 0 && /* query a batch, not a test */
1424             lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) == 0 &&
1425             lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0) == 0) {
1426                 /* all RPCs finished, and no active test */
1427                 batch->bat_state = LST_BATCH_IDLE;
1428         }
1429
1430         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1431                                           lstcon_tsbrpc_readent);
1432         lstcon_rpc_trans_destroy(trans);
1433
1434         return rc;
1435 }
1436
1437 int
1438 lstcon_statrpc_readent(int transop, srpc_msg_t *msg,
1439                        lstcon_rpc_ent_t *ent_up)
1440 {
1441         srpc_stat_reply_t *rep = &msg->msg_body.stat_reply;
1442         sfw_counters_t    *sfwk_stat;
1443         srpc_counters_t   *srpc_stat;
1444         lnet_counters_t   *lnet_stat;
1445
1446         if (rep->str_status != 0)
1447                 return 0;
1448
1449         sfwk_stat = (sfw_counters_t *)&ent_up->rpe_payload[0];
1450         srpc_stat = (srpc_counters_t *)((char *)sfwk_stat + sizeof(*sfwk_stat));
1451         lnet_stat = (lnet_counters_t *)((char *)srpc_stat + sizeof(*srpc_stat));
1452
1453         if (cfs_copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) ||
1454             cfs_copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) ||
1455             cfs_copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat)))
1456                 return -EFAULT;
1457
1458         return 0;
1459 }
1460
1461 int
1462 lstcon_ndlist_stat(cfs_list_t *ndlist,
1463                    int timeout, cfs_list_t *result_up)
1464 {
1465         cfs_list_t          head;
1466         lstcon_rpc_trans_t *trans;
1467         int                 rc;
1468
1469         CFS_INIT_LIST_HEAD(&head);
1470
1471         rc = lstcon_rpc_trans_ndlist(ndlist, &head,
1472                                      LST_TRANS_STATQRY, NULL, NULL, &trans);
1473         if (rc != 0) {
1474                 CERROR("Can't create transaction: %d\n", rc);
1475                 return rc;
1476         }
1477
1478         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1479
1480         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1481                                           lstcon_statrpc_readent);
1482         lstcon_rpc_trans_destroy(trans);
1483
1484         return rc;
1485 }
1486
1487 int
1488 lstcon_group_stat(char *grp_name, int timeout, cfs_list_t *result_up)
1489 {
1490         lstcon_group_t     *grp;
1491         int                 rc;
1492
1493         rc = lstcon_group_find(grp_name, &grp);
1494         if (rc != 0) {
1495                 CDEBUG(D_NET, "Can't find group %s\n", grp_name);
1496                 return rc;
1497         }
1498
1499         rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up);
1500
1501         lstcon_group_put(grp);
1502
1503         return rc;
1504 }
1505
1506 int
1507 lstcon_nodes_stat(int count, lnet_process_id_t *ids_up,
1508                   int timeout, cfs_list_t *result_up)
1509 {
1510         lstcon_ndlink_t         *ndl;
1511         lstcon_group_t          *tmp;
1512         lnet_process_id_t        id;
1513         int                      i;
1514         int                      rc;
1515
1516         rc = lstcon_group_alloc(NULL, &tmp);
1517         if (rc != 0) {
1518                 CERROR("Out of memory\n");
1519                 return -ENOMEM;
1520         }
1521
1522         for (i = 0 ; i < count; i++) {
1523                 if (cfs_copy_from_user(&id, &ids_up[i], sizeof(id))) {
1524                         rc = -EFAULT;
1525                         break;
1526                 }
1527
1528                 /* add to tmp group */
1529                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2);
1530                 if (rc != 0) {
1531                         CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET,
1532                                "Failed to find or create %s: %d\n",
1533                                libcfs_id2str(id), rc);
1534                         break;
1535                 }
1536         }
1537
1538         if (rc != 0) {
1539                 lstcon_group_put(tmp);
1540                 return rc;
1541         }
1542
1543         rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up);
1544
1545         lstcon_group_put(tmp);
1546
1547         return rc;
1548 }
1549
1550 int
1551 lstcon_debug_ndlist(cfs_list_t *ndlist,
1552                     cfs_list_t *translist,
1553                     int timeout, cfs_list_t *result_up)
1554 {
1555         lstcon_rpc_trans_t *trans;
1556         int                 rc;
1557
1558         rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY,
1559                                      NULL, lstcon_sesrpc_condition, &trans);
1560         if (rc != 0) {
1561                 CERROR("Can't create transaction: %d\n", rc);
1562                 return rc;
1563         }
1564
1565         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1566
1567         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1568                                           lstcon_sesrpc_readent);
1569         lstcon_rpc_trans_destroy(trans);
1570
1571         return rc;
1572 }
1573
1574 int
1575 lstcon_session_debug(int timeout, cfs_list_t *result_up)
1576 {
1577         return lstcon_debug_ndlist(&console_session.ses_ndl_list,
1578                                    NULL, timeout, result_up);
1579 }
1580
1581 int
1582 lstcon_batch_debug(int timeout, char *name,
1583                    int client, cfs_list_t *result_up)
1584 {
1585         lstcon_batch_t *bat;
1586         int             rc;
1587
1588         rc = lstcon_batch_find(name, &bat);
1589         if (rc != 0)
1590                 return -ENOENT;
1591
1592         rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list :
1593                                           &bat->bat_srv_list,
1594                                  NULL, timeout, result_up);
1595
1596         return rc;
1597 }
1598
1599 int
1600 lstcon_group_debug(int timeout, char *name,
1601                    cfs_list_t *result_up)
1602 {
1603         lstcon_group_t *grp;
1604         int             rc;
1605
1606         rc = lstcon_group_find(name, &grp);
1607         if (rc != 0)
1608                 return -ENOENT;
1609
1610         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1611                                  timeout, result_up);
1612         lstcon_group_put(grp);
1613
1614         return rc;
1615 }
1616
1617 int
1618 lstcon_nodes_debug(int timeout,
1619                    int count, lnet_process_id_t *ids_up,
1620                    cfs_list_t *result_up)
1621 {
1622         lnet_process_id_t  id;
1623         lstcon_ndlink_t   *ndl;
1624         lstcon_group_t    *grp;
1625         int                i;
1626         int                rc;
1627
1628         rc = lstcon_group_alloc(NULL, &grp);
1629         if (rc != 0) {
1630                 CDEBUG(D_NET, "Out of memory\n");
1631                 return rc;
1632         }
1633
1634         for (i = 0; i < count; i++) {
1635                 if (cfs_copy_from_user(&id, &ids_up[i], sizeof(id))) {
1636                         rc = -EFAULT;
1637                         break;
1638                 }
1639
1640                 /* node is added to tmp group */
1641                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 1);
1642                 if (rc != 0) {
1643                         CERROR("Can't create node link\n");
1644                         break;
1645                 }
1646         }
1647
1648         if (rc != 0) {
1649                 lstcon_group_put(grp);
1650                 return rc;
1651         }
1652
1653         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1654                                  timeout, result_up);
1655
1656         lstcon_group_put(grp);
1657
1658         return rc;
1659 }
1660
1661 int
1662 lstcon_session_match(lst_sid_t sid)
1663 {
1664         return (console_session.ses_id.ses_nid   == sid.ses_nid &&
1665                 console_session.ses_id.ses_stamp == sid.ses_stamp) ?  1: 0;
1666 }
1667
1668 static void
1669 lstcon_new_session_id(lst_sid_t *sid)
1670 {
1671         lnet_process_id_t      id;
1672
1673         LASSERT (console_session.ses_state == LST_SESSION_NONE);
1674
1675         LNetGetId(1, &id);
1676         sid->ses_nid   = id.nid;
1677         sid->ses_stamp = cfs_time_current();
1678 }
1679
1680 extern srpc_service_t lstcon_acceptor_service;
1681
1682 int
1683 lstcon_session_new(char *name, int key,
1684                    int timeout,int force, lst_sid_t *sid_up)
1685 {
1686         int     rc = 0;
1687         int     i;
1688
1689         if (console_session.ses_state != LST_SESSION_NONE) {
1690                 /* session exists */
1691                 if (!force) {
1692                         CERROR("Session %s already exists\n",
1693                                console_session.ses_name);
1694                         return -EEXIST;
1695                 }
1696
1697                 rc = lstcon_session_end();
1698
1699                 /* lstcon_session_end() only return local error */
1700                 if  (rc != 0)
1701                         return rc;
1702         }
1703
1704         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++) {
1705                 LASSERT (cfs_list_empty(&console_session.ses_ndl_hash[i]));
1706         }
1707
1708         rc = lstcon_batch_add(LST_DEFAULT_BATCH);
1709         if (rc != 0)
1710                 return rc;
1711
1712         rc = lstcon_rpc_pinger_start();
1713         if (rc != 0) {
1714                 lstcon_batch_t *bat = NULL;
1715
1716                 lstcon_batch_find(LST_DEFAULT_BATCH, &bat);
1717                 lstcon_batch_destroy(bat);
1718
1719                 return rc;
1720         }
1721
1722         lstcon_new_session_id(&console_session.ses_id);
1723
1724         console_session.ses_key     = key;
1725         console_session.ses_state   = LST_SESSION_ACTIVE;
1726         console_session.ses_force   = !!force;
1727         console_session.ses_timeout = (timeout <= 0)? LST_CONSOLE_TIMEOUT:
1728                                                       timeout;
1729         strcpy(console_session.ses_name, name);
1730
1731         if (cfs_copy_to_user(sid_up, &console_session.ses_id,
1732                              sizeof(lst_sid_t)) == 0)
1733                 return rc;
1734
1735         lstcon_session_end();
1736
1737         return -EFAULT;
1738 }
1739
1740 int
1741 lstcon_session_info(lst_sid_t *sid_up, int *key_up,
1742                     lstcon_ndlist_ent_t *ndinfo_up, char *name_up, int len)
1743 {
1744         lstcon_ndlist_ent_t *entp;
1745         lstcon_ndlink_t     *ndl;
1746         int                  rc = 0;
1747
1748         if (console_session.ses_state != LST_SESSION_ACTIVE)
1749                 return -ESRCH;
1750
1751         LIBCFS_ALLOC(entp, sizeof(*entp));
1752         if (entp == NULL)
1753                 return -ENOMEM;
1754
1755         memset(entp, 0, sizeof(*entp));
1756
1757         cfs_list_for_each_entry_typed(ndl, &console_session.ses_ndl_list,
1758                                       lstcon_ndlink_t, ndl_link)
1759                 LST_NODE_STATE_COUNTER(ndl->ndl_node, entp);
1760
1761         if (cfs_copy_to_user(sid_up, &console_session.ses_id,
1762                              sizeof(lst_sid_t)) ||
1763             cfs_copy_to_user(key_up, &console_session.ses_key, sizeof(int)) ||
1764             cfs_copy_to_user(ndinfo_up, entp, sizeof(*entp)) ||
1765             cfs_copy_to_user(name_up, console_session.ses_name, len))
1766                 rc = -EFAULT;
1767
1768         LIBCFS_FREE(entp, sizeof(*entp));
1769
1770         return rc;
1771 }
1772
1773 int
1774 lstcon_session_end()
1775 {
1776         lstcon_rpc_trans_t *trans;
1777         lstcon_group_t     *grp;
1778         lstcon_batch_t     *bat;
1779         int                 rc = 0;
1780
1781         LASSERT (console_session.ses_state == LST_SESSION_ACTIVE);
1782
1783         rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list,
1784                                      NULL, LST_TRANS_SESEND, NULL,
1785                                      lstcon_sesrpc_condition, &trans);
1786         if (rc != 0) {
1787                 CERROR("Can't create transaction: %d\n", rc);
1788                 return rc;
1789         }
1790
1791         console_session.ses_shutdown = 1;
1792
1793         lstcon_rpc_pinger_stop();
1794
1795         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1796
1797         lstcon_rpc_trans_destroy(trans);
1798         /* User can do nothing even rpc failed, so go on */
1799
1800         /* waiting for orphan rpcs to die */
1801         lstcon_rpc_cleanup_wait();
1802
1803         console_session.ses_id    = LST_INVALID_SID;
1804         console_session.ses_state = LST_SESSION_NONE;
1805         console_session.ses_key   = 0;
1806         console_session.ses_force = 0;
1807
1808         /* destroy all batches */
1809         while (!cfs_list_empty(&console_session.ses_bat_list)) {
1810                 bat = cfs_list_entry(console_session.ses_bat_list.next,
1811                                      lstcon_batch_t, bat_link);
1812
1813                 lstcon_batch_destroy(bat);
1814         }
1815
1816         /* destroy all groups */
1817         while (!cfs_list_empty(&console_session.ses_grp_list)) {
1818                 grp = cfs_list_entry(console_session.ses_grp_list.next,
1819                                      lstcon_group_t, grp_link);
1820                 LASSERT (grp->grp_ref == 1);
1821
1822                 lstcon_group_put(grp);
1823         }
1824
1825         /* all nodes should be released */
1826         LASSERT (cfs_list_empty(&console_session.ses_ndl_list));
1827
1828         console_session.ses_shutdown = 0;
1829         console_session.ses_expired  = 0;
1830
1831         return rc;
1832 }
1833
1834 static int
1835 lstcon_acceptor_handle (srpc_server_rpc_t *rpc)
1836 {
1837         srpc_msg_t        *rep  = &rpc->srpc_replymsg;
1838         srpc_msg_t        *req  = &rpc->srpc_reqstbuf->buf_msg;
1839         srpc_join_reqst_t *jreq = &req->msg_body.join_reqst;
1840         srpc_join_reply_t *jrep = &rep->msg_body.join_reply;
1841         lstcon_group_t    *grp  = NULL;
1842         lstcon_ndlink_t   *ndl;
1843         int                rc   = 0;
1844
1845         sfw_unpack_message(req);
1846
1847         cfs_mutex_lock(&console_session.ses_mutex);
1848
1849         jrep->join_sid = console_session.ses_id;
1850
1851         if (console_session.ses_id.ses_nid == LNET_NID_ANY) {
1852                 jrep->join_status = ESRCH;
1853                 goto out;
1854         }
1855
1856         if (jreq->join_sid.ses_nid != LNET_NID_ANY &&
1857              !lstcon_session_match(jreq->join_sid)) {
1858                 jrep->join_status = EBUSY;
1859                 goto out;
1860         }
1861
1862         if (lstcon_group_find(jreq->join_group, &grp) != 0) {
1863                 rc = lstcon_group_alloc(jreq->join_group, &grp);
1864                 if (rc != 0) {
1865                         CERROR("Out of memory\n");
1866                         goto out;
1867                 }
1868
1869                 cfs_list_add_tail(&grp->grp_link,
1870                                   &console_session.ses_grp_list);
1871                 lstcon_group_addref(grp);
1872         }
1873
1874         if (grp->grp_ref > 2) {
1875                 /* Group in using */
1876                 jrep->join_status = EBUSY;
1877                 goto out;
1878         }
1879
1880         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0);
1881         if (rc == 0) {
1882                 jrep->join_status = EEXIST;
1883                 goto out;
1884         }
1885
1886         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1);
1887         if (rc != 0) {
1888                 CERROR("Out of memory\n");
1889                 goto out;
1890         }
1891
1892         ndl->ndl_node->nd_state   = LST_NODE_ACTIVE;
1893         ndl->ndl_node->nd_timeout = console_session.ses_timeout;
1894
1895         if (grp->grp_userland == 0)
1896                 grp->grp_userland = 1;
1897
1898         strcpy(jrep->join_session, console_session.ses_name);
1899         jrep->join_timeout = console_session.ses_timeout;
1900         jrep->join_status  = 0;
1901
1902 out:
1903         if (grp != NULL)
1904                 lstcon_group_put(grp);
1905
1906         cfs_mutex_unlock(&console_session.ses_mutex);
1907
1908         return rc;
1909 }
1910
1911 srpc_service_t lstcon_acceptor_service;
1912 void lstcon_init_acceptor_service(void)
1913 {
1914         /* initialize selftest console acceptor service table */
1915         lstcon_acceptor_service.sv_name    = "join session";
1916         lstcon_acceptor_service.sv_handler = lstcon_acceptor_handle;
1917         lstcon_acceptor_service.sv_id      = SRPC_SERVICE_JOIN;
1918         lstcon_acceptor_service.sv_concur  = SFW_SERVICE_CONCURRENCY;
1919 }
1920
1921 extern int lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_data *data);
1922
1923 DECLARE_IOCTL_HANDLER(lstcon_ioctl_handler, lstcon_ioctl_entry);
1924
1925 /* initialize console */
1926 int
1927 lstcon_console_init(void)
1928 {
1929         int     i;
1930         int     n;
1931         int     rc;
1932
1933         memset(&console_session, 0, sizeof(lstcon_session_t));
1934
1935         console_session.ses_id      = LST_INVALID_SID;
1936         console_session.ses_state   = LST_SESSION_NONE;
1937         console_session.ses_timeout = 0;
1938         console_session.ses_force   = 0;
1939         console_session.ses_expired = 0;
1940         console_session.ses_laststamp = cfs_time_current_sec();
1941
1942         cfs_mutex_init(&console_session.ses_mutex);
1943
1944         CFS_INIT_LIST_HEAD(&console_session.ses_ndl_list);
1945         CFS_INIT_LIST_HEAD(&console_session.ses_grp_list);
1946         CFS_INIT_LIST_HEAD(&console_session.ses_bat_list);
1947         CFS_INIT_LIST_HEAD(&console_session.ses_trans_list);
1948
1949         LIBCFS_ALLOC(console_session.ses_ndl_hash,
1950                      sizeof(cfs_list_t) * LST_GLOBAL_HASHSIZE);
1951         if (console_session.ses_ndl_hash == NULL)
1952                 return -ENOMEM;
1953
1954         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
1955                 CFS_INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]);
1956
1957
1958         /* initialize acceptor service table */
1959         lstcon_init_acceptor_service();
1960
1961         rc = srpc_add_service(&lstcon_acceptor_service);
1962         LASSERT (rc != -EBUSY);
1963         if (rc != 0) {
1964                 LIBCFS_FREE(console_session.ses_ndl_hash,
1965                             sizeof(cfs_list_t) * LST_GLOBAL_HASHSIZE);
1966                 return rc;
1967         }
1968
1969         n = srpc_service_add_buffers(&lstcon_acceptor_service, SFW_POST_BUFFERS);
1970         if (n != SFW_POST_BUFFERS) {
1971                 rc = -ENOMEM;
1972                 goto out;
1973         }
1974
1975         rc = libcfs_register_ioctl(&lstcon_ioctl_handler);
1976
1977         if (rc == 0) {
1978                 lstcon_rpc_module_init();
1979                 return 0;
1980         }
1981
1982 out:
1983         srpc_shutdown_service(&lstcon_acceptor_service);
1984         srpc_remove_service(&lstcon_acceptor_service);
1985
1986         LIBCFS_FREE(console_session.ses_ndl_hash,
1987                     sizeof(cfs_list_t) * LST_GLOBAL_HASHSIZE);
1988
1989         srpc_wait_service_shutdown(&lstcon_acceptor_service);
1990
1991         return rc;
1992 }
1993
1994 int
1995 lstcon_console_fini(void)
1996 {
1997         int     i;
1998
1999         libcfs_deregister_ioctl(&lstcon_ioctl_handler);
2000
2001         cfs_mutex_lock(&console_session.ses_mutex);
2002
2003         srpc_shutdown_service(&lstcon_acceptor_service);
2004         srpc_remove_service(&lstcon_acceptor_service);
2005
2006         if (console_session.ses_state != LST_SESSION_NONE)
2007                 lstcon_session_end();
2008
2009         lstcon_rpc_module_fini();
2010
2011         cfs_mutex_unlock(&console_session.ses_mutex);
2012
2013         LASSERT (cfs_list_empty(&console_session.ses_ndl_list));
2014         LASSERT (cfs_list_empty(&console_session.ses_grp_list));
2015         LASSERT (cfs_list_empty(&console_session.ses_bat_list));
2016         LASSERT (cfs_list_empty(&console_session.ses_trans_list));
2017
2018         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
2019                 LASSERT (cfs_list_empty(&console_session.ses_ndl_hash[i]));
2020         }
2021
2022         LIBCFS_FREE(console_session.ses_ndl_hash,
2023                     sizeof(cfs_list_t) * LST_GLOBAL_HASHSIZE);
2024
2025         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2026
2027         return 0;
2028 }
2029
2030 #endif