lnet/selftest/console.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2012, 2014, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  *
  31  * lnet/selftest/conctl.c
  32  *
  33  * Infrastructure of LST console
  34  *
  35  * Author: Liang Zhen <liangzhen@clusterfs.com>
  36  */
  37
  38 #include <libcfs/libcfs.h>
  39 #include <lnet/lib-lnet.h>
  40 #include "console.h"
  41 #include "conrpc.h"
  42
  43 #define LST_NODE_STATE_COUNTER(nd, p)                   \
  44 do {                                                    \
  45         if ((nd)->nd_state == LST_NODE_ACTIVE)          \
  46                 (p)->nle_nactive ++;                    \
  47         else if ((nd)->nd_state == LST_NODE_BUSY)       \
  48                 (p)->nle_nbusy ++;                      \
  49         else if ((nd)->nd_state == LST_NODE_DOWN)       \
  50                 (p)->nle_ndown ++;                      \
  51         else                                            \
  52                 (p)->nle_nunknown ++;                   \
  53         (p)->nle_nnode ++;                              \
  54 } while (0)
  55
  56 struct lstcon_session console_session;
  57
  58 static void
  59 lstcon_node_get(struct lstcon_node *nd)
  60 {
  61         LASSERT (nd->nd_ref >= 1);
  62
  63         nd->nd_ref++;
  64 }
  65
  66 static int
  67 lstcon_node_find(struct lnet_process_id id, struct lstcon_node **ndpp,
  68                  int create)
  69 {
  70         struct lstcon_ndlink *ndl;
  71         unsigned int idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE;
  72
  73         LASSERT(id.nid != LNET_NID_ANY);
  74
  75         list_for_each_entry(ndl, &console_session.ses_ndl_hash[idx],
  76                             ndl_hlink) {
  77                 if (ndl->ndl_node->nd_id.nid != id.nid ||
  78                     ndl->ndl_node->nd_id.pid != id.pid)
  79                         continue;
  80
  81                 lstcon_node_get(ndl->ndl_node);
  82                 *ndpp = ndl->ndl_node;
  83                 return 0;
  84         }
  85
  86         if (!create)
  87                 return -ENOENT;
  88
  89         LIBCFS_ALLOC(*ndpp, sizeof(**ndpp) + sizeof(*ndl));
  90         if (*ndpp == NULL)
  91                 return -ENOMEM;
  92
  93         ndl = (struct lstcon_ndlink *)(*ndpp + 1);
  94
  95         ndl->ndl_node = *ndpp;
  96
  97         ndl->ndl_node->nd_ref   = 1;
  98         ndl->ndl_node->nd_id    = id;
  99         ndl->ndl_node->nd_stamp = ktime_get();
 100         ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
 101         ndl->ndl_node->nd_timeout = 0;
 102         memset(&ndl->ndl_node->nd_ping, 0, sizeof(ndl->ndl_node->nd_ping));
 103
 104         /* queued in global hash & list, no refcount is taken by
 105          * global hash & list, if caller release his refcount,
 106          * node will be released */
 107         list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]);
 108         list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list);
 109
 110         return 0;
 111 }
 112
 113 static void
 114 lstcon_node_put(struct lstcon_node *nd)
 115 {
 116         struct lstcon_ndlink *ndl;
 117
 118         LASSERT(nd->nd_ref > 0);
 119
 120         if (--nd->nd_ref > 0)
 121                 return;
 122
 123         ndl = (struct lstcon_ndlink *)(nd + 1);
 124
 125         LASSERT(!list_empty(&ndl->ndl_link));
 126         LASSERT(!list_empty(&ndl->ndl_hlink));
 127
 128         /* remove from session */
 129         list_del(&ndl->ndl_link);
 130         list_del(&ndl->ndl_hlink);
 131
 132         LIBCFS_FREE(nd, sizeof(*nd) + sizeof(*ndl));
 133 }
 134
 135 static int
 136 lstcon_ndlink_find(struct list_head *hash, struct lnet_process_id id,
 137                    struct lstcon_ndlink **ndlpp, int create)
 138 {
 139         unsigned int idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
 140         struct lstcon_ndlink *ndl;
 141         struct lstcon_node *nd;
 142         int rc;
 143
 144         if (id.nid == LNET_NID_ANY)
 145                 return -EINVAL;
 146
 147         /* search in hash */
 148         list_for_each_entry(ndl, &hash[idx], ndl_hlink) {
 149                 if (ndl->ndl_node->nd_id.nid != id.nid ||
 150                     ndl->ndl_node->nd_id.pid != id.pid)
 151                         continue;
 152
 153                 *ndlpp = ndl;
 154                 return 0;
 155         }
 156
 157         if (create == 0)
 158                 return -ENOENT;
 159
 160         /* find or create in session hash */
 161         rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0);
 162         if (rc != 0)
 163                 return rc;
 164
 165         LIBCFS_ALLOC(ndl, sizeof(*ndl));
 166         if (ndl == NULL) {
 167                 lstcon_node_put(nd);
 168                 return -ENOMEM;
 169         }
 170
 171         *ndlpp = ndl;
 172
 173         ndl->ndl_node = nd;
 174         INIT_LIST_HEAD(&ndl->ndl_link);
 175         list_add_tail(&ndl->ndl_hlink, &hash[idx]);
 176
 177         return 0;
 178 }
 179
 180 static void
 181 lstcon_ndlink_release(struct lstcon_ndlink *ndl)
 182 {
 183         LASSERT(list_empty(&ndl->ndl_link));
 184         LASSERT(!list_empty(&ndl->ndl_hlink));
 185
 186         list_del(&ndl->ndl_hlink); /* delete from hash */
 187         lstcon_node_put(ndl->ndl_node);
 188
 189         LIBCFS_FREE(ndl, sizeof(*ndl));
 190 }
 191
 192 static int
 193 lstcon_group_alloc(char *name, struct lstcon_group **grpp)
 194 {
 195         struct lstcon_group *grp;
 196         int i;
 197
 198         LIBCFS_ALLOC(grp, offsetof(struct lstcon_group,
 199                                    grp_ndl_hash[LST_NODE_HASHSIZE]));
 200         if (grp == NULL)
 201                 return -ENOMEM;
 202
 203         grp->grp_ref = 1;
 204         if (name != NULL) {
 205                 if (strlen(name) > sizeof(grp->grp_name)-1) {
 206                         LIBCFS_FREE(grp, offsetof(struct lstcon_group,
 207                                           grp_ndl_hash[LST_NODE_HASHSIZE]));
 208                         return -E2BIG;
 209                 }
 210                 strncpy(grp->grp_name, name, sizeof(grp->grp_name));
 211         }
 212
 213         INIT_LIST_HEAD(&grp->grp_link);
 214         INIT_LIST_HEAD(&grp->grp_ndl_list);
 215         INIT_LIST_HEAD(&grp->grp_trans_list);
 216
 217         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 218                 INIT_LIST_HEAD(&grp->grp_ndl_hash[i]);
 219
 220         *grpp = grp;
 221
 222         return 0;
 223 }
 224
 225 static void
 226 lstcon_group_addref(struct lstcon_group *grp)
 227 {
 228         grp->grp_ref++;
 229 }
 230
 231 static void lstcon_group_ndlink_release(struct lstcon_group *,
 232                                         struct lstcon_ndlink *);
 233
 234 static void
 235 lstcon_group_drain(struct lstcon_group *grp, int keep)
 236 {
 237         struct lstcon_ndlink *ndl;
 238         struct lstcon_ndlink *tmp;
 239
 240         list_for_each_entry_safe(ndl, tmp, &grp->grp_ndl_list, ndl_link) {
 241                 if ((ndl->ndl_node->nd_state & keep) == 0)
 242                         lstcon_group_ndlink_release(grp, ndl);
 243         }
 244 }
 245
 246 static void
 247 lstcon_group_decref(struct lstcon_group *grp)
 248 {
 249         int i;
 250
 251         if (--grp->grp_ref > 0)
 252                 return;
 253
 254         if (!list_empty(&grp->grp_link))
 255                 list_del(&grp->grp_link);
 256
 257         lstcon_group_drain(grp, 0);
 258
 259         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 260                 LASSERT(list_empty(&grp->grp_ndl_hash[i]));
 261
 262         LIBCFS_FREE(grp, offsetof(struct lstcon_group,
 263                                   grp_ndl_hash[LST_NODE_HASHSIZE]));
 264 }
 265
 266 static int
 267 lstcon_group_find(const char *name, struct lstcon_group **grpp)
 268 {
 269         struct lstcon_group *grp;
 270
 271         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 272                 if (strncmp(grp->grp_name, name, LST_NAME_SIZE) != 0)
 273                         continue;
 274
 275                 lstcon_group_addref(grp);  /* +1 ref for caller */
 276                 *grpp = grp;
 277                 return 0;
 278         }
 279
 280         return -ENOENT;
 281 }
 282
 283 static int
 284 lstcon_group_ndlink_find(struct lstcon_group *grp, struct lnet_process_id id,
 285                          struct lstcon_ndlink **ndlpp, int create)
 286 {
 287         int rc;
 288
 289         rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create);
 290         if (rc != 0)
 291                 return rc;
 292
 293         if (!list_empty(&(*ndlpp)->ndl_link))
 294                 return 0;
 295
 296         list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list);
 297         grp->grp_nnode++;
 298
 299         return 0;
 300 }
 301
 302 static void
 303 lstcon_group_ndlink_release(struct lstcon_group *grp, struct lstcon_ndlink *ndl)
 304 {
 305         list_del_init(&ndl->ndl_link);
 306         lstcon_ndlink_release(ndl);
 307         grp->grp_nnode--;
 308 }
 309
 310 static void
 311 lstcon_group_ndlink_move(struct lstcon_group *old,
 312                          struct lstcon_group *new, struct lstcon_ndlink *ndl)
 313 {
 314         unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) %
 315                                         LST_NODE_HASHSIZE;
 316
 317         old->grp_nnode--;
 318
 319         list_move_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]);
 320         list_move_tail(&ndl->ndl_link, &new->grp_ndl_list);
 321         new->grp_nnode++;
 322 }
 323
 324 static void
 325 lstcon_group_move(struct lstcon_group *old, struct lstcon_group *new)
 326 {
 327         struct lstcon_ndlink *ndl;
 328
 329         while (!list_empty(&old->grp_ndl_list)) {
 330                 ndl = list_entry(old->grp_ndl_list.next,
 331                                  struct lstcon_ndlink, ndl_link);
 332                 lstcon_group_ndlink_move(old, new, ndl);
 333         }
 334 }
 335
 336 static int
 337 lstcon_sesrpc_condition(int transop, struct lstcon_node *nd, void *arg)
 338 {
 339         struct lstcon_group *grp = arg;
 340
 341         switch (transop) {
 342         case LST_TRANS_SESNEW:
 343                 if (nd->nd_state == LST_NODE_ACTIVE)
 344                         return 0;
 345                 break;
 346
 347         case LST_TRANS_SESEND:
 348                 if (nd->nd_state != LST_NODE_ACTIVE)
 349                         return 0;
 350
 351                 if (grp != NULL && nd->nd_ref > 1)
 352                         return 0;
 353                 break;
 354
 355         case LST_TRANS_SESQRY:
 356                 break;
 357
 358         default:
 359                 LBUG();
 360         }
 361
 362         return 1;
 363 }
 364
 365 static int
 366 lstcon_sesrpc_readent(int transop, struct srpc_msg *msg,
 367                       struct lstcon_rpc_ent __user *ent_up)
 368 {
 369         struct srpc_debug_reply *rep;
 370
 371         switch (transop) {
 372         case LST_TRANS_SESNEW:
 373         case LST_TRANS_SESEND:
 374                 return 0;
 375
 376         case LST_TRANS_SESQRY:
 377                 rep = &msg->msg_body.dbg_reply;
 378
 379                 if (copy_to_user(&ent_up->rpe_priv[0],
 380                                      &rep->dbg_timeout, sizeof(int)) ||
 381                     copy_to_user(&ent_up->rpe_payload[0],
 382                                      &rep->dbg_name, LST_NAME_SIZE))
 383                         return -EFAULT;
 384
 385                 return 0;
 386
 387         default:
 388                 LBUG();
 389         }
 390
 391         return 0;
 392 }
 393
 394 static int
 395 lstcon_group_nodes_add(struct lstcon_group *grp,
 396                        int count, struct lnet_process_id __user *ids_up,
 397                        unsigned int *featp,
 398                        struct list_head __user *result_up)
 399 {
 400         struct lstcon_rpc_trans *trans;
 401         struct lstcon_ndlink *ndl;
 402         struct lstcon_group *tmp;
 403         struct lnet_process_id id;
 404         int i;
 405         int rc;
 406
 407         rc = lstcon_group_alloc(NULL, &tmp);
 408         if (rc != 0) {
 409                 CERROR("Out of memory\n");
 410                 return -ENOMEM;
 411         }
 412
 413         for (i = 0 ; i < count; i++) {
 414                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 415                         rc = -EFAULT;
 416                         break;
 417                 }
 418
 419                 /* skip if it's in this group already */
 420                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 0);
 421                 if (rc == 0)
 422                         continue;
 423
 424                 /* add to tmp group */
 425                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1);
 426                 if (rc != 0) {
 427                         CERROR("Can't create ndlink, out of memory\n");
 428                         break;
 429                 }
 430         }
 431
 432         if (rc != 0) {
 433                 lstcon_group_decref(tmp);
 434                 return rc;
 435         }
 436
 437         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 438                                      &tmp->grp_trans_list, LST_TRANS_SESNEW,
 439                                      tmp, lstcon_sesrpc_condition, &trans);
 440         if (rc != 0) {
 441                 CERROR("Can't create transaction: %d\n", rc);
 442                 lstcon_group_decref(tmp);
 443                 return rc;
 444         }
 445
 446         /* post all RPCs */
 447         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 448
 449         rc = lstcon_rpc_trans_interpreter(trans, result_up,
 450                                           lstcon_sesrpc_readent);
 451         *featp = trans->tas_features;
 452
 453         /* destroy all RPGs */
 454         lstcon_rpc_trans_destroy(trans);
 455
 456         lstcon_group_move(tmp, grp);
 457         lstcon_group_decref(tmp);
 458
 459         return rc;
 460 }
 461
 462 static int
 463 lstcon_group_nodes_remove(struct lstcon_group *grp,
 464                           int count, struct lnet_process_id __user *ids_up,
 465                           struct list_head __user *result_up)
 466 {
 467         struct lstcon_rpc_trans *trans;
 468         struct lstcon_ndlink *ndl;
 469         struct lstcon_group *tmp;
 470         struct lnet_process_id id;
 471         int rc;
 472         int i;
 473
 474         /* End session and remove node from the group */
 475
 476         rc = lstcon_group_alloc(NULL, &tmp);
 477         if (rc != 0) {
 478                 CERROR("Out of memory\n");
 479                 return -ENOMEM;
 480         }
 481
 482         for (i = 0; i < count; i++) {
 483                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 484                         rc = -EFAULT;
 485                         goto error;
 486                 }
 487
 488                 /* move node to tmp group */
 489                 if (lstcon_group_ndlink_find(grp, id, &ndl, 0) == 0)
 490                         lstcon_group_ndlink_move(grp, tmp, ndl);
 491         }
 492
 493         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 494                                      &tmp->grp_trans_list, LST_TRANS_SESEND,
 495                                      tmp, lstcon_sesrpc_condition, &trans);
 496         if (rc != 0) {
 497                 CERROR("Can't create transaction: %d\n", rc);
 498                 goto error;
 499         }
 500
 501         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 502
 503         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 504
 505         lstcon_rpc_trans_destroy(trans);
 506         /* release nodes anyway, because we can't rollback status */
 507         lstcon_group_decref(tmp);
 508
 509         return rc;
 510 error:
 511         lstcon_group_move(tmp, grp);
 512         lstcon_group_decref(tmp);
 513
 514         return rc;
 515 }
 516
 517 int
 518 lstcon_group_add(char *name)
 519 {
 520         struct lstcon_group *grp;
 521         int rc;
 522
 523         rc = (lstcon_group_find(name, &grp) == 0)? -EEXIST: 0;
 524         if (rc != 0) {
 525                 /* find a group with same name */
 526                 lstcon_group_decref(grp);
 527                 return rc;
 528         }
 529
 530         rc = lstcon_group_alloc(name, &grp);
 531         if (rc != 0) {
 532                 CERROR("Can't allocate descriptor for group %s\n", name);
 533                 return -ENOMEM;
 534         }
 535
 536         list_add_tail(&grp->grp_link, &console_session.ses_grp_list);
 537
 538         return rc;
 539 }
 540
 541 int
 542 lstcon_nodes_add(char *name, int count, struct lnet_process_id __user *ids_up,
 543                  unsigned *featp, struct list_head __user *result_up)
 544 {
 545         struct lstcon_group         *grp;
 546         int                     rc;
 547
 548         LASSERT (count > 0);
 549         LASSERT (ids_up != NULL);
 550
 551         rc = lstcon_group_find(name, &grp);
 552         if (rc != 0) {
 553                 CDEBUG(D_NET, "Can't find group %s\n", name);
 554                 return rc;
 555         }
 556
 557         if (grp->grp_ref > 2) {
 558                 /* referred by other threads or test */
 559                 CDEBUG(D_NET, "Group %s is busy\n", name);
 560                 lstcon_group_decref(grp);
 561
 562                 return -EBUSY;
 563         }
 564
 565         rc = lstcon_group_nodes_add(grp, count, ids_up, featp, result_up);
 566
 567         lstcon_group_decref(grp);
 568
 569         return rc;
 570 }
 571
 572 int
 573 lstcon_group_del(char *name)
 574 {
 575         struct lstcon_rpc_trans *trans;
 576         struct lstcon_group *grp;
 577         int rc;
 578
 579         rc = lstcon_group_find(name, &grp);
 580         if (rc != 0) {
 581                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 582                 return rc;
 583         }
 584
 585         if (grp->grp_ref > 2) {
 586                 /* referred by others threads or test */
 587                 CDEBUG(D_NET, "Group %s is busy\n", name);
 588                 lstcon_group_decref(grp);
 589                 return -EBUSY;
 590         }
 591
 592         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 593                                      &grp->grp_trans_list, LST_TRANS_SESEND,
 594                                      grp, lstcon_sesrpc_condition, &trans);
 595         if (rc != 0) {
 596                 CERROR("Can't create transaction: %d\n", rc);
 597                 lstcon_group_decref(grp);
 598                 return rc;
 599         }
 600
 601         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 602
 603         lstcon_rpc_trans_destroy(trans);
 604
 605         lstcon_group_decref(grp);
 606         /* -ref for session, it's destroyed,
 607          * status can't be rolled back, destroy group anway */
 608         lstcon_group_decref(grp);
 609
 610         return rc;
 611 }
 612
 613 int
 614 lstcon_group_clean(char *name, int args)
 615 {
 616         struct lstcon_group *grp = NULL;
 617         int rc;
 618
 619         rc = lstcon_group_find(name, &grp);
 620         if (rc != 0) {
 621                 CDEBUG(D_NET, "Can't find group %s\n", name);
 622                 return rc;
 623         }
 624
 625         if (grp->grp_ref > 2) {
 626                 /* referred by test */
 627                 CDEBUG(D_NET, "Group %s is busy\n", name);
 628                 lstcon_group_decref(grp);
 629                 return -EBUSY;
 630         }
 631
 632         args = (LST_NODE_ACTIVE | LST_NODE_BUSY |
 633                 LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args;
 634
 635         lstcon_group_drain(grp, args);
 636
 637         lstcon_group_decref(grp);
 638         /* release empty group */
 639         if (list_empty(&grp->grp_ndl_list))
 640                 lstcon_group_decref(grp);
 641
 642         return 0;
 643 }
 644
 645 int
 646 lstcon_nodes_remove(char *name, int count,
 647                     struct lnet_process_id __user *ids_up,
 648                     struct list_head __user *result_up)
 649 {
 650         struct lstcon_group *grp = NULL;
 651         int rc;
 652
 653         rc = lstcon_group_find(name, &grp);
 654         if (rc != 0) {
 655                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 656                 return rc;
 657         }
 658
 659         if (grp->grp_ref > 2) {
 660                 /* referred by test */
 661                 CDEBUG(D_NET, "Group %s is busy\n", name);
 662                 lstcon_group_decref(grp);
 663                 return -EBUSY;
 664         }
 665
 666         rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up);
 667
 668         lstcon_group_decref(grp);
 669         /* release empty group */
 670         if (list_empty(&grp->grp_ndl_list))
 671                 lstcon_group_decref(grp);
 672
 673         return rc;
 674 }
 675
 676 int
 677 lstcon_group_refresh(char *name, struct list_head __user *result_up)
 678 {
 679         struct lstcon_rpc_trans *trans;
 680         struct lstcon_group *grp;
 681         int rc;
 682
 683         rc = lstcon_group_find(name, &grp);
 684         if (rc != 0) {
 685                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 686                 return rc;
 687         }
 688
 689         if (grp->grp_ref > 2) {
 690                 /* referred by test */
 691                 CDEBUG(D_NET, "Group %s is busy\n", name);
 692                 lstcon_group_decref(grp);
 693                 return -EBUSY;
 694         }
 695
 696         /* re-invite all inactive nodes int the group */
 697         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 698                                      &grp->grp_trans_list, LST_TRANS_SESNEW,
 699                                      grp, lstcon_sesrpc_condition, &trans);
 700         if (rc != 0) {
 701                 /* local error, return */
 702                 CDEBUG(D_NET, "Can't create transaction: %d\n", rc);
 703                 lstcon_group_decref(grp);
 704                 return rc;
 705         }
 706
 707         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 708
 709         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 710
 711         lstcon_rpc_trans_destroy(trans);
 712         /* -ref for me */
 713         lstcon_group_decref(grp);
 714
 715         return rc;
 716 }
 717
 718 int
 719 lstcon_group_list(int index, int len, char __user *name_up)
 720 {
 721         struct lstcon_group *grp;
 722
 723         LASSERT(index >= 0);
 724         LASSERT(name_up != NULL);
 725
 726         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 727                 if (index-- == 0) {
 728                         return copy_to_user(name_up, grp->grp_name, len) ?
 729                                             -EFAULT : 0;
 730                 }
 731         }
 732
 733         return -ENOENT;
 734 }
 735
 736 static int
 737 lstcon_nodes_getent(struct list_head *head, int *index_p,
 738                     int *count_p, struct lstcon_node_ent __user *dents_up)
 739 {
 740         struct lstcon_ndlink *ndl;
 741         struct lstcon_node *nd;
 742         int count = 0;
 743         int index = 0;
 744
 745         LASSERT(index_p != NULL && count_p != NULL);
 746         LASSERT(dents_up != NULL);
 747         LASSERT(*index_p >= 0);
 748         LASSERT(*count_p > 0);
 749
 750         list_for_each_entry(ndl, head, ndl_link) {
 751                 if (index++ < *index_p)
 752                         continue;
 753
 754                 if (count >= *count_p)
 755                         break;
 756
 757                 nd = ndl->ndl_node;
 758                 if (copy_to_user(&dents_up[count].nde_id,
 759                                      &nd->nd_id, sizeof(nd->nd_id)) ||
 760                     copy_to_user(&dents_up[count].nde_state,
 761                                      &nd->nd_state, sizeof(nd->nd_state)))
 762                         return -EFAULT;
 763
 764                 count ++;
 765         }
 766
 767         if (index <= *index_p)
 768                 return -ENOENT;
 769
 770         *count_p = count;
 771         *index_p = index;
 772
 773         return 0;
 774 }
 775
 776 int
 777 lstcon_group_info(char *name, struct lstcon_ndlist_ent __user *gents_p,
 778                   int *index_p, int *count_p,
 779                   struct lstcon_node_ent __user *dents_up)
 780 {
 781         struct lstcon_ndlist_ent *gentp;
 782         struct lstcon_group *grp;
 783         struct lstcon_ndlink *ndl;
 784         int rc;
 785
 786         rc = lstcon_group_find(name, &grp);
 787         if (rc != 0) {
 788                 CDEBUG(D_NET, "Can't find group %s\n", name);
 789                 return rc;
 790         }
 791
 792         if (dents_up != NULL) {
 793                 /* verbose query */
 794                 rc = lstcon_nodes_getent(&grp->grp_ndl_list,
 795                                          index_p, count_p, dents_up);
 796                 lstcon_group_decref(grp);
 797
 798                 return rc;
 799         }
 800
 801         /* non-verbose query */
 802         CFS_ALLOC_PTR(gentp);
 803         if (gentp == NULL) {
 804                 CERROR("Can't allocate ndlist_ent\n");
 805                 lstcon_group_decref(grp);
 806
 807                 return -ENOMEM;
 808         }
 809
 810         list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link)
 811                 LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp);
 812
 813         rc = copy_to_user(gents_p, gentp,
 814                           sizeof(struct lstcon_ndlist_ent)) ? -EFAULT : 0;
 815
 816         CFS_FREE_PTR(gentp);
 817
 818         lstcon_group_decref(grp);
 819
 820         return 0;
 821 }
 822
 823 static int
 824 lstcon_batch_find(const char *name, struct lstcon_batch **batpp)
 825 {
 826         struct lstcon_batch *bat;
 827
 828         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 829                 if (strncmp(bat->bat_name, name, LST_NAME_SIZE) == 0) {
 830                         *batpp = bat;
 831                         return 0;
 832                 }
 833         }
 834
 835         return -ENOENT;
 836 }
 837
 838 int
 839 lstcon_batch_add(char *name)
 840 {
 841         struct lstcon_batch *bat;
 842         int i;
 843         int rc;
 844
 845         rc = (lstcon_batch_find(name, &bat) == 0)? -EEXIST: 0;
 846         if (rc != 0) {
 847                 CDEBUG(D_NET, "Batch %s already exists\n", name);
 848                 return rc;
 849         }
 850
 851         LIBCFS_ALLOC(bat, sizeof(*bat));
 852         if (bat == NULL) {
 853                 CERROR("Can't allocate descriptor for batch %s\n", name);
 854                 return -ENOMEM;
 855         }
 856
 857         CFS_ALLOC_PTR_ARRAY(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 858         if (bat->bat_cli_hash == NULL) {
 859                 CERROR("Can't allocate hash for batch %s\n", name);
 860                 LIBCFS_FREE(bat, sizeof(*bat));
 861
 862                 return -ENOMEM;
 863         }
 864
 865         CFS_ALLOC_PTR_ARRAY(bat->bat_srv_hash, LST_NODE_HASHSIZE);
 866         if (bat->bat_srv_hash == NULL) {
 867                 CERROR("Can't allocate hash for batch %s\n", name);
 868                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 869                 LIBCFS_FREE(bat, sizeof(*bat));
 870
 871                 return -ENOMEM;
 872         }
 873
 874         if (strlen(name) > sizeof(bat->bat_name)-1) {
 875                 LIBCFS_FREE(bat->bat_srv_hash, LST_NODE_HASHSIZE);
 876                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 877                 LIBCFS_FREE(bat, sizeof(*bat));
 878                 return -E2BIG;
 879         }
 880         strncpy(bat->bat_name, name, sizeof(bat->bat_name));
 881         bat->bat_hdr.tsb_index = 0;
 882         bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie;
 883
 884         bat->bat_ntest = 0;
 885         bat->bat_state = LST_BATCH_IDLE;
 886
 887         INIT_LIST_HEAD(&bat->bat_cli_list);
 888         INIT_LIST_HEAD(&bat->bat_srv_list);
 889         INIT_LIST_HEAD(&bat->bat_test_list);
 890         INIT_LIST_HEAD(&bat->bat_trans_list);
 891
 892         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 893                 INIT_LIST_HEAD(&bat->bat_cli_hash[i]);
 894                 INIT_LIST_HEAD(&bat->bat_srv_hash[i]);
 895         }
 896
 897         list_add_tail(&bat->bat_link, &console_session.ses_bat_list);
 898
 899         return rc;
 900 }
 901
 902 int
 903 lstcon_batch_list(int index, int len, char __user *name_up)
 904 {
 905         struct lstcon_batch *bat;
 906
 907         LASSERT(name_up != NULL);
 908         LASSERT(index >= 0);
 909
 910         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 911                 if (index-- == 0) {
 912                         return copy_to_user(name_up, bat->bat_name, len) ?
 913                                             -EFAULT : 0;
 914                 }
 915         }
 916
 917         return -ENOENT;
 918 }
 919
 920 int
 921 lstcon_batch_info(char *name, struct lstcon_test_batch_ent __user *ent_up,
 922                   int server, int testidx, int *index_p, int *ndent_p,
 923                   struct lstcon_node_ent __user *dents_up)
 924 {
 925         struct lstcon_test_batch_ent *entp;
 926         struct list_head *clilst;
 927         struct list_head *srvlst;
 928         struct lstcon_test *test = NULL;
 929         struct lstcon_batch *bat;
 930         struct lstcon_ndlink *ndl;
 931         int rc;
 932
 933         rc = lstcon_batch_find(name, &bat);
 934         if (rc != 0) {
 935                 CDEBUG(D_NET, "Can't find batch %s\n", name);
 936                 return -ENOENT;
 937         }
 938
 939         if (testidx > 0) {
 940                 /* query test, test index start from 1 */
 941                 list_for_each_entry(test, &bat->bat_test_list, tes_link) {
 942                         if (testidx-- == 1)
 943                                 break;
 944                 }
 945
 946                 if (testidx > 0) {
 947                         CDEBUG(D_NET, "Can't find specified test in batch\n");
 948                         return -ENOENT;
 949                 }
 950         }
 951
 952         clilst = (test == NULL) ? &bat->bat_cli_list :
 953                                   &test->tes_src_grp->grp_ndl_list;
 954         srvlst = (test == NULL) ? &bat->bat_srv_list :
 955                                   &test->tes_dst_grp->grp_ndl_list;
 956
 957         if (dents_up != NULL) {
 958                 rc = lstcon_nodes_getent((server ? srvlst: clilst),
 959                                          index_p, ndent_p, dents_up);
 960                 return rc;
 961         }
 962
 963         /* non-verbose query */
 964         CFS_ALLOC_PTR(entp);
 965         if (entp == NULL)
 966                 return -ENOMEM;
 967
 968         if (test == NULL) {
 969                 entp->u.tbe_batch.bae_ntest = bat->bat_ntest;
 970                 entp->u.tbe_batch.bae_state = bat->bat_state;
 971
 972         } else {
 973
 974                 entp->u.tbe_test.tse_type   = test->tes_type;
 975                 entp->u.tbe_test.tse_loop   = test->tes_loop;
 976                 entp->u.tbe_test.tse_concur = test->tes_concur;
 977         }
 978
 979         list_for_each_entry(ndl, clilst, ndl_link)
 980                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle);
 981
 982         list_for_each_entry(ndl, srvlst, ndl_link)
 983                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle);
 984
 985         rc = copy_to_user(ent_up, entp,
 986                           sizeof(struct lstcon_test_batch_ent)) ? -EFAULT : 0;
 987
 988         CFS_FREE_PTR(entp)
 989
 990         return rc;
 991 }
 992
 993 static int
 994 lstcon_batrpc_condition(int transop, struct lstcon_node *nd, void *arg)
 995 {
 996         switch (transop) {
 997         case LST_TRANS_TSBRUN:
 998                 if (nd->nd_state != LST_NODE_ACTIVE)
 999                         return -ENETDOWN;
1000                 break;
1001
1002         case LST_TRANS_TSBSTOP:
1003                 if (nd->nd_state != LST_NODE_ACTIVE)
1004                         return 0;
1005                 break;
1006
1007         case LST_TRANS_TSBCLIQRY:
1008         case LST_TRANS_TSBSRVQRY:
1009                 break;
1010         }
1011
1012         return 1;
1013 }
1014
1015 static int
1016 lstcon_batch_op(struct lstcon_batch *bat, int transop,
1017                 struct list_head __user *result_up)
1018 {
1019         struct lstcon_rpc_trans *trans;
1020         int                 rc;
1021
1022         rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list,
1023                                      &bat->bat_trans_list, transop,
1024                                      bat, lstcon_batrpc_condition, &trans);
1025         if (rc != 0) {
1026                 CERROR("Can't create transaction: %d\n", rc);
1027                 return rc;
1028         }
1029
1030         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1031
1032         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1033
1034         lstcon_rpc_trans_destroy(trans);
1035
1036         return rc;
1037 }
1038
1039 int
1040 lstcon_batch_run(char *name, int timeout, struct list_head __user *result_up)
1041 {
1042         struct lstcon_batch *bat;
1043         int rc;
1044
1045         if (lstcon_batch_find(name, &bat) != 0) {
1046                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1047                 return -ENOENT;
1048         }
1049
1050         bat->bat_arg = timeout;
1051
1052         rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up);
1053
1054         /* mark batch as running if it's started in any node */
1055         if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0) != 0)
1056                 bat->bat_state = LST_BATCH_RUNNING;
1057
1058         return rc;
1059 }
1060
1061 int
1062 lstcon_batch_stop(char *name, int force, struct list_head __user *result_up)
1063 {
1064         struct lstcon_batch *bat;
1065         int rc;
1066
1067         if (lstcon_batch_find(name, &bat) != 0) {
1068                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1069                 return -ENOENT;
1070         }
1071
1072         bat->bat_arg = force;
1073
1074         rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up);
1075
1076         /* mark batch as stopped if all RPCs finished */
1077         if (lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0) == 0)
1078                 bat->bat_state = LST_BATCH_IDLE;
1079
1080         return rc;
1081 }
1082
1083 static void
1084 lstcon_batch_destroy(struct lstcon_batch *bat)
1085 {
1086         struct lstcon_ndlink *ndl;
1087         struct lstcon_test *test;
1088         int i;
1089
1090         list_del(&bat->bat_link);
1091
1092         while (!list_empty(&bat->bat_test_list)) {
1093                 test = list_entry(bat->bat_test_list.next,
1094                                   struct lstcon_test, tes_link);
1095                 LASSERT(list_empty(&test->tes_trans_list));
1096
1097                 list_del(&test->tes_link);
1098
1099                 lstcon_group_decref(test->tes_src_grp);
1100                 lstcon_group_decref(test->tes_dst_grp);
1101
1102                 LIBCFS_FREE(test, offsetof(struct lstcon_test,
1103                                            tes_param[test->tes_paramlen]));
1104         }
1105
1106         LASSERT(list_empty(&bat->bat_trans_list));
1107
1108         while (!list_empty(&bat->bat_cli_list)) {
1109                 ndl = list_entry(bat->bat_cli_list.next,
1110                                  struct lstcon_ndlink, ndl_link);
1111                 list_del_init(&ndl->ndl_link);
1112
1113                 lstcon_ndlink_release(ndl);
1114         }
1115
1116         while (!list_empty(&bat->bat_srv_list)) {
1117                 ndl = list_entry(bat->bat_srv_list.next,
1118                                  struct lstcon_ndlink, ndl_link);
1119                 list_del_init(&ndl->ndl_link);
1120
1121                 lstcon_ndlink_release(ndl);
1122         }
1123
1124         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1125                 LASSERT(list_empty(&bat->bat_cli_hash[i]));
1126                 LASSERT(list_empty(&bat->bat_srv_hash[i]));
1127         }
1128
1129         LIBCFS_FREE(bat->bat_cli_hash,
1130                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1131         LIBCFS_FREE(bat->bat_srv_hash,
1132                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1133         LIBCFS_FREE(bat, sizeof(*bat));
1134 }
1135
1136 static int
1137 lstcon_testrpc_condition(int transop, struct lstcon_node *nd, void *arg)
1138 {
1139         struct lstcon_test *test = arg;
1140         struct lstcon_batch *batch;
1141         struct lstcon_ndlink *ndl;
1142         struct list_head *hash;
1143         struct list_head *head;
1144
1145         LASSERT(test != NULL);
1146
1147         batch = test->tes_batch;
1148         LASSERT(batch != NULL);
1149
1150         if (test->tes_oneside &&
1151             transop == LST_TRANS_TSBSRVADD)
1152                 return 0;
1153
1154         if (nd->nd_state != LST_NODE_ACTIVE)
1155                 return -ENETDOWN;
1156
1157         if (transop == LST_TRANS_TSBCLIADD) {
1158                 hash = batch->bat_cli_hash;
1159                 head = &batch->bat_cli_list;
1160
1161         } else {
1162                 LASSERT (transop == LST_TRANS_TSBSRVADD);
1163
1164                 hash = batch->bat_srv_hash;
1165                 head = &batch->bat_srv_list;
1166         }
1167
1168         LASSERT (nd->nd_id.nid != LNET_NID_ANY);
1169
1170         if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1) != 0)
1171                 return -ENOMEM;
1172
1173         if (list_empty(&ndl->ndl_link))
1174                 list_add_tail(&ndl->ndl_link, head);
1175
1176         return 1;
1177 }
1178
1179 static int
1180 lstcon_test_nodes_add(struct lstcon_test *test,
1181                       struct list_head __user *result_up)
1182 {
1183         struct lstcon_rpc_trans *trans;
1184         struct lstcon_group *grp;
1185         int transop;
1186         int rc;
1187
1188         LASSERT (test->tes_src_grp != NULL);
1189         LASSERT (test->tes_dst_grp != NULL);
1190
1191         transop = LST_TRANS_TSBSRVADD;
1192         grp  = test->tes_dst_grp;
1193 again:
1194         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
1195                                      &test->tes_trans_list, transop,
1196                                      test, lstcon_testrpc_condition, &trans);
1197         if (rc != 0) {
1198                 CERROR("Can't create transaction: %d\n", rc);
1199                 return rc;
1200         }
1201
1202         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1203
1204         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1205             lstcon_trans_stat()->trs_fwk_errno != 0) {
1206                 lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1207
1208                 lstcon_rpc_trans_destroy(trans);
1209                 /* return if any error */
1210                 CDEBUG(D_NET, "Failed to add test %s, "
1211                               "RPC error %d, framework error %d\n",
1212                        transop == LST_TRANS_TSBCLIADD ? "client" : "server",
1213                        lstcon_trans_stat()->trs_rpc_errno,
1214                        lstcon_trans_stat()->trs_fwk_errno);
1215
1216                 return rc;
1217         }
1218
1219         lstcon_rpc_trans_destroy(trans);
1220
1221         if (transop == LST_TRANS_TSBCLIADD)
1222                 return rc;
1223
1224         transop = LST_TRANS_TSBCLIADD;
1225         grp = test->tes_src_grp;
1226         test->tes_cliidx = 0;
1227
1228         /* requests to test clients */
1229         goto again;
1230 }
1231
1232 static int
1233 lstcon_verify_batch(const char *name, struct lstcon_batch **batch)
1234 {
1235         int rc;
1236
1237         rc = lstcon_batch_find(name, batch);
1238         if (rc != 0) {
1239                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1240                 return rc;
1241         }
1242
1243         if ((*batch)->bat_state != LST_BATCH_IDLE) {
1244                 CDEBUG(D_NET, "Can't change running batch %s\n", name);
1245                 return -EINVAL;
1246         }
1247
1248         return 0;
1249 }
1250
1251 static int
1252 lstcon_verify_group(const char *name, struct lstcon_group **grp)
1253 {
1254         int rc;
1255         struct lstcon_ndlink *ndl;
1256
1257         rc = lstcon_group_find(name, grp);
1258         if (rc != 0) {
1259                 CDEBUG(D_NET, "can't find group %s\n", name);
1260                 return rc;
1261         }
1262
1263         list_for_each_entry(ndl, &(*grp)->grp_ndl_list, ndl_link) {
1264                 if (ndl->ndl_node->nd_state == LST_NODE_ACTIVE) {
1265                         return 0;
1266                 }
1267         }
1268
1269         CDEBUG(D_NET, "Group %s has no ACTIVE nodes\n", name);
1270
1271         return -EINVAL;
1272 }
1273
1274 int
1275 lstcon_test_add(char *batch_name, int type, int loop,
1276                 int concur, int dist, int span,
1277                 char *src_name, char *dst_name,
1278                 void *param, int paramlen, int *retp,
1279                 struct list_head __user *result_up)
1280 {
1281         struct lstcon_test *test = NULL;
1282         int rc;
1283         struct lstcon_group *src_grp = NULL;
1284         struct lstcon_group *dst_grp = NULL;
1285         struct lstcon_batch *batch = NULL;
1286
1287         /*
1288          * verify that a batch of the given name exists, and the groups
1289          * that will be part of the batch exist and have at least one
1290          * active node
1291          */
1292         rc = lstcon_verify_batch(batch_name, &batch);
1293         if (rc != 0)
1294                 goto out;
1295
1296         rc = lstcon_verify_group(src_name, &src_grp);
1297         if (rc != 0)
1298                 goto out;
1299
1300         rc = lstcon_verify_group(dst_name, &dst_grp);
1301         if (rc != 0)
1302                 goto out;
1303
1304         if (dst_grp->grp_userland)
1305                 *retp = 1;
1306
1307         LIBCFS_ALLOC(test, offsetof(struct lstcon_test, tes_param[paramlen]));
1308         if (!test) {
1309                 CERROR("Can't allocate test descriptor\n");
1310                 rc = -ENOMEM;
1311
1312                 goto out;
1313         }
1314
1315         test->tes_hdr.tsb_id    = batch->bat_hdr.tsb_id;
1316         test->tes_batch         = batch;
1317         test->tes_type          = type;
1318         test->tes_oneside       = 0; /* TODO */
1319         test->tes_loop          = loop;
1320         test->tes_concur        = concur;
1321         test->tes_stop_onerr    = 1; /* TODO */
1322         test->tes_span          = span;
1323         test->tes_dist          = dist;
1324         test->tes_cliidx        = 0; /* just used for creating RPC */
1325         test->tes_src_grp       = src_grp;
1326         test->tes_dst_grp       = dst_grp;
1327         INIT_LIST_HEAD(&test->tes_trans_list);
1328
1329         if (param != NULL) {
1330                 test->tes_paramlen = paramlen;
1331                 memcpy(&test->tes_param[0], param, paramlen);
1332         }
1333
1334         rc = lstcon_test_nodes_add(test, result_up);
1335
1336         if (rc != 0)
1337                 goto out;
1338
1339         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1340             lstcon_trans_stat()->trs_fwk_errno != 0)
1341                 CDEBUG(D_NET, "Failed to add test %d to batch %s\n", type,
1342                        batch_name);
1343
1344         /* add to test list anyway, so user can check what's going on */
1345         list_add_tail(&test->tes_link, &batch->bat_test_list);
1346
1347         batch->bat_ntest++;
1348         test->tes_hdr.tsb_index = batch->bat_ntest;
1349
1350         /*  hold groups so nobody can change them */
1351         return rc;
1352 out:
1353         if (test != NULL)
1354                 LIBCFS_FREE(test, offsetof(struct lstcon_test,
1355                                            tes_param[paramlen]));
1356
1357         if (dst_grp != NULL)
1358                 lstcon_group_decref(dst_grp);
1359
1360         if (src_grp != NULL)
1361                 lstcon_group_decref(src_grp);
1362
1363         return rc;
1364 }
1365
1366 static int
1367 lstcon_test_find(struct lstcon_batch *batch, int idx,
1368                  struct lstcon_test **testpp)
1369 {
1370         struct lstcon_test *test;
1371
1372         list_for_each_entry(test, &batch->bat_test_list, tes_link) {
1373                 if (idx == test->tes_hdr.tsb_index) {
1374                         *testpp = test;
1375                         return 0;
1376                 }
1377         }
1378
1379         return -ENOENT;
1380 }
1381
1382 static int
1383 lstcon_tsbrpc_readent(int transop, struct srpc_msg *msg,
1384                       struct lstcon_rpc_ent __user *ent_up)
1385 {
1386         struct srpc_batch_reply *rep = &msg->msg_body.bat_reply;
1387
1388         LASSERT (transop == LST_TRANS_TSBCLIQRY ||
1389                  transop == LST_TRANS_TSBSRVQRY);
1390
1391         /* positive errno, framework error code */
1392         if (copy_to_user(&ent_up->rpe_priv[0],
1393                              &rep->bar_active, sizeof(rep->bar_active)))
1394                 return -EFAULT;
1395
1396         return 0;
1397 }
1398
1399 int
1400 lstcon_test_batch_query(char *name, int testidx, int client,
1401                         int timeout, struct list_head __user *result_up)
1402 {
1403         struct lstcon_rpc_trans *trans;
1404         struct list_head *translist;
1405         struct list_head *ndlist;
1406         struct lstcon_tsb_hdr *hdr;
1407         struct lstcon_batch *batch;
1408         struct lstcon_test *test = NULL;
1409         int transop;
1410         int rc;
1411
1412         rc = lstcon_batch_find(name, &batch);
1413         if (rc != 0) {
1414                 CDEBUG(D_NET, "Can't find batch: %s\n", name);
1415                 return rc;
1416         }
1417
1418         if (testidx == 0) {
1419                 translist = &batch->bat_trans_list;
1420                 ndlist    = &batch->bat_cli_list;
1421                 hdr       = &batch->bat_hdr;
1422
1423         } else {
1424                 /* query specified test only */
1425                 rc = lstcon_test_find(batch, testidx, &test);
1426                 if (rc != 0) {
1427                         CDEBUG(D_NET, "Can't find test: %d\n", testidx);
1428                         return rc;
1429                 }
1430
1431                 translist = &test->tes_trans_list;
1432                 ndlist    = &test->tes_src_grp->grp_ndl_list;
1433                 hdr       = &test->tes_hdr;
1434         }
1435
1436         transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY;
1437
1438         rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr,
1439                                      lstcon_batrpc_condition, &trans);
1440         if (rc != 0) {
1441                 CERROR("Can't create transaction: %d\n", rc);
1442                 return rc;
1443         }
1444
1445         lstcon_rpc_trans_postwait(trans, timeout);
1446
1447         if (testidx == 0 && /* query a batch, not a test */
1448             lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) == 0 &&
1449             lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0) == 0) {
1450                 /* all RPCs finished, and no active test */
1451                 batch->bat_state = LST_BATCH_IDLE;
1452         }
1453
1454         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1455                                           lstcon_tsbrpc_readent);
1456         lstcon_rpc_trans_destroy(trans);
1457
1458         return rc;
1459 }
1460
1461 static int
1462 lstcon_statrpc_readent(int transop, struct srpc_msg *msg,
1463                        struct lstcon_rpc_ent __user *ent_up)
1464 {
1465         struct srpc_stat_reply *rep = &msg->msg_body.stat_reply;
1466         struct sfw_counters __user *sfwk_stat;
1467         struct srpc_counters __user *srpc_stat;
1468         struct lnet_counters_common __user *lnet_stat;
1469
1470         if (rep->str_status != 0)
1471                 return 0;
1472
1473         sfwk_stat = (struct sfw_counters __user *)&ent_up->rpe_payload[0];
1474         srpc_stat = (struct srpc_counters __user *)
1475                 ((char __user *)sfwk_stat + sizeof(*sfwk_stat));
1476         lnet_stat = (struct lnet_counters_common __user *)
1477                 ((char __user *)srpc_stat + sizeof(*srpc_stat));
1478
1479         if (copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) ||
1480             copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) ||
1481             copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat)))
1482                 return -EFAULT;
1483
1484         return 0;
1485 }
1486
1487 static int
1488 lstcon_ndlist_stat(struct list_head *ndlist,
1489                    int timeout, struct list_head __user *result_up)
1490 {
1491         LIST_HEAD(head);
1492         struct lstcon_rpc_trans *trans;
1493         int rc;
1494
1495         rc = lstcon_rpc_trans_ndlist(ndlist, &head,
1496                                      LST_TRANS_STATQRY, NULL, NULL, &trans);
1497         if (rc != 0) {
1498                 CERROR("Can't create transaction: %d\n", rc);
1499                 return rc;
1500         }
1501
1502         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1503
1504         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1505                                           lstcon_statrpc_readent);
1506         lstcon_rpc_trans_destroy(trans);
1507
1508         return rc;
1509 }
1510
1511 int
1512 lstcon_group_stat(char *grp_name, int timeout,
1513                   struct list_head __user *result_up)
1514 {
1515         struct lstcon_group *grp;
1516         int rc;
1517
1518         rc = lstcon_group_find(grp_name, &grp);
1519         if (rc != 0) {
1520                 CDEBUG(D_NET, "Can't find group %s\n", grp_name);
1521                 return rc;
1522         }
1523
1524         rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up);
1525
1526         lstcon_group_decref(grp);
1527
1528         return rc;
1529 }
1530
1531 int
1532 lstcon_nodes_stat(int count, struct lnet_process_id __user *ids_up,
1533                   int timeout, struct list_head __user *result_up)
1534 {
1535         struct lstcon_ndlink *ndl;
1536         struct lstcon_group *tmp;
1537         struct lnet_process_id id;
1538         int i;
1539         int rc;
1540
1541         rc = lstcon_group_alloc(NULL, &tmp);
1542         if (rc != 0) {
1543                 CERROR("Out of memory\n");
1544                 return -ENOMEM;
1545         }
1546
1547         for (i = 0 ; i < count; i++) {
1548                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1549                         rc = -EFAULT;
1550                         break;
1551                 }
1552
1553                 /* add to tmp group */
1554                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2);
1555                 if (rc != 0) {
1556                         CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET,
1557                                "Failed to find or create %s: %d\n",
1558                                libcfs_id2str(id), rc);
1559                         break;
1560                 }
1561         }
1562
1563         if (rc != 0) {
1564                 lstcon_group_decref(tmp);
1565                 return rc;
1566         }
1567
1568         rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up);
1569
1570         lstcon_group_decref(tmp);
1571
1572         return rc;
1573 }
1574
1575 static int
1576 lstcon_debug_ndlist(struct list_head *ndlist,
1577                     struct list_head *translist,
1578                     int timeout, struct list_head __user *result_up)
1579 {
1580         struct lstcon_rpc_trans *trans;
1581         int rc;
1582
1583         rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY,
1584                                      NULL, lstcon_sesrpc_condition, &trans);
1585         if (rc != 0) {
1586                 CERROR("Can't create transaction: %d\n", rc);
1587                 return rc;
1588         }
1589
1590         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1591
1592         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1593                                           lstcon_sesrpc_readent);
1594         lstcon_rpc_trans_destroy(trans);
1595
1596         return rc;
1597 }
1598
1599 int
1600 lstcon_session_debug(int timeout, struct list_head __user *result_up)
1601 {
1602         return lstcon_debug_ndlist(&console_session.ses_ndl_list,
1603                                    NULL, timeout, result_up);
1604 }
1605
1606 int
1607 lstcon_batch_debug(int timeout, char *name,
1608                    int client, struct list_head __user *result_up)
1609 {
1610         struct lstcon_batch *bat;
1611         int rc;
1612
1613         rc = lstcon_batch_find(name, &bat);
1614         if (rc != 0)
1615                 return -ENOENT;
1616
1617         rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list :
1618                                           &bat->bat_srv_list,
1619                                  NULL, timeout, result_up);
1620
1621         return rc;
1622 }
1623
1624 int
1625 lstcon_group_debug(int timeout, char *name,
1626                    struct list_head __user *result_up)
1627 {
1628         struct lstcon_group *grp;
1629         int rc;
1630
1631         rc = lstcon_group_find(name, &grp);
1632         if (rc != 0)
1633                 return -ENOENT;
1634
1635         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1636                                  timeout, result_up);
1637         lstcon_group_decref(grp);
1638
1639         return rc;
1640 }
1641
1642 int
1643 lstcon_nodes_debug(int timeout, int count,
1644                    struct lnet_process_id __user *ids_up,
1645                    struct list_head __user *result_up)
1646 {
1647         struct lnet_process_id id;
1648         struct lstcon_ndlink *ndl;
1649         struct lstcon_group *grp;
1650         int i;
1651         int rc;
1652
1653         rc = lstcon_group_alloc(NULL, &grp);
1654         if (rc != 0) {
1655                 CDEBUG(D_NET, "Out of memory\n");
1656                 return rc;
1657         }
1658
1659         for (i = 0; i < count; i++) {
1660                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1661                         rc = -EFAULT;
1662                         break;
1663                 }
1664
1665                 /* node is added to tmp group */
1666                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 1);
1667                 if (rc != 0) {
1668                         CERROR("Can't create node link\n");
1669                         break;
1670                 }
1671         }
1672
1673         if (rc != 0) {
1674                 lstcon_group_decref(grp);
1675                 return rc;
1676         }
1677
1678         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1679                                  timeout, result_up);
1680
1681         lstcon_group_decref(grp);
1682
1683         return rc;
1684 }
1685
1686 int
1687 lstcon_session_match(struct lst_sid sid)
1688 {
1689         return (console_session.ses_id.ses_nid   == sid.ses_nid &&
1690                 console_session.ses_id.ses_stamp == sid.ses_stamp) ?  1: 0;
1691 }
1692
1693 static void
1694 lstcon_new_session_id(struct lst_sid *sid)
1695 {
1696         struct lnet_process_id id;
1697
1698         LASSERT(console_session.ses_state == LST_SESSION_NONE);
1699
1700         LNetGetId(1, &id);
1701         sid->ses_nid = id.nid;
1702         sid->ses_stamp = div_u64(ktime_get_ns(), NSEC_PER_MSEC);
1703 }
1704
1705 int
1706 lstcon_session_new(char *name, int key, unsigned feats,
1707                    int timeout, int force, struct lst_sid __user *sid_up)
1708 {
1709         int     rc = 0;
1710         int     i;
1711
1712         if (console_session.ses_state != LST_SESSION_NONE) {
1713                 /* session exists */
1714                 if (!force) {
1715                         CNETERR("Session %s already exists\n",
1716                                 console_session.ses_name);
1717                         return -EEXIST;
1718                 }
1719
1720                 rc = lstcon_session_end();
1721
1722                 /* lstcon_session_end() only return local error */
1723                 if  (rc != 0)
1724                         return rc;
1725         }
1726
1727         if ((feats & ~LST_FEATS_MASK) != 0) {
1728                 CNETERR("Unknown session features %x\n",
1729                         (feats & ~LST_FEATS_MASK));
1730                 return -EINVAL;
1731         }
1732
1733         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
1734                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
1735
1736         lstcon_new_session_id(&console_session.ses_id);
1737
1738         console_session.ses_key     = key;
1739         console_session.ses_state   = LST_SESSION_ACTIVE;
1740         console_session.ses_force   = !!force;
1741         console_session.ses_features = feats;
1742         console_session.ses_feats_updated = 0;
1743         console_session.ses_timeout = (timeout <= 0) ?
1744                                       LST_CONSOLE_TIMEOUT : timeout;
1745
1746         if (strlen(name) > sizeof(console_session.ses_name)-1)
1747                 return -E2BIG;
1748         strlcpy(console_session.ses_name, name,
1749                 sizeof(console_session.ses_name));
1750
1751         rc = lstcon_batch_add(LST_DEFAULT_BATCH);
1752         if (rc != 0)
1753                 return rc;
1754
1755         rc = lstcon_rpc_pinger_start();
1756         if (rc != 0) {
1757                 struct lstcon_batch *bat = NULL;
1758
1759                 lstcon_batch_find(LST_DEFAULT_BATCH, &bat);
1760                 lstcon_batch_destroy(bat);
1761
1762                 return rc;
1763         }
1764
1765         if (copy_to_user(sid_up, &console_session.ses_id,
1766                              sizeof(struct lst_sid)) == 0)
1767                 return rc;
1768
1769         lstcon_session_end();
1770
1771         return -EFAULT;
1772 }
1773
1774 int
1775 lstcon_session_info(struct lst_sid __user *sid_up, int __user *key_up,
1776                     unsigned __user *featp,
1777                     struct lstcon_ndlist_ent __user *ndinfo_up,
1778                     char __user *name_up, int len)
1779 {
1780         struct lstcon_ndlist_ent *entp;
1781         struct lstcon_ndlink *ndl;
1782         int rc = 0;
1783
1784         if (console_session.ses_state != LST_SESSION_ACTIVE)
1785                 return -ESRCH;
1786
1787         LIBCFS_ALLOC(entp, sizeof(*entp));
1788         if (entp == NULL)
1789                 return -ENOMEM;
1790
1791         list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link)
1792                 LST_NODE_STATE_COUNTER(ndl->ndl_node, entp);
1793
1794         if (copy_to_user(sid_up, &console_session.ses_id,
1795                          sizeof(struct lst_sid)) ||
1796             copy_to_user(key_up, &console_session.ses_key,
1797                              sizeof(*key_up)) ||
1798             copy_to_user(featp, &console_session.ses_features,
1799                              sizeof(*featp)) ||
1800             copy_to_user(ndinfo_up, entp, sizeof(*entp)) ||
1801             copy_to_user(name_up, console_session.ses_name, len))
1802                 rc = -EFAULT;
1803
1804         LIBCFS_FREE(entp, sizeof(*entp));
1805
1806         return rc;
1807 }
1808
1809 int
1810 lstcon_session_end(void)
1811 {
1812         struct lstcon_rpc_trans *trans;
1813         struct lstcon_group *grp;
1814         struct lstcon_batch *bat;
1815         int rc = 0;
1816
1817         LASSERT (console_session.ses_state == LST_SESSION_ACTIVE);
1818
1819         rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list,
1820                                      NULL, LST_TRANS_SESEND, NULL,
1821                                      lstcon_sesrpc_condition, &trans);
1822         if (rc != 0) {
1823                 CERROR("Can't create transaction: %d\n", rc);
1824                 return rc;
1825         }
1826
1827         console_session.ses_shutdown = 1;
1828
1829         lstcon_rpc_pinger_stop();
1830
1831         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1832
1833         lstcon_rpc_trans_destroy(trans);
1834         /* User can do nothing even rpc failed, so go on */
1835
1836         /* waiting for orphan rpcs to die */
1837         lstcon_rpc_cleanup_wait();
1838
1839         console_session.ses_id    = LST_INVALID_SID;
1840         console_session.ses_state = LST_SESSION_NONE;
1841         console_session.ses_key   = 0;
1842         console_session.ses_force = 0;
1843         console_session.ses_feats_updated = 0;
1844
1845         /* destroy all batches */
1846         while (!list_empty(&console_session.ses_bat_list)) {
1847                 bat = list_entry(console_session.ses_bat_list.next,
1848                                  struct lstcon_batch, bat_link);
1849
1850                 lstcon_batch_destroy(bat);
1851         }
1852
1853         /* destroy all groups */
1854         while (!list_empty(&console_session.ses_grp_list)) {
1855                 grp = list_entry(console_session.ses_grp_list.next,
1856                                  struct lstcon_group, grp_link);
1857                 LASSERT(grp->grp_ref == 1);
1858
1859                 lstcon_group_decref(grp);
1860         }
1861
1862         /* all nodes should be released */
1863         LASSERT(list_empty(&console_session.ses_ndl_list));
1864
1865         console_session.ses_shutdown = 0;
1866         console_session.ses_expired  = 0;
1867
1868         return rc;
1869 }
1870
1871 int
1872 lstcon_session_feats_check(unsigned feats)
1873 {
1874         int rc = 0;
1875
1876         if ((feats & ~LST_FEATS_MASK) != 0) {
1877                 CERROR("Can't support these features: %x\n",
1878                        (feats & ~LST_FEATS_MASK));
1879                 return -EPROTO;
1880         }
1881
1882         spin_lock(&console_session.ses_rpc_lock);
1883
1884         if (!console_session.ses_feats_updated) {
1885                 console_session.ses_feats_updated = 1;
1886                 console_session.ses_features = feats;
1887         }
1888
1889         if (console_session.ses_features != feats)
1890                 rc = -EPROTO;
1891
1892         spin_unlock(&console_session.ses_rpc_lock);
1893
1894         if (rc != 0) {
1895                 CERROR("remote features %x do not match with "
1896                        "session features %x of console\n",
1897                        feats, console_session.ses_features);
1898         }
1899
1900         return rc;
1901 }
1902
1903 static int
1904 lstcon_acceptor_handle(struct srpc_server_rpc *rpc)
1905 {
1906         struct srpc_msg *rep = &rpc->srpc_replymsg;
1907         struct srpc_msg *req = &rpc->srpc_reqstbuf->buf_msg;
1908         struct srpc_join_reqst *jreq = &req->msg_body.join_reqst;
1909         struct srpc_join_reply *jrep = &rep->msg_body.join_reply;
1910         struct lstcon_group *grp = NULL;
1911         struct lstcon_ndlink *ndl;
1912         int rc = 0;
1913
1914         sfw_unpack_message(req);
1915
1916         mutex_lock(&console_session.ses_mutex);
1917
1918         jrep->join_sid = console_session.ses_id;
1919
1920         if (console_session.ses_id.ses_nid == LNET_NID_ANY) {
1921                 jrep->join_status = ESRCH;
1922                 goto out;
1923         }
1924
1925         if (lstcon_session_feats_check(req->msg_ses_feats) != 0) {
1926                 jrep->join_status = EPROTO;
1927                 goto out;
1928         }
1929
1930         if (jreq->join_sid.ses_nid != LNET_NID_ANY &&
1931              !lstcon_session_match(jreq->join_sid)) {
1932                 jrep->join_status = EBUSY;
1933                 goto out;
1934         }
1935
1936         if (lstcon_group_find(jreq->join_group, &grp) != 0) {
1937                 rc = lstcon_group_alloc(jreq->join_group, &grp);
1938                 if (rc != 0) {
1939                         CERROR("Out of memory\n");
1940                         goto out;
1941                 }
1942
1943                 list_add_tail(&grp->grp_link,
1944                               &console_session.ses_grp_list);
1945                 lstcon_group_addref(grp);
1946         }
1947
1948         if (grp->grp_ref > 2) {
1949                 /* Group in using */
1950                 jrep->join_status = EBUSY;
1951                 goto out;
1952         }
1953
1954         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0);
1955         if (rc == 0) {
1956                 jrep->join_status = EEXIST;
1957                 goto out;
1958         }
1959
1960         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1);
1961         if (rc != 0) {
1962                 CERROR("Out of memory\n");
1963                 goto out;
1964         }
1965
1966         ndl->ndl_node->nd_state   = LST_NODE_ACTIVE;
1967         ndl->ndl_node->nd_timeout = console_session.ses_timeout;
1968
1969         if (grp->grp_userland == 0)
1970                 grp->grp_userland = 1;
1971
1972         strlcpy(jrep->join_session, console_session.ses_name,
1973                 sizeof(jrep->join_session));
1974         jrep->join_timeout = console_session.ses_timeout;
1975         jrep->join_status  = 0;
1976
1977 out:
1978         rep->msg_ses_feats = console_session.ses_features;
1979         if (grp != NULL)
1980                 lstcon_group_decref(grp);
1981
1982         mutex_unlock(&console_session.ses_mutex);
1983
1984         return rc;
1985 }
1986
1987 static struct srpc_service lstcon_acceptor_service;
1988
1989 static void lstcon_init_acceptor_service(void)
1990 {
1991         /* initialize selftest console acceptor service table */
1992         lstcon_acceptor_service.sv_name    = "join session";
1993         lstcon_acceptor_service.sv_handler = lstcon_acceptor_handle;
1994         lstcon_acceptor_service.sv_id      = SRPC_SERVICE_JOIN;
1995         lstcon_acceptor_service.sv_wi_total = SFW_FRWK_WI_MAX;
1996 }
1997
1998 static struct notifier_block lstcon_ioctl_handler = {
1999         .notifier_call = lstcon_ioctl_entry,
2000 };
2001
2002 /* initialize console */
2003 int
2004 lstcon_console_init(void)
2005 {
2006         int     i;
2007         int     rc;
2008
2009         console_session.ses_id              = LST_INVALID_SID;
2010         console_session.ses_state           = LST_SESSION_NONE;
2011         console_session.ses_timeout         = 0;
2012         console_session.ses_force           = 0;
2013         console_session.ses_expired         = 0;
2014         console_session.ses_feats_updated   = 0;
2015         console_session.ses_features        = LST_FEATS_MASK;
2016         console_session.ses_laststamp = ktime_get_real_seconds();
2017
2018         mutex_init(&console_session.ses_mutex);
2019
2020         INIT_LIST_HEAD(&console_session.ses_ndl_list);
2021         INIT_LIST_HEAD(&console_session.ses_grp_list);
2022         INIT_LIST_HEAD(&console_session.ses_bat_list);
2023         INIT_LIST_HEAD(&console_session.ses_trans_list);
2024
2025         CFS_ALLOC_PTR_ARRAY(console_session.ses_ndl_hash,
2026                                LST_GLOBAL_HASHSIZE);
2027         if (console_session.ses_ndl_hash == NULL)
2028                 return -ENOMEM;
2029
2030         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
2031                 INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]);
2032
2033
2034         /* initialize acceptor service table */
2035         lstcon_init_acceptor_service();
2036
2037         rc = srpc_add_service(&lstcon_acceptor_service);
2038         LASSERT(rc != -EBUSY);
2039         if (rc != 0) {
2040                 CFS_FREE_PTR_ARRAY(console_session.ses_ndl_hash,
2041                                    LST_GLOBAL_HASHSIZE);
2042                 return rc;
2043         }
2044
2045         rc = srpc_service_add_buffers(&lstcon_acceptor_service,
2046                                       lstcon_acceptor_service.sv_wi_total);
2047         if (rc != 0) {
2048                 rc = -ENOMEM;
2049                 goto out;
2050         }
2051
2052         rc = blocking_notifier_chain_register(&libcfs_ioctl_list,
2053                                               &lstcon_ioctl_handler);
2054         if (rc == 0) {
2055                 lstcon_rpc_module_init();
2056                 return 0;
2057         }
2058
2059 out:
2060         srpc_shutdown_service(&lstcon_acceptor_service);
2061         srpc_remove_service(&lstcon_acceptor_service);
2062
2063         CFS_FREE_PTR_ARRAY(console_session.ses_ndl_hash, LST_GLOBAL_HASHSIZE);
2064
2065         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2066
2067         return rc;
2068 }
2069
2070 int
2071 lstcon_console_fini(void)
2072 {
2073         int i;
2074
2075         blocking_notifier_chain_unregister(&libcfs_ioctl_list,
2076                                            &lstcon_ioctl_handler);
2077
2078         mutex_lock(&console_session.ses_mutex);
2079
2080         srpc_shutdown_service(&lstcon_acceptor_service);
2081         srpc_remove_service(&lstcon_acceptor_service);
2082
2083         if (console_session.ses_state != LST_SESSION_NONE)
2084                 lstcon_session_end();
2085
2086         lstcon_rpc_module_fini();
2087
2088         mutex_unlock(&console_session.ses_mutex);
2089
2090         LASSERT(list_empty(&console_session.ses_ndl_list));
2091         LASSERT(list_empty(&console_session.ses_grp_list));
2092         LASSERT(list_empty(&console_session.ses_bat_list));
2093         LASSERT(list_empty(&console_session.ses_trans_list));
2094
2095         for (i = 0; i < LST_NODE_HASHSIZE; i++)
2096                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
2097
2098         CFS_FREE_PTR_ARRAY(console_session.ses_ndl_hash,
2099                            LST_GLOBAL_HASHSIZE);
2100
2101         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2102
2103         return 0;
2104 }
2105