lnet/selftest/console.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2012, 2014, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  * Lustre is a trademark of Sun Microsystems, Inc.
  31  *
  32  * lnet/selftest/conctl.c
  33  *
  34  * Infrastructure of LST console
  35  *
  36  * Author: Liang Zhen <liangzhen@clusterfs.com>
  37  */
  38
  39 #include <libcfs/libcfs.h>
  40 #include <lnet/lib-lnet.h>
  41 #include "console.h"
  42 #include "conrpc.h"
  43
  44 #define LST_NODE_STATE_COUNTER(nd, p)                   \
  45 do {                                                    \
  46         if ((nd)->nd_state == LST_NODE_ACTIVE)          \
  47                 (p)->nle_nactive ++;                    \
  48         else if ((nd)->nd_state == LST_NODE_BUSY)       \
  49                 (p)->nle_nbusy ++;                      \
  50         else if ((nd)->nd_state == LST_NODE_DOWN)       \
  51                 (p)->nle_ndown ++;                      \
  52         else                                            \
  53                 (p)->nle_nunknown ++;                   \
  54         (p)->nle_nnode ++;                              \
  55 } while (0)
  56
  57 struct lstcon_session console_session;
  58
  59 static void
  60 lstcon_node_get(struct lstcon_node *nd)
  61 {
  62         LASSERT (nd->nd_ref >= 1);
  63
  64         nd->nd_ref++;
  65 }
  66
  67 static int
  68 lstcon_node_find(struct lnet_process_id id, struct lstcon_node **ndpp,
  69                  int create)
  70 {
  71         struct lstcon_ndlink *ndl;
  72         unsigned int idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE;
  73
  74         LASSERT(id.nid != LNET_NID_ANY);
  75
  76         list_for_each_entry(ndl, &console_session.ses_ndl_hash[idx],
  77                             ndl_hlink) {
  78                 if (ndl->ndl_node->nd_id.nid != id.nid ||
  79                     ndl->ndl_node->nd_id.pid != id.pid)
  80                         continue;
  81
  82                 lstcon_node_get(ndl->ndl_node);
  83                 *ndpp = ndl->ndl_node;
  84                 return 0;
  85         }
  86
  87         if (!create)
  88                 return -ENOENT;
  89
  90         LIBCFS_ALLOC(*ndpp, sizeof(**ndpp) + sizeof(*ndl));
  91         if (*ndpp == NULL)
  92                 return -ENOMEM;
  93
  94         ndl = (struct lstcon_ndlink *)(*ndpp + 1);
  95
  96         ndl->ndl_node = *ndpp;
  97
  98         ndl->ndl_node->nd_ref   = 1;
  99         ndl->ndl_node->nd_id    = id;
 100         ndl->ndl_node->nd_stamp = ktime_get();
 101         ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
 102         ndl->ndl_node->nd_timeout = 0;
 103         memset(&ndl->ndl_node->nd_ping, 0, sizeof(ndl->ndl_node->nd_ping));
 104
 105         /* queued in global hash & list, no refcount is taken by
 106          * global hash & list, if caller release his refcount,
 107          * node will be released */
 108         list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]);
 109         list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list);
 110
 111         return 0;
 112 }
 113
 114 static void
 115 lstcon_node_put(struct lstcon_node *nd)
 116 {
 117         struct lstcon_ndlink *ndl;
 118
 119         LASSERT(nd->nd_ref > 0);
 120
 121         if (--nd->nd_ref > 0)
 122                 return;
 123
 124         ndl = (struct lstcon_ndlink *)(nd + 1);
 125
 126         LASSERT(!list_empty(&ndl->ndl_link));
 127         LASSERT(!list_empty(&ndl->ndl_hlink));
 128
 129         /* remove from session */
 130         list_del(&ndl->ndl_link);
 131         list_del(&ndl->ndl_hlink);
 132
 133         LIBCFS_FREE(nd, sizeof(*nd) + sizeof(*ndl));
 134 }
 135
 136 static int
 137 lstcon_ndlink_find(struct list_head *hash, struct lnet_process_id id,
 138                    struct lstcon_ndlink **ndlpp, int create)
 139 {
 140         unsigned int idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
 141         struct lstcon_ndlink *ndl;
 142         struct lstcon_node *nd;
 143         int rc;
 144
 145         if (id.nid == LNET_NID_ANY)
 146                 return -EINVAL;
 147
 148         /* search in hash */
 149         list_for_each_entry(ndl, &hash[idx], ndl_hlink) {
 150                 if (ndl->ndl_node->nd_id.nid != id.nid ||
 151                     ndl->ndl_node->nd_id.pid != id.pid)
 152                         continue;
 153
 154                 *ndlpp = ndl;
 155                 return 0;
 156         }
 157
 158         if (create == 0)
 159                 return -ENOENT;
 160
 161         /* find or create in session hash */
 162         rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0);
 163         if (rc != 0)
 164                 return rc;
 165
 166         LIBCFS_ALLOC(ndl, sizeof(*ndl));
 167         if (ndl == NULL) {
 168                 lstcon_node_put(nd);
 169                 return -ENOMEM;
 170         }
 171
 172         *ndlpp = ndl;
 173
 174         ndl->ndl_node = nd;
 175         INIT_LIST_HEAD(&ndl->ndl_link);
 176         list_add_tail(&ndl->ndl_hlink, &hash[idx]);
 177
 178         return 0;
 179 }
 180
 181 static void
 182 lstcon_ndlink_release(struct lstcon_ndlink *ndl)
 183 {
 184         LASSERT(list_empty(&ndl->ndl_link));
 185         LASSERT(!list_empty(&ndl->ndl_hlink));
 186
 187         list_del(&ndl->ndl_hlink); /* delete from hash */
 188         lstcon_node_put(ndl->ndl_node);
 189
 190         LIBCFS_FREE(ndl, sizeof(*ndl));
 191 }
 192
 193 static int
 194 lstcon_group_alloc(char *name, struct lstcon_group **grpp)
 195 {
 196         struct lstcon_group *grp;
 197         int i;
 198
 199         LIBCFS_ALLOC(grp, offsetof(struct lstcon_group,
 200                                    grp_ndl_hash[LST_NODE_HASHSIZE]));
 201         if (grp == NULL)
 202                 return -ENOMEM;
 203
 204         grp->grp_ref = 1;
 205         if (name != NULL) {
 206                 if (strlen(name) > sizeof(grp->grp_name)-1) {
 207                         LIBCFS_FREE(grp, offsetof(struct lstcon_group,
 208                                           grp_ndl_hash[LST_NODE_HASHSIZE]));
 209                         return -E2BIG;
 210                 }
 211                 strncpy(grp->grp_name, name, sizeof(grp->grp_name));
 212         }
 213
 214         INIT_LIST_HEAD(&grp->grp_link);
 215         INIT_LIST_HEAD(&grp->grp_ndl_list);
 216         INIT_LIST_HEAD(&grp->grp_trans_list);
 217
 218         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 219                 INIT_LIST_HEAD(&grp->grp_ndl_hash[i]);
 220
 221         *grpp = grp;
 222
 223         return 0;
 224 }
 225
 226 static void
 227 lstcon_group_addref(struct lstcon_group *grp)
 228 {
 229         grp->grp_ref++;
 230 }
 231
 232 static void lstcon_group_ndlink_release(struct lstcon_group *,
 233                                         struct lstcon_ndlink *);
 234
 235 static void
 236 lstcon_group_drain(struct lstcon_group *grp, int keep)
 237 {
 238         struct lstcon_ndlink *ndl;
 239         struct lstcon_ndlink *tmp;
 240
 241         list_for_each_entry_safe(ndl, tmp, &grp->grp_ndl_list, ndl_link) {
 242                 if ((ndl->ndl_node->nd_state & keep) == 0)
 243                         lstcon_group_ndlink_release(grp, ndl);
 244         }
 245 }
 246
 247 static void
 248 lstcon_group_decref(struct lstcon_group *grp)
 249 {
 250         int i;
 251
 252         if (--grp->grp_ref > 0)
 253                 return;
 254
 255         if (!list_empty(&grp->grp_link))
 256                 list_del(&grp->grp_link);
 257
 258         lstcon_group_drain(grp, 0);
 259
 260         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 261                 LASSERT(list_empty(&grp->grp_ndl_hash[i]));
 262
 263         LIBCFS_FREE(grp, offsetof(struct lstcon_group,
 264                                   grp_ndl_hash[LST_NODE_HASHSIZE]));
 265 }
 266
 267 static int
 268 lstcon_group_find(const char *name, struct lstcon_group **grpp)
 269 {
 270         struct lstcon_group *grp;
 271
 272         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 273                 if (strncmp(grp->grp_name, name, LST_NAME_SIZE) != 0)
 274                         continue;
 275
 276                 lstcon_group_addref(grp);  /* +1 ref for caller */
 277                 *grpp = grp;
 278                 return 0;
 279         }
 280
 281         return -ENOENT;
 282 }
 283
 284 static int
 285 lstcon_group_ndlink_find(struct lstcon_group *grp, struct lnet_process_id id,
 286                          struct lstcon_ndlink **ndlpp, int create)
 287 {
 288         int rc;
 289
 290         rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create);
 291         if (rc != 0)
 292                 return rc;
 293
 294         if (!list_empty(&(*ndlpp)->ndl_link))
 295                 return 0;
 296
 297         list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list);
 298         grp->grp_nnode++;
 299
 300         return 0;
 301 }
 302
 303 static void
 304 lstcon_group_ndlink_release(struct lstcon_group *grp, struct lstcon_ndlink *ndl)
 305 {
 306         list_del_init(&ndl->ndl_link);
 307         lstcon_ndlink_release(ndl);
 308         grp->grp_nnode--;
 309 }
 310
 311 static void
 312 lstcon_group_ndlink_move(struct lstcon_group *old,
 313                          struct lstcon_group *new, struct lstcon_ndlink *ndl)
 314 {
 315         unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) %
 316                                         LST_NODE_HASHSIZE;
 317
 318         old->grp_nnode--;
 319
 320         list_move_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]);
 321         list_move_tail(&ndl->ndl_link, &new->grp_ndl_list);
 322         new->grp_nnode++;
 323 }
 324
 325 static void
 326 lstcon_group_move(struct lstcon_group *old, struct lstcon_group *new)
 327 {
 328         struct lstcon_ndlink *ndl;
 329
 330         while (!list_empty(&old->grp_ndl_list)) {
 331                 ndl = list_entry(old->grp_ndl_list.next,
 332                                  struct lstcon_ndlink, ndl_link);
 333                 lstcon_group_ndlink_move(old, new, ndl);
 334         }
 335 }
 336
 337 static int
 338 lstcon_sesrpc_condition(int transop, struct lstcon_node *nd, void *arg)
 339 {
 340         struct lstcon_group *grp = arg;
 341
 342         switch (transop) {
 343         case LST_TRANS_SESNEW:
 344                 if (nd->nd_state == LST_NODE_ACTIVE)
 345                         return 0;
 346                 break;
 347
 348         case LST_TRANS_SESEND:
 349                 if (nd->nd_state != LST_NODE_ACTIVE)
 350                         return 0;
 351
 352                 if (grp != NULL && nd->nd_ref > 1)
 353                         return 0;
 354                 break;
 355
 356         case LST_TRANS_SESQRY:
 357                 break;
 358
 359         default:
 360                 LBUG();
 361         }
 362
 363         return 1;
 364 }
 365
 366 static int
 367 lstcon_sesrpc_readent(int transop, struct srpc_msg *msg,
 368                       struct lstcon_rpc_ent __user *ent_up)
 369 {
 370         struct srpc_debug_reply *rep;
 371
 372         switch (transop) {
 373         case LST_TRANS_SESNEW:
 374         case LST_TRANS_SESEND:
 375                 return 0;
 376
 377         case LST_TRANS_SESQRY:
 378                 rep = &msg->msg_body.dbg_reply;
 379
 380                 if (copy_to_user(&ent_up->rpe_priv[0],
 381                                      &rep->dbg_timeout, sizeof(int)) ||
 382                     copy_to_user(&ent_up->rpe_payload[0],
 383                                      &rep->dbg_name, LST_NAME_SIZE))
 384                         return -EFAULT;
 385
 386                 return 0;
 387
 388         default:
 389                 LBUG();
 390         }
 391
 392         return 0;
 393 }
 394
 395 static int
 396 lstcon_group_nodes_add(struct lstcon_group *grp,
 397                        int count, struct lnet_process_id __user *ids_up,
 398                        unsigned int *featp,
 399                        struct list_head __user *result_up)
 400 {
 401         struct lstcon_rpc_trans *trans;
 402         struct lstcon_ndlink *ndl;
 403         struct lstcon_group *tmp;
 404         struct lnet_process_id id;
 405         int i;
 406         int rc;
 407
 408         rc = lstcon_group_alloc(NULL, &tmp);
 409         if (rc != 0) {
 410                 CERROR("Out of memory\n");
 411                 return -ENOMEM;
 412         }
 413
 414         for (i = 0 ; i < count; i++) {
 415                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 416                         rc = -EFAULT;
 417                         break;
 418                 }
 419
 420                 /* skip if it's in this group already */
 421                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 0);
 422                 if (rc == 0)
 423                         continue;
 424
 425                 /* add to tmp group */
 426                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1);
 427                 if (rc != 0) {
 428                         CERROR("Can't create ndlink, out of memory\n");
 429                         break;
 430                 }
 431         }
 432
 433         if (rc != 0) {
 434                 lstcon_group_decref(tmp);
 435                 return rc;
 436         }
 437
 438         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 439                                      &tmp->grp_trans_list, LST_TRANS_SESNEW,
 440                                      tmp, lstcon_sesrpc_condition, &trans);
 441         if (rc != 0) {
 442                 CERROR("Can't create transaction: %d\n", rc);
 443                 lstcon_group_decref(tmp);
 444                 return rc;
 445         }
 446
 447         /* post all RPCs */
 448         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 449
 450         rc = lstcon_rpc_trans_interpreter(trans, result_up,
 451                                           lstcon_sesrpc_readent);
 452         *featp = trans->tas_features;
 453
 454         /* destroy all RPGs */
 455         lstcon_rpc_trans_destroy(trans);
 456
 457         lstcon_group_move(tmp, grp);
 458         lstcon_group_decref(tmp);
 459
 460         return rc;
 461 }
 462
 463 static int
 464 lstcon_group_nodes_remove(struct lstcon_group *grp,
 465                           int count, struct lnet_process_id __user *ids_up,
 466                           struct list_head __user *result_up)
 467 {
 468         struct lstcon_rpc_trans *trans;
 469         struct lstcon_ndlink *ndl;
 470         struct lstcon_group *tmp;
 471         struct lnet_process_id id;
 472         int rc;
 473         int i;
 474
 475         /* End session and remove node from the group */
 476
 477         rc = lstcon_group_alloc(NULL, &tmp);
 478         if (rc != 0) {
 479                 CERROR("Out of memory\n");
 480                 return -ENOMEM;
 481         }
 482
 483         for (i = 0; i < count; i++) {
 484                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 485                         rc = -EFAULT;
 486                         goto error;
 487                 }
 488
 489                 /* move node to tmp group */
 490                 if (lstcon_group_ndlink_find(grp, id, &ndl, 0) == 0)
 491                         lstcon_group_ndlink_move(grp, tmp, ndl);
 492         }
 493
 494         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 495                                      &tmp->grp_trans_list, LST_TRANS_SESEND,
 496                                      tmp, lstcon_sesrpc_condition, &trans);
 497         if (rc != 0) {
 498                 CERROR("Can't create transaction: %d\n", rc);
 499                 goto error;
 500         }
 501
 502         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 503
 504         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 505
 506         lstcon_rpc_trans_destroy(trans);
 507         /* release nodes anyway, because we can't rollback status */
 508         lstcon_group_decref(tmp);
 509
 510         return rc;
 511 error:
 512         lstcon_group_move(tmp, grp);
 513         lstcon_group_decref(tmp);
 514
 515         return rc;
 516 }
 517
 518 int
 519 lstcon_group_add(char *name)
 520 {
 521         struct lstcon_group *grp;
 522         int rc;
 523
 524         rc = (lstcon_group_find(name, &grp) == 0)? -EEXIST: 0;
 525         if (rc != 0) {
 526                 /* find a group with same name */
 527                 lstcon_group_decref(grp);
 528                 return rc;
 529         }
 530
 531         rc = lstcon_group_alloc(name, &grp);
 532         if (rc != 0) {
 533                 CERROR("Can't allocate descriptor for group %s\n", name);
 534                 return -ENOMEM;
 535         }
 536
 537         list_add_tail(&grp->grp_link, &console_session.ses_grp_list);
 538
 539         return rc;
 540 }
 541
 542 int
 543 lstcon_nodes_add(char *name, int count, struct lnet_process_id __user *ids_up,
 544                  unsigned *featp, struct list_head __user *result_up)
 545 {
 546         struct lstcon_group         *grp;
 547         int                     rc;
 548
 549         LASSERT (count > 0);
 550         LASSERT (ids_up != NULL);
 551
 552         rc = lstcon_group_find(name, &grp);
 553         if (rc != 0) {
 554                 CDEBUG(D_NET, "Can't find group %s\n", name);
 555                 return rc;
 556         }
 557
 558         if (grp->grp_ref > 2) {
 559                 /* referred by other threads or test */
 560                 CDEBUG(D_NET, "Group %s is busy\n", name);
 561                 lstcon_group_decref(grp);
 562
 563                 return -EBUSY;
 564         }
 565
 566         rc = lstcon_group_nodes_add(grp, count, ids_up, featp, result_up);
 567
 568         lstcon_group_decref(grp);
 569
 570         return rc;
 571 }
 572
 573 int
 574 lstcon_group_del(char *name)
 575 {
 576         struct lstcon_rpc_trans *trans;
 577         struct lstcon_group *grp;
 578         int rc;
 579
 580         rc = lstcon_group_find(name, &grp);
 581         if (rc != 0) {
 582                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 583                 return rc;
 584         }
 585
 586         if (grp->grp_ref > 2) {
 587                 /* referred by others threads or test */
 588                 CDEBUG(D_NET, "Group %s is busy\n", name);
 589                 lstcon_group_decref(grp);
 590                 return -EBUSY;
 591         }
 592
 593         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 594                                      &grp->grp_trans_list, LST_TRANS_SESEND,
 595                                      grp, lstcon_sesrpc_condition, &trans);
 596         if (rc != 0) {
 597                 CERROR("Can't create transaction: %d\n", rc);
 598                 lstcon_group_decref(grp);
 599                 return rc;
 600         }
 601
 602         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 603
 604         lstcon_rpc_trans_destroy(trans);
 605
 606         lstcon_group_decref(grp);
 607         /* -ref for session, it's destroyed,
 608          * status can't be rolled back, destroy group anway */
 609         lstcon_group_decref(grp);
 610
 611         return rc;
 612 }
 613
 614 int
 615 lstcon_group_clean(char *name, int args)
 616 {
 617         struct lstcon_group *grp = NULL;
 618         int rc;
 619
 620         rc = lstcon_group_find(name, &grp);
 621         if (rc != 0) {
 622                 CDEBUG(D_NET, "Can't find group %s\n", name);
 623                 return rc;
 624         }
 625
 626         if (grp->grp_ref > 2) {
 627                 /* referred by test */
 628                 CDEBUG(D_NET, "Group %s is busy\n", name);
 629                 lstcon_group_decref(grp);
 630                 return -EBUSY;
 631         }
 632
 633         args = (LST_NODE_ACTIVE | LST_NODE_BUSY |
 634                 LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args;
 635
 636         lstcon_group_drain(grp, args);
 637
 638         lstcon_group_decref(grp);
 639         /* release empty group */
 640         if (list_empty(&grp->grp_ndl_list))
 641                 lstcon_group_decref(grp);
 642
 643         return 0;
 644 }
 645
 646 int
 647 lstcon_nodes_remove(char *name, int count,
 648                     struct lnet_process_id __user *ids_up,
 649                     struct list_head __user *result_up)
 650 {
 651         struct lstcon_group *grp = NULL;
 652         int rc;
 653
 654         rc = lstcon_group_find(name, &grp);
 655         if (rc != 0) {
 656                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 657                 return rc;
 658         }
 659
 660         if (grp->grp_ref > 2) {
 661                 /* referred by test */
 662                 CDEBUG(D_NET, "Group %s is busy\n", name);
 663                 lstcon_group_decref(grp);
 664                 return -EBUSY;
 665         }
 666
 667         rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up);
 668
 669         lstcon_group_decref(grp);
 670         /* release empty group */
 671         if (list_empty(&grp->grp_ndl_list))
 672                 lstcon_group_decref(grp);
 673
 674         return rc;
 675 }
 676
 677 int
 678 lstcon_group_refresh(char *name, struct list_head __user *result_up)
 679 {
 680         struct lstcon_rpc_trans *trans;
 681         struct lstcon_group *grp;
 682         int rc;
 683
 684         rc = lstcon_group_find(name, &grp);
 685         if (rc != 0) {
 686                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 687                 return rc;
 688         }
 689
 690         if (grp->grp_ref > 2) {
 691                 /* referred by test */
 692                 CDEBUG(D_NET, "Group %s is busy\n", name);
 693                 lstcon_group_decref(grp);
 694                 return -EBUSY;
 695         }
 696
 697         /* re-invite all inactive nodes int the group */
 698         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 699                                      &grp->grp_trans_list, LST_TRANS_SESNEW,
 700                                      grp, lstcon_sesrpc_condition, &trans);
 701         if (rc != 0) {
 702                 /* local error, return */
 703                 CDEBUG(D_NET, "Can't create transaction: %d\n", rc);
 704                 lstcon_group_decref(grp);
 705                 return rc;
 706         }
 707
 708         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 709
 710         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 711
 712         lstcon_rpc_trans_destroy(trans);
 713         /* -ref for me */
 714         lstcon_group_decref(grp);
 715
 716         return rc;
 717 }
 718
 719 int
 720 lstcon_group_list(int index, int len, char __user *name_up)
 721 {
 722         struct lstcon_group *grp;
 723
 724         LASSERT(index >= 0);
 725         LASSERT(name_up != NULL);
 726
 727         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 728                 if (index-- == 0) {
 729                         return copy_to_user(name_up, grp->grp_name, len) ?
 730                                             -EFAULT : 0;
 731                 }
 732         }
 733
 734         return -ENOENT;
 735 }
 736
 737 static int
 738 lstcon_nodes_getent(struct list_head *head, int *index_p,
 739                     int *count_p, struct lstcon_node_ent __user *dents_up)
 740 {
 741         struct lstcon_ndlink *ndl;
 742         struct lstcon_node *nd;
 743         int count = 0;
 744         int index = 0;
 745
 746         LASSERT(index_p != NULL && count_p != NULL);
 747         LASSERT(dents_up != NULL);
 748         LASSERT(*index_p >= 0);
 749         LASSERT(*count_p > 0);
 750
 751         list_for_each_entry(ndl, head, ndl_link) {
 752                 if (index++ < *index_p)
 753                         continue;
 754
 755                 if (count >= *count_p)
 756                         break;
 757
 758                 nd = ndl->ndl_node;
 759                 if (copy_to_user(&dents_up[count].nde_id,
 760                                      &nd->nd_id, sizeof(nd->nd_id)) ||
 761                     copy_to_user(&dents_up[count].nde_state,
 762                                      &nd->nd_state, sizeof(nd->nd_state)))
 763                         return -EFAULT;
 764
 765                 count ++;
 766         }
 767
 768         if (index <= *index_p)
 769                 return -ENOENT;
 770
 771         *count_p = count;
 772         *index_p = index;
 773
 774         return 0;
 775 }
 776
 777 int
 778 lstcon_group_info(char *name, struct lstcon_ndlist_ent __user *gents_p,
 779                   int *index_p, int *count_p,
 780                   struct lstcon_node_ent __user *dents_up)
 781 {
 782         struct lstcon_ndlist_ent *gentp;
 783         struct lstcon_group *grp;
 784         struct lstcon_ndlink *ndl;
 785         int rc;
 786
 787         rc = lstcon_group_find(name, &grp);
 788         if (rc != 0) {
 789                 CDEBUG(D_NET, "Can't find group %s\n", name);
 790                 return rc;
 791         }
 792
 793         if (dents_up != NULL) {
 794                 /* verbose query */
 795                 rc = lstcon_nodes_getent(&grp->grp_ndl_list,
 796                                          index_p, count_p, dents_up);
 797                 lstcon_group_decref(grp);
 798
 799                 return rc;
 800         }
 801
 802         /* non-verbose query */
 803         CFS_ALLOC_PTR(gentp);
 804         if (gentp == NULL) {
 805                 CERROR("Can't allocate ndlist_ent\n");
 806                 lstcon_group_decref(grp);
 807
 808                 return -ENOMEM;
 809         }
 810
 811         list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link)
 812                 LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp);
 813
 814         rc = copy_to_user(gents_p, gentp,
 815                           sizeof(struct lstcon_ndlist_ent)) ? -EFAULT : 0;
 816
 817         CFS_FREE_PTR(gentp);
 818
 819         lstcon_group_decref(grp);
 820
 821         return 0;
 822 }
 823
 824 static int
 825 lstcon_batch_find(const char *name, struct lstcon_batch **batpp)
 826 {
 827         struct lstcon_batch *bat;
 828
 829         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 830                 if (strncmp(bat->bat_name, name, LST_NAME_SIZE) == 0) {
 831                         *batpp = bat;
 832                         return 0;
 833                 }
 834         }
 835
 836         return -ENOENT;
 837 }
 838
 839 int
 840 lstcon_batch_add(char *name)
 841 {
 842         struct lstcon_batch *bat;
 843         int i;
 844         int rc;
 845
 846         rc = (lstcon_batch_find(name, &bat) == 0)? -EEXIST: 0;
 847         if (rc != 0) {
 848                 CDEBUG(D_NET, "Batch %s already exists\n", name);
 849                 return rc;
 850         }
 851
 852         LIBCFS_ALLOC(bat, sizeof(*bat));
 853         if (bat == NULL) {
 854                 CERROR("Can't allocate descriptor for batch %s\n", name);
 855                 return -ENOMEM;
 856         }
 857
 858         CFS_ALLOC_PTR_ARRAY(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 859         if (bat->bat_cli_hash == NULL) {
 860                 CERROR("Can't allocate hash for batch %s\n", name);
 861                 LIBCFS_FREE(bat, sizeof(*bat));
 862
 863                 return -ENOMEM;
 864         }
 865
 866         CFS_ALLOC_PTR_ARRAY(bat->bat_srv_hash, LST_NODE_HASHSIZE);
 867         if (bat->bat_srv_hash == NULL) {
 868                 CERROR("Can't allocate hash for batch %s\n", name);
 869                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 870                 LIBCFS_FREE(bat, sizeof(*bat));
 871
 872                 return -ENOMEM;
 873         }
 874
 875         if (strlen(name) > sizeof(bat->bat_name)-1) {
 876                 LIBCFS_FREE(bat->bat_srv_hash, LST_NODE_HASHSIZE);
 877                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 878                 LIBCFS_FREE(bat, sizeof(*bat));
 879                 return -E2BIG;
 880         }
 881         strncpy(bat->bat_name, name, sizeof(bat->bat_name));
 882         bat->bat_hdr.tsb_index = 0;
 883         bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie;
 884
 885         bat->bat_ntest = 0;
 886         bat->bat_state = LST_BATCH_IDLE;
 887
 888         INIT_LIST_HEAD(&bat->bat_cli_list);
 889         INIT_LIST_HEAD(&bat->bat_srv_list);
 890         INIT_LIST_HEAD(&bat->bat_test_list);
 891         INIT_LIST_HEAD(&bat->bat_trans_list);
 892
 893         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 894                 INIT_LIST_HEAD(&bat->bat_cli_hash[i]);
 895                 INIT_LIST_HEAD(&bat->bat_srv_hash[i]);
 896         }
 897
 898         list_add_tail(&bat->bat_link, &console_session.ses_bat_list);
 899
 900         return rc;
 901 }
 902
 903 int
 904 lstcon_batch_list(int index, int len, char __user *name_up)
 905 {
 906         struct lstcon_batch *bat;
 907
 908         LASSERT(name_up != NULL);
 909         LASSERT(index >= 0);
 910
 911         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 912                 if (index-- == 0) {
 913                         return copy_to_user(name_up, bat->bat_name, len) ?
 914                                             -EFAULT : 0;
 915                 }
 916         }
 917
 918         return -ENOENT;
 919 }
 920
 921 int
 922 lstcon_batch_info(char *name, struct lstcon_test_batch_ent __user *ent_up,
 923                   int server, int testidx, int *index_p, int *ndent_p,
 924                   struct lstcon_node_ent __user *dents_up)
 925 {
 926         struct lstcon_test_batch_ent *entp;
 927         struct list_head *clilst;
 928         struct list_head *srvlst;
 929         struct lstcon_test *test = NULL;
 930         struct lstcon_batch *bat;
 931         struct lstcon_ndlink *ndl;
 932         int rc;
 933
 934         rc = lstcon_batch_find(name, &bat);
 935         if (rc != 0) {
 936                 CDEBUG(D_NET, "Can't find batch %s\n", name);
 937                 return -ENOENT;
 938         }
 939
 940         if (testidx > 0) {
 941                 /* query test, test index start from 1 */
 942                 list_for_each_entry(test, &bat->bat_test_list, tes_link) {
 943                         if (testidx-- == 1)
 944                                 break;
 945                 }
 946
 947                 if (testidx > 0) {
 948                         CDEBUG(D_NET, "Can't find specified test in batch\n");
 949                         return -ENOENT;
 950                 }
 951         }
 952
 953         clilst = (test == NULL) ? &bat->bat_cli_list :
 954                                   &test->tes_src_grp->grp_ndl_list;
 955         srvlst = (test == NULL) ? &bat->bat_srv_list :
 956                                   &test->tes_dst_grp->grp_ndl_list;
 957
 958         if (dents_up != NULL) {
 959                 rc = lstcon_nodes_getent((server ? srvlst: clilst),
 960                                          index_p, ndent_p, dents_up);
 961                 return rc;
 962         }
 963
 964         /* non-verbose query */
 965         CFS_ALLOC_PTR(entp);
 966         if (entp == NULL)
 967                 return -ENOMEM;
 968
 969         if (test == NULL) {
 970                 entp->u.tbe_batch.bae_ntest = bat->bat_ntest;
 971                 entp->u.tbe_batch.bae_state = bat->bat_state;
 972
 973         } else {
 974
 975                 entp->u.tbe_test.tse_type   = test->tes_type;
 976                 entp->u.tbe_test.tse_loop   = test->tes_loop;
 977                 entp->u.tbe_test.tse_concur = test->tes_concur;
 978         }
 979
 980         list_for_each_entry(ndl, clilst, ndl_link)
 981                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle);
 982
 983         list_for_each_entry(ndl, srvlst, ndl_link)
 984                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle);
 985
 986         rc = copy_to_user(ent_up, entp,
 987                           sizeof(struct lstcon_test_batch_ent)) ? -EFAULT : 0;
 988
 989         CFS_FREE_PTR(entp)
 990
 991         return rc;
 992 }
 993
 994 static int
 995 lstcon_batrpc_condition(int transop, struct lstcon_node *nd, void *arg)
 996 {
 997         switch (transop) {
 998         case LST_TRANS_TSBRUN:
 999                 if (nd->nd_state != LST_NODE_ACTIVE)
1000                         return -ENETDOWN;
1001                 break;
1002
1003         case LST_TRANS_TSBSTOP:
1004                 if (nd->nd_state != LST_NODE_ACTIVE)
1005                         return 0;
1006                 break;
1007
1008         case LST_TRANS_TSBCLIQRY:
1009         case LST_TRANS_TSBSRVQRY:
1010                 break;
1011         }
1012
1013         return 1;
1014 }
1015
1016 static int
1017 lstcon_batch_op(struct lstcon_batch *bat, int transop,
1018                 struct list_head __user *result_up)
1019 {
1020         struct lstcon_rpc_trans *trans;
1021         int                 rc;
1022
1023         rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list,
1024                                      &bat->bat_trans_list, transop,
1025                                      bat, lstcon_batrpc_condition, &trans);
1026         if (rc != 0) {
1027                 CERROR("Can't create transaction: %d\n", rc);
1028                 return rc;
1029         }
1030
1031         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1032
1033         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1034
1035         lstcon_rpc_trans_destroy(trans);
1036
1037         return rc;
1038 }
1039
1040 int
1041 lstcon_batch_run(char *name, int timeout, struct list_head __user *result_up)
1042 {
1043         struct lstcon_batch *bat;
1044         int rc;
1045
1046         if (lstcon_batch_find(name, &bat) != 0) {
1047                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1048                 return -ENOENT;
1049         }
1050
1051         bat->bat_arg = timeout;
1052
1053         rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up);
1054
1055         /* mark batch as running if it's started in any node */
1056         if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0) != 0)
1057                 bat->bat_state = LST_BATCH_RUNNING;
1058
1059         return rc;
1060 }
1061
1062 int
1063 lstcon_batch_stop(char *name, int force, struct list_head __user *result_up)
1064 {
1065         struct lstcon_batch *bat;
1066         int rc;
1067
1068         if (lstcon_batch_find(name, &bat) != 0) {
1069                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1070                 return -ENOENT;
1071         }
1072
1073         bat->bat_arg = force;
1074
1075         rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up);
1076
1077         /* mark batch as stopped if all RPCs finished */
1078         if (lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0) == 0)
1079                 bat->bat_state = LST_BATCH_IDLE;
1080
1081         return rc;
1082 }
1083
1084 static void
1085 lstcon_batch_destroy(struct lstcon_batch *bat)
1086 {
1087         struct lstcon_ndlink *ndl;
1088         struct lstcon_test *test;
1089         int i;
1090
1091         list_del(&bat->bat_link);
1092
1093         while (!list_empty(&bat->bat_test_list)) {
1094                 test = list_entry(bat->bat_test_list.next,
1095                                   struct lstcon_test, tes_link);
1096                 LASSERT(list_empty(&test->tes_trans_list));
1097
1098                 list_del(&test->tes_link);
1099
1100                 lstcon_group_decref(test->tes_src_grp);
1101                 lstcon_group_decref(test->tes_dst_grp);
1102
1103                 LIBCFS_FREE(test, offsetof(struct lstcon_test,
1104                                            tes_param[test->tes_paramlen]));
1105         }
1106
1107         LASSERT(list_empty(&bat->bat_trans_list));
1108
1109         while (!list_empty(&bat->bat_cli_list)) {
1110                 ndl = list_entry(bat->bat_cli_list.next,
1111                                  struct lstcon_ndlink, ndl_link);
1112                 list_del_init(&ndl->ndl_link);
1113
1114                 lstcon_ndlink_release(ndl);
1115         }
1116
1117         while (!list_empty(&bat->bat_srv_list)) {
1118                 ndl = list_entry(bat->bat_srv_list.next,
1119                                  struct lstcon_ndlink, ndl_link);
1120                 list_del_init(&ndl->ndl_link);
1121
1122                 lstcon_ndlink_release(ndl);
1123         }
1124
1125         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1126                 LASSERT(list_empty(&bat->bat_cli_hash[i]));
1127                 LASSERT(list_empty(&bat->bat_srv_hash[i]));
1128         }
1129
1130         LIBCFS_FREE(bat->bat_cli_hash,
1131                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1132         LIBCFS_FREE(bat->bat_srv_hash,
1133                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1134         LIBCFS_FREE(bat, sizeof(*bat));
1135 }
1136
1137 static int
1138 lstcon_testrpc_condition(int transop, struct lstcon_node *nd, void *arg)
1139 {
1140         struct lstcon_test *test = arg;
1141         struct lstcon_batch *batch;
1142         struct lstcon_ndlink *ndl;
1143         struct list_head *hash;
1144         struct list_head *head;
1145
1146         LASSERT(test != NULL);
1147
1148         batch = test->tes_batch;
1149         LASSERT(batch != NULL);
1150
1151         if (test->tes_oneside &&
1152             transop == LST_TRANS_TSBSRVADD)
1153                 return 0;
1154
1155         if (nd->nd_state != LST_NODE_ACTIVE)
1156                 return -ENETDOWN;
1157
1158         if (transop == LST_TRANS_TSBCLIADD) {
1159                 hash = batch->bat_cli_hash;
1160                 head = &batch->bat_cli_list;
1161
1162         } else {
1163                 LASSERT (transop == LST_TRANS_TSBSRVADD);
1164
1165                 hash = batch->bat_srv_hash;
1166                 head = &batch->bat_srv_list;
1167         }
1168
1169         LASSERT (nd->nd_id.nid != LNET_NID_ANY);
1170
1171         if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1) != 0)
1172                 return -ENOMEM;
1173
1174         if (list_empty(&ndl->ndl_link))
1175                 list_add_tail(&ndl->ndl_link, head);
1176
1177         return 1;
1178 }
1179
1180 static int
1181 lstcon_test_nodes_add(struct lstcon_test *test,
1182                       struct list_head __user *result_up)
1183 {
1184         struct lstcon_rpc_trans *trans;
1185         struct lstcon_group *grp;
1186         int transop;
1187         int rc;
1188
1189         LASSERT (test->tes_src_grp != NULL);
1190         LASSERT (test->tes_dst_grp != NULL);
1191
1192         transop = LST_TRANS_TSBSRVADD;
1193         grp  = test->tes_dst_grp;
1194 again:
1195         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
1196                                      &test->tes_trans_list, transop,
1197                                      test, lstcon_testrpc_condition, &trans);
1198         if (rc != 0) {
1199                 CERROR("Can't create transaction: %d\n", rc);
1200                 return rc;
1201         }
1202
1203         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1204
1205         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1206             lstcon_trans_stat()->trs_fwk_errno != 0) {
1207                 lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1208
1209                 lstcon_rpc_trans_destroy(trans);
1210                 /* return if any error */
1211                 CDEBUG(D_NET, "Failed to add test %s, "
1212                               "RPC error %d, framework error %d\n",
1213                        transop == LST_TRANS_TSBCLIADD ? "client" : "server",
1214                        lstcon_trans_stat()->trs_rpc_errno,
1215                        lstcon_trans_stat()->trs_fwk_errno);
1216
1217                 return rc;
1218         }
1219
1220         lstcon_rpc_trans_destroy(trans);
1221
1222         if (transop == LST_TRANS_TSBCLIADD)
1223                 return rc;
1224
1225         transop = LST_TRANS_TSBCLIADD;
1226         grp = test->tes_src_grp;
1227         test->tes_cliidx = 0;
1228
1229         /* requests to test clients */
1230         goto again;
1231 }
1232
1233 static int
1234 lstcon_verify_batch(const char *name, struct lstcon_batch **batch)
1235 {
1236         int rc;
1237
1238         rc = lstcon_batch_find(name, batch);
1239         if (rc != 0) {
1240                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1241                 return rc;
1242         }
1243
1244         if ((*batch)->bat_state != LST_BATCH_IDLE) {
1245                 CDEBUG(D_NET, "Can't change running batch %s\n", name);
1246                 return -EINVAL;
1247         }
1248
1249         return 0;
1250 }
1251
1252 static int
1253 lstcon_verify_group(const char *name, struct lstcon_group **grp)
1254 {
1255         int rc;
1256         struct lstcon_ndlink *ndl;
1257
1258         rc = lstcon_group_find(name, grp);
1259         if (rc != 0) {
1260                 CDEBUG(D_NET, "can't find group %s\n", name);
1261                 return rc;
1262         }
1263
1264         list_for_each_entry(ndl, &(*grp)->grp_ndl_list, ndl_link) {
1265                 if (ndl->ndl_node->nd_state == LST_NODE_ACTIVE) {
1266                         return 0;
1267                 }
1268         }
1269
1270         CDEBUG(D_NET, "Group %s has no ACTIVE nodes\n", name);
1271
1272         return -EINVAL;
1273 }
1274
1275 int
1276 lstcon_test_add(char *batch_name, int type, int loop,
1277                 int concur, int dist, int span,
1278                 char *src_name, char *dst_name,
1279                 void *param, int paramlen, int *retp,
1280                 struct list_head __user *result_up)
1281 {
1282         struct lstcon_test *test = NULL;
1283         int rc;
1284         struct lstcon_group *src_grp = NULL;
1285         struct lstcon_group *dst_grp = NULL;
1286         struct lstcon_batch *batch = NULL;
1287
1288         /*
1289          * verify that a batch of the given name exists, and the groups
1290          * that will be part of the batch exist and have at least one
1291          * active node
1292          */
1293         rc = lstcon_verify_batch(batch_name, &batch);
1294         if (rc != 0)
1295                 goto out;
1296
1297         rc = lstcon_verify_group(src_name, &src_grp);
1298         if (rc != 0)
1299                 goto out;
1300
1301         rc = lstcon_verify_group(dst_name, &dst_grp);
1302         if (rc != 0)
1303                 goto out;
1304
1305         if (dst_grp->grp_userland)
1306                 *retp = 1;
1307
1308         LIBCFS_ALLOC(test, offsetof(struct lstcon_test, tes_param[paramlen]));
1309         if (!test) {
1310                 CERROR("Can't allocate test descriptor\n");
1311                 rc = -ENOMEM;
1312
1313                 goto out;
1314         }
1315
1316         test->tes_hdr.tsb_id    = batch->bat_hdr.tsb_id;
1317         test->tes_batch         = batch;
1318         test->tes_type          = type;
1319         test->tes_oneside       = 0; /* TODO */
1320         test->tes_loop          = loop;
1321         test->tes_concur        = concur;
1322         test->tes_stop_onerr    = 1; /* TODO */
1323         test->tes_span          = span;
1324         test->tes_dist          = dist;
1325         test->tes_cliidx        = 0; /* just used for creating RPC */
1326         test->tes_src_grp       = src_grp;
1327         test->tes_dst_grp       = dst_grp;
1328         INIT_LIST_HEAD(&test->tes_trans_list);
1329
1330         if (param != NULL) {
1331                 test->tes_paramlen = paramlen;
1332                 memcpy(&test->tes_param[0], param, paramlen);
1333         }
1334
1335         rc = lstcon_test_nodes_add(test, result_up);
1336
1337         if (rc != 0)
1338                 goto out;
1339
1340         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1341             lstcon_trans_stat()->trs_fwk_errno != 0)
1342                 CDEBUG(D_NET, "Failed to add test %d to batch %s\n", type,
1343                        batch_name);
1344
1345         /* add to test list anyway, so user can check what's going on */
1346         list_add_tail(&test->tes_link, &batch->bat_test_list);
1347
1348         batch->bat_ntest++;
1349         test->tes_hdr.tsb_index = batch->bat_ntest;
1350
1351         /*  hold groups so nobody can change them */
1352         return rc;
1353 out:
1354         if (test != NULL)
1355                 LIBCFS_FREE(test, offsetof(struct lstcon_test,
1356                                            tes_param[paramlen]));
1357
1358         if (dst_grp != NULL)
1359                 lstcon_group_decref(dst_grp);
1360
1361         if (src_grp != NULL)
1362                 lstcon_group_decref(src_grp);
1363
1364         return rc;
1365 }
1366
1367 static int
1368 lstcon_test_find(struct lstcon_batch *batch, int idx,
1369                  struct lstcon_test **testpp)
1370 {
1371         struct lstcon_test *test;
1372
1373         list_for_each_entry(test, &batch->bat_test_list, tes_link) {
1374                 if (idx == test->tes_hdr.tsb_index) {
1375                         *testpp = test;
1376                         return 0;
1377                 }
1378         }
1379
1380         return -ENOENT;
1381 }
1382
1383 static int
1384 lstcon_tsbrpc_readent(int transop, struct srpc_msg *msg,
1385                       struct lstcon_rpc_ent __user *ent_up)
1386 {
1387         struct srpc_batch_reply *rep = &msg->msg_body.bat_reply;
1388
1389         LASSERT (transop == LST_TRANS_TSBCLIQRY ||
1390                  transop == LST_TRANS_TSBSRVQRY);
1391
1392         /* positive errno, framework error code */
1393         if (copy_to_user(&ent_up->rpe_priv[0],
1394                              &rep->bar_active, sizeof(rep->bar_active)))
1395                 return -EFAULT;
1396
1397         return 0;
1398 }
1399
1400 int
1401 lstcon_test_batch_query(char *name, int testidx, int client,
1402                         int timeout, struct list_head __user *result_up)
1403 {
1404         struct lstcon_rpc_trans *trans;
1405         struct list_head *translist;
1406         struct list_head *ndlist;
1407         struct lstcon_tsb_hdr *hdr;
1408         struct lstcon_batch *batch;
1409         struct lstcon_test *test = NULL;
1410         int transop;
1411         int rc;
1412
1413         rc = lstcon_batch_find(name, &batch);
1414         if (rc != 0) {
1415                 CDEBUG(D_NET, "Can't find batch: %s\n", name);
1416                 return rc;
1417         }
1418
1419         if (testidx == 0) {
1420                 translist = &batch->bat_trans_list;
1421                 ndlist    = &batch->bat_cli_list;
1422                 hdr       = &batch->bat_hdr;
1423
1424         } else {
1425                 /* query specified test only */
1426                 rc = lstcon_test_find(batch, testidx, &test);
1427                 if (rc != 0) {
1428                         CDEBUG(D_NET, "Can't find test: %d\n", testidx);
1429                         return rc;
1430                 }
1431
1432                 translist = &test->tes_trans_list;
1433                 ndlist    = &test->tes_src_grp->grp_ndl_list;
1434                 hdr       = &test->tes_hdr;
1435         }
1436
1437         transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY;
1438
1439         rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr,
1440                                      lstcon_batrpc_condition, &trans);
1441         if (rc != 0) {
1442                 CERROR("Can't create transaction: %d\n", rc);
1443                 return rc;
1444         }
1445
1446         lstcon_rpc_trans_postwait(trans, timeout);
1447
1448         if (testidx == 0 && /* query a batch, not a test */
1449             lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) == 0 &&
1450             lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0) == 0) {
1451                 /* all RPCs finished, and no active test */
1452                 batch->bat_state = LST_BATCH_IDLE;
1453         }
1454
1455         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1456                                           lstcon_tsbrpc_readent);
1457         lstcon_rpc_trans_destroy(trans);
1458
1459         return rc;
1460 }
1461
1462 static int
1463 lstcon_statrpc_readent(int transop, struct srpc_msg *msg,
1464                        struct lstcon_rpc_ent __user *ent_up)
1465 {
1466         struct srpc_stat_reply *rep = &msg->msg_body.stat_reply;
1467         struct sfw_counters __user *sfwk_stat;
1468         struct srpc_counters __user *srpc_stat;
1469         struct lnet_counters_common __user *lnet_stat;
1470
1471         if (rep->str_status != 0)
1472                 return 0;
1473
1474         sfwk_stat = (struct sfw_counters __user *)&ent_up->rpe_payload[0];
1475         srpc_stat = (struct srpc_counters __user *)
1476                 ((char __user *)sfwk_stat + sizeof(*sfwk_stat));
1477         lnet_stat = (struct lnet_counters_common __user *)
1478                 ((char __user *)srpc_stat + sizeof(*srpc_stat));
1479
1480         if (copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) ||
1481             copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) ||
1482             copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat)))
1483                 return -EFAULT;
1484
1485         return 0;
1486 }
1487
1488 static int
1489 lstcon_ndlist_stat(struct list_head *ndlist,
1490                    int timeout, struct list_head __user *result_up)
1491 {
1492         LIST_HEAD(head);
1493         struct lstcon_rpc_trans *trans;
1494         int rc;
1495
1496         rc = lstcon_rpc_trans_ndlist(ndlist, &head,
1497                                      LST_TRANS_STATQRY, NULL, NULL, &trans);
1498         if (rc != 0) {
1499                 CERROR("Can't create transaction: %d\n", rc);
1500                 return rc;
1501         }
1502
1503         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1504
1505         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1506                                           lstcon_statrpc_readent);
1507         lstcon_rpc_trans_destroy(trans);
1508
1509         return rc;
1510 }
1511
1512 int
1513 lstcon_group_stat(char *grp_name, int timeout,
1514                   struct list_head __user *result_up)
1515 {
1516         struct lstcon_group *grp;
1517         int rc;
1518
1519         rc = lstcon_group_find(grp_name, &grp);
1520         if (rc != 0) {
1521                 CDEBUG(D_NET, "Can't find group %s\n", grp_name);
1522                 return rc;
1523         }
1524
1525         rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up);
1526
1527         lstcon_group_decref(grp);
1528
1529         return rc;
1530 }
1531
1532 int
1533 lstcon_nodes_stat(int count, struct lnet_process_id __user *ids_up,
1534                   int timeout, struct list_head __user *result_up)
1535 {
1536         struct lstcon_ndlink *ndl;
1537         struct lstcon_group *tmp;
1538         struct lnet_process_id id;
1539         int i;
1540         int rc;
1541
1542         rc = lstcon_group_alloc(NULL, &tmp);
1543         if (rc != 0) {
1544                 CERROR("Out of memory\n");
1545                 return -ENOMEM;
1546         }
1547
1548         for (i = 0 ; i < count; i++) {
1549                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1550                         rc = -EFAULT;
1551                         break;
1552                 }
1553
1554                 /* add to tmp group */
1555                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2);
1556                 if (rc != 0) {
1557                         CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET,
1558                                "Failed to find or create %s: %d\n",
1559                                libcfs_id2str(id), rc);
1560                         break;
1561                 }
1562         }
1563
1564         if (rc != 0) {
1565                 lstcon_group_decref(tmp);
1566                 return rc;
1567         }
1568
1569         rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up);
1570
1571         lstcon_group_decref(tmp);
1572
1573         return rc;
1574 }
1575
1576 static int
1577 lstcon_debug_ndlist(struct list_head *ndlist,
1578                     struct list_head *translist,
1579                     int timeout, struct list_head __user *result_up)
1580 {
1581         struct lstcon_rpc_trans *trans;
1582         int rc;
1583
1584         rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY,
1585                                      NULL, lstcon_sesrpc_condition, &trans);
1586         if (rc != 0) {
1587                 CERROR("Can't create transaction: %d\n", rc);
1588                 return rc;
1589         }
1590
1591         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1592
1593         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1594                                           lstcon_sesrpc_readent);
1595         lstcon_rpc_trans_destroy(trans);
1596
1597         return rc;
1598 }
1599
1600 int
1601 lstcon_session_debug(int timeout, struct list_head __user *result_up)
1602 {
1603         return lstcon_debug_ndlist(&console_session.ses_ndl_list,
1604                                    NULL, timeout, result_up);
1605 }
1606
1607 int
1608 lstcon_batch_debug(int timeout, char *name,
1609                    int client, struct list_head __user *result_up)
1610 {
1611         struct lstcon_batch *bat;
1612         int rc;
1613
1614         rc = lstcon_batch_find(name, &bat);
1615         if (rc != 0)
1616                 return -ENOENT;
1617
1618         rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list :
1619                                           &bat->bat_srv_list,
1620                                  NULL, timeout, result_up);
1621
1622         return rc;
1623 }
1624
1625 int
1626 lstcon_group_debug(int timeout, char *name,
1627                    struct list_head __user *result_up)
1628 {
1629         struct lstcon_group *grp;
1630         int rc;
1631
1632         rc = lstcon_group_find(name, &grp);
1633         if (rc != 0)
1634                 return -ENOENT;
1635
1636         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1637                                  timeout, result_up);
1638         lstcon_group_decref(grp);
1639
1640         return rc;
1641 }
1642
1643 int
1644 lstcon_nodes_debug(int timeout, int count,
1645                    struct lnet_process_id __user *ids_up,
1646                    struct list_head __user *result_up)
1647 {
1648         struct lnet_process_id id;
1649         struct lstcon_ndlink *ndl;
1650         struct lstcon_group *grp;
1651         int i;
1652         int rc;
1653
1654         rc = lstcon_group_alloc(NULL, &grp);
1655         if (rc != 0) {
1656                 CDEBUG(D_NET, "Out of memory\n");
1657                 return rc;
1658         }
1659
1660         for (i = 0; i < count; i++) {
1661                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1662                         rc = -EFAULT;
1663                         break;
1664                 }
1665
1666                 /* node is added to tmp group */
1667                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 1);
1668                 if (rc != 0) {
1669                         CERROR("Can't create node link\n");
1670                         break;
1671                 }
1672         }
1673
1674         if (rc != 0) {
1675                 lstcon_group_decref(grp);
1676                 return rc;
1677         }
1678
1679         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1680                                  timeout, result_up);
1681
1682         lstcon_group_decref(grp);
1683
1684         return rc;
1685 }
1686
1687 int
1688 lstcon_session_match(struct lst_sid sid)
1689 {
1690         return (console_session.ses_id.ses_nid   == sid.ses_nid &&
1691                 console_session.ses_id.ses_stamp == sid.ses_stamp) ?  1: 0;
1692 }
1693
1694 static void
1695 lstcon_new_session_id(struct lst_sid *sid)
1696 {
1697         struct lnet_process_id id;
1698
1699         LASSERT(console_session.ses_state == LST_SESSION_NONE);
1700
1701         LNetGetId(1, &id);
1702         sid->ses_nid = id.nid;
1703         sid->ses_stamp = div_u64(ktime_get_ns(), NSEC_PER_MSEC);
1704 }
1705
1706 int
1707 lstcon_session_new(char *name, int key, unsigned feats,
1708                    int timeout, int force, struct lst_sid __user *sid_up)
1709 {
1710         int     rc = 0;
1711         int     i;
1712
1713         if (console_session.ses_state != LST_SESSION_NONE) {
1714                 /* session exists */
1715                 if (!force) {
1716                         CNETERR("Session %s already exists\n",
1717                                 console_session.ses_name);
1718                         return -EEXIST;
1719                 }
1720
1721                 rc = lstcon_session_end();
1722
1723                 /* lstcon_session_end() only return local error */
1724                 if  (rc != 0)
1725                         return rc;
1726         }
1727
1728         if ((feats & ~LST_FEATS_MASK) != 0) {
1729                 CNETERR("Unknown session features %x\n",
1730                         (feats & ~LST_FEATS_MASK));
1731                 return -EINVAL;
1732         }
1733
1734         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
1735                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
1736
1737         lstcon_new_session_id(&console_session.ses_id);
1738
1739         console_session.ses_key     = key;
1740         console_session.ses_state   = LST_SESSION_ACTIVE;
1741         console_session.ses_force   = !!force;
1742         console_session.ses_features = feats;
1743         console_session.ses_feats_updated = 0;
1744         console_session.ses_timeout = (timeout <= 0) ?
1745                                       LST_CONSOLE_TIMEOUT : timeout;
1746
1747         if (strlen(name) > sizeof(console_session.ses_name)-1)
1748                 return -E2BIG;
1749         strlcpy(console_session.ses_name, name,
1750                 sizeof(console_session.ses_name));
1751
1752         rc = lstcon_batch_add(LST_DEFAULT_BATCH);
1753         if (rc != 0)
1754                 return rc;
1755
1756         rc = lstcon_rpc_pinger_start();
1757         if (rc != 0) {
1758                 struct lstcon_batch *bat = NULL;
1759
1760                 lstcon_batch_find(LST_DEFAULT_BATCH, &bat);
1761                 lstcon_batch_destroy(bat);
1762
1763                 return rc;
1764         }
1765
1766         if (copy_to_user(sid_up, &console_session.ses_id,
1767                              sizeof(struct lst_sid)) == 0)
1768                 return rc;
1769
1770         lstcon_session_end();
1771
1772         return -EFAULT;
1773 }
1774
1775 int
1776 lstcon_session_info(struct lst_sid __user *sid_up, int __user *key_up,
1777                     unsigned __user *featp,
1778                     struct lstcon_ndlist_ent __user *ndinfo_up,
1779                     char __user *name_up, int len)
1780 {
1781         struct lstcon_ndlist_ent *entp;
1782         struct lstcon_ndlink *ndl;
1783         int rc = 0;
1784
1785         if (console_session.ses_state != LST_SESSION_ACTIVE)
1786                 return -ESRCH;
1787
1788         LIBCFS_ALLOC(entp, sizeof(*entp));
1789         if (entp == NULL)
1790                 return -ENOMEM;
1791
1792         list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link)
1793                 LST_NODE_STATE_COUNTER(ndl->ndl_node, entp);
1794
1795         if (copy_to_user(sid_up, &console_session.ses_id,
1796                          sizeof(struct lst_sid)) ||
1797             copy_to_user(key_up, &console_session.ses_key,
1798                              sizeof(*key_up)) ||
1799             copy_to_user(featp, &console_session.ses_features,
1800                              sizeof(*featp)) ||
1801             copy_to_user(ndinfo_up, entp, sizeof(*entp)) ||
1802             copy_to_user(name_up, console_session.ses_name, len))
1803                 rc = -EFAULT;
1804
1805         LIBCFS_FREE(entp, sizeof(*entp));
1806
1807         return rc;
1808 }
1809
1810 int
1811 lstcon_session_end(void)
1812 {
1813         struct lstcon_rpc_trans *trans;
1814         struct lstcon_group *grp;
1815         struct lstcon_batch *bat;
1816         int rc = 0;
1817
1818         LASSERT (console_session.ses_state == LST_SESSION_ACTIVE);
1819
1820         rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list,
1821                                      NULL, LST_TRANS_SESEND, NULL,
1822                                      lstcon_sesrpc_condition, &trans);
1823         if (rc != 0) {
1824                 CERROR("Can't create transaction: %d\n", rc);
1825                 return rc;
1826         }
1827
1828         console_session.ses_shutdown = 1;
1829
1830         lstcon_rpc_pinger_stop();
1831
1832         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1833
1834         lstcon_rpc_trans_destroy(trans);
1835         /* User can do nothing even rpc failed, so go on */
1836
1837         /* waiting for orphan rpcs to die */
1838         lstcon_rpc_cleanup_wait();
1839
1840         console_session.ses_id    = LST_INVALID_SID;
1841         console_session.ses_state = LST_SESSION_NONE;
1842         console_session.ses_key   = 0;
1843         console_session.ses_force = 0;
1844         console_session.ses_feats_updated = 0;
1845
1846         /* destroy all batches */
1847         while (!list_empty(&console_session.ses_bat_list)) {
1848                 bat = list_entry(console_session.ses_bat_list.next,
1849                                  struct lstcon_batch, bat_link);
1850
1851                 lstcon_batch_destroy(bat);
1852         }
1853
1854         /* destroy all groups */
1855         while (!list_empty(&console_session.ses_grp_list)) {
1856                 grp = list_entry(console_session.ses_grp_list.next,
1857                                  struct lstcon_group, grp_link);
1858                 LASSERT(grp->grp_ref == 1);
1859
1860                 lstcon_group_decref(grp);
1861         }
1862
1863         /* all nodes should be released */
1864         LASSERT(list_empty(&console_session.ses_ndl_list));
1865
1866         console_session.ses_shutdown = 0;
1867         console_session.ses_expired  = 0;
1868
1869         return rc;
1870 }
1871
1872 int
1873 lstcon_session_feats_check(unsigned feats)
1874 {
1875         int rc = 0;
1876
1877         if ((feats & ~LST_FEATS_MASK) != 0) {
1878                 CERROR("Can't support these features: %x\n",
1879                        (feats & ~LST_FEATS_MASK));
1880                 return -EPROTO;
1881         }
1882
1883         spin_lock(&console_session.ses_rpc_lock);
1884
1885         if (!console_session.ses_feats_updated) {
1886                 console_session.ses_feats_updated = 1;
1887                 console_session.ses_features = feats;
1888         }
1889
1890         if (console_session.ses_features != feats)
1891                 rc = -EPROTO;
1892
1893         spin_unlock(&console_session.ses_rpc_lock);
1894
1895         if (rc != 0) {
1896                 CERROR("remote features %x do not match with "
1897                        "session features %x of console\n",
1898                        feats, console_session.ses_features);
1899         }
1900
1901         return rc;
1902 }
1903
1904 static int
1905 lstcon_acceptor_handle(struct srpc_server_rpc *rpc)
1906 {
1907         struct srpc_msg *rep = &rpc->srpc_replymsg;
1908         struct srpc_msg *req = &rpc->srpc_reqstbuf->buf_msg;
1909         struct srpc_join_reqst *jreq = &req->msg_body.join_reqst;
1910         struct srpc_join_reply *jrep = &rep->msg_body.join_reply;
1911         struct lstcon_group *grp = NULL;
1912         struct lstcon_ndlink *ndl;
1913         int rc = 0;
1914
1915         sfw_unpack_message(req);
1916
1917         mutex_lock(&console_session.ses_mutex);
1918
1919         jrep->join_sid = console_session.ses_id;
1920
1921         if (console_session.ses_id.ses_nid == LNET_NID_ANY) {
1922                 jrep->join_status = ESRCH;
1923                 goto out;
1924         }
1925
1926         if (lstcon_session_feats_check(req->msg_ses_feats) != 0) {
1927                 jrep->join_status = EPROTO;
1928                 goto out;
1929         }
1930
1931         if (jreq->join_sid.ses_nid != LNET_NID_ANY &&
1932              !lstcon_session_match(jreq->join_sid)) {
1933                 jrep->join_status = EBUSY;
1934                 goto out;
1935         }
1936
1937         if (lstcon_group_find(jreq->join_group, &grp) != 0) {
1938                 rc = lstcon_group_alloc(jreq->join_group, &grp);
1939                 if (rc != 0) {
1940                         CERROR("Out of memory\n");
1941                         goto out;
1942                 }
1943
1944                 list_add_tail(&grp->grp_link,
1945                               &console_session.ses_grp_list);
1946                 lstcon_group_addref(grp);
1947         }
1948
1949         if (grp->grp_ref > 2) {
1950                 /* Group in using */
1951                 jrep->join_status = EBUSY;
1952                 goto out;
1953         }
1954
1955         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0);
1956         if (rc == 0) {
1957                 jrep->join_status = EEXIST;
1958                 goto out;
1959         }
1960
1961         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1);
1962         if (rc != 0) {
1963                 CERROR("Out of memory\n");
1964                 goto out;
1965         }
1966
1967         ndl->ndl_node->nd_state   = LST_NODE_ACTIVE;
1968         ndl->ndl_node->nd_timeout = console_session.ses_timeout;
1969
1970         if (grp->grp_userland == 0)
1971                 grp->grp_userland = 1;
1972
1973         strlcpy(jrep->join_session, console_session.ses_name,
1974                 sizeof(jrep->join_session));
1975         jrep->join_timeout = console_session.ses_timeout;
1976         jrep->join_status  = 0;
1977
1978 out:
1979         rep->msg_ses_feats = console_session.ses_features;
1980         if (grp != NULL)
1981                 lstcon_group_decref(grp);
1982
1983         mutex_unlock(&console_session.ses_mutex);
1984
1985         return rc;
1986 }
1987
1988 static struct srpc_service lstcon_acceptor_service;
1989
1990 static void lstcon_init_acceptor_service(void)
1991 {
1992         /* initialize selftest console acceptor service table */
1993         lstcon_acceptor_service.sv_name    = "join session";
1994         lstcon_acceptor_service.sv_handler = lstcon_acceptor_handle;
1995         lstcon_acceptor_service.sv_id      = SRPC_SERVICE_JOIN;
1996         lstcon_acceptor_service.sv_wi_total = SFW_FRWK_WI_MAX;
1997 }
1998
1999 static struct notifier_block lstcon_ioctl_handler = {
2000         .notifier_call = lstcon_ioctl_entry,
2001 };
2002
2003 /* initialize console */
2004 int
2005 lstcon_console_init(void)
2006 {
2007         int     i;
2008         int     rc;
2009
2010         console_session.ses_id              = LST_INVALID_SID;
2011         console_session.ses_state           = LST_SESSION_NONE;
2012         console_session.ses_timeout         = 0;
2013         console_session.ses_force           = 0;
2014         console_session.ses_expired         = 0;
2015         console_session.ses_feats_updated   = 0;
2016         console_session.ses_features        = LST_FEATS_MASK;
2017         console_session.ses_laststamp = ktime_get_real_seconds();
2018
2019         mutex_init(&console_session.ses_mutex);
2020
2021         INIT_LIST_HEAD(&console_session.ses_ndl_list);
2022         INIT_LIST_HEAD(&console_session.ses_grp_list);
2023         INIT_LIST_HEAD(&console_session.ses_bat_list);
2024         INIT_LIST_HEAD(&console_session.ses_trans_list);
2025
2026         CFS_ALLOC_PTR_ARRAY(console_session.ses_ndl_hash,
2027                                LST_GLOBAL_HASHSIZE);
2028         if (console_session.ses_ndl_hash == NULL)
2029                 return -ENOMEM;
2030
2031         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
2032                 INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]);
2033
2034
2035         /* initialize acceptor service table */
2036         lstcon_init_acceptor_service();
2037
2038         rc = srpc_add_service(&lstcon_acceptor_service);
2039         LASSERT(rc != -EBUSY);
2040         if (rc != 0) {
2041                 CFS_FREE_PTR_ARRAY(console_session.ses_ndl_hash,
2042                                    LST_GLOBAL_HASHSIZE);
2043                 return rc;
2044         }
2045
2046         rc = srpc_service_add_buffers(&lstcon_acceptor_service,
2047                                       lstcon_acceptor_service.sv_wi_total);
2048         if (rc != 0) {
2049                 rc = -ENOMEM;
2050                 goto out;
2051         }
2052
2053         rc = blocking_notifier_chain_register(&libcfs_ioctl_list,
2054                                               &lstcon_ioctl_handler);
2055         if (rc == 0) {
2056                 lstcon_rpc_module_init();
2057                 return 0;
2058         }
2059
2060 out:
2061         srpc_shutdown_service(&lstcon_acceptor_service);
2062         srpc_remove_service(&lstcon_acceptor_service);
2063
2064         CFS_FREE_PTR_ARRAY(console_session.ses_ndl_hash, LST_GLOBAL_HASHSIZE);
2065
2066         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2067
2068         return rc;
2069 }
2070
2071 int
2072 lstcon_console_fini(void)
2073 {
2074         int i;
2075
2076         blocking_notifier_chain_unregister(&libcfs_ioctl_list,
2077                                            &lstcon_ioctl_handler);
2078
2079         mutex_lock(&console_session.ses_mutex);
2080
2081         srpc_shutdown_service(&lstcon_acceptor_service);
2082         srpc_remove_service(&lstcon_acceptor_service);
2083
2084         if (console_session.ses_state != LST_SESSION_NONE)
2085                 lstcon_session_end();
2086
2087         lstcon_rpc_module_fini();
2088
2089         mutex_unlock(&console_session.ses_mutex);
2090
2091         LASSERT(list_empty(&console_session.ses_ndl_list));
2092         LASSERT(list_empty(&console_session.ses_grp_list));
2093         LASSERT(list_empty(&console_session.ses_bat_list));
2094         LASSERT(list_empty(&console_session.ses_trans_list));
2095
2096         for (i = 0; i < LST_NODE_HASHSIZE; i++)
2097                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
2098
2099         CFS_FREE_PTR_ARRAY(console_session.ses_ndl_hash,
2100                            LST_GLOBAL_HASHSIZE);
2101
2102         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2103
2104         return 0;
2105 }
2106