lnet/selftest/console.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2012, 2014, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  * Lustre is a trademark of Sun Microsystems, Inc.
  31  *
  32  * lnet/selftest/conctl.c
  33  *
  34  * Infrastructure of LST console
  35  *
  36  * Author: Liang Zhen <liangzhen@clusterfs.com>
  37  */
  38
  39 #include <libcfs/libcfs.h>
  40 #include <lnet/lib-lnet.h>
  41 #include "console.h"
  42 #include "conrpc.h"
  43
  44 #define LST_NODE_STATE_COUNTER(nd, p)                   \
  45 do {                                                    \
  46         if ((nd)->nd_state == LST_NODE_ACTIVE)          \
  47                 (p)->nle_nactive ++;                    \
  48         else if ((nd)->nd_state == LST_NODE_BUSY)       \
  49                 (p)->nle_nbusy ++;                      \
  50         else if ((nd)->nd_state == LST_NODE_DOWN)       \
  51                 (p)->nle_ndown ++;                      \
  52         else                                            \
  53                 (p)->nle_nunknown ++;                   \
  54         (p)->nle_nnode ++;                              \
  55 } while (0)
  56
  57 struct lstcon_session console_session;
  58
  59 static void
  60 lstcon_node_get(struct lstcon_node *nd)
  61 {
  62         LASSERT (nd->nd_ref >= 1);
  63
  64         nd->nd_ref++;
  65 }
  66
  67 static int
  68 lstcon_node_find(struct lnet_process_id id, struct lstcon_node **ndpp,
  69                  int create)
  70 {
  71         struct lstcon_ndlink *ndl;
  72         unsigned int idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE;
  73
  74         LASSERT(id.nid != LNET_NID_ANY);
  75
  76         list_for_each_entry(ndl, &console_session.ses_ndl_hash[idx],
  77                             ndl_hlink) {
  78                 if (ndl->ndl_node->nd_id.nid != id.nid ||
  79                     ndl->ndl_node->nd_id.pid != id.pid)
  80                         continue;
  81
  82                 lstcon_node_get(ndl->ndl_node);
  83                 *ndpp = ndl->ndl_node;
  84                 return 0;
  85         }
  86
  87         if (!create)
  88                 return -ENOENT;
  89
  90         LIBCFS_ALLOC(*ndpp, sizeof(**ndpp) + sizeof(*ndl));
  91         if (*ndpp == NULL)
  92                 return -ENOMEM;
  93
  94         ndl = (struct lstcon_ndlink *)(*ndpp + 1);
  95
  96         ndl->ndl_node = *ndpp;
  97
  98         ndl->ndl_node->nd_ref   = 1;
  99         ndl->ndl_node->nd_id    = id;
 100         ndl->ndl_node->nd_stamp = ktime_get();
 101         ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
 102         ndl->ndl_node->nd_timeout = 0;
 103         memset(&ndl->ndl_node->nd_ping, 0, sizeof(ndl->ndl_node->nd_ping));
 104
 105         /* queued in global hash & list, no refcount is taken by
 106          * global hash & list, if caller release his refcount,
 107          * node will be released */
 108         list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]);
 109         list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list);
 110
 111         return 0;
 112 }
 113
 114 static void
 115 lstcon_node_put(struct lstcon_node *nd)
 116 {
 117         struct lstcon_ndlink *ndl;
 118
 119         LASSERT(nd->nd_ref > 0);
 120
 121         if (--nd->nd_ref > 0)
 122                 return;
 123
 124         ndl = (struct lstcon_ndlink *)(nd + 1);
 125
 126         LASSERT(!list_empty(&ndl->ndl_link));
 127         LASSERT(!list_empty(&ndl->ndl_hlink));
 128
 129         /* remove from session */
 130         list_del(&ndl->ndl_link);
 131         list_del(&ndl->ndl_hlink);
 132
 133         LIBCFS_FREE(nd, sizeof(*nd) + sizeof(*ndl));
 134 }
 135
 136 static int
 137 lstcon_ndlink_find(struct list_head *hash, struct lnet_process_id id,
 138                    struct lstcon_ndlink **ndlpp, int create)
 139 {
 140         unsigned int idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
 141         struct lstcon_ndlink *ndl;
 142         struct lstcon_node *nd;
 143         int rc;
 144
 145         if (id.nid == LNET_NID_ANY)
 146                 return -EINVAL;
 147
 148         /* search in hash */
 149         list_for_each_entry(ndl, &hash[idx], ndl_hlink) {
 150                 if (ndl->ndl_node->nd_id.nid != id.nid ||
 151                     ndl->ndl_node->nd_id.pid != id.pid)
 152                         continue;
 153
 154                 *ndlpp = ndl;
 155                 return 0;
 156         }
 157
 158         if (create == 0)
 159                 return -ENOENT;
 160
 161         /* find or create in session hash */
 162         rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0);
 163         if (rc != 0)
 164                 return rc;
 165
 166         LIBCFS_ALLOC(ndl, sizeof(*ndl));
 167         if (ndl == NULL) {
 168                 lstcon_node_put(nd);
 169                 return -ENOMEM;
 170         }
 171
 172         *ndlpp = ndl;
 173
 174         ndl->ndl_node = nd;
 175         INIT_LIST_HEAD(&ndl->ndl_link);
 176         list_add_tail(&ndl->ndl_hlink, &hash[idx]);
 177
 178         return 0;
 179 }
 180
 181 static void
 182 lstcon_ndlink_release(struct lstcon_ndlink *ndl)
 183 {
 184         LASSERT(list_empty(&ndl->ndl_link));
 185         LASSERT(!list_empty(&ndl->ndl_hlink));
 186
 187         list_del(&ndl->ndl_hlink); /* delete from hash */
 188         lstcon_node_put(ndl->ndl_node);
 189
 190         LIBCFS_FREE(ndl, sizeof(*ndl));
 191 }
 192
 193 static int
 194 lstcon_group_alloc(char *name, struct lstcon_group **grpp)
 195 {
 196         struct lstcon_group *grp;
 197         int i;
 198
 199         LIBCFS_ALLOC(grp, offsetof(struct lstcon_group,
 200                                    grp_ndl_hash[LST_NODE_HASHSIZE]));
 201         if (grp == NULL)
 202                 return -ENOMEM;
 203
 204         grp->grp_ref = 1;
 205         if (name != NULL) {
 206                 if (strlen(name) > sizeof(grp->grp_name)-1) {
 207                         LIBCFS_FREE(grp, offsetof(struct lstcon_group,
 208                                           grp_ndl_hash[LST_NODE_HASHSIZE]));
 209                         return -E2BIG;
 210                 }
 211                 strncpy(grp->grp_name, name, sizeof(grp->grp_name));
 212         }
 213
 214         INIT_LIST_HEAD(&grp->grp_link);
 215         INIT_LIST_HEAD(&grp->grp_ndl_list);
 216         INIT_LIST_HEAD(&grp->grp_trans_list);
 217
 218         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 219                 INIT_LIST_HEAD(&grp->grp_ndl_hash[i]);
 220
 221         *grpp = grp;
 222
 223         return 0;
 224 }
 225
 226 static void
 227 lstcon_group_addref(struct lstcon_group *grp)
 228 {
 229         grp->grp_ref++;
 230 }
 231
 232 static void lstcon_group_ndlink_release(struct lstcon_group *,
 233                                         struct lstcon_ndlink *);
 234
 235 static void
 236 lstcon_group_drain(struct lstcon_group *grp, int keep)
 237 {
 238         struct lstcon_ndlink *ndl;
 239         struct lstcon_ndlink *tmp;
 240
 241         list_for_each_entry_safe(ndl, tmp, &grp->grp_ndl_list, ndl_link) {
 242                 if ((ndl->ndl_node->nd_state & keep) == 0)
 243                         lstcon_group_ndlink_release(grp, ndl);
 244         }
 245 }
 246
 247 static void
 248 lstcon_group_decref(struct lstcon_group *grp)
 249 {
 250         int i;
 251
 252         if (--grp->grp_ref > 0)
 253                 return;
 254
 255         if (!list_empty(&grp->grp_link))
 256                 list_del(&grp->grp_link);
 257
 258         lstcon_group_drain(grp, 0);
 259
 260         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 261                 LASSERT(list_empty(&grp->grp_ndl_hash[i]));
 262
 263         LIBCFS_FREE(grp, offsetof(struct lstcon_group,
 264                                   grp_ndl_hash[LST_NODE_HASHSIZE]));
 265 }
 266
 267 static int
 268 lstcon_group_find(const char *name, struct lstcon_group **grpp)
 269 {
 270         struct lstcon_group *grp;
 271
 272         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 273                 if (strncmp(grp->grp_name, name, LST_NAME_SIZE) != 0)
 274                         continue;
 275
 276                 lstcon_group_addref(grp);  /* +1 ref for caller */
 277                 *grpp = grp;
 278                 return 0;
 279         }
 280
 281         return -ENOENT;
 282 }
 283
 284 static int
 285 lstcon_group_ndlink_find(struct lstcon_group *grp, struct lnet_process_id id,
 286                          struct lstcon_ndlink **ndlpp, int create)
 287 {
 288         int rc;
 289
 290         rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create);
 291         if (rc != 0)
 292                 return rc;
 293
 294         if (!list_empty(&(*ndlpp)->ndl_link))
 295                 return 0;
 296
 297         list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list);
 298         grp->grp_nnode++;
 299
 300         return 0;
 301 }
 302
 303 static void
 304 lstcon_group_ndlink_release(struct lstcon_group *grp, struct lstcon_ndlink *ndl)
 305 {
 306         list_del_init(&ndl->ndl_link);
 307         lstcon_ndlink_release(ndl);
 308         grp->grp_nnode--;
 309 }
 310
 311 static void
 312 lstcon_group_ndlink_move(struct lstcon_group *old,
 313                          struct lstcon_group *new, struct lstcon_ndlink *ndl)
 314 {
 315         unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) %
 316                                         LST_NODE_HASHSIZE;
 317
 318         old->grp_nnode--;
 319
 320         list_move_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]);
 321         list_move_tail(&ndl->ndl_link, &new->grp_ndl_list);
 322         new->grp_nnode++;
 323 }
 324
 325 static void
 326 lstcon_group_move(struct lstcon_group *old, struct lstcon_group *new)
 327 {
 328         struct lstcon_ndlink *ndl;
 329
 330         while (!list_empty(&old->grp_ndl_list)) {
 331                 ndl = list_entry(old->grp_ndl_list.next,
 332                                  struct lstcon_ndlink, ndl_link);
 333                 lstcon_group_ndlink_move(old, new, ndl);
 334         }
 335 }
 336
 337 static int
 338 lstcon_sesrpc_condition(int transop, struct lstcon_node *nd, void *arg)
 339 {
 340         struct lstcon_group *grp = arg;
 341
 342         switch (transop) {
 343         case LST_TRANS_SESNEW:
 344                 if (nd->nd_state == LST_NODE_ACTIVE)
 345                         return 0;
 346                 break;
 347
 348         case LST_TRANS_SESEND:
 349                 if (nd->nd_state != LST_NODE_ACTIVE)
 350                         return 0;
 351
 352                 if (grp != NULL && nd->nd_ref > 1)
 353                         return 0;
 354                 break;
 355
 356         case LST_TRANS_SESQRY:
 357                 break;
 358
 359         default:
 360                 LBUG();
 361         }
 362
 363         return 1;
 364 }
 365
 366 static int
 367 lstcon_sesrpc_readent(int transop, struct srpc_msg *msg,
 368                       struct lstcon_rpc_ent __user *ent_up)
 369 {
 370         struct srpc_debug_reply *rep;
 371
 372         switch (transop) {
 373         case LST_TRANS_SESNEW:
 374         case LST_TRANS_SESEND:
 375                 return 0;
 376
 377         case LST_TRANS_SESQRY:
 378                 rep = &msg->msg_body.dbg_reply;
 379
 380                 if (copy_to_user(&ent_up->rpe_priv[0],
 381                                      &rep->dbg_timeout, sizeof(int)) ||
 382                     copy_to_user(&ent_up->rpe_payload[0],
 383                                      &rep->dbg_name, LST_NAME_SIZE))
 384                         return -EFAULT;
 385
 386                 return 0;
 387
 388         default:
 389                 LBUG();
 390         }
 391
 392         return 0;
 393 }
 394
 395 static int
 396 lstcon_group_nodes_add(struct lstcon_group *grp,
 397                        int count, struct lnet_process_id __user *ids_up,
 398                        unsigned int *featp,
 399                        struct list_head __user *result_up)
 400 {
 401         struct lstcon_rpc_trans *trans;
 402         struct lstcon_ndlink *ndl;
 403         struct lstcon_group *tmp;
 404         struct lnet_process_id id;
 405         int i;
 406         int rc;
 407
 408         rc = lstcon_group_alloc(NULL, &tmp);
 409         if (rc != 0) {
 410                 CERROR("Out of memory\n");
 411                 return -ENOMEM;
 412         }
 413
 414         for (i = 0 ; i < count; i++) {
 415                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 416                         rc = -EFAULT;
 417                         break;
 418                 }
 419
 420                 /* skip if it's in this group already */
 421                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 0);
 422                 if (rc == 0)
 423                         continue;
 424
 425                 /* add to tmp group */
 426                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1);
 427                 if (rc != 0) {
 428                         CERROR("Can't create ndlink, out of memory\n");
 429                         break;
 430                 }
 431         }
 432
 433         if (rc != 0) {
 434                 lstcon_group_decref(tmp);
 435                 return rc;
 436         }
 437
 438         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 439                                      &tmp->grp_trans_list, LST_TRANS_SESNEW,
 440                                      tmp, lstcon_sesrpc_condition, &trans);
 441         if (rc != 0) {
 442                 CERROR("Can't create transaction: %d\n", rc);
 443                 lstcon_group_decref(tmp);
 444                 return rc;
 445         }
 446
 447         /* post all RPCs */
 448         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 449
 450         rc = lstcon_rpc_trans_interpreter(trans, result_up,
 451                                           lstcon_sesrpc_readent);
 452         *featp = trans->tas_features;
 453
 454         /* destroy all RPGs */
 455         lstcon_rpc_trans_destroy(trans);
 456
 457         lstcon_group_move(tmp, grp);
 458         lstcon_group_decref(tmp);
 459
 460         return rc;
 461 }
 462
 463 static int
 464 lstcon_group_nodes_remove(struct lstcon_group *grp,
 465                           int count, struct lnet_process_id __user *ids_up,
 466                           struct list_head __user *result_up)
 467 {
 468         struct lstcon_rpc_trans *trans;
 469         struct lstcon_ndlink *ndl;
 470         struct lstcon_group *tmp;
 471         struct lnet_process_id id;
 472         int rc;
 473         int i;
 474
 475         /* End session and remove node from the group */
 476
 477         rc = lstcon_group_alloc(NULL, &tmp);
 478         if (rc != 0) {
 479                 CERROR("Out of memory\n");
 480                 return -ENOMEM;
 481         }
 482
 483         for (i = 0; i < count; i++) {
 484                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 485                         rc = -EFAULT;
 486                         goto error;
 487                 }
 488
 489                 /* move node to tmp group */
 490                 if (lstcon_group_ndlink_find(grp, id, &ndl, 0) == 0)
 491                         lstcon_group_ndlink_move(grp, tmp, ndl);
 492         }
 493
 494         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 495                                      &tmp->grp_trans_list, LST_TRANS_SESEND,
 496                                      tmp, lstcon_sesrpc_condition, &trans);
 497         if (rc != 0) {
 498                 CERROR("Can't create transaction: %d\n", rc);
 499                 goto error;
 500         }
 501
 502         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 503
 504         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 505
 506         lstcon_rpc_trans_destroy(trans);
 507         /* release nodes anyway, because we can't rollback status */
 508         lstcon_group_decref(tmp);
 509
 510         return rc;
 511 error:
 512         lstcon_group_move(tmp, grp);
 513         lstcon_group_decref(tmp);
 514
 515         return rc;
 516 }
 517
 518 int
 519 lstcon_group_add(char *name)
 520 {
 521         struct lstcon_group *grp;
 522         int rc;
 523
 524         rc = (lstcon_group_find(name, &grp) == 0)? -EEXIST: 0;
 525         if (rc != 0) {
 526                 /* find a group with same name */
 527                 lstcon_group_decref(grp);
 528                 return rc;
 529         }
 530
 531         rc = lstcon_group_alloc(name, &grp);
 532         if (rc != 0) {
 533                 CERROR("Can't allocate descriptor for group %s\n", name);
 534                 return -ENOMEM;
 535         }
 536
 537         list_add_tail(&grp->grp_link, &console_session.ses_grp_list);
 538
 539         return rc;
 540 }
 541
 542 int
 543 lstcon_nodes_add(char *name, int count, struct lnet_process_id __user *ids_up,
 544                  unsigned *featp, struct list_head __user *result_up)
 545 {
 546         struct lstcon_group         *grp;
 547         int                     rc;
 548
 549         LASSERT (count > 0);
 550         LASSERT (ids_up != NULL);
 551
 552         rc = lstcon_group_find(name, &grp);
 553         if (rc != 0) {
 554                 CDEBUG(D_NET, "Can't find group %s\n", name);
 555                 return rc;
 556         }
 557
 558         if (grp->grp_ref > 2) {
 559                 /* referred by other threads or test */
 560                 CDEBUG(D_NET, "Group %s is busy\n", name);
 561                 lstcon_group_decref(grp);
 562
 563                 return -EBUSY;
 564         }
 565
 566         rc = lstcon_group_nodes_add(grp, count, ids_up, featp, result_up);
 567
 568         lstcon_group_decref(grp);
 569
 570         return rc;
 571 }
 572
 573 int
 574 lstcon_group_del(char *name)
 575 {
 576         struct lstcon_rpc_trans *trans;
 577         struct lstcon_group *grp;
 578         int rc;
 579
 580         rc = lstcon_group_find(name, &grp);
 581         if (rc != 0) {
 582                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 583                 return rc;
 584         }
 585
 586         if (grp->grp_ref > 2) {
 587                 /* referred by others threads or test */
 588                 CDEBUG(D_NET, "Group %s is busy\n", name);
 589                 lstcon_group_decref(grp);
 590                 return -EBUSY;
 591         }
 592
 593         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 594                                      &grp->grp_trans_list, LST_TRANS_SESEND,
 595                                      grp, lstcon_sesrpc_condition, &trans);
 596         if (rc != 0) {
 597                 CERROR("Can't create transaction: %d\n", rc);
 598                 lstcon_group_decref(grp);
 599                 return rc;
 600         }
 601
 602         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 603
 604         lstcon_rpc_trans_destroy(trans);
 605
 606         lstcon_group_decref(grp);
 607         /* -ref for session, it's destroyed,
 608          * status can't be rolled back, destroy group anway */
 609         lstcon_group_decref(grp);
 610
 611         return rc;
 612 }
 613
 614 int
 615 lstcon_group_clean(char *name, int args)
 616 {
 617         struct lstcon_group *grp = NULL;
 618         int rc;
 619
 620         rc = lstcon_group_find(name, &grp);
 621         if (rc != 0) {
 622                 CDEBUG(D_NET, "Can't find group %s\n", name);
 623                 return rc;
 624         }
 625
 626         if (grp->grp_ref > 2) {
 627                 /* referred by test */
 628                 CDEBUG(D_NET, "Group %s is busy\n", name);
 629                 lstcon_group_decref(grp);
 630                 return -EBUSY;
 631         }
 632
 633         args = (LST_NODE_ACTIVE | LST_NODE_BUSY |
 634                 LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args;
 635
 636         lstcon_group_drain(grp, args);
 637
 638         lstcon_group_decref(grp);
 639         /* release empty group */
 640         if (list_empty(&grp->grp_ndl_list))
 641                 lstcon_group_decref(grp);
 642
 643         return 0;
 644 }
 645
 646 int
 647 lstcon_nodes_remove(char *name, int count,
 648                     struct lnet_process_id __user *ids_up,
 649                     struct list_head __user *result_up)
 650 {
 651         struct lstcon_group *grp = NULL;
 652         int rc;
 653
 654         rc = lstcon_group_find(name, &grp);
 655         if (rc != 0) {
 656                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 657                 return rc;
 658         }
 659
 660         if (grp->grp_ref > 2) {
 661                 /* referred by test */
 662                 CDEBUG(D_NET, "Group %s is busy\n", name);
 663                 lstcon_group_decref(grp);
 664                 return -EBUSY;
 665         }
 666
 667         rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up);
 668
 669         lstcon_group_decref(grp);
 670         /* release empty group */
 671         if (list_empty(&grp->grp_ndl_list))
 672                 lstcon_group_decref(grp);
 673
 674         return rc;
 675 }
 676
 677 int
 678 lstcon_group_refresh(char *name, struct list_head __user *result_up)
 679 {
 680         struct lstcon_rpc_trans *trans;
 681         struct lstcon_group *grp;
 682         int rc;
 683
 684         rc = lstcon_group_find(name, &grp);
 685         if (rc != 0) {
 686                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 687                 return rc;
 688         }
 689
 690         if (grp->grp_ref > 2) {
 691                 /* referred by test */
 692                 CDEBUG(D_NET, "Group %s is busy\n", name);
 693                 lstcon_group_decref(grp);
 694                 return -EBUSY;
 695         }
 696
 697         /* re-invite all inactive nodes int the group */
 698         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 699                                      &grp->grp_trans_list, LST_TRANS_SESNEW,
 700                                      grp, lstcon_sesrpc_condition, &trans);
 701         if (rc != 0) {
 702                 /* local error, return */
 703                 CDEBUG(D_NET, "Can't create transaction: %d\n", rc);
 704                 lstcon_group_decref(grp);
 705                 return rc;
 706         }
 707
 708         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 709
 710         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 711
 712         lstcon_rpc_trans_destroy(trans);
 713         /* -ref for me */
 714         lstcon_group_decref(grp);
 715
 716         return rc;
 717 }
 718
 719 int
 720 lstcon_group_list(int index, int len, char __user *name_up)
 721 {
 722         struct lstcon_group *grp;
 723
 724         LASSERT(index >= 0);
 725         LASSERT(name_up != NULL);
 726
 727         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 728                 if (index-- == 0) {
 729                         return copy_to_user(name_up, grp->grp_name, len) ?
 730                                             -EFAULT : 0;
 731                 }
 732         }
 733
 734         return -ENOENT;
 735 }
 736
 737 static int
 738 lstcon_nodes_getent(struct list_head *head, int *index_p,
 739                     int *count_p, struct lstcon_node_ent __user *dents_up)
 740 {
 741         struct lstcon_ndlink *ndl;
 742         struct lstcon_node *nd;
 743         int count = 0;
 744         int index = 0;
 745
 746         LASSERT(index_p != NULL && count_p != NULL);
 747         LASSERT(dents_up != NULL);
 748         LASSERT(*index_p >= 0);
 749         LASSERT(*count_p > 0);
 750
 751         list_for_each_entry(ndl, head, ndl_link) {
 752                 if (index++ < *index_p)
 753                         continue;
 754
 755                 if (count >= *count_p)
 756                         break;
 757
 758                 nd = ndl->ndl_node;
 759                 if (copy_to_user(&dents_up[count].nde_id,
 760                                      &nd->nd_id, sizeof(nd->nd_id)) ||
 761                     copy_to_user(&dents_up[count].nde_state,
 762                                      &nd->nd_state, sizeof(nd->nd_state)))
 763                         return -EFAULT;
 764
 765                 count ++;
 766         }
 767
 768         if (index <= *index_p)
 769                 return -ENOENT;
 770
 771         *count_p = count;
 772         *index_p = index;
 773
 774         return 0;
 775 }
 776
 777 int
 778 lstcon_group_info(char *name, struct lstcon_ndlist_ent __user *gents_p,
 779                   int *index_p, int *count_p,
 780                   struct lstcon_node_ent __user *dents_up)
 781 {
 782         struct lstcon_ndlist_ent *gentp;
 783         struct lstcon_group *grp;
 784         struct lstcon_ndlink *ndl;
 785         int rc;
 786
 787         rc = lstcon_group_find(name, &grp);
 788         if (rc != 0) {
 789                 CDEBUG(D_NET, "Can't find group %s\n", name);
 790                 return rc;
 791         }
 792
 793         if (dents_up != NULL) {
 794                 /* verbose query */
 795                 rc = lstcon_nodes_getent(&grp->grp_ndl_list,
 796                                          index_p, count_p, dents_up);
 797                 lstcon_group_decref(grp);
 798
 799                 return rc;
 800         }
 801
 802         /* non-verbose query */
 803         CFS_ALLOC_PTR(gentp);
 804         if (gentp == NULL) {
 805                 CERROR("Can't allocate ndlist_ent\n");
 806                 lstcon_group_decref(grp);
 807
 808                 return -ENOMEM;
 809         }
 810
 811         list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link)
 812                 LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp);
 813
 814         rc = copy_to_user(gents_p, gentp,
 815                           sizeof(struct lstcon_ndlist_ent)) ? -EFAULT : 0;
 816
 817         CFS_FREE_PTR(gentp);
 818
 819         lstcon_group_decref(grp);
 820
 821         return 0;
 822 }
 823
 824 static int
 825 lstcon_batch_find(const char *name, struct lstcon_batch **batpp)
 826 {
 827         struct lstcon_batch *bat;
 828
 829         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 830                 if (strncmp(bat->bat_name, name, LST_NAME_SIZE) == 0) {
 831                         *batpp = bat;
 832                         return 0;
 833                 }
 834         }
 835
 836         return -ENOENT;
 837 }
 838
 839 int
 840 lstcon_batch_add(char *name)
 841 {
 842         struct lstcon_batch *bat;
 843         int i;
 844         int rc;
 845
 846         rc = (lstcon_batch_find(name, &bat) == 0)? -EEXIST: 0;
 847         if (rc != 0) {
 848                 CDEBUG(D_NET, "Batch %s already exists\n", name);
 849                 return rc;
 850         }
 851
 852         LIBCFS_ALLOC(bat, sizeof(*bat));
 853         if (bat == NULL) {
 854                 CERROR("Can't allocate descriptor for batch %s\n", name);
 855                 return -ENOMEM;
 856         }
 857
 858         LIBCFS_ALLOC(bat->bat_cli_hash,
 859                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 860         if (bat->bat_cli_hash == NULL) {
 861                 CERROR("Can't allocate hash for batch %s\n", name);
 862                 LIBCFS_FREE(bat, sizeof(*bat));
 863
 864                 return -ENOMEM;
 865         }
 866
 867         LIBCFS_ALLOC(bat->bat_srv_hash,
 868                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 869         if (bat->bat_srv_hash == NULL) {
 870                 CERROR("Can't allocate hash for batch %s\n", name);
 871                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 872                 LIBCFS_FREE(bat, sizeof(*bat));
 873
 874                 return -ENOMEM;
 875         }
 876
 877         if (strlen(name) > sizeof(bat->bat_name)-1) {
 878                 LIBCFS_FREE(bat->bat_srv_hash, LST_NODE_HASHSIZE);
 879                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 880                 LIBCFS_FREE(bat, sizeof(*bat));
 881                 return -E2BIG;
 882         }
 883         strncpy(bat->bat_name, name, sizeof(bat->bat_name));
 884         bat->bat_hdr.tsb_index = 0;
 885         bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie;
 886
 887         bat->bat_ntest = 0;
 888         bat->bat_state = LST_BATCH_IDLE;
 889
 890         INIT_LIST_HEAD(&bat->bat_cli_list);
 891         INIT_LIST_HEAD(&bat->bat_srv_list);
 892         INIT_LIST_HEAD(&bat->bat_test_list);
 893         INIT_LIST_HEAD(&bat->bat_trans_list);
 894
 895         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 896                 INIT_LIST_HEAD(&bat->bat_cli_hash[i]);
 897                 INIT_LIST_HEAD(&bat->bat_srv_hash[i]);
 898         }
 899
 900         list_add_tail(&bat->bat_link, &console_session.ses_bat_list);
 901
 902         return rc;
 903 }
 904
 905 int
 906 lstcon_batch_list(int index, int len, char __user *name_up)
 907 {
 908         struct lstcon_batch *bat;
 909
 910         LASSERT(name_up != NULL);
 911         LASSERT(index >= 0);
 912
 913         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 914                 if (index-- == 0) {
 915                         return copy_to_user(name_up, bat->bat_name, len) ?
 916                                             -EFAULT : 0;
 917                 }
 918         }
 919
 920         return -ENOENT;
 921 }
 922
 923 int
 924 lstcon_batch_info(char *name, struct lstcon_test_batch_ent __user *ent_up,
 925                   int server, int testidx, int *index_p, int *ndent_p,
 926                   struct lstcon_node_ent __user *dents_up)
 927 {
 928         struct lstcon_test_batch_ent *entp;
 929         struct list_head *clilst;
 930         struct list_head *srvlst;
 931         struct lstcon_test *test = NULL;
 932         struct lstcon_batch *bat;
 933         struct lstcon_ndlink *ndl;
 934         int rc;
 935
 936         rc = lstcon_batch_find(name, &bat);
 937         if (rc != 0) {
 938                 CDEBUG(D_NET, "Can't find batch %s\n", name);
 939                 return -ENOENT;
 940         }
 941
 942         if (testidx > 0) {
 943                 /* query test, test index start from 1 */
 944                 list_for_each_entry(test, &bat->bat_test_list, tes_link) {
 945                         if (testidx-- == 1)
 946                                 break;
 947                 }
 948
 949                 if (testidx > 0) {
 950                         CDEBUG(D_NET, "Can't find specified test in batch\n");
 951                         return -ENOENT;
 952                 }
 953         }
 954
 955         clilst = (test == NULL) ? &bat->bat_cli_list :
 956                                   &test->tes_src_grp->grp_ndl_list;
 957         srvlst = (test == NULL) ? &bat->bat_srv_list :
 958                                   &test->tes_dst_grp->grp_ndl_list;
 959
 960         if (dents_up != NULL) {
 961                 rc = lstcon_nodes_getent((server ? srvlst: clilst),
 962                                          index_p, ndent_p, dents_up);
 963                 return rc;
 964         }
 965
 966         /* non-verbose query */
 967         CFS_ALLOC_PTR(entp);
 968         if (entp == NULL)
 969                 return -ENOMEM;
 970
 971         if (test == NULL) {
 972                 entp->u.tbe_batch.bae_ntest = bat->bat_ntest;
 973                 entp->u.tbe_batch.bae_state = bat->bat_state;
 974
 975         } else {
 976
 977                 entp->u.tbe_test.tse_type   = test->tes_type;
 978                 entp->u.tbe_test.tse_loop   = test->tes_loop;
 979                 entp->u.tbe_test.tse_concur = test->tes_concur;
 980         }
 981
 982         list_for_each_entry(ndl, clilst, ndl_link)
 983                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle);
 984
 985         list_for_each_entry(ndl, srvlst, ndl_link)
 986                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle);
 987
 988         rc = copy_to_user(ent_up, entp,
 989                           sizeof(struct lstcon_test_batch_ent)) ? -EFAULT : 0;
 990
 991         CFS_FREE_PTR(entp)
 992
 993         return rc;
 994 }
 995
 996 static int
 997 lstcon_batrpc_condition(int transop, struct lstcon_node *nd, void *arg)
 998 {
 999         switch (transop) {
1000         case LST_TRANS_TSBRUN:
1001                 if (nd->nd_state != LST_NODE_ACTIVE)
1002                         return -ENETDOWN;
1003                 break;
1004
1005         case LST_TRANS_TSBSTOP:
1006                 if (nd->nd_state != LST_NODE_ACTIVE)
1007                         return 0;
1008                 break;
1009
1010         case LST_TRANS_TSBCLIQRY:
1011         case LST_TRANS_TSBSRVQRY:
1012                 break;
1013         }
1014
1015         return 1;
1016 }
1017
1018 static int
1019 lstcon_batch_op(struct lstcon_batch *bat, int transop,
1020                 struct list_head __user *result_up)
1021 {
1022         struct lstcon_rpc_trans *trans;
1023         int                 rc;
1024
1025         rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list,
1026                                      &bat->bat_trans_list, transop,
1027                                      bat, lstcon_batrpc_condition, &trans);
1028         if (rc != 0) {
1029                 CERROR("Can't create transaction: %d\n", rc);
1030                 return rc;
1031         }
1032
1033         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1034
1035         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1036
1037         lstcon_rpc_trans_destroy(trans);
1038
1039         return rc;
1040 }
1041
1042 int
1043 lstcon_batch_run(char *name, int timeout, struct list_head __user *result_up)
1044 {
1045         struct lstcon_batch *bat;
1046         int rc;
1047
1048         if (lstcon_batch_find(name, &bat) != 0) {
1049                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1050                 return -ENOENT;
1051         }
1052
1053         bat->bat_arg = timeout;
1054
1055         rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up);
1056
1057         /* mark batch as running if it's started in any node */
1058         if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0) != 0)
1059                 bat->bat_state = LST_BATCH_RUNNING;
1060
1061         return rc;
1062 }
1063
1064 int
1065 lstcon_batch_stop(char *name, int force, struct list_head __user *result_up)
1066 {
1067         struct lstcon_batch *bat;
1068         int rc;
1069
1070         if (lstcon_batch_find(name, &bat) != 0) {
1071                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1072                 return -ENOENT;
1073         }
1074
1075         bat->bat_arg = force;
1076
1077         rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up);
1078
1079         /* mark batch as stopped if all RPCs finished */
1080         if (lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0) == 0)
1081                 bat->bat_state = LST_BATCH_IDLE;
1082
1083         return rc;
1084 }
1085
1086 static void
1087 lstcon_batch_destroy(struct lstcon_batch *bat)
1088 {
1089         struct lstcon_ndlink *ndl;
1090         struct lstcon_test *test;
1091         int i;
1092
1093         list_del(&bat->bat_link);
1094
1095         while (!list_empty(&bat->bat_test_list)) {
1096                 test = list_entry(bat->bat_test_list.next,
1097                                   struct lstcon_test, tes_link);
1098                 LASSERT(list_empty(&test->tes_trans_list));
1099
1100                 list_del(&test->tes_link);
1101
1102                 lstcon_group_decref(test->tes_src_grp);
1103                 lstcon_group_decref(test->tes_dst_grp);
1104
1105                 LIBCFS_FREE(test, offsetof(struct lstcon_test,
1106                                            tes_param[test->tes_paramlen]));
1107         }
1108
1109         LASSERT(list_empty(&bat->bat_trans_list));
1110
1111         while (!list_empty(&bat->bat_cli_list)) {
1112                 ndl = list_entry(bat->bat_cli_list.next,
1113                                  struct lstcon_ndlink, ndl_link);
1114                 list_del_init(&ndl->ndl_link);
1115
1116                 lstcon_ndlink_release(ndl);
1117         }
1118
1119         while (!list_empty(&bat->bat_srv_list)) {
1120                 ndl = list_entry(bat->bat_srv_list.next,
1121                                  struct lstcon_ndlink, ndl_link);
1122                 list_del_init(&ndl->ndl_link);
1123
1124                 lstcon_ndlink_release(ndl);
1125         }
1126
1127         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1128                 LASSERT(list_empty(&bat->bat_cli_hash[i]));
1129                 LASSERT(list_empty(&bat->bat_srv_hash[i]));
1130         }
1131
1132         LIBCFS_FREE(bat->bat_cli_hash,
1133                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1134         LIBCFS_FREE(bat->bat_srv_hash,
1135                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1136         LIBCFS_FREE(bat, sizeof(*bat));
1137 }
1138
1139 static int
1140 lstcon_testrpc_condition(int transop, struct lstcon_node *nd, void *arg)
1141 {
1142         struct lstcon_test *test = arg;
1143         struct lstcon_batch *batch;
1144         struct lstcon_ndlink *ndl;
1145         struct list_head *hash;
1146         struct list_head *head;
1147
1148         LASSERT(test != NULL);
1149
1150         batch = test->tes_batch;
1151         LASSERT(batch != NULL);
1152
1153         if (test->tes_oneside &&
1154             transop == LST_TRANS_TSBSRVADD)
1155                 return 0;
1156
1157         if (nd->nd_state != LST_NODE_ACTIVE)
1158                 return -ENETDOWN;
1159
1160         if (transop == LST_TRANS_TSBCLIADD) {
1161                 hash = batch->bat_cli_hash;
1162                 head = &batch->bat_cli_list;
1163
1164         } else {
1165                 LASSERT (transop == LST_TRANS_TSBSRVADD);
1166
1167                 hash = batch->bat_srv_hash;
1168                 head = &batch->bat_srv_list;
1169         }
1170
1171         LASSERT (nd->nd_id.nid != LNET_NID_ANY);
1172
1173         if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1) != 0)
1174                 return -ENOMEM;
1175
1176         if (list_empty(&ndl->ndl_link))
1177                 list_add_tail(&ndl->ndl_link, head);
1178
1179         return 1;
1180 }
1181
1182 static int
1183 lstcon_test_nodes_add(struct lstcon_test *test,
1184                       struct list_head __user *result_up)
1185 {
1186         struct lstcon_rpc_trans *trans;
1187         struct lstcon_group *grp;
1188         int transop;
1189         int rc;
1190
1191         LASSERT (test->tes_src_grp != NULL);
1192         LASSERT (test->tes_dst_grp != NULL);
1193
1194         transop = LST_TRANS_TSBSRVADD;
1195         grp  = test->tes_dst_grp;
1196 again:
1197         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
1198                                      &test->tes_trans_list, transop,
1199                                      test, lstcon_testrpc_condition, &trans);
1200         if (rc != 0) {
1201                 CERROR("Can't create transaction: %d\n", rc);
1202                 return rc;
1203         }
1204
1205         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1206
1207         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1208             lstcon_trans_stat()->trs_fwk_errno != 0) {
1209                 lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1210
1211                 lstcon_rpc_trans_destroy(trans);
1212                 /* return if any error */
1213                 CDEBUG(D_NET, "Failed to add test %s, "
1214                               "RPC error %d, framework error %d\n",
1215                        transop == LST_TRANS_TSBCLIADD ? "client" : "server",
1216                        lstcon_trans_stat()->trs_rpc_errno,
1217                        lstcon_trans_stat()->trs_fwk_errno);
1218
1219                 return rc;
1220         }
1221
1222         lstcon_rpc_trans_destroy(trans);
1223
1224         if (transop == LST_TRANS_TSBCLIADD)
1225                 return rc;
1226
1227         transop = LST_TRANS_TSBCLIADD;
1228         grp = test->tes_src_grp;
1229         test->tes_cliidx = 0;
1230
1231         /* requests to test clients */
1232         goto again;
1233 }
1234
1235 static int
1236 lstcon_verify_batch(const char *name, struct lstcon_batch **batch)
1237 {
1238         int rc;
1239
1240         rc = lstcon_batch_find(name, batch);
1241         if (rc != 0) {
1242                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1243                 return rc;
1244         }
1245
1246         if ((*batch)->bat_state != LST_BATCH_IDLE) {
1247                 CDEBUG(D_NET, "Can't change running batch %s\n", name);
1248                 return -EINVAL;
1249         }
1250
1251         return 0;
1252 }
1253
1254 static int
1255 lstcon_verify_group(const char *name, struct lstcon_group **grp)
1256 {
1257         int rc;
1258         struct lstcon_ndlink *ndl;
1259
1260         rc = lstcon_group_find(name, grp);
1261         if (rc != 0) {
1262                 CDEBUG(D_NET, "can't find group %s\n", name);
1263                 return rc;
1264         }
1265
1266         list_for_each_entry(ndl, &(*grp)->grp_ndl_list, ndl_link) {
1267                 if (ndl->ndl_node->nd_state == LST_NODE_ACTIVE) {
1268                         return 0;
1269                 }
1270         }
1271
1272         CDEBUG(D_NET, "Group %s has no ACTIVE nodes\n", name);
1273
1274         return -EINVAL;
1275 }
1276
1277 int
1278 lstcon_test_add(char *batch_name, int type, int loop,
1279                 int concur, int dist, int span,
1280                 char *src_name, char *dst_name,
1281                 void *param, int paramlen, int *retp,
1282                 struct list_head __user *result_up)
1283 {
1284         struct lstcon_test *test = NULL;
1285         int rc;
1286         struct lstcon_group *src_grp = NULL;
1287         struct lstcon_group *dst_grp = NULL;
1288         struct lstcon_batch *batch = NULL;
1289
1290         /*
1291          * verify that a batch of the given name exists, and the groups
1292          * that will be part of the batch exist and have at least one
1293          * active node
1294          */
1295         rc = lstcon_verify_batch(batch_name, &batch);
1296         if (rc != 0)
1297                 goto out;
1298
1299         rc = lstcon_verify_group(src_name, &src_grp);
1300         if (rc != 0)
1301                 goto out;
1302
1303         rc = lstcon_verify_group(dst_name, &dst_grp);
1304         if (rc != 0)
1305                 goto out;
1306
1307         if (dst_grp->grp_userland)
1308                 *retp = 1;
1309
1310         LIBCFS_ALLOC(test, offsetof(struct lstcon_test, tes_param[paramlen]));
1311         if (!test) {
1312                 CERROR("Can't allocate test descriptor\n");
1313                 rc = -ENOMEM;
1314
1315                 goto out;
1316         }
1317
1318         test->tes_hdr.tsb_id    = batch->bat_hdr.tsb_id;
1319         test->tes_batch         = batch;
1320         test->tes_type          = type;
1321         test->tes_oneside       = 0; /* TODO */
1322         test->tes_loop          = loop;
1323         test->tes_concur        = concur;
1324         test->tes_stop_onerr    = 1; /* TODO */
1325         test->tes_span          = span;
1326         test->tes_dist          = dist;
1327         test->tes_cliidx        = 0; /* just used for creating RPC */
1328         test->tes_src_grp       = src_grp;
1329         test->tes_dst_grp       = dst_grp;
1330         INIT_LIST_HEAD(&test->tes_trans_list);
1331
1332         if (param != NULL) {
1333                 test->tes_paramlen = paramlen;
1334                 memcpy(&test->tes_param[0], param, paramlen);
1335         }
1336
1337         rc = lstcon_test_nodes_add(test, result_up);
1338
1339         if (rc != 0)
1340                 goto out;
1341
1342         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1343             lstcon_trans_stat()->trs_fwk_errno != 0)
1344                 CDEBUG(D_NET, "Failed to add test %d to batch %s\n", type,
1345                        batch_name);
1346
1347         /* add to test list anyway, so user can check what's going on */
1348         list_add_tail(&test->tes_link, &batch->bat_test_list);
1349
1350         batch->bat_ntest++;
1351         test->tes_hdr.tsb_index = batch->bat_ntest;
1352
1353         /*  hold groups so nobody can change them */
1354         return rc;
1355 out:
1356         if (test != NULL)
1357                 LIBCFS_FREE(test, offsetof(struct lstcon_test,
1358                                            tes_param[paramlen]));
1359
1360         if (dst_grp != NULL)
1361                 lstcon_group_decref(dst_grp);
1362
1363         if (src_grp != NULL)
1364                 lstcon_group_decref(src_grp);
1365
1366         return rc;
1367 }
1368
1369 static int
1370 lstcon_test_find(struct lstcon_batch *batch, int idx,
1371                  struct lstcon_test **testpp)
1372 {
1373         struct lstcon_test *test;
1374
1375         list_for_each_entry(test, &batch->bat_test_list, tes_link) {
1376                 if (idx == test->tes_hdr.tsb_index) {
1377                         *testpp = test;
1378                         return 0;
1379                 }
1380         }
1381
1382         return -ENOENT;
1383 }
1384
1385 static int
1386 lstcon_tsbrpc_readent(int transop, struct srpc_msg *msg,
1387                       struct lstcon_rpc_ent __user *ent_up)
1388 {
1389         struct srpc_batch_reply *rep = &msg->msg_body.bat_reply;
1390
1391         LASSERT (transop == LST_TRANS_TSBCLIQRY ||
1392                  transop == LST_TRANS_TSBSRVQRY);
1393
1394         /* positive errno, framework error code */
1395         if (copy_to_user(&ent_up->rpe_priv[0],
1396                              &rep->bar_active, sizeof(rep->bar_active)))
1397                 return -EFAULT;
1398
1399         return 0;
1400 }
1401
1402 int
1403 lstcon_test_batch_query(char *name, int testidx, int client,
1404                         int timeout, struct list_head __user *result_up)
1405 {
1406         struct lstcon_rpc_trans *trans;
1407         struct list_head *translist;
1408         struct list_head *ndlist;
1409         struct lstcon_tsb_hdr *hdr;
1410         struct lstcon_batch *batch;
1411         struct lstcon_test *test = NULL;
1412         int transop;
1413         int rc;
1414
1415         rc = lstcon_batch_find(name, &batch);
1416         if (rc != 0) {
1417                 CDEBUG(D_NET, "Can't find batch: %s\n", name);
1418                 return rc;
1419         }
1420
1421         if (testidx == 0) {
1422                 translist = &batch->bat_trans_list;
1423                 ndlist    = &batch->bat_cli_list;
1424                 hdr       = &batch->bat_hdr;
1425
1426         } else {
1427                 /* query specified test only */
1428                 rc = lstcon_test_find(batch, testidx, &test);
1429                 if (rc != 0) {
1430                         CDEBUG(D_NET, "Can't find test: %d\n", testidx);
1431                         return rc;
1432                 }
1433
1434                 translist = &test->tes_trans_list;
1435                 ndlist    = &test->tes_src_grp->grp_ndl_list;
1436                 hdr       = &test->tes_hdr;
1437         }
1438
1439         transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY;
1440
1441         rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr,
1442                                      lstcon_batrpc_condition, &trans);
1443         if (rc != 0) {
1444                 CERROR("Can't create transaction: %d\n", rc);
1445                 return rc;
1446         }
1447
1448         lstcon_rpc_trans_postwait(trans, timeout);
1449
1450         if (testidx == 0 && /* query a batch, not a test */
1451             lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) == 0 &&
1452             lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0) == 0) {
1453                 /* all RPCs finished, and no active test */
1454                 batch->bat_state = LST_BATCH_IDLE;
1455         }
1456
1457         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1458                                           lstcon_tsbrpc_readent);
1459         lstcon_rpc_trans_destroy(trans);
1460
1461         return rc;
1462 }
1463
1464 static int
1465 lstcon_statrpc_readent(int transop, struct srpc_msg *msg,
1466                        struct lstcon_rpc_ent __user *ent_up)
1467 {
1468         struct srpc_stat_reply *rep = &msg->msg_body.stat_reply;
1469         struct sfw_counters __user *sfwk_stat;
1470         struct srpc_counters __user *srpc_stat;
1471         struct lnet_counters_common __user *lnet_stat;
1472
1473         if (rep->str_status != 0)
1474                 return 0;
1475
1476         sfwk_stat = (struct sfw_counters __user *)&ent_up->rpe_payload[0];
1477         srpc_stat = (struct srpc_counters __user *)
1478                 ((char __user *)sfwk_stat + sizeof(*sfwk_stat));
1479         lnet_stat = (struct lnet_counters_common __user *)
1480                 ((char __user *)srpc_stat + sizeof(*srpc_stat));
1481
1482         if (copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) ||
1483             copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) ||
1484             copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat)))
1485                 return -EFAULT;
1486
1487         return 0;
1488 }
1489
1490 static int
1491 lstcon_ndlist_stat(struct list_head *ndlist,
1492                    int timeout, struct list_head __user *result_up)
1493 {
1494         LIST_HEAD(head);
1495         struct lstcon_rpc_trans *trans;
1496         int rc;
1497
1498         rc = lstcon_rpc_trans_ndlist(ndlist, &head,
1499                                      LST_TRANS_STATQRY, NULL, NULL, &trans);
1500         if (rc != 0) {
1501                 CERROR("Can't create transaction: %d\n", rc);
1502                 return rc;
1503         }
1504
1505         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1506
1507         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1508                                           lstcon_statrpc_readent);
1509         lstcon_rpc_trans_destroy(trans);
1510
1511         return rc;
1512 }
1513
1514 int
1515 lstcon_group_stat(char *grp_name, int timeout,
1516                   struct list_head __user *result_up)
1517 {
1518         struct lstcon_group *grp;
1519         int rc;
1520
1521         rc = lstcon_group_find(grp_name, &grp);
1522         if (rc != 0) {
1523                 CDEBUG(D_NET, "Can't find group %s\n", grp_name);
1524                 return rc;
1525         }
1526
1527         rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up);
1528
1529         lstcon_group_decref(grp);
1530
1531         return rc;
1532 }
1533
1534 int
1535 lstcon_nodes_stat(int count, struct lnet_process_id __user *ids_up,
1536                   int timeout, struct list_head __user *result_up)
1537 {
1538         struct lstcon_ndlink *ndl;
1539         struct lstcon_group *tmp;
1540         struct lnet_process_id id;
1541         int i;
1542         int rc;
1543
1544         rc = lstcon_group_alloc(NULL, &tmp);
1545         if (rc != 0) {
1546                 CERROR("Out of memory\n");
1547                 return -ENOMEM;
1548         }
1549
1550         for (i = 0 ; i < count; i++) {
1551                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1552                         rc = -EFAULT;
1553                         break;
1554                 }
1555
1556                 /* add to tmp group */
1557                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2);
1558                 if (rc != 0) {
1559                         CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET,
1560                                "Failed to find or create %s: %d\n",
1561                                libcfs_id2str(id), rc);
1562                         break;
1563                 }
1564         }
1565
1566         if (rc != 0) {
1567                 lstcon_group_decref(tmp);
1568                 return rc;
1569         }
1570
1571         rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up);
1572
1573         lstcon_group_decref(tmp);
1574
1575         return rc;
1576 }
1577
1578 static int
1579 lstcon_debug_ndlist(struct list_head *ndlist,
1580                     struct list_head *translist,
1581                     int timeout, struct list_head __user *result_up)
1582 {
1583         struct lstcon_rpc_trans *trans;
1584         int rc;
1585
1586         rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY,
1587                                      NULL, lstcon_sesrpc_condition, &trans);
1588         if (rc != 0) {
1589                 CERROR("Can't create transaction: %d\n", rc);
1590                 return rc;
1591         }
1592
1593         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1594
1595         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1596                                           lstcon_sesrpc_readent);
1597         lstcon_rpc_trans_destroy(trans);
1598
1599         return rc;
1600 }
1601
1602 int
1603 lstcon_session_debug(int timeout, struct list_head __user *result_up)
1604 {
1605         return lstcon_debug_ndlist(&console_session.ses_ndl_list,
1606                                    NULL, timeout, result_up);
1607 }
1608
1609 int
1610 lstcon_batch_debug(int timeout, char *name,
1611                    int client, struct list_head __user *result_up)
1612 {
1613         struct lstcon_batch *bat;
1614         int rc;
1615
1616         rc = lstcon_batch_find(name, &bat);
1617         if (rc != 0)
1618                 return -ENOENT;
1619
1620         rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list :
1621                                           &bat->bat_srv_list,
1622                                  NULL, timeout, result_up);
1623
1624         return rc;
1625 }
1626
1627 int
1628 lstcon_group_debug(int timeout, char *name,
1629                    struct list_head __user *result_up)
1630 {
1631         struct lstcon_group *grp;
1632         int rc;
1633
1634         rc = lstcon_group_find(name, &grp);
1635         if (rc != 0)
1636                 return -ENOENT;
1637
1638         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1639                                  timeout, result_up);
1640         lstcon_group_decref(grp);
1641
1642         return rc;
1643 }
1644
1645 int
1646 lstcon_nodes_debug(int timeout, int count,
1647                    struct lnet_process_id __user *ids_up,
1648                    struct list_head __user *result_up)
1649 {
1650         struct lnet_process_id id;
1651         struct lstcon_ndlink *ndl;
1652         struct lstcon_group *grp;
1653         int i;
1654         int rc;
1655
1656         rc = lstcon_group_alloc(NULL, &grp);
1657         if (rc != 0) {
1658                 CDEBUG(D_NET, "Out of memory\n");
1659                 return rc;
1660         }
1661
1662         for (i = 0; i < count; i++) {
1663                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1664                         rc = -EFAULT;
1665                         break;
1666                 }
1667
1668                 /* node is added to tmp group */
1669                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 1);
1670                 if (rc != 0) {
1671                         CERROR("Can't create node link\n");
1672                         break;
1673                 }
1674         }
1675
1676         if (rc != 0) {
1677                 lstcon_group_decref(grp);
1678                 return rc;
1679         }
1680
1681         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1682                                  timeout, result_up);
1683
1684         lstcon_group_decref(grp);
1685
1686         return rc;
1687 }
1688
1689 int
1690 lstcon_session_match(struct lst_sid sid)
1691 {
1692         return (console_session.ses_id.ses_nid   == sid.ses_nid &&
1693                 console_session.ses_id.ses_stamp == sid.ses_stamp) ?  1: 0;
1694 }
1695
1696 static void
1697 lstcon_new_session_id(struct lst_sid *sid)
1698 {
1699         struct lnet_process_id id;
1700
1701         LASSERT(console_session.ses_state == LST_SESSION_NONE);
1702
1703         LNetGetId(1, &id);
1704         sid->ses_nid = id.nid;
1705         sid->ses_stamp = div_u64(ktime_get_ns(), NSEC_PER_MSEC);
1706 }
1707
1708 int
1709 lstcon_session_new(char *name, int key, unsigned feats,
1710                    int timeout, int force, struct lst_sid __user *sid_up)
1711 {
1712         int     rc = 0;
1713         int     i;
1714
1715         if (console_session.ses_state != LST_SESSION_NONE) {
1716                 /* session exists */
1717                 if (!force) {
1718                         CNETERR("Session %s already exists\n",
1719                                 console_session.ses_name);
1720                         return -EEXIST;
1721                 }
1722
1723                 rc = lstcon_session_end();
1724
1725                 /* lstcon_session_end() only return local error */
1726                 if  (rc != 0)
1727                         return rc;
1728         }
1729
1730         if ((feats & ~LST_FEATS_MASK) != 0) {
1731                 CNETERR("Unknown session features %x\n",
1732                         (feats & ~LST_FEATS_MASK));
1733                 return -EINVAL;
1734         }
1735
1736         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
1737                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
1738
1739         lstcon_new_session_id(&console_session.ses_id);
1740
1741         console_session.ses_key     = key;
1742         console_session.ses_state   = LST_SESSION_ACTIVE;
1743         console_session.ses_force   = !!force;
1744         console_session.ses_features = feats;
1745         console_session.ses_feats_updated = 0;
1746         console_session.ses_timeout = (timeout <= 0) ?
1747                                       LST_CONSOLE_TIMEOUT : timeout;
1748
1749         if (strlen(name) > sizeof(console_session.ses_name)-1)
1750                 return -E2BIG;
1751         strlcpy(console_session.ses_name, name,
1752                 sizeof(console_session.ses_name));
1753
1754         rc = lstcon_batch_add(LST_DEFAULT_BATCH);
1755         if (rc != 0)
1756                 return rc;
1757
1758         rc = lstcon_rpc_pinger_start();
1759         if (rc != 0) {
1760                 struct lstcon_batch *bat = NULL;
1761
1762                 lstcon_batch_find(LST_DEFAULT_BATCH, &bat);
1763                 lstcon_batch_destroy(bat);
1764
1765                 return rc;
1766         }
1767
1768         if (copy_to_user(sid_up, &console_session.ses_id,
1769                              sizeof(struct lst_sid)) == 0)
1770                 return rc;
1771
1772         lstcon_session_end();
1773
1774         return -EFAULT;
1775 }
1776
1777 int
1778 lstcon_session_info(struct lst_sid __user *sid_up, int __user *key_up,
1779                     unsigned __user *featp,
1780                     struct lstcon_ndlist_ent __user *ndinfo_up,
1781                     char __user *name_up, int len)
1782 {
1783         struct lstcon_ndlist_ent *entp;
1784         struct lstcon_ndlink *ndl;
1785         int rc = 0;
1786
1787         if (console_session.ses_state != LST_SESSION_ACTIVE)
1788                 return -ESRCH;
1789
1790         LIBCFS_ALLOC(entp, sizeof(*entp));
1791         if (entp == NULL)
1792                 return -ENOMEM;
1793
1794         list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link)
1795                 LST_NODE_STATE_COUNTER(ndl->ndl_node, entp);
1796
1797         if (copy_to_user(sid_up, &console_session.ses_id,
1798                          sizeof(struct lst_sid)) ||
1799             copy_to_user(key_up, &console_session.ses_key,
1800                              sizeof(*key_up)) ||
1801             copy_to_user(featp, &console_session.ses_features,
1802                              sizeof(*featp)) ||
1803             copy_to_user(ndinfo_up, entp, sizeof(*entp)) ||
1804             copy_to_user(name_up, console_session.ses_name, len))
1805                 rc = -EFAULT;
1806
1807         LIBCFS_FREE(entp, sizeof(*entp));
1808
1809         return rc;
1810 }
1811
1812 int
1813 lstcon_session_end(void)
1814 {
1815         struct lstcon_rpc_trans *trans;
1816         struct lstcon_group *grp;
1817         struct lstcon_batch *bat;
1818         int rc = 0;
1819
1820         LASSERT (console_session.ses_state == LST_SESSION_ACTIVE);
1821
1822         rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list,
1823                                      NULL, LST_TRANS_SESEND, NULL,
1824                                      lstcon_sesrpc_condition, &trans);
1825         if (rc != 0) {
1826                 CERROR("Can't create transaction: %d\n", rc);
1827                 return rc;
1828         }
1829
1830         console_session.ses_shutdown = 1;
1831
1832         lstcon_rpc_pinger_stop();
1833
1834         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1835
1836         lstcon_rpc_trans_destroy(trans);
1837         /* User can do nothing even rpc failed, so go on */
1838
1839         /* waiting for orphan rpcs to die */
1840         lstcon_rpc_cleanup_wait();
1841
1842         console_session.ses_id    = LST_INVALID_SID;
1843         console_session.ses_state = LST_SESSION_NONE;
1844         console_session.ses_key   = 0;
1845         console_session.ses_force = 0;
1846         console_session.ses_feats_updated = 0;
1847
1848         /* destroy all batches */
1849         while (!list_empty(&console_session.ses_bat_list)) {
1850                 bat = list_entry(console_session.ses_bat_list.next,
1851                                  struct lstcon_batch, bat_link);
1852
1853                 lstcon_batch_destroy(bat);
1854         }
1855
1856         /* destroy all groups */
1857         while (!list_empty(&console_session.ses_grp_list)) {
1858                 grp = list_entry(console_session.ses_grp_list.next,
1859                                  struct lstcon_group, grp_link);
1860                 LASSERT(grp->grp_ref == 1);
1861
1862                 lstcon_group_decref(grp);
1863         }
1864
1865         /* all nodes should be released */
1866         LASSERT(list_empty(&console_session.ses_ndl_list));
1867
1868         console_session.ses_shutdown = 0;
1869         console_session.ses_expired  = 0;
1870
1871         return rc;
1872 }
1873
1874 int
1875 lstcon_session_feats_check(unsigned feats)
1876 {
1877         int rc = 0;
1878
1879         if ((feats & ~LST_FEATS_MASK) != 0) {
1880                 CERROR("Can't support these features: %x\n",
1881                        (feats & ~LST_FEATS_MASK));
1882                 return -EPROTO;
1883         }
1884
1885         spin_lock(&console_session.ses_rpc_lock);
1886
1887         if (!console_session.ses_feats_updated) {
1888                 console_session.ses_feats_updated = 1;
1889                 console_session.ses_features = feats;
1890         }
1891
1892         if (console_session.ses_features != feats)
1893                 rc = -EPROTO;
1894
1895         spin_unlock(&console_session.ses_rpc_lock);
1896
1897         if (rc != 0) {
1898                 CERROR("remote features %x do not match with "
1899                        "session features %x of console\n",
1900                        feats, console_session.ses_features);
1901         }
1902
1903         return rc;
1904 }
1905
1906 static int
1907 lstcon_acceptor_handle(struct srpc_server_rpc *rpc)
1908 {
1909         struct srpc_msg *rep = &rpc->srpc_replymsg;
1910         struct srpc_msg *req = &rpc->srpc_reqstbuf->buf_msg;
1911         struct srpc_join_reqst *jreq = &req->msg_body.join_reqst;
1912         struct srpc_join_reply *jrep = &rep->msg_body.join_reply;
1913         struct lstcon_group *grp = NULL;
1914         struct lstcon_ndlink *ndl;
1915         int rc = 0;
1916
1917         sfw_unpack_message(req);
1918
1919         mutex_lock(&console_session.ses_mutex);
1920
1921         jrep->join_sid = console_session.ses_id;
1922
1923         if (console_session.ses_id.ses_nid == LNET_NID_ANY) {
1924                 jrep->join_status = ESRCH;
1925                 goto out;
1926         }
1927
1928         if (lstcon_session_feats_check(req->msg_ses_feats) != 0) {
1929                 jrep->join_status = EPROTO;
1930                 goto out;
1931         }
1932
1933         if (jreq->join_sid.ses_nid != LNET_NID_ANY &&
1934              !lstcon_session_match(jreq->join_sid)) {
1935                 jrep->join_status = EBUSY;
1936                 goto out;
1937         }
1938
1939         if (lstcon_group_find(jreq->join_group, &grp) != 0) {
1940                 rc = lstcon_group_alloc(jreq->join_group, &grp);
1941                 if (rc != 0) {
1942                         CERROR("Out of memory\n");
1943                         goto out;
1944                 }
1945
1946                 list_add_tail(&grp->grp_link,
1947                               &console_session.ses_grp_list);
1948                 lstcon_group_addref(grp);
1949         }
1950
1951         if (grp->grp_ref > 2) {
1952                 /* Group in using */
1953                 jrep->join_status = EBUSY;
1954                 goto out;
1955         }
1956
1957         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0);
1958         if (rc == 0) {
1959                 jrep->join_status = EEXIST;
1960                 goto out;
1961         }
1962
1963         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1);
1964         if (rc != 0) {
1965                 CERROR("Out of memory\n");
1966                 goto out;
1967         }
1968
1969         ndl->ndl_node->nd_state   = LST_NODE_ACTIVE;
1970         ndl->ndl_node->nd_timeout = console_session.ses_timeout;
1971
1972         if (grp->grp_userland == 0)
1973                 grp->grp_userland = 1;
1974
1975         strlcpy(jrep->join_session, console_session.ses_name,
1976                 sizeof(jrep->join_session));
1977         jrep->join_timeout = console_session.ses_timeout;
1978         jrep->join_status  = 0;
1979
1980 out:
1981         rep->msg_ses_feats = console_session.ses_features;
1982         if (grp != NULL)
1983                 lstcon_group_decref(grp);
1984
1985         mutex_unlock(&console_session.ses_mutex);
1986
1987         return rc;
1988 }
1989
1990 static struct srpc_service lstcon_acceptor_service;
1991
1992 static void lstcon_init_acceptor_service(void)
1993 {
1994         /* initialize selftest console acceptor service table */
1995         lstcon_acceptor_service.sv_name    = "join session";
1996         lstcon_acceptor_service.sv_handler = lstcon_acceptor_handle;
1997         lstcon_acceptor_service.sv_id      = SRPC_SERVICE_JOIN;
1998         lstcon_acceptor_service.sv_wi_total = SFW_FRWK_WI_MAX;
1999 }
2000
2001 static struct notifier_block lstcon_ioctl_handler = {
2002         .notifier_call = lstcon_ioctl_entry,
2003 };
2004
2005 /* initialize console */
2006 int
2007 lstcon_console_init(void)
2008 {
2009         int     i;
2010         int     rc;
2011
2012         console_session.ses_id              = LST_INVALID_SID;
2013         console_session.ses_state           = LST_SESSION_NONE;
2014         console_session.ses_timeout         = 0;
2015         console_session.ses_force           = 0;
2016         console_session.ses_expired         = 0;
2017         console_session.ses_feats_updated   = 0;
2018         console_session.ses_features        = LST_FEATS_MASK;
2019         console_session.ses_laststamp = ktime_get_real_seconds();
2020
2021         mutex_init(&console_session.ses_mutex);
2022
2023         INIT_LIST_HEAD(&console_session.ses_ndl_list);
2024         INIT_LIST_HEAD(&console_session.ses_grp_list);
2025         INIT_LIST_HEAD(&console_session.ses_bat_list);
2026         INIT_LIST_HEAD(&console_session.ses_trans_list);
2027
2028         LIBCFS_ALLOC(console_session.ses_ndl_hash,
2029                      sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2030         if (console_session.ses_ndl_hash == NULL)
2031                 return -ENOMEM;
2032
2033         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
2034                 INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]);
2035
2036
2037         /* initialize acceptor service table */
2038         lstcon_init_acceptor_service();
2039
2040         rc = srpc_add_service(&lstcon_acceptor_service);
2041         LASSERT(rc != -EBUSY);
2042         if (rc != 0) {
2043                 LIBCFS_FREE(console_session.ses_ndl_hash,
2044                             sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2045                 return rc;
2046         }
2047
2048         rc = srpc_service_add_buffers(&lstcon_acceptor_service,
2049                                       lstcon_acceptor_service.sv_wi_total);
2050         if (rc != 0) {
2051                 rc = -ENOMEM;
2052                 goto out;
2053         }
2054
2055         rc = blocking_notifier_chain_register(&libcfs_ioctl_list,
2056                                               &lstcon_ioctl_handler);
2057         if (rc == 0) {
2058                 lstcon_rpc_module_init();
2059                 return 0;
2060         }
2061
2062 out:
2063         srpc_shutdown_service(&lstcon_acceptor_service);
2064         srpc_remove_service(&lstcon_acceptor_service);
2065
2066         LIBCFS_FREE(console_session.ses_ndl_hash,
2067                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2068
2069         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2070
2071         return rc;
2072 }
2073
2074 int
2075 lstcon_console_fini(void)
2076 {
2077         int i;
2078
2079         blocking_notifier_chain_unregister(&libcfs_ioctl_list,
2080                                            &lstcon_ioctl_handler);
2081
2082         mutex_lock(&console_session.ses_mutex);
2083
2084         srpc_shutdown_service(&lstcon_acceptor_service);
2085         srpc_remove_service(&lstcon_acceptor_service);
2086
2087         if (console_session.ses_state != LST_SESSION_NONE)
2088                 lstcon_session_end();
2089
2090         lstcon_rpc_module_fini();
2091
2092         mutex_unlock(&console_session.ses_mutex);
2093
2094         LASSERT(list_empty(&console_session.ses_ndl_list));
2095         LASSERT(list_empty(&console_session.ses_grp_list));
2096         LASSERT(list_empty(&console_session.ses_bat_list));
2097         LASSERT(list_empty(&console_session.ses_trans_list));
2098
2099         for (i = 0; i < LST_NODE_HASHSIZE; i++)
2100                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
2101
2102         LIBCFS_FREE(console_session.ses_ndl_hash,
2103                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2104
2105         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2106
2107         return 0;
2108 }
2109