lnet/selftest/console.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2012, 2014, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  * Lustre is a trademark of Sun Microsystems, Inc.
  31  *
  32  * lnet/selftest/conctl.c
  33  *
  34  * Infrastructure of LST console
  35  *
  36  * Author: Liang Zhen <liangzhen@clusterfs.com>
  37  */
  38
  39 #include <libcfs/libcfs.h>
  40 #include <lnet/lib-lnet.h>
  41 #include "console.h"
  42 #include "conrpc.h"
  43
  44 #define LST_NODE_STATE_COUNTER(nd, p)                   \
  45 do {                                                    \
  46         if ((nd)->nd_state == LST_NODE_ACTIVE)          \
  47                 (p)->nle_nactive ++;                    \
  48         else if ((nd)->nd_state == LST_NODE_BUSY)       \
  49                 (p)->nle_nbusy ++;                      \
  50         else if ((nd)->nd_state == LST_NODE_DOWN)       \
  51                 (p)->nle_ndown ++;                      \
  52         else                                            \
  53                 (p)->nle_nunknown ++;                   \
  54         (p)->nle_nnode ++;                              \
  55 } while (0)
  56
  57 struct lstcon_session console_session;
  58
  59 static void
  60 lstcon_node_get(struct lstcon_node *nd)
  61 {
  62         LASSERT (nd->nd_ref >= 1);
  63
  64         nd->nd_ref++;
  65 }
  66
  67 static int
  68 lstcon_node_find(struct lnet_process_id id, struct lstcon_node **ndpp,
  69                  int create)
  70 {
  71         struct lstcon_ndlink *ndl;
  72         unsigned int idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE;
  73
  74         LASSERT(id.nid != LNET_NID_ANY);
  75
  76         list_for_each_entry(ndl, &console_session.ses_ndl_hash[idx],
  77                             ndl_hlink) {
  78                 if (ndl->ndl_node->nd_id.nid != id.nid ||
  79                     ndl->ndl_node->nd_id.pid != id.pid)
  80                         continue;
  81
  82                 lstcon_node_get(ndl->ndl_node);
  83                 *ndpp = ndl->ndl_node;
  84                 return 0;
  85         }
  86
  87         if (!create)
  88                 return -ENOENT;
  89
  90         LIBCFS_ALLOC(*ndpp, sizeof(**ndpp) + sizeof(*ndl));
  91         if (*ndpp == NULL)
  92                 return -ENOMEM;
  93
  94         ndl = (struct lstcon_ndlink *)(*ndpp + 1);
  95
  96         ndl->ndl_node = *ndpp;
  97
  98         ndl->ndl_node->nd_ref   = 1;
  99         ndl->ndl_node->nd_id    = id;
 100         ndl->ndl_node->nd_stamp = ktime_get();
 101         ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
 102         ndl->ndl_node->nd_timeout = 0;
 103         memset(&ndl->ndl_node->nd_ping, 0, sizeof(ndl->ndl_node->nd_ping));
 104
 105         /* queued in global hash & list, no refcount is taken by
 106          * global hash & list, if caller release his refcount,
 107          * node will be released */
 108         list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]);
 109         list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list);
 110
 111         return 0;
 112 }
 113
 114 static void
 115 lstcon_node_put(struct lstcon_node *nd)
 116 {
 117         struct lstcon_ndlink *ndl;
 118
 119         LASSERT(nd->nd_ref > 0);
 120
 121         if (--nd->nd_ref > 0)
 122                 return;
 123
 124         ndl = (struct lstcon_ndlink *)(nd + 1);
 125
 126         LASSERT(!list_empty(&ndl->ndl_link));
 127         LASSERT(!list_empty(&ndl->ndl_hlink));
 128
 129         /* remove from session */
 130         list_del(&ndl->ndl_link);
 131         list_del(&ndl->ndl_hlink);
 132
 133         LIBCFS_FREE(nd, sizeof(*nd) + sizeof(*ndl));
 134 }
 135
 136 static int
 137 lstcon_ndlink_find(struct list_head *hash, struct lnet_process_id id,
 138                    struct lstcon_ndlink **ndlpp, int create)
 139 {
 140         unsigned int idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
 141         struct lstcon_ndlink *ndl;
 142         struct lstcon_node *nd;
 143         int rc;
 144
 145         if (id.nid == LNET_NID_ANY)
 146                 return -EINVAL;
 147
 148         /* search in hash */
 149         list_for_each_entry(ndl, &hash[idx], ndl_hlink) {
 150                 if (ndl->ndl_node->nd_id.nid != id.nid ||
 151                     ndl->ndl_node->nd_id.pid != id.pid)
 152                         continue;
 153
 154                 *ndlpp = ndl;
 155                 return 0;
 156         }
 157
 158         if (create == 0)
 159                 return -ENOENT;
 160
 161         /* find or create in session hash */
 162         rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0);
 163         if (rc != 0)
 164                 return rc;
 165
 166         LIBCFS_ALLOC(ndl, sizeof(*ndl));
 167         if (ndl == NULL) {
 168                 lstcon_node_put(nd);
 169                 return -ENOMEM;
 170         }
 171
 172         *ndlpp = ndl;
 173
 174         ndl->ndl_node = nd;
 175         INIT_LIST_HEAD(&ndl->ndl_link);
 176         list_add_tail(&ndl->ndl_hlink, &hash[idx]);
 177
 178         return 0;
 179 }
 180
 181 static void
 182 lstcon_ndlink_release(struct lstcon_ndlink *ndl)
 183 {
 184         LASSERT(list_empty(&ndl->ndl_link));
 185         LASSERT(!list_empty(&ndl->ndl_hlink));
 186
 187         list_del(&ndl->ndl_hlink); /* delete from hash */
 188         lstcon_node_put(ndl->ndl_node);
 189
 190         LIBCFS_FREE(ndl, sizeof(*ndl));
 191 }
 192
 193 static int
 194 lstcon_group_alloc(char *name, struct lstcon_group **grpp)
 195 {
 196         struct lstcon_group *grp;
 197         int i;
 198
 199         LIBCFS_ALLOC(grp, offsetof(struct lstcon_group,
 200                                    grp_ndl_hash[LST_NODE_HASHSIZE]));
 201         if (grp == NULL)
 202                 return -ENOMEM;
 203
 204         grp->grp_ref = 1;
 205         if (name != NULL) {
 206                 if (strlen(name) > sizeof(grp->grp_name)-1) {
 207                         LIBCFS_FREE(grp, offsetof(struct lstcon_group,
 208                                           grp_ndl_hash[LST_NODE_HASHSIZE]));
 209                         return -E2BIG;
 210                 }
 211                 strncpy(grp->grp_name, name, sizeof(grp->grp_name));
 212         }
 213
 214         INIT_LIST_HEAD(&grp->grp_link);
 215         INIT_LIST_HEAD(&grp->grp_ndl_list);
 216         INIT_LIST_HEAD(&grp->grp_trans_list);
 217
 218         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 219                 INIT_LIST_HEAD(&grp->grp_ndl_hash[i]);
 220
 221         *grpp = grp;
 222
 223         return 0;
 224 }
 225
 226 static void
 227 lstcon_group_addref(struct lstcon_group *grp)
 228 {
 229         grp->grp_ref++;
 230 }
 231
 232 static void lstcon_group_ndlink_release(struct lstcon_group *,
 233                                         struct lstcon_ndlink *);
 234
 235 static void
 236 lstcon_group_drain(struct lstcon_group *grp, int keep)
 237 {
 238         struct lstcon_ndlink *ndl;
 239         struct lstcon_ndlink *tmp;
 240
 241         list_for_each_entry_safe(ndl, tmp, &grp->grp_ndl_list, ndl_link) {
 242                 if ((ndl->ndl_node->nd_state & keep) == 0)
 243                         lstcon_group_ndlink_release(grp, ndl);
 244         }
 245 }
 246
 247 static void
 248 lstcon_group_decref(struct lstcon_group *grp)
 249 {
 250         int i;
 251
 252         if (--grp->grp_ref > 0)
 253                 return;
 254
 255         if (!list_empty(&grp->grp_link))
 256                 list_del(&grp->grp_link);
 257
 258         lstcon_group_drain(grp, 0);
 259
 260         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 261                 LASSERT(list_empty(&grp->grp_ndl_hash[i]));
 262
 263         LIBCFS_FREE(grp, offsetof(struct lstcon_group,
 264                                   grp_ndl_hash[LST_NODE_HASHSIZE]));
 265 }
 266
 267 static int
 268 lstcon_group_find(const char *name, struct lstcon_group **grpp)
 269 {
 270         struct lstcon_group *grp;
 271
 272         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 273                 if (strncmp(grp->grp_name, name, LST_NAME_SIZE) != 0)
 274                         continue;
 275
 276                 lstcon_group_addref(grp);  /* +1 ref for caller */
 277                 *grpp = grp;
 278                 return 0;
 279         }
 280
 281         return -ENOENT;
 282 }
 283
 284 static int
 285 lstcon_group_ndlink_find(struct lstcon_group *grp, struct lnet_process_id id,
 286                          struct lstcon_ndlink **ndlpp, int create)
 287 {
 288         int rc;
 289
 290         rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create);
 291         if (rc != 0)
 292                 return rc;
 293
 294         if (!list_empty(&(*ndlpp)->ndl_link))
 295                 return 0;
 296
 297         list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list);
 298         grp->grp_nnode++;
 299
 300         return 0;
 301 }
 302
 303 static void
 304 lstcon_group_ndlink_release(struct lstcon_group *grp, struct lstcon_ndlink *ndl)
 305 {
 306         list_del_init(&ndl->ndl_link);
 307         lstcon_ndlink_release(ndl);
 308         grp->grp_nnode--;
 309 }
 310
 311 static void
 312 lstcon_group_ndlink_move(struct lstcon_group *old,
 313                          struct lstcon_group *new, struct lstcon_ndlink *ndl)
 314 {
 315         unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) %
 316                                         LST_NODE_HASHSIZE;
 317
 318         list_del(&ndl->ndl_hlink);
 319         list_del(&ndl->ndl_link);
 320         old->grp_nnode--;
 321
 322         list_add_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]);
 323         list_add_tail(&ndl->ndl_link, &new->grp_ndl_list);
 324         new->grp_nnode++;
 325
 326         return;
 327 }
 328
 329 static void
 330 lstcon_group_move(struct lstcon_group *old, struct lstcon_group *new)
 331 {
 332         struct lstcon_ndlink *ndl;
 333
 334         while (!list_empty(&old->grp_ndl_list)) {
 335                 ndl = list_entry(old->grp_ndl_list.next,
 336                                  struct lstcon_ndlink, ndl_link);
 337                 lstcon_group_ndlink_move(old, new, ndl);
 338         }
 339 }
 340
 341 static int
 342 lstcon_sesrpc_condition(int transop, struct lstcon_node *nd, void *arg)
 343 {
 344         struct lstcon_group *grp = arg;
 345
 346         switch (transop) {
 347         case LST_TRANS_SESNEW:
 348                 if (nd->nd_state == LST_NODE_ACTIVE)
 349                         return 0;
 350                 break;
 351
 352         case LST_TRANS_SESEND:
 353                 if (nd->nd_state != LST_NODE_ACTIVE)
 354                         return 0;
 355
 356                 if (grp != NULL && nd->nd_ref > 1)
 357                         return 0;
 358                 break;
 359
 360         case LST_TRANS_SESQRY:
 361                 break;
 362
 363         default:
 364                 LBUG();
 365         }
 366
 367         return 1;
 368 }
 369
 370 static int
 371 lstcon_sesrpc_readent(int transop, struct srpc_msg *msg,
 372                       struct lstcon_rpc_ent __user *ent_up)
 373 {
 374         struct srpc_debug_reply *rep;
 375
 376         switch (transop) {
 377         case LST_TRANS_SESNEW:
 378         case LST_TRANS_SESEND:
 379                 return 0;
 380
 381         case LST_TRANS_SESQRY:
 382                 rep = &msg->msg_body.dbg_reply;
 383
 384                 if (copy_to_user(&ent_up->rpe_priv[0],
 385                                      &rep->dbg_timeout, sizeof(int)) ||
 386                     copy_to_user(&ent_up->rpe_payload[0],
 387                                      &rep->dbg_name, LST_NAME_SIZE))
 388                         return -EFAULT;
 389
 390                 return 0;
 391
 392         default:
 393                 LBUG();
 394         }
 395
 396         return 0;
 397 }
 398
 399 static int
 400 lstcon_group_nodes_add(struct lstcon_group *grp,
 401                        int count, struct lnet_process_id __user *ids_up,
 402                        unsigned int *featp,
 403                        struct list_head __user *result_up)
 404 {
 405         struct lstcon_rpc_trans *trans;
 406         struct lstcon_ndlink *ndl;
 407         struct lstcon_group *tmp;
 408         struct lnet_process_id id;
 409         int i;
 410         int rc;
 411
 412         rc = lstcon_group_alloc(NULL, &tmp);
 413         if (rc != 0) {
 414                 CERROR("Out of memory\n");
 415                 return -ENOMEM;
 416         }
 417
 418         for (i = 0 ; i < count; i++) {
 419                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 420                         rc = -EFAULT;
 421                         break;
 422                 }
 423
 424                 /* skip if it's in this group already */
 425                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 0);
 426                 if (rc == 0)
 427                         continue;
 428
 429                 /* add to tmp group */
 430                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1);
 431                 if (rc != 0) {
 432                         CERROR("Can't create ndlink, out of memory\n");
 433                         break;
 434                 }
 435         }
 436
 437         if (rc != 0) {
 438                 lstcon_group_decref(tmp);
 439                 return rc;
 440         }
 441
 442         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 443                                      &tmp->grp_trans_list, LST_TRANS_SESNEW,
 444                                      tmp, lstcon_sesrpc_condition, &trans);
 445         if (rc != 0) {
 446                 CERROR("Can't create transaction: %d\n", rc);
 447                 lstcon_group_decref(tmp);
 448                 return rc;
 449         }
 450
 451         /* post all RPCs */
 452         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 453
 454         rc = lstcon_rpc_trans_interpreter(trans, result_up,
 455                                           lstcon_sesrpc_readent);
 456         *featp = trans->tas_features;
 457
 458         /* destroy all RPGs */
 459         lstcon_rpc_trans_destroy(trans);
 460
 461         lstcon_group_move(tmp, grp);
 462         lstcon_group_decref(tmp);
 463
 464         return rc;
 465 }
 466
 467 static int
 468 lstcon_group_nodes_remove(struct lstcon_group *grp,
 469                           int count, struct lnet_process_id __user *ids_up,
 470                           struct list_head __user *result_up)
 471 {
 472         struct lstcon_rpc_trans *trans;
 473         struct lstcon_ndlink *ndl;
 474         struct lstcon_group *tmp;
 475         struct lnet_process_id id;
 476         int rc;
 477         int i;
 478
 479         /* End session and remove node from the group */
 480
 481         rc = lstcon_group_alloc(NULL, &tmp);
 482         if (rc != 0) {
 483                 CERROR("Out of memory\n");
 484                 return -ENOMEM;
 485         }
 486
 487         for (i = 0; i < count; i++) {
 488                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 489                         rc = -EFAULT;
 490                         goto error;
 491                 }
 492
 493                 /* move node to tmp group */
 494                 if (lstcon_group_ndlink_find(grp, id, &ndl, 0) == 0)
 495                         lstcon_group_ndlink_move(grp, tmp, ndl);
 496         }
 497
 498         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 499                                      &tmp->grp_trans_list, LST_TRANS_SESEND,
 500                                      tmp, lstcon_sesrpc_condition, &trans);
 501         if (rc != 0) {
 502                 CERROR("Can't create transaction: %d\n", rc);
 503                 goto error;
 504         }
 505
 506         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 507
 508         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 509
 510         lstcon_rpc_trans_destroy(trans);
 511         /* release nodes anyway, because we can't rollback status */
 512         lstcon_group_decref(tmp);
 513
 514         return rc;
 515 error:
 516         lstcon_group_move(tmp, grp);
 517         lstcon_group_decref(tmp);
 518
 519         return rc;
 520 }
 521
 522 int
 523 lstcon_group_add(char *name)
 524 {
 525         struct lstcon_group *grp;
 526         int rc;
 527
 528         rc = (lstcon_group_find(name, &grp) == 0)? -EEXIST: 0;
 529         if (rc != 0) {
 530                 /* find a group with same name */
 531                 lstcon_group_decref(grp);
 532                 return rc;
 533         }
 534
 535         rc = lstcon_group_alloc(name, &grp);
 536         if (rc != 0) {
 537                 CERROR("Can't allocate descriptor for group %s\n", name);
 538                 return -ENOMEM;
 539         }
 540
 541         list_add_tail(&grp->grp_link, &console_session.ses_grp_list);
 542
 543         return rc;
 544 }
 545
 546 int
 547 lstcon_nodes_add(char *name, int count, struct lnet_process_id __user *ids_up,
 548                  unsigned *featp, struct list_head __user *result_up)
 549 {
 550         struct lstcon_group         *grp;
 551         int                     rc;
 552
 553         LASSERT (count > 0);
 554         LASSERT (ids_up != NULL);
 555
 556         rc = lstcon_group_find(name, &grp);
 557         if (rc != 0) {
 558                 CDEBUG(D_NET, "Can't find group %s\n", name);
 559                 return rc;
 560         }
 561
 562         if (grp->grp_ref > 2) {
 563                 /* referred by other threads or test */
 564                 CDEBUG(D_NET, "Group %s is busy\n", name);
 565                 lstcon_group_decref(grp);
 566
 567                 return -EBUSY;
 568         }
 569
 570         rc = lstcon_group_nodes_add(grp, count, ids_up, featp, result_up);
 571
 572         lstcon_group_decref(grp);
 573
 574         return rc;
 575 }
 576
 577 int
 578 lstcon_group_del(char *name)
 579 {
 580         struct lstcon_rpc_trans *trans;
 581         struct lstcon_group *grp;
 582         int rc;
 583
 584         rc = lstcon_group_find(name, &grp);
 585         if (rc != 0) {
 586                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 587                 return rc;
 588         }
 589
 590         if (grp->grp_ref > 2) {
 591                 /* referred by others threads or test */
 592                 CDEBUG(D_NET, "Group %s is busy\n", name);
 593                 lstcon_group_decref(grp);
 594                 return -EBUSY;
 595         }
 596
 597         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 598                                      &grp->grp_trans_list, LST_TRANS_SESEND,
 599                                      grp, lstcon_sesrpc_condition, &trans);
 600         if (rc != 0) {
 601                 CERROR("Can't create transaction: %d\n", rc);
 602                 lstcon_group_decref(grp);
 603                 return rc;
 604         }
 605
 606         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 607
 608         lstcon_rpc_trans_destroy(trans);
 609
 610         lstcon_group_decref(grp);
 611         /* -ref for session, it's destroyed,
 612          * status can't be rolled back, destroy group anway */
 613         lstcon_group_decref(grp);
 614
 615         return rc;
 616 }
 617
 618 int
 619 lstcon_group_clean(char *name, int args)
 620 {
 621         struct lstcon_group *grp = NULL;
 622         int rc;
 623
 624         rc = lstcon_group_find(name, &grp);
 625         if (rc != 0) {
 626                 CDEBUG(D_NET, "Can't find group %s\n", name);
 627                 return rc;
 628         }
 629
 630         if (grp->grp_ref > 2) {
 631                 /* referred by test */
 632                 CDEBUG(D_NET, "Group %s is busy\n", name);
 633                 lstcon_group_decref(grp);
 634                 return -EBUSY;
 635         }
 636
 637         args = (LST_NODE_ACTIVE | LST_NODE_BUSY |
 638                 LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args;
 639
 640         lstcon_group_drain(grp, args);
 641
 642         lstcon_group_decref(grp);
 643         /* release empty group */
 644         if (list_empty(&grp->grp_ndl_list))
 645                 lstcon_group_decref(grp);
 646
 647         return 0;
 648 }
 649
 650 int
 651 lstcon_nodes_remove(char *name, int count,
 652                     struct lnet_process_id __user *ids_up,
 653                     struct list_head __user *result_up)
 654 {
 655         struct lstcon_group *grp = NULL;
 656         int rc;
 657
 658         rc = lstcon_group_find(name, &grp);
 659         if (rc != 0) {
 660                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 661                 return rc;
 662         }
 663
 664         if (grp->grp_ref > 2) {
 665                 /* referred by test */
 666                 CDEBUG(D_NET, "Group %s is busy\n", name);
 667                 lstcon_group_decref(grp);
 668                 return -EBUSY;
 669         }
 670
 671         rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up);
 672
 673         lstcon_group_decref(grp);
 674         /* release empty group */
 675         if (list_empty(&grp->grp_ndl_list))
 676                 lstcon_group_decref(grp);
 677
 678         return rc;
 679 }
 680
 681 int
 682 lstcon_group_refresh(char *name, struct list_head __user *result_up)
 683 {
 684         struct lstcon_rpc_trans *trans;
 685         struct lstcon_group *grp;
 686         int rc;
 687
 688         rc = lstcon_group_find(name, &grp);
 689         if (rc != 0) {
 690                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 691                 return rc;
 692         }
 693
 694         if (grp->grp_ref > 2) {
 695                 /* referred by test */
 696                 CDEBUG(D_NET, "Group %s is busy\n", name);
 697                 lstcon_group_decref(grp);
 698                 return -EBUSY;
 699         }
 700
 701         /* re-invite all inactive nodes int the group */
 702         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 703                                      &grp->grp_trans_list, LST_TRANS_SESNEW,
 704                                      grp, lstcon_sesrpc_condition, &trans);
 705         if (rc != 0) {
 706                 /* local error, return */
 707                 CDEBUG(D_NET, "Can't create transaction: %d\n", rc);
 708                 lstcon_group_decref(grp);
 709                 return rc;
 710         }
 711
 712         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 713
 714         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 715
 716         lstcon_rpc_trans_destroy(trans);
 717         /* -ref for me */
 718         lstcon_group_decref(grp);
 719
 720         return rc;
 721 }
 722
 723 int
 724 lstcon_group_list(int index, int len, char __user *name_up)
 725 {
 726         struct lstcon_group *grp;
 727
 728         LASSERT(index >= 0);
 729         LASSERT(name_up != NULL);
 730
 731         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 732                 if (index-- == 0) {
 733                         return copy_to_user(name_up, grp->grp_name, len) ?
 734                                             -EFAULT : 0;
 735                 }
 736         }
 737
 738         return -ENOENT;
 739 }
 740
 741 static int
 742 lstcon_nodes_getent(struct list_head *head, int *index_p,
 743                     int *count_p, struct lstcon_node_ent __user *dents_up)
 744 {
 745         struct lstcon_ndlink *ndl;
 746         struct lstcon_node *nd;
 747         int count = 0;
 748         int index = 0;
 749
 750         LASSERT(index_p != NULL && count_p != NULL);
 751         LASSERT(dents_up != NULL);
 752         LASSERT(*index_p >= 0);
 753         LASSERT(*count_p > 0);
 754
 755         list_for_each_entry(ndl, head, ndl_link) {
 756                 if (index++ < *index_p)
 757                         continue;
 758
 759                 if (count >= *count_p)
 760                         break;
 761
 762                 nd = ndl->ndl_node;
 763                 if (copy_to_user(&dents_up[count].nde_id,
 764                                      &nd->nd_id, sizeof(nd->nd_id)) ||
 765                     copy_to_user(&dents_up[count].nde_state,
 766                                      &nd->nd_state, sizeof(nd->nd_state)))
 767                         return -EFAULT;
 768
 769                 count ++;
 770         }
 771
 772         if (index <= *index_p)
 773                 return -ENOENT;
 774
 775         *count_p = count;
 776         *index_p = index;
 777
 778         return 0;
 779 }
 780
 781 int
 782 lstcon_group_info(char *name, struct lstcon_ndlist_ent __user *gents_p,
 783                   int *index_p, int *count_p,
 784                   struct lstcon_node_ent __user *dents_up)
 785 {
 786         struct lstcon_ndlist_ent *gentp;
 787         struct lstcon_group *grp;
 788         struct lstcon_ndlink *ndl;
 789         int rc;
 790
 791         rc = lstcon_group_find(name, &grp);
 792         if (rc != 0) {
 793                 CDEBUG(D_NET, "Can't find group %s\n", name);
 794                 return rc;
 795         }
 796
 797         if (dents_up != NULL) {
 798                 /* verbose query */
 799                 rc = lstcon_nodes_getent(&grp->grp_ndl_list,
 800                                          index_p, count_p, dents_up);
 801                 lstcon_group_decref(grp);
 802
 803                 return rc;
 804         }
 805
 806         /* non-verbose query */
 807         LIBCFS_ALLOC(gentp, sizeof(struct lstcon_ndlist_ent));
 808         if (gentp == NULL) {
 809                 CERROR("Can't allocate ndlist_ent\n");
 810                 lstcon_group_decref(grp);
 811
 812                 return -ENOMEM;
 813         }
 814
 815         list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link)
 816                 LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp);
 817
 818         rc = copy_to_user(gents_p, gentp,
 819                           sizeof(struct lstcon_ndlist_ent)) ? -EFAULT : 0;
 820
 821         LIBCFS_FREE(gentp, sizeof(struct lstcon_ndlist_ent));
 822
 823         lstcon_group_decref(grp);
 824
 825         return 0;
 826 }
 827
 828 static int
 829 lstcon_batch_find(const char *name, struct lstcon_batch **batpp)
 830 {
 831         struct lstcon_batch *bat;
 832
 833         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 834                 if (strncmp(bat->bat_name, name, LST_NAME_SIZE) == 0) {
 835                         *batpp = bat;
 836                         return 0;
 837                 }
 838         }
 839
 840         return -ENOENT;
 841 }
 842
 843 int
 844 lstcon_batch_add(char *name)
 845 {
 846         struct lstcon_batch *bat;
 847         int i;
 848         int rc;
 849
 850         rc = (lstcon_batch_find(name, &bat) == 0)? -EEXIST: 0;
 851         if (rc != 0) {
 852                 CDEBUG(D_NET, "Batch %s already exists\n", name);
 853                 return rc;
 854         }
 855
 856         LIBCFS_ALLOC(bat, sizeof(*bat));
 857         if (bat == NULL) {
 858                 CERROR("Can't allocate descriptor for batch %s\n", name);
 859                 return -ENOMEM;
 860         }
 861
 862         LIBCFS_ALLOC(bat->bat_cli_hash,
 863                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 864         if (bat->bat_cli_hash == NULL) {
 865                 CERROR("Can't allocate hash for batch %s\n", name);
 866                 LIBCFS_FREE(bat, sizeof(*bat));
 867
 868                 return -ENOMEM;
 869         }
 870
 871         LIBCFS_ALLOC(bat->bat_srv_hash,
 872                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 873         if (bat->bat_srv_hash == NULL) {
 874                 CERROR("Can't allocate hash for batch %s\n", name);
 875                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 876                 LIBCFS_FREE(bat, sizeof(*bat));
 877
 878                 return -ENOMEM;
 879         }
 880
 881         if (strlen(name) > sizeof(bat->bat_name)-1) {
 882                 LIBCFS_FREE(bat->bat_srv_hash, LST_NODE_HASHSIZE);
 883                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 884                 LIBCFS_FREE(bat, sizeof(*bat));
 885                 return -E2BIG;
 886         }
 887         strncpy(bat->bat_name, name, sizeof(bat->bat_name));
 888         bat->bat_hdr.tsb_index = 0;
 889         bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie;
 890
 891         bat->bat_ntest = 0;
 892         bat->bat_state = LST_BATCH_IDLE;
 893
 894         INIT_LIST_HEAD(&bat->bat_cli_list);
 895         INIT_LIST_HEAD(&bat->bat_srv_list);
 896         INIT_LIST_HEAD(&bat->bat_test_list);
 897         INIT_LIST_HEAD(&bat->bat_trans_list);
 898
 899         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 900                 INIT_LIST_HEAD(&bat->bat_cli_hash[i]);
 901                 INIT_LIST_HEAD(&bat->bat_srv_hash[i]);
 902         }
 903
 904         list_add_tail(&bat->bat_link, &console_session.ses_bat_list);
 905
 906         return rc;
 907 }
 908
 909 int
 910 lstcon_batch_list(int index, int len, char __user *name_up)
 911 {
 912         struct lstcon_batch *bat;
 913
 914         LASSERT(name_up != NULL);
 915         LASSERT(index >= 0);
 916
 917         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 918                 if (index-- == 0) {
 919                         return copy_to_user(name_up, bat->bat_name, len) ?
 920                                             -EFAULT : 0;
 921                 }
 922         }
 923
 924         return -ENOENT;
 925 }
 926
 927 int
 928 lstcon_batch_info(char *name, struct lstcon_test_batch_ent __user *ent_up,
 929                   int server, int testidx, int *index_p, int *ndent_p,
 930                   struct lstcon_node_ent __user *dents_up)
 931 {
 932         struct lstcon_test_batch_ent *entp;
 933         struct list_head *clilst;
 934         struct list_head *srvlst;
 935         struct lstcon_test *test = NULL;
 936         struct lstcon_batch *bat;
 937         struct lstcon_ndlink *ndl;
 938         int rc;
 939
 940         rc = lstcon_batch_find(name, &bat);
 941         if (rc != 0) {
 942                 CDEBUG(D_NET, "Can't find batch %s\n", name);
 943                 return -ENOENT;
 944         }
 945
 946         if (testidx > 0) {
 947                 /* query test, test index start from 1 */
 948                 list_for_each_entry(test, &bat->bat_test_list, tes_link) {
 949                         if (testidx-- == 1)
 950                                 break;
 951                 }
 952
 953                 if (testidx > 0) {
 954                         CDEBUG(D_NET, "Can't find specified test in batch\n");
 955                         return -ENOENT;
 956                 }
 957         }
 958
 959         clilst = (test == NULL) ? &bat->bat_cli_list :
 960                                   &test->tes_src_grp->grp_ndl_list;
 961         srvlst = (test == NULL) ? &bat->bat_srv_list :
 962                                   &test->tes_dst_grp->grp_ndl_list;
 963
 964         if (dents_up != NULL) {
 965                 rc = lstcon_nodes_getent((server ? srvlst: clilst),
 966                                          index_p, ndent_p, dents_up);
 967                 return rc;
 968         }
 969
 970         /* non-verbose query */
 971         LIBCFS_ALLOC(entp, sizeof(struct lstcon_test_batch_ent));
 972         if (entp == NULL)
 973                 return -ENOMEM;
 974
 975         if (test == NULL) {
 976                 entp->u.tbe_batch.bae_ntest = bat->bat_ntest;
 977                 entp->u.tbe_batch.bae_state = bat->bat_state;
 978
 979         } else {
 980
 981                 entp->u.tbe_test.tse_type   = test->tes_type;
 982                 entp->u.tbe_test.tse_loop   = test->tes_loop;
 983                 entp->u.tbe_test.tse_concur = test->tes_concur;
 984         }
 985
 986         list_for_each_entry(ndl, clilst, ndl_link)
 987                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle);
 988
 989         list_for_each_entry(ndl, srvlst, ndl_link)
 990                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle);
 991
 992         rc = copy_to_user(ent_up, entp,
 993                           sizeof(struct lstcon_test_batch_ent)) ? -EFAULT : 0;
 994
 995         LIBCFS_FREE(entp, sizeof(struct lstcon_test_batch_ent));
 996
 997         return rc;
 998 }
 999
1000 static int
1001 lstcon_batrpc_condition(int transop, struct lstcon_node *nd, void *arg)
1002 {
1003         switch (transop) {
1004         case LST_TRANS_TSBRUN:
1005                 if (nd->nd_state != LST_NODE_ACTIVE)
1006                         return -ENETDOWN;
1007                 break;
1008
1009         case LST_TRANS_TSBSTOP:
1010                 if (nd->nd_state != LST_NODE_ACTIVE)
1011                         return 0;
1012                 break;
1013
1014         case LST_TRANS_TSBCLIQRY:
1015         case LST_TRANS_TSBSRVQRY:
1016                 break;
1017         }
1018
1019         return 1;
1020 }
1021
1022 static int
1023 lstcon_batch_op(struct lstcon_batch *bat, int transop,
1024                 struct list_head __user *result_up)
1025 {
1026         struct lstcon_rpc_trans *trans;
1027         int                 rc;
1028
1029         rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list,
1030                                      &bat->bat_trans_list, transop,
1031                                      bat, lstcon_batrpc_condition, &trans);
1032         if (rc != 0) {
1033                 CERROR("Can't create transaction: %d\n", rc);
1034                 return rc;
1035         }
1036
1037         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1038
1039         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1040
1041         lstcon_rpc_trans_destroy(trans);
1042
1043         return rc;
1044 }
1045
1046 int
1047 lstcon_batch_run(char *name, int timeout, struct list_head __user *result_up)
1048 {
1049         struct lstcon_batch *bat;
1050         int rc;
1051
1052         if (lstcon_batch_find(name, &bat) != 0) {
1053                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1054                 return -ENOENT;
1055         }
1056
1057         bat->bat_arg = timeout;
1058
1059         rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up);
1060
1061         /* mark batch as running if it's started in any node */
1062         if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0) != 0)
1063                 bat->bat_state = LST_BATCH_RUNNING;
1064
1065         return rc;
1066 }
1067
1068 int
1069 lstcon_batch_stop(char *name, int force, struct list_head __user *result_up)
1070 {
1071         struct lstcon_batch *bat;
1072         int rc;
1073
1074         if (lstcon_batch_find(name, &bat) != 0) {
1075                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1076                 return -ENOENT;
1077         }
1078
1079         bat->bat_arg = force;
1080
1081         rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up);
1082
1083         /* mark batch as stopped if all RPCs finished */
1084         if (lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0) == 0)
1085                 bat->bat_state = LST_BATCH_IDLE;
1086
1087         return rc;
1088 }
1089
1090 static void
1091 lstcon_batch_destroy(struct lstcon_batch *bat)
1092 {
1093         struct lstcon_ndlink *ndl;
1094         struct lstcon_test *test;
1095         int i;
1096
1097         list_del(&bat->bat_link);
1098
1099         while (!list_empty(&bat->bat_test_list)) {
1100                 test = list_entry(bat->bat_test_list.next,
1101                                   struct lstcon_test, tes_link);
1102                 LASSERT(list_empty(&test->tes_trans_list));
1103
1104                 list_del(&test->tes_link);
1105
1106                 lstcon_group_decref(test->tes_src_grp);
1107                 lstcon_group_decref(test->tes_dst_grp);
1108
1109                 LIBCFS_FREE(test, offsetof(struct lstcon_test,
1110                                            tes_param[test->tes_paramlen]));
1111         }
1112
1113         LASSERT(list_empty(&bat->bat_trans_list));
1114
1115         while (!list_empty(&bat->bat_cli_list)) {
1116                 ndl = list_entry(bat->bat_cli_list.next,
1117                                  struct lstcon_ndlink, ndl_link);
1118                 list_del_init(&ndl->ndl_link);
1119
1120                 lstcon_ndlink_release(ndl);
1121         }
1122
1123         while (!list_empty(&bat->bat_srv_list)) {
1124                 ndl = list_entry(bat->bat_srv_list.next,
1125                                  struct lstcon_ndlink, ndl_link);
1126                 list_del_init(&ndl->ndl_link);
1127
1128                 lstcon_ndlink_release(ndl);
1129         }
1130
1131         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1132                 LASSERT(list_empty(&bat->bat_cli_hash[i]));
1133                 LASSERT(list_empty(&bat->bat_srv_hash[i]));
1134         }
1135
1136         LIBCFS_FREE(bat->bat_cli_hash,
1137                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1138         LIBCFS_FREE(bat->bat_srv_hash,
1139                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1140         LIBCFS_FREE(bat, sizeof(*bat));
1141 }
1142
1143 static int
1144 lstcon_testrpc_condition(int transop, struct lstcon_node *nd, void *arg)
1145 {
1146         struct lstcon_test *test = arg;
1147         struct lstcon_batch *batch;
1148         struct lstcon_ndlink *ndl;
1149         struct list_head *hash;
1150         struct list_head *head;
1151
1152         LASSERT(test != NULL);
1153
1154         batch = test->tes_batch;
1155         LASSERT(batch != NULL);
1156
1157         if (test->tes_oneside &&
1158             transop == LST_TRANS_TSBSRVADD)
1159                 return 0;
1160
1161         if (nd->nd_state != LST_NODE_ACTIVE)
1162                 return -ENETDOWN;
1163
1164         if (transop == LST_TRANS_TSBCLIADD) {
1165                 hash = batch->bat_cli_hash;
1166                 head = &batch->bat_cli_list;
1167
1168         } else {
1169                 LASSERT (transop == LST_TRANS_TSBSRVADD);
1170
1171                 hash = batch->bat_srv_hash;
1172                 head = &batch->bat_srv_list;
1173         }
1174
1175         LASSERT (nd->nd_id.nid != LNET_NID_ANY);
1176
1177         if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1) != 0)
1178                 return -ENOMEM;
1179
1180         if (list_empty(&ndl->ndl_link))
1181                 list_add_tail(&ndl->ndl_link, head);
1182
1183         return 1;
1184 }
1185
1186 static int
1187 lstcon_test_nodes_add(struct lstcon_test *test,
1188                       struct list_head __user *result_up)
1189 {
1190         struct lstcon_rpc_trans *trans;
1191         struct lstcon_group *grp;
1192         int transop;
1193         int rc;
1194
1195         LASSERT (test->tes_src_grp != NULL);
1196         LASSERT (test->tes_dst_grp != NULL);
1197
1198         transop = LST_TRANS_TSBSRVADD;
1199         grp  = test->tes_dst_grp;
1200 again:
1201         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
1202                                      &test->tes_trans_list, transop,
1203                                      test, lstcon_testrpc_condition, &trans);
1204         if (rc != 0) {
1205                 CERROR("Can't create transaction: %d\n", rc);
1206                 return rc;
1207         }
1208
1209         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1210
1211         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1212             lstcon_trans_stat()->trs_fwk_errno != 0) {
1213                 lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1214
1215                 lstcon_rpc_trans_destroy(trans);
1216                 /* return if any error */
1217                 CDEBUG(D_NET, "Failed to add test %s, "
1218                               "RPC error %d, framework error %d\n",
1219                        transop == LST_TRANS_TSBCLIADD ? "client" : "server",
1220                        lstcon_trans_stat()->trs_rpc_errno,
1221                        lstcon_trans_stat()->trs_fwk_errno);
1222
1223                 return rc;
1224         }
1225
1226         lstcon_rpc_trans_destroy(trans);
1227
1228         if (transop == LST_TRANS_TSBCLIADD)
1229                 return rc;
1230
1231         transop = LST_TRANS_TSBCLIADD;
1232         grp = test->tes_src_grp;
1233         test->tes_cliidx = 0;
1234
1235         /* requests to test clients */
1236         goto again;
1237 }
1238
1239 static int
1240 lstcon_verify_batch(const char *name, struct lstcon_batch **batch)
1241 {
1242         int rc;
1243
1244         rc = lstcon_batch_find(name, batch);
1245         if (rc != 0) {
1246                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1247                 return rc;
1248         }
1249
1250         if ((*batch)->bat_state != LST_BATCH_IDLE) {
1251                 CDEBUG(D_NET, "Can't change running batch %s\n", name);
1252                 return -EINVAL;
1253         }
1254
1255         return 0;
1256 }
1257
1258 static int
1259 lstcon_verify_group(const char *name, struct lstcon_group **grp)
1260 {
1261         int rc;
1262         struct lstcon_ndlink *ndl;
1263
1264         rc = lstcon_group_find(name, grp);
1265         if (rc != 0) {
1266                 CDEBUG(D_NET, "can't find group %s\n", name);
1267                 return rc;
1268         }
1269
1270         list_for_each_entry(ndl, &(*grp)->grp_ndl_list, ndl_link) {
1271                 if (ndl->ndl_node->nd_state == LST_NODE_ACTIVE) {
1272                         return 0;
1273                 }
1274         }
1275
1276         CDEBUG(D_NET, "Group %s has no ACTIVE nodes\n", name);
1277
1278         return -EINVAL;
1279 }
1280
1281 int
1282 lstcon_test_add(char *batch_name, int type, int loop,
1283                 int concur, int dist, int span,
1284                 char *src_name, char *dst_name,
1285                 void *param, int paramlen, int *retp,
1286                 struct list_head __user *result_up)
1287 {
1288         struct lstcon_test *test = NULL;
1289         int rc;
1290         struct lstcon_group *src_grp = NULL;
1291         struct lstcon_group *dst_grp = NULL;
1292         struct lstcon_batch *batch = NULL;
1293
1294         /*
1295          * verify that a batch of the given name exists, and the groups
1296          * that will be part of the batch exist and have at least one
1297          * active node
1298          */
1299         rc = lstcon_verify_batch(batch_name, &batch);
1300         if (rc != 0)
1301                 goto out;
1302
1303         rc = lstcon_verify_group(src_name, &src_grp);
1304         if (rc != 0)
1305                 goto out;
1306
1307         rc = lstcon_verify_group(dst_name, &dst_grp);
1308         if (rc != 0)
1309                 goto out;
1310
1311         if (dst_grp->grp_userland)
1312                 *retp = 1;
1313
1314         LIBCFS_ALLOC(test, offsetof(struct lstcon_test, tes_param[paramlen]));
1315         if (!test) {
1316                 CERROR("Can't allocate test descriptor\n");
1317                 rc = -ENOMEM;
1318
1319                 goto out;
1320         }
1321
1322         test->tes_hdr.tsb_id    = batch->bat_hdr.tsb_id;
1323         test->tes_batch         = batch;
1324         test->tes_type          = type;
1325         test->tes_oneside       = 0; /* TODO */
1326         test->tes_loop          = loop;
1327         test->tes_concur        = concur;
1328         test->tes_stop_onerr    = 1; /* TODO */
1329         test->tes_span          = span;
1330         test->tes_dist          = dist;
1331         test->tes_cliidx        = 0; /* just used for creating RPC */
1332         test->tes_src_grp       = src_grp;
1333         test->tes_dst_grp       = dst_grp;
1334         INIT_LIST_HEAD(&test->tes_trans_list);
1335
1336         if (param != NULL) {
1337                 test->tes_paramlen = paramlen;
1338                 memcpy(&test->tes_param[0], param, paramlen);
1339         }
1340
1341         rc = lstcon_test_nodes_add(test, result_up);
1342
1343         if (rc != 0)
1344                 goto out;
1345
1346         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1347             lstcon_trans_stat()->trs_fwk_errno != 0)
1348                 CDEBUG(D_NET, "Failed to add test %d to batch %s\n", type,
1349                        batch_name);
1350
1351         /* add to test list anyway, so user can check what's going on */
1352         list_add_tail(&test->tes_link, &batch->bat_test_list);
1353
1354         batch->bat_ntest++;
1355         test->tes_hdr.tsb_index = batch->bat_ntest;
1356
1357         /*  hold groups so nobody can change them */
1358         return rc;
1359 out:
1360         if (test != NULL)
1361                 LIBCFS_FREE(test, offsetof(struct lstcon_test,
1362                                            tes_param[paramlen]));
1363
1364         if (dst_grp != NULL)
1365                 lstcon_group_decref(dst_grp);
1366
1367         if (src_grp != NULL)
1368                 lstcon_group_decref(src_grp);
1369
1370         return rc;
1371 }
1372
1373 static int
1374 lstcon_test_find(struct lstcon_batch *batch, int idx,
1375                  struct lstcon_test **testpp)
1376 {
1377         struct lstcon_test *test;
1378
1379         list_for_each_entry(test, &batch->bat_test_list, tes_link) {
1380                 if (idx == test->tes_hdr.tsb_index) {
1381                         *testpp = test;
1382                         return 0;
1383                 }
1384         }
1385
1386         return -ENOENT;
1387 }
1388
1389 static int
1390 lstcon_tsbrpc_readent(int transop, struct srpc_msg *msg,
1391                       struct lstcon_rpc_ent __user *ent_up)
1392 {
1393         struct srpc_batch_reply *rep = &msg->msg_body.bat_reply;
1394
1395         LASSERT (transop == LST_TRANS_TSBCLIQRY ||
1396                  transop == LST_TRANS_TSBSRVQRY);
1397
1398         /* positive errno, framework error code */
1399         if (copy_to_user(&ent_up->rpe_priv[0],
1400                              &rep->bar_active, sizeof(rep->bar_active)))
1401                 return -EFAULT;
1402
1403         return 0;
1404 }
1405
1406 int
1407 lstcon_test_batch_query(char *name, int testidx, int client,
1408                         int timeout, struct list_head __user *result_up)
1409 {
1410         struct lstcon_rpc_trans *trans;
1411         struct list_head *translist;
1412         struct list_head *ndlist;
1413         struct lstcon_tsb_hdr *hdr;
1414         struct lstcon_batch *batch;
1415         struct lstcon_test *test = NULL;
1416         int transop;
1417         int rc;
1418
1419         rc = lstcon_batch_find(name, &batch);
1420         if (rc != 0) {
1421                 CDEBUG(D_NET, "Can't find batch: %s\n", name);
1422                 return rc;
1423         }
1424
1425         if (testidx == 0) {
1426                 translist = &batch->bat_trans_list;
1427                 ndlist    = &batch->bat_cli_list;
1428                 hdr       = &batch->bat_hdr;
1429
1430         } else {
1431                 /* query specified test only */
1432                 rc = lstcon_test_find(batch, testidx, &test);
1433                 if (rc != 0) {
1434                         CDEBUG(D_NET, "Can't find test: %d\n", testidx);
1435                         return rc;
1436                 }
1437
1438                 translist = &test->tes_trans_list;
1439                 ndlist    = &test->tes_src_grp->grp_ndl_list;
1440                 hdr       = &test->tes_hdr;
1441         }
1442
1443         transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY;
1444
1445         rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr,
1446                                      lstcon_batrpc_condition, &trans);
1447         if (rc != 0) {
1448                 CERROR("Can't create transaction: %d\n", rc);
1449                 return rc;
1450         }
1451
1452         lstcon_rpc_trans_postwait(trans, timeout);
1453
1454         if (testidx == 0 && /* query a batch, not a test */
1455             lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) == 0 &&
1456             lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0) == 0) {
1457                 /* all RPCs finished, and no active test */
1458                 batch->bat_state = LST_BATCH_IDLE;
1459         }
1460
1461         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1462                                           lstcon_tsbrpc_readent);
1463         lstcon_rpc_trans_destroy(trans);
1464
1465         return rc;
1466 }
1467
1468 static int
1469 lstcon_statrpc_readent(int transop, struct srpc_msg *msg,
1470                        struct lstcon_rpc_ent __user *ent_up)
1471 {
1472         struct srpc_stat_reply *rep = &msg->msg_body.stat_reply;
1473         struct sfw_counters __user *sfwk_stat;
1474         struct srpc_counters __user *srpc_stat;
1475         struct lnet_counters __user *lnet_stat;
1476
1477         if (rep->str_status != 0)
1478                 return 0;
1479
1480         sfwk_stat = (struct sfw_counters __user *)&ent_up->rpe_payload[0];
1481         srpc_stat = (struct srpc_counters __user *)
1482                 ((char __user *)sfwk_stat + sizeof(*sfwk_stat));
1483         lnet_stat = (struct lnet_counters __user *)
1484                 ((char __user *)srpc_stat + sizeof(*srpc_stat));
1485
1486         if (copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) ||
1487             copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) ||
1488             copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat)))
1489                 return -EFAULT;
1490
1491         return 0;
1492 }
1493
1494 static int
1495 lstcon_ndlist_stat(struct list_head *ndlist,
1496                    int timeout, struct list_head __user *result_up)
1497 {
1498         struct list_head    head;
1499         struct lstcon_rpc_trans *trans;
1500         int                 rc;
1501
1502         INIT_LIST_HEAD(&head);
1503
1504         rc = lstcon_rpc_trans_ndlist(ndlist, &head,
1505                                      LST_TRANS_STATQRY, NULL, NULL, &trans);
1506         if (rc != 0) {
1507                 CERROR("Can't create transaction: %d\n", rc);
1508                 return rc;
1509         }
1510
1511         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1512
1513         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1514                                           lstcon_statrpc_readent);
1515         lstcon_rpc_trans_destroy(trans);
1516
1517         return rc;
1518 }
1519
1520 int
1521 lstcon_group_stat(char *grp_name, int timeout,
1522                   struct list_head __user *result_up)
1523 {
1524         struct lstcon_group *grp;
1525         int rc;
1526
1527         rc = lstcon_group_find(grp_name, &grp);
1528         if (rc != 0) {
1529                 CDEBUG(D_NET, "Can't find group %s\n", grp_name);
1530                 return rc;
1531         }
1532
1533         rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up);
1534
1535         lstcon_group_decref(grp);
1536
1537         return rc;
1538 }
1539
1540 int
1541 lstcon_nodes_stat(int count, struct lnet_process_id __user *ids_up,
1542                   int timeout, struct list_head __user *result_up)
1543 {
1544         struct lstcon_ndlink *ndl;
1545         struct lstcon_group *tmp;
1546         struct lnet_process_id id;
1547         int i;
1548         int rc;
1549
1550         rc = lstcon_group_alloc(NULL, &tmp);
1551         if (rc != 0) {
1552                 CERROR("Out of memory\n");
1553                 return -ENOMEM;
1554         }
1555
1556         for (i = 0 ; i < count; i++) {
1557                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1558                         rc = -EFAULT;
1559                         break;
1560                 }
1561
1562                 /* add to tmp group */
1563                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2);
1564                 if (rc != 0) {
1565                         CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET,
1566                                "Failed to find or create %s: %d\n",
1567                                libcfs_id2str(id), rc);
1568                         break;
1569                 }
1570         }
1571
1572         if (rc != 0) {
1573                 lstcon_group_decref(tmp);
1574                 return rc;
1575         }
1576
1577         rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up);
1578
1579         lstcon_group_decref(tmp);
1580
1581         return rc;
1582 }
1583
1584 static int
1585 lstcon_debug_ndlist(struct list_head *ndlist,
1586                     struct list_head *translist,
1587                     int timeout, struct list_head __user *result_up)
1588 {
1589         struct lstcon_rpc_trans *trans;
1590         int rc;
1591
1592         rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY,
1593                                      NULL, lstcon_sesrpc_condition, &trans);
1594         if (rc != 0) {
1595                 CERROR("Can't create transaction: %d\n", rc);
1596                 return rc;
1597         }
1598
1599         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1600
1601         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1602                                           lstcon_sesrpc_readent);
1603         lstcon_rpc_trans_destroy(trans);
1604
1605         return rc;
1606 }
1607
1608 int
1609 lstcon_session_debug(int timeout, struct list_head __user *result_up)
1610 {
1611         return lstcon_debug_ndlist(&console_session.ses_ndl_list,
1612                                    NULL, timeout, result_up);
1613 }
1614
1615 int
1616 lstcon_batch_debug(int timeout, char *name,
1617                    int client, struct list_head __user *result_up)
1618 {
1619         struct lstcon_batch *bat;
1620         int rc;
1621
1622         rc = lstcon_batch_find(name, &bat);
1623         if (rc != 0)
1624                 return -ENOENT;
1625
1626         rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list :
1627                                           &bat->bat_srv_list,
1628                                  NULL, timeout, result_up);
1629
1630         return rc;
1631 }
1632
1633 int
1634 lstcon_group_debug(int timeout, char *name,
1635                    struct list_head __user *result_up)
1636 {
1637         struct lstcon_group *grp;
1638         int rc;
1639
1640         rc = lstcon_group_find(name, &grp);
1641         if (rc != 0)
1642                 return -ENOENT;
1643
1644         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1645                                  timeout, result_up);
1646         lstcon_group_decref(grp);
1647
1648         return rc;
1649 }
1650
1651 int
1652 lstcon_nodes_debug(int timeout, int count,
1653                    struct lnet_process_id __user *ids_up,
1654                    struct list_head __user *result_up)
1655 {
1656         struct lnet_process_id id;
1657         struct lstcon_ndlink *ndl;
1658         struct lstcon_group *grp;
1659         int i;
1660         int rc;
1661
1662         rc = lstcon_group_alloc(NULL, &grp);
1663         if (rc != 0) {
1664                 CDEBUG(D_NET, "Out of memory\n");
1665                 return rc;
1666         }
1667
1668         for (i = 0; i < count; i++) {
1669                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1670                         rc = -EFAULT;
1671                         break;
1672                 }
1673
1674                 /* node is added to tmp group */
1675                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 1);
1676                 if (rc != 0) {
1677                         CERROR("Can't create node link\n");
1678                         break;
1679                 }
1680         }
1681
1682         if (rc != 0) {
1683                 lstcon_group_decref(grp);
1684                 return rc;
1685         }
1686
1687         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1688                                  timeout, result_up);
1689
1690         lstcon_group_decref(grp);
1691
1692         return rc;
1693 }
1694
1695 int
1696 lstcon_session_match(struct lst_sid sid)
1697 {
1698         return (console_session.ses_id.ses_nid   == sid.ses_nid &&
1699                 console_session.ses_id.ses_stamp == sid.ses_stamp) ?  1: 0;
1700 }
1701
1702 static void
1703 lstcon_new_session_id(struct lst_sid *sid)
1704 {
1705         struct lnet_process_id id;
1706
1707         LASSERT(console_session.ses_state == LST_SESSION_NONE);
1708
1709         LNetGetId(1, &id);
1710         sid->ses_nid = id.nid;
1711         sid->ses_stamp = div_u64(ktime_get_ns(), NSEC_PER_MSEC);
1712 }
1713
1714 int
1715 lstcon_session_new(char *name, int key, unsigned feats,
1716                    int timeout, int force, struct lst_sid __user *sid_up)
1717 {
1718         int     rc = 0;
1719         int     i;
1720
1721         if (console_session.ses_state != LST_SESSION_NONE) {
1722                 /* session exists */
1723                 if (!force) {
1724                         CNETERR("Session %s already exists\n",
1725                                 console_session.ses_name);
1726                         return -EEXIST;
1727                 }
1728
1729                 rc = lstcon_session_end();
1730
1731                 /* lstcon_session_end() only return local error */
1732                 if  (rc != 0)
1733                         return rc;
1734         }
1735
1736         if ((feats & ~LST_FEATS_MASK) != 0) {
1737                 CNETERR("Unknown session features %x\n",
1738                         (feats & ~LST_FEATS_MASK));
1739                 return -EINVAL;
1740         }
1741
1742         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
1743                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
1744
1745         lstcon_new_session_id(&console_session.ses_id);
1746
1747         console_session.ses_key     = key;
1748         console_session.ses_state   = LST_SESSION_ACTIVE;
1749         console_session.ses_force   = !!force;
1750         console_session.ses_features = feats;
1751         console_session.ses_feats_updated = 0;
1752         console_session.ses_timeout = (timeout <= 0) ?
1753                                       LST_CONSOLE_TIMEOUT : timeout;
1754
1755         if (strlen(name) > sizeof(console_session.ses_name)-1)
1756                 return -E2BIG;
1757         strlcpy(console_session.ses_name, name,
1758                 sizeof(console_session.ses_name));
1759
1760         rc = lstcon_batch_add(LST_DEFAULT_BATCH);
1761         if (rc != 0)
1762                 return rc;
1763
1764         rc = lstcon_rpc_pinger_start();
1765         if (rc != 0) {
1766                 struct lstcon_batch *bat = NULL;
1767
1768                 lstcon_batch_find(LST_DEFAULT_BATCH, &bat);
1769                 lstcon_batch_destroy(bat);
1770
1771                 return rc;
1772         }
1773
1774         if (copy_to_user(sid_up, &console_session.ses_id,
1775                              sizeof(struct lst_sid)) == 0)
1776                 return rc;
1777
1778         lstcon_session_end();
1779
1780         return -EFAULT;
1781 }
1782
1783 int
1784 lstcon_session_info(struct lst_sid __user *sid_up, int __user *key_up,
1785                     unsigned __user *featp,
1786                     struct lstcon_ndlist_ent __user *ndinfo_up,
1787                     char __user *name_up, int len)
1788 {
1789         struct lstcon_ndlist_ent *entp;
1790         struct lstcon_ndlink *ndl;
1791         int rc = 0;
1792
1793         if (console_session.ses_state != LST_SESSION_ACTIVE)
1794                 return -ESRCH;
1795
1796         LIBCFS_ALLOC(entp, sizeof(*entp));
1797         if (entp == NULL)
1798                 return -ENOMEM;
1799
1800         list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link)
1801                 LST_NODE_STATE_COUNTER(ndl->ndl_node, entp);
1802
1803         if (copy_to_user(sid_up, &console_session.ses_id,
1804                          sizeof(struct lst_sid)) ||
1805             copy_to_user(key_up, &console_session.ses_key,
1806                              sizeof(*key_up)) ||
1807             copy_to_user(featp, &console_session.ses_features,
1808                              sizeof(*featp)) ||
1809             copy_to_user(ndinfo_up, entp, sizeof(*entp)) ||
1810             copy_to_user(name_up, console_session.ses_name, len))
1811                 rc = -EFAULT;
1812
1813         LIBCFS_FREE(entp, sizeof(*entp));
1814
1815         return rc;
1816 }
1817
1818 int
1819 lstcon_session_end()
1820 {
1821         struct lstcon_rpc_trans *trans;
1822         struct lstcon_group *grp;
1823         struct lstcon_batch *bat;
1824         int rc = 0;
1825
1826         LASSERT (console_session.ses_state == LST_SESSION_ACTIVE);
1827
1828         rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list,
1829                                      NULL, LST_TRANS_SESEND, NULL,
1830                                      lstcon_sesrpc_condition, &trans);
1831         if (rc != 0) {
1832                 CERROR("Can't create transaction: %d\n", rc);
1833                 return rc;
1834         }
1835
1836         console_session.ses_shutdown = 1;
1837
1838         lstcon_rpc_pinger_stop();
1839
1840         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1841
1842         lstcon_rpc_trans_destroy(trans);
1843         /* User can do nothing even rpc failed, so go on */
1844
1845         /* waiting for orphan rpcs to die */
1846         lstcon_rpc_cleanup_wait();
1847
1848         console_session.ses_id    = LST_INVALID_SID;
1849         console_session.ses_state = LST_SESSION_NONE;
1850         console_session.ses_key   = 0;
1851         console_session.ses_force = 0;
1852         console_session.ses_feats_updated = 0;
1853
1854         /* destroy all batches */
1855         while (!list_empty(&console_session.ses_bat_list)) {
1856                 bat = list_entry(console_session.ses_bat_list.next,
1857                                  struct lstcon_batch, bat_link);
1858
1859                 lstcon_batch_destroy(bat);
1860         }
1861
1862         /* destroy all groups */
1863         while (!list_empty(&console_session.ses_grp_list)) {
1864                 grp = list_entry(console_session.ses_grp_list.next,
1865                                  struct lstcon_group, grp_link);
1866                 LASSERT(grp->grp_ref == 1);
1867
1868                 lstcon_group_decref(grp);
1869         }
1870
1871         /* all nodes should be released */
1872         LASSERT(list_empty(&console_session.ses_ndl_list));
1873
1874         console_session.ses_shutdown = 0;
1875         console_session.ses_expired  = 0;
1876
1877         return rc;
1878 }
1879
1880 int
1881 lstcon_session_feats_check(unsigned feats)
1882 {
1883         int rc = 0;
1884
1885         if ((feats & ~LST_FEATS_MASK) != 0) {
1886                 CERROR("Can't support these features: %x\n",
1887                        (feats & ~LST_FEATS_MASK));
1888                 return -EPROTO;
1889         }
1890
1891         spin_lock(&console_session.ses_rpc_lock);
1892
1893         if (!console_session.ses_feats_updated) {
1894                 console_session.ses_feats_updated = 1;
1895                 console_session.ses_features = feats;
1896         }
1897
1898         if (console_session.ses_features != feats)
1899                 rc = -EPROTO;
1900
1901         spin_unlock(&console_session.ses_rpc_lock);
1902
1903         if (rc != 0) {
1904                 CERROR("remote features %x do not match with "
1905                        "session features %x of console\n",
1906                        feats, console_session.ses_features);
1907         }
1908
1909         return rc;
1910 }
1911
1912 static int
1913 lstcon_acceptor_handle(struct srpc_server_rpc *rpc)
1914 {
1915         struct srpc_msg *rep = &rpc->srpc_replymsg;
1916         struct srpc_msg *req = &rpc->srpc_reqstbuf->buf_msg;
1917         struct srpc_join_reqst *jreq = &req->msg_body.join_reqst;
1918         struct srpc_join_reply *jrep = &rep->msg_body.join_reply;
1919         struct lstcon_group *grp = NULL;
1920         struct lstcon_ndlink *ndl;
1921         int rc = 0;
1922
1923         sfw_unpack_message(req);
1924
1925         mutex_lock(&console_session.ses_mutex);
1926
1927         jrep->join_sid = console_session.ses_id;
1928
1929         if (console_session.ses_id.ses_nid == LNET_NID_ANY) {
1930                 jrep->join_status = ESRCH;
1931                 goto out;
1932         }
1933
1934         if (lstcon_session_feats_check(req->msg_ses_feats) != 0) {
1935                 jrep->join_status = EPROTO;
1936                 goto out;
1937         }
1938
1939         if (jreq->join_sid.ses_nid != LNET_NID_ANY &&
1940              !lstcon_session_match(jreq->join_sid)) {
1941                 jrep->join_status = EBUSY;
1942                 goto out;
1943         }
1944
1945         if (lstcon_group_find(jreq->join_group, &grp) != 0) {
1946                 rc = lstcon_group_alloc(jreq->join_group, &grp);
1947                 if (rc != 0) {
1948                         CERROR("Out of memory\n");
1949                         goto out;
1950                 }
1951
1952                 list_add_tail(&grp->grp_link,
1953                               &console_session.ses_grp_list);
1954                 lstcon_group_addref(grp);
1955         }
1956
1957         if (grp->grp_ref > 2) {
1958                 /* Group in using */
1959                 jrep->join_status = EBUSY;
1960                 goto out;
1961         }
1962
1963         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0);
1964         if (rc == 0) {
1965                 jrep->join_status = EEXIST;
1966                 goto out;
1967         }
1968
1969         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1);
1970         if (rc != 0) {
1971                 CERROR("Out of memory\n");
1972                 goto out;
1973         }
1974
1975         ndl->ndl_node->nd_state   = LST_NODE_ACTIVE;
1976         ndl->ndl_node->nd_timeout = console_session.ses_timeout;
1977
1978         if (grp->grp_userland == 0)
1979                 grp->grp_userland = 1;
1980
1981         strlcpy(jrep->join_session, console_session.ses_name,
1982                 sizeof(jrep->join_session));
1983         jrep->join_timeout = console_session.ses_timeout;
1984         jrep->join_status  = 0;
1985
1986 out:
1987         rep->msg_ses_feats = console_session.ses_features;
1988         if (grp != NULL)
1989                 lstcon_group_decref(grp);
1990
1991         mutex_unlock(&console_session.ses_mutex);
1992
1993         return rc;
1994 }
1995
1996 static struct srpc_service lstcon_acceptor_service;
1997
1998 static void lstcon_init_acceptor_service(void)
1999 {
2000         /* initialize selftest console acceptor service table */
2001         lstcon_acceptor_service.sv_name    = "join session";
2002         lstcon_acceptor_service.sv_handler = lstcon_acceptor_handle;
2003         lstcon_acceptor_service.sv_id      = SRPC_SERVICE_JOIN;
2004         lstcon_acceptor_service.sv_wi_total = SFW_FRWK_WI_MAX;
2005 }
2006
2007 int lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_hdr *hdr);
2008
2009 DECLARE_IOCTL_HANDLER(lstcon_ioctl_handler, lstcon_ioctl_entry);
2010
2011 /* initialize console */
2012 int
2013 lstcon_console_init(void)
2014 {
2015         int     i;
2016         int     rc;
2017
2018         console_session.ses_id              = LST_INVALID_SID;
2019         console_session.ses_state           = LST_SESSION_NONE;
2020         console_session.ses_timeout         = 0;
2021         console_session.ses_force           = 0;
2022         console_session.ses_expired         = 0;
2023         console_session.ses_feats_updated   = 0;
2024         console_session.ses_features        = LST_FEATS_MASK;
2025         console_session.ses_laststamp = ktime_get_real_seconds();
2026
2027         mutex_init(&console_session.ses_mutex);
2028
2029         INIT_LIST_HEAD(&console_session.ses_ndl_list);
2030         INIT_LIST_HEAD(&console_session.ses_grp_list);
2031         INIT_LIST_HEAD(&console_session.ses_bat_list);
2032         INIT_LIST_HEAD(&console_session.ses_trans_list);
2033
2034         LIBCFS_ALLOC(console_session.ses_ndl_hash,
2035                      sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2036         if (console_session.ses_ndl_hash == NULL)
2037                 return -ENOMEM;
2038
2039         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
2040                 INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]);
2041
2042
2043         /* initialize acceptor service table */
2044         lstcon_init_acceptor_service();
2045
2046         rc = srpc_add_service(&lstcon_acceptor_service);
2047         LASSERT(rc != -EBUSY);
2048         if (rc != 0) {
2049                 LIBCFS_FREE(console_session.ses_ndl_hash,
2050                             sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2051                 return rc;
2052         }
2053
2054         rc = srpc_service_add_buffers(&lstcon_acceptor_service,
2055                                       lstcon_acceptor_service.sv_wi_total);
2056         if (rc != 0) {
2057                 rc = -ENOMEM;
2058                 goto out;
2059         }
2060
2061         rc = libcfs_register_ioctl(&lstcon_ioctl_handler);
2062
2063         if (rc == 0) {
2064                 lstcon_rpc_module_init();
2065                 return 0;
2066         }
2067
2068 out:
2069         srpc_shutdown_service(&lstcon_acceptor_service);
2070         srpc_remove_service(&lstcon_acceptor_service);
2071
2072         LIBCFS_FREE(console_session.ses_ndl_hash,
2073                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2074
2075         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2076
2077         return rc;
2078 }
2079
2080 int
2081 lstcon_console_fini(void)
2082 {
2083         int     i;
2084
2085         libcfs_deregister_ioctl(&lstcon_ioctl_handler);
2086
2087         mutex_lock(&console_session.ses_mutex);
2088
2089         srpc_shutdown_service(&lstcon_acceptor_service);
2090         srpc_remove_service(&lstcon_acceptor_service);
2091
2092         if (console_session.ses_state != LST_SESSION_NONE)
2093                 lstcon_session_end();
2094
2095         lstcon_rpc_module_fini();
2096
2097         mutex_unlock(&console_session.ses_mutex);
2098
2099         LASSERT(list_empty(&console_session.ses_ndl_list));
2100         LASSERT(list_empty(&console_session.ses_grp_list));
2101         LASSERT(list_empty(&console_session.ses_bat_list));
2102         LASSERT(list_empty(&console_session.ses_trans_list));
2103
2104         for (i = 0; i < LST_NODE_HASHSIZE; i++)
2105                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
2106
2107         LIBCFS_FREE(console_session.ses_ndl_hash,
2108                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2109
2110         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2111
2112         return 0;
2113 }
2114