lnet/selftest/console.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  *
  30  * Copyright (c) 2012, 2014, Intel Corporation.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * lnet/selftest/conctl.c
  37  *
  38  * Infrastructure of LST console
  39  *
  40  * Author: Liang Zhen <liangzhen@clusterfs.com>
  41  */
  42
  43
  44 #include <libcfs/libcfs.h>
  45 #include <lnet/lib-lnet.h>
  46 #include "console.h"
  47 #include "conrpc.h"
  48
  49 #define LST_NODE_STATE_COUNTER(nd, p)                   \
  50 do {                                                    \
  51         if ((nd)->nd_state == LST_NODE_ACTIVE)          \
  52                 (p)->nle_nactive ++;                    \
  53         else if ((nd)->nd_state == LST_NODE_BUSY)       \
  54                 (p)->nle_nbusy ++;                      \
  55         else if ((nd)->nd_state == LST_NODE_DOWN)       \
  56                 (p)->nle_ndown ++;                      \
  57         else                                            \
  58                 (p)->nle_nunknown ++;                   \
  59         (p)->nle_nnode ++;                              \
  60 } while (0)
  61
  62 lstcon_session_t        console_session;
  63
  64 static void
  65 lstcon_node_get(lstcon_node_t *nd)
  66 {
  67         LASSERT (nd->nd_ref >= 1);
  68
  69         nd->nd_ref++;
  70 }
  71
  72 static int
  73 lstcon_node_find(lnet_process_id_t id, lstcon_node_t **ndpp, int create)
  74 {
  75         lstcon_ndlink_t *ndl;
  76         unsigned int     idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE;
  77
  78         LASSERT(id.nid != LNET_NID_ANY);
  79
  80         list_for_each_entry(ndl, &console_session.ses_ndl_hash[idx],
  81                             ndl_hlink) {
  82                 if (ndl->ndl_node->nd_id.nid != id.nid ||
  83                     ndl->ndl_node->nd_id.pid != id.pid)
  84                         continue;
  85
  86                 lstcon_node_get(ndl->ndl_node);
  87                 *ndpp = ndl->ndl_node;
  88                 return 0;
  89         }
  90
  91         if (!create)
  92                 return -ENOENT;
  93
  94         LIBCFS_ALLOC(*ndpp, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
  95         if (*ndpp == NULL)
  96                 return -ENOMEM;
  97
  98         ndl = (lstcon_ndlink_t *)(*ndpp + 1);
  99
 100         ndl->ndl_node = *ndpp;
 101
 102         ndl->ndl_node->nd_ref   = 1;
 103         ndl->ndl_node->nd_id    = id;
 104         ndl->ndl_node->nd_stamp = cfs_time_current();
 105         ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
 106         ndl->ndl_node->nd_timeout = 0;
 107         memset(&ndl->ndl_node->nd_ping, 0, sizeof(lstcon_rpc_t));
 108
 109         /* queued in global hash & list, no refcount is taken by
 110          * global hash & list, if caller release his refcount,
 111          * node will be released */
 112         list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]);
 113         list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list);
 114
 115         return 0;
 116 }
 117
 118 static void
 119 lstcon_node_put(lstcon_node_t *nd)
 120 {
 121         lstcon_ndlink_t *ndl;
 122
 123         LASSERT(nd->nd_ref > 0);
 124
 125         if (--nd->nd_ref > 0)
 126                 return;
 127
 128         ndl = (lstcon_ndlink_t *)(nd + 1);
 129
 130         LASSERT(!list_empty(&ndl->ndl_link));
 131         LASSERT(!list_empty(&ndl->ndl_hlink));
 132
 133         /* remove from session */
 134         list_del(&ndl->ndl_link);
 135         list_del(&ndl->ndl_hlink);
 136
 137         LIBCFS_FREE(nd, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
 138 }
 139
 140 static int
 141 lstcon_ndlink_find(struct list_head *hash,
 142                    lnet_process_id_t id, lstcon_ndlink_t **ndlpp, int create)
 143 {
 144         unsigned int     idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
 145         lstcon_ndlink_t *ndl;
 146         lstcon_node_t   *nd;
 147         int              rc;
 148
 149         if (id.nid == LNET_NID_ANY)
 150                 return -EINVAL;
 151
 152         /* search in hash */
 153         list_for_each_entry(ndl, &hash[idx], ndl_hlink) {
 154                 if (ndl->ndl_node->nd_id.nid != id.nid ||
 155                     ndl->ndl_node->nd_id.pid != id.pid)
 156                         continue;
 157
 158                 *ndlpp = ndl;
 159                 return 0;
 160         }
 161
 162         if (create == 0)
 163                 return -ENOENT;
 164
 165         /* find or create in session hash */
 166         rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0);
 167         if (rc != 0)
 168                 return rc;
 169
 170         LIBCFS_ALLOC(ndl, sizeof(lstcon_ndlink_t));
 171         if (ndl == NULL) {
 172                 lstcon_node_put(nd);
 173                 return -ENOMEM;
 174         }
 175
 176         *ndlpp = ndl;
 177
 178         ndl->ndl_node = nd;
 179         INIT_LIST_HEAD(&ndl->ndl_link);
 180         list_add_tail(&ndl->ndl_hlink, &hash[idx]);
 181
 182         return 0;
 183 }
 184
 185 static void
 186 lstcon_ndlink_release(lstcon_ndlink_t *ndl)
 187 {
 188         LASSERT(list_empty(&ndl->ndl_link));
 189         LASSERT(!list_empty(&ndl->ndl_hlink));
 190
 191         list_del(&ndl->ndl_hlink); /* delete from hash */
 192         lstcon_node_put(ndl->ndl_node);
 193
 194         LIBCFS_FREE(ndl, sizeof(*ndl));
 195 }
 196
 197 static int
 198 lstcon_group_alloc(char *name, lstcon_group_t **grpp)
 199 {
 200         lstcon_group_t *grp;
 201         int             i;
 202
 203         LIBCFS_ALLOC(grp, offsetof(lstcon_group_t,
 204                                    grp_ndl_hash[LST_NODE_HASHSIZE]));
 205         if (grp == NULL)
 206                 return -ENOMEM;
 207
 208         memset(grp, 0, offsetof(lstcon_group_t,
 209                                 grp_ndl_hash[LST_NODE_HASHSIZE]));
 210
 211         grp->grp_ref = 1;
 212         if (name != NULL) {
 213                 if (strlen(name) > sizeof(grp->grp_name)-1) {
 214                         LIBCFS_FREE(grp, offsetof(lstcon_group_t,
 215                                           grp_ndl_hash[LST_NODE_HASHSIZE]));
 216                         return -E2BIG;
 217                 }
 218                 strncpy(grp->grp_name, name, sizeof(grp->grp_name));
 219         }
 220
 221         INIT_LIST_HEAD(&grp->grp_link);
 222         INIT_LIST_HEAD(&grp->grp_ndl_list);
 223         INIT_LIST_HEAD(&grp->grp_trans_list);
 224
 225         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 226                 INIT_LIST_HEAD(&grp->grp_ndl_hash[i]);
 227
 228         *grpp = grp;
 229
 230         return 0;
 231 }
 232
 233 static void
 234 lstcon_group_addref(lstcon_group_t *grp)
 235 {
 236         grp->grp_ref++;
 237 }
 238
 239 static void lstcon_group_ndlink_release(lstcon_group_t *, lstcon_ndlink_t *);
 240
 241 static void
 242 lstcon_group_drain(lstcon_group_t *grp, int keep)
 243 {
 244         lstcon_ndlink_t *ndl;
 245         lstcon_ndlink_t *tmp;
 246
 247         list_for_each_entry_safe(ndl, tmp, &grp->grp_ndl_list, ndl_link) {
 248                 if ((ndl->ndl_node->nd_state & keep) == 0)
 249                         lstcon_group_ndlink_release(grp, ndl);
 250         }
 251 }
 252
 253 static void
 254 lstcon_group_decref(lstcon_group_t *grp)
 255 {
 256         int i;
 257
 258         if (--grp->grp_ref > 0)
 259                 return;
 260
 261         if (!list_empty(&grp->grp_link))
 262                 list_del(&grp->grp_link);
 263
 264         lstcon_group_drain(grp, 0);
 265
 266         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 267                 LASSERT(list_empty(&grp->grp_ndl_hash[i]));
 268
 269         LIBCFS_FREE(grp, offsetof(lstcon_group_t,
 270                                   grp_ndl_hash[LST_NODE_HASHSIZE]));
 271 }
 272
 273 static int
 274 lstcon_group_find(const char *name, lstcon_group_t **grpp)
 275 {
 276         lstcon_group_t *grp;
 277
 278         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 279                 if (strncmp(grp->grp_name, name, LST_NAME_SIZE) != 0)
 280                         continue;
 281
 282                 lstcon_group_addref(grp);  /* +1 ref for caller */
 283                 *grpp = grp;
 284                 return 0;
 285         }
 286
 287         return -ENOENT;
 288 }
 289
 290 static void
 291 lstcon_group_put(lstcon_group_t *grp)
 292 {
 293         lstcon_group_decref(grp);
 294 }
 295
 296 static int
 297 lstcon_group_ndlink_find(lstcon_group_t *grp, lnet_process_id_t id,
 298                          lstcon_ndlink_t **ndlpp, int create)
 299 {
 300         int rc;
 301
 302         rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create);
 303         if (rc != 0)
 304                 return rc;
 305
 306         if (!list_empty(&(*ndlpp)->ndl_link))
 307                 return 0;
 308
 309         list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list);
 310         grp->grp_nnode++;
 311
 312         return 0;
 313 }
 314
 315 static void
 316 lstcon_group_ndlink_release(lstcon_group_t *grp, lstcon_ndlink_t *ndl)
 317 {
 318         list_del_init(&ndl->ndl_link);
 319         lstcon_ndlink_release(ndl);
 320         grp->grp_nnode--;
 321 }
 322
 323 static void
 324 lstcon_group_ndlink_move(lstcon_group_t *old,
 325                          lstcon_group_t *new, lstcon_ndlink_t *ndl)
 326 {
 327         unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) %
 328                                         LST_NODE_HASHSIZE;
 329
 330         list_del(&ndl->ndl_hlink);
 331         list_del(&ndl->ndl_link);
 332         old->grp_nnode--;
 333
 334         list_add_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]);
 335         list_add_tail(&ndl->ndl_link, &new->grp_ndl_list);
 336         new->grp_nnode++;
 337
 338         return;
 339 }
 340
 341 static void
 342 lstcon_group_move(lstcon_group_t *old, lstcon_group_t *new)
 343 {
 344         lstcon_ndlink_t *ndl;
 345
 346         while (!list_empty(&old->grp_ndl_list)) {
 347                 ndl = list_entry(old->grp_ndl_list.next,
 348                                  lstcon_ndlink_t, ndl_link);
 349                 lstcon_group_ndlink_move(old, new, ndl);
 350         }
 351 }
 352
 353 static int
 354 lstcon_sesrpc_condition(int transop, lstcon_node_t *nd, void *arg)
 355 {
 356         lstcon_group_t *grp = (lstcon_group_t *)arg;
 357
 358         switch (transop) {
 359         case LST_TRANS_SESNEW:
 360                 if (nd->nd_state == LST_NODE_ACTIVE)
 361                         return 0;
 362                 break;
 363
 364         case LST_TRANS_SESEND:
 365                 if (nd->nd_state != LST_NODE_ACTIVE)
 366                         return 0;
 367
 368                 if (grp != NULL && nd->nd_ref > 1)
 369                         return 0;
 370                 break;
 371
 372         case LST_TRANS_SESQRY:
 373                 break;
 374
 375         default:
 376                 LBUG();
 377         }
 378
 379         return 1;
 380 }
 381
 382 static int
 383 lstcon_sesrpc_readent(int transop, srpc_msg_t *msg,
 384                       lstcon_rpc_ent_t __user *ent_up)
 385 {
 386         srpc_debug_reply_t *rep;
 387
 388         switch (transop) {
 389         case LST_TRANS_SESNEW:
 390         case LST_TRANS_SESEND:
 391                 return 0;
 392
 393         case LST_TRANS_SESQRY:
 394                 rep = &msg->msg_body.dbg_reply;
 395
 396                 if (copy_to_user(&ent_up->rpe_priv[0],
 397                                      &rep->dbg_timeout, sizeof(int)) ||
 398                     copy_to_user(&ent_up->rpe_payload[0],
 399                                      &rep->dbg_name, LST_NAME_SIZE))
 400                         return -EFAULT;
 401
 402                 return 0;
 403
 404         default:
 405                 LBUG();
 406         }
 407
 408         return 0;
 409 }
 410
 411 static int
 412 lstcon_group_nodes_add(lstcon_group_t *grp,
 413                        int count, lnet_process_id_t __user *ids_up,
 414                        unsigned *featp, struct list_head __user *result_up)
 415 {
 416         lstcon_rpc_trans_t      *trans;
 417         lstcon_ndlink_t         *ndl;
 418         lstcon_group_t          *tmp;
 419         lnet_process_id_t        id;
 420         int                      i;
 421         int                      rc;
 422
 423         rc = lstcon_group_alloc(NULL, &tmp);
 424         if (rc != 0) {
 425                 CERROR("Out of memory\n");
 426                 return -ENOMEM;
 427         }
 428
 429         for (i = 0 ; i < count; i++) {
 430                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 431                         rc = -EFAULT;
 432                         break;
 433                 }
 434
 435                 /* skip if it's in this group already */
 436                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 0);
 437                 if (rc == 0)
 438                         continue;
 439
 440                 /* add to tmp group */
 441                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1);
 442                 if (rc != 0) {
 443                         CERROR("Can't create ndlink, out of memory\n");
 444                         break;
 445                 }
 446         }
 447
 448         if (rc != 0) {
 449                 lstcon_group_put(tmp);
 450                 return rc;
 451         }
 452
 453         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 454                                      &tmp->grp_trans_list, LST_TRANS_SESNEW,
 455                                      tmp, lstcon_sesrpc_condition, &trans);
 456         if (rc != 0) {
 457                 CERROR("Can't create transaction: %d\n", rc);
 458                 lstcon_group_put(tmp);
 459                 return rc;
 460         }
 461
 462         /* post all RPCs */
 463         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 464
 465         rc = lstcon_rpc_trans_interpreter(trans, result_up,
 466                                           lstcon_sesrpc_readent);
 467         *featp = trans->tas_features;
 468
 469         /* destroy all RPGs */
 470         lstcon_rpc_trans_destroy(trans);
 471
 472         lstcon_group_move(tmp, grp);
 473         lstcon_group_put(tmp);
 474
 475         return rc;
 476 }
 477
 478 static int
 479 lstcon_group_nodes_remove(lstcon_group_t *grp,
 480                           int count, lnet_process_id_t __user *ids_up,
 481                           struct list_head __user *result_up)
 482 {
 483         lstcon_rpc_trans_t     *trans;
 484         lstcon_ndlink_t        *ndl;
 485         lstcon_group_t         *tmp;
 486         lnet_process_id_t       id;
 487         int                     rc;
 488         int                     i;
 489
 490         /* End session and remove node from the group */
 491
 492         rc = lstcon_group_alloc(NULL, &tmp);
 493         if (rc != 0) {
 494                 CERROR("Out of memory\n");
 495                 return -ENOMEM;
 496         }
 497
 498         for (i = 0; i < count; i++) {
 499                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 500                         rc = -EFAULT;
 501                         goto error;
 502                 }
 503
 504                 /* move node to tmp group */
 505                 if (lstcon_group_ndlink_find(grp, id, &ndl, 0) == 0)
 506                         lstcon_group_ndlink_move(grp, tmp, ndl);
 507         }
 508
 509         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 510                                      &tmp->grp_trans_list, LST_TRANS_SESEND,
 511                                      tmp, lstcon_sesrpc_condition, &trans);
 512         if (rc != 0) {
 513                 CERROR("Can't create transaction: %d\n", rc);
 514                 goto error;
 515         }
 516
 517         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 518
 519         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 520
 521         lstcon_rpc_trans_destroy(trans);
 522         /* release nodes anyway, because we can't rollback status */
 523         lstcon_group_put(tmp);
 524
 525         return rc;
 526 error:
 527         lstcon_group_move(tmp, grp);
 528         lstcon_group_put(tmp);
 529
 530         return rc;
 531 }
 532
 533 int
 534 lstcon_group_add(char *name)
 535 {
 536         lstcon_group_t *grp;
 537         int             rc;
 538
 539         rc = (lstcon_group_find(name, &grp) == 0)? -EEXIST: 0;
 540         if (rc != 0) {
 541                 /* find a group with same name */
 542                 lstcon_group_put(grp);
 543                 return rc;
 544         }
 545
 546         rc = lstcon_group_alloc(name, &grp);
 547         if (rc != 0) {
 548                 CERROR("Can't allocate descriptor for group %s\n", name);
 549                 return -ENOMEM;
 550         }
 551
 552         list_add_tail(&grp->grp_link, &console_session.ses_grp_list);
 553
 554         return rc;
 555 }
 556
 557 int
 558 lstcon_nodes_add(char *name, int count, lnet_process_id_t __user *ids_up,
 559                  unsigned *featp, struct list_head __user *result_up)
 560 {
 561         lstcon_group_t         *grp;
 562         int                     rc;
 563
 564         LASSERT (count > 0);
 565         LASSERT (ids_up != NULL);
 566
 567         rc = lstcon_group_find(name, &grp);
 568         if (rc != 0) {
 569                 CDEBUG(D_NET, "Can't find group %s\n", name);
 570                 return rc;
 571         }
 572
 573         if (grp->grp_ref > 2) {
 574                 /* referred by other threads or test */
 575                 CDEBUG(D_NET, "Group %s is busy\n", name);
 576                 lstcon_group_put(grp);
 577
 578                 return -EBUSY;
 579         }
 580
 581         rc = lstcon_group_nodes_add(grp, count, ids_up, featp, result_up);
 582
 583         lstcon_group_put(grp);
 584
 585         return rc;
 586 }
 587
 588 int
 589 lstcon_group_del(char *name)
 590 {
 591         lstcon_rpc_trans_t *trans;
 592         lstcon_group_t     *grp;
 593         int                 rc;
 594
 595         rc = lstcon_group_find(name, &grp);
 596         if (rc != 0) {
 597                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 598                 return rc;
 599         }
 600
 601         if (grp->grp_ref > 2) {
 602                 /* referred by others threads or test */
 603                 CDEBUG(D_NET, "Group %s is busy\n", name);
 604                 lstcon_group_put(grp);
 605                 return -EBUSY;
 606         }
 607
 608         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 609                                      &grp->grp_trans_list, LST_TRANS_SESEND,
 610                                      grp, lstcon_sesrpc_condition, &trans);
 611         if (rc != 0) {
 612                 CERROR("Can't create transaction: %d\n", rc);
 613                 lstcon_group_put(grp);
 614                 return rc;
 615         }
 616
 617         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 618
 619         lstcon_rpc_trans_destroy(trans);
 620
 621         lstcon_group_put(grp);
 622         /* -ref for session, it's destroyed,
 623          * status can't be rolled back, destroy group anway */
 624         lstcon_group_put(grp);
 625
 626         return rc;
 627 }
 628
 629 int
 630 lstcon_group_clean(char *name, int args)
 631 {
 632         lstcon_group_t *grp = NULL;
 633         int             rc;
 634
 635         rc = lstcon_group_find(name, &grp);
 636         if (rc != 0) {
 637                 CDEBUG(D_NET, "Can't find group %s\n", name);
 638                 return rc;
 639         }
 640
 641         if (grp->grp_ref > 2) {
 642                 /* referred by test */
 643                 CDEBUG(D_NET, "Group %s is busy\n", name);
 644                 lstcon_group_put(grp);
 645                 return -EBUSY;
 646         }
 647
 648         args = (LST_NODE_ACTIVE | LST_NODE_BUSY |
 649                 LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args;
 650
 651         lstcon_group_drain(grp, args);
 652
 653         lstcon_group_put(grp);
 654         /* release empty group */
 655         if (list_empty(&grp->grp_ndl_list))
 656                 lstcon_group_put(grp);
 657
 658         return 0;
 659 }
 660
 661 int
 662 lstcon_nodes_remove(char *name, int count,
 663                     lnet_process_id_t __user *ids_up,
 664                     struct list_head __user *result_up)
 665 {
 666         lstcon_group_t *grp = NULL;
 667         int             rc;
 668
 669         rc = lstcon_group_find(name, &grp);
 670         if (rc != 0) {
 671                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 672                 return rc;
 673         }
 674
 675         if (grp->grp_ref > 2) {
 676                 /* referred by test */
 677                 CDEBUG(D_NET, "Group %s is busy\n", name);
 678                 lstcon_group_put(grp);
 679                 return -EBUSY;
 680         }
 681
 682         rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up);
 683
 684         lstcon_group_put(grp);
 685         /* release empty group */
 686         if (list_empty(&grp->grp_ndl_list))
 687                 lstcon_group_put(grp);
 688
 689         return rc;
 690 }
 691
 692 int
 693 lstcon_group_refresh(char *name, struct list_head __user *result_up)
 694 {
 695         lstcon_rpc_trans_t      *trans;
 696         lstcon_group_t          *grp;
 697         int                      rc;
 698
 699         rc = lstcon_group_find(name, &grp);
 700         if (rc != 0) {
 701                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 702                 return rc;
 703         }
 704
 705         if (grp->grp_ref > 2) {
 706                 /* referred by test */
 707                 CDEBUG(D_NET, "Group %s is busy\n", name);
 708                 lstcon_group_put(grp);
 709                 return -EBUSY;
 710         }
 711
 712         /* re-invite all inactive nodes int the group */
 713         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 714                                      &grp->grp_trans_list, LST_TRANS_SESNEW,
 715                                      grp, lstcon_sesrpc_condition, &trans);
 716         if (rc != 0) {
 717                 /* local error, return */
 718                 CDEBUG(D_NET, "Can't create transaction: %d\n", rc);
 719                 lstcon_group_put(grp);
 720                 return rc;
 721         }
 722
 723         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 724
 725         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 726
 727         lstcon_rpc_trans_destroy(trans);
 728         /* -ref for me */
 729         lstcon_group_put(grp);
 730
 731         return rc;
 732 }
 733
 734 int
 735 lstcon_group_list(int index, int len, char __user *name_up)
 736 {
 737         lstcon_group_t *grp;
 738
 739         LASSERT(index >= 0);
 740         LASSERT(name_up != NULL);
 741
 742         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 743                 if (index-- == 0) {
 744                         return copy_to_user(name_up, grp->grp_name, len) ?
 745                                             -EFAULT : 0;
 746                 }
 747         }
 748
 749         return -ENOENT;
 750 }
 751
 752 static int
 753 lstcon_nodes_getent(struct list_head *head, int *index_p,
 754                     int *count_p, lstcon_node_ent_t __user *dents_up)
 755 {
 756         lstcon_ndlink_t  *ndl;
 757         lstcon_node_t    *nd;
 758         int               count = 0;
 759         int               index = 0;
 760
 761         LASSERT(index_p != NULL && count_p != NULL);
 762         LASSERT(dents_up != NULL);
 763         LASSERT(*index_p >= 0);
 764         LASSERT(*count_p > 0);
 765
 766         list_for_each_entry(ndl, head, ndl_link) {
 767                 if (index++ < *index_p)
 768                         continue;
 769
 770                 if (count >= *count_p)
 771                         break;
 772
 773                 nd = ndl->ndl_node;
 774                 if (copy_to_user(&dents_up[count].nde_id,
 775                                      &nd->nd_id, sizeof(nd->nd_id)) ||
 776                     copy_to_user(&dents_up[count].nde_state,
 777                                      &nd->nd_state, sizeof(nd->nd_state)))
 778                         return -EFAULT;
 779
 780                 count ++;
 781         }
 782
 783         if (index <= *index_p)
 784                 return -ENOENT;
 785
 786         *count_p = count;
 787         *index_p = index;
 788
 789         return 0;
 790 }
 791
 792 int
 793 lstcon_group_info(char *name, lstcon_ndlist_ent_t __user *gents_p,
 794                   int *index_p, int *count_p,
 795                   lstcon_node_ent_t __user *dents_up)
 796 {
 797         lstcon_ndlist_ent_t *gentp;
 798         lstcon_group_t      *grp;
 799         lstcon_ndlink_t     *ndl;
 800         int                  rc;
 801
 802         rc = lstcon_group_find(name, &grp);
 803         if (rc != 0) {
 804                 CDEBUG(D_NET, "Can't find group %s\n", name);
 805                 return rc;
 806         }
 807
 808         if (dents_up != NULL) {
 809                 /* verbose query */
 810                 rc = lstcon_nodes_getent(&grp->grp_ndl_list,
 811                                          index_p, count_p, dents_up);
 812                 lstcon_group_put(grp);
 813
 814                 return rc;
 815         }
 816
 817         /* non-verbose query */
 818         LIBCFS_ALLOC(gentp, sizeof(lstcon_ndlist_ent_t));
 819         if (gentp == NULL) {
 820                 CERROR("Can't allocate ndlist_ent\n");
 821                 lstcon_group_put(grp);
 822
 823                 return -ENOMEM;
 824         }
 825
 826         memset(gentp, 0, sizeof(lstcon_ndlist_ent_t));
 827
 828         list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link)
 829                 LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp);
 830
 831         rc = copy_to_user(gents_p, gentp,
 832                           sizeof(lstcon_ndlist_ent_t)) ? -EFAULT : 0;
 833
 834         LIBCFS_FREE(gentp, sizeof(lstcon_ndlist_ent_t));
 835
 836         lstcon_group_put(grp);
 837
 838         return 0;
 839 }
 840
 841 static int
 842 lstcon_batch_find(const char *name, lstcon_batch_t **batpp)
 843 {
 844         lstcon_batch_t *bat;
 845
 846         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 847                 if (strncmp(bat->bat_name, name, LST_NAME_SIZE) == 0) {
 848                         *batpp = bat;
 849                         return 0;
 850                 }
 851         }
 852
 853         return -ENOENT;
 854 }
 855
 856 int
 857 lstcon_batch_add(char *name)
 858 {
 859         lstcon_batch_t   *bat;
 860         int               i;
 861         int               rc;
 862
 863         rc = (lstcon_batch_find(name, &bat) == 0)? -EEXIST: 0;
 864         if (rc != 0) {
 865                 CDEBUG(D_NET, "Batch %s already exists\n", name);
 866                 return rc;
 867         }
 868
 869         LIBCFS_ALLOC(bat, sizeof(lstcon_batch_t));
 870         if (bat == NULL) {
 871                 CERROR("Can't allocate descriptor for batch %s\n", name);
 872                 return -ENOMEM;
 873         }
 874
 875         LIBCFS_ALLOC(bat->bat_cli_hash,
 876                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 877         if (bat->bat_cli_hash == NULL) {
 878                 CERROR("Can't allocate hash for batch %s\n", name);
 879                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 880
 881                 return -ENOMEM;
 882         }
 883
 884         LIBCFS_ALLOC(bat->bat_srv_hash,
 885                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 886         if (bat->bat_srv_hash == NULL) {
 887                 CERROR("Can't allocate hash for batch %s\n", name);
 888                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 889                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 890
 891                 return -ENOMEM;
 892         }
 893
 894         if (strlen(name) > sizeof(bat->bat_name)-1) {
 895                 LIBCFS_FREE(bat->bat_srv_hash, LST_NODE_HASHSIZE);
 896                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 897                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 898                 return -E2BIG;
 899         }
 900         strncpy(bat->bat_name, name, sizeof(bat->bat_name));
 901         bat->bat_hdr.tsb_index = 0;
 902         bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie;
 903
 904         bat->bat_ntest = 0;
 905         bat->bat_state = LST_BATCH_IDLE;
 906
 907         INIT_LIST_HEAD(&bat->bat_cli_list);
 908         INIT_LIST_HEAD(&bat->bat_srv_list);
 909         INIT_LIST_HEAD(&bat->bat_test_list);
 910         INIT_LIST_HEAD(&bat->bat_trans_list);
 911
 912         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 913                 INIT_LIST_HEAD(&bat->bat_cli_hash[i]);
 914                 INIT_LIST_HEAD(&bat->bat_srv_hash[i]);
 915         }
 916
 917         list_add_tail(&bat->bat_link, &console_session.ses_bat_list);
 918
 919         return rc;
 920 }
 921
 922 int
 923 lstcon_batch_list(int index, int len, char __user *name_up)
 924 {
 925         lstcon_batch_t *bat;
 926
 927         LASSERT(name_up != NULL);
 928         LASSERT(index >= 0);
 929
 930         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 931                 if (index-- == 0) {
 932                         return copy_to_user(name_up, bat->bat_name, len) ?
 933                                             -EFAULT : 0;
 934                 }
 935         }
 936
 937         return -ENOENT;
 938 }
 939
 940 int
 941 lstcon_batch_info(char *name, lstcon_test_batch_ent_t __user *ent_up,
 942                   int server, int testidx, int *index_p, int *ndent_p,
 943                   lstcon_node_ent_t __user *dents_up)
 944 {
 945         lstcon_test_batch_ent_t *entp;
 946         struct list_head        *clilst;
 947         struct list_head        *srvlst;
 948         lstcon_test_t           *test = NULL;
 949         lstcon_batch_t          *bat;
 950         lstcon_ndlink_t         *ndl;
 951         int                      rc;
 952
 953         rc = lstcon_batch_find(name, &bat);
 954         if (rc != 0) {
 955                 CDEBUG(D_NET, "Can't find batch %s\n", name);
 956                 return -ENOENT;
 957         }
 958
 959         if (testidx > 0) {
 960                 /* query test, test index start from 1 */
 961                 list_for_each_entry(test, &bat->bat_test_list, tes_link) {
 962                         if (testidx-- == 1)
 963                                 break;
 964                 }
 965
 966                 if (testidx > 0) {
 967                         CDEBUG(D_NET, "Can't find specified test in batch\n");
 968                         return -ENOENT;
 969                 }
 970         }
 971
 972         clilst = (test == NULL) ? &bat->bat_cli_list :
 973                                   &test->tes_src_grp->grp_ndl_list;
 974         srvlst = (test == NULL) ? &bat->bat_srv_list :
 975                                   &test->tes_dst_grp->grp_ndl_list;
 976
 977         if (dents_up != NULL) {
 978                 rc = lstcon_nodes_getent((server ? srvlst: clilst),
 979                                          index_p, ndent_p, dents_up);
 980                 return rc;
 981         }
 982
 983         /* non-verbose query */
 984         LIBCFS_ALLOC(entp, sizeof(lstcon_test_batch_ent_t));
 985         if (entp == NULL)
 986                 return -ENOMEM;
 987
 988         memset(entp, 0, sizeof(lstcon_test_batch_ent_t));
 989
 990         if (test == NULL) {
 991                 entp->u.tbe_batch.bae_ntest = bat->bat_ntest;
 992                 entp->u.tbe_batch.bae_state = bat->bat_state;
 993
 994         } else {
 995
 996                 entp->u.tbe_test.tse_type   = test->tes_type;
 997                 entp->u.tbe_test.tse_loop   = test->tes_loop;
 998                 entp->u.tbe_test.tse_concur = test->tes_concur;
 999         }
1000
1001         list_for_each_entry(ndl, clilst, ndl_link)
1002                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle);
1003
1004         list_for_each_entry(ndl, srvlst, ndl_link)
1005                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle);
1006
1007         rc = copy_to_user(ent_up, entp,
1008                           sizeof(lstcon_test_batch_ent_t)) ? -EFAULT : 0;
1009
1010         LIBCFS_FREE(entp, sizeof(lstcon_test_batch_ent_t));
1011
1012         return rc;
1013 }
1014
1015 static int
1016 lstcon_batrpc_condition(int transop, lstcon_node_t *nd, void *arg)
1017 {
1018         switch (transop) {
1019         case LST_TRANS_TSBRUN:
1020                 if (nd->nd_state != LST_NODE_ACTIVE)
1021                         return -ENETDOWN;
1022                 break;
1023
1024         case LST_TRANS_TSBSTOP:
1025                 if (nd->nd_state != LST_NODE_ACTIVE)
1026                         return 0;
1027                 break;
1028
1029         case LST_TRANS_TSBCLIQRY:
1030         case LST_TRANS_TSBSRVQRY:
1031                 break;
1032         }
1033
1034         return 1;
1035 }
1036
1037 static int
1038 lstcon_batch_op(lstcon_batch_t *bat, int transop,
1039                 struct list_head __user *result_up)
1040 {
1041         lstcon_rpc_trans_t *trans;
1042         int                 rc;
1043
1044         rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list,
1045                                      &bat->bat_trans_list, transop,
1046                                      bat, lstcon_batrpc_condition, &trans);
1047         if (rc != 0) {
1048                 CERROR("Can't create transaction: %d\n", rc);
1049                 return rc;
1050         }
1051
1052         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1053
1054         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1055
1056         lstcon_rpc_trans_destroy(trans);
1057
1058         return rc;
1059 }
1060
1061 int
1062 lstcon_batch_run(char *name, int timeout, struct list_head __user *result_up)
1063 {
1064         lstcon_batch_t *bat;
1065         int             rc;
1066
1067         if (lstcon_batch_find(name, &bat) != 0) {
1068                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1069                 return -ENOENT;
1070         }
1071
1072         bat->bat_arg = timeout;
1073
1074         rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up);
1075
1076         /* mark batch as running if it's started in any node */
1077         if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0) != 0)
1078                 bat->bat_state = LST_BATCH_RUNNING;
1079
1080         return rc;
1081 }
1082
1083 int
1084 lstcon_batch_stop(char *name, int force, struct list_head __user *result_up)
1085 {
1086         lstcon_batch_t *bat;
1087         int             rc;
1088
1089         if (lstcon_batch_find(name, &bat) != 0) {
1090                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1091                 return -ENOENT;
1092         }
1093
1094         bat->bat_arg = force;
1095
1096         rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up);
1097
1098         /* mark batch as stopped if all RPCs finished */
1099         if (lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0) == 0)
1100                 bat->bat_state = LST_BATCH_IDLE;
1101
1102         return rc;
1103 }
1104
1105 static void
1106 lstcon_batch_destroy(lstcon_batch_t *bat)
1107 {
1108         lstcon_ndlink_t    *ndl;
1109         lstcon_test_t      *test;
1110         int                 i;
1111
1112         list_del(&bat->bat_link);
1113
1114         while (!list_empty(&bat->bat_test_list)) {
1115                 test = list_entry(bat->bat_test_list.next,
1116                                   lstcon_test_t, tes_link);
1117                 LASSERT(list_empty(&test->tes_trans_list));
1118
1119                 list_del(&test->tes_link);
1120
1121                 lstcon_group_put(test->tes_src_grp);
1122                 lstcon_group_put(test->tes_dst_grp);
1123
1124                 LIBCFS_FREE(test, offsetof(lstcon_test_t,
1125                                            tes_param[test->tes_paramlen]));
1126         }
1127
1128         LASSERT(list_empty(&bat->bat_trans_list));
1129
1130         while (!list_empty(&bat->bat_cli_list)) {
1131                 ndl = list_entry(bat->bat_cli_list.next,
1132                                  lstcon_ndlink_t, ndl_link);
1133                 list_del_init(&ndl->ndl_link);
1134
1135                 lstcon_ndlink_release(ndl);
1136         }
1137
1138         while (!list_empty(&bat->bat_srv_list)) {
1139                 ndl = list_entry(bat->bat_srv_list.next,
1140                                  lstcon_ndlink_t, ndl_link);
1141                 list_del_init(&ndl->ndl_link);
1142
1143                 lstcon_ndlink_release(ndl);
1144         }
1145
1146         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1147                 LASSERT(list_empty(&bat->bat_cli_hash[i]));
1148                 LASSERT(list_empty(&bat->bat_srv_hash[i]));
1149         }
1150
1151         LIBCFS_FREE(bat->bat_cli_hash,
1152                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1153         LIBCFS_FREE(bat->bat_srv_hash,
1154                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1155         LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
1156 }
1157
1158 static int
1159 lstcon_testrpc_condition(int transop, lstcon_node_t *nd, void *arg)
1160 {
1161         lstcon_test_t    *test;
1162         lstcon_batch_t   *batch;
1163         lstcon_ndlink_t  *ndl;
1164         struct list_head *hash;
1165         struct list_head *head;
1166
1167         test = (lstcon_test_t *)arg;
1168         LASSERT(test != NULL);
1169
1170         batch = test->tes_batch;
1171         LASSERT(batch != NULL);
1172
1173         if (test->tes_oneside &&
1174             transop == LST_TRANS_TSBSRVADD)
1175                 return 0;
1176
1177         if (nd->nd_state != LST_NODE_ACTIVE)
1178                 return -ENETDOWN;
1179
1180         if (transop == LST_TRANS_TSBCLIADD) {
1181                 hash = batch->bat_cli_hash;
1182                 head = &batch->bat_cli_list;
1183
1184         } else {
1185                 LASSERT (transop == LST_TRANS_TSBSRVADD);
1186
1187                 hash = batch->bat_srv_hash;
1188                 head = &batch->bat_srv_list;
1189         }
1190
1191         LASSERT (nd->nd_id.nid != LNET_NID_ANY);
1192
1193         if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1) != 0)
1194                 return -ENOMEM;
1195
1196         if (list_empty(&ndl->ndl_link))
1197                 list_add_tail(&ndl->ndl_link, head);
1198
1199         return 1;
1200 }
1201
1202 static int
1203 lstcon_test_nodes_add(lstcon_test_t *test, struct list_head __user *result_up)
1204 {
1205         lstcon_rpc_trans_t     *trans;
1206         lstcon_group_t         *grp;
1207         int                     transop;
1208         int                     rc;
1209
1210         LASSERT (test->tes_src_grp != NULL);
1211         LASSERT (test->tes_dst_grp != NULL);
1212
1213         transop = LST_TRANS_TSBSRVADD;
1214         grp  = test->tes_dst_grp;
1215 again:
1216         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
1217                                      &test->tes_trans_list, transop,
1218                                      test, lstcon_testrpc_condition, &trans);
1219         if (rc != 0) {
1220                 CERROR("Can't create transaction: %d\n", rc);
1221                 return rc;
1222         }
1223
1224         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1225
1226         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1227             lstcon_trans_stat()->trs_fwk_errno != 0) {
1228                 lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1229
1230                 lstcon_rpc_trans_destroy(trans);
1231                 /* return if any error */
1232                 CDEBUG(D_NET, "Failed to add test %s, "
1233                               "RPC error %d, framework error %d\n",
1234                        transop == LST_TRANS_TSBCLIADD ? "client" : "server",
1235                        lstcon_trans_stat()->trs_rpc_errno,
1236                        lstcon_trans_stat()->trs_fwk_errno);
1237
1238                 return rc;
1239         }
1240
1241         lstcon_rpc_trans_destroy(trans);
1242
1243         if (transop == LST_TRANS_TSBCLIADD)
1244                 return rc;
1245
1246         transop = LST_TRANS_TSBCLIADD;
1247         grp = test->tes_src_grp;
1248         test->tes_cliidx = 0;
1249
1250         /* requests to test clients */
1251         goto again;
1252 }
1253
1254 static int
1255 lstcon_verify_batch(const char *name, lstcon_batch_t **batch)
1256 {
1257         int rc;
1258
1259         rc = lstcon_batch_find(name, batch);
1260         if (rc != 0) {
1261                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1262                 return rc;
1263         }
1264
1265         if ((*batch)->bat_state != LST_BATCH_IDLE) {
1266                 CDEBUG(D_NET, "Can't change running batch %s\n", name);
1267                 return -EINVAL;
1268         }
1269
1270         return 0;
1271 }
1272
1273 static int
1274 lstcon_verify_group(const char *name, lstcon_group_t **grp)
1275 {
1276         int                     rc;
1277         lstcon_ndlink_t         *ndl;
1278
1279         rc = lstcon_group_find(name, grp);
1280         if (rc != 0) {
1281                 CDEBUG(D_NET, "can't find group %s\n", name);
1282                 return rc;
1283         }
1284
1285         list_for_each_entry(ndl, &(*grp)->grp_ndl_list, ndl_link) {
1286                 if (ndl->ndl_node->nd_state == LST_NODE_ACTIVE) {
1287                         return 0;
1288                 }
1289         }
1290
1291         CDEBUG(D_NET, "Group %s has no ACTIVE nodes\n", name);
1292
1293         return -EINVAL;
1294 }
1295
1296 int
1297 lstcon_test_add(char *batch_name, int type, int loop,
1298                 int concur, int dist, int span,
1299                 char *src_name, char *dst_name,
1300                 void *param, int paramlen, int *retp,
1301                 struct list_head __user *result_up)
1302 {
1303         lstcon_test_t    *test   = NULL;
1304         int              rc;
1305         lstcon_group_t   *src_grp = NULL;
1306         lstcon_group_t   *dst_grp = NULL;
1307         lstcon_batch_t   *batch = NULL;
1308
1309         /*
1310          * verify that a batch of the given name exists, and the groups
1311          * that will be part of the batch exist and have at least one
1312          * active node
1313          */
1314         rc = lstcon_verify_batch(batch_name, &batch);
1315         if (rc != 0)
1316                 goto out;
1317
1318         rc = lstcon_verify_group(src_name, &src_grp);
1319         if (rc != 0)
1320                 goto out;
1321
1322         rc = lstcon_verify_group(dst_name, &dst_grp);
1323         if (rc != 0)
1324                 goto out;
1325
1326         if (dst_grp->grp_userland)
1327                 *retp = 1;
1328
1329         LIBCFS_ALLOC(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1330         if (!test) {
1331                 CERROR("Can't allocate test descriptor\n");
1332                 rc = -ENOMEM;
1333
1334                 goto out;
1335         }
1336
1337         memset(test, 0, offsetof(lstcon_test_t, tes_param[paramlen]));
1338         test->tes_hdr.tsb_id    = batch->bat_hdr.tsb_id;
1339         test->tes_batch         = batch;
1340         test->tes_type          = type;
1341         test->tes_oneside       = 0; /* TODO */
1342         test->tes_loop          = loop;
1343         test->tes_concur        = concur;
1344         test->tes_stop_onerr    = 1; /* TODO */
1345         test->tes_span          = span;
1346         test->tes_dist          = dist;
1347         test->tes_cliidx        = 0; /* just used for creating RPC */
1348         test->tes_src_grp       = src_grp;
1349         test->tes_dst_grp       = dst_grp;
1350         INIT_LIST_HEAD(&test->tes_trans_list);
1351
1352         if (param != NULL) {
1353                 test->tes_paramlen = paramlen;
1354                 memcpy(&test->tes_param[0], param, paramlen);
1355         }
1356
1357         rc = lstcon_test_nodes_add(test, result_up);
1358
1359         if (rc != 0)
1360                 goto out;
1361
1362         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1363             lstcon_trans_stat()->trs_fwk_errno != 0)
1364                 CDEBUG(D_NET, "Failed to add test %d to batch %s\n", type,
1365                        batch_name);
1366
1367         /* add to test list anyway, so user can check what's going on */
1368         list_add_tail(&test->tes_link, &batch->bat_test_list);
1369
1370         batch->bat_ntest++;
1371         test->tes_hdr.tsb_index = batch->bat_ntest;
1372
1373         /*  hold groups so nobody can change them */
1374         return rc;
1375 out:
1376         if (test != NULL)
1377                 LIBCFS_FREE(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1378
1379         if (dst_grp != NULL)
1380                 lstcon_group_put(dst_grp);
1381
1382         if (src_grp != NULL)
1383                 lstcon_group_put(src_grp);
1384
1385         return rc;
1386 }
1387
1388 static int
1389 lstcon_test_find(lstcon_batch_t *batch, int idx, lstcon_test_t **testpp)
1390 {
1391         lstcon_test_t *test;
1392
1393         list_for_each_entry(test, &batch->bat_test_list, tes_link) {
1394                 if (idx == test->tes_hdr.tsb_index) {
1395                         *testpp = test;
1396                         return 0;
1397                 }
1398         }
1399
1400         return -ENOENT;
1401 }
1402
1403 static int
1404 lstcon_tsbrpc_readent(int transop, srpc_msg_t *msg,
1405                       lstcon_rpc_ent_t __user *ent_up)
1406 {
1407         srpc_batch_reply_t *rep = &msg->msg_body.bat_reply;
1408
1409         LASSERT (transop == LST_TRANS_TSBCLIQRY ||
1410                  transop == LST_TRANS_TSBSRVQRY);
1411
1412         /* positive errno, framework error code */
1413         if (copy_to_user(&ent_up->rpe_priv[0],
1414                              &rep->bar_active, sizeof(rep->bar_active)))
1415                 return -EFAULT;
1416
1417         return 0;
1418 }
1419
1420 int
1421 lstcon_test_batch_query(char *name, int testidx, int client,
1422                         int timeout, struct list_head __user *result_up)
1423 {
1424         lstcon_rpc_trans_t *trans;
1425         struct list_head   *translist;
1426         struct list_head   *ndlist;
1427         lstcon_tsb_hdr_t   *hdr;
1428         lstcon_batch_t     *batch;
1429         lstcon_test_t      *test = NULL;
1430         int                 transop;
1431         int                 rc;
1432
1433         rc = lstcon_batch_find(name, &batch);
1434         if (rc != 0) {
1435                 CDEBUG(D_NET, "Can't find batch: %s\n", name);
1436                 return rc;
1437         }
1438
1439         if (testidx == 0) {
1440                 translist = &batch->bat_trans_list;
1441                 ndlist    = &batch->bat_cli_list;
1442                 hdr       = &batch->bat_hdr;
1443
1444         } else {
1445                 /* query specified test only */
1446                 rc = lstcon_test_find(batch, testidx, &test);
1447                 if (rc != 0) {
1448                         CDEBUG(D_NET, "Can't find test: %d\n", testidx);
1449                         return rc;
1450                 }
1451
1452                 translist = &test->tes_trans_list;
1453                 ndlist    = &test->tes_src_grp->grp_ndl_list;
1454                 hdr       = &test->tes_hdr;
1455         }
1456
1457         transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY;
1458
1459         rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr,
1460                                      lstcon_batrpc_condition, &trans);
1461         if (rc != 0) {
1462                 CERROR("Can't create transaction: %d\n", rc);
1463                 return rc;
1464         }
1465
1466         lstcon_rpc_trans_postwait(trans, timeout);
1467
1468         if (testidx == 0 && /* query a batch, not a test */
1469             lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) == 0 &&
1470             lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0) == 0) {
1471                 /* all RPCs finished, and no active test */
1472                 batch->bat_state = LST_BATCH_IDLE;
1473         }
1474
1475         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1476                                           lstcon_tsbrpc_readent);
1477         lstcon_rpc_trans_destroy(trans);
1478
1479         return rc;
1480 }
1481
1482 static int
1483 lstcon_statrpc_readent(int transop, srpc_msg_t *msg,
1484                        lstcon_rpc_ent_t __user *ent_up)
1485 {
1486         srpc_stat_reply_t *rep = &msg->msg_body.stat_reply;
1487         sfw_counters_t __user  *sfwk_stat;
1488         srpc_counters_t __user *srpc_stat;
1489         lnet_counters_t __user *lnet_stat;
1490
1491         if (rep->str_status != 0)
1492                 return 0;
1493
1494         sfwk_stat = (sfw_counters_t __user *)&ent_up->rpe_payload[0];
1495         srpc_stat = (srpc_counters_t __user *)
1496                 ((char __user *)sfwk_stat + sizeof(*sfwk_stat));
1497         lnet_stat = (lnet_counters_t __user *)
1498                 ((char __user *)srpc_stat + sizeof(*srpc_stat));
1499
1500         if (copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) ||
1501             copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) ||
1502             copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat)))
1503                 return -EFAULT;
1504
1505         return 0;
1506 }
1507
1508 static int
1509 lstcon_ndlist_stat(struct list_head *ndlist,
1510                    int timeout, struct list_head __user *result_up)
1511 {
1512         struct list_head    head;
1513         lstcon_rpc_trans_t *trans;
1514         int                 rc;
1515
1516         INIT_LIST_HEAD(&head);
1517
1518         rc = lstcon_rpc_trans_ndlist(ndlist, &head,
1519                                      LST_TRANS_STATQRY, NULL, NULL, &trans);
1520         if (rc != 0) {
1521                 CERROR("Can't create transaction: %d\n", rc);
1522                 return rc;
1523         }
1524
1525         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1526
1527         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1528                                           lstcon_statrpc_readent);
1529         lstcon_rpc_trans_destroy(trans);
1530
1531         return rc;
1532 }
1533
1534 int
1535 lstcon_group_stat(char *grp_name, int timeout,
1536                   struct list_head __user *result_up)
1537 {
1538         lstcon_group_t     *grp;
1539         int                 rc;
1540
1541         rc = lstcon_group_find(grp_name, &grp);
1542         if (rc != 0) {
1543                 CDEBUG(D_NET, "Can't find group %s\n", grp_name);
1544                 return rc;
1545         }
1546
1547         rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up);
1548
1549         lstcon_group_put(grp);
1550
1551         return rc;
1552 }
1553
1554 int
1555 lstcon_nodes_stat(int count, lnet_process_id_t __user *ids_up,
1556                   int timeout, struct list_head __user *result_up)
1557 {
1558         lstcon_ndlink_t         *ndl;
1559         lstcon_group_t          *tmp;
1560         lnet_process_id_t        id;
1561         int                      i;
1562         int                      rc;
1563
1564         rc = lstcon_group_alloc(NULL, &tmp);
1565         if (rc != 0) {
1566                 CERROR("Out of memory\n");
1567                 return -ENOMEM;
1568         }
1569
1570         for (i = 0 ; i < count; i++) {
1571                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1572                         rc = -EFAULT;
1573                         break;
1574                 }
1575
1576                 /* add to tmp group */
1577                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2);
1578                 if (rc != 0) {
1579                         CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET,
1580                                "Failed to find or create %s: %d\n",
1581                                libcfs_id2str(id), rc);
1582                         break;
1583                 }
1584         }
1585
1586         if (rc != 0) {
1587                 lstcon_group_put(tmp);
1588                 return rc;
1589         }
1590
1591         rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up);
1592
1593         lstcon_group_put(tmp);
1594
1595         return rc;
1596 }
1597
1598 static int
1599 lstcon_debug_ndlist(struct list_head *ndlist,
1600                     struct list_head *translist,
1601                     int timeout, struct list_head __user *result_up)
1602 {
1603         lstcon_rpc_trans_t *trans;
1604         int                 rc;
1605
1606         rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY,
1607                                      NULL, lstcon_sesrpc_condition, &trans);
1608         if (rc != 0) {
1609                 CERROR("Can't create transaction: %d\n", rc);
1610                 return rc;
1611         }
1612
1613         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1614
1615         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1616                                           lstcon_sesrpc_readent);
1617         lstcon_rpc_trans_destroy(trans);
1618
1619         return rc;
1620 }
1621
1622 int
1623 lstcon_session_debug(int timeout, struct list_head __user *result_up)
1624 {
1625         return lstcon_debug_ndlist(&console_session.ses_ndl_list,
1626                                    NULL, timeout, result_up);
1627 }
1628
1629 int
1630 lstcon_batch_debug(int timeout, char *name,
1631                    int client, struct list_head __user *result_up)
1632 {
1633         lstcon_batch_t *bat;
1634         int             rc;
1635
1636         rc = lstcon_batch_find(name, &bat);
1637         if (rc != 0)
1638                 return -ENOENT;
1639
1640         rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list :
1641                                           &bat->bat_srv_list,
1642                                  NULL, timeout, result_up);
1643
1644         return rc;
1645 }
1646
1647 int
1648 lstcon_group_debug(int timeout, char *name,
1649                    struct list_head __user *result_up)
1650 {
1651         lstcon_group_t *grp;
1652         int             rc;
1653
1654         rc = lstcon_group_find(name, &grp);
1655         if (rc != 0)
1656                 return -ENOENT;
1657
1658         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1659                                  timeout, result_up);
1660         lstcon_group_put(grp);
1661
1662         return rc;
1663 }
1664
1665 int
1666 lstcon_nodes_debug(int timeout,
1667                    int count, lnet_process_id_t __user *ids_up,
1668                    struct list_head __user *result_up)
1669 {
1670         lnet_process_id_t  id;
1671         lstcon_ndlink_t   *ndl;
1672         lstcon_group_t    *grp;
1673         int                i;
1674         int                rc;
1675
1676         rc = lstcon_group_alloc(NULL, &grp);
1677         if (rc != 0) {
1678                 CDEBUG(D_NET, "Out of memory\n");
1679                 return rc;
1680         }
1681
1682         for (i = 0; i < count; i++) {
1683                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1684                         rc = -EFAULT;
1685                         break;
1686                 }
1687
1688                 /* node is added to tmp group */
1689                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 1);
1690                 if (rc != 0) {
1691                         CERROR("Can't create node link\n");
1692                         break;
1693                 }
1694         }
1695
1696         if (rc != 0) {
1697                 lstcon_group_put(grp);
1698                 return rc;
1699         }
1700
1701         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1702                                  timeout, result_up);
1703
1704         lstcon_group_put(grp);
1705
1706         return rc;
1707 }
1708
1709 int
1710 lstcon_session_match(lst_sid_t sid)
1711 {
1712         return (console_session.ses_id.ses_nid   == sid.ses_nid &&
1713                 console_session.ses_id.ses_stamp == sid.ses_stamp) ?  1: 0;
1714 }
1715
1716 static void
1717 lstcon_new_session_id(lst_sid_t *sid)
1718 {
1719         lnet_process_id_t      id;
1720
1721         LASSERT (console_session.ses_state == LST_SESSION_NONE);
1722
1723         LNetGetId(1, &id);
1724         sid->ses_nid   = id.nid;
1725         sid->ses_stamp = cfs_time_current();
1726 }
1727
1728 int
1729 lstcon_session_new(char *name, int key, unsigned feats,
1730                    int timeout, int force, lst_sid_t __user *sid_up)
1731 {
1732         int     rc = 0;
1733         int     i;
1734
1735         if (console_session.ses_state != LST_SESSION_NONE) {
1736                 /* session exists */
1737                 if (!force) {
1738                         CNETERR("Session %s already exists\n",
1739                                 console_session.ses_name);
1740                         return -EEXIST;
1741                 }
1742
1743                 rc = lstcon_session_end();
1744
1745                 /* lstcon_session_end() only return local error */
1746                 if  (rc != 0)
1747                         return rc;
1748         }
1749
1750         if ((feats & ~LST_FEATS_MASK) != 0) {
1751                 CNETERR("Unknown session features %x\n",
1752                         (feats & ~LST_FEATS_MASK));
1753                 return -EINVAL;
1754         }
1755
1756         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
1757                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
1758
1759         lstcon_new_session_id(&console_session.ses_id);
1760
1761         console_session.ses_key     = key;
1762         console_session.ses_state   = LST_SESSION_ACTIVE;
1763         console_session.ses_force   = !!force;
1764         console_session.ses_features = feats;
1765         console_session.ses_feats_updated = 0;
1766         console_session.ses_timeout = (timeout <= 0) ?
1767                                       LST_CONSOLE_TIMEOUT : timeout;
1768
1769         if (strlen(name) > sizeof(console_session.ses_name)-1)
1770                 return -E2BIG;
1771         strlcpy(console_session.ses_name, name,
1772                 sizeof(console_session.ses_name));
1773
1774         rc = lstcon_batch_add(LST_DEFAULT_BATCH);
1775         if (rc != 0)
1776                 return rc;
1777
1778         rc = lstcon_rpc_pinger_start();
1779         if (rc != 0) {
1780                 lstcon_batch_t *bat = NULL;
1781
1782                 lstcon_batch_find(LST_DEFAULT_BATCH, &bat);
1783                 lstcon_batch_destroy(bat);
1784
1785                 return rc;
1786         }
1787
1788         if (copy_to_user(sid_up, &console_session.ses_id,
1789                              sizeof(lst_sid_t)) == 0)
1790                 return rc;
1791
1792         lstcon_session_end();
1793
1794         return -EFAULT;
1795 }
1796
1797 int
1798 lstcon_session_info(lst_sid_t __user *sid_up, int __user *key_up,
1799                     unsigned __user *featp,
1800                     lstcon_ndlist_ent_t __user *ndinfo_up,
1801                     char __user *name_up, int len)
1802 {
1803         lstcon_ndlist_ent_t *entp;
1804         lstcon_ndlink_t     *ndl;
1805         int                  rc = 0;
1806
1807         if (console_session.ses_state != LST_SESSION_ACTIVE)
1808                 return -ESRCH;
1809
1810         LIBCFS_ALLOC(entp, sizeof(*entp));
1811         if (entp == NULL)
1812                 return -ENOMEM;
1813
1814         memset(entp, 0, sizeof(*entp));
1815
1816         list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link)
1817                 LST_NODE_STATE_COUNTER(ndl->ndl_node, entp);
1818
1819         if (copy_to_user(sid_up, &console_session.ses_id,
1820                              sizeof(lst_sid_t)) ||
1821             copy_to_user(key_up, &console_session.ses_key,
1822                              sizeof(*key_up)) ||
1823             copy_to_user(featp, &console_session.ses_features,
1824                              sizeof(*featp)) ||
1825             copy_to_user(ndinfo_up, entp, sizeof(*entp)) ||
1826             copy_to_user(name_up, console_session.ses_name, len))
1827                 rc = -EFAULT;
1828
1829         LIBCFS_FREE(entp, sizeof(*entp));
1830
1831         return rc;
1832 }
1833
1834 int
1835 lstcon_session_end()
1836 {
1837         lstcon_rpc_trans_t *trans;
1838         lstcon_group_t     *grp;
1839         lstcon_batch_t     *bat;
1840         int                 rc = 0;
1841
1842         LASSERT (console_session.ses_state == LST_SESSION_ACTIVE);
1843
1844         rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list,
1845                                      NULL, LST_TRANS_SESEND, NULL,
1846                                      lstcon_sesrpc_condition, &trans);
1847         if (rc != 0) {
1848                 CERROR("Can't create transaction: %d\n", rc);
1849                 return rc;
1850         }
1851
1852         console_session.ses_shutdown = 1;
1853
1854         lstcon_rpc_pinger_stop();
1855
1856         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1857
1858         lstcon_rpc_trans_destroy(trans);
1859         /* User can do nothing even rpc failed, so go on */
1860
1861         /* waiting for orphan rpcs to die */
1862         lstcon_rpc_cleanup_wait();
1863
1864         console_session.ses_id    = LST_INVALID_SID;
1865         console_session.ses_state = LST_SESSION_NONE;
1866         console_session.ses_key   = 0;
1867         console_session.ses_force = 0;
1868         console_session.ses_feats_updated = 0;
1869
1870         /* destroy all batches */
1871         while (!list_empty(&console_session.ses_bat_list)) {
1872                 bat = list_entry(console_session.ses_bat_list.next,
1873                                  lstcon_batch_t, bat_link);
1874
1875                 lstcon_batch_destroy(bat);
1876         }
1877
1878         /* destroy all groups */
1879         while (!list_empty(&console_session.ses_grp_list)) {
1880                 grp = list_entry(console_session.ses_grp_list.next,
1881                                  lstcon_group_t, grp_link);
1882                 LASSERT(grp->grp_ref == 1);
1883
1884                 lstcon_group_put(grp);
1885         }
1886
1887         /* all nodes should be released */
1888         LASSERT(list_empty(&console_session.ses_ndl_list));
1889
1890         console_session.ses_shutdown = 0;
1891         console_session.ses_expired  = 0;
1892
1893         return rc;
1894 }
1895
1896 int
1897 lstcon_session_feats_check(unsigned feats)
1898 {
1899         int rc = 0;
1900
1901         if ((feats & ~LST_FEATS_MASK) != 0) {
1902                 CERROR("Can't support these features: %x\n",
1903                        (feats & ~LST_FEATS_MASK));
1904                 return -EPROTO;
1905         }
1906
1907         spin_lock(&console_session.ses_rpc_lock);
1908
1909         if (!console_session.ses_feats_updated) {
1910                 console_session.ses_feats_updated = 1;
1911                 console_session.ses_features = feats;
1912         }
1913
1914         if (console_session.ses_features != feats)
1915                 rc = -EPROTO;
1916
1917         spin_unlock(&console_session.ses_rpc_lock);
1918
1919         if (rc != 0) {
1920                 CERROR("remote features %x do not match with "
1921                        "session features %x of console\n",
1922                        feats, console_session.ses_features);
1923         }
1924
1925         return rc;
1926 }
1927
1928 static int
1929 lstcon_acceptor_handle (srpc_server_rpc_t *rpc)
1930 {
1931         srpc_msg_t        *rep  = &rpc->srpc_replymsg;
1932         srpc_msg_t        *req  = &rpc->srpc_reqstbuf->buf_msg;
1933         srpc_join_reqst_t *jreq = &req->msg_body.join_reqst;
1934         srpc_join_reply_t *jrep = &rep->msg_body.join_reply;
1935         lstcon_group_t    *grp  = NULL;
1936         lstcon_ndlink_t   *ndl;
1937         int                rc   = 0;
1938
1939         sfw_unpack_message(req);
1940
1941         mutex_lock(&console_session.ses_mutex);
1942
1943         jrep->join_sid = console_session.ses_id;
1944
1945         if (console_session.ses_id.ses_nid == LNET_NID_ANY) {
1946                 jrep->join_status = ESRCH;
1947                 goto out;
1948         }
1949
1950         if (lstcon_session_feats_check(req->msg_ses_feats) != 0) {
1951                 jrep->join_status = EPROTO;
1952                 goto out;
1953         }
1954
1955         if (jreq->join_sid.ses_nid != LNET_NID_ANY &&
1956              !lstcon_session_match(jreq->join_sid)) {
1957                 jrep->join_status = EBUSY;
1958                 goto out;
1959         }
1960
1961         if (lstcon_group_find(jreq->join_group, &grp) != 0) {
1962                 rc = lstcon_group_alloc(jreq->join_group, &grp);
1963                 if (rc != 0) {
1964                         CERROR("Out of memory\n");
1965                         goto out;
1966                 }
1967
1968                 list_add_tail(&grp->grp_link,
1969                               &console_session.ses_grp_list);
1970                 lstcon_group_addref(grp);
1971         }
1972
1973         if (grp->grp_ref > 2) {
1974                 /* Group in using */
1975                 jrep->join_status = EBUSY;
1976                 goto out;
1977         }
1978
1979         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0);
1980         if (rc == 0) {
1981                 jrep->join_status = EEXIST;
1982                 goto out;
1983         }
1984
1985         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1);
1986         if (rc != 0) {
1987                 CERROR("Out of memory\n");
1988                 goto out;
1989         }
1990
1991         ndl->ndl_node->nd_state   = LST_NODE_ACTIVE;
1992         ndl->ndl_node->nd_timeout = console_session.ses_timeout;
1993
1994         if (grp->grp_userland == 0)
1995                 grp->grp_userland = 1;
1996
1997         strlcpy(jrep->join_session, console_session.ses_name,
1998                 sizeof(jrep->join_session));
1999         jrep->join_timeout = console_session.ses_timeout;
2000         jrep->join_status  = 0;
2001
2002 out:
2003         rep->msg_ses_feats = console_session.ses_features;
2004         if (grp != NULL)
2005                 lstcon_group_put(grp);
2006
2007         mutex_unlock(&console_session.ses_mutex);
2008
2009         return rc;
2010 }
2011
2012 static srpc_service_t lstcon_acceptor_service;
2013 static void lstcon_init_acceptor_service(void)
2014 {
2015         /* initialize selftest console acceptor service table */
2016         lstcon_acceptor_service.sv_name    = "join session";
2017         lstcon_acceptor_service.sv_handler = lstcon_acceptor_handle;
2018         lstcon_acceptor_service.sv_id      = SRPC_SERVICE_JOIN;
2019         lstcon_acceptor_service.sv_wi_total = SFW_FRWK_WI_MAX;
2020 }
2021
2022 int lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_hdr *hdr);
2023
2024 DECLARE_IOCTL_HANDLER(lstcon_ioctl_handler, lstcon_ioctl_entry);
2025
2026 /* initialize console */
2027 int
2028 lstcon_console_init(void)
2029 {
2030         int     i;
2031         int     rc;
2032
2033         memset(&console_session, 0, sizeof(lstcon_session_t));
2034
2035         console_session.ses_id              = LST_INVALID_SID;
2036         console_session.ses_state           = LST_SESSION_NONE;
2037         console_session.ses_timeout         = 0;
2038         console_session.ses_force           = 0;
2039         console_session.ses_expired         = 0;
2040         console_session.ses_feats_updated   = 0;
2041         console_session.ses_features        = LST_FEATS_MASK;
2042         console_session.ses_laststamp       = cfs_time_current_sec();
2043
2044         mutex_init(&console_session.ses_mutex);
2045
2046         INIT_LIST_HEAD(&console_session.ses_ndl_list);
2047         INIT_LIST_HEAD(&console_session.ses_grp_list);
2048         INIT_LIST_HEAD(&console_session.ses_bat_list);
2049         INIT_LIST_HEAD(&console_session.ses_trans_list);
2050
2051         LIBCFS_ALLOC(console_session.ses_ndl_hash,
2052                      sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2053         if (console_session.ses_ndl_hash == NULL)
2054                 return -ENOMEM;
2055
2056         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
2057                 INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]);
2058
2059
2060         /* initialize acceptor service table */
2061         lstcon_init_acceptor_service();
2062
2063         rc = srpc_add_service(&lstcon_acceptor_service);
2064         LASSERT(rc != -EBUSY);
2065         if (rc != 0) {
2066                 LIBCFS_FREE(console_session.ses_ndl_hash,
2067                             sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2068                 return rc;
2069         }
2070
2071         rc = srpc_service_add_buffers(&lstcon_acceptor_service,
2072                                       lstcon_acceptor_service.sv_wi_total);
2073         if (rc != 0) {
2074                 rc = -ENOMEM;
2075                 goto out;
2076         }
2077
2078         rc = libcfs_register_ioctl(&lstcon_ioctl_handler);
2079
2080         if (rc == 0) {
2081                 lstcon_rpc_module_init();
2082                 return 0;
2083         }
2084
2085 out:
2086         srpc_shutdown_service(&lstcon_acceptor_service);
2087         srpc_remove_service(&lstcon_acceptor_service);
2088
2089         LIBCFS_FREE(console_session.ses_ndl_hash,
2090                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2091
2092         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2093
2094         return rc;
2095 }
2096
2097 int
2098 lstcon_console_fini(void)
2099 {
2100         int     i;
2101
2102         libcfs_deregister_ioctl(&lstcon_ioctl_handler);
2103
2104         mutex_lock(&console_session.ses_mutex);
2105
2106         srpc_shutdown_service(&lstcon_acceptor_service);
2107         srpc_remove_service(&lstcon_acceptor_service);
2108
2109         if (console_session.ses_state != LST_SESSION_NONE)
2110                 lstcon_session_end();
2111
2112         lstcon_rpc_module_fini();
2113
2114         mutex_unlock(&console_session.ses_mutex);
2115
2116         LASSERT(list_empty(&console_session.ses_ndl_list));
2117         LASSERT(list_empty(&console_session.ses_grp_list));
2118         LASSERT(list_empty(&console_session.ses_bat_list));
2119         LASSERT(list_empty(&console_session.ses_trans_list));
2120
2121         for (i = 0; i < LST_NODE_HASHSIZE; i++)
2122                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
2123
2124         LIBCFS_FREE(console_session.ses_ndl_hash,
2125                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2126
2127         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2128
2129         return 0;
2130 }
2131