lnet/selftest/console.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2012, 2014, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  * Lustre is a trademark of Sun Microsystems, Inc.
  31  *
  32  * lnet/selftest/conctl.c
  33  *
  34  * Infrastructure of LST console
  35  *
  36  * Author: Liang Zhen <liangzhen@clusterfs.com>
  37  */
  38
  39 #include <libcfs/libcfs.h>
  40 #include <lnet/lib-lnet.h>
  41 #include "console.h"
  42 #include "conrpc.h"
  43
  44 #define LST_NODE_STATE_COUNTER(nd, p)                   \
  45 do {                                                    \
  46         if ((nd)->nd_state == LST_NODE_ACTIVE)          \
  47                 (p)->nle_nactive ++;                    \
  48         else if ((nd)->nd_state == LST_NODE_BUSY)       \
  49                 (p)->nle_nbusy ++;                      \
  50         else if ((nd)->nd_state == LST_NODE_DOWN)       \
  51                 (p)->nle_ndown ++;                      \
  52         else                                            \
  53                 (p)->nle_nunknown ++;                   \
  54         (p)->nle_nnode ++;                              \
  55 } while (0)
  56
  57 struct lstcon_session console_session;
  58
  59 static void
  60 lstcon_node_get(struct lstcon_node *nd)
  61 {
  62         LASSERT (nd->nd_ref >= 1);
  63
  64         nd->nd_ref++;
  65 }
  66
  67 static int
  68 lstcon_node_find(struct lnet_process_id id, struct lstcon_node **ndpp,
  69                  int create)
  70 {
  71         struct lstcon_ndlink *ndl;
  72         unsigned int idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE;
  73
  74         LASSERT(id.nid != LNET_NID_ANY);
  75
  76         list_for_each_entry(ndl, &console_session.ses_ndl_hash[idx],
  77                             ndl_hlink) {
  78                 if (ndl->ndl_node->nd_id.nid != id.nid ||
  79                     ndl->ndl_node->nd_id.pid != id.pid)
  80                         continue;
  81
  82                 lstcon_node_get(ndl->ndl_node);
  83                 *ndpp = ndl->ndl_node;
  84                 return 0;
  85         }
  86
  87         if (!create)
  88                 return -ENOENT;
  89
  90         LIBCFS_ALLOC(*ndpp, sizeof(**ndpp) + sizeof(*ndl));
  91         if (*ndpp == NULL)
  92                 return -ENOMEM;
  93
  94         ndl = (struct lstcon_ndlink *)(*ndpp + 1);
  95
  96         ndl->ndl_node = *ndpp;
  97
  98         ndl->ndl_node->nd_ref   = 1;
  99         ndl->ndl_node->nd_id    = id;
 100         ndl->ndl_node->nd_stamp = ktime_get();
 101         ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
 102         ndl->ndl_node->nd_timeout = 0;
 103         memset(&ndl->ndl_node->nd_ping, 0, sizeof(ndl->ndl_node->nd_ping));
 104
 105         /* queued in global hash & list, no refcount is taken by
 106          * global hash & list, if caller release his refcount,
 107          * node will be released */
 108         list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]);
 109         list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list);
 110
 111         return 0;
 112 }
 113
 114 static void
 115 lstcon_node_put(struct lstcon_node *nd)
 116 {
 117         struct lstcon_ndlink *ndl;
 118
 119         LASSERT(nd->nd_ref > 0);
 120
 121         if (--nd->nd_ref > 0)
 122                 return;
 123
 124         ndl = (struct lstcon_ndlink *)(nd + 1);
 125
 126         LASSERT(!list_empty(&ndl->ndl_link));
 127         LASSERT(!list_empty(&ndl->ndl_hlink));
 128
 129         /* remove from session */
 130         list_del(&ndl->ndl_link);
 131         list_del(&ndl->ndl_hlink);
 132
 133         LIBCFS_FREE(nd, sizeof(*nd) + sizeof(*ndl));
 134 }
 135
 136 static int
 137 lstcon_ndlink_find(struct list_head *hash, struct lnet_process_id id,
 138                    struct lstcon_ndlink **ndlpp, int create)
 139 {
 140         unsigned int idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
 141         struct lstcon_ndlink *ndl;
 142         struct lstcon_node *nd;
 143         int rc;
 144
 145         if (id.nid == LNET_NID_ANY)
 146                 return -EINVAL;
 147
 148         /* search in hash */
 149         list_for_each_entry(ndl, &hash[idx], ndl_hlink) {
 150                 if (ndl->ndl_node->nd_id.nid != id.nid ||
 151                     ndl->ndl_node->nd_id.pid != id.pid)
 152                         continue;
 153
 154                 *ndlpp = ndl;
 155                 return 0;
 156         }
 157
 158         if (create == 0)
 159                 return -ENOENT;
 160
 161         /* find or create in session hash */
 162         rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0);
 163         if (rc != 0)
 164                 return rc;
 165
 166         LIBCFS_ALLOC(ndl, sizeof(*ndl));
 167         if (ndl == NULL) {
 168                 lstcon_node_put(nd);
 169                 return -ENOMEM;
 170         }
 171
 172         *ndlpp = ndl;
 173
 174         ndl->ndl_node = nd;
 175         INIT_LIST_HEAD(&ndl->ndl_link);
 176         list_add_tail(&ndl->ndl_hlink, &hash[idx]);
 177
 178         return 0;
 179 }
 180
 181 static void
 182 lstcon_ndlink_release(struct lstcon_ndlink *ndl)
 183 {
 184         LASSERT(list_empty(&ndl->ndl_link));
 185         LASSERT(!list_empty(&ndl->ndl_hlink));
 186
 187         list_del(&ndl->ndl_hlink); /* delete from hash */
 188         lstcon_node_put(ndl->ndl_node);
 189
 190         LIBCFS_FREE(ndl, sizeof(*ndl));
 191 }
 192
 193 static int
 194 lstcon_group_alloc(char *name, struct lstcon_group **grpp)
 195 {
 196         struct lstcon_group *grp;
 197         int i;
 198
 199         LIBCFS_ALLOC(grp, offsetof(struct lstcon_group,
 200                                    grp_ndl_hash[LST_NODE_HASHSIZE]));
 201         if (grp == NULL)
 202                 return -ENOMEM;
 203
 204         grp->grp_ref = 1;
 205         if (name != NULL) {
 206                 if (strlen(name) > sizeof(grp->grp_name)-1) {
 207                         LIBCFS_FREE(grp, offsetof(struct lstcon_group,
 208                                           grp_ndl_hash[LST_NODE_HASHSIZE]));
 209                         return -E2BIG;
 210                 }
 211                 strncpy(grp->grp_name, name, sizeof(grp->grp_name));
 212         }
 213
 214         INIT_LIST_HEAD(&grp->grp_link);
 215         INIT_LIST_HEAD(&grp->grp_ndl_list);
 216         INIT_LIST_HEAD(&grp->grp_trans_list);
 217
 218         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 219                 INIT_LIST_HEAD(&grp->grp_ndl_hash[i]);
 220
 221         *grpp = grp;
 222
 223         return 0;
 224 }
 225
 226 static void
 227 lstcon_group_addref(struct lstcon_group *grp)
 228 {
 229         grp->grp_ref++;
 230 }
 231
 232 static void lstcon_group_ndlink_release(struct lstcon_group *,
 233                                         struct lstcon_ndlink *);
 234
 235 static void
 236 lstcon_group_drain(struct lstcon_group *grp, int keep)
 237 {
 238         struct lstcon_ndlink *ndl;
 239         struct lstcon_ndlink *tmp;
 240
 241         list_for_each_entry_safe(ndl, tmp, &grp->grp_ndl_list, ndl_link) {
 242                 if ((ndl->ndl_node->nd_state & keep) == 0)
 243                         lstcon_group_ndlink_release(grp, ndl);
 244         }
 245 }
 246
 247 static void
 248 lstcon_group_decref(struct lstcon_group *grp)
 249 {
 250         int i;
 251
 252         if (--grp->grp_ref > 0)
 253                 return;
 254
 255         if (!list_empty(&grp->grp_link))
 256                 list_del(&grp->grp_link);
 257
 258         lstcon_group_drain(grp, 0);
 259
 260         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 261                 LASSERT(list_empty(&grp->grp_ndl_hash[i]));
 262
 263         LIBCFS_FREE(grp, offsetof(struct lstcon_group,
 264                                   grp_ndl_hash[LST_NODE_HASHSIZE]));
 265 }
 266
 267 static int
 268 lstcon_group_find(const char *name, struct lstcon_group **grpp)
 269 {
 270         struct lstcon_group *grp;
 271
 272         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 273                 if (strncmp(grp->grp_name, name, LST_NAME_SIZE) != 0)
 274                         continue;
 275
 276                 lstcon_group_addref(grp);  /* +1 ref for caller */
 277                 *grpp = grp;
 278                 return 0;
 279         }
 280
 281         return -ENOENT;
 282 }
 283
 284 static int
 285 lstcon_group_ndlink_find(struct lstcon_group *grp, struct lnet_process_id id,
 286                          struct lstcon_ndlink **ndlpp, int create)
 287 {
 288         int rc;
 289
 290         rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create);
 291         if (rc != 0)
 292                 return rc;
 293
 294         if (!list_empty(&(*ndlpp)->ndl_link))
 295                 return 0;
 296
 297         list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list);
 298         grp->grp_nnode++;
 299
 300         return 0;
 301 }
 302
 303 static void
 304 lstcon_group_ndlink_release(struct lstcon_group *grp, struct lstcon_ndlink *ndl)
 305 {
 306         list_del_init(&ndl->ndl_link);
 307         lstcon_ndlink_release(ndl);
 308         grp->grp_nnode--;
 309 }
 310
 311 static void
 312 lstcon_group_ndlink_move(struct lstcon_group *old,
 313                          struct lstcon_group *new, struct lstcon_ndlink *ndl)
 314 {
 315         unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) %
 316                                         LST_NODE_HASHSIZE;
 317
 318         list_del(&ndl->ndl_hlink);
 319         list_del(&ndl->ndl_link);
 320         old->grp_nnode--;
 321
 322         list_add_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]);
 323         list_add_tail(&ndl->ndl_link, &new->grp_ndl_list);
 324         new->grp_nnode++;
 325 }
 326
 327 static void
 328 lstcon_group_move(struct lstcon_group *old, struct lstcon_group *new)
 329 {
 330         struct lstcon_ndlink *ndl;
 331
 332         while (!list_empty(&old->grp_ndl_list)) {
 333                 ndl = list_entry(old->grp_ndl_list.next,
 334                                  struct lstcon_ndlink, ndl_link);
 335                 lstcon_group_ndlink_move(old, new, ndl);
 336         }
 337 }
 338
 339 static int
 340 lstcon_sesrpc_condition(int transop, struct lstcon_node *nd, void *arg)
 341 {
 342         struct lstcon_group *grp = arg;
 343
 344         switch (transop) {
 345         case LST_TRANS_SESNEW:
 346                 if (nd->nd_state == LST_NODE_ACTIVE)
 347                         return 0;
 348                 break;
 349
 350         case LST_TRANS_SESEND:
 351                 if (nd->nd_state != LST_NODE_ACTIVE)
 352                         return 0;
 353
 354                 if (grp != NULL && nd->nd_ref > 1)
 355                         return 0;
 356                 break;
 357
 358         case LST_TRANS_SESQRY:
 359                 break;
 360
 361         default:
 362                 LBUG();
 363         }
 364
 365         return 1;
 366 }
 367
 368 static int
 369 lstcon_sesrpc_readent(int transop, struct srpc_msg *msg,
 370                       struct lstcon_rpc_ent __user *ent_up)
 371 {
 372         struct srpc_debug_reply *rep;
 373
 374         switch (transop) {
 375         case LST_TRANS_SESNEW:
 376         case LST_TRANS_SESEND:
 377                 return 0;
 378
 379         case LST_TRANS_SESQRY:
 380                 rep = &msg->msg_body.dbg_reply;
 381
 382                 if (copy_to_user(&ent_up->rpe_priv[0],
 383                                      &rep->dbg_timeout, sizeof(int)) ||
 384                     copy_to_user(&ent_up->rpe_payload[0],
 385                                      &rep->dbg_name, LST_NAME_SIZE))
 386                         return -EFAULT;
 387
 388                 return 0;
 389
 390         default:
 391                 LBUG();
 392         }
 393
 394         return 0;
 395 }
 396
 397 static int
 398 lstcon_group_nodes_add(struct lstcon_group *grp,
 399                        int count, struct lnet_process_id __user *ids_up,
 400                        unsigned int *featp,
 401                        struct list_head __user *result_up)
 402 {
 403         struct lstcon_rpc_trans *trans;
 404         struct lstcon_ndlink *ndl;
 405         struct lstcon_group *tmp;
 406         struct lnet_process_id id;
 407         int i;
 408         int rc;
 409
 410         rc = lstcon_group_alloc(NULL, &tmp);
 411         if (rc != 0) {
 412                 CERROR("Out of memory\n");
 413                 return -ENOMEM;
 414         }
 415
 416         for (i = 0 ; i < count; i++) {
 417                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 418                         rc = -EFAULT;
 419                         break;
 420                 }
 421
 422                 /* skip if it's in this group already */
 423                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 0);
 424                 if (rc == 0)
 425                         continue;
 426
 427                 /* add to tmp group */
 428                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1);
 429                 if (rc != 0) {
 430                         CERROR("Can't create ndlink, out of memory\n");
 431                         break;
 432                 }
 433         }
 434
 435         if (rc != 0) {
 436                 lstcon_group_decref(tmp);
 437                 return rc;
 438         }
 439
 440         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 441                                      &tmp->grp_trans_list, LST_TRANS_SESNEW,
 442                                      tmp, lstcon_sesrpc_condition, &trans);
 443         if (rc != 0) {
 444                 CERROR("Can't create transaction: %d\n", rc);
 445                 lstcon_group_decref(tmp);
 446                 return rc;
 447         }
 448
 449         /* post all RPCs */
 450         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 451
 452         rc = lstcon_rpc_trans_interpreter(trans, result_up,
 453                                           lstcon_sesrpc_readent);
 454         *featp = trans->tas_features;
 455
 456         /* destroy all RPGs */
 457         lstcon_rpc_trans_destroy(trans);
 458
 459         lstcon_group_move(tmp, grp);
 460         lstcon_group_decref(tmp);
 461
 462         return rc;
 463 }
 464
 465 static int
 466 lstcon_group_nodes_remove(struct lstcon_group *grp,
 467                           int count, struct lnet_process_id __user *ids_up,
 468                           struct list_head __user *result_up)
 469 {
 470         struct lstcon_rpc_trans *trans;
 471         struct lstcon_ndlink *ndl;
 472         struct lstcon_group *tmp;
 473         struct lnet_process_id id;
 474         int rc;
 475         int i;
 476
 477         /* End session and remove node from the group */
 478
 479         rc = lstcon_group_alloc(NULL, &tmp);
 480         if (rc != 0) {
 481                 CERROR("Out of memory\n");
 482                 return -ENOMEM;
 483         }
 484
 485         for (i = 0; i < count; i++) {
 486                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 487                         rc = -EFAULT;
 488                         goto error;
 489                 }
 490
 491                 /* move node to tmp group */
 492                 if (lstcon_group_ndlink_find(grp, id, &ndl, 0) == 0)
 493                         lstcon_group_ndlink_move(grp, tmp, ndl);
 494         }
 495
 496         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 497                                      &tmp->grp_trans_list, LST_TRANS_SESEND,
 498                                      tmp, lstcon_sesrpc_condition, &trans);
 499         if (rc != 0) {
 500                 CERROR("Can't create transaction: %d\n", rc);
 501                 goto error;
 502         }
 503
 504         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 505
 506         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 507
 508         lstcon_rpc_trans_destroy(trans);
 509         /* release nodes anyway, because we can't rollback status */
 510         lstcon_group_decref(tmp);
 511
 512         return rc;
 513 error:
 514         lstcon_group_move(tmp, grp);
 515         lstcon_group_decref(tmp);
 516
 517         return rc;
 518 }
 519
 520 int
 521 lstcon_group_add(char *name)
 522 {
 523         struct lstcon_group *grp;
 524         int rc;
 525
 526         rc = (lstcon_group_find(name, &grp) == 0)? -EEXIST: 0;
 527         if (rc != 0) {
 528                 /* find a group with same name */
 529                 lstcon_group_decref(grp);
 530                 return rc;
 531         }
 532
 533         rc = lstcon_group_alloc(name, &grp);
 534         if (rc != 0) {
 535                 CERROR("Can't allocate descriptor for group %s\n", name);
 536                 return -ENOMEM;
 537         }
 538
 539         list_add_tail(&grp->grp_link, &console_session.ses_grp_list);
 540
 541         return rc;
 542 }
 543
 544 int
 545 lstcon_nodes_add(char *name, int count, struct lnet_process_id __user *ids_up,
 546                  unsigned *featp, struct list_head __user *result_up)
 547 {
 548         struct lstcon_group         *grp;
 549         int                     rc;
 550
 551         LASSERT (count > 0);
 552         LASSERT (ids_up != NULL);
 553
 554         rc = lstcon_group_find(name, &grp);
 555         if (rc != 0) {
 556                 CDEBUG(D_NET, "Can't find group %s\n", name);
 557                 return rc;
 558         }
 559
 560         if (grp->grp_ref > 2) {
 561                 /* referred by other threads or test */
 562                 CDEBUG(D_NET, "Group %s is busy\n", name);
 563                 lstcon_group_decref(grp);
 564
 565                 return -EBUSY;
 566         }
 567
 568         rc = lstcon_group_nodes_add(grp, count, ids_up, featp, result_up);
 569
 570         lstcon_group_decref(grp);
 571
 572         return rc;
 573 }
 574
 575 int
 576 lstcon_group_del(char *name)
 577 {
 578         struct lstcon_rpc_trans *trans;
 579         struct lstcon_group *grp;
 580         int rc;
 581
 582         rc = lstcon_group_find(name, &grp);
 583         if (rc != 0) {
 584                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 585                 return rc;
 586         }
 587
 588         if (grp->grp_ref > 2) {
 589                 /* referred by others threads or test */
 590                 CDEBUG(D_NET, "Group %s is busy\n", name);
 591                 lstcon_group_decref(grp);
 592                 return -EBUSY;
 593         }
 594
 595         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 596                                      &grp->grp_trans_list, LST_TRANS_SESEND,
 597                                      grp, lstcon_sesrpc_condition, &trans);
 598         if (rc != 0) {
 599                 CERROR("Can't create transaction: %d\n", rc);
 600                 lstcon_group_decref(grp);
 601                 return rc;
 602         }
 603
 604         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 605
 606         lstcon_rpc_trans_destroy(trans);
 607
 608         lstcon_group_decref(grp);
 609         /* -ref for session, it's destroyed,
 610          * status can't be rolled back, destroy group anway */
 611         lstcon_group_decref(grp);
 612
 613         return rc;
 614 }
 615
 616 int
 617 lstcon_group_clean(char *name, int args)
 618 {
 619         struct lstcon_group *grp = NULL;
 620         int rc;
 621
 622         rc = lstcon_group_find(name, &grp);
 623         if (rc != 0) {
 624                 CDEBUG(D_NET, "Can't find group %s\n", name);
 625                 return rc;
 626         }
 627
 628         if (grp->grp_ref > 2) {
 629                 /* referred by test */
 630                 CDEBUG(D_NET, "Group %s is busy\n", name);
 631                 lstcon_group_decref(grp);
 632                 return -EBUSY;
 633         }
 634
 635         args = (LST_NODE_ACTIVE | LST_NODE_BUSY |
 636                 LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args;
 637
 638         lstcon_group_drain(grp, args);
 639
 640         lstcon_group_decref(grp);
 641         /* release empty group */
 642         if (list_empty(&grp->grp_ndl_list))
 643                 lstcon_group_decref(grp);
 644
 645         return 0;
 646 }
 647
 648 int
 649 lstcon_nodes_remove(char *name, int count,
 650                     struct lnet_process_id __user *ids_up,
 651                     struct list_head __user *result_up)
 652 {
 653         struct lstcon_group *grp = NULL;
 654         int rc;
 655
 656         rc = lstcon_group_find(name, &grp);
 657         if (rc != 0) {
 658                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 659                 return rc;
 660         }
 661
 662         if (grp->grp_ref > 2) {
 663                 /* referred by test */
 664                 CDEBUG(D_NET, "Group %s is busy\n", name);
 665                 lstcon_group_decref(grp);
 666                 return -EBUSY;
 667         }
 668
 669         rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up);
 670
 671         lstcon_group_decref(grp);
 672         /* release empty group */
 673         if (list_empty(&grp->grp_ndl_list))
 674                 lstcon_group_decref(grp);
 675
 676         return rc;
 677 }
 678
 679 int
 680 lstcon_group_refresh(char *name, struct list_head __user *result_up)
 681 {
 682         struct lstcon_rpc_trans *trans;
 683         struct lstcon_group *grp;
 684         int rc;
 685
 686         rc = lstcon_group_find(name, &grp);
 687         if (rc != 0) {
 688                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 689                 return rc;
 690         }
 691
 692         if (grp->grp_ref > 2) {
 693                 /* referred by test */
 694                 CDEBUG(D_NET, "Group %s is busy\n", name);
 695                 lstcon_group_decref(grp);
 696                 return -EBUSY;
 697         }
 698
 699         /* re-invite all inactive nodes int the group */
 700         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 701                                      &grp->grp_trans_list, LST_TRANS_SESNEW,
 702                                      grp, lstcon_sesrpc_condition, &trans);
 703         if (rc != 0) {
 704                 /* local error, return */
 705                 CDEBUG(D_NET, "Can't create transaction: %d\n", rc);
 706                 lstcon_group_decref(grp);
 707                 return rc;
 708         }
 709
 710         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 711
 712         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 713
 714         lstcon_rpc_trans_destroy(trans);
 715         /* -ref for me */
 716         lstcon_group_decref(grp);
 717
 718         return rc;
 719 }
 720
 721 int
 722 lstcon_group_list(int index, int len, char __user *name_up)
 723 {
 724         struct lstcon_group *grp;
 725
 726         LASSERT(index >= 0);
 727         LASSERT(name_up != NULL);
 728
 729         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 730                 if (index-- == 0) {
 731                         return copy_to_user(name_up, grp->grp_name, len) ?
 732                                             -EFAULT : 0;
 733                 }
 734         }
 735
 736         return -ENOENT;
 737 }
 738
 739 static int
 740 lstcon_nodes_getent(struct list_head *head, int *index_p,
 741                     int *count_p, struct lstcon_node_ent __user *dents_up)
 742 {
 743         struct lstcon_ndlink *ndl;
 744         struct lstcon_node *nd;
 745         int count = 0;
 746         int index = 0;
 747
 748         LASSERT(index_p != NULL && count_p != NULL);
 749         LASSERT(dents_up != NULL);
 750         LASSERT(*index_p >= 0);
 751         LASSERT(*count_p > 0);
 752
 753         list_for_each_entry(ndl, head, ndl_link) {
 754                 if (index++ < *index_p)
 755                         continue;
 756
 757                 if (count >= *count_p)
 758                         break;
 759
 760                 nd = ndl->ndl_node;
 761                 if (copy_to_user(&dents_up[count].nde_id,
 762                                      &nd->nd_id, sizeof(nd->nd_id)) ||
 763                     copy_to_user(&dents_up[count].nde_state,
 764                                      &nd->nd_state, sizeof(nd->nd_state)))
 765                         return -EFAULT;
 766
 767                 count ++;
 768         }
 769
 770         if (index <= *index_p)
 771                 return -ENOENT;
 772
 773         *count_p = count;
 774         *index_p = index;
 775
 776         return 0;
 777 }
 778
 779 int
 780 lstcon_group_info(char *name, struct lstcon_ndlist_ent __user *gents_p,
 781                   int *index_p, int *count_p,
 782                   struct lstcon_node_ent __user *dents_up)
 783 {
 784         struct lstcon_ndlist_ent *gentp;
 785         struct lstcon_group *grp;
 786         struct lstcon_ndlink *ndl;
 787         int rc;
 788
 789         rc = lstcon_group_find(name, &grp);
 790         if (rc != 0) {
 791                 CDEBUG(D_NET, "Can't find group %s\n", name);
 792                 return rc;
 793         }
 794
 795         if (dents_up != NULL) {
 796                 /* verbose query */
 797                 rc = lstcon_nodes_getent(&grp->grp_ndl_list,
 798                                          index_p, count_p, dents_up);
 799                 lstcon_group_decref(grp);
 800
 801                 return rc;
 802         }
 803
 804         /* non-verbose query */
 805         CFS_ALLOC_PTR(gentp);
 806         if (gentp == NULL) {
 807                 CERROR("Can't allocate ndlist_ent\n");
 808                 lstcon_group_decref(grp);
 809
 810                 return -ENOMEM;
 811         }
 812
 813         list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link)
 814                 LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp);
 815
 816         rc = copy_to_user(gents_p, gentp,
 817                           sizeof(struct lstcon_ndlist_ent)) ? -EFAULT : 0;
 818
 819         CFS_FREE_PTR(gentp);
 820
 821         lstcon_group_decref(grp);
 822
 823         return 0;
 824 }
 825
 826 static int
 827 lstcon_batch_find(const char *name, struct lstcon_batch **batpp)
 828 {
 829         struct lstcon_batch *bat;
 830
 831         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 832                 if (strncmp(bat->bat_name, name, LST_NAME_SIZE) == 0) {
 833                         *batpp = bat;
 834                         return 0;
 835                 }
 836         }
 837
 838         return -ENOENT;
 839 }
 840
 841 int
 842 lstcon_batch_add(char *name)
 843 {
 844         struct lstcon_batch *bat;
 845         int i;
 846         int rc;
 847
 848         rc = (lstcon_batch_find(name, &bat) == 0)? -EEXIST: 0;
 849         if (rc != 0) {
 850                 CDEBUG(D_NET, "Batch %s already exists\n", name);
 851                 return rc;
 852         }
 853
 854         LIBCFS_ALLOC(bat, sizeof(*bat));
 855         if (bat == NULL) {
 856                 CERROR("Can't allocate descriptor for batch %s\n", name);
 857                 return -ENOMEM;
 858         }
 859
 860         LIBCFS_ALLOC(bat->bat_cli_hash,
 861                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 862         if (bat->bat_cli_hash == NULL) {
 863                 CERROR("Can't allocate hash for batch %s\n", name);
 864                 LIBCFS_FREE(bat, sizeof(*bat));
 865
 866                 return -ENOMEM;
 867         }
 868
 869         LIBCFS_ALLOC(bat->bat_srv_hash,
 870                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 871         if (bat->bat_srv_hash == NULL) {
 872                 CERROR("Can't allocate hash for batch %s\n", name);
 873                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 874                 LIBCFS_FREE(bat, sizeof(*bat));
 875
 876                 return -ENOMEM;
 877         }
 878
 879         if (strlen(name) > sizeof(bat->bat_name)-1) {
 880                 LIBCFS_FREE(bat->bat_srv_hash, LST_NODE_HASHSIZE);
 881                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 882                 LIBCFS_FREE(bat, sizeof(*bat));
 883                 return -E2BIG;
 884         }
 885         strncpy(bat->bat_name, name, sizeof(bat->bat_name));
 886         bat->bat_hdr.tsb_index = 0;
 887         bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie;
 888
 889         bat->bat_ntest = 0;
 890         bat->bat_state = LST_BATCH_IDLE;
 891
 892         INIT_LIST_HEAD(&bat->bat_cli_list);
 893         INIT_LIST_HEAD(&bat->bat_srv_list);
 894         INIT_LIST_HEAD(&bat->bat_test_list);
 895         INIT_LIST_HEAD(&bat->bat_trans_list);
 896
 897         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 898                 INIT_LIST_HEAD(&bat->bat_cli_hash[i]);
 899                 INIT_LIST_HEAD(&bat->bat_srv_hash[i]);
 900         }
 901
 902         list_add_tail(&bat->bat_link, &console_session.ses_bat_list);
 903
 904         return rc;
 905 }
 906
 907 int
 908 lstcon_batch_list(int index, int len, char __user *name_up)
 909 {
 910         struct lstcon_batch *bat;
 911
 912         LASSERT(name_up != NULL);
 913         LASSERT(index >= 0);
 914
 915         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 916                 if (index-- == 0) {
 917                         return copy_to_user(name_up, bat->bat_name, len) ?
 918                                             -EFAULT : 0;
 919                 }
 920         }
 921
 922         return -ENOENT;
 923 }
 924
 925 int
 926 lstcon_batch_info(char *name, struct lstcon_test_batch_ent __user *ent_up,
 927                   int server, int testidx, int *index_p, int *ndent_p,
 928                   struct lstcon_node_ent __user *dents_up)
 929 {
 930         struct lstcon_test_batch_ent *entp;
 931         struct list_head *clilst;
 932         struct list_head *srvlst;
 933         struct lstcon_test *test = NULL;
 934         struct lstcon_batch *bat;
 935         struct lstcon_ndlink *ndl;
 936         int rc;
 937
 938         rc = lstcon_batch_find(name, &bat);
 939         if (rc != 0) {
 940                 CDEBUG(D_NET, "Can't find batch %s\n", name);
 941                 return -ENOENT;
 942         }
 943
 944         if (testidx > 0) {
 945                 /* query test, test index start from 1 */
 946                 list_for_each_entry(test, &bat->bat_test_list, tes_link) {
 947                         if (testidx-- == 1)
 948                                 break;
 949                 }
 950
 951                 if (testidx > 0) {
 952                         CDEBUG(D_NET, "Can't find specified test in batch\n");
 953                         return -ENOENT;
 954                 }
 955         }
 956
 957         clilst = (test == NULL) ? &bat->bat_cli_list :
 958                                   &test->tes_src_grp->grp_ndl_list;
 959         srvlst = (test == NULL) ? &bat->bat_srv_list :
 960                                   &test->tes_dst_grp->grp_ndl_list;
 961
 962         if (dents_up != NULL) {
 963                 rc = lstcon_nodes_getent((server ? srvlst: clilst),
 964                                          index_p, ndent_p, dents_up);
 965                 return rc;
 966         }
 967
 968         /* non-verbose query */
 969         CFS_ALLOC_PTR(entp);
 970         if (entp == NULL)
 971                 return -ENOMEM;
 972
 973         if (test == NULL) {
 974                 entp->u.tbe_batch.bae_ntest = bat->bat_ntest;
 975                 entp->u.tbe_batch.bae_state = bat->bat_state;
 976
 977         } else {
 978
 979                 entp->u.tbe_test.tse_type   = test->tes_type;
 980                 entp->u.tbe_test.tse_loop   = test->tes_loop;
 981                 entp->u.tbe_test.tse_concur = test->tes_concur;
 982         }
 983
 984         list_for_each_entry(ndl, clilst, ndl_link)
 985                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle);
 986
 987         list_for_each_entry(ndl, srvlst, ndl_link)
 988                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle);
 989
 990         rc = copy_to_user(ent_up, entp,
 991                           sizeof(struct lstcon_test_batch_ent)) ? -EFAULT : 0;
 992
 993         CFS_FREE_PTR(entp)
 994
 995         return rc;
 996 }
 997
 998 static int
 999 lstcon_batrpc_condition(int transop, struct lstcon_node *nd, void *arg)
1000 {
1001         switch (transop) {
1002         case LST_TRANS_TSBRUN:
1003                 if (nd->nd_state != LST_NODE_ACTIVE)
1004                         return -ENETDOWN;
1005                 break;
1006
1007         case LST_TRANS_TSBSTOP:
1008                 if (nd->nd_state != LST_NODE_ACTIVE)
1009                         return 0;
1010                 break;
1011
1012         case LST_TRANS_TSBCLIQRY:
1013         case LST_TRANS_TSBSRVQRY:
1014                 break;
1015         }
1016
1017         return 1;
1018 }
1019
1020 static int
1021 lstcon_batch_op(struct lstcon_batch *bat, int transop,
1022                 struct list_head __user *result_up)
1023 {
1024         struct lstcon_rpc_trans *trans;
1025         int                 rc;
1026
1027         rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list,
1028                                      &bat->bat_trans_list, transop,
1029                                      bat, lstcon_batrpc_condition, &trans);
1030         if (rc != 0) {
1031                 CERROR("Can't create transaction: %d\n", rc);
1032                 return rc;
1033         }
1034
1035         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1036
1037         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1038
1039         lstcon_rpc_trans_destroy(trans);
1040
1041         return rc;
1042 }
1043
1044 int
1045 lstcon_batch_run(char *name, int timeout, struct list_head __user *result_up)
1046 {
1047         struct lstcon_batch *bat;
1048         int rc;
1049
1050         if (lstcon_batch_find(name, &bat) != 0) {
1051                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1052                 return -ENOENT;
1053         }
1054
1055         bat->bat_arg = timeout;
1056
1057         rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up);
1058
1059         /* mark batch as running if it's started in any node */
1060         if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0) != 0)
1061                 bat->bat_state = LST_BATCH_RUNNING;
1062
1063         return rc;
1064 }
1065
1066 int
1067 lstcon_batch_stop(char *name, int force, struct list_head __user *result_up)
1068 {
1069         struct lstcon_batch *bat;
1070         int rc;
1071
1072         if (lstcon_batch_find(name, &bat) != 0) {
1073                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1074                 return -ENOENT;
1075         }
1076
1077         bat->bat_arg = force;
1078
1079         rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up);
1080
1081         /* mark batch as stopped if all RPCs finished */
1082         if (lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0) == 0)
1083                 bat->bat_state = LST_BATCH_IDLE;
1084
1085         return rc;
1086 }
1087
1088 static void
1089 lstcon_batch_destroy(struct lstcon_batch *bat)
1090 {
1091         struct lstcon_ndlink *ndl;
1092         struct lstcon_test *test;
1093         int i;
1094
1095         list_del(&bat->bat_link);
1096
1097         while (!list_empty(&bat->bat_test_list)) {
1098                 test = list_entry(bat->bat_test_list.next,
1099                                   struct lstcon_test, tes_link);
1100                 LASSERT(list_empty(&test->tes_trans_list));
1101
1102                 list_del(&test->tes_link);
1103
1104                 lstcon_group_decref(test->tes_src_grp);
1105                 lstcon_group_decref(test->tes_dst_grp);
1106
1107                 LIBCFS_FREE(test, offsetof(struct lstcon_test,
1108                                            tes_param[test->tes_paramlen]));
1109         }
1110
1111         LASSERT(list_empty(&bat->bat_trans_list));
1112
1113         while (!list_empty(&bat->bat_cli_list)) {
1114                 ndl = list_entry(bat->bat_cli_list.next,
1115                                  struct lstcon_ndlink, ndl_link);
1116                 list_del_init(&ndl->ndl_link);
1117
1118                 lstcon_ndlink_release(ndl);
1119         }
1120
1121         while (!list_empty(&bat->bat_srv_list)) {
1122                 ndl = list_entry(bat->bat_srv_list.next,
1123                                  struct lstcon_ndlink, ndl_link);
1124                 list_del_init(&ndl->ndl_link);
1125
1126                 lstcon_ndlink_release(ndl);
1127         }
1128
1129         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1130                 LASSERT(list_empty(&bat->bat_cli_hash[i]));
1131                 LASSERT(list_empty(&bat->bat_srv_hash[i]));
1132         }
1133
1134         LIBCFS_FREE(bat->bat_cli_hash,
1135                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1136         LIBCFS_FREE(bat->bat_srv_hash,
1137                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1138         LIBCFS_FREE(bat, sizeof(*bat));
1139 }
1140
1141 static int
1142 lstcon_testrpc_condition(int transop, struct lstcon_node *nd, void *arg)
1143 {
1144         struct lstcon_test *test = arg;
1145         struct lstcon_batch *batch;
1146         struct lstcon_ndlink *ndl;
1147         struct list_head *hash;
1148         struct list_head *head;
1149
1150         LASSERT(test != NULL);
1151
1152         batch = test->tes_batch;
1153         LASSERT(batch != NULL);
1154
1155         if (test->tes_oneside &&
1156             transop == LST_TRANS_TSBSRVADD)
1157                 return 0;
1158
1159         if (nd->nd_state != LST_NODE_ACTIVE)
1160                 return -ENETDOWN;
1161
1162         if (transop == LST_TRANS_TSBCLIADD) {
1163                 hash = batch->bat_cli_hash;
1164                 head = &batch->bat_cli_list;
1165
1166         } else {
1167                 LASSERT (transop == LST_TRANS_TSBSRVADD);
1168
1169                 hash = batch->bat_srv_hash;
1170                 head = &batch->bat_srv_list;
1171         }
1172
1173         LASSERT (nd->nd_id.nid != LNET_NID_ANY);
1174
1175         if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1) != 0)
1176                 return -ENOMEM;
1177
1178         if (list_empty(&ndl->ndl_link))
1179                 list_add_tail(&ndl->ndl_link, head);
1180
1181         return 1;
1182 }
1183
1184 static int
1185 lstcon_test_nodes_add(struct lstcon_test *test,
1186                       struct list_head __user *result_up)
1187 {
1188         struct lstcon_rpc_trans *trans;
1189         struct lstcon_group *grp;
1190         int transop;
1191         int rc;
1192
1193         LASSERT (test->tes_src_grp != NULL);
1194         LASSERT (test->tes_dst_grp != NULL);
1195
1196         transop = LST_TRANS_TSBSRVADD;
1197         grp  = test->tes_dst_grp;
1198 again:
1199         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
1200                                      &test->tes_trans_list, transop,
1201                                      test, lstcon_testrpc_condition, &trans);
1202         if (rc != 0) {
1203                 CERROR("Can't create transaction: %d\n", rc);
1204                 return rc;
1205         }
1206
1207         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1208
1209         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1210             lstcon_trans_stat()->trs_fwk_errno != 0) {
1211                 lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1212
1213                 lstcon_rpc_trans_destroy(trans);
1214                 /* return if any error */
1215                 CDEBUG(D_NET, "Failed to add test %s, "
1216                               "RPC error %d, framework error %d\n",
1217                        transop == LST_TRANS_TSBCLIADD ? "client" : "server",
1218                        lstcon_trans_stat()->trs_rpc_errno,
1219                        lstcon_trans_stat()->trs_fwk_errno);
1220
1221                 return rc;
1222         }
1223
1224         lstcon_rpc_trans_destroy(trans);
1225
1226         if (transop == LST_TRANS_TSBCLIADD)
1227                 return rc;
1228
1229         transop = LST_TRANS_TSBCLIADD;
1230         grp = test->tes_src_grp;
1231         test->tes_cliidx = 0;
1232
1233         /* requests to test clients */
1234         goto again;
1235 }
1236
1237 static int
1238 lstcon_verify_batch(const char *name, struct lstcon_batch **batch)
1239 {
1240         int rc;
1241
1242         rc = lstcon_batch_find(name, batch);
1243         if (rc != 0) {
1244                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1245                 return rc;
1246         }
1247
1248         if ((*batch)->bat_state != LST_BATCH_IDLE) {
1249                 CDEBUG(D_NET, "Can't change running batch %s\n", name);
1250                 return -EINVAL;
1251         }
1252
1253         return 0;
1254 }
1255
1256 static int
1257 lstcon_verify_group(const char *name, struct lstcon_group **grp)
1258 {
1259         int rc;
1260         struct lstcon_ndlink *ndl;
1261
1262         rc = lstcon_group_find(name, grp);
1263         if (rc != 0) {
1264                 CDEBUG(D_NET, "can't find group %s\n", name);
1265                 return rc;
1266         }
1267
1268         list_for_each_entry(ndl, &(*grp)->grp_ndl_list, ndl_link) {
1269                 if (ndl->ndl_node->nd_state == LST_NODE_ACTIVE) {
1270                         return 0;
1271                 }
1272         }
1273
1274         CDEBUG(D_NET, "Group %s has no ACTIVE nodes\n", name);
1275
1276         return -EINVAL;
1277 }
1278
1279 int
1280 lstcon_test_add(char *batch_name, int type, int loop,
1281                 int concur, int dist, int span,
1282                 char *src_name, char *dst_name,
1283                 void *param, int paramlen, int *retp,
1284                 struct list_head __user *result_up)
1285 {
1286         struct lstcon_test *test = NULL;
1287         int rc;
1288         struct lstcon_group *src_grp = NULL;
1289         struct lstcon_group *dst_grp = NULL;
1290         struct lstcon_batch *batch = NULL;
1291
1292         /*
1293          * verify that a batch of the given name exists, and the groups
1294          * that will be part of the batch exist and have at least one
1295          * active node
1296          */
1297         rc = lstcon_verify_batch(batch_name, &batch);
1298         if (rc != 0)
1299                 goto out;
1300
1301         rc = lstcon_verify_group(src_name, &src_grp);
1302         if (rc != 0)
1303                 goto out;
1304
1305         rc = lstcon_verify_group(dst_name, &dst_grp);
1306         if (rc != 0)
1307                 goto out;
1308
1309         if (dst_grp->grp_userland)
1310                 *retp = 1;
1311
1312         LIBCFS_ALLOC(test, offsetof(struct lstcon_test, tes_param[paramlen]));
1313         if (!test) {
1314                 CERROR("Can't allocate test descriptor\n");
1315                 rc = -ENOMEM;
1316
1317                 goto out;
1318         }
1319
1320         test->tes_hdr.tsb_id    = batch->bat_hdr.tsb_id;
1321         test->tes_batch         = batch;
1322         test->tes_type          = type;
1323         test->tes_oneside       = 0; /* TODO */
1324         test->tes_loop          = loop;
1325         test->tes_concur        = concur;
1326         test->tes_stop_onerr    = 1; /* TODO */
1327         test->tes_span          = span;
1328         test->tes_dist          = dist;
1329         test->tes_cliidx        = 0; /* just used for creating RPC */
1330         test->tes_src_grp       = src_grp;
1331         test->tes_dst_grp       = dst_grp;
1332         INIT_LIST_HEAD(&test->tes_trans_list);
1333
1334         if (param != NULL) {
1335                 test->tes_paramlen = paramlen;
1336                 memcpy(&test->tes_param[0], param, paramlen);
1337         }
1338
1339         rc = lstcon_test_nodes_add(test, result_up);
1340
1341         if (rc != 0)
1342                 goto out;
1343
1344         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1345             lstcon_trans_stat()->trs_fwk_errno != 0)
1346                 CDEBUG(D_NET, "Failed to add test %d to batch %s\n", type,
1347                        batch_name);
1348
1349         /* add to test list anyway, so user can check what's going on */
1350         list_add_tail(&test->tes_link, &batch->bat_test_list);
1351
1352         batch->bat_ntest++;
1353         test->tes_hdr.tsb_index = batch->bat_ntest;
1354
1355         /*  hold groups so nobody can change them */
1356         return rc;
1357 out:
1358         if (test != NULL)
1359                 LIBCFS_FREE(test, offsetof(struct lstcon_test,
1360                                            tes_param[paramlen]));
1361
1362         if (dst_grp != NULL)
1363                 lstcon_group_decref(dst_grp);
1364
1365         if (src_grp != NULL)
1366                 lstcon_group_decref(src_grp);
1367
1368         return rc;
1369 }
1370
1371 static int
1372 lstcon_test_find(struct lstcon_batch *batch, int idx,
1373                  struct lstcon_test **testpp)
1374 {
1375         struct lstcon_test *test;
1376
1377         list_for_each_entry(test, &batch->bat_test_list, tes_link) {
1378                 if (idx == test->tes_hdr.tsb_index) {
1379                         *testpp = test;
1380                         return 0;
1381                 }
1382         }
1383
1384         return -ENOENT;
1385 }
1386
1387 static int
1388 lstcon_tsbrpc_readent(int transop, struct srpc_msg *msg,
1389                       struct lstcon_rpc_ent __user *ent_up)
1390 {
1391         struct srpc_batch_reply *rep = &msg->msg_body.bat_reply;
1392
1393         LASSERT (transop == LST_TRANS_TSBCLIQRY ||
1394                  transop == LST_TRANS_TSBSRVQRY);
1395
1396         /* positive errno, framework error code */
1397         if (copy_to_user(&ent_up->rpe_priv[0],
1398                              &rep->bar_active, sizeof(rep->bar_active)))
1399                 return -EFAULT;
1400
1401         return 0;
1402 }
1403
1404 int
1405 lstcon_test_batch_query(char *name, int testidx, int client,
1406                         int timeout, struct list_head __user *result_up)
1407 {
1408         struct lstcon_rpc_trans *trans;
1409         struct list_head *translist;
1410         struct list_head *ndlist;
1411         struct lstcon_tsb_hdr *hdr;
1412         struct lstcon_batch *batch;
1413         struct lstcon_test *test = NULL;
1414         int transop;
1415         int rc;
1416
1417         rc = lstcon_batch_find(name, &batch);
1418         if (rc != 0) {
1419                 CDEBUG(D_NET, "Can't find batch: %s\n", name);
1420                 return rc;
1421         }
1422
1423         if (testidx == 0) {
1424                 translist = &batch->bat_trans_list;
1425                 ndlist    = &batch->bat_cli_list;
1426                 hdr       = &batch->bat_hdr;
1427
1428         } else {
1429                 /* query specified test only */
1430                 rc = lstcon_test_find(batch, testidx, &test);
1431                 if (rc != 0) {
1432                         CDEBUG(D_NET, "Can't find test: %d\n", testidx);
1433                         return rc;
1434                 }
1435
1436                 translist = &test->tes_trans_list;
1437                 ndlist    = &test->tes_src_grp->grp_ndl_list;
1438                 hdr       = &test->tes_hdr;
1439         }
1440
1441         transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY;
1442
1443         rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr,
1444                                      lstcon_batrpc_condition, &trans);
1445         if (rc != 0) {
1446                 CERROR("Can't create transaction: %d\n", rc);
1447                 return rc;
1448         }
1449
1450         lstcon_rpc_trans_postwait(trans, timeout);
1451
1452         if (testidx == 0 && /* query a batch, not a test */
1453             lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) == 0 &&
1454             lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0) == 0) {
1455                 /* all RPCs finished, and no active test */
1456                 batch->bat_state = LST_BATCH_IDLE;
1457         }
1458
1459         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1460                                           lstcon_tsbrpc_readent);
1461         lstcon_rpc_trans_destroy(trans);
1462
1463         return rc;
1464 }
1465
1466 static int
1467 lstcon_statrpc_readent(int transop, struct srpc_msg *msg,
1468                        struct lstcon_rpc_ent __user *ent_up)
1469 {
1470         struct srpc_stat_reply *rep = &msg->msg_body.stat_reply;
1471         struct sfw_counters __user *sfwk_stat;
1472         struct srpc_counters __user *srpc_stat;
1473         struct lnet_counters_common __user *lnet_stat;
1474
1475         if (rep->str_status != 0)
1476                 return 0;
1477
1478         sfwk_stat = (struct sfw_counters __user *)&ent_up->rpe_payload[0];
1479         srpc_stat = (struct srpc_counters __user *)
1480                 ((char __user *)sfwk_stat + sizeof(*sfwk_stat));
1481         lnet_stat = (struct lnet_counters_common __user *)
1482                 ((char __user *)srpc_stat + sizeof(*srpc_stat));
1483
1484         if (copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) ||
1485             copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) ||
1486             copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat)))
1487                 return -EFAULT;
1488
1489         return 0;
1490 }
1491
1492 static int
1493 lstcon_ndlist_stat(struct list_head *ndlist,
1494                    int timeout, struct list_head __user *result_up)
1495 {
1496         struct list_head    head;
1497         struct lstcon_rpc_trans *trans;
1498         int                 rc;
1499
1500         INIT_LIST_HEAD(&head);
1501
1502         rc = lstcon_rpc_trans_ndlist(ndlist, &head,
1503                                      LST_TRANS_STATQRY, NULL, NULL, &trans);
1504         if (rc != 0) {
1505                 CERROR("Can't create transaction: %d\n", rc);
1506                 return rc;
1507         }
1508
1509         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1510
1511         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1512                                           lstcon_statrpc_readent);
1513         lstcon_rpc_trans_destroy(trans);
1514
1515         return rc;
1516 }
1517
1518 int
1519 lstcon_group_stat(char *grp_name, int timeout,
1520                   struct list_head __user *result_up)
1521 {
1522         struct lstcon_group *grp;
1523         int rc;
1524
1525         rc = lstcon_group_find(grp_name, &grp);
1526         if (rc != 0) {
1527                 CDEBUG(D_NET, "Can't find group %s\n", grp_name);
1528                 return rc;
1529         }
1530
1531         rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up);
1532
1533         lstcon_group_decref(grp);
1534
1535         return rc;
1536 }
1537
1538 int
1539 lstcon_nodes_stat(int count, struct lnet_process_id __user *ids_up,
1540                   int timeout, struct list_head __user *result_up)
1541 {
1542         struct lstcon_ndlink *ndl;
1543         struct lstcon_group *tmp;
1544         struct lnet_process_id id;
1545         int i;
1546         int rc;
1547
1548         rc = lstcon_group_alloc(NULL, &tmp);
1549         if (rc != 0) {
1550                 CERROR("Out of memory\n");
1551                 return -ENOMEM;
1552         }
1553
1554         for (i = 0 ; i < count; i++) {
1555                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1556                         rc = -EFAULT;
1557                         break;
1558                 }
1559
1560                 /* add to tmp group */
1561                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2);
1562                 if (rc != 0) {
1563                         CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET,
1564                                "Failed to find or create %s: %d\n",
1565                                libcfs_id2str(id), rc);
1566                         break;
1567                 }
1568         }
1569
1570         if (rc != 0) {
1571                 lstcon_group_decref(tmp);
1572                 return rc;
1573         }
1574
1575         rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up);
1576
1577         lstcon_group_decref(tmp);
1578
1579         return rc;
1580 }
1581
1582 static int
1583 lstcon_debug_ndlist(struct list_head *ndlist,
1584                     struct list_head *translist,
1585                     int timeout, struct list_head __user *result_up)
1586 {
1587         struct lstcon_rpc_trans *trans;
1588         int rc;
1589
1590         rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY,
1591                                      NULL, lstcon_sesrpc_condition, &trans);
1592         if (rc != 0) {
1593                 CERROR("Can't create transaction: %d\n", rc);
1594                 return rc;
1595         }
1596
1597         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1598
1599         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1600                                           lstcon_sesrpc_readent);
1601         lstcon_rpc_trans_destroy(trans);
1602
1603         return rc;
1604 }
1605
1606 int
1607 lstcon_session_debug(int timeout, struct list_head __user *result_up)
1608 {
1609         return lstcon_debug_ndlist(&console_session.ses_ndl_list,
1610                                    NULL, timeout, result_up);
1611 }
1612
1613 int
1614 lstcon_batch_debug(int timeout, char *name,
1615                    int client, struct list_head __user *result_up)
1616 {
1617         struct lstcon_batch *bat;
1618         int rc;
1619
1620         rc = lstcon_batch_find(name, &bat);
1621         if (rc != 0)
1622                 return -ENOENT;
1623
1624         rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list :
1625                                           &bat->bat_srv_list,
1626                                  NULL, timeout, result_up);
1627
1628         return rc;
1629 }
1630
1631 int
1632 lstcon_group_debug(int timeout, char *name,
1633                    struct list_head __user *result_up)
1634 {
1635         struct lstcon_group *grp;
1636         int rc;
1637
1638         rc = lstcon_group_find(name, &grp);
1639         if (rc != 0)
1640                 return -ENOENT;
1641
1642         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1643                                  timeout, result_up);
1644         lstcon_group_decref(grp);
1645
1646         return rc;
1647 }
1648
1649 int
1650 lstcon_nodes_debug(int timeout, int count,
1651                    struct lnet_process_id __user *ids_up,
1652                    struct list_head __user *result_up)
1653 {
1654         struct lnet_process_id id;
1655         struct lstcon_ndlink *ndl;
1656         struct lstcon_group *grp;
1657         int i;
1658         int rc;
1659
1660         rc = lstcon_group_alloc(NULL, &grp);
1661         if (rc != 0) {
1662                 CDEBUG(D_NET, "Out of memory\n");
1663                 return rc;
1664         }
1665
1666         for (i = 0; i < count; i++) {
1667                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1668                         rc = -EFAULT;
1669                         break;
1670                 }
1671
1672                 /* node is added to tmp group */
1673                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 1);
1674                 if (rc != 0) {
1675                         CERROR("Can't create node link\n");
1676                         break;
1677                 }
1678         }
1679
1680         if (rc != 0) {
1681                 lstcon_group_decref(grp);
1682                 return rc;
1683         }
1684
1685         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1686                                  timeout, result_up);
1687
1688         lstcon_group_decref(grp);
1689
1690         return rc;
1691 }
1692
1693 int
1694 lstcon_session_match(struct lst_sid sid)
1695 {
1696         return (console_session.ses_id.ses_nid   == sid.ses_nid &&
1697                 console_session.ses_id.ses_stamp == sid.ses_stamp) ?  1: 0;
1698 }
1699
1700 static void
1701 lstcon_new_session_id(struct lst_sid *sid)
1702 {
1703         struct lnet_process_id id;
1704
1705         LASSERT(console_session.ses_state == LST_SESSION_NONE);
1706
1707         LNetGetId(1, &id);
1708         sid->ses_nid = id.nid;
1709         sid->ses_stamp = div_u64(ktime_get_ns(), NSEC_PER_MSEC);
1710 }
1711
1712 int
1713 lstcon_session_new(char *name, int key, unsigned feats,
1714                    int timeout, int force, struct lst_sid __user *sid_up)
1715 {
1716         int     rc = 0;
1717         int     i;
1718
1719         if (console_session.ses_state != LST_SESSION_NONE) {
1720                 /* session exists */
1721                 if (!force) {
1722                         CNETERR("Session %s already exists\n",
1723                                 console_session.ses_name);
1724                         return -EEXIST;
1725                 }
1726
1727                 rc = lstcon_session_end();
1728
1729                 /* lstcon_session_end() only return local error */
1730                 if  (rc != 0)
1731                         return rc;
1732         }
1733
1734         if ((feats & ~LST_FEATS_MASK) != 0) {
1735                 CNETERR("Unknown session features %x\n",
1736                         (feats & ~LST_FEATS_MASK));
1737                 return -EINVAL;
1738         }
1739
1740         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
1741                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
1742
1743         lstcon_new_session_id(&console_session.ses_id);
1744
1745         console_session.ses_key     = key;
1746         console_session.ses_state   = LST_SESSION_ACTIVE;
1747         console_session.ses_force   = !!force;
1748         console_session.ses_features = feats;
1749         console_session.ses_feats_updated = 0;
1750         console_session.ses_timeout = (timeout <= 0) ?
1751                                       LST_CONSOLE_TIMEOUT : timeout;
1752
1753         if (strlen(name) > sizeof(console_session.ses_name)-1)
1754                 return -E2BIG;
1755         strlcpy(console_session.ses_name, name,
1756                 sizeof(console_session.ses_name));
1757
1758         rc = lstcon_batch_add(LST_DEFAULT_BATCH);
1759         if (rc != 0)
1760                 return rc;
1761
1762         rc = lstcon_rpc_pinger_start();
1763         if (rc != 0) {
1764                 struct lstcon_batch *bat = NULL;
1765
1766                 lstcon_batch_find(LST_DEFAULT_BATCH, &bat);
1767                 lstcon_batch_destroy(bat);
1768
1769                 return rc;
1770         }
1771
1772         if (copy_to_user(sid_up, &console_session.ses_id,
1773                              sizeof(struct lst_sid)) == 0)
1774                 return rc;
1775
1776         lstcon_session_end();
1777
1778         return -EFAULT;
1779 }
1780
1781 int
1782 lstcon_session_info(struct lst_sid __user *sid_up, int __user *key_up,
1783                     unsigned __user *featp,
1784                     struct lstcon_ndlist_ent __user *ndinfo_up,
1785                     char __user *name_up, int len)
1786 {
1787         struct lstcon_ndlist_ent *entp;
1788         struct lstcon_ndlink *ndl;
1789         int rc = 0;
1790
1791         if (console_session.ses_state != LST_SESSION_ACTIVE)
1792                 return -ESRCH;
1793
1794         LIBCFS_ALLOC(entp, sizeof(*entp));
1795         if (entp == NULL)
1796                 return -ENOMEM;
1797
1798         list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link)
1799                 LST_NODE_STATE_COUNTER(ndl->ndl_node, entp);
1800
1801         if (copy_to_user(sid_up, &console_session.ses_id,
1802                          sizeof(struct lst_sid)) ||
1803             copy_to_user(key_up, &console_session.ses_key,
1804                              sizeof(*key_up)) ||
1805             copy_to_user(featp, &console_session.ses_features,
1806                              sizeof(*featp)) ||
1807             copy_to_user(ndinfo_up, entp, sizeof(*entp)) ||
1808             copy_to_user(name_up, console_session.ses_name, len))
1809                 rc = -EFAULT;
1810
1811         LIBCFS_FREE(entp, sizeof(*entp));
1812
1813         return rc;
1814 }
1815
1816 int
1817 lstcon_session_end(void)
1818 {
1819         struct lstcon_rpc_trans *trans;
1820         struct lstcon_group *grp;
1821         struct lstcon_batch *bat;
1822         int rc = 0;
1823
1824         LASSERT (console_session.ses_state == LST_SESSION_ACTIVE);
1825
1826         rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list,
1827                                      NULL, LST_TRANS_SESEND, NULL,
1828                                      lstcon_sesrpc_condition, &trans);
1829         if (rc != 0) {
1830                 CERROR("Can't create transaction: %d\n", rc);
1831                 return rc;
1832         }
1833
1834         console_session.ses_shutdown = 1;
1835
1836         lstcon_rpc_pinger_stop();
1837
1838         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1839
1840         lstcon_rpc_trans_destroy(trans);
1841         /* User can do nothing even rpc failed, so go on */
1842
1843         /* waiting for orphan rpcs to die */
1844         lstcon_rpc_cleanup_wait();
1845
1846         console_session.ses_id    = LST_INVALID_SID;
1847         console_session.ses_state = LST_SESSION_NONE;
1848         console_session.ses_key   = 0;
1849         console_session.ses_force = 0;
1850         console_session.ses_feats_updated = 0;
1851
1852         /* destroy all batches */
1853         while (!list_empty(&console_session.ses_bat_list)) {
1854                 bat = list_entry(console_session.ses_bat_list.next,
1855                                  struct lstcon_batch, bat_link);
1856
1857                 lstcon_batch_destroy(bat);
1858         }
1859
1860         /* destroy all groups */
1861         while (!list_empty(&console_session.ses_grp_list)) {
1862                 grp = list_entry(console_session.ses_grp_list.next,
1863                                  struct lstcon_group, grp_link);
1864                 LASSERT(grp->grp_ref == 1);
1865
1866                 lstcon_group_decref(grp);
1867         }
1868
1869         /* all nodes should be released */
1870         LASSERT(list_empty(&console_session.ses_ndl_list));
1871
1872         console_session.ses_shutdown = 0;
1873         console_session.ses_expired  = 0;
1874
1875         return rc;
1876 }
1877
1878 int
1879 lstcon_session_feats_check(unsigned feats)
1880 {
1881         int rc = 0;
1882
1883         if ((feats & ~LST_FEATS_MASK) != 0) {
1884                 CERROR("Can't support these features: %x\n",
1885                        (feats & ~LST_FEATS_MASK));
1886                 return -EPROTO;
1887         }
1888
1889         spin_lock(&console_session.ses_rpc_lock);
1890
1891         if (!console_session.ses_feats_updated) {
1892                 console_session.ses_feats_updated = 1;
1893                 console_session.ses_features = feats;
1894         }
1895
1896         if (console_session.ses_features != feats)
1897                 rc = -EPROTO;
1898
1899         spin_unlock(&console_session.ses_rpc_lock);
1900
1901         if (rc != 0) {
1902                 CERROR("remote features %x do not match with "
1903                        "session features %x of console\n",
1904                        feats, console_session.ses_features);
1905         }
1906
1907         return rc;
1908 }
1909
1910 static int
1911 lstcon_acceptor_handle(struct srpc_server_rpc *rpc)
1912 {
1913         struct srpc_msg *rep = &rpc->srpc_replymsg;
1914         struct srpc_msg *req = &rpc->srpc_reqstbuf->buf_msg;
1915         struct srpc_join_reqst *jreq = &req->msg_body.join_reqst;
1916         struct srpc_join_reply *jrep = &rep->msg_body.join_reply;
1917         struct lstcon_group *grp = NULL;
1918         struct lstcon_ndlink *ndl;
1919         int rc = 0;
1920
1921         sfw_unpack_message(req);
1922
1923         mutex_lock(&console_session.ses_mutex);
1924
1925         jrep->join_sid = console_session.ses_id;
1926
1927         if (console_session.ses_id.ses_nid == LNET_NID_ANY) {
1928                 jrep->join_status = ESRCH;
1929                 goto out;
1930         }
1931
1932         if (lstcon_session_feats_check(req->msg_ses_feats) != 0) {
1933                 jrep->join_status = EPROTO;
1934                 goto out;
1935         }
1936
1937         if (jreq->join_sid.ses_nid != LNET_NID_ANY &&
1938              !lstcon_session_match(jreq->join_sid)) {
1939                 jrep->join_status = EBUSY;
1940                 goto out;
1941         }
1942
1943         if (lstcon_group_find(jreq->join_group, &grp) != 0) {
1944                 rc = lstcon_group_alloc(jreq->join_group, &grp);
1945                 if (rc != 0) {
1946                         CERROR("Out of memory\n");
1947                         goto out;
1948                 }
1949
1950                 list_add_tail(&grp->grp_link,
1951                               &console_session.ses_grp_list);
1952                 lstcon_group_addref(grp);
1953         }
1954
1955         if (grp->grp_ref > 2) {
1956                 /* Group in using */
1957                 jrep->join_status = EBUSY;
1958                 goto out;
1959         }
1960
1961         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0);
1962         if (rc == 0) {
1963                 jrep->join_status = EEXIST;
1964                 goto out;
1965         }
1966
1967         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1);
1968         if (rc != 0) {
1969                 CERROR("Out of memory\n");
1970                 goto out;
1971         }
1972
1973         ndl->ndl_node->nd_state   = LST_NODE_ACTIVE;
1974         ndl->ndl_node->nd_timeout = console_session.ses_timeout;
1975
1976         if (grp->grp_userland == 0)
1977                 grp->grp_userland = 1;
1978
1979         strlcpy(jrep->join_session, console_session.ses_name,
1980                 sizeof(jrep->join_session));
1981         jrep->join_timeout = console_session.ses_timeout;
1982         jrep->join_status  = 0;
1983
1984 out:
1985         rep->msg_ses_feats = console_session.ses_features;
1986         if (grp != NULL)
1987                 lstcon_group_decref(grp);
1988
1989         mutex_unlock(&console_session.ses_mutex);
1990
1991         return rc;
1992 }
1993
1994 static struct srpc_service lstcon_acceptor_service;
1995
1996 static void lstcon_init_acceptor_service(void)
1997 {
1998         /* initialize selftest console acceptor service table */
1999         lstcon_acceptor_service.sv_name    = "join session";
2000         lstcon_acceptor_service.sv_handler = lstcon_acceptor_handle;
2001         lstcon_acceptor_service.sv_id      = SRPC_SERVICE_JOIN;
2002         lstcon_acceptor_service.sv_wi_total = SFW_FRWK_WI_MAX;
2003 }
2004
2005 static struct notifier_block lstcon_ioctl_handler = {
2006         .notifier_call = lstcon_ioctl_entry,
2007 };
2008
2009 /* initialize console */
2010 int
2011 lstcon_console_init(void)
2012 {
2013         int     i;
2014         int     rc;
2015
2016         console_session.ses_id              = LST_INVALID_SID;
2017         console_session.ses_state           = LST_SESSION_NONE;
2018         console_session.ses_timeout         = 0;
2019         console_session.ses_force           = 0;
2020         console_session.ses_expired         = 0;
2021         console_session.ses_feats_updated   = 0;
2022         console_session.ses_features        = LST_FEATS_MASK;
2023         console_session.ses_laststamp = ktime_get_real_seconds();
2024
2025         mutex_init(&console_session.ses_mutex);
2026
2027         INIT_LIST_HEAD(&console_session.ses_ndl_list);
2028         INIT_LIST_HEAD(&console_session.ses_grp_list);
2029         INIT_LIST_HEAD(&console_session.ses_bat_list);
2030         INIT_LIST_HEAD(&console_session.ses_trans_list);
2031
2032         LIBCFS_ALLOC(console_session.ses_ndl_hash,
2033                      sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2034         if (console_session.ses_ndl_hash == NULL)
2035                 return -ENOMEM;
2036
2037         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
2038                 INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]);
2039
2040
2041         /* initialize acceptor service table */
2042         lstcon_init_acceptor_service();
2043
2044         rc = srpc_add_service(&lstcon_acceptor_service);
2045         LASSERT(rc != -EBUSY);
2046         if (rc != 0) {
2047                 LIBCFS_FREE(console_session.ses_ndl_hash,
2048                             sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2049                 return rc;
2050         }
2051
2052         rc = srpc_service_add_buffers(&lstcon_acceptor_service,
2053                                       lstcon_acceptor_service.sv_wi_total);
2054         if (rc != 0) {
2055                 rc = -ENOMEM;
2056                 goto out;
2057         }
2058
2059         rc = blocking_notifier_chain_register(&libcfs_ioctl_list,
2060                                               &lstcon_ioctl_handler);
2061         if (rc == 0) {
2062                 lstcon_rpc_module_init();
2063                 return 0;
2064         }
2065
2066 out:
2067         srpc_shutdown_service(&lstcon_acceptor_service);
2068         srpc_remove_service(&lstcon_acceptor_service);
2069
2070         LIBCFS_FREE(console_session.ses_ndl_hash,
2071                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2072
2073         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2074
2075         return rc;
2076 }
2077
2078 int
2079 lstcon_console_fini(void)
2080 {
2081         int i;
2082
2083         blocking_notifier_chain_unregister(&libcfs_ioctl_list,
2084                                            &lstcon_ioctl_handler);
2085
2086         mutex_lock(&console_session.ses_mutex);
2087
2088         srpc_shutdown_service(&lstcon_acceptor_service);
2089         srpc_remove_service(&lstcon_acceptor_service);
2090
2091         if (console_session.ses_state != LST_SESSION_NONE)
2092                 lstcon_session_end();
2093
2094         lstcon_rpc_module_fini();
2095
2096         mutex_unlock(&console_session.ses_mutex);
2097
2098         LASSERT(list_empty(&console_session.ses_ndl_list));
2099         LASSERT(list_empty(&console_session.ses_grp_list));
2100         LASSERT(list_empty(&console_session.ses_bat_list));
2101         LASSERT(list_empty(&console_session.ses_trans_list));
2102
2103         for (i = 0; i < LST_NODE_HASHSIZE; i++)
2104                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
2105
2106         LIBCFS_FREE(console_session.ses_ndl_hash,
2107                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2108
2109         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2110
2111         return 0;
2112 }
2113