lnet/selftest/console.c

   1 /*
   2  * -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   3  * vim:expandtab:shiftwidth=8:tabstop=8:
   4  *
   5  * GPL HEADER START
   6  *
   7  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 only,
  11  * as published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful, but
  14  * WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * General Public License version 2 for more details (a copy is included
  17  * in the LICENSE file that accompanied this code).
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * version 2 along with this program; If not, see [sun.com URL with a
  21  * copy of GPLv2].
  22  *
  23  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  24  * CA 95054 USA or visit www.sun.com if you need additional information or
  25  * have any questions.
  26  *
  27  * GPL HEADER END
  28  */
  29 /*
  30  * Copyright  2008 Sun Microsystems, Inc. All rights reserved
  31  * Use is subject to license terms.
  32  */
  33 /*
  34  * This file is part of Lustre, http://www.lustre.org/
  35  * Lustre is a trademark of Sun Microsystems, Inc.
  36  *
  37  * lnet/selftest/conctl.c
  38  *
  39  * Infrastructure of LST console
  40  *
  41  * Author: Liang Zhen <liangzhen@clusterfs.com>
  42  */
  43
  44 #ifdef __KERNEL__
  45
  46 #include <libcfs/libcfs.h>
  47 #include <lnet/lib-lnet.h>
  48 #include "console.h"
  49 #include "conrpc.h"
  50
  51 #define LST_NODE_STATE_COUNTER(nd, p)                   \
  52 do {                                                    \
  53         if ((nd)->nd_state == LST_NODE_ACTIVE)          \
  54                 (p)->nle_nactive ++;                    \
  55         else if ((nd)->nd_state == LST_NODE_BUSY)       \
  56                 (p)->nle_nbusy ++;                      \
  57         else if ((nd)->nd_state == LST_NODE_DOWN)       \
  58                 (p)->nle_ndown ++;                      \
  59         else                                            \
  60                 (p)->nle_nunknown ++;                   \
  61         (p)->nle_nnode ++;                              \
  62 } while (0)
  63
  64 lstcon_session_t        console_session;
  65
  66 void
  67 lstcon_node_get(lstcon_node_t *nd)
  68 {
  69         LASSERT (nd->nd_ref >= 1);
  70
  71         nd->nd_ref++;
  72 }
  73
  74 static int
  75 lstcon_node_find(lnet_process_id_t id, lstcon_node_t **ndpp, int create)
  76 {
  77         lstcon_ndlink_t *ndl;
  78         unsigned int     idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE;
  79
  80         LASSERT (id.nid != LNET_NID_ANY);
  81
  82         list_for_each_entry(ndl, &console_session.ses_ndl_hash[idx], ndl_hlink) {
  83                 if (ndl->ndl_node->nd_id.nid != id.nid ||
  84                     ndl->ndl_node->nd_id.pid != id.pid)
  85                         continue;
  86
  87                 lstcon_node_get(ndl->ndl_node);
  88                 *ndpp = ndl->ndl_node;
  89                 return 0;
  90         }
  91
  92         if (!create)
  93                 return -ENOENT;
  94
  95         LIBCFS_ALLOC(*ndpp, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
  96         if (*ndpp == NULL)
  97                 return -ENOMEM;
  98
  99         ndl = (lstcon_ndlink_t *)(*ndpp + 1);
 100
 101         ndl->ndl_node = *ndpp;
 102
 103         ndl->ndl_node->nd_ref   = 1;
 104         ndl->ndl_node->nd_id    = id;
 105         ndl->ndl_node->nd_stamp = cfs_time_current();
 106         ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
 107         ndl->ndl_node->nd_timeout = 0;
 108         memset(&ndl->ndl_node->nd_ping, 0, sizeof(lstcon_rpc_t));
 109
 110         /* queued in global hash & list, no refcount is taken by
 111          * global hash & list, if caller release his refcount,
 112          * node will be released */
 113         list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]);
 114         list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list);
 115
 116         return 0;
 117 }
 118
 119 void
 120 lstcon_node_put(lstcon_node_t *nd)
 121 {
 122         lstcon_ndlink_t  *ndl;
 123
 124         LASSERT (nd->nd_ref > 0);
 125
 126         if (--nd->nd_ref > 0)
 127                 return;
 128
 129         ndl = (lstcon_ndlink_t *)(nd + 1);
 130
 131         LASSERT (!list_empty(&ndl->ndl_link));
 132         LASSERT (!list_empty(&ndl->ndl_hlink));
 133
 134         /* remove from session */
 135         list_del(&ndl->ndl_link);
 136         list_del(&ndl->ndl_hlink);
 137
 138         LIBCFS_FREE(nd, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
 139 }
 140
 141 static int
 142 lstcon_ndlink_find(struct list_head *hash,
 143                    lnet_process_id_t id, lstcon_ndlink_t **ndlpp, int create)
 144 {
 145         unsigned int     idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
 146         lstcon_ndlink_t *ndl;
 147         lstcon_node_t   *nd;
 148         int              rc;
 149
 150         if (id.nid == LNET_NID_ANY)
 151                 return -EINVAL;
 152
 153         /* search in hash */
 154         list_for_each_entry(ndl, &hash[idx], ndl_hlink) {
 155                 if (ndl->ndl_node->nd_id.nid != id.nid ||
 156                     ndl->ndl_node->nd_id.pid != id.pid)
 157                         continue;
 158
 159                 *ndlpp = ndl;
 160                 return 0;
 161         }
 162
 163         if (create == 0)
 164                 return -ENOENT;
 165
 166         /* find or create in session hash */
 167         rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0);
 168         if (rc != 0)
 169                 return rc;
 170
 171         LIBCFS_ALLOC(ndl, sizeof(lstcon_ndlink_t));
 172         if (ndl == NULL) {
 173                 lstcon_node_put(nd);
 174                 return -ENOMEM;
 175         }
 176
 177         *ndlpp = ndl;
 178
 179         ndl->ndl_node = nd;
 180         CFS_INIT_LIST_HEAD(&ndl->ndl_link);
 181         list_add_tail(&ndl->ndl_hlink, &hash[idx]);
 182
 183         return  0;
 184 }
 185
 186 static void
 187 lstcon_ndlink_release(lstcon_ndlink_t *ndl)
 188 {
 189         LASSERT (list_empty(&ndl->ndl_link));
 190         LASSERT (!list_empty(&ndl->ndl_hlink));
 191
 192         list_del(&ndl->ndl_hlink); /* delete from hash */
 193         lstcon_node_put(ndl->ndl_node);
 194
 195         LIBCFS_FREE(ndl, sizeof(*ndl));
 196 }
 197
 198 static int
 199 lstcon_group_alloc(char *name, lstcon_group_t **grpp)
 200 {
 201         lstcon_group_t *grp;
 202         int             i;
 203
 204         LIBCFS_ALLOC(grp, offsetof(lstcon_group_t,
 205                                    grp_ndl_hash[LST_NODE_HASHSIZE]));
 206         if (grp == NULL)
 207                 return -ENOMEM;
 208
 209         memset(grp, 0, offsetof(lstcon_group_t,
 210                                 grp_ndl_hash[LST_NODE_HASHSIZE]));
 211
 212         grp->grp_ref = 1;
 213         if (name != NULL)
 214                 strcpy(grp->grp_name, name);
 215
 216         CFS_INIT_LIST_HEAD(&grp->grp_link);
 217         CFS_INIT_LIST_HEAD(&grp->grp_ndl_list);
 218         CFS_INIT_LIST_HEAD(&grp->grp_trans_list);
 219
 220         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 221                 CFS_INIT_LIST_HEAD(&grp->grp_ndl_hash[i]);
 222
 223         *grpp = grp;
 224
 225         return 0;
 226 }
 227
 228 static void
 229 lstcon_group_addref(lstcon_group_t *grp)
 230 {
 231         grp->grp_ref ++;
 232 }
 233
 234 static void lstcon_group_ndlink_release(lstcon_group_t *, lstcon_ndlink_t *);
 235
 236 static void
 237 lstcon_group_drain(lstcon_group_t *grp, int keep)
 238 {
 239         lstcon_ndlink_t *ndl;
 240         lstcon_ndlink_t *tmp;
 241
 242         list_for_each_entry_safe(ndl, tmp, &grp->grp_ndl_list, ndl_link) {
 243                 if ((ndl->ndl_node->nd_state & keep) == 0)
 244                         lstcon_group_ndlink_release(grp, ndl);
 245         }
 246 }
 247
 248 static void
 249 lstcon_group_decref(lstcon_group_t *grp)
 250 {
 251         int     i;
 252
 253         if (--grp->grp_ref > 0)
 254                 return;
 255
 256         if (!list_empty(&grp->grp_link))
 257                 list_del(&grp->grp_link);
 258
 259         lstcon_group_drain(grp, 0);
 260
 261         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 262                 LASSERT (list_empty(&grp->grp_ndl_hash[i]));
 263         }
 264
 265         LIBCFS_FREE(grp, offsetof(lstcon_group_t,
 266                                   grp_ndl_hash[LST_NODE_HASHSIZE]));
 267 }
 268
 269 static int
 270 lstcon_group_find(char *name, lstcon_group_t **grpp)
 271 {
 272         lstcon_group_t   *grp;
 273
 274         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 275                 if (strncmp(grp->grp_name, name, LST_NAME_SIZE) != 0)
 276                         continue;
 277
 278                 lstcon_group_addref(grp);  /* +1 ref for caller */
 279                 *grpp = grp;
 280                 return 0;
 281         }
 282
 283         return -ENOENT;
 284 }
 285
 286 static void
 287 lstcon_group_put(lstcon_group_t *grp)
 288 {
 289         lstcon_group_decref(grp);
 290 }
 291
 292 static int
 293 lstcon_group_ndlink_find(lstcon_group_t *grp, lnet_process_id_t id,
 294                          lstcon_ndlink_t **ndlpp, int create)
 295 {
 296         int     rc;
 297
 298         rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create);
 299         if (rc != 0)
 300                 return rc;
 301
 302         if (!list_empty(&(*ndlpp)->ndl_link))
 303                 return 0;
 304
 305         list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list);
 306         grp->grp_nnode ++;
 307
 308         return 0;
 309 }
 310
 311 static void
 312 lstcon_group_ndlink_release(lstcon_group_t *grp, lstcon_ndlink_t *ndl)
 313 {
 314         list_del_init(&ndl->ndl_link);
 315         lstcon_ndlink_release(ndl);
 316         grp->grp_nnode --;
 317 }
 318
 319 static void
 320 lstcon_group_ndlink_move(lstcon_group_t *old,
 321                          lstcon_group_t *new, lstcon_ndlink_t *ndl)
 322 {
 323         unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) %
 324                            LST_NODE_HASHSIZE;
 325
 326         list_del(&ndl->ndl_hlink);
 327         list_del(&ndl->ndl_link);
 328         old->grp_nnode --;
 329
 330         list_add_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]);
 331         list_add_tail(&ndl->ndl_link, &new->grp_ndl_list);
 332         new->grp_nnode ++;
 333
 334         return;
 335 }
 336
 337 static void
 338 lstcon_group_move(lstcon_group_t *old, lstcon_group_t *new)
 339 {
 340         lstcon_ndlink_t *ndl;
 341
 342         while (!list_empty(&old->grp_ndl_list)) {
 343                 ndl = list_entry(old->grp_ndl_list.next,
 344                                  lstcon_ndlink_t, ndl_link);
 345                 lstcon_group_ndlink_move(old, new, ndl);
 346         }
 347 }
 348
 349 int
 350 lstcon_sesrpc_condition(int transop, lstcon_node_t *nd, void *arg)
 351 {
 352         lstcon_group_t *grp = (lstcon_group_t *)arg;
 353
 354         switch (transop) {
 355         case LST_TRANS_SESNEW:
 356                 if (nd->nd_state == LST_NODE_ACTIVE)
 357                         return 0;
 358                 break;
 359
 360         case LST_TRANS_SESEND:
 361                 if (nd->nd_state != LST_NODE_ACTIVE)
 362                         return 0;
 363
 364                 if (grp != NULL && nd->nd_ref > 1)
 365                         return 0;
 366                 break;
 367
 368         case LST_TRANS_SESQRY:
 369                 break;
 370
 371         default:
 372                 LBUG();
 373         }
 374
 375         return 1;
 376 }
 377
 378 int
 379 lstcon_sesrpc_readent(int transop, srpc_msg_t *msg,
 380                       lstcon_rpc_ent_t *ent_up)
 381 {
 382         srpc_debug_reply_t *rep;
 383
 384         switch (transop) {
 385         case LST_TRANS_SESNEW:
 386         case LST_TRANS_SESEND:
 387                 return 0;
 388
 389         case LST_TRANS_SESQRY:
 390                 rep = &msg->msg_body.dbg_reply;
 391
 392                 if (copy_to_user(&ent_up->rpe_priv[0],
 393                                  &rep->dbg_timeout, sizeof(int)) ||
 394                     copy_to_user(&ent_up->rpe_payload[0],
 395                                  &rep->dbg_name, LST_NAME_SIZE))
 396                         return -EFAULT;
 397
 398                 return 0;
 399
 400         default:
 401                 LBUG();
 402         }
 403
 404         return 0;
 405 }
 406
 407 static int
 408 lstcon_group_nodes_add(lstcon_group_t *grp, int count,
 409                        lnet_process_id_t *ids_up, struct list_head *result_up)
 410 {
 411         lstcon_rpc_trans_t      *trans;
 412         lstcon_ndlink_t         *ndl;
 413         lstcon_group_t          *tmp;
 414         lnet_process_id_t        id;
 415         int                      i;
 416         int                      rc;
 417
 418         rc = lstcon_group_alloc(NULL, &tmp);
 419         if (rc != 0) {
 420                 CERROR("Out of memory\n");
 421                 return -ENOMEM;
 422         }
 423
 424         for (i = 0 ; i < count; i++) {
 425                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 426                         rc = -EFAULT;
 427                         break;
 428                 }
 429
 430                 /* skip if it's in this group already */
 431                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 0);
 432                 if (rc == 0)
 433                         continue;
 434
 435                 /* add to tmp group */
 436                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1);
 437                 if (rc != 0) {
 438                         CERROR("Can't create ndlink, out of memory\n");
 439                         break;
 440                 }
 441         }
 442
 443         if (rc != 0) {
 444                 lstcon_group_put(tmp);
 445                 return rc;
 446         }
 447
 448         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 449                                      &tmp->grp_trans_list, LST_TRANS_SESNEW,
 450                                      tmp, lstcon_sesrpc_condition, &trans);
 451         if (rc != 0) {
 452                 CERROR("Can't create transaction: %d\n", rc);
 453                 lstcon_group_put(tmp);
 454                 return rc;
 455         }
 456
 457         /* post all RPCs */
 458         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 459
 460         rc = lstcon_rpc_trans_interpreter(trans, result_up,
 461                                           lstcon_sesrpc_readent);
 462         /* destroy all RPGs */
 463         lstcon_rpc_trans_destroy(trans);
 464
 465         lstcon_group_move(tmp, grp);
 466         lstcon_group_put(tmp);
 467
 468         return rc;
 469 }
 470
 471 static int
 472 lstcon_group_nodes_remove(lstcon_group_t *grp,
 473                           int count, lnet_process_id_t *ids_up,
 474                           struct list_head *result_up)
 475 {
 476         lstcon_rpc_trans_t     *trans;
 477         lstcon_ndlink_t        *ndl;
 478         lstcon_group_t         *tmp;
 479         lnet_process_id_t       id;
 480         int                     rc;
 481         int                     i;
 482
 483         /* End session and remove node from the group */
 484
 485         rc = lstcon_group_alloc(NULL, &tmp);
 486         if (rc != 0) {
 487                 CERROR("Out of memory\n");
 488                 return -ENOMEM;
 489         }
 490
 491         for (i = 0; i < count; i++) {
 492                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 493                         rc = -EFAULT;
 494                         goto error;
 495                 }
 496
 497                 /* move node to tmp group */
 498                 if (lstcon_group_ndlink_find(grp, id, &ndl, 0) == 0)
 499                         lstcon_group_ndlink_move(grp, tmp, ndl);
 500         }
 501
 502         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 503                                      &tmp->grp_trans_list, LST_TRANS_SESEND,
 504                                      tmp, lstcon_sesrpc_condition, &trans);
 505         if (rc != 0) {
 506                 CERROR("Can't create transaction: %d\n", rc);
 507                 goto error;
 508         }
 509
 510         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 511
 512         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 513
 514         lstcon_rpc_trans_destroy(trans);
 515         /* release nodes anyway, because we can't rollback status */
 516         lstcon_group_put(tmp);
 517
 518         return rc;
 519 error:
 520         lstcon_group_move(tmp, grp);
 521         lstcon_group_put(tmp);
 522
 523         return rc;
 524 }
 525
 526 int
 527 lstcon_group_add(char *name)
 528 {
 529         lstcon_group_t *grp;
 530         int             rc;
 531
 532         rc = (lstcon_group_find(name, &grp) == 0)? -EEXIST: 0;
 533         if (rc != 0) {
 534                 /* find a group with same name */
 535                 lstcon_group_put(grp);
 536                 return rc;
 537         }
 538
 539         rc = lstcon_group_alloc(name, &grp);
 540         if (rc != 0) {
 541                 CERROR("Can't allocate descriptor for group %s\n", name);
 542                 return -ENOMEM;
 543         }
 544
 545         list_add_tail(&grp->grp_link, &console_session.ses_grp_list);
 546
 547         return rc;
 548 }
 549
 550 int
 551 lstcon_nodes_add(char *name, int count,
 552                  lnet_process_id_t *ids_up, struct list_head *result_up)
 553 {
 554         lstcon_group_t         *grp;
 555         int                     rc;
 556
 557         LASSERT (count > 0);
 558         LASSERT (ids_up != NULL);
 559
 560         rc = lstcon_group_find(name, &grp);
 561         if (rc != 0) {
 562                 CDEBUG(D_NET, "Can't find group %s\n", name);
 563                 return rc;
 564         }
 565
 566         if (grp->grp_ref > 2) {
 567                 /* referred by other threads or test */
 568                 CDEBUG(D_NET, "Group %s is busy\n", name);
 569                 lstcon_group_put(grp);
 570
 571                 return -EBUSY;
 572         }
 573
 574         rc = lstcon_group_nodes_add(grp, count, ids_up, result_up);
 575
 576         lstcon_group_put(grp);
 577
 578         return rc;
 579 }
 580
 581 int
 582 lstcon_group_del(char *name)
 583 {
 584         lstcon_rpc_trans_t *trans;
 585         lstcon_group_t     *grp;
 586         int                 rc;
 587
 588         rc = lstcon_group_find(name, &grp);
 589         if (rc != 0) {
 590                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 591                 return rc;
 592         }
 593
 594         if (grp->grp_ref > 2) {
 595                 /* referred by others threads or test */
 596                 CDEBUG(D_NET, "Group %s is busy\n", name);
 597                 lstcon_group_put(grp);
 598                 return -EBUSY;
 599         }
 600
 601         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 602                                      &grp->grp_trans_list, LST_TRANS_SESEND,
 603                                      grp, lstcon_sesrpc_condition, &trans);
 604         if (rc != 0) {
 605                 CERROR("Can't create transaction: %d\n", rc);
 606                 lstcon_group_put(grp);
 607                 return rc;
 608         }
 609
 610         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 611
 612         lstcon_rpc_trans_destroy(trans);
 613
 614         lstcon_group_put(grp);
 615         /* -ref for session, it's destroyed,
 616          * status can't be rolled back, destroy group anway */
 617         lstcon_group_put(grp);
 618
 619         return rc;
 620 }
 621
 622 int
 623 lstcon_group_clean(char *name, int args)
 624 {
 625         lstcon_group_t *grp = NULL;
 626         int             rc;
 627
 628         rc = lstcon_group_find(name, &grp);
 629         if (rc != 0) {
 630                 CDEBUG(D_NET, "Can't find group %s\n", name);
 631                 return rc;
 632         }
 633
 634         if (grp->grp_ref > 2) {
 635                 /* referred by test */
 636                 CDEBUG(D_NET, "Group %s is busy\n", name);
 637                 lstcon_group_put(grp);
 638                 return -EBUSY;
 639         }
 640
 641         args = (LST_NODE_ACTIVE | LST_NODE_BUSY |
 642                 LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args;
 643
 644         lstcon_group_drain(grp, args);
 645
 646         lstcon_group_put(grp);
 647         /* release empty group */
 648         if (list_empty(&grp->grp_ndl_list))
 649                 lstcon_group_put(grp);
 650
 651         return 0;
 652 }
 653
 654 int
 655 lstcon_nodes_remove(char *name, int count,
 656                     lnet_process_id_t *ids_up, struct list_head *result_up)
 657 {
 658         lstcon_group_t *grp = NULL;
 659         int             rc;
 660
 661         rc = lstcon_group_find(name, &grp);
 662         if (rc != 0) {
 663                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 664                 return rc;
 665         }
 666
 667         if (grp->grp_ref > 2) {
 668                 /* referred by test */
 669                 CDEBUG(D_NET, "Group %s is busy\n", name);
 670                 lstcon_group_put(grp);
 671                 return -EBUSY;
 672         }
 673
 674         rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up);
 675
 676         lstcon_group_put(grp);
 677         /* release empty group */
 678         if (list_empty(&grp->grp_ndl_list))
 679                 lstcon_group_put(grp);
 680
 681         return rc;
 682 }
 683
 684 int
 685 lstcon_group_refresh(char *name, struct list_head *result_up)
 686 {
 687         lstcon_rpc_trans_t      *trans;
 688         lstcon_group_t          *grp;
 689         int                      rc;
 690
 691         rc = lstcon_group_find(name, &grp);
 692         if (rc != 0) {
 693                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 694                 return rc;
 695         }
 696
 697         if (grp->grp_ref > 2) {
 698                 /* referred by test */
 699                 CDEBUG(D_NET, "Group %s is busy\n", name);
 700                 lstcon_group_put(grp);
 701                 return -EBUSY;
 702         }
 703
 704         /* re-invite all inactive nodes int the group */
 705         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 706                                      &grp->grp_trans_list, LST_TRANS_SESNEW,
 707                                      grp, lstcon_sesrpc_condition, &trans);
 708         if (rc != 0) {
 709                 /* local error, return */
 710                 CDEBUG(D_NET, "Can't create transaction: %d\n", rc);
 711                 lstcon_group_put(grp);
 712                 return rc;
 713         }
 714
 715         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 716
 717         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 718
 719         lstcon_rpc_trans_destroy(trans);
 720         /* -ref for me */
 721         lstcon_group_put(grp);
 722
 723         return rc;
 724 }
 725
 726 int
 727 lstcon_group_list(int index, int len, char *name_up)
 728 {
 729         lstcon_group_t *grp;
 730
 731         LASSERT (index >= 0);
 732         LASSERT (name_up != NULL);
 733
 734         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 735                 if (index-- == 0) {
 736                         return copy_to_user(name_up, grp->grp_name, len) ?
 737                                -EFAULT : 0;
 738                 }
 739         }
 740
 741         return -ENOENT;
 742 }
 743
 744 static int
 745 lstcon_nodes_getent(struct list_head *head, int *index_p,
 746                     int *count_p, lstcon_node_ent_t *dents_up)
 747 {
 748         lstcon_ndlink_t  *ndl;
 749         lstcon_node_t    *nd;
 750         int               count = 0;
 751         int               index = 0;
 752
 753         LASSERT (index_p != NULL && count_p != NULL);
 754         LASSERT (dents_up != NULL);
 755         LASSERT (*index_p >= 0);
 756         LASSERT (*count_p > 0);
 757
 758         list_for_each_entry(ndl, head, ndl_link) {
 759                 if (index++ < *index_p)
 760                         continue;
 761
 762                 if (count >= *count_p)
 763                         break;
 764
 765                 nd = ndl->ndl_node;
 766                 if (copy_to_user(&dents_up[count].nde_id,
 767                                  &nd->nd_id, sizeof(nd->nd_id)) ||
 768                     copy_to_user(&dents_up[count].nde_state,
 769                                  &nd->nd_state, sizeof(nd->nd_state)))
 770                         return -EFAULT;
 771
 772                 count ++;
 773         }
 774
 775         if (index <= *index_p)
 776                 return -ENOENT;
 777
 778         *count_p = count;
 779         *index_p = index;
 780
 781         return 0;
 782 }
 783
 784 int
 785 lstcon_group_info(char *name, lstcon_ndlist_ent_t *gents_p,
 786                   int *index_p, int *count_p, lstcon_node_ent_t *dents_up)
 787 {
 788         lstcon_ndlist_ent_t *gentp;
 789         lstcon_group_t      *grp;
 790         lstcon_ndlink_t     *ndl;
 791         int                  rc;
 792
 793         rc = lstcon_group_find(name, &grp);
 794         if (rc != 0) {
 795                 CDEBUG(D_NET, "Can't find group %s\n", name);
 796                 return rc;
 797         }
 798
 799         if (dents_up != 0) {
 800                 /* verbose query */
 801                 rc = lstcon_nodes_getent(&grp->grp_ndl_list,
 802                                          index_p, count_p, dents_up);
 803                 lstcon_group_put(grp);
 804
 805                 return rc;
 806         }
 807
 808         /* non-verbose query */
 809         LIBCFS_ALLOC(gentp, sizeof(lstcon_ndlist_ent_t));
 810         if (gentp == NULL) {
 811                 CERROR("Can't allocate ndlist_ent\n");
 812                 lstcon_group_put(grp);
 813
 814                 return -ENOMEM;
 815         }
 816
 817         memset(gentp, 0, sizeof(lstcon_ndlist_ent_t));
 818
 819         list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link)
 820                 LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp);
 821
 822         rc = copy_to_user(gents_p, gentp,
 823                           sizeof(lstcon_ndlist_ent_t)) ? -EFAULT: 0;
 824
 825         LIBCFS_FREE(gentp, sizeof(lstcon_ndlist_ent_t));
 826
 827         lstcon_group_put(grp);
 828
 829         return 0;
 830 }
 831
 832 int
 833 lstcon_batch_find(char *name, lstcon_batch_t **batpp)
 834 {
 835         lstcon_batch_t   *bat;
 836
 837         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 838                 if (strncmp(bat->bat_name, name, LST_NAME_SIZE) == 0) {
 839                         *batpp = bat;
 840                         return 0;
 841                 }
 842         }
 843
 844         return -ENOENT;
 845 }
 846
 847 int
 848 lstcon_batch_add(char *name)
 849 {
 850         lstcon_batch_t   *bat;
 851         int               i;
 852         int               rc;
 853
 854         rc = (lstcon_batch_find(name, &bat) == 0)? -EEXIST: 0;
 855         if (rc != 0) {
 856                 CDEBUG(D_NET, "Batch %s already exists\n", name);
 857                 return rc;
 858         }
 859
 860         LIBCFS_ALLOC(bat, sizeof(lstcon_batch_t));
 861         if (bat == NULL) {
 862                 CERROR("Can't allocate descriptor for batch %s\n", name);
 863                 return -ENOMEM;
 864         }
 865
 866         LIBCFS_ALLOC(bat->bat_cli_hash,
 867                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 868         if (bat->bat_cli_hash == NULL) {
 869                 CERROR("Can't allocate hash for batch %s\n", name);
 870                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 871
 872                 return -ENOMEM;
 873         }
 874
 875         LIBCFS_ALLOC(bat->bat_srv_hash,
 876                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 877         if (bat->bat_srv_hash == NULL) {
 878                 CERROR("Can't allocate hash for batch %s\n", name);
 879                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 880                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 881
 882                 return -ENOMEM;
 883         }
 884
 885         strcpy(bat->bat_name, name);
 886         bat->bat_hdr.tsb_index = 0;
 887         bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie;
 888
 889         bat->bat_ntest = 0;
 890         bat->bat_state = LST_BATCH_IDLE;
 891
 892         CFS_INIT_LIST_HEAD(&bat->bat_cli_list);
 893         CFS_INIT_LIST_HEAD(&bat->bat_srv_list);
 894         CFS_INIT_LIST_HEAD(&bat->bat_test_list);
 895         CFS_INIT_LIST_HEAD(&bat->bat_trans_list);
 896
 897         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 898                 CFS_INIT_LIST_HEAD(&bat->bat_cli_hash[i]);
 899                 CFS_INIT_LIST_HEAD(&bat->bat_srv_hash[i]);
 900         }
 901
 902         list_add_tail(&bat->bat_link, &console_session.ses_bat_list);
 903
 904         return rc;
 905 }
 906
 907 int
 908 lstcon_batch_list(int index, int len, char *name_up)
 909 {
 910         lstcon_batch_t    *bat;
 911
 912         LASSERT (name_up != NULL);
 913         LASSERT (index >= 0);
 914
 915         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 916                 if (index-- == 0) {
 917                         return copy_to_user(name_up,bat->bat_name, len) ?
 918                                -EFAULT: 0;
 919                 }
 920         }
 921
 922         return -ENOENT;
 923 }
 924
 925 int
 926 lstcon_batch_info(char *name, lstcon_test_batch_ent_t *ent_up, int server,
 927                   int testidx, int *index_p, int *ndent_p,
 928                   lstcon_node_ent_t *dents_up)
 929 {
 930         lstcon_test_batch_ent_t *entp;
 931         struct list_head        *clilst;
 932         struct list_head        *srvlst;
 933         lstcon_test_t           *test = NULL;
 934         lstcon_batch_t          *bat;
 935         lstcon_ndlink_t         *ndl;
 936         int                      rc;
 937
 938         rc = lstcon_batch_find(name, &bat);
 939         if (rc != 0) {
 940                 CDEBUG(D_NET, "Can't find batch %s\n", name);
 941                 return -ENOENT;
 942         }
 943
 944         if (testidx > 0) {
 945                 /* query test, test index start from 1 */
 946                 list_for_each_entry(test, &bat->bat_test_list, tes_link) {
 947                         if (testidx-- == 1)
 948                                 break;
 949                 }
 950
 951                 if (testidx > 0) {
 952                         CDEBUG(D_NET, "Can't find specified test in batch\n");
 953                         return -ENOENT;
 954                 }
 955         }
 956
 957         clilst = (test == NULL) ? &bat->bat_cli_list :
 958                                   &test->tes_src_grp->grp_ndl_list;
 959         srvlst = (test == NULL) ? &bat->bat_srv_list :
 960                                   &test->tes_dst_grp->grp_ndl_list;
 961
 962         if (dents_up != NULL) {
 963                 rc = lstcon_nodes_getent((server ? srvlst: clilst),
 964                                          index_p, ndent_p, dents_up);
 965                 return rc;
 966         }
 967
 968         /* non-verbose query */
 969         LIBCFS_ALLOC(entp, sizeof(lstcon_test_batch_ent_t));
 970         if (entp == NULL)
 971                 return -ENOMEM;
 972
 973         memset(entp, 0, sizeof(lstcon_test_batch_ent_t));
 974
 975         if (test == NULL) {
 976                 entp->u.tbe_batch.bae_ntest = bat->bat_ntest;
 977                 entp->u.tbe_batch.bae_state = bat->bat_state;
 978
 979         } else {
 980
 981                 entp->u.tbe_test.tse_type   = test->tes_type;
 982                 entp->u.tbe_test.tse_loop   = test->tes_loop;
 983                 entp->u.tbe_test.tse_concur = test->tes_concur;
 984         }
 985
 986         list_for_each_entry(ndl, clilst, ndl_link)
 987                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle);
 988
 989         list_for_each_entry(ndl, srvlst, ndl_link)
 990                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle);
 991
 992         rc = copy_to_user(ent_up, entp,
 993                           sizeof(lstcon_test_batch_ent_t)) ? -EFAULT : 0;
 994
 995         LIBCFS_FREE(entp, sizeof(lstcon_test_batch_ent_t));
 996
 997         return rc;
 998 }
 999
1000 int
1001 lstcon_batrpc_condition(int transop, lstcon_node_t *nd, void *arg)
1002 {
1003         switch (transop) {
1004         case LST_TRANS_TSBRUN:
1005                 if (nd->nd_state != LST_NODE_ACTIVE)
1006                         return -ENETDOWN;
1007                 break;
1008
1009         case LST_TRANS_TSBSTOP:
1010                 if (nd->nd_state != LST_NODE_ACTIVE)
1011                         return 0;
1012                 break;
1013
1014         case LST_TRANS_TSBCLIQRY:
1015         case LST_TRANS_TSBSRVQRY:
1016                 break;
1017         }
1018
1019         return 1;
1020 }
1021
1022 static int
1023 lstcon_batch_op(lstcon_batch_t *bat, int transop, struct list_head *result_up)
1024 {
1025         lstcon_rpc_trans_t *trans;
1026         int                 rc;
1027
1028         rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list,
1029                                      &bat->bat_trans_list, transop,
1030                                      bat, lstcon_batrpc_condition, &trans);
1031         if (rc != 0) {
1032                 CERROR("Can't create transaction: %d\n", rc);
1033                 return rc;
1034         }
1035
1036         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1037
1038         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1039
1040         lstcon_rpc_trans_destroy(trans);
1041
1042         return rc;
1043 }
1044
1045 int
1046 lstcon_batch_run(char *name, int timeout, struct list_head *result_up)
1047 {
1048         lstcon_batch_t *bat;
1049         int             rc;
1050
1051         if (lstcon_batch_find(name, &bat) != 0) {
1052                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1053                 return -ENOENT;
1054         }
1055
1056         bat->bat_arg = timeout;
1057
1058         rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up);
1059
1060         /* mark batch as running if it's started in any node */
1061         if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0) != 0)
1062                 bat->bat_state = LST_BATCH_RUNNING;
1063
1064         return rc;
1065 }
1066
1067 int
1068 lstcon_batch_stop(char *name, int force, struct list_head *result_up)
1069 {
1070         lstcon_batch_t *bat;
1071         int             rc;
1072
1073         if (lstcon_batch_find(name, &bat) != 0) {
1074                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1075                 return -ENOENT;
1076         }
1077
1078         bat->bat_arg = force;
1079
1080         rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up);
1081
1082         /* mark batch as stopped if all RPCs finished */
1083         if (lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0) == 0)
1084                 bat->bat_state = LST_BATCH_IDLE;
1085
1086         return rc;
1087 }
1088
1089 static void
1090 lstcon_batch_destroy(lstcon_batch_t *bat)
1091 {
1092         lstcon_ndlink_t    *ndl;
1093         lstcon_test_t      *test;
1094         int                 i;
1095
1096         list_del(&bat->bat_link);
1097
1098         while (!list_empty(&bat->bat_test_list)) {
1099                 test = list_entry(bat->bat_test_list.next,
1100                                   lstcon_test_t, tes_link);
1101                 LASSERT (list_empty(&test->tes_trans_list));
1102
1103                 list_del(&test->tes_link);
1104
1105                 lstcon_group_put(test->tes_src_grp);
1106                 lstcon_group_put(test->tes_dst_grp);
1107
1108                 LIBCFS_FREE(test, offsetof(lstcon_test_t,
1109                                            tes_param[test->tes_paramlen]));
1110         }
1111
1112         LASSERT (list_empty(&bat->bat_trans_list));
1113
1114         while (!list_empty(&bat->bat_cli_list)) {
1115                 ndl = list_entry(bat->bat_cli_list.next,
1116                                  lstcon_ndlink_t, ndl_link);
1117                 list_del_init(&ndl->ndl_link);
1118
1119                 lstcon_ndlink_release(ndl);
1120         }
1121
1122         while (!list_empty(&bat->bat_srv_list)) {
1123                 ndl = list_entry(bat->bat_srv_list.next,
1124                                  lstcon_ndlink_t, ndl_link);
1125                 list_del_init(&ndl->ndl_link);
1126
1127                 lstcon_ndlink_release(ndl);
1128         }
1129
1130         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1131                 LASSERT (list_empty(&bat->bat_cli_hash[i]));
1132                 LASSERT (list_empty(&bat->bat_srv_hash[i]));
1133         }
1134
1135         LIBCFS_FREE(bat->bat_cli_hash,
1136                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1137         LIBCFS_FREE(bat->bat_srv_hash,
1138                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1139         LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
1140 }
1141
1142 int
1143 lstcon_testrpc_condition(int transop, lstcon_node_t *nd, void *arg)
1144 {
1145         lstcon_test_t    *test;
1146         lstcon_batch_t   *batch;
1147         lstcon_ndlink_t  *ndl;
1148         struct list_head *hash;
1149         struct list_head *head;
1150
1151         test = (lstcon_test_t *)arg;
1152         LASSERT (test != NULL);
1153
1154         batch = test->tes_batch;
1155         LASSERT (batch != NULL);
1156
1157         if (test->tes_oneside &&
1158             transop == LST_TRANS_TSBSRVADD)
1159                 return 0;
1160
1161         if (nd->nd_state != LST_NODE_ACTIVE)
1162                 return -ENETDOWN;
1163
1164         if (transop == LST_TRANS_TSBCLIADD) {
1165                 hash = batch->bat_cli_hash;
1166                 head = &batch->bat_cli_list;
1167
1168         } else {
1169                 LASSERT (transop == LST_TRANS_TSBSRVADD);
1170
1171                 hash = batch->bat_srv_hash;
1172                 head = &batch->bat_srv_list;
1173         }
1174
1175         LASSERT (nd->nd_id.nid != LNET_NID_ANY);
1176
1177         if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1) != 0)
1178                 return -ENOMEM;
1179
1180         if (list_empty(&ndl->ndl_link))
1181                 list_add_tail(&ndl->ndl_link, head);
1182
1183         return 1;
1184 }
1185
1186 static int
1187 lstcon_test_nodes_add(lstcon_test_t *test, struct list_head *result_up)
1188 {
1189         lstcon_rpc_trans_t     *trans;
1190         lstcon_group_t         *grp;
1191         int                     transop;
1192         int                     rc;
1193
1194         LASSERT (test->tes_src_grp != NULL);
1195         LASSERT (test->tes_dst_grp != NULL);
1196
1197         transop = LST_TRANS_TSBSRVADD;
1198         grp  = test->tes_dst_grp;
1199 again:
1200         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
1201                                      &test->tes_trans_list, transop,
1202                                      test, lstcon_testrpc_condition, &trans);
1203         if (rc != 0) {
1204                 CERROR("Can't create transaction: %d\n", rc);
1205                 return rc;
1206         }
1207
1208         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1209
1210         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1211             lstcon_trans_stat()->trs_fwk_errno != 0) {
1212                 lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1213
1214                 lstcon_rpc_trans_destroy(trans);
1215                 /* return if any error */
1216                 CDEBUG(D_NET, "Failed to add test %s, "
1217                               "RPC error %d, framework error %d\n",
1218                        transop == LST_TRANS_TSBCLIADD ? "client" : "server",
1219                        lstcon_trans_stat()->trs_rpc_errno,
1220                        lstcon_trans_stat()->trs_fwk_errno);
1221
1222                 return rc;
1223         }
1224
1225         lstcon_rpc_trans_destroy(trans);
1226
1227         if (transop == LST_TRANS_TSBCLIADD)
1228                 return rc;
1229
1230         transop = LST_TRANS_TSBCLIADD;
1231         grp = test->tes_src_grp;
1232         test->tes_cliidx = 0;
1233
1234         /* requests to test clients */
1235         goto again;
1236 }
1237
1238 int
1239 lstcon_test_add(char *name, int type, int loop, int concur,
1240                 int dist, int span, char *src_name, char * dst_name,
1241                 void *param, int paramlen, int *retp, struct list_head *result_up)
1242
1243 {
1244         lstcon_group_t  *src_grp = NULL;
1245         lstcon_group_t  *dst_grp = NULL;
1246         lstcon_test_t   *test    = NULL;
1247         lstcon_batch_t  *batch;
1248         int              rc;
1249
1250         rc = lstcon_batch_find(name, &batch);
1251         if (rc != 0) {
1252                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1253                 return rc;
1254         }
1255
1256         if (batch->bat_state != LST_BATCH_IDLE) {
1257                 CDEBUG(D_NET, "Can't change running batch %s\n", name);
1258                 return rc;
1259         }
1260
1261         rc = lstcon_group_find(src_name, &src_grp);
1262         if (rc != 0) {
1263                 CDEBUG(D_NET, "Can't find group %s\n", src_name);
1264                 goto out;
1265         }
1266
1267         rc = lstcon_group_find(dst_name, &dst_grp);
1268         if (rc != 0) {
1269                 CDEBUG(D_NET, "Can't find group %s\n", dst_name);
1270                 goto out;
1271         }
1272
1273         if (dst_grp->grp_userland)
1274                 *retp = 1;
1275
1276         LIBCFS_ALLOC(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1277         if (!test) {
1278                 CERROR("Can't allocate test descriptor\n");
1279                 rc = -ENOMEM;
1280
1281                 goto out;
1282         }
1283
1284         memset(test, 0, offsetof(lstcon_test_t, tes_param[paramlen]));
1285         test->tes_hdr.tsb_id    = batch->bat_hdr.tsb_id;
1286         test->tes_batch         = batch;
1287         test->tes_type          = type;
1288         test->tes_oneside       = 0; /* TODO */
1289         test->tes_loop          = loop;
1290         test->tes_concur        = concur;
1291         test->tes_stop_onerr    = 1; /* TODO */
1292         test->tes_span          = span;
1293         test->tes_dist          = dist;
1294         test->tes_cliidx        = 0; /* just used for creating RPC */
1295         test->tes_src_grp       = src_grp;
1296         test->tes_dst_grp       = dst_grp;
1297         CFS_INIT_LIST_HEAD(&test->tes_trans_list);
1298
1299         if (param != NULL) {
1300                 test->tes_paramlen = paramlen;
1301                 memcpy(&test->tes_param[0], param, paramlen);
1302         }
1303
1304         rc = lstcon_test_nodes_add(test, result_up);
1305
1306         if (rc != 0)
1307                 goto out;
1308
1309         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1310             lstcon_trans_stat()->trs_fwk_errno != 0)
1311                 CDEBUG(D_NET, "Failed to add test %d to batch %s\n", type, name);
1312
1313         /* add to test list anyway, so user can check what's going on */
1314         list_add_tail(&test->tes_link, &batch->bat_test_list);
1315
1316         batch->bat_ntest ++;
1317         test->tes_hdr.tsb_index = batch->bat_ntest;
1318
1319         /*  hold groups so nobody can change them */
1320         return rc;
1321 out:
1322         if (test != NULL)
1323                 LIBCFS_FREE(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1324
1325         if (dst_grp != NULL)
1326                 lstcon_group_put(dst_grp);
1327
1328         if (src_grp != NULL)
1329                 lstcon_group_put(src_grp);
1330
1331         return rc;
1332 }
1333
1334 int
1335 lstcon_test_find(lstcon_batch_t *batch, int idx, lstcon_test_t **testpp)
1336 {
1337         lstcon_test_t *test;
1338
1339         list_for_each_entry(test, &batch->bat_test_list, tes_link) {
1340                 if (idx == test->tes_hdr.tsb_index) {
1341                         *testpp = test;
1342                         return 0;
1343                 }
1344         }
1345
1346         return -ENOENT;
1347 }
1348
1349 int
1350 lstcon_tsbrpc_readent(int transop, srpc_msg_t *msg,
1351                       lstcon_rpc_ent_t *ent_up)
1352 {
1353         srpc_batch_reply_t *rep = &msg->msg_body.bat_reply;
1354
1355         LASSERT (transop == LST_TRANS_TSBCLIQRY ||
1356                  transop == LST_TRANS_TSBSRVQRY);
1357
1358         /* positive errno, framework error code */
1359         if (copy_to_user(&ent_up->rpe_priv[0],
1360                          &rep->bar_active, sizeof(rep->bar_active)))
1361                 return -EFAULT;
1362
1363         return 0;
1364 }
1365
1366 int
1367 lstcon_test_batch_query(char *name, int testidx, int client,
1368                         int timeout, struct list_head *result_up)
1369 {
1370         lstcon_rpc_trans_t *trans;
1371         struct list_head   *translist;
1372         struct list_head   *ndlist;
1373         lstcon_tsb_hdr_t   *hdr;
1374         lstcon_batch_t     *batch;
1375         lstcon_test_t      *test = NULL;
1376         int                 transop;
1377         int                 rc;
1378
1379         rc = lstcon_batch_find(name, &batch);
1380         if (rc != 0) {
1381                 CDEBUG(D_NET, "Can't find batch: %s\n", name);
1382                 return rc;
1383         }
1384
1385         if (testidx == 0) {
1386                 translist = &batch->bat_trans_list;
1387                 ndlist    = &batch->bat_cli_list;
1388                 hdr       = &batch->bat_hdr;
1389
1390         } else {
1391                 /* query specified test only */
1392                 rc = lstcon_test_find(batch, testidx, &test);
1393                 if (rc != 0) {
1394                         CDEBUG(D_NET, "Can't find test: %d\n", testidx);
1395                         return rc;
1396                 }
1397
1398                 translist = &test->tes_trans_list;
1399                 ndlist    = &test->tes_src_grp->grp_ndl_list;
1400                 hdr       = &test->tes_hdr;
1401         }
1402
1403         transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY;
1404
1405         rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr,
1406                                      lstcon_batrpc_condition, &trans);
1407         if (rc != 0) {
1408                 CERROR("Can't create transaction: %d\n", rc);
1409                 return rc;
1410         }
1411
1412         lstcon_rpc_trans_postwait(trans, timeout);
1413
1414         if (testidx == 0 && /* query a batch, not a test */
1415             lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) == 0 &&
1416             lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0) == 0) {
1417                 /* all RPCs finished, and no active test */
1418                 batch->bat_state = LST_BATCH_IDLE;
1419         }
1420
1421         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1422                                           lstcon_tsbrpc_readent);
1423         lstcon_rpc_trans_destroy(trans);
1424
1425         return rc;
1426 }
1427
1428 int
1429 lstcon_statrpc_readent(int transop, srpc_msg_t *msg,
1430                        lstcon_rpc_ent_t *ent_up)
1431 {
1432         srpc_stat_reply_t *rep = &msg->msg_body.stat_reply;
1433         sfw_counters_t    *sfwk_stat;
1434         srpc_counters_t   *srpc_stat;
1435         lnet_counters_t   *lnet_stat;
1436
1437         if (rep->str_status != 0)
1438                 return 0;
1439
1440         sfwk_stat = (sfw_counters_t *)&ent_up->rpe_payload[0];
1441         srpc_stat = (srpc_counters_t *)((char *)sfwk_stat + sizeof(*sfwk_stat));
1442         lnet_stat = (lnet_counters_t *)((char *)srpc_stat + sizeof(*srpc_stat));
1443
1444         if (copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) ||
1445             copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) ||
1446             copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat)))
1447                 return -EFAULT;
1448
1449         return 0;
1450 }
1451
1452 int
1453 lstcon_ndlist_stat(struct list_head *ndlist,
1454                    int timeout, struct list_head *result_up)
1455 {
1456         struct list_head    head;
1457         lstcon_rpc_trans_t *trans;
1458         int                 rc;
1459
1460         CFS_INIT_LIST_HEAD(&head);
1461
1462         rc = lstcon_rpc_trans_ndlist(ndlist, &head,
1463                                      LST_TRANS_STATQRY, NULL, NULL, &trans);
1464         if (rc != 0) {
1465                 CERROR("Can't create transaction: %d\n", rc);
1466                 return rc;
1467         }
1468
1469         timeout = (timeout > LST_TRANS_MIN_TIMEOUT) ? timeout :
1470                                                       LST_TRANS_MIN_TIMEOUT;
1471         lstcon_rpc_trans_postwait(trans, timeout);
1472
1473         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1474                                           lstcon_statrpc_readent);
1475         lstcon_rpc_trans_destroy(trans);
1476
1477         return rc;
1478 }
1479
1480 int
1481 lstcon_group_stat(char *grp_name, int timeout, struct list_head *result_up)
1482 {
1483         lstcon_group_t     *grp;
1484         int                 rc;
1485
1486         rc = lstcon_group_find(grp_name, &grp);
1487         if (rc != 0) {
1488                 CDEBUG(D_NET, "Can't find group %s\n", grp_name);
1489                 return rc;
1490         }
1491
1492         rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up);
1493
1494         lstcon_group_put(grp);
1495
1496         return rc;
1497 }
1498
1499 int
1500 lstcon_nodes_stat(int count, lnet_process_id_t *ids_up,
1501                   int timeout, struct list_head *result_up)
1502 {
1503         lstcon_ndlink_t         *ndl;
1504         lstcon_group_t          *tmp;
1505         lnet_process_id_t        id;
1506         int                      i;
1507         int                      rc;
1508
1509         rc = lstcon_group_alloc(NULL, &tmp);
1510         if (rc != 0) {
1511                 CERROR("Out of memory\n");
1512                 return -ENOMEM;
1513         }
1514
1515         for (i = 0 ; i < count; i++) {
1516                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1517                         rc = -EFAULT;
1518                         break;
1519                 }
1520
1521                 /* add to tmp group */
1522                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2);
1523                 if (rc != 0) {
1524                         CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET,
1525                                "Failed to find or create %s: %d\n",
1526                                libcfs_id2str(id), rc);
1527                         break;
1528                 }
1529         }
1530
1531         if (rc != 0) {
1532                 lstcon_group_put(tmp);
1533                 return rc;
1534         }
1535
1536         rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up);
1537
1538         lstcon_group_put(tmp);
1539
1540         return rc;
1541 }
1542
1543 int
1544 lstcon_debug_ndlist(struct list_head *ndlist,
1545                     struct list_head *translist,
1546                     int timeout, struct list_head *result_up)
1547 {
1548         lstcon_rpc_trans_t *trans;
1549         int                 rc;
1550
1551         rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY,
1552                                      NULL, lstcon_sesrpc_condition, &trans);
1553         if (rc != 0) {
1554                 CERROR("Can't create transaction: %d\n", rc);
1555                 return rc;
1556         }
1557
1558         timeout = (timeout > LST_TRANS_MIN_TIMEOUT) ? timeout :
1559                                                       LST_TRANS_MIN_TIMEOUT;
1560
1561         lstcon_rpc_trans_postwait(trans, timeout);
1562
1563         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1564                                           lstcon_sesrpc_readent);
1565         lstcon_rpc_trans_destroy(trans);
1566
1567         return rc;
1568 }
1569
1570 int
1571 lstcon_session_debug(int timeout, struct list_head *result_up)
1572 {
1573         return lstcon_debug_ndlist(&console_session.ses_ndl_list,
1574                                    NULL, timeout, result_up);
1575 }
1576
1577 int
1578 lstcon_batch_debug(int timeout, char *name,
1579                    int client, struct list_head *result_up)
1580 {
1581         lstcon_batch_t *bat;
1582         int             rc;
1583
1584         rc = lstcon_batch_find(name, &bat);
1585         if (rc != 0)
1586                 return -ENOENT;
1587
1588         rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list :
1589                                           &bat->bat_srv_list,
1590                                  NULL, timeout, result_up);
1591
1592         return rc;
1593 }
1594
1595 int
1596 lstcon_group_debug(int timeout, char *name,
1597                    struct list_head *result_up)
1598 {
1599         lstcon_group_t *grp;
1600         int             rc;
1601
1602         rc = lstcon_group_find(name, &grp);
1603         if (rc != 0)
1604                 return -ENOENT;
1605
1606         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1607                                  timeout, result_up);
1608         lstcon_group_put(grp);
1609
1610         return rc;
1611 }
1612
1613 int
1614 lstcon_nodes_debug(int timeout,
1615                    int count, lnet_process_id_t *ids_up,
1616                    struct list_head *result_up)
1617 {
1618         lnet_process_id_t  id;
1619         lstcon_ndlink_t   *ndl;
1620         lstcon_group_t    *grp;
1621         int                i;
1622         int                rc;
1623
1624         rc = lstcon_group_alloc(NULL, &grp);
1625         if (rc != 0) {
1626                 CDEBUG(D_NET, "Out of memory\n");
1627                 return rc;
1628         }
1629
1630         for (i = 0; i < count; i++) {
1631                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1632                         rc = -EFAULT;
1633                         break;
1634                 }
1635
1636                 /* node is added to tmp group */
1637                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 1);
1638                 if (rc != 0) {
1639                         CERROR("Can't create node link\n");
1640                         break;
1641                 }
1642         }
1643
1644         if (rc != 0) {
1645                 lstcon_group_put(grp);
1646                 return rc;
1647         }
1648
1649         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1650                                  timeout, result_up);
1651
1652         lstcon_group_put(grp);
1653
1654         return rc;
1655 }
1656
1657 int
1658 lstcon_session_match(lst_sid_t sid)
1659 {
1660         return (console_session.ses_id.ses_nid   == sid.ses_nid &&
1661                 console_session.ses_id.ses_stamp == sid.ses_stamp) ?  1: 0;
1662 }
1663
1664 static void
1665 lstcon_new_session_id(lst_sid_t *sid)
1666 {
1667         lnet_process_id_t      id;
1668
1669         LASSERT (console_session.ses_state == LST_SESSION_NONE);
1670
1671         LNetGetId(1, &id);
1672         sid->ses_nid   = id.nid;
1673         sid->ses_stamp = cfs_time_current();
1674 }
1675
1676 extern srpc_service_t lstcon_acceptor_service;
1677
1678 int
1679 lstcon_session_new(char *name, int key,
1680                    int timeout,int force, lst_sid_t *sid_up)
1681 {
1682         int     rc = 0;
1683         int     i;
1684
1685         if (console_session.ses_state != LST_SESSION_NONE) {
1686                 /* session exists */
1687                 if (!force) {
1688                         CERROR("Session %s already exists\n",
1689                                console_session.ses_name);
1690                         return -EEXIST;
1691                 }
1692
1693                 rc = lstcon_session_end();
1694
1695                 /* lstcon_session_end() only return local error */
1696                 if  (rc != 0)
1697                         return rc;
1698         }
1699
1700         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++) {
1701                 LASSERT (list_empty(&console_session.ses_ndl_hash[i]));
1702         }
1703
1704         rc = lstcon_batch_add(LST_DEFAULT_BATCH);
1705         if (rc != 0)
1706                 return rc;
1707
1708         rc = lstcon_rpc_pinger_start();
1709         if (rc != 0) {
1710                 lstcon_batch_t *bat;
1711
1712                 lstcon_batch_find(LST_DEFAULT_BATCH, &bat);
1713                 lstcon_batch_destroy(bat);
1714
1715                 return rc;
1716         }
1717
1718         lstcon_new_session_id(&console_session.ses_id);
1719
1720         console_session.ses_key     = key;
1721         console_session.ses_state   = LST_SESSION_ACTIVE;
1722         console_session.ses_force   = !!force;
1723         console_session.ses_timeout = (timeout <= 0)? LST_CONSOLE_TIMEOUT:
1724                                                       timeout;
1725         strcpy(console_session.ses_name, name);
1726
1727         if (copy_to_user(sid_up, &console_session.ses_id,
1728                          sizeof(lst_sid_t)) == 0)
1729                 return rc;
1730
1731         lstcon_session_end();
1732
1733         return -EFAULT;
1734 }
1735
1736 int
1737 lstcon_session_info(lst_sid_t *sid_up, int *key_up,
1738                     lstcon_ndlist_ent_t *ndinfo_up, char *name_up, int len)
1739 {
1740         lstcon_ndlist_ent_t *entp;
1741         lstcon_ndlink_t     *ndl;
1742         int                  rc = 0;
1743
1744         if (console_session.ses_state != LST_SESSION_ACTIVE)
1745                 return -ESRCH;
1746
1747         LIBCFS_ALLOC(entp, sizeof(*entp));
1748         if (entp == NULL)
1749                 return -ENOMEM;
1750
1751         memset(entp, 0, sizeof(*entp));
1752
1753         list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link)
1754                 LST_NODE_STATE_COUNTER(ndl->ndl_node, entp);
1755
1756         if (copy_to_user(sid_up, &console_session.ses_id, sizeof(lst_sid_t)) ||
1757             copy_to_user(key_up, &console_session.ses_key, sizeof(int)) ||
1758             copy_to_user(ndinfo_up, entp, sizeof(*entp)) ||
1759             copy_to_user(name_up, console_session.ses_name, len))
1760                 rc = -EFAULT;
1761
1762         LIBCFS_FREE(entp, sizeof(*entp));
1763
1764         return rc;
1765 }
1766
1767 int
1768 lstcon_session_end()
1769 {
1770         lstcon_rpc_trans_t *trans;
1771         lstcon_group_t     *grp;
1772         lstcon_batch_t     *bat;
1773         int                 rc = 0;
1774
1775         LASSERT (console_session.ses_state == LST_SESSION_ACTIVE);
1776
1777         rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list,
1778                                      NULL, LST_TRANS_SESEND, NULL,
1779                                      lstcon_sesrpc_condition, &trans);
1780         if (rc != 0) {
1781                 CERROR("Can't create transaction: %d\n", rc);
1782                 return rc;
1783         }
1784
1785         console_session.ses_shutdown = 1;
1786
1787         lstcon_rpc_pinger_stop();
1788
1789         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1790
1791         lstcon_rpc_trans_destroy(trans);
1792         /* User can do nothing even rpc failed, so go on */
1793
1794         /* waiting for orphan rpcs to die */
1795         lstcon_rpc_cleanup_wait();
1796
1797         console_session.ses_id    = LST_INVALID_SID;
1798         console_session.ses_state = LST_SESSION_NONE;
1799         console_session.ses_key   = 0;
1800         console_session.ses_force = 0;
1801
1802         /* destroy all batches */
1803         while (!list_empty(&console_session.ses_bat_list)) {
1804                 bat = list_entry(console_session.ses_bat_list.next,
1805                                  lstcon_batch_t, bat_link);
1806
1807                 lstcon_batch_destroy(bat);
1808         }
1809
1810         /* destroy all groups */
1811         while (!list_empty(&console_session.ses_grp_list)) {
1812                 grp = list_entry(console_session.ses_grp_list.next,
1813                                  lstcon_group_t, grp_link);
1814                 LASSERT (grp->grp_ref == 1);
1815
1816                 lstcon_group_put(grp);
1817         }
1818
1819         /* all nodes should be released */
1820         LASSERT (list_empty(&console_session.ses_ndl_list));
1821
1822         console_session.ses_shutdown = 0;
1823         console_session.ses_expired  = 0;
1824
1825         return rc;
1826 }
1827
1828 static int
1829 lstcon_acceptor_handle (srpc_server_rpc_t *rpc)
1830 {
1831         srpc_msg_t        *rep  = &rpc->srpc_replymsg;
1832         srpc_msg_t        *req  = &rpc->srpc_reqstbuf->buf_msg;
1833         srpc_join_reqst_t *jreq = &req->msg_body.join_reqst;
1834         srpc_join_reply_t *jrep = &rep->msg_body.join_reply;
1835         lstcon_group_t    *grp  = NULL;
1836         lstcon_ndlink_t   *ndl;
1837         int                rc   = 0;
1838
1839         sfw_unpack_message(req);
1840
1841         mutex_down(&console_session.ses_mutex);
1842
1843         jrep->join_sid = console_session.ses_id;
1844
1845         if (console_session.ses_id.ses_nid == LNET_NID_ANY) {
1846                 jrep->join_status = ESRCH;
1847                 goto out;
1848         }
1849
1850         if (jreq->join_sid.ses_nid != LNET_NID_ANY &&
1851              !lstcon_session_match(jreq->join_sid)) {
1852                 jrep->join_status = EBUSY;
1853                 goto out;
1854         }
1855
1856         if (lstcon_group_find(jreq->join_group, &grp) != 0) {
1857                 rc = lstcon_group_alloc(jreq->join_group, &grp);
1858                 if (rc != 0) {
1859                         CERROR("Out of memory\n");
1860                         goto out;
1861                 }
1862
1863                 list_add_tail(&grp->grp_link,
1864                               &console_session.ses_grp_list);
1865                 lstcon_group_addref(grp);
1866         }
1867
1868         if (grp->grp_ref > 2) {
1869                 /* Group in using */
1870                 jrep->join_status = EBUSY;
1871                 goto out;
1872         }
1873
1874         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0);
1875         if (rc == 0) {
1876                 jrep->join_status = EEXIST;
1877                 goto out;
1878         }
1879
1880         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1);
1881         if (rc != 0) {
1882                 CERROR("Out of memory\n");
1883                 goto out;
1884         }
1885
1886         ndl->ndl_node->nd_state   = LST_NODE_ACTIVE;
1887         ndl->ndl_node->nd_timeout = console_session.ses_timeout;
1888
1889         if (grp->grp_userland == 0)
1890                 grp->grp_userland = 1;
1891
1892         strcpy(jrep->join_session, console_session.ses_name);
1893         jrep->join_timeout = console_session.ses_timeout;
1894         jrep->join_status  = 0;
1895
1896 out:
1897         if (grp != NULL)
1898                 lstcon_group_put(grp);
1899
1900         mutex_up(&console_session.ses_mutex);
1901
1902         return rc;
1903 }
1904
1905 srpc_service_t lstcon_acceptor_service =
1906 {
1907         .sv_name        = "join session",
1908         .sv_handler     = lstcon_acceptor_handle,
1909         .sv_bulk_ready  = NULL,
1910         .sv_id          = SRPC_SERVICE_JOIN,
1911         .sv_concur      = SFW_SERVICE_CONCURRENCY,
1912 };
1913
1914 extern int lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_data *data);
1915
1916 DECLARE_IOCTL_HANDLER(lstcon_ioctl_handler, lstcon_ioctl_entry);
1917
1918 /* initialize console */
1919 int
1920 lstcon_console_init(void)
1921 {
1922         int     i;
1923         int     n;
1924         int     rc;
1925
1926         memset(&console_session, 0, sizeof(lstcon_session_t));
1927
1928         console_session.ses_id      = LST_INVALID_SID;
1929         console_session.ses_state   = LST_SESSION_NONE;
1930         console_session.ses_timeout = 0;
1931         console_session.ses_force   = 0;
1932         console_session.ses_expired = 0;
1933         console_session.ses_laststamp = cfs_time_current_sec();
1934
1935         init_mutex(&console_session.ses_mutex);
1936
1937         CFS_INIT_LIST_HEAD(&console_session.ses_ndl_list);
1938         CFS_INIT_LIST_HEAD(&console_session.ses_grp_list);
1939         CFS_INIT_LIST_HEAD(&console_session.ses_bat_list);
1940         CFS_INIT_LIST_HEAD(&console_session.ses_trans_list);
1941
1942         LIBCFS_ALLOC(console_session.ses_ndl_hash,
1943                      sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
1944         if (console_session.ses_ndl_hash == NULL)
1945                 return -ENOMEM;
1946
1947         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
1948                 CFS_INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]);
1949
1950         rc = srpc_add_service(&lstcon_acceptor_service);
1951         LASSERT (rc != -EBUSY);
1952         if (rc != 0) {
1953                 LIBCFS_FREE(console_session.ses_ndl_hash,
1954                             sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
1955                 return rc;
1956         }
1957
1958         n = srpc_service_add_buffers(&lstcon_acceptor_service, SFW_POST_BUFFERS);
1959         if (n != SFW_POST_BUFFERS) {
1960                 rc = -ENOMEM;
1961                 goto out;
1962         }
1963
1964         rc = libcfs_register_ioctl(&lstcon_ioctl_handler);
1965
1966         if (rc == 0) {
1967                 lstcon_rpc_module_init();
1968                 return 0;
1969         }
1970
1971 out:
1972         srpc_shutdown_service(&lstcon_acceptor_service);
1973         srpc_remove_service(&lstcon_acceptor_service);
1974
1975         LIBCFS_FREE(console_session.ses_ndl_hash,
1976                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
1977
1978         srpc_wait_service_shutdown(&lstcon_acceptor_service);
1979
1980         return rc;
1981 }
1982
1983 int
1984 lstcon_console_fini(void)
1985 {
1986         int     i;
1987
1988         mutex_down(&console_session.ses_mutex);
1989
1990         libcfs_deregister_ioctl(&lstcon_ioctl_handler);
1991
1992         srpc_shutdown_service(&lstcon_acceptor_service);
1993         srpc_remove_service(&lstcon_acceptor_service);
1994
1995         if (console_session.ses_state != LST_SESSION_NONE)
1996                 lstcon_session_end();
1997
1998         lstcon_rpc_module_fini();
1999
2000         mutex_up(&console_session.ses_mutex);
2001
2002         LASSERT (list_empty(&console_session.ses_ndl_list));
2003         LASSERT (list_empty(&console_session.ses_grp_list));
2004         LASSERT (list_empty(&console_session.ses_bat_list));
2005         LASSERT (list_empty(&console_session.ses_trans_list));
2006
2007         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
2008                 LASSERT (list_empty(&console_session.ses_ndl_hash[i]));
2009         }
2010
2011         LIBCFS_FREE(console_session.ses_ndl_hash,
2012                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2013
2014         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2015
2016         return 0;
2017 }
2018
2019 #endif