lnet/selftest/console.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * Author: Liang Zhen <liangzhen@clusterfs.com>
   5  *
   6  * This file is part of Lustre, http://www.lustre.org
   7  *
   8  * Infrastructure of LST console
   9  */
  10 #ifdef __KERNEL__
  11
  12 #include <libcfs/libcfs.h>
  13 #include <lnet/lib-lnet.h>
  14 #include "console.h"
  15 #include "conrpc.h"
  16
  17 #define LST_NODE_STATE_COUNTER(nd, p)                   \
  18 do {                                                    \
  19         if ((nd)->nd_state == LST_NODE_ACTIVE)          \
  20                 (p)->nle_nactive ++;                    \
  21         else if ((nd)->nd_state == LST_NODE_BUSY)       \
  22                 (p)->nle_nbusy ++;                      \
  23         else if ((nd)->nd_state == LST_NODE_DOWN)       \
  24                 (p)->nle_ndown ++;                      \
  25         else                                            \
  26                 (p)->nle_nunknown ++;                   \
  27         (p)->nle_nnode ++;                              \
  28 } while (0)
  29
  30 lstcon_session_t        console_session;
  31
  32 void
  33 lstcon_node_get(lstcon_node_t *nd)
  34 {
  35         LASSERT (nd->nd_ref >= 1);
  36
  37         nd->nd_ref++;
  38 }
  39
  40 static int
  41 lstcon_node_find(lnet_process_id_t id, lstcon_node_t **ndpp, int create)
  42 {
  43         lstcon_ndlink_t *ndl;
  44         unsigned int     idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE;
  45
  46         LASSERT (id.nid != LNET_NID_ANY);
  47
  48         list_for_each_entry(ndl, &console_session.ses_ndl_hash[idx], ndl_hlink) {
  49                 if (ndl->ndl_node->nd_id.nid != id.nid ||
  50                     ndl->ndl_node->nd_id.pid != id.pid)
  51                         continue;
  52
  53                 lstcon_node_get(ndl->ndl_node);
  54                 *ndpp = ndl->ndl_node;
  55                 return 0;
  56         }
  57
  58         if (!create)
  59                 return -ENOENT;
  60
  61         LIBCFS_ALLOC(*ndpp, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
  62         if (*ndpp == NULL)
  63                 return -ENOMEM;
  64
  65         ndl = (lstcon_ndlink_t *)(*ndpp + 1);
  66
  67         ndl->ndl_node = *ndpp;
  68
  69         ndl->ndl_node->nd_ref   = 1;
  70         ndl->ndl_node->nd_id    = id;
  71         ndl->ndl_node->nd_stamp = cfs_time_current();
  72         ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
  73         ndl->ndl_node->nd_timeout = 0;
  74         memset(&ndl->ndl_node->nd_ping, 0, sizeof(lstcon_rpc_t));
  75
  76         /* queued in global hash & list, no refcount is taken by
  77          * global hash & list, if caller release his refcount,
  78          * node will be released */
  79         list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]);
  80         list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list);
  81
  82         return 0;
  83 }
  84
  85 void
  86 lstcon_node_put(lstcon_node_t *nd)
  87 {
  88         lstcon_ndlink_t  *ndl;
  89
  90         LASSERT (nd->nd_ref > 0);
  91
  92         if (--nd->nd_ref > 0)
  93                 return;
  94
  95         ndl = (lstcon_ndlink_t *)(nd + 1);
  96
  97         LASSERT (!list_empty(&ndl->ndl_link));
  98         LASSERT (!list_empty(&ndl->ndl_hlink));
  99
 100         /* remove from session */
 101         list_del(&ndl->ndl_link);
 102         list_del(&ndl->ndl_hlink);
 103
 104         LIBCFS_FREE(nd, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
 105 }
 106
 107 static int
 108 lstcon_ndlink_find(struct list_head *hash,
 109                    lnet_process_id_t id, lstcon_ndlink_t **ndlpp, int create)
 110 {
 111         unsigned int     idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
 112         lstcon_ndlink_t *ndl;
 113         lstcon_node_t   *nd;
 114         int              rc;
 115
 116         if (id.nid == LNET_NID_ANY)
 117                 return -EINVAL;
 118
 119         /* search in hash */
 120         list_for_each_entry(ndl, &hash[idx], ndl_hlink) {
 121                 if (ndl->ndl_node->nd_id.nid != id.nid ||
 122                     ndl->ndl_node->nd_id.pid != id.pid)
 123                         continue;
 124
 125                 *ndlpp = ndl;
 126                 return 0;
 127         }
 128
 129         if (create == 0)
 130                 return -ENOENT;
 131
 132         /* find or create in session hash */
 133         rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0);
 134         if (rc != 0)
 135                 return rc;
 136
 137         LIBCFS_ALLOC(ndl, sizeof(lstcon_ndlink_t));
 138         if (ndl == NULL) {
 139                 lstcon_node_put(nd);
 140                 return -ENOMEM;
 141         }
 142
 143         *ndlpp = ndl;
 144
 145         ndl->ndl_node = nd;
 146         CFS_INIT_LIST_HEAD(&ndl->ndl_link);
 147         list_add_tail(&ndl->ndl_hlink, &hash[idx]);
 148
 149         return  0;
 150 }
 151
 152 static void
 153 lstcon_ndlink_release(lstcon_ndlink_t *ndl)
 154 {
 155         LASSERT (list_empty(&ndl->ndl_link));
 156         LASSERT (!list_empty(&ndl->ndl_hlink));
 157
 158         list_del(&ndl->ndl_hlink); /* delete from hash */
 159         lstcon_node_put(ndl->ndl_node);
 160
 161         LIBCFS_FREE(ndl, sizeof(*ndl));
 162 }
 163
 164 static int
 165 lstcon_group_alloc(char *name, lstcon_group_t **grpp)
 166 {
 167         lstcon_group_t *grp;
 168         int             i;
 169
 170         LIBCFS_ALLOC(grp, offsetof(lstcon_group_t,
 171                                    grp_ndl_hash[LST_NODE_HASHSIZE]));
 172         if (grp == NULL)
 173                 return -ENOMEM;
 174
 175         memset(grp, 0, offsetof(lstcon_group_t,
 176                                 grp_ndl_hash[LST_NODE_HASHSIZE]));
 177
 178         grp->grp_ref = 1;
 179         if (name != NULL)
 180                 strcpy(grp->grp_name, name);
 181
 182         CFS_INIT_LIST_HEAD(&grp->grp_link);
 183         CFS_INIT_LIST_HEAD(&grp->grp_ndl_list);
 184         CFS_INIT_LIST_HEAD(&grp->grp_trans_list);
 185
 186         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 187                 CFS_INIT_LIST_HEAD(&grp->grp_ndl_hash[i]);
 188
 189         *grpp = grp;
 190
 191         return 0;
 192 }
 193
 194 static void
 195 lstcon_group_addref(lstcon_group_t *grp)
 196 {
 197         grp->grp_ref ++;
 198 }
 199
 200 static void lstcon_group_ndlink_release(lstcon_group_t *, lstcon_ndlink_t *);
 201
 202 static void
 203 lstcon_group_drain(lstcon_group_t *grp, int keep)
 204 {
 205         lstcon_ndlink_t *ndl;
 206         lstcon_ndlink_t *tmp;
 207
 208         list_for_each_entry_safe(ndl, tmp, &grp->grp_ndl_list, ndl_link) {
 209                 if ((ndl->ndl_node->nd_state & keep) == 0)
 210                         lstcon_group_ndlink_release(grp, ndl);
 211         }
 212 }
 213
 214 static void
 215 lstcon_group_decref(lstcon_group_t *grp)
 216 {
 217         int     i;
 218
 219         if (--grp->grp_ref > 0)
 220                 return;
 221
 222         if (!list_empty(&grp->grp_link))
 223                 list_del(&grp->grp_link);
 224
 225         lstcon_group_drain(grp, 0);
 226
 227         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 228                 LASSERT (list_empty(&grp->grp_ndl_hash[i]));
 229         }
 230
 231         LIBCFS_FREE(grp, offsetof(lstcon_group_t,
 232                                   grp_ndl_hash[LST_NODE_HASHSIZE]));
 233 }
 234
 235 static int
 236 lstcon_group_find(char *name, lstcon_group_t **grpp)
 237 {
 238         lstcon_group_t   *grp;
 239
 240         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 241                 if (strncmp(grp->grp_name, name, LST_NAME_SIZE) != 0)
 242                         continue;
 243
 244                 lstcon_group_addref(grp);  /* +1 ref for caller */
 245                 *grpp = grp;
 246                 return 0;
 247         }
 248
 249         return -ENOENT;
 250 }
 251
 252 static void
 253 lstcon_group_put(lstcon_group_t *grp)
 254 {
 255         lstcon_group_decref(grp);
 256 }
 257
 258 static int
 259 lstcon_group_ndlink_find(lstcon_group_t *grp, lnet_process_id_t id,
 260                          lstcon_ndlink_t **ndlpp, int create)
 261 {
 262         int     rc;
 263
 264         rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create);
 265         if (rc != 0)
 266                 return rc;
 267
 268         if (!list_empty(&(*ndlpp)->ndl_link))
 269                 return 0;
 270
 271         list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list);
 272         grp->grp_nnode ++;
 273
 274         return 0;
 275 }
 276
 277 static void
 278 lstcon_group_ndlink_release(lstcon_group_t *grp, lstcon_ndlink_t *ndl)
 279 {
 280         list_del_init(&ndl->ndl_link);
 281         lstcon_ndlink_release(ndl);
 282         grp->grp_nnode --;
 283 }
 284
 285 static void
 286 lstcon_group_ndlink_move(lstcon_group_t *old,
 287                          lstcon_group_t *new, lstcon_ndlink_t *ndl)
 288 {
 289         unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) %
 290                            LST_NODE_HASHSIZE;
 291
 292         list_del(&ndl->ndl_hlink);
 293         list_del(&ndl->ndl_link);
 294         old->grp_nnode --;
 295
 296         list_add_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]);
 297         list_add_tail(&ndl->ndl_link, &new->grp_ndl_list);
 298         new->grp_nnode ++;
 299
 300         return;
 301 }
 302
 303 static void
 304 lstcon_group_move(lstcon_group_t *old, lstcon_group_t *new)
 305 {
 306         lstcon_ndlink_t *ndl;
 307
 308         while (!list_empty(&old->grp_ndl_list)) {
 309                 ndl = list_entry(old->grp_ndl_list.next,
 310                                  lstcon_ndlink_t, ndl_link);
 311                 lstcon_group_ndlink_move(old, new, ndl);
 312         }
 313 }
 314
 315 int
 316 lstcon_sesrpc_condition(int transop, lstcon_node_t *nd, void *arg)
 317 {
 318         lstcon_group_t *grp = (lstcon_group_t *)arg;
 319
 320         switch (transop) {
 321         case LST_TRANS_SESNEW:
 322                 if (nd->nd_state == LST_NODE_ACTIVE)
 323                         return 0;
 324                 break;
 325
 326         case LST_TRANS_SESEND:
 327                 if (nd->nd_state != LST_NODE_ACTIVE)
 328                         return 0;
 329
 330                 if (grp != NULL && nd->nd_ref > 1)
 331                         return 0;
 332                 break;
 333
 334         case LST_TRANS_SESQRY:
 335                 break;
 336
 337         default:
 338                 LBUG();
 339         }
 340
 341         return 1;
 342 }
 343
 344 int
 345 lstcon_sesrpc_readent(int transop, srpc_msg_t *msg,
 346                       lstcon_rpc_ent_t *ent_up)
 347 {
 348         srpc_debug_reply_t *rep;
 349
 350         switch (transop) {
 351         case LST_TRANS_SESNEW:
 352         case LST_TRANS_SESEND:
 353                 return 0;
 354
 355         case LST_TRANS_SESQRY:
 356                 rep = &msg->msg_body.dbg_reply;
 357
 358                 if (copy_to_user(&ent_up->rpe_priv[0],
 359                                  &rep->dbg_timeout, sizeof(int)) ||
 360                     copy_to_user(&ent_up->rpe_payload[0],
 361                                  &rep->dbg_name, LST_NAME_SIZE))
 362                         return -EFAULT;
 363
 364                 return 0;
 365
 366         default:
 367                 LBUG();
 368         }
 369
 370         return 0;
 371 }
 372
 373 static int
 374 lstcon_group_nodes_add(lstcon_group_t *grp, int count,
 375                        lnet_process_id_t *ids_up, struct list_head *result_up)
 376 {
 377         lstcon_rpc_trans_t      *trans;
 378         lstcon_ndlink_t         *ndl;
 379         lstcon_group_t          *tmp;
 380         lnet_process_id_t        id;
 381         int                      i;
 382         int                      rc;
 383
 384         rc = lstcon_group_alloc(NULL, &tmp);
 385         if (rc != 0) {
 386                 CERROR("Out of memory\n");
 387                 return -ENOMEM;
 388         }
 389
 390         for (i = 0 ; i < count; i++) {
 391                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 392                         rc = -EFAULT;
 393                         break;
 394                 }
 395
 396                 /* skip if it's in this group already */
 397                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 0);
 398                 if (rc == 0)
 399                         continue;
 400
 401                 /* add to tmp group */
 402                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1);
 403                 if (rc != 0) {
 404                         CERROR("Can't create ndlink, out of memory\n");
 405                         break;
 406                 }
 407         }
 408
 409         if (rc != 0) {
 410                 lstcon_group_put(tmp);
 411                 return rc;
 412         }
 413
 414         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 415                                      &tmp->grp_trans_list, LST_TRANS_SESNEW,
 416                                      tmp, lstcon_sesrpc_condition, &trans);
 417         if (rc != 0) {
 418                 CERROR("Can't create transaction: %d\n", rc);
 419                 lstcon_group_put(tmp);
 420                 return rc;
 421         }
 422
 423         /* post all RPCs */
 424         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 425
 426         rc = lstcon_rpc_trans_interpreter(trans, result_up,
 427                                           lstcon_sesrpc_readent);
 428         /* destroy all RPGs */
 429         lstcon_rpc_trans_destroy(trans);
 430
 431         lstcon_group_move(tmp, grp);
 432         lstcon_group_put(tmp);
 433
 434         return rc;
 435 }
 436
 437 static int
 438 lstcon_group_nodes_remove(lstcon_group_t *grp,
 439                           int count, lnet_process_id_t *ids_up,
 440                           struct list_head *result_up)
 441 {
 442         lstcon_rpc_trans_t     *trans;
 443         lstcon_ndlink_t        *ndl;
 444         lstcon_group_t         *tmp;
 445         lnet_process_id_t       id;
 446         int                     rc;
 447         int                     i;
 448
 449         /* End session and remove node from the group */
 450
 451         rc = lstcon_group_alloc(NULL, &tmp);
 452         if (rc != 0) {
 453                 CERROR("Out of memory\n");
 454                 return -ENOMEM;
 455         }
 456
 457         for (i = 0; i < count; i++) {
 458                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 459                         rc = -EFAULT;
 460                         goto error;
 461                 }
 462
 463                 /* move node to tmp group */
 464                 if (lstcon_group_ndlink_find(grp, id, &ndl, 0) == 0)
 465                         lstcon_group_ndlink_move(grp, tmp, ndl);
 466         }
 467
 468         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 469                                      &tmp->grp_trans_list, LST_TRANS_SESEND,
 470                                      tmp, lstcon_sesrpc_condition, &trans);
 471         if (rc != 0) {
 472                 CERROR("Can't create transaction: %d\n", rc);
 473                 goto error;
 474         }
 475
 476         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 477
 478         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 479
 480         lstcon_rpc_trans_destroy(trans);
 481         /* release nodes anyway, because we can't rollback status */
 482         lstcon_group_put(tmp);
 483
 484         return rc;
 485 error:
 486         lstcon_group_move(tmp, grp);
 487         lstcon_group_put(tmp);
 488
 489         return rc;
 490 }
 491
 492 int
 493 lstcon_group_add(char *name)
 494 {
 495         lstcon_group_t *grp;
 496         int             rc;
 497
 498         rc = (lstcon_group_find(name, &grp) == 0)? -EEXIST: 0;
 499         if (rc != 0) {
 500                 /* find a group with same name */
 501                 lstcon_group_put(grp);
 502                 return rc;
 503         }
 504
 505         rc = lstcon_group_alloc(name, &grp);
 506         if (rc != 0) {
 507                 CERROR("Can't allocate descriptor for group %s\n", name);
 508                 return -ENOMEM;
 509         }
 510
 511         list_add_tail(&grp->grp_link, &console_session.ses_grp_list);
 512
 513         return rc;
 514 }
 515
 516 int
 517 lstcon_nodes_add(char *name, int count,
 518                  lnet_process_id_t *ids_up, struct list_head *result_up)
 519 {
 520         lstcon_group_t         *grp;
 521         int                     rc;
 522
 523         LASSERT (count > 0);
 524         LASSERT (ids_up != NULL);
 525
 526         rc = lstcon_group_find(name, &grp);
 527         if (rc != 0) {
 528                 CDEBUG(D_NET, "Can't find group %s\n", name);
 529                 return rc;
 530         }
 531
 532         if (grp->grp_ref > 2) {
 533                 /* referred by other threads or test */
 534                 CDEBUG(D_NET, "Group %s is busy\n", name);
 535                 lstcon_group_put(grp);
 536
 537                 return -EBUSY;
 538         }
 539
 540         rc = lstcon_group_nodes_add(grp, count, ids_up, result_up);
 541
 542         lstcon_group_put(grp);
 543
 544         return rc;
 545 }
 546
 547 int
 548 lstcon_group_del(char *name)
 549 {
 550         lstcon_rpc_trans_t *trans;
 551         lstcon_group_t     *grp;
 552         int                 rc;
 553
 554         rc = lstcon_group_find(name, &grp);
 555         if (rc != 0) {
 556                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 557                 return rc;
 558         }
 559
 560         if (grp->grp_ref > 2) {
 561                 /* referred by others threads or test */
 562                 CDEBUG(D_NET, "Group %s is busy\n", name);
 563                 lstcon_group_put(grp);
 564                 return -EBUSY;
 565         }
 566
 567         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 568                                      &grp->grp_trans_list, LST_TRANS_SESEND,
 569                                      grp, lstcon_sesrpc_condition, &trans);
 570         if (rc != 0) {
 571                 CERROR("Can't create transaction: %d\n", rc);
 572                 lstcon_group_put(grp);
 573                 return rc;
 574         }
 575
 576         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 577
 578         lstcon_rpc_trans_destroy(trans);
 579
 580         lstcon_group_put(grp);
 581         /* -ref for session, it's destroyed,
 582          * status can't be rolled back, destroy group anway */
 583         lstcon_group_put(grp);
 584
 585         return rc;
 586 }
 587
 588 int
 589 lstcon_group_clean(char *name, int args)
 590 {
 591         lstcon_group_t *grp = NULL;
 592         int             rc;
 593
 594         rc = lstcon_group_find(name, &grp);
 595         if (rc != 0) {
 596                 CDEBUG(D_NET, "Can't find group %s\n", name);
 597                 return rc;
 598         }
 599
 600         if (grp->grp_ref > 2) {
 601                 /* referred by test */
 602                 CDEBUG(D_NET, "Group %s is busy\n", name);
 603                 lstcon_group_put(grp);
 604                 return -EBUSY;
 605         }
 606
 607         args = (LST_NODE_ACTIVE | LST_NODE_BUSY |
 608                 LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args;
 609
 610         lstcon_group_drain(grp, args);
 611
 612         lstcon_group_put(grp);
 613         /* release empty group */
 614         if (list_empty(&grp->grp_ndl_list))
 615                 lstcon_group_put(grp);
 616
 617         return 0;
 618 }
 619
 620 int
 621 lstcon_nodes_remove(char *name, int count,
 622                     lnet_process_id_t *ids_up, struct list_head *result_up)
 623 {
 624         lstcon_group_t *grp = NULL;
 625         int             rc;
 626
 627         rc = lstcon_group_find(name, &grp);
 628         if (rc != 0) {
 629                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 630                 return rc;
 631         }
 632
 633         if (grp->grp_ref > 2) {
 634                 /* referred by test */
 635                 CDEBUG(D_NET, "Group %s is busy\n", name);
 636                 lstcon_group_put(grp);
 637                 return -EBUSY;
 638         }
 639
 640         rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up);
 641
 642         lstcon_group_put(grp);
 643         /* release empty group */
 644         if (list_empty(&grp->grp_ndl_list))
 645                 lstcon_group_put(grp);
 646
 647         return rc;
 648 }
 649
 650 int
 651 lstcon_group_refresh(char *name, struct list_head *result_up)
 652 {
 653         lstcon_rpc_trans_t      *trans;
 654         lstcon_group_t          *grp;
 655         int                      rc;
 656
 657         rc = lstcon_group_find(name, &grp);
 658         if (rc != 0) {
 659                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 660                 return rc;
 661         }
 662
 663         if (grp->grp_ref > 2) {
 664                 /* referred by test */
 665                 CDEBUG(D_NET, "Group %s is busy\n", name);
 666                 lstcon_group_put(grp);
 667                 return -EBUSY;
 668         }
 669
 670         /* re-invite all inactive nodes int the group */
 671         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 672                                      &grp->grp_trans_list, LST_TRANS_SESNEW,
 673                                      grp, lstcon_sesrpc_condition, &trans);
 674         if (rc != 0) {
 675                 /* local error, return */
 676                 CDEBUG(D_NET, "Can't create transaction: %d\n", rc);
 677                 lstcon_group_put(grp);
 678                 return rc;
 679         }
 680
 681         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 682
 683         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 684
 685         lstcon_rpc_trans_destroy(trans);
 686         /* -ref for me */
 687         lstcon_group_put(grp);
 688
 689         return rc;
 690 }
 691
 692 int
 693 lstcon_group_list(int index, int len, char *name_up)
 694 {
 695         lstcon_group_t *grp;
 696
 697         LASSERT (index >= 0);
 698         LASSERT (name_up != NULL);
 699
 700         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 701                 if (index-- == 0) {
 702                         return copy_to_user(name_up, grp->grp_name, len) ?
 703                                -EFAULT : 0;
 704                 }
 705         }
 706
 707         return -ENOENT;
 708 }
 709
 710 static int
 711 lstcon_nodes_getent(struct list_head *head, int *index_p,
 712                     int *count_p, lstcon_node_ent_t *dents_up)
 713 {
 714         lstcon_ndlink_t  *ndl;
 715         lstcon_node_t    *nd;
 716         int               count = 0;
 717         int               index = 0;
 718
 719         LASSERT (index_p != NULL && count_p != NULL);
 720         LASSERT (dents_up != NULL);
 721         LASSERT (*index_p >= 0);
 722         LASSERT (*count_p > 0);
 723
 724         list_for_each_entry(ndl, head, ndl_link) {
 725                 if (index++ < *index_p)
 726                         continue;
 727
 728                 if (count >= *count_p)
 729                         break;
 730
 731                 nd = ndl->ndl_node;
 732                 if (copy_to_user(&dents_up[count].nde_id,
 733                                  &nd->nd_id, sizeof(nd->nd_id)) ||
 734                     copy_to_user(&dents_up[count].nde_state,
 735                                  &nd->nd_state, sizeof(nd->nd_state)))
 736                         return -EFAULT;
 737
 738                 count ++;
 739         }
 740
 741         if (index <= *index_p)
 742                 return -ENOENT;
 743
 744         *count_p = count;
 745         *index_p = index;
 746
 747         return 0;
 748 }
 749
 750 int
 751 lstcon_group_info(char *name, lstcon_ndlist_ent_t *gents_p,
 752                   int *index_p, int *count_p, lstcon_node_ent_t *dents_up)
 753 {
 754         lstcon_ndlist_ent_t *gentp;
 755         lstcon_group_t      *grp;
 756         lstcon_ndlink_t     *ndl;
 757         int                  rc;
 758
 759         rc = lstcon_group_find(name, &grp);
 760         if (rc != 0) {
 761                 CDEBUG(D_NET, "Can't find group %s\n", name);
 762                 return rc;
 763         }
 764
 765         if (dents_up != 0) {
 766                 /* verbose query */
 767                 rc = lstcon_nodes_getent(&grp->grp_ndl_list,
 768                                          index_p, count_p, dents_up);
 769                 lstcon_group_put(grp);
 770
 771                 return rc;
 772         }
 773
 774         /* non-verbose query */
 775         LIBCFS_ALLOC(gentp, sizeof(lstcon_ndlist_ent_t));
 776         if (gentp == NULL) {
 777                 CERROR("Can't allocate ndlist_ent\n");
 778                 lstcon_group_put(grp);
 779
 780                 return -ENOMEM;
 781         }
 782
 783         memset(gentp, 0, sizeof(lstcon_ndlist_ent_t));
 784
 785         list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link)
 786                 LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp);
 787
 788         rc = copy_to_user(gents_p, gentp,
 789                           sizeof(lstcon_ndlist_ent_t)) ? -EFAULT: 0;
 790
 791         LIBCFS_FREE(gentp, sizeof(lstcon_ndlist_ent_t));
 792
 793         lstcon_group_put(grp);
 794
 795         return 0;
 796 }
 797
 798 int
 799 lstcon_batch_find(char *name, lstcon_batch_t **batpp)
 800 {
 801         lstcon_batch_t   *bat;
 802
 803         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 804                 if (strncmp(bat->bat_name, name, LST_NAME_SIZE) == 0) {
 805                         *batpp = bat;
 806                         return 0;
 807                 }
 808         }
 809
 810         return -ENOENT;
 811 }
 812
 813 int
 814 lstcon_batch_add(char *name)
 815 {
 816         lstcon_batch_t   *bat;
 817         int               i;
 818         int               rc;
 819
 820         rc = (lstcon_batch_find(name, &bat) == 0)? -EEXIST: 0;
 821         if (rc != 0) {
 822                 CDEBUG(D_NET, "Batch %s already exists\n", name);
 823                 return rc;
 824         }
 825
 826         LIBCFS_ALLOC(bat, sizeof(lstcon_batch_t));
 827         if (bat == NULL) {
 828                 CERROR("Can't allocate descriptor for batch %s\n", name);
 829                 return -ENOMEM;
 830         }
 831
 832         LIBCFS_ALLOC(bat->bat_cli_hash,
 833                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 834         if (bat->bat_cli_hash == NULL) {
 835                 CERROR("Can't allocate hash for batch %s\n", name);
 836                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 837
 838                 return -ENOMEM;
 839         }
 840
 841         LIBCFS_ALLOC(bat->bat_srv_hash,
 842                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 843         if (bat->bat_srv_hash == NULL) {
 844                 CERROR("Can't allocate hash for batch %s\n", name);
 845                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 846                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 847
 848                 return -ENOMEM;
 849         }
 850
 851         strcpy(bat->bat_name, name);
 852         bat->bat_hdr.tsb_index = 0;
 853         bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie;
 854
 855         bat->bat_ntest = 0;
 856         bat->bat_state = LST_BATCH_IDLE;
 857
 858         CFS_INIT_LIST_HEAD(&bat->bat_cli_list);
 859         CFS_INIT_LIST_HEAD(&bat->bat_srv_list);
 860         CFS_INIT_LIST_HEAD(&bat->bat_test_list);
 861         CFS_INIT_LIST_HEAD(&bat->bat_trans_list);
 862
 863         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 864                 CFS_INIT_LIST_HEAD(&bat->bat_cli_hash[i]);
 865                 CFS_INIT_LIST_HEAD(&bat->bat_srv_hash[i]);
 866         }
 867
 868         list_add_tail(&bat->bat_link, &console_session.ses_bat_list);
 869
 870         return rc;
 871 }
 872
 873 int
 874 lstcon_batch_list(int index, int len, char *name_up)
 875 {
 876         lstcon_batch_t    *bat;
 877
 878         LASSERT (name_up != NULL);
 879         LASSERT (index >= 0);
 880
 881         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 882                 if (index-- == 0) {
 883                         return copy_to_user(name_up,bat->bat_name, len) ?
 884                                -EFAULT: 0;
 885                 }
 886         }
 887
 888         return -ENOENT;
 889 }
 890
 891 int
 892 lstcon_batch_info(char *name, lstcon_test_batch_ent_t *ent_up, int server,
 893                   int testidx, int *index_p, int *ndent_p,
 894                   lstcon_node_ent_t *dents_up)
 895 {
 896         lstcon_test_batch_ent_t *entp;
 897         struct list_head        *clilst;
 898         struct list_head        *srvlst;
 899         lstcon_test_t           *test = NULL;
 900         lstcon_batch_t          *bat;
 901         lstcon_ndlink_t         *ndl;
 902         int                      rc;
 903
 904         rc = lstcon_batch_find(name, &bat);
 905         if (rc != 0) {
 906                 CDEBUG(D_NET, "Can't find batch %s\n", name);
 907                 return -ENOENT;
 908         }
 909
 910         if (testidx > 0) {
 911                 /* query test, test index start from 1 */
 912                 list_for_each_entry(test, &bat->bat_test_list, tes_link) {
 913                         if (testidx-- == 1)
 914                                 break;
 915                 }
 916
 917                 if (testidx > 0) {
 918                         CDEBUG(D_NET, "Can't find specified test in batch\n");
 919                         return -ENOENT;
 920                 }
 921         }
 922
 923         clilst = (test == NULL) ? &bat->bat_cli_list :
 924                                   &test->tes_src_grp->grp_ndl_list;
 925         srvlst = (test == NULL) ? &bat->bat_srv_list :
 926                                   &test->tes_dst_grp->grp_ndl_list;
 927
 928         if (dents_up != NULL) {
 929                 rc = lstcon_nodes_getent((server ? srvlst: clilst),
 930                                          index_p, ndent_p, dents_up);
 931                 return rc;
 932         }
 933
 934         /* non-verbose query */
 935         LIBCFS_ALLOC(entp, sizeof(lstcon_test_batch_ent_t));
 936         if (entp == NULL)
 937                 return -ENOMEM;
 938
 939         memset(entp, 0, sizeof(lstcon_test_batch_ent_t));
 940
 941         if (test == NULL) {
 942                 entp->u.tbe_batch.bae_ntest = bat->bat_ntest;
 943                 entp->u.tbe_batch.bae_state = bat->bat_state;
 944
 945         } else {
 946
 947                 entp->u.tbe_test.tse_type   = test->tes_type;
 948                 entp->u.tbe_test.tse_loop   = test->tes_loop;
 949                 entp->u.tbe_test.tse_concur = test->tes_concur;
 950         }
 951
 952         list_for_each_entry(ndl, clilst, ndl_link)
 953                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle);
 954
 955         list_for_each_entry(ndl, srvlst, ndl_link)
 956                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle);
 957
 958         rc = copy_to_user(ent_up, entp,
 959                           sizeof(lstcon_test_batch_ent_t)) ? -EFAULT : 0;
 960
 961         LIBCFS_FREE(entp, sizeof(lstcon_test_batch_ent_t));
 962
 963         return rc;
 964 }
 965
 966 int
 967 lstcon_batrpc_condition(int transop, lstcon_node_t *nd, void *arg)
 968 {
 969         switch (transop) {
 970         case LST_TRANS_TSBRUN:
 971                 if (nd->nd_state != LST_NODE_ACTIVE)
 972                         return -ENETDOWN;
 973                 break;
 974
 975         case LST_TRANS_TSBSTOP:
 976                 if (nd->nd_state != LST_NODE_ACTIVE)
 977                         return 0;
 978                 break;
 979
 980         case LST_TRANS_TSBCLIQRY:
 981         case LST_TRANS_TSBSRVQRY:
 982                 break;
 983         }
 984
 985         return 1;
 986 }
 987
 988 static int
 989 lstcon_batch_op(lstcon_batch_t *bat, int transop, struct list_head *result_up)
 990 {
 991         lstcon_rpc_trans_t *trans;
 992         int                 rc;
 993
 994         rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list,
 995                                      &bat->bat_trans_list, transop,
 996                                      bat, lstcon_batrpc_condition, &trans);
 997         if (rc != 0) {
 998                 CERROR("Can't create transaction: %d\n", rc);
 999                 return rc;
1000         }
1001
1002         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1003
1004         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1005
1006         lstcon_rpc_trans_destroy(trans);
1007
1008         return rc;
1009 }
1010
1011 int
1012 lstcon_batch_run(char *name, int timeout, struct list_head *result_up)
1013 {
1014         lstcon_batch_t *bat;
1015         int             rc;
1016
1017         if (lstcon_batch_find(name, &bat) != 0) {
1018                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1019                 return -ENOENT;
1020         }
1021
1022         bat->bat_arg = timeout;
1023
1024         rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up);
1025
1026         /* mark batch as running if it's started in any node */
1027         if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0) != 0)
1028                 bat->bat_state = LST_BATCH_RUNNING;
1029
1030         return rc;
1031 }
1032
1033 int
1034 lstcon_batch_stop(char *name, int force, struct list_head *result_up)
1035 {
1036         lstcon_batch_t *bat;
1037         int             rc;
1038
1039         if (lstcon_batch_find(name, &bat) != 0) {
1040                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1041                 return -ENOENT;
1042         }
1043
1044         bat->bat_arg = force;
1045
1046         rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up);
1047
1048         /* mark batch as stopped if all RPCs finished */
1049         if (lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0) == 0)
1050                 bat->bat_state = LST_BATCH_IDLE;
1051
1052         return rc;
1053 }
1054
1055 static void
1056 lstcon_batch_destroy(lstcon_batch_t *bat)
1057 {
1058         lstcon_ndlink_t    *ndl;
1059         lstcon_test_t      *test;
1060         int                 i;
1061
1062         list_del(&bat->bat_link);
1063
1064         while (!list_empty(&bat->bat_test_list)) {
1065                 test = list_entry(bat->bat_test_list.next,
1066                                   lstcon_test_t, tes_link);
1067                 LASSERT (list_empty(&test->tes_trans_list));
1068
1069                 list_del(&test->tes_link);
1070
1071                 lstcon_group_put(test->tes_src_grp);
1072                 lstcon_group_put(test->tes_dst_grp);
1073
1074                 LIBCFS_FREE(test, offsetof(lstcon_test_t,
1075                                            tes_param[test->tes_paramlen]));
1076         }
1077
1078         LASSERT (list_empty(&bat->bat_trans_list));
1079
1080         while (!list_empty(&bat->bat_cli_list)) {
1081                 ndl = list_entry(bat->bat_cli_list.next,
1082                                  lstcon_ndlink_t, ndl_link);
1083                 list_del_init(&ndl->ndl_link);
1084
1085                 lstcon_ndlink_release(ndl);
1086         }
1087
1088         while (!list_empty(&bat->bat_srv_list)) {
1089                 ndl = list_entry(bat->bat_srv_list.next,
1090                                  lstcon_ndlink_t, ndl_link);
1091                 list_del_init(&ndl->ndl_link);
1092
1093                 lstcon_ndlink_release(ndl);
1094         }
1095
1096         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1097                 LASSERT (list_empty(&bat->bat_cli_hash[i]));
1098                 LASSERT (list_empty(&bat->bat_srv_hash[i]));
1099         }
1100
1101         LIBCFS_FREE(bat->bat_cli_hash,
1102                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1103         LIBCFS_FREE(bat->bat_srv_hash,
1104                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1105         LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
1106 }
1107
1108 int
1109 lstcon_testrpc_condition(int transop, lstcon_node_t *nd, void *arg)
1110 {
1111         lstcon_test_t    *test;
1112         lstcon_batch_t   *batch;
1113         lstcon_ndlink_t  *ndl;
1114         struct list_head *hash;
1115         struct list_head *head;
1116
1117         test = (lstcon_test_t *)arg;
1118         LASSERT (test != NULL);
1119
1120         batch = test->tes_batch;
1121         LASSERT (batch != NULL);
1122
1123         if (test->tes_oneside &&
1124             transop == LST_TRANS_TSBSRVADD)
1125                 return 0;
1126
1127         if (nd->nd_state != LST_NODE_ACTIVE)
1128                 return -ENETDOWN;
1129
1130         if (transop == LST_TRANS_TSBCLIADD) {
1131                 hash = batch->bat_cli_hash;
1132                 head = &batch->bat_cli_list;
1133
1134         } else {
1135                 LASSERT (transop == LST_TRANS_TSBSRVADD);
1136
1137                 hash = batch->bat_srv_hash;
1138                 head = &batch->bat_srv_list;
1139         }
1140
1141         LASSERT (nd->nd_id.nid != LNET_NID_ANY);
1142
1143         if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1) != 0)
1144                 return -ENOMEM;
1145
1146         if (list_empty(&ndl->ndl_link))
1147                 list_add_tail(&ndl->ndl_link, head);
1148
1149         return 1;
1150 }
1151
1152 static int
1153 lstcon_test_nodes_add(lstcon_test_t *test, struct list_head *result_up)
1154 {
1155         lstcon_rpc_trans_t     *trans;
1156         lstcon_group_t         *grp;
1157         int                     transop;
1158         int                     rc;
1159
1160         LASSERT (test->tes_src_grp != NULL);
1161         LASSERT (test->tes_dst_grp != NULL);
1162
1163         transop = LST_TRANS_TSBSRVADD;
1164         grp  = test->tes_dst_grp;
1165 again:
1166         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
1167                                      &test->tes_trans_list, transop,
1168                                      test, lstcon_testrpc_condition, &trans);
1169         if (rc != 0) {
1170                 CERROR("Can't create transaction: %d\n", rc);
1171                 return rc;
1172         }
1173
1174         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1175
1176         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1177             lstcon_trans_stat()->trs_fwk_errno != 0) {
1178                 lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1179
1180                 lstcon_rpc_trans_destroy(trans);
1181                 /* return if any error */
1182                 CDEBUG(D_NET, "Failed to add test %s, "
1183                               "RPC error %d, framework error %d\n",
1184                        transop == LST_TRANS_TSBCLIADD ? "client" : "server",
1185                        lstcon_trans_stat()->trs_rpc_errno,
1186                        lstcon_trans_stat()->trs_fwk_errno);
1187
1188                 return rc;
1189         }
1190
1191         lstcon_rpc_trans_destroy(trans);
1192
1193         if (transop == LST_TRANS_TSBCLIADD)
1194                 return rc;
1195
1196         transop = LST_TRANS_TSBCLIADD;
1197         grp = test->tes_src_grp;
1198         test->tes_cliidx = 0;
1199
1200         /* requests to test clients */
1201         goto again;
1202 }
1203
1204 int
1205 lstcon_test_add(char *name, int type, int loop, int concur,
1206                 int dist, int span, char *src_name, char * dst_name,
1207                 void *param, int paramlen, int *retp, struct list_head *result_up)
1208
1209 {
1210         lstcon_group_t  *src_grp = NULL;
1211         lstcon_group_t  *dst_grp = NULL;
1212         lstcon_test_t   *test    = NULL;
1213         lstcon_batch_t  *batch;
1214         int              rc;
1215
1216         rc = lstcon_batch_find(name, &batch);
1217         if (rc != 0) {
1218                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1219                 return rc;
1220         }
1221
1222         if (batch->bat_state != LST_BATCH_IDLE) {
1223                 CDEBUG(D_NET, "Can't change running batch %s\n", name);
1224                 return rc;
1225         }
1226
1227         rc = lstcon_group_find(src_name, &src_grp);
1228         if (rc != 0) {
1229                 CDEBUG(D_NET, "Can't find group %s\n", src_name);
1230                 goto out;
1231         }
1232
1233         rc = lstcon_group_find(dst_name, &dst_grp);
1234         if (rc != 0) {
1235                 CDEBUG(D_NET, "Can't find group %s\n", dst_name);
1236                 goto out;
1237         }
1238
1239         if (dst_grp->grp_userland)
1240                 *retp = 1;
1241
1242         LIBCFS_ALLOC(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1243         if (!test) {
1244                 CERROR("Can't allocate test descriptor\n");
1245                 rc = -ENOMEM;
1246
1247                 goto out;
1248         }
1249
1250         memset(test, 0, offsetof(lstcon_test_t, tes_param[paramlen]));
1251         test->tes_hdr.tsb_id    = batch->bat_hdr.tsb_id;
1252         test->tes_batch         = batch;
1253         test->tes_type          = type;
1254         test->tes_oneside       = 0; /* TODO */
1255         test->tes_loop          = loop;
1256         test->tes_concur        = concur;
1257         test->tes_stop_onerr    = 1; /* TODO */
1258         test->tes_span          = span;
1259         test->tes_dist          = dist;
1260         test->tes_cliidx        = 0; /* just used for creating RPC */
1261         test->tes_src_grp       = src_grp;
1262         test->tes_dst_grp       = dst_grp;
1263         CFS_INIT_LIST_HEAD(&test->tes_trans_list);
1264
1265         if (param != NULL) {
1266                 test->tes_paramlen = paramlen;
1267                 memcpy(&test->tes_param[0], param, paramlen);
1268         }
1269
1270         rc = lstcon_test_nodes_add(test, result_up);
1271
1272         if (rc != 0)
1273                 goto out;
1274
1275         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1276             lstcon_trans_stat()->trs_fwk_errno != 0)
1277                 CDEBUG(D_NET, "Failed to add test %d to batch %s\n", type, name);
1278
1279         /* add to test list anyway, so user can check what's going on */
1280         list_add_tail(&test->tes_link, &batch->bat_test_list);
1281
1282         batch->bat_ntest ++;
1283         test->tes_hdr.tsb_index = batch->bat_ntest;
1284
1285         /*  hold groups so nobody can change them */
1286         return rc;
1287 out:
1288         if (test != NULL)
1289                 LIBCFS_FREE(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1290
1291         if (dst_grp != NULL)
1292                 lstcon_group_put(dst_grp);
1293
1294         if (src_grp != NULL)
1295                 lstcon_group_put(src_grp);
1296
1297         return rc;
1298 }
1299
1300 int
1301 lstcon_test_find(lstcon_batch_t *batch, int idx, lstcon_test_t **testpp)
1302 {
1303         lstcon_test_t *test;
1304
1305         list_for_each_entry(test, &batch->bat_test_list, tes_link) {
1306                 if (idx == test->tes_hdr.tsb_index) {
1307                         *testpp = test;
1308                         return 0;
1309                 }
1310         }
1311
1312         return -ENOENT;
1313 }
1314
1315 int
1316 lstcon_tsbrpc_readent(int transop, srpc_msg_t *msg,
1317                       lstcon_rpc_ent_t *ent_up)
1318 {
1319         srpc_batch_reply_t *rep = &msg->msg_body.bat_reply;
1320
1321         LASSERT (transop == LST_TRANS_TSBCLIQRY ||
1322                  transop == LST_TRANS_TSBSRVQRY);
1323
1324         /* positive errno, framework error code */
1325         if (copy_to_user(&ent_up->rpe_priv[0],
1326                          &rep->bar_active, sizeof(rep->bar_active)))
1327                 return -EFAULT;
1328
1329         return 0;
1330 }
1331
1332 int
1333 lstcon_test_batch_query(char *name, int testidx, int client,
1334                         int timeout, struct list_head *result_up)
1335 {
1336         lstcon_rpc_trans_t *trans;
1337         struct list_head   *translist;
1338         struct list_head   *ndlist;
1339         lstcon_tsb_hdr_t   *hdr;
1340         lstcon_batch_t     *batch;
1341         lstcon_test_t      *test = NULL;
1342         int                 transop;
1343         int                 rc;
1344
1345         rc = lstcon_batch_find(name, &batch);
1346         if (rc != 0) {
1347                 CDEBUG(D_NET, "Can't find batch: %s\n", name);
1348                 return rc;
1349         }
1350
1351         if (testidx == 0) {
1352                 translist = &batch->bat_trans_list;
1353                 ndlist    = &batch->bat_cli_list;
1354                 hdr       = &batch->bat_hdr;
1355
1356         } else {
1357                 /* query specified test only */
1358                 rc = lstcon_test_find(batch, testidx, &test);
1359                 if (rc != 0) {
1360                         CDEBUG(D_NET, "Can't find test: %d\n", testidx);
1361                         return rc;
1362                 }
1363
1364                 translist = &test->tes_trans_list;
1365                 ndlist    = &test->tes_src_grp->grp_ndl_list;
1366                 hdr       = &test->tes_hdr;
1367         }
1368
1369         transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY;
1370
1371         rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr,
1372                                      lstcon_batrpc_condition, &trans);
1373         if (rc != 0) {
1374                 CERROR("Can't create transaction: %d\n", rc);
1375                 return rc;
1376         }
1377
1378         lstcon_rpc_trans_postwait(trans, timeout);
1379
1380         if (testidx == 0 && /* query a batch, not a test */
1381             lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) == 0 &&
1382             lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0) == 0) {
1383                 /* all RPCs finished, and no active test */
1384                 batch->bat_state = LST_BATCH_IDLE;
1385         }
1386
1387         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1388                                           lstcon_tsbrpc_readent);
1389         lstcon_rpc_trans_destroy(trans);
1390
1391         return rc;
1392 }
1393
1394 int
1395 lstcon_statrpc_readent(int transop, srpc_msg_t *msg,
1396                        lstcon_rpc_ent_t *ent_up)
1397 {
1398         srpc_stat_reply_t *rep = &msg->msg_body.stat_reply;
1399         sfw_counters_t    *sfwk_stat;
1400         srpc_counters_t   *srpc_stat;
1401         lnet_counters_t   *lnet_stat;
1402
1403         if (rep->str_status != 0)
1404                 return 0;
1405
1406         sfwk_stat = (sfw_counters_t *)&ent_up->rpe_payload[0];
1407         srpc_stat = (srpc_counters_t *)((char *)sfwk_stat + sizeof(*sfwk_stat));
1408         lnet_stat = (lnet_counters_t *)((char *)srpc_stat + sizeof(*srpc_stat));
1409
1410         if (copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) ||
1411             copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) ||
1412             copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat)))
1413                 return -EFAULT;
1414
1415         return 0;
1416 }
1417
1418 int
1419 lstcon_ndlist_stat(struct list_head *ndlist,
1420                    int timeout, struct list_head *result_up)
1421 {
1422         struct list_head    head;
1423         lstcon_rpc_trans_t *trans;
1424         int                 rc;
1425
1426         CFS_INIT_LIST_HEAD(&head);
1427
1428         rc = lstcon_rpc_trans_ndlist(ndlist, &head,
1429                                      LST_TRANS_STATQRY, NULL, NULL, &trans);
1430         if (rc != 0) {
1431                 CERROR("Can't create transaction: %d\n", rc);
1432                 return rc;
1433         }
1434
1435         timeout = (timeout > LST_TRANS_MIN_TIMEOUT) ? timeout :
1436                                                       LST_TRANS_MIN_TIMEOUT;
1437         lstcon_rpc_trans_postwait(trans, timeout);
1438
1439         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1440                                           lstcon_statrpc_readent);
1441         lstcon_rpc_trans_destroy(trans);
1442
1443         return rc;
1444 }
1445
1446 int
1447 lstcon_group_stat(char *grp_name, int timeout, struct list_head *result_up)
1448 {
1449         lstcon_group_t     *grp;
1450         int                 rc;
1451
1452         rc = lstcon_group_find(grp_name, &grp);
1453         if (rc != 0) {
1454                 CDEBUG(D_NET, "Can't find group %s\n", grp_name);
1455                 return rc;
1456         }
1457
1458         rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up);
1459
1460         lstcon_group_put(grp);
1461
1462         return rc;
1463 }
1464
1465 int
1466 lstcon_nodes_stat(int count, lnet_process_id_t *ids_up,
1467                   int timeout, struct list_head *result_up)
1468 {
1469         lstcon_ndlink_t         *ndl;
1470         lstcon_group_t          *tmp;
1471         lnet_process_id_t        id;
1472         int                      i;
1473         int                      rc;
1474
1475         rc = lstcon_group_alloc(NULL, &tmp);
1476         if (rc != 0) {
1477                 CERROR("Out of memory\n");
1478                 return -ENOMEM;
1479         }
1480
1481         for (i = 0 ; i < count; i++) {
1482                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1483                         rc = -EFAULT;
1484                         break;
1485                 }
1486
1487                 /* add to tmp group */
1488                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2);
1489                 if (rc != 0) {
1490                         CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET,
1491                                "Failed to find or create %s: %d\n",
1492                                libcfs_id2str(id), rc);
1493                         break;
1494                 }
1495         }
1496
1497         if (rc != 0) {
1498                 lstcon_group_put(tmp);
1499                 return rc;
1500         }
1501
1502         rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up);
1503
1504         lstcon_group_put(tmp);
1505
1506         return rc;
1507 }
1508
1509 int
1510 lstcon_debug_ndlist(struct list_head *ndlist,
1511                     struct list_head *translist,
1512                     int timeout, struct list_head *result_up)
1513 {
1514         lstcon_rpc_trans_t *trans;
1515         int                 rc;
1516
1517         rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY,
1518                                      NULL, lstcon_sesrpc_condition, &trans);
1519         if (rc != 0) {
1520                 CERROR("Can't create transaction: %d\n", rc);
1521                 return rc;
1522         }
1523
1524         timeout = (timeout > LST_TRANS_MIN_TIMEOUT) ? timeout :
1525                                                       LST_TRANS_MIN_TIMEOUT;
1526
1527         lstcon_rpc_trans_postwait(trans, timeout);
1528
1529         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1530                                           lstcon_sesrpc_readent);
1531         lstcon_rpc_trans_destroy(trans);
1532
1533         return rc;
1534 }
1535
1536 int
1537 lstcon_session_debug(int timeout, struct list_head *result_up)
1538 {
1539         return lstcon_debug_ndlist(&console_session.ses_ndl_list,
1540                                    NULL, timeout, result_up);
1541 }
1542
1543 int
1544 lstcon_batch_debug(int timeout, char *name,
1545                    int client, struct list_head *result_up)
1546 {
1547         lstcon_batch_t *bat;
1548         int             rc;
1549
1550         rc = lstcon_batch_find(name, &bat);
1551         if (rc != 0)
1552                 return -ENOENT;
1553
1554         rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list :
1555                                           &bat->bat_srv_list,
1556                                  NULL, timeout, result_up);
1557
1558         return rc;
1559 }
1560
1561 int
1562 lstcon_group_debug(int timeout, char *name,
1563                    struct list_head *result_up)
1564 {
1565         lstcon_group_t *grp;
1566         int             rc;
1567
1568         rc = lstcon_group_find(name, &grp);
1569         if (rc != 0)
1570                 return -ENOENT;
1571
1572         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1573                                  timeout, result_up);
1574         lstcon_group_put(grp);
1575
1576         return rc;
1577 }
1578
1579 int
1580 lstcon_nodes_debug(int timeout,
1581                    int count, lnet_process_id_t *ids_up,
1582                    struct list_head *result_up)
1583 {
1584         lnet_process_id_t  id;
1585         lstcon_ndlink_t   *ndl;
1586         lstcon_group_t    *grp;
1587         int                i;
1588         int                rc;
1589
1590         rc = lstcon_group_alloc(NULL, &grp);
1591         if (rc != 0) {
1592                 CDEBUG(D_NET, "Out of memory\n");
1593                 return rc;
1594         }
1595
1596         for (i = 0; i < count; i++) {
1597                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1598                         rc = -EFAULT;
1599                         break;
1600                 }
1601
1602                 /* node is added to tmp group */
1603                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 1);
1604                 if (rc != 0) {
1605                         CERROR("Can't create node link\n");
1606                         break;
1607                 }
1608         }
1609
1610         if (rc != 0) {
1611                 lstcon_group_put(grp);
1612                 return rc;
1613         }
1614
1615         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1616                                  timeout, result_up);
1617
1618         lstcon_group_put(grp);
1619
1620         return rc;
1621 }
1622
1623 int
1624 lstcon_session_match(lst_sid_t sid)
1625 {
1626         return (console_session.ses_id.ses_nid   == sid.ses_nid &&
1627                 console_session.ses_id.ses_stamp == sid.ses_stamp) ?  1: 0;
1628 }
1629
1630 static void
1631 lstcon_new_session_id(lst_sid_t *sid)
1632 {
1633         lnet_process_id_t      id;
1634
1635         LASSERT (console_session.ses_state == LST_SESSION_NONE);
1636
1637         LNetGetId(1, &id);
1638         sid->ses_nid   = id.nid;
1639         sid->ses_stamp = cfs_time_current();
1640 }
1641
1642 extern srpc_service_t lstcon_acceptor_service;
1643
1644 int
1645 lstcon_session_new(char *name, int key,
1646                    int timeout,int force, lst_sid_t *sid_up)
1647 {
1648         int     rc = 0;
1649         int     i;
1650
1651         if (console_session.ses_state != LST_SESSION_NONE) {
1652                 /* session exists */
1653                 if (!force) {
1654                         CERROR("Session %s already exists\n",
1655                                console_session.ses_name);
1656                         return -EEXIST;
1657                 }
1658
1659                 rc = lstcon_session_end();
1660
1661                 /* lstcon_session_end() only return local error */
1662                 if  (rc != 0)
1663                         return rc;
1664         }
1665
1666         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++) {
1667                 LASSERT (list_empty(&console_session.ses_ndl_hash[i]));
1668         }
1669
1670         rc = lstcon_batch_add(LST_DEFAULT_BATCH);
1671         if (rc != 0)
1672                 return rc;
1673
1674         rc = lstcon_rpc_pinger_start();
1675         if (rc != 0) {
1676                 lstcon_batch_t *bat;
1677
1678                 lstcon_batch_find(LST_DEFAULT_BATCH, &bat);
1679                 lstcon_batch_destroy(bat);
1680
1681                 return rc;
1682         }
1683
1684         lstcon_new_session_id(&console_session.ses_id);
1685
1686         console_session.ses_key     = key;
1687         console_session.ses_state   = LST_SESSION_ACTIVE;
1688         console_session.ses_force   = !!force;
1689         console_session.ses_timeout = (timeout <= 0)? LST_CONSOLE_TIMEOUT:
1690                                                       timeout;
1691         strcpy(console_session.ses_name, name);
1692
1693         if (copy_to_user(sid_up, &console_session.ses_id,
1694                          sizeof(lst_sid_t)) == 0)
1695                 return rc;
1696
1697         lstcon_session_end();
1698
1699         return -EFAULT;
1700 }
1701
1702 int
1703 lstcon_session_info(lst_sid_t *sid_up, int *key_up,
1704                     lstcon_ndlist_ent_t *ndinfo_up, char *name_up, int len)
1705 {
1706         lstcon_ndlist_ent_t *entp;
1707         lstcon_ndlink_t     *ndl;
1708         int                  rc = 0;
1709
1710         if (console_session.ses_state != LST_SESSION_ACTIVE)
1711                 return -ESRCH;
1712
1713         LIBCFS_ALLOC(entp, sizeof(*entp));
1714         if (entp == NULL)
1715                 return -ENOMEM;
1716
1717         memset(entp, 0, sizeof(*entp));
1718
1719         list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link)
1720                 LST_NODE_STATE_COUNTER(ndl->ndl_node, entp);
1721
1722         if (copy_to_user(sid_up, &console_session.ses_id, sizeof(lst_sid_t)) ||
1723             copy_to_user(key_up, &console_session.ses_key, sizeof(int)) ||
1724             copy_to_user(ndinfo_up, entp, sizeof(*entp)) ||
1725             copy_to_user(name_up, console_session.ses_name, len))
1726                 rc = -EFAULT;
1727
1728         LIBCFS_FREE(entp, sizeof(*entp));
1729
1730         return rc;
1731 }
1732
1733 int
1734 lstcon_session_end()
1735 {
1736         lstcon_rpc_trans_t *trans;
1737         lstcon_group_t     *grp;
1738         lstcon_batch_t     *bat;
1739         int                 rc = 0;
1740
1741         LASSERT (console_session.ses_state == LST_SESSION_ACTIVE);
1742
1743         rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list,
1744                                      NULL, LST_TRANS_SESEND, NULL,
1745                                      lstcon_sesrpc_condition, &trans);
1746         if (rc != 0) {
1747                 CERROR("Can't create transaction: %d\n", rc);
1748                 return rc;
1749         }
1750
1751         console_session.ses_shutdown = 1;
1752
1753         lstcon_rpc_pinger_stop();
1754
1755         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1756
1757         lstcon_rpc_trans_destroy(trans);
1758         /* User can do nothing even rpc failed, so go on */
1759
1760         /* waiting for orphan rpcs to die */
1761         lstcon_rpc_cleanup_wait();
1762
1763         console_session.ses_id    = LST_INVALID_SID;
1764         console_session.ses_state = LST_SESSION_NONE;
1765         console_session.ses_key   = 0;
1766         console_session.ses_force = 0;
1767
1768         /* destroy all batches */
1769         while (!list_empty(&console_session.ses_bat_list)) {
1770                 bat = list_entry(console_session.ses_bat_list.next,
1771                                  lstcon_batch_t, bat_link);
1772
1773                 lstcon_batch_destroy(bat);
1774         }
1775
1776         /* destroy all groups */
1777         while (!list_empty(&console_session.ses_grp_list)) {
1778                 grp = list_entry(console_session.ses_grp_list.next,
1779                                  lstcon_group_t, grp_link);
1780                 LASSERT (grp->grp_ref == 1);
1781
1782                 lstcon_group_put(grp);
1783         }
1784
1785         /* all nodes should be released */
1786         LASSERT (list_empty(&console_session.ses_ndl_list));
1787
1788         console_session.ses_shutdown = 0;
1789         console_session.ses_expired  = 0;
1790
1791         return rc;
1792 }
1793
1794 static int
1795 lstcon_acceptor_handle (srpc_server_rpc_t *rpc)
1796 {
1797         srpc_msg_t        *rep  = &rpc->srpc_replymsg;
1798         srpc_msg_t        *req  = &rpc->srpc_reqstbuf->buf_msg;
1799         srpc_join_reqst_t *jreq = &req->msg_body.join_reqst;
1800         srpc_join_reply_t *jrep = &rep->msg_body.join_reply;
1801         lstcon_group_t    *grp  = NULL;
1802         lstcon_ndlink_t   *ndl;
1803         int                rc   = 0;
1804
1805         sfw_unpack_message(req);
1806
1807         mutex_down(&console_session.ses_mutex);
1808
1809         jrep->join_sid = console_session.ses_id;
1810
1811         if (console_session.ses_id.ses_nid == LNET_NID_ANY) {
1812                 jrep->join_status = ESRCH;
1813                 goto out;
1814         }
1815
1816         if (jreq->join_sid.ses_nid != LNET_NID_ANY &&
1817              !lstcon_session_match(jreq->join_sid)) {
1818                 jrep->join_status = EBUSY;
1819                 goto out;
1820         }
1821
1822         if (lstcon_group_find(jreq->join_group, &grp) != 0) {
1823                 rc = lstcon_group_alloc(jreq->join_group, &grp);
1824                 if (rc != 0) {
1825                         CERROR("Out of memory\n");
1826                         goto out;
1827                 }
1828
1829                 list_add_tail(&grp->grp_link,
1830                               &console_session.ses_grp_list);
1831                 lstcon_group_addref(grp);
1832         }
1833
1834         if (grp->grp_ref > 2) {
1835                 /* Group in using */
1836                 jrep->join_status = EBUSY;
1837                 goto out;
1838         }
1839
1840         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0);
1841         if (rc == 0) {
1842                 jrep->join_status = EEXIST;
1843                 goto out;
1844         }
1845
1846         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1);
1847         if (rc != 0) {
1848                 CERROR("Out of memory\n");
1849                 goto out;
1850         }
1851
1852         ndl->ndl_node->nd_state   = LST_NODE_ACTIVE;
1853         ndl->ndl_node->nd_timeout = console_session.ses_timeout;
1854
1855         if (grp->grp_userland == 0)
1856                 grp->grp_userland = 1;
1857
1858         strcpy(jrep->join_session, console_session.ses_name);
1859         jrep->join_timeout = console_session.ses_timeout;
1860         jrep->join_status  = 0;
1861
1862 out:
1863         if (grp != NULL)
1864                 lstcon_group_put(grp);
1865
1866         mutex_up(&console_session.ses_mutex);
1867
1868         return rc;
1869 }
1870
1871 srpc_service_t lstcon_acceptor_service =
1872 {
1873         .sv_name        = "join session",
1874         .sv_handler     = lstcon_acceptor_handle,
1875         .sv_bulk_ready  = NULL,
1876         .sv_id          = SRPC_SERVICE_JOIN,
1877         .sv_concur      = SFW_SERVICE_CONCURRENCY,
1878 };
1879
1880 extern int lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_data *data);
1881
1882 DECLARE_IOCTL_HANDLER(lstcon_ioctl_handler, lstcon_ioctl_entry);
1883
1884 /* initialize console */
1885 int
1886 lstcon_console_init(void)
1887 {
1888         int     i;
1889         int     n;
1890         int     rc;
1891
1892         memset(&console_session, 0, sizeof(lstcon_session_t));
1893
1894         console_session.ses_id      = LST_INVALID_SID;
1895         console_session.ses_state   = LST_SESSION_NONE;
1896         console_session.ses_timeout = 0;
1897         console_session.ses_force   = 0;
1898         console_session.ses_expired = 0;
1899         console_session.ses_laststamp = cfs_time_current_sec();
1900
1901         init_mutex(&console_session.ses_mutex);
1902
1903         CFS_INIT_LIST_HEAD(&console_session.ses_ndl_list);
1904         CFS_INIT_LIST_HEAD(&console_session.ses_grp_list);
1905         CFS_INIT_LIST_HEAD(&console_session.ses_bat_list);
1906         CFS_INIT_LIST_HEAD(&console_session.ses_trans_list);
1907
1908         LIBCFS_ALLOC(console_session.ses_ndl_hash,
1909                      sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
1910         if (console_session.ses_ndl_hash == NULL)
1911                 return -ENOMEM;
1912
1913         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
1914                 CFS_INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]);
1915
1916         rc = srpc_add_service(&lstcon_acceptor_service);
1917         LASSERT (rc != -EBUSY);
1918         if (rc != 0) {
1919                 LIBCFS_FREE(console_session.ses_ndl_hash,
1920                             sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
1921                 return rc;
1922         }
1923
1924         n = srpc_service_add_buffers(&lstcon_acceptor_service, SFW_POST_BUFFERS);
1925         if (n != SFW_POST_BUFFERS) {
1926                 rc = -ENOMEM;
1927                 goto out;
1928         }
1929
1930         rc = libcfs_register_ioctl(&lstcon_ioctl_handler);
1931
1932         if (rc == 0) {
1933                 lstcon_rpc_module_init();
1934                 return 0;
1935         }
1936
1937 out:
1938         srpc_shutdown_service(&lstcon_acceptor_service);
1939         srpc_remove_service(&lstcon_acceptor_service);
1940
1941         LIBCFS_FREE(console_session.ses_ndl_hash,
1942                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
1943
1944         srpc_wait_service_shutdown(&lstcon_acceptor_service);
1945
1946         return rc;
1947 }
1948
1949 int
1950 lstcon_console_fini(void)
1951 {
1952         int     i;
1953
1954         mutex_down(&console_session.ses_mutex);
1955
1956         libcfs_deregister_ioctl(&lstcon_ioctl_handler);
1957
1958         srpc_shutdown_service(&lstcon_acceptor_service);
1959         srpc_remove_service(&lstcon_acceptor_service);
1960
1961         if (console_session.ses_state != LST_SESSION_NONE)
1962                 lstcon_session_end();
1963
1964         lstcon_rpc_module_fini();
1965
1966         mutex_up(&console_session.ses_mutex);
1967
1968         LASSERT (list_empty(&console_session.ses_ndl_list));
1969         LASSERT (list_empty(&console_session.ses_grp_list));
1970         LASSERT (list_empty(&console_session.ses_bat_list));
1971         LASSERT (list_empty(&console_session.ses_trans_list));
1972
1973         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1974                 LASSERT (list_empty(&console_session.ses_ndl_hash[i]));
1975         }
1976
1977         LIBCFS_FREE(console_session.ses_ndl_hash,
1978                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
1979
1980         srpc_wait_service_shutdown(&lstcon_acceptor_service);
1981
1982         return 0;
1983 }
1984
1985 #endif