lnet/selftest/console.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * Author: Liang Zhen <liangzhen@clusterfs.com>
   5  *
   6  * This file is part of Lustre, http://www.lustre.org
   7  *
   8  * Infrastructure of LST console
   9  */
  10 #ifdef __KERNEL__
  11
  12 #include <libcfs/libcfs.h>
  13 #include <lnet/lib-lnet.h>
  14 #include "console.h"
  15 #include "conrpc.h"
  16
  17 #define LST_NODE_STATE_COUNTER(nd, p)                   \
  18 do {                                                    \
  19         if ((nd)->nd_state == LST_NODE_ACTIVE)          \
  20                 (p)->nle_nactive ++;                    \
  21         else if ((nd)->nd_state == LST_NODE_BUSY)       \
  22                 (p)->nle_nbusy ++;                      \
  23         else if ((nd)->nd_state == LST_NODE_DOWN)       \
  24                 (p)->nle_ndown ++;                      \
  25         else                                            \
  26                 (p)->nle_nunknown ++;                   \
  27         (p)->nle_nnode ++;                              \
  28 } while (0)
  29
  30 lstcon_session_t        console_session;
  31
  32 void
  33 lstcon_node_get(lstcon_node_t *nd)
  34 {
  35         LASSERT (nd->nd_ref >= 1);
  36
  37         nd->nd_ref++;
  38 }
  39
  40 static int
  41 lstcon_node_find(lnet_process_id_t id, lstcon_node_t **ndpp, int create)
  42 {
  43         lstcon_ndlink_t *ndl;
  44         unsigned int     idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE;
  45
  46         LASSERT (id.nid != LNET_NID_ANY);
  47
  48         list_for_each_entry(ndl, &console_session.ses_ndl_hash[idx], ndl_hlink) {
  49                 if (ndl->ndl_node->nd_id.nid != id.nid ||
  50                     ndl->ndl_node->nd_id.pid != id.pid)
  51                         continue;
  52
  53                 lstcon_node_get(ndl->ndl_node);
  54                 *ndpp = ndl->ndl_node;
  55                 return 0;
  56         }
  57
  58         if (!create)
  59                 return -ENOENT;
  60
  61         LIBCFS_ALLOC(*ndpp, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
  62         if (*ndpp == NULL)
  63                 return -ENOMEM;
  64
  65         ndl = (lstcon_ndlink_t *)(*ndpp + 1);
  66
  67         ndl->ndl_node = *ndpp;
  68
  69         ndl->ndl_node->nd_ref   = 1;
  70         ndl->ndl_node->nd_id    = id;
  71         ndl->ndl_node->nd_stamp = cfs_time_current();
  72         ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
  73         ndl->ndl_node->nd_timeout = 0;
  74         memset(&ndl->ndl_node->nd_ping, 0, sizeof(lstcon_rpc_t));
  75
  76         /* queued in global hash & list, no refcount is taken by
  77          * global hash & list, if caller release his refcount,
  78          * node will be released */
  79         list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]);
  80         list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list);
  81
  82         return 0;
  83 }
  84
  85 void
  86 lstcon_node_put(lstcon_node_t *nd)
  87 {
  88         lstcon_ndlink_t  *ndl;
  89
  90         LASSERT (nd->nd_ref > 0);
  91
  92         if (--nd->nd_ref > 0)
  93                 return;
  94
  95         ndl = (lstcon_ndlink_t *)(nd + 1);
  96
  97         LASSERT (!list_empty(&ndl->ndl_link));
  98         LASSERT (!list_empty(&ndl->ndl_hlink));
  99
 100         /* remove from session */
 101         list_del(&ndl->ndl_link);
 102         list_del(&ndl->ndl_hlink);
 103
 104         LIBCFS_FREE(nd, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
 105 }
 106
 107 static int
 108 lstcon_ndlink_find(struct list_head *hash,
 109                    lnet_process_id_t id, lstcon_ndlink_t **ndlpp, int create)
 110 {
 111         unsigned int     idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
 112         lstcon_ndlink_t *ndl;
 113         lstcon_node_t   *nd;
 114         int              rc;
 115
 116         if (id.nid == LNET_NID_ANY)
 117                 return -EINVAL;
 118
 119         /* search in hash */
 120         list_for_each_entry(ndl, &hash[idx], ndl_hlink) {
 121                 if (ndl->ndl_node->nd_id.nid != id.nid ||
 122                     ndl->ndl_node->nd_id.pid != id.pid)
 123                         continue;
 124
 125                 *ndlpp = ndl;
 126                 return 0;
 127         }
 128
 129         if (create == 0)
 130                 return -ENOENT;
 131
 132         /* find or create in session hash */
 133         rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0);
 134         if (rc != 0)
 135                 return rc;
 136
 137         LIBCFS_ALLOC(ndl, sizeof(lstcon_ndlink_t));
 138         if (ndl == NULL) {
 139                 lstcon_node_put(nd);
 140                 return -ENOMEM;
 141         }
 142
 143         *ndlpp = ndl;
 144
 145         ndl->ndl_node = nd;
 146         CFS_INIT_LIST_HEAD(&ndl->ndl_link);
 147         list_add_tail(&ndl->ndl_hlink, &hash[idx]);
 148
 149         return  0;
 150 }
 151
 152 static void
 153 lstcon_ndlink_release(lstcon_ndlink_t *ndl)
 154 {
 155         LASSERT (list_empty(&ndl->ndl_link));
 156         LASSERT (!list_empty(&ndl->ndl_hlink));
 157
 158         list_del(&ndl->ndl_hlink); /* delete from hash */
 159         lstcon_node_put(ndl->ndl_node);
 160
 161         LIBCFS_FREE(ndl, sizeof(*ndl));
 162 }
 163
 164 static int
 165 lstcon_group_alloc(char *name, lstcon_group_t **grpp)
 166 {
 167         lstcon_group_t *grp;
 168         int             i;
 169
 170         LIBCFS_ALLOC(grp, offsetof(lstcon_group_t,
 171                                    grp_ndl_hash[LST_NODE_HASHSIZE]));
 172         if (grp == NULL)
 173                 return -ENOMEM;
 174
 175         memset(grp, 0, offsetof(lstcon_group_t,
 176                                 grp_ndl_hash[LST_NODE_HASHSIZE]));
 177
 178         grp->grp_ref = 1;
 179         if (name != NULL)
 180                 strcpy(grp->grp_name, name);
 181
 182         CFS_INIT_LIST_HEAD(&grp->grp_link);
 183         CFS_INIT_LIST_HEAD(&grp->grp_ndl_list);
 184         CFS_INIT_LIST_HEAD(&grp->grp_trans_list);
 185
 186         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 187                 CFS_INIT_LIST_HEAD(&grp->grp_ndl_hash[i]);
 188
 189         *grpp = grp;
 190
 191         return 0;
 192 }
 193
 194 static void
 195 lstcon_group_addref(lstcon_group_t *grp)
 196 {
 197         grp->grp_ref ++;
 198 }
 199
 200 static void lstcon_group_ndlink_release(lstcon_group_t *, lstcon_ndlink_t *);
 201
 202 static void
 203 lstcon_group_drain(lstcon_group_t *grp, int keep)
 204 {
 205         lstcon_ndlink_t *ndl;
 206         lstcon_ndlink_t *tmp;
 207
 208         list_for_each_entry_safe(ndl, tmp, &grp->grp_ndl_list, ndl_link) {
 209                 if ((ndl->ndl_node->nd_state & keep) == 0)
 210                         lstcon_group_ndlink_release(grp, ndl);
 211         }
 212 }
 213
 214 static void
 215 lstcon_group_decref(lstcon_group_t *grp)
 216 {
 217         int     i;
 218
 219         if (--grp->grp_ref > 0)
 220                 return;
 221
 222         if (!list_empty(&grp->grp_link))
 223                 list_del(&grp->grp_link);
 224
 225         lstcon_group_drain(grp, 0);
 226
 227         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 228                 LASSERT (list_empty(&grp->grp_ndl_hash[i]));
 229         }
 230
 231         LIBCFS_FREE(grp, offsetof(lstcon_group_t,
 232                                   grp_ndl_hash[LST_NODE_HASHSIZE]));
 233 }
 234
 235 static int
 236 lstcon_group_find(char *name, lstcon_group_t **grpp)
 237 {
 238         lstcon_group_t   *grp;
 239
 240         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 241                 if (strncmp(grp->grp_name, name, LST_NAME_SIZE) != 0)
 242                         continue;
 243
 244                 lstcon_group_addref(grp);  /* +1 ref for caller */
 245                 *grpp = grp;
 246                 return 0;
 247         }
 248
 249         return -ENOENT;
 250 }
 251
 252 static void
 253 lstcon_group_put(lstcon_group_t *grp)
 254 {
 255         lstcon_group_decref(grp);
 256 }
 257
 258 static int
 259 lstcon_group_ndlink_find(lstcon_group_t *grp, lnet_process_id_t id,
 260                          lstcon_ndlink_t **ndlpp, int create)
 261 {
 262         int     rc;
 263
 264         rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create);
 265         if (rc != 0)
 266                 return rc;
 267
 268         if (!list_empty(&(*ndlpp)->ndl_link))
 269                 return 0;
 270
 271         list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list);
 272         grp->grp_nnode ++;
 273
 274         return 0;
 275 }
 276
 277 static void
 278 lstcon_group_ndlink_release(lstcon_group_t *grp, lstcon_ndlink_t *ndl)
 279 {
 280         list_del_init(&ndl->ndl_link);
 281         lstcon_ndlink_release(ndl);
 282         grp->grp_nnode --;
 283 }
 284
 285 static void
 286 lstcon_group_ndlink_move(lstcon_group_t *old,
 287                          lstcon_group_t *new, lstcon_ndlink_t *ndl)
 288 {
 289         unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) %
 290                            LST_NODE_HASHSIZE;
 291
 292         list_del(&ndl->ndl_hlink);
 293         list_del(&ndl->ndl_link);
 294         old->grp_nnode --;
 295
 296         list_add_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]);
 297         list_add_tail(&ndl->ndl_link, &new->grp_ndl_list);
 298         new->grp_nnode ++;
 299
 300         return;
 301 }
 302
 303 static void
 304 lstcon_group_move(lstcon_group_t *old, lstcon_group_t *new)
 305 {
 306         lstcon_ndlink_t *ndl;
 307
 308         while (!list_empty(&old->grp_ndl_list)) {
 309                 ndl = list_entry(old->grp_ndl_list.next,
 310                                  lstcon_ndlink_t, ndl_link);
 311                 lstcon_group_ndlink_move(old, new, ndl);
 312         }
 313 }
 314
 315 int
 316 lstcon_sesrpc_condition(int transop, lstcon_node_t *nd, void *arg)
 317 {
 318         lstcon_group_t *grp = (lstcon_group_t *)arg;
 319
 320         switch (transop) {
 321         case LST_TRANS_SESNEW:
 322                 if (nd->nd_state == LST_NODE_ACTIVE)
 323                         return 0;
 324                 break;
 325
 326         case LST_TRANS_SESEND:
 327                 if (nd->nd_state != LST_NODE_ACTIVE)
 328                         return 0;
 329
 330                 if (grp != NULL && nd->nd_ref > 1)
 331                         return 0;
 332                 break;
 333
 334         case LST_TRANS_SESQRY:
 335                 break;
 336
 337         default:
 338                 LBUG();
 339         }
 340
 341         return 1;
 342 }
 343
 344 int
 345 lstcon_sesrpc_readent(int transop, srpc_msg_t *msg,
 346                       lstcon_rpc_ent_t *ent_up)
 347 {
 348         srpc_debug_reply_t *rep;
 349
 350         switch (transop) {
 351         case LST_TRANS_SESNEW:
 352         case LST_TRANS_SESEND:
 353                 return 0;
 354
 355         case LST_TRANS_SESQRY:
 356                 rep = &msg->msg_body.dbg_reply;
 357
 358                 if (copy_to_user(&ent_up->rpe_priv[0],
 359                                  &rep->dbg_timeout, sizeof(int)) ||
 360                     copy_to_user(&ent_up->rpe_payload[0],
 361                                  &rep->dbg_name, LST_NAME_SIZE))
 362                         return -EFAULT;
 363
 364                 return 0;
 365
 366         default:
 367                 LBUG();
 368         }
 369
 370         return 0;
 371 }
 372
 373 static int
 374 lstcon_group_nodes_add(lstcon_group_t *grp, int count,
 375                        lnet_process_id_t *ids_up, struct list_head *result_up)
 376 {
 377         lstcon_rpc_trans_t      *trans;
 378         lstcon_ndlink_t         *ndl;
 379         lstcon_group_t          *tmp;
 380         lnet_process_id_t        id;
 381         int                      i;
 382         int                      rc;
 383
 384         rc = lstcon_group_alloc(NULL, &tmp);
 385         if (rc != 0) {
 386                 CERROR("Out of memory\n");
 387                 return -ENOMEM;
 388         }
 389
 390         for (i = 0 ; i < count; i++) {
 391                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 392                         rc = -EFAULT;
 393                         break;
 394                 }
 395
 396                 /* skip if it's in this group already */
 397                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 0);
 398                 if (rc == 0)
 399                         continue;
 400
 401                 /* add to tmp group */
 402                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1);
 403                 if (rc != 0) {
 404                         CERROR("Can't create ndlink, out of memory\n");
 405                         break;
 406                 }
 407         }
 408
 409         if (rc != 0) {
 410                 lstcon_group_put(tmp);
 411                 return rc;
 412         }
 413
 414         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 415                                      &tmp->grp_trans_list, LST_TRANS_SESNEW,
 416                                      tmp, lstcon_sesrpc_condition, &trans);
 417         if (rc != 0) {
 418                 CERROR("Can't create transaction: %d\n", rc);
 419                 lstcon_group_put(tmp);
 420                 return rc;
 421         }
 422
 423         /* post all RPCs */
 424         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 425
 426         rc = lstcon_rpc_trans_interpreter(trans, result_up,
 427                                           lstcon_sesrpc_readent);
 428         /* destroy all RPGs */
 429         lstcon_rpc_trans_destroy(trans);
 430
 431         lstcon_group_move(tmp, grp);
 432         lstcon_group_put(tmp);
 433
 434         return rc;
 435 }
 436
 437 static int
 438 lstcon_group_nodes_remove(lstcon_group_t *grp,
 439                           int count, lnet_process_id_t *ids_up,
 440                           struct list_head *result_up)
 441 {
 442         lstcon_rpc_trans_t     *trans;
 443         lstcon_ndlink_t        *ndl;
 444         lstcon_group_t         *tmp;
 445         lnet_process_id_t       id;
 446         int                     rc;
 447         int                     i;
 448
 449         /* End session and remove node from the group */
 450
 451         rc = lstcon_group_alloc(NULL, &tmp);
 452         if (rc != 0) {
 453                 CERROR("Out of memory\n");
 454                 return -ENOMEM;
 455         }
 456
 457         for (i = 0; i < count; i++) {
 458                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 459                         rc = -EFAULT;
 460                         goto error;
 461                 }
 462
 463                 /* move node to tmp group */
 464                 if (lstcon_group_ndlink_find(grp, id, &ndl, 0) == 0)
 465                         lstcon_group_ndlink_move(grp, tmp, ndl);
 466         }
 467
 468         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 469                                      &tmp->grp_trans_list, LST_TRANS_SESEND,
 470                                      tmp, lstcon_sesrpc_condition, &trans);
 471         if (rc != 0) {
 472                 CERROR("Can't create transaction: %d\n", rc);
 473                 goto error;
 474         }
 475
 476         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 477
 478         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 479
 480         lstcon_rpc_trans_destroy(trans);
 481         /* release nodes anyway, because we can't rollback status */
 482         lstcon_group_put(tmp);
 483
 484         return rc;
 485 error:
 486         lstcon_group_move(tmp, grp);
 487         lstcon_group_put(tmp);
 488
 489         return rc;
 490 }
 491
 492 int
 493 lstcon_group_add(char *name)
 494 {
 495         lstcon_group_t *grp;
 496         int             rc;
 497
 498         rc = (lstcon_group_find(name, &grp) == 0)? -EEXIST: 0;
 499         if (rc != 0) {
 500                 /* find a group with same name */
 501                 lstcon_group_put(grp);
 502                 return rc;
 503         }
 504
 505         rc = lstcon_group_alloc(name, &grp);
 506         if (rc != 0) {
 507                 CERROR("Can't allocate descriptor for group %s\n", name);
 508                 return -ENOMEM;
 509         }
 510
 511         list_add_tail(&grp->grp_link, &console_session.ses_grp_list);
 512
 513         return rc;
 514 }
 515
 516 int
 517 lstcon_nodes_add(char *name, int count,
 518                  lnet_process_id_t *ids_up, struct list_head *result_up)
 519 {
 520         lstcon_group_t         *grp;
 521         int                     rc;
 522
 523         LASSERT (count > 0);
 524         LASSERT (ids_up != NULL);
 525
 526         rc = lstcon_group_find(name, &grp);
 527         if (rc != 0) {
 528                 CDEBUG(D_NET, "Can't find group %s\n", name);
 529                 return rc;
 530         }
 531
 532         if (grp->grp_ref > 2) {
 533                 /* referred by other threads or test */
 534                 CDEBUG(D_NET, "Group %s is busy\n", name);
 535                 lstcon_group_put(grp);
 536
 537                 return -EBUSY;
 538         }
 539
 540         rc = lstcon_group_nodes_add(grp, count, ids_up, result_up);
 541
 542         lstcon_group_put(grp);
 543
 544         return rc;
 545 }
 546
 547 int
 548 lstcon_group_del(char *name)
 549 {
 550         lstcon_rpc_trans_t *trans;
 551         lstcon_group_t     *grp;
 552         int                 rc;
 553
 554         rc = lstcon_group_find(name, &grp);
 555         if (rc != 0) {
 556                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 557                 return rc;
 558         }
 559
 560         if (grp->grp_ref > 2) {
 561                 /* referred by others threads or test */
 562                 CDEBUG(D_NET, "Group %s is busy\n", name);
 563                 lstcon_group_put(grp);
 564                 return -EBUSY;
 565         }
 566
 567         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 568                                      &grp->grp_trans_list, LST_TRANS_SESEND,
 569                                      grp, lstcon_sesrpc_condition, &trans);
 570         if (rc != 0) {
 571                 CERROR("Can't create transaction: %d\n", rc);
 572                 lstcon_group_put(grp);
 573                 return rc;
 574         }
 575
 576         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 577
 578         lstcon_rpc_trans_destroy(trans);
 579
 580         lstcon_group_put(grp);
 581         /* -ref for session, it's destroyed,
 582          * status can't be rolled back, destroy group anway */
 583         lstcon_group_put(grp);
 584
 585         return rc;
 586 }
 587
 588 int
 589 lstcon_group_clean(char *name, int args)
 590 {
 591         lstcon_group_t *grp = NULL;
 592         int             rc;
 593
 594         rc = lstcon_group_find(name, &grp);
 595         if (rc != 0) {
 596                 CDEBUG(D_NET, "Can't find group %s\n", name);
 597                 return rc;
 598         }
 599
 600         if (grp->grp_ref > 2) {
 601                 /* referred by test */
 602                 CDEBUG(D_NET, "Group %s is busy\n", name);
 603                 lstcon_group_put(grp);
 604                 return -EBUSY;
 605         }
 606
 607         args = (LST_NODE_ACTIVE | LST_NODE_BUSY |
 608                 LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args;
 609
 610         lstcon_group_drain(grp, args);
 611
 612         lstcon_group_put(grp);
 613         /* release empty group */
 614         if (list_empty(&grp->grp_ndl_list))
 615                 lstcon_group_put(grp);
 616
 617         return 0;
 618 }
 619
 620 int
 621 lstcon_nodes_remove(char *name, int count,
 622                     lnet_process_id_t *ids_up, struct list_head *result_up)
 623 {
 624         lstcon_group_t *grp = NULL;
 625         int             rc;
 626
 627         rc = lstcon_group_find(name, &grp);
 628         if (rc != 0) {
 629                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 630                 return rc;
 631         }
 632
 633         if (grp->grp_ref > 2) {
 634                 /* referred by test */
 635                 CDEBUG(D_NET, "Group %s is busy\n", name);
 636                 lstcon_group_put(grp);
 637                 return -EBUSY;
 638         }
 639
 640         rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up);
 641
 642         lstcon_group_put(grp);
 643         /* release empty group */
 644         if (list_empty(&grp->grp_ndl_list))
 645                 lstcon_group_put(grp);
 646
 647         return rc;
 648 }
 649
 650 int
 651 lstcon_group_refresh(char *name, struct list_head *result_up)
 652 {
 653         lstcon_rpc_trans_t      *trans;
 654         lstcon_group_t          *grp;
 655         int                      rc;
 656
 657         rc = lstcon_group_find(name, &grp);
 658         if (rc != 0) {
 659                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 660                 return rc;
 661         }
 662
 663         if (grp->grp_ref > 2) {
 664                 /* referred by test */
 665                 CDEBUG(D_NET, "Group %s is busy\n", name);
 666                 lstcon_group_put(grp);
 667                 return -EBUSY;
 668         }
 669
 670         /* re-invite all inactive nodes int the group */
 671         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 672                                      &grp->grp_trans_list, LST_TRANS_SESNEW,
 673                                      grp, lstcon_sesrpc_condition, &trans);
 674         if (rc != 0) {
 675                 /* local error, return */
 676                 CDEBUG(D_NET, "Can't create transaction: %d\n", rc);
 677                 lstcon_group_put(grp);
 678                 return rc;
 679         }
 680
 681         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 682
 683         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 684
 685         lstcon_rpc_trans_destroy(trans);
 686         /* -ref for me */
 687         lstcon_group_put(grp);
 688
 689         return rc;
 690 }
 691
 692 int
 693 lstcon_group_list(int index, int len, char *name_up)
 694 {
 695         lstcon_group_t *grp;
 696
 697         LASSERT (index >= 0);
 698         LASSERT (name_up != NULL);
 699
 700         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 701                 if (index-- == 0) {
 702                         return copy_to_user(name_up, grp->grp_name, len) ?
 703                                -EFAULT : 0;
 704                 }
 705         }
 706
 707         return -ENOENT;
 708 }
 709
 710 static int
 711 lstcon_nodes_getent(struct list_head *head, int *index_p,
 712                     int *count_p, lstcon_node_ent_t *dents_up)
 713 {
 714         lstcon_ndlink_t  *ndl;
 715         lstcon_node_t    *nd;
 716         int               count = 0;
 717         int               index = 0;
 718
 719         LASSERT (index_p != NULL && count_p != NULL);
 720         LASSERT (dents_up != NULL);
 721         LASSERT (*index_p >= 0);
 722         LASSERT (*count_p > 0);
 723
 724         list_for_each_entry(ndl, head, ndl_link) {
 725                 if (index++ < *index_p)
 726                         continue;
 727
 728                 if (count >= *count_p)
 729                         break;
 730
 731                 nd = ndl->ndl_node;
 732                 if (copy_to_user(&dents_up[count].nde_id,
 733                                  &nd->nd_id, sizeof(nd->nd_id)) ||
 734                     copy_to_user(&dents_up[count].nde_state,
 735                                  &nd->nd_state, sizeof(nd->nd_state)))
 736                         return -EFAULT;
 737
 738                 count ++;
 739         }
 740
 741         if (index <= *index_p)
 742                 return -ENOENT;
 743
 744         *count_p = count;
 745         *index_p = index;
 746
 747         return 0;
 748 }
 749
 750 int
 751 lstcon_group_info(char *name, lstcon_ndlist_ent_t *gents_p,
 752                   int *index_p, int *count_p, lstcon_node_ent_t *dents_up)
 753 {
 754         lstcon_ndlist_ent_t *gentp;
 755         lstcon_group_t      *grp;
 756         lstcon_ndlink_t     *ndl;
 757         int                  rc;
 758
 759         rc = lstcon_group_find(name, &grp);
 760         if (rc != 0) {
 761                 CDEBUG(D_NET, "Can't find group %s\n", name);
 762                 return rc;
 763         }
 764
 765         if (dents_up != 0) {
 766                 /* verbose query */
 767                 rc = lstcon_nodes_getent(&grp->grp_ndl_list,
 768                                          index_p, count_p, dents_up);
 769                 lstcon_group_put(grp);
 770
 771                 return rc;
 772         }
 773
 774         /* non-verbose query */
 775         LIBCFS_ALLOC(gentp, sizeof(lstcon_ndlist_ent_t));
 776         if (gentp == NULL) {
 777                 CERROR("Can't allocate ndlist_ent\n");
 778                 lstcon_group_put(grp);
 779
 780                 return -ENOMEM;
 781         }
 782
 783         memset(gentp, 0, sizeof(lstcon_ndlist_ent_t));
 784
 785         list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link)
 786                 LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp);
 787
 788         rc = copy_to_user(gents_p, gentp,
 789                           sizeof(lstcon_ndlist_ent_t)) ? -EFAULT: 0;
 790
 791         LIBCFS_FREE(gentp, sizeof(lstcon_ndlist_ent_t));
 792
 793         lstcon_group_put(grp);
 794
 795         return 0;
 796 }
 797
 798 int
 799 lstcon_batch_find(char *name, lstcon_batch_t **batpp)
 800 {
 801         lstcon_batch_t   *bat;
 802
 803         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 804                 if (strncmp(bat->bat_name, name, LST_NAME_SIZE) == 0) {
 805                         *batpp = bat;
 806                         return 0;
 807                 }
 808         }
 809
 810         return -ENOENT;
 811 }
 812
 813 int
 814 lstcon_batch_add(char *name)
 815 {
 816         lstcon_batch_t   *bat;
 817         int               i;
 818         int               rc;
 819
 820         rc = (lstcon_batch_find(name, &bat) == 0)? -EEXIST: 0;
 821         if (rc != 0) {
 822                 CDEBUG(D_NET, "Batch %s already exists\n", name);
 823                 return rc;
 824         }
 825
 826         LIBCFS_ALLOC(bat, sizeof(lstcon_batch_t));
 827         if (bat == NULL) {
 828                 CERROR("Can't allocate descriptor for batch %s\n", name);
 829                 return -ENOMEM;
 830         }
 831
 832         LIBCFS_ALLOC(bat->bat_cli_hash,
 833                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 834         if (bat->bat_cli_hash == NULL) {
 835                 CERROR("Can't allocate hash for batch %s\n", name);
 836                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 837
 838                 return -ENOMEM;
 839         }
 840
 841         LIBCFS_ALLOC(bat->bat_srv_hash,
 842                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 843         if (bat->bat_srv_hash == NULL) {
 844                 CERROR("Can't allocate hash for batch %s\n", name);
 845                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 846                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 847
 848                 return -ENOMEM;
 849         }
 850
 851         strcpy(bat->bat_name, name);
 852         bat->bat_hdr.tsb_index = 0;
 853         bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie;
 854
 855         bat->bat_ntest = 0;
 856         bat->bat_state = LST_BATCH_IDLE;
 857
 858         CFS_INIT_LIST_HEAD(&bat->bat_cli_list);
 859         CFS_INIT_LIST_HEAD(&bat->bat_srv_list);
 860         CFS_INIT_LIST_HEAD(&bat->bat_test_list);
 861         CFS_INIT_LIST_HEAD(&bat->bat_trans_list);
 862
 863         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 864                 CFS_INIT_LIST_HEAD(&bat->bat_cli_hash[i]);
 865                 CFS_INIT_LIST_HEAD(&bat->bat_srv_hash[i]);
 866         }
 867
 868         list_add_tail(&bat->bat_link, &console_session.ses_bat_list);
 869
 870         return rc;
 871 }
 872
 873 int
 874 lstcon_batch_list(int index, int len, char *name_up)
 875 {
 876         lstcon_batch_t    *bat;
 877
 878         LASSERT (name_up != NULL);
 879         LASSERT (index >= 0);
 880
 881         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 882                 if (index-- == 0) {
 883                         return copy_to_user(name_up,bat->bat_name, len) ?
 884                                -EFAULT: 0;
 885                 }
 886         }
 887
 888         return -ENOENT;
 889 }
 890
 891 int
 892 lstcon_batch_info(char *name, lstcon_test_batch_ent_t *ent_up, int server,
 893                   int testidx, int *index_p, int *ndent_p,
 894                   lstcon_node_ent_t *dents_up)
 895 {
 896         lstcon_test_batch_ent_t *entp;
 897         struct list_head        *clilst;
 898         struct list_head        *srvlst;
 899         lstcon_test_t           *test = NULL;
 900         lstcon_batch_t          *bat;
 901         lstcon_ndlink_t         *ndl;
 902         int                      rc;
 903
 904         rc = lstcon_batch_find(name, &bat);
 905         if (rc != 0) {
 906                 CDEBUG(D_NET, "Can't find batch %s\n", name);
 907                 return -ENOENT;
 908         }
 909
 910         if (testidx > 0) {
 911                 /* query test, test index start from 1 */
 912                 list_for_each_entry(test, &bat->bat_test_list, tes_link) {
 913                         if (testidx-- == 1)
 914                                 break;
 915                 }
 916
 917                 if (testidx > 0) {
 918                         CDEBUG(D_NET, "Can't find specified test in batch\n");
 919                         return -ENOENT;
 920                 }
 921         }
 922
 923         clilst = (test == NULL) ? &bat->bat_cli_list :
 924                                   &test->tes_src_grp->grp_ndl_list;
 925         srvlst = (test == NULL) ? &bat->bat_srv_list :
 926                                   &test->tes_dst_grp->grp_ndl_list;
 927
 928         if (dents_up != NULL) {
 929                 rc = lstcon_nodes_getent((server ? srvlst: clilst),
 930                                          index_p, ndent_p, dents_up);
 931                 return rc;
 932         }
 933
 934         /* non-verbose query */
 935         LIBCFS_ALLOC(entp, sizeof(lstcon_test_batch_ent_t));
 936         if (entp == NULL)
 937                 return -ENOMEM;
 938
 939         memset(entp, 0, sizeof(lstcon_test_batch_ent_t));
 940
 941         if (test == NULL) {
 942                 entp->u.tbe_batch.bae_ntest = bat->bat_ntest;
 943                 entp->u.tbe_batch.bae_state = bat->bat_state;
 944
 945         } else {
 946
 947                 entp->u.tbe_test.tse_type   = test->tes_type;
 948                 entp->u.tbe_test.tse_loop   = test->tes_loop;
 949                 entp->u.tbe_test.tse_concur = test->tes_concur;
 950         }
 951
 952         list_for_each_entry(ndl, clilst, ndl_link)
 953                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle);
 954
 955         list_for_each_entry(ndl, srvlst, ndl_link)
 956                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle);
 957
 958         rc = copy_to_user(ent_up, entp,
 959                           sizeof(lstcon_test_batch_ent_t)) ? -EFAULT : 0;
 960
 961         LIBCFS_FREE(entp, sizeof(lstcon_test_batch_ent_t));
 962
 963         return rc;
 964 }
 965
 966 int
 967 lstcon_batrpc_condition(int transop, lstcon_node_t *nd, void *arg)
 968 {
 969         switch (transop) {
 970         case LST_TRANS_TSBRUN:
 971                 if (nd->nd_state != LST_NODE_ACTIVE)
 972                         return -ENETDOWN;
 973                 break;
 974
 975         case LST_TRANS_TSBSTOP:
 976                 if (nd->nd_state != LST_NODE_ACTIVE)
 977                         return 0;
 978                 break;
 979
 980         case LST_TRANS_TSBCLIQRY:
 981         case LST_TRANS_TSBSRVQRY:
 982                 break;
 983         }
 984
 985         return 1;
 986 }
 987
 988 static int
 989 lstcon_batch_op(lstcon_batch_t *bat, int transop, struct list_head *result_up)
 990 {
 991         lstcon_rpc_trans_t *trans;
 992         int                 rc;
 993
 994         rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list,
 995                                      &bat->bat_trans_list, transop,
 996                                      bat, lstcon_batrpc_condition, &trans);
 997         if (rc != 0) {
 998                 CERROR("Can't create transaction: %d\n", rc);
 999                 return rc;
1000         }
1001
1002         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1003
1004         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1005
1006         lstcon_rpc_trans_destroy(trans);
1007
1008         return rc;
1009 }
1010
1011 int
1012 lstcon_batch_run(char *name, int timeout, struct list_head *result_up)
1013 {
1014         lstcon_batch_t *bat;
1015         int             rc;
1016
1017         if (lstcon_batch_find(name, &bat) != 0) {
1018                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1019                 return -ENOENT;
1020         }
1021
1022         bat->bat_arg = timeout;
1023
1024         rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up);
1025
1026         /* mark batch as running if it's started in any node */
1027         if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0) != 0)
1028                 bat->bat_state = LST_BATCH_RUNNING;
1029
1030         return rc;
1031 }
1032
1033 int
1034 lstcon_batch_stop(char *name, int force, struct list_head *result_up)
1035 {
1036         lstcon_batch_t *bat;
1037         int             rc;
1038
1039         if (lstcon_batch_find(name, &bat) != 0) {
1040                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1041                 return -ENOENT;
1042         }
1043
1044         bat->bat_arg = force;
1045
1046         rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up);
1047
1048         /* mark batch as stopped if all RPCs finished */
1049         if (lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0) == 0)
1050                 bat->bat_state = LST_BATCH_IDLE;
1051
1052         return rc;
1053 }
1054
1055 static void
1056 lstcon_batch_destroy(lstcon_batch_t *bat)
1057 {
1058         lstcon_ndlink_t    *ndl;
1059         lstcon_test_t      *test;
1060         int                 i;
1061
1062         list_del(&bat->bat_link);
1063
1064         while (!list_empty(&bat->bat_test_list)) {
1065                 test = list_entry(bat->bat_test_list.next,
1066                                   lstcon_test_t, tes_link);
1067                 LASSERT (list_empty(&test->tes_trans_list));
1068
1069                 list_del(&test->tes_link);
1070
1071                 lstcon_group_put(test->tes_src_grp);
1072                 lstcon_group_put(test->tes_dst_grp);
1073
1074                 LIBCFS_FREE(test, offsetof(lstcon_test_t,
1075                                            tes_param[test->tes_paramlen]));
1076         }
1077
1078         LASSERT (list_empty(&bat->bat_trans_list));
1079
1080         while (!list_empty(&bat->bat_cli_list)) {
1081                 ndl = list_entry(bat->bat_cli_list.next,
1082                                  lstcon_ndlink_t, ndl_link);
1083                 list_del_init(&ndl->ndl_link);
1084
1085                 lstcon_ndlink_release(ndl);
1086         }
1087
1088         while (!list_empty(&bat->bat_srv_list)) {
1089                 ndl = list_entry(bat->bat_srv_list.next,
1090                                  lstcon_ndlink_t, ndl_link);
1091                 list_del_init(&ndl->ndl_link);
1092
1093                 lstcon_ndlink_release(ndl);
1094         }
1095
1096         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1097                 LASSERT (list_empty(&bat->bat_cli_hash[i]));
1098                 LASSERT (list_empty(&bat->bat_srv_hash[i]));
1099         }
1100
1101         LIBCFS_FREE(bat->bat_cli_hash,
1102                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1103         LIBCFS_FREE(bat->bat_srv_hash,
1104                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1105         LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
1106 }
1107
1108 int
1109 lstcon_testrpc_condition(int transop, lstcon_node_t *nd, void *arg)
1110 {
1111         lstcon_test_t    *test;
1112         lstcon_batch_t   *batch;
1113         lstcon_ndlink_t  *ndl;
1114         struct list_head *hash;
1115         struct list_head *head;
1116
1117         test = (lstcon_test_t *)arg;
1118         LASSERT (test != NULL);
1119
1120         batch = test->tes_batch;
1121         LASSERT (batch != NULL);
1122
1123         if (test->tes_oneside &&
1124             transop == LST_TRANS_TSBSRVADD)
1125                 return 0;
1126
1127         if (nd->nd_state != LST_NODE_ACTIVE)
1128                 return -ENETDOWN;
1129
1130         if (transop == LST_TRANS_TSBCLIADD) {
1131                 hash = batch->bat_cli_hash;
1132                 head = &batch->bat_cli_list;
1133
1134         } else {
1135                 LASSERT (transop == LST_TRANS_TSBSRVADD);
1136
1137                 hash = batch->bat_srv_hash;
1138                 head = &batch->bat_srv_list;
1139         }
1140
1141         LASSERT (nd->nd_id.nid != LNET_NID_ANY);
1142
1143         if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1) != 0)
1144                 return -ENOMEM;
1145
1146         if (list_empty(&ndl->ndl_link))
1147                 list_add_tail(&ndl->ndl_link, head);
1148
1149         return 1;
1150 }
1151
1152 static int
1153 lstcon_test_nodes_add(lstcon_test_t *test, struct list_head *result_up)
1154 {
1155         lstcon_rpc_trans_t     *trans;
1156         lstcon_group_t         *grp;
1157         int                     transop;
1158         int                     rc;
1159
1160         LASSERT (test->tes_src_grp != NULL);
1161         LASSERT (test->tes_dst_grp != NULL);
1162
1163         transop = LST_TRANS_TSBSRVADD;
1164         grp  = test->tes_dst_grp;
1165 again:
1166         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
1167                                      &test->tes_trans_list, transop,
1168                                      test, lstcon_testrpc_condition, &trans);
1169         if (rc != 0) {
1170                 CERROR("Can't create transaction: %d\n", rc);
1171                 return rc;
1172         }
1173
1174         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1175
1176         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1177             lstcon_trans_stat()->trs_fwk_errno != 0) {
1178                 lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1179
1180                 lstcon_rpc_trans_destroy(trans);
1181                 /* return if any error */
1182                 CDEBUG(D_NET, "Failed to add test %s, "
1183                               "RPC error %d, framework error %d\n",
1184                        transop == LST_TRANS_TSBCLIADD ? "client" : "server",
1185                        lstcon_trans_stat()->trs_rpc_errno,
1186                        lstcon_trans_stat()->trs_fwk_errno);
1187
1188                 return rc;
1189         }
1190
1191         lstcon_rpc_trans_destroy(trans);
1192
1193         if (transop == LST_TRANS_TSBCLIADD)
1194                 return rc;
1195
1196         transop = LST_TRANS_TSBCLIADD;
1197         grp = test->tes_src_grp;
1198         test->tes_cliidx = 0;
1199
1200         /* requests to test clients */
1201         goto again;
1202 }
1203
1204 int
1205 lstcon_test_add(char *name, int type, int loop, int concur,
1206                 int dist, int span, char *src_name, char * dst_name,
1207                 void *param, int paramlen, struct list_head *result_up)
1208
1209 {
1210         lstcon_group_t  *src_grp = NULL;
1211         lstcon_group_t  *dst_grp = NULL;
1212         lstcon_test_t   *test    = NULL;
1213         lstcon_batch_t  *batch;
1214         int              rc;
1215
1216         rc = lstcon_batch_find(name, &batch);
1217         if (rc != 0) {
1218                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1219                 return rc;
1220         }
1221
1222         if (batch->bat_state != LST_BATCH_IDLE) {
1223                 CDEBUG(D_NET, "Can't change running batch %s\n", name);
1224                 return rc;
1225         }
1226
1227         rc = lstcon_group_find(src_name, &src_grp);
1228         if (rc != 0) {
1229                 CDEBUG(D_NET, "Can't find group %s\n", src_name);
1230                 goto out;
1231         }
1232
1233         rc = lstcon_group_find(dst_name, &dst_grp);
1234         if (rc != 0) {
1235                 CDEBUG(D_NET, "Can't find group %s\n", dst_name);
1236                 goto out;
1237         }
1238
1239         LIBCFS_ALLOC(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1240         if (!test) {
1241                 CERROR("Can't allocate test descriptor\n");
1242                 rc = -ENOMEM;
1243
1244                 goto out;
1245         }
1246
1247         memset(test, 0, offsetof(lstcon_test_t, tes_param[paramlen]));
1248         test->tes_hdr.tsb_id    = batch->bat_hdr.tsb_id;
1249         test->tes_batch         = batch;
1250         test->tes_type          = type;
1251         test->tes_oneside       = 0; /* TODO */
1252         test->tes_loop          = loop;
1253         test->tes_concur        = concur;
1254         test->tes_stop_onerr    = 1; /* TODO */
1255         test->tes_span          = span;
1256         test->tes_dist          = dist;
1257         test->tes_cliidx        = 0; /* just used for creating RPC */
1258         test->tes_src_grp       = src_grp;
1259         test->tes_dst_grp       = dst_grp;
1260         CFS_INIT_LIST_HEAD(&test->tes_trans_list);
1261
1262         if (param != NULL) {
1263                 test->tes_paramlen = paramlen;
1264                 memcpy(&test->tes_param[0], param, paramlen);
1265         }
1266
1267         rc = lstcon_test_nodes_add(test, result_up);
1268
1269         if (rc != 0)
1270                 goto out;
1271
1272         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1273             lstcon_trans_stat()->trs_fwk_errno != 0)
1274                 CDEBUG(D_NET, "Failed to add test %d to batch %s", type, name);
1275
1276         /* add to test list anyway, so user can check what's going on */
1277         list_add_tail(&test->tes_link, &batch->bat_test_list);
1278
1279         batch->bat_ntest ++;
1280         test->tes_hdr.tsb_index = batch->bat_ntest;
1281
1282         /*  hold groups so nobody can change them */
1283         return rc;
1284 out:
1285         if (test != NULL)
1286                 LIBCFS_FREE(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1287
1288         if (dst_grp != NULL)
1289                 lstcon_group_put(dst_grp);
1290
1291         if (src_grp != NULL)
1292                 lstcon_group_put(src_grp);
1293
1294         return rc;
1295 }
1296
1297 int
1298 lstcon_test_find(lstcon_batch_t *batch, int idx, lstcon_test_t **testpp)
1299 {
1300         lstcon_test_t *test;
1301
1302         list_for_each_entry(test, &batch->bat_test_list, tes_link) {
1303                 if (idx == test->tes_hdr.tsb_index) {
1304                         *testpp = test;
1305                         return 0;
1306                 }
1307         }
1308
1309         return -ENOENT;
1310 }
1311
1312 int
1313 lstcon_tsbrpc_readent(int transop, srpc_msg_t *msg,
1314                       lstcon_rpc_ent_t *ent_up)
1315 {
1316         srpc_batch_reply_t *rep = &msg->msg_body.bat_reply;
1317
1318         LASSERT (transop == LST_TRANS_TSBCLIQRY ||
1319                  transop == LST_TRANS_TSBSRVQRY);
1320
1321         /* positive errno, framework error code */
1322         if (copy_to_user(&ent_up->rpe_priv[0],
1323                          &rep->bar_active, sizeof(rep->bar_active)))
1324                 return -EFAULT;
1325
1326         return 0;
1327 }
1328
1329 int
1330 lstcon_test_batch_query(char *name, int testidx, int client,
1331                         int timeout, struct list_head *result_up)
1332 {
1333         lstcon_rpc_trans_t *trans;
1334         struct list_head   *translist;
1335         struct list_head   *ndlist;
1336         lstcon_tsb_hdr_t   *hdr;
1337         lstcon_batch_t     *batch;
1338         lstcon_test_t      *test = NULL;
1339         int                 transop;
1340         int                 rc;
1341
1342         rc = lstcon_batch_find(name, &batch);
1343         if (rc != 0) {
1344                 CDEBUG(D_NET, "Can't find batch: %s\n", name);
1345                 return rc;
1346         }
1347
1348         if (testidx == 0) {
1349                 translist = &batch->bat_trans_list;
1350                 ndlist    = &batch->bat_cli_list;
1351                 hdr       = &batch->bat_hdr;
1352
1353         } else {
1354                 /* query specified test only */
1355                 rc = lstcon_test_find(batch, testidx, &test);
1356                 if (rc != 0) {
1357                         CDEBUG(D_NET, "Can't find test: %d\n", testidx);
1358                         return rc;
1359                 }
1360
1361                 translist = &test->tes_trans_list;
1362                 ndlist    = &test->tes_src_grp->grp_ndl_list;
1363                 hdr       = &test->tes_hdr;
1364         }
1365
1366         transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY;
1367
1368         rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr,
1369                                      lstcon_batrpc_condition, &trans);
1370         if (rc != 0) {
1371                 CERROR("Can't create transaction: %d\n", rc);
1372                 return rc;
1373         }
1374
1375         lstcon_rpc_trans_postwait(trans, timeout);
1376
1377         if (testidx == 0 && /* query a batch, not a test */
1378             lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) == 0 &&
1379             lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0) == 0) {
1380                 /* all RPCs finished, and no active test */
1381                 batch->bat_state = LST_BATCH_IDLE;
1382         }
1383
1384         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1385                                           lstcon_tsbrpc_readent);
1386         lstcon_rpc_trans_destroy(trans);
1387
1388         return rc;
1389 }
1390
1391 int
1392 lstcon_statrpc_readent(int transop, srpc_msg_t *msg,
1393                        lstcon_rpc_ent_t *ent_up)
1394 {
1395         srpc_stat_reply_t *rep = &msg->msg_body.stat_reply;
1396         sfw_counters_t    *sfwk_stat;
1397         srpc_counters_t   *srpc_stat;
1398         lnet_counters_t   *lnet_stat;
1399
1400         if (rep->str_status != 0)
1401                 return 0;
1402
1403         sfwk_stat = (sfw_counters_t *)&ent_up->rpe_payload[0];
1404         srpc_stat = (srpc_counters_t *)((char *)sfwk_stat + sizeof(*sfwk_stat));
1405         lnet_stat = (lnet_counters_t *)((char *)srpc_stat + sizeof(*srpc_stat));
1406
1407         if (copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) ||
1408             copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) ||
1409             copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat)))
1410                 return -EFAULT;
1411
1412         return 0;
1413 }
1414
1415 int
1416 lstcon_ndlist_stat(struct list_head *ndlist,
1417                    int timeout, struct list_head *result_up)
1418 {
1419         struct list_head    head;
1420         lstcon_rpc_trans_t *trans;
1421         int                 rc;
1422
1423         CFS_INIT_LIST_HEAD(&head);
1424
1425         rc = lstcon_rpc_trans_ndlist(ndlist, &head,
1426                                      LST_TRANS_STATQRY, NULL, NULL, &trans);
1427         if (rc != 0) {
1428                 CERROR("Can't create transaction: %d\n", rc);
1429                 return rc;
1430         }
1431
1432         timeout = (timeout > LST_TRANS_MIN_TIMEOUT) ? timeout :
1433                                                       LST_TRANS_MIN_TIMEOUT;
1434         lstcon_rpc_trans_postwait(trans, timeout);
1435
1436         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1437                                           lstcon_statrpc_readent);
1438         lstcon_rpc_trans_destroy(trans);
1439
1440         return rc;
1441 }
1442
1443 int
1444 lstcon_group_stat(char *grp_name, int timeout, struct list_head *result_up)
1445 {
1446         lstcon_group_t     *grp;
1447         int                 rc;
1448
1449         rc = lstcon_group_find(grp_name, &grp);
1450         if (rc != 0) {
1451                 CDEBUG(D_NET, "Can't find group %s\n", grp_name);
1452                 return rc;
1453         }
1454
1455         rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up);
1456
1457         lstcon_group_put(grp);
1458
1459         return rc;
1460 }
1461
1462 int
1463 lstcon_nodes_stat(int count, lnet_process_id_t *ids_up,
1464                   int timeout, struct list_head *result_up)
1465 {
1466         lstcon_ndlink_t         *ndl;
1467         lstcon_group_t          *tmp;
1468         lnet_process_id_t        id;
1469         int                      i;
1470         int                      rc;
1471
1472         rc = lstcon_group_alloc(NULL, &tmp);
1473         if (rc != 0) {
1474                 CERROR("Out of memory\n");
1475                 return -ENOMEM;
1476         }
1477
1478         for (i = 0 ; i < count; i++) {
1479                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1480                         rc = -EFAULT;
1481                         break;
1482                 }
1483
1484                 /* add to tmp group */
1485                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2);
1486                 if (rc != 0) {
1487                         CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET,
1488                                "Failed to find or create %s: %d\n",
1489                                libcfs_id2str(id), rc);
1490                         break;
1491                 }
1492         }
1493
1494         if (rc != 0) {
1495                 lstcon_group_put(tmp);
1496                 return rc;
1497         }
1498
1499         rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up);
1500
1501         lstcon_group_put(tmp);
1502
1503         return rc;
1504 }
1505
1506 int
1507 lstcon_debug_ndlist(struct list_head *ndlist,
1508                     struct list_head *translist,
1509                     int timeout, struct list_head *result_up)
1510 {
1511         lstcon_rpc_trans_t *trans;
1512         int                 rc;
1513
1514         rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY,
1515                                      NULL, lstcon_sesrpc_condition, &trans);
1516         if (rc != 0) {
1517                 CERROR("Can't create transaction: %d\n", rc);
1518                 return rc;
1519         }
1520
1521         timeout = (timeout > LST_TRANS_MIN_TIMEOUT) ? timeout :
1522                                                       LST_TRANS_MIN_TIMEOUT;
1523
1524         lstcon_rpc_trans_postwait(trans, timeout);
1525
1526         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1527                                           lstcon_sesrpc_readent);
1528         lstcon_rpc_trans_destroy(trans);
1529
1530         return rc;
1531 }
1532
1533 int
1534 lstcon_session_debug(int timeout, struct list_head *result_up)
1535 {
1536         return lstcon_debug_ndlist(&console_session.ses_ndl_list,
1537                                    NULL, timeout, result_up);
1538 }
1539
1540 int
1541 lstcon_batch_debug(int timeout, char *name,
1542                    int client, struct list_head *result_up)
1543 {
1544         lstcon_batch_t *bat;
1545         int             rc;
1546
1547         rc = lstcon_batch_find(name, &bat);
1548         if (rc != 0)
1549                 return -ENOENT;
1550
1551         rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list :
1552                                           &bat->bat_srv_list,
1553                                  NULL, timeout, result_up);
1554
1555         return rc;
1556 }
1557
1558 int
1559 lstcon_group_debug(int timeout, char *name,
1560                    struct list_head *result_up)
1561 {
1562         lstcon_group_t *grp;
1563         int             rc;
1564
1565         rc = lstcon_group_find(name, &grp);
1566         if (rc != 0)
1567                 return -ENOENT;
1568
1569         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1570                                  timeout, result_up);
1571         lstcon_group_put(grp);
1572
1573         return rc;
1574 }
1575
1576 int
1577 lstcon_nodes_debug(int timeout,
1578                    int count, lnet_process_id_t *ids_up,
1579                    struct list_head *result_up)
1580 {
1581         lnet_process_id_t  id;
1582         lstcon_ndlink_t   *ndl;
1583         lstcon_group_t    *grp;
1584         int                i;
1585         int                rc;
1586
1587         rc = lstcon_group_alloc(NULL, &grp);
1588         if (rc != 0) {
1589                 CDEBUG(D_NET, "Out of memory\n");
1590                 return rc;
1591         }
1592
1593         for (i = 0; i < count; i++) {
1594                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1595                         rc = -EFAULT;
1596                         break;
1597                 }
1598
1599                 /* node is added to tmp group */
1600                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 1);
1601                 if (rc != 0) {
1602                         CERROR("Can't create node link\n");
1603                         break;
1604                 }
1605         }
1606
1607         if (rc != 0) {
1608                 lstcon_group_put(grp);
1609                 return rc;
1610         }
1611
1612         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1613                                  timeout, result_up);
1614
1615         lstcon_group_put(grp);
1616
1617         return rc;
1618 }
1619
1620 int
1621 lstcon_session_match(lst_sid_t sid)
1622 {
1623         return (console_session.ses_id.ses_nid   == sid.ses_nid &&
1624                 console_session.ses_id.ses_stamp == sid.ses_stamp) ?  1: 0;
1625 }
1626
1627 static void
1628 lstcon_new_session_id(lst_sid_t *sid)
1629 {
1630         lnet_process_id_t      id;
1631
1632         LASSERT (console_session.ses_state == LST_SESSION_NONE);
1633
1634         LNetGetId(1, &id);
1635         sid->ses_nid   = id.nid;
1636         sid->ses_stamp = cfs_time_current();
1637 }
1638
1639 extern srpc_service_t lstcon_acceptor_service;
1640
1641 int
1642 lstcon_session_new(char *name, int key,
1643                    int timeout,int force, lst_sid_t *sid_up)
1644 {
1645         int     rc = 0;
1646         int     i;
1647
1648         if (console_session.ses_state != LST_SESSION_NONE) {
1649                 /* session exists */
1650                 if (!force) {
1651                         CERROR("Session %s already exists\n",
1652                                console_session.ses_name);
1653                         return -EEXIST;
1654                 }
1655
1656                 rc = lstcon_session_end();
1657
1658                 /* lstcon_session_end() only return local error */
1659                 if  (rc != 0)
1660                         return rc;
1661         }
1662
1663         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++) {
1664                 LASSERT (list_empty(&console_session.ses_ndl_hash[i]));
1665         }
1666
1667         rc = lstcon_batch_add(LST_DEFAULT_BATCH);
1668         if (rc != 0)
1669                 return rc;
1670
1671         rc = lstcon_rpc_pinger_start();
1672         if (rc != 0) {
1673                 lstcon_batch_t *bat;
1674
1675                 lstcon_batch_find(LST_DEFAULT_BATCH, &bat);
1676                 lstcon_batch_destroy(bat);
1677
1678                 return rc;
1679         }
1680
1681         lstcon_new_session_id(&console_session.ses_id);
1682
1683         console_session.ses_key     = key;
1684         console_session.ses_state   = LST_SESSION_ACTIVE;
1685         console_session.ses_force   = !!force;
1686         console_session.ses_timeout = (timeout <= 0)? LST_CONSOLE_TIMEOUT:
1687                                                       timeout;
1688         strcpy(console_session.ses_name, name);
1689
1690         if (copy_to_user(sid_up, &console_session.ses_id,
1691                          sizeof(lst_sid_t)) == 0)
1692                 return rc;
1693
1694         lstcon_session_end();
1695
1696         return -EFAULT;
1697 }
1698
1699 int
1700 lstcon_session_info(lst_sid_t *sid_up, int *key_up,
1701                     lstcon_ndlist_ent_t *ndinfo_up, char *name_up, int len)
1702 {
1703         lstcon_ndlist_ent_t *entp;
1704         lstcon_ndlink_t     *ndl;
1705         int                  rc = 0;
1706
1707         if (console_session.ses_state != LST_SESSION_ACTIVE)
1708                 return -ESRCH;
1709
1710         LIBCFS_ALLOC(entp, sizeof(*entp));
1711         if (entp == NULL)
1712                 return -ENOMEM;
1713
1714         memset(entp, 0, sizeof(*entp));
1715
1716         list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link)
1717                 LST_NODE_STATE_COUNTER(ndl->ndl_node, entp);
1718
1719         if (copy_to_user(sid_up, &console_session.ses_id, sizeof(lst_sid_t)) ||
1720             copy_to_user(key_up, &console_session.ses_key, sizeof(int)) ||
1721             copy_to_user(ndinfo_up, entp, sizeof(*entp)) ||
1722             copy_to_user(name_up, console_session.ses_name, len))
1723                 rc = -EFAULT;
1724
1725         LIBCFS_FREE(entp, sizeof(*entp));
1726
1727         return rc;
1728 }
1729
1730 int
1731 lstcon_session_end()
1732 {
1733         lstcon_rpc_trans_t *trans;
1734         lstcon_group_t     *grp;
1735         lstcon_batch_t     *bat;
1736         int                 rc = 0;
1737
1738         LASSERT (console_session.ses_state == LST_SESSION_ACTIVE);
1739
1740         rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list,
1741                                      NULL, LST_TRANS_SESEND, NULL,
1742                                      lstcon_sesrpc_condition, &trans);
1743         if (rc != 0) {
1744                 CERROR("Can't create transaction: %d\n", rc);
1745                 return rc;
1746         }
1747
1748         console_session.ses_shutdown = 1;
1749
1750         lstcon_rpc_pinger_stop();
1751
1752         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1753
1754         lstcon_rpc_trans_destroy(trans);
1755         /* User can do nothing even rpc failed, so go on */
1756
1757         /* waiting for orphan rpcs to die */
1758         lstcon_rpc_cleanup_wait();
1759
1760         console_session.ses_id    = LST_INVALID_SID;
1761         console_session.ses_state = LST_SESSION_NONE;
1762         console_session.ses_key   = 0;
1763         console_session.ses_force = 0;
1764
1765         /* destroy all batches */
1766         while (!list_empty(&console_session.ses_bat_list)) {
1767                 bat = list_entry(console_session.ses_bat_list.next,
1768                                  lstcon_batch_t, bat_link);
1769
1770                 lstcon_batch_destroy(bat);
1771         }
1772
1773         /* destroy all groups */
1774         while (!list_empty(&console_session.ses_grp_list)) {
1775                 grp = list_entry(console_session.ses_grp_list.next,
1776                                  lstcon_group_t, grp_link);
1777                 LASSERT (grp->grp_ref == 1);
1778
1779                 lstcon_group_put(grp);
1780         }
1781
1782         /* all nodes should be released */
1783         LASSERT (list_empty(&console_session.ses_ndl_list));
1784
1785         console_session.ses_shutdown = 0;
1786         console_session.ses_expired  = 0;
1787
1788         return rc;
1789 }
1790
1791 static int
1792 lstcon_acceptor_handle (srpc_server_rpc_t *rpc)
1793 {
1794         srpc_msg_t        *rep  = &rpc->srpc_replymsg;
1795         srpc_msg_t        *req  = &rpc->srpc_reqstbuf->buf_msg;
1796         srpc_join_reqst_t *jreq = &req->msg_body.join_reqst;
1797         srpc_join_reply_t *jrep = &rep->msg_body.join_reply;
1798         lstcon_group_t    *grp  = NULL;
1799         lstcon_ndlink_t   *ndl;
1800         int                rc   = 0;
1801
1802         sfw_unpack_message(req);
1803
1804         mutex_down(&console_session.ses_mutex);
1805
1806         jrep->join_sid = console_session.ses_id;
1807
1808         if (console_session.ses_id.ses_nid == LNET_NID_ANY) {
1809                 jrep->join_status = ESRCH;
1810                 goto out;
1811         }
1812
1813         if (jreq->join_sid.ses_nid != LNET_NID_ANY &&
1814              !lstcon_session_match(jreq->join_sid)) {
1815                 jrep->join_status = EBUSY;
1816                 goto out;
1817         }
1818
1819         if (lstcon_group_find(jreq->join_group, &grp) != 0) {
1820                 rc = lstcon_group_alloc(jreq->join_group, &grp);
1821                 if (rc != 0) {
1822                         CERROR("Out of memory\n");
1823                         goto out;
1824                 }
1825
1826                 list_add_tail(&grp->grp_link,
1827                               &console_session.ses_grp_list);
1828                 lstcon_group_addref(grp);
1829         }
1830
1831         if (grp->grp_ref > 2) {
1832                 /* Group in using */
1833                 jrep->join_status = EBUSY;
1834                 goto out;
1835         }
1836
1837         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0);
1838         if (rc == 0) {
1839                 jrep->join_status = EEXIST;
1840                 goto out;
1841         }
1842
1843         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1);
1844         if (rc != 0) {
1845                 CERROR("Out of memory\n");
1846                 goto out;
1847         }
1848
1849         ndl->ndl_node->nd_state   = LST_NODE_ACTIVE;
1850         ndl->ndl_node->nd_timeout = console_session.ses_timeout;
1851
1852         strcpy(jrep->join_session, console_session.ses_name);
1853         jrep->join_timeout = console_session.ses_timeout;
1854         jrep->join_status  = 0;
1855
1856 out:
1857         if (grp != NULL)
1858                 lstcon_group_put(grp);
1859
1860         mutex_up(&console_session.ses_mutex);
1861
1862         return rc;
1863 }
1864
1865 srpc_service_t lstcon_acceptor_service =
1866 {
1867         .sv_name        = "join session",
1868         .sv_handler     = lstcon_acceptor_handle,
1869         .sv_bulk_ready  = NULL,
1870         .sv_id          = SRPC_SERVICE_JOIN,
1871         .sv_concur      = SFW_SERVICE_CONCURRENCY,
1872 };
1873
1874 extern int lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_data *data);
1875
1876 DECLARE_IOCTL_HANDLER(lstcon_ioctl_handler, lstcon_ioctl_entry);
1877
1878 /* initialize console */
1879 int
1880 lstcon_console_init(void)
1881 {
1882         int     i;
1883         int     n;
1884         int     rc;
1885
1886         memset(&console_session, 0, sizeof(lstcon_session_t));
1887
1888         console_session.ses_id      = LST_INVALID_SID;
1889         console_session.ses_state   = LST_SESSION_NONE;
1890         console_session.ses_timeout = 0;
1891         console_session.ses_force   = 0;
1892         console_session.ses_expired = 0;
1893         console_session.ses_laststamp = cfs_time_current_sec();
1894
1895         init_mutex(&console_session.ses_mutex);
1896
1897         CFS_INIT_LIST_HEAD(&console_session.ses_ndl_list);
1898         CFS_INIT_LIST_HEAD(&console_session.ses_grp_list);
1899         CFS_INIT_LIST_HEAD(&console_session.ses_bat_list);
1900         CFS_INIT_LIST_HEAD(&console_session.ses_trans_list);
1901
1902         LIBCFS_ALLOC(console_session.ses_ndl_hash,
1903                      sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
1904         if (console_session.ses_ndl_hash == NULL)
1905                 return -ENOMEM;
1906
1907         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
1908                 CFS_INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]);
1909
1910         rc = srpc_add_service(&lstcon_acceptor_service);
1911         LASSERT (rc != -EBUSY);
1912         if (rc != 0) {
1913                 LIBCFS_FREE(console_session.ses_ndl_hash,
1914                             sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
1915                 return rc;
1916         }
1917
1918         n = srpc_service_add_buffers(&lstcon_acceptor_service, SFW_POST_BUFFERS);
1919         if (n != SFW_POST_BUFFERS) {
1920                 rc = -ENOMEM;
1921                 goto out;
1922         }
1923
1924         rc = libcfs_register_ioctl(&lstcon_ioctl_handler);
1925
1926         if (rc == 0) {
1927                 lstcon_rpc_module_init();
1928                 return 0;
1929         }
1930
1931 out:
1932         srpc_shutdown_service(&lstcon_acceptor_service);
1933         srpc_remove_service(&lstcon_acceptor_service);
1934
1935         LIBCFS_FREE(console_session.ses_ndl_hash,
1936                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
1937
1938         srpc_wait_service_shutdown(&lstcon_acceptor_service);
1939
1940         return rc;
1941 }
1942
1943 int
1944 lstcon_console_fini(void)
1945 {
1946         int     i;
1947
1948         mutex_down(&console_session.ses_mutex);
1949
1950         libcfs_deregister_ioctl(&lstcon_ioctl_handler);
1951
1952         srpc_shutdown_service(&lstcon_acceptor_service);
1953         srpc_remove_service(&lstcon_acceptor_service);
1954
1955         if (console_session.ses_state != LST_SESSION_NONE)
1956                 lstcon_session_end();
1957
1958         lstcon_rpc_module_fini();
1959
1960         mutex_up(&console_session.ses_mutex);
1961
1962         LASSERT (list_empty(&console_session.ses_ndl_list));
1963         LASSERT (list_empty(&console_session.ses_grp_list));
1964         LASSERT (list_empty(&console_session.ses_bat_list));
1965         LASSERT (list_empty(&console_session.ses_trans_list));
1966
1967         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1968                 LASSERT (list_empty(&console_session.ses_ndl_hash[i]));
1969         }
1970
1971         LIBCFS_FREE(console_session.ses_ndl_hash,
1972                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
1973
1974         srpc_wait_service_shutdown(&lstcon_acceptor_service);
1975
1976         return 0;
1977 }
1978
1979 #endif