lnet/utils/lst.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * Author: Liang Zhen <liangzhen@clusterfs.com>
   5  *
   6  * This file is part of Lustre, http://www.lustre.org
   7  */
   8
   9 #include <stdio.h>
  10 #include <stdlib.h>
  11 #include <getopt.h>
  12 #include <errno.h>
  13 #include <pwd.h>
  14 #include <lnet/lnetctl.h>
  15 #include <lnet/lnetst.h>
  16 #include "parser.h"
  17
  18 static command_t           lst_cmdlist[];
  19 static lst_sid_t           session_id;
  20 static int                 session_key;
  21 static lstcon_trans_stat_t trans_stat;
  22
  23 typedef struct list_string {
  24         struct list_string *lstr_next;
  25         int                 lstr_sz;
  26         char                lstr_str[0];
  27 } lstr_t;
  28
  29 #define offsetof(typ,memb)     ((unsigned long)((char *)&(((typ *)0)->memb)))
  30
  31 static int alloc_count = 0;
  32 static int alloc_nob   = 0;
  33
  34 lstr_t *
  35 alloc_lstr(int sz)
  36 {
  37         lstr_t  *lstr = malloc(offsetof(lstr_t, lstr_str[sz]));
  38
  39         if (lstr == NULL) {
  40                 fprintf(stderr, "Can't allocate lstr\n");
  41                 abort();
  42         }
  43
  44         alloc_nob += sz;
  45         alloc_count++;
  46
  47         lstr->lstr_str[0] = 0;
  48         lstr->lstr_sz = sz;
  49         return lstr;
  50 }
  51
  52 void
  53 free_lstr(lstr_t *lstr)
  54 {
  55         alloc_count--;
  56         alloc_nob -= lstr->lstr_sz;
  57         free(lstr);
  58 }
  59
  60 void
  61 free_lstrs(lstr_t **list)
  62 {
  63         lstr_t   *lstr;
  64
  65         while ((lstr = *list) != NULL) {
  66                 *list = lstr->lstr_next;
  67                 free_lstr(lstr);
  68         }
  69 }
  70
  71 new_lstrs(lstr_t **list, char *prefix, char *postfix,
  72           int lo, int hi, int stride)
  73 {
  74         int    n1 = strlen(prefix);
  75         int    n2 = strlen(postfix);
  76         int    sz = n1 + 20 + n2 + 1;
  77
  78         do {
  79                 lstr_t *n = alloc_lstr(sz);
  80
  81                 snprintf(n->lstr_str, sz - 1, "%s%u%s",
  82                          prefix, lo, postfix);
  83
  84                 n->lstr_next = *list;
  85                 *list = n;
  86
  87                 lo += stride;
  88         } while (lo <= hi);
  89 }
  90
  91 int
  92 expand_lstr(lstr_t **list, lstr_t *l)
  93 {
  94         int          nob = strlen(l->lstr_str);
  95         char        *b1;
  96         char        *b2;
  97         char        *expr;
  98         char        *sep;
  99         int          x;
 100         int          y;
 101         int          z;
 102         int          n;
 103
 104         b1 = strchr(l->lstr_str, '[');
 105         if (b1 == NULL) {
 106                 l->lstr_next = *list;
 107                 *list = l;
 108                 return 0;
 109         }
 110
 111         b2 = strchr(b1, ']');
 112         if (b2 == NULL || b2 == b1 + 1)
 113                 return -1;
 114
 115         *b1++ = 0;
 116         *b2++ = 0;
 117         expr = b1;
 118         do {
 119
 120                 sep = strchr(expr, ',');
 121                 if (sep != NULL)
 122                         *sep++ = 0;
 123
 124                 nob = strlen(expr);
 125                 n = nob;
 126                 if (sscanf(expr, "%u%n", &x, &n) >= 1 && n == nob) {
 127                         /* simple number */
 128                         new_lstrs(list, l->lstr_str, b2, x, x, 1);
 129                         continue;
 130                 }
 131
 132                 n = nob;
 133                 if (sscanf(expr, "%u-%u%n", &x, &y, &n) >= 2 && n == nob &&
 134                     x < y) {
 135                         /* simple range */
 136                         new_lstrs(list, l->lstr_str, b2, x, y, 1);
 137                         continue;
 138                 }
 139
 140                 n = nob;
 141                 if (sscanf(expr, "%u-%u/%u%n", &x, &y, &z, &n) >= 3 && n == nob &&
 142                     x < y) {
 143                         /* strided range */
 144                         new_lstrs(list, l->lstr_str, b2, x, y, z);
 145                         continue;
 146                 }
 147
 148                 /* syntax error */
 149                 return -1;
 150         } while ((expr = sep) != NULL);
 151
 152         free_lstr(l);
 153
 154         return 1;
 155 }
 156
 157 int
 158 expand_strs(char *str, lstr_t **head)
 159 {
 160         lstr_t  *list = NULL;
 161         lstr_t  *nlist;
 162         lstr_t  *l;
 163         int      rc;
 164         int      expanded;
 165
 166         l = alloc_lstr(strlen(str) + 1);
 167         memcpy(l->lstr_str, str, strlen(str) + 1);
 168         l->lstr_next = NULL;
 169         list = l;
 170
 171         do {
 172                 expanded = 0;
 173                 nlist = NULL;
 174
 175                 while ((l = list) != NULL) {
 176                         list = l->lstr_next;
 177
 178                         rc = expand_lstr(&nlist, l);
 179                         if (rc < 0) {
 180                                 fprintf(stderr, "Syntax error in \"%s\"\n", str);
 181                                 free_lstr(l);
 182                                 break;
 183                         }
 184
 185                         expanded |= rc > 0;
 186                 }
 187
 188                 /* re-order onto 'list' */
 189                 while ((l = nlist) != NULL) {
 190                         nlist = l->lstr_next;
 191                         l->lstr_next = list;
 192                         list = l;
 193                 }
 194
 195         } while (expanded && rc > 0);
 196
 197         if (rc >= 0) {
 198                 *head = list;
 199                 return 0;
 200         }
 201
 202         while ((l = list) != NULL) {
 203                 list = l->lstr_next;
 204
 205                 free_lstr(l);
 206         }
 207         return rc;
 208 }
 209
 210 int
 211 lst_parse_nids(char *str, int *countp, lnet_process_id_t **idspp)
 212 {
 213         lstr_t  *head = NULL;
 214         lstr_t  *l;
 215         int      c = 0;
 216         int      i;
 217         int      rc;
 218
 219         rc = expand_strs(str, &head);
 220         if (rc != 0)
 221                 goto out;
 222
 223         l = head;
 224         while (l != NULL) {
 225                 l = l->lstr_next;
 226                 c++;
 227         }
 228
 229         *idspp = malloc(c * sizeof(lnet_process_id_t));
 230         if (*idspp == NULL) {
 231                 fprintf(stderr, "Out of memory\n");
 232                 rc = -1;
 233         }
 234
 235         *countp = c;
 236 out:
 237         i = 0;
 238         while ((l = head) != NULL) {
 239                 head = l->lstr_next;
 240
 241                 if (rc == 0) {
 242                         (*idspp)[i].nid = libcfs_str2nid(l->lstr_str);
 243                         if ((*idspp)[i].nid == LNET_NID_ANY) {
 244                                 fprintf(stderr, "Invalid nid: %s\n",
 245                                         l->lstr_str);
 246                                 rc = -1;
 247                         }
 248
 249                         (*idspp)[i].pid = LUSTRE_LNET_PID;
 250                         i++;
 251                 }
 252
 253                 free_lstr(l);
 254         }
 255
 256         if (rc == 0)
 257                 return 0;
 258
 259         free(*idspp);
 260         *idspp = NULL;
 261
 262         return rc;
 263 }
 264
 265 char *
 266 lst_node_state2str(int state)
 267 {
 268         if (state == LST_NODE_ACTIVE)
 269                 return "Active";
 270         if (state == LST_NODE_BUSY)
 271                 return "Busy";
 272         if (state == LST_NODE_DOWN)
 273                 return "Down";
 274
 275         return "Unknown";
 276 }
 277
 278 int
 279 lst_node_str2state(char *str)
 280 {
 281         if (strcasecmp(str, "active") == 0)
 282                 return LST_NODE_ACTIVE;
 283         if (strcasecmp(str, "busy") == 0)
 284                 return LST_NODE_BUSY;
 285         if (strcasecmp(str, "down") == 0)
 286                 return LST_NODE_DOWN;
 287         if (strcasecmp(str, "unknown") == 0)
 288                 return LST_NODE_UNKNOWN;
 289         if (strcasecmp(str, "invalid") == 0)
 290                 return (LST_NODE_UNKNOWN | LST_NODE_DOWN | LST_NODE_BUSY);
 291
 292         return -1;
 293 }
 294
 295 char *
 296 lst_test_type2name(int type)
 297 {
 298         if (type == LST_TEST_PING)
 299                 return "ping";
 300         if (type == LST_TEST_BULK)
 301                 return "brw";
 302
 303         return "unknown";
 304 }
 305
 306 int
 307 lst_test_name2type(char *name)
 308 {
 309         if (strcasecmp(name, "ping") == 0)
 310                 return LST_TEST_PING;
 311         if (strcasecmp(name, "brw") == 0)
 312                 return LST_TEST_BULK;
 313
 314         return -1;
 315 }
 316
 317 void
 318 lst_print_usage(char *cmd)
 319 {
 320         Parser_printhelp(cmd);
 321 }
 322
 323 void
 324 lst_print_error(char *sub, const char *def_format, ...)
 325 {
 326         va_list ap;
 327
 328         /* local error returned from kernel */
 329         switch (errno) {
 330         case ESRCH:
 331                 fprintf(stderr, "No session exists\n");
 332                 return;
 333         case ESHUTDOWN:
 334                 fprintf(stderr, "Session is shutting down\n");
 335                 return;
 336         case EACCES:
 337                 fprintf(stderr, "Unmatched session key or not root\n");
 338                 return;
 339         case ENOENT:
 340                 fprintf(stderr, "Can't find %s in current session\n", sub);
 341                 return;
 342         case EINVAL:
 343                 fprintf(stderr, "Invalid parameters list in command line\n");
 344                 return;
 345         case EFAULT:
 346                 fprintf(stderr, "Bad parameter address\n");
 347                 return;
 348         case EEXIST:
 349                 fprintf(stderr, "%s already exists\n", sub);
 350                 return;
 351         default:
 352                 va_start(ap, def_format);
 353                 vfprintf(stderr, def_format, ap);
 354                 va_end(ap);
 355
 356                 return;
 357         }
 358 }
 359
 360 void
 361 lst_free_rpcent(struct list_head *head)
 362 {
 363         lstcon_rpc_ent_t *ent;
 364
 365         while (!list_empty(head)) {
 366                 ent = list_entry(head->next, lstcon_rpc_ent_t, rpe_link);
 367
 368                 list_del(&ent->rpe_link);
 369                 free(ent);
 370         }
 371 }
 372
 373 int
 374 lst_reset_rpcent(struct list_head *head)
 375 {
 376         lstcon_rpc_ent_t *ent;
 377
 378         list_for_each_entry(ent, head, rpe_link) {
 379                 ent->rpe_sid      = LST_INVALID_SID;
 380                 ent->rpe_peer.nid = LNET_NID_ANY;
 381                 ent->rpe_peer.pid = LNET_PID_ANY;
 382                 ent->rpe_rpc_errno = ent->rpe_fwk_errno = 0;
 383         }
 384 }
 385
 386 int
 387 lst_alloc_rpcent(struct list_head *head, int count, int offset)
 388 {
 389         lstcon_rpc_ent_t *ent;
 390         int               i;
 391
 392         for (i = 0; i < count; i++) {
 393                 ent = malloc(offsetof(lstcon_rpc_ent_t, rpe_payload[offset]));
 394                 if (ent == NULL) {
 395                         lst_free_rpcent(head);
 396                         return -1;
 397                 }
 398
 399                 memset(ent, 0, offsetof(lstcon_rpc_ent_t, rpe_payload[offset]));
 400
 401                 ent->rpe_sid      = LST_INVALID_SID;
 402                 ent->rpe_peer.nid = LNET_NID_ANY;
 403                 ent->rpe_peer.pid = LNET_PID_ANY;
 404                 list_add(&ent->rpe_link, head);
 405         }
 406
 407         return 0;
 408 }
 409
 410 void
 411 lst_print_transerr(struct list_head *head, char *optstr)
 412 {
 413         lstcon_rpc_ent_t  *ent;
 414
 415         list_for_each_entry(ent, head, rpe_link) {
 416                 if (ent->rpe_rpc_errno == 0 && ent->rpe_fwk_errno == 0)
 417                         continue;
 418
 419                 if (ent->rpe_rpc_errno != 0) {
 420                         fprintf(stderr, "%s RPC failed on %s: %s\n",
 421                                 optstr, libcfs_id2str(ent->rpe_peer),
 422                                 strerror(ent->rpe_rpc_errno));
 423                         continue;
 424                 }
 425
 426                 fprintf(stderr, "%s failed on %s: %s\n",
 427                         optstr, libcfs_id2str(ent->rpe_peer),
 428                         strerror(ent->rpe_fwk_errno));
 429         }
 430 }
 431
 432 int
 433 lst_ioctl(unsigned int opc, void *buf, int len)
 434 {
 435         struct libcfs_ioctl_data data;
 436         int    rc;
 437
 438         LIBCFS_IOC_INIT (data);
 439         data.ioc_u32[0]  = opc;
 440         data.ioc_plen1   = len;
 441         data.ioc_pbuf1   = (char *)buf;
 442         data.ioc_plen2   = sizeof(trans_stat);
 443         data.ioc_pbuf2   = (char *)&trans_stat;
 444
 445         memset(&trans_stat, 0, sizeof(trans_stat));
 446
 447         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNETST, &data);
 448
 449         /* local error, no valid RPC result */
 450         if (rc != 0)
 451                 return -1;
 452
 453         /* RPC error */
 454         if (trans_stat.trs_rpc_errno != 0)
 455                 return -2;
 456
 457         /* Framework error */
 458         if (trans_stat.trs_fwk_errno != 0)
 459                 return -3;
 460
 461         return 0;
 462 }
 463
 464 int
 465 lst_new_session_ioctl (char *name, int timeout, int force, lst_sid_t *sid)
 466 {
 467         lstio_session_new_args_t        args = {
 468                 .lstio_ses_key          = session_key,
 469                 .lstio_ses_timeout      = timeout,
 470                 .lstio_ses_force        = force,
 471                 .lstio_ses_idp          = sid,
 472                 .lstio_ses_namep        = name,
 473                 .lstio_ses_nmlen        = strlen(name),
 474         };
 475
 476         return lst_ioctl (LSTIO_SESSION_NEW, &args, sizeof(args));
 477 }
 478
 479 int
 480 jt_lst_new_session(int argc,  char **argv)
 481 {
 482         char  buf[LST_NAME_SIZE];
 483         char *name;
 484         int   optidx  = 0;
 485         int   timeout = 300;
 486         int   force   = 0;
 487         int   c;
 488         int   rc;
 489
 490         static struct option session_opts[] =
 491         {
 492                 {"timeout", required_argument,  0, 't' },
 493                 {"force",   no_argument,        0, 'f' },
 494                 {0,         0,                  0,  0  }
 495         };
 496
 497         if (session_key == 0) {
 498                 fprintf(stderr,
 499                         "Can't find env LST_SESSION or value is not valid\n");
 500                 return -1;
 501         }
 502
 503         while (1) {
 504
 505                 c = getopt_long(argc, argv, "ft:",
 506                                 session_opts, &optidx);
 507
 508                 if (c == -1)
 509                         break;
 510
 511                 switch (c) {
 512                 case 'f':
 513                         force = 1;
 514                         break;
 515                 case 't':
 516                         timeout = atoi(optarg);
 517                         break;
 518                 default:
 519                         lst_print_usage(argv[0]);
 520                         return -1;
 521                 }
 522         }
 523
 524         if (timeout <= 0) {
 525                 fprintf(stderr, "Invalid timeout value\n");
 526                 return -1;
 527         }
 528
 529         if (optind == argc - 1) {
 530                 name = argv[optind ++];
 531                 if (strlen(name) >= LST_NAME_SIZE) {
 532                         fprintf(stderr, "Name size is limited to %d\n",
 533                                 LST_NAME_SIZE - 1);
 534                         return -1;
 535                 }
 536
 537         } else if (optind == argc) {
 538                 char           user[LST_NAME_SIZE];
 539                 char           host[LST_NAME_SIZE];
 540                 struct passwd *pw = getpwuid(getuid());
 541
 542                 if (pw == NULL)
 543                         snprintf(user, sizeof(user), "%d", (int)getuid());
 544                 else
 545                         snprintf(user, sizeof(user), "%s", pw->pw_name);
 546
 547                 rc = gethostname(host, sizeof(host));
 548                 if (rc != 0)
 549                         snprintf(host, sizeof(host), "unknown_host");
 550
 551                 snprintf(buf, LST_NAME_SIZE, "%s@%s", user, host);
 552                 name = buf;
 553
 554         } else {
 555                 lst_print_usage(argv[0]);
 556                 return -1;
 557         }
 558
 559         rc = lst_new_session_ioctl(name, timeout, force, &session_id);
 560
 561         if (rc != 0) {
 562                 lst_print_error("session", "Failed to create session: %s\n",
 563                                 strerror(errno));
 564                 return rc;
 565         }
 566
 567         fprintf(stdout, "SESSION: %s TIMEOUT: %d FORCE: %s\n",
 568                 name, timeout, force ? "Yes": "No");
 569
 570         return rc;
 571 }
 572
 573 int
 574 lst_session_info_ioctl(char *name, int len, int *key,
 575                        lst_sid_t *sid, lstcon_ndlist_ent_t *ndinfo)
 576 {
 577         lstio_session_info_args_t args = {
 578                 .lstio_ses_keyp         = key,
 579                 .lstio_ses_idp          = sid,
 580                 .lstio_ses_ndinfo       = ndinfo,
 581                 .lstio_ses_nmlen        = len,
 582                 .lstio_ses_namep        = name,
 583         };
 584
 585         return lst_ioctl(LSTIO_SESSION_INFO, &args, sizeof(args));
 586 }
 587
 588 int
 589 jt_lst_show_session(int argc, char **argv)
 590 {
 591         lstcon_ndlist_ent_t ndinfo;
 592         lst_sid_t           sid;
 593         char                name[LST_NAME_SIZE];
 594         int                 key;
 595         int                 rc;
 596
 597         rc = lst_session_info_ioctl(name, LST_NAME_SIZE, &key, &sid, &ndinfo);
 598
 599         if (rc != 0) {
 600                 lst_print_error("session", "Failed to show session: %s\n",
 601                                 strerror(errno));
 602                 return -1;
 603         }
 604
 605         fprintf(stdout, "%s ID: %Lu@%s, KEY: %d NODES: %d\n",
 606                 name, sid.ses_stamp, libcfs_nid2str(sid.ses_nid),
 607                 key, ndinfo.nle_nnode);
 608
 609         return 0;
 610 }
 611
 612 int
 613 lst_end_session_ioctl(void)
 614 {
 615         lstio_session_end_args_t args = {
 616                 .lstio_ses_key           = session_key,
 617         };
 618
 619         return lst_ioctl (LSTIO_SESSION_END, &args, sizeof(args));
 620 }
 621
 622 int
 623 jt_lst_end_session(int argc, char **argv)
 624 {
 625         int             rc;
 626
 627         if (session_key == 0) {
 628                 fprintf(stderr,
 629                         "Can't find env LST_SESSION or value is not valid\n");
 630                 return -1;
 631         }
 632
 633         rc = lst_end_session_ioctl();
 634
 635         if (rc == 0) {
 636                 fprintf(stdout, "session is ended\n");
 637                 return 0;
 638         }
 639
 640         if (rc == -1) {
 641                 lst_print_error("session", "Failed to end session: %s\n",
 642                                 strerror(errno));
 643                 return rc;
 644         }
 645
 646         if (trans_stat.trs_rpc_errno != 0) {
 647                 fprintf(stderr,
 648                         "[RPC] Failed to send %d session RPCs: %s\n",
 649                         lstcon_rpc_stat_failure(&trans_stat, 0),
 650                         strerror(trans_stat.trs_rpc_errno));
 651         }
 652
 653         if (trans_stat.trs_fwk_errno != 0) {
 654                 fprintf(stderr,
 655                         "[FWK] Failed to end session on %d nodes: %s\n",
 656                         lstcon_sesop_stat_failure(&trans_stat, 0),
 657                         strerror(trans_stat.trs_fwk_errno));
 658         }
 659
 660         return rc;
 661 }
 662
 663 int
 664 lst_ping_ioctl(char *str, int type, int timeout,
 665                int count, lnet_process_id_t *ids, struct list_head *head)
 666 {
 667         lstio_debug_args_t args = {
 668                 .lstio_dbg_key          = session_key,
 669                 .lstio_dbg_type         = type,
 670                 .lstio_dbg_flags        = 0,
 671                 .lstio_dbg_timeout      = timeout,
 672                 .lstio_dbg_nmlen        = (str == NULL) ? 0: strlen(str),
 673                 .lstio_dbg_namep        = str,
 674                 .lstio_dbg_count        = count,
 675                 .lstio_dbg_idsp         = ids,
 676                 .lstio_dbg_resultp      = head,
 677         };
 678
 679         return lst_ioctl (LSTIO_DEBUG, &args, sizeof(args));
 680 }
 681
 682 int
 683 lst_get_node_count(int type, char *str, int *countp, lnet_process_id_t **idspp)
 684 {
 685         char                    buf[LST_NAME_SIZE];
 686         lstcon_test_batch_ent_t ent;
 687         lstcon_ndlist_ent_t    *entp = &ent.tbe_cli_nle;
 688         lst_sid_t               sid;
 689         int                     key;
 690         int                     rc;
 691
 692         switch (type) {
 693         case LST_OPC_SESSION:
 694                 rc = lst_session_info_ioctl(buf, LST_NAME_SIZE,
 695                                             &key, &sid, entp);
 696                 break;
 697
 698         case LST_OPC_BATCHSRV:
 699                 entp = &ent.tbe_srv_nle;
 700         case LST_OPC_BATCHCLI:
 701                 rc = lst_info_batch_ioctl(str, 0, 0, &ent, NULL, NULL, NULL);
 702                 break;
 703
 704         case LST_OPC_GROUP:
 705                 rc = lst_info_group_ioctl(str, entp, NULL, NULL, NULL);
 706                 break;
 707
 708         case LST_OPC_NODES:
 709                 rc = lst_parse_nids(str, &entp->nle_nnode, idspp) < 0 ? -1 : 0;
 710                 break;
 711
 712         default:
 713                 rc = -1;
 714                 break;
 715         }
 716
 717         if (rc == 0)
 718                 *countp = entp->nle_nnode;
 719
 720         return rc;
 721 }
 722
 723 int
 724 jt_lst_ping(int argc,  char **argv)
 725 {
 726         struct list_head   head;
 727         lnet_process_id_t *ids = NULL;
 728         lstcon_rpc_ent_t  *ent = NULL;
 729         char              *str = NULL;
 730         int                optidx  = 0;
 731         int                server  = 0;
 732         int                timeout = 5;
 733         int                count   = 0;
 734         int                type    = 0;
 735         int                rc      = 0;
 736         int                c;
 737         int                i;
 738
 739         static struct option ping_opts[] =
 740         {
 741                 {"session", no_argument,       0, 's' },
 742                 {"server",  no_argument,       0, 'v' },
 743                 {"batch",   required_argument, 0, 'b' },
 744                 {"group",   required_argument, 0, 'g' },
 745                 {"nodes",   required_argument, 0, 'n' },
 746                 {"timeout", required_argument, 0, 't' },
 747                 {0,         0,                 0,  0  }
 748         };
 749
 750         if (session_key == 0) {
 751                 fprintf(stderr,
 752                         "Can't find env LST_SESSION or value is not valid\n");
 753                 return -1;
 754         }
 755
 756         while (1) {
 757
 758                 c = getopt_long(argc, argv, "g:b:n:t:sv",
 759                                 ping_opts, &optidx);
 760
 761                 if (c == -1)
 762                         break;
 763
 764                 switch (c) {
 765                 case 's':
 766                         type = LST_OPC_SESSION;
 767                         break;
 768
 769                 case 'g':
 770                         type = LST_OPC_GROUP;
 771                         str = optarg;
 772                         break;
 773
 774                 case 'b':
 775                         type = LST_OPC_BATCHCLI;
 776                         str = optarg;
 777                         break;
 778
 779                 case 'n':
 780                         type = LST_OPC_NODES;
 781                         str = optarg;
 782                         break;
 783
 784                 case 't':
 785                         timeout = atoi(optarg);
 786                         break;
 787
 788                 case 'v':
 789                         server = 1;
 790                         break;
 791
 792                 default:
 793                         lst_print_usage(argv[0]);
 794                         return -1;
 795                 }
 796         }
 797
 798         if (type == 0 || timeout <= 0 || optind != argc) {
 799                 lst_print_usage(argv[0]);
 800                 return -1;
 801         }
 802
 803         if (type == LST_OPC_BATCHCLI && server)
 804                 type = LST_OPC_BATCHSRV;
 805
 806         rc = lst_get_node_count(type, str, &count, &ids);
 807         if (rc < 0) {
 808                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
 809                         (str == NULL) ? "session" : str, strerror(errno));
 810                 return -1;
 811         }
 812
 813         CFS_INIT_LIST_HEAD(&head);
 814
 815         rc = lst_alloc_rpcent(&head, count, LST_NAME_SIZE);
 816         if (rc != 0) {
 817                 fprintf(stderr, "Out of memory\n");
 818                 goto out;
 819         }
 820
 821         if (count == 0) {
 822                 fprintf(stdout, "Target %s is empty\n",
 823                         (str == NULL) ? "session" : str);
 824                 goto out;
 825         }
 826
 827         rc = lst_ping_ioctl(str, type, timeout, count, ids, &head);
 828         if (rc == -1) { /* local failure */
 829                 lst_print_error("debug", "Failed to ping %s: %s\n",
 830                                 (str == NULL) ? "session" : str,
 831                                 strerror(errno));
 832                 rc = -1;
 833                 goto out;
 834         }
 835
 836         /* ignore RPC errors and framwork errors */
 837         list_for_each_entry(ent, &head, rpe_link) {
 838                 fprintf(stdout, "\t%s: %s [session: %s id: %s]\n",
 839                         libcfs_id2str(ent->rpe_peer),
 840                         lst_node_state2str(ent->rpe_state),
 841                         (ent->rpe_state == LST_NODE_ACTIVE ||
 842                          ent->rpe_state == LST_NODE_BUSY)?
 843                                  (ent->rpe_rpc_errno == 0 ?
 844                                          &ent->rpe_payload[0] : "Unknown") :
 845                                  "<NULL>", libcfs_nid2str(ent->rpe_sid.ses_nid));
 846         }
 847
 848 out:
 849         lst_free_rpcent(&head);
 850
 851         if (ids != NULL)
 852                 free(ids);
 853
 854         return rc;
 855
 856 }
 857
 858 int
 859 lst_add_nodes_ioctl (char *name, int count, lnet_process_id_t *ids,
 860                      struct list_head *resultp)
 861 {
 862         lstio_group_nodes_args_t        args = {
 863                 .lstio_grp_key          = session_key,
 864                 .lstio_grp_nmlen        = strlen(name),
 865                 .lstio_grp_namep        = name,
 866                 .lstio_grp_count        = count,
 867                 .lstio_grp_idsp         = ids,
 868                 .lstio_grp_resultp      = resultp,
 869         };
 870
 871         return lst_ioctl(LSTIO_NODES_ADD, &args, sizeof(args));
 872 }
 873
 874 int
 875 lst_add_group_ioctl (char *name)
 876 {
 877         lstio_group_add_args_t  args = {
 878                 .lstio_grp_key          = session_key,
 879                 .lstio_grp_nmlen        = strlen(name),
 880                 .lstio_grp_namep        = name,
 881         };
 882
 883         return lst_ioctl(LSTIO_GROUP_ADD, &args, sizeof(args));
 884 }
 885
 886 int
 887 jt_lst_add_group(int argc, char **argv)
 888 {
 889         struct list_head   head;
 890         lnet_process_id_t *ids;
 891         char              *name;
 892         int                count;
 893         int                rc;
 894         int                i;
 895
 896         if (session_key == 0) {
 897                 fprintf(stderr,
 898                         "Can't find env LST_SESSION or value is not valid\n");
 899                 return -1;
 900         }
 901
 902         if (argc < 3) {
 903                 lst_print_usage(argv[0]);
 904                 return -1;
 905         }
 906
 907         name = argv[1];
 908         if (strlen(name) >= LST_NAME_SIZE) {
 909                 fprintf(stderr, "Name length is limited to %d\n",
 910                         LST_NAME_SIZE - 1);
 911                 return -1;
 912         }
 913
 914         rc = lst_add_group_ioctl(name);
 915         if (rc != 0) {
 916                 lst_print_error("group", "Failed to add group %s: %s\n",
 917                                 name, strerror(errno));
 918                 return -1;
 919         }
 920
 921         CFS_INIT_LIST_HEAD(&head);
 922
 923         for (i = 2; i < argc; i++) {
 924                 /* parse address list */
 925                 rc = lst_parse_nids(argv[i], &count, &ids);
 926                 if (rc < 0) {
 927                         fprintf(stderr, "Ignore invalid id list %s\n",
 928                                 argv[i]);
 929                         continue;
 930                 }
 931
 932                 if (count == 0)
 933                         continue;
 934
 935                 rc = lst_alloc_rpcent(&head, count, 0);
 936                 if (rc != 0) {
 937                         fprintf(stderr, "Out of memory\n");
 938                         break;
 939                 }
 940
 941                 rc = lst_add_nodes_ioctl(name, count, ids, &head);
 942
 943                 free(ids);
 944
 945                 if (rc == 0) {
 946                         lst_free_rpcent(&head);
 947                         fprintf(stderr, "%s are added to session\n", argv[i]);
 948                         continue;
 949                 }
 950
 951                 if (rc == -1) {
 952                         lst_free_rpcent(&head);
 953                         lst_print_error("group", "Failed to add nodes %s: %s\n",
 954                                         argv[i], strerror(errno));
 955                         break;
 956                 }
 957
 958                 lst_print_transerr(&head, "create session");
 959                 lst_free_rpcent(&head);
 960         }
 961
 962         return rc;
 963 }
 964
 965 int
 966 lst_del_group_ioctl (char *name)
 967 {
 968         lstio_group_del_args_t  args = {
 969                 .lstio_grp_key          = session_key,
 970                 .lstio_grp_nmlen        = strlen(name),
 971                 .lstio_grp_namep        = name,
 972         };
 973
 974         return lst_ioctl(LSTIO_GROUP_DEL, &args, sizeof(args));
 975 }
 976
 977 int
 978 jt_lst_del_group(int argc, char **argv)
 979 {
 980         int     rc;
 981
 982         if (session_key == 0) {
 983                 fprintf(stderr,
 984                         "Can't find env LST_SESSION or value is not valid\n");
 985                 return -1;
 986         }
 987
 988         if (argc != 2) {
 989                 lst_print_usage(argv[0]);
 990                 return -1;
 991         }
 992
 993         rc = lst_del_group_ioctl(argv[1]);
 994         if (rc == 0) {
 995                 fprintf(stdout, "Group is deleted\n");
 996                 return 0;
 997         }
 998
 999         if (rc == -1) {
1000                 lst_print_error("group", "Failed to delete group: %s\n",
1001                                 strerror(errno));
1002                 return rc;
1003         }
1004
1005         fprintf(stderr, "Group is deleted with some errors\n");
1006
1007         if (trans_stat.trs_rpc_errno != 0) {
1008                 fprintf(stderr, "[RPC] Failed to send %d end session RPCs: %s\n",
1009                         lstcon_rpc_stat_failure(&trans_stat, 0),
1010                         strerror(trans_stat.trs_rpc_errno));
1011         }
1012
1013         if (trans_stat.trs_fwk_errno != 0) {
1014                 fprintf(stderr,
1015                         "[FWK] Failed to end session on %d nodes: %s\n",
1016                         lstcon_sesop_stat_failure(&trans_stat, 0),
1017                         strerror(trans_stat.trs_fwk_errno));
1018         }
1019
1020         return -1;
1021 }
1022
1023 int
1024 lst_update_group_ioctl(int opc, char *name, int clean, int count,
1025                        lnet_process_id_t *ids, struct list_head *resultp)
1026 {
1027         lstio_group_update_args_t  args = {
1028                 .lstio_grp_key          = session_key,
1029                 .lstio_grp_opc          = opc,
1030                 .lstio_grp_args         = clean,
1031                 .lstio_grp_nmlen        = strlen(name),
1032                 .lstio_grp_namep        = name,
1033                 .lstio_grp_count        = count,
1034                 .lstio_grp_idsp         = ids,
1035                 .lstio_grp_resultp      = resultp,
1036         };
1037
1038         return lst_ioctl(LSTIO_GROUP_UPDATE, &args, sizeof(args));
1039 }
1040
1041 int
1042 jt_lst_update_group(int argc, char **argv)
1043 {
1044         struct list_head   head;
1045         lnet_process_id_t *ids = NULL;
1046         char              *str = NULL;
1047         char              *grp = NULL;
1048         int                optidx = 0;
1049         int                count = 0;
1050         int                clean = 0;
1051         int                opc = 0;
1052         int                rc;
1053         int                c;
1054
1055         static struct option update_group_opts[] =
1056         {
1057                 {"refresh", no_argument,       0, 'f' },
1058                 {"clean",   required_argument, 0, 'c' },
1059                 {"remove",  required_argument, 0, 'r' },
1060                 {0,         0,                 0,  0  }
1061         };
1062
1063         if (session_key == 0) {
1064                 fprintf(stderr,
1065                         "Can't find env LST_SESSION or value is not valid\n");
1066                 return -1;
1067         }
1068
1069         while (1) {
1070                 c = getopt_long(argc, argv, "fc:r:",
1071                                 update_group_opts, &optidx);
1072
1073                 /* Detect the end of the options. */
1074                 if (c == -1)
1075                         break;
1076
1077                 switch (c) {
1078                 case 'f':
1079                         if (opc != 0) {
1080                                 lst_print_usage(argv[0]);
1081                                 return -1;
1082                         }
1083                         opc = LST_GROUP_REFRESH;
1084                         break;
1085
1086                 case 'r':
1087                         if (opc != 0) {
1088                                 lst_print_usage(argv[0]);
1089                                 return -1;
1090                         }
1091                         opc = LST_GROUP_RMND;
1092                         str = optarg;
1093                         break;
1094
1095                 case 'c':
1096                         clean = lst_node_str2state(optarg);
1097                         if (opc != 0 || clean <= 0) {
1098                                 lst_print_usage(argv[0]);
1099                                 return -1;
1100                         }
1101                         opc = LST_GROUP_CLEAN;
1102                         break;
1103
1104                 default:
1105                         lst_print_usage(argv[0]);
1106                         return -1;
1107                 }
1108         }
1109
1110         /* no OPC or group is specified */
1111         if (opc == 0 || optind != argc - 1) {
1112                 lst_print_usage(argv[0]);
1113                 return -1;
1114         }
1115
1116         grp = argv[optind];
1117
1118         CFS_INIT_LIST_HEAD(&head);
1119
1120         if (opc == LST_GROUP_RMND || opc == LST_GROUP_REFRESH) {
1121                 rc = lst_get_node_count(opc == LST_GROUP_RMND ? LST_OPC_NODES :
1122                                                                 LST_OPC_GROUP,
1123                                         opc == LST_GROUP_RMND ? str : grp,
1124                                         &count, &ids);
1125
1126                 if (rc != 0) {
1127                         fprintf(stderr, "Can't get count of nodes from %s: %s\n",
1128                                 opc == LST_GROUP_RMND ? str : grp,
1129                                 strerror(errno));
1130                         return -1;
1131                 }
1132
1133                 rc = lst_alloc_rpcent(&head, count, 0);
1134                 if (rc != 0) {
1135                         fprintf(stderr, "Out of memory\n");
1136                         free(ids);
1137                         return -1;
1138                 }
1139
1140         }
1141
1142         rc = lst_update_group_ioctl(opc, grp, clean, count, ids, &head);
1143
1144         if (ids != NULL)
1145                 free(ids);
1146
1147         if (rc == 0) {
1148                 lst_free_rpcent(&head);
1149                 return 0;
1150         }
1151
1152         if (rc == -1) {
1153                 lst_free_rpcent(&head);
1154                 lst_print_error("group", "Failed to update group: %s\n",
1155                                 strerror(errno));
1156                 return rc;
1157         }
1158
1159         lst_print_transerr(&head, "Updating group");
1160
1161         lst_free_rpcent(&head);
1162
1163         return rc;
1164 }
1165
1166 int
1167 lst_list_group_ioctl(int len, char *name, int idx)
1168 {
1169         lstio_group_list_args_t         args = {
1170                 .lstio_grp_key          = session_key,
1171                 .lstio_grp_idx          = idx,
1172                 .lstio_grp_nmlen        = len,
1173                 .lstio_grp_namep        = name,
1174         };
1175
1176         return lst_ioctl(LSTIO_GROUP_LIST, &args, sizeof(args));
1177 }
1178
1179 int
1180 lst_info_group_ioctl(char *name, lstcon_ndlist_ent_t *gent,
1181                      int *idx, int *count, lstcon_node_ent_t *dents)
1182 {
1183         lstio_group_info_args_t         args = {
1184                 .lstio_grp_key          = session_key,
1185                 .lstio_grp_nmlen        = strlen(name),
1186                 .lstio_grp_namep        = name,
1187                 .lstio_grp_entp         = gent,
1188                 .lstio_grp_idxp         = idx,
1189                 .lstio_grp_ndentp       = count,
1190                 .lstio_grp_dentsp       = dents,
1191         };
1192
1193         return lst_ioctl(LSTIO_GROUP_INFO, &args, sizeof(args));
1194 }
1195
1196 int
1197 lst_list_group_all(void)
1198 {
1199         char  name[LST_NAME_SIZE];
1200         int   rc;
1201         int   i;
1202
1203         /* no group is specified, list name of all groups */
1204         for (i = 0; ; i++) {
1205                 rc = lst_list_group_ioctl(LST_NAME_SIZE, name, i);
1206                 if (rc == 0) {
1207                         fprintf(stdout, "%d) %s\n", i + 1, name);
1208                         continue;
1209                 }
1210
1211                 if (errno == ENOENT)
1212                         break;
1213
1214                 lst_print_error("group", "Failed to list group: %s\n",
1215                                 strerror(errno));
1216                 return -1;
1217         }
1218
1219         fprintf(stdout, "Total %d groups\n", i);
1220
1221         return 0;
1222 }
1223
1224 #define LST_NODES_TITLE "\tACTIVE\tBUSY\tDOWN\tUNKNOWN\tTOTAL\n"
1225
1226 int
1227 jt_lst_list_group(int argc, char **argv)
1228 {
1229         lstcon_ndlist_ent_t  gent;
1230         lstcon_node_ent_t   *dents;
1231         int               optidx  = 0;
1232         int               verbose = 0;
1233         int               active  = 0;
1234         int               busy    = 0;
1235         int               down    = 0;
1236         int               unknown = 0;
1237         int               all     = 0;
1238         int               count;
1239         int               index;
1240         int               i;
1241         int               j;
1242         int               c;
1243         int               rc;
1244
1245         static struct option list_group_opts[] =
1246         {
1247                 {"active",  no_argument, 0, 'a' },
1248                 {"busy",    no_argument, 0, 'b' },
1249                 {"down",    no_argument, 0, 'd' },
1250                 {"unknown", no_argument, 0, 'u' },
1251                 {"all",     no_argument, 0, 'l' },
1252                 {0,         0,           0,  0  }
1253         };
1254
1255         if (session_key == 0) {
1256                 fprintf(stderr,
1257                         "Can't find env LST_SESSION or value is not valid\n");
1258                 return -1;
1259         }
1260
1261         while (1) {
1262                 c = getopt_long(argc, argv, "abdul",
1263                                 list_group_opts, &optidx);
1264
1265                 if (c == -1)
1266                         break;
1267
1268                 switch (c) {
1269                 case 'a':
1270                         verbose = active = 1;
1271                         all = 0;
1272                         break;
1273                 case 'b':
1274                         verbose = busy = 1;
1275                         all = 0;
1276                         break;
1277                 case 'd':
1278                         verbose = down = 1;
1279                         all = 0;
1280                         break;
1281                 case 'u':
1282                         verbose = unknown = 1;
1283                         all = 0;
1284                         break;
1285                 case 'l':
1286                         verbose = all = 1;
1287                         break;
1288                 default:
1289                         lst_print_usage(argv[0]);
1290                         return -1;
1291                 }
1292         }
1293
1294         if (optind == argc) {
1295                 /* no group is specified, list name of all groups */
1296                 rc = lst_list_group_all();
1297
1298                 return rc;
1299         }
1300
1301         if (!verbose)
1302                 fprintf(stdout, LST_NODES_TITLE);
1303
1304         /* list nodes in specified groups */
1305         for (i = optind; i < argc; i++) {
1306                 rc = lst_info_group_ioctl(argv[i], &gent, NULL, NULL, NULL);
1307                 if (rc != 0) {
1308                         if (errno == ENOENT) {
1309                                 rc = 0;
1310                                 break;
1311                         }
1312
1313                         lst_print_error("group", "Failed to list group\n",
1314                                         strerror(errno));
1315                         break;
1316                 }
1317
1318                 if (!verbose) {
1319                         fprintf(stdout, "\t%d\t%d\t%d\t%d\t%d\t%s\n",
1320                                 gent.nle_nactive, gent.nle_nbusy,
1321                                 gent.nle_ndown, gent.nle_nunknown,
1322                                 gent.nle_nnode, argv[i]);
1323                         continue;
1324                 }
1325
1326                 fprintf(stdout, "Group [ %s ]\n", argv[i]);
1327
1328                 if (gent.nle_nnode == 0) {
1329                         fprintf(stdout, "No nodes found [ %s ]\n", argv[i]);
1330                         continue;
1331                 }
1332
1333                 count = gent.nle_nnode;
1334
1335                 dents = malloc(count * sizeof(lstcon_node_ent_t));
1336                 if (dents == NULL) {
1337                         fprintf(stderr, "Failed to malloc: %s\n",
1338                                 strerror(errno));
1339                         return -1;
1340                 }
1341
1342                 index = 0;
1343                 rc = lst_info_group_ioctl(argv[i], &gent, &index, &count, dents);
1344                 if (rc != 0) {
1345                         lst_print_error("group", "Failed to list group: %s\n",
1346                                         strerror(errno));
1347                         free(dents);
1348                         return -1;
1349                 }
1350
1351                 for (j = 0, c = 0; j < count; j++) {
1352                         if (all ||
1353                             ((active  &&  dents[j].nde_state == LST_NODE_ACTIVE) ||
1354                              (busy    &&  dents[j].nde_state == LST_NODE_BUSY)   ||
1355                              (down    &&  dents[j].nde_state == LST_NODE_DOWN)   ||
1356                              (unknown &&  dents[j].nde_state == LST_NODE_UNKNOWN))) {
1357
1358                                 fprintf(stdout, "\t%s: %s\n",
1359                                         libcfs_id2str(dents[j].nde_id),
1360                                         lst_node_state2str(dents[j].nde_state));
1361                                 c++;
1362                         }
1363                 }
1364
1365                 fprintf(stdout, "Total %d nodes [ %s ]\n", c, argv[i]);
1366
1367                 free(dents);
1368         }
1369
1370         return rc;
1371 }
1372
1373 int
1374 lst_stat_ioctl (char *name, int count, lnet_process_id_t *idsp,
1375                 int timeout, struct list_head *resultp)
1376 {
1377         lstio_stat_args_t  args = {
1378                 .lstio_sta_key           = session_key,
1379                 .lstio_sta_timeout       = timeout,
1380                 .lstio_sta_nmlen         = strlen(name),
1381                 .lstio_sta_namep         = name,
1382                 .lstio_sta_count         = count,
1383                 .lstio_sta_idsp          = idsp,
1384                 .lstio_sta_resultp       = resultp,
1385         };
1386
1387         return lst_ioctl (LSTIO_STAT_QUERY, &args, sizeof(args));
1388 }
1389
1390 typedef struct {
1391         struct list_head        srp_link;
1392         int                     srp_count;
1393         char                   *srp_name;
1394         lnet_process_id_t      *srp_ids;
1395         struct list_head        srp_result[2];
1396 } lst_stat_req_param_t;
1397
1398 static void
1399 lst_stat_req_param_free(lst_stat_req_param_t *srp)
1400 {
1401         int     i;
1402
1403         for (i = 0; i < 2; i++)
1404                 lst_free_rpcent(&srp->srp_result[i]);
1405
1406         if (srp->srp_ids != NULL)
1407                 free(srp->srp_ids);
1408
1409         free(srp);
1410 }
1411
1412 static int
1413 lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp)
1414 {
1415         lst_stat_req_param_t *srp = NULL;
1416         int                   rc;
1417         int                   i;
1418
1419         srp = malloc(sizeof(*srp));
1420         if (srp == NULL)
1421                 return -ENOMEM;
1422
1423         memset(srp, 0, sizeof(*srp));
1424         CFS_INIT_LIST_HEAD(&srp->srp_result[0]);
1425         CFS_INIT_LIST_HEAD(&srp->srp_result[1]);
1426
1427         rc = lst_get_node_count(LST_OPC_GROUP, name,
1428                                 &srp->srp_count, NULL);
1429         if (rc != 0) {
1430                 rc = lst_get_node_count(LST_OPC_NODES, name,
1431                                         &srp->srp_count, &srp->srp_ids);
1432         }
1433
1434         if (rc != 0) {
1435                 fprintf(stderr,
1436                         "Failed to get count of nodes from %s\n", name);
1437                 lst_stat_req_param_free(srp);
1438
1439                 return rc;
1440         }
1441
1442         srp->srp_name = name;
1443
1444         for (i = 0; i < 2; i++) {
1445                 rc = lst_alloc_rpcent(&srp->srp_result[i], srp->srp_count,
1446                                       sizeof(sfw_counters_t)  +
1447                                       sizeof(srpc_counters_t) +
1448                                       sizeof(lnet_counters_t));
1449                 if (rc != 0) {
1450                         fprintf(stderr, "Out of memory\n");
1451                         break;
1452                 }
1453         }
1454
1455         if (rc == 0) {
1456                 *srpp = srp;
1457                 return 0;
1458         }
1459
1460         lst_stat_req_param_free(srp);
1461
1462         return rc;
1463 }
1464
1465 typedef struct {
1466         /* TODO */
1467 } lst_srpc_stat_result;
1468
1469 #define LST_LNET_AVG    0
1470 #define LST_LNET_MIN    1
1471 #define LST_LNET_MAX    2
1472
1473 typedef struct {
1474         float           lnet_avg_sndrate;
1475         float           lnet_min_sndrate;
1476         float           lnet_max_sndrate;
1477         float           lnet_total_sndrate;
1478
1479         float           lnet_avg_rcvrate;
1480         float           lnet_min_rcvrate;
1481         float           lnet_max_rcvrate;
1482         float           lnet_total_rcvrate;
1483
1484         float           lnet_avg_sndperf;
1485         float           lnet_min_sndperf;
1486         float           lnet_max_sndperf;
1487         float           lnet_total_sndperf;
1488
1489         float           lnet_avg_rcvperf;
1490         float           lnet_min_rcvperf;
1491         float           lnet_max_rcvperf;
1492         float           lnet_total_rcvperf;
1493
1494         int             lnet_stat_count;
1495 } lst_lnet_stat_result_t;
1496
1497 lst_lnet_stat_result_t lnet_stat_result;
1498
1499 static float
1500 lst_lnet_stat_value(int bw, int send, int off)
1501 {
1502         float  *p;
1503
1504         p = bw ? &lnet_stat_result.lnet_avg_sndperf :
1505                  &lnet_stat_result.lnet_avg_sndrate;
1506
1507         if (!send)
1508                 p += 4;
1509
1510         p += off;
1511
1512         return *p;
1513 }
1514
1515 static void
1516 lst_timeval_diff(struct timeval *tv1,
1517                  struct timeval *tv2, struct timeval *df)
1518 {
1519         if (tv1->tv_usec >= tv2->tv_usec) {
1520                 df->tv_sec  = tv1->tv_sec - tv2->tv_sec;
1521                 df->tv_usec = tv1->tv_usec - tv2->tv_usec;
1522                 return;
1523         }
1524
1525         df->tv_sec  = tv1->tv_sec - 1 - tv2->tv_sec;
1526         df->tv_usec = tv1->tv_sec + 1000000 - tv2->tv_usec;
1527
1528         return;
1529 }
1530
1531 void
1532 lst_cal_lnet_stat(float delta, lnet_counters_t *lnet_new,
1533                   lnet_counters_t *lnet_old)
1534 {
1535         float perf;
1536         float rate;
1537
1538         perf = (float)(lnet_new->send_length -
1539                        lnet_old->send_length) / (1024 * 1024) / delta;
1540         lnet_stat_result.lnet_total_sndperf += perf;
1541
1542         if (lnet_stat_result.lnet_min_sndperf > perf ||
1543             lnet_stat_result.lnet_min_sndperf == 0)
1544                 lnet_stat_result.lnet_min_sndperf = perf;
1545
1546         if (lnet_stat_result.lnet_max_sndperf < perf)
1547                 lnet_stat_result.lnet_max_sndperf = perf;
1548
1549         perf = (float)(lnet_new->recv_length -
1550                        lnet_old->recv_length) / (1024 * 1024) / delta;
1551         lnet_stat_result.lnet_total_rcvperf += perf;
1552
1553         if (lnet_stat_result.lnet_min_rcvperf > perf ||
1554             lnet_stat_result.lnet_min_rcvperf == 0)
1555                 lnet_stat_result.lnet_min_rcvperf = perf;
1556
1557         if (lnet_stat_result.lnet_max_rcvperf < perf)
1558                 lnet_stat_result.lnet_max_rcvperf = perf;
1559
1560         rate = (lnet_new->send_count - lnet_old->send_count) / delta;
1561         lnet_stat_result.lnet_total_sndrate += rate;
1562
1563         if (lnet_stat_result.lnet_min_sndrate > rate ||
1564             lnet_stat_result.lnet_min_sndrate == 0)
1565                 lnet_stat_result.lnet_min_sndrate = rate;
1566
1567         if (lnet_stat_result.lnet_max_sndrate < rate)
1568                 lnet_stat_result.lnet_max_sndrate = rate;
1569
1570         rate = (lnet_new->recv_count - lnet_old->recv_count) / delta;
1571         lnet_stat_result.lnet_total_rcvrate += rate;
1572
1573         if (lnet_stat_result.lnet_min_rcvrate > rate ||
1574             lnet_stat_result.lnet_min_rcvrate == 0)
1575                 lnet_stat_result.lnet_min_rcvrate = rate;
1576
1577         if (lnet_stat_result.lnet_max_rcvrate < rate)
1578                 lnet_stat_result.lnet_max_rcvrate = rate;
1579
1580         lnet_stat_result.lnet_stat_count ++;
1581
1582         lnet_stat_result.lnet_avg_sndrate = lnet_stat_result.lnet_total_sndrate /
1583                                             lnet_stat_result.lnet_stat_count;
1584         lnet_stat_result.lnet_avg_rcvrate = lnet_stat_result.lnet_total_rcvrate /
1585                                             lnet_stat_result.lnet_stat_count;
1586
1587         lnet_stat_result.lnet_avg_sndperf = lnet_stat_result.lnet_total_sndperf /
1588                                             lnet_stat_result.lnet_stat_count;
1589         lnet_stat_result.lnet_avg_rcvperf = lnet_stat_result.lnet_total_rcvperf /
1590                                             lnet_stat_result.lnet_stat_count;
1591
1592 }
1593
1594 void
1595 lst_print_lnet_stat(char *name, int bwrt, int rdwr, int type)
1596 {
1597         int     start1 = 0;
1598         int     end1   = 1;
1599         int     start2 = 0;
1600         int     end2   = 1;
1601         int     start3 = 0;
1602         int     end3   = 2;
1603         int     i;
1604         int     j;
1605
1606         if (lnet_stat_result.lnet_stat_count == 0)
1607                 return;
1608
1609         if (bwrt == 1) /* bw only */
1610                 start1 = 1;
1611
1612         if (bwrt == 2) /* rates only */
1613                 end1 = 0;
1614
1615         if (rdwr == 1) /* recv only */
1616                 start2 = 1;
1617
1618         if (rdwr == 2) /* send only */
1619                 end2 = 0;
1620
1621         for (i = start1; i <= end1; i++) {
1622                 fprintf(stdout, "[LNet %s of %s]\n",
1623                         i == 0 ? "Rates" : "Bandwidth", name);
1624
1625                 for (j = start2; j <= end2; j++) {
1626                         fprintf(stdout, "[%c] ", j == 0 ? 'W' : 'R');
1627
1628                         if ((type & 1) != 0) {
1629                                 fprintf(stdout, i == 0 ? "Avg: %-8.0f RPC/s " :
1630                                                          "Avg: %-8.2f MB/s  ",
1631                                         lst_lnet_stat_value(i, j, 0));
1632                         }
1633
1634                         if ((type & 2) != 0) {
1635                                 fprintf(stdout, i == 0 ? "Min: %-8.0f RPC/s " :
1636                                                          "Min: %-8.2f MB/s  ",
1637                                         lst_lnet_stat_value(i, j, 1));
1638                         }
1639
1640                         if ((type & 4) != 0) {
1641                                 fprintf(stdout, i == 0 ? "Max: %-8.0f RPC/s" :
1642                                                          "Max: %-8.2f MB/s",
1643                                         lst_lnet_stat_value(i, j, 2));
1644                         }
1645
1646                         fprintf(stdout, "\n");
1647                 }
1648         }
1649 }
1650
1651 void
1652 lst_print_stat(char *name, struct list_head *resultp,
1653                int idx, int lnet, int bwrt, int rdwr, int type)
1654 {
1655         struct list_head  tmp[2];
1656         lstcon_rpc_ent_t *new;
1657         lstcon_rpc_ent_t *old;
1658         sfw_counters_t   *sfwk_new;
1659         sfw_counters_t   *sfwk_old;
1660         srpc_counters_t  *srpc_new;
1661         srpc_counters_t  *srpc_old;
1662         lnet_counters_t  *lnet_new;
1663         lnet_counters_t  *lnet_old;
1664         struct timeval    tv;
1665         float             delta;
1666         int               errcount = 0;
1667
1668         CFS_INIT_LIST_HEAD(&tmp[0]);
1669         CFS_INIT_LIST_HEAD(&tmp[1]);
1670
1671         memset(&lnet_stat_result, 0, sizeof(lnet_stat_result));
1672
1673         while (!list_empty(&resultp[idx])) {
1674                 if (list_empty(&resultp[1 - idx])) {
1675                         fprintf(stderr, "Group is changed, re-run stat\n");
1676                         break;
1677                 }
1678
1679                 new = list_entry(resultp[idx].next, lstcon_rpc_ent_t, rpe_link);
1680                 old = list_entry(resultp[1 - idx].next, lstcon_rpc_ent_t, rpe_link);
1681
1682                 /* first time get stats result, can't calculate diff */
1683                 if (new->rpe_peer.nid == LNET_NID_ANY)
1684                         break;
1685
1686                 if (new->rpe_peer.nid != old->rpe_peer.nid ||
1687                     new->rpe_peer.pid != old->rpe_peer.pid) {
1688                         /* Something wrong. i.e, somebody change the group */
1689                         break;
1690                 }
1691
1692                 list_del(&new->rpe_link);
1693                 list_add_tail(&new->rpe_link, &tmp[idx]);
1694
1695                 list_del(&old->rpe_link);
1696                 list_add_tail(&old->rpe_link, &tmp[1 - idx]);
1697
1698                 if (new->rpe_rpc_errno != 0 || new->rpe_fwk_errno != 0 ||
1699                     old->rpe_rpc_errno != 0 || old->rpe_fwk_errno != 0) {
1700                         errcount ++;
1701                         continue;
1702                 }
1703
1704                 sfwk_new = (sfw_counters_t *)&new->rpe_payload[0];
1705                 sfwk_old = (sfw_counters_t *)&old->rpe_payload[0];
1706
1707                 srpc_new = (srpc_counters_t *)((char *)sfwk_new + sizeof(*sfwk_new));
1708                 srpc_old = (srpc_counters_t *)((char *)sfwk_old + sizeof(*sfwk_old));
1709
1710                 lnet_new = (lnet_counters_t *)((char *)srpc_new + sizeof(*srpc_new));
1711                 lnet_old = (lnet_counters_t *)((char *)srpc_old + sizeof(*srpc_old));
1712
1713                 lst_timeval_diff(&new->rpe_stamp, &old->rpe_stamp, &tv);
1714
1715                 delta = tv.tv_sec + (float)tv.tv_usec/1000000;
1716
1717                 if (!lnet) /* TODO */
1718                         continue;
1719
1720                 lst_cal_lnet_stat(delta, lnet_new, lnet_old);
1721         }
1722
1723         list_splice(&tmp[idx], &resultp[idx]);
1724         list_splice(&tmp[1 - idx], &resultp[1 - idx]);
1725
1726         if (errcount > 0)
1727                 fprintf(stdout, "Failed to stat on %d nodes\n", errcount);
1728
1729         if (!lnet)  /* TODO */
1730                 return;
1731
1732         lst_print_lnet_stat(name, bwrt, rdwr, type);
1733 }
1734
1735 int
1736 jt_lst_stat(int argc, char **argv)
1737 {
1738         struct list_head      head;
1739         lst_stat_req_param_t *srp;
1740         lstcon_rpc_ent_t     *ent;
1741         char                 *name;
1742         time_t                last    = 0;
1743         int                   optidx  = 0;
1744         int                   count   = 0;
1745         int                   timeout = 5; /* default timeout, 5 sec */
1746         int                   delay   = 5; /* default delay, 5 sec */
1747         int                   lnet    = 1; /* lnet stat by default */
1748         int                   bwrt    = 0;
1749         int                   rdwr    = 0;
1750         int                   type    = -1;
1751         int                   idx     = 0;
1752         int                   rc;
1753         int                   i;
1754         int                   c;
1755
1756         static struct option stat_opts[] =
1757         {
1758                 {"timeout", required_argument, 0, 't' },
1759                 {"delay"  , required_argument, 0, 'd' },
1760                 {"lnet"   , no_argument,       0, 'l' },
1761                 {"rpc"    , no_argument,       0, 'c' },
1762                 {"bw"     , no_argument,       0, 'b' },
1763                 {"rate"   , no_argument,       0, 'a' },
1764                 {"read"   , no_argument,       0, 'r' },
1765                 {"write"  , no_argument,       0, 'w' },
1766                 {"avg"    , no_argument,       0, 'g' },
1767                 {"min"    , no_argument,       0, 'n' },
1768                 {"max"    , no_argument,       0, 'x' },
1769                 {0,         0,                 0,  0  }
1770         };
1771
1772         if (session_key == 0) {
1773                 fprintf(stderr,
1774                         "Can't find env LST_SESSION or value is not valid\n");
1775                 return -1;
1776         }
1777
1778         while (1) {
1779                 c = getopt_long(argc, argv, "t:d:lcbarwgnx", stat_opts, &optidx);
1780
1781                 if (c == -1)
1782                         break;
1783
1784                 switch (c) {
1785                 case 't':
1786                         timeout = atoi(optarg);
1787                         break;
1788                 case 'd':
1789                         delay = atoi(optarg);
1790                         break;
1791                 case 'l':
1792                         lnet = 1;
1793                         break;
1794                 case 'c':
1795                         lnet = 0;
1796                         break;
1797                 case 'b':
1798                         bwrt |= 1;
1799                         break;
1800                 case 'a':
1801                         bwrt |= 2;
1802                         break;
1803                 case 'r':
1804                         rdwr |= 1;
1805                         break;
1806                 case 'w':
1807                         rdwr |= 2;
1808                         break;
1809                 case 'g':
1810                         if (type == -1) {
1811                                 type = 1;
1812                                 break;
1813                         }
1814                         type |= 1;
1815                         break;
1816                 case 'n':
1817                         if (type == -1) {
1818                                 type = 2;
1819                                 break;
1820                         }
1821                         type |= 2;
1822                         break;
1823                 case 'x':
1824                         if (type == -1) {
1825                                 type = 4;
1826                                 break;
1827                         }
1828                         type |= 4;
1829                         break;
1830                 default:
1831                         lst_print_usage(argv[0]);
1832                         return -1;
1833                 }
1834         }
1835
1836         if (optind == argc) {
1837                 lst_print_usage(argv[0]);
1838                 return -1;
1839         }
1840
1841         if (timeout <= 0 || delay <= 0) {
1842                 fprintf(stderr, "Invalid timeout or delay value\n");
1843                 return -1;
1844         }
1845
1846         CFS_INIT_LIST_HEAD(&head);
1847
1848         while (optind < argc) {
1849                 name = argv[optind++];
1850
1851                 rc = lst_stat_req_param_alloc(name, &srp);
1852                 if (rc != 0)
1853                         goto out;
1854
1855                 list_add_tail(&srp->srp_link, &head);
1856         }
1857
1858         while (1) {
1859                 time_t  now = time(NULL);
1860
1861                 if (now - last < delay) {
1862                         sleep(delay - now + last);
1863                         time(&now);
1864                 }
1865
1866                 last = now;
1867
1868                 list_for_each_entry(srp, &head, srp_link) {
1869                         rc = lst_stat_ioctl(srp->srp_name,
1870                                             srp->srp_count, srp->srp_ids,
1871                                             timeout, &srp->srp_result[idx]);
1872                         if (rc == -1) {
1873                                 lst_print_error("stat", "Failed to stat %s: %s\n",
1874                                                 name, strerror(errno));
1875                                 goto out;
1876                         }
1877
1878                         lst_print_stat(srp->srp_name, srp->srp_result,
1879                                        idx, lnet, bwrt, rdwr, type);
1880
1881                         lst_reset_rpcent(&srp->srp_result[1 - idx]);
1882                 }
1883
1884                 idx = 1 - idx;
1885         }
1886
1887 out:
1888         while (!list_empty(&head)) {
1889                 srp = list_entry(head.next, lst_stat_req_param_t, srp_link);
1890
1891                 list_del(&srp->srp_link);
1892                 lst_stat_req_param_free(srp);
1893         }
1894
1895         return rc;
1896 }
1897
1898 int
1899 jt_lst_show_error(int argc, char **argv)
1900 {
1901         struct list_head      head;
1902         lst_stat_req_param_t *srp;
1903         lstcon_rpc_ent_t     *ent;
1904         sfw_counters_t       *sfwk;
1905         srpc_counters_t      *srpc;
1906         lnet_counters_t      *lnet;
1907         lnet_process_id_t    *idsp   = NULL;
1908         char                 *name   = NULL;
1909         int                   optidx = 0;
1910         int                   count  = 0;
1911         int                   type   = 0;
1912         int                   timeout = 5;
1913         int                   ecount = 0;
1914         int                   rc;
1915         int                   c;
1916
1917
1918         static struct option show_error_opts[] =
1919         {
1920                 {"group"  , required_argument, 0, 'g' },
1921                 {"nodes"  , required_argument, 0, 'n' },
1922                 {0,         0,                 0,  0  }
1923         };
1924
1925         if (session_key == 0) {
1926                 fprintf(stderr,
1927                         "Can't find env LST_SESSION or value is not valid\n");
1928                 return -1;
1929         }
1930
1931         while (1) {
1932                 c = getopt_long(argc, argv, "g:n:", show_error_opts, &optidx);
1933
1934                 if (c == -1)
1935                         break;
1936
1937                 switch (c) {
1938                 case 'g':
1939                         type = LST_OPC_GROUP;
1940                         name = optarg;
1941                         break;
1942                 case 'n':
1943                         type = LST_OPC_NODES;
1944                         name = optarg;
1945                         break;
1946                 default:
1947                         lst_print_usage(argv[0]);
1948                         return -1;
1949                 }
1950         }
1951
1952         if (optind != argc || type == 0) {
1953                 lst_print_usage(argv[0]);
1954                 return -1;
1955         }
1956
1957         if (name == NULL) {
1958                 fprintf(stderr, "Missing name of target (group | nodes)\n");
1959                 return -1;
1960         }
1961
1962         rc = lst_get_node_count(type, name, &count, &idsp);
1963         if (rc < 0) {
1964                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
1965                         name, strerror(errno));
1966                 return -1;
1967         }
1968
1969         CFS_INIT_LIST_HEAD(&head);
1970
1971         rc = lst_alloc_rpcent(&head, count, sizeof(sfw_counters_t) +
1972                                             sizeof(srpc_counters_t) +
1973                                             sizeof(lnet_counters_t));
1974         if (rc != 0) {
1975                 fprintf(stderr, "Out of memory\n");
1976                 goto out;
1977         }
1978
1979         rc = lst_stat_ioctl(name, count, idsp, timeout, &head);
1980         if (rc == -1) {
1981                 lst_print_error(name, "Failed to show errors of %s: %s\n",
1982                                 name, strerror(errno));
1983                 goto out;
1984         }
1985
1986         list_for_each_entry(ent, &head, rpe_link) {
1987                 if (ent->rpe_rpc_errno != 0) {
1988                         ecount ++;
1989                         fprintf(stderr, "RPC failure, can't show error on %s\n",
1990                                 libcfs_id2str(ent->rpe_peer));
1991                         continue;
1992                 }
1993
1994                 if (ent->rpe_fwk_errno != 0) {
1995                         ecount ++;
1996                         fprintf(stderr, "Framework failure, can't show error on %s\n",
1997                                 libcfs_id2str(ent->rpe_peer));
1998                         continue;
1999                 }
2000
2001                 sfwk = (sfw_counters_t *)&ent->rpe_payload[0];
2002                 srpc = (srpc_counters_t *)((char *)sfwk + sizeof(*sfwk));
2003                 lnet = (lnet_counters_t *)((char *)srpc + sizeof(*srpc));
2004
2005                 if (srpc->errors == 0 &&
2006                     sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2007                         continue;
2008
2009                 ecount ++;
2010                 fprintf(stderr, "[%s]: %d RPC errors, %d brw errors, %d ping errors\n",
2011                         libcfs_id2str(ent->rpe_peer), srpc->errors,
2012                         sfwk->brw_errors, sfwk->ping_errors);
2013         }
2014
2015         fprintf(stdout, "Total %d errors in %s\n", ecount, name);
2016 out:
2017         lst_free_rpcent(&head);
2018         if (idsp != NULL)
2019                 free(idsp);
2020
2021         return 0;
2022 }
2023
2024 int
2025 lst_add_batch_ioctl (char *name)
2026 {
2027         lstio_batch_add_args_t  args = {
2028                 .lstio_bat_key           = session_key,
2029                 .lstio_bat_nmlen         = strlen(name),
2030                 .lstio_bat_namep         = name,
2031         };
2032
2033         return lst_ioctl (LSTIO_BATCH_ADD, &args, sizeof(args));
2034 }
2035
2036 int
2037 jt_lst_add_batch(int argc, char **argv)
2038 {
2039         char   *name;
2040         int     rc;
2041
2042         if (session_key == 0) {
2043                 fprintf(stderr,
2044                         "Can't find env LST_SESSION or value is not valid\n");
2045                 return -1;
2046         }
2047
2048         if (argc != 2) {
2049                 lst_print_usage(argv[0]);
2050                 return -1;
2051         }
2052
2053         name = argv[1];
2054         if (strlen(name) >= LST_NAME_SIZE) {
2055                 fprintf(stderr, "Name length is limited to %d\n",
2056                         LST_NAME_SIZE - 1);
2057                 return -1;
2058         }
2059
2060         rc = lst_add_batch_ioctl(name);
2061         if (rc == 0)
2062                 return 0;
2063
2064         lst_print_error("batch", "Failed to create batch: %s\n",
2065                         strerror(errno));
2066
2067         return -1;
2068 }
2069
2070 int
2071 lst_start_batch_ioctl (char *name, int timeout, struct list_head *resultp)
2072 {
2073         lstio_batch_run_args_t   args = {
2074                 .lstio_bat_key          = session_key,
2075                 .lstio_bat_timeout      = timeout,
2076                 .lstio_bat_nmlen        = strlen(name),
2077                 .lstio_bat_namep        = name,
2078                 .lstio_bat_resultp      = resultp,
2079         };
2080
2081         return lst_ioctl(LSTIO_BATCH_START, &args, sizeof(args));
2082 }
2083
2084 int
2085 jt_lst_start_batch(int argc, char **argv)
2086 {
2087         struct list_head  head;
2088         char             *batch;
2089         int               optidx  = 0;
2090         int               timeout = 0;
2091         int               count = 0;
2092         int               rc;
2093         int               c;
2094
2095         static struct option start_batch_opts[] =
2096         {
2097                 {"timeout", required_argument, 0, 't' },
2098                 {0,         0,                 0,  0  }
2099         };
2100
2101         if (session_key == 0) {
2102                 fprintf(stderr,
2103                         "Can't find env LST_SESSION or value is not valid\n");
2104                 return -1;
2105         }
2106
2107         while (1) {
2108                 c = getopt_long(argc, argv, "t:",
2109                                 start_batch_opts, &optidx);
2110
2111                 /* Detect the end of the options. */
2112                 if (c == -1)
2113                         break;
2114
2115                 switch (c) {
2116                 case 't':
2117                         timeout = atoi(optarg);
2118                         break;
2119                 default:
2120                         lst_print_usage(argv[0]);
2121                         return -1;
2122                 }
2123         }
2124
2125         if (optind == argc) {
2126                 batch = LST_DEFAULT_BATCH;
2127
2128         } else if (optind == argc - 1) {
2129                 batch = argv[optind];
2130
2131         } else {
2132                 lst_print_usage(argv[0]);
2133                 return -1;
2134         }
2135
2136         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2137         if (rc != 0) {
2138                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2139                         batch, strerror(errno));
2140                 return -1;
2141         }
2142
2143         CFS_INIT_LIST_HEAD(&head);
2144
2145         rc = lst_alloc_rpcent(&head, count, 0);
2146         if (rc != 0) {
2147                 fprintf(stderr, "Out of memory\n");
2148                 return -1;
2149         }
2150
2151         rc = lst_start_batch_ioctl(batch, timeout, &head);
2152
2153         if (rc == 0) {
2154                 fprintf(stdout, "%s is running now\n", batch);
2155                 lst_free_rpcent(&head);
2156                 return 0;
2157         }
2158
2159         if (rc == -1) {
2160                 lst_print_error("batch", "Failed to start batch: %s\n",
2161                                 strerror(errno));
2162                 lst_free_rpcent(&head);
2163                 return rc;
2164         }
2165
2166         lst_print_transerr(&head, "Run batch");
2167
2168         lst_free_rpcent(&head);
2169
2170         return rc;
2171 }
2172
2173 int
2174 lst_stop_batch_ioctl(char *name, int force, struct list_head *resultp)
2175 {
2176         lstio_batch_stop_args_t   args = {
2177                 .lstio_bat_key          = session_key,
2178                 .lstio_bat_force        = force,
2179                 .lstio_bat_nmlen        = strlen(name),
2180                 .lstio_bat_namep        = name,
2181                 .lstio_bat_resultp      = resultp,
2182         };
2183
2184         return lst_ioctl(LSTIO_BATCH_STOP, &args, sizeof(args));
2185 }
2186
2187 int
2188 jt_lst_stop_batch(int argc, char **argv)
2189 {
2190         struct list_head  head;
2191         char             *batch;
2192         int               force = 0;
2193         int               optidx;
2194         int               count;
2195         int               rc;
2196         int               c;
2197
2198         static struct option stop_batch_opts[] =
2199         {
2200                 {"force",   no_argument,   0, 'f' },
2201                 {0,         0,             0,  0  }
2202         };
2203
2204         if (session_key == 0) {
2205                 fprintf(stderr,
2206                         "Can't find env LST_SESSION or value is not valid\n");
2207                 return -1;
2208         }
2209
2210         while (1) {
2211                 c = getopt_long(argc, argv, "f",
2212                                 stop_batch_opts, &optidx);
2213
2214                 /* Detect the end of the options. */
2215                 if (c == -1)
2216                         break;
2217
2218                 switch (c) {
2219                 case 'f':
2220                         force = 1;
2221                         break;
2222                 default:
2223                         lst_print_usage(argv[0]);
2224                         return -1;
2225                 }
2226         }
2227
2228         if (optind == argc) {
2229                 batch = LST_DEFAULT_BATCH;
2230
2231         } else if (optind == argc - 1) {
2232                 batch = argv[optind];
2233
2234         } else {
2235                 lst_print_usage(argv[0]);
2236                 return -1;
2237         }
2238
2239         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2240         if (rc != 0) {
2241                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2242                         batch, strerror(errno));
2243                 return -1;
2244         }
2245
2246         CFS_INIT_LIST_HEAD(&head);
2247
2248         rc = lst_alloc_rpcent(&head, count, 0);
2249         if (rc != 0) {
2250                 fprintf(stderr, "Out of memory\n");
2251                 return -1;
2252         }
2253
2254         rc = lst_stop_batch_ioctl(batch, force, &head);
2255         if (rc != 0)
2256                 goto out;
2257
2258         while (1) {
2259                 lst_reset_rpcent(&head);
2260
2261                 rc = lst_query_batch_ioctl(batch, 0, 0, 30, &head);
2262                 if (rc != 0)
2263                         goto out;
2264
2265                 if (lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0 &&
2266                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0)
2267                         break;
2268
2269                 fprintf(stdout, "%d batch in stopping\n",
2270                         lstcon_tsbqry_stat_run(&trans_stat, 0));
2271                 sleep(1);
2272         }
2273
2274         fprintf(stdout, "Batch is stopped\n");
2275         lst_free_rpcent(&head);
2276
2277         return 0;
2278 out:
2279         if (rc == -1) {
2280                 lst_print_error("batch", "Failed to stop batch: %s\n",
2281                                 strerror(errno));
2282                 lst_free_rpcent(&head);
2283                 return -1;
2284         }
2285
2286         lst_print_transerr(&head, "stop batch");
2287
2288         lst_free_rpcent(&head);
2289
2290         return rc;
2291 }
2292
2293 int
2294 lst_list_batch_ioctl(int len, char *name, int index)
2295 {
2296         lstio_batch_list_args_t         args = {
2297                 .lstio_bat_key          = session_key,
2298                 .lstio_bat_idx          = index,
2299                 .lstio_bat_nmlen        = len,
2300                 .lstio_bat_namep        = name,
2301         };
2302
2303         return lst_ioctl(LSTIO_BATCH_LIST, &args, sizeof(args));
2304 }
2305
2306 int
2307 lst_info_batch_ioctl(char *batch, int test, int server,
2308                      lstcon_test_batch_ent_t *entp, int *idxp,
2309                      int *ndentp, lstcon_node_ent_t *dentsp)
2310 {
2311         lstio_batch_info_args_t         args = {
2312                 .lstio_bat_key          = session_key,
2313                 .lstio_bat_nmlen        = strlen(batch),
2314                 .lstio_bat_namep        = batch,
2315                 .lstio_bat_server       = server,
2316                 .lstio_bat_testidx      = test,
2317                 .lstio_bat_entp         = entp,
2318                 .lstio_bat_idxp         = idxp,
2319                 .lstio_bat_ndentp       = ndentp,
2320                 .lstio_bat_dentsp       = dentsp,
2321         };
2322
2323         return lst_ioctl(LSTIO_BATCH_INFO, &args, sizeof(args));
2324 }
2325
2326 int
2327 lst_list_batch_all(void)
2328 {
2329         char name[LST_NAME_SIZE];
2330         int  rc;
2331         int  i;
2332
2333         for (i = 0; ; i++) {
2334                 rc = lst_list_batch_ioctl(LST_NAME_SIZE, name, i);
2335                 if (rc == 0) {
2336                         fprintf(stdout, "%d) %s\n", i + 1, name);
2337                         continue;
2338                 }
2339
2340                 if (errno == ENOENT)
2341                         break;
2342
2343                 lst_print_error("batch", "Failed to list batch: %s\n",
2344                                 strerror(errno));
2345                 return rc;
2346         }
2347
2348         fprintf(stdout, "Total %d batches\n", i);
2349
2350         return 0;
2351 }
2352
2353 int
2354 lst_list_tsb_nodes(char *batch, int test, int server,
2355                    int count, int active, int invalid)
2356 {
2357         lstcon_node_ent_t *dents;
2358         int                index = 0;
2359         int                rc;
2360         int                c;
2361         int                i;
2362
2363         if (count == 0)
2364                 return 0;
2365
2366         /* verbose list, show nodes in batch or test */
2367         dents = malloc(count * sizeof(lstcon_node_ent_t));
2368         if (dents == NULL) {
2369                 fprintf(stdout, "Can't allocate memory\n");
2370                 return -1;
2371         }
2372
2373         rc = lst_info_batch_ioctl(batch, test, server,
2374                                   NULL, &index, &count, dents);
2375         if (rc != 0) {
2376                 free(dents);
2377                 lst_print_error((test > 0) ? "test" : "batch",
2378                                 (test > 0) ? "Failed to query test: %s\n" :
2379                                              "Failed to query batch: %s\n",
2380                                 strerror(errno));
2381                 return -1;
2382         }
2383
2384         for (i = 0, c = 0; i < count; i++) {
2385                 if ((!active  && dents[i].nde_state == LST_NODE_ACTIVE) ||
2386                     (!invalid && (dents[i].nde_state == LST_NODE_BUSY  ||
2387                                   dents[i].nde_state == LST_NODE_DOWN  ||
2388                                   dents[i].nde_state == LST_NODE_UNKNOWN)))
2389                         continue;
2390
2391                 fprintf(stdout, "\t%s: %s\n",
2392                         libcfs_id2str(dents[i].nde_id),
2393                         lst_node_state2str(dents[i].nde_state));
2394                 c++;
2395         }
2396
2397         fprintf(stdout, "Total %d nodes\n", c);
2398         free(dents);
2399
2400         return 0;
2401 }
2402
2403 int
2404 jt_lst_list_batch(int argc, char **argv)
2405 {
2406         lstcon_test_batch_ent_t ent;
2407         char                *batch   = NULL;
2408         int                  optidx  = 0;
2409         int                  verbose = 0; /* list nodes in batch or test */
2410         int                  invalid = 0;
2411         int                  active  = 0;
2412         int                  server  = 0;
2413         int                  ntest   = 0;
2414         int                  test    = 0;
2415         int                  c       = 0;
2416         int                  i;
2417         int                  rc;
2418
2419         static struct option list_batch_opts[] =
2420         {
2421                 {"test",    required_argument, 0, 't' },
2422                 {"invalid", no_argument,       0, 'i' },
2423                 {"active",  no_argument,       0, 'a' },
2424                 {"all",     no_argument,       0, 'l' },
2425                 {"server",  no_argument,       0, 's' },
2426                 {0,         0,                 0,  0  }
2427         };
2428
2429         if (session_key == 0) {
2430                 fprintf(stderr,
2431                         "Can't find env LST_SESSION or value is not valid\n");
2432                 return -1;
2433         }
2434
2435         while (1) {
2436                 c = getopt_long(argc, argv, "ailst:",
2437                                 list_batch_opts, &optidx);
2438
2439                 if (c == -1)
2440                         break;
2441
2442                 switch (c) {
2443                 case 'a':
2444                         verbose = active = 1;
2445                         break;
2446                 case 'i':
2447                         verbose = invalid = 1;
2448                         break;
2449                 case 'l':
2450                         verbose = active = invalid = 1;
2451                         break;
2452                 case 's':
2453                         server = 1;
2454                         break;
2455                 case 't':
2456                         test = atoi(optarg);
2457                         ntest = 1;
2458                         break;
2459                 default:
2460                         lst_print_usage(argv[0]);
2461                         return -1;
2462                 }
2463         }
2464
2465         if (optind == argc) {
2466                 /* list all batches */
2467                 rc = lst_list_batch_all();
2468                 return rc;
2469         }
2470
2471         if (ntest == 1 && test <= 0) {
2472                 fprintf(stderr, "Invalid test id, test id starts from 1\n");
2473                 return -1;
2474         }
2475
2476         if (optind != argc - 1) {
2477                 lst_print_usage(argv[0]);
2478                 return -1;
2479         }
2480
2481         batch = argv[optind];
2482
2483 loop:
2484         /* show detail of specified batch or test */
2485         rc = lst_info_batch_ioctl(batch, test, server,
2486                                   &ent, NULL, NULL, NULL);
2487         if (rc != 0) {
2488                 lst_print_error((test > 0) ? "test" : "batch",
2489                                 (test > 0) ? "Failed to query test: %s\n" :
2490                                              "Failed to query batch: %s\n",
2491                                 strerror(errno));
2492                 return -1;
2493         }
2494
2495         if (verbose) {
2496                 /* list nodes in test or batch */
2497                 rc = lst_list_tsb_nodes(batch, test, server,
2498                                         server ? ent.tbe_srv_nle.nle_nnode :
2499                                                  ent.tbe_cli_nle.nle_nnode,
2500                                         active, invalid);
2501                 return rc;
2502         }
2503
2504         /* only show number of hosts in batch or test */
2505         if (test == 0) {
2506                 fprintf(stdout, "Batch: %s Tests: %d State: %d\n",
2507                         batch, ent.u.tbe_batch.bae_ntest,
2508                         ent.u.tbe_batch.bae_state);
2509                 ntest = ent.u.tbe_batch.bae_ntest;
2510                 test = 1; /* starting from test 1 */
2511
2512         } else {
2513                 fprintf(stdout,
2514                         "\tTest %d(%s) (loop: %d, concurrency: %d)\n",
2515                         test, lst_test_type2name(ent.u.tbe_test.tse_type),
2516                         ent.u.tbe_test.tse_loop,
2517                         ent.u.tbe_test.tse_concur);
2518                 ntest --;
2519                 test ++;
2520         }
2521
2522         fprintf(stdout, LST_NODES_TITLE);
2523         fprintf(stdout, "client\t%d\t%d\t%d\t%d\t%d\n"
2524                         "server\t%d\t%d\t%d\t%d\t%d\n",
2525                 ent.tbe_cli_nle.nle_nactive,
2526                 ent.tbe_cli_nle.nle_nbusy,
2527                 ent.tbe_cli_nle.nle_ndown,
2528                 ent.tbe_cli_nle.nle_nunknown,
2529                 ent.tbe_cli_nle.nle_nnode,
2530                 ent.tbe_srv_nle.nle_nactive,
2531                 ent.tbe_srv_nle.nle_nbusy,
2532                 ent.tbe_srv_nle.nle_ndown,
2533                 ent.tbe_srv_nle.nle_nunknown,
2534                 ent.tbe_srv_nle.nle_nnode);
2535
2536         if (ntest != 0)
2537                 goto loop;
2538
2539         return 0;
2540 }
2541
2542 int
2543 lst_query_batch_ioctl(char *batch, int test, int server,
2544                       int timeout, struct list_head *head)
2545 {
2546         lstio_batch_query_args_t args = {
2547                 .lstio_bat_key     = session_key,
2548                 .lstio_bat_testidx = test,
2549                 .lstio_bat_client  = !(server),
2550                 .lstio_bat_timeout = timeout,
2551                 .lstio_bat_nmlen   = strlen(batch),
2552                 .lstio_bat_namep   = batch,
2553                 .lstio_bat_resultp = head,
2554         };
2555
2556         return lst_ioctl(LSTIO_BATCH_QUERY, &args, sizeof(args));
2557 }
2558
2559 void
2560 lst_print_tsb_verbose(struct list_head *head,
2561                       int active, int idle, int error)
2562 {
2563         lstcon_rpc_ent_t *ent;
2564
2565         list_for_each_entry(ent, head, rpe_link) {
2566                 if (ent->rpe_priv[0] == 0 && active)
2567                         continue;
2568
2569                 if (ent->rpe_priv[0] != 0 && idle)
2570                         continue;
2571
2572                 if (ent->rpe_fwk_errno == 0 && error)
2573                         continue;
2574
2575                 fprintf(stdout, "%s [%s]: %s\n",
2576                         libcfs_id2str(ent->rpe_peer),
2577                         lst_node_state2str(ent->rpe_state),
2578                         ent->rpe_rpc_errno != 0 ?
2579                                 strerror(ent->rpe_rpc_errno) :
2580                                 (ent->rpe_priv[0] > 0 ? "Running" : "Idle"));
2581         }
2582 }
2583
2584 int
2585 jt_lst_query_batch(int argc, char **argv)
2586 {
2587         lstcon_test_batch_ent_t ent;
2588         struct list_head     head;
2589         lstcon_rpc_ent_t    *rent    = NULL;
2590         char                *batch   = NULL;
2591         time_t               last    = 0;
2592         int                  optidx  = 0;
2593         int                  index   = 0;
2594         int                  verbose = 0;
2595         int                  server  = 0;
2596         int                  timeout = 5; /* default 5 seconds */
2597         int                  delay   = 5; /* default 5 seconds */
2598         int                  loop    = 1; /* default 1 loop */
2599         int                  active  = 0;
2600         int                  error   = 0;
2601         int                  idle    = 0;
2602         int                  count   = 0;
2603         int                  test    = 0;
2604         int                  rc      = 0;
2605         int                  c       = 0;
2606         int                  i;
2607
2608         static struct option query_batch_opts[] =
2609         {
2610                 {"timeout", required_argument, 0, 'o' },
2611                 {"delay",   required_argument, 0, 'd' },
2612                 {"loop",    required_argument, 0, 'c' },
2613                 {"test",    required_argument, 0, 't' },
2614                 {"server",  no_argument,       0, 's' },
2615                 {"active",  no_argument,       0, 'a' },
2616                 {"idle",    no_argument,       0, 'i' },
2617                 {"error",   no_argument,       0, 'e' },
2618                 {"all",     no_argument,       0, 'l' },
2619                 {0,         0,                 0,  0  }
2620         };
2621
2622         if (session_key == 0) {
2623                 fprintf(stderr,
2624                         "Can't find env LST_SESSION or value is not valid\n");
2625                 return -1;
2626         }
2627
2628         while (1) {
2629                 c = getopt_long(argc, argv, "o:d:c:t:saiel",
2630                                 query_batch_opts, &optidx);
2631
2632                 /* Detect the end of the options. */
2633                 if (c == -1)
2634                         break;
2635
2636                 switch (c) {
2637                 case 'o':
2638                         timeout = atoi(optarg);
2639                         break;
2640                 case 'd':
2641                         delay = atoi(optarg);
2642                         break;
2643                 case 'c':
2644                         loop = atoi(optarg);
2645                         break;
2646                 case 't':
2647                         test = atoi(optarg);
2648                         break;
2649                 case 's':
2650                         server = 1;
2651                         break;
2652                 case 'a':
2653                         active = verbose = 1;
2654                         break;
2655                 case 'i':
2656                         idle = verbose = 1;
2657                         break;
2658                 case 'e':
2659                         error = verbose = 1;
2660                         break;
2661                 case 'l':
2662                         verbose = 1;
2663                         break;
2664                 default:
2665                         lst_print_usage(argv[0]);
2666                         return -1;
2667                 }
2668         }
2669
2670         if (test < 0 || timeout <= 0 || delay <= 0 || loop <= 0) {
2671                 lst_print_usage(argv[0]);
2672                 return -1;
2673         }
2674
2675         if (optind == argc) {
2676                 batch = LST_DEFAULT_BATCH;
2677
2678         } else if (optind == argc - 1) {
2679                 batch = argv[optind];
2680
2681         } else {
2682                 lst_print_usage(argv[0]);
2683                 return -1;
2684         }
2685
2686
2687         CFS_INIT_LIST_HEAD(&head);
2688
2689         if (verbose) {
2690                 rc = lst_info_batch_ioctl(batch, test, server,
2691                                           &ent, NULL, NULL, NULL);
2692                 if (rc != 0) {
2693                         fprintf(stderr, "Failed to query %s [%d]: %s\n",
2694                                 batch, test, strerror(errno));
2695                         return -1;
2696                 }
2697
2698                 count = server ? ent.tbe_srv_nle.nle_nnode :
2699                                  ent.tbe_cli_nle.nle_nnode;
2700                 if (count == 0) {
2701                         fprintf(stdout, "Batch or test is empty\n");
2702                         return 0;
2703                 }
2704         }
2705
2706         rc = lst_alloc_rpcent(&head, count, 0);
2707         if (rc != 0) {
2708                 fprintf(stderr, "Out of memory\n");
2709                 return rc;
2710         }
2711
2712         for (i = 0; i < loop; i++) {
2713                 time_t  now = time(NULL);
2714
2715                 if (now - last < delay) {
2716                         sleep(delay - now + last);
2717                         time(&now);
2718                 }
2719
2720                 last = now;
2721
2722                 rc = lst_query_batch_ioctl(batch, test,
2723                                            server, timeout, &head);
2724                 if (rc == -1) {
2725                         fprintf(stderr, "Failed to query batch: %s\n",
2726                                 strerror(errno));
2727                         break;
2728                 }
2729
2730                 if (verbose) {
2731                         /* Verbose mode */
2732                         lst_print_tsb_verbose(&head, active, idle, error);
2733                         continue;
2734                 }
2735
2736                 fprintf(stdout, "%s [%d] ", batch, test);
2737
2738                 if (lstcon_rpc_stat_failure(&trans_stat, 0) != 0) {
2739                         fprintf(stdout, "%d of %d nodes are unknown, ",
2740                                 lstcon_rpc_stat_failure(&trans_stat, 0),
2741                                 lstcon_rpc_stat_total(&trans_stat, 0));
2742                 }
2743
2744                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
2745                     lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0  &&
2746                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
2747                         fprintf(stdout, "is stopped\n");
2748                         continue;
2749                 }
2750
2751                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
2752                     lstcon_tsbqry_stat_idle(&trans_stat, 0) == 0 &&
2753                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
2754                         fprintf(stdout, "is running\n");
2755                         continue;
2756                 }
2757
2758                 fprintf(stdout, "stopped: %d , running: %d, failed: %d\n",
2759                                 lstcon_tsbqry_stat_idle(&trans_stat, 0),
2760                                 lstcon_tsbqry_stat_run(&trans_stat, 0),
2761                                 lstcon_tsbqry_stat_failure(&trans_stat, 0));
2762         }
2763
2764         lst_free_rpcent(&head);
2765
2766         return rc;
2767 }
2768
2769 int
2770 lst_parse_distribute(char *dstr, int *dist, int *span)
2771 {
2772         *dist = atoi(dstr);
2773         if (*dist <= 0)
2774                 return -1;
2775
2776         dstr = strchr(dstr, ':');
2777         if (dstr == NULL)
2778                 return -1;
2779
2780         *span = atoi(dstr + 1);
2781         if (*span <= 0)
2782                 return -1;
2783
2784         return 0;
2785 }
2786
2787 int
2788 lst_get_test_param(char *test, int argc, char **argv, void **param, int *plen)
2789 {
2790         lst_test_bulk_param_t *bulk = NULL;
2791         lst_test_ping_param_t *ping = NULL;
2792         int                    type;
2793         int                    i = 0;
2794
2795         type = lst_test_name2type(test);
2796         if (type < 0)
2797                 return -EINVAL;
2798
2799         switch (type) {
2800         case LST_TEST_PING:
2801                 break;
2802
2803         case LST_TEST_BULK:
2804                 if (i == argc)
2805                         return -EINVAL;
2806
2807                 bulk = malloc(sizeof(*bulk));
2808                 if (bulk == NULL)
2809                         return -ENOMEM;
2810
2811                 memset(bulk, 0, sizeof(*bulk));
2812
2813                 if (strcmp(argv[i], "w") == 0)
2814                         bulk->blk_opc = LST_BRW_WRITE;
2815                 else  /* read by default */
2816                         bulk->blk_opc = LST_BRW_READ;
2817
2818                 if (++i == argc) {
2819                         /* 1 page by default */
2820                         bulk->blk_flags = LST_BRW_CHECK_NONE;
2821                         bulk->blk_npg   = 1;
2822                         *param = bulk;
2823                         *plen  = sizeof(*bulk);
2824                         break;
2825
2826                 }
2827
2828                 bulk->blk_npg = atoi(argv[i]);
2829                 if (bulk->blk_npg <= 0 ||
2830                     bulk->blk_npg >= LNET_MAX_IOV) {
2831                         free(bulk);
2832                         return -EINVAL;
2833                 }
2834
2835                 if (++i == argc) {
2836                         bulk->blk_flags = LST_BRW_CHECK_NONE;
2837                         *param = bulk;
2838                         *plen  = sizeof(*bulk);
2839                         break;
2840                 }
2841
2842                 /* posion pages */
2843                 if (strcmp(argv[i], "s") == 0)
2844                         bulk->blk_flags = LST_BRW_CHECK_SIMPLE;
2845                 else if (strcmp(argv[i], "f") == 0)
2846                         bulk->blk_flags = LST_BRW_CHECK_FULL;
2847                 else
2848                         bulk->blk_flags = LST_BRW_CHECK_NONE;
2849
2850                 *param = bulk;
2851                 *plen  = sizeof(*bulk);
2852
2853                 break;
2854
2855         default:
2856                 break;
2857         }
2858
2859         /* TODO: parse more parameter */
2860         return type;
2861 }
2862
2863 int
2864 lst_add_test_ioctl(char *batch, int type, int loop, int concur,
2865                    int dist, int span, char *sgrp, char *dgrp,
2866                    void *param, int plen, int *retp, struct list_head *resultp)
2867 {
2868         lstio_test_args_t args = {
2869                 .lstio_tes_key          = session_key,
2870                 .lstio_tes_bat_nmlen    = strlen(batch),
2871                 .lstio_tes_bat_name     = batch,
2872                 .lstio_tes_type         = type,
2873                 .lstio_tes_loop         = loop,
2874                 .lstio_tes_concur       = concur,
2875                 .lstio_tes_dist         = dist,
2876                 .lstio_tes_span         = span,
2877                 .lstio_tes_sgrp_nmlen   = strlen(sgrp),
2878                 .lstio_tes_sgrp_name    = sgrp,
2879                 .lstio_tes_dgrp_nmlen   = strlen(dgrp),
2880                 .lstio_tes_dgrp_name    = dgrp,
2881                 .lstio_tes_param_len    = plen,
2882                 .lstio_tes_param        = param,
2883                 .lstio_tes_retp         = retp,
2884                 .lstio_tes_resultp      = resultp,
2885         };
2886
2887         return lst_ioctl(LSTIO_TEST_ADD, &args, sizeof(args));
2888 }
2889
2890 int
2891 jt_lst_add_test(int argc, char **argv)
2892 {
2893         struct list_head  head;
2894         char             *batch  = NULL;
2895         char             *test   = NULL;
2896         char             *dstr   = NULL;
2897         char             *from   = NULL;
2898         char             *to     = NULL;
2899         void             *param  = NULL;
2900         int               optidx = 0;
2901         int               concur = 1;
2902         int               loop   = -1;
2903         int               dist   = 1;
2904         int               span   = 1;
2905         int               plen   = 0;
2906         int               fcount = 0;
2907         int               tcount = 0;
2908         int               ret    = 0;
2909         int               type;
2910         int               rc;
2911         int               c;
2912
2913         static struct option add_test_opts[] =
2914         {
2915                 {"batch",       required_argument, 0, 'b' },
2916                 {"concurrency", required_argument, 0, 'c' },
2917                 {"distribute",  required_argument, 0, 'd' },
2918                 {"from",        required_argument, 0, 'f' },
2919                 {"to",          required_argument, 0, 't' },
2920                 {"loop",        required_argument, 0, 'l' },
2921                 {0,             0,                 0,  0  }
2922         };
2923
2924         if (session_key == 0) {
2925                 fprintf(stderr,
2926                         "Can't find env LST_SESSION or value is not valid\n");
2927                 return -1;
2928         }
2929
2930         while (1) {
2931                 c = getopt_long(argc, argv, "b:c:d:f:l:t:",
2932                                 add_test_opts, &optidx);
2933
2934                 /* Detect the end of the options. */
2935                 if (c == -1)
2936                         break;
2937
2938                 switch (c) {
2939                 case 'b':
2940                         batch = optarg;
2941                         break;
2942                 case 'c':
2943                         concur = atoi(optarg);
2944                         break;
2945                 case 'd':
2946                         dstr = optarg;
2947                         break;
2948                 case 'f':
2949                         from = optarg;
2950                         break;
2951                 case 'l':
2952                         loop = atoi(optarg);
2953                         break;
2954                 case 't':
2955                         to = optarg;
2956                         break;
2957                 default:
2958                         lst_print_usage(argv[0]);
2959                         return -1;
2960                 }
2961         }
2962
2963         if (optind == argc || from == NULL || to == NULL) {
2964                 lst_print_usage(argv[0]);
2965                 return -1;
2966         }
2967
2968         if (concur <= 0 || concur > LST_MAX_CONCUR) {
2969                 fprintf(stderr, "Invalid concurrency of test: %d\n", concur);
2970                 return -1;
2971         }
2972
2973         if (batch == NULL)
2974                 batch = LST_DEFAULT_BATCH;
2975
2976         if (dstr != NULL) {
2977                 rc = lst_parse_distribute(dstr, &dist, &span);
2978                 if (rc != 0) {
2979                         fprintf(stderr, "Invalid distribution: %s\n", dstr);
2980                         return -1;
2981                 }
2982         }
2983
2984         test = argv[optind++];
2985
2986         argc -= optind;
2987         argv += optind;
2988
2989         type = lst_get_test_param(test, argc, argv, &param, &plen);
2990         if (type < 0) {
2991                 fprintf(stderr, "Can't parse test (%s)  parameter: %s\n",
2992                         test, strerror(-type));
2993                 return -1;
2994         }
2995
2996         CFS_INIT_LIST_HEAD(&head);
2997
2998         rc = lst_get_node_count(LST_OPC_GROUP, from, &fcount, NULL);
2999         if (rc != 0) {
3000                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3001                         from, strerror(errno));
3002                 goto out;
3003         }
3004
3005         rc = lst_get_node_count(LST_OPC_GROUP, to, &tcount, NULL);
3006         if (rc != 0) {
3007                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3008                         to, strerror(errno));
3009                 goto out;
3010         }
3011
3012         rc = lst_alloc_rpcent(&head, fcount > tcount ? fcount : tcount, 0);
3013         if (rc != 0) {
3014                 fprintf(stderr, "Out of memory\n");
3015                 goto out;
3016         }
3017
3018         rc = lst_add_test_ioctl(batch, type, loop, concur,
3019                                 dist, span, from, to, param, plen, &ret, &head);
3020
3021         if (rc == 0) {
3022                 fprintf(stdout, "Test was added successfully\n");
3023                 if (ret != 0) {
3024                         fprintf(stdout, "Server group contains userland test "
3025                                 "nodes, old version of tcplnd can't accept "
3026                                 "connection request\n");
3027                 }
3028
3029                 goto out;
3030         }
3031
3032         if (rc == -1) {
3033                 lst_print_error("test", "Failed to add test: %s\n",
3034                                 strerror(errno));
3035                 goto out;
3036         }
3037
3038         lst_print_transerr(&head, "add test");
3039 out:
3040         lst_free_rpcent(&head);
3041
3042         if (param != NULL)
3043                 free(param);
3044
3045         return rc;
3046 }
3047
3048 static command_t lst_cmdlist[] = {
3049         {"new_session",         jt_lst_new_session,     NULL,
3050          "Usage: lst new_session [--timeout TIME] [--force] [NAME]"                     },
3051         {"end_session",         jt_lst_end_session,     NULL,
3052          "Usage: lst end_session"                                                       },
3053         {"show_session",        jt_lst_show_session,    NULL,
3054          "Usage: lst show_session"                                                      },
3055         {"ping",                jt_lst_ping ,           NULL,
3056          "Usage: lst ping  [--group NAME] [--batch NAME] [--session] [--nodes IDS]"     },
3057         {"add_group",           jt_lst_add_group,       NULL,
3058          "Usage: lst group NAME IDs [IDs]..."                                           },
3059         {"del_group",           jt_lst_del_group,       NULL,
3060          "Usage: lst del_group NAME"                                                    },
3061         {"update_group",        jt_lst_update_group,    NULL,
3062          "Usage: lst update_group NAME [--clean] [--refresh] [--remove IDs]"            },
3063         {"list_group",          jt_lst_list_group,      NULL,
3064           "Usage: lst list_group [--active] [--busy] [--down] [--unknown] GROUP ..."    },
3065         {"stat",                jt_lst_stat,            NULL,
3066          "Usage: lst stat [--bw] [--rate] [--read] [--write] [--max] [--min] [--avg] "
3067          " [--timeout #] [--delay #] GROUP [GROUP]"                                     },
3068         {"show_error",          jt_lst_show_error,      NULL,
3069          "Usage: lst show_error [--group NAME] | [--nodes IDS]"                         },
3070         {"add_batch",           jt_lst_add_batch,       NULL,
3071          "Usage: lst add_batch NAME"                                                    },
3072         {"run",                 jt_lst_start_batch,     NULL,
3073          "Usage: lst run [--timeout TIME] [NAME]"                                       },
3074         {"stop",                jt_lst_stop_batch,      NULL,
3075          "Usage: lst stop [--force] BATCH_NAME"                                         },
3076         {"list_batch",          jt_lst_list_batch,      NULL,
3077          "Usage: lst list_batch NAME [--test ID] [--server]"                            },
3078         {"query",               jt_lst_query_batch,     NULL,
3079          "Usage: lst query [--test ID] [--server] [--timeout TIME] NAME"                },
3080         {"add_test",            jt_lst_add_test,        NULL,
3081          "Usage: lst add_test [--batch BATCH] [--loop #] [--concurrency #] "
3082          " [--distribute #:#] [--from GROUP] [--to GROUP] TEST..."                      },
3083         {"help",                Parser_help,            0,     "help"                   },
3084         {0,                     0,                      0,      NULL                    }
3085 };
3086
3087 int
3088 lst_initialize(void)
3089 {
3090         char   *key;
3091
3092         key = getenv("LST_SESSION");
3093
3094         if (key == NULL) {
3095                 session_key = 0;
3096                 return 0;
3097         }
3098
3099         session_key = atoi(key);
3100
3101         return 0;
3102 }
3103
3104 int
3105 main(int argc, char **argv)
3106 {
3107         int     rc;
3108
3109         setlinebuf(stdout);
3110
3111         if (lst_initialize() < 0)
3112                 exit(0);
3113
3114         if (ptl_initialize(argc, argv) < 0)
3115                 exit(0);
3116
3117         Parser_init("lst > ", lst_cmdlist);
3118
3119         if (argc != 1)
3120                 return Parser_execarg(argc - 1, argv + 1, lst_cmdlist);
3121
3122         Parser_commands();
3123
3124         return 0;
3125 }