lnet/utils/lst.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * Author: Liang Zhen <liangzhen@clusterfs.com>
   5  *
   6  * This file is part of Lustre, http://www.lustre.org
   7  */
   8
   9 #include <stdio.h>
  10 #include <stdlib.h>
  11 #include <getopt.h>
  12 #include <errno.h>
  13 #include <pwd.h>
  14 #include <lnet/lnetctl.h>
  15 #include <lnet/lnetst.h>
  16 #include "parser.h"
  17
  18 static command_t           lst_cmdlist[];
  19 static lst_sid_t           session_id;
  20 static int                 session_key;
  21 static lstcon_trans_stat_t trans_stat;
  22
  23 typedef struct list_string {
  24         struct list_string *lstr_next;
  25         int                 lstr_sz;
  26         char                lstr_str[0];
  27 } lstr_t;
  28
  29 #define offsetof(typ,memb)     ((unsigned long)((char *)&(((typ *)0)->memb)))
  30
  31 static int alloc_count = 0;
  32 static int alloc_nob   = 0;
  33
  34 lstr_t *
  35 alloc_lstr(int sz)
  36 {
  37         lstr_t  *lstr = malloc(offsetof(lstr_t, lstr_str[sz]));
  38
  39         if (lstr == NULL) {
  40                 fprintf(stderr, "Can't allocate lstr\n");
  41                 abort();
  42         }
  43
  44         alloc_nob += sz;
  45         alloc_count++;
  46
  47         lstr->lstr_str[0] = 0;
  48         lstr->lstr_sz = sz;
  49         return lstr;
  50 }
  51
  52 void
  53 free_lstr(lstr_t *lstr)
  54 {
  55         alloc_count--;
  56         alloc_nob -= lstr->lstr_sz;
  57         free(lstr);
  58 }
  59
  60 void
  61 free_lstrs(lstr_t **list)
  62 {
  63         lstr_t   *lstr;
  64
  65         while ((lstr = *list) != NULL) {
  66                 *list = lstr->lstr_next;
  67                 free_lstr(lstr);
  68         }
  69 }
  70
  71 void
  72 new_lstrs(lstr_t **list, char *prefix, char *postfix,
  73           int lo, int hi, int stride)
  74 {
  75         int    n1 = strlen(prefix);
  76         int    n2 = strlen(postfix);
  77         int    sz = n1 + 20 + n2 + 1;
  78
  79         do {
  80                 lstr_t *n = alloc_lstr(sz);
  81
  82                 snprintf(n->lstr_str, sz - 1, "%s%u%s",
  83                          prefix, lo, postfix);
  84
  85                 n->lstr_next = *list;
  86                 *list = n;
  87
  88                 lo += stride;
  89         } while (lo <= hi);
  90 }
  91
  92 int
  93 expand_lstr(lstr_t **list, lstr_t *l)
  94 {
  95         int          nob = strlen(l->lstr_str);
  96         char        *b1;
  97         char        *b2;
  98         char        *expr;
  99         char        *sep;
 100         int          x;
 101         int          y;
 102         int          z;
 103         int          n;
 104
 105         b1 = strchr(l->lstr_str, '[');
 106         if (b1 == NULL) {
 107                 l->lstr_next = *list;
 108                 *list = l;
 109                 return 0;
 110         }
 111
 112         b2 = strchr(b1, ']');
 113         if (b2 == NULL || b2 == b1 + 1)
 114                 return -1;
 115
 116         *b1++ = 0;
 117         *b2++ = 0;
 118         expr = b1;
 119         do {
 120
 121                 sep = strchr(expr, ',');
 122                 if (sep != NULL)
 123                         *sep++ = 0;
 124
 125                 nob = strlen(expr);
 126                 n = nob;
 127                 if (sscanf(expr, "%u%n", &x, &n) >= 1 && n == nob) {
 128                         /* simple number */
 129                         new_lstrs(list, l->lstr_str, b2, x, x, 1);
 130                         continue;
 131                 }
 132
 133                 n = nob;
 134                 if (sscanf(expr, "%u-%u%n", &x, &y, &n) >= 2 && n == nob &&
 135                     x < y) {
 136                         /* simple range */
 137                         new_lstrs(list, l->lstr_str, b2, x, y, 1);
 138                         continue;
 139                 }
 140
 141                 n = nob;
 142                 if (sscanf(expr, "%u-%u/%u%n", &x, &y, &z, &n) >= 3 && n == nob &&
 143                     x < y) {
 144                         /* strided range */
 145                         new_lstrs(list, l->lstr_str, b2, x, y, z);
 146                         continue;
 147                 }
 148
 149                 /* syntax error */
 150                 return -1;
 151         } while ((expr = sep) != NULL);
 152
 153         free_lstr(l);
 154
 155         return 1;
 156 }
 157
 158 int
 159 expand_strs(char *str, lstr_t **head)
 160 {
 161         lstr_t  *list = NULL;
 162         lstr_t  *nlist;
 163         lstr_t  *l;
 164         int      rc = 0;
 165         int      expanded;
 166
 167         l = alloc_lstr(strlen(str) + 1);
 168         memcpy(l->lstr_str, str, strlen(str) + 1);
 169         l->lstr_next = NULL;
 170         list = l;
 171
 172         do {
 173                 expanded = 0;
 174                 nlist = NULL;
 175
 176                 while ((l = list) != NULL) {
 177                         list = l->lstr_next;
 178
 179                         rc = expand_lstr(&nlist, l);
 180                         if (rc < 0) {
 181                                 fprintf(stderr, "Syntax error in \"%s\"\n", str);
 182                                 free_lstr(l);
 183                                 break;
 184                         }
 185
 186                         expanded |= rc > 0;
 187                 }
 188
 189                 /* re-order onto 'list' */
 190                 while ((l = nlist) != NULL) {
 191                         nlist = l->lstr_next;
 192                         l->lstr_next = list;
 193                         list = l;
 194                 }
 195
 196         } while (expanded && rc > 0);
 197
 198         if (rc >= 0) {
 199                 *head = list;
 200                 return 0;
 201         }
 202
 203         while ((l = list) != NULL) {
 204                 list = l->lstr_next;
 205
 206                 free_lstr(l);
 207         }
 208         return rc;
 209 }
 210
 211 int
 212 lst_parse_nids(char *str, int *countp, lnet_process_id_t **idspp)
 213 {
 214         lstr_t  *head = NULL;
 215         lstr_t  *l;
 216         int      c = 0;
 217         int      i;
 218         int      rc;
 219
 220         rc = expand_strs(str, &head);
 221         if (rc != 0)
 222                 goto out;
 223
 224         l = head;
 225         while (l != NULL) {
 226                 l = l->lstr_next;
 227                 c++;
 228         }
 229
 230         *idspp = malloc(c * sizeof(lnet_process_id_t));
 231         if (*idspp == NULL) {
 232                 fprintf(stderr, "Out of memory\n");
 233                 rc = -1;
 234         }
 235
 236         *countp = c;
 237 out:
 238         i = 0;
 239         while ((l = head) != NULL) {
 240                 head = l->lstr_next;
 241
 242                 if (rc == 0) {
 243                         (*idspp)[i].nid = libcfs_str2nid(l->lstr_str);
 244                         if ((*idspp)[i].nid == LNET_NID_ANY) {
 245                                 fprintf(stderr, "Invalid nid: %s\n",
 246                                         l->lstr_str);
 247                                 rc = -1;
 248                         }
 249
 250                         (*idspp)[i].pid = LUSTRE_LNET_PID;
 251                         i++;
 252                 }
 253
 254                 free_lstr(l);
 255         }
 256
 257         if (rc == 0)
 258                 return 0;
 259
 260         free(*idspp);
 261         *idspp = NULL;
 262
 263         return rc;
 264 }
 265
 266 char *
 267 lst_node_state2str(int state)
 268 {
 269         if (state == LST_NODE_ACTIVE)
 270                 return "Active";
 271         if (state == LST_NODE_BUSY)
 272                 return "Busy";
 273         if (state == LST_NODE_DOWN)
 274                 return "Down";
 275
 276         return "Unknown";
 277 }
 278
 279 int
 280 lst_node_str2state(char *str)
 281 {
 282         if (strcasecmp(str, "active") == 0)
 283                 return LST_NODE_ACTIVE;
 284         if (strcasecmp(str, "busy") == 0)
 285                 return LST_NODE_BUSY;
 286         if (strcasecmp(str, "down") == 0)
 287                 return LST_NODE_DOWN;
 288         if (strcasecmp(str, "unknown") == 0)
 289                 return LST_NODE_UNKNOWN;
 290         if (strcasecmp(str, "invalid") == 0)
 291                 return (LST_NODE_UNKNOWN | LST_NODE_DOWN | LST_NODE_BUSY);
 292
 293         return -1;
 294 }
 295
 296 char *
 297 lst_test_type2name(int type)
 298 {
 299         if (type == LST_TEST_PING)
 300                 return "ping";
 301         if (type == LST_TEST_BULK)
 302                 return "brw";
 303
 304         return "unknown";
 305 }
 306
 307 int
 308 lst_test_name2type(char *name)
 309 {
 310         if (strcasecmp(name, "ping") == 0)
 311                 return LST_TEST_PING;
 312         if (strcasecmp(name, "brw") == 0)
 313                 return LST_TEST_BULK;
 314
 315         return -1;
 316 }
 317
 318 void
 319 lst_print_usage(char *cmd)
 320 {
 321         Parser_printhelp(cmd);
 322 }
 323
 324 void
 325 lst_print_error(char *sub, const char *def_format, ...)
 326 {
 327         va_list ap;
 328
 329         /* local error returned from kernel */
 330         switch (errno) {
 331         case ESRCH:
 332                 fprintf(stderr, "No session exists\n");
 333                 return;
 334         case ESHUTDOWN:
 335                 fprintf(stderr, "Session is shutting down\n");
 336                 return;
 337         case EACCES:
 338                 fprintf(stderr, "Unmatched session key or not root\n");
 339                 return;
 340         case ENOENT:
 341                 fprintf(stderr, "Can't find %s in current session\n", sub);
 342                 return;
 343         case EINVAL:
 344                 fprintf(stderr, "Invalid parameters list in command line\n");
 345                 return;
 346         case EFAULT:
 347                 fprintf(stderr, "Bad parameter address\n");
 348                 return;
 349         case EEXIST:
 350                 fprintf(stderr, "%s already exists\n", sub);
 351                 return;
 352         default:
 353                 va_start(ap, def_format);
 354                 vfprintf(stderr, def_format, ap);
 355                 va_end(ap);
 356
 357                 return;
 358         }
 359 }
 360
 361 void
 362 lst_free_rpcent(struct list_head *head)
 363 {
 364         lstcon_rpc_ent_t *ent;
 365
 366         while (!list_empty(head)) {
 367                 ent = list_entry(head->next, lstcon_rpc_ent_t, rpe_link);
 368
 369                 list_del(&ent->rpe_link);
 370                 free(ent);
 371         }
 372 }
 373
 374 void
 375 lst_reset_rpcent(struct list_head *head)
 376 {
 377         lstcon_rpc_ent_t *ent;
 378
 379         list_for_each_entry(ent, head, rpe_link) {
 380                 ent->rpe_sid      = LST_INVALID_SID;
 381                 ent->rpe_peer.nid = LNET_NID_ANY;
 382                 ent->rpe_peer.pid = LNET_PID_ANY;
 383                 ent->rpe_rpc_errno = ent->rpe_fwk_errno = 0;
 384         }
 385 }
 386
 387 int
 388 lst_alloc_rpcent(struct list_head *head, int count, int offset)
 389 {
 390         lstcon_rpc_ent_t *ent;
 391         int               i;
 392
 393         for (i = 0; i < count; i++) {
 394                 ent = malloc(offsetof(lstcon_rpc_ent_t, rpe_payload[offset]));
 395                 if (ent == NULL) {
 396                         lst_free_rpcent(head);
 397                         return -1;
 398                 }
 399
 400                 memset(ent, 0, offsetof(lstcon_rpc_ent_t, rpe_payload[offset]));
 401
 402                 ent->rpe_sid      = LST_INVALID_SID;
 403                 ent->rpe_peer.nid = LNET_NID_ANY;
 404                 ent->rpe_peer.pid = LNET_PID_ANY;
 405                 list_add(&ent->rpe_link, head);
 406         }
 407
 408         return 0;
 409 }
 410
 411 void
 412 lst_print_transerr(struct list_head *head, char *optstr)
 413 {
 414         lstcon_rpc_ent_t  *ent;
 415
 416         list_for_each_entry(ent, head, rpe_link) {
 417                 if (ent->rpe_rpc_errno == 0 && ent->rpe_fwk_errno == 0)
 418                         continue;
 419
 420                 if (ent->rpe_rpc_errno != 0) {
 421                         fprintf(stderr, "%s RPC failed on %s: %s\n",
 422                                 optstr, libcfs_id2str(ent->rpe_peer),
 423                                 strerror(ent->rpe_rpc_errno));
 424                         continue;
 425                 }
 426
 427                 fprintf(stderr, "%s failed on %s: %s\n",
 428                         optstr, libcfs_id2str(ent->rpe_peer),
 429                         strerror(ent->rpe_fwk_errno));
 430         }
 431 }
 432
 433 int
 434 lst_ioctl(unsigned int opc, void *buf, int len)
 435 {
 436         struct libcfs_ioctl_data data;
 437         int    rc;
 438
 439         LIBCFS_IOC_INIT (data);
 440         data.ioc_u32[0]  = opc;
 441         data.ioc_plen1   = len;
 442         data.ioc_pbuf1   = (char *)buf;
 443         data.ioc_plen2   = sizeof(trans_stat);
 444         data.ioc_pbuf2   = (char *)&trans_stat;
 445
 446         memset(&trans_stat, 0, sizeof(trans_stat));
 447
 448         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNETST, &data);
 449
 450         /* local error, no valid RPC result */
 451         if (rc != 0)
 452                 return -1;
 453
 454         /* RPC error */
 455         if (trans_stat.trs_rpc_errno != 0)
 456                 return -2;
 457
 458         /* Framework error */
 459         if (trans_stat.trs_fwk_errno != 0)
 460                 return -3;
 461
 462         return 0;
 463 }
 464
 465 int
 466 lst_new_session_ioctl (char *name, int timeout, int force, lst_sid_t *sid)
 467 {
 468         lstio_session_new_args_t        args = {
 469                 .lstio_ses_key          = session_key,
 470                 .lstio_ses_timeout      = timeout,
 471                 .lstio_ses_force        = force,
 472                 .lstio_ses_idp          = sid,
 473                 .lstio_ses_namep        = name,
 474                 .lstio_ses_nmlen        = strlen(name),
 475         };
 476
 477         return lst_ioctl (LSTIO_SESSION_NEW, &args, sizeof(args));
 478 }
 479
 480 int
 481 jt_lst_new_session(int argc,  char **argv)
 482 {
 483         char  buf[LST_NAME_SIZE];
 484         char *name;
 485         int   optidx  = 0;
 486         int   timeout = 300;
 487         int   force   = 0;
 488         int   c;
 489         int   rc;
 490
 491         static struct option session_opts[] =
 492         {
 493                 {"timeout", required_argument,  0, 't' },
 494                 {"force",   no_argument,        0, 'f' },
 495                 {0,         0,                  0,  0  }
 496         };
 497
 498         if (session_key == 0) {
 499                 fprintf(stderr,
 500                         "Can't find env LST_SESSION or value is not valid\n");
 501                 return -1;
 502         }
 503
 504         while (1) {
 505
 506                 c = getopt_long(argc, argv, "ft:",
 507                                 session_opts, &optidx);
 508
 509                 if (c == -1)
 510                         break;
 511
 512                 switch (c) {
 513                 case 'f':
 514                         force = 1;
 515                         break;
 516                 case 't':
 517                         timeout = atoi(optarg);
 518                         break;
 519                 default:
 520                         lst_print_usage(argv[0]);
 521                         return -1;
 522                 }
 523         }
 524
 525         if (timeout <= 0) {
 526                 fprintf(stderr, "Invalid timeout value\n");
 527                 return -1;
 528         }
 529
 530         if (optind == argc - 1) {
 531                 name = argv[optind ++];
 532                 if (strlen(name) >= LST_NAME_SIZE) {
 533                         fprintf(stderr, "Name size is limited to %d\n",
 534                                 LST_NAME_SIZE - 1);
 535                         return -1;
 536                 }
 537
 538         } else if (optind == argc) {
 539                 char           user[LST_NAME_SIZE];
 540                 char           host[LST_NAME_SIZE];
 541                 struct passwd *pw = getpwuid(getuid());
 542
 543                 if (pw == NULL)
 544                         snprintf(user, sizeof(user), "%d", (int)getuid());
 545                 else
 546                         snprintf(user, sizeof(user), "%s", pw->pw_name);
 547
 548                 rc = gethostname(host, sizeof(host));
 549                 if (rc != 0)
 550                         snprintf(host, sizeof(host), "unknown_host");
 551
 552                 snprintf(buf, LST_NAME_SIZE, "%s@%s", user, host);
 553                 name = buf;
 554
 555         } else {
 556                 lst_print_usage(argv[0]);
 557                 return -1;
 558         }
 559
 560         rc = lst_new_session_ioctl(name, timeout, force, &session_id);
 561
 562         if (rc != 0) {
 563                 lst_print_error("session", "Failed to create session: %s\n",
 564                                 strerror(errno));
 565                 return rc;
 566         }
 567
 568         fprintf(stdout, "SESSION: %s TIMEOUT: %d FORCE: %s\n",
 569                 name, timeout, force ? "Yes": "No");
 570
 571         return rc;
 572 }
 573
 574 int
 575 lst_session_info_ioctl(char *name, int len, int *key,
 576                        lst_sid_t *sid, lstcon_ndlist_ent_t *ndinfo)
 577 {
 578         lstio_session_info_args_t args = {
 579                 .lstio_ses_keyp         = key,
 580                 .lstio_ses_idp          = sid,
 581                 .lstio_ses_ndinfo       = ndinfo,
 582                 .lstio_ses_nmlen        = len,
 583                 .lstio_ses_namep        = name,
 584         };
 585
 586         return lst_ioctl(LSTIO_SESSION_INFO, &args, sizeof(args));
 587 }
 588
 589 int
 590 jt_lst_show_session(int argc, char **argv)
 591 {
 592         lstcon_ndlist_ent_t ndinfo;
 593         lst_sid_t           sid;
 594         char                name[LST_NAME_SIZE];
 595         int                 key;
 596         int                 rc;
 597
 598         rc = lst_session_info_ioctl(name, LST_NAME_SIZE, &key, &sid, &ndinfo);
 599
 600         if (rc != 0) {
 601                 lst_print_error("session", "Failed to show session: %s\n",
 602                                 strerror(errno));
 603                 return -1;
 604         }
 605
 606         fprintf(stdout, "%s ID: %Lu@%s, KEY: %d NODES: %d\n",
 607                 name, sid.ses_stamp, libcfs_nid2str(sid.ses_nid),
 608                 key, ndinfo.nle_nnode);
 609
 610         return 0;
 611 }
 612
 613 int
 614 lst_end_session_ioctl(void)
 615 {
 616         lstio_session_end_args_t args = {
 617                 .lstio_ses_key           = session_key,
 618         };
 619
 620         return lst_ioctl (LSTIO_SESSION_END, &args, sizeof(args));
 621 }
 622
 623 int
 624 jt_lst_end_session(int argc, char **argv)
 625 {
 626         int             rc;
 627
 628         if (session_key == 0) {
 629                 fprintf(stderr,
 630                         "Can't find env LST_SESSION or value is not valid\n");
 631                 return -1;
 632         }
 633
 634         rc = lst_end_session_ioctl();
 635
 636         if (rc == 0) {
 637                 fprintf(stdout, "session is ended\n");
 638                 return 0;
 639         }
 640
 641         if (rc == -1) {
 642                 lst_print_error("session", "Failed to end session: %s\n",
 643                                 strerror(errno));
 644                 return rc;
 645         }
 646
 647         if (trans_stat.trs_rpc_errno != 0) {
 648                 fprintf(stderr,
 649                         "[RPC] Failed to send %d session RPCs: %s\n",
 650                         lstcon_rpc_stat_failure(&trans_stat, 0),
 651                         strerror(trans_stat.trs_rpc_errno));
 652         }
 653
 654         if (trans_stat.trs_fwk_errno != 0) {
 655                 fprintf(stderr,
 656                         "[FWK] Failed to end session on %d nodes: %s\n",
 657                         lstcon_sesop_stat_failure(&trans_stat, 0),
 658                         strerror(trans_stat.trs_fwk_errno));
 659         }
 660
 661         return rc;
 662 }
 663
 664 int
 665 lst_ping_ioctl(char *str, int type, int timeout,
 666                int count, lnet_process_id_t *ids, struct list_head *head)
 667 {
 668         lstio_debug_args_t args = {
 669                 .lstio_dbg_key          = session_key,
 670                 .lstio_dbg_type         = type,
 671                 .lstio_dbg_flags        = 0,
 672                 .lstio_dbg_timeout      = timeout,
 673                 .lstio_dbg_nmlen        = (str == NULL) ? 0: strlen(str),
 674                 .lstio_dbg_namep        = str,
 675                 .lstio_dbg_count        = count,
 676                 .lstio_dbg_idsp         = ids,
 677                 .lstio_dbg_resultp      = head,
 678         };
 679
 680         return lst_ioctl (LSTIO_DEBUG, &args, sizeof(args));
 681 }
 682
 683 int lst_info_batch_ioctl(char *batch, int test, int server,
 684                         lstcon_test_batch_ent_t *entp, int *idxp,
 685                         int *ndentp, lstcon_node_ent_t *dentsp);
 686
 687 int lst_info_group_ioctl(char *name, lstcon_ndlist_ent_t *gent,
 688                          int *idx, int *count, lstcon_node_ent_t *dents);
 689
 690 int lst_query_batch_ioctl(char *batch, int test, int server,
 691                           int timeout, struct list_head *head);
 692
 693 int
 694 lst_get_node_count(int type, char *str, int *countp, lnet_process_id_t **idspp)
 695 {
 696         char                    buf[LST_NAME_SIZE];
 697         lstcon_test_batch_ent_t ent;
 698         lstcon_ndlist_ent_t    *entp = &ent.tbe_cli_nle;
 699         lst_sid_t               sid;
 700         int                     key;
 701         int                     rc;
 702
 703         switch (type) {
 704         case LST_OPC_SESSION:
 705                 rc = lst_session_info_ioctl(buf, LST_NAME_SIZE,
 706                                             &key, &sid, entp);
 707                 break;
 708
 709         case LST_OPC_BATCHSRV:
 710                 entp = &ent.tbe_srv_nle;
 711         case LST_OPC_BATCHCLI:
 712                 rc = lst_info_batch_ioctl(str, 0, 0, &ent, NULL, NULL, NULL);
 713                 break;
 714
 715         case LST_OPC_GROUP:
 716                 rc = lst_info_group_ioctl(str, entp, NULL, NULL, NULL);
 717                 break;
 718
 719         case LST_OPC_NODES:
 720                 rc = lst_parse_nids(str, &entp->nle_nnode, idspp) < 0 ? -1 : 0;
 721                 break;
 722
 723         default:
 724                 rc = -1;
 725                 break;
 726         }
 727
 728         if (rc == 0)
 729                 *countp = entp->nle_nnode;
 730
 731         return rc;
 732 }
 733
 734 int
 735 jt_lst_ping(int argc,  char **argv)
 736 {
 737         struct list_head   head;
 738         lnet_process_id_t *ids = NULL;
 739         lstcon_rpc_ent_t  *ent = NULL;
 740         char              *str = NULL;
 741         int                optidx  = 0;
 742         int                server  = 0;
 743         int                timeout = 5;
 744         int                count   = 0;
 745         int                type    = 0;
 746         int                rc      = 0;
 747         int                c;
 748
 749         static struct option ping_opts[] =
 750         {
 751                 {"session", no_argument,       0, 's' },
 752                 {"server",  no_argument,       0, 'v' },
 753                 {"batch",   required_argument, 0, 'b' },
 754                 {"group",   required_argument, 0, 'g' },
 755                 {"nodes",   required_argument, 0, 'n' },
 756                 {"timeout", required_argument, 0, 't' },
 757                 {0,         0,                 0,  0  }
 758         };
 759
 760         if (session_key == 0) {
 761                 fprintf(stderr,
 762                         "Can't find env LST_SESSION or value is not valid\n");
 763                 return -1;
 764         }
 765
 766         while (1) {
 767
 768                 c = getopt_long(argc, argv, "g:b:n:t:sv",
 769                                 ping_opts, &optidx);
 770
 771                 if (c == -1)
 772                         break;
 773
 774                 switch (c) {
 775                 case 's':
 776                         type = LST_OPC_SESSION;
 777                         break;
 778
 779                 case 'g':
 780                         type = LST_OPC_GROUP;
 781                         str = optarg;
 782                         break;
 783
 784                 case 'b':
 785                         type = LST_OPC_BATCHCLI;
 786                         str = optarg;
 787                         break;
 788
 789                 case 'n':
 790                         type = LST_OPC_NODES;
 791                         str = optarg;
 792                         break;
 793
 794                 case 't':
 795                         timeout = atoi(optarg);
 796                         break;
 797
 798                 case 'v':
 799                         server = 1;
 800                         break;
 801
 802                 default:
 803                         lst_print_usage(argv[0]);
 804                         return -1;
 805                 }
 806         }
 807
 808         if (type == 0 || timeout <= 0 || optind != argc) {
 809                 lst_print_usage(argv[0]);
 810                 return -1;
 811         }
 812
 813         if (type == LST_OPC_BATCHCLI && server)
 814                 type = LST_OPC_BATCHSRV;
 815
 816         rc = lst_get_node_count(type, str, &count, &ids);
 817         if (rc < 0) {
 818                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
 819                         (str == NULL) ? "session" : str, strerror(errno));
 820                 return -1;
 821         }
 822
 823         CFS_INIT_LIST_HEAD(&head);
 824
 825         rc = lst_alloc_rpcent(&head, count, LST_NAME_SIZE);
 826         if (rc != 0) {
 827                 fprintf(stderr, "Out of memory\n");
 828                 goto out;
 829         }
 830
 831         if (count == 0) {
 832                 fprintf(stdout, "Target %s is empty\n",
 833                         (str == NULL) ? "session" : str);
 834                 goto out;
 835         }
 836
 837         rc = lst_ping_ioctl(str, type, timeout, count, ids, &head);
 838         if (rc == -1) { /* local failure */
 839                 lst_print_error("debug", "Failed to ping %s: %s\n",
 840                                 (str == NULL) ? "session" : str,
 841                                 strerror(errno));
 842                 rc = -1;
 843                 goto out;
 844         }
 845
 846         /* ignore RPC errors and framwork errors */
 847         list_for_each_entry(ent, &head, rpe_link) {
 848                 fprintf(stdout, "\t%s: %s [session: %s id: %s]\n",
 849                         libcfs_id2str(ent->rpe_peer),
 850                         lst_node_state2str(ent->rpe_state),
 851                         (ent->rpe_state == LST_NODE_ACTIVE ||
 852                          ent->rpe_state == LST_NODE_BUSY)?
 853                                  (ent->rpe_rpc_errno == 0 ?
 854                                          &ent->rpe_payload[0] : "Unknown") :
 855                                  "<NULL>", libcfs_nid2str(ent->rpe_sid.ses_nid));
 856         }
 857
 858 out:
 859         lst_free_rpcent(&head);
 860
 861         if (ids != NULL)
 862                 free(ids);
 863
 864         return rc;
 865
 866 }
 867
 868 int
 869 lst_add_nodes_ioctl (char *name, int count, lnet_process_id_t *ids,
 870                      struct list_head *resultp)
 871 {
 872         lstio_group_nodes_args_t        args = {
 873                 .lstio_grp_key          = session_key,
 874                 .lstio_grp_nmlen        = strlen(name),
 875                 .lstio_grp_namep        = name,
 876                 .lstio_grp_count        = count,
 877                 .lstio_grp_idsp         = ids,
 878                 .lstio_grp_resultp      = resultp,
 879         };
 880
 881         return lst_ioctl(LSTIO_NODES_ADD, &args, sizeof(args));
 882 }
 883
 884 int
 885 lst_add_group_ioctl (char *name)
 886 {
 887         lstio_group_add_args_t  args = {
 888                 .lstio_grp_key          = session_key,
 889                 .lstio_grp_nmlen        = strlen(name),
 890                 .lstio_grp_namep        = name,
 891         };
 892
 893         return lst_ioctl(LSTIO_GROUP_ADD, &args, sizeof(args));
 894 }
 895
 896 int
 897 jt_lst_add_group(int argc, char **argv)
 898 {
 899         struct list_head   head;
 900         lnet_process_id_t *ids;
 901         char              *name;
 902         int                count;
 903         int                rc;
 904         int                i;
 905
 906         if (session_key == 0) {
 907                 fprintf(stderr,
 908                         "Can't find env LST_SESSION or value is not valid\n");
 909                 return -1;
 910         }
 911
 912         if (argc < 3) {
 913                 lst_print_usage(argv[0]);
 914                 return -1;
 915         }
 916
 917         name = argv[1];
 918         if (strlen(name) >= LST_NAME_SIZE) {
 919                 fprintf(stderr, "Name length is limited to %d\n",
 920                         LST_NAME_SIZE - 1);
 921                 return -1;
 922         }
 923
 924         rc = lst_add_group_ioctl(name);
 925         if (rc != 0) {
 926                 lst_print_error("group", "Failed to add group %s: %s\n",
 927                                 name, strerror(errno));
 928                 return -1;
 929         }
 930
 931         CFS_INIT_LIST_HEAD(&head);
 932
 933         for (i = 2; i < argc; i++) {
 934                 /* parse address list */
 935                 rc = lst_parse_nids(argv[i], &count, &ids);
 936                 if (rc < 0) {
 937                         fprintf(stderr, "Ignore invalid id list %s\n",
 938                                 argv[i]);
 939                         continue;
 940                 }
 941
 942                 if (count == 0)
 943                         continue;
 944
 945                 rc = lst_alloc_rpcent(&head, count, 0);
 946                 if (rc != 0) {
 947                         fprintf(stderr, "Out of memory\n");
 948                         break;
 949                 }
 950
 951                 rc = lst_add_nodes_ioctl(name, count, ids, &head);
 952
 953                 free(ids);
 954
 955                 if (rc == 0) {
 956                         lst_free_rpcent(&head);
 957                         fprintf(stderr, "%s are added to session\n", argv[i]);
 958                         continue;
 959                 }
 960
 961                 if (rc == -1) {
 962                         lst_free_rpcent(&head);
 963                         lst_print_error("group", "Failed to add nodes %s: %s\n",
 964                                         argv[i], strerror(errno));
 965                         break;
 966                 }
 967
 968                 lst_print_transerr(&head, "create session");
 969                 lst_free_rpcent(&head);
 970         }
 971
 972         return rc;
 973 }
 974
 975 int
 976 lst_del_group_ioctl (char *name)
 977 {
 978         lstio_group_del_args_t  args = {
 979                 .lstio_grp_key          = session_key,
 980                 .lstio_grp_nmlen        = strlen(name),
 981                 .lstio_grp_namep        = name,
 982         };
 983
 984         return lst_ioctl(LSTIO_GROUP_DEL, &args, sizeof(args));
 985 }
 986
 987 int
 988 jt_lst_del_group(int argc, char **argv)
 989 {
 990         int     rc;
 991
 992         if (session_key == 0) {
 993                 fprintf(stderr,
 994                         "Can't find env LST_SESSION or value is not valid\n");
 995                 return -1;
 996         }
 997
 998         if (argc != 2) {
 999                 lst_print_usage(argv[0]);
1000                 return -1;
1001         }
1002
1003         rc = lst_del_group_ioctl(argv[1]);
1004         if (rc == 0) {
1005                 fprintf(stdout, "Group is deleted\n");
1006                 return 0;
1007         }
1008
1009         if (rc == -1) {
1010                 lst_print_error("group", "Failed to delete group: %s\n",
1011                                 strerror(errno));
1012                 return rc;
1013         }
1014
1015         fprintf(stderr, "Group is deleted with some errors\n");
1016
1017         if (trans_stat.trs_rpc_errno != 0) {
1018                 fprintf(stderr, "[RPC] Failed to send %d end session RPCs: %s\n",
1019                         lstcon_rpc_stat_failure(&trans_stat, 0),
1020                         strerror(trans_stat.trs_rpc_errno));
1021         }
1022
1023         if (trans_stat.trs_fwk_errno != 0) {
1024                 fprintf(stderr,
1025                         "[FWK] Failed to end session on %d nodes: %s\n",
1026                         lstcon_sesop_stat_failure(&trans_stat, 0),
1027                         strerror(trans_stat.trs_fwk_errno));
1028         }
1029
1030         return -1;
1031 }
1032
1033 int
1034 lst_update_group_ioctl(int opc, char *name, int clean, int count,
1035                        lnet_process_id_t *ids, struct list_head *resultp)
1036 {
1037         lstio_group_update_args_t  args = {
1038                 .lstio_grp_key          = session_key,
1039                 .lstio_grp_opc          = opc,
1040                 .lstio_grp_args         = clean,
1041                 .lstio_grp_nmlen        = strlen(name),
1042                 .lstio_grp_namep        = name,
1043                 .lstio_grp_count        = count,
1044                 .lstio_grp_idsp         = ids,
1045                 .lstio_grp_resultp      = resultp,
1046         };
1047
1048         return lst_ioctl(LSTIO_GROUP_UPDATE, &args, sizeof(args));
1049 }
1050
1051 int
1052 jt_lst_update_group(int argc, char **argv)
1053 {
1054         struct list_head   head;
1055         lnet_process_id_t *ids = NULL;
1056         char              *str = NULL;
1057         char              *grp = NULL;
1058         int                optidx = 0;
1059         int                count = 0;
1060         int                clean = 0;
1061         int                opc = 0;
1062         int                rc;
1063         int                c;
1064
1065         static struct option update_group_opts[] =
1066         {
1067                 {"refresh", no_argument,       0, 'f' },
1068                 {"clean",   required_argument, 0, 'c' },
1069                 {"remove",  required_argument, 0, 'r' },
1070                 {0,         0,                 0,  0  }
1071         };
1072
1073         if (session_key == 0) {
1074                 fprintf(stderr,
1075                         "Can't find env LST_SESSION or value is not valid\n");
1076                 return -1;
1077         }
1078
1079         while (1) {
1080                 c = getopt_long(argc, argv, "fc:r:",
1081                                 update_group_opts, &optidx);
1082
1083                 /* Detect the end of the options. */
1084                 if (c == -1)
1085                         break;
1086
1087                 switch (c) {
1088                 case 'f':
1089                         if (opc != 0) {
1090                                 lst_print_usage(argv[0]);
1091                                 return -1;
1092                         }
1093                         opc = LST_GROUP_REFRESH;
1094                         break;
1095
1096                 case 'r':
1097                         if (opc != 0) {
1098                                 lst_print_usage(argv[0]);
1099                                 return -1;
1100                         }
1101                         opc = LST_GROUP_RMND;
1102                         str = optarg;
1103                         break;
1104
1105                 case 'c':
1106                         clean = lst_node_str2state(optarg);
1107                         if (opc != 0 || clean <= 0) {
1108                                 lst_print_usage(argv[0]);
1109                                 return -1;
1110                         }
1111                         opc = LST_GROUP_CLEAN;
1112                         break;
1113
1114                 default:
1115                         lst_print_usage(argv[0]);
1116                         return -1;
1117                 }
1118         }
1119
1120         /* no OPC or group is specified */
1121         if (opc == 0 || optind != argc - 1) {
1122                 lst_print_usage(argv[0]);
1123                 return -1;
1124         }
1125
1126         grp = argv[optind];
1127
1128         CFS_INIT_LIST_HEAD(&head);
1129
1130         if (opc == LST_GROUP_RMND || opc == LST_GROUP_REFRESH) {
1131                 rc = lst_get_node_count(opc == LST_GROUP_RMND ? LST_OPC_NODES :
1132                                                                 LST_OPC_GROUP,
1133                                         opc == LST_GROUP_RMND ? str : grp,
1134                                         &count, &ids);
1135
1136                 if (rc != 0) {
1137                         fprintf(stderr, "Can't get count of nodes from %s: %s\n",
1138                                 opc == LST_GROUP_RMND ? str : grp,
1139                                 strerror(errno));
1140                         return -1;
1141                 }
1142
1143                 rc = lst_alloc_rpcent(&head, count, 0);
1144                 if (rc != 0) {
1145                         fprintf(stderr, "Out of memory\n");
1146                         free(ids);
1147                         return -1;
1148                 }
1149
1150         }
1151
1152         rc = lst_update_group_ioctl(opc, grp, clean, count, ids, &head);
1153
1154         if (ids != NULL)
1155                 free(ids);
1156
1157         if (rc == 0) {
1158                 lst_free_rpcent(&head);
1159                 return 0;
1160         }
1161
1162         if (rc == -1) {
1163                 lst_free_rpcent(&head);
1164                 lst_print_error("group", "Failed to update group: %s\n",
1165                                 strerror(errno));
1166                 return rc;
1167         }
1168
1169         lst_print_transerr(&head, "Updating group");
1170
1171         lst_free_rpcent(&head);
1172
1173         return rc;
1174 }
1175
1176 int
1177 lst_list_group_ioctl(int len, char *name, int idx)
1178 {
1179         lstio_group_list_args_t         args = {
1180                 .lstio_grp_key          = session_key,
1181                 .lstio_grp_idx          = idx,
1182                 .lstio_grp_nmlen        = len,
1183                 .lstio_grp_namep        = name,
1184         };
1185
1186         return lst_ioctl(LSTIO_GROUP_LIST, &args, sizeof(args));
1187 }
1188
1189 int
1190 lst_info_group_ioctl(char *name, lstcon_ndlist_ent_t *gent,
1191                      int *idx, int *count, lstcon_node_ent_t *dents)
1192 {
1193         lstio_group_info_args_t         args = {
1194                 .lstio_grp_key          = session_key,
1195                 .lstio_grp_nmlen        = strlen(name),
1196                 .lstio_grp_namep        = name,
1197                 .lstio_grp_entp         = gent,
1198                 .lstio_grp_idxp         = idx,
1199                 .lstio_grp_ndentp       = count,
1200                 .lstio_grp_dentsp       = dents,
1201         };
1202
1203         return lst_ioctl(LSTIO_GROUP_INFO, &args, sizeof(args));
1204 }
1205
1206 int
1207 lst_list_group_all(void)
1208 {
1209         char  name[LST_NAME_SIZE];
1210         int   rc;
1211         int   i;
1212
1213         /* no group is specified, list name of all groups */
1214         for (i = 0; ; i++) {
1215                 rc = lst_list_group_ioctl(LST_NAME_SIZE, name, i);
1216                 if (rc == 0) {
1217                         fprintf(stdout, "%d) %s\n", i + 1, name);
1218                         continue;
1219                 }
1220
1221                 if (errno == ENOENT)
1222                         break;
1223
1224                 lst_print_error("group", "Failed to list group: %s\n",
1225                                 strerror(errno));
1226                 return -1;
1227         }
1228
1229         fprintf(stdout, "Total %d groups\n", i);
1230
1231         return 0;
1232 }
1233
1234 #define LST_NODES_TITLE "\tACTIVE\tBUSY\tDOWN\tUNKNOWN\tTOTAL\n"
1235
1236 int
1237 jt_lst_list_group(int argc, char **argv)
1238 {
1239         lstcon_ndlist_ent_t  gent;
1240         lstcon_node_ent_t   *dents;
1241         int               optidx  = 0;
1242         int               verbose = 0;
1243         int               active  = 0;
1244         int               busy    = 0;
1245         int               down    = 0;
1246         int               unknown = 0;
1247         int               all     = 0;
1248         int               count;
1249         int               index;
1250         int               i;
1251         int               j;
1252         int               c;
1253         int               rc = 0;
1254
1255         static struct option list_group_opts[] =
1256         {
1257                 {"active",  no_argument, 0, 'a' },
1258                 {"busy",    no_argument, 0, 'b' },
1259                 {"down",    no_argument, 0, 'd' },
1260                 {"unknown", no_argument, 0, 'u' },
1261                 {"all",     no_argument, 0, 'l' },
1262                 {0,         0,           0,  0  }
1263         };
1264
1265         if (session_key == 0) {
1266                 fprintf(stderr,
1267                         "Can't find env LST_SESSION or value is not valid\n");
1268                 return -1;
1269         }
1270
1271         while (1) {
1272                 c = getopt_long(argc, argv, "abdul",
1273                                 list_group_opts, &optidx);
1274
1275                 if (c == -1)
1276                         break;
1277
1278                 switch (c) {
1279                 case 'a':
1280                         verbose = active = 1;
1281                         all = 0;
1282                         break;
1283                 case 'b':
1284                         verbose = busy = 1;
1285                         all = 0;
1286                         break;
1287                 case 'd':
1288                         verbose = down = 1;
1289                         all = 0;
1290                         break;
1291                 case 'u':
1292                         verbose = unknown = 1;
1293                         all = 0;
1294                         break;
1295                 case 'l':
1296                         verbose = all = 1;
1297                         break;
1298                 default:
1299                         lst_print_usage(argv[0]);
1300                         return -1;
1301                 }
1302         }
1303
1304         if (optind == argc) {
1305                 /* no group is specified, list name of all groups */
1306                 rc = lst_list_group_all();
1307
1308                 return rc;
1309         }
1310
1311         if (!verbose)
1312                 fprintf(stdout, LST_NODES_TITLE);
1313
1314         /* list nodes in specified groups */
1315         for (i = optind; i < argc; i++) {
1316                 rc = lst_info_group_ioctl(argv[i], &gent, NULL, NULL, NULL);
1317                 if (rc != 0) {
1318                         if (errno == ENOENT) {
1319                                 rc = 0;
1320                                 break;
1321                         }
1322
1323                         lst_print_error("group", "Failed to list group\n",
1324                                         strerror(errno));
1325                         break;
1326                 }
1327
1328                 if (!verbose) {
1329                         fprintf(stdout, "\t%d\t%d\t%d\t%d\t%d\t%s\n",
1330                                 gent.nle_nactive, gent.nle_nbusy,
1331                                 gent.nle_ndown, gent.nle_nunknown,
1332                                 gent.nle_nnode, argv[i]);
1333                         continue;
1334                 }
1335
1336                 fprintf(stdout, "Group [ %s ]\n", argv[i]);
1337
1338                 if (gent.nle_nnode == 0) {
1339                         fprintf(stdout, "No nodes found [ %s ]\n", argv[i]);
1340                         continue;
1341                 }
1342
1343                 count = gent.nle_nnode;
1344
1345                 dents = malloc(count * sizeof(lstcon_node_ent_t));
1346                 if (dents == NULL) {
1347                         fprintf(stderr, "Failed to malloc: %s\n",
1348                                 strerror(errno));
1349                         return -1;
1350                 }
1351
1352                 index = 0;
1353                 rc = lst_info_group_ioctl(argv[i], &gent, &index, &count, dents);
1354                 if (rc != 0) {
1355                         lst_print_error("group", "Failed to list group: %s\n",
1356                                         strerror(errno));
1357                         free(dents);
1358                         return -1;
1359                 }
1360
1361                 for (j = 0, c = 0; j < count; j++) {
1362                         if (all ||
1363                             ((active  &&  dents[j].nde_state == LST_NODE_ACTIVE) ||
1364                              (busy    &&  dents[j].nde_state == LST_NODE_BUSY)   ||
1365                              (down    &&  dents[j].nde_state == LST_NODE_DOWN)   ||
1366                              (unknown &&  dents[j].nde_state == LST_NODE_UNKNOWN))) {
1367
1368                                 fprintf(stdout, "\t%s: %s\n",
1369                                         libcfs_id2str(dents[j].nde_id),
1370                                         lst_node_state2str(dents[j].nde_state));
1371                                 c++;
1372                         }
1373                 }
1374
1375                 fprintf(stdout, "Total %d nodes [ %s ]\n", c, argv[i]);
1376
1377                 free(dents);
1378         }
1379
1380         return rc;
1381 }
1382
1383 int
1384 lst_stat_ioctl (char *name, int count, lnet_process_id_t *idsp,
1385                 int timeout, struct list_head *resultp)
1386 {
1387         lstio_stat_args_t  args = {
1388                 .lstio_sta_key           = session_key,
1389                 .lstio_sta_timeout       = timeout,
1390                 .lstio_sta_nmlen         = strlen(name),
1391                 .lstio_sta_namep         = name,
1392                 .lstio_sta_count         = count,
1393                 .lstio_sta_idsp          = idsp,
1394                 .lstio_sta_resultp       = resultp,
1395         };
1396
1397         return lst_ioctl (LSTIO_STAT_QUERY, &args, sizeof(args));
1398 }
1399
1400 typedef struct {
1401         struct list_head        srp_link;
1402         int                     srp_count;
1403         char                   *srp_name;
1404         lnet_process_id_t      *srp_ids;
1405         struct list_head        srp_result[2];
1406 } lst_stat_req_param_t;
1407
1408 static void
1409 lst_stat_req_param_free(lst_stat_req_param_t *srp)
1410 {
1411         int     i;
1412
1413         for (i = 0; i < 2; i++)
1414                 lst_free_rpcent(&srp->srp_result[i]);
1415
1416         if (srp->srp_ids != NULL)
1417                 free(srp->srp_ids);
1418
1419         free(srp);
1420 }
1421
1422 static int
1423 lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp)
1424 {
1425         lst_stat_req_param_t *srp = NULL;
1426         int                   rc;
1427         int                   i;
1428
1429         srp = malloc(sizeof(*srp));
1430         if (srp == NULL)
1431                 return -ENOMEM;
1432
1433         memset(srp, 0, sizeof(*srp));
1434         CFS_INIT_LIST_HEAD(&srp->srp_result[0]);
1435         CFS_INIT_LIST_HEAD(&srp->srp_result[1]);
1436
1437         rc = lst_get_node_count(LST_OPC_GROUP, name,
1438                                 &srp->srp_count, NULL);
1439         if (rc != 0) {
1440                 rc = lst_get_node_count(LST_OPC_NODES, name,
1441                                         &srp->srp_count, &srp->srp_ids);
1442         }
1443
1444         if (rc != 0) {
1445                 fprintf(stderr,
1446                         "Failed to get count of nodes from %s\n", name);
1447                 lst_stat_req_param_free(srp);
1448
1449                 return rc;
1450         }
1451
1452         srp->srp_name = name;
1453
1454         for (i = 0; i < 2; i++) {
1455                 rc = lst_alloc_rpcent(&srp->srp_result[i], srp->srp_count,
1456                                       sizeof(sfw_counters_t)  +
1457                                       sizeof(srpc_counters_t) +
1458                                       sizeof(lnet_counters_t));
1459                 if (rc != 0) {
1460                         fprintf(stderr, "Out of memory\n");
1461                         break;
1462                 }
1463         }
1464
1465         if (rc == 0) {
1466                 *srpp = srp;
1467                 return 0;
1468         }
1469
1470         lst_stat_req_param_free(srp);
1471
1472         return rc;
1473 }
1474
1475 typedef struct {
1476         /* TODO */
1477 } lst_srpc_stat_result;
1478
1479 #define LST_LNET_AVG    0
1480 #define LST_LNET_MIN    1
1481 #define LST_LNET_MAX    2
1482
1483 typedef struct {
1484         float           lnet_avg_sndrate;
1485         float           lnet_min_sndrate;
1486         float           lnet_max_sndrate;
1487         float           lnet_total_sndrate;
1488
1489         float           lnet_avg_rcvrate;
1490         float           lnet_min_rcvrate;
1491         float           lnet_max_rcvrate;
1492         float           lnet_total_rcvrate;
1493
1494         float           lnet_avg_sndperf;
1495         float           lnet_min_sndperf;
1496         float           lnet_max_sndperf;
1497         float           lnet_total_sndperf;
1498
1499         float           lnet_avg_rcvperf;
1500         float           lnet_min_rcvperf;
1501         float           lnet_max_rcvperf;
1502         float           lnet_total_rcvperf;
1503
1504         int             lnet_stat_count;
1505 } lst_lnet_stat_result_t;
1506
1507 lst_lnet_stat_result_t lnet_stat_result;
1508
1509 static float
1510 lst_lnet_stat_value(int bw, int send, int off)
1511 {
1512         float  *p;
1513
1514         p = bw ? &lnet_stat_result.lnet_avg_sndperf :
1515                  &lnet_stat_result.lnet_avg_sndrate;
1516
1517         if (!send)
1518                 p += 4;
1519
1520         p += off;
1521
1522         return *p;
1523 }
1524
1525 static void
1526 lst_timeval_diff(struct timeval *tv1,
1527                  struct timeval *tv2, struct timeval *df)
1528 {
1529         if (tv1->tv_usec >= tv2->tv_usec) {
1530                 df->tv_sec  = tv1->tv_sec - tv2->tv_sec;
1531                 df->tv_usec = tv1->tv_usec - tv2->tv_usec;
1532                 return;
1533         }
1534
1535         df->tv_sec  = tv1->tv_sec - 1 - tv2->tv_sec;
1536         df->tv_usec = tv1->tv_sec + 1000000 - tv2->tv_usec;
1537
1538         return;
1539 }
1540
1541 void
1542 lst_cal_lnet_stat(float delta, lnet_counters_t *lnet_new,
1543                   lnet_counters_t *lnet_old)
1544 {
1545         float perf;
1546         float rate;
1547
1548         perf = (float)(lnet_new->send_length -
1549                        lnet_old->send_length) / (1024 * 1024) / delta;
1550         lnet_stat_result.lnet_total_sndperf += perf;
1551
1552         if (lnet_stat_result.lnet_min_sndperf > perf ||
1553             lnet_stat_result.lnet_min_sndperf == 0)
1554                 lnet_stat_result.lnet_min_sndperf = perf;
1555
1556         if (lnet_stat_result.lnet_max_sndperf < perf)
1557                 lnet_stat_result.lnet_max_sndperf = perf;
1558
1559         perf = (float)(lnet_new->recv_length -
1560                        lnet_old->recv_length) / (1024 * 1024) / delta;
1561         lnet_stat_result.lnet_total_rcvperf += perf;
1562
1563         if (lnet_stat_result.lnet_min_rcvperf > perf ||
1564             lnet_stat_result.lnet_min_rcvperf == 0)
1565                 lnet_stat_result.lnet_min_rcvperf = perf;
1566
1567         if (lnet_stat_result.lnet_max_rcvperf < perf)
1568                 lnet_stat_result.lnet_max_rcvperf = perf;
1569
1570         rate = (lnet_new->send_count - lnet_old->send_count) / delta;
1571         lnet_stat_result.lnet_total_sndrate += rate;
1572
1573         if (lnet_stat_result.lnet_min_sndrate > rate ||
1574             lnet_stat_result.lnet_min_sndrate == 0)
1575                 lnet_stat_result.lnet_min_sndrate = rate;
1576
1577         if (lnet_stat_result.lnet_max_sndrate < rate)
1578                 lnet_stat_result.lnet_max_sndrate = rate;
1579
1580         rate = (lnet_new->recv_count - lnet_old->recv_count) / delta;
1581         lnet_stat_result.lnet_total_rcvrate += rate;
1582
1583         if (lnet_stat_result.lnet_min_rcvrate > rate ||
1584             lnet_stat_result.lnet_min_rcvrate == 0)
1585                 lnet_stat_result.lnet_min_rcvrate = rate;
1586
1587         if (lnet_stat_result.lnet_max_rcvrate < rate)
1588                 lnet_stat_result.lnet_max_rcvrate = rate;
1589
1590         lnet_stat_result.lnet_stat_count ++;
1591
1592         lnet_stat_result.lnet_avg_sndrate = lnet_stat_result.lnet_total_sndrate /
1593                                             lnet_stat_result.lnet_stat_count;
1594         lnet_stat_result.lnet_avg_rcvrate = lnet_stat_result.lnet_total_rcvrate /
1595                                             lnet_stat_result.lnet_stat_count;
1596
1597         lnet_stat_result.lnet_avg_sndperf = lnet_stat_result.lnet_total_sndperf /
1598                                             lnet_stat_result.lnet_stat_count;
1599         lnet_stat_result.lnet_avg_rcvperf = lnet_stat_result.lnet_total_rcvperf /
1600                                             lnet_stat_result.lnet_stat_count;
1601
1602 }
1603
1604 void
1605 lst_print_lnet_stat(char *name, int bwrt, int rdwr, int type)
1606 {
1607         int     start1 = 0;
1608         int     end1   = 1;
1609         int     start2 = 0;
1610         int     end2   = 1;
1611         int     i;
1612         int     j;
1613
1614         if (lnet_stat_result.lnet_stat_count == 0)
1615                 return;
1616
1617         if (bwrt == 1) /* bw only */
1618                 start1 = 1;
1619
1620         if (bwrt == 2) /* rates only */
1621                 end1 = 0;
1622
1623         if (rdwr == 1) /* recv only */
1624                 start2 = 1;
1625
1626         if (rdwr == 2) /* send only */
1627                 end2 = 0;
1628
1629         for (i = start1; i <= end1; i++) {
1630                 fprintf(stdout, "[LNet %s of %s]\n",
1631                         i == 0 ? "Rates" : "Bandwidth", name);
1632
1633                 for (j = start2; j <= end2; j++) {
1634                         fprintf(stdout, "[%c] ", j == 0 ? 'W' : 'R');
1635
1636                         if ((type & 1) != 0) {
1637                                 fprintf(stdout, i == 0 ? "Avg: %-8.0f RPC/s " :
1638                                                          "Avg: %-8.2f MB/s  ",
1639                                         lst_lnet_stat_value(i, j, 0));
1640                         }
1641
1642                         if ((type & 2) != 0) {
1643                                 fprintf(stdout, i == 0 ? "Min: %-8.0f RPC/s " :
1644                                                          "Min: %-8.2f MB/s  ",
1645                                         lst_lnet_stat_value(i, j, 1));
1646                         }
1647
1648                         if ((type & 4) != 0) {
1649                                 fprintf(stdout, i == 0 ? "Max: %-8.0f RPC/s" :
1650                                                          "Max: %-8.2f MB/s",
1651                                         lst_lnet_stat_value(i, j, 2));
1652                         }
1653
1654                         fprintf(stdout, "\n");
1655                 }
1656         }
1657 }
1658
1659 void
1660 lst_print_stat(char *name, struct list_head *resultp,
1661                int idx, int lnet, int bwrt, int rdwr, int type)
1662 {
1663         struct list_head  tmp[2];
1664         lstcon_rpc_ent_t *new;
1665         lstcon_rpc_ent_t *old;
1666         sfw_counters_t   *sfwk_new;
1667         sfw_counters_t   *sfwk_old;
1668         srpc_counters_t  *srpc_new;
1669         srpc_counters_t  *srpc_old;
1670         lnet_counters_t  *lnet_new;
1671         lnet_counters_t  *lnet_old;
1672         struct timeval    tv;
1673         float             delta;
1674         int               errcount = 0;
1675
1676         CFS_INIT_LIST_HEAD(&tmp[0]);
1677         CFS_INIT_LIST_HEAD(&tmp[1]);
1678
1679         memset(&lnet_stat_result, 0, sizeof(lnet_stat_result));
1680
1681         while (!list_empty(&resultp[idx])) {
1682                 if (list_empty(&resultp[1 - idx])) {
1683                         fprintf(stderr, "Group is changed, re-run stat\n");
1684                         break;
1685                 }
1686
1687                 new = list_entry(resultp[idx].next, lstcon_rpc_ent_t, rpe_link);
1688                 old = list_entry(resultp[1 - idx].next, lstcon_rpc_ent_t, rpe_link);
1689
1690                 /* first time get stats result, can't calculate diff */
1691                 if (new->rpe_peer.nid == LNET_NID_ANY)
1692                         break;
1693
1694                 if (new->rpe_peer.nid != old->rpe_peer.nid ||
1695                     new->rpe_peer.pid != old->rpe_peer.pid) {
1696                         /* Something wrong. i.e, somebody change the group */
1697                         break;
1698                 }
1699
1700                 list_del(&new->rpe_link);
1701                 list_add_tail(&new->rpe_link, &tmp[idx]);
1702
1703                 list_del(&old->rpe_link);
1704                 list_add_tail(&old->rpe_link, &tmp[1 - idx]);
1705
1706                 if (new->rpe_rpc_errno != 0 || new->rpe_fwk_errno != 0 ||
1707                     old->rpe_rpc_errno != 0 || old->rpe_fwk_errno != 0) {
1708                         errcount ++;
1709                         continue;
1710                 }
1711
1712                 sfwk_new = (sfw_counters_t *)&new->rpe_payload[0];
1713                 sfwk_old = (sfw_counters_t *)&old->rpe_payload[0];
1714
1715                 srpc_new = (srpc_counters_t *)((char *)sfwk_new + sizeof(*sfwk_new));
1716                 srpc_old = (srpc_counters_t *)((char *)sfwk_old + sizeof(*sfwk_old));
1717
1718                 lnet_new = (lnet_counters_t *)((char *)srpc_new + sizeof(*srpc_new));
1719                 lnet_old = (lnet_counters_t *)((char *)srpc_old + sizeof(*srpc_old));
1720
1721                 lst_timeval_diff(&new->rpe_stamp, &old->rpe_stamp, &tv);
1722
1723                 delta = tv.tv_sec + (float)tv.tv_usec/1000000;
1724
1725                 if (!lnet) /* TODO */
1726                         continue;
1727
1728                 lst_cal_lnet_stat(delta, lnet_new, lnet_old);
1729         }
1730
1731         list_splice(&tmp[idx], &resultp[idx]);
1732         list_splice(&tmp[1 - idx], &resultp[1 - idx]);
1733
1734         if (errcount > 0)
1735                 fprintf(stdout, "Failed to stat on %d nodes\n", errcount);
1736
1737         if (!lnet)  /* TODO */
1738                 return;
1739
1740         lst_print_lnet_stat(name, bwrt, rdwr, type);
1741 }
1742
1743 int
1744 jt_lst_stat(int argc, char **argv)
1745 {
1746         struct list_head      head;
1747         lst_stat_req_param_t *srp;
1748         char                 *name    = NULL;
1749         time_t                last    = 0;
1750         int                   optidx  = 0;
1751         int                   timeout = 5; /* default timeout, 5 sec */
1752         int                   delay   = 5; /* default delay, 5 sec */
1753         int                   lnet    = 1; /* lnet stat by default */
1754         int                   bwrt    = 0;
1755         int                   rdwr    = 0;
1756         int                   type    = -1;
1757         int                   idx     = 0;
1758         int                   rc;
1759         int                   c;
1760
1761         static struct option stat_opts[] =
1762         {
1763                 {"timeout", required_argument, 0, 't' },
1764                 {"delay"  , required_argument, 0, 'd' },
1765                 {"lnet"   , no_argument,       0, 'l' },
1766                 {"rpc"    , no_argument,       0, 'c' },
1767                 {"bw"     , no_argument,       0, 'b' },
1768                 {"rate"   , no_argument,       0, 'a' },
1769                 {"read"   , no_argument,       0, 'r' },
1770                 {"write"  , no_argument,       0, 'w' },
1771                 {"avg"    , no_argument,       0, 'g' },
1772                 {"min"    , no_argument,       0, 'n' },
1773                 {"max"    , no_argument,       0, 'x' },
1774                 {0,         0,                 0,  0  }
1775         };
1776
1777         if (session_key == 0) {
1778                 fprintf(stderr,
1779                         "Can't find env LST_SESSION or value is not valid\n");
1780                 return -1;
1781         }
1782
1783         while (1) {
1784                 c = getopt_long(argc, argv, "t:d:lcbarwgnx", stat_opts, &optidx);
1785
1786                 if (c == -1)
1787                         break;
1788
1789                 switch (c) {
1790                 case 't':
1791                         timeout = atoi(optarg);
1792                         break;
1793                 case 'd':
1794                         delay = atoi(optarg);
1795                         break;
1796                 case 'l':
1797                         lnet = 1;
1798                         break;
1799                 case 'c':
1800                         lnet = 0;
1801                         break;
1802                 case 'b':
1803                         bwrt |= 1;
1804                         break;
1805                 case 'a':
1806                         bwrt |= 2;
1807                         break;
1808                 case 'r':
1809                         rdwr |= 1;
1810                         break;
1811                 case 'w':
1812                         rdwr |= 2;
1813                         break;
1814                 case 'g':
1815                         if (type == -1) {
1816                                 type = 1;
1817                                 break;
1818                         }
1819                         type |= 1;
1820                         break;
1821                 case 'n':
1822                         if (type == -1) {
1823                                 type = 2;
1824                                 break;
1825                         }
1826                         type |= 2;
1827                         break;
1828                 case 'x':
1829                         if (type == -1) {
1830                                 type = 4;
1831                                 break;
1832                         }
1833                         type |= 4;
1834                         break;
1835                 default:
1836                         lst_print_usage(argv[0]);
1837                         return -1;
1838                 }
1839         }
1840
1841         if (optind == argc) {
1842                 lst_print_usage(argv[0]);
1843                 return -1;
1844         }
1845
1846         if (timeout <= 0 || delay <= 0) {
1847                 fprintf(stderr, "Invalid timeout or delay value\n");
1848                 return -1;
1849         }
1850
1851         CFS_INIT_LIST_HEAD(&head);
1852
1853         while (optind < argc) {
1854                 name = argv[optind++];
1855
1856                 rc = lst_stat_req_param_alloc(name, &srp);
1857                 if (rc != 0)
1858                         goto out;
1859
1860                 list_add_tail(&srp->srp_link, &head);
1861         }
1862
1863         while (1) {
1864                 time_t  now = time(NULL);
1865
1866                 if (now - last < delay) {
1867                         sleep(delay - now + last);
1868                         time(&now);
1869                 }
1870
1871                 last = now;
1872
1873                 list_for_each_entry(srp, &head, srp_link) {
1874                         rc = lst_stat_ioctl(srp->srp_name,
1875                                             srp->srp_count, srp->srp_ids,
1876                                             timeout, &srp->srp_result[idx]);
1877                         if (rc == -1) {
1878                                 lst_print_error("stat", "Failed to stat %s: %s\n",
1879                                                 name, strerror(errno));
1880                                 goto out;
1881                         }
1882
1883                         lst_print_stat(srp->srp_name, srp->srp_result,
1884                                        idx, lnet, bwrt, rdwr, type);
1885
1886                         lst_reset_rpcent(&srp->srp_result[1 - idx]);
1887                 }
1888
1889                 idx = 1 - idx;
1890         }
1891
1892 out:
1893         while (!list_empty(&head)) {
1894                 srp = list_entry(head.next, lst_stat_req_param_t, srp_link);
1895
1896                 list_del(&srp->srp_link);
1897                 lst_stat_req_param_free(srp);
1898         }
1899
1900         return rc;
1901 }
1902
1903 int
1904 jt_lst_show_error(int argc, char **argv)
1905 {
1906         struct list_head      head;
1907         lstcon_rpc_ent_t     *ent;
1908         sfw_counters_t       *sfwk;
1909         srpc_counters_t      *srpc;
1910         lnet_counters_t      *lnet;
1911         lnet_process_id_t    *idsp   = NULL;
1912         char                 *name   = NULL;
1913         int                   optidx = 0;
1914         int                   count  = 0;
1915         int                   type   = 0;
1916         int                   timeout = 5;
1917         int                   ecount = 0;
1918         int                   rc;
1919         int                   c;
1920
1921
1922         static struct option show_error_opts[] =
1923         {
1924                 {"group"  , required_argument, 0, 'g' },
1925                 {"nodes"  , required_argument, 0, 'n' },
1926                 {0,         0,                 0,  0  }
1927         };
1928
1929         if (session_key == 0) {
1930                 fprintf(stderr,
1931                         "Can't find env LST_SESSION or value is not valid\n");
1932                 return -1;
1933         }
1934
1935         while (1) {
1936                 c = getopt_long(argc, argv, "g:n:", show_error_opts, &optidx);
1937
1938                 if (c == -1)
1939                         break;
1940
1941                 switch (c) {
1942                 case 'g':
1943                         type = LST_OPC_GROUP;
1944                         name = optarg;
1945                         break;
1946                 case 'n':
1947                         type = LST_OPC_NODES;
1948                         name = optarg;
1949                         break;
1950                 default:
1951                         lst_print_usage(argv[0]);
1952                         return -1;
1953                 }
1954         }
1955
1956         if (optind != argc || type == 0) {
1957                 lst_print_usage(argv[0]);
1958                 return -1;
1959         }
1960
1961         if (name == NULL) {
1962                 fprintf(stderr, "Missing name of target (group | nodes)\n");
1963                 return -1;
1964         }
1965
1966         rc = lst_get_node_count(type, name, &count, &idsp);
1967         if (rc < 0) {
1968                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
1969                         name, strerror(errno));
1970                 return -1;
1971         }
1972
1973         CFS_INIT_LIST_HEAD(&head);
1974
1975         rc = lst_alloc_rpcent(&head, count, sizeof(sfw_counters_t) +
1976                                             sizeof(srpc_counters_t) +
1977                                             sizeof(lnet_counters_t));
1978         if (rc != 0) {
1979                 fprintf(stderr, "Out of memory\n");
1980                 goto out;
1981         }
1982
1983         rc = lst_stat_ioctl(name, count, idsp, timeout, &head);
1984         if (rc == -1) {
1985                 lst_print_error(name, "Failed to show errors of %s: %s\n",
1986                                 name, strerror(errno));
1987                 goto out;
1988         }
1989
1990         list_for_each_entry(ent, &head, rpe_link) {
1991                 if (ent->rpe_rpc_errno != 0) {
1992                         ecount ++;
1993                         fprintf(stderr, "RPC failure, can't show error on %s\n",
1994                                 libcfs_id2str(ent->rpe_peer));
1995                         continue;
1996                 }
1997
1998                 if (ent->rpe_fwk_errno != 0) {
1999                         ecount ++;
2000                         fprintf(stderr, "Framework failure, can't show error on %s\n",
2001                                 libcfs_id2str(ent->rpe_peer));
2002                         continue;
2003                 }
2004
2005                 sfwk = (sfw_counters_t *)&ent->rpe_payload[0];
2006                 srpc = (srpc_counters_t *)((char *)sfwk + sizeof(*sfwk));
2007                 lnet = (lnet_counters_t *)((char *)srpc + sizeof(*srpc));
2008
2009                 if (srpc->errors == 0 &&
2010                     sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2011                         continue;
2012
2013                 ecount ++;
2014                 fprintf(stderr, "[%s]: %d RPC errors, %d brw errors, %d ping errors\n",
2015                         libcfs_id2str(ent->rpe_peer), srpc->errors,
2016                         sfwk->brw_errors, sfwk->ping_errors);
2017         }
2018
2019         fprintf(stdout, "Total %d errors in %s\n", ecount, name);
2020 out:
2021         lst_free_rpcent(&head);
2022         if (idsp != NULL)
2023                 free(idsp);
2024
2025         return 0;
2026 }
2027
2028 int
2029 lst_add_batch_ioctl (char *name)
2030 {
2031         lstio_batch_add_args_t  args = {
2032                 .lstio_bat_key           = session_key,
2033                 .lstio_bat_nmlen         = strlen(name),
2034                 .lstio_bat_namep         = name,
2035         };
2036
2037         return lst_ioctl (LSTIO_BATCH_ADD, &args, sizeof(args));
2038 }
2039
2040 int
2041 jt_lst_add_batch(int argc, char **argv)
2042 {
2043         char   *name;
2044         int     rc;
2045
2046         if (session_key == 0) {
2047                 fprintf(stderr,
2048                         "Can't find env LST_SESSION or value is not valid\n");
2049                 return -1;
2050         }
2051
2052         if (argc != 2) {
2053                 lst_print_usage(argv[0]);
2054                 return -1;
2055         }
2056
2057         name = argv[1];
2058         if (strlen(name) >= LST_NAME_SIZE) {
2059                 fprintf(stderr, "Name length is limited to %d\n",
2060                         LST_NAME_SIZE - 1);
2061                 return -1;
2062         }
2063
2064         rc = lst_add_batch_ioctl(name);
2065         if (rc == 0)
2066                 return 0;
2067
2068         lst_print_error("batch", "Failed to create batch: %s\n",
2069                         strerror(errno));
2070
2071         return -1;
2072 }
2073
2074 int
2075 lst_start_batch_ioctl (char *name, int timeout, struct list_head *resultp)
2076 {
2077         lstio_batch_run_args_t   args = {
2078                 .lstio_bat_key          = session_key,
2079                 .lstio_bat_timeout      = timeout,
2080                 .lstio_bat_nmlen        = strlen(name),
2081                 .lstio_bat_namep        = name,
2082                 .lstio_bat_resultp      = resultp,
2083         };
2084
2085         return lst_ioctl(LSTIO_BATCH_START, &args, sizeof(args));
2086 }
2087
2088 int
2089 jt_lst_start_batch(int argc, char **argv)
2090 {
2091         struct list_head  head;
2092         char             *batch;
2093         int               optidx  = 0;
2094         int               timeout = 0;
2095         int               count = 0;
2096         int               rc;
2097         int               c;
2098
2099         static struct option start_batch_opts[] =
2100         {
2101                 {"timeout", required_argument, 0, 't' },
2102                 {0,         0,                 0,  0  }
2103         };
2104
2105         if (session_key == 0) {
2106                 fprintf(stderr,
2107                         "Can't find env LST_SESSION or value is not valid\n");
2108                 return -1;
2109         }
2110
2111         while (1) {
2112                 c = getopt_long(argc, argv, "t:",
2113                                 start_batch_opts, &optidx);
2114
2115                 /* Detect the end of the options. */
2116                 if (c == -1)
2117                         break;
2118
2119                 switch (c) {
2120                 case 't':
2121                         timeout = atoi(optarg);
2122                         break;
2123                 default:
2124                         lst_print_usage(argv[0]);
2125                         return -1;
2126                 }
2127         }
2128
2129         if (optind == argc) {
2130                 batch = LST_DEFAULT_BATCH;
2131
2132         } else if (optind == argc - 1) {
2133                 batch = argv[optind];
2134
2135         } else {
2136                 lst_print_usage(argv[0]);
2137                 return -1;
2138         }
2139
2140         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2141         if (rc != 0) {
2142                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2143                         batch, strerror(errno));
2144                 return -1;
2145         }
2146
2147         CFS_INIT_LIST_HEAD(&head);
2148
2149         rc = lst_alloc_rpcent(&head, count, 0);
2150         if (rc != 0) {
2151                 fprintf(stderr, "Out of memory\n");
2152                 return -1;
2153         }
2154
2155         rc = lst_start_batch_ioctl(batch, timeout, &head);
2156
2157         if (rc == 0) {
2158                 fprintf(stdout, "%s is running now\n", batch);
2159                 lst_free_rpcent(&head);
2160                 return 0;
2161         }
2162
2163         if (rc == -1) {
2164                 lst_print_error("batch", "Failed to start batch: %s\n",
2165                                 strerror(errno));
2166                 lst_free_rpcent(&head);
2167                 return rc;
2168         }
2169
2170         lst_print_transerr(&head, "Run batch");
2171
2172         lst_free_rpcent(&head);
2173
2174         return rc;
2175 }
2176
2177 int
2178 lst_stop_batch_ioctl(char *name, int force, struct list_head *resultp)
2179 {
2180         lstio_batch_stop_args_t   args = {
2181                 .lstio_bat_key          = session_key,
2182                 .lstio_bat_force        = force,
2183                 .lstio_bat_nmlen        = strlen(name),
2184                 .lstio_bat_namep        = name,
2185                 .lstio_bat_resultp      = resultp,
2186         };
2187
2188         return lst_ioctl(LSTIO_BATCH_STOP, &args, sizeof(args));
2189 }
2190
2191 int
2192 jt_lst_stop_batch(int argc, char **argv)
2193 {
2194         struct list_head  head;
2195         char             *batch;
2196         int               force = 0;
2197         int               optidx;
2198         int               count;
2199         int               rc;
2200         int               c;
2201
2202         static struct option stop_batch_opts[] =
2203         {
2204                 {"force",   no_argument,   0, 'f' },
2205                 {0,         0,             0,  0  }
2206         };
2207
2208         if (session_key == 0) {
2209                 fprintf(stderr,
2210                         "Can't find env LST_SESSION or value is not valid\n");
2211                 return -1;
2212         }
2213
2214         while (1) {
2215                 c = getopt_long(argc, argv, "f",
2216                                 stop_batch_opts, &optidx);
2217
2218                 /* Detect the end of the options. */
2219                 if (c == -1)
2220                         break;
2221
2222                 switch (c) {
2223                 case 'f':
2224                         force = 1;
2225                         break;
2226                 default:
2227                         lst_print_usage(argv[0]);
2228                         return -1;
2229                 }
2230         }
2231
2232         if (optind == argc) {
2233                 batch = LST_DEFAULT_BATCH;
2234
2235         } else if (optind == argc - 1) {
2236                 batch = argv[optind];
2237
2238         } else {
2239                 lst_print_usage(argv[0]);
2240                 return -1;
2241         }
2242
2243         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2244         if (rc != 0) {
2245                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2246                         batch, strerror(errno));
2247                 return -1;
2248         }
2249
2250         CFS_INIT_LIST_HEAD(&head);
2251
2252         rc = lst_alloc_rpcent(&head, count, 0);
2253         if (rc != 0) {
2254                 fprintf(stderr, "Out of memory\n");
2255                 return -1;
2256         }
2257
2258         rc = lst_stop_batch_ioctl(batch, force, &head);
2259         if (rc != 0)
2260                 goto out;
2261
2262         while (1) {
2263                 lst_reset_rpcent(&head);
2264
2265                 rc = lst_query_batch_ioctl(batch, 0, 0, 30, &head);
2266                 if (rc != 0)
2267                         goto out;
2268
2269                 if (lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0 &&
2270                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0)
2271                         break;
2272
2273                 fprintf(stdout, "%d batch in stopping\n",
2274                         lstcon_tsbqry_stat_run(&trans_stat, 0));
2275                 sleep(1);
2276         }
2277
2278         fprintf(stdout, "Batch is stopped\n");
2279         lst_free_rpcent(&head);
2280
2281         return 0;
2282 out:
2283         if (rc == -1) {
2284                 lst_print_error("batch", "Failed to stop batch: %s\n",
2285                                 strerror(errno));
2286                 lst_free_rpcent(&head);
2287                 return -1;
2288         }
2289
2290         lst_print_transerr(&head, "stop batch");
2291
2292         lst_free_rpcent(&head);
2293
2294         return rc;
2295 }
2296
2297 int
2298 lst_list_batch_ioctl(int len, char *name, int index)
2299 {
2300         lstio_batch_list_args_t         args = {
2301                 .lstio_bat_key          = session_key,
2302                 .lstio_bat_idx          = index,
2303                 .lstio_bat_nmlen        = len,
2304                 .lstio_bat_namep        = name,
2305         };
2306
2307         return lst_ioctl(LSTIO_BATCH_LIST, &args, sizeof(args));
2308 }
2309
2310 int
2311 lst_info_batch_ioctl(char *batch, int test, int server,
2312                      lstcon_test_batch_ent_t *entp, int *idxp,
2313                      int *ndentp, lstcon_node_ent_t *dentsp)
2314 {
2315         lstio_batch_info_args_t         args = {
2316                 .lstio_bat_key          = session_key,
2317                 .lstio_bat_nmlen        = strlen(batch),
2318                 .lstio_bat_namep        = batch,
2319                 .lstio_bat_server       = server,
2320                 .lstio_bat_testidx      = test,
2321                 .lstio_bat_entp         = entp,
2322                 .lstio_bat_idxp         = idxp,
2323                 .lstio_bat_ndentp       = ndentp,
2324                 .lstio_bat_dentsp       = dentsp,
2325         };
2326
2327         return lst_ioctl(LSTIO_BATCH_INFO, &args, sizeof(args));
2328 }
2329
2330 int
2331 lst_list_batch_all(void)
2332 {
2333         char name[LST_NAME_SIZE];
2334         int  rc;
2335         int  i;
2336
2337         for (i = 0; ; i++) {
2338                 rc = lst_list_batch_ioctl(LST_NAME_SIZE, name, i);
2339                 if (rc == 0) {
2340                         fprintf(stdout, "%d) %s\n", i + 1, name);
2341                         continue;
2342                 }
2343
2344                 if (errno == ENOENT)
2345                         break;
2346
2347                 lst_print_error("batch", "Failed to list batch: %s\n",
2348                                 strerror(errno));
2349                 return rc;
2350         }
2351
2352         fprintf(stdout, "Total %d batches\n", i);
2353
2354         return 0;
2355 }
2356
2357 int
2358 lst_list_tsb_nodes(char *batch, int test, int server,
2359                    int count, int active, int invalid)
2360 {
2361         lstcon_node_ent_t *dents;
2362         int                index = 0;
2363         int                rc;
2364         int                c;
2365         int                i;
2366
2367         if (count == 0)
2368                 return 0;
2369
2370         /* verbose list, show nodes in batch or test */
2371         dents = malloc(count * sizeof(lstcon_node_ent_t));
2372         if (dents == NULL) {
2373                 fprintf(stdout, "Can't allocate memory\n");
2374                 return -1;
2375         }
2376
2377         rc = lst_info_batch_ioctl(batch, test, server,
2378                                   NULL, &index, &count, dents);
2379         if (rc != 0) {
2380                 free(dents);
2381                 lst_print_error((test > 0) ? "test" : "batch",
2382                                 (test > 0) ? "Failed to query test: %s\n" :
2383                                              "Failed to query batch: %s\n",
2384                                 strerror(errno));
2385                 return -1;
2386         }
2387
2388         for (i = 0, c = 0; i < count; i++) {
2389                 if ((!active  && dents[i].nde_state == LST_NODE_ACTIVE) ||
2390                     (!invalid && (dents[i].nde_state == LST_NODE_BUSY  ||
2391                                   dents[i].nde_state == LST_NODE_DOWN  ||
2392                                   dents[i].nde_state == LST_NODE_UNKNOWN)))
2393                         continue;
2394
2395                 fprintf(stdout, "\t%s: %s\n",
2396                         libcfs_id2str(dents[i].nde_id),
2397                         lst_node_state2str(dents[i].nde_state));
2398                 c++;
2399         }
2400
2401         fprintf(stdout, "Total %d nodes\n", c);
2402         free(dents);
2403
2404         return 0;
2405 }
2406
2407 int
2408 jt_lst_list_batch(int argc, char **argv)
2409 {
2410         lstcon_test_batch_ent_t ent;
2411         char                *batch   = NULL;
2412         int                  optidx  = 0;
2413         int                  verbose = 0; /* list nodes in batch or test */
2414         int                  invalid = 0;
2415         int                  active  = 0;
2416         int                  server  = 0;
2417         int                  ntest   = 0;
2418         int                  test    = 0;
2419         int                  c       = 0;
2420         int                  rc;
2421
2422         static struct option list_batch_opts[] =
2423         {
2424                 {"test",    required_argument, 0, 't' },
2425                 {"invalid", no_argument,       0, 'i' },
2426                 {"active",  no_argument,       0, 'a' },
2427                 {"all",     no_argument,       0, 'l' },
2428                 {"server",  no_argument,       0, 's' },
2429                 {0,         0,                 0,  0  }
2430         };
2431
2432         if (session_key == 0) {
2433                 fprintf(stderr,
2434                         "Can't find env LST_SESSION or value is not valid\n");
2435                 return -1;
2436         }
2437
2438         while (1) {
2439                 c = getopt_long(argc, argv, "ailst:",
2440                                 list_batch_opts, &optidx);
2441
2442                 if (c == -1)
2443                         break;
2444
2445                 switch (c) {
2446                 case 'a':
2447                         verbose = active = 1;
2448                         break;
2449                 case 'i':
2450                         verbose = invalid = 1;
2451                         break;
2452                 case 'l':
2453                         verbose = active = invalid = 1;
2454                         break;
2455                 case 's':
2456                         server = 1;
2457                         break;
2458                 case 't':
2459                         test = atoi(optarg);
2460                         ntest = 1;
2461                         break;
2462                 default:
2463                         lst_print_usage(argv[0]);
2464                         return -1;
2465                 }
2466         }
2467
2468         if (optind == argc) {
2469                 /* list all batches */
2470                 rc = lst_list_batch_all();
2471                 return rc;
2472         }
2473
2474         if (ntest == 1 && test <= 0) {
2475                 fprintf(stderr, "Invalid test id, test id starts from 1\n");
2476                 return -1;
2477         }
2478
2479         if (optind != argc - 1) {
2480                 lst_print_usage(argv[0]);
2481                 return -1;
2482         }
2483
2484         batch = argv[optind];
2485
2486 loop:
2487         /* show detail of specified batch or test */
2488         rc = lst_info_batch_ioctl(batch, test, server,
2489                                   &ent, NULL, NULL, NULL);
2490         if (rc != 0) {
2491                 lst_print_error((test > 0) ? "test" : "batch",
2492                                 (test > 0) ? "Failed to query test: %s\n" :
2493                                              "Failed to query batch: %s\n",
2494                                 strerror(errno));
2495                 return -1;
2496         }
2497
2498         if (verbose) {
2499                 /* list nodes in test or batch */
2500                 rc = lst_list_tsb_nodes(batch, test, server,
2501                                         server ? ent.tbe_srv_nle.nle_nnode :
2502                                                  ent.tbe_cli_nle.nle_nnode,
2503                                         active, invalid);
2504                 return rc;
2505         }
2506
2507         /* only show number of hosts in batch or test */
2508         if (test == 0) {
2509                 fprintf(stdout, "Batch: %s Tests: %d State: %d\n",
2510                         batch, ent.u.tbe_batch.bae_ntest,
2511                         ent.u.tbe_batch.bae_state);
2512                 ntest = ent.u.tbe_batch.bae_ntest;
2513                 test = 1; /* starting from test 1 */
2514
2515         } else {
2516                 fprintf(stdout,
2517                         "\tTest %d(%s) (loop: %d, concurrency: %d)\n",
2518                         test, lst_test_type2name(ent.u.tbe_test.tse_type),
2519                         ent.u.tbe_test.tse_loop,
2520                         ent.u.tbe_test.tse_concur);
2521                 ntest --;
2522                 test ++;
2523         }
2524
2525         fprintf(stdout, LST_NODES_TITLE);
2526         fprintf(stdout, "client\t%d\t%d\t%d\t%d\t%d\n"
2527                         "server\t%d\t%d\t%d\t%d\t%d\n",
2528                 ent.tbe_cli_nle.nle_nactive,
2529                 ent.tbe_cli_nle.nle_nbusy,
2530                 ent.tbe_cli_nle.nle_ndown,
2531                 ent.tbe_cli_nle.nle_nunknown,
2532                 ent.tbe_cli_nle.nle_nnode,
2533                 ent.tbe_srv_nle.nle_nactive,
2534                 ent.tbe_srv_nle.nle_nbusy,
2535                 ent.tbe_srv_nle.nle_ndown,
2536                 ent.tbe_srv_nle.nle_nunknown,
2537                 ent.tbe_srv_nle.nle_nnode);
2538
2539         if (ntest != 0)
2540                 goto loop;
2541
2542         return 0;
2543 }
2544
2545 int
2546 lst_query_batch_ioctl(char *batch, int test, int server,
2547                       int timeout, struct list_head *head)
2548 {
2549         lstio_batch_query_args_t args = {
2550                 .lstio_bat_key     = session_key,
2551                 .lstio_bat_testidx = test,
2552                 .lstio_bat_client  = !(server),
2553                 .lstio_bat_timeout = timeout,
2554                 .lstio_bat_nmlen   = strlen(batch),
2555                 .lstio_bat_namep   = batch,
2556                 .lstio_bat_resultp = head,
2557         };
2558
2559         return lst_ioctl(LSTIO_BATCH_QUERY, &args, sizeof(args));
2560 }
2561
2562 void
2563 lst_print_tsb_verbose(struct list_head *head,
2564                       int active, int idle, int error)
2565 {
2566         lstcon_rpc_ent_t *ent;
2567
2568         list_for_each_entry(ent, head, rpe_link) {
2569                 if (ent->rpe_priv[0] == 0 && active)
2570                         continue;
2571
2572                 if (ent->rpe_priv[0] != 0 && idle)
2573                         continue;
2574
2575                 if (ent->rpe_fwk_errno == 0 && error)
2576                         continue;
2577
2578                 fprintf(stdout, "%s [%s]: %s\n",
2579                         libcfs_id2str(ent->rpe_peer),
2580                         lst_node_state2str(ent->rpe_state),
2581                         ent->rpe_rpc_errno != 0 ?
2582                                 strerror(ent->rpe_rpc_errno) :
2583                                 (ent->rpe_priv[0] > 0 ? "Running" : "Idle"));
2584         }
2585 }
2586
2587 int
2588 jt_lst_query_batch(int argc, char **argv)
2589 {
2590         lstcon_test_batch_ent_t ent;
2591         struct list_head     head;
2592         char                *batch   = NULL;
2593         time_t               last    = 0;
2594         int                  optidx  = 0;
2595         int                  verbose = 0;
2596         int                  server  = 0;
2597         int                  timeout = 5; /* default 5 seconds */
2598         int                  delay   = 5; /* default 5 seconds */
2599         int                  loop    = 1; /* default 1 loop */
2600         int                  active  = 0;
2601         int                  error   = 0;
2602         int                  idle    = 0;
2603         int                  count   = 0;
2604         int                  test    = 0;
2605         int                  rc      = 0;
2606         int                  c       = 0;
2607         int                  i;
2608
2609         static struct option query_batch_opts[] =
2610         {
2611                 {"timeout", required_argument, 0, 'o' },
2612                 {"delay",   required_argument, 0, 'd' },
2613                 {"loop",    required_argument, 0, 'c' },
2614                 {"test",    required_argument, 0, 't' },
2615                 {"server",  no_argument,       0, 's' },
2616                 {"active",  no_argument,       0, 'a' },
2617                 {"idle",    no_argument,       0, 'i' },
2618                 {"error",   no_argument,       0, 'e' },
2619                 {"all",     no_argument,       0, 'l' },
2620                 {0,         0,                 0,  0  }
2621         };
2622
2623         if (session_key == 0) {
2624                 fprintf(stderr,
2625                         "Can't find env LST_SESSION or value is not valid\n");
2626                 return -1;
2627         }
2628
2629         while (1) {
2630                 c = getopt_long(argc, argv, "o:d:c:t:saiel",
2631                                 query_batch_opts, &optidx);
2632
2633                 /* Detect the end of the options. */
2634                 if (c == -1)
2635                         break;
2636
2637                 switch (c) {
2638                 case 'o':
2639                         timeout = atoi(optarg);
2640                         break;
2641                 case 'd':
2642                         delay = atoi(optarg);
2643                         break;
2644                 case 'c':
2645                         loop = atoi(optarg);
2646                         break;
2647                 case 't':
2648                         test = atoi(optarg);
2649                         break;
2650                 case 's':
2651                         server = 1;
2652                         break;
2653                 case 'a':
2654                         active = verbose = 1;
2655                         break;
2656                 case 'i':
2657                         idle = verbose = 1;
2658                         break;
2659                 case 'e':
2660                         error = verbose = 1;
2661                         break;
2662                 case 'l':
2663                         verbose = 1;
2664                         break;
2665                 default:
2666                         lst_print_usage(argv[0]);
2667                         return -1;
2668                 }
2669         }
2670
2671         if (test < 0 || timeout <= 0 || delay <= 0 || loop <= 0) {
2672                 lst_print_usage(argv[0]);
2673                 return -1;
2674         }
2675
2676         if (optind == argc) {
2677                 batch = LST_DEFAULT_BATCH;
2678
2679         } else if (optind == argc - 1) {
2680                 batch = argv[optind];
2681
2682         } else {
2683                 lst_print_usage(argv[0]);
2684                 return -1;
2685         }
2686
2687
2688         CFS_INIT_LIST_HEAD(&head);
2689
2690         if (verbose) {
2691                 rc = lst_info_batch_ioctl(batch, test, server,
2692                                           &ent, NULL, NULL, NULL);
2693                 if (rc != 0) {
2694                         fprintf(stderr, "Failed to query %s [%d]: %s\n",
2695                                 batch, test, strerror(errno));
2696                         return -1;
2697                 }
2698
2699                 count = server ? ent.tbe_srv_nle.nle_nnode :
2700                                  ent.tbe_cli_nle.nle_nnode;
2701                 if (count == 0) {
2702                         fprintf(stdout, "Batch or test is empty\n");
2703                         return 0;
2704                 }
2705         }
2706
2707         rc = lst_alloc_rpcent(&head, count, 0);
2708         if (rc != 0) {
2709                 fprintf(stderr, "Out of memory\n");
2710                 return rc;
2711         }
2712
2713         for (i = 0; i < loop; i++) {
2714                 time_t  now = time(NULL);
2715
2716                 if (now - last < delay) {
2717                         sleep(delay - now + last);
2718                         time(&now);
2719                 }
2720
2721                 last = now;
2722
2723                 rc = lst_query_batch_ioctl(batch, test,
2724                                            server, timeout, &head);
2725                 if (rc == -1) {
2726                         fprintf(stderr, "Failed to query batch: %s\n",
2727                                 strerror(errno));
2728                         break;
2729                 }
2730
2731                 if (verbose) {
2732                         /* Verbose mode */
2733                         lst_print_tsb_verbose(&head, active, idle, error);
2734                         continue;
2735                 }
2736
2737                 fprintf(stdout, "%s [%d] ", batch, test);
2738
2739                 if (lstcon_rpc_stat_failure(&trans_stat, 0) != 0) {
2740                         fprintf(stdout, "%d of %d nodes are unknown, ",
2741                                 lstcon_rpc_stat_failure(&trans_stat, 0),
2742                                 lstcon_rpc_stat_total(&trans_stat, 0));
2743                 }
2744
2745                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
2746                     lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0  &&
2747                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
2748                         fprintf(stdout, "is stopped\n");
2749                         continue;
2750                 }
2751
2752                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
2753                     lstcon_tsbqry_stat_idle(&trans_stat, 0) == 0 &&
2754                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
2755                         fprintf(stdout, "is running\n");
2756                         continue;
2757                 }
2758
2759                 fprintf(stdout, "stopped: %d , running: %d, failed: %d\n",
2760                                 lstcon_tsbqry_stat_idle(&trans_stat, 0),
2761                                 lstcon_tsbqry_stat_run(&trans_stat, 0),
2762                                 lstcon_tsbqry_stat_failure(&trans_stat, 0));
2763         }
2764
2765         lst_free_rpcent(&head);
2766
2767         return rc;
2768 }
2769
2770 int
2771 lst_parse_distribute(char *dstr, int *dist, int *span)
2772 {
2773         *dist = atoi(dstr);
2774         if (*dist <= 0)
2775                 return -1;
2776
2777         dstr = strchr(dstr, ':');
2778         if (dstr == NULL)
2779                 return -1;
2780
2781         *span = atoi(dstr + 1);
2782         if (*span <= 0)
2783                 return -1;
2784
2785         return 0;
2786 }
2787
2788 int
2789 lst_get_test_param(char *test, int argc, char **argv, void **param, int *plen)
2790 {
2791         lst_test_bulk_param_t *bulk = NULL;
2792         int                    type;
2793         int                    i = 0;
2794
2795         type = lst_test_name2type(test);
2796         if (type < 0)
2797                 return -EINVAL;
2798
2799         switch (type) {
2800         case LST_TEST_PING:
2801                 break;
2802
2803         case LST_TEST_BULK:
2804                 if (i == argc)
2805                         return -EINVAL;
2806
2807                 bulk = malloc(sizeof(*bulk));
2808                 if (bulk == NULL)
2809                         return -ENOMEM;
2810
2811                 memset(bulk, 0, sizeof(*bulk));
2812
2813                 if (strcmp(argv[i], "w") == 0)
2814                         bulk->blk_opc = LST_BRW_WRITE;
2815                 else  /* read by default */
2816                         bulk->blk_opc = LST_BRW_READ;
2817
2818                 if (++i == argc) {
2819                         /* 1 page by default */
2820                         bulk->blk_flags = LST_BRW_CHECK_NONE;
2821                         bulk->blk_npg   = 1;
2822                         *param = bulk;
2823                         *plen  = sizeof(*bulk);
2824                         break;
2825
2826                 }
2827
2828                 bulk->blk_npg = atoi(argv[i]);
2829                 if (bulk->blk_npg <= 0 ||
2830                     bulk->blk_npg >= LNET_MAX_IOV) {
2831                         free(bulk);
2832                         return -EINVAL;
2833                 }
2834
2835                 if (++i == argc) {
2836                         bulk->blk_flags = LST_BRW_CHECK_NONE;
2837                         *param = bulk;
2838                         *plen  = sizeof(*bulk);
2839                         break;
2840                 }
2841
2842                 /* posion pages */
2843                 if (strcmp(argv[i], "s") == 0)
2844                         bulk->blk_flags = LST_BRW_CHECK_SIMPLE;
2845                 else if (strcmp(argv[i], "f") == 0)
2846                         bulk->blk_flags = LST_BRW_CHECK_FULL;
2847                 else
2848                         bulk->blk_flags = LST_BRW_CHECK_NONE;
2849
2850                 *param = bulk;
2851                 *plen  = sizeof(*bulk);
2852
2853                 break;
2854
2855         default:
2856                 break;
2857         }
2858
2859         /* TODO: parse more parameter */
2860         return type;
2861 }
2862
2863 int
2864 lst_add_test_ioctl(char *batch, int type, int loop, int concur,
2865                    int dist, int span, char *sgrp, char *dgrp,
2866                    void *param, int plen, int *retp, struct list_head *resultp)
2867 {
2868         lstio_test_args_t args = {
2869                 .lstio_tes_key          = session_key,
2870                 .lstio_tes_bat_nmlen    = strlen(batch),
2871                 .lstio_tes_bat_name     = batch,
2872                 .lstio_tes_type         = type,
2873                 .lstio_tes_loop         = loop,
2874                 .lstio_tes_concur       = concur,
2875                 .lstio_tes_dist         = dist,
2876                 .lstio_tes_span         = span,
2877                 .lstio_tes_sgrp_nmlen   = strlen(sgrp),
2878                 .lstio_tes_sgrp_name    = sgrp,
2879                 .lstio_tes_dgrp_nmlen   = strlen(dgrp),
2880                 .lstio_tes_dgrp_name    = dgrp,
2881                 .lstio_tes_param_len    = plen,
2882                 .lstio_tes_param        = param,
2883                 .lstio_tes_retp         = retp,
2884                 .lstio_tes_resultp      = resultp,
2885         };
2886
2887         return lst_ioctl(LSTIO_TEST_ADD, &args, sizeof(args));
2888 }
2889
2890 int
2891 jt_lst_add_test(int argc, char **argv)
2892 {
2893         struct list_head  head;
2894         char             *batch  = NULL;
2895         char             *test   = NULL;
2896         char             *dstr   = NULL;
2897         char             *from   = NULL;
2898         char             *to     = NULL;
2899         void             *param  = NULL;
2900         int               optidx = 0;
2901         int               concur = 1;
2902         int               loop   = -1;
2903         int               dist   = 1;
2904         int               span   = 1;
2905         int               plen   = 0;
2906         int               fcount = 0;
2907         int               tcount = 0;
2908         int               ret    = 0;
2909         int               type;
2910         int               rc;
2911         int               c;
2912
2913         static struct option add_test_opts[] =
2914         {
2915                 {"batch",       required_argument, 0, 'b' },
2916                 {"concurrency", required_argument, 0, 'c' },
2917                 {"distribute",  required_argument, 0, 'd' },
2918                 {"from",        required_argument, 0, 'f' },
2919                 {"to",          required_argument, 0, 't' },
2920                 {"loop",        required_argument, 0, 'l' },
2921                 {0,             0,                 0,  0  }
2922         };
2923
2924         if (session_key == 0) {
2925                 fprintf(stderr,
2926                         "Can't find env LST_SESSION or value is not valid\n");
2927                 return -1;
2928         }
2929
2930         while (1) {
2931                 c = getopt_long(argc, argv, "b:c:d:f:l:t:",
2932                                 add_test_opts, &optidx);
2933
2934                 /* Detect the end of the options. */
2935                 if (c == -1)
2936                         break;
2937
2938                 switch (c) {
2939                 case 'b':
2940                         batch = optarg;
2941                         break;
2942                 case 'c':
2943                         concur = atoi(optarg);
2944                         break;
2945                 case 'd':
2946                         dstr = optarg;
2947                         break;
2948                 case 'f':
2949                         from = optarg;
2950                         break;
2951                 case 'l':
2952                         loop = atoi(optarg);
2953                         break;
2954                 case 't':
2955                         to = optarg;
2956                         break;
2957                 default:
2958                         lst_print_usage(argv[0]);
2959                         return -1;
2960                 }
2961         }
2962
2963         if (optind == argc || from == NULL || to == NULL) {
2964                 lst_print_usage(argv[0]);
2965                 return -1;
2966         }
2967
2968         if (concur <= 0 || concur > LST_MAX_CONCUR) {
2969                 fprintf(stderr, "Invalid concurrency of test: %d\n", concur);
2970                 return -1;
2971         }
2972
2973         if (batch == NULL)
2974                 batch = LST_DEFAULT_BATCH;
2975
2976         if (dstr != NULL) {
2977                 rc = lst_parse_distribute(dstr, &dist, &span);
2978                 if (rc != 0) {
2979                         fprintf(stderr, "Invalid distribution: %s\n", dstr);
2980                         return -1;
2981                 }
2982         }
2983
2984         test = argv[optind++];
2985
2986         argc -= optind;
2987         argv += optind;
2988
2989         type = lst_get_test_param(test, argc, argv, &param, &plen);
2990         if (type < 0) {
2991                 fprintf(stderr, "Can't parse test (%s)  parameter: %s\n",
2992                         test, strerror(-type));
2993                 return -1;
2994         }
2995
2996         CFS_INIT_LIST_HEAD(&head);
2997
2998         rc = lst_get_node_count(LST_OPC_GROUP, from, &fcount, NULL);
2999         if (rc != 0) {
3000                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3001                         from, strerror(errno));
3002                 goto out;
3003         }
3004
3005         rc = lst_get_node_count(LST_OPC_GROUP, to, &tcount, NULL);
3006         if (rc != 0) {
3007                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3008                         to, strerror(errno));
3009                 goto out;
3010         }
3011
3012         rc = lst_alloc_rpcent(&head, fcount > tcount ? fcount : tcount, 0);
3013         if (rc != 0) {
3014                 fprintf(stderr, "Out of memory\n");
3015                 goto out;
3016         }
3017
3018         rc = lst_add_test_ioctl(batch, type, loop, concur,
3019                                 dist, span, from, to, param, plen, &ret, &head);
3020
3021         if (rc == 0) {
3022                 fprintf(stdout, "Test was added successfully\n");
3023                 if (ret != 0) {
3024                         fprintf(stdout, "Server group contains userland test "
3025                                 "nodes, old version of tcplnd can't accept "
3026                                 "connection request\n");
3027                 }
3028
3029                 goto out;
3030         }
3031
3032         if (rc == -1) {
3033                 lst_print_error("test", "Failed to add test: %s\n",
3034                                 strerror(errno));
3035                 goto out;
3036         }
3037
3038         lst_print_transerr(&head, "add test");
3039 out:
3040         lst_free_rpcent(&head);
3041
3042         if (param != NULL)
3043                 free(param);
3044
3045         return rc;
3046 }
3047
3048 static command_t lst_cmdlist[] = {
3049         {"new_session",         jt_lst_new_session,     NULL,
3050          "Usage: lst new_session [--timeout TIME] [--force] [NAME]"                     },
3051         {"end_session",         jt_lst_end_session,     NULL,
3052          "Usage: lst end_session"                                                       },
3053         {"show_session",        jt_lst_show_session,    NULL,
3054          "Usage: lst show_session"                                                      },
3055         {"ping",                jt_lst_ping ,           NULL,
3056          "Usage: lst ping  [--group NAME] [--batch NAME] [--session] [--nodes IDS]"     },
3057         {"add_group",           jt_lst_add_group,       NULL,
3058          "Usage: lst group NAME IDs [IDs]..."                                           },
3059         {"del_group",           jt_lst_del_group,       NULL,
3060          "Usage: lst del_group NAME"                                                    },
3061         {"update_group",        jt_lst_update_group,    NULL,
3062          "Usage: lst update_group NAME [--clean] [--refresh] [--remove IDs]"            },
3063         {"list_group",          jt_lst_list_group,      NULL,
3064           "Usage: lst list_group [--active] [--busy] [--down] [--unknown] GROUP ..."    },
3065         {"stat",                jt_lst_stat,            NULL,
3066          "Usage: lst stat [--bw] [--rate] [--read] [--write] [--max] [--min] [--avg] "
3067          " [--timeout #] [--delay #] GROUP [GROUP]"                                     },
3068         {"show_error",          jt_lst_show_error,      NULL,
3069          "Usage: lst show_error [--group NAME] | [--nodes IDS]"                         },
3070         {"add_batch",           jt_lst_add_batch,       NULL,
3071          "Usage: lst add_batch NAME"                                                    },
3072         {"run",                 jt_lst_start_batch,     NULL,
3073          "Usage: lst run [--timeout TIME] [NAME]"                                       },
3074         {"stop",                jt_lst_stop_batch,      NULL,
3075          "Usage: lst stop [--force] BATCH_NAME"                                         },
3076         {"list_batch",          jt_lst_list_batch,      NULL,
3077          "Usage: lst list_batch NAME [--test ID] [--server]"                            },
3078         {"query",               jt_lst_query_batch,     NULL,
3079          "Usage: lst query [--test ID] [--server] [--timeout TIME] NAME"                },
3080         {"add_test",            jt_lst_add_test,        NULL,
3081          "Usage: lst add_test [--batch BATCH] [--loop #] [--concurrency #] "
3082          " [--distribute #:#] [--from GROUP] [--to GROUP] TEST..."                      },
3083         {"help",                Parser_help,            0,     "help"                   },
3084         {0,                     0,                      0,      NULL                    }
3085 };
3086
3087 int
3088 lst_initialize(void)
3089 {
3090         char   *key;
3091
3092         key = getenv("LST_SESSION");
3093
3094         if (key == NULL) {
3095                 session_key = 0;
3096                 return 0;
3097         }
3098
3099         session_key = atoi(key);
3100
3101         return 0;
3102 }
3103
3104 int
3105 main(int argc, char **argv)
3106 {
3107         setlinebuf(stdout);
3108
3109         if (lst_initialize() < 0)
3110                 exit(0);
3111
3112         if (ptl_initialize(argc, argv) < 0)
3113                 exit(0);
3114
3115         Parser_init("lst > ", lst_cmdlist);
3116
3117         if (argc != 1)
3118                 return Parser_execarg(argc - 1, argv + 1, lst_cmdlist);
3119
3120         Parser_commands();
3121
3122         return 0;
3123 }