lnet/utils/lst.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  *
  30  * Copyright (c) 2012, 2014, Intel Corporation.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * lnet/selftest/conctl.c
  37  *
  38  * Author: Liang Zhen <liangzhen@clusterfs.com>
  39  */
  40 #include <errno.h>
  41 #include <getopt.h>
  42 #include <inttypes.h>
  43 #include <pwd.h>
  44 #include <unistd.h>
  45 #include <stdarg.h>
  46 #include <stdio.h>
  47 #include <stdlib.h>
  48 #include <string.h>
  49 #include <sys/ioctl.h>
  50 #include <time.h>
  51
  52 #include <libcfs/util/list.h>
  53 #include <libcfs/util/ioctl.h>
  54 #include <libcfs/util/parser.h>
  55 #include <lnet/lnetctl.h>
  56 #include <lnet/lnetst.h>
  57 #include <lnet/lnet.h>
  58 #include <utils/obdctl.h>
  59
  60 lst_sid_t LST_INVALID_SID = {LNET_NID_ANY, -1};
  61 static lst_sid_t           session_id;
  62 static int                 session_key;
  63
  64 /* All nodes running 2.6.50 or later understand feature LST_FEAT_BULK_LEN */
  65 static unsigned         session_features = LST_FEATS_MASK;
  66 static lstcon_trans_stat_t      trans_stat;
  67
  68 typedef struct list_string {
  69         struct list_string *lstr_next;
  70         int                 lstr_sz;
  71         char                lstr_str[0];
  72 } lstr_t;
  73
  74 #ifndef offsetof
  75 # define offsetof(typ,memb)     ((unsigned long)((char *)&(((typ *)0)->memb)))
  76 #endif
  77
  78 static int alloc_count = 0;
  79 static int alloc_nob   = 0;
  80
  81 lstr_t *
  82 alloc_lstr(int sz)
  83 {
  84         lstr_t  *lstr = malloc(offsetof(lstr_t, lstr_str[sz]));
  85
  86         if (lstr == NULL) {
  87                 fprintf(stderr, "Can't allocate lstr\n");
  88                 abort();
  89         }
  90
  91         alloc_nob += sz;
  92         alloc_count++;
  93
  94         lstr->lstr_str[0] = 0;
  95         lstr->lstr_sz = sz;
  96         return lstr;
  97 }
  98
  99 void
 100 free_lstr(lstr_t *lstr)
 101 {
 102         alloc_count--;
 103         alloc_nob -= lstr->lstr_sz;
 104         free(lstr);
 105 }
 106
 107 void
 108 free_lstrs(lstr_t **list)
 109 {
 110         lstr_t   *lstr;
 111
 112         while ((lstr = *list) != NULL) {
 113                 *list = lstr->lstr_next;
 114                 free_lstr(lstr);
 115         }
 116 }
 117
 118 void
 119 new_lstrs(lstr_t **list, char *prefix, char *postfix,
 120           int lo, int hi, int stride)
 121 {
 122         int    n1 = strlen(prefix);
 123         int    n2 = strlen(postfix);
 124         int    sz = n1 + 20 + n2 + 1;
 125
 126         do {
 127                 lstr_t *n = alloc_lstr(sz);
 128
 129                 snprintf(n->lstr_str, sz - 1, "%s%u%s",
 130                          prefix, lo, postfix);
 131
 132                 n->lstr_next = *list;
 133                 *list = n;
 134
 135                 lo += stride;
 136         } while (lo <= hi);
 137 }
 138
 139 int
 140 expand_lstr(lstr_t **list, lstr_t *l)
 141 {
 142         int          nob = strlen(l->lstr_str);
 143         char        *b1;
 144         char        *b2;
 145         char        *expr;
 146         char        *sep;
 147         int          x;
 148         int          y;
 149         int          z;
 150         int          n;
 151
 152         b1 = strchr(l->lstr_str, '[');
 153         if (b1 == NULL) {
 154                 l->lstr_next = *list;
 155                 *list = l;
 156                 return 0;
 157         }
 158
 159         b2 = strchr(b1, ']');
 160         if (b2 == NULL || b2 == b1 + 1)
 161                 return -1;
 162
 163         *b1++ = 0;
 164         *b2++ = 0;
 165         expr = b1;
 166         do {
 167
 168                 sep = strchr(expr, ',');
 169                 if (sep != NULL)
 170                         *sep++ = 0;
 171
 172                 nob = strlen(expr);
 173                 n = nob;
 174                 if (sscanf(expr, "%u%n", &x, &n) >= 1 && n == nob) {
 175                         /* simple number */
 176                         new_lstrs(list, l->lstr_str, b2, x, x, 1);
 177                         continue;
 178                 }
 179
 180                 n = nob;
 181                 if (sscanf(expr, "%u-%u%n", &x, &y, &n) >= 2 && n == nob &&
 182                     x < y) {
 183                         /* simple range */
 184                         new_lstrs(list, l->lstr_str, b2, x, y, 1);
 185                         continue;
 186                 }
 187
 188                 n = nob;
 189                 if (sscanf(expr, "%u-%u/%u%n", &x, &y, &z, &n) >= 3 && n == nob &&
 190                     x < y) {
 191                         /* strided range */
 192                         new_lstrs(list, l->lstr_str, b2, x, y, z);
 193                         continue;
 194                 }
 195
 196                 /* syntax error */
 197                 return -1;
 198         } while ((expr = sep) != NULL);
 199
 200         free_lstr(l);
 201
 202         return 1;
 203 }
 204
 205 int
 206 expand_strs(char *str, lstr_t **head)
 207 {
 208         lstr_t  *list = NULL;
 209         lstr_t  *nlist;
 210         lstr_t  *l;
 211         int      rc = 0;
 212         int      expanded;
 213
 214         l = alloc_lstr(strlen(str) + 1);
 215         memcpy(l->lstr_str, str, strlen(str) + 1);
 216         l->lstr_next = NULL;
 217         list = l;
 218
 219         do {
 220                 expanded = 0;
 221                 nlist = NULL;
 222
 223                 while ((l = list) != NULL) {
 224                         list = l->lstr_next;
 225
 226                         rc = expand_lstr(&nlist, l);
 227                         if (rc < 0) {
 228                                 fprintf(stderr, "Syntax error in \"%s\"\n", str);
 229                                 free_lstr(l);
 230                                 break;
 231                         }
 232
 233                         expanded |= rc > 0;
 234                 }
 235
 236                 /* re-order onto 'list' */
 237                 while ((l = nlist) != NULL) {
 238                         nlist = l->lstr_next;
 239                         l->lstr_next = list;
 240                         list = l;
 241                 }
 242
 243         } while (expanded && rc > 0);
 244
 245         if (rc >= 0) {
 246                 *head = list;
 247                 return 0;
 248         }
 249
 250         while ((l = list) != NULL) {
 251                 list = l->lstr_next;
 252
 253                 free_lstr(l);
 254         }
 255         return rc;
 256 }
 257
 258 int
 259 lst_parse_nids(char *str, int *countp, lnet_process_id_t **idspp)
 260 {
 261         lstr_t  *head = NULL;
 262         lstr_t  *l;
 263         int      c = 0;
 264         int      i;
 265         int      rc;
 266
 267         rc = expand_strs(str, &head);
 268         if (rc != 0)
 269                 goto out;
 270
 271         l = head;
 272         while (l != NULL) {
 273                 l = l->lstr_next;
 274                 c++;
 275         }
 276
 277         *idspp = malloc(c * sizeof(lnet_process_id_t));
 278         if (*idspp == NULL) {
 279                 fprintf(stderr, "Out of memory\n");
 280                 rc = -1;
 281         }
 282
 283         *countp = c;
 284 out:
 285         i = 0;
 286         while ((l = head) != NULL) {
 287                 head = l->lstr_next;
 288
 289                 if (rc == 0) {
 290                         (*idspp)[i].nid = libcfs_str2nid(l->lstr_str);
 291                         if ((*idspp)[i].nid == LNET_NID_ANY) {
 292                                 fprintf(stderr, "Invalid nid: %s\n",
 293                                         l->lstr_str);
 294                                 rc = -1;
 295                         }
 296
 297                         (*idspp)[i].pid = LNET_PID_LUSTRE;
 298                         i++;
 299                 }
 300
 301                 free_lstr(l);
 302         }
 303
 304         if (rc == 0)
 305                 return 0;
 306
 307         free(*idspp);
 308         *idspp = NULL;
 309
 310         return rc;
 311 }
 312
 313 char *
 314 lst_node_state2str(int state)
 315 {
 316         if (state == LST_NODE_ACTIVE)
 317                 return "Active";
 318         if (state == LST_NODE_BUSY)
 319                 return "Busy";
 320         if (state == LST_NODE_DOWN)
 321                 return "Down";
 322
 323         return "Unknown";
 324 }
 325
 326 int
 327 lst_node_str2state(char *str)
 328 {
 329         if (strcasecmp(str, "active") == 0)
 330                 return LST_NODE_ACTIVE;
 331         if (strcasecmp(str, "busy") == 0)
 332                 return LST_NODE_BUSY;
 333         if (strcasecmp(str, "down") == 0)
 334                 return LST_NODE_DOWN;
 335         if (strcasecmp(str, "unknown") == 0)
 336                 return LST_NODE_UNKNOWN;
 337         if (strcasecmp(str, "invalid") == 0)
 338                 return (LST_NODE_UNKNOWN | LST_NODE_DOWN | LST_NODE_BUSY);
 339
 340         return -1;
 341 }
 342
 343 char *
 344 lst_test_type2name(int type)
 345 {
 346         if (type == LST_TEST_PING)
 347                 return "ping";
 348         if (type == LST_TEST_BULK)
 349                 return "brw";
 350
 351         return "unknown";
 352 }
 353
 354 int
 355 lst_test_name2type(char *name)
 356 {
 357         if (strcasecmp(name, "ping") == 0)
 358                 return LST_TEST_PING;
 359         if (strcasecmp(name, "brw") == 0)
 360                 return LST_TEST_BULK;
 361
 362         return -1;
 363 }
 364
 365 void
 366 lst_print_usage(char *cmd)
 367 {
 368         Parser_printhelp(cmd);
 369 }
 370
 371 void
 372 lst_print_error(char *sub, const char *def_format, ...)
 373 {
 374         va_list ap;
 375
 376         /* local error returned from kernel */
 377         switch (errno) {
 378         case ESRCH:
 379                 fprintf(stderr, "No session exists\n");
 380                 return;
 381         case ESHUTDOWN:
 382                 fprintf(stderr, "Session is shutting down\n");
 383                 return;
 384         case EACCES:
 385                 fprintf(stderr, "Unmatched session key or not root\n");
 386                 return;
 387         case ENOENT:
 388                 fprintf(stderr, "Can't find %s in current session\n", sub);
 389                 return;
 390         case EINVAL:
 391                 fprintf(stderr, "Invalid parameters list in command line\n");
 392                 return;
 393         case EFAULT:
 394                 fprintf(stderr, "Bad parameter address\n");
 395                 return;
 396         case EEXIST:
 397                 fprintf(stderr, "%s already exists\n", sub);
 398                 return;
 399         default:
 400                 va_start(ap, def_format);
 401                 vfprintf(stderr, def_format, ap);
 402                 va_end(ap);
 403
 404                 return;
 405         }
 406 }
 407
 408 void
 409 lst_free_rpcent(struct list_head *head)
 410 {
 411         lstcon_rpc_ent_t *ent;
 412
 413         while (!list_empty(head)) {
 414                 ent = list_entry(head->next, lstcon_rpc_ent_t, rpe_link);
 415
 416                 list_del(&ent->rpe_link);
 417                 free(ent);
 418         }
 419 }
 420
 421 void
 422 lst_reset_rpcent(struct list_head *head)
 423 {
 424         lstcon_rpc_ent_t *ent;
 425
 426         list_for_each_entry(ent, head, rpe_link) {
 427                 ent->rpe_sid       = LST_INVALID_SID;
 428                 ent->rpe_peer.nid  = LNET_NID_ANY;
 429                 ent->rpe_peer.pid  = LNET_PID_ANY;
 430                 ent->rpe_rpc_errno = ent->rpe_fwk_errno = 0;
 431         }
 432 }
 433
 434 int
 435 lst_alloc_rpcent(struct list_head *head, int count, int offset)
 436 {
 437         lstcon_rpc_ent_t *ent;
 438         int               i;
 439
 440         for (i = 0; i < count; i++) {
 441                 ent = malloc(offsetof(lstcon_rpc_ent_t, rpe_payload[offset]));
 442                 if (ent == NULL) {
 443                         lst_free_rpcent(head);
 444                         return -1;
 445                 }
 446
 447                 memset(ent, 0, offsetof(lstcon_rpc_ent_t, rpe_payload[offset]));
 448
 449                 ent->rpe_sid      = LST_INVALID_SID;
 450                 ent->rpe_peer.nid = LNET_NID_ANY;
 451                 ent->rpe_peer.pid = LNET_PID_ANY;
 452                 list_add(&ent->rpe_link, head);
 453         }
 454
 455         return 0;
 456 }
 457
 458 void
 459 lst_print_transerr(struct list_head *head, char *optstr)
 460 {
 461         lstcon_rpc_ent_t *ent;
 462
 463         list_for_each_entry(ent, head, rpe_link) {
 464                 if (ent->rpe_rpc_errno == 0 && ent->rpe_fwk_errno == 0)
 465                         continue;
 466
 467                 if (ent->rpe_rpc_errno != 0) {
 468                         fprintf(stderr, "%s RPC failed on %s: %s\n",
 469                                 optstr, libcfs_id2str(ent->rpe_peer),
 470                                 strerror(ent->rpe_rpc_errno));
 471                         continue;
 472                 }
 473
 474                 fprintf(stderr, "operation %s failed on %s: %s\n",
 475                         optstr, libcfs_id2str(ent->rpe_peer),
 476                         strerror(ent->rpe_fwk_errno));
 477         }
 478 }
 479
 480 int lst_info_batch_ioctl(char *batch, int test, int server,
 481                         lstcon_test_batch_ent_t *entp, int *idxp,
 482                         int *ndentp, lstcon_node_ent_t *dentsp);
 483
 484 int lst_info_group_ioctl(char *name, lstcon_ndlist_ent_t *gent,
 485                          int *idx, int *count, lstcon_node_ent_t *dents);
 486
 487 int lst_query_batch_ioctl(char *batch, int test, int server,
 488                           int timeout, struct list_head *head);
 489
 490 int
 491 lst_ioctl(unsigned int opc, void *buf, int len)
 492 {
 493         struct libcfs_ioctl_data data;
 494         int    rc;
 495
 496         LIBCFS_IOC_INIT (data);
 497         data.ioc_u32[0]  = opc;
 498         data.ioc_plen1   = len;
 499         data.ioc_pbuf1   = (char *)buf;
 500         data.ioc_plen2   = sizeof(trans_stat);
 501         data.ioc_pbuf2   = (char *)&trans_stat;
 502
 503         memset(&trans_stat, 0, sizeof(trans_stat));
 504
 505         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNETST, &data);
 506
 507         /* local error, no valid RPC result */
 508         if (rc != 0)
 509                 return -1;
 510
 511         /* RPC error */
 512         if (trans_stat.trs_rpc_errno != 0)
 513                 return -2;
 514
 515         /* Framework error */
 516         if (trans_stat.trs_fwk_errno != 0)
 517                 return -3;
 518
 519         return 0;
 520 }
 521
 522 int
 523 lst_new_session_ioctl(char *name, int timeout, int force, lst_sid_t *sid)
 524 {
 525         lstio_session_new_args_t args = {0};
 526
 527         args.lstio_ses_key     = session_key;
 528         args.lstio_ses_timeout = timeout;
 529         args.lstio_ses_force   = force;
 530         args.lstio_ses_idp     = sid;
 531         args.lstio_ses_feats   = session_features;
 532         args.lstio_ses_nmlen   = strlen(name);
 533         args.lstio_ses_namep   = name;
 534
 535         return lst_ioctl (LSTIO_SESSION_NEW, &args, sizeof(args));
 536 }
 537
 538 int
 539 jt_lst_new_session(int argc,  char **argv)
 540 {
 541         char  buf[LST_NAME_SIZE];
 542         char *name;
 543         int   optidx  = 0;
 544         int   timeout = 300;
 545         int   force   = 0;
 546         int   c;
 547         int   rc;
 548
 549         static struct option session_opts[] =
 550         {
 551                 {"timeout", required_argument,  0, 't' },
 552                 {"force",   no_argument,        0, 'f' },
 553                 {0,         0,                  0,  0  }
 554         };
 555
 556         if (session_key == 0) {
 557                 fprintf(stderr,
 558                         "Can't find env LST_SESSION or value is not valid\n");
 559                 return -1;
 560         }
 561
 562         while (1) {
 563
 564                 c = getopt_long(argc, argv, "ft:",
 565                                 session_opts, &optidx);
 566
 567                 if (c == -1)
 568                         break;
 569
 570                 switch (c) {
 571                 case 'f':
 572                         force = 1;
 573                         break;
 574                 case 't':
 575                         timeout = atoi(optarg);
 576                         break;
 577                 default:
 578                         lst_print_usage(argv[0]);
 579                         return -1;
 580                 }
 581         }
 582
 583         if (timeout <= 0) {
 584                 fprintf(stderr, "Invalid timeout value\n");
 585                 return -1;
 586         }
 587
 588         if (optind == argc - 1) {
 589                 name = argv[optind ++];
 590                 if (strlen(name) >= LST_NAME_SIZE) {
 591                         fprintf(stderr, "Name size is limited to %d\n",
 592                                 LST_NAME_SIZE - 1);
 593                         return -1;
 594                 }
 595
 596         } else if (optind == argc) {
 597                 char           user[LST_NAME_SIZE];
 598                 char           host[LST_NAME_SIZE];
 599                 struct passwd *pw = getpwuid(getuid());
 600
 601                 if (pw == NULL)
 602                         snprintf(user, sizeof(user), "%d", (int)getuid());
 603                 else
 604                         snprintf(user, sizeof(user), "%s", pw->pw_name);
 605
 606                 rc = gethostname(host, sizeof(host));
 607                 if (rc != 0)
 608                         snprintf(host, sizeof(host), "unknown_host");
 609
 610                 snprintf(buf, sizeof(buf), "%s@%s", user, host);
 611                 name = buf;
 612
 613         } else {
 614                 lst_print_usage(argv[0]);
 615                 return -1;
 616         }
 617
 618         rc = lst_new_session_ioctl(name, timeout, force, &session_id);
 619         if (rc != 0) {
 620                 lst_print_error("session", "Failed to create session: %s\n",
 621                                 strerror(errno));
 622                 return rc;
 623         }
 624
 625         fprintf(stdout, "SESSION: %s FEATURES: %x TIMEOUT: %d FORCE: %s\n",
 626                 name, session_features, timeout, force ? "Yes" : "No");
 627         return 0;
 628 }
 629
 630 int
 631 lst_session_info_ioctl(char *name, int len, int *key, unsigned *featp,
 632                        lst_sid_t *sid, lstcon_ndlist_ent_t *ndinfo)
 633 {
 634         lstio_session_info_args_t args = {0};
 635
 636         args.lstio_ses_idp     = sid;
 637         args.lstio_ses_keyp    = key;
 638         args.lstio_ses_featp   = featp;
 639         args.lstio_ses_ndinfo  = ndinfo;
 640         args.lstio_ses_nmlen   = len;
 641         args.lstio_ses_namep   = name;
 642
 643         return lst_ioctl(LSTIO_SESSION_INFO, &args, sizeof(args));
 644 }
 645
 646 int
 647 jt_lst_show_session(int argc, char **argv)
 648 {
 649         lstcon_ndlist_ent_t ndinfo;
 650         lst_sid_t           sid;
 651         char                name[LST_NAME_SIZE];
 652         unsigned            feats;
 653         int                 key;
 654         int                 rc;
 655
 656         rc = lst_session_info_ioctl(name, sizeof(name), &key,
 657                                     &feats, &sid, &ndinfo);
 658
 659         if (rc != 0) {
 660                 lst_print_error("session", "Failed to show session: %s\n",
 661                                 strerror(errno));
 662                 return -1;
 663         }
 664
 665         fprintf(stdout, "%s ID: %ju@%s, KEY: %d FEATURES: %x NODES: %d\n",
 666                 name, (uintmax_t)sid.ses_stamp, libcfs_nid2str(sid.ses_nid),
 667                 key, feats, ndinfo.nle_nnode);
 668
 669         return 0;
 670 }
 671
 672 int
 673 lst_end_session_ioctl(void)
 674 {
 675         lstio_session_end_args_t args = {0};
 676
 677         args.lstio_ses_key =  session_key;
 678         return lst_ioctl (LSTIO_SESSION_END, &args, sizeof(args));
 679 }
 680
 681 int
 682 jt_lst_end_session(int argc, char **argv)
 683 {
 684         int             rc;
 685
 686         if (session_key == 0) {
 687                 fprintf(stderr,
 688                         "Can't find env LST_SESSION or value is not valid\n");
 689                 return -1;
 690         }
 691
 692         rc = lst_end_session_ioctl();
 693
 694         if (rc == 0) {
 695                 fprintf(stdout, "session is ended\n");
 696                 return 0;
 697         }
 698
 699         if (rc == -1) {
 700                 lst_print_error("session", "Failed to end session: %s\n",
 701                                 strerror(errno));
 702                 return rc;
 703         }
 704
 705         if (trans_stat.trs_rpc_errno != 0) {
 706                 fprintf(stderr,
 707                         "[RPC] Failed to send %d session RPCs: %s\n",
 708                         lstcon_rpc_stat_failure(&trans_stat, 0),
 709                         strerror(trans_stat.trs_rpc_errno));
 710         }
 711
 712         if (trans_stat.trs_fwk_errno != 0) {
 713                 fprintf(stderr,
 714                         "[FWK] Failed to end session on %d nodes: %s\n",
 715                         lstcon_sesop_stat_failure(&trans_stat, 0),
 716                         strerror(trans_stat.trs_fwk_errno));
 717         }
 718
 719         return rc;
 720 }
 721
 722 int
 723 lst_ping_ioctl(char *str, int type, int timeout,
 724                int count, lnet_process_id_t *ids, struct list_head *head)
 725 {
 726         lstio_debug_args_t args = {0};
 727
 728         args.lstio_dbg_key     = session_key;
 729         args.lstio_dbg_type    = type;
 730         args.lstio_dbg_flags   = 0;
 731         args.lstio_dbg_timeout = timeout;
 732         args.lstio_dbg_nmlen   = (str == NULL) ? 0: strlen(str);
 733         args.lstio_dbg_namep   = str;
 734         args.lstio_dbg_count   = count;
 735         args.lstio_dbg_idsp    = ids;
 736         args.lstio_dbg_resultp = head;
 737
 738         return lst_ioctl (LSTIO_DEBUG, &args, sizeof(args));
 739 }
 740
 741 int
 742 lst_get_node_count(int type, char *str, int *countp, lnet_process_id_t **idspp)
 743 {
 744         char                    buf[LST_NAME_SIZE];
 745         lstcon_test_batch_ent_t ent;
 746         lstcon_ndlist_ent_t    *entp = &ent.tbe_cli_nle;
 747         lst_sid_t               sid;
 748         unsigned                feats;
 749         int                     key;
 750         int                     rc;
 751
 752         switch (type) {
 753         case LST_OPC_SESSION:
 754                 rc = lst_session_info_ioctl(buf, LST_NAME_SIZE,
 755                                             &key, &feats, &sid, entp);
 756                 break;
 757
 758         case LST_OPC_BATCHSRV:
 759                 entp = &ent.tbe_srv_nle;
 760         case LST_OPC_BATCHCLI:
 761                 rc = lst_info_batch_ioctl(str, 0, 0, &ent, NULL, NULL, NULL);
 762                 break;
 763
 764         case LST_OPC_GROUP:
 765                 rc = lst_info_group_ioctl(str, entp, NULL, NULL, NULL);
 766                 break;
 767
 768         case LST_OPC_NODES:
 769                 rc = lst_parse_nids(str, &entp->nle_nnode, idspp) < 0 ? -1 : 0;
 770                 break;
 771
 772         default:
 773                 rc = -1;
 774                 break;
 775         }
 776
 777         if (rc == 0)
 778                 *countp = entp->nle_nnode;
 779
 780         return rc;
 781 }
 782
 783 int
 784 jt_lst_ping(int argc,  char **argv)
 785 {
 786         struct list_head   head;
 787         lnet_process_id_t *ids = NULL;
 788         lstcon_rpc_ent_t  *ent = NULL;
 789         char              *str = NULL;
 790         int                optidx  = 0;
 791         int                server  = 0;
 792         int                timeout = 5;
 793         int                count   = 0;
 794         int                type    = 0;
 795         int                rc      = 0;
 796         int                c;
 797
 798         static struct option ping_opts[] =
 799         {
 800                 {"session", no_argument,       0, 's' },
 801                 {"server",  no_argument,       0, 'v' },
 802                 {"batch",   required_argument, 0, 'b' },
 803                 {"group",   required_argument, 0, 'g' },
 804                 {"nodes",   required_argument, 0, 'n' },
 805                 {"timeout", required_argument, 0, 't' },
 806                 {0,         0,                 0,  0  }
 807         };
 808
 809         if (session_key == 0) {
 810                 fprintf(stderr,
 811                         "Can't find env LST_SESSION or value is not valid\n");
 812                 return -1;
 813         }
 814
 815         while (1) {
 816
 817                 c = getopt_long(argc, argv, "g:b:n:t:sv",
 818                                 ping_opts, &optidx);
 819
 820                 if (c == -1)
 821                         break;
 822
 823                 switch (c) {
 824                 case 's':
 825                         type = LST_OPC_SESSION;
 826                         break;
 827
 828                 case 'g':
 829                         type = LST_OPC_GROUP;
 830                         str = optarg;
 831                         break;
 832
 833                 case 'b':
 834                         type = LST_OPC_BATCHCLI;
 835                         str = optarg;
 836                         break;
 837
 838                 case 'n':
 839                         type = LST_OPC_NODES;
 840                         str = optarg;
 841                         break;
 842
 843                 case 't':
 844                         timeout = atoi(optarg);
 845                         break;
 846
 847                 case 'v':
 848                         server = 1;
 849                         break;
 850
 851                 default:
 852                         lst_print_usage(argv[0]);
 853                         return -1;
 854                 }
 855         }
 856
 857         if (type == 0 || timeout <= 0 || optind != argc) {
 858                 lst_print_usage(argv[0]);
 859                 return -1;
 860         }
 861
 862         if (type == LST_OPC_BATCHCLI && server)
 863                 type = LST_OPC_BATCHSRV;
 864
 865         rc = lst_get_node_count(type, str, &count, &ids);
 866         if (rc < 0) {
 867                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
 868                         (str == NULL) ? "session" : str, strerror(errno));
 869                 return -1;
 870         }
 871
 872         INIT_LIST_HEAD(&head);
 873
 874         rc = lst_alloc_rpcent(&head, count, LST_NAME_SIZE);
 875         if (rc != 0) {
 876                 fprintf(stderr, "Out of memory\n");
 877                 goto out;
 878         }
 879
 880         if (count == 0) {
 881                 fprintf(stdout, "Target %s is empty\n",
 882                         (str == NULL) ? "session" : str);
 883                 goto out;
 884         }
 885
 886         rc = lst_ping_ioctl(str, type, timeout, count, ids, &head);
 887         if (rc == -1) { /* local failure */
 888                 lst_print_error("debug", "Failed to ping %s: %s\n",
 889                                 (str == NULL) ? "session" : str,
 890                                 strerror(errno));
 891                 rc = -1;
 892                 goto out;
 893         }
 894
 895         /* ignore RPC errors and framwork errors */
 896         list_for_each_entry(ent, &head, rpe_link) {
 897                 fprintf(stdout, "\t%s: %s [session: %s id: %s]\n",
 898                         libcfs_id2str(ent->rpe_peer),
 899                         lst_node_state2str(ent->rpe_state),
 900                         (ent->rpe_state == LST_NODE_ACTIVE ||
 901                          ent->rpe_state == LST_NODE_BUSY) ?
 902                                 (ent->rpe_rpc_errno == 0 ?
 903                                         &ent->rpe_payload[0] : "Unknown") :
 904                                 "<NULL>", libcfs_nid2str(ent->rpe_sid.ses_nid));
 905         }
 906
 907 out:
 908         lst_free_rpcent(&head);
 909
 910         if (ids != NULL)
 911                 free(ids);
 912
 913         return rc;
 914
 915 }
 916
 917 int
 918 lst_add_nodes_ioctl (char *name, int count, lnet_process_id_t *ids,
 919                      unsigned *featp, struct list_head *resultp)
 920 {
 921         lstio_group_nodes_args_t args = {0};
 922
 923         args.lstio_grp_key     = session_key;
 924         args.lstio_grp_nmlen   = strlen(name);
 925         args.lstio_grp_namep   = name;
 926         args.lstio_grp_count   = count;
 927         args.lstio_grp_featp   = featp;
 928         args.lstio_grp_idsp    = ids;
 929         args.lstio_grp_resultp = resultp;
 930
 931         return lst_ioctl(LSTIO_NODES_ADD, &args, sizeof(args));
 932 }
 933
 934 int
 935 lst_del_group_ioctl(char *name)
 936 {
 937         lstio_group_del_args_t args = {0};
 938
 939         args.lstio_grp_key   = session_key;
 940         args.lstio_grp_nmlen = strlen(name);
 941         args.lstio_grp_namep = name;
 942
 943         return lst_ioctl(LSTIO_GROUP_DEL, &args, sizeof(args));
 944 }
 945
 946 int
 947 lst_del_group(char *grp_name)
 948 {
 949         int     rc;
 950
 951         rc = lst_del_group_ioctl(grp_name);
 952         if (rc == 0) {
 953                 fprintf(stdout, "Group is deleted\n");
 954                 return 0;
 955         }
 956
 957         if (rc == -1) {
 958                 lst_print_error("group", "Failed to delete group: %s\n",
 959                                 strerror(errno));
 960                 return rc;
 961         }
 962
 963         fprintf(stderr, "Group is deleted with some errors\n");
 964
 965         if (trans_stat.trs_rpc_errno != 0) {
 966                 fprintf(stderr,
 967                         "[RPC] Failed to send %d end session RPCs: %s\n",
 968                         lstcon_rpc_stat_failure(&trans_stat, 0),
 969                         strerror(trans_stat.trs_rpc_errno));
 970         }
 971
 972         if (trans_stat.trs_fwk_errno != 0) {
 973                 fprintf(stderr,
 974                         "[FWK] Failed to end session on %d nodes: %s\n",
 975                 lstcon_sesop_stat_failure(&trans_stat, 0),
 976                 strerror(trans_stat.trs_fwk_errno));
 977         }
 978
 979         return -1;
 980 }
 981
 982 int
 983 lst_add_group_ioctl (char *name)
 984 {
 985         lstio_group_add_args_t args = {0};
 986
 987         args.lstio_grp_key     =  session_key;
 988         args.lstio_grp_nmlen   =  strlen(name);
 989         args.lstio_grp_namep   =  name;
 990
 991         return lst_ioctl(LSTIO_GROUP_ADD, &args, sizeof(args));
 992 }
 993
 994 int
 995 jt_lst_add_group(int argc, char **argv)
 996 {
 997         struct list_head   head;
 998         lnet_process_id_t *ids;
 999         char              *name;
1000         unsigned           feats = session_features;
1001         int                count;
1002         int                rc;
1003         int                i;
1004         bool               nodes_added = false;
1005
1006         if (session_key == 0) {
1007                 fprintf(stderr,
1008                         "Can't find env LST_SESSION or value is not valid\n");
1009                 return -1;
1010         }
1011
1012         if (argc < 3) {
1013                 lst_print_usage(argv[0]);
1014                 return -1;
1015         }
1016
1017         name = argv[1];
1018         if (strlen(name) >= LST_NAME_SIZE) {
1019                 fprintf(stderr, "Name length is limited to %d\n",
1020                         LST_NAME_SIZE - 1);
1021                 return -1;
1022         }
1023
1024         rc = lst_add_group_ioctl(name);
1025         if (rc != 0) {
1026                 lst_print_error("group", "Failed to add group %s: %s\n",
1027                                 name, strerror(errno));
1028                 return -1;
1029         }
1030
1031         INIT_LIST_HEAD(&head);
1032
1033         for (i = 2; i < argc; i++) {
1034                 /* parse address list */
1035                 rc = lst_parse_nids(argv[i], &count, &ids);
1036                 if (rc < 0) {
1037                         fprintf(stderr, "Ignore invalid id list %s\n",
1038                                 argv[i]);
1039                         continue;
1040                 }
1041
1042                 if (count == 0)
1043                         continue;
1044
1045                 rc = lst_alloc_rpcent(&head, count, 0);
1046                 if (rc != 0) {
1047                         fprintf(stderr, "Out of memory\n");
1048                         free(ids);
1049                         rc = -1;
1050                         goto failed;
1051                 }
1052
1053                 rc = lst_add_nodes_ioctl(name, count, ids, &feats, &head);
1054
1055                 free(ids);
1056
1057                 if (rc != 0)
1058                         goto failed;
1059
1060                 fprintf(stdout, "%s are added to session\n", argv[i]);
1061
1062                 nodes_added = true;
1063
1064                 if ((feats & session_features) != session_features) {
1065                         fprintf(stdout,
1066                                 "Warning, this session will run with "
1067                                 "compatible mode because some test nodes "
1068                                 "might not understand these features: %x\n",
1069                                 (~feats & session_features));
1070                 }
1071
1072                 lst_free_rpcent(&head);
1073         }
1074
1075         if (!nodes_added) {
1076                 /*
1077                  * The selftest kernel module expects that a group should
1078                  * have at least one node, since it doesn't make sense for
1079                  * an empty group to be added to a test.
1080                  */
1081                 fprintf(stderr,
1082                         "No nodes added successfully, deleting group %s\n",
1083                         name);
1084                 rc = lst_del_group(name);
1085                 if (rc != 0) {
1086                         fprintf(stderr,
1087                                 "Failed to delete group %s."
1088                                 "  Group is empty.\n", name);
1089                 }
1090         }
1091
1092         return rc;
1093
1094 failed:
1095         if (rc == -1) {
1096                 lst_print_error("group", "Failed to add nodes %s: %s\n",
1097                                 argv[i], strerror(errno));
1098
1099         } else {
1100                 if (trans_stat.trs_fwk_errno == EPROTO) {
1101                         fprintf(stderr,
1102                                 "test nodes might have different LST "
1103                                 "features, please disable some features by "
1104                                 "setting LST_FEATURES\n");
1105                 }
1106
1107                 lst_print_transerr(&head, "create session");
1108         }
1109
1110         lst_free_rpcent(&head);
1111
1112         if (!nodes_added) {
1113                 fprintf(stderr,
1114                         "No nodes added successfully, deleting group %s\n",
1115                         name);
1116                 if (lst_del_group(name) != 0) {
1117                         fprintf(stderr,
1118                                 "Failed to delete group %s."
1119                                 "  Group is empty.\n", name);
1120                 }
1121         }
1122
1123         return rc;
1124 }
1125
1126 int
1127 jt_lst_del_group(int argc, char **argv)
1128 {
1129         int     rc;
1130
1131         if (session_key == 0) {
1132                 fprintf(stderr,
1133                         "Can't find env LST_SESSION or value is not valid\n");
1134                 return -1;
1135         }
1136
1137         if (argc != 2) {
1138                 lst_print_usage(argv[0]);
1139                 return -1;
1140         }
1141
1142         rc = lst_del_group(argv[1]);
1143
1144         return rc;
1145 }
1146
1147 int
1148 lst_update_group_ioctl(int opc, char *name, int clean, int count,
1149                        lnet_process_id_t *ids, struct list_head *resultp)
1150 {
1151         lstio_group_update_args_t args = {0};
1152
1153         args.lstio_grp_key      = session_key;
1154         args.lstio_grp_opc      = opc;
1155         args.lstio_grp_args     = clean;
1156         args.lstio_grp_nmlen    = strlen(name);
1157         args.lstio_grp_namep    = name;
1158         args.lstio_grp_count    = count;
1159         args.lstio_grp_idsp     = ids;
1160         args.lstio_grp_resultp  = resultp;
1161
1162         return lst_ioctl(LSTIO_GROUP_UPDATE, &args, sizeof(args));
1163 }
1164
1165 int
1166 jt_lst_update_group(int argc, char **argv)
1167 {
1168         struct list_head         head;
1169         lnet_process_id_t *ids = NULL;
1170         char              *str = NULL;
1171         char              *grp = NULL;
1172         int                optidx = 0;
1173         int                count = 0;
1174         int                clean = 0;
1175         int                opc = 0;
1176         int                rc;
1177         int                c;
1178
1179         static struct option update_group_opts[] =
1180         {
1181                 {"refresh", no_argument,       0, 'f' },
1182                 {"clean",   required_argument, 0, 'c' },
1183                 {"remove",  required_argument, 0, 'r' },
1184                 {0,         0,                 0,  0  }
1185         };
1186
1187         if (session_key == 0) {
1188                 fprintf(stderr,
1189                         "Can't find env LST_SESSION or value is not valid\n");
1190                 return -1;
1191         }
1192
1193         while (1) {
1194                 c = getopt_long(argc, argv, "fc:r:",
1195                                 update_group_opts, &optidx);
1196
1197                 /* Detect the end of the options. */
1198                 if (c == -1)
1199                         break;
1200
1201                 switch (c) {
1202                 case 'f':
1203                         if (opc != 0) {
1204                                 lst_print_usage(argv[0]);
1205                                 return -1;
1206                         }
1207                         opc = LST_GROUP_REFRESH;
1208                         break;
1209
1210                 case 'r':
1211                         if (opc != 0) {
1212                                 lst_print_usage(argv[0]);
1213                                 return -1;
1214                         }
1215                         opc = LST_GROUP_RMND;
1216                         str = optarg;
1217                         break;
1218
1219                 case 'c':
1220                         clean = lst_node_str2state(optarg);
1221                         if (opc != 0 || clean <= 0) {
1222                                 lst_print_usage(argv[0]);
1223                                 return -1;
1224                         }
1225                         opc = LST_GROUP_CLEAN;
1226                         break;
1227
1228                 default:
1229                         lst_print_usage(argv[0]);
1230                         return -1;
1231                 }
1232         }
1233
1234         /* no OPC or group is specified */
1235         if (opc == 0 || optind != argc - 1) {
1236                 lst_print_usage(argv[0]);
1237                 return -1;
1238         }
1239
1240         grp = argv[optind];
1241
1242         INIT_LIST_HEAD(&head);
1243
1244         if (opc == LST_GROUP_RMND || opc == LST_GROUP_REFRESH) {
1245                 rc = lst_get_node_count(opc == LST_GROUP_RMND ? LST_OPC_NODES :
1246                                                                 LST_OPC_GROUP,
1247                                         opc == LST_GROUP_RMND ? str : grp,
1248                                         &count, &ids);
1249
1250                 if (rc != 0) {
1251                         fprintf(stderr, "Can't get count of nodes from %s: %s\n",
1252                                 opc == LST_GROUP_RMND ? str : grp,
1253                                 strerror(errno));
1254                         return -1;
1255                 }
1256
1257                 rc = lst_alloc_rpcent(&head, count, 0);
1258                 if (rc != 0) {
1259                         fprintf(stderr, "Out of memory\n");
1260                         free(ids);
1261                         return -1;
1262                 }
1263
1264         }
1265
1266         rc = lst_update_group_ioctl(opc, grp, clean, count, ids, &head);
1267
1268         if (ids != NULL)
1269                 free(ids);
1270
1271         if (rc == 0) {
1272                 lst_free_rpcent(&head);
1273                 return 0;
1274         }
1275
1276         if (rc == -1) {
1277                 lst_free_rpcent(&head);
1278                 lst_print_error("group", "Failed to update group: %s\n",
1279                                 strerror(errno));
1280                 return rc;
1281         }
1282
1283         lst_print_transerr(&head, "Updating group");
1284
1285         lst_free_rpcent(&head);
1286
1287         return rc;
1288 }
1289
1290 int
1291 lst_list_group_ioctl(int len, char *name, int idx)
1292 {
1293         lstio_group_list_args_t args = {0};
1294
1295         args.lstio_grp_key   = session_key;
1296         args.lstio_grp_idx   = idx;
1297         args.lstio_grp_nmlen = len;
1298         args.lstio_grp_namep = name;
1299
1300         return lst_ioctl(LSTIO_GROUP_LIST, &args, sizeof(args));
1301 }
1302
1303 int
1304 lst_info_group_ioctl(char *name, lstcon_ndlist_ent_t *gent,
1305                      int *idx, int *count, lstcon_node_ent_t *dents)
1306 {
1307         lstio_group_info_args_t args = {0};
1308
1309         args.lstio_grp_key    = session_key;
1310         args.lstio_grp_nmlen  = strlen(name);
1311         args.lstio_grp_namep  = name;
1312         args.lstio_grp_entp   = gent;
1313         args.lstio_grp_idxp   = idx;
1314         args.lstio_grp_ndentp = count;
1315         args.lstio_grp_dentsp = dents;
1316
1317         return lst_ioctl(LSTIO_GROUP_INFO, &args, sizeof(args));
1318 }
1319
1320 int
1321 lst_list_group_all(void)
1322 {
1323         char  name[LST_NAME_SIZE];
1324         int   rc;
1325         int   i;
1326
1327         /* no group is specified, list name of all groups */
1328         for (i = 0; ; i++) {
1329                 rc = lst_list_group_ioctl(LST_NAME_SIZE, name, i);
1330                 if (rc == 0) {
1331                         fprintf(stdout, "%d) %s\n", i + 1, name);
1332                         continue;
1333                 }
1334
1335                 if (errno == ENOENT)
1336                         break;
1337
1338                 lst_print_error("group", "Failed to list group: %s\n",
1339                                 strerror(errno));
1340                 return -1;
1341         }
1342
1343         fprintf(stdout, "Total %d groups\n", i);
1344
1345         return 0;
1346 }
1347
1348 #define LST_NODES_TITLE "\tACTIVE\tBUSY\tDOWN\tUNKNOWN\tTOTAL\n"
1349
1350 int
1351 jt_lst_list_group(int argc, char **argv)
1352 {
1353         lstcon_ndlist_ent_t  gent;
1354         lstcon_node_ent_t   *dents;
1355         int               optidx  = 0;
1356         int               verbose = 0;
1357         int               active  = 0;
1358         int               busy    = 0;
1359         int               down    = 0;
1360         int               unknown = 0;
1361         int               all     = 0;
1362         int               count;
1363         int               index;
1364         int               i;
1365         int               j;
1366         int               c;
1367         int               rc = 0;
1368
1369         static struct option list_group_opts[] =
1370         {
1371                 {"active",  no_argument, 0, 'a' },
1372                 {"busy",    no_argument, 0, 'b' },
1373                 {"down",    no_argument, 0, 'd' },
1374                 {"unknown", no_argument, 0, 'u' },
1375                 {"all",     no_argument, 0, 'l' },
1376                 {0,         0,           0,  0  }
1377         };
1378
1379         if (session_key == 0) {
1380                 fprintf(stderr,
1381                         "Can't find env LST_SESSION or value is not valid\n");
1382                 return -1;
1383         }
1384
1385         while (1) {
1386                 c = getopt_long(argc, argv, "abdul",
1387                                 list_group_opts, &optidx);
1388
1389                 if (c == -1)
1390                         break;
1391
1392                 switch (c) {
1393                 case 'a':
1394                         verbose = active = 1;
1395                         all = 0;
1396                         break;
1397                 case 'b':
1398                         verbose = busy = 1;
1399                         all = 0;
1400                         break;
1401                 case 'd':
1402                         verbose = down = 1;
1403                         all = 0;
1404                         break;
1405                 case 'u':
1406                         verbose = unknown = 1;
1407                         all = 0;
1408                         break;
1409                 case 'l':
1410                         verbose = all = 1;
1411                         break;
1412                 default:
1413                         lst_print_usage(argv[0]);
1414                         return -1;
1415                 }
1416         }
1417
1418         if (optind == argc) {
1419                 /* no group is specified, list name of all groups */
1420                 rc = lst_list_group_all();
1421
1422                 return rc;
1423         }
1424
1425         if (!verbose)
1426                 fprintf(stdout, LST_NODES_TITLE);
1427
1428         /* list nodes in specified groups */
1429         for (i = optind; i < argc; i++) {
1430                 rc = lst_info_group_ioctl(argv[i], &gent, NULL, NULL, NULL);
1431                 if (rc != 0) {
1432                         if (errno == ENOENT) {
1433                                 rc = 0;
1434                                 break;
1435                         }
1436
1437                         lst_print_error("group", "Failed to list group\n",
1438                                         strerror(errno));
1439                         break;
1440                 }
1441
1442                 if (!verbose) {
1443                         fprintf(stdout, "\t%d\t%d\t%d\t%d\t%d\t%s\n",
1444                                 gent.nle_nactive, gent.nle_nbusy,
1445                                 gent.nle_ndown, gent.nle_nunknown,
1446                                 gent.nle_nnode, argv[i]);
1447                         continue;
1448                 }
1449
1450                 fprintf(stdout, "Group [ %s ]\n", argv[i]);
1451
1452                 if (gent.nle_nnode == 0) {
1453                         fprintf(stdout, "No nodes found [ %s ]\n", argv[i]);
1454                         continue;
1455                 }
1456
1457                 count = gent.nle_nnode;
1458
1459                 dents = malloc(count * sizeof(lstcon_node_ent_t));
1460                 if (dents == NULL) {
1461                         fprintf(stderr, "Failed to malloc: %s\n",
1462                                 strerror(errno));
1463                         return -1;
1464                 }
1465
1466                 index = 0;
1467                 rc = lst_info_group_ioctl(argv[i], &gent, &index, &count, dents);
1468                 if (rc != 0) {
1469                         lst_print_error("group", "Failed to list group: %s\n",
1470                                         strerror(errno));
1471                         free(dents);
1472                         return -1;
1473                 }
1474
1475                 for (j = 0, c = 0; j < count; j++) {
1476                         if (all ||
1477                             ((active  &&  dents[j].nde_state == LST_NODE_ACTIVE) ||
1478                              (busy    &&  dents[j].nde_state == LST_NODE_BUSY)   ||
1479                              (down    &&  dents[j].nde_state == LST_NODE_DOWN)   ||
1480                              (unknown &&  dents[j].nde_state == LST_NODE_UNKNOWN))) {
1481
1482                                 fprintf(stdout, "\t%s: %s\n",
1483                                         libcfs_id2str(dents[j].nde_id),
1484                                         lst_node_state2str(dents[j].nde_state));
1485                                 c++;
1486                         }
1487                 }
1488
1489                 fprintf(stdout, "Total %d nodes [ %s ]\n", c, argv[i]);
1490
1491                 free(dents);
1492         }
1493
1494         return rc;
1495 }
1496
1497 int
1498 lst_stat_ioctl (char *name, int count, lnet_process_id_t *idsp,
1499                 int timeout, struct list_head *resultp)
1500 {
1501         lstio_stat_args_t args = {0};
1502
1503         args.lstio_sta_key     = session_key;
1504         args.lstio_sta_timeout = timeout;
1505         args.lstio_sta_nmlen   = strlen(name);
1506         args.lstio_sta_namep   = name;
1507         args.lstio_sta_count   = count;
1508         args.lstio_sta_idsp    = idsp;
1509         args.lstio_sta_resultp = resultp;
1510
1511         return lst_ioctl(LSTIO_STAT_QUERY, &args, sizeof(args));
1512 }
1513
1514 typedef struct {
1515         struct list_head              srp_link;
1516         int                     srp_count;
1517         char                   *srp_name;
1518         lnet_process_id_t      *srp_ids;
1519         struct list_head              srp_result[2];
1520 } lst_stat_req_param_t;
1521
1522 static void
1523 lst_stat_req_param_free(lst_stat_req_param_t *srp)
1524 {
1525         int     i;
1526
1527         for (i = 0; i < 2; i++)
1528                 lst_free_rpcent(&srp->srp_result[i]);
1529
1530         if (srp->srp_ids != NULL)
1531                 free(srp->srp_ids);
1532
1533         free(srp);
1534 }
1535
1536 static int
1537 lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp, int save_old)
1538 {
1539         lst_stat_req_param_t *srp = NULL;
1540         int                   count = save_old ? 2 : 1;
1541         int                   rc;
1542         int                   i;
1543
1544         srp = malloc(sizeof(*srp));
1545         if (srp == NULL)
1546                 return -ENOMEM;
1547
1548         memset(srp, 0, sizeof(*srp));
1549         INIT_LIST_HEAD(&srp->srp_result[0]);
1550         INIT_LIST_HEAD(&srp->srp_result[1]);
1551
1552         rc = lst_get_node_count(LST_OPC_GROUP, name,
1553                                 &srp->srp_count, NULL);
1554         if (rc != 0 && errno == ENOENT) {
1555                 rc = lst_get_node_count(LST_OPC_NODES, name,
1556                                         &srp->srp_count, &srp->srp_ids);
1557         }
1558
1559         if (rc != 0) {
1560                 fprintf(stderr,
1561                         "Failed to get count of nodes from %s: %s\n",
1562                         name, strerror(errno));
1563                 lst_stat_req_param_free(srp);
1564
1565                 return rc;
1566         }
1567
1568         srp->srp_name = name;
1569
1570         for (i = 0; i < count; i++) {
1571                 rc = lst_alloc_rpcent(&srp->srp_result[i], srp->srp_count,
1572                                       sizeof(sfw_counters_t)  +
1573                                       sizeof(srpc_counters_t) +
1574                                       sizeof(lnet_counters_t));
1575                 if (rc != 0) {
1576                         fprintf(stderr, "Out of memory\n");
1577                         break;
1578                 }
1579         }
1580
1581         if (rc == 0) {
1582                 *srpp = srp;
1583                 return 0;
1584         }
1585
1586         lst_stat_req_param_free(srp);
1587
1588         return rc;
1589 }
1590
1591 typedef struct {
1592         /* TODO */
1593         int foo;
1594 } lst_srpc_stat_result;
1595
1596 #define LST_LNET_AVG    0
1597 #define LST_LNET_MIN    1
1598 #define LST_LNET_MAX    2
1599
1600 typedef struct {
1601         float           lnet_avg_sndrate;
1602         float           lnet_min_sndrate;
1603         float           lnet_max_sndrate;
1604         float           lnet_total_sndrate;
1605
1606         float           lnet_avg_rcvrate;
1607         float           lnet_min_rcvrate;
1608         float           lnet_max_rcvrate;
1609         float           lnet_total_rcvrate;
1610
1611         float           lnet_avg_sndperf;
1612         float           lnet_min_sndperf;
1613         float           lnet_max_sndperf;
1614         float           lnet_total_sndperf;
1615
1616         float           lnet_avg_rcvperf;
1617         float           lnet_min_rcvperf;
1618         float           lnet_max_rcvperf;
1619         float           lnet_total_rcvperf;
1620
1621         int             lnet_stat_count;
1622 } lst_lnet_stat_result_t;
1623
1624 lst_lnet_stat_result_t lnet_stat_result;
1625
1626 static float
1627 lst_lnet_stat_value(int bw, int send, int off)
1628 {
1629         float  *p;
1630
1631         p = bw ? &lnet_stat_result.lnet_avg_sndperf :
1632                  &lnet_stat_result.lnet_avg_sndrate;
1633
1634         if (!send)
1635                 p += 4;
1636
1637         p += off;
1638
1639         return *p;
1640 }
1641
1642 static void
1643 lst_timeval_diff(struct timeval *tv1,
1644                  struct timeval *tv2, struct timeval *df)
1645 {
1646         if (tv1->tv_usec >= tv2->tv_usec) {
1647                 df->tv_sec  = tv1->tv_sec - tv2->tv_sec;
1648                 df->tv_usec = tv1->tv_usec - tv2->tv_usec;
1649                 return;
1650         }
1651
1652         df->tv_sec  = tv1->tv_sec - 1 - tv2->tv_sec;
1653         df->tv_usec = tv1->tv_usec + 1000000 - tv2->tv_usec;
1654
1655         return;
1656 }
1657
1658 static void
1659 lst_cal_lnet_stat(float delta, lnet_counters_t *lnet_new,
1660                   lnet_counters_t *lnet_old, int mbs)
1661 {
1662         float perf;
1663         float rate;
1664         unsigned int unit_divisor;
1665
1666         unit_divisor = (mbs) ? (1000 * 1000) : (1024 * 1024);
1667         perf = (float)(lnet_new->send_length -
1668                        lnet_old->send_length) / unit_divisor / delta;
1669         lnet_stat_result.lnet_total_sndperf += perf;
1670
1671         if (lnet_stat_result.lnet_min_sndperf > perf ||
1672             lnet_stat_result.lnet_min_sndperf == 0)
1673                 lnet_stat_result.lnet_min_sndperf = perf;
1674
1675         if (lnet_stat_result.lnet_max_sndperf < perf)
1676                 lnet_stat_result.lnet_max_sndperf = perf;
1677
1678         perf = (float)(lnet_new->recv_length -
1679                        lnet_old->recv_length) / unit_divisor / delta;
1680         lnet_stat_result.lnet_total_rcvperf += perf;
1681
1682         if (lnet_stat_result.lnet_min_rcvperf > perf ||
1683             lnet_stat_result.lnet_min_rcvperf == 0)
1684                 lnet_stat_result.lnet_min_rcvperf = perf;
1685
1686         if (lnet_stat_result.lnet_max_rcvperf < perf)
1687                 lnet_stat_result.lnet_max_rcvperf = perf;
1688
1689         rate = (lnet_new->send_count - lnet_old->send_count) / delta;
1690         lnet_stat_result.lnet_total_sndrate += rate;
1691
1692         if (lnet_stat_result.lnet_min_sndrate > rate ||
1693             lnet_stat_result.lnet_min_sndrate == 0)
1694                 lnet_stat_result.lnet_min_sndrate = rate;
1695
1696         if (lnet_stat_result.lnet_max_sndrate < rate)
1697                 lnet_stat_result.lnet_max_sndrate = rate;
1698
1699         rate = (lnet_new->recv_count - lnet_old->recv_count) / delta;
1700         lnet_stat_result.lnet_total_rcvrate += rate;
1701
1702         if (lnet_stat_result.lnet_min_rcvrate > rate ||
1703             lnet_stat_result.lnet_min_rcvrate == 0)
1704                 lnet_stat_result.lnet_min_rcvrate = rate;
1705
1706         if (lnet_stat_result.lnet_max_rcvrate < rate)
1707                 lnet_stat_result.lnet_max_rcvrate = rate;
1708
1709         lnet_stat_result.lnet_stat_count++;
1710
1711         lnet_stat_result.lnet_avg_sndrate = lnet_stat_result.lnet_total_sndrate /
1712                                             lnet_stat_result.lnet_stat_count;
1713         lnet_stat_result.lnet_avg_rcvrate = lnet_stat_result.lnet_total_rcvrate /
1714                                             lnet_stat_result.lnet_stat_count;
1715
1716         lnet_stat_result.lnet_avg_sndperf = lnet_stat_result.lnet_total_sndperf /
1717                                             lnet_stat_result.lnet_stat_count;
1718         lnet_stat_result.lnet_avg_rcvperf = lnet_stat_result.lnet_total_rcvperf /
1719                                             lnet_stat_result.lnet_stat_count;
1720 }
1721
1722 static void
1723 lst_print_lnet_stat(char *name, int bwrt, int rdwr, int type, int mbs)
1724 {
1725         int     start1 = 0;
1726         int     end1   = 1;
1727         int     start2 = 0;
1728         int     end2   = 1;
1729         int     i;
1730         int     j;
1731         char   *units;
1732
1733         if (lnet_stat_result.lnet_stat_count == 0)
1734                 return;
1735
1736         units = (mbs) ? "MB/s  " : "MiB/s ";
1737
1738         if (bwrt == 1) /* bw only */
1739                 start1 = 1;
1740
1741         if (bwrt == 2) /* rates only */
1742                 end1 = 0;
1743
1744         if (rdwr == 1) /* recv only */
1745                 start2 = 1;
1746
1747         if (rdwr == 2) /* send only */
1748                 end2 = 0;
1749
1750         for (i = start1; i <= end1; i++) {
1751                 fprintf(stdout, "[LNet %s of %s]\n",
1752                         i == 0 ? "Rates" : "Bandwidth", name);
1753
1754                 for (j = start2; j <= end2; j++) {
1755                         fprintf(stdout, "[%c] ", j == 0 ? 'R' : 'W');
1756
1757                         if ((type & 1) != 0) {
1758                                 fprintf(stdout, i == 0 ? "Avg: %-8.0f RPC/s " :
1759                                                          "Avg: %-8.2f %s",
1760                                         lst_lnet_stat_value(i, j, 0), units);
1761                         }
1762
1763                         if ((type & 2) != 0) {
1764                                 fprintf(stdout, i == 0 ? "Min: %-8.0f RPC/s " :
1765                                                          "Min: %-8.2f %s",
1766                                         lst_lnet_stat_value(i, j, 1), units);
1767                         }
1768
1769                         if ((type & 4) != 0) {
1770                                 fprintf(stdout, i == 0 ? "Max: %-8.0f RPC/s" :
1771                                                          "Max: %-8.2f %s",
1772                                         lst_lnet_stat_value(i, j, 2), units);
1773                         }
1774
1775                         fprintf(stdout, "\n");
1776                 }
1777         }
1778 }
1779
1780 static void
1781 lst_print_stat(char *name, struct list_head *resultp,
1782                int idx, int lnet, int bwrt, int rdwr, int type,
1783                int mbs)
1784 {
1785         struct list_head        tmp[2];
1786         lstcon_rpc_ent_t *new;
1787         lstcon_rpc_ent_t *old;
1788         sfw_counters_t   *sfwk_new;
1789         sfw_counters_t   *sfwk_old;
1790         srpc_counters_t  *srpc_new;
1791         srpc_counters_t  *srpc_old;
1792         lnet_counters_t  *lnet_new;
1793         lnet_counters_t  *lnet_old;
1794         float             delta;
1795         int               errcount = 0;
1796
1797         INIT_LIST_HEAD(&tmp[0]);
1798         INIT_LIST_HEAD(&tmp[1]);
1799
1800         memset(&lnet_stat_result, 0, sizeof(lnet_stat_result));
1801
1802         while (!list_empty(&resultp[idx])) {
1803                 if (list_empty(&resultp[1 - idx])) {
1804                         fprintf(stderr, "Group is changed, re-run stat\n");
1805                         break;
1806                 }
1807
1808                 new = list_entry(resultp[idx].next, lstcon_rpc_ent_t,
1809                                      rpe_link);
1810                 old = list_entry(resultp[1 - idx].next, lstcon_rpc_ent_t,
1811                                      rpe_link);
1812
1813                 /* first time get stats result, can't calculate diff */
1814                 if (new->rpe_peer.nid == LNET_NID_ANY)
1815                         break;
1816
1817                 if (new->rpe_peer.nid != old->rpe_peer.nid ||
1818                     new->rpe_peer.pid != old->rpe_peer.pid) {
1819                         /* Something wrong. i.e, somebody change the group */
1820                         break;
1821                 }
1822
1823                 list_del(&new->rpe_link);
1824                 list_add_tail(&new->rpe_link, &tmp[idx]);
1825
1826                 list_del(&old->rpe_link);
1827                 list_add_tail(&old->rpe_link, &tmp[1 - idx]);
1828
1829                 if (new->rpe_rpc_errno != 0 || new->rpe_fwk_errno != 0 ||
1830                     old->rpe_rpc_errno != 0 || old->rpe_fwk_errno != 0) {
1831                         errcount ++;
1832                         continue;
1833                 }
1834
1835                 sfwk_new = (sfw_counters_t *)&new->rpe_payload[0];
1836                 sfwk_old = (sfw_counters_t *)&old->rpe_payload[0];
1837
1838                 srpc_new = (srpc_counters_t *)((char *)sfwk_new + sizeof(*sfwk_new));
1839                 srpc_old = (srpc_counters_t *)((char *)sfwk_old + sizeof(*sfwk_old));
1840
1841                 lnet_new = (lnet_counters_t *)((char *)srpc_new + sizeof(*srpc_new));
1842                 lnet_old = (lnet_counters_t *)((char *)srpc_old + sizeof(*srpc_old));
1843
1844                 /* Prior to version 2.3, the running_ms field was a counter for
1845                  * the number of running tests.  We are looking at this value
1846                  * to determine if it is a millisecond timestamep (>= 2.3) or a
1847                  * test counter (< 2.3).  The number 500 is being used for this
1848                  * barrier as the test counter should never get this high, and
1849                  * the timestamp should never get this low. */
1850
1851                 if (sfwk_new->running_ms > 500) {
1852                         /* use the timestamp from the remote node, not our
1853                          * rpe_stamp from when we copied up the data out of
1854                          * the kernel */
1855
1856                         delta = (float) (sfwk_new->running_ms -
1857                                         sfwk_old->running_ms) / 1000;
1858                 } else {
1859                         struct timeval    tv;
1860
1861                         lst_timeval_diff(&new->rpe_stamp, &old->rpe_stamp, &tv);
1862                         delta = tv.tv_sec + (float)tv.tv_usec / 1000000;
1863                 }
1864
1865                 if (!lnet) /* TODO */
1866                         continue;
1867
1868                 lst_cal_lnet_stat(delta, lnet_new, lnet_old, mbs);
1869         }
1870
1871         list_splice(&tmp[idx], &resultp[idx]);
1872         list_splice(&tmp[1 - idx], &resultp[1 - idx]);
1873
1874         if (errcount > 0)
1875                 fprintf(stdout, "Failed to stat on %d nodes\n", errcount);
1876
1877         if (!lnet)  /* TODO */
1878                 return;
1879
1880         lst_print_lnet_stat(name, bwrt, rdwr, type, mbs);
1881 }
1882
1883 int
1884 jt_lst_stat(int argc, char **argv)
1885 {
1886         struct list_head        head;
1887         lst_stat_req_param_t *srp;
1888         time_t                last    = 0;
1889         int                   optidx  = 0;
1890         int                   timeout = 5; /* default timeout, 5 sec */
1891         int                   delay   = 5; /* default delay, 5 sec */
1892         int                   count   = -1; /* run forever */
1893         int                   lnet    = 1; /* lnet stat by default */
1894         int                   bwrt    = 0;
1895         int                   rdwr    = 0;
1896         int                   type    = -1;
1897         int                   idx     = 0;
1898         int                   rc;
1899         int                   c;
1900         int                   mbs     = 0; /* report as MB/s */
1901
1902         static struct option stat_opts[] = {
1903                 {"timeout"   , required_argument, 0, 't' },
1904                 {"delay"     , required_argument, 0, 'd' },
1905                 {"count"     , required_argument, 0, 'o' },
1906                 {"lnet"      , no_argument,      0, 'l' },
1907                 {"rpc"       , no_argument,      0, 'c' },
1908                 {"bw"        , no_argument,      0, 'b' },
1909                 {"rate"      , no_argument,      0, 'a' },
1910                 {"read"      , no_argument,      0, 'r' },
1911                 {"write"     , no_argument,      0, 'w' },
1912                 {"avg"       , no_argument,      0, 'g' },
1913                 {"min"       , no_argument,      0, 'n' },
1914                 {"max"       , no_argument,      0, 'x' },
1915                 {"mbs"       , no_argument,      0, 'm' },
1916                 {0,            0,                0,  0  }
1917         };
1918
1919         if (session_key == 0) {
1920                 fprintf(stderr,
1921                         "Can't find env LST_SESSION or value is not valid\n");
1922                 return -1;
1923         }
1924
1925         while (1) {
1926                 c = getopt_long(argc, argv, "t:d:lcbarwgnxm", stat_opts,
1927                                 &optidx);
1928
1929                 if (c == -1)
1930                         break;
1931
1932                 switch (c) {
1933                 case 't':
1934                         timeout = atoi(optarg);
1935                         break;
1936                 case 'd':
1937                         delay = atoi(optarg);
1938                         break;
1939                 case 'o':
1940                         count = atoi(optarg);
1941                         break;
1942                 case 'l':
1943                         lnet = 1;
1944                         break;
1945                 case 'c':
1946                         lnet = 0;
1947                         break;
1948                 case 'b':
1949                         bwrt |= 1;
1950                         break;
1951                 case 'a':
1952                         bwrt |= 2;
1953                         break;
1954                 case 'r':
1955                         rdwr |= 1;
1956                         break;
1957                 case 'w':
1958                         rdwr |= 2;
1959                         break;
1960                 case 'g':
1961                         if (type == -1) {
1962                                 type = 1;
1963                                 break;
1964                         }
1965                         type |= 1;
1966                         break;
1967                 case 'n':
1968                         if (type == -1) {
1969                                 type = 2;
1970                                 break;
1971                         }
1972                         type |= 2;
1973                         break;
1974                 case 'x':
1975                         if (type == -1) {
1976                                 type = 4;
1977                                 break;
1978                         }
1979                         type |= 4;
1980                         break;
1981                 case 'm':
1982                         mbs = 1;
1983                         break;
1984
1985                 default:
1986                         lst_print_usage(argv[0]);
1987                         return -1;
1988                 }
1989         }
1990
1991         if (optind == argc) {
1992                 lst_print_usage(argv[0]);
1993                 return -1;
1994         }
1995
1996         if (timeout <= 0 || delay <= 0) {
1997                 fprintf(stderr, "Invalid timeout or delay value\n");
1998                 return -1;
1999         }
2000
2001         if (count < -1) {
2002             fprintf(stderr, "Invalid count value\n");
2003             return -1;
2004         }
2005
2006         /* extra count to get first data point */
2007         if (count != -1)
2008             count++;
2009
2010         INIT_LIST_HEAD(&head);
2011
2012         while (optind < argc) {
2013                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 1);
2014                 if (rc != 0)
2015                         goto out;
2016
2017                 list_add_tail(&srp->srp_link, &head);
2018         }
2019
2020         do {
2021                 time_t  now = time(NULL);
2022
2023                 if (now - last < delay) {
2024                         sleep(delay - now + last);
2025                         time(&now);
2026                 }
2027                 last = now;
2028
2029                 list_for_each_entry(srp, &head, srp_link) {
2030                         rc = lst_stat_ioctl(srp->srp_name,
2031                                             srp->srp_count, srp->srp_ids,
2032                                             timeout, &srp->srp_result[idx]);
2033                         if (rc == -1) {
2034                                 lst_print_error("stat", "Failed to stat %s: %s\n",
2035                                                 srp->srp_name, strerror(errno));
2036                                 goto out;
2037                         }
2038
2039                         lst_print_stat(srp->srp_name, srp->srp_result,
2040                                        idx, lnet, bwrt, rdwr, type, mbs);
2041
2042                         lst_reset_rpcent(&srp->srp_result[1 - idx]);
2043                 }
2044
2045                 idx = 1 - idx;
2046
2047                 if (count > 0)
2048                         count--;
2049         } while (count == -1 || count > 0);
2050
2051 out:
2052         while (!list_empty(&head)) {
2053                 srp = list_entry(head.next, lst_stat_req_param_t, srp_link);
2054
2055                 list_del(&srp->srp_link);
2056                 lst_stat_req_param_free(srp);
2057         }
2058
2059         return rc;
2060 }
2061
2062 int
2063 jt_lst_show_error(int argc, char **argv)
2064 {
2065         struct list_head            head;
2066         lst_stat_req_param_t *srp;
2067         lstcon_rpc_ent_t     *ent;
2068         sfw_counters_t       *sfwk;
2069         srpc_counters_t      *srpc;
2070         int                   show_rpc = 1;
2071         int                   optidx   = 0;
2072         int                   rc       = 0;
2073         int                   ecount;
2074         int                   c;
2075
2076         static struct option  show_error_opts[] =
2077         {
2078                 {"session", no_argument,       0, 's' },
2079                 {0,         0,                 0,  0  }
2080         };
2081
2082         if (session_key == 0) {
2083                 fprintf(stderr,
2084                         "Can't find env LST_SESSION or value is not valid\n");
2085                 return -1;
2086         }
2087
2088         while (1) {
2089                 c = getopt_long(argc, argv, "s", show_error_opts, &optidx);
2090
2091                 if (c == -1)
2092                         break;
2093
2094                 switch (c) {
2095                 case 's':
2096                         show_rpc  = 0;
2097                         break;
2098
2099                 default:
2100                         lst_print_usage(argv[0]);
2101                         return -1;
2102                 }
2103         }
2104
2105         if (optind == argc) {
2106                 lst_print_usage(argv[0]);
2107                 return -1;
2108         }
2109
2110         INIT_LIST_HEAD(&head);
2111
2112         while (optind < argc) {
2113                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 0);
2114                 if (rc != 0)
2115                         goto out;
2116
2117                 list_add_tail(&srp->srp_link, &head);
2118         }
2119
2120         list_for_each_entry(srp, &head, srp_link) {
2121                 rc = lst_stat_ioctl(srp->srp_name, srp->srp_count,
2122                                     srp->srp_ids, 10, &srp->srp_result[0]);
2123
2124                 if (rc == -1) {
2125                         lst_print_error(srp->srp_name, "Failed to show errors of %s: %s\n",
2126                                         srp->srp_name, strerror(errno));
2127                         goto out;
2128                 }
2129
2130                 fprintf(stdout, "%s:\n", srp->srp_name);
2131
2132                 ecount = 0;
2133
2134                 list_for_each_entry(ent, &srp->srp_result[0], rpe_link) {
2135                         if (ent->rpe_rpc_errno != 0) {
2136                                 ecount ++;
2137                                 fprintf(stderr, "RPC failure, can't show error on %s\n",
2138                                         libcfs_id2str(ent->rpe_peer));
2139                                 continue;
2140                         }
2141
2142                         if (ent->rpe_fwk_errno != 0) {
2143                                 ecount ++;
2144                                 fprintf(stderr, "Framework failure, can't show error on %s\n",
2145                                         libcfs_id2str(ent->rpe_peer));
2146                                 continue;
2147                         }
2148
2149                         sfwk = (sfw_counters_t *)&ent->rpe_payload[0];
2150                         srpc = (srpc_counters_t *)((char *)sfwk + sizeof(*sfwk));
2151
2152                         if (srpc->errors == 0 &&
2153                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2154                                 continue;
2155
2156                         if (!show_rpc  &&
2157                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2158                                 continue;
2159
2160                         ecount ++;
2161
2162                         fprintf(stderr, "%s: [Session %d brw errors, %d ping errors]%c",
2163                                 libcfs_id2str(ent->rpe_peer),
2164                                 sfwk->brw_errors, sfwk->ping_errors,
2165                                 show_rpc  ? ' ' : '\n');
2166
2167                         if (!show_rpc)
2168                                 continue;
2169
2170                         fprintf(stderr, "[RPC: %d errors, %d dropped, %d expired]\n",
2171                                 srpc->errors, srpc->rpcs_dropped, srpc->rpcs_expired);
2172                 }
2173
2174                 fprintf(stdout, "Total %d error nodes in %s\n", ecount, srp->srp_name);
2175         }
2176 out:
2177         while (!list_empty(&head)) {
2178                 srp = list_entry(head.next, lst_stat_req_param_t, srp_link);
2179
2180                 list_del(&srp->srp_link);
2181                 lst_stat_req_param_free(srp);
2182         }
2183
2184         return rc;
2185 }
2186
2187 int
2188 lst_add_batch_ioctl (char *name)
2189 {
2190         lstio_batch_add_args_t args = {0};
2191
2192         args.lstio_bat_key   = session_key;
2193         args.lstio_bat_nmlen = strlen(name);
2194         args.lstio_bat_namep = name;
2195
2196         return lst_ioctl (LSTIO_BATCH_ADD, &args, sizeof(args));
2197 }
2198
2199 int
2200 jt_lst_add_batch(int argc, char **argv)
2201 {
2202         char   *name;
2203         int     rc;
2204
2205         if (session_key == 0) {
2206                 fprintf(stderr,
2207                         "Can't find env LST_SESSION or value is not valid\n");
2208                 return -1;
2209         }
2210
2211         if (argc != 2) {
2212                 lst_print_usage(argv[0]);
2213                 return -1;
2214         }
2215
2216         name = argv[1];
2217         if (strlen(name) >= LST_NAME_SIZE) {
2218                 fprintf(stderr, "Name length is limited to %d\n",
2219                         LST_NAME_SIZE - 1);
2220                 return -1;
2221         }
2222
2223         rc = lst_add_batch_ioctl(name);
2224         if (rc == 0)
2225                 return 0;
2226
2227         lst_print_error("batch", "Failed to create batch: %s\n",
2228                         strerror(errno));
2229
2230         return -1;
2231 }
2232
2233 int
2234 lst_start_batch_ioctl(char *name, int timeout, struct list_head *resultp)
2235 {
2236         lstio_batch_run_args_t args = {0};
2237
2238         args.lstio_bat_key     = session_key;
2239         args.lstio_bat_timeout = timeout;
2240         args.lstio_bat_nmlen   = strlen(name);
2241         args.lstio_bat_namep   = name;
2242         args.lstio_bat_resultp = resultp;
2243
2244         return lst_ioctl(LSTIO_BATCH_START, &args, sizeof(args));
2245 }
2246
2247 int
2248 jt_lst_start_batch(int argc, char **argv)
2249 {
2250         struct list_head        head;
2251         char             *batch;
2252         int               optidx  = 0;
2253         int               timeout = 0;
2254         int               count = 0;
2255         int               rc;
2256         int               c;
2257
2258         static struct option start_batch_opts[] =
2259         {
2260                 {"timeout", required_argument, 0, 't' },
2261                 {0,         0,                 0,  0  }
2262         };
2263
2264         if (session_key == 0) {
2265                 fprintf(stderr,
2266                         "Can't find env LST_SESSION or value is not valid\n");
2267                 return -1;
2268         }
2269
2270         while (1) {
2271                 c = getopt_long(argc, argv, "t:",
2272                                 start_batch_opts, &optidx);
2273
2274                 /* Detect the end of the options. */
2275                 if (c == -1)
2276                         break;
2277
2278                 switch (c) {
2279                 case 't':
2280                         timeout = atoi(optarg);
2281                         break;
2282                 default:
2283                         lst_print_usage(argv[0]);
2284                         return -1;
2285                 }
2286         }
2287
2288         if (optind == argc) {
2289                 batch = LST_DEFAULT_BATCH;
2290
2291         } else if (optind == argc - 1) {
2292                 batch = argv[optind];
2293
2294         } else {
2295                 lst_print_usage(argv[0]);
2296                 return -1;
2297         }
2298
2299         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2300         if (rc != 0) {
2301                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2302                         batch, strerror(errno));
2303                 return -1;
2304         }
2305
2306         INIT_LIST_HEAD(&head);
2307
2308         rc = lst_alloc_rpcent(&head, count, 0);
2309         if (rc != 0) {
2310                 fprintf(stderr, "Out of memory\n");
2311                 return -1;
2312         }
2313
2314         rc = lst_start_batch_ioctl(batch, timeout, &head);
2315
2316         if (rc == 0) {
2317                 fprintf(stdout, "%s is running now\n", batch);
2318                 lst_free_rpcent(&head);
2319                 return 0;
2320         }
2321
2322         if (rc == -1) {
2323                 lst_print_error("batch", "Failed to start batch: %s\n",
2324                                 strerror(errno));
2325                 lst_free_rpcent(&head);
2326                 return rc;
2327         }
2328
2329         lst_print_transerr(&head, "Run batch");
2330
2331         lst_free_rpcent(&head);
2332
2333         return rc;
2334 }
2335
2336 int
2337 lst_stop_batch_ioctl(char *name, int force, struct list_head *resultp)
2338 {
2339         lstio_batch_stop_args_t args = {0};
2340
2341         args.lstio_bat_key     = session_key;
2342         args.lstio_bat_force   = force;
2343         args.lstio_bat_nmlen   = strlen(name);
2344         args.lstio_bat_namep   = name;
2345         args.lstio_bat_resultp = resultp;
2346
2347         return lst_ioctl(LSTIO_BATCH_STOP, &args, sizeof(args));
2348 }
2349
2350 int
2351 jt_lst_stop_batch(int argc, char **argv)
2352 {
2353         struct list_head        head;
2354         char             *batch;
2355         int               force = 0;
2356         int               optidx;
2357         int               count;
2358         int               rc;
2359         int               c;
2360
2361         static struct option stop_batch_opts[] =
2362         {
2363                 {"force",   no_argument,   0, 'f' },
2364                 {0,         0,             0,  0  }
2365         };
2366
2367         if (session_key == 0) {
2368                 fprintf(stderr,
2369                         "Can't find env LST_SESSION or value is not valid\n");
2370                 return -1;
2371         }
2372
2373         while (1) {
2374                 c = getopt_long(argc, argv, "f",
2375                                 stop_batch_opts, &optidx);
2376
2377                 /* Detect the end of the options. */
2378                 if (c == -1)
2379                         break;
2380
2381                 switch (c) {
2382                 case 'f':
2383                         force = 1;
2384                         break;
2385                 default:
2386                         lst_print_usage(argv[0]);
2387                         return -1;
2388                 }
2389         }
2390
2391         if (optind == argc) {
2392                 batch = LST_DEFAULT_BATCH;
2393
2394         } else if (optind == argc - 1) {
2395                 batch = argv[optind];
2396
2397         } else {
2398                 lst_print_usage(argv[0]);
2399                 return -1;
2400         }
2401
2402         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2403         if (rc != 0) {
2404                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2405                         batch, strerror(errno));
2406                 return -1;
2407         }
2408
2409         INIT_LIST_HEAD(&head);
2410
2411         rc = lst_alloc_rpcent(&head, count, 0);
2412         if (rc != 0) {
2413                 fprintf(stderr, "Out of memory\n");
2414                 return -1;
2415         }
2416
2417         rc = lst_stop_batch_ioctl(batch, force, &head);
2418         if (rc != 0)
2419                 goto out;
2420
2421         while (1) {
2422                 lst_reset_rpcent(&head);
2423
2424                 rc = lst_query_batch_ioctl(batch, 0, 0, 30, &head);
2425                 if (rc != 0)
2426                         goto out;
2427
2428                 if (lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0 &&
2429                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0)
2430                         break;
2431
2432                 fprintf(stdout, "%d batch in stopping\n",
2433                         lstcon_tsbqry_stat_run(&trans_stat, 0));
2434                 sleep(1);
2435         }
2436
2437         fprintf(stdout, "Batch is stopped\n");
2438         lst_free_rpcent(&head);
2439
2440         return 0;
2441 out:
2442         if (rc == -1) {
2443                 lst_print_error("batch", "Failed to stop batch: %s\n",
2444                                 strerror(errno));
2445                 lst_free_rpcent(&head);
2446                 return -1;
2447         }
2448
2449         lst_print_transerr(&head, "stop batch");
2450
2451         lst_free_rpcent(&head);
2452
2453         return rc;
2454 }
2455
2456 int
2457 lst_list_batch_ioctl(int len, char *name, int index)
2458 {
2459         lstio_batch_list_args_t args = {0};
2460
2461         args.lstio_bat_key   = session_key;
2462         args.lstio_bat_idx   = index;
2463         args.lstio_bat_nmlen = len;
2464         args.lstio_bat_namep = name;
2465
2466         return lst_ioctl(LSTIO_BATCH_LIST, &args, sizeof(args));
2467 }
2468
2469 int
2470 lst_info_batch_ioctl(char *batch, int test, int server,
2471                      lstcon_test_batch_ent_t *entp, int *idxp,
2472                      int *ndentp, lstcon_node_ent_t *dentsp)
2473 {
2474         lstio_batch_info_args_t args = {0};
2475
2476         args.lstio_bat_key     = session_key;
2477         args.lstio_bat_nmlen   = strlen(batch);
2478         args.lstio_bat_namep   = batch;
2479         args.lstio_bat_server  = server;
2480         args.lstio_bat_testidx = test;
2481         args.lstio_bat_entp    = entp;
2482         args.lstio_bat_idxp    = idxp;
2483         args.lstio_bat_ndentp  = ndentp;
2484         args.lstio_bat_dentsp  = dentsp;
2485
2486         return lst_ioctl(LSTIO_BATCH_INFO, &args, sizeof(args));
2487 }
2488
2489 int
2490 lst_list_batch_all(void)
2491 {
2492         char name[LST_NAME_SIZE];
2493         int  rc;
2494         int  i;
2495
2496         for (i = 0; ; i++) {
2497                 rc = lst_list_batch_ioctl(LST_NAME_SIZE, name, i);
2498                 if (rc == 0) {
2499                         fprintf(stdout, "%d) %s\n", i + 1, name);
2500                         continue;
2501                 }
2502
2503                 if (errno == ENOENT)
2504                         break;
2505
2506                 lst_print_error("batch", "Failed to list batch: %s\n",
2507                                 strerror(errno));
2508                 return rc;
2509         }
2510
2511         fprintf(stdout, "Total %d batches\n", i);
2512
2513         return 0;
2514 }
2515
2516 int
2517 lst_list_tsb_nodes(char *batch, int test, int server,
2518                    int count, int active, int invalid)
2519 {
2520         lstcon_node_ent_t *dents;
2521         int                index = 0;
2522         int                rc;
2523         int                c;
2524         int                i;
2525
2526         if (count == 0)
2527                 return 0;
2528
2529         /* verbose list, show nodes in batch or test */
2530         dents = malloc(count * sizeof(lstcon_node_ent_t));
2531         if (dents == NULL) {
2532                 fprintf(stdout, "Can't allocate memory\n");
2533                 return -1;
2534         }
2535
2536         rc = lst_info_batch_ioctl(batch, test, server,
2537                                   NULL, &index, &count, dents);
2538         if (rc != 0) {
2539                 free(dents);
2540                 lst_print_error((test > 0) ? "test" : "batch",
2541                                 (test > 0) ? "Failed to query test: %s\n" :
2542                                              "Failed to query batch: %s\n",
2543                                 strerror(errno));
2544                 return -1;
2545         }
2546
2547         for (i = 0, c = 0; i < count; i++) {
2548                 if ((!active  && dents[i].nde_state == LST_NODE_ACTIVE) ||
2549                     (!invalid && (dents[i].nde_state == LST_NODE_BUSY  ||
2550                                   dents[i].nde_state == LST_NODE_DOWN  ||
2551                                   dents[i].nde_state == LST_NODE_UNKNOWN)))
2552                         continue;
2553
2554                 fprintf(stdout, "\t%s: %s\n",
2555                         libcfs_id2str(dents[i].nde_id),
2556                         lst_node_state2str(dents[i].nde_state));
2557                 c++;
2558         }
2559
2560         fprintf(stdout, "Total %d nodes\n", c);
2561         free(dents);
2562
2563         return 0;
2564 }
2565
2566 int
2567 jt_lst_list_batch(int argc, char **argv)
2568 {
2569         lstcon_test_batch_ent_t ent;
2570         char                *batch   = NULL;
2571         int                  optidx  = 0;
2572         int                  verbose = 0; /* list nodes in batch or test */
2573         int                  invalid = 0;
2574         int                  active  = 0;
2575         int                  server  = 0;
2576         int                  ntest   = 0;
2577         int                  test    = 0;
2578         int                  c       = 0;
2579         int                  rc;
2580
2581         static struct option list_batch_opts[] =
2582         {
2583                 {"test",    required_argument, 0, 't' },
2584                 {"invalid", no_argument,       0, 'i' },
2585                 {"active",  no_argument,       0, 'a' },
2586                 {"all",     no_argument,       0, 'l' },
2587                 {"server",  no_argument,       0, 's' },
2588                 {0,         0,                 0,  0  }
2589         };
2590
2591         if (session_key == 0) {
2592                 fprintf(stderr,
2593                         "Can't find env LST_SESSION or value is not valid\n");
2594                 return -1;
2595         }
2596
2597         while (1) {
2598                 c = getopt_long(argc, argv, "ailst:",
2599                                 list_batch_opts, &optidx);
2600
2601                 if (c == -1)
2602                         break;
2603
2604                 switch (c) {
2605                 case 'a':
2606                         verbose = active = 1;
2607                         break;
2608                 case 'i':
2609                         verbose = invalid = 1;
2610                         break;
2611                 case 'l':
2612                         verbose = active = invalid = 1;
2613                         break;
2614                 case 's':
2615                         server = 1;
2616                         break;
2617                 case 't':
2618                         test = atoi(optarg);
2619                         ntest = 1;
2620                         break;
2621                 default:
2622                         lst_print_usage(argv[0]);
2623                         return -1;
2624                 }
2625         }
2626
2627         if (optind == argc) {
2628                 /* list all batches */
2629                 rc = lst_list_batch_all();
2630                 return rc;
2631         }
2632
2633         if (ntest == 1 && test <= 0) {
2634                 fprintf(stderr, "Invalid test id, test id starts from 1\n");
2635                 return -1;
2636         }
2637
2638         if (optind != argc - 1) {
2639                 lst_print_usage(argv[0]);
2640                 return -1;
2641         }
2642
2643         batch = argv[optind];
2644
2645 loop:
2646         /* show detail of specified batch or test */
2647         rc = lst_info_batch_ioctl(batch, test, server,
2648                                   &ent, NULL, NULL, NULL);
2649         if (rc != 0) {
2650                 lst_print_error((test > 0) ? "test" : "batch",
2651                                 (test > 0) ? "Failed to query test: %s\n" :
2652                                              "Failed to query batch: %s\n",
2653                                 strerror(errno));
2654                 return -1;
2655         }
2656
2657         if (verbose) {
2658                 /* list nodes in test or batch */
2659                 rc = lst_list_tsb_nodes(batch, test, server,
2660                                         server ? ent.tbe_srv_nle.nle_nnode :
2661                                                  ent.tbe_cli_nle.nle_nnode,
2662                                         active, invalid);
2663                 return rc;
2664         }
2665
2666         /* only show number of hosts in batch or test */
2667         if (test == 0) {
2668                 fprintf(stdout, "Batch: %s Tests: %d State: %d\n",
2669                         batch, ent.u.tbe_batch.bae_ntest,
2670                         ent.u.tbe_batch.bae_state);
2671                 ntest = ent.u.tbe_batch.bae_ntest;
2672                 test = 1; /* starting from test 1 */
2673
2674         } else {
2675                 fprintf(stdout,
2676                         "\tTest %d(%s) (loop: %d, concurrency: %d)\n",
2677                         test, lst_test_type2name(ent.u.tbe_test.tse_type),
2678                         ent.u.tbe_test.tse_loop,
2679                         ent.u.tbe_test.tse_concur);
2680                 ntest --;
2681                 test ++;
2682         }
2683
2684         fprintf(stdout, LST_NODES_TITLE);
2685         fprintf(stdout, "client\t%d\t%d\t%d\t%d\t%d\n"
2686                         "server\t%d\t%d\t%d\t%d\t%d\n",
2687                 ent.tbe_cli_nle.nle_nactive,
2688                 ent.tbe_cli_nle.nle_nbusy,
2689                 ent.tbe_cli_nle.nle_ndown,
2690                 ent.tbe_cli_nle.nle_nunknown,
2691                 ent.tbe_cli_nle.nle_nnode,
2692                 ent.tbe_srv_nle.nle_nactive,
2693                 ent.tbe_srv_nle.nle_nbusy,
2694                 ent.tbe_srv_nle.nle_ndown,
2695                 ent.tbe_srv_nle.nle_nunknown,
2696                 ent.tbe_srv_nle.nle_nnode);
2697
2698         if (ntest != 0)
2699                 goto loop;
2700
2701         return 0;
2702 }
2703
2704 int
2705 lst_query_batch_ioctl(char *batch, int test, int server,
2706                       int timeout, struct list_head *head)
2707 {
2708         lstio_batch_query_args_t args = {0};
2709
2710         args.lstio_bat_key     = session_key;
2711         args.lstio_bat_testidx = test;
2712         args.lstio_bat_client  = !(server);
2713         args.lstio_bat_timeout = timeout;
2714         args.lstio_bat_nmlen   = strlen(batch);
2715         args.lstio_bat_namep   = batch;
2716         args.lstio_bat_resultp = head;
2717
2718         return lst_ioctl(LSTIO_BATCH_QUERY, &args, sizeof(args));
2719 }
2720
2721 void
2722 lst_print_tsb_verbose(struct list_head *head,
2723                       int active, int idle, int error)
2724 {
2725         lstcon_rpc_ent_t *ent;
2726
2727         list_for_each_entry(ent, head, rpe_link) {
2728                 if (ent->rpe_priv[0] == 0 && active)
2729                         continue;
2730
2731                 if (ent->rpe_priv[0] != 0 && idle)
2732                         continue;
2733
2734                 if (ent->rpe_fwk_errno == 0 && error)
2735                         continue;
2736
2737                 fprintf(stdout, "%s [%s]: %s\n",
2738                         libcfs_id2str(ent->rpe_peer),
2739                         lst_node_state2str(ent->rpe_state),
2740                         ent->rpe_rpc_errno != 0 ?
2741                                 strerror(ent->rpe_rpc_errno) :
2742                                 (ent->rpe_priv[0] > 0 ? "Running" : "Idle"));
2743         }
2744 }
2745
2746 int
2747 jt_lst_query_batch(int argc, char **argv)
2748 {
2749         lstcon_test_batch_ent_t ent;
2750         struct list_head              head;
2751         char                   *batch   = NULL;
2752         time_t                  last    = 0;
2753         int                     optidx  = 0;
2754         int                     verbose = 0;
2755         int                     server  = 0;
2756         int                     timeout = 5; /* default 5 seconds */
2757         int                     delay   = 5; /* default 5 seconds */
2758         int                     loop    = 1; /* default 1 loop */
2759         int                     active  = 0;
2760         int                     error   = 0;
2761         int                     idle    = 0;
2762         int                     count   = 0;
2763         int                     test    = 0;
2764         int                     rc      = 0;
2765         int                     c       = 0;
2766         int                     i;
2767
2768         static struct option query_batch_opts[] =
2769         {
2770                 {"timeout", required_argument, 0, 'o' },
2771                 {"delay",   required_argument, 0, 'd' },
2772                 {"loop",    required_argument, 0, 'c' },
2773                 {"test",    required_argument, 0, 't' },
2774                 {"server",  no_argument,       0, 's' },
2775                 {"active",  no_argument,       0, 'a' },
2776                 {"idle",    no_argument,       0, 'i' },
2777                 {"error",   no_argument,       0, 'e' },
2778                 {"all",     no_argument,       0, 'l' },
2779                 {0,         0,                 0,  0  }
2780         };
2781
2782         if (session_key == 0) {
2783                 fprintf(stderr,
2784                         "Can't find env LST_SESSION or value is not valid\n");
2785                 return -1;
2786         }
2787
2788         while (1) {
2789                 c = getopt_long(argc, argv, "o:d:c:t:saiel",
2790                                 query_batch_opts, &optidx);
2791
2792                 /* Detect the end of the options. */
2793                 if (c == -1)
2794                         break;
2795
2796                 switch (c) {
2797                 case 'o':
2798                         timeout = atoi(optarg);
2799                         break;
2800                 case 'd':
2801                         delay = atoi(optarg);
2802                         break;
2803                 case 'c':
2804                         loop = atoi(optarg);
2805                         break;
2806                 case 't':
2807                         test = atoi(optarg);
2808                         break;
2809                 case 's':
2810                         server = 1;
2811                         break;
2812                 case 'a':
2813                         active = verbose = 1;
2814                         break;
2815                 case 'i':
2816                         idle = verbose = 1;
2817                         break;
2818                 case 'e':
2819                         error = verbose = 1;
2820                         break;
2821                 case 'l':
2822                         verbose = 1;
2823                         break;
2824                 default:
2825                         lst_print_usage(argv[0]);
2826                         return -1;
2827                 }
2828         }
2829
2830         if (test < 0 || timeout <= 0 || delay <= 0 || loop <= 0) {
2831                 lst_print_usage(argv[0]);
2832                 return -1;
2833         }
2834
2835         if (optind == argc) {
2836                 batch = LST_DEFAULT_BATCH;
2837
2838         } else if (optind == argc - 1) {
2839                 batch = argv[optind];
2840
2841         } else {
2842                 lst_print_usage(argv[0]);
2843                 return -1;
2844         }
2845
2846
2847         INIT_LIST_HEAD(&head);
2848
2849         if (verbose) {
2850                 rc = lst_info_batch_ioctl(batch, test, server,
2851                                           &ent, NULL, NULL, NULL);
2852                 if (rc != 0) {
2853                         fprintf(stderr, "Failed to query %s [%d]: %s\n",
2854                                 batch, test, strerror(errno));
2855                         return -1;
2856                 }
2857
2858                 count = server ? ent.tbe_srv_nle.nle_nnode :
2859                                  ent.tbe_cli_nle.nle_nnode;
2860                 if (count == 0) {
2861                         fprintf(stdout, "Batch or test is empty\n");
2862                         return 0;
2863                 }
2864         }
2865
2866         rc = lst_alloc_rpcent(&head, count, 0);
2867         if (rc != 0) {
2868                 fprintf(stderr, "Out of memory\n");
2869                 return rc;
2870         }
2871
2872         for (i = 0; i < loop; i++) {
2873                 time_t  now = time(NULL);
2874
2875                 if (now - last < delay) {
2876                         sleep(delay - now + last);
2877                         time(&now);
2878                 }
2879
2880                 last = now;
2881
2882                 rc = lst_query_batch_ioctl(batch, test,
2883                                            server, timeout, &head);
2884                 if (rc == -1) {
2885                         fprintf(stderr, "Failed to query batch: %s\n",
2886                                 strerror(errno));
2887                         break;
2888                 }
2889
2890                 if (verbose) {
2891                         /* Verbose mode */
2892                         lst_print_tsb_verbose(&head, active, idle, error);
2893                         continue;
2894                 }
2895
2896                 fprintf(stdout, "%s [%d] ", batch, test);
2897
2898                 if (lstcon_rpc_stat_failure(&trans_stat, 0) != 0) {
2899                         fprintf(stdout, "%d of %d nodes are unknown, ",
2900                                 lstcon_rpc_stat_failure(&trans_stat, 0),
2901                                 lstcon_rpc_stat_total(&trans_stat, 0));
2902                 }
2903
2904                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
2905                     lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0  &&
2906                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
2907                         fprintf(stdout, "is stopped\n");
2908                         continue;
2909                 }
2910
2911                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
2912                     lstcon_tsbqry_stat_idle(&trans_stat, 0) == 0 &&
2913                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
2914                         fprintf(stdout, "is running\n");
2915                         continue;
2916                 }
2917
2918                 fprintf(stdout, "stopped: %d , running: %d, failed: %d\n",
2919                                 lstcon_tsbqry_stat_idle(&trans_stat, 0),
2920                                 lstcon_tsbqry_stat_run(&trans_stat, 0),
2921                                 lstcon_tsbqry_stat_failure(&trans_stat, 0));
2922         }
2923
2924         lst_free_rpcent(&head);
2925
2926         return rc;
2927 }
2928
2929 int
2930 lst_parse_distribute(char *dstr, int *dist, int *span)
2931 {
2932         *dist = atoi(dstr);
2933         if (*dist <= 0)
2934                 return -1;
2935
2936         dstr = strchr(dstr, ':');
2937         if (dstr == NULL)
2938                 return -1;
2939
2940         *span = atoi(dstr + 1);
2941         if (*span <= 0)
2942                 return -1;
2943
2944         return 0;
2945 }
2946
2947 int
2948 lst_get_bulk_param(int argc, char **argv, lst_test_bulk_param_t *bulk)
2949 {
2950         char   *tok = NULL;
2951         char   *end = NULL;
2952         int     rc  = 0;
2953         int     i   = 0;
2954
2955         bulk->blk_size  = 4096;
2956         bulk->blk_opc   = LST_BRW_READ;
2957         bulk->blk_flags = LST_BRW_CHECK_NONE;
2958         bulk->blk_srv_off = bulk->blk_cli_off = 0;
2959
2960         while (i < argc) {
2961                 if (strcasestr(argv[i], "check=") == argv[i] ||
2962                     strcasestr(argv[i], "c=") == argv[i]) {
2963                         tok = strchr(argv[i], '=') + 1;
2964
2965                         if (strcasecmp(tok, "full") == 0) {
2966                                 bulk->blk_flags = LST_BRW_CHECK_FULL;
2967                         } else if (strcasecmp(tok, "simple") == 0) {
2968                                 bulk->blk_flags = LST_BRW_CHECK_SIMPLE;
2969                         } else {
2970                                 fprintf(stderr, "Unknow flag %s\n", tok);
2971                                 return -1;
2972                         }
2973
2974                 } else if (strcasestr(argv[i], "size=") == argv[i] ||
2975                            strcasestr(argv[i], "s=") == argv[i]) {
2976                         int max_size = sysconf(_SC_PAGESIZE) * LNET_MAX_IOV;
2977
2978                         tok = strchr(argv[i], '=') + 1;
2979
2980                         bulk->blk_size = strtol(tok, &end, 0);
2981                         if (bulk->blk_size <= 0) {
2982                                 fprintf(stderr, "Invalid size %s\n", tok);
2983                                 return -1;
2984                         }
2985
2986                         if (end == NULL)
2987                                 return 0;
2988
2989                         if (*end == 'k' || *end == 'K')
2990                                 bulk->blk_size *= 1024;
2991                         else if (*end == 'm' || *end == 'M')
2992                                 bulk->blk_size *= 1024 * 1024;
2993
2994                         if (bulk->blk_size > max_size) {
2995                                 fprintf(stderr, "Size exceed limitation: %d bytes\n",
2996                                         bulk->blk_size);
2997                                 return -1;
2998                         }
2999
3000                 } else if (strcasestr(argv[i], "off=") == argv[i]) {
3001                         int     off;
3002
3003                         tok = strchr(argv[i], '=') + 1;
3004
3005                         off = strtol(tok, &end, 0);
3006                         /* NB: align with sizeof(__u64) to simplify page
3007                          * checking implementation */
3008                         if (off < 0 || off % sizeof(__u64) != 0) {
3009                                 fprintf(stderr,
3010                                         "Invalid offset %s/%d, it should be "
3011                                         "postive value and multiple of %d\n",
3012                                         tok, off, (int)sizeof(__u64));
3013                                 return -1;
3014                         }
3015
3016                         /* NB: blk_srv_off is reserved so far */
3017                         bulk->blk_cli_off = bulk->blk_srv_off = off;
3018                         if (end == NULL)
3019                                 return 0;
3020
3021                 } else if (strcasecmp(argv[i], "read") == 0 ||
3022                            strcasecmp(argv[i], "r") == 0) {
3023                         bulk->blk_opc = LST_BRW_READ;
3024
3025                 } else if (strcasecmp(argv[i], "write") == 0 ||
3026                            strcasecmp(argv[i], "w") == 0) {
3027                         bulk->blk_opc = LST_BRW_WRITE;
3028
3029                 } else {
3030                         fprintf(stderr, "Unknow parameter: %s\n", argv[i]);
3031                         return -1;
3032                 }
3033
3034                 i++;
3035         }
3036
3037         return rc;
3038 }
3039
3040 int
3041 lst_get_test_param(char *test, int argc, char **argv, void **param, int *plen)
3042 {
3043         lst_test_bulk_param_t *bulk = NULL;
3044         int                    type;
3045
3046         type = lst_test_name2type(test);
3047         if (type < 0) {
3048                 fprintf(stderr, "Unknow test name %s\n", test);
3049                 return -1;
3050         }
3051
3052         switch (type) {
3053         case LST_TEST_PING:
3054                 break;
3055
3056         case LST_TEST_BULK:
3057                 bulk = malloc(sizeof(*bulk));
3058                 if (bulk == NULL) {
3059                         fprintf(stderr, "Out of memory\n");
3060                         return -1;
3061                 }
3062
3063                 memset(bulk, 0, sizeof(*bulk));
3064
3065                 if (lst_get_bulk_param(argc, argv, bulk) != 0) {
3066                         free(bulk);
3067                         return -1;
3068                 }
3069
3070                 *param = bulk;
3071                 *plen  = sizeof(*bulk);
3072
3073                 break;
3074
3075         default:
3076                 break;
3077         }
3078
3079         /* TODO: parse more parameter */
3080         return type;
3081 }
3082
3083 int
3084 lst_add_test_ioctl(char *batch, int type, int loop, int concur,
3085                    int dist, int span, char *sgrp, char *dgrp,
3086                    void *param, int plen, int *retp, struct list_head *resultp)
3087 {
3088         lstio_test_args_t args = {0};
3089
3090         args.lstio_tes_key        = session_key;
3091         args.lstio_tes_bat_nmlen  = strlen(batch);
3092         args.lstio_tes_bat_name   = batch;
3093         args.lstio_tes_type       = type;
3094         args.lstio_tes_oneside    = 0;
3095         args.lstio_tes_loop       = loop;
3096         args.lstio_tes_concur     = concur;
3097         args.lstio_tes_dist       = dist;
3098         args.lstio_tes_span       = span;
3099         args.lstio_tes_sgrp_nmlen = strlen(sgrp);
3100         args.lstio_tes_sgrp_name  = sgrp;
3101         args.lstio_tes_dgrp_nmlen = strlen(dgrp);
3102         args.lstio_tes_dgrp_name  = dgrp;
3103         args.lstio_tes_param_len  = plen;
3104         args.lstio_tes_param      = param;
3105         args.lstio_tes_retp       = retp;
3106         args.lstio_tes_resultp    = resultp;
3107
3108         return lst_ioctl(LSTIO_TEST_ADD, &args, sizeof(args));
3109 }
3110
3111 int
3112 jt_lst_add_test(int argc, char **argv)
3113 {
3114         struct list_head    head;
3115         char         *batch  = NULL;
3116         char         *test   = NULL;
3117         char         *dstr   = NULL;
3118         char         *from   = NULL;
3119         char         *to     = NULL;
3120         void         *param  = NULL;
3121         int           optidx = 0;
3122         int           concur = 1;
3123         int           loop   = -1;
3124         int           dist   = 1;
3125         int           span   = 1;
3126         int           plen   = 0;
3127         int           fcount = 0;
3128         int           tcount = 0;
3129         int           ret    = 0;
3130         int           type;
3131         int           rc;
3132         int           c;
3133
3134         static struct option add_test_opts[] =
3135         {
3136                 {"batch",       required_argument, 0, 'b' },
3137                 {"concurrency", required_argument, 0, 'c' },
3138                 {"distribute",  required_argument, 0, 'd' },
3139                 {"from",        required_argument, 0, 'f' },
3140                 {"to",          required_argument, 0, 't' },
3141                 {"loop",        required_argument, 0, 'l' },
3142                 {0,             0,                 0,  0  }
3143         };
3144
3145         if (session_key == 0) {
3146                 fprintf(stderr,
3147                         "Can't find env LST_SESSION or value is not valid\n");
3148                 return -1;
3149         }
3150
3151         while (1) {
3152                 c = getopt_long(argc, argv, "b:c:d:f:l:t:",
3153                                 add_test_opts, &optidx);
3154
3155                 /* Detect the end of the options. */
3156                 if (c == -1)
3157                         break;
3158
3159                 switch (c) {
3160                 case 'b':
3161                         batch = optarg;
3162                         break;
3163                 case 'c':
3164                         concur = atoi(optarg);
3165                         break;
3166                 case 'd':
3167                         dstr = optarg;
3168                         break;
3169                 case 'f':
3170                         from = optarg;
3171                         break;
3172                 case 'l':
3173                         loop = atoi(optarg);
3174                         break;
3175                 case 't':
3176                         to = optarg;
3177                         break;
3178                 default:
3179                         lst_print_usage(argv[0]);
3180                         return -1;
3181                 }
3182         }
3183
3184         if (optind == argc || from == NULL || to == NULL) {
3185                 lst_print_usage(argv[0]);
3186                 return -1;
3187         }
3188
3189         if (concur <= 0 || concur > LST_MAX_CONCUR) {
3190                 fprintf(stderr, "Invalid concurrency of test: %d\n", concur);
3191                 return -1;
3192         }
3193
3194         if (batch == NULL)
3195                 batch = LST_DEFAULT_BATCH;
3196
3197         if (dstr != NULL) {
3198                 rc = lst_parse_distribute(dstr, &dist, &span);
3199                 if (rc != 0) {
3200                         fprintf(stderr, "Invalid distribution: %s\n", dstr);
3201                         return -1;
3202                 }
3203         }
3204
3205         test = argv[optind++];
3206
3207         argc -= optind;
3208         argv += optind;
3209
3210         type = lst_get_test_param(test, argc, argv, &param, &plen);
3211         if (type < 0) {
3212                 fprintf(stderr, "Failed to add test (%s)\n", test);
3213                 return -1;
3214         }
3215
3216         INIT_LIST_HEAD(&head);
3217
3218         rc = lst_get_node_count(LST_OPC_GROUP, from, &fcount, NULL);
3219         if (rc != 0) {
3220                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3221                         from, strerror(errno));
3222                 goto out;
3223         }
3224
3225         rc = lst_get_node_count(LST_OPC_GROUP, to, &tcount, NULL);
3226         if (rc != 0) {
3227                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3228                         to, strerror(errno));
3229                 goto out;
3230         }
3231
3232         rc = lst_alloc_rpcent(&head, fcount > tcount ? fcount : tcount, 0);
3233         if (rc != 0) {
3234                 fprintf(stderr, "Out of memory\n");
3235                 goto out;
3236         }
3237
3238         rc = lst_add_test_ioctl(batch, type, loop, concur,
3239                                 dist, span, from, to, param, plen, &ret, &head);
3240
3241         if (rc == 0) {
3242                 fprintf(stdout, "Test was added successfully\n");
3243                 if (ret != 0) {
3244                         fprintf(stdout, "Server group contains userland test "
3245                                 "nodes, old version of tcplnd can't accept "
3246                                 "connection request\n");
3247                 }
3248
3249                 goto out;
3250         }
3251
3252         if (rc == -1) {
3253                 lst_print_error("test", "Failed to add test: %s\n",
3254                                 strerror(errno));
3255                 goto out;
3256         }
3257
3258         lst_print_transerr(&head, "add test");
3259 out:
3260         lst_free_rpcent(&head);
3261
3262         if (param != NULL)
3263                 free(param);
3264
3265         return rc;
3266 }
3267
3268 static command_t lst_cmdlist[] = {
3269         {"new_session",         jt_lst_new_session,     NULL,
3270          "Usage: lst new_session [--timeout TIME] [--force] [NAME]"                     },
3271         {"end_session",         jt_lst_end_session,     NULL,
3272          "Usage: lst end_session"                                                       },
3273         {"show_session",        jt_lst_show_session,    NULL,
3274          "Usage: lst show_session"                                                      },
3275         {"ping",                jt_lst_ping ,           NULL,
3276          "Usage: lst ping  [--group NAME] [--batch NAME] [--session] [--nodes IDS]"     },
3277         {"add_group",           jt_lst_add_group,       NULL,
3278          "Usage: lst group NAME IDs [IDs]..."                                           },
3279         {"del_group",           jt_lst_del_group,       NULL,
3280          "Usage: lst del_group NAME"                                                    },
3281         {"update_group",        jt_lst_update_group,    NULL,
3282          "Usage: lst update_group NAME [--clean] [--refresh] [--remove IDs]"            },
3283         {"list_group",          jt_lst_list_group,      NULL,
3284           "Usage: lst list_group [--active] [--busy] [--down] [--unknown] GROUP ..."    },
3285         {"stat",                jt_lst_stat,            NULL,
3286          "Usage: lst stat [--bw] [--rate] [--read] [--write] [--max] [--min] [--avg] "
3287          " [--mbs] [--timeout #] [--delay #] [--count #] GROUP [GROUP]"                 },
3288         {"show_error",          jt_lst_show_error,      NULL,
3289          "Usage: lst show_error NAME | IDS ..."                                         },
3290         {"add_batch",           jt_lst_add_batch,       NULL,
3291          "Usage: lst add_batch NAME"                                                    },
3292         {"run",                 jt_lst_start_batch,     NULL,
3293          "Usage: lst run [--timeout TIME] [NAME]"                                       },
3294         {"stop",                jt_lst_stop_batch,      NULL,
3295          "Usage: lst stop [--force] BATCH_NAME"                                         },
3296         {"list_batch",          jt_lst_list_batch,      NULL,
3297          "Usage: lst list_batch NAME [--test ID] [--server]"                            },
3298         {"query",               jt_lst_query_batch,     NULL,
3299          "Usage: lst query [--test ID] [--server] [--timeout TIME] NAME"                },
3300         {"add_test",            jt_lst_add_test,        NULL,
3301          "Usage: lst add_test [--batch BATCH] [--loop #] [--concurrency #] "
3302          " [--distribute #:#] [--from GROUP] [--to GROUP] TEST..."                      },
3303         {"help",                Parser_help,            0,     "help"                   },
3304         {0,                     0,                      0,      NULL                    }
3305 };
3306
3307 int
3308 lst_initialize(void)
3309 {
3310         char   *key;
3311         char   *feats;
3312
3313         feats = getenv("LST_FEATURES");
3314         if (feats != NULL)
3315                 session_features = strtol(feats, NULL, 16);
3316
3317         if ((session_features & ~LST_FEATS_MASK) != 0) {
3318                 fprintf(stderr,
3319                         "Unsupported session features %x, "
3320                         "only support these features so far: %x\n",
3321                         (session_features & ~LST_FEATS_MASK), LST_FEATS_MASK);
3322                 return -1;
3323         }
3324
3325         key = getenv("LST_SESSION");
3326
3327         if (key == NULL) {
3328                 session_key = 0;
3329                 return 0;
3330         }
3331
3332         session_key = atoi(key);
3333
3334         return 0;
3335 }
3336
3337 int
3338 main(int argc, char **argv)
3339 {
3340         int rc = 0;
3341
3342         setlinebuf(stdout);
3343
3344         rc = lst_initialize();
3345         if (rc < 0)
3346                 goto errorout;
3347
3348         rc = ptl_initialize(argc, argv);
3349         if (rc < 0)
3350                 goto errorout;
3351
3352         Parser_init("lst > ", lst_cmdlist);
3353
3354         if (argc != 1)  {
3355                 rc = Parser_execarg(argc - 1, argv + 1, lst_cmdlist);
3356                 goto errorout;
3357         }
3358
3359         Parser_commands();
3360
3361 errorout:
3362         return rc;
3363 }