lnet/utils/lst.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2012, 2016, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  * Lustre is a trademark of Sun Microsystems, Inc.
  31  *
  32  * lnet/selftest/conctl.c
  33  *
  34  * Author: Liang Zhen <liangzhen@clusterfs.com>
  35  */
  36 #include <errno.h>
  37 #include <getopt.h>
  38 #include <inttypes.h>
  39 #include <pwd.h>
  40 #include <unistd.h>
  41 #include <stdarg.h>
  42 #include <stdio.h>
  43 #include <stdlib.h>
  44 #include <string.h>
  45 #include <sys/ioctl.h>
  46 #include <time.h>
  47
  48 #include <libcfs/util/list.h>
  49 #include <libcfs/util/ioctl.h>
  50 #include <libcfs/util/parser.h>
  51 #include <lnet/lnetctl.h>
  52 #include <lnet/lnetst.h>
  53 #include <lnet/lnet.h>
  54 #include <utils/obdctl.h>
  55
  56 struct lst_sid LST_INVALID_SID = {LNET_NID_ANY, -1};
  57 static struct lst_sid session_id;
  58 static int                 session_key;
  59
  60 /* All nodes running 2.6.50 or later understand feature LST_FEAT_BULK_LEN */
  61 static unsigned         session_features = LST_FEATS_MASK;
  62 static struct lstcon_trans_stat trans_stat;
  63
  64 typedef struct list_string {
  65         struct list_string *lstr_next;
  66         int                 lstr_sz;
  67         char                lstr_str[0];
  68 } lstr_t;
  69
  70 #ifndef offsetof
  71 # define offsetof(typ,memb)     ((unsigned long)((char *)&(((typ *)0)->memb)))
  72 #endif
  73
  74 static int alloc_count = 0;
  75 static int alloc_nob   = 0;
  76
  77 lstr_t *
  78 alloc_lstr(int sz)
  79 {
  80         lstr_t  *lstr = malloc(offsetof(lstr_t, lstr_str[sz]));
  81
  82         if (lstr == NULL) {
  83                 fprintf(stderr, "Can't allocate lstr\n");
  84                 abort();
  85         }
  86
  87         alloc_nob += sz;
  88         alloc_count++;
  89
  90         lstr->lstr_str[0] = 0;
  91         lstr->lstr_sz = sz;
  92         return lstr;
  93 }
  94
  95 void
  96 free_lstr(lstr_t *lstr)
  97 {
  98         alloc_count--;
  99         alloc_nob -= lstr->lstr_sz;
 100         free(lstr);
 101 }
 102
 103 void
 104 free_lstrs(lstr_t **list)
 105 {
 106         lstr_t   *lstr;
 107
 108         while ((lstr = *list) != NULL) {
 109                 *list = lstr->lstr_next;
 110                 free_lstr(lstr);
 111         }
 112 }
 113
 114 void
 115 new_lstrs(lstr_t **list, char *prefix, char *postfix,
 116           int lo, int hi, int stride)
 117 {
 118         int    n1 = strlen(prefix);
 119         int    n2 = strlen(postfix);
 120         int    sz = n1 + 20 + n2 + 1;
 121
 122         do {
 123                 lstr_t *n = alloc_lstr(sz);
 124
 125                 snprintf(n->lstr_str, sz - 1, "%s%u%s",
 126                          prefix, lo, postfix);
 127
 128                 n->lstr_next = *list;
 129                 *list = n;
 130
 131                 lo += stride;
 132         } while (lo <= hi);
 133 }
 134
 135 int
 136 expand_lstr(lstr_t **list, lstr_t *l)
 137 {
 138         int          nob = strlen(l->lstr_str);
 139         char        *b1;
 140         char        *b2;
 141         char        *expr;
 142         char        *sep;
 143         int          x;
 144         int          y;
 145         int          z;
 146         int          n;
 147
 148         b1 = strchr(l->lstr_str, '[');
 149         if (b1 == NULL) {
 150                 l->lstr_next = *list;
 151                 *list = l;
 152                 return 0;
 153         }
 154
 155         b2 = strchr(b1, ']');
 156         if (b2 == NULL || b2 == b1 + 1)
 157                 return -1;
 158
 159         *b1++ = 0;
 160         *b2++ = 0;
 161         expr = b1;
 162         do {
 163
 164                 sep = strchr(expr, ',');
 165                 if (sep != NULL)
 166                         *sep++ = 0;
 167
 168                 nob = strlen(expr);
 169                 n = nob;
 170                 if (sscanf(expr, "%u%n", &x, &n) >= 1 && n == nob) {
 171                         /* simple number */
 172                         new_lstrs(list, l->lstr_str, b2, x, x, 1);
 173                         continue;
 174                 }
 175
 176                 n = nob;
 177                 if (sscanf(expr, "%u-%u%n", &x, &y, &n) >= 2 && n == nob &&
 178                     x < y) {
 179                         /* simple range */
 180                         new_lstrs(list, l->lstr_str, b2, x, y, 1);
 181                         continue;
 182                 }
 183
 184                 n = nob;
 185                 if (sscanf(expr, "%u-%u/%u%n", &x, &y, &z, &n) >= 3 && n == nob &&
 186                     x < y) {
 187                         /* strided range */
 188                         new_lstrs(list, l->lstr_str, b2, x, y, z);
 189                         continue;
 190                 }
 191
 192                 /* syntax error */
 193                 return -1;
 194         } while ((expr = sep) != NULL);
 195
 196         free_lstr(l);
 197
 198         return 1;
 199 }
 200
 201 int
 202 expand_strs(char *str, lstr_t **head)
 203 {
 204         lstr_t  *list = NULL;
 205         lstr_t  *nlist;
 206         lstr_t  *l;
 207         int      rc = 0;
 208         int      expanded;
 209
 210         l = alloc_lstr(strlen(str) + 1);
 211         memcpy(l->lstr_str, str, strlen(str) + 1);
 212         l->lstr_next = NULL;
 213         list = l;
 214
 215         do {
 216                 expanded = 0;
 217                 nlist = NULL;
 218
 219                 while ((l = list) != NULL) {
 220                         list = l->lstr_next;
 221
 222                         rc = expand_lstr(&nlist, l);
 223                         if (rc < 0) {
 224                                 fprintf(stderr, "Syntax error in \"%s\"\n", str);
 225                                 free_lstr(l);
 226                                 break;
 227                         }
 228
 229                         expanded |= rc > 0;
 230                 }
 231
 232                 /* re-order onto 'list' */
 233                 while ((l = nlist) != NULL) {
 234                         nlist = l->lstr_next;
 235                         l->lstr_next = list;
 236                         list = l;
 237                 }
 238
 239         } while (expanded && rc > 0);
 240
 241         if (rc >= 0) {
 242                 *head = list;
 243                 return 0;
 244         }
 245
 246         while ((l = list) != NULL) {
 247                 list = l->lstr_next;
 248
 249                 free_lstr(l);
 250         }
 251         return rc;
 252 }
 253
 254 int
 255 lst_parse_nids(char *str, int *countp, lnet_process_id_t **idspp)
 256 {
 257         lstr_t  *head = NULL;
 258         lstr_t  *l;
 259         int      c = 0;
 260         int      i;
 261         int      rc;
 262
 263         rc = expand_strs(str, &head);
 264         if (rc != 0)
 265                 goto out;
 266
 267         l = head;
 268         while (l != NULL) {
 269                 l = l->lstr_next;
 270                 c++;
 271         }
 272
 273         *idspp = malloc(c * sizeof(lnet_process_id_t));
 274         if (*idspp == NULL) {
 275                 fprintf(stderr, "Out of memory\n");
 276                 rc = -1;
 277         }
 278
 279         *countp = c;
 280 out:
 281         i = 0;
 282         while ((l = head) != NULL) {
 283                 head = l->lstr_next;
 284
 285                 if (rc == 0) {
 286                         (*idspp)[i].nid = libcfs_str2nid(l->lstr_str);
 287                         if ((*idspp)[i].nid == LNET_NID_ANY) {
 288                                 fprintf(stderr, "Invalid nid: %s\n",
 289                                         l->lstr_str);
 290                                 rc = -1;
 291                         }
 292
 293                         (*idspp)[i].pid = LNET_PID_LUSTRE;
 294                         i++;
 295                 }
 296
 297                 free_lstr(l);
 298         }
 299
 300         if (rc == 0)
 301                 return 0;
 302
 303         free(*idspp);
 304         *idspp = NULL;
 305
 306         return rc;
 307 }
 308
 309 char *
 310 lst_node_state2str(int state)
 311 {
 312         if (state == LST_NODE_ACTIVE)
 313                 return "Active";
 314         if (state == LST_NODE_BUSY)
 315                 return "Busy";
 316         if (state == LST_NODE_DOWN)
 317                 return "Down";
 318
 319         return "Unknown";
 320 }
 321
 322 int
 323 lst_node_str2state(char *str)
 324 {
 325         if (strcasecmp(str, "active") == 0)
 326                 return LST_NODE_ACTIVE;
 327         if (strcasecmp(str, "busy") == 0)
 328                 return LST_NODE_BUSY;
 329         if (strcasecmp(str, "down") == 0)
 330                 return LST_NODE_DOWN;
 331         if (strcasecmp(str, "unknown") == 0)
 332                 return LST_NODE_UNKNOWN;
 333         if (strcasecmp(str, "invalid") == 0)
 334                 return (LST_NODE_UNKNOWN | LST_NODE_DOWN | LST_NODE_BUSY);
 335
 336         return -1;
 337 }
 338
 339 char *
 340 lst_test_type2name(int type)
 341 {
 342         if (type == LST_TEST_PING)
 343                 return "ping";
 344         if (type == LST_TEST_BULK)
 345                 return "brw";
 346
 347         return "unknown";
 348 }
 349
 350 int
 351 lst_test_name2type(char *name)
 352 {
 353         if (strcasecmp(name, "ping") == 0)
 354                 return LST_TEST_PING;
 355         if (strcasecmp(name, "brw") == 0)
 356                 return LST_TEST_BULK;
 357
 358         return -1;
 359 }
 360
 361 void
 362 lst_print_usage(char *cmd)
 363 {
 364         Parser_printhelp(cmd);
 365 }
 366
 367 void
 368 lst_print_error(char *sub, const char *def_format, ...)
 369 {
 370         va_list ap;
 371
 372         /* local error returned from kernel */
 373         switch (errno) {
 374         case ESRCH:
 375                 fprintf(stderr, "No session exists\n");
 376                 return;
 377         case ESHUTDOWN:
 378                 fprintf(stderr, "Session is shutting down\n");
 379                 return;
 380         case EACCES:
 381                 fprintf(stderr, "Unmatched session key or not root\n");
 382                 return;
 383         case ENOENT:
 384                 fprintf(stderr, "Can't find %s in current session\n", sub);
 385                 return;
 386         case EINVAL:
 387                 fprintf(stderr, "Invalid parameters list in command line\n");
 388                 return;
 389         case EFAULT:
 390                 fprintf(stderr, "Bad parameter address\n");
 391                 return;
 392         case EEXIST:
 393                 fprintf(stderr, "%s already exists\n", sub);
 394                 return;
 395         default:
 396                 va_start(ap, def_format);
 397                 vfprintf(stderr, def_format, ap);
 398                 va_end(ap);
 399
 400                 return;
 401         }
 402 }
 403
 404 void
 405 lst_free_rpcent(struct list_head *head)
 406 {
 407         struct lstcon_rpc_ent *ent;
 408
 409         while (!list_empty(head)) {
 410                 ent = list_entry(head->next, struct lstcon_rpc_ent, rpe_link);
 411
 412                 list_del(&ent->rpe_link);
 413                 free(ent);
 414         }
 415 }
 416
 417 void
 418 lst_reset_rpcent(struct list_head *head)
 419 {
 420         struct lstcon_rpc_ent *ent;
 421
 422         list_for_each_entry(ent, head, rpe_link) {
 423                 ent->rpe_sid       = LST_INVALID_SID;
 424                 ent->rpe_peer.nid  = LNET_NID_ANY;
 425                 ent->rpe_peer.pid  = LNET_PID_ANY;
 426                 ent->rpe_rpc_errno = ent->rpe_fwk_errno = 0;
 427         }
 428 }
 429
 430 int
 431 lst_alloc_rpcent(struct list_head *head, int count, int offset)
 432 {
 433         struct lstcon_rpc_ent *ent;
 434         int               i;
 435
 436         for (i = 0; i < count; i++) {
 437                 ent = malloc(offsetof(struct lstcon_rpc_ent, rpe_payload[offset]));
 438                 if (ent == NULL) {
 439                         lst_free_rpcent(head);
 440                         return -1;
 441                 }
 442
 443                 memset(ent, 0, offsetof(struct lstcon_rpc_ent, rpe_payload[offset]));
 444
 445                 ent->rpe_sid      = LST_INVALID_SID;
 446                 ent->rpe_peer.nid = LNET_NID_ANY;
 447                 ent->rpe_peer.pid = LNET_PID_ANY;
 448                 list_add(&ent->rpe_link, head);
 449         }
 450
 451         return 0;
 452 }
 453
 454 void
 455 lst_print_transerr(struct list_head *head, char *optstr)
 456 {
 457         struct lstcon_rpc_ent *ent;
 458
 459         list_for_each_entry(ent, head, rpe_link) {
 460                 if (ent->rpe_rpc_errno == 0 && ent->rpe_fwk_errno == 0)
 461                         continue;
 462
 463                 if (ent->rpe_rpc_errno != 0) {
 464                         fprintf(stderr, "%s RPC failed on %s: %s\n",
 465                                 optstr, libcfs_id2str(ent->rpe_peer),
 466                                 strerror(ent->rpe_rpc_errno));
 467                         continue;
 468                 }
 469
 470                 fprintf(stderr, "operation %s failed on %s: %s\n",
 471                         optstr, libcfs_id2str(ent->rpe_peer),
 472                         strerror(ent->rpe_fwk_errno));
 473         }
 474 }
 475
 476 int lst_info_batch_ioctl(char *batch, int test, int server,
 477                         struct lstcon_test_batch_ent *entp, int *idxp,
 478                         int *ndentp, struct lstcon_node_ent *dentsp);
 479
 480 int lst_info_group_ioctl(char *name, struct lstcon_ndlist_ent *gent,
 481                          int *idx, int *count, struct lstcon_node_ent *dents);
 482
 483 int lst_query_batch_ioctl(char *batch, int test, int server,
 484                           int timeout, struct list_head *head);
 485
 486 int
 487 lst_ioctl(unsigned int opc, void *buf, int len)
 488 {
 489         struct libcfs_ioctl_data data;
 490         int    rc;
 491
 492         LIBCFS_IOC_INIT (data);
 493         data.ioc_u32[0]  = opc;
 494         data.ioc_plen1   = len;
 495         data.ioc_pbuf1   = (char *)buf;
 496         data.ioc_plen2   = sizeof(trans_stat);
 497         data.ioc_pbuf2   = (char *)&trans_stat;
 498
 499         memset(&trans_stat, 0, sizeof(trans_stat));
 500
 501         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNETST, &data);
 502
 503         /* local error, no valid RPC result */
 504         if (rc != 0)
 505                 return -1;
 506
 507         /* RPC error */
 508         if (trans_stat.trs_rpc_errno != 0)
 509                 return -2;
 510
 511         /* Framework error */
 512         if (trans_stat.trs_fwk_errno != 0)
 513                 return -3;
 514
 515         return 0;
 516 }
 517
 518 int
 519 lst_new_session_ioctl(char *name, int timeout, int force, struct lst_sid *sid)
 520 {
 521         struct lstio_session_new_args args = { 0 };
 522
 523         args.lstio_ses_key     = session_key;
 524         args.lstio_ses_timeout = timeout;
 525         args.lstio_ses_force   = force;
 526         args.lstio_ses_idp     = sid;
 527         args.lstio_ses_feats   = session_features;
 528         args.lstio_ses_nmlen   = strlen(name);
 529         args.lstio_ses_namep   = name;
 530
 531         return lst_ioctl (LSTIO_SESSION_NEW, &args, sizeof(args));
 532 }
 533
 534 int
 535 jt_lst_new_session(int argc,  char **argv)
 536 {
 537         char  buf[LST_NAME_SIZE];
 538         char *name;
 539         int   optidx  = 0;
 540         int   timeout = 300;
 541         int   force   = 0;
 542         int   c;
 543         int   rc;
 544
 545         static struct option session_opts[] =
 546         {
 547                 {"timeout", required_argument,  0, 't' },
 548                 {"force",   no_argument,        0, 'f' },
 549                 {0,         0,                  0,  0  }
 550         };
 551
 552         if (session_key == 0) {
 553                 fprintf(stderr,
 554                         "Can't find env LST_SESSION or value is not valid\n");
 555                 return -1;
 556         }
 557
 558         while (1) {
 559
 560                 c = getopt_long(argc, argv, "ft:",
 561                                 session_opts, &optidx);
 562
 563                 if (c == -1)
 564                         break;
 565
 566                 switch (c) {
 567                 case 'f':
 568                         force = 1;
 569                         break;
 570                 case 't':
 571                         timeout = atoi(optarg);
 572                         break;
 573                 default:
 574                         lst_print_usage(argv[0]);
 575                         return -1;
 576                 }
 577         }
 578
 579         if (timeout <= 0) {
 580                 fprintf(stderr, "Invalid timeout value\n");
 581                 return -1;
 582         }
 583
 584         if (optind == argc - 1) {
 585                 name = argv[optind ++];
 586                 if (strlen(name) >= LST_NAME_SIZE) {
 587                         fprintf(stderr, "Name size is limited to %d\n",
 588                                 LST_NAME_SIZE - 1);
 589                         return -1;
 590                 }
 591
 592         } else if (optind == argc) {
 593                 char           user[LST_NAME_SIZE];
 594                 char           host[LST_NAME_SIZE];
 595                 struct passwd *pw = getpwuid(getuid());
 596
 597                 if (pw == NULL)
 598                         snprintf(user, sizeof(user), "%d", (int)getuid());
 599                 else
 600                         snprintf(user, sizeof(user), "%s", pw->pw_name);
 601
 602                 rc = gethostname(host, sizeof(host));
 603                 if (rc != 0)
 604                         snprintf(host, sizeof(host), "unknown_host");
 605
 606                 snprintf(buf, sizeof(buf), "%s@%s", user, host);
 607                 name = buf;
 608
 609         } else {
 610                 lst_print_usage(argv[0]);
 611                 return -1;
 612         }
 613
 614         rc = lst_new_session_ioctl(name, timeout, force, &session_id);
 615         if (rc != 0) {
 616                 lst_print_error("session", "Failed to create session: %s\n",
 617                                 strerror(errno));
 618                 return rc;
 619         }
 620
 621         fprintf(stdout, "SESSION: %s FEATURES: %x TIMEOUT: %d FORCE: %s\n",
 622                 name, session_features, timeout, force ? "Yes" : "No");
 623         return 0;
 624 }
 625
 626 int
 627 lst_session_info_ioctl(char *name, int len, int *key, unsigned *featp,
 628                        struct lst_sid *sid, struct lstcon_ndlist_ent *ndinfo)
 629 {
 630         struct lstio_session_info_args args = { 0 };
 631
 632         args.lstio_ses_idp     = sid;
 633         args.lstio_ses_keyp    = key;
 634         args.lstio_ses_featp   = featp;
 635         args.lstio_ses_ndinfo  = ndinfo;
 636         args.lstio_ses_nmlen   = len;
 637         args.lstio_ses_namep   = name;
 638
 639         return lst_ioctl(LSTIO_SESSION_INFO, &args, sizeof(args));
 640 }
 641
 642 int
 643 jt_lst_show_session(int argc, char **argv)
 644 {
 645         struct lstcon_ndlist_ent ndinfo;
 646         struct lst_sid sid;
 647         char                name[LST_NAME_SIZE];
 648         unsigned            feats;
 649         int                 key;
 650         int                 rc;
 651
 652         rc = lst_session_info_ioctl(name, sizeof(name), &key,
 653                                     &feats, &sid, &ndinfo);
 654
 655         if (rc != 0) {
 656                 lst_print_error("session", "Failed to show session: %s\n",
 657                                 strerror(errno));
 658                 return -1;
 659         }
 660
 661         fprintf(stdout, "%s ID: %ju@%s, KEY: %d FEATURES: %x NODES: %d\n",
 662                 name, (uintmax_t)sid.ses_stamp, libcfs_nid2str(sid.ses_nid),
 663                 key, feats, ndinfo.nle_nnode);
 664
 665         return 0;
 666 }
 667
 668 int
 669 lst_end_session_ioctl(void)
 670 {
 671         struct lstio_session_end_args args = { 0 };
 672
 673         args.lstio_ses_key = session_key;
 674         return lst_ioctl(LSTIO_SESSION_END, &args, sizeof(args));
 675 }
 676
 677 int
 678 jt_lst_end_session(int argc, char **argv)
 679 {
 680         int             rc;
 681
 682         if (session_key == 0) {
 683                 fprintf(stderr,
 684                         "Can't find env LST_SESSION or value is not valid\n");
 685                 return -1;
 686         }
 687
 688         rc = lst_end_session_ioctl();
 689
 690         if (rc == 0) {
 691                 fprintf(stdout, "session is ended\n");
 692                 return 0;
 693         }
 694
 695         if (rc == -1) {
 696                 lst_print_error("session", "Failed to end session: %s\n",
 697                                 strerror(errno));
 698                 return rc;
 699         }
 700
 701         if (trans_stat.trs_rpc_errno != 0) {
 702                 fprintf(stderr,
 703                         "[RPC] Failed to send %d session RPCs: %s\n",
 704                         lstcon_rpc_stat_failure(&trans_stat, 0),
 705                         strerror(trans_stat.trs_rpc_errno));
 706         }
 707
 708         if (trans_stat.trs_fwk_errno != 0) {
 709                 fprintf(stderr,
 710                         "[FWK] Failed to end session on %d nodes: %s\n",
 711                         lstcon_sesop_stat_failure(&trans_stat, 0),
 712                         strerror(trans_stat.trs_fwk_errno));
 713         }
 714
 715         return rc;
 716 }
 717
 718 int
 719 lst_ping_ioctl(char *str, int type, int timeout,
 720                int count, lnet_process_id_t *ids, struct list_head *head)
 721 {
 722         struct lstio_debug_args args = { 0 };
 723
 724         args.lstio_dbg_key     = session_key;
 725         args.lstio_dbg_type    = type;
 726         args.lstio_dbg_flags   = 0;
 727         args.lstio_dbg_timeout = timeout;
 728         args.lstio_dbg_nmlen   = (str == NULL) ? 0: strlen(str);
 729         args.lstio_dbg_namep   = str;
 730         args.lstio_dbg_count   = count;
 731         args.lstio_dbg_idsp    = ids;
 732         args.lstio_dbg_resultp = head;
 733
 734         return lst_ioctl (LSTIO_DEBUG, &args, sizeof(args));
 735 }
 736
 737 int
 738 lst_get_node_count(int type, char *str, int *countp, lnet_process_id_t **idspp)
 739 {
 740         char                    buf[LST_NAME_SIZE];
 741         struct lstcon_test_batch_ent ent;
 742         struct lstcon_ndlist_ent    *entp = &ent.tbe_cli_nle;
 743         struct lst_sid sid;
 744         unsigned                feats;
 745         int                     key;
 746         int                     rc;
 747
 748         switch (type) {
 749         case LST_OPC_SESSION:
 750                 rc = lst_session_info_ioctl(buf, LST_NAME_SIZE,
 751                                             &key, &feats, &sid, entp);
 752                 break;
 753
 754         case LST_OPC_BATCHSRV:
 755                 entp = &ent.tbe_srv_nle;
 756         case LST_OPC_BATCHCLI:
 757                 rc = lst_info_batch_ioctl(str, 0, 0, &ent, NULL, NULL, NULL);
 758                 break;
 759
 760         case LST_OPC_GROUP:
 761                 rc = lst_info_group_ioctl(str, entp, NULL, NULL, NULL);
 762                 break;
 763
 764         case LST_OPC_NODES:
 765                 rc = lst_parse_nids(str, &entp->nle_nnode, idspp) < 0 ? -1 : 0;
 766                 break;
 767
 768         default:
 769                 rc = -1;
 770                 break;
 771         }
 772
 773         if (rc == 0)
 774                 *countp = entp->nle_nnode;
 775
 776         return rc;
 777 }
 778
 779 int
 780 jt_lst_ping(int argc,  char **argv)
 781 {
 782         struct list_head   head;
 783         lnet_process_id_t *ids = NULL;
 784         struct lstcon_rpc_ent  *ent = NULL;
 785         char              *str = NULL;
 786         int                optidx  = 0;
 787         int                server  = 0;
 788         int                timeout = 5;
 789         int                count   = 0;
 790         int                type    = 0;
 791         int                rc      = 0;
 792         int                c;
 793
 794         static struct option ping_opts[] =
 795         {
 796                 {"session", no_argument,       0, 's' },
 797                 {"server",  no_argument,       0, 'v' },
 798                 {"batch",   required_argument, 0, 'b' },
 799                 {"group",   required_argument, 0, 'g' },
 800                 {"nodes",   required_argument, 0, 'n' },
 801                 {"timeout", required_argument, 0, 't' },
 802                 {0,         0,                 0,  0  }
 803         };
 804
 805         if (session_key == 0) {
 806                 fprintf(stderr,
 807                         "Can't find env LST_SESSION or value is not valid\n");
 808                 return -1;
 809         }
 810
 811         while (1) {
 812
 813                 c = getopt_long(argc, argv, "g:b:n:t:sv",
 814                                 ping_opts, &optidx);
 815
 816                 if (c == -1)
 817                         break;
 818
 819                 switch (c) {
 820                 case 's':
 821                         type = LST_OPC_SESSION;
 822                         break;
 823
 824                 case 'g':
 825                         type = LST_OPC_GROUP;
 826                         str = optarg;
 827                         break;
 828
 829                 case 'b':
 830                         type = LST_OPC_BATCHCLI;
 831                         str = optarg;
 832                         break;
 833
 834                 case 'n':
 835                         type = LST_OPC_NODES;
 836                         str = optarg;
 837                         break;
 838
 839                 case 't':
 840                         timeout = atoi(optarg);
 841                         break;
 842
 843                 case 'v':
 844                         server = 1;
 845                         break;
 846
 847                 default:
 848                         lst_print_usage(argv[0]);
 849                         return -1;
 850                 }
 851         }
 852
 853         if (type == 0 || timeout <= 0 || optind != argc) {
 854                 lst_print_usage(argv[0]);
 855                 return -1;
 856         }
 857
 858         if (type == LST_OPC_BATCHCLI && server)
 859                 type = LST_OPC_BATCHSRV;
 860
 861         rc = lst_get_node_count(type, str, &count, &ids);
 862         if (rc < 0) {
 863                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
 864                         (str == NULL) ? "session" : str, strerror(errno));
 865                 return -1;
 866         }
 867
 868         INIT_LIST_HEAD(&head);
 869
 870         rc = lst_alloc_rpcent(&head, count, LST_NAME_SIZE);
 871         if (rc != 0) {
 872                 fprintf(stderr, "Out of memory\n");
 873                 goto out;
 874         }
 875
 876         if (count == 0) {
 877                 fprintf(stdout, "Target %s is empty\n",
 878                         (str == NULL) ? "session" : str);
 879                 goto out;
 880         }
 881
 882         rc = lst_ping_ioctl(str, type, timeout, count, ids, &head);
 883         if (rc == -1) { /* local failure */
 884                 lst_print_error("debug", "Failed to ping %s: %s\n",
 885                                 (str == NULL) ? "session" : str,
 886                                 strerror(errno));
 887                 rc = -1;
 888                 goto out;
 889         }
 890
 891         /* ignore RPC errors and framwork errors */
 892         list_for_each_entry(ent, &head, rpe_link) {
 893                 fprintf(stdout, "\t%s: %s [session: %s id: %s]\n",
 894                         libcfs_id2str(ent->rpe_peer),
 895                         lst_node_state2str(ent->rpe_state),
 896                         (ent->rpe_state == LST_NODE_ACTIVE ||
 897                          ent->rpe_state == LST_NODE_BUSY) ?
 898                                 (ent->rpe_rpc_errno == 0 ?
 899                                         &ent->rpe_payload[0] : "Unknown") :
 900                                 "<NULL>", libcfs_nid2str(ent->rpe_sid.ses_nid));
 901         }
 902
 903 out:
 904         lst_free_rpcent(&head);
 905
 906         if (ids != NULL)
 907                 free(ids);
 908
 909         return rc;
 910
 911 }
 912
 913 int
 914 lst_add_nodes_ioctl(char *name, int count, lnet_process_id_t *ids,
 915                     unsigned *featp, struct list_head *resultp)
 916 {
 917         struct lstio_group_nodes_args args = { 0 };
 918
 919         args.lstio_grp_key     = session_key;
 920         args.lstio_grp_nmlen   = strlen(name);
 921         args.lstio_grp_namep   = name;
 922         args.lstio_grp_count   = count;
 923         args.lstio_grp_featp   = featp;
 924         args.lstio_grp_idsp    = ids;
 925         args.lstio_grp_resultp = resultp;
 926
 927         return lst_ioctl(LSTIO_NODES_ADD, &args, sizeof(args));
 928 }
 929
 930 int
 931 lst_del_group_ioctl(char *name)
 932 {
 933         struct lstio_group_del_args args = { 0 };
 934
 935         args.lstio_grp_key   = session_key;
 936         args.lstio_grp_nmlen = strlen(name);
 937         args.lstio_grp_namep = name;
 938
 939         return lst_ioctl(LSTIO_GROUP_DEL, &args, sizeof(args));
 940 }
 941
 942 int
 943 lst_del_group(char *grp_name)
 944 {
 945         int     rc;
 946
 947         rc = lst_del_group_ioctl(grp_name);
 948         if (rc == 0) {
 949                 fprintf(stdout, "Group is deleted\n");
 950                 return 0;
 951         }
 952
 953         if (rc == -1) {
 954                 lst_print_error("group", "Failed to delete group: %s\n",
 955                                 strerror(errno));
 956                 return rc;
 957         }
 958
 959         fprintf(stderr, "Group is deleted with some errors\n");
 960
 961         if (trans_stat.trs_rpc_errno != 0) {
 962                 fprintf(stderr,
 963                         "[RPC] Failed to send %d end session RPCs: %s\n",
 964                         lstcon_rpc_stat_failure(&trans_stat, 0),
 965                         strerror(trans_stat.trs_rpc_errno));
 966         }
 967
 968         if (trans_stat.trs_fwk_errno != 0) {
 969                 fprintf(stderr,
 970                         "[FWK] Failed to end session on %d nodes: %s\n",
 971                 lstcon_sesop_stat_failure(&trans_stat, 0),
 972                 strerror(trans_stat.trs_fwk_errno));
 973         }
 974
 975         return -1;
 976 }
 977
 978 int
 979 lst_add_group_ioctl(char *name)
 980 {
 981         struct lstio_group_add_args args = { 0 };
 982
 983         args.lstio_grp_key     =  session_key;
 984         args.lstio_grp_nmlen   =  strlen(name);
 985         args.lstio_grp_namep   =  name;
 986
 987         return lst_ioctl(LSTIO_GROUP_ADD, &args, sizeof(args));
 988 }
 989
 990 int
 991 jt_lst_add_group(int argc, char **argv)
 992 {
 993         struct list_head   head;
 994         lnet_process_id_t *ids;
 995         char              *name;
 996         unsigned           feats = session_features;
 997         int                count;
 998         int                rc;
 999         int                i;
1000         bool               nodes_added = false;
1001
1002         if (session_key == 0) {
1003                 fprintf(stderr,
1004                         "Can't find env LST_SESSION or value is not valid\n");
1005                 return -1;
1006         }
1007
1008         if (argc < 3) {
1009                 lst_print_usage(argv[0]);
1010                 return -1;
1011         }
1012
1013         name = argv[1];
1014         if (strlen(name) >= LST_NAME_SIZE) {
1015                 fprintf(stderr, "Name length is limited to %d\n",
1016                         LST_NAME_SIZE - 1);
1017                 return -1;
1018         }
1019
1020         rc = lst_add_group_ioctl(name);
1021         if (rc != 0) {
1022                 lst_print_error("group", "Failed to add group %s: %s\n",
1023                                 name, strerror(errno));
1024                 return -1;
1025         }
1026
1027         INIT_LIST_HEAD(&head);
1028
1029         for (i = 2; i < argc; i++) {
1030                 /* parse address list */
1031                 rc = lst_parse_nids(argv[i], &count, &ids);
1032                 if (rc < 0) {
1033                         fprintf(stderr, "Ignore invalid id list %s\n",
1034                                 argv[i]);
1035                         continue;
1036                 }
1037
1038                 if (count == 0)
1039                         continue;
1040
1041                 rc = lst_alloc_rpcent(&head, count, 0);
1042                 if (rc != 0) {
1043                         fprintf(stderr, "Out of memory\n");
1044                         free(ids);
1045                         rc = -1;
1046                         goto failed;
1047                 }
1048
1049                 rc = lst_add_nodes_ioctl(name, count, ids, &feats, &head);
1050
1051                 free(ids);
1052
1053                 if (rc != 0)
1054                         goto failed;
1055
1056                 fprintf(stdout, "%s are added to session\n", argv[i]);
1057
1058                 nodes_added = true;
1059
1060                 if ((feats & session_features) != session_features) {
1061                         fprintf(stdout,
1062                                 "Warning, this session will run with "
1063                                 "compatible mode because some test nodes "
1064                                 "might not understand these features: %x\n",
1065                                 (~feats & session_features));
1066                 }
1067
1068                 lst_free_rpcent(&head);
1069         }
1070
1071         if (!nodes_added) {
1072                 /*
1073                  * The selftest kernel module expects that a group should
1074                  * have at least one node, since it doesn't make sense for
1075                  * an empty group to be added to a test.
1076                  */
1077                 fprintf(stderr,
1078                         "No nodes added successfully, deleting group %s\n",
1079                         name);
1080                 rc = lst_del_group(name);
1081                 if (rc != 0) {
1082                         fprintf(stderr,
1083                                 "Failed to delete group %s."
1084                                 "  Group is empty.\n", name);
1085                 }
1086         }
1087
1088         return rc;
1089
1090 failed:
1091         if (rc == -1) {
1092                 lst_print_error("group", "Failed to add nodes %s: %s\n",
1093                                 argv[i], strerror(errno));
1094
1095         } else {
1096                 if (trans_stat.trs_fwk_errno == EPROTO) {
1097                         fprintf(stderr,
1098                                 "test nodes might have different LST "
1099                                 "features, please disable some features by "
1100                                 "setting LST_FEATURES\n");
1101                 }
1102
1103                 lst_print_transerr(&head, "create session");
1104         }
1105
1106         lst_free_rpcent(&head);
1107
1108         if (!nodes_added) {
1109                 fprintf(stderr,
1110                         "No nodes added successfully, deleting group %s\n",
1111                         name);
1112                 if (lst_del_group(name) != 0) {
1113                         fprintf(stderr,
1114                                 "Failed to delete group %s."
1115                                 "  Group is empty.\n", name);
1116                 }
1117         }
1118
1119         return rc;
1120 }
1121
1122 int
1123 jt_lst_del_group(int argc, char **argv)
1124 {
1125         int     rc;
1126
1127         if (session_key == 0) {
1128                 fprintf(stderr,
1129                         "Can't find env LST_SESSION or value is not valid\n");
1130                 return -1;
1131         }
1132
1133         if (argc != 2) {
1134                 lst_print_usage(argv[0]);
1135                 return -1;
1136         }
1137
1138         rc = lst_del_group(argv[1]);
1139
1140         return rc;
1141 }
1142
1143 int
1144 lst_update_group_ioctl(int opc, char *name, int clean, int count,
1145                        lnet_process_id_t *ids, struct list_head *resultp)
1146 {
1147         struct lstio_group_update_args args = { 0 };
1148
1149         args.lstio_grp_key      = session_key;
1150         args.lstio_grp_opc      = opc;
1151         args.lstio_grp_args     = clean;
1152         args.lstio_grp_nmlen    = strlen(name);
1153         args.lstio_grp_namep    = name;
1154         args.lstio_grp_count    = count;
1155         args.lstio_grp_idsp     = ids;
1156         args.lstio_grp_resultp  = resultp;
1157
1158         return lst_ioctl(LSTIO_GROUP_UPDATE, &args, sizeof(args));
1159 }
1160
1161 int
1162 jt_lst_update_group(int argc, char **argv)
1163 {
1164         struct list_head         head;
1165         lnet_process_id_t *ids = NULL;
1166         char              *str = NULL;
1167         char              *grp = NULL;
1168         int                optidx = 0;
1169         int                count = 0;
1170         int                clean = 0;
1171         int                opc = 0;
1172         int                rc;
1173         int                c;
1174
1175         static struct option update_group_opts[] =
1176         {
1177                 {"refresh", no_argument,       0, 'f' },
1178                 {"clean",   required_argument, 0, 'c' },
1179                 {"remove",  required_argument, 0, 'r' },
1180                 {0,         0,                 0,  0  }
1181         };
1182
1183         if (session_key == 0) {
1184                 fprintf(stderr,
1185                         "Can't find env LST_SESSION or value is not valid\n");
1186                 return -1;
1187         }
1188
1189         while (1) {
1190                 c = getopt_long(argc, argv, "fc:r:",
1191                                 update_group_opts, &optidx);
1192
1193                 /* Detect the end of the options. */
1194                 if (c == -1)
1195                         break;
1196
1197                 switch (c) {
1198                 case 'f':
1199                         if (opc != 0) {
1200                                 lst_print_usage(argv[0]);
1201                                 return -1;
1202                         }
1203                         opc = LST_GROUP_REFRESH;
1204                         break;
1205
1206                 case 'r':
1207                         if (opc != 0) {
1208                                 lst_print_usage(argv[0]);
1209                                 return -1;
1210                         }
1211                         opc = LST_GROUP_RMND;
1212                         str = optarg;
1213                         break;
1214
1215                 case 'c':
1216                         clean = lst_node_str2state(optarg);
1217                         if (opc != 0 || clean <= 0) {
1218                                 lst_print_usage(argv[0]);
1219                                 return -1;
1220                         }
1221                         opc = LST_GROUP_CLEAN;
1222                         break;
1223
1224                 default:
1225                         lst_print_usage(argv[0]);
1226                         return -1;
1227                 }
1228         }
1229
1230         /* no OPC or group is specified */
1231         if (opc == 0 || optind != argc - 1) {
1232                 lst_print_usage(argv[0]);
1233                 return -1;
1234         }
1235
1236         grp = argv[optind];
1237
1238         INIT_LIST_HEAD(&head);
1239
1240         if (opc == LST_GROUP_RMND || opc == LST_GROUP_REFRESH) {
1241                 rc = lst_get_node_count(opc == LST_GROUP_RMND ? LST_OPC_NODES :
1242                                                                 LST_OPC_GROUP,
1243                                         opc == LST_GROUP_RMND ? str : grp,
1244                                         &count, &ids);
1245
1246                 if (rc != 0) {
1247                         fprintf(stderr, "Can't get count of nodes from %s: %s\n",
1248                                 opc == LST_GROUP_RMND ? str : grp,
1249                                 strerror(errno));
1250                         return -1;
1251                 }
1252
1253                 rc = lst_alloc_rpcent(&head, count, 0);
1254                 if (rc != 0) {
1255                         fprintf(stderr, "Out of memory\n");
1256                         free(ids);
1257                         return -1;
1258                 }
1259
1260         }
1261
1262         rc = lst_update_group_ioctl(opc, grp, clean, count, ids, &head);
1263
1264         if (ids != NULL)
1265                 free(ids);
1266
1267         if (rc == 0) {
1268                 lst_free_rpcent(&head);
1269                 return 0;
1270         }
1271
1272         if (rc == -1) {
1273                 lst_free_rpcent(&head);
1274                 lst_print_error("group", "Failed to update group: %s\n",
1275                                 strerror(errno));
1276                 return rc;
1277         }
1278
1279         lst_print_transerr(&head, "Updating group");
1280
1281         lst_free_rpcent(&head);
1282
1283         return rc;
1284 }
1285
1286 int
1287 lst_list_group_ioctl(int len, char *name, int idx)
1288 {
1289         struct lstio_group_list_args args = { 0 };
1290
1291         args.lstio_grp_key   = session_key;
1292         args.lstio_grp_idx   = idx;
1293         args.lstio_grp_nmlen = len;
1294         args.lstio_grp_namep = name;
1295
1296         return lst_ioctl(LSTIO_GROUP_LIST, &args, sizeof(args));
1297 }
1298
1299 int
1300 lst_info_group_ioctl(char *name, struct lstcon_ndlist_ent *gent,
1301                      int *idx, int *count, struct lstcon_node_ent *dents)
1302 {
1303         struct lstio_group_info_args args = { 0 };
1304
1305         args.lstio_grp_key    = session_key;
1306         args.lstio_grp_nmlen  = strlen(name);
1307         args.lstio_grp_namep  = name;
1308         args.lstio_grp_entp   = gent;
1309         args.lstio_grp_idxp   = idx;
1310         args.lstio_grp_ndentp = count;
1311         args.lstio_grp_dentsp = dents;
1312
1313         return lst_ioctl(LSTIO_GROUP_INFO, &args, sizeof(args));
1314 }
1315
1316 int
1317 lst_list_group_all(void)
1318 {
1319         char  name[LST_NAME_SIZE];
1320         int   rc;
1321         int   i;
1322
1323         /* no group is specified, list name of all groups */
1324         for (i = 0; ; i++) {
1325                 rc = lst_list_group_ioctl(LST_NAME_SIZE, name, i);
1326                 if (rc == 0) {
1327                         fprintf(stdout, "%d) %s\n", i + 1, name);
1328                         continue;
1329                 }
1330
1331                 if (errno == ENOENT)
1332                         break;
1333
1334                 lst_print_error("group", "Failed to list group: %s\n",
1335                                 strerror(errno));
1336                 return -1;
1337         }
1338
1339         fprintf(stdout, "Total %d groups\n", i);
1340
1341         return 0;
1342 }
1343
1344 #define LST_NODES_TITLE "\tACTIVE\tBUSY\tDOWN\tUNKNOWN\tTOTAL\n"
1345
1346 int
1347 jt_lst_list_group(int argc, char **argv)
1348 {
1349         struct lstcon_ndlist_ent gent;
1350         struct lstcon_node_ent   *dents;
1351         int               optidx  = 0;
1352         int               verbose = 0;
1353         int               active  = 0;
1354         int               busy    = 0;
1355         int               down    = 0;
1356         int               unknown = 0;
1357         int               all     = 0;
1358         int               count;
1359         int               index;
1360         int               i;
1361         int               j;
1362         int               c;
1363         int               rc = 0;
1364
1365         static struct option list_group_opts[] =
1366         {
1367                 {"active",  no_argument, 0, 'a' },
1368                 {"busy",    no_argument, 0, 'b' },
1369                 {"down",    no_argument, 0, 'd' },
1370                 {"unknown", no_argument, 0, 'u' },
1371                 {"all",     no_argument, 0, 'l' },
1372                 {0,         0,           0,  0  }
1373         };
1374
1375         if (session_key == 0) {
1376                 fprintf(stderr,
1377                         "Can't find env LST_SESSION or value is not valid\n");
1378                 return -1;
1379         }
1380
1381         while (1) {
1382                 c = getopt_long(argc, argv, "abdul",
1383                                 list_group_opts, &optidx);
1384
1385                 if (c == -1)
1386                         break;
1387
1388                 switch (c) {
1389                 case 'a':
1390                         verbose = active = 1;
1391                         all = 0;
1392                         break;
1393                 case 'b':
1394                         verbose = busy = 1;
1395                         all = 0;
1396                         break;
1397                 case 'd':
1398                         verbose = down = 1;
1399                         all = 0;
1400                         break;
1401                 case 'u':
1402                         verbose = unknown = 1;
1403                         all = 0;
1404                         break;
1405                 case 'l':
1406                         verbose = all = 1;
1407                         break;
1408                 default:
1409                         lst_print_usage(argv[0]);
1410                         return -1;
1411                 }
1412         }
1413
1414         if (optind == argc) {
1415                 /* no group is specified, list name of all groups */
1416                 rc = lst_list_group_all();
1417
1418                 return rc;
1419         }
1420
1421         if (!verbose)
1422                 fprintf(stdout, LST_NODES_TITLE);
1423
1424         /* list nodes in specified groups */
1425         for (i = optind; i < argc; i++) {
1426                 rc = lst_info_group_ioctl(argv[i], &gent, NULL, NULL, NULL);
1427                 if (rc != 0) {
1428                         if (errno == ENOENT) {
1429                                 rc = 0;
1430                                 break;
1431                         }
1432
1433                         lst_print_error("group", "Failed to list group\n",
1434                                         strerror(errno));
1435                         break;
1436                 }
1437
1438                 if (!verbose) {
1439                         fprintf(stdout, "\t%d\t%d\t%d\t%d\t%d\t%s\n",
1440                                 gent.nle_nactive, gent.nle_nbusy,
1441                                 gent.nle_ndown, gent.nle_nunknown,
1442                                 gent.nle_nnode, argv[i]);
1443                         continue;
1444                 }
1445
1446                 fprintf(stdout, "Group [ %s ]\n", argv[i]);
1447
1448                 if (gent.nle_nnode == 0) {
1449                         fprintf(stdout, "No nodes found [ %s ]\n", argv[i]);
1450                         continue;
1451                 }
1452
1453                 count = gent.nle_nnode;
1454
1455                 dents = malloc(count * sizeof(struct lstcon_node_ent));
1456                 if (dents == NULL) {
1457                         fprintf(stderr, "Failed to malloc: %s\n",
1458                                 strerror(errno));
1459                         return -1;
1460                 }
1461
1462                 index = 0;
1463                 rc = lst_info_group_ioctl(argv[i], &gent, &index, &count, dents);
1464                 if (rc != 0) {
1465                         lst_print_error("group", "Failed to list group: %s\n",
1466                                         strerror(errno));
1467                         free(dents);
1468                         return -1;
1469                 }
1470
1471                 for (j = 0, c = 0; j < count; j++) {
1472                         if (all ||
1473                             ((active  &&  dents[j].nde_state == LST_NODE_ACTIVE) ||
1474                              (busy    &&  dents[j].nde_state == LST_NODE_BUSY)   ||
1475                              (down    &&  dents[j].nde_state == LST_NODE_DOWN)   ||
1476                              (unknown &&  dents[j].nde_state == LST_NODE_UNKNOWN))) {
1477
1478                                 fprintf(stdout, "\t%s: %s\n",
1479                                         libcfs_id2str(dents[j].nde_id),
1480                                         lst_node_state2str(dents[j].nde_state));
1481                                 c++;
1482                         }
1483                 }
1484
1485                 fprintf(stdout, "Total %d nodes [ %s ]\n", c, argv[i]);
1486
1487                 free(dents);
1488         }
1489
1490         return rc;
1491 }
1492
1493 int
1494 lst_stat_ioctl (char *name, int count, lnet_process_id_t *idsp,
1495                 int timeout, struct list_head *resultp)
1496 {
1497         struct lstio_stat_args args = { 0 };
1498
1499         args.lstio_sta_key     = session_key;
1500         args.lstio_sta_timeout = timeout;
1501         args.lstio_sta_nmlen   = strlen(name);
1502         args.lstio_sta_namep   = name;
1503         args.lstio_sta_count   = count;
1504         args.lstio_sta_idsp    = idsp;
1505         args.lstio_sta_resultp = resultp;
1506
1507         return lst_ioctl(LSTIO_STAT_QUERY, &args, sizeof(args));
1508 }
1509
1510 typedef struct {
1511         struct list_head              srp_link;
1512         int                     srp_count;
1513         char                   *srp_name;
1514         lnet_process_id_t      *srp_ids;
1515         struct list_head              srp_result[2];
1516 } lst_stat_req_param_t;
1517
1518 static void
1519 lst_stat_req_param_free(lst_stat_req_param_t *srp)
1520 {
1521         int     i;
1522
1523         for (i = 0; i < 2; i++)
1524                 lst_free_rpcent(&srp->srp_result[i]);
1525
1526         if (srp->srp_ids != NULL)
1527                 free(srp->srp_ids);
1528
1529         free(srp);
1530 }
1531
1532 static int
1533 lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp, int save_old)
1534 {
1535         lst_stat_req_param_t *srp = NULL;
1536         int                   count = save_old ? 2 : 1;
1537         int                   rc;
1538         int                   i;
1539
1540         srp = malloc(sizeof(*srp));
1541         if (srp == NULL)
1542                 return -ENOMEM;
1543
1544         memset(srp, 0, sizeof(*srp));
1545         INIT_LIST_HEAD(&srp->srp_result[0]);
1546         INIT_LIST_HEAD(&srp->srp_result[1]);
1547
1548         rc = lst_get_node_count(LST_OPC_GROUP, name,
1549                                 &srp->srp_count, NULL);
1550         if (rc != 0 && errno == ENOENT) {
1551                 rc = lst_get_node_count(LST_OPC_NODES, name,
1552                                         &srp->srp_count, &srp->srp_ids);
1553         }
1554
1555         if (rc != 0) {
1556                 fprintf(stderr,
1557                         "Failed to get count of nodes from %s: %s\n",
1558                         name, strerror(errno));
1559                 lst_stat_req_param_free(srp);
1560
1561                 return rc;
1562         }
1563
1564         srp->srp_name = name;
1565
1566         for (i = 0; i < count; i++) {
1567                 rc = lst_alloc_rpcent(&srp->srp_result[i], srp->srp_count,
1568                                       sizeof(struct sfw_counters)  +
1569                                       sizeof(struct srpc_counters) +
1570                                       sizeof(lnet_counters_t));
1571                 if (rc != 0) {
1572                         fprintf(stderr, "Out of memory\n");
1573                         break;
1574                 }
1575         }
1576
1577         if (rc == 0) {
1578                 *srpp = srp;
1579                 return 0;
1580         }
1581
1582         lst_stat_req_param_free(srp);
1583
1584         return rc;
1585 }
1586
1587 typedef struct {
1588         /* TODO */
1589         int foo;
1590 } lst_srpc_stat_result;
1591
1592 #define LST_LNET_AVG    0
1593 #define LST_LNET_MIN    1
1594 #define LST_LNET_MAX    2
1595
1596 typedef struct {
1597         float           lnet_avg_sndrate;
1598         float           lnet_min_sndrate;
1599         float           lnet_max_sndrate;
1600         float           lnet_total_sndrate;
1601
1602         float           lnet_avg_rcvrate;
1603         float           lnet_min_rcvrate;
1604         float           lnet_max_rcvrate;
1605         float           lnet_total_rcvrate;
1606
1607         float           lnet_avg_sndperf;
1608         float           lnet_min_sndperf;
1609         float           lnet_max_sndperf;
1610         float           lnet_total_sndperf;
1611
1612         float           lnet_avg_rcvperf;
1613         float           lnet_min_rcvperf;
1614         float           lnet_max_rcvperf;
1615         float           lnet_total_rcvperf;
1616
1617         int             lnet_stat_count;
1618 } lst_lnet_stat_result_t;
1619
1620 lst_lnet_stat_result_t lnet_stat_result;
1621
1622 static float
1623 lst_lnet_stat_value(int bw, int send, int off)
1624 {
1625         float  *p;
1626
1627         p = bw ? &lnet_stat_result.lnet_avg_sndperf :
1628                  &lnet_stat_result.lnet_avg_sndrate;
1629
1630         if (!send)
1631                 p += 4;
1632
1633         p += off;
1634
1635         return *p;
1636 }
1637
1638 static void
1639 lst_timeval_diff(struct timeval *tv1,
1640                  struct timeval *tv2, struct timeval *df)
1641 {
1642         if (tv1->tv_usec >= tv2->tv_usec) {
1643                 df->tv_sec  = tv1->tv_sec - tv2->tv_sec;
1644                 df->tv_usec = tv1->tv_usec - tv2->tv_usec;
1645                 return;
1646         }
1647
1648         df->tv_sec  = tv1->tv_sec - 1 - tv2->tv_sec;
1649         df->tv_usec = tv1->tv_usec + 1000000 - tv2->tv_usec;
1650
1651         return;
1652 }
1653
1654 static void
1655 lst_cal_lnet_stat(float delta, lnet_counters_t *lnet_new,
1656                   lnet_counters_t *lnet_old, int mbs)
1657 {
1658         float perf;
1659         float rate;
1660         unsigned int unit_divisor;
1661
1662         unit_divisor = (mbs) ? (1000 * 1000) : (1024 * 1024);
1663         perf = (float)(lnet_new->send_length -
1664                        lnet_old->send_length) / unit_divisor / delta;
1665         lnet_stat_result.lnet_total_sndperf += perf;
1666
1667         if (lnet_stat_result.lnet_min_sndperf > perf ||
1668             lnet_stat_result.lnet_min_sndperf == 0)
1669                 lnet_stat_result.lnet_min_sndperf = perf;
1670
1671         if (lnet_stat_result.lnet_max_sndperf < perf)
1672                 lnet_stat_result.lnet_max_sndperf = perf;
1673
1674         perf = (float)(lnet_new->recv_length -
1675                        lnet_old->recv_length) / unit_divisor / delta;
1676         lnet_stat_result.lnet_total_rcvperf += perf;
1677
1678         if (lnet_stat_result.lnet_min_rcvperf > perf ||
1679             lnet_stat_result.lnet_min_rcvperf == 0)
1680                 lnet_stat_result.lnet_min_rcvperf = perf;
1681
1682         if (lnet_stat_result.lnet_max_rcvperf < perf)
1683                 lnet_stat_result.lnet_max_rcvperf = perf;
1684
1685         rate = (lnet_new->send_count - lnet_old->send_count) / delta;
1686         lnet_stat_result.lnet_total_sndrate += rate;
1687
1688         if (lnet_stat_result.lnet_min_sndrate > rate ||
1689             lnet_stat_result.lnet_min_sndrate == 0)
1690                 lnet_stat_result.lnet_min_sndrate = rate;
1691
1692         if (lnet_stat_result.lnet_max_sndrate < rate)
1693                 lnet_stat_result.lnet_max_sndrate = rate;
1694
1695         rate = (lnet_new->recv_count - lnet_old->recv_count) / delta;
1696         lnet_stat_result.lnet_total_rcvrate += rate;
1697
1698         if (lnet_stat_result.lnet_min_rcvrate > rate ||
1699             lnet_stat_result.lnet_min_rcvrate == 0)
1700                 lnet_stat_result.lnet_min_rcvrate = rate;
1701
1702         if (lnet_stat_result.lnet_max_rcvrate < rate)
1703                 lnet_stat_result.lnet_max_rcvrate = rate;
1704
1705         lnet_stat_result.lnet_stat_count++;
1706
1707         lnet_stat_result.lnet_avg_sndrate = lnet_stat_result.lnet_total_sndrate /
1708                                             lnet_stat_result.lnet_stat_count;
1709         lnet_stat_result.lnet_avg_rcvrate = lnet_stat_result.lnet_total_rcvrate /
1710                                             lnet_stat_result.lnet_stat_count;
1711
1712         lnet_stat_result.lnet_avg_sndperf = lnet_stat_result.lnet_total_sndperf /
1713                                             lnet_stat_result.lnet_stat_count;
1714         lnet_stat_result.lnet_avg_rcvperf = lnet_stat_result.lnet_total_rcvperf /
1715                                             lnet_stat_result.lnet_stat_count;
1716 }
1717
1718 static void
1719 lst_print_lnet_stat(char *name, int bwrt, int rdwr, int type, int mbs)
1720 {
1721         int     start1 = 0;
1722         int     end1   = 1;
1723         int     start2 = 0;
1724         int     end2   = 1;
1725         int     i;
1726         int     j;
1727         char   *units;
1728
1729         if (lnet_stat_result.lnet_stat_count == 0)
1730                 return;
1731
1732         units = (mbs) ? "MB/s  " : "MiB/s ";
1733
1734         if (bwrt == 1) /* bw only */
1735                 start1 = 1;
1736
1737         if (bwrt == 2) /* rates only */
1738                 end1 = 0;
1739
1740         if (rdwr == 1) /* recv only */
1741                 start2 = 1;
1742
1743         if (rdwr == 2) /* send only */
1744                 end2 = 0;
1745
1746         for (i = start1; i <= end1; i++) {
1747                 fprintf(stdout, "[LNet %s of %s]\n",
1748                         i == 0 ? "Rates" : "Bandwidth", name);
1749
1750                 for (j = start2; j <= end2; j++) {
1751                         fprintf(stdout, "[%c] ", j == 0 ? 'R' : 'W');
1752
1753                         if ((type & 1) != 0) {
1754                                 fprintf(stdout, i == 0 ? "Avg: %-8.0f RPC/s " :
1755                                                          "Avg: %-8.2f %s",
1756                                         lst_lnet_stat_value(i, j, 0), units);
1757                         }
1758
1759                         if ((type & 2) != 0) {
1760                                 fprintf(stdout, i == 0 ? "Min: %-8.0f RPC/s " :
1761                                                          "Min: %-8.2f %s",
1762                                         lst_lnet_stat_value(i, j, 1), units);
1763                         }
1764
1765                         if ((type & 4) != 0) {
1766                                 fprintf(stdout, i == 0 ? "Max: %-8.0f RPC/s" :
1767                                                          "Max: %-8.2f %s",
1768                                         lst_lnet_stat_value(i, j, 2), units);
1769                         }
1770
1771                         fprintf(stdout, "\n");
1772                 }
1773         }
1774 }
1775
1776 static void
1777 lst_print_stat(char *name, struct list_head *resultp,
1778                int idx, int lnet, int bwrt, int rdwr, int type,
1779                int mbs)
1780 {
1781         struct list_head        tmp[2];
1782         struct lstcon_rpc_ent *new;
1783         struct lstcon_rpc_ent *old;
1784         struct sfw_counters   *sfwk_new;
1785         struct sfw_counters   *sfwk_old;
1786         struct srpc_counters  *srpc_new;
1787         struct srpc_counters  *srpc_old;
1788         lnet_counters_t  *lnet_new;
1789         lnet_counters_t  *lnet_old;
1790         float             delta;
1791         int               errcount = 0;
1792
1793         INIT_LIST_HEAD(&tmp[0]);
1794         INIT_LIST_HEAD(&tmp[1]);
1795
1796         memset(&lnet_stat_result, 0, sizeof(lnet_stat_result));
1797
1798         while (!list_empty(&resultp[idx])) {
1799                 if (list_empty(&resultp[1 - idx])) {
1800                         fprintf(stderr, "Group is changed, re-run stat\n");
1801                         break;
1802                 }
1803
1804                 new = list_entry(resultp[idx].next, struct lstcon_rpc_ent,
1805                                      rpe_link);
1806                 old = list_entry(resultp[1 - idx].next, struct lstcon_rpc_ent,
1807                                      rpe_link);
1808
1809                 /* first time get stats result, can't calculate diff */
1810                 if (new->rpe_peer.nid == LNET_NID_ANY)
1811                         break;
1812
1813                 if (new->rpe_peer.nid != old->rpe_peer.nid ||
1814                     new->rpe_peer.pid != old->rpe_peer.pid) {
1815                         /* Something wrong. i.e, somebody change the group */
1816                         break;
1817                 }
1818
1819                 list_del(&new->rpe_link);
1820                 list_add_tail(&new->rpe_link, &tmp[idx]);
1821
1822                 list_del(&old->rpe_link);
1823                 list_add_tail(&old->rpe_link, &tmp[1 - idx]);
1824
1825                 if (new->rpe_rpc_errno != 0 || new->rpe_fwk_errno != 0 ||
1826                     old->rpe_rpc_errno != 0 || old->rpe_fwk_errno != 0) {
1827                         errcount ++;
1828                         continue;
1829                 }
1830
1831                 sfwk_new = (struct sfw_counters *)&new->rpe_payload[0];
1832                 sfwk_old = (struct sfw_counters *)&old->rpe_payload[0];
1833
1834                 srpc_new = (struct srpc_counters *)((char *)sfwk_new + sizeof(*sfwk_new));
1835                 srpc_old = (struct srpc_counters *)((char *)sfwk_old + sizeof(*sfwk_old));
1836
1837                 lnet_new = (lnet_counters_t *)((char *)srpc_new + sizeof(*srpc_new));
1838                 lnet_old = (lnet_counters_t *)((char *)srpc_old + sizeof(*srpc_old));
1839
1840                 /* Prior to version 2.3, the running_ms field was a counter for
1841                  * the number of running tests.  We are looking at this value
1842                  * to determine if it is a millisecond timestamep (>= 2.3) or a
1843                  * test counter (< 2.3).  The number 500 is being used for this
1844                  * barrier as the test counter should never get this high, and
1845                  * the timestamp should never get this low. */
1846
1847                 if (sfwk_new->running_ms > 500) {
1848                         /* use the timestamp from the remote node, not our
1849                          * rpe_stamp from when we copied up the data out of
1850                          * the kernel */
1851
1852                         delta = (float) (sfwk_new->running_ms -
1853                                         sfwk_old->running_ms) / 1000;
1854                 } else {
1855                         struct timeval    tv;
1856
1857                         lst_timeval_diff(&new->rpe_stamp, &old->rpe_stamp, &tv);
1858                         delta = tv.tv_sec + (float)tv.tv_usec / 1000000;
1859                 }
1860
1861                 if (!lnet) /* TODO */
1862                         continue;
1863
1864                 lst_cal_lnet_stat(delta, lnet_new, lnet_old, mbs);
1865         }
1866
1867         list_splice(&tmp[idx], &resultp[idx]);
1868         list_splice(&tmp[1 - idx], &resultp[1 - idx]);
1869
1870         if (errcount > 0)
1871                 fprintf(stdout, "Failed to stat on %d nodes\n", errcount);
1872
1873         if (!lnet)  /* TODO */
1874                 return;
1875
1876         lst_print_lnet_stat(name, bwrt, rdwr, type, mbs);
1877 }
1878
1879 int
1880 jt_lst_stat(int argc, char **argv)
1881 {
1882         struct list_head        head;
1883         lst_stat_req_param_t *srp;
1884         time_t                last    = 0;
1885         int                   optidx  = 0;
1886         int                   timeout = 5; /* default timeout, 5 sec */
1887         int                   delay   = 5; /* default delay, 5 sec */
1888         int                   count   = -1; /* run forever */
1889         int                   lnet    = 1; /* lnet stat by default */
1890         int                   bwrt    = 0;
1891         int                   rdwr    = 0;
1892         int                   type    = -1;
1893         int                   idx     = 0;
1894         int                   rc;
1895         int                   c;
1896         int                   mbs     = 0; /* report as MB/s */
1897
1898         static struct option stat_opts[] = {
1899                 {"timeout"   , required_argument, 0, 't' },
1900                 {"delay"     , required_argument, 0, 'd' },
1901                 {"count"     , required_argument, 0, 'o' },
1902                 {"lnet"      , no_argument,      0, 'l' },
1903                 {"rpc"       , no_argument,      0, 'c' },
1904                 {"bw"        , no_argument,      0, 'b' },
1905                 {"rate"      , no_argument,      0, 'a' },
1906                 {"read"      , no_argument,      0, 'r' },
1907                 {"write"     , no_argument,      0, 'w' },
1908                 {"avg"       , no_argument,      0, 'g' },
1909                 {"min"       , no_argument,      0, 'n' },
1910                 {"max"       , no_argument,      0, 'x' },
1911                 {"mbs"       , no_argument,      0, 'm' },
1912                 {0,            0,                0,  0  }
1913         };
1914
1915         if (session_key == 0) {
1916                 fprintf(stderr,
1917                         "Can't find env LST_SESSION or value is not valid\n");
1918                 return -1;
1919         }
1920
1921         while (1) {
1922                 c = getopt_long(argc, argv, "t:d:lcbarwgnxm", stat_opts,
1923                                 &optidx);
1924
1925                 if (c == -1)
1926                         break;
1927
1928                 switch (c) {
1929                 case 't':
1930                         timeout = atoi(optarg);
1931                         break;
1932                 case 'd':
1933                         delay = atoi(optarg);
1934                         break;
1935                 case 'o':
1936                         count = atoi(optarg);
1937                         break;
1938                 case 'l':
1939                         lnet = 1;
1940                         break;
1941                 case 'c':
1942                         lnet = 0;
1943                         break;
1944                 case 'b':
1945                         bwrt |= 1;
1946                         break;
1947                 case 'a':
1948                         bwrt |= 2;
1949                         break;
1950                 case 'r':
1951                         rdwr |= 1;
1952                         break;
1953                 case 'w':
1954                         rdwr |= 2;
1955                         break;
1956                 case 'g':
1957                         if (type == -1) {
1958                                 type = 1;
1959                                 break;
1960                         }
1961                         type |= 1;
1962                         break;
1963                 case 'n':
1964                         if (type == -1) {
1965                                 type = 2;
1966                                 break;
1967                         }
1968                         type |= 2;
1969                         break;
1970                 case 'x':
1971                         if (type == -1) {
1972                                 type = 4;
1973                                 break;
1974                         }
1975                         type |= 4;
1976                         break;
1977                 case 'm':
1978                         mbs = 1;
1979                         break;
1980
1981                 default:
1982                         lst_print_usage(argv[0]);
1983                         return -1;
1984                 }
1985         }
1986
1987         if (optind == argc) {
1988                 lst_print_usage(argv[0]);
1989                 return -1;
1990         }
1991
1992         if (timeout <= 0 || delay <= 0) {
1993                 fprintf(stderr, "Invalid timeout or delay value\n");
1994                 return -1;
1995         }
1996
1997         if (count < -1) {
1998             fprintf(stderr, "Invalid count value\n");
1999             return -1;
2000         }
2001
2002         /* extra count to get first data point */
2003         if (count != -1)
2004             count++;
2005
2006         INIT_LIST_HEAD(&head);
2007
2008         while (optind < argc) {
2009                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 1);
2010                 if (rc != 0)
2011                         goto out;
2012
2013                 list_add_tail(&srp->srp_link, &head);
2014         }
2015
2016         do {
2017                 time_t  now = time(NULL);
2018
2019                 if (now - last < delay) {
2020                         sleep(delay - now + last);
2021                         time(&now);
2022                 }
2023                 last = now;
2024
2025                 list_for_each_entry(srp, &head, srp_link) {
2026                         rc = lst_stat_ioctl(srp->srp_name,
2027                                             srp->srp_count, srp->srp_ids,
2028                                             timeout, &srp->srp_result[idx]);
2029                         if (rc == -1) {
2030                                 lst_print_error("stat", "Failed to stat %s: %s\n",
2031                                                 srp->srp_name, strerror(errno));
2032                                 goto out;
2033                         }
2034
2035                         lst_print_stat(srp->srp_name, srp->srp_result,
2036                                        idx, lnet, bwrt, rdwr, type, mbs);
2037
2038                         lst_reset_rpcent(&srp->srp_result[1 - idx]);
2039                 }
2040
2041                 idx = 1 - idx;
2042
2043                 if (count > 0)
2044                         count--;
2045         } while (count == -1 || count > 0);
2046
2047 out:
2048         while (!list_empty(&head)) {
2049                 srp = list_entry(head.next, lst_stat_req_param_t, srp_link);
2050
2051                 list_del(&srp->srp_link);
2052                 lst_stat_req_param_free(srp);
2053         }
2054
2055         return rc;
2056 }
2057
2058 int
2059 jt_lst_show_error(int argc, char **argv)
2060 {
2061         struct list_head            head;
2062         lst_stat_req_param_t *srp;
2063         struct lstcon_rpc_ent     *ent;
2064         struct sfw_counters       *sfwk;
2065         struct srpc_counters      *srpc;
2066         int                   show_rpc = 1;
2067         int                   optidx   = 0;
2068         int                   rc       = 0;
2069         int                   ecount;
2070         int                   c;
2071
2072         static struct option  show_error_opts[] =
2073         {
2074                 {"session", no_argument,       0, 's' },
2075                 {0,         0,                 0,  0  }
2076         };
2077
2078         if (session_key == 0) {
2079                 fprintf(stderr,
2080                         "Can't find env LST_SESSION or value is not valid\n");
2081                 return -1;
2082         }
2083
2084         while (1) {
2085                 c = getopt_long(argc, argv, "s", show_error_opts, &optidx);
2086
2087                 if (c == -1)
2088                         break;
2089
2090                 switch (c) {
2091                 case 's':
2092                         show_rpc  = 0;
2093                         break;
2094
2095                 default:
2096                         lst_print_usage(argv[0]);
2097                         return -1;
2098                 }
2099         }
2100
2101         if (optind == argc) {
2102                 lst_print_usage(argv[0]);
2103                 return -1;
2104         }
2105
2106         INIT_LIST_HEAD(&head);
2107
2108         while (optind < argc) {
2109                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 0);
2110                 if (rc != 0)
2111                         goto out;
2112
2113                 list_add_tail(&srp->srp_link, &head);
2114         }
2115
2116         list_for_each_entry(srp, &head, srp_link) {
2117                 rc = lst_stat_ioctl(srp->srp_name, srp->srp_count,
2118                                     srp->srp_ids, 10, &srp->srp_result[0]);
2119
2120                 if (rc == -1) {
2121                         lst_print_error(srp->srp_name, "Failed to show errors of %s: %s\n",
2122                                         srp->srp_name, strerror(errno));
2123                         goto out;
2124                 }
2125
2126                 fprintf(stdout, "%s:\n", srp->srp_name);
2127
2128                 ecount = 0;
2129
2130                 list_for_each_entry(ent, &srp->srp_result[0], rpe_link) {
2131                         if (ent->rpe_rpc_errno != 0) {
2132                                 ecount ++;
2133                                 fprintf(stderr, "RPC failure, can't show error on %s\n",
2134                                         libcfs_id2str(ent->rpe_peer));
2135                                 continue;
2136                         }
2137
2138                         if (ent->rpe_fwk_errno != 0) {
2139                                 ecount ++;
2140                                 fprintf(stderr, "Framework failure, can't show error on %s\n",
2141                                         libcfs_id2str(ent->rpe_peer));
2142                                 continue;
2143                         }
2144
2145                         sfwk = (struct sfw_counters *)&ent->rpe_payload[0];
2146                         srpc = (struct srpc_counters *)((char *)sfwk + sizeof(*sfwk));
2147
2148                         if (srpc->errors == 0 &&
2149                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2150                                 continue;
2151
2152                         if (!show_rpc  &&
2153                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2154                                 continue;
2155
2156                         ecount ++;
2157
2158                         fprintf(stderr, "%s: [Session %d brw errors, %d ping errors]%c",
2159                                 libcfs_id2str(ent->rpe_peer),
2160                                 sfwk->brw_errors, sfwk->ping_errors,
2161                                 show_rpc  ? ' ' : '\n');
2162
2163                         if (!show_rpc)
2164                                 continue;
2165
2166                         fprintf(stderr, "[RPC: %d errors, %d dropped, %d expired]\n",
2167                                 srpc->errors, srpc->rpcs_dropped, srpc->rpcs_expired);
2168                 }
2169
2170                 fprintf(stdout, "Total %d error nodes in %s\n", ecount, srp->srp_name);
2171         }
2172 out:
2173         while (!list_empty(&head)) {
2174                 srp = list_entry(head.next, lst_stat_req_param_t, srp_link);
2175
2176                 list_del(&srp->srp_link);
2177                 lst_stat_req_param_free(srp);
2178         }
2179
2180         return rc;
2181 }
2182
2183 int
2184 lst_add_batch_ioctl(char *name)
2185 {
2186         struct lstio_batch_add_args args = { 0 };
2187
2188         args.lstio_bat_key   = session_key;
2189         args.lstio_bat_nmlen = strlen(name);
2190         args.lstio_bat_namep = name;
2191
2192         return lst_ioctl (LSTIO_BATCH_ADD, &args, sizeof(args));
2193 }
2194
2195 int
2196 jt_lst_add_batch(int argc, char **argv)
2197 {
2198         char   *name;
2199         int     rc;
2200
2201         if (session_key == 0) {
2202                 fprintf(stderr,
2203                         "Can't find env LST_SESSION or value is not valid\n");
2204                 return -1;
2205         }
2206
2207         if (argc != 2) {
2208                 lst_print_usage(argv[0]);
2209                 return -1;
2210         }
2211
2212         name = argv[1];
2213         if (strlen(name) >= LST_NAME_SIZE) {
2214                 fprintf(stderr, "Name length is limited to %d\n",
2215                         LST_NAME_SIZE - 1);
2216                 return -1;
2217         }
2218
2219         rc = lst_add_batch_ioctl(name);
2220         if (rc == 0)
2221                 return 0;
2222
2223         lst_print_error("batch", "Failed to create batch: %s\n",
2224                         strerror(errno));
2225
2226         return -1;
2227 }
2228
2229 int
2230 lst_start_batch_ioctl(char *name, int timeout, struct list_head *resultp)
2231 {
2232         struct lstio_batch_run_args args = { 0 };
2233
2234         args.lstio_bat_key     = session_key;
2235         args.lstio_bat_timeout = timeout;
2236         args.lstio_bat_nmlen   = strlen(name);
2237         args.lstio_bat_namep   = name;
2238         args.lstio_bat_resultp = resultp;
2239
2240         return lst_ioctl(LSTIO_BATCH_START, &args, sizeof(args));
2241 }
2242
2243 int
2244 jt_lst_start_batch(int argc, char **argv)
2245 {
2246         struct list_head        head;
2247         char             *batch;
2248         int               optidx  = 0;
2249         int               timeout = 0;
2250         int               count = 0;
2251         int               rc;
2252         int               c;
2253
2254         static struct option start_batch_opts[] =
2255         {
2256                 {"timeout", required_argument, 0, 't' },
2257                 {0,         0,                 0,  0  }
2258         };
2259
2260         if (session_key == 0) {
2261                 fprintf(stderr,
2262                         "Can't find env LST_SESSION or value is not valid\n");
2263                 return -1;
2264         }
2265
2266         while (1) {
2267                 c = getopt_long(argc, argv, "t:",
2268                                 start_batch_opts, &optidx);
2269
2270                 /* Detect the end of the options. */
2271                 if (c == -1)
2272                         break;
2273
2274                 switch (c) {
2275                 case 't':
2276                         timeout = atoi(optarg);
2277                         break;
2278                 default:
2279                         lst_print_usage(argv[0]);
2280                         return -1;
2281                 }
2282         }
2283
2284         if (optind == argc) {
2285                 batch = LST_DEFAULT_BATCH;
2286
2287         } else if (optind == argc - 1) {
2288                 batch = argv[optind];
2289
2290         } else {
2291                 lst_print_usage(argv[0]);
2292                 return -1;
2293         }
2294
2295         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2296         if (rc != 0) {
2297                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2298                         batch, strerror(errno));
2299                 return -1;
2300         }
2301
2302         INIT_LIST_HEAD(&head);
2303
2304         rc = lst_alloc_rpcent(&head, count, 0);
2305         if (rc != 0) {
2306                 fprintf(stderr, "Out of memory\n");
2307                 return -1;
2308         }
2309
2310         rc = lst_start_batch_ioctl(batch, timeout, &head);
2311
2312         if (rc == 0) {
2313                 fprintf(stdout, "%s is running now\n", batch);
2314                 lst_free_rpcent(&head);
2315                 return 0;
2316         }
2317
2318         if (rc == -1) {
2319                 lst_print_error("batch", "Failed to start batch: %s\n",
2320                                 strerror(errno));
2321                 lst_free_rpcent(&head);
2322                 return rc;
2323         }
2324
2325         lst_print_transerr(&head, "Run batch");
2326
2327         lst_free_rpcent(&head);
2328
2329         return rc;
2330 }
2331
2332 int
2333 lst_stop_batch_ioctl(char *name, int force, struct list_head *resultp)
2334 {
2335         struct lstio_batch_stop_args args = { 0 };
2336
2337         args.lstio_bat_key     = session_key;
2338         args.lstio_bat_force   = force;
2339         args.lstio_bat_nmlen   = strlen(name);
2340         args.lstio_bat_namep   = name;
2341         args.lstio_bat_resultp = resultp;
2342
2343         return lst_ioctl(LSTIO_BATCH_STOP, &args, sizeof(args));
2344 }
2345
2346 int
2347 jt_lst_stop_batch(int argc, char **argv)
2348 {
2349         struct list_head        head;
2350         char             *batch;
2351         int               force = 0;
2352         int               optidx;
2353         int               count;
2354         int               rc;
2355         int               c;
2356
2357         static struct option stop_batch_opts[] =
2358         {
2359                 {"force",   no_argument,   0, 'f' },
2360                 {0,         0,             0,  0  }
2361         };
2362
2363         if (session_key == 0) {
2364                 fprintf(stderr,
2365                         "Can't find env LST_SESSION or value is not valid\n");
2366                 return -1;
2367         }
2368
2369         while (1) {
2370                 c = getopt_long(argc, argv, "f",
2371                                 stop_batch_opts, &optidx);
2372
2373                 /* Detect the end of the options. */
2374                 if (c == -1)
2375                         break;
2376
2377                 switch (c) {
2378                 case 'f':
2379                         force = 1;
2380                         break;
2381                 default:
2382                         lst_print_usage(argv[0]);
2383                         return -1;
2384                 }
2385         }
2386
2387         if (optind == argc) {
2388                 batch = LST_DEFAULT_BATCH;
2389
2390         } else if (optind == argc - 1) {
2391                 batch = argv[optind];
2392
2393         } else {
2394                 lst_print_usage(argv[0]);
2395                 return -1;
2396         }
2397
2398         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2399         if (rc != 0) {
2400                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2401                         batch, strerror(errno));
2402                 return -1;
2403         }
2404
2405         INIT_LIST_HEAD(&head);
2406
2407         rc = lst_alloc_rpcent(&head, count, 0);
2408         if (rc != 0) {
2409                 fprintf(stderr, "Out of memory\n");
2410                 return -1;
2411         }
2412
2413         rc = lst_stop_batch_ioctl(batch, force, &head);
2414         if (rc != 0)
2415                 goto out;
2416
2417         while (1) {
2418                 lst_reset_rpcent(&head);
2419
2420                 rc = lst_query_batch_ioctl(batch, 0, 0, 30, &head);
2421                 if (rc != 0)
2422                         goto out;
2423
2424                 if (lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0 &&
2425                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0)
2426                         break;
2427
2428                 fprintf(stdout, "%d batch in stopping\n",
2429                         lstcon_tsbqry_stat_run(&trans_stat, 0));
2430                 sleep(1);
2431         }
2432
2433         fprintf(stdout, "Batch is stopped\n");
2434         lst_free_rpcent(&head);
2435
2436         return 0;
2437 out:
2438         if (rc == -1) {
2439                 lst_print_error("batch", "Failed to stop batch: %s\n",
2440                                 strerror(errno));
2441                 lst_free_rpcent(&head);
2442                 return -1;
2443         }
2444
2445         lst_print_transerr(&head, "stop batch");
2446
2447         lst_free_rpcent(&head);
2448
2449         return rc;
2450 }
2451
2452 int
2453 lst_list_batch_ioctl(int len, char *name, int index)
2454 {
2455         struct lstio_batch_list_args args = { 0 };
2456
2457         args.lstio_bat_key   = session_key;
2458         args.lstio_bat_idx   = index;
2459         args.lstio_bat_nmlen = len;
2460         args.lstio_bat_namep = name;
2461
2462         return lst_ioctl(LSTIO_BATCH_LIST, &args, sizeof(args));
2463 }
2464
2465 int
2466 lst_info_batch_ioctl(char *batch, int test, int server,
2467                      struct lstcon_test_batch_ent *entp, int *idxp,
2468                      int *ndentp, struct lstcon_node_ent *dentsp)
2469 {
2470         struct lstio_batch_info_args args = { 0 };
2471
2472         args.lstio_bat_key     = session_key;
2473         args.lstio_bat_nmlen   = strlen(batch);
2474         args.lstio_bat_namep   = batch;
2475         args.lstio_bat_server  = server;
2476         args.lstio_bat_testidx = test;
2477         args.lstio_bat_entp    = entp;
2478         args.lstio_bat_idxp    = idxp;
2479         args.lstio_bat_ndentp  = ndentp;
2480         args.lstio_bat_dentsp  = dentsp;
2481
2482         return lst_ioctl(LSTIO_BATCH_INFO, &args, sizeof(args));
2483 }
2484
2485 int
2486 lst_list_batch_all(void)
2487 {
2488         char name[LST_NAME_SIZE];
2489         int  rc;
2490         int  i;
2491
2492         for (i = 0; ; i++) {
2493                 rc = lst_list_batch_ioctl(LST_NAME_SIZE, name, i);
2494                 if (rc == 0) {
2495                         fprintf(stdout, "%d) %s\n", i + 1, name);
2496                         continue;
2497                 }
2498
2499                 if (errno == ENOENT)
2500                         break;
2501
2502                 lst_print_error("batch", "Failed to list batch: %s\n",
2503                                 strerror(errno));
2504                 return rc;
2505         }
2506
2507         fprintf(stdout, "Total %d batches\n", i);
2508
2509         return 0;
2510 }
2511
2512 int
2513 lst_list_tsb_nodes(char *batch, int test, int server,
2514                    int count, int active, int invalid)
2515 {
2516         struct lstcon_node_ent *dents;
2517         int                index = 0;
2518         int                rc;
2519         int                c;
2520         int                i;
2521
2522         if (count == 0)
2523                 return 0;
2524
2525         /* verbose list, show nodes in batch or test */
2526         dents = malloc(count * sizeof(struct lstcon_node_ent));
2527         if (dents == NULL) {
2528                 fprintf(stdout, "Can't allocate memory\n");
2529                 return -1;
2530         }
2531
2532         rc = lst_info_batch_ioctl(batch, test, server,
2533                                   NULL, &index, &count, dents);
2534         if (rc != 0) {
2535                 free(dents);
2536                 lst_print_error((test > 0) ? "test" : "batch",
2537                                 (test > 0) ? "Failed to query test: %s\n" :
2538                                              "Failed to query batch: %s\n",
2539                                 strerror(errno));
2540                 return -1;
2541         }
2542
2543         for (i = 0, c = 0; i < count; i++) {
2544                 if ((!active  && dents[i].nde_state == LST_NODE_ACTIVE) ||
2545                     (!invalid && (dents[i].nde_state == LST_NODE_BUSY  ||
2546                                   dents[i].nde_state == LST_NODE_DOWN  ||
2547                                   dents[i].nde_state == LST_NODE_UNKNOWN)))
2548                         continue;
2549
2550                 fprintf(stdout, "\t%s: %s\n",
2551                         libcfs_id2str(dents[i].nde_id),
2552                         lst_node_state2str(dents[i].nde_state));
2553                 c++;
2554         }
2555
2556         fprintf(stdout, "Total %d nodes\n", c);
2557         free(dents);
2558
2559         return 0;
2560 }
2561
2562 int
2563 jt_lst_list_batch(int argc, char **argv)
2564 {
2565         struct lstcon_test_batch_ent ent;
2566         char                *batch   = NULL;
2567         int                  optidx  = 0;
2568         int                  verbose = 0; /* list nodes in batch or test */
2569         int                  invalid = 0;
2570         int                  active  = 0;
2571         int                  server  = 0;
2572         int                  ntest   = 0;
2573         int                  test    = 0;
2574         int                  c       = 0;
2575         int                  rc;
2576
2577         static struct option list_batch_opts[] =
2578         {
2579                 {"test",    required_argument, 0, 't' },
2580                 {"invalid", no_argument,       0, 'i' },
2581                 {"active",  no_argument,       0, 'a' },
2582                 {"all",     no_argument,       0, 'l' },
2583                 {"server",  no_argument,       0, 's' },
2584                 {0,         0,                 0,  0  }
2585         };
2586
2587         if (session_key == 0) {
2588                 fprintf(stderr,
2589                         "Can't find env LST_SESSION or value is not valid\n");
2590                 return -1;
2591         }
2592
2593         while (1) {
2594                 c = getopt_long(argc, argv, "ailst:",
2595                                 list_batch_opts, &optidx);
2596
2597                 if (c == -1)
2598                         break;
2599
2600                 switch (c) {
2601                 case 'a':
2602                         verbose = active = 1;
2603                         break;
2604                 case 'i':
2605                         verbose = invalid = 1;
2606                         break;
2607                 case 'l':
2608                         verbose = active = invalid = 1;
2609                         break;
2610                 case 's':
2611                         server = 1;
2612                         break;
2613                 case 't':
2614                         test = atoi(optarg);
2615                         ntest = 1;
2616                         break;
2617                 default:
2618                         lst_print_usage(argv[0]);
2619                         return -1;
2620                 }
2621         }
2622
2623         if (optind == argc) {
2624                 /* list all batches */
2625                 rc = lst_list_batch_all();
2626                 return rc;
2627         }
2628
2629         if (ntest == 1 && test <= 0) {
2630                 fprintf(stderr, "Invalid test id, test id starts from 1\n");
2631                 return -1;
2632         }
2633
2634         if (optind != argc - 1) {
2635                 lst_print_usage(argv[0]);
2636                 return -1;
2637         }
2638
2639         batch = argv[optind];
2640
2641 loop:
2642         /* show detail of specified batch or test */
2643         rc = lst_info_batch_ioctl(batch, test, server,
2644                                   &ent, NULL, NULL, NULL);
2645         if (rc != 0) {
2646                 lst_print_error((test > 0) ? "test" : "batch",
2647                                 (test > 0) ? "Failed to query test: %s\n" :
2648                                              "Failed to query batch: %s\n",
2649                                 strerror(errno));
2650                 return -1;
2651         }
2652
2653         if (verbose) {
2654                 /* list nodes in test or batch */
2655                 rc = lst_list_tsb_nodes(batch, test, server,
2656                                         server ? ent.tbe_srv_nle.nle_nnode :
2657                                                  ent.tbe_cli_nle.nle_nnode,
2658                                         active, invalid);
2659                 return rc;
2660         }
2661
2662         /* only show number of hosts in batch or test */
2663         if (test == 0) {
2664                 fprintf(stdout, "Batch: %s Tests: %d State: %d\n",
2665                         batch, ent.u.tbe_batch.bae_ntest,
2666                         ent.u.tbe_batch.bae_state);
2667                 ntest = ent.u.tbe_batch.bae_ntest;
2668                 test = 1; /* starting from test 1 */
2669
2670         } else {
2671                 fprintf(stdout,
2672                         "\tTest %d(%s) (loop: %d, concurrency: %d)\n",
2673                         test, lst_test_type2name(ent.u.tbe_test.tse_type),
2674                         ent.u.tbe_test.tse_loop,
2675                         ent.u.tbe_test.tse_concur);
2676                 ntest --;
2677                 test ++;
2678         }
2679
2680         fprintf(stdout, LST_NODES_TITLE);
2681         fprintf(stdout, "client\t%d\t%d\t%d\t%d\t%d\n"
2682                         "server\t%d\t%d\t%d\t%d\t%d\n",
2683                 ent.tbe_cli_nle.nle_nactive,
2684                 ent.tbe_cli_nle.nle_nbusy,
2685                 ent.tbe_cli_nle.nle_ndown,
2686                 ent.tbe_cli_nle.nle_nunknown,
2687                 ent.tbe_cli_nle.nle_nnode,
2688                 ent.tbe_srv_nle.nle_nactive,
2689                 ent.tbe_srv_nle.nle_nbusy,
2690                 ent.tbe_srv_nle.nle_ndown,
2691                 ent.tbe_srv_nle.nle_nunknown,
2692                 ent.tbe_srv_nle.nle_nnode);
2693
2694         if (ntest != 0)
2695                 goto loop;
2696
2697         return 0;
2698 }
2699
2700 int
2701 lst_query_batch_ioctl(char *batch, int test, int server,
2702                       int timeout, struct list_head *head)
2703 {
2704         struct lstio_batch_query_args args = { 0 };
2705
2706         args.lstio_bat_key     = session_key;
2707         args.lstio_bat_testidx = test;
2708         args.lstio_bat_client  = !(server);
2709         args.lstio_bat_timeout = timeout;
2710         args.lstio_bat_nmlen   = strlen(batch);
2711         args.lstio_bat_namep   = batch;
2712         args.lstio_bat_resultp = head;
2713
2714         return lst_ioctl(LSTIO_BATCH_QUERY, &args, sizeof(args));
2715 }
2716
2717 void
2718 lst_print_tsb_verbose(struct list_head *head,
2719                       int active, int idle, int error)
2720 {
2721         struct lstcon_rpc_ent *ent;
2722
2723         list_for_each_entry(ent, head, rpe_link) {
2724                 if (ent->rpe_priv[0] == 0 && active)
2725                         continue;
2726
2727                 if (ent->rpe_priv[0] != 0 && idle)
2728                         continue;
2729
2730                 if (ent->rpe_fwk_errno == 0 && error)
2731                         continue;
2732
2733                 fprintf(stdout, "%s [%s]: %s\n",
2734                         libcfs_id2str(ent->rpe_peer),
2735                         lst_node_state2str(ent->rpe_state),
2736                         ent->rpe_rpc_errno != 0 ?
2737                                 strerror(ent->rpe_rpc_errno) :
2738                                 (ent->rpe_priv[0] > 0 ? "Running" : "Idle"));
2739         }
2740 }
2741
2742 int
2743 jt_lst_query_batch(int argc, char **argv)
2744 {
2745         struct lstcon_test_batch_ent ent;
2746         struct list_head              head;
2747         char                   *batch   = NULL;
2748         time_t                  last    = 0;
2749         int                     optidx  = 0;
2750         int                     verbose = 0;
2751         int                     server  = 0;
2752         int                     timeout = 5; /* default 5 seconds */
2753         int                     delay   = 5; /* default 5 seconds */
2754         int                     loop    = 1; /* default 1 loop */
2755         int                     active  = 0;
2756         int                     error   = 0;
2757         int                     idle    = 0;
2758         int                     count   = 0;
2759         int                     test    = 0;
2760         int                     rc      = 0;
2761         int                     c       = 0;
2762         int                     i;
2763
2764         static struct option query_batch_opts[] =
2765         {
2766                 {"timeout", required_argument, 0, 'o' },
2767                 {"delay",   required_argument, 0, 'd' },
2768                 {"loop",    required_argument, 0, 'c' },
2769                 {"test",    required_argument, 0, 't' },
2770                 {"server",  no_argument,       0, 's' },
2771                 {"active",  no_argument,       0, 'a' },
2772                 {"idle",    no_argument,       0, 'i' },
2773                 {"error",   no_argument,       0, 'e' },
2774                 {"all",     no_argument,       0, 'l' },
2775                 {0,         0,                 0,  0  }
2776         };
2777
2778         if (session_key == 0) {
2779                 fprintf(stderr,
2780                         "Can't find env LST_SESSION or value is not valid\n");
2781                 return -1;
2782         }
2783
2784         while (1) {
2785                 c = getopt_long(argc, argv, "o:d:c:t:saiel",
2786                                 query_batch_opts, &optidx);
2787
2788                 /* Detect the end of the options. */
2789                 if (c == -1)
2790                         break;
2791
2792                 switch (c) {
2793                 case 'o':
2794                         timeout = atoi(optarg);
2795                         break;
2796                 case 'd':
2797                         delay = atoi(optarg);
2798                         break;
2799                 case 'c':
2800                         loop = atoi(optarg);
2801                         break;
2802                 case 't':
2803                         test = atoi(optarg);
2804                         break;
2805                 case 's':
2806                         server = 1;
2807                         break;
2808                 case 'a':
2809                         active = verbose = 1;
2810                         break;
2811                 case 'i':
2812                         idle = verbose = 1;
2813                         break;
2814                 case 'e':
2815                         error = verbose = 1;
2816                         break;
2817                 case 'l':
2818                         verbose = 1;
2819                         break;
2820                 default:
2821                         lst_print_usage(argv[0]);
2822                         return -1;
2823                 }
2824         }
2825
2826         if (test < 0 || timeout <= 0 || delay <= 0 || loop <= 0) {
2827                 lst_print_usage(argv[0]);
2828                 return -1;
2829         }
2830
2831         if (optind == argc) {
2832                 batch = LST_DEFAULT_BATCH;
2833
2834         } else if (optind == argc - 1) {
2835                 batch = argv[optind];
2836
2837         } else {
2838                 lst_print_usage(argv[0]);
2839                 return -1;
2840         }
2841
2842
2843         INIT_LIST_HEAD(&head);
2844
2845         if (verbose) {
2846                 rc = lst_info_batch_ioctl(batch, test, server,
2847                                           &ent, NULL, NULL, NULL);
2848                 if (rc != 0) {
2849                         fprintf(stderr, "Failed to query %s [%d]: %s\n",
2850                                 batch, test, strerror(errno));
2851                         return -1;
2852                 }
2853
2854                 count = server ? ent.tbe_srv_nle.nle_nnode :
2855                                  ent.tbe_cli_nle.nle_nnode;
2856                 if (count == 0) {
2857                         fprintf(stdout, "Batch or test is empty\n");
2858                         return 0;
2859                 }
2860         }
2861
2862         rc = lst_alloc_rpcent(&head, count, 0);
2863         if (rc != 0) {
2864                 fprintf(stderr, "Out of memory\n");
2865                 return rc;
2866         }
2867
2868         for (i = 0; i < loop; i++) {
2869                 time_t  now = time(NULL);
2870
2871                 if (now - last < delay) {
2872                         sleep(delay - now + last);
2873                         time(&now);
2874                 }
2875
2876                 last = now;
2877
2878                 rc = lst_query_batch_ioctl(batch, test,
2879                                            server, timeout, &head);
2880                 if (rc == -1) {
2881                         fprintf(stderr, "Failed to query batch: %s\n",
2882                                 strerror(errno));
2883                         break;
2884                 }
2885
2886                 if (verbose) {
2887                         /* Verbose mode */
2888                         lst_print_tsb_verbose(&head, active, idle, error);
2889                         continue;
2890                 }
2891
2892                 fprintf(stdout, "%s [%d] ", batch, test);
2893
2894                 if (lstcon_rpc_stat_failure(&trans_stat, 0) != 0) {
2895                         fprintf(stdout, "%d of %d nodes are unknown, ",
2896                                 lstcon_rpc_stat_failure(&trans_stat, 0),
2897                                 lstcon_rpc_stat_total(&trans_stat, 0));
2898                 }
2899
2900                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
2901                     lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0  &&
2902                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
2903                         fprintf(stdout, "is stopped\n");
2904                         continue;
2905                 }
2906
2907                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
2908                     lstcon_tsbqry_stat_idle(&trans_stat, 0) == 0 &&
2909                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
2910                         fprintf(stdout, "is running\n");
2911                         continue;
2912                 }
2913
2914                 fprintf(stdout, "stopped: %d , running: %d, failed: %d\n",
2915                                 lstcon_tsbqry_stat_idle(&trans_stat, 0),
2916                                 lstcon_tsbqry_stat_run(&trans_stat, 0),
2917                                 lstcon_tsbqry_stat_failure(&trans_stat, 0));
2918         }
2919
2920         lst_free_rpcent(&head);
2921
2922         return rc;
2923 }
2924
2925 int
2926 lst_parse_distribute(char *dstr, int *dist, int *span)
2927 {
2928         *dist = atoi(dstr);
2929         if (*dist <= 0)
2930                 return -1;
2931
2932         dstr = strchr(dstr, ':');
2933         if (dstr == NULL)
2934                 return -1;
2935
2936         *span = atoi(dstr + 1);
2937         if (*span <= 0)
2938                 return -1;
2939
2940         return 0;
2941 }
2942
2943 int
2944 lst_get_bulk_param(int argc, char **argv, struct lst_test_bulk_param *bulk)
2945 {
2946         char   *tok = NULL;
2947         char   *end = NULL;
2948         int     rc  = 0;
2949         int     i   = 0;
2950
2951         bulk->blk_size  = 4096;
2952         bulk->blk_opc   = LST_BRW_READ;
2953         bulk->blk_flags = LST_BRW_CHECK_NONE;
2954         bulk->blk_srv_off = bulk->blk_cli_off = 0;
2955
2956         while (i < argc) {
2957                 if (strcasestr(argv[i], "check=") == argv[i] ||
2958                     strcasestr(argv[i], "c=") == argv[i]) {
2959                         tok = strchr(argv[i], '=') + 1;
2960
2961                         if (strcasecmp(tok, "full") == 0) {
2962                                 bulk->blk_flags = LST_BRW_CHECK_FULL;
2963                         } else if (strcasecmp(tok, "simple") == 0) {
2964                                 bulk->blk_flags = LST_BRW_CHECK_SIMPLE;
2965                         } else {
2966                                 fprintf(stderr, "Unknow flag %s\n", tok);
2967                                 return -1;
2968                         }
2969
2970                 } else if (strcasestr(argv[i], "size=") == argv[i] ||
2971                            strcasestr(argv[i], "s=") == argv[i]) {
2972                         int max_size = sysconf(_SC_PAGESIZE) * LNET_MAX_IOV;
2973
2974                         tok = strchr(argv[i], '=') + 1;
2975
2976                         bulk->blk_size = strtol(tok, &end, 0);
2977                         if (bulk->blk_size <= 0) {
2978                                 fprintf(stderr, "Invalid size %s\n", tok);
2979                                 return -1;
2980                         }
2981
2982                         if (end == NULL)
2983                                 return 0;
2984
2985                         if (*end == 'k' || *end == 'K')
2986                                 bulk->blk_size *= 1024;
2987                         else if (*end == 'm' || *end == 'M')
2988                                 bulk->blk_size *= 1024 * 1024;
2989
2990                         if (bulk->blk_size > max_size) {
2991                                 fprintf(stderr, "Size exceed limitation: %d bytes\n",
2992                                         bulk->blk_size);
2993                                 return -1;
2994                         }
2995
2996                 } else if (strcasestr(argv[i], "off=") == argv[i]) {
2997                         int     off;
2998
2999                         tok = strchr(argv[i], '=') + 1;
3000
3001                         off = strtol(tok, &end, 0);
3002                         /* NB: align with sizeof(__u64) to simplify page
3003                          * checking implementation */
3004                         if (off < 0 || off % sizeof(__u64) != 0) {
3005                                 fprintf(stderr,
3006                                         "Invalid offset %s/%d, it should be "
3007                                         "postive value and multiple of %d\n",
3008                                         tok, off, (int)sizeof(__u64));
3009                                 return -1;
3010                         }
3011
3012                         /* NB: blk_srv_off is reserved so far */
3013                         bulk->blk_cli_off = bulk->blk_srv_off = off;
3014                         if (end == NULL)
3015                                 return 0;
3016
3017                 } else if (strcasecmp(argv[i], "read") == 0 ||
3018                            strcasecmp(argv[i], "r") == 0) {
3019                         bulk->blk_opc = LST_BRW_READ;
3020
3021                 } else if (strcasecmp(argv[i], "write") == 0 ||
3022                            strcasecmp(argv[i], "w") == 0) {
3023                         bulk->blk_opc = LST_BRW_WRITE;
3024
3025                 } else {
3026                         fprintf(stderr, "Unknow parameter: %s\n", argv[i]);
3027                         return -1;
3028                 }
3029
3030                 i++;
3031         }
3032
3033         return rc;
3034 }
3035
3036 int
3037 lst_get_test_param(char *test, int argc, char **argv, void **param, int *plen)
3038 {
3039         struct lst_test_bulk_param *bulk = NULL;
3040         int                    type;
3041
3042         type = lst_test_name2type(test);
3043         if (type < 0) {
3044                 fprintf(stderr, "Unknow test name %s\n", test);
3045                 return -1;
3046         }
3047
3048         switch (type) {
3049         case LST_TEST_PING:
3050                 break;
3051
3052         case LST_TEST_BULK:
3053                 bulk = malloc(sizeof(*bulk));
3054                 if (bulk == NULL) {
3055                         fprintf(stderr, "Out of memory\n");
3056                         return -1;
3057                 }
3058
3059                 memset(bulk, 0, sizeof(*bulk));
3060
3061                 if (lst_get_bulk_param(argc, argv, bulk) != 0) {
3062                         free(bulk);
3063                         return -1;
3064                 }
3065
3066                 *param = bulk;
3067                 *plen  = sizeof(*bulk);
3068
3069                 break;
3070
3071         default:
3072                 break;
3073         }
3074
3075         /* TODO: parse more parameter */
3076         return type;
3077 }
3078
3079 int
3080 lst_add_test_ioctl(char *batch, int type, int loop, int concur,
3081                    int dist, int span, char *sgrp, char *dgrp,
3082                    void *param, int plen, int *retp, struct list_head *resultp)
3083 {
3084         struct lstio_test_args args = { 0 };
3085
3086         args.lstio_tes_key        = session_key;
3087         args.lstio_tes_bat_nmlen  = strlen(batch);
3088         args.lstio_tes_bat_name   = batch;
3089         args.lstio_tes_type       = type;
3090         args.lstio_tes_oneside    = 0;
3091         args.lstio_tes_loop       = loop;
3092         args.lstio_tes_concur     = concur;
3093         args.lstio_tes_dist       = dist;
3094         args.lstio_tes_span       = span;
3095         args.lstio_tes_sgrp_nmlen = strlen(sgrp);
3096         args.lstio_tes_sgrp_name  = sgrp;
3097         args.lstio_tes_dgrp_nmlen = strlen(dgrp);
3098         args.lstio_tes_dgrp_name  = dgrp;
3099         args.lstio_tes_param_len  = plen;
3100         args.lstio_tes_param      = param;
3101         args.lstio_tes_retp       = retp;
3102         args.lstio_tes_resultp    = resultp;
3103
3104         return lst_ioctl(LSTIO_TEST_ADD, &args, sizeof(args));
3105 }
3106
3107 int
3108 jt_lst_add_test(int argc, char **argv)
3109 {
3110         struct list_head    head;
3111         char         *batch  = NULL;
3112         char         *test   = NULL;
3113         char         *dstr   = NULL;
3114         char         *from   = NULL;
3115         char         *to     = NULL;
3116         void         *param  = NULL;
3117         int           optidx = 0;
3118         int           concur = 1;
3119         int           loop   = -1;
3120         int           dist   = 1;
3121         int           span   = 1;
3122         int           plen   = 0;
3123         int           fcount = 0;
3124         int           tcount = 0;
3125         int           ret    = 0;
3126         int           type;
3127         int           rc;
3128         int           c;
3129
3130         static struct option add_test_opts[] =
3131         {
3132                 {"batch",       required_argument, 0, 'b' },
3133                 {"concurrency", required_argument, 0, 'c' },
3134                 {"distribute",  required_argument, 0, 'd' },
3135                 {"from",        required_argument, 0, 'f' },
3136                 {"to",          required_argument, 0, 't' },
3137                 {"loop",        required_argument, 0, 'l' },
3138                 {0,             0,                 0,  0  }
3139         };
3140
3141         if (session_key == 0) {
3142                 fprintf(stderr,
3143                         "Can't find env LST_SESSION or value is not valid\n");
3144                 return -1;
3145         }
3146
3147         while (1) {
3148                 c = getopt_long(argc, argv, "b:c:d:f:l:t:",
3149                                 add_test_opts, &optidx);
3150
3151                 /* Detect the end of the options. */
3152                 if (c == -1)
3153                         break;
3154
3155                 switch (c) {
3156                 case 'b':
3157                         batch = optarg;
3158                         break;
3159                 case 'c':
3160                         concur = atoi(optarg);
3161                         break;
3162                 case 'd':
3163                         dstr = optarg;
3164                         break;
3165                 case 'f':
3166                         from = optarg;
3167                         break;
3168                 case 'l':
3169                         loop = atoi(optarg);
3170                         break;
3171                 case 't':
3172                         to = optarg;
3173                         break;
3174                 default:
3175                         lst_print_usage(argv[0]);
3176                         return -1;
3177                 }
3178         }
3179
3180         if (optind == argc || from == NULL || to == NULL) {
3181                 lst_print_usage(argv[0]);
3182                 return -1;
3183         }
3184
3185         if (concur <= 0 || concur > LST_MAX_CONCUR) {
3186                 fprintf(stderr, "Invalid concurrency of test: %d\n", concur);
3187                 return -1;
3188         }
3189
3190         if (batch == NULL)
3191                 batch = LST_DEFAULT_BATCH;
3192
3193         if (dstr != NULL) {
3194                 rc = lst_parse_distribute(dstr, &dist, &span);
3195                 if (rc != 0) {
3196                         fprintf(stderr, "Invalid distribution: %s\n", dstr);
3197                         return -1;
3198                 }
3199         }
3200
3201         test = argv[optind++];
3202
3203         argc -= optind;
3204         argv += optind;
3205
3206         type = lst_get_test_param(test, argc, argv, &param, &plen);
3207         if (type < 0) {
3208                 fprintf(stderr, "Failed to add test (%s)\n", test);
3209                 return -1;
3210         }
3211
3212         INIT_LIST_HEAD(&head);
3213
3214         rc = lst_get_node_count(LST_OPC_GROUP, from, &fcount, NULL);
3215         if (rc != 0) {
3216                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3217                         from, strerror(errno));
3218                 goto out;
3219         }
3220
3221         rc = lst_get_node_count(LST_OPC_GROUP, to, &tcount, NULL);
3222         if (rc != 0) {
3223                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3224                         to, strerror(errno));
3225                 goto out;
3226         }
3227
3228         rc = lst_alloc_rpcent(&head, fcount > tcount ? fcount : tcount, 0);
3229         if (rc != 0) {
3230                 fprintf(stderr, "Out of memory\n");
3231                 goto out;
3232         }
3233
3234         rc = lst_add_test_ioctl(batch, type, loop, concur,
3235                                 dist, span, from, to, param, plen, &ret, &head);
3236
3237         if (rc == 0) {
3238                 fprintf(stdout, "Test was added successfully\n");
3239                 if (ret != 0) {
3240                         fprintf(stdout, "Server group contains userland test "
3241                                 "nodes, old version of tcplnd can't accept "
3242                                 "connection request\n");
3243                 }
3244
3245                 goto out;
3246         }
3247
3248         if (rc == -1) {
3249                 lst_print_error("test", "Failed to add test: %s\n",
3250                                 strerror(errno));
3251                 goto out;
3252         }
3253
3254         lst_print_transerr(&head, "add test");
3255 out:
3256         lst_free_rpcent(&head);
3257
3258         if (param != NULL)
3259                 free(param);
3260
3261         return rc;
3262 }
3263
3264 static command_t lst_cmdlist[] = {
3265         {"new_session",         jt_lst_new_session,     NULL,
3266          "Usage: lst new_session [--timeout TIME] [--force] [NAME]"                     },
3267         {"end_session",         jt_lst_end_session,     NULL,
3268          "Usage: lst end_session"                                                       },
3269         {"show_session",        jt_lst_show_session,    NULL,
3270          "Usage: lst show_session"                                                      },
3271         {"ping",                jt_lst_ping ,           NULL,
3272          "Usage: lst ping  [--group NAME] [--batch NAME] [--session] [--nodes IDS]"     },
3273         {"add_group",           jt_lst_add_group,       NULL,
3274          "Usage: lst group NAME IDs [IDs]..."                                           },
3275         {"del_group",           jt_lst_del_group,       NULL,
3276          "Usage: lst del_group NAME"                                                    },
3277         {"update_group",        jt_lst_update_group,    NULL,
3278          "Usage: lst update_group NAME [--clean] [--refresh] [--remove IDs]"            },
3279         {"list_group",          jt_lst_list_group,      NULL,
3280           "Usage: lst list_group [--active] [--busy] [--down] [--unknown] GROUP ..."    },
3281         {"stat",                jt_lst_stat,            NULL,
3282          "Usage: lst stat [--bw] [--rate] [--read] [--write] [--max] [--min] [--avg] "
3283          " [--mbs] [--timeout #] [--delay #] [--count #] GROUP [GROUP]"                 },
3284         {"show_error",          jt_lst_show_error,      NULL,
3285          "Usage: lst show_error NAME | IDS ..."                                         },
3286         {"add_batch",           jt_lst_add_batch,       NULL,
3287          "Usage: lst add_batch NAME"                                                    },
3288         {"run",                 jt_lst_start_batch,     NULL,
3289          "Usage: lst run [--timeout TIME] [NAME]"                                       },
3290         {"stop",                jt_lst_stop_batch,      NULL,
3291          "Usage: lst stop [--force] BATCH_NAME"                                         },
3292         {"list_batch",          jt_lst_list_batch,      NULL,
3293          "Usage: lst list_batch NAME [--test ID] [--server]"                            },
3294         {"query",               jt_lst_query_batch,     NULL,
3295          "Usage: lst query [--test ID] [--server] [--timeout TIME] NAME"                },
3296         {"add_test",            jt_lst_add_test,        NULL,
3297          "Usage: lst add_test [--batch BATCH] [--loop #] [--concurrency #] "
3298          " [--distribute #:#] [--from GROUP] [--to GROUP] TEST..."                      },
3299         {"help",                Parser_help,            0,     "help"                   },
3300         {0,                     0,                      0,      NULL                    }
3301 };
3302
3303 int
3304 lst_initialize(void)
3305 {
3306         char   *key;
3307         char   *feats;
3308
3309         feats = getenv("LST_FEATURES");
3310         if (feats != NULL)
3311                 session_features = strtol(feats, NULL, 16);
3312
3313         if ((session_features & ~LST_FEATS_MASK) != 0) {
3314                 fprintf(stderr,
3315                         "Unsupported session features %x, "
3316                         "only support these features so far: %x\n",
3317                         (session_features & ~LST_FEATS_MASK), LST_FEATS_MASK);
3318                 return -1;
3319         }
3320
3321         key = getenv("LST_SESSION");
3322
3323         if (key == NULL) {
3324                 session_key = 0;
3325                 return 0;
3326         }
3327
3328         session_key = atoi(key);
3329
3330         return 0;
3331 }
3332
3333 int
3334 main(int argc, char **argv)
3335 {
3336         int rc = 0;
3337
3338         setlinebuf(stdout);
3339
3340         rc = lst_initialize();
3341         if (rc < 0)
3342                 goto errorout;
3343
3344         rc = ptl_initialize(argc, argv);
3345         if (rc < 0)
3346                 goto errorout;
3347
3348         Parser_init("lst > ", lst_cmdlist);
3349
3350         if (argc != 1)  {
3351                 rc = Parser_execarg(argc - 1, argv + 1, lst_cmdlist);
3352                 goto errorout;
3353         }
3354
3355         Parser_commands();
3356
3357 errorout:
3358         return rc;
3359 }