lnet/utils/lst.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  *
  30  * Copyright (c) 2012, 2014, Intel Corporation.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * lnet/selftest/conctl.c
  37  *
  38  * Author: Liang Zhen <liangzhen@clusterfs.com>
  39  */
  40 #include <errno.h>
  41 #include <getopt.h>
  42 #include <inttypes.h>
  43 #include <pwd.h>
  44 #include <unistd.h>
  45 #include <stdarg.h>
  46 #include <stdio.h>
  47 #include <stdlib.h>
  48 #include <string.h>
  49 #include <sys/ioctl.h>
  50 #include <time.h>
  51
  52 #include <libcfs/list.h>
  53 #include <libcfs/util/ioctl.h>
  54 #include <libcfs/util/parser.h>
  55 #include <lnet/lnetctl.h>
  56 #include <lnet/lnetst.h>
  57 #include <lnet/lnet.h>
  58 #include <utils/obdctl.h>
  59
  60 lst_sid_t LST_INVALID_SID = {LNET_NID_ANY, -1};
  61 static lst_sid_t           session_id;
  62 static int                 session_key;
  63
  64 /* All nodes running 2.6.50 or later understand feature LST_FEAT_BULK_LEN */
  65 static unsigned         session_features = LST_FEATS_MASK;
  66 static lstcon_trans_stat_t      trans_stat;
  67
  68 typedef struct list_string {
  69         struct list_string *lstr_next;
  70         int                 lstr_sz;
  71         char                lstr_str[0];
  72 } lstr_t;
  73
  74 #ifndef offsetof
  75 # define offsetof(typ,memb)     ((unsigned long)((char *)&(((typ *)0)->memb)))
  76 #endif
  77
  78 static int alloc_count = 0;
  79 static int alloc_nob   = 0;
  80
  81 lstr_t *
  82 alloc_lstr(int sz)
  83 {
  84         lstr_t  *lstr = malloc(offsetof(lstr_t, lstr_str[sz]));
  85
  86         if (lstr == NULL) {
  87                 fprintf(stderr, "Can't allocate lstr\n");
  88                 abort();
  89         }
  90
  91         alloc_nob += sz;
  92         alloc_count++;
  93
  94         lstr->lstr_str[0] = 0;
  95         lstr->lstr_sz = sz;
  96         return lstr;
  97 }
  98
  99 void
 100 free_lstr(lstr_t *lstr)
 101 {
 102         alloc_count--;
 103         alloc_nob -= lstr->lstr_sz;
 104         free(lstr);
 105 }
 106
 107 void
 108 free_lstrs(lstr_t **list)
 109 {
 110         lstr_t   *lstr;
 111
 112         while ((lstr = *list) != NULL) {
 113                 *list = lstr->lstr_next;
 114                 free_lstr(lstr);
 115         }
 116 }
 117
 118 void
 119 new_lstrs(lstr_t **list, char *prefix, char *postfix,
 120           int lo, int hi, int stride)
 121 {
 122         int    n1 = strlen(prefix);
 123         int    n2 = strlen(postfix);
 124         int    sz = n1 + 20 + n2 + 1;
 125
 126         do {
 127                 lstr_t *n = alloc_lstr(sz);
 128
 129                 snprintf(n->lstr_str, sz - 1, "%s%u%s",
 130                          prefix, lo, postfix);
 131
 132                 n->lstr_next = *list;
 133                 *list = n;
 134
 135                 lo += stride;
 136         } while (lo <= hi);
 137 }
 138
 139 int
 140 expand_lstr(lstr_t **list, lstr_t *l)
 141 {
 142         int          nob = strlen(l->lstr_str);
 143         char        *b1;
 144         char        *b2;
 145         char        *expr;
 146         char        *sep;
 147         int          x;
 148         int          y;
 149         int          z;
 150         int          n;
 151
 152         b1 = strchr(l->lstr_str, '[');
 153         if (b1 == NULL) {
 154                 l->lstr_next = *list;
 155                 *list = l;
 156                 return 0;
 157         }
 158
 159         b2 = strchr(b1, ']');
 160         if (b2 == NULL || b2 == b1 + 1)
 161                 return -1;
 162
 163         *b1++ = 0;
 164         *b2++ = 0;
 165         expr = b1;
 166         do {
 167
 168                 sep = strchr(expr, ',');
 169                 if (sep != NULL)
 170                         *sep++ = 0;
 171
 172                 nob = strlen(expr);
 173                 n = nob;
 174                 if (sscanf(expr, "%u%n", &x, &n) >= 1 && n == nob) {
 175                         /* simple number */
 176                         new_lstrs(list, l->lstr_str, b2, x, x, 1);
 177                         continue;
 178                 }
 179
 180                 n = nob;
 181                 if (sscanf(expr, "%u-%u%n", &x, &y, &n) >= 2 && n == nob &&
 182                     x < y) {
 183                         /* simple range */
 184                         new_lstrs(list, l->lstr_str, b2, x, y, 1);
 185                         continue;
 186                 }
 187
 188                 n = nob;
 189                 if (sscanf(expr, "%u-%u/%u%n", &x, &y, &z, &n) >= 3 && n == nob &&
 190                     x < y) {
 191                         /* strided range */
 192                         new_lstrs(list, l->lstr_str, b2, x, y, z);
 193                         continue;
 194                 }
 195
 196                 /* syntax error */
 197                 return -1;
 198         } while ((expr = sep) != NULL);
 199
 200         free_lstr(l);
 201
 202         return 1;
 203 }
 204
 205 int
 206 expand_strs(char *str, lstr_t **head)
 207 {
 208         lstr_t  *list = NULL;
 209         lstr_t  *nlist;
 210         lstr_t  *l;
 211         int      rc = 0;
 212         int      expanded;
 213
 214         l = alloc_lstr(strlen(str) + 1);
 215         memcpy(l->lstr_str, str, strlen(str) + 1);
 216         l->lstr_next = NULL;
 217         list = l;
 218
 219         do {
 220                 expanded = 0;
 221                 nlist = NULL;
 222
 223                 while ((l = list) != NULL) {
 224                         list = l->lstr_next;
 225
 226                         rc = expand_lstr(&nlist, l);
 227                         if (rc < 0) {
 228                                 fprintf(stderr, "Syntax error in \"%s\"\n", str);
 229                                 free_lstr(l);
 230                                 break;
 231                         }
 232
 233                         expanded |= rc > 0;
 234                 }
 235
 236                 /* re-order onto 'list' */
 237                 while ((l = nlist) != NULL) {
 238                         nlist = l->lstr_next;
 239                         l->lstr_next = list;
 240                         list = l;
 241                 }
 242
 243         } while (expanded && rc > 0);
 244
 245         if (rc >= 0) {
 246                 *head = list;
 247                 return 0;
 248         }
 249
 250         while ((l = list) != NULL) {
 251                 list = l->lstr_next;
 252
 253                 free_lstr(l);
 254         }
 255         return rc;
 256 }
 257
 258 int
 259 lst_parse_nids(char *str, int *countp, lnet_process_id_t **idspp)
 260 {
 261         lstr_t  *head = NULL;
 262         lstr_t  *l;
 263         int      c = 0;
 264         int      i;
 265         int      rc;
 266
 267         rc = expand_strs(str, &head);
 268         if (rc != 0)
 269                 goto out;
 270
 271         l = head;
 272         while (l != NULL) {
 273                 l = l->lstr_next;
 274                 c++;
 275         }
 276
 277         *idspp = malloc(c * sizeof(lnet_process_id_t));
 278         if (*idspp == NULL) {
 279                 fprintf(stderr, "Out of memory\n");
 280                 rc = -1;
 281         }
 282
 283         *countp = c;
 284 out:
 285         i = 0;
 286         while ((l = head) != NULL) {
 287                 head = l->lstr_next;
 288
 289                 if (rc == 0) {
 290                         (*idspp)[i].nid = libcfs_str2nid(l->lstr_str);
 291                         if ((*idspp)[i].nid == LNET_NID_ANY) {
 292                                 fprintf(stderr, "Invalid nid: %s\n",
 293                                         l->lstr_str);
 294                                 rc = -1;
 295                         }
 296
 297                         (*idspp)[i].pid = LNET_PID_LUSTRE;
 298                         i++;
 299                 }
 300
 301                 free_lstr(l);
 302         }
 303
 304         if (rc == 0)
 305                 return 0;
 306
 307         free(*idspp);
 308         *idspp = NULL;
 309
 310         return rc;
 311 }
 312
 313 char *
 314 lst_node_state2str(int state)
 315 {
 316         if (state == LST_NODE_ACTIVE)
 317                 return "Active";
 318         if (state == LST_NODE_BUSY)
 319                 return "Busy";
 320         if (state == LST_NODE_DOWN)
 321                 return "Down";
 322
 323         return "Unknown";
 324 }
 325
 326 int
 327 lst_node_str2state(char *str)
 328 {
 329         if (strcasecmp(str, "active") == 0)
 330                 return LST_NODE_ACTIVE;
 331         if (strcasecmp(str, "busy") == 0)
 332                 return LST_NODE_BUSY;
 333         if (strcasecmp(str, "down") == 0)
 334                 return LST_NODE_DOWN;
 335         if (strcasecmp(str, "unknown") == 0)
 336                 return LST_NODE_UNKNOWN;
 337         if (strcasecmp(str, "invalid") == 0)
 338                 return (LST_NODE_UNKNOWN | LST_NODE_DOWN | LST_NODE_BUSY);
 339
 340         return -1;
 341 }
 342
 343 char *
 344 lst_test_type2name(int type)
 345 {
 346         if (type == LST_TEST_PING)
 347                 return "ping";
 348         if (type == LST_TEST_BULK)
 349                 return "brw";
 350
 351         return "unknown";
 352 }
 353
 354 int
 355 lst_test_name2type(char *name)
 356 {
 357         if (strcasecmp(name, "ping") == 0)
 358                 return LST_TEST_PING;
 359         if (strcasecmp(name, "brw") == 0)
 360                 return LST_TEST_BULK;
 361
 362         return -1;
 363 }
 364
 365 void
 366 lst_print_usage(char *cmd)
 367 {
 368         Parser_printhelp(cmd);
 369 }
 370
 371 void
 372 lst_print_error(char *sub, const char *def_format, ...)
 373 {
 374         va_list ap;
 375
 376         /* local error returned from kernel */
 377         switch (errno) {
 378         case ESRCH:
 379                 fprintf(stderr, "No session exists\n");
 380                 return;
 381         case ESHUTDOWN:
 382                 fprintf(stderr, "Session is shutting down\n");
 383                 return;
 384         case EACCES:
 385                 fprintf(stderr, "Unmatched session key or not root\n");
 386                 return;
 387         case ENOENT:
 388                 fprintf(stderr, "Can't find %s in current session\n", sub);
 389                 return;
 390         case EINVAL:
 391                 fprintf(stderr, "Invalid parameters list in command line\n");
 392                 return;
 393         case EFAULT:
 394                 fprintf(stderr, "Bad parameter address\n");
 395                 return;
 396         case EEXIST:
 397                 fprintf(stderr, "%s already exists\n", sub);
 398                 return;
 399         default:
 400                 va_start(ap, def_format);
 401                 vfprintf(stderr, def_format, ap);
 402                 va_end(ap);
 403
 404                 return;
 405         }
 406 }
 407
 408 void
 409 lst_free_rpcent(struct list_head *head)
 410 {
 411         lstcon_rpc_ent_t *ent;
 412
 413         while (!list_empty(head)) {
 414                 ent = list_entry(head->next, lstcon_rpc_ent_t, rpe_link);
 415
 416                 list_del(&ent->rpe_link);
 417                 free(ent);
 418         }
 419 }
 420
 421 void
 422 lst_reset_rpcent(struct list_head *head)
 423 {
 424         lstcon_rpc_ent_t *ent;
 425
 426         list_for_each_entry(ent, head, rpe_link) {
 427                 ent->rpe_sid       = LST_INVALID_SID;
 428                 ent->rpe_peer.nid  = LNET_NID_ANY;
 429                 ent->rpe_peer.pid  = LNET_PID_ANY;
 430                 ent->rpe_rpc_errno = ent->rpe_fwk_errno = 0;
 431         }
 432 }
 433
 434 int
 435 lst_alloc_rpcent(struct list_head *head, int count, int offset)
 436 {
 437         lstcon_rpc_ent_t *ent;
 438         int               i;
 439
 440         for (i = 0; i < count; i++) {
 441                 ent = malloc(offsetof(lstcon_rpc_ent_t, rpe_payload[offset]));
 442                 if (ent == NULL) {
 443                         lst_free_rpcent(head);
 444                         return -1;
 445                 }
 446
 447                 memset(ent, 0, offsetof(lstcon_rpc_ent_t, rpe_payload[offset]));
 448
 449                 ent->rpe_sid      = LST_INVALID_SID;
 450                 ent->rpe_peer.nid = LNET_NID_ANY;
 451                 ent->rpe_peer.pid = LNET_PID_ANY;
 452                 list_add(&ent->rpe_link, head);
 453         }
 454
 455         return 0;
 456 }
 457
 458 void
 459 lst_print_transerr(struct list_head *head, char *optstr)
 460 {
 461         lstcon_rpc_ent_t *ent;
 462
 463         list_for_each_entry(ent, head, rpe_link) {
 464                 if (ent->rpe_rpc_errno == 0 && ent->rpe_fwk_errno == 0)
 465                         continue;
 466
 467                 if (ent->rpe_rpc_errno != 0) {
 468                         fprintf(stderr, "%s RPC failed on %s: %s\n",
 469                                 optstr, libcfs_id2str(ent->rpe_peer),
 470                                 strerror(ent->rpe_rpc_errno));
 471                         continue;
 472                 }
 473
 474                 fprintf(stderr, "operation %s failed on %s: %s\n",
 475                         optstr, libcfs_id2str(ent->rpe_peer),
 476                         strerror(ent->rpe_fwk_errno));
 477         }
 478 }
 479
 480 int lst_info_batch_ioctl(char *batch, int test, int server,
 481                         lstcon_test_batch_ent_t *entp, int *idxp,
 482                         int *ndentp, lstcon_node_ent_t *dentsp);
 483
 484 int lst_info_group_ioctl(char *name, lstcon_ndlist_ent_t *gent,
 485                          int *idx, int *count, lstcon_node_ent_t *dents);
 486
 487 int lst_query_batch_ioctl(char *batch, int test, int server,
 488                           int timeout, struct list_head *head);
 489
 490 int
 491 lst_ioctl(unsigned int opc, void *buf, int len)
 492 {
 493         struct libcfs_ioctl_data data;
 494         int    rc;
 495
 496         LIBCFS_IOC_INIT (data);
 497         data.ioc_u32[0]  = opc;
 498         data.ioc_plen1   = len;
 499         data.ioc_pbuf1   = (char *)buf;
 500         data.ioc_plen2   = sizeof(trans_stat);
 501         data.ioc_pbuf2   = (char *)&trans_stat;
 502
 503         memset(&trans_stat, 0, sizeof(trans_stat));
 504
 505         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNETST, &data);
 506
 507         /* local error, no valid RPC result */
 508         if (rc != 0)
 509                 return -1;
 510
 511         /* RPC error */
 512         if (trans_stat.trs_rpc_errno != 0)
 513                 return -2;
 514
 515         /* Framework error */
 516         if (trans_stat.trs_fwk_errno != 0)
 517                 return -3;
 518
 519         return 0;
 520 }
 521
 522 int
 523 lst_new_session_ioctl(char *name, int timeout, int force, lst_sid_t *sid)
 524 {
 525         lstio_session_new_args_t args = {0};
 526
 527         args.lstio_ses_key     = session_key;
 528         args.lstio_ses_timeout = timeout;
 529         args.lstio_ses_force   = force;
 530         args.lstio_ses_idp     = sid;
 531         args.lstio_ses_feats   = session_features;
 532         args.lstio_ses_nmlen   = strlen(name);
 533         args.lstio_ses_namep   = name;
 534
 535         return lst_ioctl (LSTIO_SESSION_NEW, &args, sizeof(args));
 536 }
 537
 538 int
 539 jt_lst_new_session(int argc,  char **argv)
 540 {
 541         char  buf[LST_NAME_SIZE];
 542         char *name;
 543         int   optidx  = 0;
 544         int   timeout = 300;
 545         int   force   = 0;
 546         int   c;
 547         int   rc;
 548
 549         static struct option session_opts[] =
 550         {
 551                 {"timeout", required_argument,  0, 't' },
 552                 {"force",   no_argument,        0, 'f' },
 553                 {0,         0,                  0,  0  }
 554         };
 555
 556         if (session_key == 0) {
 557                 fprintf(stderr,
 558                         "Can't find env LST_SESSION or value is not valid\n");
 559                 return -1;
 560         }
 561
 562         while (1) {
 563
 564                 c = getopt_long(argc, argv, "ft:",
 565                                 session_opts, &optidx);
 566
 567                 if (c == -1)
 568                         break;
 569
 570                 switch (c) {
 571                 case 'f':
 572                         force = 1;
 573                         break;
 574                 case 't':
 575                         timeout = atoi(optarg);
 576                         break;
 577                 default:
 578                         lst_print_usage(argv[0]);
 579                         return -1;
 580                 }
 581         }
 582
 583         if (timeout <= 0) {
 584                 fprintf(stderr, "Invalid timeout value\n");
 585                 return -1;
 586         }
 587
 588         if (optind == argc - 1) {
 589                 name = argv[optind ++];
 590                 if (strlen(name) >= LST_NAME_SIZE) {
 591                         fprintf(stderr, "Name size is limited to %d\n",
 592                                 LST_NAME_SIZE - 1);
 593                         return -1;
 594                 }
 595
 596         } else if (optind == argc) {
 597                 char           user[LST_NAME_SIZE];
 598                 char           host[LST_NAME_SIZE];
 599                 struct passwd *pw = getpwuid(getuid());
 600
 601                 if (pw == NULL)
 602                         snprintf(user, sizeof(user), "%d", (int)getuid());
 603                 else
 604                         snprintf(user, sizeof(user), "%s", pw->pw_name);
 605
 606                 rc = gethostname(host, sizeof(host));
 607                 if (rc != 0)
 608                         snprintf(host, sizeof(host), "unknown_host");
 609
 610                 snprintf(buf, sizeof(buf), "%s@%s", user, host);
 611                 name = buf;
 612
 613         } else {
 614                 lst_print_usage(argv[0]);
 615                 return -1;
 616         }
 617
 618         rc = lst_new_session_ioctl(name, timeout, force, &session_id);
 619         if (rc != 0) {
 620                 lst_print_error("session", "Failed to create session: %s\n",
 621                                 strerror(errno));
 622                 return rc;
 623         }
 624
 625         fprintf(stdout, "SESSION: %s FEATURES: %x TIMEOUT: %d FORCE: %s\n",
 626                 name, session_features, timeout, force ? "Yes" : "No");
 627         return 0;
 628 }
 629
 630 int
 631 lst_session_info_ioctl(char *name, int len, int *key, unsigned *featp,
 632                        lst_sid_t *sid, lstcon_ndlist_ent_t *ndinfo)
 633 {
 634         lstio_session_info_args_t args = {0};
 635
 636         args.lstio_ses_idp     = sid;
 637         args.lstio_ses_keyp    = key;
 638         args.lstio_ses_featp   = featp;
 639         args.lstio_ses_ndinfo  = ndinfo;
 640         args.lstio_ses_nmlen   = len;
 641         args.lstio_ses_namep   = name;
 642
 643         return lst_ioctl(LSTIO_SESSION_INFO, &args, sizeof(args));
 644 }
 645
 646 int
 647 jt_lst_show_session(int argc, char **argv)
 648 {
 649         lstcon_ndlist_ent_t ndinfo;
 650         lst_sid_t           sid;
 651         char                name[LST_NAME_SIZE];
 652         unsigned            feats;
 653         int                 key;
 654         int                 rc;
 655
 656         rc = lst_session_info_ioctl(name, sizeof(name), &key,
 657                                     &feats, &sid, &ndinfo);
 658
 659         if (rc != 0) {
 660                 lst_print_error("session", "Failed to show session: %s\n",
 661                                 strerror(errno));
 662                 return -1;
 663         }
 664
 665         fprintf(stdout, "%s ID: %ju@%s, KEY: %d FEATURES: %x NODES: %d\n",
 666                 name, (uintmax_t)sid.ses_stamp, libcfs_nid2str(sid.ses_nid),
 667                 key, feats, ndinfo.nle_nnode);
 668
 669         return 0;
 670 }
 671
 672 int
 673 lst_end_session_ioctl(void)
 674 {
 675         lstio_session_end_args_t args = {0};
 676
 677         args.lstio_ses_key =  session_key;
 678         return lst_ioctl (LSTIO_SESSION_END, &args, sizeof(args));
 679 }
 680
 681 int
 682 jt_lst_end_session(int argc, char **argv)
 683 {
 684         int             rc;
 685
 686         if (session_key == 0) {
 687                 fprintf(stderr,
 688                         "Can't find env LST_SESSION or value is not valid\n");
 689                 return -1;
 690         }
 691
 692         rc = lst_end_session_ioctl();
 693
 694         if (rc == 0) {
 695                 fprintf(stdout, "session is ended\n");
 696                 return 0;
 697         }
 698
 699         if (rc == -1) {
 700                 lst_print_error("session", "Failed to end session: %s\n",
 701                                 strerror(errno));
 702                 return rc;
 703         }
 704
 705         if (trans_stat.trs_rpc_errno != 0) {
 706                 fprintf(stderr,
 707                         "[RPC] Failed to send %d session RPCs: %s\n",
 708                         lstcon_rpc_stat_failure(&trans_stat, 0),
 709                         strerror(trans_stat.trs_rpc_errno));
 710         }
 711
 712         if (trans_stat.trs_fwk_errno != 0) {
 713                 fprintf(stderr,
 714                         "[FWK] Failed to end session on %d nodes: %s\n",
 715                         lstcon_sesop_stat_failure(&trans_stat, 0),
 716                         strerror(trans_stat.trs_fwk_errno));
 717         }
 718
 719         return rc;
 720 }
 721
 722 int
 723 lst_ping_ioctl(char *str, int type, int timeout,
 724                int count, lnet_process_id_t *ids, struct list_head *head)
 725 {
 726         lstio_debug_args_t args = {0};
 727
 728         args.lstio_dbg_key     = session_key;
 729         args.lstio_dbg_type    = type;
 730         args.lstio_dbg_flags   = 0;
 731         args.lstio_dbg_timeout = timeout;
 732         args.lstio_dbg_nmlen   = (str == NULL) ? 0: strlen(str);
 733         args.lstio_dbg_namep   = str;
 734         args.lstio_dbg_count   = count;
 735         args.lstio_dbg_idsp    = ids;
 736         args.lstio_dbg_resultp = head;
 737
 738         return lst_ioctl (LSTIO_DEBUG, &args, sizeof(args));
 739 }
 740
 741 int
 742 lst_get_node_count(int type, char *str, int *countp, lnet_process_id_t **idspp)
 743 {
 744         char                    buf[LST_NAME_SIZE];
 745         lstcon_test_batch_ent_t ent;
 746         lstcon_ndlist_ent_t    *entp = &ent.tbe_cli_nle;
 747         lst_sid_t               sid;
 748         unsigned                feats;
 749         int                     key;
 750         int                     rc;
 751
 752         switch (type) {
 753         case LST_OPC_SESSION:
 754                 rc = lst_session_info_ioctl(buf, LST_NAME_SIZE,
 755                                             &key, &feats, &sid, entp);
 756                 break;
 757
 758         case LST_OPC_BATCHSRV:
 759                 entp = &ent.tbe_srv_nle;
 760         case LST_OPC_BATCHCLI:
 761                 rc = lst_info_batch_ioctl(str, 0, 0, &ent, NULL, NULL, NULL);
 762                 break;
 763
 764         case LST_OPC_GROUP:
 765                 rc = lst_info_group_ioctl(str, entp, NULL, NULL, NULL);
 766                 break;
 767
 768         case LST_OPC_NODES:
 769                 rc = lst_parse_nids(str, &entp->nle_nnode, idspp) < 0 ? -1 : 0;
 770                 break;
 771
 772         default:
 773                 rc = -1;
 774                 break;
 775         }
 776
 777         if (rc == 0)
 778                 *countp = entp->nle_nnode;
 779
 780         return rc;
 781 }
 782
 783 int
 784 jt_lst_ping(int argc,  char **argv)
 785 {
 786         struct list_head   head;
 787         lnet_process_id_t *ids = NULL;
 788         lstcon_rpc_ent_t  *ent = NULL;
 789         char              *str = NULL;
 790         int                optidx  = 0;
 791         int                server  = 0;
 792         int                timeout = 5;
 793         int                count   = 0;
 794         int                type    = 0;
 795         int                rc      = 0;
 796         int                c;
 797
 798         static struct option ping_opts[] =
 799         {
 800                 {"session", no_argument,       0, 's' },
 801                 {"server",  no_argument,       0, 'v' },
 802                 {"batch",   required_argument, 0, 'b' },
 803                 {"group",   required_argument, 0, 'g' },
 804                 {"nodes",   required_argument, 0, 'n' },
 805                 {"timeout", required_argument, 0, 't' },
 806                 {0,         0,                 0,  0  }
 807         };
 808
 809         if (session_key == 0) {
 810                 fprintf(stderr,
 811                         "Can't find env LST_SESSION or value is not valid\n");
 812                 return -1;
 813         }
 814
 815         while (1) {
 816
 817                 c = getopt_long(argc, argv, "g:b:n:t:sv",
 818                                 ping_opts, &optidx);
 819
 820                 if (c == -1)
 821                         break;
 822
 823                 switch (c) {
 824                 case 's':
 825                         type = LST_OPC_SESSION;
 826                         break;
 827
 828                 case 'g':
 829                         type = LST_OPC_GROUP;
 830                         str = optarg;
 831                         break;
 832
 833                 case 'b':
 834                         type = LST_OPC_BATCHCLI;
 835                         str = optarg;
 836                         break;
 837
 838                 case 'n':
 839                         type = LST_OPC_NODES;
 840                         str = optarg;
 841                         break;
 842
 843                 case 't':
 844                         timeout = atoi(optarg);
 845                         break;
 846
 847                 case 'v':
 848                         server = 1;
 849                         break;
 850
 851                 default:
 852                         lst_print_usage(argv[0]);
 853                         return -1;
 854                 }
 855         }
 856
 857         if (type == 0 || timeout <= 0 || optind != argc) {
 858                 lst_print_usage(argv[0]);
 859                 return -1;
 860         }
 861
 862         if (type == LST_OPC_BATCHCLI && server)
 863                 type = LST_OPC_BATCHSRV;
 864
 865         rc = lst_get_node_count(type, str, &count, &ids);
 866         if (rc < 0) {
 867                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
 868                         (str == NULL) ? "session" : str, strerror(errno));
 869                 return -1;
 870         }
 871
 872         INIT_LIST_HEAD(&head);
 873
 874         rc = lst_alloc_rpcent(&head, count, LST_NAME_SIZE);
 875         if (rc != 0) {
 876                 fprintf(stderr, "Out of memory\n");
 877                 goto out;
 878         }
 879
 880         if (count == 0) {
 881                 fprintf(stdout, "Target %s is empty\n",
 882                         (str == NULL) ? "session" : str);
 883                 goto out;
 884         }
 885
 886         rc = lst_ping_ioctl(str, type, timeout, count, ids, &head);
 887         if (rc == -1) { /* local failure */
 888                 lst_print_error("debug", "Failed to ping %s: %s\n",
 889                                 (str == NULL) ? "session" : str,
 890                                 strerror(errno));
 891                 rc = -1;
 892                 goto out;
 893         }
 894
 895         /* ignore RPC errors and framwork errors */
 896         list_for_each_entry(ent, &head, rpe_link) {
 897                 fprintf(stdout, "\t%s: %s [session: %s id: %s]\n",
 898                         libcfs_id2str(ent->rpe_peer),
 899                         lst_node_state2str(ent->rpe_state),
 900                         (ent->rpe_state == LST_NODE_ACTIVE ||
 901                          ent->rpe_state == LST_NODE_BUSY) ?
 902                                 (ent->rpe_rpc_errno == 0 ?
 903                                         &ent->rpe_payload[0] : "Unknown") :
 904                                 "<NULL>", libcfs_nid2str(ent->rpe_sid.ses_nid));
 905         }
 906
 907 out:
 908         lst_free_rpcent(&head);
 909
 910         if (ids != NULL)
 911                 free(ids);
 912
 913         return rc;
 914
 915 }
 916
 917 int
 918 lst_add_nodes_ioctl (char *name, int count, lnet_process_id_t *ids,
 919                      unsigned *featp, struct list_head *resultp)
 920 {
 921         lstio_group_nodes_args_t args = {0};
 922
 923         args.lstio_grp_key     = session_key;
 924         args.lstio_grp_nmlen   = strlen(name);
 925         args.lstio_grp_namep   = name;
 926         args.lstio_grp_count   = count;
 927         args.lstio_grp_featp   = featp;
 928         args.lstio_grp_idsp    = ids;
 929         args.lstio_grp_resultp = resultp;
 930
 931         return lst_ioctl(LSTIO_NODES_ADD, &args, sizeof(args));
 932 }
 933
 934 int
 935 lst_del_group_ioctl(char *name)
 936 {
 937         lstio_group_del_args_t args = {0};
 938
 939         args.lstio_grp_key   = session_key;
 940         args.lstio_grp_nmlen = strlen(name);
 941         args.lstio_grp_namep = name;
 942
 943         return lst_ioctl(LSTIO_GROUP_DEL, &args, sizeof(args));
 944 }
 945
 946 int
 947 lst_del_group(char *grp_name)
 948 {
 949         int     rc;
 950
 951         rc = lst_del_group_ioctl(grp_name);
 952         if (rc == 0) {
 953                 fprintf(stdout, "Group is deleted\n");
 954                 return 0;
 955         }
 956
 957         if (rc == -1) {
 958                 lst_print_error("group", "Failed to delete group: %s\n",
 959                                 strerror(errno));
 960                 return rc;
 961         }
 962
 963         fprintf(stderr, "Group is deleted with some errors\n");
 964
 965         if (trans_stat.trs_rpc_errno != 0) {
 966                 fprintf(stderr,
 967                         "[RPC] Failed to send %d end session RPCs: %s\n",
 968                         lstcon_rpc_stat_failure(&trans_stat, 0),
 969                         strerror(trans_stat.trs_rpc_errno));
 970         }
 971
 972         if (trans_stat.trs_fwk_errno != 0) {
 973                 fprintf(stderr,
 974                         "[FWK] Failed to end session on %d nodes: %s\n",
 975                 lstcon_sesop_stat_failure(&trans_stat, 0),
 976                 strerror(trans_stat.trs_fwk_errno));
 977         }
 978
 979         return -1;
 980 }
 981
 982 int
 983 lst_add_group_ioctl (char *name)
 984 {
 985         lstio_group_add_args_t args = {0};
 986
 987         args.lstio_grp_key     =  session_key;
 988         args.lstio_grp_nmlen   =  strlen(name);
 989         args.lstio_grp_namep   =  name;
 990
 991         return lst_ioctl(LSTIO_GROUP_ADD, &args, sizeof(args));
 992 }
 993
 994 int
 995 jt_lst_add_group(int argc, char **argv)
 996 {
 997         struct list_head   head;
 998         lnet_process_id_t *ids;
 999         char              *name;
1000         unsigned           feats = session_features;
1001         int                count;
1002         int                rc;
1003         int                i;
1004         bool               nodes_added = false;
1005
1006         if (session_key == 0) {
1007                 fprintf(stderr,
1008                         "Can't find env LST_SESSION or value is not valid\n");
1009                 return -1;
1010         }
1011
1012         if (argc < 3) {
1013                 lst_print_usage(argv[0]);
1014                 return -1;
1015         }
1016
1017         name = argv[1];
1018         if (strlen(name) >= LST_NAME_SIZE) {
1019                 fprintf(stderr, "Name length is limited to %d\n",
1020                         LST_NAME_SIZE - 1);
1021                 return -1;
1022         }
1023
1024         rc = lst_add_group_ioctl(name);
1025         if (rc != 0) {
1026                 lst_print_error("group", "Failed to add group %s: %s\n",
1027                                 name, strerror(errno));
1028                 return -1;
1029         }
1030
1031         INIT_LIST_HEAD(&head);
1032
1033         for (i = 2; i < argc; i++) {
1034                 /* parse address list */
1035                 rc = lst_parse_nids(argv[i], &count, &ids);
1036                 if (rc < 0) {
1037                         fprintf(stderr, "Ignore invalid id list %s\n",
1038                                 argv[i]);
1039                         continue;
1040                 }
1041
1042                 if (count == 0)
1043                         continue;
1044
1045                 rc = lst_alloc_rpcent(&head, count, 0);
1046                 if (rc != 0) {
1047                         fprintf(stderr, "Out of memory\n");
1048                         free(ids);
1049                         rc = -1;
1050                         goto failed;
1051                 }
1052
1053                 rc = lst_add_nodes_ioctl(name, count, ids, &feats, &head);
1054
1055                 free(ids);
1056
1057                 if (rc != 0)
1058                         goto failed;
1059
1060                 fprintf(stdout, "%s are added to session\n", argv[i]);
1061
1062                 nodes_added = true;
1063
1064                 if ((feats & session_features) != session_features) {
1065                         fprintf(stdout,
1066                                 "Warning, this session will run with "
1067                                 "compatible mode because some test nodes "
1068                                 "might not understand these features: %x\n",
1069                                 (~feats & session_features));
1070                 }
1071
1072                 lst_free_rpcent(&head);
1073         }
1074
1075         if (!nodes_added) {
1076                 /*
1077                  * The selftest kernel module expects that a group should
1078                  * have at least one node, since it doesn't make sense for
1079                  * an empty group to be added to a test.
1080                  */
1081                 fprintf(stderr,
1082                         "No nodes added successfully, deleting group %s\n",
1083                         name);
1084                 rc = lst_del_group(name);
1085                 if (rc != 0) {
1086                         fprintf(stderr,
1087                                 "Failed to delete group %s."
1088                                 "  Group is empty.\n", name);
1089                 }
1090         }
1091
1092         return rc;
1093
1094 failed:
1095         if (rc == -1) {
1096                 lst_print_error("group", "Failed to add nodes %s: %s\n",
1097                                 argv[i], strerror(errno));
1098
1099         } else {
1100                 if (trans_stat.trs_fwk_errno == EPROTO) {
1101                         fprintf(stderr,
1102                                 "test nodes might have different LST "
1103                                 "features, please disable some features by "
1104                                 "setting LST_FEATURES\n");
1105                 }
1106
1107                 lst_print_transerr(&head, "create session");
1108         }
1109
1110         lst_free_rpcent(&head);
1111
1112         if (!nodes_added) {
1113                 fprintf(stderr,
1114                         "No nodes added successfully, deleting group %s\n",
1115                         name);
1116                 if (lst_del_group(name) != 0) {
1117                         fprintf(stderr,
1118                                 "Failed to delete group %s."
1119                                 "  Group is empty.\n", name);
1120                 }
1121         }
1122
1123         return rc;
1124 }
1125
1126 int
1127 jt_lst_del_group(int argc, char **argv)
1128 {
1129         int     rc;
1130
1131         if (session_key == 0) {
1132                 fprintf(stderr,
1133                         "Can't find env LST_SESSION or value is not valid\n");
1134                 return -1;
1135         }
1136
1137         if (argc != 2) {
1138                 lst_print_usage(argv[0]);
1139                 return -1;
1140         }
1141
1142         rc = lst_del_group(argv[1]);
1143
1144         return rc;
1145 }
1146
1147 int
1148 lst_update_group_ioctl(int opc, char *name, int clean, int count,
1149                        lnet_process_id_t *ids, struct list_head *resultp)
1150 {
1151         lstio_group_update_args_t args = {0};
1152
1153         args.lstio_grp_key      = session_key;
1154         args.lstio_grp_opc      = opc;
1155         args.lstio_grp_args     = clean;
1156         args.lstio_grp_nmlen    = strlen(name);
1157         args.lstio_grp_namep    = name;
1158         args.lstio_grp_count    = count;
1159         args.lstio_grp_idsp     = ids;
1160         args.lstio_grp_resultp  = resultp;
1161
1162         return lst_ioctl(LSTIO_GROUP_UPDATE, &args, sizeof(args));
1163 }
1164
1165 int
1166 jt_lst_update_group(int argc, char **argv)
1167 {
1168         struct list_head         head;
1169         lnet_process_id_t *ids = NULL;
1170         char              *str = NULL;
1171         char              *grp = NULL;
1172         int                optidx = 0;
1173         int                count = 0;
1174         int                clean = 0;
1175         int                opc = 0;
1176         int                rc;
1177         int                c;
1178
1179         static struct option update_group_opts[] =
1180         {
1181                 {"refresh", no_argument,       0, 'f' },
1182                 {"clean",   required_argument, 0, 'c' },
1183                 {"remove",  required_argument, 0, 'r' },
1184                 {0,         0,                 0,  0  }
1185         };
1186
1187         if (session_key == 0) {
1188                 fprintf(stderr,
1189                         "Can't find env LST_SESSION or value is not valid\n");
1190                 return -1;
1191         }
1192
1193         while (1) {
1194                 c = getopt_long(argc, argv, "fc:r:",
1195                                 update_group_opts, &optidx);
1196
1197                 /* Detect the end of the options. */
1198                 if (c == -1)
1199                         break;
1200
1201                 switch (c) {
1202                 case 'f':
1203                         if (opc != 0) {
1204                                 lst_print_usage(argv[0]);
1205                                 return -1;
1206                         }
1207                         opc = LST_GROUP_REFRESH;
1208                         break;
1209
1210                 case 'r':
1211                         if (opc != 0) {
1212                                 lst_print_usage(argv[0]);
1213                                 return -1;
1214                         }
1215                         opc = LST_GROUP_RMND;
1216                         str = optarg;
1217                         break;
1218
1219                 case 'c':
1220                         clean = lst_node_str2state(optarg);
1221                         if (opc != 0 || clean <= 0) {
1222                                 lst_print_usage(argv[0]);
1223                                 return -1;
1224                         }
1225                         opc = LST_GROUP_CLEAN;
1226                         break;
1227
1228                 default:
1229                         lst_print_usage(argv[0]);
1230                         return -1;
1231                 }
1232         }
1233
1234         /* no OPC or group is specified */
1235         if (opc == 0 || optind != argc - 1) {
1236                 lst_print_usage(argv[0]);
1237                 return -1;
1238         }
1239
1240         grp = argv[optind];
1241
1242         INIT_LIST_HEAD(&head);
1243
1244         if (opc == LST_GROUP_RMND || opc == LST_GROUP_REFRESH) {
1245                 rc = lst_get_node_count(opc == LST_GROUP_RMND ? LST_OPC_NODES :
1246                                                                 LST_OPC_GROUP,
1247                                         opc == LST_GROUP_RMND ? str : grp,
1248                                         &count, &ids);
1249
1250                 if (rc != 0) {
1251                         fprintf(stderr, "Can't get count of nodes from %s: %s\n",
1252                                 opc == LST_GROUP_RMND ? str : grp,
1253                                 strerror(errno));
1254                         return -1;
1255                 }
1256
1257                 rc = lst_alloc_rpcent(&head, count, 0);
1258                 if (rc != 0) {
1259                         fprintf(stderr, "Out of memory\n");
1260                         free(ids);
1261                         return -1;
1262                 }
1263
1264         }
1265
1266         rc = lst_update_group_ioctl(opc, grp, clean, count, ids, &head);
1267
1268         if (ids != NULL)
1269                 free(ids);
1270
1271         if (rc == 0) {
1272                 lst_free_rpcent(&head);
1273                 return 0;
1274         }
1275
1276         if (rc == -1) {
1277                 lst_free_rpcent(&head);
1278                 lst_print_error("group", "Failed to update group: %s\n",
1279                                 strerror(errno));
1280                 return rc;
1281         }
1282
1283         lst_print_transerr(&head, "Updating group");
1284
1285         lst_free_rpcent(&head);
1286
1287         return rc;
1288 }
1289
1290 int
1291 lst_list_group_ioctl(int len, char *name, int idx)
1292 {
1293         lstio_group_list_args_t args = {0};
1294
1295         args.lstio_grp_key   = session_key;
1296         args.lstio_grp_idx   = idx;
1297         args.lstio_grp_nmlen = len;
1298         args.lstio_grp_namep = name;
1299
1300         return lst_ioctl(LSTIO_GROUP_LIST, &args, sizeof(args));
1301 }
1302
1303 int
1304 lst_info_group_ioctl(char *name, lstcon_ndlist_ent_t *gent,
1305                      int *idx, int *count, lstcon_node_ent_t *dents)
1306 {
1307         lstio_group_info_args_t args = {0};
1308
1309         args.lstio_grp_key    = session_key;
1310         args.lstio_grp_nmlen  = strlen(name);
1311         args.lstio_grp_namep  = name;
1312         args.lstio_grp_entp   = gent;
1313         args.lstio_grp_idxp   = idx;
1314         args.lstio_grp_ndentp = count;
1315         args.lstio_grp_dentsp = dents;
1316
1317         return lst_ioctl(LSTIO_GROUP_INFO, &args, sizeof(args));
1318 }
1319
1320 int
1321 lst_list_group_all(void)
1322 {
1323         char  name[LST_NAME_SIZE];
1324         int   rc;
1325         int   i;
1326
1327         /* no group is specified, list name of all groups */
1328         for (i = 0; ; i++) {
1329                 rc = lst_list_group_ioctl(LST_NAME_SIZE, name, i);
1330                 if (rc == 0) {
1331                         fprintf(stdout, "%d) %s\n", i + 1, name);
1332                         continue;
1333                 }
1334
1335                 if (errno == ENOENT)
1336                         break;
1337
1338                 lst_print_error("group", "Failed to list group: %s\n",
1339                                 strerror(errno));
1340                 return -1;
1341         }
1342
1343         fprintf(stdout, "Total %d groups\n", i);
1344
1345         return 0;
1346 }
1347
1348 #define LST_NODES_TITLE "\tACTIVE\tBUSY\tDOWN\tUNKNOWN\tTOTAL\n"
1349
1350 int
1351 jt_lst_list_group(int argc, char **argv)
1352 {
1353         lstcon_ndlist_ent_t  gent;
1354         lstcon_node_ent_t   *dents;
1355         int               optidx  = 0;
1356         int               verbose = 0;
1357         int               active  = 0;
1358         int               busy    = 0;
1359         int               down    = 0;
1360         int               unknown = 0;
1361         int               all     = 0;
1362         int               count;
1363         int               index;
1364         int               i;
1365         int               j;
1366         int               c;
1367         int               rc = 0;
1368
1369         static struct option list_group_opts[] =
1370         {
1371                 {"active",  no_argument, 0, 'a' },
1372                 {"busy",    no_argument, 0, 'b' },
1373                 {"down",    no_argument, 0, 'd' },
1374                 {"unknown", no_argument, 0, 'u' },
1375                 {"all",     no_argument, 0, 'l' },
1376                 {0,         0,           0,  0  }
1377         };
1378
1379         if (session_key == 0) {
1380                 fprintf(stderr,
1381                         "Can't find env LST_SESSION or value is not valid\n");
1382                 return -1;
1383         }
1384
1385         while (1) {
1386                 c = getopt_long(argc, argv, "abdul",
1387                                 list_group_opts, &optidx);
1388
1389                 if (c == -1)
1390                         break;
1391
1392                 switch (c) {
1393                 case 'a':
1394                         verbose = active = 1;
1395                         all = 0;
1396                         break;
1397                 case 'b':
1398                         verbose = busy = 1;
1399                         all = 0;
1400                         break;
1401                 case 'd':
1402                         verbose = down = 1;
1403                         all = 0;
1404                         break;
1405                 case 'u':
1406                         verbose = unknown = 1;
1407                         all = 0;
1408                         break;
1409                 case 'l':
1410                         verbose = all = 1;
1411                         break;
1412                 default:
1413                         lst_print_usage(argv[0]);
1414                         return -1;
1415                 }
1416         }
1417
1418         if (optind == argc) {
1419                 /* no group is specified, list name of all groups */
1420                 rc = lst_list_group_all();
1421
1422                 return rc;
1423         }
1424
1425         if (!verbose)
1426                 fprintf(stdout, LST_NODES_TITLE);
1427
1428         /* list nodes in specified groups */
1429         for (i = optind; i < argc; i++) {
1430                 rc = lst_info_group_ioctl(argv[i], &gent, NULL, NULL, NULL);
1431                 if (rc != 0) {
1432                         if (errno == ENOENT) {
1433                                 rc = 0;
1434                                 break;
1435                         }
1436
1437                         lst_print_error("group", "Failed to list group\n",
1438                                         strerror(errno));
1439                         break;
1440                 }
1441
1442                 if (!verbose) {
1443                         fprintf(stdout, "\t%d\t%d\t%d\t%d\t%d\t%s\n",
1444                                 gent.nle_nactive, gent.nle_nbusy,
1445                                 gent.nle_ndown, gent.nle_nunknown,
1446                                 gent.nle_nnode, argv[i]);
1447                         continue;
1448                 }
1449
1450                 fprintf(stdout, "Group [ %s ]\n", argv[i]);
1451
1452                 if (gent.nle_nnode == 0) {
1453                         fprintf(stdout, "No nodes found [ %s ]\n", argv[i]);
1454                         continue;
1455                 }
1456
1457                 count = gent.nle_nnode;
1458
1459                 dents = malloc(count * sizeof(lstcon_node_ent_t));
1460                 if (dents == NULL) {
1461                         fprintf(stderr, "Failed to malloc: %s\n",
1462                                 strerror(errno));
1463                         return -1;
1464                 }
1465
1466                 index = 0;
1467                 rc = lst_info_group_ioctl(argv[i], &gent, &index, &count, dents);
1468                 if (rc != 0) {
1469                         lst_print_error("group", "Failed to list group: %s\n",
1470                                         strerror(errno));
1471                         free(dents);
1472                         return -1;
1473                 }
1474
1475                 for (j = 0, c = 0; j < count; j++) {
1476                         if (all ||
1477                             ((active  &&  dents[j].nde_state == LST_NODE_ACTIVE) ||
1478                              (busy    &&  dents[j].nde_state == LST_NODE_BUSY)   ||
1479                              (down    &&  dents[j].nde_state == LST_NODE_DOWN)   ||
1480                              (unknown &&  dents[j].nde_state == LST_NODE_UNKNOWN))) {
1481
1482                                 fprintf(stdout, "\t%s: %s\n",
1483                                         libcfs_id2str(dents[j].nde_id),
1484                                         lst_node_state2str(dents[j].nde_state));
1485                                 c++;
1486                         }
1487                 }
1488
1489                 fprintf(stdout, "Total %d nodes [ %s ]\n", c, argv[i]);
1490
1491                 free(dents);
1492         }
1493
1494         return rc;
1495 }
1496
1497 int
1498 lst_stat_ioctl (char *name, int count, lnet_process_id_t *idsp,
1499                 int timeout, struct list_head *resultp)
1500 {
1501         lstio_stat_args_t args = {0};
1502
1503         args.lstio_sta_key     = session_key;
1504         args.lstio_sta_timeout = timeout;
1505         args.lstio_sta_nmlen   = strlen(name);
1506         args.lstio_sta_namep   = name;
1507         args.lstio_sta_count   = count;
1508         args.lstio_sta_idsp    = idsp;
1509         args.lstio_sta_resultp = resultp;
1510
1511         return lst_ioctl (LSTIO_STAT_QUERY, &args, sizeof(args));
1512 }
1513
1514 typedef struct {
1515         struct list_head              srp_link;
1516         int                     srp_count;
1517         char                   *srp_name;
1518         lnet_process_id_t      *srp_ids;
1519         struct list_head              srp_result[2];
1520 } lst_stat_req_param_t;
1521
1522 static void
1523 lst_stat_req_param_free(lst_stat_req_param_t *srp)
1524 {
1525         int     i;
1526
1527         for (i = 0; i < 2; i++)
1528                 lst_free_rpcent(&srp->srp_result[i]);
1529
1530         if (srp->srp_ids != NULL)
1531                 free(srp->srp_ids);
1532
1533         free(srp);
1534 }
1535
1536 static int
1537 lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp, int save_old)
1538 {
1539         lst_stat_req_param_t *srp = NULL;
1540         int                   count = save_old ? 2 : 1;
1541         int                   rc;
1542         int                   i;
1543
1544         srp = malloc(sizeof(*srp));
1545         if (srp == NULL)
1546                 return -ENOMEM;
1547
1548         memset(srp, 0, sizeof(*srp));
1549         INIT_LIST_HEAD(&srp->srp_result[0]);
1550         INIT_LIST_HEAD(&srp->srp_result[1]);
1551
1552         rc = lst_get_node_count(LST_OPC_GROUP, name,
1553                                 &srp->srp_count, NULL);
1554         if (rc != 0 && errno == ENOENT) {
1555                 rc = lst_get_node_count(LST_OPC_NODES, name,
1556                                         &srp->srp_count, &srp->srp_ids);
1557         }
1558
1559         if (rc != 0) {
1560                 fprintf(stderr,
1561                         "Failed to get count of nodes from %s: %s\n",
1562                         name, strerror(errno));
1563                 lst_stat_req_param_free(srp);
1564
1565                 return rc;
1566         }
1567
1568         srp->srp_name = name;
1569
1570         for (i = 0; i < count; i++) {
1571                 rc = lst_alloc_rpcent(&srp->srp_result[i], srp->srp_count,
1572                                       sizeof(sfw_counters_t)  +
1573                                       sizeof(srpc_counters_t) +
1574                                       sizeof(lnet_counters_t));
1575                 if (rc != 0) {
1576                         fprintf(stderr, "Out of memory\n");
1577                         break;
1578                 }
1579         }
1580
1581         if (rc == 0) {
1582                 *srpp = srp;
1583                 return 0;
1584         }
1585
1586         lst_stat_req_param_free(srp);
1587
1588         return rc;
1589 }
1590
1591 typedef struct {
1592         /* TODO */
1593         int foo;
1594 } lst_srpc_stat_result;
1595
1596 #define LST_LNET_AVG    0
1597 #define LST_LNET_MIN    1
1598 #define LST_LNET_MAX    2
1599
1600 typedef struct {
1601         float           lnet_avg_sndrate;
1602         float           lnet_min_sndrate;
1603         float           lnet_max_sndrate;
1604         float           lnet_total_sndrate;
1605
1606         float           lnet_avg_rcvrate;
1607         float           lnet_min_rcvrate;
1608         float           lnet_max_rcvrate;
1609         float           lnet_total_rcvrate;
1610
1611         float           lnet_avg_sndperf;
1612         float           lnet_min_sndperf;
1613         float           lnet_max_sndperf;
1614         float           lnet_total_sndperf;
1615
1616         float           lnet_avg_rcvperf;
1617         float           lnet_min_rcvperf;
1618         float           lnet_max_rcvperf;
1619         float           lnet_total_rcvperf;
1620
1621         int             lnet_stat_count;
1622 } lst_lnet_stat_result_t;
1623
1624 lst_lnet_stat_result_t lnet_stat_result;
1625
1626 static float
1627 lst_lnet_stat_value(int bw, int send, int off)
1628 {
1629         float  *p;
1630
1631         p = bw ? &lnet_stat_result.lnet_avg_sndperf :
1632                  &lnet_stat_result.lnet_avg_sndrate;
1633
1634         if (!send)
1635                 p += 4;
1636
1637         p += off;
1638
1639         return *p;
1640 }
1641
1642 static void
1643 lst_timeval_diff(struct timeval *tv1,
1644                  struct timeval *tv2, struct timeval *df)
1645 {
1646         if (tv1->tv_usec >= tv2->tv_usec) {
1647                 df->tv_sec  = tv1->tv_sec - tv2->tv_sec;
1648                 df->tv_usec = tv1->tv_usec - tv2->tv_usec;
1649                 return;
1650         }
1651
1652         df->tv_sec  = tv1->tv_sec - 1 - tv2->tv_sec;
1653         df->tv_usec = tv1->tv_usec + 1000000 - tv2->tv_usec;
1654
1655         return;
1656 }
1657
1658 void
1659 lst_cal_lnet_stat(float delta, lnet_counters_t *lnet_new,
1660                   lnet_counters_t *lnet_old)
1661 {
1662         float perf;
1663         float rate;
1664
1665         perf = (float)(lnet_new->send_length -
1666                        lnet_old->send_length) / (1024 * 1024) / delta;
1667         lnet_stat_result.lnet_total_sndperf += perf;
1668
1669         if (lnet_stat_result.lnet_min_sndperf > perf ||
1670             lnet_stat_result.lnet_min_sndperf == 0)
1671                 lnet_stat_result.lnet_min_sndperf = perf;
1672
1673         if (lnet_stat_result.lnet_max_sndperf < perf)
1674                 lnet_stat_result.lnet_max_sndperf = perf;
1675
1676         perf = (float)(lnet_new->recv_length -
1677                        lnet_old->recv_length) / (1024 * 1024) / delta;
1678         lnet_stat_result.lnet_total_rcvperf += perf;
1679
1680         if (lnet_stat_result.lnet_min_rcvperf > perf ||
1681             lnet_stat_result.lnet_min_rcvperf == 0)
1682                 lnet_stat_result.lnet_min_rcvperf = perf;
1683
1684         if (lnet_stat_result.lnet_max_rcvperf < perf)
1685                 lnet_stat_result.lnet_max_rcvperf = perf;
1686
1687         rate = (lnet_new->send_count - lnet_old->send_count) / delta;
1688         lnet_stat_result.lnet_total_sndrate += rate;
1689
1690         if (lnet_stat_result.lnet_min_sndrate > rate ||
1691             lnet_stat_result.lnet_min_sndrate == 0)
1692                 lnet_stat_result.lnet_min_sndrate = rate;
1693
1694         if (lnet_stat_result.lnet_max_sndrate < rate)
1695                 lnet_stat_result.lnet_max_sndrate = rate;
1696
1697         rate = (lnet_new->recv_count - lnet_old->recv_count) / delta;
1698         lnet_stat_result.lnet_total_rcvrate += rate;
1699
1700         if (lnet_stat_result.lnet_min_rcvrate > rate ||
1701             lnet_stat_result.lnet_min_rcvrate == 0)
1702                 lnet_stat_result.lnet_min_rcvrate = rate;
1703
1704         if (lnet_stat_result.lnet_max_rcvrate < rate)
1705                 lnet_stat_result.lnet_max_rcvrate = rate;
1706
1707         lnet_stat_result.lnet_stat_count ++;
1708
1709         lnet_stat_result.lnet_avg_sndrate = lnet_stat_result.lnet_total_sndrate /
1710                                             lnet_stat_result.lnet_stat_count;
1711         lnet_stat_result.lnet_avg_rcvrate = lnet_stat_result.lnet_total_rcvrate /
1712                                             lnet_stat_result.lnet_stat_count;
1713
1714         lnet_stat_result.lnet_avg_sndperf = lnet_stat_result.lnet_total_sndperf /
1715                                             lnet_stat_result.lnet_stat_count;
1716         lnet_stat_result.lnet_avg_rcvperf = lnet_stat_result.lnet_total_rcvperf /
1717                                             lnet_stat_result.lnet_stat_count;
1718
1719 }
1720
1721 void
1722 lst_print_lnet_stat(char *name, int bwrt, int rdwr, int type)
1723 {
1724         int     start1 = 0;
1725         int     end1   = 1;
1726         int     start2 = 0;
1727         int     end2   = 1;
1728         int     i;
1729         int     j;
1730
1731         if (lnet_stat_result.lnet_stat_count == 0)
1732                 return;
1733
1734         if (bwrt == 1) /* bw only */
1735                 start1 = 1;
1736
1737         if (bwrt == 2) /* rates only */
1738                 end1 = 0;
1739
1740         if (rdwr == 1) /* recv only */
1741                 start2 = 1;
1742
1743         if (rdwr == 2) /* send only */
1744                 end2 = 0;
1745
1746         for (i = start1; i <= end1; i++) {
1747                 fprintf(stdout, "[LNet %s of %s]\n",
1748                         i == 0 ? "Rates" : "Bandwidth", name);
1749
1750                 for (j = start2; j <= end2; j++) {
1751                         fprintf(stdout, "[%c] ", j == 0 ? 'R' : 'W');
1752
1753                         if ((type & 1) != 0) {
1754                                 fprintf(stdout, i == 0 ? "Avg: %-8.0f RPC/s " :
1755                                                          "Avg: %-8.2f MB/s  ",
1756                                         lst_lnet_stat_value(i, j, 0));
1757                         }
1758
1759                         if ((type & 2) != 0) {
1760                                 fprintf(stdout, i == 0 ? "Min: %-8.0f RPC/s " :
1761                                                          "Min: %-8.2f MB/s  ",
1762                                         lst_lnet_stat_value(i, j, 1));
1763                         }
1764
1765                         if ((type & 4) != 0) {
1766                                 fprintf(stdout, i == 0 ? "Max: %-8.0f RPC/s" :
1767                                                          "Max: %-8.2f MB/s",
1768                                         lst_lnet_stat_value(i, j, 2));
1769                         }
1770
1771                         fprintf(stdout, "\n");
1772                 }
1773         }
1774 }
1775
1776 void
1777 lst_print_stat(char *name, struct list_head *resultp,
1778                int idx, int lnet, int bwrt, int rdwr, int type)
1779 {
1780         struct list_head        tmp[2];
1781         lstcon_rpc_ent_t *new;
1782         lstcon_rpc_ent_t *old;
1783         sfw_counters_t   *sfwk_new;
1784         sfw_counters_t   *sfwk_old;
1785         srpc_counters_t  *srpc_new;
1786         srpc_counters_t  *srpc_old;
1787         lnet_counters_t  *lnet_new;
1788         lnet_counters_t  *lnet_old;
1789         float             delta;
1790         int               errcount = 0;
1791
1792         INIT_LIST_HEAD(&tmp[0]);
1793         INIT_LIST_HEAD(&tmp[1]);
1794
1795         memset(&lnet_stat_result, 0, sizeof(lnet_stat_result));
1796
1797         while (!list_empty(&resultp[idx])) {
1798                 if (list_empty(&resultp[1 - idx])) {
1799                         fprintf(stderr, "Group is changed, re-run stat\n");
1800                         break;
1801                 }
1802
1803                 new = list_entry(resultp[idx].next, lstcon_rpc_ent_t,
1804                                      rpe_link);
1805                 old = list_entry(resultp[1 - idx].next, lstcon_rpc_ent_t,
1806                                      rpe_link);
1807
1808                 /* first time get stats result, can't calculate diff */
1809                 if (new->rpe_peer.nid == LNET_NID_ANY)
1810                         break;
1811
1812                 if (new->rpe_peer.nid != old->rpe_peer.nid ||
1813                     new->rpe_peer.pid != old->rpe_peer.pid) {
1814                         /* Something wrong. i.e, somebody change the group */
1815                         break;
1816                 }
1817
1818                 list_del(&new->rpe_link);
1819                 list_add_tail(&new->rpe_link, &tmp[idx]);
1820
1821                 list_del(&old->rpe_link);
1822                 list_add_tail(&old->rpe_link, &tmp[1 - idx]);
1823
1824                 if (new->rpe_rpc_errno != 0 || new->rpe_fwk_errno != 0 ||
1825                     old->rpe_rpc_errno != 0 || old->rpe_fwk_errno != 0) {
1826                         errcount ++;
1827                         continue;
1828                 }
1829
1830                 sfwk_new = (sfw_counters_t *)&new->rpe_payload[0];
1831                 sfwk_old = (sfw_counters_t *)&old->rpe_payload[0];
1832
1833                 srpc_new = (srpc_counters_t *)((char *)sfwk_new + sizeof(*sfwk_new));
1834                 srpc_old = (srpc_counters_t *)((char *)sfwk_old + sizeof(*sfwk_old));
1835
1836                 lnet_new = (lnet_counters_t *)((char *)srpc_new + sizeof(*srpc_new));
1837                 lnet_old = (lnet_counters_t *)((char *)srpc_old + sizeof(*srpc_old));
1838
1839                 /* Prior to version 2.3, the running_ms field was a counter for
1840                  * the number of running tests.  We are looking at this value
1841                  * to determine if it is a millisecond timestamep (>= 2.3) or a
1842                  * test counter (< 2.3).  The number 500 is being used for this
1843                  * barrier as the test counter should never get this high, and
1844                  * the timestamp should never get this low. */
1845
1846                 if (sfwk_new->running_ms > 500) {
1847                         /* use the timestamp from the remote node, not our
1848                          * rpe_stamp from when we copied up the data out of
1849                          * the kernel */
1850
1851                         delta = (float) (sfwk_new->running_ms -
1852                                         sfwk_old->running_ms) / 1000;
1853                 } else {
1854                         struct timeval    tv;
1855
1856                         lst_timeval_diff(&new->rpe_stamp, &old->rpe_stamp, &tv);
1857                         delta = tv.tv_sec + (float)tv.tv_usec / 1000000;
1858                 }
1859
1860                 if (!lnet) /* TODO */
1861                         continue;
1862
1863                 lst_cal_lnet_stat(delta, lnet_new, lnet_old);
1864         }
1865
1866         list_splice(&tmp[idx], &resultp[idx]);
1867         list_splice(&tmp[1 - idx], &resultp[1 - idx]);
1868
1869         if (errcount > 0)
1870                 fprintf(stdout, "Failed to stat on %d nodes\n", errcount);
1871
1872         if (!lnet)  /* TODO */
1873                 return;
1874
1875         lst_print_lnet_stat(name, bwrt, rdwr, type);
1876 }
1877
1878 int
1879 jt_lst_stat(int argc, char **argv)
1880 {
1881         struct list_head        head;
1882         lst_stat_req_param_t *srp;
1883         time_t                last    = 0;
1884         int                   optidx  = 0;
1885         int                   timeout = 5; /* default timeout, 5 sec */
1886         int                   delay   = 5; /* default delay, 5 sec */
1887         int                   count   = -1; /* run forever */
1888         int                   lnet    = 1; /* lnet stat by default */
1889         int                   bwrt    = 0;
1890         int                   rdwr    = 0;
1891         int                   type    = -1;
1892         int                   idx     = 0;
1893         int                   rc;
1894         int                   c;
1895
1896         static struct option stat_opts[] =
1897         {
1898                 {"timeout"   , required_argument, 0, 't' },
1899                 {"delay"     , required_argument, 0, 'd' },
1900                 {"count"     , required_argument, 0, 'o' },
1901                 {"lnet"      , no_argument,      0, 'l' },
1902                 {"rpc"       , no_argument,      0, 'c' },
1903                 {"bw"        , no_argument,      0, 'b' },
1904                 {"rate"      , no_argument,      0, 'a' },
1905                 {"read"      , no_argument,      0, 'r' },
1906                 {"write"     , no_argument,      0, 'w' },
1907                 {"avg"       , no_argument,      0, 'g' },
1908                 {"min"       , no_argument,      0, 'n' },
1909                 {"max"       , no_argument,      0, 'x' },
1910                 {0,            0,                0,  0  }
1911         };
1912
1913         if (session_key == 0) {
1914                 fprintf(stderr,
1915                         "Can't find env LST_SESSION or value is not valid\n");
1916                 return -1;
1917         }
1918
1919         while (1) {
1920                 c = getopt_long(argc, argv, "t:d:lcbarwgnx", stat_opts, &optidx);
1921
1922                 if (c == -1)
1923                         break;
1924
1925                 switch (c) {
1926                 case 't':
1927                         timeout = atoi(optarg);
1928                         break;
1929                 case 'd':
1930                         delay = atoi(optarg);
1931                         break;
1932                 case 'o':
1933                         count = atoi(optarg);
1934                         break;
1935                 case 'l':
1936                         lnet = 1;
1937                         break;
1938                 case 'c':
1939                         lnet = 0;
1940                         break;
1941                 case 'b':
1942                         bwrt |= 1;
1943                         break;
1944                 case 'a':
1945                         bwrt |= 2;
1946                         break;
1947                 case 'r':
1948                         rdwr |= 1;
1949                         break;
1950                 case 'w':
1951                         rdwr |= 2;
1952                         break;
1953                 case 'g':
1954                         if (type == -1) {
1955                                 type = 1;
1956                                 break;
1957                         }
1958                         type |= 1;
1959                         break;
1960                 case 'n':
1961                         if (type == -1) {
1962                                 type = 2;
1963                                 break;
1964                         }
1965                         type |= 2;
1966                         break;
1967                 case 'x':
1968                         if (type == -1) {
1969                                 type = 4;
1970                                 break;
1971                         }
1972                         type |= 4;
1973                         break;
1974
1975                 default:
1976                         lst_print_usage(argv[0]);
1977                         return -1;
1978                 }
1979         }
1980
1981         if (optind == argc) {
1982                 lst_print_usage(argv[0]);
1983                 return -1;
1984         }
1985
1986         if (timeout <= 0 || delay <= 0) {
1987                 fprintf(stderr, "Invalid timeout or delay value\n");
1988                 return -1;
1989         }
1990
1991         if (count < -1) {
1992             fprintf(stderr, "Invalid count value\n");
1993             return -1;
1994         }
1995
1996         /* extra count to get first data point */
1997         if (count != -1)
1998             count++;
1999
2000         INIT_LIST_HEAD(&head);
2001
2002         while (optind < argc) {
2003                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 1);
2004                 if (rc != 0)
2005                         goto out;
2006
2007                 list_add_tail(&srp->srp_link, &head);
2008         }
2009
2010         do {
2011                 time_t  now = time(NULL);
2012
2013                 if (now - last < delay) {
2014                         sleep(delay - now + last);
2015                         time(&now);
2016                 }
2017                 last = now;
2018
2019                 list_for_each_entry(srp, &head, srp_link) {
2020                         rc = lst_stat_ioctl(srp->srp_name,
2021                                             srp->srp_count, srp->srp_ids,
2022                                             timeout, &srp->srp_result[idx]);
2023                         if (rc == -1) {
2024                                 lst_print_error("stat", "Failed to stat %s: %s\n",
2025                                                 srp->srp_name, strerror(errno));
2026                                 goto out;
2027                         }
2028
2029                         lst_print_stat(srp->srp_name, srp->srp_result,
2030                                        idx, lnet, bwrt, rdwr, type);
2031
2032                         lst_reset_rpcent(&srp->srp_result[1 - idx]);
2033                 }
2034
2035                 idx = 1 - idx;
2036
2037                 if (count > 0)
2038                         count--;
2039         } while (count == -1 || count > 0);
2040
2041 out:
2042         while (!list_empty(&head)) {
2043                 srp = list_entry(head.next, lst_stat_req_param_t, srp_link);
2044
2045                 list_del(&srp->srp_link);
2046                 lst_stat_req_param_free(srp);
2047         }
2048
2049         return rc;
2050 }
2051
2052 int
2053 jt_lst_show_error(int argc, char **argv)
2054 {
2055         struct list_head            head;
2056         lst_stat_req_param_t *srp;
2057         lstcon_rpc_ent_t     *ent;
2058         sfw_counters_t       *sfwk;
2059         srpc_counters_t      *srpc;
2060         int                   show_rpc = 1;
2061         int                   optidx   = 0;
2062         int                   rc       = 0;
2063         int                   ecount;
2064         int                   c;
2065
2066         static struct option  show_error_opts[] =
2067         {
2068                 {"session", no_argument,       0, 's' },
2069                 {0,         0,                 0,  0  }
2070         };
2071
2072         if (session_key == 0) {
2073                 fprintf(stderr,
2074                         "Can't find env LST_SESSION or value is not valid\n");
2075                 return -1;
2076         }
2077
2078         while (1) {
2079                 c = getopt_long(argc, argv, "s", show_error_opts, &optidx);
2080
2081                 if (c == -1)
2082                         break;
2083
2084                 switch (c) {
2085                 case 's':
2086                         show_rpc  = 0;
2087                         break;
2088
2089                 default:
2090                         lst_print_usage(argv[0]);
2091                         return -1;
2092                 }
2093         }
2094
2095         if (optind == argc) {
2096                 lst_print_usage(argv[0]);
2097                 return -1;
2098         }
2099
2100         INIT_LIST_HEAD(&head);
2101
2102         while (optind < argc) {
2103                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 0);
2104                 if (rc != 0)
2105                         goto out;
2106
2107                 list_add_tail(&srp->srp_link, &head);
2108         }
2109
2110         list_for_each_entry(srp, &head, srp_link) {
2111                 rc = lst_stat_ioctl(srp->srp_name, srp->srp_count,
2112                                     srp->srp_ids, 10, &srp->srp_result[0]);
2113
2114                 if (rc == -1) {
2115                         lst_print_error(srp->srp_name, "Failed to show errors of %s: %s\n",
2116                                         srp->srp_name, strerror(errno));
2117                         goto out;
2118                 }
2119
2120                 fprintf(stdout, "%s:\n", srp->srp_name);
2121
2122                 ecount = 0;
2123
2124                 list_for_each_entry(ent, &srp->srp_result[0], rpe_link) {
2125                         if (ent->rpe_rpc_errno != 0) {
2126                                 ecount ++;
2127                                 fprintf(stderr, "RPC failure, can't show error on %s\n",
2128                                         libcfs_id2str(ent->rpe_peer));
2129                                 continue;
2130                         }
2131
2132                         if (ent->rpe_fwk_errno != 0) {
2133                                 ecount ++;
2134                                 fprintf(stderr, "Framework failure, can't show error on %s\n",
2135                                         libcfs_id2str(ent->rpe_peer));
2136                                 continue;
2137                         }
2138
2139                         sfwk = (sfw_counters_t *)&ent->rpe_payload[0];
2140                         srpc = (srpc_counters_t *)((char *)sfwk + sizeof(*sfwk));
2141
2142                         if (srpc->errors == 0 &&
2143                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2144                                 continue;
2145
2146                         if (!show_rpc  &&
2147                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2148                                 continue;
2149
2150                         ecount ++;
2151
2152                         fprintf(stderr, "%s: [Session %d brw errors, %d ping errors]%c",
2153                                 libcfs_id2str(ent->rpe_peer),
2154                                 sfwk->brw_errors, sfwk->ping_errors,
2155                                 show_rpc  ? ' ' : '\n');
2156
2157                         if (!show_rpc)
2158                                 continue;
2159
2160                         fprintf(stderr, "[RPC: %d errors, %d dropped, %d expired]\n",
2161                                 srpc->errors, srpc->rpcs_dropped, srpc->rpcs_expired);
2162                 }
2163
2164                 fprintf(stdout, "Total %d error nodes in %s\n", ecount, srp->srp_name);
2165         }
2166 out:
2167         while (!list_empty(&head)) {
2168                 srp = list_entry(head.next, lst_stat_req_param_t, srp_link);
2169
2170                 list_del(&srp->srp_link);
2171                 lst_stat_req_param_free(srp);
2172         }
2173
2174         return rc;
2175 }
2176
2177 int
2178 lst_add_batch_ioctl (char *name)
2179 {
2180         lstio_batch_add_args_t args = {0};
2181
2182         args.lstio_bat_key   = session_key;
2183         args.lstio_bat_nmlen = strlen(name);
2184         args.lstio_bat_namep = name;
2185
2186         return lst_ioctl (LSTIO_BATCH_ADD, &args, sizeof(args));
2187 }
2188
2189 int
2190 jt_lst_add_batch(int argc, char **argv)
2191 {
2192         char   *name;
2193         int     rc;
2194
2195         if (session_key == 0) {
2196                 fprintf(stderr,
2197                         "Can't find env LST_SESSION or value is not valid\n");
2198                 return -1;
2199         }
2200
2201         if (argc != 2) {
2202                 lst_print_usage(argv[0]);
2203                 return -1;
2204         }
2205
2206         name = argv[1];
2207         if (strlen(name) >= LST_NAME_SIZE) {
2208                 fprintf(stderr, "Name length is limited to %d\n",
2209                         LST_NAME_SIZE - 1);
2210                 return -1;
2211         }
2212
2213         rc = lst_add_batch_ioctl(name);
2214         if (rc == 0)
2215                 return 0;
2216
2217         lst_print_error("batch", "Failed to create batch: %s\n",
2218                         strerror(errno));
2219
2220         return -1;
2221 }
2222
2223 int
2224 lst_start_batch_ioctl(char *name, int timeout, struct list_head *resultp)
2225 {
2226         lstio_batch_run_args_t args = {0};
2227
2228         args.lstio_bat_key     = session_key;
2229         args.lstio_bat_timeout = timeout;
2230         args.lstio_bat_nmlen   = strlen(name);
2231         args.lstio_bat_namep   = name;
2232         args.lstio_bat_resultp = resultp;
2233
2234         return lst_ioctl(LSTIO_BATCH_START, &args, sizeof(args));
2235 }
2236
2237 int
2238 jt_lst_start_batch(int argc, char **argv)
2239 {
2240         struct list_head        head;
2241         char             *batch;
2242         int               optidx  = 0;
2243         int               timeout = 0;
2244         int               count = 0;
2245         int               rc;
2246         int               c;
2247
2248         static struct option start_batch_opts[] =
2249         {
2250                 {"timeout", required_argument, 0, 't' },
2251                 {0,         0,                 0,  0  }
2252         };
2253
2254         if (session_key == 0) {
2255                 fprintf(stderr,
2256                         "Can't find env LST_SESSION or value is not valid\n");
2257                 return -1;
2258         }
2259
2260         while (1) {
2261                 c = getopt_long(argc, argv, "t:",
2262                                 start_batch_opts, &optidx);
2263
2264                 /* Detect the end of the options. */
2265                 if (c == -1)
2266                         break;
2267
2268                 switch (c) {
2269                 case 't':
2270                         timeout = atoi(optarg);
2271                         break;
2272                 default:
2273                         lst_print_usage(argv[0]);
2274                         return -1;
2275                 }
2276         }
2277
2278         if (optind == argc) {
2279                 batch = LST_DEFAULT_BATCH;
2280
2281         } else if (optind == argc - 1) {
2282                 batch = argv[optind];
2283
2284         } else {
2285                 lst_print_usage(argv[0]);
2286                 return -1;
2287         }
2288
2289         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2290         if (rc != 0) {
2291                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2292                         batch, strerror(errno));
2293                 return -1;
2294         }
2295
2296         INIT_LIST_HEAD(&head);
2297
2298         rc = lst_alloc_rpcent(&head, count, 0);
2299         if (rc != 0) {
2300                 fprintf(stderr, "Out of memory\n");
2301                 return -1;
2302         }
2303
2304         rc = lst_start_batch_ioctl(batch, timeout, &head);
2305
2306         if (rc == 0) {
2307                 fprintf(stdout, "%s is running now\n", batch);
2308                 lst_free_rpcent(&head);
2309                 return 0;
2310         }
2311
2312         if (rc == -1) {
2313                 lst_print_error("batch", "Failed to start batch: %s\n",
2314                                 strerror(errno));
2315                 lst_free_rpcent(&head);
2316                 return rc;
2317         }
2318
2319         lst_print_transerr(&head, "Run batch");
2320
2321         lst_free_rpcent(&head);
2322
2323         return rc;
2324 }
2325
2326 int
2327 lst_stop_batch_ioctl(char *name, int force, struct list_head *resultp)
2328 {
2329         lstio_batch_stop_args_t args = {0};
2330
2331         args.lstio_bat_key     = session_key;
2332         args.lstio_bat_force   = force;
2333         args.lstio_bat_nmlen   = strlen(name);
2334         args.lstio_bat_namep   = name;
2335         args.lstio_bat_resultp = resultp;
2336
2337         return lst_ioctl(LSTIO_BATCH_STOP, &args, sizeof(args));
2338 }
2339
2340 int
2341 jt_lst_stop_batch(int argc, char **argv)
2342 {
2343         struct list_head        head;
2344         char             *batch;
2345         int               force = 0;
2346         int               optidx;
2347         int               count;
2348         int               rc;
2349         int               c;
2350
2351         static struct option stop_batch_opts[] =
2352         {
2353                 {"force",   no_argument,   0, 'f' },
2354                 {0,         0,             0,  0  }
2355         };
2356
2357         if (session_key == 0) {
2358                 fprintf(stderr,
2359                         "Can't find env LST_SESSION or value is not valid\n");
2360                 return -1;
2361         }
2362
2363         while (1) {
2364                 c = getopt_long(argc, argv, "f",
2365                                 stop_batch_opts, &optidx);
2366
2367                 /* Detect the end of the options. */
2368                 if (c == -1)
2369                         break;
2370
2371                 switch (c) {
2372                 case 'f':
2373                         force = 1;
2374                         break;
2375                 default:
2376                         lst_print_usage(argv[0]);
2377                         return -1;
2378                 }
2379         }
2380
2381         if (optind == argc) {
2382                 batch = LST_DEFAULT_BATCH;
2383
2384         } else if (optind == argc - 1) {
2385                 batch = argv[optind];
2386
2387         } else {
2388                 lst_print_usage(argv[0]);
2389                 return -1;
2390         }
2391
2392         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2393         if (rc != 0) {
2394                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2395                         batch, strerror(errno));
2396                 return -1;
2397         }
2398
2399         INIT_LIST_HEAD(&head);
2400
2401         rc = lst_alloc_rpcent(&head, count, 0);
2402         if (rc != 0) {
2403                 fprintf(stderr, "Out of memory\n");
2404                 return -1;
2405         }
2406
2407         rc = lst_stop_batch_ioctl(batch, force, &head);
2408         if (rc != 0)
2409                 goto out;
2410
2411         while (1) {
2412                 lst_reset_rpcent(&head);
2413
2414                 rc = lst_query_batch_ioctl(batch, 0, 0, 30, &head);
2415                 if (rc != 0)
2416                         goto out;
2417
2418                 if (lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0 &&
2419                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0)
2420                         break;
2421
2422                 fprintf(stdout, "%d batch in stopping\n",
2423                         lstcon_tsbqry_stat_run(&trans_stat, 0));
2424                 sleep(1);
2425         }
2426
2427         fprintf(stdout, "Batch is stopped\n");
2428         lst_free_rpcent(&head);
2429
2430         return 0;
2431 out:
2432         if (rc == -1) {
2433                 lst_print_error("batch", "Failed to stop batch: %s\n",
2434                                 strerror(errno));
2435                 lst_free_rpcent(&head);
2436                 return -1;
2437         }
2438
2439         lst_print_transerr(&head, "stop batch");
2440
2441         lst_free_rpcent(&head);
2442
2443         return rc;
2444 }
2445
2446 int
2447 lst_list_batch_ioctl(int len, char *name, int index)
2448 {
2449         lstio_batch_list_args_t args = {0};
2450
2451         args.lstio_bat_key   = session_key;
2452         args.lstio_bat_idx   = index;
2453         args.lstio_bat_nmlen = len;
2454         args.lstio_bat_namep = name;
2455
2456         return lst_ioctl(LSTIO_BATCH_LIST, &args, sizeof(args));
2457 }
2458
2459 int
2460 lst_info_batch_ioctl(char *batch, int test, int server,
2461                      lstcon_test_batch_ent_t *entp, int *idxp,
2462                      int *ndentp, lstcon_node_ent_t *dentsp)
2463 {
2464         lstio_batch_info_args_t args = {0};
2465
2466         args.lstio_bat_key     = session_key;
2467         args.lstio_bat_nmlen   = strlen(batch);
2468         args.lstio_bat_namep   = batch;
2469         args.lstio_bat_server  = server;
2470         args.lstio_bat_testidx = test;
2471         args.lstio_bat_entp    = entp;
2472         args.lstio_bat_idxp    = idxp;
2473         args.lstio_bat_ndentp  = ndentp;
2474         args.lstio_bat_dentsp  = dentsp;
2475
2476         return lst_ioctl(LSTIO_BATCH_INFO, &args, sizeof(args));
2477 }
2478
2479 int
2480 lst_list_batch_all(void)
2481 {
2482         char name[LST_NAME_SIZE];
2483         int  rc;
2484         int  i;
2485
2486         for (i = 0; ; i++) {
2487                 rc = lst_list_batch_ioctl(LST_NAME_SIZE, name, i);
2488                 if (rc == 0) {
2489                         fprintf(stdout, "%d) %s\n", i + 1, name);
2490                         continue;
2491                 }
2492
2493                 if (errno == ENOENT)
2494                         break;
2495
2496                 lst_print_error("batch", "Failed to list batch: %s\n",
2497                                 strerror(errno));
2498                 return rc;
2499         }
2500
2501         fprintf(stdout, "Total %d batches\n", i);
2502
2503         return 0;
2504 }
2505
2506 int
2507 lst_list_tsb_nodes(char *batch, int test, int server,
2508                    int count, int active, int invalid)
2509 {
2510         lstcon_node_ent_t *dents;
2511         int                index = 0;
2512         int                rc;
2513         int                c;
2514         int                i;
2515
2516         if (count == 0)
2517                 return 0;
2518
2519         /* verbose list, show nodes in batch or test */
2520         dents = malloc(count * sizeof(lstcon_node_ent_t));
2521         if (dents == NULL) {
2522                 fprintf(stdout, "Can't allocate memory\n");
2523                 return -1;
2524         }
2525
2526         rc = lst_info_batch_ioctl(batch, test, server,
2527                                   NULL, &index, &count, dents);
2528         if (rc != 0) {
2529                 free(dents);
2530                 lst_print_error((test > 0) ? "test" : "batch",
2531                                 (test > 0) ? "Failed to query test: %s\n" :
2532                                              "Failed to query batch: %s\n",
2533                                 strerror(errno));
2534                 return -1;
2535         }
2536
2537         for (i = 0, c = 0; i < count; i++) {
2538                 if ((!active  && dents[i].nde_state == LST_NODE_ACTIVE) ||
2539                     (!invalid && (dents[i].nde_state == LST_NODE_BUSY  ||
2540                                   dents[i].nde_state == LST_NODE_DOWN  ||
2541                                   dents[i].nde_state == LST_NODE_UNKNOWN)))
2542                         continue;
2543
2544                 fprintf(stdout, "\t%s: %s\n",
2545                         libcfs_id2str(dents[i].nde_id),
2546                         lst_node_state2str(dents[i].nde_state));
2547                 c++;
2548         }
2549
2550         fprintf(stdout, "Total %d nodes\n", c);
2551         free(dents);
2552
2553         return 0;
2554 }
2555
2556 int
2557 jt_lst_list_batch(int argc, char **argv)
2558 {
2559         lstcon_test_batch_ent_t ent;
2560         char                *batch   = NULL;
2561         int                  optidx  = 0;
2562         int                  verbose = 0; /* list nodes in batch or test */
2563         int                  invalid = 0;
2564         int                  active  = 0;
2565         int                  server  = 0;
2566         int                  ntest   = 0;
2567         int                  test    = 0;
2568         int                  c       = 0;
2569         int                  rc;
2570
2571         static struct option list_batch_opts[] =
2572         {
2573                 {"test",    required_argument, 0, 't' },
2574                 {"invalid", no_argument,       0, 'i' },
2575                 {"active",  no_argument,       0, 'a' },
2576                 {"all",     no_argument,       0, 'l' },
2577                 {"server",  no_argument,       0, 's' },
2578                 {0,         0,                 0,  0  }
2579         };
2580
2581         if (session_key == 0) {
2582                 fprintf(stderr,
2583                         "Can't find env LST_SESSION or value is not valid\n");
2584                 return -1;
2585         }
2586
2587         while (1) {
2588                 c = getopt_long(argc, argv, "ailst:",
2589                                 list_batch_opts, &optidx);
2590
2591                 if (c == -1)
2592                         break;
2593
2594                 switch (c) {
2595                 case 'a':
2596                         verbose = active = 1;
2597                         break;
2598                 case 'i':
2599                         verbose = invalid = 1;
2600                         break;
2601                 case 'l':
2602                         verbose = active = invalid = 1;
2603                         break;
2604                 case 's':
2605                         server = 1;
2606                         break;
2607                 case 't':
2608                         test = atoi(optarg);
2609                         ntest = 1;
2610                         break;
2611                 default:
2612                         lst_print_usage(argv[0]);
2613                         return -1;
2614                 }
2615         }
2616
2617         if (optind == argc) {
2618                 /* list all batches */
2619                 rc = lst_list_batch_all();
2620                 return rc;
2621         }
2622
2623         if (ntest == 1 && test <= 0) {
2624                 fprintf(stderr, "Invalid test id, test id starts from 1\n");
2625                 return -1;
2626         }
2627
2628         if (optind != argc - 1) {
2629                 lst_print_usage(argv[0]);
2630                 return -1;
2631         }
2632
2633         batch = argv[optind];
2634
2635 loop:
2636         /* show detail of specified batch or test */
2637         rc = lst_info_batch_ioctl(batch, test, server,
2638                                   &ent, NULL, NULL, NULL);
2639         if (rc != 0) {
2640                 lst_print_error((test > 0) ? "test" : "batch",
2641                                 (test > 0) ? "Failed to query test: %s\n" :
2642                                              "Failed to query batch: %s\n",
2643                                 strerror(errno));
2644                 return -1;
2645         }
2646
2647         if (verbose) {
2648                 /* list nodes in test or batch */
2649                 rc = lst_list_tsb_nodes(batch, test, server,
2650                                         server ? ent.tbe_srv_nle.nle_nnode :
2651                                                  ent.tbe_cli_nle.nle_nnode,
2652                                         active, invalid);
2653                 return rc;
2654         }
2655
2656         /* only show number of hosts in batch or test */
2657         if (test == 0) {
2658                 fprintf(stdout, "Batch: %s Tests: %d State: %d\n",
2659                         batch, ent.u.tbe_batch.bae_ntest,
2660                         ent.u.tbe_batch.bae_state);
2661                 ntest = ent.u.tbe_batch.bae_ntest;
2662                 test = 1; /* starting from test 1 */
2663
2664         } else {
2665                 fprintf(stdout,
2666                         "\tTest %d(%s) (loop: %d, concurrency: %d)\n",
2667                         test, lst_test_type2name(ent.u.tbe_test.tse_type),
2668                         ent.u.tbe_test.tse_loop,
2669                         ent.u.tbe_test.tse_concur);
2670                 ntest --;
2671                 test ++;
2672         }
2673
2674         fprintf(stdout, LST_NODES_TITLE);
2675         fprintf(stdout, "client\t%d\t%d\t%d\t%d\t%d\n"
2676                         "server\t%d\t%d\t%d\t%d\t%d\n",
2677                 ent.tbe_cli_nle.nle_nactive,
2678                 ent.tbe_cli_nle.nle_nbusy,
2679                 ent.tbe_cli_nle.nle_ndown,
2680                 ent.tbe_cli_nle.nle_nunknown,
2681                 ent.tbe_cli_nle.nle_nnode,
2682                 ent.tbe_srv_nle.nle_nactive,
2683                 ent.tbe_srv_nle.nle_nbusy,
2684                 ent.tbe_srv_nle.nle_ndown,
2685                 ent.tbe_srv_nle.nle_nunknown,
2686                 ent.tbe_srv_nle.nle_nnode);
2687
2688         if (ntest != 0)
2689                 goto loop;
2690
2691         return 0;
2692 }
2693
2694 int
2695 lst_query_batch_ioctl(char *batch, int test, int server,
2696                       int timeout, struct list_head *head)
2697 {
2698         lstio_batch_query_args_t args = {0};
2699
2700         args.lstio_bat_key     = session_key;
2701         args.lstio_bat_testidx = test;
2702         args.lstio_bat_client  = !(server);
2703         args.lstio_bat_timeout = timeout;
2704         args.lstio_bat_nmlen   = strlen(batch);
2705         args.lstio_bat_namep   = batch;
2706         args.lstio_bat_resultp = head;
2707
2708         return lst_ioctl(LSTIO_BATCH_QUERY, &args, sizeof(args));
2709 }
2710
2711 void
2712 lst_print_tsb_verbose(struct list_head *head,
2713                       int active, int idle, int error)
2714 {
2715         lstcon_rpc_ent_t *ent;
2716
2717         list_for_each_entry(ent, head, rpe_link) {
2718                 if (ent->rpe_priv[0] == 0 && active)
2719                         continue;
2720
2721                 if (ent->rpe_priv[0] != 0 && idle)
2722                         continue;
2723
2724                 if (ent->rpe_fwk_errno == 0 && error)
2725                         continue;
2726
2727                 fprintf(stdout, "%s [%s]: %s\n",
2728                         libcfs_id2str(ent->rpe_peer),
2729                         lst_node_state2str(ent->rpe_state),
2730                         ent->rpe_rpc_errno != 0 ?
2731                                 strerror(ent->rpe_rpc_errno) :
2732                                 (ent->rpe_priv[0] > 0 ? "Running" : "Idle"));
2733         }
2734 }
2735
2736 int
2737 jt_lst_query_batch(int argc, char **argv)
2738 {
2739         lstcon_test_batch_ent_t ent;
2740         struct list_head              head;
2741         char                   *batch   = NULL;
2742         time_t                  last    = 0;
2743         int                     optidx  = 0;
2744         int                     verbose = 0;
2745         int                     server  = 0;
2746         int                     timeout = 5; /* default 5 seconds */
2747         int                     delay   = 5; /* default 5 seconds */
2748         int                     loop    = 1; /* default 1 loop */
2749         int                     active  = 0;
2750         int                     error   = 0;
2751         int                     idle    = 0;
2752         int                     count   = 0;
2753         int                     test    = 0;
2754         int                     rc      = 0;
2755         int                     c       = 0;
2756         int                     i;
2757
2758         static struct option query_batch_opts[] =
2759         {
2760                 {"timeout", required_argument, 0, 'o' },
2761                 {"delay",   required_argument, 0, 'd' },
2762                 {"loop",    required_argument, 0, 'c' },
2763                 {"test",    required_argument, 0, 't' },
2764                 {"server",  no_argument,       0, 's' },
2765                 {"active",  no_argument,       0, 'a' },
2766                 {"idle",    no_argument,       0, 'i' },
2767                 {"error",   no_argument,       0, 'e' },
2768                 {"all",     no_argument,       0, 'l' },
2769                 {0,         0,                 0,  0  }
2770         };
2771
2772         if (session_key == 0) {
2773                 fprintf(stderr,
2774                         "Can't find env LST_SESSION or value is not valid\n");
2775                 return -1;
2776         }
2777
2778         while (1) {
2779                 c = getopt_long(argc, argv, "o:d:c:t:saiel",
2780                                 query_batch_opts, &optidx);
2781
2782                 /* Detect the end of the options. */
2783                 if (c == -1)
2784                         break;
2785
2786                 switch (c) {
2787                 case 'o':
2788                         timeout = atoi(optarg);
2789                         break;
2790                 case 'd':
2791                         delay = atoi(optarg);
2792                         break;
2793                 case 'c':
2794                         loop = atoi(optarg);
2795                         break;
2796                 case 't':
2797                         test = atoi(optarg);
2798                         break;
2799                 case 's':
2800                         server = 1;
2801                         break;
2802                 case 'a':
2803                         active = verbose = 1;
2804                         break;
2805                 case 'i':
2806                         idle = verbose = 1;
2807                         break;
2808                 case 'e':
2809                         error = verbose = 1;
2810                         break;
2811                 case 'l':
2812                         verbose = 1;
2813                         break;
2814                 default:
2815                         lst_print_usage(argv[0]);
2816                         return -1;
2817                 }
2818         }
2819
2820         if (test < 0 || timeout <= 0 || delay <= 0 || loop <= 0) {
2821                 lst_print_usage(argv[0]);
2822                 return -1;
2823         }
2824
2825         if (optind == argc) {
2826                 batch = LST_DEFAULT_BATCH;
2827
2828         } else if (optind == argc - 1) {
2829                 batch = argv[optind];
2830
2831         } else {
2832                 lst_print_usage(argv[0]);
2833                 return -1;
2834         }
2835
2836
2837         INIT_LIST_HEAD(&head);
2838
2839         if (verbose) {
2840                 rc = lst_info_batch_ioctl(batch, test, server,
2841                                           &ent, NULL, NULL, NULL);
2842                 if (rc != 0) {
2843                         fprintf(stderr, "Failed to query %s [%d]: %s\n",
2844                                 batch, test, strerror(errno));
2845                         return -1;
2846                 }
2847
2848                 count = server ? ent.tbe_srv_nle.nle_nnode :
2849                                  ent.tbe_cli_nle.nle_nnode;
2850                 if (count == 0) {
2851                         fprintf(stdout, "Batch or test is empty\n");
2852                         return 0;
2853                 }
2854         }
2855
2856         rc = lst_alloc_rpcent(&head, count, 0);
2857         if (rc != 0) {
2858                 fprintf(stderr, "Out of memory\n");
2859                 return rc;
2860         }
2861
2862         for (i = 0; i < loop; i++) {
2863                 time_t  now = time(NULL);
2864
2865                 if (now - last < delay) {
2866                         sleep(delay - now + last);
2867                         time(&now);
2868                 }
2869
2870                 last = now;
2871
2872                 rc = lst_query_batch_ioctl(batch, test,
2873                                            server, timeout, &head);
2874                 if (rc == -1) {
2875                         fprintf(stderr, "Failed to query batch: %s\n",
2876                                 strerror(errno));
2877                         break;
2878                 }
2879
2880                 if (verbose) {
2881                         /* Verbose mode */
2882                         lst_print_tsb_verbose(&head, active, idle, error);
2883                         continue;
2884                 }
2885
2886                 fprintf(stdout, "%s [%d] ", batch, test);
2887
2888                 if (lstcon_rpc_stat_failure(&trans_stat, 0) != 0) {
2889                         fprintf(stdout, "%d of %d nodes are unknown, ",
2890                                 lstcon_rpc_stat_failure(&trans_stat, 0),
2891                                 lstcon_rpc_stat_total(&trans_stat, 0));
2892                 }
2893
2894                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
2895                     lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0  &&
2896                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
2897                         fprintf(stdout, "is stopped\n");
2898                         continue;
2899                 }
2900
2901                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
2902                     lstcon_tsbqry_stat_idle(&trans_stat, 0) == 0 &&
2903                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
2904                         fprintf(stdout, "is running\n");
2905                         continue;
2906                 }
2907
2908                 fprintf(stdout, "stopped: %d , running: %d, failed: %d\n",
2909                                 lstcon_tsbqry_stat_idle(&trans_stat, 0),
2910                                 lstcon_tsbqry_stat_run(&trans_stat, 0),
2911                                 lstcon_tsbqry_stat_failure(&trans_stat, 0));
2912         }
2913
2914         lst_free_rpcent(&head);
2915
2916         return rc;
2917 }
2918
2919 int
2920 lst_parse_distribute(char *dstr, int *dist, int *span)
2921 {
2922         *dist = atoi(dstr);
2923         if (*dist <= 0)
2924                 return -1;
2925
2926         dstr = strchr(dstr, ':');
2927         if (dstr == NULL)
2928                 return -1;
2929
2930         *span = atoi(dstr + 1);
2931         if (*span <= 0)
2932                 return -1;
2933
2934         return 0;
2935 }
2936
2937 int
2938 lst_get_bulk_param(int argc, char **argv, lst_test_bulk_param_t *bulk)
2939 {
2940         char   *tok = NULL;
2941         char   *end = NULL;
2942         int     rc  = 0;
2943         int     i   = 0;
2944
2945         bulk->blk_size  = 4096;
2946         bulk->blk_opc   = LST_BRW_READ;
2947         bulk->blk_flags = LST_BRW_CHECK_NONE;
2948
2949         while (i < argc) {
2950                 if (strcasestr(argv[i], "check=") == argv[i] ||
2951                     strcasestr(argv[i], "c=") == argv[i]) {
2952                         tok = strchr(argv[i], '=') + 1;
2953
2954                         if (strcasecmp(tok, "full") == 0) {
2955                                 bulk->blk_flags = LST_BRW_CHECK_FULL;
2956                         } else if (strcasecmp(tok, "simple") == 0) {
2957                                 bulk->blk_flags = LST_BRW_CHECK_SIMPLE;
2958                         } else {
2959                                 fprintf(stderr, "Unknow flag %s\n", tok);
2960                                 return -1;
2961                         }
2962
2963                 } else if (strcasestr(argv[i], "size=") == argv[i] ||
2964                            strcasestr(argv[i], "s=") == argv[i]) {
2965                         int max_size = sysconf(_SC_PAGESIZE) * LNET_MAX_IOV;
2966
2967                         tok = strchr(argv[i], '=') + 1;
2968
2969                         bulk->blk_size = strtol(tok, &end, 0);
2970                         if (bulk->blk_size <= 0) {
2971                                 fprintf(stderr, "Invalid size %s\n", tok);
2972                                 return -1;
2973                         }
2974
2975                         if (end == NULL)
2976                                 return 0;
2977
2978                         if (*end == 'k' || *end == 'K')
2979                                 bulk->blk_size *= 1024;
2980                         else if (*end == 'm' || *end == 'M')
2981                                 bulk->blk_size *= 1024 * 1024;
2982
2983                         if (bulk->blk_size > max_size) {
2984                                 fprintf(stderr, "Size exceed limitation: %d bytes\n",
2985                                         bulk->blk_size);
2986                                 return -1;
2987                         }
2988
2989                 } else if (strcasecmp(argv[i], "read") == 0 ||
2990                            strcasecmp(argv[i], "r") == 0) {
2991                         bulk->blk_opc = LST_BRW_READ;
2992
2993                 } else if (strcasecmp(argv[i], "write") == 0 ||
2994                            strcasecmp(argv[i], "w") == 0) {
2995                         bulk->blk_opc = LST_BRW_WRITE;
2996
2997                 } else {
2998                         fprintf(stderr, "Unknow parameter: %s\n", argv[i]);
2999                         return -1;
3000                 }
3001
3002                 i++;
3003         }
3004
3005         return rc;
3006 }
3007
3008 int
3009 lst_get_test_param(char *test, int argc, char **argv, void **param, int *plen)
3010 {
3011         lst_test_bulk_param_t *bulk = NULL;
3012         int                    type;
3013
3014         type = lst_test_name2type(test);
3015         if (type < 0) {
3016                 fprintf(stderr, "Unknow test name %s\n", test);
3017                 return -1;
3018         }
3019
3020         switch (type) {
3021         case LST_TEST_PING:
3022                 break;
3023
3024         case LST_TEST_BULK:
3025                 bulk = malloc(sizeof(*bulk));
3026                 if (bulk == NULL) {
3027                         fprintf(stderr, "Out of memory\n");
3028                         return -1;
3029                 }
3030
3031                 memset(bulk, 0, sizeof(*bulk));
3032
3033                 if (lst_get_bulk_param(argc, argv, bulk) != 0) {
3034                         free(bulk);
3035                         return -1;
3036                 }
3037
3038                 *param = bulk;
3039                 *plen  = sizeof(*bulk);
3040
3041                 break;
3042
3043         default:
3044                 break;
3045         }
3046
3047         /* TODO: parse more parameter */
3048         return type;
3049 }
3050
3051 int
3052 lst_add_test_ioctl(char *batch, int type, int loop, int concur,
3053                    int dist, int span, char *sgrp, char *dgrp,
3054                    void *param, int plen, int *retp, struct list_head *resultp)
3055 {
3056         lstio_test_args_t args = {0};
3057
3058         args.lstio_tes_key        = session_key;
3059         args.lstio_tes_bat_nmlen  = strlen(batch);
3060         args.lstio_tes_bat_name   = batch;
3061         args.lstio_tes_type       = type;
3062         args.lstio_tes_oneside    = 0;
3063         args.lstio_tes_loop       = loop;
3064         args.lstio_tes_concur     = concur;
3065         args.lstio_tes_dist       = dist;
3066         args.lstio_tes_span       = span;
3067         args.lstio_tes_sgrp_nmlen = strlen(sgrp);
3068         args.lstio_tes_sgrp_name  = sgrp;
3069         args.lstio_tes_dgrp_nmlen = strlen(dgrp);
3070         args.lstio_tes_dgrp_name  = dgrp;
3071         args.lstio_tes_param_len  = plen;
3072         args.lstio_tes_param      = param;
3073         args.lstio_tes_retp       = retp;
3074         args.lstio_tes_resultp    = resultp;
3075
3076         return lst_ioctl(LSTIO_TEST_ADD, &args, sizeof(args));
3077 }
3078
3079 int
3080 jt_lst_add_test(int argc, char **argv)
3081 {
3082         struct list_head    head;
3083         char         *batch  = NULL;
3084         char         *test   = NULL;
3085         char         *dstr   = NULL;
3086         char         *from   = NULL;
3087         char         *to     = NULL;
3088         void         *param  = NULL;
3089         int           optidx = 0;
3090         int           concur = 1;
3091         int           loop   = -1;
3092         int           dist   = 1;
3093         int           span   = 1;
3094         int           plen   = 0;
3095         int           fcount = 0;
3096         int           tcount = 0;
3097         int           ret    = 0;
3098         int           type;
3099         int           rc;
3100         int           c;
3101
3102         static struct option add_test_opts[] =
3103         {
3104                 {"batch",       required_argument, 0, 'b' },
3105                 {"concurrency", required_argument, 0, 'c' },
3106                 {"distribute",  required_argument, 0, 'd' },
3107                 {"from",        required_argument, 0, 'f' },
3108                 {"to",          required_argument, 0, 't' },
3109                 {"loop",        required_argument, 0, 'l' },
3110                 {0,             0,                 0,  0  }
3111         };
3112
3113         if (session_key == 0) {
3114                 fprintf(stderr,
3115                         "Can't find env LST_SESSION or value is not valid\n");
3116                 return -1;
3117         }
3118
3119         while (1) {
3120                 c = getopt_long(argc, argv, "b:c:d:f:l:t:",
3121                                 add_test_opts, &optidx);
3122
3123                 /* Detect the end of the options. */
3124                 if (c == -1)
3125                         break;
3126
3127                 switch (c) {
3128                 case 'b':
3129                         batch = optarg;
3130                         break;
3131                 case 'c':
3132                         concur = atoi(optarg);
3133                         break;
3134                 case 'd':
3135                         dstr = optarg;
3136                         break;
3137                 case 'f':
3138                         from = optarg;
3139                         break;
3140                 case 'l':
3141                         loop = atoi(optarg);
3142                         break;
3143                 case 't':
3144                         to = optarg;
3145                         break;
3146                 default:
3147                         lst_print_usage(argv[0]);
3148                         return -1;
3149                 }
3150         }
3151
3152         if (optind == argc || from == NULL || to == NULL) {
3153                 lst_print_usage(argv[0]);
3154                 return -1;
3155         }
3156
3157         if (concur <= 0 || concur > LST_MAX_CONCUR) {
3158                 fprintf(stderr, "Invalid concurrency of test: %d\n", concur);
3159                 return -1;
3160         }
3161
3162         if (batch == NULL)
3163                 batch = LST_DEFAULT_BATCH;
3164
3165         if (dstr != NULL) {
3166                 rc = lst_parse_distribute(dstr, &dist, &span);
3167                 if (rc != 0) {
3168                         fprintf(stderr, "Invalid distribution: %s\n", dstr);
3169                         return -1;
3170                 }
3171         }
3172
3173         test = argv[optind++];
3174
3175         argc -= optind;
3176         argv += optind;
3177
3178         type = lst_get_test_param(test, argc, argv, &param, &plen);
3179         if (type < 0) {
3180                 fprintf(stderr, "Failed to add test (%s)\n", test);
3181                 return -1;
3182         }
3183
3184         INIT_LIST_HEAD(&head);
3185
3186         rc = lst_get_node_count(LST_OPC_GROUP, from, &fcount, NULL);
3187         if (rc != 0) {
3188                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3189                         from, strerror(errno));
3190                 goto out;
3191         }
3192
3193         rc = lst_get_node_count(LST_OPC_GROUP, to, &tcount, NULL);
3194         if (rc != 0) {
3195                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3196                         to, strerror(errno));
3197                 goto out;
3198         }
3199
3200         rc = lst_alloc_rpcent(&head, fcount > tcount ? fcount : tcount, 0);
3201         if (rc != 0) {
3202                 fprintf(stderr, "Out of memory\n");
3203                 goto out;
3204         }
3205
3206         rc = lst_add_test_ioctl(batch, type, loop, concur,
3207                                 dist, span, from, to, param, plen, &ret, &head);
3208
3209         if (rc == 0) {
3210                 fprintf(stdout, "Test was added successfully\n");
3211                 if (ret != 0) {
3212                         fprintf(stdout, "Server group contains userland test "
3213                                 "nodes, old version of tcplnd can't accept "
3214                                 "connection request\n");
3215                 }
3216
3217                 goto out;
3218         }
3219
3220         if (rc == -1) {
3221                 lst_print_error("test", "Failed to add test: %s\n",
3222                                 strerror(errno));
3223                 goto out;
3224         }
3225
3226         lst_print_transerr(&head, "add test");
3227 out:
3228         lst_free_rpcent(&head);
3229
3230         if (param != NULL)
3231                 free(param);
3232
3233         return rc;
3234 }
3235
3236 static command_t lst_cmdlist[] = {
3237         {"new_session",         jt_lst_new_session,     NULL,
3238          "Usage: lst new_session [--timeout TIME] [--force] [NAME]"                     },
3239         {"end_session",         jt_lst_end_session,     NULL,
3240          "Usage: lst end_session"                                                       },
3241         {"show_session",        jt_lst_show_session,    NULL,
3242          "Usage: lst show_session"                                                      },
3243         {"ping",                jt_lst_ping ,           NULL,
3244          "Usage: lst ping  [--group NAME] [--batch NAME] [--session] [--nodes IDS]"     },
3245         {"add_group",           jt_lst_add_group,       NULL,
3246          "Usage: lst group NAME IDs [IDs]..."                                           },
3247         {"del_group",           jt_lst_del_group,       NULL,
3248          "Usage: lst del_group NAME"                                                    },
3249         {"update_group",        jt_lst_update_group,    NULL,
3250          "Usage: lst update_group NAME [--clean] [--refresh] [--remove IDs]"            },
3251         {"list_group",          jt_lst_list_group,      NULL,
3252           "Usage: lst list_group [--active] [--busy] [--down] [--unknown] GROUP ..."    },
3253         {"stat",                jt_lst_stat,            NULL,
3254          "Usage: lst stat [--bw] [--rate] [--read] [--write] [--max] [--min] [--avg] "
3255          " [--timeout #] [--delay #] [--count #] GROUP [GROUP]"                         },
3256         {"show_error",          jt_lst_show_error,      NULL,
3257          "Usage: lst show_error NAME | IDS ..."                                         },
3258         {"add_batch",           jt_lst_add_batch,       NULL,
3259          "Usage: lst add_batch NAME"                                                    },
3260         {"run",                 jt_lst_start_batch,     NULL,
3261          "Usage: lst run [--timeout TIME] [NAME]"                                       },
3262         {"stop",                jt_lst_stop_batch,      NULL,
3263          "Usage: lst stop [--force] BATCH_NAME"                                         },
3264         {"list_batch",          jt_lst_list_batch,      NULL,
3265          "Usage: lst list_batch NAME [--test ID] [--server]"                            },
3266         {"query",               jt_lst_query_batch,     NULL,
3267          "Usage: lst query [--test ID] [--server] [--timeout TIME] NAME"                },
3268         {"add_test",            jt_lst_add_test,        NULL,
3269          "Usage: lst add_test [--batch BATCH] [--loop #] [--concurrency #] "
3270          " [--distribute #:#] [--from GROUP] [--to GROUP] TEST..."                      },
3271         {"help",                Parser_help,            0,     "help"                   },
3272         {0,                     0,                      0,      NULL                    }
3273 };
3274
3275 int
3276 lst_initialize(void)
3277 {
3278         char   *key;
3279         char   *feats;
3280
3281         feats = getenv("LST_FEATURES");
3282         if (feats != NULL)
3283                 session_features = strtol(feats, NULL, 16);
3284
3285         if ((session_features & ~LST_FEATS_MASK) != 0) {
3286                 fprintf(stderr,
3287                         "Unsupported session features %x, "
3288                         "only support these features so far: %x\n",
3289                         (session_features & ~LST_FEATS_MASK), LST_FEATS_MASK);
3290                 return -1;
3291         }
3292
3293         key = getenv("LST_SESSION");
3294
3295         if (key == NULL) {
3296                 session_key = 0;
3297                 return 0;
3298         }
3299
3300         session_key = atoi(key);
3301
3302         return 0;
3303 }
3304
3305 int
3306 main(int argc, char **argv)
3307 {
3308         int rc = 0;
3309
3310         setlinebuf(stdout);
3311
3312         rc = lst_initialize();
3313         if (rc < 0)
3314                 goto errorout;
3315
3316         rc = ptl_initialize(argc, argv);
3317         if (rc < 0)
3318                 goto errorout;
3319
3320         Parser_init("lst > ", lst_cmdlist);
3321
3322         if (argc != 1)  {
3323                 rc = Parser_execarg(argc - 1, argv + 1, lst_cmdlist);
3324                 goto errorout;
3325         }
3326
3327         Parser_commands();
3328
3329 errorout:
3330         return rc;
3331 }