lnet/utils/lst.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  *
  30  * Copyright (c) 2012, 2014, Intel Corporation.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * lnet/selftest/conctl.c
  37  *
  38  * Author: Liang Zhen <liangzhen@clusterfs.com>
  39  */
  40 #include <errno.h>
  41 #include <getopt.h>
  42 #include <pwd.h>
  43 #include <unistd.h>
  44 #include <stdarg.h>
  45 #include <stdio.h>
  46 #include <stdlib.h>
  47 #include <string.h>
  48 #include <sys/ioctl.h>
  49 #include <time.h>
  50
  51 #include <libcfs/list.h>
  52 #include <libcfs/util/ioctl.h>
  53 #include <libcfs/util/parser.h>
  54 #include <lnet/lnetctl.h>
  55 #include <lnet/lnetst.h>
  56 #include <lnet/lnet.h>
  57 #include <utils/obdctl.h>
  58
  59 lst_sid_t LST_INVALID_SID = {LNET_NID_ANY, -1};
  60 static lst_sid_t           session_id;
  61 static int                 session_key;
  62
  63 /* All nodes running 2.6.50 or later understand feature LST_FEAT_BULK_LEN */
  64 static unsigned         session_features = LST_FEATS_MASK;
  65 static lstcon_trans_stat_t      trans_stat;
  66
  67 typedef struct list_string {
  68         struct list_string *lstr_next;
  69         int                 lstr_sz;
  70         char                lstr_str[0];
  71 } lstr_t;
  72
  73 #ifndef offsetof
  74 # define offsetof(typ,memb)     ((unsigned long)((char *)&(((typ *)0)->memb)))
  75 #endif
  76
  77 static int alloc_count = 0;
  78 static int alloc_nob   = 0;
  79
  80 lstr_t *
  81 alloc_lstr(int sz)
  82 {
  83         lstr_t  *lstr = malloc(offsetof(lstr_t, lstr_str[sz]));
  84
  85         if (lstr == NULL) {
  86                 fprintf(stderr, "Can't allocate lstr\n");
  87                 abort();
  88         }
  89
  90         alloc_nob += sz;
  91         alloc_count++;
  92
  93         lstr->lstr_str[0] = 0;
  94         lstr->lstr_sz = sz;
  95         return lstr;
  96 }
  97
  98 void
  99 free_lstr(lstr_t *lstr)
 100 {
 101         alloc_count--;
 102         alloc_nob -= lstr->lstr_sz;
 103         free(lstr);
 104 }
 105
 106 void
 107 free_lstrs(lstr_t **list)
 108 {
 109         lstr_t   *lstr;
 110
 111         while ((lstr = *list) != NULL) {
 112                 *list = lstr->lstr_next;
 113                 free_lstr(lstr);
 114         }
 115 }
 116
 117 void
 118 new_lstrs(lstr_t **list, char *prefix, char *postfix,
 119           int lo, int hi, int stride)
 120 {
 121         int    n1 = strlen(prefix);
 122         int    n2 = strlen(postfix);
 123         int    sz = n1 + 20 + n2 + 1;
 124
 125         do {
 126                 lstr_t *n = alloc_lstr(sz);
 127
 128                 snprintf(n->lstr_str, sz - 1, "%s%u%s",
 129                          prefix, lo, postfix);
 130
 131                 n->lstr_next = *list;
 132                 *list = n;
 133
 134                 lo += stride;
 135         } while (lo <= hi);
 136 }
 137
 138 int
 139 expand_lstr(lstr_t **list, lstr_t *l)
 140 {
 141         int          nob = strlen(l->lstr_str);
 142         char        *b1;
 143         char        *b2;
 144         char        *expr;
 145         char        *sep;
 146         int          x;
 147         int          y;
 148         int          z;
 149         int          n;
 150
 151         b1 = strchr(l->lstr_str, '[');
 152         if (b1 == NULL) {
 153                 l->lstr_next = *list;
 154                 *list = l;
 155                 return 0;
 156         }
 157
 158         b2 = strchr(b1, ']');
 159         if (b2 == NULL || b2 == b1 + 1)
 160                 return -1;
 161
 162         *b1++ = 0;
 163         *b2++ = 0;
 164         expr = b1;
 165         do {
 166
 167                 sep = strchr(expr, ',');
 168                 if (sep != NULL)
 169                         *sep++ = 0;
 170
 171                 nob = strlen(expr);
 172                 n = nob;
 173                 if (sscanf(expr, "%u%n", &x, &n) >= 1 && n == nob) {
 174                         /* simple number */
 175                         new_lstrs(list, l->lstr_str, b2, x, x, 1);
 176                         continue;
 177                 }
 178
 179                 n = nob;
 180                 if (sscanf(expr, "%u-%u%n", &x, &y, &n) >= 2 && n == nob &&
 181                     x < y) {
 182                         /* simple range */
 183                         new_lstrs(list, l->lstr_str, b2, x, y, 1);
 184                         continue;
 185                 }
 186
 187                 n = nob;
 188                 if (sscanf(expr, "%u-%u/%u%n", &x, &y, &z, &n) >= 3 && n == nob &&
 189                     x < y) {
 190                         /* strided range */
 191                         new_lstrs(list, l->lstr_str, b2, x, y, z);
 192                         continue;
 193                 }
 194
 195                 /* syntax error */
 196                 return -1;
 197         } while ((expr = sep) != NULL);
 198
 199         free_lstr(l);
 200
 201         return 1;
 202 }
 203
 204 int
 205 expand_strs(char *str, lstr_t **head)
 206 {
 207         lstr_t  *list = NULL;
 208         lstr_t  *nlist;
 209         lstr_t  *l;
 210         int      rc = 0;
 211         int      expanded;
 212
 213         l = alloc_lstr(strlen(str) + 1);
 214         memcpy(l->lstr_str, str, strlen(str) + 1);
 215         l->lstr_next = NULL;
 216         list = l;
 217
 218         do {
 219                 expanded = 0;
 220                 nlist = NULL;
 221
 222                 while ((l = list) != NULL) {
 223                         list = l->lstr_next;
 224
 225                         rc = expand_lstr(&nlist, l);
 226                         if (rc < 0) {
 227                                 fprintf(stderr, "Syntax error in \"%s\"\n", str);
 228                                 free_lstr(l);
 229                                 break;
 230                         }
 231
 232                         expanded |= rc > 0;
 233                 }
 234
 235                 /* re-order onto 'list' */
 236                 while ((l = nlist) != NULL) {
 237                         nlist = l->lstr_next;
 238                         l->lstr_next = list;
 239                         list = l;
 240                 }
 241
 242         } while (expanded && rc > 0);
 243
 244         if (rc >= 0) {
 245                 *head = list;
 246                 return 0;
 247         }
 248
 249         while ((l = list) != NULL) {
 250                 list = l->lstr_next;
 251
 252                 free_lstr(l);
 253         }
 254         return rc;
 255 }
 256
 257 int
 258 lst_parse_nids(char *str, int *countp, lnet_process_id_t **idspp)
 259 {
 260         lstr_t  *head = NULL;
 261         lstr_t  *l;
 262         int      c = 0;
 263         int      i;
 264         int      rc;
 265
 266         rc = expand_strs(str, &head);
 267         if (rc != 0)
 268                 goto out;
 269
 270         l = head;
 271         while (l != NULL) {
 272                 l = l->lstr_next;
 273                 c++;
 274         }
 275
 276         *idspp = malloc(c * sizeof(lnet_process_id_t));
 277         if (*idspp == NULL) {
 278                 fprintf(stderr, "Out of memory\n");
 279                 rc = -1;
 280         }
 281
 282         *countp = c;
 283 out:
 284         i = 0;
 285         while ((l = head) != NULL) {
 286                 head = l->lstr_next;
 287
 288                 if (rc == 0) {
 289                         (*idspp)[i].nid = libcfs_str2nid(l->lstr_str);
 290                         if ((*idspp)[i].nid == LNET_NID_ANY) {
 291                                 fprintf(stderr, "Invalid nid: %s\n",
 292                                         l->lstr_str);
 293                                 rc = -1;
 294                         }
 295
 296                         (*idspp)[i].pid = LNET_PID_LUSTRE;
 297                         i++;
 298                 }
 299
 300                 free_lstr(l);
 301         }
 302
 303         if (rc == 0)
 304                 return 0;
 305
 306         free(*idspp);
 307         *idspp = NULL;
 308
 309         return rc;
 310 }
 311
 312 char *
 313 lst_node_state2str(int state)
 314 {
 315         if (state == LST_NODE_ACTIVE)
 316                 return "Active";
 317         if (state == LST_NODE_BUSY)
 318                 return "Busy";
 319         if (state == LST_NODE_DOWN)
 320                 return "Down";
 321
 322         return "Unknown";
 323 }
 324
 325 int
 326 lst_node_str2state(char *str)
 327 {
 328         if (strcasecmp(str, "active") == 0)
 329                 return LST_NODE_ACTIVE;
 330         if (strcasecmp(str, "busy") == 0)
 331                 return LST_NODE_BUSY;
 332         if (strcasecmp(str, "down") == 0)
 333                 return LST_NODE_DOWN;
 334         if (strcasecmp(str, "unknown") == 0)
 335                 return LST_NODE_UNKNOWN;
 336         if (strcasecmp(str, "invalid") == 0)
 337                 return (LST_NODE_UNKNOWN | LST_NODE_DOWN | LST_NODE_BUSY);
 338
 339         return -1;
 340 }
 341
 342 char *
 343 lst_test_type2name(int type)
 344 {
 345         if (type == LST_TEST_PING)
 346                 return "ping";
 347         if (type == LST_TEST_BULK)
 348                 return "brw";
 349
 350         return "unknown";
 351 }
 352
 353 int
 354 lst_test_name2type(char *name)
 355 {
 356         if (strcasecmp(name, "ping") == 0)
 357                 return LST_TEST_PING;
 358         if (strcasecmp(name, "brw") == 0)
 359                 return LST_TEST_BULK;
 360
 361         return -1;
 362 }
 363
 364 void
 365 lst_print_usage(char *cmd)
 366 {
 367         Parser_printhelp(cmd);
 368 }
 369
 370 void
 371 lst_print_error(char *sub, const char *def_format, ...)
 372 {
 373         va_list ap;
 374
 375         /* local error returned from kernel */
 376         switch (errno) {
 377         case ESRCH:
 378                 fprintf(stderr, "No session exists\n");
 379                 return;
 380         case ESHUTDOWN:
 381                 fprintf(stderr, "Session is shutting down\n");
 382                 return;
 383         case EACCES:
 384                 fprintf(stderr, "Unmatched session key or not root\n");
 385                 return;
 386         case ENOENT:
 387                 fprintf(stderr, "Can't find %s in current session\n", sub);
 388                 return;
 389         case EINVAL:
 390                 fprintf(stderr, "Invalid parameters list in command line\n");
 391                 return;
 392         case EFAULT:
 393                 fprintf(stderr, "Bad parameter address\n");
 394                 return;
 395         case EEXIST:
 396                 fprintf(stderr, "%s already exists\n", sub);
 397                 return;
 398         default:
 399                 va_start(ap, def_format);
 400                 vfprintf(stderr, def_format, ap);
 401                 va_end(ap);
 402
 403                 return;
 404         }
 405 }
 406
 407 void
 408 lst_free_rpcent(struct list_head *head)
 409 {
 410         lstcon_rpc_ent_t *ent;
 411
 412         while (!list_empty(head)) {
 413                 ent = list_entry(head->next, lstcon_rpc_ent_t, rpe_link);
 414
 415                 list_del(&ent->rpe_link);
 416                 free(ent);
 417         }
 418 }
 419
 420 void
 421 lst_reset_rpcent(struct list_head *head)
 422 {
 423         lstcon_rpc_ent_t *ent;
 424
 425         list_for_each_entry(ent, head, rpe_link) {
 426                 ent->rpe_sid       = LST_INVALID_SID;
 427                 ent->rpe_peer.nid  = LNET_NID_ANY;
 428                 ent->rpe_peer.pid  = LNET_PID_ANY;
 429                 ent->rpe_rpc_errno = ent->rpe_fwk_errno = 0;
 430         }
 431 }
 432
 433 int
 434 lst_alloc_rpcent(struct list_head *head, int count, int offset)
 435 {
 436         lstcon_rpc_ent_t *ent;
 437         int               i;
 438
 439         for (i = 0; i < count; i++) {
 440                 ent = malloc(offsetof(lstcon_rpc_ent_t, rpe_payload[offset]));
 441                 if (ent == NULL) {
 442                         lst_free_rpcent(head);
 443                         return -1;
 444                 }
 445
 446                 memset(ent, 0, offsetof(lstcon_rpc_ent_t, rpe_payload[offset]));
 447
 448                 ent->rpe_sid      = LST_INVALID_SID;
 449                 ent->rpe_peer.nid = LNET_NID_ANY;
 450                 ent->rpe_peer.pid = LNET_PID_ANY;
 451                 list_add(&ent->rpe_link, head);
 452         }
 453
 454         return 0;
 455 }
 456
 457 void
 458 lst_print_transerr(struct list_head *head, char *optstr)
 459 {
 460         lstcon_rpc_ent_t *ent;
 461
 462         list_for_each_entry(ent, head, rpe_link) {
 463                 if (ent->rpe_rpc_errno == 0 && ent->rpe_fwk_errno == 0)
 464                         continue;
 465
 466                 if (ent->rpe_rpc_errno != 0) {
 467                         fprintf(stderr, "%s RPC failed on %s: %s\n",
 468                                 optstr, libcfs_id2str(ent->rpe_peer),
 469                                 strerror(ent->rpe_rpc_errno));
 470                         continue;
 471                 }
 472
 473                 fprintf(stderr, "operation %s failed on %s: %s\n",
 474                         optstr, libcfs_id2str(ent->rpe_peer),
 475                         strerror(ent->rpe_fwk_errno));
 476         }
 477 }
 478
 479 int lst_info_batch_ioctl(char *batch, int test, int server,
 480                         lstcon_test_batch_ent_t *entp, int *idxp,
 481                         int *ndentp, lstcon_node_ent_t *dentsp);
 482
 483 int lst_info_group_ioctl(char *name, lstcon_ndlist_ent_t *gent,
 484                          int *idx, int *count, lstcon_node_ent_t *dents);
 485
 486 int lst_query_batch_ioctl(char *batch, int test, int server,
 487                           int timeout, struct list_head *head);
 488
 489 int
 490 lst_ioctl(unsigned int opc, void *buf, int len)
 491 {
 492         struct libcfs_ioctl_data data;
 493         int    rc;
 494
 495         LIBCFS_IOC_INIT (data);
 496         data.ioc_u32[0]  = opc;
 497         data.ioc_plen1   = len;
 498         data.ioc_pbuf1   = (char *)buf;
 499         data.ioc_plen2   = sizeof(trans_stat);
 500         data.ioc_pbuf2   = (char *)&trans_stat;
 501
 502         memset(&trans_stat, 0, sizeof(trans_stat));
 503
 504         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNETST, &data);
 505
 506         /* local error, no valid RPC result */
 507         if (rc != 0)
 508                 return -1;
 509
 510         /* RPC error */
 511         if (trans_stat.trs_rpc_errno != 0)
 512                 return -2;
 513
 514         /* Framework error */
 515         if (trans_stat.trs_fwk_errno != 0)
 516                 return -3;
 517
 518         return 0;
 519 }
 520
 521 int
 522 lst_new_session_ioctl(char *name, int timeout, int force, lst_sid_t *sid)
 523 {
 524         lstio_session_new_args_t args = {0};
 525
 526         args.lstio_ses_key     = session_key;
 527         args.lstio_ses_timeout = timeout;
 528         args.lstio_ses_force   = force;
 529         args.lstio_ses_idp     = sid;
 530         args.lstio_ses_feats   = session_features;
 531         args.lstio_ses_nmlen   = strlen(name);
 532         args.lstio_ses_namep   = name;
 533
 534         return lst_ioctl (LSTIO_SESSION_NEW, &args, sizeof(args));
 535 }
 536
 537 int
 538 jt_lst_new_session(int argc,  char **argv)
 539 {
 540         char  buf[LST_NAME_SIZE];
 541         char *name;
 542         int   optidx  = 0;
 543         int   timeout = 300;
 544         int   force   = 0;
 545         int   c;
 546         int   rc;
 547
 548         static struct option session_opts[] =
 549         {
 550                 {"timeout", required_argument,  0, 't' },
 551                 {"force",   no_argument,        0, 'f' },
 552                 {0,         0,                  0,  0  }
 553         };
 554
 555         if (session_key == 0) {
 556                 fprintf(stderr,
 557                         "Can't find env LST_SESSION or value is not valid\n");
 558                 return -1;
 559         }
 560
 561         while (1) {
 562
 563                 c = getopt_long(argc, argv, "ft:",
 564                                 session_opts, &optidx);
 565
 566                 if (c == -1)
 567                         break;
 568
 569                 switch (c) {
 570                 case 'f':
 571                         force = 1;
 572                         break;
 573                 case 't':
 574                         timeout = atoi(optarg);
 575                         break;
 576                 default:
 577                         lst_print_usage(argv[0]);
 578                         return -1;
 579                 }
 580         }
 581
 582         if (timeout <= 0) {
 583                 fprintf(stderr, "Invalid timeout value\n");
 584                 return -1;
 585         }
 586
 587         if (optind == argc - 1) {
 588                 name = argv[optind ++];
 589                 if (strlen(name) >= LST_NAME_SIZE) {
 590                         fprintf(stderr, "Name size is limited to %d\n",
 591                                 LST_NAME_SIZE - 1);
 592                         return -1;
 593                 }
 594
 595         } else if (optind == argc) {
 596                 char           user[LST_NAME_SIZE];
 597                 char           host[LST_NAME_SIZE];
 598                 struct passwd *pw = getpwuid(getuid());
 599
 600                 if (pw == NULL)
 601                         snprintf(user, sizeof(user), "%d", (int)getuid());
 602                 else
 603                         snprintf(user, sizeof(user), "%s", pw->pw_name);
 604
 605                 rc = gethostname(host, sizeof(host));
 606                 if (rc != 0)
 607                         snprintf(host, sizeof(host), "unknown_host");
 608
 609                 snprintf(buf, sizeof(buf), "%s@%s", user, host);
 610                 name = buf;
 611
 612         } else {
 613                 lst_print_usage(argv[0]);
 614                 return -1;
 615         }
 616
 617         rc = lst_new_session_ioctl(name, timeout, force, &session_id);
 618         if (rc != 0) {
 619                 lst_print_error("session", "Failed to create session: %s\n",
 620                                 strerror(errno));
 621                 return rc;
 622         }
 623
 624         fprintf(stdout, "SESSION: %s FEATURES: %x TIMEOUT: %d FORCE: %s\n",
 625                 name, session_features, timeout, force ? "Yes" : "No");
 626         return 0;
 627 }
 628
 629 int
 630 lst_session_info_ioctl(char *name, int len, int *key, unsigned *featp,
 631                        lst_sid_t *sid, lstcon_ndlist_ent_t *ndinfo)
 632 {
 633         lstio_session_info_args_t args = {0};
 634
 635         args.lstio_ses_idp     = sid;
 636         args.lstio_ses_keyp    = key;
 637         args.lstio_ses_featp   = featp;
 638         args.lstio_ses_ndinfo  = ndinfo;
 639         args.lstio_ses_nmlen   = len;
 640         args.lstio_ses_namep   = name;
 641
 642         return lst_ioctl(LSTIO_SESSION_INFO, &args, sizeof(args));
 643 }
 644
 645 int
 646 jt_lst_show_session(int argc, char **argv)
 647 {
 648         lstcon_ndlist_ent_t ndinfo;
 649         lst_sid_t           sid;
 650         char                name[LST_NAME_SIZE];
 651         unsigned            feats;
 652         int                 key;
 653         int                 rc;
 654
 655         rc = lst_session_info_ioctl(name, sizeof(name), &key,
 656                                     &feats, &sid, &ndinfo);
 657
 658         if (rc != 0) {
 659                 lst_print_error("session", "Failed to show session: %s\n",
 660                                 strerror(errno));
 661                 return -1;
 662         }
 663
 664         fprintf(stdout, "%s ID: "LPU64"@%s, KEY: %d FEATURES: %x NODES: %d\n",
 665                 name, sid.ses_stamp, libcfs_nid2str(sid.ses_nid),
 666                 key, feats, ndinfo.nle_nnode);
 667
 668         return 0;
 669 }
 670
 671 int
 672 lst_end_session_ioctl(void)
 673 {
 674         lstio_session_end_args_t args = {0};
 675
 676         args.lstio_ses_key =  session_key;
 677         return lst_ioctl (LSTIO_SESSION_END, &args, sizeof(args));
 678 }
 679
 680 int
 681 jt_lst_end_session(int argc, char **argv)
 682 {
 683         int             rc;
 684
 685         if (session_key == 0) {
 686                 fprintf(stderr,
 687                         "Can't find env LST_SESSION or value is not valid\n");
 688                 return -1;
 689         }
 690
 691         rc = lst_end_session_ioctl();
 692
 693         if (rc == 0) {
 694                 fprintf(stdout, "session is ended\n");
 695                 return 0;
 696         }
 697
 698         if (rc == -1) {
 699                 lst_print_error("session", "Failed to end session: %s\n",
 700                                 strerror(errno));
 701                 return rc;
 702         }
 703
 704         if (trans_stat.trs_rpc_errno != 0) {
 705                 fprintf(stderr,
 706                         "[RPC] Failed to send %d session RPCs: %s\n",
 707                         lstcon_rpc_stat_failure(&trans_stat, 0),
 708                         strerror(trans_stat.trs_rpc_errno));
 709         }
 710
 711         if (trans_stat.trs_fwk_errno != 0) {
 712                 fprintf(stderr,
 713                         "[FWK] Failed to end session on %d nodes: %s\n",
 714                         lstcon_sesop_stat_failure(&trans_stat, 0),
 715                         strerror(trans_stat.trs_fwk_errno));
 716         }
 717
 718         return rc;
 719 }
 720
 721 int
 722 lst_ping_ioctl(char *str, int type, int timeout,
 723                int count, lnet_process_id_t *ids, struct list_head *head)
 724 {
 725         lstio_debug_args_t args = {0};
 726
 727         args.lstio_dbg_key     = session_key;
 728         args.lstio_dbg_type    = type;
 729         args.lstio_dbg_flags   = 0;
 730         args.lstio_dbg_timeout = timeout;
 731         args.lstio_dbg_nmlen   = (str == NULL) ? 0: strlen(str);
 732         args.lstio_dbg_namep   = str;
 733         args.lstio_dbg_count   = count;
 734         args.lstio_dbg_idsp    = ids;
 735         args.lstio_dbg_resultp = head;
 736
 737         return lst_ioctl (LSTIO_DEBUG, &args, sizeof(args));
 738 }
 739
 740 int
 741 lst_get_node_count(int type, char *str, int *countp, lnet_process_id_t **idspp)
 742 {
 743         char                    buf[LST_NAME_SIZE];
 744         lstcon_test_batch_ent_t ent;
 745         lstcon_ndlist_ent_t    *entp = &ent.tbe_cli_nle;
 746         lst_sid_t               sid;
 747         unsigned                feats;
 748         int                     key;
 749         int                     rc;
 750
 751         switch (type) {
 752         case LST_OPC_SESSION:
 753                 rc = lst_session_info_ioctl(buf, LST_NAME_SIZE,
 754                                             &key, &feats, &sid, entp);
 755                 break;
 756
 757         case LST_OPC_BATCHSRV:
 758                 entp = &ent.tbe_srv_nle;
 759         case LST_OPC_BATCHCLI:
 760                 rc = lst_info_batch_ioctl(str, 0, 0, &ent, NULL, NULL, NULL);
 761                 break;
 762
 763         case LST_OPC_GROUP:
 764                 rc = lst_info_group_ioctl(str, entp, NULL, NULL, NULL);
 765                 break;
 766
 767         case LST_OPC_NODES:
 768                 rc = lst_parse_nids(str, &entp->nle_nnode, idspp) < 0 ? -1 : 0;
 769                 break;
 770
 771         default:
 772                 rc = -1;
 773                 break;
 774         }
 775
 776         if (rc == 0)
 777                 *countp = entp->nle_nnode;
 778
 779         return rc;
 780 }
 781
 782 int
 783 jt_lst_ping(int argc,  char **argv)
 784 {
 785         struct list_head   head;
 786         lnet_process_id_t *ids = NULL;
 787         lstcon_rpc_ent_t  *ent = NULL;
 788         char              *str = NULL;
 789         int                optidx  = 0;
 790         int                server  = 0;
 791         int                timeout = 5;
 792         int                count   = 0;
 793         int                type    = 0;
 794         int                rc      = 0;
 795         int                c;
 796
 797         static struct option ping_opts[] =
 798         {
 799                 {"session", no_argument,       0, 's' },
 800                 {"server",  no_argument,       0, 'v' },
 801                 {"batch",   required_argument, 0, 'b' },
 802                 {"group",   required_argument, 0, 'g' },
 803                 {"nodes",   required_argument, 0, 'n' },
 804                 {"timeout", required_argument, 0, 't' },
 805                 {0,         0,                 0,  0  }
 806         };
 807
 808         if (session_key == 0) {
 809                 fprintf(stderr,
 810                         "Can't find env LST_SESSION or value is not valid\n");
 811                 return -1;
 812         }
 813
 814         while (1) {
 815
 816                 c = getopt_long(argc, argv, "g:b:n:t:sv",
 817                                 ping_opts, &optidx);
 818
 819                 if (c == -1)
 820                         break;
 821
 822                 switch (c) {
 823                 case 's':
 824                         type = LST_OPC_SESSION;
 825                         break;
 826
 827                 case 'g':
 828                         type = LST_OPC_GROUP;
 829                         str = optarg;
 830                         break;
 831
 832                 case 'b':
 833                         type = LST_OPC_BATCHCLI;
 834                         str = optarg;
 835                         break;
 836
 837                 case 'n':
 838                         type = LST_OPC_NODES;
 839                         str = optarg;
 840                         break;
 841
 842                 case 't':
 843                         timeout = atoi(optarg);
 844                         break;
 845
 846                 case 'v':
 847                         server = 1;
 848                         break;
 849
 850                 default:
 851                         lst_print_usage(argv[0]);
 852                         return -1;
 853                 }
 854         }
 855
 856         if (type == 0 || timeout <= 0 || optind != argc) {
 857                 lst_print_usage(argv[0]);
 858                 return -1;
 859         }
 860
 861         if (type == LST_OPC_BATCHCLI && server)
 862                 type = LST_OPC_BATCHSRV;
 863
 864         rc = lst_get_node_count(type, str, &count, &ids);
 865         if (rc < 0) {
 866                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
 867                         (str == NULL) ? "session" : str, strerror(errno));
 868                 return -1;
 869         }
 870
 871         INIT_LIST_HEAD(&head);
 872
 873         rc = lst_alloc_rpcent(&head, count, LST_NAME_SIZE);
 874         if (rc != 0) {
 875                 fprintf(stderr, "Out of memory\n");
 876                 goto out;
 877         }
 878
 879         if (count == 0) {
 880                 fprintf(stdout, "Target %s is empty\n",
 881                         (str == NULL) ? "session" : str);
 882                 goto out;
 883         }
 884
 885         rc = lst_ping_ioctl(str, type, timeout, count, ids, &head);
 886         if (rc == -1) { /* local failure */
 887                 lst_print_error("debug", "Failed to ping %s: %s\n",
 888                                 (str == NULL) ? "session" : str,
 889                                 strerror(errno));
 890                 rc = -1;
 891                 goto out;
 892         }
 893
 894         /* ignore RPC errors and framwork errors */
 895         list_for_each_entry(ent, &head, rpe_link) {
 896                 fprintf(stdout, "\t%s: %s [session: %s id: %s]\n",
 897                         libcfs_id2str(ent->rpe_peer),
 898                         lst_node_state2str(ent->rpe_state),
 899                         (ent->rpe_state == LST_NODE_ACTIVE ||
 900                          ent->rpe_state == LST_NODE_BUSY) ?
 901                                 (ent->rpe_rpc_errno == 0 ?
 902                                         &ent->rpe_payload[0] : "Unknown") :
 903                                 "<NULL>", libcfs_nid2str(ent->rpe_sid.ses_nid));
 904         }
 905
 906 out:
 907         lst_free_rpcent(&head);
 908
 909         if (ids != NULL)
 910                 free(ids);
 911
 912         return rc;
 913
 914 }
 915
 916 int
 917 lst_add_nodes_ioctl (char *name, int count, lnet_process_id_t *ids,
 918                      unsigned *featp, struct list_head *resultp)
 919 {
 920         lstio_group_nodes_args_t args = {0};
 921
 922         args.lstio_grp_key     = session_key;
 923         args.lstio_grp_nmlen   = strlen(name);
 924         args.lstio_grp_namep   = name;
 925         args.lstio_grp_count   = count;
 926         args.lstio_grp_featp   = featp;
 927         args.lstio_grp_idsp    = ids;
 928         args.lstio_grp_resultp = resultp;
 929
 930         return lst_ioctl(LSTIO_NODES_ADD, &args, sizeof(args));
 931 }
 932
 933 int
 934 lst_del_group_ioctl(char *name)
 935 {
 936         lstio_group_del_args_t args = {0};
 937
 938         args.lstio_grp_key   = session_key;
 939         args.lstio_grp_nmlen = strlen(name);
 940         args.lstio_grp_namep = name;
 941
 942         return lst_ioctl(LSTIO_GROUP_DEL, &args, sizeof(args));
 943 }
 944
 945 int
 946 lst_del_group(char *grp_name)
 947 {
 948         int     rc;
 949
 950         rc = lst_del_group_ioctl(grp_name);
 951         if (rc == 0) {
 952                 fprintf(stdout, "Group is deleted\n");
 953                 return 0;
 954         }
 955
 956         if (rc == -1) {
 957                 lst_print_error("group", "Failed to delete group: %s\n",
 958                                 strerror(errno));
 959                 return rc;
 960         }
 961
 962         fprintf(stderr, "Group is deleted with some errors\n");
 963
 964         if (trans_stat.trs_rpc_errno != 0) {
 965                 fprintf(stderr,
 966                         "[RPC] Failed to send %d end session RPCs: %s\n",
 967                         lstcon_rpc_stat_failure(&trans_stat, 0),
 968                         strerror(trans_stat.trs_rpc_errno));
 969         }
 970
 971         if (trans_stat.trs_fwk_errno != 0) {
 972                 fprintf(stderr,
 973                         "[FWK] Failed to end session on %d nodes: %s\n",
 974                 lstcon_sesop_stat_failure(&trans_stat, 0),
 975                 strerror(trans_stat.trs_fwk_errno));
 976         }
 977
 978         return -1;
 979 }
 980
 981 int
 982 lst_add_group_ioctl (char *name)
 983 {
 984         lstio_group_add_args_t args = {0};
 985
 986         args.lstio_grp_key     =  session_key;
 987         args.lstio_grp_nmlen   =  strlen(name);
 988         args.lstio_grp_namep   =  name;
 989
 990         return lst_ioctl(LSTIO_GROUP_ADD, &args, sizeof(args));
 991 }
 992
 993 int
 994 jt_lst_add_group(int argc, char **argv)
 995 {
 996         struct list_head   head;
 997         lnet_process_id_t *ids;
 998         char              *name;
 999         unsigned           feats = session_features;
1000         int                count;
1001         int                rc;
1002         int                i;
1003         bool               nodes_added = false;
1004
1005         if (session_key == 0) {
1006                 fprintf(stderr,
1007                         "Can't find env LST_SESSION or value is not valid\n");
1008                 return -1;
1009         }
1010
1011         if (argc < 3) {
1012                 lst_print_usage(argv[0]);
1013                 return -1;
1014         }
1015
1016         name = argv[1];
1017         if (strlen(name) >= LST_NAME_SIZE) {
1018                 fprintf(stderr, "Name length is limited to %d\n",
1019                         LST_NAME_SIZE - 1);
1020                 return -1;
1021         }
1022
1023         rc = lst_add_group_ioctl(name);
1024         if (rc != 0) {
1025                 lst_print_error("group", "Failed to add group %s: %s\n",
1026                                 name, strerror(errno));
1027                 return -1;
1028         }
1029
1030         INIT_LIST_HEAD(&head);
1031
1032         for (i = 2; i < argc; i++) {
1033                 /* parse address list */
1034                 rc = lst_parse_nids(argv[i], &count, &ids);
1035                 if (rc < 0) {
1036                         fprintf(stderr, "Ignore invalid id list %s\n",
1037                                 argv[i]);
1038                         continue;
1039                 }
1040
1041                 if (count == 0)
1042                         continue;
1043
1044                 rc = lst_alloc_rpcent(&head, count, 0);
1045                 if (rc != 0) {
1046                         fprintf(stderr, "Out of memory\n");
1047                         free(ids);
1048                         rc = -1;
1049                         goto failed;
1050                 }
1051
1052                 rc = lst_add_nodes_ioctl(name, count, ids, &feats, &head);
1053
1054                 free(ids);
1055
1056                 if (rc != 0)
1057                         goto failed;
1058
1059                 fprintf(stdout, "%s are added to session\n", argv[i]);
1060
1061                 nodes_added = true;
1062
1063                 if ((feats & session_features) != session_features) {
1064                         fprintf(stdout,
1065                                 "Warning, this session will run with "
1066                                 "compatible mode because some test nodes "
1067                                 "might not understand these features: %x\n",
1068                                 (~feats & session_features));
1069                 }
1070
1071                 lst_free_rpcent(&head);
1072         }
1073
1074         if (!nodes_added) {
1075                 /*
1076                  * The selftest kernel module expects that a group should
1077                  * have at least one node, since it doesn't make sense for
1078                  * an empty group to be added to a test.
1079                  */
1080                 fprintf(stderr,
1081                         "No nodes added successfully, deleting group %s\n",
1082                         name);
1083                 rc = lst_del_group(name);
1084                 if (rc != 0) {
1085                         fprintf(stderr,
1086                                 "Failed to delete group %s."
1087                                 "  Group is empty.\n", name);
1088                 }
1089         }
1090
1091         return rc;
1092
1093 failed:
1094         if (rc == -1) {
1095                 lst_print_error("group", "Failed to add nodes %s: %s\n",
1096                                 argv[i], strerror(errno));
1097
1098         } else {
1099                 if (trans_stat.trs_fwk_errno == EPROTO) {
1100                         fprintf(stderr,
1101                                 "test nodes might have different LST "
1102                                 "features, please disable some features by "
1103                                 "setting LST_FEATURES\n");
1104                 }
1105
1106                 lst_print_transerr(&head, "create session");
1107         }
1108
1109         lst_free_rpcent(&head);
1110
1111         if (!nodes_added) {
1112                 fprintf(stderr,
1113                         "No nodes added successfully, deleting group %s\n",
1114                         name);
1115                 if (lst_del_group(name) != 0) {
1116                         fprintf(stderr,
1117                                 "Failed to delete group %s."
1118                                 "  Group is empty.\n", name);
1119                 }
1120         }
1121
1122         return rc;
1123 }
1124
1125 int
1126 jt_lst_del_group(int argc, char **argv)
1127 {
1128         int     rc;
1129
1130         if (session_key == 0) {
1131                 fprintf(stderr,
1132                         "Can't find env LST_SESSION or value is not valid\n");
1133                 return -1;
1134         }
1135
1136         if (argc != 2) {
1137                 lst_print_usage(argv[0]);
1138                 return -1;
1139         }
1140
1141         rc = lst_del_group(argv[1]);
1142
1143         return rc;
1144 }
1145
1146 int
1147 lst_update_group_ioctl(int opc, char *name, int clean, int count,
1148                        lnet_process_id_t *ids, struct list_head *resultp)
1149 {
1150         lstio_group_update_args_t args = {0};
1151
1152         args.lstio_grp_key      = session_key;
1153         args.lstio_grp_opc      = opc;
1154         args.lstio_grp_args     = clean;
1155         args.lstio_grp_nmlen    = strlen(name);
1156         args.lstio_grp_namep    = name;
1157         args.lstio_grp_count    = count;
1158         args.lstio_grp_idsp     = ids;
1159         args.lstio_grp_resultp  = resultp;
1160
1161         return lst_ioctl(LSTIO_GROUP_UPDATE, &args, sizeof(args));
1162 }
1163
1164 int
1165 jt_lst_update_group(int argc, char **argv)
1166 {
1167         struct list_head         head;
1168         lnet_process_id_t *ids = NULL;
1169         char              *str = NULL;
1170         char              *grp = NULL;
1171         int                optidx = 0;
1172         int                count = 0;
1173         int                clean = 0;
1174         int                opc = 0;
1175         int                rc;
1176         int                c;
1177
1178         static struct option update_group_opts[] =
1179         {
1180                 {"refresh", no_argument,       0, 'f' },
1181                 {"clean",   required_argument, 0, 'c' },
1182                 {"remove",  required_argument, 0, 'r' },
1183                 {0,         0,                 0,  0  }
1184         };
1185
1186         if (session_key == 0) {
1187                 fprintf(stderr,
1188                         "Can't find env LST_SESSION or value is not valid\n");
1189                 return -1;
1190         }
1191
1192         while (1) {
1193                 c = getopt_long(argc, argv, "fc:r:",
1194                                 update_group_opts, &optidx);
1195
1196                 /* Detect the end of the options. */
1197                 if (c == -1)
1198                         break;
1199
1200                 switch (c) {
1201                 case 'f':
1202                         if (opc != 0) {
1203                                 lst_print_usage(argv[0]);
1204                                 return -1;
1205                         }
1206                         opc = LST_GROUP_REFRESH;
1207                         break;
1208
1209                 case 'r':
1210                         if (opc != 0) {
1211                                 lst_print_usage(argv[0]);
1212                                 return -1;
1213                         }
1214                         opc = LST_GROUP_RMND;
1215                         str = optarg;
1216                         break;
1217
1218                 case 'c':
1219                         clean = lst_node_str2state(optarg);
1220                         if (opc != 0 || clean <= 0) {
1221                                 lst_print_usage(argv[0]);
1222                                 return -1;
1223                         }
1224                         opc = LST_GROUP_CLEAN;
1225                         break;
1226
1227                 default:
1228                         lst_print_usage(argv[0]);
1229                         return -1;
1230                 }
1231         }
1232
1233         /* no OPC or group is specified */
1234         if (opc == 0 || optind != argc - 1) {
1235                 lst_print_usage(argv[0]);
1236                 return -1;
1237         }
1238
1239         grp = argv[optind];
1240
1241         INIT_LIST_HEAD(&head);
1242
1243         if (opc == LST_GROUP_RMND || opc == LST_GROUP_REFRESH) {
1244                 rc = lst_get_node_count(opc == LST_GROUP_RMND ? LST_OPC_NODES :
1245                                                                 LST_OPC_GROUP,
1246                                         opc == LST_GROUP_RMND ? str : grp,
1247                                         &count, &ids);
1248
1249                 if (rc != 0) {
1250                         fprintf(stderr, "Can't get count of nodes from %s: %s\n",
1251                                 opc == LST_GROUP_RMND ? str : grp,
1252                                 strerror(errno));
1253                         return -1;
1254                 }
1255
1256                 rc = lst_alloc_rpcent(&head, count, 0);
1257                 if (rc != 0) {
1258                         fprintf(stderr, "Out of memory\n");
1259                         free(ids);
1260                         return -1;
1261                 }
1262
1263         }
1264
1265         rc = lst_update_group_ioctl(opc, grp, clean, count, ids, &head);
1266
1267         if (ids != NULL)
1268                 free(ids);
1269
1270         if (rc == 0) {
1271                 lst_free_rpcent(&head);
1272                 return 0;
1273         }
1274
1275         if (rc == -1) {
1276                 lst_free_rpcent(&head);
1277                 lst_print_error("group", "Failed to update group: %s\n",
1278                                 strerror(errno));
1279                 return rc;
1280         }
1281
1282         lst_print_transerr(&head, "Updating group");
1283
1284         lst_free_rpcent(&head);
1285
1286         return rc;
1287 }
1288
1289 int
1290 lst_list_group_ioctl(int len, char *name, int idx)
1291 {
1292         lstio_group_list_args_t args = {0};
1293
1294         args.lstio_grp_key   = session_key;
1295         args.lstio_grp_idx   = idx;
1296         args.lstio_grp_nmlen = len;
1297         args.lstio_grp_namep = name;
1298
1299         return lst_ioctl(LSTIO_GROUP_LIST, &args, sizeof(args));
1300 }
1301
1302 int
1303 lst_info_group_ioctl(char *name, lstcon_ndlist_ent_t *gent,
1304                      int *idx, int *count, lstcon_node_ent_t *dents)
1305 {
1306         lstio_group_info_args_t args = {0};
1307
1308         args.lstio_grp_key    = session_key;
1309         args.lstio_grp_nmlen  = strlen(name);
1310         args.lstio_grp_namep  = name;
1311         args.lstio_grp_entp   = gent;
1312         args.lstio_grp_idxp   = idx;
1313         args.lstio_grp_ndentp = count;
1314         args.lstio_grp_dentsp = dents;
1315
1316         return lst_ioctl(LSTIO_GROUP_INFO, &args, sizeof(args));
1317 }
1318
1319 int
1320 lst_list_group_all(void)
1321 {
1322         char  name[LST_NAME_SIZE];
1323         int   rc;
1324         int   i;
1325
1326         /* no group is specified, list name of all groups */
1327         for (i = 0; ; i++) {
1328                 rc = lst_list_group_ioctl(LST_NAME_SIZE, name, i);
1329                 if (rc == 0) {
1330                         fprintf(stdout, "%d) %s\n", i + 1, name);
1331                         continue;
1332                 }
1333
1334                 if (errno == ENOENT)
1335                         break;
1336
1337                 lst_print_error("group", "Failed to list group: %s\n",
1338                                 strerror(errno));
1339                 return -1;
1340         }
1341
1342         fprintf(stdout, "Total %d groups\n", i);
1343
1344         return 0;
1345 }
1346
1347 #define LST_NODES_TITLE "\tACTIVE\tBUSY\tDOWN\tUNKNOWN\tTOTAL\n"
1348
1349 int
1350 jt_lst_list_group(int argc, char **argv)
1351 {
1352         lstcon_ndlist_ent_t  gent;
1353         lstcon_node_ent_t   *dents;
1354         int               optidx  = 0;
1355         int               verbose = 0;
1356         int               active  = 0;
1357         int               busy    = 0;
1358         int               down    = 0;
1359         int               unknown = 0;
1360         int               all     = 0;
1361         int               count;
1362         int               index;
1363         int               i;
1364         int               j;
1365         int               c;
1366         int               rc = 0;
1367
1368         static struct option list_group_opts[] =
1369         {
1370                 {"active",  no_argument, 0, 'a' },
1371                 {"busy",    no_argument, 0, 'b' },
1372                 {"down",    no_argument, 0, 'd' },
1373                 {"unknown", no_argument, 0, 'u' },
1374                 {"all",     no_argument, 0, 'l' },
1375                 {0,         0,           0,  0  }
1376         };
1377
1378         if (session_key == 0) {
1379                 fprintf(stderr,
1380                         "Can't find env LST_SESSION or value is not valid\n");
1381                 return -1;
1382         }
1383
1384         while (1) {
1385                 c = getopt_long(argc, argv, "abdul",
1386                                 list_group_opts, &optidx);
1387
1388                 if (c == -1)
1389                         break;
1390
1391                 switch (c) {
1392                 case 'a':
1393                         verbose = active = 1;
1394                         all = 0;
1395                         break;
1396                 case 'b':
1397                         verbose = busy = 1;
1398                         all = 0;
1399                         break;
1400                 case 'd':
1401                         verbose = down = 1;
1402                         all = 0;
1403                         break;
1404                 case 'u':
1405                         verbose = unknown = 1;
1406                         all = 0;
1407                         break;
1408                 case 'l':
1409                         verbose = all = 1;
1410                         break;
1411                 default:
1412                         lst_print_usage(argv[0]);
1413                         return -1;
1414                 }
1415         }
1416
1417         if (optind == argc) {
1418                 /* no group is specified, list name of all groups */
1419                 rc = lst_list_group_all();
1420
1421                 return rc;
1422         }
1423
1424         if (!verbose)
1425                 fprintf(stdout, LST_NODES_TITLE);
1426
1427         /* list nodes in specified groups */
1428         for (i = optind; i < argc; i++) {
1429                 rc = lst_info_group_ioctl(argv[i], &gent, NULL, NULL, NULL);
1430                 if (rc != 0) {
1431                         if (errno == ENOENT) {
1432                                 rc = 0;
1433                                 break;
1434                         }
1435
1436                         lst_print_error("group", "Failed to list group\n",
1437                                         strerror(errno));
1438                         break;
1439                 }
1440
1441                 if (!verbose) {
1442                         fprintf(stdout, "\t%d\t%d\t%d\t%d\t%d\t%s\n",
1443                                 gent.nle_nactive, gent.nle_nbusy,
1444                                 gent.nle_ndown, gent.nle_nunknown,
1445                                 gent.nle_nnode, argv[i]);
1446                         continue;
1447                 }
1448
1449                 fprintf(stdout, "Group [ %s ]\n", argv[i]);
1450
1451                 if (gent.nle_nnode == 0) {
1452                         fprintf(stdout, "No nodes found [ %s ]\n", argv[i]);
1453                         continue;
1454                 }
1455
1456                 count = gent.nle_nnode;
1457
1458                 dents = malloc(count * sizeof(lstcon_node_ent_t));
1459                 if (dents == NULL) {
1460                         fprintf(stderr, "Failed to malloc: %s\n",
1461                                 strerror(errno));
1462                         return -1;
1463                 }
1464
1465                 index = 0;
1466                 rc = lst_info_group_ioctl(argv[i], &gent, &index, &count, dents);
1467                 if (rc != 0) {
1468                         lst_print_error("group", "Failed to list group: %s\n",
1469                                         strerror(errno));
1470                         free(dents);
1471                         return -1;
1472                 }
1473
1474                 for (j = 0, c = 0; j < count; j++) {
1475                         if (all ||
1476                             ((active  &&  dents[j].nde_state == LST_NODE_ACTIVE) ||
1477                              (busy    &&  dents[j].nde_state == LST_NODE_BUSY)   ||
1478                              (down    &&  dents[j].nde_state == LST_NODE_DOWN)   ||
1479                              (unknown &&  dents[j].nde_state == LST_NODE_UNKNOWN))) {
1480
1481                                 fprintf(stdout, "\t%s: %s\n",
1482                                         libcfs_id2str(dents[j].nde_id),
1483                                         lst_node_state2str(dents[j].nde_state));
1484                                 c++;
1485                         }
1486                 }
1487
1488                 fprintf(stdout, "Total %d nodes [ %s ]\n", c, argv[i]);
1489
1490                 free(dents);
1491         }
1492
1493         return rc;
1494 }
1495
1496 int
1497 lst_stat_ioctl (char *name, int count, lnet_process_id_t *idsp,
1498                 int timeout, struct list_head *resultp)
1499 {
1500         lstio_stat_args_t args = {0};
1501
1502         args.lstio_sta_key     = session_key;
1503         args.lstio_sta_timeout = timeout;
1504         args.lstio_sta_nmlen   = strlen(name);
1505         args.lstio_sta_namep   = name;
1506         args.lstio_sta_count   = count;
1507         args.lstio_sta_idsp    = idsp;
1508         args.lstio_sta_resultp = resultp;
1509
1510         return lst_ioctl (LSTIO_STAT_QUERY, &args, sizeof(args));
1511 }
1512
1513 typedef struct {
1514         struct list_head              srp_link;
1515         int                     srp_count;
1516         char                   *srp_name;
1517         lnet_process_id_t      *srp_ids;
1518         struct list_head              srp_result[2];
1519 } lst_stat_req_param_t;
1520
1521 static void
1522 lst_stat_req_param_free(lst_stat_req_param_t *srp)
1523 {
1524         int     i;
1525
1526         for (i = 0; i < 2; i++)
1527                 lst_free_rpcent(&srp->srp_result[i]);
1528
1529         if (srp->srp_ids != NULL)
1530                 free(srp->srp_ids);
1531
1532         free(srp);
1533 }
1534
1535 static int
1536 lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp, int save_old)
1537 {
1538         lst_stat_req_param_t *srp = NULL;
1539         int                   count = save_old ? 2 : 1;
1540         int                   rc;
1541         int                   i;
1542
1543         srp = malloc(sizeof(*srp));
1544         if (srp == NULL)
1545                 return -ENOMEM;
1546
1547         memset(srp, 0, sizeof(*srp));
1548         INIT_LIST_HEAD(&srp->srp_result[0]);
1549         INIT_LIST_HEAD(&srp->srp_result[1]);
1550
1551         rc = lst_get_node_count(LST_OPC_GROUP, name,
1552                                 &srp->srp_count, NULL);
1553         if (rc != 0 && errno == ENOENT) {
1554                 rc = lst_get_node_count(LST_OPC_NODES, name,
1555                                         &srp->srp_count, &srp->srp_ids);
1556         }
1557
1558         if (rc != 0) {
1559                 fprintf(stderr,
1560                         "Failed to get count of nodes from %s: %s\n",
1561                         name, strerror(errno));
1562                 lst_stat_req_param_free(srp);
1563
1564                 return rc;
1565         }
1566
1567         srp->srp_name = name;
1568
1569         for (i = 0; i < count; i++) {
1570                 rc = lst_alloc_rpcent(&srp->srp_result[i], srp->srp_count,
1571                                       sizeof(sfw_counters_t)  +
1572                                       sizeof(srpc_counters_t) +
1573                                       sizeof(lnet_counters_t));
1574                 if (rc != 0) {
1575                         fprintf(stderr, "Out of memory\n");
1576                         break;
1577                 }
1578         }
1579
1580         if (rc == 0) {
1581                 *srpp = srp;
1582                 return 0;
1583         }
1584
1585         lst_stat_req_param_free(srp);
1586
1587         return rc;
1588 }
1589
1590 typedef struct {
1591         /* TODO */
1592         int foo;
1593 } lst_srpc_stat_result;
1594
1595 #define LST_LNET_AVG    0
1596 #define LST_LNET_MIN    1
1597 #define LST_LNET_MAX    2
1598
1599 typedef struct {
1600         float           lnet_avg_sndrate;
1601         float           lnet_min_sndrate;
1602         float           lnet_max_sndrate;
1603         float           lnet_total_sndrate;
1604
1605         float           lnet_avg_rcvrate;
1606         float           lnet_min_rcvrate;
1607         float           lnet_max_rcvrate;
1608         float           lnet_total_rcvrate;
1609
1610         float           lnet_avg_sndperf;
1611         float           lnet_min_sndperf;
1612         float           lnet_max_sndperf;
1613         float           lnet_total_sndperf;
1614
1615         float           lnet_avg_rcvperf;
1616         float           lnet_min_rcvperf;
1617         float           lnet_max_rcvperf;
1618         float           lnet_total_rcvperf;
1619
1620         int             lnet_stat_count;
1621 } lst_lnet_stat_result_t;
1622
1623 lst_lnet_stat_result_t lnet_stat_result;
1624
1625 static float
1626 lst_lnet_stat_value(int bw, int send, int off)
1627 {
1628         float  *p;
1629
1630         p = bw ? &lnet_stat_result.lnet_avg_sndperf :
1631                  &lnet_stat_result.lnet_avg_sndrate;
1632
1633         if (!send)
1634                 p += 4;
1635
1636         p += off;
1637
1638         return *p;
1639 }
1640
1641 static void
1642 lst_timeval_diff(struct timeval *tv1,
1643                  struct timeval *tv2, struct timeval *df)
1644 {
1645         if (tv1->tv_usec >= tv2->tv_usec) {
1646                 df->tv_sec  = tv1->tv_sec - tv2->tv_sec;
1647                 df->tv_usec = tv1->tv_usec - tv2->tv_usec;
1648                 return;
1649         }
1650
1651         df->tv_sec  = tv1->tv_sec - 1 - tv2->tv_sec;
1652         df->tv_usec = tv1->tv_usec + 1000000 - tv2->tv_usec;
1653
1654         return;
1655 }
1656
1657 void
1658 lst_cal_lnet_stat(float delta, lnet_counters_t *lnet_new,
1659                   lnet_counters_t *lnet_old)
1660 {
1661         float perf;
1662         float rate;
1663
1664         perf = (float)(lnet_new->send_length -
1665                        lnet_old->send_length) / (1024 * 1024) / delta;
1666         lnet_stat_result.lnet_total_sndperf += perf;
1667
1668         if (lnet_stat_result.lnet_min_sndperf > perf ||
1669             lnet_stat_result.lnet_min_sndperf == 0)
1670                 lnet_stat_result.lnet_min_sndperf = perf;
1671
1672         if (lnet_stat_result.lnet_max_sndperf < perf)
1673                 lnet_stat_result.lnet_max_sndperf = perf;
1674
1675         perf = (float)(lnet_new->recv_length -
1676                        lnet_old->recv_length) / (1024 * 1024) / delta;
1677         lnet_stat_result.lnet_total_rcvperf += perf;
1678
1679         if (lnet_stat_result.lnet_min_rcvperf > perf ||
1680             lnet_stat_result.lnet_min_rcvperf == 0)
1681                 lnet_stat_result.lnet_min_rcvperf = perf;
1682
1683         if (lnet_stat_result.lnet_max_rcvperf < perf)
1684                 lnet_stat_result.lnet_max_rcvperf = perf;
1685
1686         rate = (lnet_new->send_count - lnet_old->send_count) / delta;
1687         lnet_stat_result.lnet_total_sndrate += rate;
1688
1689         if (lnet_stat_result.lnet_min_sndrate > rate ||
1690             lnet_stat_result.lnet_min_sndrate == 0)
1691                 lnet_stat_result.lnet_min_sndrate = rate;
1692
1693         if (lnet_stat_result.lnet_max_sndrate < rate)
1694                 lnet_stat_result.lnet_max_sndrate = rate;
1695
1696         rate = (lnet_new->recv_count - lnet_old->recv_count) / delta;
1697         lnet_stat_result.lnet_total_rcvrate += rate;
1698
1699         if (lnet_stat_result.lnet_min_rcvrate > rate ||
1700             lnet_stat_result.lnet_min_rcvrate == 0)
1701                 lnet_stat_result.lnet_min_rcvrate = rate;
1702
1703         if (lnet_stat_result.lnet_max_rcvrate < rate)
1704                 lnet_stat_result.lnet_max_rcvrate = rate;
1705
1706         lnet_stat_result.lnet_stat_count ++;
1707
1708         lnet_stat_result.lnet_avg_sndrate = lnet_stat_result.lnet_total_sndrate /
1709                                             lnet_stat_result.lnet_stat_count;
1710         lnet_stat_result.lnet_avg_rcvrate = lnet_stat_result.lnet_total_rcvrate /
1711                                             lnet_stat_result.lnet_stat_count;
1712
1713         lnet_stat_result.lnet_avg_sndperf = lnet_stat_result.lnet_total_sndperf /
1714                                             lnet_stat_result.lnet_stat_count;
1715         lnet_stat_result.lnet_avg_rcvperf = lnet_stat_result.lnet_total_rcvperf /
1716                                             lnet_stat_result.lnet_stat_count;
1717
1718 }
1719
1720 void
1721 lst_print_lnet_stat(char *name, int bwrt, int rdwr, int type)
1722 {
1723         int     start1 = 0;
1724         int     end1   = 1;
1725         int     start2 = 0;
1726         int     end2   = 1;
1727         int     i;
1728         int     j;
1729
1730         if (lnet_stat_result.lnet_stat_count == 0)
1731                 return;
1732
1733         if (bwrt == 1) /* bw only */
1734                 start1 = 1;
1735
1736         if (bwrt == 2) /* rates only */
1737                 end1 = 0;
1738
1739         if (rdwr == 1) /* recv only */
1740                 start2 = 1;
1741
1742         if (rdwr == 2) /* send only */
1743                 end2 = 0;
1744
1745         for (i = start1; i <= end1; i++) {
1746                 fprintf(stdout, "[LNet %s of %s]\n",
1747                         i == 0 ? "Rates" : "Bandwidth", name);
1748
1749                 for (j = start2; j <= end2; j++) {
1750                         fprintf(stdout, "[%c] ", j == 0 ? 'R' : 'W');
1751
1752                         if ((type & 1) != 0) {
1753                                 fprintf(stdout, i == 0 ? "Avg: %-8.0f RPC/s " :
1754                                                          "Avg: %-8.2f MB/s  ",
1755                                         lst_lnet_stat_value(i, j, 0));
1756                         }
1757
1758                         if ((type & 2) != 0) {
1759                                 fprintf(stdout, i == 0 ? "Min: %-8.0f RPC/s " :
1760                                                          "Min: %-8.2f MB/s  ",
1761                                         lst_lnet_stat_value(i, j, 1));
1762                         }
1763
1764                         if ((type & 4) != 0) {
1765                                 fprintf(stdout, i == 0 ? "Max: %-8.0f RPC/s" :
1766                                                          "Max: %-8.2f MB/s",
1767                                         lst_lnet_stat_value(i, j, 2));
1768                         }
1769
1770                         fprintf(stdout, "\n");
1771                 }
1772         }
1773 }
1774
1775 void
1776 lst_print_stat(char *name, struct list_head *resultp,
1777                int idx, int lnet, int bwrt, int rdwr, int type)
1778 {
1779         struct list_head        tmp[2];
1780         lstcon_rpc_ent_t *new;
1781         lstcon_rpc_ent_t *old;
1782         sfw_counters_t   *sfwk_new;
1783         sfw_counters_t   *sfwk_old;
1784         srpc_counters_t  *srpc_new;
1785         srpc_counters_t  *srpc_old;
1786         lnet_counters_t  *lnet_new;
1787         lnet_counters_t  *lnet_old;
1788         float             delta;
1789         int               errcount = 0;
1790
1791         INIT_LIST_HEAD(&tmp[0]);
1792         INIT_LIST_HEAD(&tmp[1]);
1793
1794         memset(&lnet_stat_result, 0, sizeof(lnet_stat_result));
1795
1796         while (!list_empty(&resultp[idx])) {
1797                 if (list_empty(&resultp[1 - idx])) {
1798                         fprintf(stderr, "Group is changed, re-run stat\n");
1799                         break;
1800                 }
1801
1802                 new = list_entry(resultp[idx].next, lstcon_rpc_ent_t,
1803                                      rpe_link);
1804                 old = list_entry(resultp[1 - idx].next, lstcon_rpc_ent_t,
1805                                      rpe_link);
1806
1807                 /* first time get stats result, can't calculate diff */
1808                 if (new->rpe_peer.nid == LNET_NID_ANY)
1809                         break;
1810
1811                 if (new->rpe_peer.nid != old->rpe_peer.nid ||
1812                     new->rpe_peer.pid != old->rpe_peer.pid) {
1813                         /* Something wrong. i.e, somebody change the group */
1814                         break;
1815                 }
1816
1817                 list_del(&new->rpe_link);
1818                 list_add_tail(&new->rpe_link, &tmp[idx]);
1819
1820                 list_del(&old->rpe_link);
1821                 list_add_tail(&old->rpe_link, &tmp[1 - idx]);
1822
1823                 if (new->rpe_rpc_errno != 0 || new->rpe_fwk_errno != 0 ||
1824                     old->rpe_rpc_errno != 0 || old->rpe_fwk_errno != 0) {
1825                         errcount ++;
1826                         continue;
1827                 }
1828
1829                 sfwk_new = (sfw_counters_t *)&new->rpe_payload[0];
1830                 sfwk_old = (sfw_counters_t *)&old->rpe_payload[0];
1831
1832                 srpc_new = (srpc_counters_t *)((char *)sfwk_new + sizeof(*sfwk_new));
1833                 srpc_old = (srpc_counters_t *)((char *)sfwk_old + sizeof(*sfwk_old));
1834
1835                 lnet_new = (lnet_counters_t *)((char *)srpc_new + sizeof(*srpc_new));
1836                 lnet_old = (lnet_counters_t *)((char *)srpc_old + sizeof(*srpc_old));
1837
1838                 /* Prior to version 2.3, the running_ms field was a counter for
1839                  * the number of running tests.  We are looking at this value
1840                  * to determine if it is a millisecond timestamep (>= 2.3) or a
1841                  * test counter (< 2.3).  The number 500 is being used for this
1842                  * barrier as the test counter should never get this high, and
1843                  * the timestamp should never get this low. */
1844
1845                 if (sfwk_new->running_ms > 500) {
1846                         /* use the timestamp from the remote node, not our
1847                          * rpe_stamp from when we copied up the data out of
1848                          * the kernel */
1849
1850                         delta = (float) (sfwk_new->running_ms -
1851                                         sfwk_old->running_ms) / 1000;
1852                 } else {
1853                         struct timeval    tv;
1854
1855                         lst_timeval_diff(&new->rpe_stamp, &old->rpe_stamp, &tv);
1856                         delta = tv.tv_sec + (float)tv.tv_usec / 1000000;
1857                 }
1858
1859                 if (!lnet) /* TODO */
1860                         continue;
1861
1862                 lst_cal_lnet_stat(delta, lnet_new, lnet_old);
1863         }
1864
1865         list_splice(&tmp[idx], &resultp[idx]);
1866         list_splice(&tmp[1 - idx], &resultp[1 - idx]);
1867
1868         if (errcount > 0)
1869                 fprintf(stdout, "Failed to stat on %d nodes\n", errcount);
1870
1871         if (!lnet)  /* TODO */
1872                 return;
1873
1874         lst_print_lnet_stat(name, bwrt, rdwr, type);
1875 }
1876
1877 int
1878 jt_lst_stat(int argc, char **argv)
1879 {
1880         struct list_head        head;
1881         lst_stat_req_param_t *srp;
1882         time_t                last    = 0;
1883         int                   optidx  = 0;
1884         int                   timeout = 5; /* default timeout, 5 sec */
1885         int                   delay   = 5; /* default delay, 5 sec */
1886         int                   count   = -1; /* run forever */
1887         int                   lnet    = 1; /* lnet stat by default */
1888         int                   bwrt    = 0;
1889         int                   rdwr    = 0;
1890         int                   type    = -1;
1891         int                   idx     = 0;
1892         int                   rc;
1893         int                   c;
1894
1895         static struct option stat_opts[] =
1896         {
1897                 {"timeout"   , required_argument, 0, 't' },
1898                 {"delay"     , required_argument, 0, 'd' },
1899                 {"count"     , required_argument, 0, 'o' },
1900                 {"lnet"      , no_argument,      0, 'l' },
1901                 {"rpc"       , no_argument,      0, 'c' },
1902                 {"bw"        , no_argument,      0, 'b' },
1903                 {"rate"      , no_argument,      0, 'a' },
1904                 {"read"      , no_argument,      0, 'r' },
1905                 {"write"     , no_argument,      0, 'w' },
1906                 {"avg"       , no_argument,      0, 'g' },
1907                 {"min"       , no_argument,      0, 'n' },
1908                 {"max"       , no_argument,      0, 'x' },
1909                 {0,            0,                0,  0  }
1910         };
1911
1912         if (session_key == 0) {
1913                 fprintf(stderr,
1914                         "Can't find env LST_SESSION or value is not valid\n");
1915                 return -1;
1916         }
1917
1918         while (1) {
1919                 c = getopt_long(argc, argv, "t:d:lcbarwgnx", stat_opts, &optidx);
1920
1921                 if (c == -1)
1922                         break;
1923
1924                 switch (c) {
1925                 case 't':
1926                         timeout = atoi(optarg);
1927                         break;
1928                 case 'd':
1929                         delay = atoi(optarg);
1930                         break;
1931                 case 'o':
1932                         count = atoi(optarg);
1933                         break;
1934                 case 'l':
1935                         lnet = 1;
1936                         break;
1937                 case 'c':
1938                         lnet = 0;
1939                         break;
1940                 case 'b':
1941                         bwrt |= 1;
1942                         break;
1943                 case 'a':
1944                         bwrt |= 2;
1945                         break;
1946                 case 'r':
1947                         rdwr |= 1;
1948                         break;
1949                 case 'w':
1950                         rdwr |= 2;
1951                         break;
1952                 case 'g':
1953                         if (type == -1) {
1954                                 type = 1;
1955                                 break;
1956                         }
1957                         type |= 1;
1958                         break;
1959                 case 'n':
1960                         if (type == -1) {
1961                                 type = 2;
1962                                 break;
1963                         }
1964                         type |= 2;
1965                         break;
1966                 case 'x':
1967                         if (type == -1) {
1968                                 type = 4;
1969                                 break;
1970                         }
1971                         type |= 4;
1972                         break;
1973
1974                 default:
1975                         lst_print_usage(argv[0]);
1976                         return -1;
1977                 }
1978         }
1979
1980         if (optind == argc) {
1981                 lst_print_usage(argv[0]);
1982                 return -1;
1983         }
1984
1985         if (timeout <= 0 || delay <= 0) {
1986                 fprintf(stderr, "Invalid timeout or delay value\n");
1987                 return -1;
1988         }
1989
1990         if (count < -1) {
1991             fprintf(stderr, "Invalid count value\n");
1992             return -1;
1993         }
1994
1995         /* extra count to get first data point */
1996         if (count != -1)
1997             count++;
1998
1999         INIT_LIST_HEAD(&head);
2000
2001         while (optind < argc) {
2002                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 1);
2003                 if (rc != 0)
2004                         goto out;
2005
2006                 list_add_tail(&srp->srp_link, &head);
2007         }
2008
2009         do {
2010                 time_t  now = time(NULL);
2011
2012                 if (now - last < delay) {
2013                         sleep(delay - now + last);
2014                         time(&now);
2015                 }
2016                 last = now;
2017
2018                 list_for_each_entry(srp, &head, srp_link) {
2019                         rc = lst_stat_ioctl(srp->srp_name,
2020                                             srp->srp_count, srp->srp_ids,
2021                                             timeout, &srp->srp_result[idx]);
2022                         if (rc == -1) {
2023                                 lst_print_error("stat", "Failed to stat %s: %s\n",
2024                                                 srp->srp_name, strerror(errno));
2025                                 goto out;
2026                         }
2027
2028                         lst_print_stat(srp->srp_name, srp->srp_result,
2029                                        idx, lnet, bwrt, rdwr, type);
2030
2031                         lst_reset_rpcent(&srp->srp_result[1 - idx]);
2032                 }
2033
2034                 idx = 1 - idx;
2035
2036                 if (count > 0)
2037                         count--;
2038         } while (count == -1 || count > 0);
2039
2040 out:
2041         while (!list_empty(&head)) {
2042                 srp = list_entry(head.next, lst_stat_req_param_t, srp_link);
2043
2044                 list_del(&srp->srp_link);
2045                 lst_stat_req_param_free(srp);
2046         }
2047
2048         return rc;
2049 }
2050
2051 int
2052 jt_lst_show_error(int argc, char **argv)
2053 {
2054         struct list_head            head;
2055         lst_stat_req_param_t *srp;
2056         lstcon_rpc_ent_t     *ent;
2057         sfw_counters_t       *sfwk;
2058         srpc_counters_t      *srpc;
2059         int                   show_rpc = 1;
2060         int                   optidx   = 0;
2061         int                   rc       = 0;
2062         int                   ecount;
2063         int                   c;
2064
2065         static struct option  show_error_opts[] =
2066         {
2067                 {"session", no_argument,       0, 's' },
2068                 {0,         0,                 0,  0  }
2069         };
2070
2071         if (session_key == 0) {
2072                 fprintf(stderr,
2073                         "Can't find env LST_SESSION or value is not valid\n");
2074                 return -1;
2075         }
2076
2077         while (1) {
2078                 c = getopt_long(argc, argv, "s", show_error_opts, &optidx);
2079
2080                 if (c == -1)
2081                         break;
2082
2083                 switch (c) {
2084                 case 's':
2085                         show_rpc  = 0;
2086                         break;
2087
2088                 default:
2089                         lst_print_usage(argv[0]);
2090                         return -1;
2091                 }
2092         }
2093
2094         if (optind == argc) {
2095                 lst_print_usage(argv[0]);
2096                 return -1;
2097         }
2098
2099         INIT_LIST_HEAD(&head);
2100
2101         while (optind < argc) {
2102                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 0);
2103                 if (rc != 0)
2104                         goto out;
2105
2106                 list_add_tail(&srp->srp_link, &head);
2107         }
2108
2109         list_for_each_entry(srp, &head, srp_link) {
2110                 rc = lst_stat_ioctl(srp->srp_name, srp->srp_count,
2111                                     srp->srp_ids, 10, &srp->srp_result[0]);
2112
2113                 if (rc == -1) {
2114                         lst_print_error(srp->srp_name, "Failed to show errors of %s: %s\n",
2115                                         srp->srp_name, strerror(errno));
2116                         goto out;
2117                 }
2118
2119                 fprintf(stdout, "%s:\n", srp->srp_name);
2120
2121                 ecount = 0;
2122
2123                 list_for_each_entry(ent, &srp->srp_result[0], rpe_link) {
2124                         if (ent->rpe_rpc_errno != 0) {
2125                                 ecount ++;
2126                                 fprintf(stderr, "RPC failure, can't show error on %s\n",
2127                                         libcfs_id2str(ent->rpe_peer));
2128                                 continue;
2129                         }
2130
2131                         if (ent->rpe_fwk_errno != 0) {
2132                                 ecount ++;
2133                                 fprintf(stderr, "Framework failure, can't show error on %s\n",
2134                                         libcfs_id2str(ent->rpe_peer));
2135                                 continue;
2136                         }
2137
2138                         sfwk = (sfw_counters_t *)&ent->rpe_payload[0];
2139                         srpc = (srpc_counters_t *)((char *)sfwk + sizeof(*sfwk));
2140
2141                         if (srpc->errors == 0 &&
2142                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2143                                 continue;
2144
2145                         if (!show_rpc  &&
2146                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2147                                 continue;
2148
2149                         ecount ++;
2150
2151                         fprintf(stderr, "%s: [Session %d brw errors, %d ping errors]%c",
2152                                 libcfs_id2str(ent->rpe_peer),
2153                                 sfwk->brw_errors, sfwk->ping_errors,
2154                                 show_rpc  ? ' ' : '\n');
2155
2156                         if (!show_rpc)
2157                                 continue;
2158
2159                         fprintf(stderr, "[RPC: %d errors, %d dropped, %d expired]\n",
2160                                 srpc->errors, srpc->rpcs_dropped, srpc->rpcs_expired);
2161                 }
2162
2163                 fprintf(stdout, "Total %d error nodes in %s\n", ecount, srp->srp_name);
2164         }
2165 out:
2166         while (!list_empty(&head)) {
2167                 srp = list_entry(head.next, lst_stat_req_param_t, srp_link);
2168
2169                 list_del(&srp->srp_link);
2170                 lst_stat_req_param_free(srp);
2171         }
2172
2173         return rc;
2174 }
2175
2176 int
2177 lst_add_batch_ioctl (char *name)
2178 {
2179         lstio_batch_add_args_t args = {0};
2180
2181         args.lstio_bat_key   = session_key;
2182         args.lstio_bat_nmlen = strlen(name);
2183         args.lstio_bat_namep = name;
2184
2185         return lst_ioctl (LSTIO_BATCH_ADD, &args, sizeof(args));
2186 }
2187
2188 int
2189 jt_lst_add_batch(int argc, char **argv)
2190 {
2191         char   *name;
2192         int     rc;
2193
2194         if (session_key == 0) {
2195                 fprintf(stderr,
2196                         "Can't find env LST_SESSION or value is not valid\n");
2197                 return -1;
2198         }
2199
2200         if (argc != 2) {
2201                 lst_print_usage(argv[0]);
2202                 return -1;
2203         }
2204
2205         name = argv[1];
2206         if (strlen(name) >= LST_NAME_SIZE) {
2207                 fprintf(stderr, "Name length is limited to %d\n",
2208                         LST_NAME_SIZE - 1);
2209                 return -1;
2210         }
2211
2212         rc = lst_add_batch_ioctl(name);
2213         if (rc == 0)
2214                 return 0;
2215
2216         lst_print_error("batch", "Failed to create batch: %s\n",
2217                         strerror(errno));
2218
2219         return -1;
2220 }
2221
2222 int
2223 lst_start_batch_ioctl(char *name, int timeout, struct list_head *resultp)
2224 {
2225         lstio_batch_run_args_t args = {0};
2226
2227         args.lstio_bat_key     = session_key;
2228         args.lstio_bat_timeout = timeout;
2229         args.lstio_bat_nmlen   = strlen(name);
2230         args.lstio_bat_namep   = name;
2231         args.lstio_bat_resultp = resultp;
2232
2233         return lst_ioctl(LSTIO_BATCH_START, &args, sizeof(args));
2234 }
2235
2236 int
2237 jt_lst_start_batch(int argc, char **argv)
2238 {
2239         struct list_head        head;
2240         char             *batch;
2241         int               optidx  = 0;
2242         int               timeout = 0;
2243         int               count = 0;
2244         int               rc;
2245         int               c;
2246
2247         static struct option start_batch_opts[] =
2248         {
2249                 {"timeout", required_argument, 0, 't' },
2250                 {0,         0,                 0,  0  }
2251         };
2252
2253         if (session_key == 0) {
2254                 fprintf(stderr,
2255                         "Can't find env LST_SESSION or value is not valid\n");
2256                 return -1;
2257         }
2258
2259         while (1) {
2260                 c = getopt_long(argc, argv, "t:",
2261                                 start_batch_opts, &optidx);
2262
2263                 /* Detect the end of the options. */
2264                 if (c == -1)
2265                         break;
2266
2267                 switch (c) {
2268                 case 't':
2269                         timeout = atoi(optarg);
2270                         break;
2271                 default:
2272                         lst_print_usage(argv[0]);
2273                         return -1;
2274                 }
2275         }
2276
2277         if (optind == argc) {
2278                 batch = LST_DEFAULT_BATCH;
2279
2280         } else if (optind == argc - 1) {
2281                 batch = argv[optind];
2282
2283         } else {
2284                 lst_print_usage(argv[0]);
2285                 return -1;
2286         }
2287
2288         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2289         if (rc != 0) {
2290                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2291                         batch, strerror(errno));
2292                 return -1;
2293         }
2294
2295         INIT_LIST_HEAD(&head);
2296
2297         rc = lst_alloc_rpcent(&head, count, 0);
2298         if (rc != 0) {
2299                 fprintf(stderr, "Out of memory\n");
2300                 return -1;
2301         }
2302
2303         rc = lst_start_batch_ioctl(batch, timeout, &head);
2304
2305         if (rc == 0) {
2306                 fprintf(stdout, "%s is running now\n", batch);
2307                 lst_free_rpcent(&head);
2308                 return 0;
2309         }
2310
2311         if (rc == -1) {
2312                 lst_print_error("batch", "Failed to start batch: %s\n",
2313                                 strerror(errno));
2314                 lst_free_rpcent(&head);
2315                 return rc;
2316         }
2317
2318         lst_print_transerr(&head, "Run batch");
2319
2320         lst_free_rpcent(&head);
2321
2322         return rc;
2323 }
2324
2325 int
2326 lst_stop_batch_ioctl(char *name, int force, struct list_head *resultp)
2327 {
2328         lstio_batch_stop_args_t args = {0};
2329
2330         args.lstio_bat_key     = session_key;
2331         args.lstio_bat_force   = force;
2332         args.lstio_bat_nmlen   = strlen(name);
2333         args.lstio_bat_namep   = name;
2334         args.lstio_bat_resultp = resultp;
2335
2336         return lst_ioctl(LSTIO_BATCH_STOP, &args, sizeof(args));
2337 }
2338
2339 int
2340 jt_lst_stop_batch(int argc, char **argv)
2341 {
2342         struct list_head        head;
2343         char             *batch;
2344         int               force = 0;
2345         int               optidx;
2346         int               count;
2347         int               rc;
2348         int               c;
2349
2350         static struct option stop_batch_opts[] =
2351         {
2352                 {"force",   no_argument,   0, 'f' },
2353                 {0,         0,             0,  0  }
2354         };
2355
2356         if (session_key == 0) {
2357                 fprintf(stderr,
2358                         "Can't find env LST_SESSION or value is not valid\n");
2359                 return -1;
2360         }
2361
2362         while (1) {
2363                 c = getopt_long(argc, argv, "f",
2364                                 stop_batch_opts, &optidx);
2365
2366                 /* Detect the end of the options. */
2367                 if (c == -1)
2368                         break;
2369
2370                 switch (c) {
2371                 case 'f':
2372                         force = 1;
2373                         break;
2374                 default:
2375                         lst_print_usage(argv[0]);
2376                         return -1;
2377                 }
2378         }
2379
2380         if (optind == argc) {
2381                 batch = LST_DEFAULT_BATCH;
2382
2383         } else if (optind == argc - 1) {
2384                 batch = argv[optind];
2385
2386         } else {
2387                 lst_print_usage(argv[0]);
2388                 return -1;
2389         }
2390
2391         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2392         if (rc != 0) {
2393                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2394                         batch, strerror(errno));
2395                 return -1;
2396         }
2397
2398         INIT_LIST_HEAD(&head);
2399
2400         rc = lst_alloc_rpcent(&head, count, 0);
2401         if (rc != 0) {
2402                 fprintf(stderr, "Out of memory\n");
2403                 return -1;
2404         }
2405
2406         rc = lst_stop_batch_ioctl(batch, force, &head);
2407         if (rc != 0)
2408                 goto out;
2409
2410         while (1) {
2411                 lst_reset_rpcent(&head);
2412
2413                 rc = lst_query_batch_ioctl(batch, 0, 0, 30, &head);
2414                 if (rc != 0)
2415                         goto out;
2416
2417                 if (lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0 &&
2418                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0)
2419                         break;
2420
2421                 fprintf(stdout, "%d batch in stopping\n",
2422                         lstcon_tsbqry_stat_run(&trans_stat, 0));
2423                 sleep(1);
2424         }
2425
2426         fprintf(stdout, "Batch is stopped\n");
2427         lst_free_rpcent(&head);
2428
2429         return 0;
2430 out:
2431         if (rc == -1) {
2432                 lst_print_error("batch", "Failed to stop batch: %s\n",
2433                                 strerror(errno));
2434                 lst_free_rpcent(&head);
2435                 return -1;
2436         }
2437
2438         lst_print_transerr(&head, "stop batch");
2439
2440         lst_free_rpcent(&head);
2441
2442         return rc;
2443 }
2444
2445 int
2446 lst_list_batch_ioctl(int len, char *name, int index)
2447 {
2448         lstio_batch_list_args_t args = {0};
2449
2450         args.lstio_bat_key   = session_key;
2451         args.lstio_bat_idx   = index;
2452         args.lstio_bat_nmlen = len;
2453         args.lstio_bat_namep = name;
2454
2455         return lst_ioctl(LSTIO_BATCH_LIST, &args, sizeof(args));
2456 }
2457
2458 int
2459 lst_info_batch_ioctl(char *batch, int test, int server,
2460                      lstcon_test_batch_ent_t *entp, int *idxp,
2461                      int *ndentp, lstcon_node_ent_t *dentsp)
2462 {
2463         lstio_batch_info_args_t args = {0};
2464
2465         args.lstio_bat_key     = session_key;
2466         args.lstio_bat_nmlen   = strlen(batch);
2467         args.lstio_bat_namep   = batch;
2468         args.lstio_bat_server  = server;
2469         args.lstio_bat_testidx = test;
2470         args.lstio_bat_entp    = entp;
2471         args.lstio_bat_idxp    = idxp;
2472         args.lstio_bat_ndentp  = ndentp;
2473         args.lstio_bat_dentsp  = dentsp;
2474
2475         return lst_ioctl(LSTIO_BATCH_INFO, &args, sizeof(args));
2476 }
2477
2478 int
2479 lst_list_batch_all(void)
2480 {
2481         char name[LST_NAME_SIZE];
2482         int  rc;
2483         int  i;
2484
2485         for (i = 0; ; i++) {
2486                 rc = lst_list_batch_ioctl(LST_NAME_SIZE, name, i);
2487                 if (rc == 0) {
2488                         fprintf(stdout, "%d) %s\n", i + 1, name);
2489                         continue;
2490                 }
2491
2492                 if (errno == ENOENT)
2493                         break;
2494
2495                 lst_print_error("batch", "Failed to list batch: %s\n",
2496                                 strerror(errno));
2497                 return rc;
2498         }
2499
2500         fprintf(stdout, "Total %d batches\n", i);
2501
2502         return 0;
2503 }
2504
2505 int
2506 lst_list_tsb_nodes(char *batch, int test, int server,
2507                    int count, int active, int invalid)
2508 {
2509         lstcon_node_ent_t *dents;
2510         int                index = 0;
2511         int                rc;
2512         int                c;
2513         int                i;
2514
2515         if (count == 0)
2516                 return 0;
2517
2518         /* verbose list, show nodes in batch or test */
2519         dents = malloc(count * sizeof(lstcon_node_ent_t));
2520         if (dents == NULL) {
2521                 fprintf(stdout, "Can't allocate memory\n");
2522                 return -1;
2523         }
2524
2525         rc = lst_info_batch_ioctl(batch, test, server,
2526                                   NULL, &index, &count, dents);
2527         if (rc != 0) {
2528                 free(dents);
2529                 lst_print_error((test > 0) ? "test" : "batch",
2530                                 (test > 0) ? "Failed to query test: %s\n" :
2531                                              "Failed to query batch: %s\n",
2532                                 strerror(errno));
2533                 return -1;
2534         }
2535
2536         for (i = 0, c = 0; i < count; i++) {
2537                 if ((!active  && dents[i].nde_state == LST_NODE_ACTIVE) ||
2538                     (!invalid && (dents[i].nde_state == LST_NODE_BUSY  ||
2539                                   dents[i].nde_state == LST_NODE_DOWN  ||
2540                                   dents[i].nde_state == LST_NODE_UNKNOWN)))
2541                         continue;
2542
2543                 fprintf(stdout, "\t%s: %s\n",
2544                         libcfs_id2str(dents[i].nde_id),
2545                         lst_node_state2str(dents[i].nde_state));
2546                 c++;
2547         }
2548
2549         fprintf(stdout, "Total %d nodes\n", c);
2550         free(dents);
2551
2552         return 0;
2553 }
2554
2555 int
2556 jt_lst_list_batch(int argc, char **argv)
2557 {
2558         lstcon_test_batch_ent_t ent;
2559         char                *batch   = NULL;
2560         int                  optidx  = 0;
2561         int                  verbose = 0; /* list nodes in batch or test */
2562         int                  invalid = 0;
2563         int                  active  = 0;
2564         int                  server  = 0;
2565         int                  ntest   = 0;
2566         int                  test    = 0;
2567         int                  c       = 0;
2568         int                  rc;
2569
2570         static struct option list_batch_opts[] =
2571         {
2572                 {"test",    required_argument, 0, 't' },
2573                 {"invalid", no_argument,       0, 'i' },
2574                 {"active",  no_argument,       0, 'a' },
2575                 {"all",     no_argument,       0, 'l' },
2576                 {"server",  no_argument,       0, 's' },
2577                 {0,         0,                 0,  0  }
2578         };
2579
2580         if (session_key == 0) {
2581                 fprintf(stderr,
2582                         "Can't find env LST_SESSION or value is not valid\n");
2583                 return -1;
2584         }
2585
2586         while (1) {
2587                 c = getopt_long(argc, argv, "ailst:",
2588                                 list_batch_opts, &optidx);
2589
2590                 if (c == -1)
2591                         break;
2592
2593                 switch (c) {
2594                 case 'a':
2595                         verbose = active = 1;
2596                         break;
2597                 case 'i':
2598                         verbose = invalid = 1;
2599                         break;
2600                 case 'l':
2601                         verbose = active = invalid = 1;
2602                         break;
2603                 case 's':
2604                         server = 1;
2605                         break;
2606                 case 't':
2607                         test = atoi(optarg);
2608                         ntest = 1;
2609                         break;
2610                 default:
2611                         lst_print_usage(argv[0]);
2612                         return -1;
2613                 }
2614         }
2615
2616         if (optind == argc) {
2617                 /* list all batches */
2618                 rc = lst_list_batch_all();
2619                 return rc;
2620         }
2621
2622         if (ntest == 1 && test <= 0) {
2623                 fprintf(stderr, "Invalid test id, test id starts from 1\n");
2624                 return -1;
2625         }
2626
2627         if (optind != argc - 1) {
2628                 lst_print_usage(argv[0]);
2629                 return -1;
2630         }
2631
2632         batch = argv[optind];
2633
2634 loop:
2635         /* show detail of specified batch or test */
2636         rc = lst_info_batch_ioctl(batch, test, server,
2637                                   &ent, NULL, NULL, NULL);
2638         if (rc != 0) {
2639                 lst_print_error((test > 0) ? "test" : "batch",
2640                                 (test > 0) ? "Failed to query test: %s\n" :
2641                                              "Failed to query batch: %s\n",
2642                                 strerror(errno));
2643                 return -1;
2644         }
2645
2646         if (verbose) {
2647                 /* list nodes in test or batch */
2648                 rc = lst_list_tsb_nodes(batch, test, server,
2649                                         server ? ent.tbe_srv_nle.nle_nnode :
2650                                                  ent.tbe_cli_nle.nle_nnode,
2651                                         active, invalid);
2652                 return rc;
2653         }
2654
2655         /* only show number of hosts in batch or test */
2656         if (test == 0) {
2657                 fprintf(stdout, "Batch: %s Tests: %d State: %d\n",
2658                         batch, ent.u.tbe_batch.bae_ntest,
2659                         ent.u.tbe_batch.bae_state);
2660                 ntest = ent.u.tbe_batch.bae_ntest;
2661                 test = 1; /* starting from test 1 */
2662
2663         } else {
2664                 fprintf(stdout,
2665                         "\tTest %d(%s) (loop: %d, concurrency: %d)\n",
2666                         test, lst_test_type2name(ent.u.tbe_test.tse_type),
2667                         ent.u.tbe_test.tse_loop,
2668                         ent.u.tbe_test.tse_concur);
2669                 ntest --;
2670                 test ++;
2671         }
2672
2673         fprintf(stdout, LST_NODES_TITLE);
2674         fprintf(stdout, "client\t%d\t%d\t%d\t%d\t%d\n"
2675                         "server\t%d\t%d\t%d\t%d\t%d\n",
2676                 ent.tbe_cli_nle.nle_nactive,
2677                 ent.tbe_cli_nle.nle_nbusy,
2678                 ent.tbe_cli_nle.nle_ndown,
2679                 ent.tbe_cli_nle.nle_nunknown,
2680                 ent.tbe_cli_nle.nle_nnode,
2681                 ent.tbe_srv_nle.nle_nactive,
2682                 ent.tbe_srv_nle.nle_nbusy,
2683                 ent.tbe_srv_nle.nle_ndown,
2684                 ent.tbe_srv_nle.nle_nunknown,
2685                 ent.tbe_srv_nle.nle_nnode);
2686
2687         if (ntest != 0)
2688                 goto loop;
2689
2690         return 0;
2691 }
2692
2693 int
2694 lst_query_batch_ioctl(char *batch, int test, int server,
2695                       int timeout, struct list_head *head)
2696 {
2697         lstio_batch_query_args_t args = {0};
2698
2699         args.lstio_bat_key     = session_key;
2700         args.lstio_bat_testidx = test;
2701         args.lstio_bat_client  = !(server);
2702         args.lstio_bat_timeout = timeout;
2703         args.lstio_bat_nmlen   = strlen(batch);
2704         args.lstio_bat_namep   = batch;
2705         args.lstio_bat_resultp = head;
2706
2707         return lst_ioctl(LSTIO_BATCH_QUERY, &args, sizeof(args));
2708 }
2709
2710 void
2711 lst_print_tsb_verbose(struct list_head *head,
2712                       int active, int idle, int error)
2713 {
2714         lstcon_rpc_ent_t *ent;
2715
2716         list_for_each_entry(ent, head, rpe_link) {
2717                 if (ent->rpe_priv[0] == 0 && active)
2718                         continue;
2719
2720                 if (ent->rpe_priv[0] != 0 && idle)
2721                         continue;
2722
2723                 if (ent->rpe_fwk_errno == 0 && error)
2724                         continue;
2725
2726                 fprintf(stdout, "%s [%s]: %s\n",
2727                         libcfs_id2str(ent->rpe_peer),
2728                         lst_node_state2str(ent->rpe_state),
2729                         ent->rpe_rpc_errno != 0 ?
2730                                 strerror(ent->rpe_rpc_errno) :
2731                                 (ent->rpe_priv[0] > 0 ? "Running" : "Idle"));
2732         }
2733 }
2734
2735 int
2736 jt_lst_query_batch(int argc, char **argv)
2737 {
2738         lstcon_test_batch_ent_t ent;
2739         struct list_head              head;
2740         char                   *batch   = NULL;
2741         time_t                  last    = 0;
2742         int                     optidx  = 0;
2743         int                     verbose = 0;
2744         int                     server  = 0;
2745         int                     timeout = 5; /* default 5 seconds */
2746         int                     delay   = 5; /* default 5 seconds */
2747         int                     loop    = 1; /* default 1 loop */
2748         int                     active  = 0;
2749         int                     error   = 0;
2750         int                     idle    = 0;
2751         int                     count   = 0;
2752         int                     test    = 0;
2753         int                     rc      = 0;
2754         int                     c       = 0;
2755         int                     i;
2756
2757         static struct option query_batch_opts[] =
2758         {
2759                 {"timeout", required_argument, 0, 'o' },
2760                 {"delay",   required_argument, 0, 'd' },
2761                 {"loop",    required_argument, 0, 'c' },
2762                 {"test",    required_argument, 0, 't' },
2763                 {"server",  no_argument,       0, 's' },
2764                 {"active",  no_argument,       0, 'a' },
2765                 {"idle",    no_argument,       0, 'i' },
2766                 {"error",   no_argument,       0, 'e' },
2767                 {"all",     no_argument,       0, 'l' },
2768                 {0,         0,                 0,  0  }
2769         };
2770
2771         if (session_key == 0) {
2772                 fprintf(stderr,
2773                         "Can't find env LST_SESSION or value is not valid\n");
2774                 return -1;
2775         }
2776
2777         while (1) {
2778                 c = getopt_long(argc, argv, "o:d:c:t:saiel",
2779                                 query_batch_opts, &optidx);
2780
2781                 /* Detect the end of the options. */
2782                 if (c == -1)
2783                         break;
2784
2785                 switch (c) {
2786                 case 'o':
2787                         timeout = atoi(optarg);
2788                         break;
2789                 case 'd':
2790                         delay = atoi(optarg);
2791                         break;
2792                 case 'c':
2793                         loop = atoi(optarg);
2794                         break;
2795                 case 't':
2796                         test = atoi(optarg);
2797                         break;
2798                 case 's':
2799                         server = 1;
2800                         break;
2801                 case 'a':
2802                         active = verbose = 1;
2803                         break;
2804                 case 'i':
2805                         idle = verbose = 1;
2806                         break;
2807                 case 'e':
2808                         error = verbose = 1;
2809                         break;
2810                 case 'l':
2811                         verbose = 1;
2812                         break;
2813                 default:
2814                         lst_print_usage(argv[0]);
2815                         return -1;
2816                 }
2817         }
2818
2819         if (test < 0 || timeout <= 0 || delay <= 0 || loop <= 0) {
2820                 lst_print_usage(argv[0]);
2821                 return -1;
2822         }
2823
2824         if (optind == argc) {
2825                 batch = LST_DEFAULT_BATCH;
2826
2827         } else if (optind == argc - 1) {
2828                 batch = argv[optind];
2829
2830         } else {
2831                 lst_print_usage(argv[0]);
2832                 return -1;
2833         }
2834
2835
2836         INIT_LIST_HEAD(&head);
2837
2838         if (verbose) {
2839                 rc = lst_info_batch_ioctl(batch, test, server,
2840                                           &ent, NULL, NULL, NULL);
2841                 if (rc != 0) {
2842                         fprintf(stderr, "Failed to query %s [%d]: %s\n",
2843                                 batch, test, strerror(errno));
2844                         return -1;
2845                 }
2846
2847                 count = server ? ent.tbe_srv_nle.nle_nnode :
2848                                  ent.tbe_cli_nle.nle_nnode;
2849                 if (count == 0) {
2850                         fprintf(stdout, "Batch or test is empty\n");
2851                         return 0;
2852                 }
2853         }
2854
2855         rc = lst_alloc_rpcent(&head, count, 0);
2856         if (rc != 0) {
2857                 fprintf(stderr, "Out of memory\n");
2858                 return rc;
2859         }
2860
2861         for (i = 0; i < loop; i++) {
2862                 time_t  now = time(NULL);
2863
2864                 if (now - last < delay) {
2865                         sleep(delay - now + last);
2866                         time(&now);
2867                 }
2868
2869                 last = now;
2870
2871                 rc = lst_query_batch_ioctl(batch, test,
2872                                            server, timeout, &head);
2873                 if (rc == -1) {
2874                         fprintf(stderr, "Failed to query batch: %s\n",
2875                                 strerror(errno));
2876                         break;
2877                 }
2878
2879                 if (verbose) {
2880                         /* Verbose mode */
2881                         lst_print_tsb_verbose(&head, active, idle, error);
2882                         continue;
2883                 }
2884
2885                 fprintf(stdout, "%s [%d] ", batch, test);
2886
2887                 if (lstcon_rpc_stat_failure(&trans_stat, 0) != 0) {
2888                         fprintf(stdout, "%d of %d nodes are unknown, ",
2889                                 lstcon_rpc_stat_failure(&trans_stat, 0),
2890                                 lstcon_rpc_stat_total(&trans_stat, 0));
2891                 }
2892
2893                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
2894                     lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0  &&
2895                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
2896                         fprintf(stdout, "is stopped\n");
2897                         continue;
2898                 }
2899
2900                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
2901                     lstcon_tsbqry_stat_idle(&trans_stat, 0) == 0 &&
2902                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
2903                         fprintf(stdout, "is running\n");
2904                         continue;
2905                 }
2906
2907                 fprintf(stdout, "stopped: %d , running: %d, failed: %d\n",
2908                                 lstcon_tsbqry_stat_idle(&trans_stat, 0),
2909                                 lstcon_tsbqry_stat_run(&trans_stat, 0),
2910                                 lstcon_tsbqry_stat_failure(&trans_stat, 0));
2911         }
2912
2913         lst_free_rpcent(&head);
2914
2915         return rc;
2916 }
2917
2918 int
2919 lst_parse_distribute(char *dstr, int *dist, int *span)
2920 {
2921         *dist = atoi(dstr);
2922         if (*dist <= 0)
2923                 return -1;
2924
2925         dstr = strchr(dstr, ':');
2926         if (dstr == NULL)
2927                 return -1;
2928
2929         *span = atoi(dstr + 1);
2930         if (*span <= 0)
2931                 return -1;
2932
2933         return 0;
2934 }
2935
2936 int
2937 lst_get_bulk_param(int argc, char **argv, lst_test_bulk_param_t *bulk)
2938 {
2939         char   *tok = NULL;
2940         char   *end = NULL;
2941         int     rc  = 0;
2942         int     i   = 0;
2943
2944         bulk->blk_size  = 4096;
2945         bulk->blk_opc   = LST_BRW_READ;
2946         bulk->blk_flags = LST_BRW_CHECK_NONE;
2947
2948         while (i < argc) {
2949                 if (strcasestr(argv[i], "check=") == argv[i] ||
2950                     strcasestr(argv[i], "c=") == argv[i]) {
2951                         tok = strchr(argv[i], '=') + 1;
2952
2953                         if (strcasecmp(tok, "full") == 0) {
2954                                 bulk->blk_flags = LST_BRW_CHECK_FULL;
2955                         } else if (strcasecmp(tok, "simple") == 0) {
2956                                 bulk->blk_flags = LST_BRW_CHECK_SIMPLE;
2957                         } else {
2958                                 fprintf(stderr, "Unknow flag %s\n", tok);
2959                                 return -1;
2960                         }
2961
2962                 } else if (strcasestr(argv[i], "size=") == argv[i] ||
2963                            strcasestr(argv[i], "s=") == argv[i]) {
2964                         int max_size = sysconf(_SC_PAGESIZE) * LNET_MAX_IOV;
2965
2966                         tok = strchr(argv[i], '=') + 1;
2967
2968                         bulk->blk_size = strtol(tok, &end, 0);
2969                         if (bulk->blk_size <= 0) {
2970                                 fprintf(stderr, "Invalid size %s\n", tok);
2971                                 return -1;
2972                         }
2973
2974                         if (end == NULL)
2975                                 return 0;
2976
2977                         if (*end == 'k' || *end == 'K')
2978                                 bulk->blk_size *= 1024;
2979                         else if (*end == 'm' || *end == 'M')
2980                                 bulk->blk_size *= 1024 * 1024;
2981
2982                         if (bulk->blk_size > max_size) {
2983                                 fprintf(stderr, "Size exceed limitation: %d bytes\n",
2984                                         bulk->blk_size);
2985                                 return -1;
2986                         }
2987
2988                 } else if (strcasecmp(argv[i], "read") == 0 ||
2989                            strcasecmp(argv[i], "r") == 0) {
2990                         bulk->blk_opc = LST_BRW_READ;
2991
2992                 } else if (strcasecmp(argv[i], "write") == 0 ||
2993                            strcasecmp(argv[i], "w") == 0) {
2994                         bulk->blk_opc = LST_BRW_WRITE;
2995
2996                 } else {
2997                         fprintf(stderr, "Unknow parameter: %s\n", argv[i]);
2998                         return -1;
2999                 }
3000
3001                 i++;
3002         }
3003
3004         return rc;
3005 }
3006
3007 int
3008 lst_get_test_param(char *test, int argc, char **argv, void **param, int *plen)
3009 {
3010         lst_test_bulk_param_t *bulk = NULL;
3011         int                    type;
3012
3013         type = lst_test_name2type(test);
3014         if (type < 0) {
3015                 fprintf(stderr, "Unknow test name %s\n", test);
3016                 return -1;
3017         }
3018
3019         switch (type) {
3020         case LST_TEST_PING:
3021                 break;
3022
3023         case LST_TEST_BULK:
3024                 bulk = malloc(sizeof(*bulk));
3025                 if (bulk == NULL) {
3026                         fprintf(stderr, "Out of memory\n");
3027                         return -1;
3028                 }
3029
3030                 memset(bulk, 0, sizeof(*bulk));
3031
3032                 if (lst_get_bulk_param(argc, argv, bulk) != 0) {
3033                         free(bulk);
3034                         return -1;
3035                 }
3036
3037                 *param = bulk;
3038                 *plen  = sizeof(*bulk);
3039
3040                 break;
3041
3042         default:
3043                 break;
3044         }
3045
3046         /* TODO: parse more parameter */
3047         return type;
3048 }
3049
3050 int
3051 lst_add_test_ioctl(char *batch, int type, int loop, int concur,
3052                    int dist, int span, char *sgrp, char *dgrp,
3053                    void *param, int plen, int *retp, struct list_head *resultp)
3054 {
3055         lstio_test_args_t args = {0};
3056
3057         args.lstio_tes_key        = session_key;
3058         args.lstio_tes_bat_nmlen  = strlen(batch);
3059         args.lstio_tes_bat_name   = batch;
3060         args.lstio_tes_type       = type;
3061         args.lstio_tes_oneside    = 0;
3062         args.lstio_tes_loop       = loop;
3063         args.lstio_tes_concur     = concur;
3064         args.lstio_tes_dist       = dist;
3065         args.lstio_tes_span       = span;
3066         args.lstio_tes_sgrp_nmlen = strlen(sgrp);
3067         args.lstio_tes_sgrp_name  = sgrp;
3068         args.lstio_tes_dgrp_nmlen = strlen(dgrp);
3069         args.lstio_tes_dgrp_name  = dgrp;
3070         args.lstio_tes_param_len  = plen;
3071         args.lstio_tes_param      = param;
3072         args.lstio_tes_retp       = retp;
3073         args.lstio_tes_resultp    = resultp;
3074
3075         return lst_ioctl(LSTIO_TEST_ADD, &args, sizeof(args));
3076 }
3077
3078 int
3079 jt_lst_add_test(int argc, char **argv)
3080 {
3081         struct list_head    head;
3082         char         *batch  = NULL;
3083         char         *test   = NULL;
3084         char         *dstr   = NULL;
3085         char         *from   = NULL;
3086         char         *to     = NULL;
3087         void         *param  = NULL;
3088         int           optidx = 0;
3089         int           concur = 1;
3090         int           loop   = -1;
3091         int           dist   = 1;
3092         int           span   = 1;
3093         int           plen   = 0;
3094         int           fcount = 0;
3095         int           tcount = 0;
3096         int           ret    = 0;
3097         int           type;
3098         int           rc;
3099         int           c;
3100
3101         static struct option add_test_opts[] =
3102         {
3103                 {"batch",       required_argument, 0, 'b' },
3104                 {"concurrency", required_argument, 0, 'c' },
3105                 {"distribute",  required_argument, 0, 'd' },
3106                 {"from",        required_argument, 0, 'f' },
3107                 {"to",          required_argument, 0, 't' },
3108                 {"loop",        required_argument, 0, 'l' },
3109                 {0,             0,                 0,  0  }
3110         };
3111
3112         if (session_key == 0) {
3113                 fprintf(stderr,
3114                         "Can't find env LST_SESSION or value is not valid\n");
3115                 return -1;
3116         }
3117
3118         while (1) {
3119                 c = getopt_long(argc, argv, "b:c:d:f:l:t:",
3120                                 add_test_opts, &optidx);
3121
3122                 /* Detect the end of the options. */
3123                 if (c == -1)
3124                         break;
3125
3126                 switch (c) {
3127                 case 'b':
3128                         batch = optarg;
3129                         break;
3130                 case 'c':
3131                         concur = atoi(optarg);
3132                         break;
3133                 case 'd':
3134                         dstr = optarg;
3135                         break;
3136                 case 'f':
3137                         from = optarg;
3138                         break;
3139                 case 'l':
3140                         loop = atoi(optarg);
3141                         break;
3142                 case 't':
3143                         to = optarg;
3144                         break;
3145                 default:
3146                         lst_print_usage(argv[0]);
3147                         return -1;
3148                 }
3149         }
3150
3151         if (optind == argc || from == NULL || to == NULL) {
3152                 lst_print_usage(argv[0]);
3153                 return -1;
3154         }
3155
3156         if (concur <= 0 || concur > LST_MAX_CONCUR) {
3157                 fprintf(stderr, "Invalid concurrency of test: %d\n", concur);
3158                 return -1;
3159         }
3160
3161         if (batch == NULL)
3162                 batch = LST_DEFAULT_BATCH;
3163
3164         if (dstr != NULL) {
3165                 rc = lst_parse_distribute(dstr, &dist, &span);
3166                 if (rc != 0) {
3167                         fprintf(stderr, "Invalid distribution: %s\n", dstr);
3168                         return -1;
3169                 }
3170         }
3171
3172         test = argv[optind++];
3173
3174         argc -= optind;
3175         argv += optind;
3176
3177         type = lst_get_test_param(test, argc, argv, &param, &plen);
3178         if (type < 0) {
3179                 fprintf(stderr, "Failed to add test (%s)\n", test);
3180                 return -1;
3181         }
3182
3183         INIT_LIST_HEAD(&head);
3184
3185         rc = lst_get_node_count(LST_OPC_GROUP, from, &fcount, NULL);
3186         if (rc != 0) {
3187                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3188                         from, strerror(errno));
3189                 goto out;
3190         }
3191
3192         rc = lst_get_node_count(LST_OPC_GROUP, to, &tcount, NULL);
3193         if (rc != 0) {
3194                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3195                         to, strerror(errno));
3196                 goto out;
3197         }
3198
3199         rc = lst_alloc_rpcent(&head, fcount > tcount ? fcount : tcount, 0);
3200         if (rc != 0) {
3201                 fprintf(stderr, "Out of memory\n");
3202                 goto out;
3203         }
3204
3205         rc = lst_add_test_ioctl(batch, type, loop, concur,
3206                                 dist, span, from, to, param, plen, &ret, &head);
3207
3208         if (rc == 0) {
3209                 fprintf(stdout, "Test was added successfully\n");
3210                 if (ret != 0) {
3211                         fprintf(stdout, "Server group contains userland test "
3212                                 "nodes, old version of tcplnd can't accept "
3213                                 "connection request\n");
3214                 }
3215
3216                 goto out;
3217         }
3218
3219         if (rc == -1) {
3220                 lst_print_error("test", "Failed to add test: %s\n",
3221                                 strerror(errno));
3222                 goto out;
3223         }
3224
3225         lst_print_transerr(&head, "add test");
3226 out:
3227         lst_free_rpcent(&head);
3228
3229         if (param != NULL)
3230                 free(param);
3231
3232         return rc;
3233 }
3234
3235 static command_t lst_cmdlist[] = {
3236         {"new_session",         jt_lst_new_session,     NULL,
3237          "Usage: lst new_session [--timeout TIME] [--force] [NAME]"                     },
3238         {"end_session",         jt_lst_end_session,     NULL,
3239          "Usage: lst end_session"                                                       },
3240         {"show_session",        jt_lst_show_session,    NULL,
3241          "Usage: lst show_session"                                                      },
3242         {"ping",                jt_lst_ping ,           NULL,
3243          "Usage: lst ping  [--group NAME] [--batch NAME] [--session] [--nodes IDS]"     },
3244         {"add_group",           jt_lst_add_group,       NULL,
3245          "Usage: lst group NAME IDs [IDs]..."                                           },
3246         {"del_group",           jt_lst_del_group,       NULL,
3247          "Usage: lst del_group NAME"                                                    },
3248         {"update_group",        jt_lst_update_group,    NULL,
3249          "Usage: lst update_group NAME [--clean] [--refresh] [--remove IDs]"            },
3250         {"list_group",          jt_lst_list_group,      NULL,
3251           "Usage: lst list_group [--active] [--busy] [--down] [--unknown] GROUP ..."    },
3252         {"stat",                jt_lst_stat,            NULL,
3253          "Usage: lst stat [--bw] [--rate] [--read] [--write] [--max] [--min] [--avg] "
3254          " [--timeout #] [--delay #] [--count #] GROUP [GROUP]"                         },
3255         {"show_error",          jt_lst_show_error,      NULL,
3256          "Usage: lst show_error NAME | IDS ..."                                         },
3257         {"add_batch",           jt_lst_add_batch,       NULL,
3258          "Usage: lst add_batch NAME"                                                    },
3259         {"run",                 jt_lst_start_batch,     NULL,
3260          "Usage: lst run [--timeout TIME] [NAME]"                                       },
3261         {"stop",                jt_lst_stop_batch,      NULL,
3262          "Usage: lst stop [--force] BATCH_NAME"                                         },
3263         {"list_batch",          jt_lst_list_batch,      NULL,
3264          "Usage: lst list_batch NAME [--test ID] [--server]"                            },
3265         {"query",               jt_lst_query_batch,     NULL,
3266          "Usage: lst query [--test ID] [--server] [--timeout TIME] NAME"                },
3267         {"add_test",            jt_lst_add_test,        NULL,
3268          "Usage: lst add_test [--batch BATCH] [--loop #] [--concurrency #] "
3269          " [--distribute #:#] [--from GROUP] [--to GROUP] TEST..."                      },
3270         {"help",                Parser_help,            0,     "help"                   },
3271         {0,                     0,                      0,      NULL                    }
3272 };
3273
3274 int
3275 lst_initialize(void)
3276 {
3277         char   *key;
3278         char   *feats;
3279
3280         feats = getenv("LST_FEATURES");
3281         if (feats != NULL)
3282                 session_features = strtol(feats, NULL, 16);
3283
3284         if ((session_features & ~LST_FEATS_MASK) != 0) {
3285                 fprintf(stderr,
3286                         "Unsupported session features %x, "
3287                         "only support these features so far: %x\n",
3288                         (session_features & ~LST_FEATS_MASK), LST_FEATS_MASK);
3289                 return -1;
3290         }
3291
3292         key = getenv("LST_SESSION");
3293
3294         if (key == NULL) {
3295                 session_key = 0;
3296                 return 0;
3297         }
3298
3299         session_key = atoi(key);
3300
3301         return 0;
3302 }
3303
3304 int
3305 main(int argc, char **argv)
3306 {
3307         int rc = 0;
3308
3309         setlinebuf(stdout);
3310
3311         rc = lst_initialize();
3312         if (rc < 0)
3313                 goto errorout;
3314
3315         rc = ptl_initialize(argc, argv);
3316         if (rc < 0)
3317                 goto errorout;
3318
3319         Parser_init("lst > ", lst_cmdlist);
3320
3321         if (argc != 1)  {
3322                 rc = Parser_execarg(argc - 1, argv + 1, lst_cmdlist);
3323                 goto errorout;
3324         }
3325
3326         Parser_commands();
3327
3328 errorout:
3329         return rc;
3330 }