lnet/utils/lst.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
   5  * Use is subject to license terms.
   6  *
   7  * Copyright (c) 2012, 2017, Intel Corporation.
   8  */
   9
  10 /*
  11  * This file is part of Lustre, http://www.lustre.org/
  12  *
  13  * Author: Liang Zhen <liangzhen@clusterfs.com>
  14  */
  15
  16 #include <errno.h>
  17 #include <getopt.h>
  18 #include <inttypes.h>
  19 #include <pwd.h>
  20 #include <unistd.h>
  21 #include <stdarg.h>
  22 #include <stdio.h>
  23 #include <stdlib.h>
  24 #include <string.h>
  25 #include <sys/ioctl.h>
  26 #include <time.h>
  27 #include <linux/types.h>
  28
  29 #include <libcfs/util/list.h>
  30 #include <libcfs/util/ioctl.h>
  31 #include <libcfs/util/parser.h>
  32 #include <linux/lnet/lnetctl.h>
  33 #include <linux/lnet/lnetst.h>
  34 #include <linux/lnet/nidstr.h>
  35 #include "lnetconfig/liblnetconfig.h"
  36
  37 static void lst_print_usage(char *cmd);
  38
  39 static int lst_info_batch_ioctl(char *batch, int test, int server,
  40                        struct lstcon_test_batch_ent *entp, int *idxp,
  41                        int *ndentp, struct lstcon_node_ent *dentsp);
  42 static int lst_info_group_ioctl(char *name, struct lstcon_ndlist_ent *gent,
  43                         int *idx, int *count, struct lstcon_node_ent *dents);
  44 static int lst_query_batch_ioctl(char *batch, int test, int server,
  45                          int timeout, struct list_head *head);
  46
  47 struct lst_sid LST_INVALID_SID = { .ses_nid = LNET_NID_ANY, .ses_stamp = -1 };
  48 static unsigned int session_key;
  49
  50 /* All nodes running 2.6.50 or later understand feature LST_FEAT_BULK_LEN */
  51 static unsigned int session_features = LST_FEATS_MASK;
  52 static struct lstcon_trans_stat trans_stat;
  53
  54 typedef struct list_string {
  55         struct list_string *lstr_next;
  56         int                 lstr_sz;
  57         char                lstr_str[];
  58 } lstr_t;
  59
  60 #ifndef offsetof
  61 # define offsetof(typ, memb)     ((unsigned long)((char *)&(((typ *)0)->memb)))
  62 #endif
  63
  64 static int alloc_count;
  65 static int alloc_nob;
  66
  67 static lstr_t *
  68 alloc_lstr(int sz)
  69 {
  70         lstr_t  *lstr = malloc(offsetof(lstr_t, lstr_str[sz]));
  71
  72         if (lstr == NULL) {
  73                 fprintf(stderr, "Can't allocate lstr\n");
  74                 abort();
  75         }
  76
  77         alloc_nob += sz;
  78         alloc_count++;
  79
  80         lstr->lstr_str[0] = 0;
  81         lstr->lstr_sz = sz;
  82         return lstr;
  83 }
  84
  85 static void
  86 free_lstr(lstr_t *lstr)
  87 {
  88         alloc_count--;
  89         alloc_nob -= lstr->lstr_sz;
  90         free(lstr);
  91 }
  92
  93 static void
  94 new_lstrs(lstr_t **list, char *prefix, char *postfix,
  95           int lo, int hi, int stride)
  96 {
  97         int n1 = strlen(prefix);
  98         int n2 = strlen(postfix);
  99         int sz = n1 + 20 + n2 + 1;
 100
 101         do {
 102                 lstr_t *n = alloc_lstr(sz);
 103
 104                 snprintf(n->lstr_str, sz - 1, "%s%u%s", prefix, lo, postfix);
 105
 106                 n->lstr_next = *list;
 107                 *list = n;
 108
 109                 lo += stride;
 110         } while (lo <= hi);
 111 }
 112
 113 static int
 114 expand_lstr(lstr_t **list, lstr_t *l)
 115 {
 116         int nob = strlen(l->lstr_str);
 117         char *b1, *b2, *expr, *sep;
 118         int x, y, z, n;
 119
 120         b1 = strchr(l->lstr_str, '[');
 121         if (b1 == NULL) {
 122                 l->lstr_next = *list;
 123                 *list = l;
 124                 return 0;
 125         }
 126
 127         b2 = strchr(b1, ']');
 128         if (b2 == NULL || b2 == b1 + 1)
 129                 return -1;
 130
 131         *b1++ = 0;
 132         *b2++ = 0;
 133         expr = b1;
 134         do {
 135                 sep = strchr(expr, ',');
 136                 if (sep != NULL)
 137                         *sep++ = 0;
 138
 139                 nob = strlen(expr);
 140                 n = nob;
 141                 if (sscanf(expr, "%u%n", &x, &n) >= 1 && n == nob) {
 142                         /* simple number */
 143                         new_lstrs(list, l->lstr_str, b2, x, x, 1);
 144                         continue;
 145                 }
 146
 147                 n = nob;
 148                 if (sscanf(expr, "%u-%u%n", &x, &y, &n) >= 2 && n == nob &&
 149                     x < y) {
 150                         /* simple range */
 151                         new_lstrs(list, l->lstr_str, b2, x, y, 1);
 152                         continue;
 153                 }
 154
 155                 n = nob;
 156                 if (sscanf(expr, "%u-%u/%u%n", &x, &y, &z, &n) >= 3 &&
 157                     n == nob && x < y) {
 158                         /* strided range */
 159                         new_lstrs(list, l->lstr_str, b2, x, y, z);
 160                         continue;
 161                 }
 162
 163                 /* syntax error */
 164                 return -1;
 165         } while ((expr = sep) != NULL);
 166
 167         free_lstr(l);
 168
 169         return 1;
 170 }
 171
 172 static int
 173 expand_strs(char *str, lstr_t **head)
 174 {
 175         lstr_t *list = NULL;
 176         lstr_t *nlist;
 177         lstr_t *l;
 178         int rc = 0;
 179         int expanded;
 180
 181         l = alloc_lstr(strlen(str) + 1);
 182         memcpy(l->lstr_str, str, strlen(str) + 1);
 183         l->lstr_next = NULL;
 184         list = l;
 185
 186         do {
 187                 expanded = 0;
 188                 nlist = NULL;
 189
 190                 while ((l = list) != NULL) {
 191                         list = l->lstr_next;
 192
 193                         rc = expand_lstr(&nlist, l);
 194                         if (rc < 0) {
 195                                 fprintf(stderr, "Syntax error in \"%s\"\n",
 196                                         str);
 197                                 free_lstr(l);
 198                                 break;
 199                         }
 200
 201                         expanded |= rc > 0;
 202                 }
 203
 204                 /* re-order onto 'list' */
 205                 while ((l = nlist) != NULL) {
 206                         nlist = l->lstr_next;
 207                         l->lstr_next = list;
 208                         list = l;
 209                 }
 210
 211         } while (expanded && rc > 0);
 212
 213         if (rc >= 0) {
 214                 *head = list;
 215                 return 0;
 216         }
 217
 218         while ((l = list) != NULL) {
 219                 list = l->lstr_next;
 220
 221                 free_lstr(l);
 222         }
 223         return rc;
 224 }
 225
 226 static int
 227 lst_parse_nids(char *str, int *countp, struct lnet_process_id **idspp)
 228 {
 229         lstr_t *head = NULL;
 230         lstr_t *l;
 231         int c = 0;
 232         int i;
 233         int rc;
 234
 235         rc = expand_strs(str, &head);
 236         if (rc != 0)
 237                 goto out;
 238
 239         l = head;
 240         while (l != NULL) {
 241                 l = l->lstr_next;
 242                 c++;
 243         }
 244
 245         *idspp = malloc(c * sizeof(struct lnet_process_id));
 246         if (*idspp == NULL) {
 247                 fprintf(stderr, "Out of memory\n");
 248                 rc = -1;
 249         }
 250
 251         *countp = c;
 252 out:
 253         i = 0;
 254         while ((l = head) != NULL) {
 255                 head = l->lstr_next;
 256
 257                 if (rc == 0) {
 258                         (*idspp)[i].nid = libcfs_str2nid(l->lstr_str);
 259                         if ((*idspp)[i].nid == LNET_NID_ANY) {
 260                                 fprintf(stderr, "Invalid nid: %s\n",
 261                                         l->lstr_str);
 262                                 rc = -1;
 263                         }
 264
 265                         (*idspp)[i].pid = LNET_PID_LUSTRE;
 266                         i++;
 267                 }
 268
 269                 free_lstr(l);
 270         }
 271
 272         if (rc == 0)
 273                 return 0;
 274
 275         free(*idspp);
 276         *idspp = NULL;
 277
 278         return rc;
 279 }
 280
 281 static char *
 282 lst_node_state2str(int state)
 283 {
 284         if (state == LST_NODE_ACTIVE)
 285                 return "Active";
 286         if (state == LST_NODE_BUSY)
 287                 return "Busy";
 288         if (state == LST_NODE_DOWN)
 289                 return "Down";
 290
 291         return "Unknown";
 292 }
 293
 294 static int
 295 lst_node_str2state(char *str)
 296 {
 297         if (strcasecmp(str, "active") == 0)
 298                 return LST_NODE_ACTIVE;
 299         if (strcasecmp(str, "busy") == 0)
 300                 return LST_NODE_BUSY;
 301         if (strcasecmp(str, "down") == 0)
 302                 return LST_NODE_DOWN;
 303         if (strcasecmp(str, "unknown") == 0)
 304                 return LST_NODE_UNKNOWN;
 305         if (strcasecmp(str, "invalid") == 0)
 306                 return (LST_NODE_UNKNOWN | LST_NODE_DOWN | LST_NODE_BUSY);
 307
 308         return -1;
 309 }
 310
 311 static char *
 312 lst_test_type2name(int type)
 313 {
 314         if (type == LST_TEST_PING)
 315                 return "ping";
 316         if (type == LST_TEST_BULK)
 317                 return "brw";
 318
 319         return "unknown";
 320 }
 321
 322 static int
 323 lst_test_name2type(char *name)
 324 {
 325         if (strcasecmp(name, "ping") == 0)
 326                 return LST_TEST_PING;
 327         if (strcasecmp(name, "brw") == 0)
 328                 return LST_TEST_BULK;
 329
 330         return -1;
 331 }
 332
 333 static void
 334 lst_print_error(char *sub, const char *def_format, ...)
 335 {
 336         va_list ap;
 337
 338         /* local error returned from kernel */
 339         switch (errno) {
 340         case ESRCH:
 341                 fprintf(stderr, "No session exists\n");
 342                 return;
 343         case ESHUTDOWN:
 344                 fprintf(stderr, "Session is shutting down\n");
 345                 return;
 346         case EACCES:
 347                 fprintf(stderr, "Unmatched session key or not root\n");
 348                 return;
 349         case ENOENT:
 350                 fprintf(stderr, "Can't find %s in current session\n", sub);
 351                 return;
 352         case EINVAL:
 353                 fprintf(stderr, "Invalid parameters list in command line\n");
 354                 return;
 355         case EFAULT:
 356                 fprintf(stderr, "Bad parameter address\n");
 357                 return;
 358         case EEXIST:
 359                 fprintf(stderr, "%s already exists\n", sub);
 360                 return;
 361         default:
 362                 va_start(ap, def_format);
 363                 vfprintf(stderr, def_format, ap);
 364                 va_end(ap);
 365
 366                 return;
 367         }
 368 }
 369
 370 static void
 371 lst_free_rpcent(struct list_head *head)
 372 {
 373         struct lstcon_rpc_ent *ent;
 374
 375         while (!list_empty(head)) {
 376                 ent = list_first_entry(head, struct lstcon_rpc_ent, rpe_link);
 377
 378                 list_del(&ent->rpe_link);
 379                 free(ent);
 380         }
 381 }
 382
 383 static void
 384 lst_reset_rpcent(struct list_head *head)
 385 {
 386         struct lstcon_rpc_ent *ent;
 387
 388         list_for_each_entry(ent, head, rpe_link) {
 389                 ent->rpe_sid       = LST_INVALID_SID;
 390                 ent->rpe_peer.nid  = LNET_NID_ANY;
 391                 ent->rpe_peer.pid  = LNET_PID_ANY;
 392                 ent->rpe_rpc_errno = ent->rpe_fwk_errno = 0;
 393         }
 394 }
 395
 396 static int
 397 lst_alloc_rpcent(struct list_head *head, int count, int offset)
 398 {
 399         struct lstcon_rpc_ent *ent;
 400         int i;
 401
 402         for (i = 0; i < count; i++) {
 403                 ent = malloc(offsetof(struct lstcon_rpc_ent,
 404                                       rpe_payload[offset]));
 405                 if (ent == NULL) {
 406                         lst_free_rpcent(head);
 407                         return -1;
 408                 }
 409
 410                 memset(ent, 0, offsetof(struct lstcon_rpc_ent,
 411                                         rpe_payload[offset]));
 412
 413                 ent->rpe_sid      = LST_INVALID_SID;
 414                 ent->rpe_peer.nid = LNET_NID_ANY;
 415                 ent->rpe_peer.pid = LNET_PID_ANY;
 416                 list_add(&ent->rpe_link, head);
 417         }
 418
 419         return 0;
 420 }
 421
 422 static void
 423 lst_print_transerr(struct list_head *head, char *optstr)
 424 {
 425         struct lstcon_rpc_ent *ent;
 426
 427         list_for_each_entry(ent, head, rpe_link) {
 428                 if (ent->rpe_rpc_errno == 0 && ent->rpe_fwk_errno == 0)
 429                         continue;
 430
 431                 if (ent->rpe_rpc_errno != 0) {
 432                         fprintf(stderr, "%s RPC failed on %s: %s\n",
 433                                 optstr, libcfs_id2str(ent->rpe_peer),
 434                                 strerror(ent->rpe_rpc_errno));
 435                         continue;
 436                 }
 437
 438                 fprintf(stderr, "operation %s failed on %s: %s\n",
 439                         optstr, libcfs_id2str(ent->rpe_peer),
 440                         strerror(ent->rpe_fwk_errno));
 441         }
 442 }
 443
 444 static int
 445 lst_ioctl(unsigned int opc, void *buf, int len)
 446 {
 447         struct libcfs_ioctl_data data;
 448         int rc;
 449
 450         LIBCFS_IOC_INIT(data);
 451         data.ioc_u32[0]  = opc;
 452         data.ioc_plen1   = len;
 453         data.ioc_pbuf1   = (char *)buf;
 454         data.ioc_plen2   = sizeof(trans_stat);
 455         data.ioc_pbuf2   = (char *)&trans_stat;
 456
 457         memset(&trans_stat, 0, sizeof(trans_stat));
 458
 459         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNETST, &data);
 460
 461         /* local error, no valid RPC result */
 462         if (rc != 0)
 463                 return -1;
 464
 465         /* RPC error */
 466         if (trans_stat.trs_rpc_errno != 0)
 467                 return -2;
 468
 469         /* Framework error */
 470         if (trans_stat.trs_fwk_errno != 0)
 471                 return -3;
 472
 473         return 0;
 474 }
 475
 476 static int lst_yaml_session(const char *label, const char *timeout, int nlflags,
 477                             const char *errmsg)
 478 {
 479         struct lstcon_ndlist_ent ndinfo = { };
 480         struct lst_sid sid = LST_INVALID_SID;
 481         /* nlflags being zero means we are destroying the session.
 482          * No parsing of reply needed.
 483          */
 484         bool done = nlflags ? false : true;
 485         char nid[LNET_NIDSTR_SIZE];
 486         char name[LST_NAME_SIZE];
 487         unsigned int key = 0;
 488         yaml_emitter_t request;
 489         yaml_parser_t reply;
 490         yaml_event_t event;
 491         struct nl_sock *sk;
 492         int rc;
 493
 494         sk = nl_socket_alloc();
 495         if (!sk)
 496                 return -1;
 497
 498         /* Note: NL_AUTO_PID == zero which we use by default for the
 499          * session_key when creating a new session. This is considered
 500          * an invalid key so we need to get the real session key from
 501          * the yaml parser yet to be created. If the user did request
 502          * a specific session key then set the socket's port id to this
 503          * value.
 504          */
 505         if (session_key)
 506                 nl_socket_set_local_port(sk, session_key);
 507
 508         /* Setup reply parser to recieve Netlink packets */
 509         rc = yaml_parser_initialize(&reply);
 510         if (rc == 0) {
 511                 nl_socket_free(sk);
 512                 return -1;
 513         }
 514
 515         rc = yaml_parser_set_input_netlink(&reply, sk, false);
 516         if (rc == 0)
 517                 goto parser_error;
 518
 519         /* Create Netlink emitter to send request to kernel */
 520         yaml_emitter_initialize(&request);
 521         rc = yaml_emitter_set_output_netlink(&request, sk,
 522                                              LNET_SELFTEST_GENL_NAME,
 523                                              LNET_SELFTEST_GENL_VERSION,
 524                                              LNET_SELFTEST_CMD_SESSIONS,
 525                                              nlflags);
 526         if (rc == 0)
 527                 goto emitter_error;
 528
 529         yaml_emitter_open(&request);
 530         yaml_document_start_event_initialize(&event, NULL, NULL, NULL, 0);
 531         rc = yaml_emitter_emit(&request, &event);
 532         if (rc == 0)
 533                 goto emitter_error;
 534
 535         yaml_mapping_start_event_initialize(&event, NULL,
 536                                             (yaml_char_t *)YAML_MAP_TAG,
 537                                             1, YAML_BLOCK_MAPPING_STYLE);
 538         rc = yaml_emitter_emit(&request, &event);
 539         if (rc == 0)
 540                 goto emitter_error;
 541
 542         yaml_scalar_event_initialize(&event, NULL,
 543                                      (yaml_char_t *)YAML_STR_TAG,
 544                                      (yaml_char_t *)"sessions",
 545                                      strlen("sessions"), 1, 0,
 546                                      YAML_PLAIN_SCALAR_STYLE);
 547         rc = yaml_emitter_emit(&request, &event);
 548         if (rc == 0)
 549                 goto emitter_error;
 550
 551         if (!label) {
 552                 yaml_scalar_event_initialize(&event, NULL,
 553                                              (yaml_char_t *)YAML_STR_TAG,
 554                                              (yaml_char_t *)"",
 555                                              strlen(""), 1, 0,
 556                                              YAML_PLAIN_SCALAR_STYLE);
 557                 rc = yaml_emitter_emit(&request, &event);
 558                 if (rc == 0)
 559                         goto emitter_error;
 560                 goto skip_params;
 561         }
 562
 563         /* sessions: { name: 'name', timeout: 300 }
 564          * or
 565          * sessions:
 566          *   name: 'name'
 567          *   timeout: 300
 568          */
 569         yaml_mapping_start_event_initialize(&event, NULL,
 570                                             (yaml_char_t *)YAML_MAP_TAG,
 571                                             1, YAML_FLOW_MAPPING_STYLE);
 572         rc = yaml_emitter_emit(&request, &event);
 573         if (rc == 0)
 574                 goto emitter_error;
 575
 576         yaml_scalar_event_initialize(&event, NULL,
 577                                      (yaml_char_t *)YAML_STR_TAG,
 578                                      (yaml_char_t *)"name",
 579                                      strlen("name"), 1, 0,
 580                                      YAML_PLAIN_SCALAR_STYLE);
 581         rc = yaml_emitter_emit(&request, &event);
 582         if (rc == 0)
 583                 goto emitter_error;
 584
 585         yaml_scalar_event_initialize(&event, NULL,
 586                                      (yaml_char_t *)YAML_STR_TAG,
 587                                      (yaml_char_t *)label,
 588                                      strlen(label), 1, 0,
 589                                      YAML_PLAIN_SCALAR_STYLE);
 590         rc = yaml_emitter_emit(&request, &event);
 591         if (rc == 0)
 592                 goto emitter_error;
 593
 594         if (timeout) {
 595                 yaml_scalar_event_initialize(&event, NULL,
 596                                              (yaml_char_t *)YAML_STR_TAG,
 597                                              (yaml_char_t *)"timeout",
 598                                              strlen("timeout"), 1, 0,
 599                                              YAML_PLAIN_SCALAR_STYLE);
 600                 rc = yaml_emitter_emit(&request, &event);
 601                 if (rc == 0)
 602                         goto emitter_error;
 603
 604                 yaml_scalar_event_initialize(&event, NULL,
 605                                              (yaml_char_t *)YAML_STR_TAG,
 606                                              (yaml_char_t *)timeout,
 607                                              strlen(timeout), 1, 0,
 608                                              YAML_PLAIN_SCALAR_STYLE);
 609                 rc = yaml_emitter_emit(&request, &event);
 610                 if (rc == 0)
 611                         goto emitter_error;
 612         }
 613
 614         yaml_mapping_end_event_initialize(&event);
 615         rc = yaml_emitter_emit(&request, &event);
 616         if (rc == 0)
 617                 goto emitter_error;
 618
 619 skip_params:
 620         yaml_mapping_end_event_initialize(&event);
 621         rc = yaml_emitter_emit(&request, &event);
 622         if (rc == 0)
 623                 goto emitter_error;
 624
 625         yaml_document_end_event_initialize(&event, 0);
 626         rc = yaml_emitter_emit(&request, &event);
 627         if (rc == 0)
 628                 goto emitter_error;
 629
 630         rc = yaml_emitter_close(&request);
 631         if (rc == 0) {
 632 emitter_error:
 633                 yaml_emitter_log_error(&request, stderr);
 634                 yaml_emitter_delete(&request);
 635                 errmsg = NULL;
 636                 goto parser_error;
 637         }
 638         yaml_emitter_delete(&request);
 639
 640         while (!done) {
 641                 rc = yaml_parser_parse(&reply, &event);
 642                 if (rc == 0)
 643                         goto parser_error;
 644
 645                 if (event.type == YAML_SCALAR_EVENT) {
 646                         char *tmp, *endp = NULL;
 647
 648                         if (strcmp((char *)event.data.scalar.value,
 649                                    "name") == 0) {
 650                                 yaml_event_delete(&event);
 651                                 rc = yaml_parser_parse(&reply, &event);
 652                                 if (rc == 0)
 653                                         goto parser_error;
 654
 655                                 strncpy(name, (char *)event.data.scalar.value,
 656                                         sizeof(name) - 1);
 657                         }
 658
 659                         if (strcmp((char *)event.data.scalar.value,
 660                                    "key") == 0) {
 661                                 yaml_event_delete(&event);
 662                                 rc = yaml_parser_parse(&reply, &event);
 663                                 if (rc == 0)
 664                                         goto parser_error;
 665
 666                                 tmp = (char *)event.data.scalar.value;
 667                                 key = strtoul(tmp, &endp, 10);
 668                                 if (endp == tmp)
 669                                         goto parser_error;
 670                         }
 671
 672                         if (strcmp((char *)event.data.scalar.value,
 673                                    "timestamp") == 0) {
 674                                 yaml_event_delete(&event);
 675                                 rc = yaml_parser_parse(&reply, &event);
 676                                 if (rc == 0)
 677                                         goto parser_error;
 678
 679                                 tmp = (char *)event.data.scalar.value;
 680                                 sid.ses_stamp = strtoll(tmp, &endp, 10);
 681                                 if (endp == tmp)
 682                                         goto parser_error;
 683                         }
 684
 685                         if (strcmp((char *)event.data.scalar.value,
 686                                    "nid") == 0) {
 687                                 yaml_event_delete(&event);
 688                                 rc = yaml_parser_parse(&reply, &event);
 689                                 if (rc == 0)
 690                                         goto parser_error;
 691
 692                                 strncpy(nid, (char *)event.data.scalar.value,
 693                                         sizeof(nid) - 1);
 694                         }
 695
 696                         if (strcmp((char *)event.data.scalar.value,
 697                                    "nodes") == 0) {
 698                                 yaml_event_delete(&event);
 699                                 rc = yaml_parser_parse(&reply, &event);
 700                                 if (rc == 0)
 701                                         goto parser_error;
 702
 703                                 tmp = (char *)event.data.scalar.value;
 704                                 ndinfo.nle_nnode = strtoul(tmp, &endp, 10);
 705                                 if (endp == tmp)
 706                                         goto parser_error;
 707                         }
 708                 }
 709
 710                 done = (event.type == YAML_STREAM_END_EVENT);
 711
 712                 yaml_event_delete(&event);
 713         }
 714
 715         if (nlflags & NLM_F_CREATE) {
 716                 session_features = yaml_parser_get_reader_proto_version(&reply);
 717                 session_key = key;
 718         }
 719 parser_error:
 720         if (rc == 0 && errmsg)
 721                 yaml_parser_log_error(&reply, stderr, errmsg);
 722         yaml_parser_delete(&reply);
 723         nl_socket_free(sk);
 724
 725         if (((nlflags & NLM_F_DUMP) == NLM_F_DUMP) && rc != 0) {
 726                 fprintf(stdout,
 727                         "%s ID: %ju@%s, KEY: %u FEATURES: %x NODES: %d\n",
 728                         name, (uintmax_t)sid.ses_stamp, nid,
 729                         key, session_features, ndinfo.nle_nnode);
 730         }
 731
 732         return rc == 0 ? -1 : 0;
 733 }
 734
 735 static int
 736 lst_new_session_ioctl(char *name, int timeout, int force, struct lst_sid *sid)
 737 {
 738         struct lstio_session_new_args args = { 0 };
 739
 740         args.lstio_ses_key     = session_key;
 741         args.lstio_ses_timeout = timeout;
 742         args.lstio_ses_force   = force;
 743         args.lstio_ses_idp     = sid;
 744         args.lstio_ses_feats   = session_features;
 745         args.lstio_ses_nmlen   = strlen(name);
 746         args.lstio_ses_namep   = name;
 747
 748         return lst_ioctl(LSTIO_SESSION_NEW, &args, sizeof(args));
 749 }
 750
 751 static int
 752 jt_lst_new_session(int argc, char **argv)
 753 {
 754         char buf[LST_NAME_SIZE * 2 + 1];
 755         char *name, *timeout_s = NULL;
 756         int nlflags = NLM_F_CREATE;
 757         struct lst_sid session_id;
 758         int optidx = 0;
 759         int timeout = 300;
 760         int force = 0;
 761         int c;
 762         int rc;
 763
 764         static const struct option session_opts[] = {
 765                 { .name = "timeout", .has_arg = required_argument, .val = 't' },
 766                 { .name = "force",   .has_arg = no_argument,       .val = 'f' },
 767                 { .name = NULL } };
 768
 769         while (1) {
 770                 c = getopt_long(argc, argv, "ft:",
 771                                 session_opts, &optidx);
 772                 if (c == -1)
 773                         break;
 774
 775                 switch (c) {
 776                 case 'f':
 777                         nlflags |= NLM_F_REPLACE;
 778                         force = 1;
 779                         break;
 780                 case 't':
 781                         timeout_s = optarg;
 782                         break;
 783                 default:
 784                         lst_print_usage(argv[0]);
 785                         return -1;
 786                 }
 787         }
 788
 789         if (timeout_s) {
 790                 timeout = atoi(timeout_s);
 791                 if (timeout <= 0) {
 792                         fprintf(stderr, "Invalid timeout value\n");
 793                         return -1;
 794                 }
 795         }
 796
 797         if (optind == argc - 1) {
 798                 name = argv[optind++];
 799                 if (strlen(name) >= LST_NAME_SIZE) {
 800                         fprintf(stderr, "Name size is limited to %d\n",
 801                                 LST_NAME_SIZE - 1);
 802                         return -1;
 803                 }
 804         } else if (optind == argc) {
 805                 char user[LST_NAME_SIZE];
 806                 char host[LST_NAME_SIZE];
 807                 struct passwd *pw = getpwuid(getuid());
 808
 809                 if (pw == NULL)
 810                         snprintf(user, sizeof(user), "%d", (int)getuid());
 811                 else
 812                         snprintf(user, sizeof(user), "%s", pw->pw_name);
 813
 814                 rc = gethostname(host, sizeof(host));
 815                 if (rc != 0)
 816                         snprintf(host, sizeof(host), "unknown_host");
 817
 818                 snprintf(buf, sizeof(buf), "%s@%s", user, host);
 819                 name = buf;
 820         } else {
 821                 lst_print_usage(argv[0]);
 822                 return -1;
 823         }
 824
 825         rc = lst_yaml_session(name, timeout_s, nlflags, "new session");
 826         if (rc == 0)
 827                 goto success;
 828
 829         if (session_key == 0) {
 830                 fprintf(stderr,
 831                         "Can't find env LST_SESSION or value is not valid\n");
 832                 return -1;
 833         }
 834
 835         rc = lst_new_session_ioctl(name, timeout, force, &session_id);
 836         if (rc != 0) {
 837                 lst_print_error("session", "Failed to create session: %s\n",
 838                                 strerror(errno));
 839                 return rc;
 840         }
 841 success:
 842         fprintf(stdout, "SESSION: %s FEATURES: %x TIMEOUT: %d FORCE: %s\n",
 843                 name, session_features, timeout, force ? "Yes" : "No");
 844         return 0;
 845 }
 846
 847 static int
 848 lst_session_info_ioctl(char *name, int len, int *key, unsigned int *featp,
 849                        struct lst_sid *sid, struct lstcon_ndlist_ent *ndinfo)
 850 {
 851         struct lstio_session_info_args args = { 0 };
 852
 853         args.lstio_ses_idp     = sid;
 854         args.lstio_ses_keyp    = key;
 855         args.lstio_ses_featp   = featp;
 856         args.lstio_ses_ndinfo  = ndinfo;
 857         args.lstio_ses_nmlen   = len;
 858         args.lstio_ses_namep   = name;
 859
 860         return lst_ioctl(LSTIO_SESSION_INFO, &args, sizeof(args));
 861 }
 862
 863 static int
 864 jt_lst_show_session(int argc, char **argv)
 865 {
 866         struct lstcon_ndlist_ent ndinfo;
 867         struct lst_sid sid;
 868         char name[LST_NAME_SIZE];
 869         unsigned int feats;
 870         int key;
 871         int rc;
 872
 873         rc = lst_yaml_session(NULL, NULL, NLM_F_DUMP, "show session");
 874         if (rc == 0)
 875                 return 0;
 876
 877         rc = lst_session_info_ioctl(name, sizeof(name), &key,
 878                                     &feats, &sid, &ndinfo);
 879
 880         if (rc != 0) {
 881                 lst_print_error("session", "Failed to show session: %s\n",
 882                                 strerror(errno));
 883                 return -1;
 884         }
 885
 886         fprintf(stdout, "%s ID: %ju@%s, KEY: %d FEATURES: %x NODES: %d\n",
 887                 name, (uintmax_t)sid.ses_stamp, libcfs_nid2str(sid.ses_nid),
 888                 key, feats, ndinfo.nle_nnode);
 889
 890         return 0;
 891 }
 892
 893 static int
 894 lst_end_session_ioctl(void)
 895 {
 896         struct lstio_session_end_args args = { 0 };
 897
 898         args.lstio_ses_key = session_key;
 899         return lst_ioctl(LSTIO_SESSION_END, &args, sizeof(args));
 900 }
 901
 902 static int
 903 jt_lst_end_session(int argc, char **argv)
 904 {
 905         int rc;
 906
 907         if (session_key == 0) {
 908                 fprintf(stderr,
 909                         "Can't find env LST_SESSION or value is not valid\n");
 910                 return -1;
 911         }
 912
 913         rc = lst_yaml_session(NULL, NULL, 0, "end session");
 914         if (rc == 0)
 915                 goto finish;
 916
 917         rc = lst_end_session_ioctl();
 918
 919         if (rc == 0) {
 920                 fprintf(stdout, "session is ended\n");
 921                 return 0;
 922         }
 923
 924         if (rc == -1) {
 925                 lst_print_error("session", "Failed to end session: %s\n",
 926                                 strerror(errno));
 927                 return rc;
 928         }
 929 finish:
 930         if (trans_stat.trs_rpc_errno != 0) {
 931                 fprintf(stderr,
 932                         "[RPC] Failed to send %d session RPCs: %s\n",
 933                         lstcon_rpc_stat_failure(&trans_stat, 0),
 934                         strerror(trans_stat.trs_rpc_errno));
 935         }
 936
 937         if (trans_stat.trs_fwk_errno != 0) {
 938                 fprintf(stderr,
 939                         "[FWK] Failed to end session on %d nodes: %s\n",
 940                         lstcon_sesop_stat_failure(&trans_stat, 0),
 941                         strerror(trans_stat.trs_fwk_errno));
 942         }
 943
 944         return rc;
 945 }
 946
 947 #define LST_NODES_TITLE "\tACTIVE\tBUSY\tDOWN\tUNKNOWN\tTOTAL\n"
 948
 949 static int lst_yaml_display_groups(yaml_parser_t *reply, char *group,
 950                                    int states, bool print)
 951 {
 952         yaml_event_t event;
 953         int rc;
 954
 955         if (states) {
 956                 char name[LST_NAME_SIZE] = {};
 957                 bool done = false;
 958                 int count = 0;
 959
 960                 if (print)
 961                         fprintf(stdout, LST_NODES_TITLE);
 962
 963                 while (!done) {
 964                         rc = yaml_parser_parse(reply, &event);
 965                         if (rc == 0)
 966                                 goto parser_error;
 967
 968                         if (event.type == YAML_MAPPING_START_EVENT) {
 969                                 char *value = NULL;
 970                                 yaml_event_t next;
 971
 972                                 rc = yaml_parser_parse(reply, &next);
 973                                 if (rc == 0)
 974                                         goto parser_error;
 975
 976                                 if (next.type != YAML_SCALAR_EVENT) {
 977                                         yaml_event_delete(&next);
 978                                         continue;
 979                                 }
 980
 981                                 value = (char *)next.data.scalar.value;
 982                                 if (strcmp(value, "groups") == 0) {
 983                                         yaml_event_delete(&next);
 984                                 } else if (strcmp(value, "nid") == 0) {
 985                                         yaml_event_t next;
 986                                         char *tmp;
 987
 988                                         rc = yaml_parser_parse(reply, &next);
 989                                         if (rc == 0)
 990                                                 goto parser_error;
 991
 992                                         fprintf(stdout, "\t%s: ",
 993                                                 (char *)next.data.scalar.value);
 994
 995                                         rc = yaml_parser_parse(reply, &next);
 996                                         if (rc == 0)
 997                                                 goto parser_error;
 998
 999                                         tmp = (char *)next.data.scalar.value;
1000                                         if (strcmp(tmp, "status") == 0) {
1001                                                 yaml_event_t state;
1002
1003                                                 rc = yaml_parser_parse(reply,
1004                                                                        &state);
1005                                                 if (rc == 0)
1006                                                         goto parser_error;
1007
1008                                                 fprintf(stdout, "%s\n",
1009                                                         (char *)state.data.scalar.value);
1010                                                 count++;
1011                                         }
1012                                 } else {
1013                                         yaml_event_t next;
1014
1015                                         rc = yaml_parser_parse(reply, &next);
1016                                         if (rc == 0)
1017                                                 goto parser_error;
1018
1019                                         strncpy(name, value, sizeof(name) - 1);
1020                                         fprintf(stdout, "Group [ %s ]\n", name);
1021                                         count = 0;
1022
1023                                         if (next.type !=
1024                                             YAML_SEQUENCE_START_EVENT) {
1025                                                 fprintf(stdout,
1026                                                         "No nodes found [ %s ]\n",
1027                                                         name);
1028                                         }
1029                                 }
1030                         } else if (event.type == YAML_SEQUENCE_END_EVENT &&
1031                                    count) {
1032                                 fprintf(stdout, "Total %d nodes [ %s ]\n",
1033                                         count, name);
1034                                 count = 0;
1035                         }
1036
1037                         done = (event.type == YAML_STREAM_END_EVENT);
1038                         yaml_event_delete(&event);
1039                 }
1040         } else if (group) {
1041                 int active = 0, busy = 0, down = 0, unknown = 0;
1042                 char group[LST_NAME_SIZE];
1043                 bool done = false;
1044
1045                 while (!done) {
1046                         rc = yaml_parser_parse(reply, &event);
1047                         if (rc == 0)
1048                                 goto parser_error;
1049
1050                         if (event.type == YAML_SCALAR_EVENT) {
1051                                 char *value = (char *)event.data.scalar.value;
1052                                 yaml_event_t next;
1053
1054                                 value = (char *)event.data.scalar.value;
1055                                 if (strcmp(value, "groups") == 0) {
1056                                         yaml_event_delete(&event);
1057                                         continue;
1058                                 }
1059
1060                                 rc = yaml_parser_parse(reply, &next);
1061                                 if (rc == 0)
1062                                         goto parser_error;
1063
1064                                 if (next.type == YAML_SCALAR_EVENT) {
1065                                         int status;
1066
1067                                         status = lst_node_str2state((char *)next.data.scalar.value);
1068                                         switch (status) {
1069                                         case LST_NODE_ACTIVE:
1070                                                 active++;
1071                                                 break;
1072                                         case LST_NODE_BUSY:
1073                                                 busy++;
1074                                                 break;
1075                                         case LST_NODE_DOWN:
1076                                                 down++;
1077                                                 break;
1078                                         case LST_NODE_UNKNOWN:
1079                                                 unknown++;
1080                                         default:
1081                                                 break;
1082                                         }
1083                                 } else if (next.type == YAML_SEQUENCE_START_EVENT) {
1084                                         strncpy(group, value,
1085                                                 sizeof(group) - 1);
1086                                         active = 0;
1087                                         busy = 0;
1088                                         down = 0;
1089                                         unknown = 0;
1090                                 }
1091                                 yaml_event_delete(&next);
1092                         } else if (event.type == YAML_SEQUENCE_END_EVENT) {
1093                                 if (strlen(group)) {
1094                                         fprintf(stdout, "\t%d\t%d\t%d\t%d\t%d\t%s\n",
1095                                                 active, busy, down, unknown,
1096                                                 active + busy + down + unknown,
1097                                                 group);
1098                                 }
1099                                 memset(group, 0, sizeof(group));
1100                         }
1101                         done = (event.type == YAML_STREAM_END_EVENT);
1102                         yaml_event_delete(&event);
1103                 }
1104         } else {
1105                 bool done = false;
1106                 unsigned int i = 1;
1107
1108                 while (!done) {
1109                         rc = yaml_parser_parse(reply, &event);
1110                         if (rc == 0)
1111                                 goto parser_error;
1112
1113                         if (event.type == YAML_SCALAR_EVENT) {
1114                                 char *value;
1115
1116                                 value = (char *)event.data.scalar.value;
1117                                 if (strlen(value) &&
1118                                     strcmp(value, "groups") != 0) {
1119                                         if (print)
1120                                                 fprintf(stdout, "%d) %s\n",
1121                                                         i, value);
1122                                         i++;
1123                                 }
1124                         }
1125                         done = (event.type == YAML_STREAM_END_EVENT);
1126
1127                         yaml_event_delete(&event);
1128                 }
1129                 if (print)
1130                         fprintf(stdout, "Total %d groups\n", i - 1);
1131                 else
1132                         rc = i - 1;
1133         }
1134 parser_error:
1135         return rc;
1136 }
1137
1138 #define LST_NODES_TITLE "\tACTIVE\tBUSY\tDOWN\tUNKNOWN\tTOTAL\n"
1139
1140 static int lst_yaml_groups(int nlflags, char *name, int states, bool print)
1141 {
1142         yaml_emitter_t request;
1143         yaml_parser_t reply;
1144         yaml_event_t event;
1145         struct nl_sock *sk;
1146         int rc;
1147
1148         sk = nl_socket_alloc();
1149         if (!sk)
1150                 return -EOPNOTSUPP;
1151
1152         /* Setup reply parser to recieve Netlink packets */
1153         rc = yaml_parser_initialize(&reply);
1154         if (rc == 0) {
1155                 nl_socket_free(sk);
1156                 return -EOPNOTSUPP;
1157         }
1158
1159         rc = yaml_parser_set_input_netlink(&reply, sk, false);
1160         if (rc == 0)
1161                 goto parser_error;
1162
1163         /* Create Netlink emitter to send request to kernel */
1164         yaml_emitter_initialize(&request);
1165         rc = yaml_emitter_set_output_netlink(&request, sk,
1166                                              LNET_SELFTEST_GENL_NAME,
1167                                              LNET_SELFTEST_GENL_VERSION,
1168                                              LNET_SELFTEST_CMD_GROUPS,
1169                                              nlflags);
1170         if (rc == 0)
1171                 goto emitter_error;
1172
1173         yaml_emitter_open(&request);
1174         yaml_document_start_event_initialize(&event, NULL, NULL, NULL, 0);
1175         rc = yaml_emitter_emit(&request, &event);
1176         if (rc == 0)
1177                 goto emitter_error;
1178
1179         yaml_mapping_start_event_initialize(&event, NULL,
1180                                             (yaml_char_t *)YAML_MAP_TAG,
1181                                             1, YAML_BLOCK_MAPPING_STYLE);
1182         rc = yaml_emitter_emit(&request, &event);
1183         if (rc == 0)
1184                 goto emitter_error;
1185
1186         yaml_scalar_event_initialize(&event, NULL,
1187                                      (yaml_char_t *)YAML_STR_TAG,
1188                                      (yaml_char_t *)"groups",
1189                                      strlen("groups"), 1, 0,
1190                                      YAML_PLAIN_SCALAR_STYLE);
1191         rc = yaml_emitter_emit(&request, &event);
1192         if (rc == 0)
1193                 goto emitter_error;
1194
1195         if (name) {
1196                 yaml_sequence_start_event_initialize(&event, NULL,
1197                                                      (yaml_char_t *)YAML_SEQ_TAG,
1198                                                      1, YAML_BLOCK_SEQUENCE_STYLE);
1199                 rc = yaml_emitter_emit(&request, &event);
1200                 if (rc == 0)
1201                         goto emitter_error;
1202
1203                 yaml_mapping_start_event_initialize(&event, NULL,
1204                                                     (yaml_char_t *)YAML_MAP_TAG,
1205                                                     1, YAML_BLOCK_MAPPING_STYLE);
1206                 rc = yaml_emitter_emit(&request, &event);
1207                 if (rc == 0)
1208                         goto emitter_error;
1209
1210                 yaml_scalar_event_initialize(&event, NULL,
1211                                              (yaml_char_t *)YAML_STR_TAG,
1212                                              (yaml_char_t *)name,
1213                                              strlen(name), 1, 0,
1214                                              YAML_PLAIN_SCALAR_STYLE);
1215                 rc = yaml_emitter_emit(&request, &event);
1216                 if (rc == 0)
1217                         goto emitter_error;
1218
1219                 if (states) {
1220                         int max = ffs(LST_NODE_UNKNOWN) + 1, i;
1221                         char *state;
1222
1223                         yaml_sequence_start_event_initialize(&event, NULL,
1224                                                              (yaml_char_t *)YAML_SEQ_TAG,
1225                                                              1, YAML_BLOCK_SEQUENCE_STYLE);
1226                         rc = yaml_emitter_emit(&request, &event);
1227                         if (rc == 0)
1228                                 goto emitter_error;
1229
1230
1231                         for (i = 0; i < max; i++) {
1232                                 int mask = states & (1 << i);
1233
1234                                 state = lst_node_state2str(mask);
1235                                 if (mask != LST_NODE_UNKNOWN && strcmp(state, "Unknown") == 0)
1236                                         continue;
1237
1238                                 yaml_mapping_start_event_initialize(&event, NULL,
1239                                                                     (yaml_char_t *)YAML_MAP_TAG,
1240                                                                     1, YAML_BLOCK_MAPPING_STYLE);
1241                                 rc = yaml_emitter_emit(&request, &event);
1242                                 if (rc == 0)
1243                                         goto emitter_error;
1244
1245                                 yaml_scalar_event_initialize(&event, NULL,
1246                                                              (yaml_char_t *)YAML_STR_TAG,
1247                                                              (yaml_char_t *)"status",
1248                                                              strlen("status"), 1, 0,
1249                                                              YAML_PLAIN_SCALAR_STYLE);
1250                                 rc = yaml_emitter_emit(&request, &event);
1251                                 if (rc == 0)
1252                                         goto emitter_error;
1253
1254                                 yaml_scalar_event_initialize(&event, NULL,
1255                                                              (yaml_char_t *)YAML_STR_TAG,
1256                                                              (yaml_char_t *)state,
1257                                                              strlen(state), 1, 0,
1258                                                              YAML_PLAIN_SCALAR_STYLE);
1259                                 rc = yaml_emitter_emit(&request, &event);
1260                                 if (rc == 0)
1261                                         goto emitter_error;
1262
1263                                 yaml_mapping_end_event_initialize(&event);
1264                                 rc = yaml_emitter_emit(&request, &event);
1265                                 if (rc == 0)
1266                                         goto emitter_error;
1267                         }
1268
1269                         yaml_sequence_end_event_initialize(&event);
1270                         rc = yaml_emitter_emit(&request, &event);
1271                         if (rc == 0)
1272                                 goto emitter_error;
1273                 } else {
1274                         yaml_scalar_event_initialize(&event, NULL,
1275                                                      (yaml_char_t *)YAML_STR_TAG,
1276                                                      (yaml_char_t *)"",
1277                                                      strlen(""), 1, 0,
1278                                                      YAML_PLAIN_SCALAR_STYLE);
1279                         rc = yaml_emitter_emit(&request, &event);
1280                         if (rc == 0)
1281                                 goto emitter_error;
1282                 }
1283
1284                 yaml_mapping_end_event_initialize(&event);
1285                 rc = yaml_emitter_emit(&request, &event);
1286                 if (rc == 0)
1287                         goto emitter_error;
1288
1289                 yaml_sequence_end_event_initialize(&event);
1290                 rc = yaml_emitter_emit(&request, &event);
1291                 if (rc == 0)
1292                         goto emitter_error;
1293         } else {
1294                 yaml_scalar_event_initialize(&event, NULL,
1295                                              (yaml_char_t *)YAML_STR_TAG,
1296                                              (yaml_char_t *)"",
1297                                              strlen(""), 1, 0,
1298                                              YAML_PLAIN_SCALAR_STYLE);
1299                 rc = yaml_emitter_emit(&request, &event);
1300                 if (rc == 0)
1301                         goto emitter_error;
1302         }
1303         yaml_mapping_end_event_initialize(&event);
1304         rc = yaml_emitter_emit(&request, &event);
1305         if (rc == 0)
1306                 goto emitter_error;
1307
1308         yaml_document_end_event_initialize(&event, 0);
1309         rc = yaml_emitter_emit(&request, &event);
1310         if (rc == 0)
1311                 goto emitter_error;
1312
1313         rc = yaml_emitter_close(&request);
1314         if (rc == 0) {
1315 emitter_error:
1316                 yaml_emitter_log_error(&request, stderr);
1317                 yaml_emitter_delete(&request);
1318                 rc = -EINVAL;
1319                 goto parser_error;
1320         }
1321         yaml_emitter_delete(&request);
1322
1323         /* display output */
1324         if (nlflags == NLM_F_DUMP)
1325                 rc = lst_yaml_display_groups(&reply, name, states, print);
1326 parser_error:
1327         if (rc == 0)
1328                 yaml_parser_log_error(&reply, stderr, NULL);
1329         yaml_parser_delete(&reply);
1330         nl_socket_free(sk);
1331
1332         if (print)
1333                 rc = rc == 1 ? 0 : -EINVAL;
1334
1335         return rc;
1336 }
1337
1338 static int
1339 lst_ping_ioctl(char *str, int type, int timeout,
1340                int count, struct lnet_process_id *ids, struct list_head *head)
1341 {
1342         struct lstio_debug_args args = { 0 };
1343
1344         args.lstio_dbg_key     = session_key;
1345         args.lstio_dbg_type    = type;
1346         args.lstio_dbg_flags   = 0;
1347         args.lstio_dbg_timeout = timeout;
1348         args.lstio_dbg_nmlen   = (str == NULL) ? 0 : strlen(str);
1349         args.lstio_dbg_namep   = str;
1350         args.lstio_dbg_count   = count;
1351         args.lstio_dbg_idsp    = ids;
1352         args.lstio_dbg_resultp = head;
1353
1354         return lst_ioctl(LSTIO_DEBUG, &args, sizeof(args));
1355 }
1356
1357 static int
1358 lst_get_node_count(int type, char *str, int *countp,
1359                    struct lnet_process_id **idspp)
1360 {
1361         char buf[LST_NAME_SIZE];
1362         struct lstcon_test_batch_ent ent;
1363         struct lstcon_ndlist_ent *entp = &ent.tbe_cli_nle;
1364         struct lst_sid sid;
1365         unsigned int feats;
1366         int key, rc;
1367
1368         switch (type) {
1369         case LST_OPC_SESSION:
1370                 rc = lst_session_info_ioctl(buf, LST_NAME_SIZE,
1371                                             &key, &feats, &sid, entp);
1372                 break;
1373
1374         case LST_OPC_BATCHSRV:
1375                 entp = &ent.tbe_srv_nle;
1376         case LST_OPC_BATCHCLI:
1377                 rc = lst_info_batch_ioctl(str, 0, 0, &ent, NULL, NULL, NULL);
1378                 break;
1379
1380         case LST_OPC_GROUP:
1381                 rc = lst_yaml_groups(NLM_F_DUMP, NULL, 0, false);
1382                 if (rc == -EOPNOTSUPP) {
1383                         rc = lst_info_group_ioctl(str, entp, NULL, NULL, NULL);
1384                 } else if (rc > 0) {
1385                         entp->nle_nnode = rc;
1386                         rc = 0;
1387                 }
1388                 break;
1389
1390         case LST_OPC_NODES:
1391                 rc = lst_parse_nids(str, &entp->nle_nnode, idspp) < 0 ? -1 : 0;
1392                 break;
1393
1394         default:
1395                 rc = -1;
1396                 break;
1397         }
1398
1399         if (rc == 0)
1400                 *countp = entp->nle_nnode;
1401
1402         return rc;
1403 }
1404
1405 static int
1406 jt_lst_ping(int argc,  char **argv)
1407 {
1408         struct list_head   head;
1409         struct lnet_process_id *ids = NULL;
1410         struct lstcon_rpc_ent  *ent = NULL;
1411         char *str = NULL;
1412         int optidx  = 0;
1413         int server  = 0;
1414         int timeout = 5;
1415         int count   = 0;
1416         int type    = 0;
1417         int rc      = 0;
1418         int c;
1419
1420         static const struct option ping_opts[] = {
1421                 { .name = "session", .has_arg = no_argument,       .val = 's' },
1422                 { .name = "server",  .has_arg = no_argument,       .val = 'v' },
1423                 { .name = "batch",   .has_arg = required_argument, .val = 'b' },
1424                 { .name = "group",   .has_arg = required_argument, .val = 'g' },
1425                 { .name = "nodes",   .has_arg = required_argument, .val = 'n' },
1426                 { .name = "timeout", .has_arg = required_argument, .val = 't' },
1427                 { .name = NULL, } };
1428
1429         if (session_key == 0) {
1430                 fprintf(stderr,
1431                         "Can't find env LST_SESSION or value is not valid\n");
1432                 return -1;
1433         }
1434
1435         while (1) {
1436
1437                 c = getopt_long(argc, argv, "g:b:n:t:sv", ping_opts, &optidx);
1438                 if (c == -1)
1439                         break;
1440
1441                 switch (c) {
1442                 case 's':
1443                         type = LST_OPC_SESSION;
1444                         break;
1445
1446                 case 'g':
1447                         type = LST_OPC_GROUP;
1448                         str = optarg;
1449                         break;
1450
1451                 case 'b':
1452                         type = LST_OPC_BATCHCLI;
1453                         str = optarg;
1454                         break;
1455
1456                 case 'n':
1457                         type = LST_OPC_NODES;
1458                         str = optarg;
1459                         break;
1460
1461                 case 't':
1462                         timeout = atoi(optarg);
1463                         break;
1464
1465                 case 'v':
1466                         server = 1;
1467                         break;
1468
1469                 default:
1470                         lst_print_usage(argv[0]);
1471                         return -1;
1472                 }
1473         }
1474
1475         if (type == 0 || timeout <= 0 || optind != argc) {
1476                 lst_print_usage(argv[0]);
1477                 return -1;
1478         }
1479
1480         if (type == LST_OPC_BATCHCLI && server)
1481                 type = LST_OPC_BATCHSRV;
1482
1483         rc = lst_get_node_count(type, str, &count, &ids);
1484         if (rc < 0) {
1485                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
1486                         (str == NULL) ? "session" : str, strerror(errno));
1487                 return -1;
1488         }
1489
1490         INIT_LIST_HEAD(&head);
1491
1492         rc = lst_alloc_rpcent(&head, count, LST_NAME_SIZE);
1493         if (rc != 0) {
1494                 fprintf(stderr, "Out of memory\n");
1495                 goto out;
1496         }
1497
1498         if (count == 0) {
1499                 fprintf(stdout, "Target %s is empty\n",
1500                         (str == NULL) ? "session" : str);
1501                 goto out;
1502         }
1503
1504         rc = lst_ping_ioctl(str, type, timeout, count, ids, &head);
1505         if (rc == -1) { /* local failure */
1506                 lst_print_error("debug", "Failed to ping %s: %s\n",
1507                                 (str == NULL) ? "session" : str,
1508                                 strerror(errno));
1509                 rc = -1;
1510                 goto out;
1511         }
1512
1513         /* ignore RPC errors and framwork errors */
1514         list_for_each_entry(ent, &head, rpe_link) {
1515                 fprintf(stdout, "\t%s: %s [session: %s id: %s]\n",
1516                         libcfs_id2str(ent->rpe_peer),
1517                         lst_node_state2str(ent->rpe_state),
1518                         (ent->rpe_state == LST_NODE_ACTIVE ||
1519                          ent->rpe_state == LST_NODE_BUSY) ?
1520                                 (ent->rpe_rpc_errno == 0 ?
1521                                         &ent->rpe_payload[0] : "Unknown") :
1522                                 "<NULL>", libcfs_nid2str(ent->rpe_sid.ses_nid));
1523         }
1524
1525 out:
1526         lst_free_rpcent(&head);
1527
1528         if (ids != NULL)
1529                 free(ids);
1530
1531         return rc;
1532 }
1533
1534 static int
1535 lst_add_nodes_ioctl(char *name, int count, struct lnet_process_id *ids,
1536                     unsigned int *featp, struct list_head *resultp)
1537 {
1538         struct lstio_group_nodes_args args = { 0 };
1539
1540         args.lstio_grp_key     = session_key;
1541         args.lstio_grp_nmlen   = strlen(name);
1542         args.lstio_grp_namep   = name;
1543         args.lstio_grp_count   = count;
1544         args.lstio_grp_featp   = featp;
1545         args.lstio_grp_idsp    = ids;
1546         args.lstio_grp_resultp = resultp;
1547
1548         return lst_ioctl(LSTIO_NODES_ADD, &args, sizeof(args));
1549 }
1550
1551 static int
1552 lst_del_group_ioctl(char *name)
1553 {
1554         struct lstio_group_del_args args = { 0 };
1555
1556         args.lstio_grp_key   = session_key;
1557         args.lstio_grp_nmlen = strlen(name);
1558         args.lstio_grp_namep = name;
1559
1560         return lst_ioctl(LSTIO_GROUP_DEL, &args, sizeof(args));
1561 }
1562
1563 static int
1564 lst_del_group(char *grp_name)
1565 {
1566         int rc;
1567
1568         rc = lst_del_group_ioctl(grp_name);
1569         if (rc == 0) {
1570                 fprintf(stdout, "Group is deleted\n");
1571                 return 0;
1572         }
1573
1574         if (rc == -1) {
1575                 lst_print_error("group", "Failed to delete group: %s\n",
1576                                 strerror(errno));
1577                 return rc;
1578         }
1579
1580         fprintf(stderr, "Group is deleted with some errors\n");
1581
1582         if (trans_stat.trs_rpc_errno != 0) {
1583                 fprintf(stderr,
1584                         "[RPC] Failed to send %d end session RPCs: %s\n",
1585                         lstcon_rpc_stat_failure(&trans_stat, 0),
1586                         strerror(trans_stat.trs_rpc_errno));
1587         }
1588
1589         if (trans_stat.trs_fwk_errno != 0) {
1590                 fprintf(stderr,
1591                         "[FWK] Failed to end session on %d nodes: %s\n",
1592                 lstcon_sesop_stat_failure(&trans_stat, 0),
1593                 strerror(trans_stat.trs_fwk_errno));
1594         }
1595
1596         return -1;
1597 }
1598
1599 static int
1600 lst_add_group_ioctl(char *name)
1601 {
1602         struct lstio_group_add_args args = { 0 };
1603
1604         args.lstio_grp_key     =  session_key;
1605         args.lstio_grp_nmlen   =  strlen(name);
1606         args.lstio_grp_namep   =  name;
1607
1608         return lst_ioctl(LSTIO_GROUP_ADD, &args, sizeof(args));
1609 }
1610
1611 static int
1612 jt_lst_add_group(int argc, char **argv)
1613 {
1614         struct list_head head;
1615         struct lnet_process_id *ids;
1616         char *name;
1617         unsigned int feats = session_features;
1618         int count;
1619         int rc;
1620         int i;
1621         bool nodes_added = false;
1622
1623         if (session_key == 0) {
1624                 fprintf(stderr,
1625                         "Can't find env LST_SESSION or value is not valid\n");
1626                 return -1;
1627         }
1628
1629         if (argc < 3) {
1630                 lst_print_usage(argv[0]);
1631                 return -1;
1632         }
1633
1634         name = argv[1];
1635         if (strlen(name) >= LST_NAME_SIZE) {
1636                 fprintf(stderr, "Name length is limited to %d\n",
1637                         LST_NAME_SIZE - 1);
1638                 return -1;
1639         }
1640
1641         rc = lst_add_group_ioctl(name);
1642         if (rc != 0) {
1643                 lst_print_error("group", "Failed to add group %s: %s\n",
1644                                 name, strerror(errno));
1645                 return -1;
1646         }
1647
1648         INIT_LIST_HEAD(&head);
1649
1650         for (i = 2; i < argc; i++) {
1651                 /* parse address list */
1652                 rc = lst_parse_nids(argv[i], &count, &ids);
1653                 if (rc < 0) {
1654                         fprintf(stderr, "Ignore invalid id list %s\n",
1655                                 argv[i]);
1656                         continue;
1657                 }
1658
1659                 if (count == 0)
1660                         continue;
1661
1662                 rc = lst_alloc_rpcent(&head, count, 0);
1663                 if (rc != 0) {
1664                         fprintf(stderr, "Out of memory\n");
1665                         free(ids);
1666                         rc = -1;
1667                         goto failed;
1668                 }
1669
1670                 rc = lst_add_nodes_ioctl(name, count, ids, &feats, &head);
1671
1672                 free(ids);
1673
1674                 if (rc != 0)
1675                         goto failed;
1676
1677                 fprintf(stdout, "%s are added to session\n", argv[i]);
1678
1679                 nodes_added = true;
1680
1681                 if ((feats & session_features) != session_features) {
1682                         fprintf(stdout,
1683                                 "Warning, this session will run with compatible mode because some test nodes might not understand these features: %x\n",
1684                                 (~feats & session_features));
1685                 }
1686
1687                 lst_free_rpcent(&head);
1688         }
1689
1690         if (!nodes_added) {
1691                 /* The selftest kernel module expects that a group should
1692                  * have at least one node, since it doesn't make sense for
1693                  * an empty group to be added to a test.
1694                  */
1695                 fprintf(stderr,
1696                         "No nodes added successfully, deleting group %s\n",
1697                         name);
1698                 rc = lst_del_group(name);
1699                 if (rc != 0) {
1700                         fprintf(stderr,
1701                                 "Failed to delete group %s. Group is empty.\n",
1702                                 name);
1703                 }
1704         }
1705
1706         return rc;
1707
1708 failed:
1709         if (rc == -1) {
1710                 lst_print_error("group", "Failed to add nodes %s: %s\n",
1711                                 argv[i], strerror(errno));
1712
1713         } else {
1714                 if (trans_stat.trs_fwk_errno == EPROTO) {
1715                         fprintf(stderr,
1716                                 "test nodes might have different LST features, please disable some features by setting LST_FEATURES\n");
1717                 }
1718
1719                 lst_print_transerr(&head, "create session");
1720         }
1721
1722         lst_free_rpcent(&head);
1723
1724         if (!nodes_added) {
1725                 fprintf(stderr,
1726                         "No nodes added successfully, deleting group %s\n",
1727                         name);
1728                 if (lst_del_group(name) != 0) {
1729                         fprintf(stderr,
1730                                 "Failed to delete group %s. Group is empty.\n",
1731                                 name);
1732                 }
1733         }
1734
1735         return rc;
1736 }
1737
1738 static int
1739 jt_lst_del_group(int argc, char **argv)
1740 {
1741         int rc;
1742
1743         if (session_key == 0) {
1744                 fprintf(stderr,
1745                         "Can't find env LST_SESSION or value is not valid\n");
1746                 return -1;
1747         }
1748
1749         if (argc != 2) {
1750                 lst_print_usage(argv[0]);
1751                 return -1;
1752         }
1753
1754         rc = lst_del_group(argv[1]);
1755
1756         return rc;
1757 }
1758
1759 static int
1760 lst_update_group_ioctl(int opc, char *name, int clean, int count,
1761                        struct lnet_process_id *ids, struct list_head *resultp)
1762 {
1763         struct lstio_group_update_args args = { 0 };
1764
1765         args.lstio_grp_key      = session_key;
1766         args.lstio_grp_opc      = opc;
1767         args.lstio_grp_args     = clean;
1768         args.lstio_grp_nmlen    = strlen(name);
1769         args.lstio_grp_namep    = name;
1770         args.lstio_grp_count    = count;
1771         args.lstio_grp_idsp     = ids;
1772         args.lstio_grp_resultp  = resultp;
1773
1774         return lst_ioctl(LSTIO_GROUP_UPDATE, &args, sizeof(args));
1775 }
1776
1777 static int
1778 jt_lst_update_group(int argc, char **argv)
1779 {
1780         struct list_head   head;
1781         struct lnet_process_id *ids = NULL;
1782         char *str = NULL;
1783         char *grp = NULL;
1784         int optidx = 0;
1785         int count = 0;
1786         int clean = 0;
1787         int opc = 0;
1788         int rc;
1789         int c;
1790
1791         static const struct option update_group_opts[] = {
1792                 { .name = "refresh", .has_arg = no_argument,       .val = 'f' },
1793                 { .name = "clean",   .has_arg = required_argument, .val = 'c' },
1794                 { .name = "remove",  .has_arg = required_argument, .val = 'r' },
1795                 { .name = NULL } };
1796
1797         if (session_key == 0) {
1798                 fprintf(stderr,
1799                         "Can't find env LST_SESSION or value is not valid\n");
1800                 return -1;
1801         }
1802
1803         while (1) {
1804                 c = getopt_long(argc, argv, "fc:r:", update_group_opts,
1805                                 &optidx);
1806
1807                 /* Detect the end of the options. */
1808                 if (c == -1)
1809                         break;
1810
1811                 switch (c) {
1812                 case 'f':
1813                         if (opc != 0) {
1814                                 lst_print_usage(argv[0]);
1815                                 return -1;
1816                         }
1817                         opc = LST_GROUP_REFRESH;
1818                         break;
1819
1820                 case 'r':
1821                         if (opc != 0) {
1822                                 lst_print_usage(argv[0]);
1823                                 return -1;
1824                         }
1825                         opc = LST_GROUP_RMND;
1826                         str = optarg;
1827                         break;
1828
1829                 case 'c':
1830                         clean = lst_node_str2state(optarg);
1831                         if (opc != 0 || clean <= 0) {
1832                                 lst_print_usage(argv[0]);
1833                                 return -1;
1834                         }
1835                         opc = LST_GROUP_CLEAN;
1836                         break;
1837
1838                 default:
1839                         lst_print_usage(argv[0]);
1840                         return -1;
1841                 }
1842         }
1843
1844         /* no OPC or group is specified */
1845         if (opc == 0 || optind != argc - 1) {
1846                 lst_print_usage(argv[0]);
1847                 return -1;
1848         }
1849
1850         grp = argv[optind];
1851
1852         INIT_LIST_HEAD(&head);
1853
1854         if (opc == LST_GROUP_RMND || opc == LST_GROUP_REFRESH) {
1855                 rc = lst_get_node_count(opc == LST_GROUP_RMND ? LST_OPC_NODES :
1856                                                                 LST_OPC_GROUP,
1857                                         opc == LST_GROUP_RMND ? str : grp,
1858                                         &count, &ids);
1859
1860                 if (rc != 0) {
1861                         fprintf(stderr, "Can't get count of nodes from %s: %s\n",
1862                                 opc == LST_GROUP_RMND ? str : grp,
1863                                 strerror(errno));
1864                         return -1;
1865                 }
1866
1867                 rc = lst_alloc_rpcent(&head, count, 0);
1868                 if (rc != 0) {
1869                         fprintf(stderr, "Out of memory\n");
1870                         free(ids);
1871                         return -1;
1872                 }
1873         }
1874
1875         rc = lst_update_group_ioctl(opc, grp, clean, count, ids, &head);
1876
1877         if (ids != NULL)
1878                 free(ids);
1879
1880         if (rc == 0) {
1881                 lst_free_rpcent(&head);
1882                 return 0;
1883         }
1884
1885         if (rc == -1) {
1886                 lst_free_rpcent(&head);
1887                 lst_print_error("group", "Failed to update group: %s\n",
1888                                 strerror(errno));
1889                 return rc;
1890         }
1891
1892         lst_print_transerr(&head, "Updating group");
1893
1894         lst_free_rpcent(&head);
1895
1896         return rc;
1897 }
1898
1899 static int
1900 lst_list_group_ioctl(int len, char *name, int idx)
1901 {
1902         struct lstio_group_list_args args = { 0 };
1903
1904         args.lstio_grp_key   = session_key;
1905         args.lstio_grp_idx   = idx;
1906         args.lstio_grp_nmlen = len;
1907         args.lstio_grp_namep = name;
1908
1909         return lst_ioctl(LSTIO_GROUP_LIST, &args, sizeof(args));
1910 }
1911
1912 static int
1913 lst_info_group_ioctl(char *name, struct lstcon_ndlist_ent *gent,
1914                      int *idx, int *count, struct lstcon_node_ent *dents)
1915 {
1916         struct lstio_group_info_args args = { 0 };
1917
1918         args.lstio_grp_key    = session_key;
1919         args.lstio_grp_nmlen  = strlen(name);
1920         args.lstio_grp_namep  = name;
1921         args.lstio_grp_entp   = gent;
1922         args.lstio_grp_idxp   = idx;
1923         args.lstio_grp_ndentp = count;
1924         args.lstio_grp_dentsp = dents;
1925
1926         return lst_ioctl(LSTIO_GROUP_INFO, &args, sizeof(args));
1927 }
1928
1929 static int
1930 lst_list_group_all(void)
1931 {
1932         char name[LST_NAME_SIZE];
1933         int rc, i;
1934
1935         /* no group is specified, list name of all groups */
1936         for (i = 0; ; i++) {
1937                 rc = lst_list_group_ioctl(LST_NAME_SIZE, name, i);
1938                 if (rc == 0) {
1939                         fprintf(stdout, "%d) %s\n", i + 1, name);
1940                         continue;
1941                 }
1942
1943                 if (errno == ENOENT)
1944                         break;
1945
1946                 lst_print_error("group", "Failed to list group: %s\n",
1947                                 strerror(errno));
1948                 return -1;
1949         }
1950
1951         fprintf(stdout, "Total %d groups\n", i);
1952
1953         return 0;
1954 }
1955
1956 static int
1957 jt_lst_list_group(int argc, char **argv)
1958 {
1959         struct lstcon_ndlist_ent gent;
1960         struct lstcon_node_ent   *dents;
1961         int states = 0;
1962         int optidx  = 0;
1963         int verbose = 0;
1964         int active  = 0;
1965         int busy    = 0;
1966         int down    = 0;
1967         int unknown = 0;
1968         int all     = 0;
1969         int count;
1970         int index;
1971         int i;
1972         int j;
1973         int c;
1974         int rc      = 0;
1975         static const struct option list_group_opts[] = {
1976                 { .name = "active",  .has_arg = no_argument, .val = 'a' },
1977                 { .name = "busy",    .has_arg = no_argument, .val = 'b' },
1978                 { .name = "down",    .has_arg = no_argument, .val = 'd' },
1979                 { .name = "unknown", .has_arg = no_argument, .val = 'u' },
1980                 { .name = "all",     .has_arg = no_argument, .val = 'l' },
1981                 { .name = NULL, } };
1982
1983         if (session_key == 0) {
1984                 fprintf(stderr,
1985                         "Can't find env LST_SESSION or value is not valid\n");
1986                 return -1;
1987         }
1988
1989         while (1) {
1990                 c = getopt_long(argc, argv, "abdul", list_group_opts, &optidx);
1991                 if (c == -1)
1992                         break;
1993
1994                 switch (c) {
1995                 case 'a':
1996                         verbose = active = 1;
1997                         states |= LST_NODE_ACTIVE;
1998                         all = 0;
1999                         break;
2000                 case 'b':
2001                         verbose = busy = 1;
2002                         states |= LST_NODE_BUSY;
2003                         all = 0;
2004                         break;
2005                 case 'd':
2006                         verbose = down = 1;
2007                         states |= LST_NODE_DOWN;
2008                         all = 0;
2009                         break;
2010                 case 'u':
2011                         verbose = unknown = 1;
2012                         states |= LST_NODE_UNKNOWN;
2013                         all = 0;
2014                         break;
2015                 case 'l':
2016                         states |= LST_NODE_ACTIVE | LST_NODE_BUSY |
2017                                   LST_NODE_DOWN | LST_NODE_UNKNOWN;
2018                         verbose = all = 1;
2019                         break;
2020                 default:
2021                         lst_print_usage(argv[0]);
2022                         return -1;
2023                 }
2024         }
2025
2026         if (optind == argc) {
2027                 rc = lst_yaml_groups(NLM_F_DUMP, NULL, 0, true);
2028                 if (rc <= 0) {
2029                         if (rc == -EOPNOTSUPP)
2030                                 goto old_api;
2031                         return rc;
2032                 }
2033         } else {
2034                 for (i = optind; i < argc; i++) {
2035                         rc = lst_yaml_groups(NLM_F_DUMP, argv[i], states,
2036                                              i == optind ? true : false);
2037                         if (rc < 0) {
2038                                 if (rc == -EOPNOTSUPP)
2039                                         goto old_api;
2040                                 return rc;
2041                         }
2042                 }
2043                 return 0;
2044         }
2045 old_api:
2046         if (optind == argc) {
2047                 /* no group is specified, list name of all groups */
2048                 rc = lst_list_group_all();
2049
2050                 return rc;
2051         }
2052
2053         if (!verbose)
2054                 fprintf(stdout, LST_NODES_TITLE);
2055
2056         /* list nodes in specified groups */
2057         for (i = optind; i < argc; i++) {
2058                 rc = lst_info_group_ioctl(argv[i], &gent, NULL, NULL, NULL);
2059                 if (rc != 0) {
2060                         if (errno == ENOENT) {
2061                                 rc = 0;
2062                                 break;
2063                         }
2064
2065                         lst_print_error("group", "Failed to list group: %s\n",
2066                                         strerror(errno));
2067                         break;
2068                 }
2069
2070                 if (!verbose) {
2071                         fprintf(stdout, "\t%d\t%d\t%d\t%d\t%d\t%s\n",
2072                                 gent.nle_nactive, gent.nle_nbusy,
2073                                 gent.nle_ndown, gent.nle_nunknown,
2074                                 gent.nle_nnode, argv[i]);
2075                         continue;
2076                 }
2077
2078                 fprintf(stdout, "Group [ %s ]\n", argv[i]);
2079
2080                 if (gent.nle_nnode == 0) {
2081                         fprintf(stdout, "No nodes found [ %s ]\n", argv[i]);
2082                         continue;
2083                 }
2084
2085                 count = gent.nle_nnode;
2086
2087                 dents = malloc(count * sizeof(struct lstcon_node_ent));
2088                 if (dents == NULL) {
2089                         fprintf(stderr, "Failed to malloc: %s\n",
2090                                 strerror(errno));
2091                         return -1;
2092                 }
2093
2094                 index = 0;
2095                 rc = lst_info_group_ioctl(argv[i], &gent, &index, &count,
2096                                           dents);
2097                 if (rc != 0) {
2098                         lst_print_error("group", "Failed to list group: %s\n",
2099                                         strerror(errno));
2100                         free(dents);
2101                         return -1;
2102                 }
2103
2104                 for (j = 0, c = 0; j < count; j++) {
2105                         if (all ||
2106                             ((active && dents[j].nde_state == LST_NODE_ACTIVE) ||
2107                              (busy && dents[j].nde_state == LST_NODE_BUSY) ||
2108                              (down && dents[j].nde_state == LST_NODE_DOWN) ||
2109                              (unknown && dents[j].nde_state == LST_NODE_UNKNOWN))) {
2110
2111                                 fprintf(stdout, "\t%s: %s\n",
2112                                         libcfs_id2str(dents[j].nde_id),
2113                                         lst_node_state2str(dents[j].nde_state));
2114                                 c++;
2115                         }
2116                 }
2117
2118                 fprintf(stdout, "Total %d nodes [ %s ]\n", c, argv[i]);
2119
2120                 free(dents);
2121         }
2122
2123         return rc;
2124 }
2125
2126 static int
2127 lst_stat_ioctl(char *name, int count, struct lnet_process_id *idsp,
2128                int timeout, struct list_head *resultp)
2129 {
2130         struct lstio_stat_args args = { 0 };
2131
2132         args.lstio_sta_key     = session_key;
2133         args.lstio_sta_timeout = timeout;
2134         args.lstio_sta_nmlen   = strlen(name);
2135         args.lstio_sta_namep   = name;
2136         args.lstio_sta_count   = count;
2137         args.lstio_sta_idsp    = idsp;
2138         args.lstio_sta_resultp = resultp;
2139
2140         return lst_ioctl(LSTIO_STAT_QUERY, &args, sizeof(args));
2141 }
2142
2143 typedef struct {
2144         struct list_head        srp_link;
2145         int                     srp_count;
2146         char                    *srp_name;
2147         struct lnet_process_id  *srp_ids;
2148         struct list_head        srp_result[2];
2149 } lst_stat_req_param_t;
2150
2151 static void
2152 lst_stat_req_param_free(lst_stat_req_param_t *srp)
2153 {
2154         int i;
2155
2156         for (i = 0; i < 2; i++)
2157                 lst_free_rpcent(&srp->srp_result[i]);
2158
2159         if (srp->srp_ids != NULL)
2160                 free(srp->srp_ids);
2161
2162         free(srp);
2163 }
2164
2165 static int
2166 lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp, int save_old)
2167 {
2168         lst_stat_req_param_t *srp = NULL;
2169         int count = save_old ? 2 : 1;
2170         int rc, i;
2171
2172         srp = malloc(sizeof(*srp));
2173         if (srp == NULL)
2174                 return -ENOMEM;
2175
2176         memset(srp, 0, sizeof(*srp));
2177         INIT_LIST_HEAD(&srp->srp_result[0]);
2178         INIT_LIST_HEAD(&srp->srp_result[1]);
2179
2180         rc = lst_get_node_count(LST_OPC_GROUP, name, &srp->srp_count, NULL);
2181         if (rc != 0 && errno == ENOENT) {
2182                 rc = lst_get_node_count(LST_OPC_NODES, name,
2183                                         &srp->srp_count, &srp->srp_ids);
2184         }
2185
2186         if (rc != 0) {
2187                 fprintf(stderr,
2188                         "Failed to get count of nodes from %s: %s\n",
2189                         name, strerror(errno));
2190                 lst_stat_req_param_free(srp);
2191
2192                 return rc;
2193         }
2194
2195         srp->srp_name = name;
2196
2197         for (i = 0; i < count; i++) {
2198                 rc = lst_alloc_rpcent(&srp->srp_result[i], srp->srp_count,
2199                                       sizeof(struct sfw_counters)  +
2200                                       sizeof(struct srpc_counters) +
2201                                       sizeof(struct lnet_counters_common));
2202                 if (rc != 0) {
2203                         fprintf(stderr, "Out of memory\n");
2204                         break;
2205                 }
2206         }
2207
2208         if (rc == 0) {
2209                 *srpp = srp;
2210                 return 0;
2211         }
2212
2213         lst_stat_req_param_free(srp);
2214
2215         return rc;
2216 }
2217
2218 typedef struct {
2219         /* TODO */
2220         int foo;
2221 } lst_srpc_stat_result;
2222
2223 #define LST_LNET_AVG    0
2224 #define LST_LNET_MIN    1
2225 #define LST_LNET_MAX    2
2226
2227 typedef struct {
2228         float           lnet_avg_sndrate;
2229         float           lnet_min_sndrate;
2230         float           lnet_max_sndrate;
2231         float           lnet_total_sndrate;
2232
2233         float           lnet_avg_rcvrate;
2234         float           lnet_min_rcvrate;
2235         float           lnet_max_rcvrate;
2236         float           lnet_total_rcvrate;
2237
2238         float           lnet_avg_sndperf;
2239         float           lnet_min_sndperf;
2240         float           lnet_max_sndperf;
2241         float           lnet_total_sndperf;
2242
2243         float           lnet_avg_rcvperf;
2244         float           lnet_min_rcvperf;
2245         float           lnet_max_rcvperf;
2246         float           lnet_total_rcvperf;
2247
2248         int             lnet_stat_count;
2249 } lst_lnet_stat_result_t;
2250
2251 lst_lnet_stat_result_t lnet_stat_result;
2252
2253 static float
2254 lst_lnet_stat_value(int bw, int send, int off)
2255 {
2256         float *p;
2257
2258         p = bw ? &lnet_stat_result.lnet_avg_sndperf :
2259             &lnet_stat_result.lnet_avg_sndrate;
2260
2261         if (!send)
2262                 p += 4;
2263
2264         p += off;
2265
2266         return *p;
2267 }
2268
2269 static void
2270 lst_cal_lnet_stat(float delta, struct lnet_counters_common *lnet_new,
2271                   struct lnet_counters_common *lnet_old, int mbs)
2272 {
2273         float perf;
2274         float rate;
2275         unsigned int unit_divisor;
2276
2277         unit_divisor = (mbs) ? (1000 * 1000) : (1024 * 1024);
2278         perf = (float)(lnet_new->lcc_send_length -
2279                        lnet_old->lcc_send_length) / unit_divisor / delta;
2280         lnet_stat_result.lnet_total_sndperf += perf;
2281
2282         if (lnet_stat_result.lnet_min_sndperf > perf ||
2283             lnet_stat_result.lnet_min_sndperf == 0)
2284                 lnet_stat_result.lnet_min_sndperf = perf;
2285
2286         if (lnet_stat_result.lnet_max_sndperf < perf)
2287                 lnet_stat_result.lnet_max_sndperf = perf;
2288
2289         perf = (float)(lnet_new->lcc_recv_length -
2290                        lnet_old->lcc_recv_length) / unit_divisor / delta;
2291         lnet_stat_result.lnet_total_rcvperf += perf;
2292
2293         if (lnet_stat_result.lnet_min_rcvperf > perf ||
2294             lnet_stat_result.lnet_min_rcvperf == 0)
2295                 lnet_stat_result.lnet_min_rcvperf = perf;
2296
2297         if (lnet_stat_result.lnet_max_rcvperf < perf)
2298                 lnet_stat_result.lnet_max_rcvperf = perf;
2299
2300         rate = (lnet_new->lcc_send_count - lnet_old->lcc_send_count) / delta;
2301         lnet_stat_result.lnet_total_sndrate += rate;
2302
2303         if (lnet_stat_result.lnet_min_sndrate > rate ||
2304             lnet_stat_result.lnet_min_sndrate == 0)
2305                 lnet_stat_result.lnet_min_sndrate = rate;
2306
2307         if (lnet_stat_result.lnet_max_sndrate < rate)
2308                 lnet_stat_result.lnet_max_sndrate = rate;
2309
2310         rate = (lnet_new->lcc_recv_count - lnet_old->lcc_recv_count) / delta;
2311         lnet_stat_result.lnet_total_rcvrate += rate;
2312
2313         if (lnet_stat_result.lnet_min_rcvrate > rate ||
2314             lnet_stat_result.lnet_min_rcvrate == 0)
2315                 lnet_stat_result.lnet_min_rcvrate = rate;
2316
2317         if (lnet_stat_result.lnet_max_rcvrate < rate)
2318                 lnet_stat_result.lnet_max_rcvrate = rate;
2319
2320         lnet_stat_result.lnet_stat_count++;
2321
2322         lnet_stat_result.lnet_avg_sndrate = lnet_stat_result.lnet_total_sndrate /
2323                                             lnet_stat_result.lnet_stat_count;
2324         lnet_stat_result.lnet_avg_rcvrate = lnet_stat_result.lnet_total_rcvrate /
2325                                             lnet_stat_result.lnet_stat_count;
2326
2327         lnet_stat_result.lnet_avg_sndperf = lnet_stat_result.lnet_total_sndperf /
2328                                             lnet_stat_result.lnet_stat_count;
2329         lnet_stat_result.lnet_avg_rcvperf = lnet_stat_result.lnet_total_rcvperf /
2330                                             lnet_stat_result.lnet_stat_count;
2331 }
2332
2333 static void
2334 lst_print_lnet_stat(char *name, int bwrt, int rdwr, int type, int mbs)
2335 {
2336         int start1 = 0;
2337         int end1   = 1;
2338         int start2 = 0;
2339         int end2   = 1;
2340         int i;
2341         int j;
2342         char *units;
2343
2344         if (lnet_stat_result.lnet_stat_count == 0)
2345                 return;
2346
2347         units = (mbs) ? "MB/s  " : "MiB/s ";
2348
2349         if (bwrt == 1) /* bw only */
2350                 start1 = 1;
2351
2352         if (bwrt == 2) /* rates only */
2353                 end1 = 0;
2354
2355         if (rdwr == 1) /* recv only */
2356                 start2 = 1;
2357
2358         if (rdwr == 2) /* send only */
2359                 end2 = 0;
2360
2361         for (i = start1; i <= end1; i++) {
2362                 fprintf(stdout, "[LNet %s of %s]\n",
2363                         i == 0 ? "Rates" : "Bandwidth", name);
2364
2365                 for (j = start2; j <= end2; j++) {
2366                         fprintf(stdout, "[%c] ", j == 0 ? 'R' : 'W');
2367
2368                         if ((type & 1) != 0) {
2369                                 fprintf(stdout, i == 0 ? "Avg: %-8.0f RPC/s " :
2370                                                          "Avg: %-8.2f %s",
2371                                         lst_lnet_stat_value(i, j, 0), units);
2372                         }
2373
2374                         if ((type & 2) != 0) {
2375                                 fprintf(stdout, i == 0 ? "Min: %-8.0f RPC/s " :
2376                                                          "Min: %-8.2f %s",
2377                                         lst_lnet_stat_value(i, j, 1), units);
2378                         }
2379
2380                         if ((type & 4) != 0) {
2381                                 fprintf(stdout, i == 0 ? "Max: %-8.0f RPC/s" :
2382                                                          "Max: %-8.2f %s",
2383                                         lst_lnet_stat_value(i, j, 2), units);
2384                         }
2385
2386                         fprintf(stdout, "\n");
2387                 }
2388         }
2389 }
2390
2391 static void
2392 lst_print_stat(char *name, struct list_head *resultp,
2393                int idx, int lnet, int bwrt, int rdwr, int type,
2394                int mbs)
2395 {
2396         struct list_head tmp[2];
2397         struct lstcon_rpc_ent *new;
2398         struct lstcon_rpc_ent *old;
2399         struct sfw_counters *sfwk_new;
2400         struct sfw_counters *sfwk_old;
2401         struct srpc_counters *srpc_new;
2402         struct srpc_counters *srpc_old;
2403         struct lnet_counters_common *lnet_new;
2404         struct lnet_counters_common *lnet_old;
2405         float delta;
2406         int errcount = 0;
2407
2408         INIT_LIST_HEAD(&tmp[0]);
2409         INIT_LIST_HEAD(&tmp[1]);
2410
2411         memset(&lnet_stat_result, 0, sizeof(lnet_stat_result));
2412
2413         while (!list_empty(&resultp[idx])) {
2414                 if (list_empty(&resultp[1 - idx])) {
2415                         fprintf(stderr, "Group is changed, re-run stat\n");
2416                         break;
2417                 }
2418
2419                 new = list_first_entry(&resultp[idx], struct lstcon_rpc_ent,
2420                                        rpe_link);
2421                 old = list_first_entry(&resultp[1 - idx], struct lstcon_rpc_ent,
2422                                        rpe_link);
2423
2424                 /* first time get stats result, can't calculate diff */
2425                 if (new->rpe_peer.nid == LNET_NID_ANY)
2426                         break;
2427
2428                 if (new->rpe_peer.nid != old->rpe_peer.nid ||
2429                     new->rpe_peer.pid != old->rpe_peer.pid) {
2430                         /* Something wrong. i.e, somebody change the group */
2431                         break;
2432                 }
2433
2434                 list_move_tail(&new->rpe_link, &tmp[idx]);
2435
2436                 list_move_tail(&old->rpe_link, &tmp[1 - idx]);
2437
2438                 if (new->rpe_rpc_errno != 0 || new->rpe_fwk_errno != 0 ||
2439                     old->rpe_rpc_errno != 0 || old->rpe_fwk_errno != 0) {
2440                         errcount++;
2441                         continue;
2442                 }
2443
2444                 sfwk_new = (struct sfw_counters *)&new->rpe_payload[0];
2445                 sfwk_old = (struct sfw_counters *)&old->rpe_payload[0];
2446
2447                 srpc_new = (struct srpc_counters *)((char *)sfwk_new +
2448                                                     sizeof(*sfwk_new));
2449                 srpc_old = (struct srpc_counters *)((char *)sfwk_old +
2450                                                     sizeof(*sfwk_old));
2451
2452                 lnet_new = (struct lnet_counters_common *)((char *)srpc_new +
2453                                                            sizeof(*srpc_new));
2454                 lnet_old = (struct lnet_counters_common *)((char *)srpc_old +
2455                                                            sizeof(*srpc_old));
2456
2457                 /* Prior to version 2.3, the running_ms was a counter for
2458                  * the number of running tests. Since 2.3, running_ms is
2459                  * changed to hold the millisecond since the start of
2460                  * the work item. The rpe_stamp field was formerly used,
2461                  * but is no longer. In 2.12 rpe_stamp was changed to
2462                  * struct timespec64 and has nanosecond resolution, in
2463                  * case it is needed in the future.
2464                  */
2465                 delta = (float)(sfwk_new->running_ms -
2466                                 sfwk_old->running_ms) / 1000;
2467
2468                 if (!lnet) /* TODO */
2469                         continue;
2470
2471                 lst_cal_lnet_stat(delta, lnet_new, lnet_old, mbs);
2472         }
2473
2474         list_splice(&tmp[idx], &resultp[idx]);
2475         list_splice(&tmp[1 - idx], &resultp[1 - idx]);
2476
2477         if (errcount > 0)
2478                 fprintf(stdout, "Failed to stat on %d nodes\n", errcount);
2479
2480         if (!lnet)  /* TODO */
2481                 return;
2482
2483         lst_print_lnet_stat(name, bwrt, rdwr, type, mbs);
2484 }
2485
2486 static int
2487 jt_lst_stat(int argc, char **argv)
2488 {
2489         struct list_head head;
2490         lst_stat_req_param_t *srp;
2491         time_t last    = 0;
2492         int optidx = 0;
2493         int timeout = 5; /* default timeout, 5 sec */
2494         int delay = 5; /* default delay, 5 sec */
2495         int count = -1; /* run forever */
2496         int lnet = 1; /* lnet stat by default */
2497         int bwrt = 0;
2498         int rdwr = 0;
2499         int type = -1;
2500         int idx = 0;
2501         int mbs = 0; /* report as MB/s */
2502         int rc, c;
2503
2504         static const struct option stat_opts[] = {
2505                 { .name = "timeout", .has_arg = required_argument, .val = 't' },
2506                 { .name = "delay",   .has_arg = required_argument, .val = 'd' },
2507                 { .name = "count",   .has_arg = required_argument, .val = 'o' },
2508                 { .name = "lnet",    .has_arg = no_argument,       .val = 'l' },
2509                 { .name = "rpc",     .has_arg = no_argument,       .val = 'c' },
2510                 { .name = "bw",      .has_arg = no_argument,       .val = 'b' },
2511                 { .name = "rate",    .has_arg = no_argument,       .val = 'a' },
2512                 { .name = "read",    .has_arg = no_argument,       .val = 'r' },
2513                 { .name = "write",   .has_arg = no_argument,       .val = 'w' },
2514                 { .name = "avg",     .has_arg = no_argument,       .val = 'g' },
2515                 { .name = "min",     .has_arg = no_argument,       .val = 'n' },
2516                 { .name = "max",     .has_arg = no_argument,       .val = 'x' },
2517                 { .name = "mbs",     .has_arg = no_argument,       .val = 'm' },
2518                 { .name = NULL } };
2519
2520         if (session_key == 0) {
2521                 fprintf(stderr,
2522                         "Can't find env LST_SESSION or value is not valid\n");
2523                 return -1;
2524         }
2525
2526         while (1) {
2527                 c = getopt_long(argc, argv, "t:d:lcbarwgnxm", stat_opts,
2528                                 &optidx);
2529
2530                 if (c == -1)
2531                         break;
2532
2533                 switch (c) {
2534                 case 't':
2535                         timeout = atoi(optarg);
2536                         break;
2537                 case 'd':
2538                         delay = atoi(optarg);
2539                         break;
2540                 case 'o':
2541                         count = atoi(optarg);
2542                         break;
2543                 case 'l':
2544                         lnet = 1;
2545                         break;
2546                 case 'c':
2547                         lnet = 0;
2548                         break;
2549                 case 'b':
2550                         bwrt |= 1;
2551                         break;
2552                 case 'a':
2553                         bwrt |= 2;
2554                         break;
2555                 case 'r':
2556                         rdwr |= 1;
2557                         break;
2558                 case 'w':
2559                         rdwr |= 2;
2560                         break;
2561                 case 'g':
2562                         if (type == -1) {
2563                                 type = 1;
2564                                 break;
2565                         }
2566                         type |= 1;
2567                         break;
2568                 case 'n':
2569                         if (type == -1) {
2570                                 type = 2;
2571                                 break;
2572                         }
2573                         type |= 2;
2574                         break;
2575                 case 'x':
2576                         if (type == -1) {
2577                                 type = 4;
2578                                 break;
2579                         }
2580                         type |= 4;
2581                         break;
2582                 case 'm':
2583                         mbs = 1;
2584                         break;
2585
2586                 default:
2587                         lst_print_usage(argv[0]);
2588                         return -1;
2589                 }
2590         }
2591
2592         if (optind == argc) {
2593                 lst_print_usage(argv[0]);
2594                 return -1;
2595         }
2596
2597         if (timeout <= 0 || delay <= 0) {
2598                 fprintf(stderr, "Invalid timeout or delay value\n");
2599                 return -1;
2600         }
2601
2602         if (count < -1) {
2603                 fprintf(stderr, "Invalid count value\n");
2604                 return -1;
2605         }
2606
2607         /* extra count to get first data point */
2608         if (count != -1)
2609                 count++;
2610
2611         INIT_LIST_HEAD(&head);
2612
2613         while (optind < argc) {
2614                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 1);
2615                 if (rc != 0)
2616                         goto out;
2617
2618                 list_add_tail(&srp->srp_link, &head);
2619         }
2620
2621         do {
2622                 time_t  now = time(NULL);
2623
2624                 if (now - last < delay) {
2625                         sleep(delay - now + last);
2626                         time(&now);
2627                 }
2628                 last = now;
2629
2630                 list_for_each_entry(srp, &head, srp_link) {
2631                         rc = lst_stat_ioctl(srp->srp_name, srp->srp_count,
2632                                             srp->srp_ids, timeout,
2633                                             &srp->srp_result[idx]);
2634                         if (rc == -1) {
2635                                 lst_print_error("stat",
2636                                                 "Failed to stat %s: %s\n",
2637                                                 srp->srp_name, strerror(errno));
2638                                 goto out;
2639                         }
2640
2641                         lst_print_stat(srp->srp_name, srp->srp_result,
2642                                        idx, lnet, bwrt, rdwr, type, mbs);
2643
2644                         lst_reset_rpcent(&srp->srp_result[1 - idx]);
2645                 }
2646
2647                 idx = 1 - idx;
2648
2649                 if (count > 0)
2650                         count--;
2651         } while (count == -1 || count > 0);
2652
2653 out:
2654         while (!list_empty(&head)) {
2655                 srp = list_first_entry(&head, lst_stat_req_param_t, srp_link);
2656
2657                 list_del(&srp->srp_link);
2658                 lst_stat_req_param_free(srp);
2659         }
2660
2661         return rc;
2662 }
2663
2664 static int
2665 jt_lst_show_error(int argc, char **argv)
2666 {
2667         struct list_head head;
2668         lst_stat_req_param_t *srp;
2669         struct lstcon_rpc_ent *ent;
2670         struct sfw_counters *sfwk;
2671         struct srpc_counters *srpc;
2672         int show_rpc = 1;
2673         int optidx = 0;
2674         int rc = 0;
2675         int ecount, c;
2676
2677         static const struct option show_error_opts[] = {
2678                 { .name = "session", .has_arg = no_argument, .val = 's' },
2679                 { .name = NULL, } };
2680
2681         if (session_key == 0) {
2682                 fprintf(stderr,
2683                         "Can't find env LST_SESSION or value is not valid\n");
2684                 return -1;
2685         }
2686
2687         while (1) {
2688                 c = getopt_long(argc, argv, "s", show_error_opts, &optidx);
2689
2690                 if (c == -1)
2691                         break;
2692
2693                 switch (c) {
2694                 case 's':
2695                         show_rpc  = 0;
2696                         break;
2697
2698                 default:
2699                         lst_print_usage(argv[0]);
2700                         return -1;
2701                 }
2702         }
2703
2704         if (optind == argc) {
2705                 lst_print_usage(argv[0]);
2706                 return -1;
2707         }
2708
2709         INIT_LIST_HEAD(&head);
2710
2711         while (optind < argc) {
2712                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 0);
2713                 if (rc != 0)
2714                         goto out;
2715
2716                 list_add_tail(&srp->srp_link, &head);
2717         }
2718
2719         list_for_each_entry(srp, &head, srp_link) {
2720                 rc = lst_stat_ioctl(srp->srp_name, srp->srp_count,
2721                                     srp->srp_ids, 10, &srp->srp_result[0]);
2722
2723                 if (rc == -1) {
2724                         lst_print_error(srp->srp_name,
2725                                         "Failed to show errors of %s: %s\n",
2726                                         srp->srp_name, strerror(errno));
2727                         goto out;
2728                 }
2729
2730                 fprintf(stdout, "%s:\n", srp->srp_name);
2731
2732                 ecount = 0;
2733
2734                 list_for_each_entry(ent, &srp->srp_result[0], rpe_link) {
2735                         if (ent->rpe_rpc_errno != 0) {
2736                                 ecount++;
2737                                 fprintf(stderr,
2738                                         "RPC failure, can't show error on %s\n",
2739                                         libcfs_id2str(ent->rpe_peer));
2740                                 continue;
2741                         }
2742
2743                         if (ent->rpe_fwk_errno != 0) {
2744                                 ecount++;
2745                                 fprintf(stderr,
2746                                         "Framework failure, can't show error on %s\n",
2747                                         libcfs_id2str(ent->rpe_peer));
2748                                 continue;
2749                         }
2750
2751                         sfwk = (struct sfw_counters *)&ent->rpe_payload[0];
2752                         srpc = (struct srpc_counters *)((char *)sfwk +
2753                                 sizeof(*sfwk));
2754
2755                         if (srpc->errors == 0 &&
2756                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2757                                 continue;
2758
2759                         if (!show_rpc &&
2760                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2761                                 continue;
2762
2763                         ecount++;
2764
2765                         fprintf(stderr,
2766                                 "%s: [Session %d brw errors, %d ping errors]%c",
2767                                 libcfs_id2str(ent->rpe_peer),
2768                                 sfwk->brw_errors, sfwk->ping_errors,
2769                                 show_rpc  ? ' ' : '\n');
2770
2771                         if (!show_rpc)
2772                                 continue;
2773
2774                         fprintf(stderr,
2775                                 "[RPC: %d errors, %d dropped, %d expired]\n",
2776                                 srpc->errors, srpc->rpcs_dropped,
2777                                 srpc->rpcs_expired);
2778                 }
2779
2780                 fprintf(stdout, "Total %d error nodes in %s\n", ecount,
2781                         srp->srp_name);
2782         }
2783 out:
2784         while (!list_empty(&head)) {
2785                 srp = list_first_entry(&head, lst_stat_req_param_t, srp_link);
2786
2787                 list_del(&srp->srp_link);
2788                 lst_stat_req_param_free(srp);
2789         }
2790
2791         return rc;
2792 }
2793
2794 static int
2795 lst_add_batch_ioctl(char *name)
2796 {
2797         struct lstio_batch_add_args args = { 0 };
2798
2799         args.lstio_bat_key   = session_key;
2800         args.lstio_bat_nmlen = strlen(name);
2801         args.lstio_bat_namep = name;
2802
2803         return lst_ioctl(LSTIO_BATCH_ADD, &args, sizeof(args));
2804 }
2805
2806 static int
2807 jt_lst_add_batch(int argc, char **argv)
2808 {
2809         char *name;
2810         int rc;
2811
2812         if (session_key == 0) {
2813                 fprintf(stderr,
2814                         "Can't find env LST_SESSION or value is not valid\n");
2815                 return -1;
2816         }
2817
2818         if (argc != 2) {
2819                 lst_print_usage(argv[0]);
2820                 return -1;
2821         }
2822
2823         name = argv[1];
2824         if (strlen(name) >= LST_NAME_SIZE) {
2825                 fprintf(stderr, "Name length is limited to %d\n",
2826                         LST_NAME_SIZE - 1);
2827                 return -1;
2828         }
2829
2830         rc = lst_add_batch_ioctl(name);
2831         if (rc == 0)
2832                 return 0;
2833
2834         lst_print_error("batch", "Failed to create batch: %s\n",
2835                         strerror(errno));
2836
2837         return -1;
2838 }
2839
2840 static int
2841 lst_start_batch_ioctl(char *name, int timeout, struct list_head *resultp)
2842 {
2843         struct lstio_batch_run_args args = { 0 };
2844
2845         args.lstio_bat_key     = session_key;
2846         args.lstio_bat_timeout = timeout;
2847         args.lstio_bat_nmlen   = strlen(name);
2848         args.lstio_bat_namep   = name;
2849         args.lstio_bat_resultp = resultp;
2850
2851         return lst_ioctl(LSTIO_BATCH_START, &args, sizeof(args));
2852 }
2853
2854 static int
2855 jt_lst_start_batch(int argc, char **argv)
2856 {
2857         struct list_head head;
2858         char *batch;
2859         int optidx = 0;
2860         int timeout = 0;
2861         int count = 0;
2862         int rc;
2863         int c;
2864
2865         static const struct option start_batch_opts[] = {
2866                 { .name = "timeout", .has_arg = required_argument, .val = 't' },
2867                 { .name = NULL } };
2868
2869         if (session_key == 0) {
2870                 fprintf(stderr,
2871                         "Can't find env LST_SESSION or value is not valid\n");
2872                 return -1;
2873         }
2874
2875         while (1) {
2876                 c = getopt_long(argc, argv, "t:",
2877                                 start_batch_opts, &optidx);
2878
2879                 /* Detect the end of the options. */
2880                 if (c == -1)
2881                         break;
2882
2883                 switch (c) {
2884                 case 't':
2885                         timeout = atoi(optarg);
2886                         break;
2887                 default:
2888                         lst_print_usage(argv[0]);
2889                         return -1;
2890                 }
2891         }
2892
2893         if (optind == argc) {
2894                 batch = LST_DEFAULT_BATCH;
2895
2896         } else if (optind == argc - 1) {
2897                 batch = argv[optind];
2898
2899         } else {
2900                 lst_print_usage(argv[0]);
2901                 return -1;
2902         }
2903
2904         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2905         if (rc != 0) {
2906                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2907                         batch, strerror(errno));
2908                 return -1;
2909         }
2910
2911         INIT_LIST_HEAD(&head);
2912
2913         rc = lst_alloc_rpcent(&head, count, 0);
2914         if (rc != 0) {
2915                 fprintf(stderr, "Out of memory\n");
2916                 return -1;
2917         }
2918
2919         rc = lst_start_batch_ioctl(batch, timeout, &head);
2920
2921         if (rc == 0) {
2922                 fprintf(stdout, "%s is running now\n", batch);
2923                 lst_free_rpcent(&head);
2924                 return 0;
2925         }
2926
2927         if (rc == -1) {
2928                 lst_print_error("batch", "Failed to start batch: %s\n",
2929                                 strerror(errno));
2930                 lst_free_rpcent(&head);
2931                 return rc;
2932         }
2933
2934         lst_print_transerr(&head, "Run batch");
2935
2936         lst_free_rpcent(&head);
2937
2938         return rc;
2939 }
2940
2941 static int
2942 lst_stop_batch_ioctl(char *name, int force, struct list_head *resultp)
2943 {
2944         struct lstio_batch_stop_args args = { 0 };
2945
2946         args.lstio_bat_key     = session_key;
2947         args.lstio_bat_force   = force;
2948         args.lstio_bat_nmlen   = strlen(name);
2949         args.lstio_bat_namep   = name;
2950         args.lstio_bat_resultp = resultp;
2951
2952         return lst_ioctl(LSTIO_BATCH_STOP, &args, sizeof(args));
2953 }
2954
2955 static int
2956 jt_lst_stop_batch(int argc, char **argv)
2957 {
2958         struct list_head  head;
2959         char *batch;
2960         int force = 0;
2961         int optidx;
2962         int count;
2963         int rc, c;
2964
2965         static const struct option stop_batch_opts[] = {
2966                 { .name = "force", .has_arg = no_argument, .val = 'f' },
2967                 { .name = NULL } };
2968
2969         if (session_key == 0) {
2970                 fprintf(stderr,
2971                         "Can't find env LST_SESSION or value is not valid\n");
2972                 return -1;
2973         }
2974
2975         while (1) {
2976                 c = getopt_long(argc, argv, "f", stop_batch_opts, &optidx);
2977                 /* Detect the end of the options. */
2978                 if (c == -1)
2979                         break;
2980
2981                 switch (c) {
2982                 case 'f':
2983                         force = 1;
2984                         break;
2985                 default:
2986                         lst_print_usage(argv[0]);
2987                         return -1;
2988                 }
2989         }
2990
2991         if (optind == argc) {
2992                 batch = LST_DEFAULT_BATCH;
2993
2994         } else if (optind == argc - 1) {
2995                 batch = argv[optind];
2996
2997         } else {
2998                 lst_print_usage(argv[0]);
2999                 return -1;
3000         }
3001
3002         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
3003         if (rc != 0) {
3004                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
3005                         batch, strerror(errno));
3006                 return -1;
3007         }
3008
3009         INIT_LIST_HEAD(&head);
3010
3011         rc = lst_alloc_rpcent(&head, count, 0);
3012         if (rc != 0) {
3013                 fprintf(stderr, "Out of memory\n");
3014                 return -1;
3015         }
3016
3017         rc = lst_stop_batch_ioctl(batch, force, &head);
3018         if (rc != 0)
3019                 goto out;
3020
3021         while (1) {
3022                 lst_reset_rpcent(&head);
3023
3024                 rc = lst_query_batch_ioctl(batch, 0, 0, 30, &head);
3025                 if (rc != 0)
3026                         goto out;
3027
3028                 if (lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0 &&
3029                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0)
3030                         break;
3031
3032                 fprintf(stdout, "%d batch in stopping\n",
3033                         lstcon_tsbqry_stat_run(&trans_stat, 0));
3034                 sleep(1);
3035         }
3036
3037         fprintf(stdout, "Batch is stopped\n");
3038         lst_free_rpcent(&head);
3039
3040         return 0;
3041 out:
3042         if (rc == -1) {
3043                 lst_print_error("batch", "Failed to stop batch: %s\n",
3044                                 strerror(errno));
3045                 lst_free_rpcent(&head);
3046                 return -1;
3047         }
3048
3049         lst_print_transerr(&head, "stop batch");
3050
3051         lst_free_rpcent(&head);
3052
3053         return rc;
3054 }
3055
3056 static int
3057 lst_list_batch_ioctl(int len, char *name, int index)
3058 {
3059         struct lstio_batch_list_args args = { 0 };
3060
3061         args.lstio_bat_key   = session_key;
3062         args.lstio_bat_idx   = index;
3063         args.lstio_bat_nmlen = len;
3064         args.lstio_bat_namep = name;
3065
3066         return lst_ioctl(LSTIO_BATCH_LIST, &args, sizeof(args));
3067 }
3068
3069 static int
3070 lst_info_batch_ioctl(char *batch, int test, int server,
3071                      struct lstcon_test_batch_ent *entp, int *idxp,
3072                      int *ndentp, struct lstcon_node_ent *dentsp)
3073 {
3074         struct lstio_batch_info_args args = { 0 };
3075
3076         args.lstio_bat_key     = session_key;
3077         args.lstio_bat_nmlen   = strlen(batch);
3078         args.lstio_bat_namep   = batch;
3079         args.lstio_bat_server  = server;
3080         args.lstio_bat_testidx = test;
3081         args.lstio_bat_entp    = entp;
3082         args.lstio_bat_idxp    = idxp;
3083         args.lstio_bat_ndentp  = ndentp;
3084         args.lstio_bat_dentsp  = dentsp;
3085
3086         return lst_ioctl(LSTIO_BATCH_INFO, &args, sizeof(args));
3087 }
3088
3089 static int
3090 lst_list_batch_all(void)
3091 {
3092         char name[LST_NAME_SIZE];
3093         int rc, i;
3094
3095         for (i = 0; ; i++) {
3096                 rc = lst_list_batch_ioctl(LST_NAME_SIZE, name, i);
3097                 if (rc == 0) {
3098                         fprintf(stdout, "%d) %s\n", i + 1, name);
3099                         continue;
3100                 }
3101
3102                 if (errno == ENOENT)
3103                         break;
3104
3105                 lst_print_error("batch", "Failed to list batch: %s\n",
3106                                 strerror(errno));
3107                 return rc;
3108         }
3109
3110         fprintf(stdout, "Total %d batches\n", i);
3111
3112         return 0;
3113 }
3114
3115 static int
3116 lst_list_tsb_nodes(char *batch, int test, int server,
3117                    int count, int active, int invalid)
3118 {
3119         struct lstcon_node_ent *dents;
3120         int index = 0;
3121         int rc, c, i;
3122
3123         if (count == 0)
3124                 return 0;
3125
3126         /* verbose list, show nodes in batch or test */
3127         dents = malloc(count * sizeof(struct lstcon_node_ent));
3128         if (dents == NULL) {
3129                 fprintf(stdout, "Can't allocate memory\n");
3130                 return -1;
3131         }
3132
3133         rc = lst_info_batch_ioctl(batch, test, server,
3134                                   NULL, &index, &count, dents);
3135         if (rc != 0) {
3136                 free(dents);
3137                 lst_print_error((test > 0) ? "test" : "batch",
3138                                 (test > 0) ? "Failed to query test: %s\n" :
3139                                  "Failed to query batch: %s\n",
3140                                 strerror(errno));
3141                 return -1;
3142         }
3143
3144         for (i = 0, c = 0; i < count; i++) {
3145                 if ((!active  && dents[i].nde_state == LST_NODE_ACTIVE) ||
3146                      (!invalid && (dents[i].nde_state == LST_NODE_BUSY  ||
3147                                    dents[i].nde_state == LST_NODE_DOWN  ||
3148                                    dents[i].nde_state == LST_NODE_UNKNOWN)))
3149                         continue;
3150
3151                 fprintf(stdout, "\t%s: %s\n",
3152                         libcfs_id2str(dents[i].nde_id),
3153                         lst_node_state2str(dents[i].nde_state));
3154                 c++;
3155         }
3156
3157         fprintf(stdout, "Total %d nodes\n", c);
3158         free(dents);
3159
3160         return 0;
3161 }
3162
3163 static int
3164 jt_lst_list_batch(int argc, char **argv)
3165 {
3166         struct lstcon_test_batch_ent ent;
3167         char *batch   = NULL;
3168         int optidx = 0;
3169         int verbose = 0; /* list nodes in batch or test */
3170         int invalid = 0;
3171         int active = 0;
3172         int server = 0;
3173         int ntest = 0;
3174         int test = 0;
3175         int c = 0;
3176         int rc;
3177
3178         static const struct option list_batch_opts[] = {
3179                 { .name = "test",    .has_arg = required_argument, .val = 't' },
3180                 { .name = "invalid", .has_arg = no_argument,       .val = 'i' },
3181                 { .name = "active",  .has_arg = no_argument,       .val = 'a' },
3182                 { .name = "all",     .has_arg = no_argument,       .val = 'l' },
3183                 { .name = "server",  .has_arg = no_argument,       .val = 's' },
3184                 { .name = NULL, } };
3185
3186         if (session_key == 0) {
3187                 fprintf(stderr,
3188                         "Can't find env LST_SESSION or value is not valid\n");
3189                 return -1;
3190         }
3191
3192         while (1) {
3193                 c = getopt_long(argc, argv, "ailst:", list_batch_opts, &optidx);
3194                 if (c == -1)
3195                         break;
3196
3197                 switch (c) {
3198                 case 'a':
3199                         verbose = active = 1;
3200                         break;
3201                 case 'i':
3202                         verbose = invalid = 1;
3203                         break;
3204                 case 'l':
3205                         verbose = active = invalid = 1;
3206                         break;
3207                 case 's':
3208                         server = 1;
3209                         break;
3210                 case 't':
3211                         test = atoi(optarg);
3212                         ntest = 1;
3213                         break;
3214                 default:
3215                         lst_print_usage(argv[0]);
3216                         return -1;
3217                 }
3218         }
3219
3220         if (optind == argc) {
3221                 /* list all batches */
3222                 rc = lst_list_batch_all();
3223                 return rc;
3224         }
3225
3226         if (ntest == 1 && test <= 0) {
3227                 fprintf(stderr, "Invalid test id, test id starts from 1\n");
3228                 return -1;
3229         }
3230
3231         if (optind != argc - 1) {
3232                 lst_print_usage(argv[0]);
3233                 return -1;
3234         }
3235
3236         batch = argv[optind];
3237
3238 loop:
3239         /* show detail of specified batch or test */
3240         rc = lst_info_batch_ioctl(batch, test, server, &ent, NULL, NULL, NULL);
3241         if (rc != 0) {
3242                 lst_print_error((test > 0) ? "test" : "batch",
3243                                 (test > 0) ? "Failed to query test: %s\n" :
3244                                  "Failed to query batch: %s\n",
3245                                 strerror(errno));
3246                 return -1;
3247         }
3248
3249         if (verbose) {
3250                 /* list nodes in test or batch */
3251                 rc = lst_list_tsb_nodes(batch, test, server,
3252                                         server ? ent.tbe_srv_nle.nle_nnode :
3253                                                  ent.tbe_cli_nle.nle_nnode,
3254                                         active, invalid);
3255                 return rc;
3256         }
3257
3258         /* only show number of hosts in batch or test */
3259         if (test == 0) {
3260                 fprintf(stdout, "Batch: %s Tests: %d State: %d\n",
3261                         batch, ent.u.tbe_batch.bae_ntest,
3262                         ent.u.tbe_batch.bae_state);
3263                 ntest = ent.u.tbe_batch.bae_ntest;
3264                 test = 1; /* starting from test 1 */
3265         } else {
3266                 fprintf(stdout,
3267                         "\tTest %d(%s) (loop: %d, concurrency: %d)\n",
3268                         test, lst_test_type2name(ent.u.tbe_test.tse_type),
3269                         ent.u.tbe_test.tse_loop,
3270                         ent.u.tbe_test.tse_concur);
3271                 ntest--;
3272                 test++;
3273         }
3274
3275         fprintf(stdout, LST_NODES_TITLE);
3276         fprintf(stdout, "client\t%d\t%d\t%d\t%d\t%d\n"
3277                         "server\t%d\t%d\t%d\t%d\t%d\n",
3278                 ent.tbe_cli_nle.nle_nactive,
3279                 ent.tbe_cli_nle.nle_nbusy,
3280                 ent.tbe_cli_nle.nle_ndown,
3281                 ent.tbe_cli_nle.nle_nunknown,
3282                 ent.tbe_cli_nle.nle_nnode,
3283                 ent.tbe_srv_nle.nle_nactive,
3284                 ent.tbe_srv_nle.nle_nbusy,
3285                 ent.tbe_srv_nle.nle_ndown,
3286                 ent.tbe_srv_nle.nle_nunknown,
3287                 ent.tbe_srv_nle.nle_nnode);
3288
3289         if (ntest != 0)
3290                 goto loop;
3291
3292         return 0;
3293 }
3294
3295 static int
3296 lst_query_batch_ioctl(char *batch, int test, int server,
3297                       int timeout, struct list_head *head)
3298 {
3299         struct lstio_batch_query_args args = { 0 };
3300
3301         args.lstio_bat_key     = session_key;
3302         args.lstio_bat_testidx = test;
3303         args.lstio_bat_client  = !(server);
3304         args.lstio_bat_timeout = timeout;
3305         args.lstio_bat_nmlen   = strlen(batch);
3306         args.lstio_bat_namep   = batch;
3307         args.lstio_bat_resultp = head;
3308
3309         return lst_ioctl(LSTIO_BATCH_QUERY, &args, sizeof(args));
3310 }
3311
3312 static void
3313 lst_print_tsb_verbose(struct list_head *head, int active, int idle, int error)
3314 {
3315         struct lstcon_rpc_ent *ent;
3316
3317         list_for_each_entry(ent, head, rpe_link) {
3318                 if (ent->rpe_priv[0] == 0 && active)
3319                         continue;
3320
3321                 if (ent->rpe_priv[0] != 0 && idle)
3322                         continue;
3323
3324                 if (ent->rpe_fwk_errno == 0 && error)
3325                         continue;
3326
3327                 fprintf(stdout, "%s [%s]: %s\n",
3328                         libcfs_id2str(ent->rpe_peer),
3329                         lst_node_state2str(ent->rpe_state),
3330                         ent->rpe_rpc_errno != 0 ?
3331                                 strerror(ent->rpe_rpc_errno) :
3332                                 (ent->rpe_priv[0] > 0 ? "Running" : "Idle"));
3333         }
3334 }
3335
3336 static int
3337 jt_lst_query_batch(int argc, char **argv)
3338 {
3339         struct lstcon_test_batch_ent ent;
3340         struct list_head head;
3341         char *batch = NULL;
3342         time_t last = 0;
3343         int optidx  = 0;
3344         int verbose = 0;
3345         int server  = 0;
3346         int timeout = 5; /* default 5 seconds */
3347         int delay = 5; /* default 5 seconds */
3348         int loop = 1; /* default 1 loop */
3349         int active = 0;
3350         int error = 0;
3351         int idle = 0;
3352         int count = 0;
3353         int test = 0;
3354         int rc = 0;
3355         int c = 0;
3356         int i;
3357
3358         static const struct option query_batch_opts[] = {
3359                 { .name = "timeout", .has_arg = required_argument, .val = 'o' },
3360                 { .name = "delay",   .has_arg = required_argument, .val = 'd' },
3361                 { .name = "loop",    .has_arg = required_argument, .val = 'c' },
3362                 { .name = "test",    .has_arg = required_argument, .val = 't' },
3363                 { .name = "server",  .has_arg = no_argument,       .val = 's' },
3364                 { .name = "active",  .has_arg = no_argument,       .val = 'a' },
3365                 { .name = "idle",    .has_arg = no_argument,       .val = 'i' },
3366                 { .name = "error",   .has_arg = no_argument,       .val = 'e' },
3367                 { .name = "all",     .has_arg = no_argument,       .val = 'l' },
3368                 { .name = NULL, } };
3369
3370         if (session_key == 0) {
3371                 fprintf(stderr,
3372                         "Can't find env LST_SESSION or value is not valid\n");
3373                 return -1;
3374         }
3375
3376         while (1) {
3377                 c = getopt_long(argc, argv, "o:d:c:t:saiel",
3378                                 query_batch_opts, &optidx);
3379
3380                 /* Detect the end of the options. */
3381                 if (c == -1)
3382                         break;
3383
3384                 switch (c) {
3385                 case 'o':
3386                         timeout = atoi(optarg);
3387                         break;
3388                 case 'd':
3389                         delay = atoi(optarg);
3390                         break;
3391                 case 'c':
3392                         loop = atoi(optarg);
3393                         break;
3394                 case 't':
3395                         test = atoi(optarg);
3396                         break;
3397                 case 's':
3398                         server = 1;
3399                         break;
3400                 case 'a':
3401                         active = verbose = 1;
3402                         break;
3403                 case 'i':
3404                         idle = verbose = 1;
3405                         break;
3406                 case 'e':
3407                         error = verbose = 1;
3408                         break;
3409                 case 'l':
3410                         verbose = 1;
3411                         break;
3412                 default:
3413                         lst_print_usage(argv[0]);
3414                         return -1;
3415                 }
3416         }
3417
3418         if (test < 0 || timeout <= 0 || delay <= 0 || loop <= 0) {
3419                 lst_print_usage(argv[0]);
3420                 return -1;
3421         }
3422
3423         if (optind == argc) {
3424                 batch = LST_DEFAULT_BATCH;
3425         } else if (optind == argc - 1) {
3426                 batch = argv[optind];
3427         } else {
3428                 lst_print_usage(argv[0]);
3429                 return -1;
3430         }
3431
3432
3433         INIT_LIST_HEAD(&head);
3434
3435         if (verbose) {
3436                 rc = lst_info_batch_ioctl(batch, test, server,
3437                                           &ent, NULL, NULL, NULL);
3438                 if (rc != 0) {
3439                         fprintf(stderr, "Failed to query %s [%d]: %s\n",
3440                                 batch, test, strerror(errno));
3441                         return -1;
3442                 }
3443
3444                 count = server ? ent.tbe_srv_nle.nle_nnode :
3445                                  ent.tbe_cli_nle.nle_nnode;
3446                 if (count == 0) {
3447                         fprintf(stdout, "Batch or test is empty\n");
3448                         return 0;
3449                 }
3450         }
3451
3452         rc = lst_alloc_rpcent(&head, count, 0);
3453         if (rc != 0) {
3454                 fprintf(stderr, "Out of memory\n");
3455                 return rc;
3456         }
3457
3458         for (i = 0; i < loop; i++) {
3459                 time_t  now = time(NULL);
3460
3461                 if (now - last < delay) {
3462                         sleep(delay - now + last);
3463                         time(&now);
3464                 }
3465
3466                 last = now;
3467
3468                 rc = lst_query_batch_ioctl(batch, test, server, timeout, &head);
3469                 if (rc == -1) {
3470                         fprintf(stderr, "Failed to query batch: %s\n",
3471                                 strerror(errno));
3472                         break;
3473                 }
3474
3475                 if (verbose) {
3476                         /* Verbose mode */
3477                         lst_print_tsb_verbose(&head, active, idle, error);
3478                         continue;
3479                 }
3480
3481                 fprintf(stdout, "%s [%d] ", batch, test);
3482
3483                 if (lstcon_rpc_stat_failure(&trans_stat, 0) != 0) {
3484                         fprintf(stdout, "%d of %d nodes are unknown, ",
3485                                 lstcon_rpc_stat_failure(&trans_stat, 0),
3486                                 lstcon_rpc_stat_total(&trans_stat, 0));
3487                 }
3488
3489                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
3490                     lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0  &&
3491                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
3492                         fprintf(stdout, "is stopped\n");
3493                         continue;
3494                 }
3495
3496                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
3497                     lstcon_tsbqry_stat_idle(&trans_stat, 0) == 0 &&
3498                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
3499                         fprintf(stdout, "is running\n");
3500                         continue;
3501                 }
3502
3503                 fprintf(stdout, "stopped: %d , running: %d, failed: %d\n",
3504                                 lstcon_tsbqry_stat_idle(&trans_stat, 0),
3505                                 lstcon_tsbqry_stat_run(&trans_stat, 0),
3506                                 lstcon_tsbqry_stat_failure(&trans_stat, 0));
3507         }
3508
3509         lst_free_rpcent(&head);
3510
3511         return rc;
3512 }
3513
3514 static int
3515 lst_parse_distribute(char *dstr, int *dist, int *span)
3516 {
3517         *dist = atoi(dstr);
3518         if (*dist <= 0)
3519                 return -1;
3520
3521         dstr = strchr(dstr, ':');
3522         if (dstr == NULL)
3523                 return -1;
3524
3525         *span = atoi(dstr + 1);
3526         if (*span <= 0)
3527                 return -1;
3528
3529         return 0;
3530 }
3531
3532 static int
3533 lst_get_bulk_param(int argc, char **argv, struct lst_test_bulk_param *bulk)
3534 {
3535         char *tok = NULL;
3536         char *end = NULL;
3537         int rc = 0;
3538         int i = 0;
3539
3540         bulk->blk_size  = 4096;
3541         bulk->blk_opc   = LST_BRW_READ;
3542         bulk->blk_flags = LST_BRW_CHECK_NONE;
3543         bulk->blk_srv_off = bulk->blk_cli_off = 0;
3544
3545         while (i < argc) {
3546                 if (strcasestr(argv[i], "check=") == argv[i] ||
3547                     strcasestr(argv[i], "c=") == argv[i]) {
3548                         tok = strchr(argv[i], '=') + 1;
3549
3550                         if (strcasecmp(tok, "full") == 0) {
3551                                 bulk->blk_flags = LST_BRW_CHECK_FULL;
3552                         } else if (strcasecmp(tok, "simple") == 0) {
3553                                 bulk->blk_flags = LST_BRW_CHECK_SIMPLE;
3554                         } else {
3555                                 fprintf(stderr, "Unknow flag %s\n", tok);
3556                                 return -1;
3557                         }
3558
3559                 } else if (strcasestr(argv[i], "size=") == argv[i] ||
3560                            strcasestr(argv[i], "s=") == argv[i]) {
3561                         tok = strchr(argv[i], '=') + 1;
3562
3563                         bulk->blk_size = strtol(tok, &end, 0);
3564                         if (bulk->blk_size <= 0) {
3565                                 fprintf(stderr, "Invalid size %s\n", tok);
3566                                 return -1;
3567                         }
3568
3569                         if (end == NULL)
3570                                 return 0;
3571
3572                         if (*end == 'k' || *end == 'K')
3573                                 bulk->blk_size *= 1024;
3574                         else if (*end == 'm' || *end == 'M')
3575                                 bulk->blk_size *= 1024 * 1024;
3576
3577                         if (bulk->blk_size > LNET_MTU) {
3578                                 fprintf(stderr, "Size exceed limitation: %d bytes\n",
3579                                         bulk->blk_size);
3580                                 return -1;
3581                         }
3582
3583                 } else if (strcasestr(argv[i], "off=") == argv[i]) {
3584                         int off;
3585
3586                         tok = strchr(argv[i], '=') + 1;
3587
3588                         off = strtol(tok, &end, 0);
3589                         /* NB: align with sizeof(__u64) to simplify page
3590                          * checking implementation
3591                          */
3592                         if (off < 0 || off % sizeof(__u64) != 0) {
3593                                 fprintf(stderr,
3594                                         "Invalid offset %s/%d, it should be postive value and multiple of %d\n",
3595                                         tok, off, (int)sizeof(__u64));
3596                                 return -1;
3597                         }
3598
3599                         /* NB: blk_srv_off is reserved so far */
3600                         bulk->blk_cli_off = bulk->blk_srv_off = off;
3601                         if (end == NULL)
3602                                 return 0;
3603
3604                 } else if (strcasecmp(argv[i], "read") == 0 ||
3605                            strcasecmp(argv[i], "r") == 0) {
3606                         bulk->blk_opc = LST_BRW_READ;
3607
3608                 } else if (strcasecmp(argv[i], "write") == 0 ||
3609                            strcasecmp(argv[i], "w") == 0) {
3610                         bulk->blk_opc = LST_BRW_WRITE;
3611
3612                 } else {
3613                         fprintf(stderr, "Unknow parameter: %s\n", argv[i]);
3614                         return -1;
3615                 }
3616
3617                 i++;
3618         }
3619
3620         return rc;
3621 }
3622
3623 static int
3624 lst_get_test_param(char *test, int argc, char **argv, void **param, int *plen)
3625 {
3626         struct lst_test_bulk_param *bulk = NULL;
3627         struct lst_test_ping_param *ping = NULL;
3628         int type;
3629
3630         type = lst_test_name2type(test);
3631         if (type < 0) {
3632                 fprintf(stderr, "Unknow test name %s\n", test);
3633                 return -1;
3634         }
3635
3636         switch (type) {
3637         case LST_TEST_PING:
3638                 /* unused but needs for kernel part */
3639                 ping = malloc(sizeof(*ping));
3640                 if (ping == NULL) {
3641                         fprintf(stderr, "Out of memory\n");
3642                         return -1;
3643                 }
3644                 memset(ping, 0, sizeof(*ping));
3645
3646                 *param = ping;
3647                 *plen  = sizeof(*ping);
3648
3649                 break;
3650
3651         case LST_TEST_BULK:
3652                 bulk = malloc(sizeof(*bulk));
3653                 if (bulk == NULL) {
3654                         fprintf(stderr, "Out of memory\n");
3655                         return -1;
3656                 }
3657
3658                 memset(bulk, 0, sizeof(*bulk));
3659
3660                 if (lst_get_bulk_param(argc, argv, bulk) != 0) {
3661                         free(bulk);
3662                         return -1;
3663                 }
3664                 *param = bulk;
3665                 *plen  = sizeof(*bulk);
3666                 break;
3667         default:
3668                 break;
3669         }
3670
3671         /* TODO: parse more parameter */
3672         return type;
3673 }
3674
3675 static int
3676 lst_add_test_ioctl(char *batch, int type, int loop, int concur,
3677                    int dist, int span, char *sgrp, char *dgrp,
3678                    void *param, int plen, int *retp, struct list_head *resultp)
3679 {
3680         struct lstio_test_args args = { 0 };
3681
3682         args.lstio_tes_key        = session_key;
3683         args.lstio_tes_bat_nmlen  = strlen(batch);
3684         args.lstio_tes_bat_name   = batch;
3685         args.lstio_tes_type       = type;
3686         args.lstio_tes_oneside    = 0;
3687         args.lstio_tes_loop       = loop;
3688         args.lstio_tes_concur     = concur;
3689         args.lstio_tes_dist       = dist;
3690         args.lstio_tes_span       = span;
3691         args.lstio_tes_sgrp_nmlen = strlen(sgrp);
3692         args.lstio_tes_sgrp_name  = sgrp;
3693         args.lstio_tes_dgrp_nmlen = strlen(dgrp);
3694         args.lstio_tes_dgrp_name  = dgrp;
3695         args.lstio_tes_param_len  = plen;
3696         args.lstio_tes_param      = param;
3697         args.lstio_tes_retp       = retp;
3698         args.lstio_tes_resultp    = resultp;
3699
3700         return lst_ioctl(LSTIO_TEST_ADD, &args, sizeof(args));
3701 }
3702
3703 static int
3704 jt_lst_add_test(int argc, char **argv)
3705 {
3706         struct list_head head;
3707         char *batch = NULL;
3708         char *test = NULL;
3709         char *dstr = NULL;
3710         char *from = NULL;
3711         char *to = NULL;
3712         void *param = NULL;
3713         int optidx = 0;
3714         int concur = 1;
3715         int loop = -1;
3716         int dist = 1;
3717         int span = 1;
3718         int plen = 0;
3719         int fcount = 0;
3720         int tcount = 0;
3721         int ret = 0;
3722         int type;
3723         int rc;
3724         int c;
3725
3726         static const struct option add_test_opts[] = {
3727         { .name = "batch",       .has_arg = required_argument, .val = 'b' },
3728         { .name = "concurrency", .has_arg = required_argument, .val = 'c' },
3729         { .name = "distribute",  .has_arg = required_argument, .val = 'd' },
3730         { .name = "from",        .has_arg = required_argument, .val = 'f' },
3731         { .name = "to",          .has_arg = required_argument, .val = 't' },
3732         { .name = "loop",        .has_arg = required_argument, .val = 'l' },
3733         { .name = NULL } };
3734
3735         if (session_key == 0) {
3736                 fprintf(stderr,
3737                         "Can't find env LST_SESSION or value is not valid\n");
3738                 return -1;
3739         }
3740
3741         while (1) {
3742                 c = getopt_long(argc, argv, "b:c:d:f:l:t:",
3743                                 add_test_opts, &optidx);
3744
3745                 /* Detect the end of the options. */
3746                 if (c == -1)
3747                         break;
3748
3749                 switch (c) {
3750                 case 'b':
3751                         batch = optarg;
3752                         break;
3753                 case 'c':
3754                         concur = atoi(optarg);
3755                         break;
3756                 case 'd':
3757                         dstr = optarg;
3758                         break;
3759                 case 'f':
3760                         from = optarg;
3761                         break;
3762                 case 'l':
3763                         loop = atoi(optarg);
3764                         break;
3765                 case 't':
3766                         to = optarg;
3767                         break;
3768                 default:
3769                         lst_print_usage(argv[0]);
3770                         return -1;
3771                 }
3772         }
3773
3774         if (optind == argc || from == NULL || to == NULL) {
3775                 lst_print_usage(argv[0]);
3776                 return -1;
3777         }
3778
3779         if (concur <= 0 || concur > LST_MAX_CONCUR) {
3780                 fprintf(stderr, "Invalid concurrency of test: %d\n", concur);
3781                 return -1;
3782         }
3783
3784         if (batch == NULL)
3785                 batch = LST_DEFAULT_BATCH;
3786
3787         if (dstr != NULL) {
3788                 rc = lst_parse_distribute(dstr, &dist, &span);
3789                 if (rc != 0) {
3790                         fprintf(stderr, "Invalid distribution: %s\n", dstr);
3791                         return -1;
3792                 }
3793         }
3794
3795         test = argv[optind++];
3796
3797         argc -= optind;
3798         argv += optind;
3799
3800         type = lst_get_test_param(test, argc, argv, &param, &plen);
3801         if (type < 0) {
3802                 fprintf(stderr, "Failed to add test (%s)\n", test);
3803                 return -1;
3804         }
3805
3806         INIT_LIST_HEAD(&head);
3807
3808         rc = lst_get_node_count(LST_OPC_GROUP, from, &fcount, NULL);
3809         if (rc != 0) {
3810                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3811                         from, strerror(errno));
3812                 goto out;
3813         }
3814
3815         rc = lst_get_node_count(LST_OPC_GROUP, to, &tcount, NULL);
3816         if (rc != 0) {
3817                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3818                         to, strerror(errno));
3819                 goto out;
3820         }
3821
3822         rc = lst_alloc_rpcent(&head, fcount > tcount ? fcount : tcount, 0);
3823         if (rc != 0) {
3824                 fprintf(stderr, "Out of memory\n");
3825                 goto out;
3826         }
3827
3828         rc = lst_add_test_ioctl(batch, type, loop, concur,
3829                                 dist, span, from, to, param, plen, &ret, &head);
3830
3831         if (rc == 0) {
3832                 fprintf(stdout, "Test was added successfully\n");
3833                 if (ret != 0) {
3834                         fprintf(stdout,
3835                                 "Server group contains userland test nodes, old version of tcplnd can't accept connection request\n");
3836                 }
3837
3838                 goto out;
3839         }
3840
3841         if (rc == -1) {
3842                 lst_print_error("test", "Failed to add test: %s\n",
3843                                 strerror(errno));
3844                 goto out;
3845         }
3846
3847         lst_print_transerr(&head, "add test");
3848 out:
3849         lst_free_rpcent(&head);
3850
3851         if (param != NULL)
3852                 free(param);
3853
3854         return rc;
3855 }
3856
3857 static command_t lst_cmdlist[] = {
3858         {"new_session", jt_lst_new_session, NULL,
3859          "Usage: lst new_session [--timeout TIME] [--force] [NAME]"},
3860         {"end_session", jt_lst_end_session, NULL,
3861          "Usage: lst end_session" },
3862         {"show_session", jt_lst_show_session, NULL,
3863          "Usage: lst show_session" },
3864         {"ping", jt_lst_ping, NULL,
3865          "Usage: lst ping  [--group NAME] [--batch NAME] [--session] "
3866          " [--nodes IDS]" },
3867         {"add_group", jt_lst_add_group, NULL,
3868          "Usage: lst group NAME IDs [IDs]..." },
3869         {"del_group", jt_lst_del_group, NULL,
3870          "Usage: lst del_group NAME" },
3871         {"update_group", jt_lst_update_group, NULL,
3872          "Usage: lst update_group NAME [--clean] [--refresh] [--remove IDs]" },
3873         {"list_group", jt_lst_list_group, NULL,
3874          "Usage: lst list_group [--active] [--busy] [--down] [--unknown] "
3875          "GROUP ..." },
3876         {"stat", jt_lst_stat, NULL,
3877          "Usage: lst stat [--bw] [--rate] [--read] [--write] [--max] [--min] "
3878          " [--avg] [--mbs] [--timeout #] [--delay #] [--count #] GROUP [GROUP]"
3879         },
3880         {"show_error", jt_lst_show_error, NULL,
3881          "Usage: lst show_error NAME | IDS ..." },
3882         {"add_batch", jt_lst_add_batch, NULL,
3883          "Usage: lst add_batch NAME" },
3884         {"run", jt_lst_start_batch, NULL,
3885          "Usage: lst run [--timeout TIME] [NAME]" },
3886         {"stop", jt_lst_stop_batch, NULL,
3887          "Usage: lst stop [--force] BATCH_NAME" },
3888         {"list_batch", jt_lst_list_batch, NULL,
3889          "Usage: lst list_batch NAME [--test ID] [--server]" },
3890         {"query", jt_lst_query_batch, NULL,
3891          "Usage: lst query [--test ID] [--server] [--timeout TIME] NAME" },
3892         {"add_test", jt_lst_add_test, NULL,
3893          "Usage: lst add_test [--batch BATCH] [--loop #] [--concurrency #] "
3894          " [--distribute #:#] [--from GROUP] [--to GROUP] TEST..." },
3895         {0, 0, 0, NULL }
3896 };
3897
3898 static int
3899 lst_initialize(void)
3900 {
3901         char *key;
3902         char *feats;
3903
3904         feats = getenv("LST_FEATURES");
3905         if (feats != NULL)
3906                 session_features = strtol(feats, NULL, 16);
3907
3908         if ((session_features & ~LST_FEATS_MASK) != 0) {
3909                 fprintf(stderr,
3910                         "Unsupported session features %x, only support these features so far: %x\n",
3911                         (session_features & ~LST_FEATS_MASK), LST_FEATS_MASK);
3912                 return -1;
3913         }
3914
3915         key = getenv("LST_SESSION");
3916
3917         if (key == NULL) {
3918                 session_key = 0;
3919                 return 0;
3920         }
3921
3922         session_key = atoi(key);
3923
3924         return 0;
3925 }
3926
3927 static void
3928 lst_print_usage(char *cmd)
3929 {
3930         char *argv[] = { "help", cmd };
3931
3932         cfs_parser(2, argv, lst_cmdlist);
3933 }
3934
3935 int main(int argc, char **argv)
3936 {
3937         int rc = 0;
3938
3939         setlinebuf(stdout);
3940
3941         rc = lst_initialize();
3942         if (rc < 0)
3943                 goto errorout;
3944
3945         rc = lustre_lnet_config_lib_init();
3946         if (rc < 0)
3947                 goto errorout;
3948
3949         rc = cfs_parser(argc, argv, lst_cmdlist);
3950
3951 errorout:
3952         return rc;
3953 }