lnet/utils/lst.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
   5  * Use is subject to license terms.
   6  *
   7  * Copyright (c) 2012, 2017, Intel Corporation.
   8  */
   9
  10 /*
  11  * This file is part of Lustre, http://www.lustre.org/
  12  *
  13  * Author: Liang Zhen <liangzhen@clusterfs.com>
  14  */
  15
  16 #include <errno.h>
  17 #include <getopt.h>
  18 #include <inttypes.h>
  19 #include <pwd.h>
  20 #include <unistd.h>
  21 #include <stdarg.h>
  22 #include <stdio.h>
  23 #include <stdlib.h>
  24 #include <string.h>
  25 #include <sys/ioctl.h>
  26 #include <time.h>
  27 #include <linux/types.h>
  28
  29 #include <libcfs/util/list.h>
  30 #include <libcfs/util/ioctl.h>
  31 #include <libcfs/util/parser.h>
  32 #include <linux/lnet/lnetctl.h>
  33 #include <linux/lnet/lnetst.h>
  34 #include <linux/lnet/nidstr.h>
  35 #include "lnetconfig/liblnetconfig.h"
  36
  37 static int lst_info_batch_ioctl(char *batch, int test, int server,
  38                        struct lstcon_test_batch_ent *entp, int *idxp,
  39                        int *ndentp, struct lstcon_node_ent *dentsp);
  40 static int lst_info_group_ioctl(char *name, struct lstcon_ndlist_ent *gent,
  41                         int *idx, int *count, struct lstcon_node_ent *dents);
  42 static int lst_query_batch_ioctl(char *batch, int test, int server,
  43                          int timeout, struct list_head *head);
  44
  45 struct lst_sid LST_INVALID_SID = { .ses_nid = LNET_NID_ANY, .ses_stamp = -1 };
  46 static unsigned int session_key;
  47
  48 /* All nodes running 2.6.50 or later understand feature LST_FEAT_BULK_LEN */
  49 static unsigned int session_features = LST_FEATS_MASK;
  50 static struct lstcon_trans_stat trans_stat;
  51
  52 typedef struct list_string {
  53         struct list_string *lstr_next;
  54         int                 lstr_sz;
  55         char                lstr_str[0];
  56 } lstr_t;
  57
  58 #ifndef offsetof
  59 # define offsetof(typ,memb)     ((unsigned long)((char *)&(((typ *)0)->memb)))
  60 #endif
  61
  62 static int alloc_count = 0;
  63 static int alloc_nob   = 0;
  64
  65 static lstr_t *
  66 alloc_lstr(int sz)
  67 {
  68         lstr_t  *lstr = malloc(offsetof(lstr_t, lstr_str[sz]));
  69
  70         if (lstr == NULL) {
  71                 fprintf(stderr, "Can't allocate lstr\n");
  72                 abort();
  73         }
  74
  75         alloc_nob += sz;
  76         alloc_count++;
  77
  78         lstr->lstr_str[0] = 0;
  79         lstr->lstr_sz = sz;
  80         return lstr;
  81 }
  82
  83 static void
  84 free_lstr(lstr_t *lstr)
  85 {
  86         alloc_count--;
  87         alloc_nob -= lstr->lstr_sz;
  88         free(lstr);
  89 }
  90
  91 static void
  92 new_lstrs(lstr_t **list, char *prefix, char *postfix,
  93           int lo, int hi, int stride)
  94 {
  95         int    n1 = strlen(prefix);
  96         int    n2 = strlen(postfix);
  97         int    sz = n1 + 20 + n2 + 1;
  98
  99         do {
 100                 lstr_t *n = alloc_lstr(sz);
 101
 102                 snprintf(n->lstr_str, sz - 1, "%s%u%s",
 103                          prefix, lo, postfix);
 104
 105                 n->lstr_next = *list;
 106                 *list = n;
 107
 108                 lo += stride;
 109         } while (lo <= hi);
 110 }
 111
 112 static int
 113 expand_lstr(lstr_t **list, lstr_t *l)
 114 {
 115         int          nob = strlen(l->lstr_str);
 116         char        *b1;
 117         char        *b2;
 118         char        *expr;
 119         char        *sep;
 120         int          x;
 121         int          y;
 122         int          z;
 123         int          n;
 124
 125         b1 = strchr(l->lstr_str, '[');
 126         if (b1 == NULL) {
 127                 l->lstr_next = *list;
 128                 *list = l;
 129                 return 0;
 130         }
 131
 132         b2 = strchr(b1, ']');
 133         if (b2 == NULL || b2 == b1 + 1)
 134                 return -1;
 135
 136         *b1++ = 0;
 137         *b2++ = 0;
 138         expr = b1;
 139         do {
 140
 141                 sep = strchr(expr, ',');
 142                 if (sep != NULL)
 143                         *sep++ = 0;
 144
 145                 nob = strlen(expr);
 146                 n = nob;
 147                 if (sscanf(expr, "%u%n", &x, &n) >= 1 && n == nob) {
 148                         /* simple number */
 149                         new_lstrs(list, l->lstr_str, b2, x, x, 1);
 150                         continue;
 151                 }
 152
 153                 n = nob;
 154                 if (sscanf(expr, "%u-%u%n", &x, &y, &n) >= 2 && n == nob &&
 155                     x < y) {
 156                         /* simple range */
 157                         new_lstrs(list, l->lstr_str, b2, x, y, 1);
 158                         continue;
 159                 }
 160
 161                 n = nob;
 162                 if (sscanf(expr, "%u-%u/%u%n", &x, &y, &z, &n) >= 3 && n == nob &&
 163                     x < y) {
 164                         /* strided range */
 165                         new_lstrs(list, l->lstr_str, b2, x, y, z);
 166                         continue;
 167                 }
 168
 169                 /* syntax error */
 170                 return -1;
 171         } while ((expr = sep) != NULL);
 172
 173         free_lstr(l);
 174
 175         return 1;
 176 }
 177
 178 static int
 179 expand_strs(char *str, lstr_t **head)
 180 {
 181         lstr_t  *list = NULL;
 182         lstr_t  *nlist;
 183         lstr_t  *l;
 184         int      rc = 0;
 185         int      expanded;
 186
 187         l = alloc_lstr(strlen(str) + 1);
 188         memcpy(l->lstr_str, str, strlen(str) + 1);
 189         l->lstr_next = NULL;
 190         list = l;
 191
 192         do {
 193                 expanded = 0;
 194                 nlist = NULL;
 195
 196                 while ((l = list) != NULL) {
 197                         list = l->lstr_next;
 198
 199                         rc = expand_lstr(&nlist, l);
 200                         if (rc < 0) {
 201                                 fprintf(stderr, "Syntax error in \"%s\"\n", str);
 202                                 free_lstr(l);
 203                                 break;
 204                         }
 205
 206                         expanded |= rc > 0;
 207                 }
 208
 209                 /* re-order onto 'list' */
 210                 while ((l = nlist) != NULL) {
 211                         nlist = l->lstr_next;
 212                         l->lstr_next = list;
 213                         list = l;
 214                 }
 215
 216         } while (expanded && rc > 0);
 217
 218         if (rc >= 0) {
 219                 *head = list;
 220                 return 0;
 221         }
 222
 223         while ((l = list) != NULL) {
 224                 list = l->lstr_next;
 225
 226                 free_lstr(l);
 227         }
 228         return rc;
 229 }
 230
 231 static int
 232 lst_parse_nids(char *str, int *countp, struct lnet_process_id **idspp)
 233 {
 234         lstr_t  *head = NULL;
 235         lstr_t  *l;
 236         int      c = 0;
 237         int      i;
 238         int      rc;
 239
 240         rc = expand_strs(str, &head);
 241         if (rc != 0)
 242                 goto out;
 243
 244         l = head;
 245         while (l != NULL) {
 246                 l = l->lstr_next;
 247                 c++;
 248         }
 249
 250         *idspp = malloc(c * sizeof(struct lnet_process_id));
 251         if (*idspp == NULL) {
 252                 fprintf(stderr, "Out of memory\n");
 253                 rc = -1;
 254         }
 255
 256         *countp = c;
 257 out:
 258         i = 0;
 259         while ((l = head) != NULL) {
 260                 head = l->lstr_next;
 261
 262                 if (rc == 0) {
 263                         (*idspp)[i].nid = libcfs_str2nid(l->lstr_str);
 264                         if ((*idspp)[i].nid == LNET_NID_ANY) {
 265                                 fprintf(stderr, "Invalid nid: %s\n",
 266                                         l->lstr_str);
 267                                 rc = -1;
 268                         }
 269
 270                         (*idspp)[i].pid = LNET_PID_LUSTRE;
 271                         i++;
 272                 }
 273
 274                 free_lstr(l);
 275         }
 276
 277         if (rc == 0)
 278                 return 0;
 279
 280         free(*idspp);
 281         *idspp = NULL;
 282
 283         return rc;
 284 }
 285
 286 static char *
 287 lst_node_state2str(int state)
 288 {
 289         if (state == LST_NODE_ACTIVE)
 290                 return "Active";
 291         if (state == LST_NODE_BUSY)
 292                 return "Busy";
 293         if (state == LST_NODE_DOWN)
 294                 return "Down";
 295
 296         return "Unknown";
 297 }
 298
 299 static int
 300 lst_node_str2state(char *str)
 301 {
 302         if (strcasecmp(str, "active") == 0)
 303                 return LST_NODE_ACTIVE;
 304         if (strcasecmp(str, "busy") == 0)
 305                 return LST_NODE_BUSY;
 306         if (strcasecmp(str, "down") == 0)
 307                 return LST_NODE_DOWN;
 308         if (strcasecmp(str, "unknown") == 0)
 309                 return LST_NODE_UNKNOWN;
 310         if (strcasecmp(str, "invalid") == 0)
 311                 return (LST_NODE_UNKNOWN | LST_NODE_DOWN | LST_NODE_BUSY);
 312
 313         return -1;
 314 }
 315
 316 static char *
 317 lst_test_type2name(int type)
 318 {
 319         if (type == LST_TEST_PING)
 320                 return "ping";
 321         if (type == LST_TEST_BULK)
 322                 return "brw";
 323
 324         return "unknown";
 325 }
 326
 327 static int
 328 lst_test_name2type(char *name)
 329 {
 330         if (strcasecmp(name, "ping") == 0)
 331                 return LST_TEST_PING;
 332         if (strcasecmp(name, "brw") == 0)
 333                 return LST_TEST_BULK;
 334
 335         return -1;
 336 }
 337
 338 static void
 339 lst_print_usage(char *cmd)
 340 {
 341         char *argv[] = { "help", cmd };
 342
 343         cfs_parser(2, argv, NULL);
 344 }
 345
 346 static void
 347 lst_print_error(char *sub, const char *def_format, ...)
 348 {
 349         va_list ap;
 350
 351         /* local error returned from kernel */
 352         switch (errno) {
 353         case ESRCH:
 354                 fprintf(stderr, "No session exists\n");
 355                 return;
 356         case ESHUTDOWN:
 357                 fprintf(stderr, "Session is shutting down\n");
 358                 return;
 359         case EACCES:
 360                 fprintf(stderr, "Unmatched session key or not root\n");
 361                 return;
 362         case ENOENT:
 363                 fprintf(stderr, "Can't find %s in current session\n", sub);
 364                 return;
 365         case EINVAL:
 366                 fprintf(stderr, "Invalid parameters list in command line\n");
 367                 return;
 368         case EFAULT:
 369                 fprintf(stderr, "Bad parameter address\n");
 370                 return;
 371         case EEXIST:
 372                 fprintf(stderr, "%s already exists\n", sub);
 373                 return;
 374         default:
 375                 va_start(ap, def_format);
 376                 vfprintf(stderr, def_format, ap);
 377                 va_end(ap);
 378
 379                 return;
 380         }
 381 }
 382
 383 static void
 384 lst_free_rpcent(struct list_head *head)
 385 {
 386         struct lstcon_rpc_ent *ent;
 387
 388         while (!list_empty(head)) {
 389                 ent = list_first_entry(head, struct lstcon_rpc_ent, rpe_link);
 390
 391                 list_del(&ent->rpe_link);
 392                 free(ent);
 393         }
 394 }
 395
 396 static void
 397 lst_reset_rpcent(struct list_head *head)
 398 {
 399         struct lstcon_rpc_ent *ent;
 400
 401         list_for_each_entry(ent, head, rpe_link) {
 402                 ent->rpe_sid       = LST_INVALID_SID;
 403                 ent->rpe_peer.nid  = LNET_NID_ANY;
 404                 ent->rpe_peer.pid  = LNET_PID_ANY;
 405                 ent->rpe_rpc_errno = ent->rpe_fwk_errno = 0;
 406         }
 407 }
 408
 409 static int
 410 lst_alloc_rpcent(struct list_head *head, int count, int offset)
 411 {
 412         struct lstcon_rpc_ent *ent;
 413         int               i;
 414
 415         for (i = 0; i < count; i++) {
 416                 ent = malloc(offsetof(struct lstcon_rpc_ent, rpe_payload[offset]));
 417                 if (ent == NULL) {
 418                         lst_free_rpcent(head);
 419                         return -1;
 420                 }
 421
 422                 memset(ent, 0, offsetof(struct lstcon_rpc_ent, rpe_payload[offset]));
 423
 424                 ent->rpe_sid      = LST_INVALID_SID;
 425                 ent->rpe_peer.nid = LNET_NID_ANY;
 426                 ent->rpe_peer.pid = LNET_PID_ANY;
 427                 list_add(&ent->rpe_link, head);
 428         }
 429
 430         return 0;
 431 }
 432
 433 static void
 434 lst_print_transerr(struct list_head *head, char *optstr)
 435 {
 436         struct lstcon_rpc_ent *ent;
 437
 438         list_for_each_entry(ent, head, rpe_link) {
 439                 if (ent->rpe_rpc_errno == 0 && ent->rpe_fwk_errno == 0)
 440                         continue;
 441
 442                 if (ent->rpe_rpc_errno != 0) {
 443                         fprintf(stderr, "%s RPC failed on %s: %s\n",
 444                                 optstr, libcfs_id2str(ent->rpe_peer),
 445                                 strerror(ent->rpe_rpc_errno));
 446                         continue;
 447                 }
 448
 449                 fprintf(stderr, "operation %s failed on %s: %s\n",
 450                         optstr, libcfs_id2str(ent->rpe_peer),
 451                         strerror(ent->rpe_fwk_errno));
 452         }
 453 }
 454
 455 static int
 456 lst_ioctl(unsigned int opc, void *buf, int len)
 457 {
 458         struct libcfs_ioctl_data data;
 459         int    rc;
 460
 461         LIBCFS_IOC_INIT (data);
 462         data.ioc_u32[0]  = opc;
 463         data.ioc_plen1   = len;
 464         data.ioc_pbuf1   = (char *)buf;
 465         data.ioc_plen2   = sizeof(trans_stat);
 466         data.ioc_pbuf2   = (char *)&trans_stat;
 467
 468         memset(&trans_stat, 0, sizeof(trans_stat));
 469
 470         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNETST, &data);
 471
 472         /* local error, no valid RPC result */
 473         if (rc != 0)
 474                 return -1;
 475
 476         /* RPC error */
 477         if (trans_stat.trs_rpc_errno != 0)
 478                 return -2;
 479
 480         /* Framework error */
 481         if (trans_stat.trs_fwk_errno != 0)
 482                 return -3;
 483
 484         return 0;
 485 }
 486
 487 static int lst_yaml_session(const char *label, const char *timeout, int nlflags,
 488                             const char *errmsg)
 489 {
 490         struct lstcon_ndlist_ent ndinfo = { };
 491         struct lst_sid sid = LST_INVALID_SID;
 492         /* nlflags being zero means we are destroying the session.
 493          * No parsing of reply needed.
 494          */
 495         bool done = nlflags ? false : true;
 496         char nid[LNET_NIDSTR_SIZE];
 497         char name[LST_NAME_SIZE];
 498         unsigned int key = 0;
 499         yaml_emitter_t request;
 500         yaml_parser_t reply;
 501         yaml_event_t event;
 502         struct nl_sock *sk;
 503         int rc;
 504
 505         sk = nl_socket_alloc();
 506         if (!sk)
 507                 return -1;
 508
 509         /* Note: NL_AUTO_PID == zero which we use by default for the
 510          * session_key when creating a new session. This is considered
 511          * an invalid key so we need to get the real session key from
 512          * the yaml parser yet to be created. If the user did request
 513          * a specific session key then set the socket's port id to this
 514          * value.
 515          */
 516         if (session_key)
 517                 nl_socket_set_local_port(sk, session_key);
 518
 519         /* Setup reply parser to recieve Netlink packets */
 520         rc = yaml_parser_initialize(&reply);
 521         if (rc == 0) {
 522                 nl_socket_free(sk);
 523                 return -1;
 524         }
 525
 526         rc = yaml_parser_set_input_netlink(&reply, sk, false);
 527         if (rc == 0)
 528                 goto parser_error;
 529
 530         /* Create Netlink emitter to send request to kernel */
 531         yaml_emitter_initialize(&request);
 532         rc = yaml_emitter_set_output_netlink(&request, sk,
 533                                              LNET_SELFTEST_GENL_NAME,
 534                                              LNET_SELFTEST_GENL_VERSION,
 535                                              LNET_SELFTEST_CMD_SESSIONS,
 536                                              nlflags);
 537         if (rc == 0)
 538                 goto emitter_error;
 539
 540         yaml_emitter_open(&request);
 541         yaml_document_start_event_initialize(&event, NULL, NULL, NULL, 0);
 542         rc = yaml_emitter_emit(&request, &event);
 543         if (rc == 0)
 544                 goto emitter_error;
 545
 546         yaml_mapping_start_event_initialize(&event, NULL,
 547                                             (yaml_char_t *)YAML_MAP_TAG,
 548                                             1, YAML_BLOCK_MAPPING_STYLE);
 549         rc = yaml_emitter_emit(&request, &event);
 550         if (rc == 0)
 551                 goto emitter_error;
 552
 553         yaml_scalar_event_initialize(&event, NULL,
 554                                      (yaml_char_t *)YAML_STR_TAG,
 555                                      (yaml_char_t *)"sessions",
 556                                      strlen("sessions"), 1, 0,
 557                                      YAML_PLAIN_SCALAR_STYLE);
 558         rc = yaml_emitter_emit(&request, &event);
 559         if (rc == 0)
 560                 goto emitter_error;
 561
 562         if (!label) {
 563                 yaml_scalar_event_initialize(&event, NULL,
 564                                              (yaml_char_t *)YAML_STR_TAG,
 565                                              (yaml_char_t *)"",
 566                                              strlen(""), 1, 0,
 567                                              YAML_PLAIN_SCALAR_STYLE);
 568                 rc = yaml_emitter_emit(&request, &event);
 569                 if (rc == 0)
 570                         goto emitter_error;
 571                 goto skip_params;
 572         }
 573
 574         /* sessions: { name: 'name', timeout: 300 }
 575          * or
 576          * sessions:
 577          *   name: 'name'
 578          *   timeout: 300
 579          */
 580         yaml_mapping_start_event_initialize(&event, NULL,
 581                                             (yaml_char_t *)YAML_MAP_TAG,
 582                                             1, YAML_FLOW_MAPPING_STYLE);
 583         rc = yaml_emitter_emit(&request, &event);
 584         if (rc == 0)
 585                 goto emitter_error;
 586
 587         yaml_scalar_event_initialize(&event, NULL,
 588                                      (yaml_char_t *)YAML_STR_TAG,
 589                                      (yaml_char_t *)"name",
 590                                      strlen("name"), 1, 0,
 591                                      YAML_PLAIN_SCALAR_STYLE);
 592         rc = yaml_emitter_emit(&request, &event);
 593         if (rc == 0)
 594                 goto emitter_error;
 595
 596         yaml_scalar_event_initialize(&event, NULL,
 597                                      (yaml_char_t *)YAML_STR_TAG,
 598                                      (yaml_char_t *)label,
 599                                      strlen(label), 1, 0,
 600                                      YAML_PLAIN_SCALAR_STYLE);
 601         rc = yaml_emitter_emit(&request, &event);
 602         if (rc == 0)
 603                 goto emitter_error;
 604
 605         if (timeout) {
 606                 yaml_scalar_event_initialize(&event, NULL,
 607                                              (yaml_char_t *)YAML_STR_TAG,
 608                                              (yaml_char_t *)"timeout",
 609                                              strlen("timeout"), 1, 0,
 610                                              YAML_PLAIN_SCALAR_STYLE);
 611                 rc = yaml_emitter_emit(&request, &event);
 612                 if (rc == 0)
 613                         goto emitter_error;
 614
 615                 yaml_scalar_event_initialize(&event, NULL,
 616                                              (yaml_char_t *)YAML_STR_TAG,
 617                                              (yaml_char_t *)timeout,
 618                                              strlen(timeout), 1, 0,
 619                                              YAML_PLAIN_SCALAR_STYLE);
 620                 rc = yaml_emitter_emit(&request, &event);
 621                 if (rc == 0)
 622                         goto emitter_error;
 623         }
 624
 625         yaml_mapping_end_event_initialize(&event);
 626         rc = yaml_emitter_emit(&request, &event);
 627         if (rc == 0)
 628                 goto emitter_error;
 629
 630 skip_params:
 631         yaml_mapping_end_event_initialize(&event);
 632         rc = yaml_emitter_emit(&request, &event);
 633         if (rc == 0)
 634                 goto emitter_error;
 635
 636         yaml_document_end_event_initialize(&event, 0);
 637         rc = yaml_emitter_emit(&request, &event);
 638         if (rc == 0)
 639                 goto emitter_error;
 640
 641         rc = yaml_emitter_close(&request);
 642         if (rc == 0) {
 643 emitter_error:
 644                 yaml_emitter_log_error(&request, stderr);
 645                 yaml_emitter_delete(&request);
 646                 errmsg = NULL;
 647                 goto parser_error;
 648         }
 649         yaml_emitter_delete(&request);
 650
 651         while (!done) {
 652                 rc = yaml_parser_parse(&reply, &event);
 653                 if (rc == 0)
 654                         goto parser_error;
 655
 656                 if (event.type == YAML_SCALAR_EVENT) {
 657                         char *tmp, *endp = NULL;
 658
 659                         if (strcmp((char *)event.data.scalar.value,
 660                                    "name") == 0) {
 661                                 yaml_event_delete(&event);
 662                                 rc = yaml_parser_parse(&reply, &event);
 663                                 if (rc == 0)
 664                                         goto parser_error;
 665
 666                                 strncpy(name, (char *)event.data.scalar.value,
 667                                         sizeof(name) - 1);
 668                         }
 669
 670                         if (strcmp((char *)event.data.scalar.value,
 671                                    "key") == 0) {
 672                                 yaml_event_delete(&event);
 673                                 rc = yaml_parser_parse(&reply, &event);
 674                                 if (rc == 0)
 675                                         goto parser_error;
 676
 677                                 tmp = (char *)event.data.scalar.value;
 678                                 key = strtoul(tmp, &endp, 10);
 679                                 if (endp == tmp)
 680                                         goto parser_error;
 681                         }
 682
 683                         if (strcmp((char *)event.data.scalar.value,
 684                                    "timestamp") == 0) {
 685                                 yaml_event_delete(&event);
 686                                 rc = yaml_parser_parse(&reply, &event);
 687                                 if (rc == 0)
 688                                         goto parser_error;
 689
 690                                 tmp = (char *)event.data.scalar.value;
 691                                 sid.ses_stamp = strtoll(tmp, &endp, 10);
 692                                 if (endp == tmp)
 693                                         goto parser_error;
 694                         }
 695
 696                         if (strcmp((char *)event.data.scalar.value,
 697                                    "nid") == 0) {
 698                                 yaml_event_delete(&event);
 699                                 rc = yaml_parser_parse(&reply, &event);
 700                                 if (rc == 0)
 701                                         goto parser_error;
 702
 703                                 strncpy(nid, (char *)event.data.scalar.value,
 704                                         sizeof(nid) - 1);
 705                         }
 706
 707                         if (strcmp((char *)event.data.scalar.value,
 708                                    "nodes") == 0) {
 709                                 yaml_event_delete(&event);
 710                                 rc = yaml_parser_parse(&reply, &event);
 711                                 if (rc == 0)
 712                                         goto parser_error;
 713
 714                                 tmp = (char *)event.data.scalar.value;
 715                                 ndinfo.nle_nnode = strtoul(tmp, &endp, 10);
 716                                 if (endp == tmp)
 717                                         goto parser_error;
 718                         }
 719                 }
 720
 721                 done = (event.type == YAML_STREAM_END_EVENT);
 722
 723                 yaml_event_delete(&event);
 724         }
 725
 726         if (nlflags & NLM_F_CREATE) {
 727                 session_features = yaml_parser_get_reader_proto_version(&reply);
 728                 session_key = key;
 729         }
 730 parser_error:
 731         if (rc == 0 && errmsg)
 732                 yaml_parser_log_error(&reply, stderr, errmsg);
 733         yaml_parser_delete(&reply);
 734         nl_socket_free(sk);
 735
 736         if (((nlflags & NLM_F_DUMP) == NLM_F_DUMP) && rc != 0) {
 737                 fprintf(stdout,
 738                         "%s ID: %ju@%s, KEY: %u FEATURES: %x NODES: %d\n",
 739                         name, (uintmax_t)sid.ses_stamp, nid,
 740                         key, session_features, ndinfo.nle_nnode);
 741         }
 742
 743         return rc == 0 ? -1 : 0;
 744 }
 745
 746 static int
 747 lst_new_session_ioctl(char *name, int timeout, int force, struct lst_sid *sid)
 748 {
 749         struct lstio_session_new_args args = { 0 };
 750
 751         args.lstio_ses_key     = session_key;
 752         args.lstio_ses_timeout = timeout;
 753         args.lstio_ses_force   = force;
 754         args.lstio_ses_idp     = sid;
 755         args.lstio_ses_feats   = session_features;
 756         args.lstio_ses_nmlen   = strlen(name);
 757         args.lstio_ses_namep   = name;
 758
 759         return lst_ioctl (LSTIO_SESSION_NEW, &args, sizeof(args));
 760 }
 761
 762 static int
 763 jt_lst_new_session(int argc, char **argv)
 764 {
 765         char  buf[LST_NAME_SIZE * 2 + 1];
 766         char *name, *timeout_s = NULL;
 767         int nlflags = NLM_F_CREATE;
 768         struct lst_sid session_id;
 769         int   optidx = 0;
 770         int   timeout = 300;
 771         int   force = 0;
 772         int   c;
 773         int   rc;
 774
 775         static const struct option session_opts[] = {
 776                 { .name = "timeout", .has_arg = required_argument, .val = 't' },
 777                 { .name = "force",   .has_arg = no_argument,       .val = 'f' },
 778                 { .name = NULL } };
 779
 780         while (1) {
 781                 c = getopt_long(argc, argv, "ft:",
 782                                 session_opts, &optidx);
 783                 if (c == -1)
 784                         break;
 785
 786                 switch (c) {
 787                 case 'f':
 788                         nlflags |= NLM_F_REPLACE;
 789                         force = 1;
 790                         break;
 791                 case 't':
 792                         timeout_s = optarg;
 793                         break;
 794                 default:
 795                         lst_print_usage(argv[0]);
 796                         return -1;
 797                 }
 798         }
 799
 800         if (timeout_s) {
 801                 timeout = atoi(timeout_s);
 802                 if (timeout <= 0) {
 803                         fprintf(stderr, "Invalid timeout value\n");
 804                         return -1;
 805                 }
 806         }
 807
 808         if (optind == argc - 1) {
 809                 name = argv[optind ++];
 810                 if (strlen(name) >= LST_NAME_SIZE) {
 811                         fprintf(stderr, "Name size is limited to %d\n",
 812                                 LST_NAME_SIZE - 1);
 813                         return -1;
 814                 }
 815         } else if (optind == argc) {
 816                 char           user[LST_NAME_SIZE];
 817                 char           host[LST_NAME_SIZE];
 818                 struct passwd *pw = getpwuid(getuid());
 819
 820                 if (pw == NULL)
 821                         snprintf(user, sizeof(user), "%d", (int)getuid());
 822                 else
 823                         snprintf(user, sizeof(user), "%s", pw->pw_name);
 824
 825                 rc = gethostname(host, sizeof(host));
 826                 if (rc != 0)
 827                         snprintf(host, sizeof(host), "unknown_host");
 828
 829                 snprintf(buf, sizeof(buf), "%s@%s", user, host);
 830                 name = buf;
 831         } else {
 832                 lst_print_usage(argv[0]);
 833                 return -1;
 834         }
 835
 836         rc = lst_yaml_session(name, timeout_s, nlflags, "new session");
 837         if (rc == 0)
 838                 goto success;
 839
 840         if (session_key == 0) {
 841                 fprintf(stderr,
 842                         "Can't find env LST_SESSION or value is not valid\n");
 843                 return -1;
 844         }
 845
 846         rc = lst_new_session_ioctl(name, timeout, force, &session_id);
 847         if (rc != 0) {
 848                 lst_print_error("session", "Failed to create session: %s\n",
 849                                 strerror(errno));
 850                 return rc;
 851         }
 852 success:
 853         fprintf(stdout, "SESSION: %s FEATURES: %x TIMEOUT: %d FORCE: %s\n",
 854                 name, session_features, timeout, force ? "Yes" : "No");
 855         return 0;
 856 }
 857
 858 static int
 859 lst_session_info_ioctl(char *name, int len, int *key, unsigned *featp,
 860                        struct lst_sid *sid, struct lstcon_ndlist_ent *ndinfo)
 861 {
 862         struct lstio_session_info_args args = { 0 };
 863
 864         args.lstio_ses_idp     = sid;
 865         args.lstio_ses_keyp    = key;
 866         args.lstio_ses_featp   = featp;
 867         args.lstio_ses_ndinfo  = ndinfo;
 868         args.lstio_ses_nmlen   = len;
 869         args.lstio_ses_namep   = name;
 870
 871         return lst_ioctl(LSTIO_SESSION_INFO, &args, sizeof(args));
 872 }
 873
 874 static int
 875 jt_lst_show_session(int argc, char **argv)
 876 {
 877         struct lstcon_ndlist_ent ndinfo;
 878         struct lst_sid sid;
 879         char name[LST_NAME_SIZE];
 880         unsigned int feats;
 881         int key;
 882         int rc;
 883
 884         rc = lst_yaml_session(NULL, NULL, NLM_F_DUMP, "show session");
 885         if (rc == 0)
 886                 return 0;
 887
 888         rc = lst_session_info_ioctl(name, sizeof(name), &key,
 889                                     &feats, &sid, &ndinfo);
 890
 891         if (rc != 0) {
 892                 lst_print_error("session", "Failed to show session: %s\n",
 893                                 strerror(errno));
 894                 return -1;
 895         }
 896
 897         fprintf(stdout, "%s ID: %ju@%s, KEY: %d FEATURES: %x NODES: %d\n",
 898                 name, (uintmax_t)sid.ses_stamp, libcfs_nid2str(sid.ses_nid),
 899                 key, feats, ndinfo.nle_nnode);
 900
 901         return 0;
 902 }
 903
 904 static int
 905 lst_end_session_ioctl(void)
 906 {
 907         struct lstio_session_end_args args = { 0 };
 908
 909         args.lstio_ses_key = session_key;
 910         return lst_ioctl(LSTIO_SESSION_END, &args, sizeof(args));
 911 }
 912
 913 static int
 914 jt_lst_end_session(int argc, char **argv)
 915 {
 916         int             rc;
 917
 918         if (session_key == 0) {
 919                 fprintf(stderr,
 920                         "Can't find env LST_SESSION or value is not valid\n");
 921                 return -1;
 922         }
 923
 924         rc = lst_yaml_session(NULL, NULL, 0, "end session");
 925         if (rc == 0)
 926                 goto finish;
 927
 928         rc = lst_end_session_ioctl();
 929
 930         if (rc == 0) {
 931                 fprintf(stdout, "session is ended\n");
 932                 return 0;
 933         }
 934
 935         if (rc == -1) {
 936                 lst_print_error("session", "Failed to end session: %s\n",
 937                                 strerror(errno));
 938                 return rc;
 939         }
 940 finish:
 941         if (trans_stat.trs_rpc_errno != 0) {
 942                 fprintf(stderr,
 943                         "[RPC] Failed to send %d session RPCs: %s\n",
 944                         lstcon_rpc_stat_failure(&trans_stat, 0),
 945                         strerror(trans_stat.trs_rpc_errno));
 946         }
 947
 948         if (trans_stat.trs_fwk_errno != 0) {
 949                 fprintf(stderr,
 950                         "[FWK] Failed to end session on %d nodes: %s\n",
 951                         lstcon_sesop_stat_failure(&trans_stat, 0),
 952                         strerror(trans_stat.trs_fwk_errno));
 953         }
 954
 955         return rc;
 956 }
 957
 958 #define LST_NODES_TITLE "\tACTIVE\tBUSY\tDOWN\tUNKNOWN\tTOTAL\n"
 959
 960 static int lst_yaml_display_groups(yaml_parser_t *reply, char *group,
 961                                    int states, bool print)
 962 {
 963         yaml_event_t event;
 964         int rc;
 965
 966         if (states) {
 967                 char name[LST_NAME_SIZE] = {};
 968                 bool done = false;
 969                 int count = 0;
 970
 971                 if (print)
 972                         fprintf(stdout, LST_NODES_TITLE);
 973
 974                 while (!done) {
 975                         rc = yaml_parser_parse(reply, &event);
 976                         if (rc == 0)
 977                                 goto parser_error;
 978
 979                         if (event.type == YAML_MAPPING_START_EVENT) {
 980                                 char *value = NULL;
 981                                 yaml_event_t next;
 982
 983                                 rc = yaml_parser_parse(reply, &next);
 984                                 if (rc == 0)
 985                                         goto parser_error;
 986
 987                                 if (next.type != YAML_SCALAR_EVENT) {
 988                                         yaml_event_delete(&next);
 989                                         continue;
 990                                 }
 991
 992                                 value = (char *)next.data.scalar.value;
 993                                 if (strcmp(value, "groups") == 0) {
 994                                         yaml_event_delete(&next);
 995                                 } else if (strcmp(value, "nid") == 0) {
 996                                         yaml_event_t next;
 997                                         char *tmp;
 998
 999                                         rc = yaml_parser_parse(reply, &next);
1000                                         if (rc == 0)
1001                                                 goto parser_error;
1002
1003                                         fprintf(stdout, "\t%s: ",
1004                                                 (char *)next.data.scalar.value);
1005
1006                                         rc = yaml_parser_parse(reply, &next);
1007                                         if (rc == 0)
1008                                                 goto parser_error;
1009
1010                                         tmp = (char *)next.data.scalar.value;
1011                                         if (strcmp(tmp, "status") == 0) {
1012                                                 yaml_event_t state;
1013
1014                                                 rc = yaml_parser_parse(reply, &state);
1015                                                 if (rc == 0)
1016                                                         goto parser_error;
1017
1018                                                 fprintf(stdout, "%s\n",
1019                                                         (char *)state.data.scalar.value);
1020                                                 count++;
1021                                         }
1022                                 } else {
1023                                         yaml_event_t next;
1024
1025                                         rc = yaml_parser_parse(reply, &next);
1026                                         if (rc == 0)
1027                                                 goto parser_error;
1028
1029                                         strncpy(name, value, sizeof(name) - 1);
1030                                         fprintf(stdout, "Group [ %s ]\n", name);
1031                                         count = 0;
1032
1033                                         if (next.type != YAML_SEQUENCE_START_EVENT) {
1034                                                 fprintf(stdout,
1035                                                         "No nodes found [ %s ]\n",
1036                                                         name);
1037                                         }
1038                                 }
1039                         } else if (event.type == YAML_SEQUENCE_END_EVENT &&
1040                                    count) {
1041                                 fprintf(stdout, "Total %d nodes [ %s ]\n",
1042                                         count, name);
1043                                 count = 0;
1044                         }
1045
1046                         done = (event.type == YAML_STREAM_END_EVENT);
1047                         yaml_event_delete(&event);
1048                 }
1049         } else if (group) {
1050                 int active = 0, busy = 0, down = 0, unknown = 0;
1051                 char group[LST_NAME_SIZE];
1052                 bool done = false;
1053
1054                 while (!done) {
1055                         rc = yaml_parser_parse(reply, &event);
1056                         if (rc == 0)
1057                                 goto parser_error;
1058
1059                         if (event.type == YAML_SCALAR_EVENT) {
1060                                 char *value = (char *)event.data.scalar.value;
1061                                 yaml_event_t next;
1062
1063                                 value = (char *)event.data.scalar.value;
1064                                 if (strcmp(value, "groups") == 0) {
1065                                         yaml_event_delete(&event);
1066                                         continue;
1067                                 }
1068
1069                                 rc = yaml_parser_parse(reply, &next);
1070                                 if (rc == 0)
1071                                         goto parser_error;
1072
1073                                 if (next.type == YAML_SCALAR_EVENT) {
1074                                         int status;
1075
1076                                         status = lst_node_str2state((char *)next.data.scalar.value);
1077                                         switch (status) {
1078                                         case LST_NODE_ACTIVE:
1079                                                 active++;
1080                                                 break;
1081                                         case LST_NODE_BUSY:
1082                                                 busy++;
1083                                                 break;
1084                                         case LST_NODE_DOWN:
1085                                                 down++;
1086                                                 break;
1087                                         case LST_NODE_UNKNOWN:
1088                                                 unknown++;
1089                                         default:
1090                                                 break;
1091                                         }
1092                                 } else if (next.type == YAML_SEQUENCE_START_EVENT) {
1093                                         strncpy(group, value, sizeof(group) - 1);
1094                                         active = 0;
1095                                         busy = 0;
1096                                         down = 0;
1097                                         unknown = 0;
1098                                 }
1099                                 yaml_event_delete(&next);
1100                         } else if (event.type == YAML_SEQUENCE_END_EVENT) {
1101                                 if (strlen(group)) {
1102                                         fprintf(stdout, "\t%d\t%d\t%d\t%d\t%d\t%s\n",
1103                                                 active, busy, down, unknown,
1104                                                 active + busy + down + unknown, group);
1105                                 }
1106                                 memset(group, 0, sizeof(group));
1107                         }
1108                         done = (event.type == YAML_STREAM_END_EVENT);
1109                         yaml_event_delete(&event);
1110                 }
1111         } else {
1112                 bool done = false;
1113                 unsigned int i = 1;
1114
1115                 while (!done) {
1116                         rc = yaml_parser_parse(reply, &event);
1117                         if (rc == 0)
1118                                 goto parser_error;
1119
1120                         if (event.type == YAML_SCALAR_EVENT) {
1121                                 char *value;
1122
1123                                 value = (char *)event.data.scalar.value;
1124                                 if (strlen(value) &&
1125                                     strcmp(value, "groups") != 0) {
1126                                         if (print)
1127                                                 fprintf(stdout, "%d) %s\n",
1128                                                         i, value);
1129                                         i++;
1130                                 }
1131                         }
1132                         done = (event.type == YAML_STREAM_END_EVENT);
1133
1134                         yaml_event_delete(&event);
1135                 }
1136                 if (print)
1137                         fprintf(stdout, "Total %d groups\n", i - 1);
1138                 else
1139                         rc = i - 1;
1140         }
1141 parser_error:
1142         return rc;
1143 }
1144
1145 #define LST_NODES_TITLE "\tACTIVE\tBUSY\tDOWN\tUNKNOWN\tTOTAL\n"
1146
1147 static int lst_yaml_groups(int nlflags, char *name, int states, bool print)
1148 {
1149         yaml_emitter_t request;
1150         yaml_parser_t reply;
1151         yaml_event_t event;
1152         struct nl_sock *sk;
1153         int rc;
1154
1155         sk = nl_socket_alloc();
1156         if (!sk)
1157                 return -EOPNOTSUPP;
1158
1159         /* Setup reply parser to recieve Netlink packets */
1160         rc = yaml_parser_initialize(&reply);
1161         if (rc == 0) {
1162                 nl_socket_free(sk);
1163                 return -EOPNOTSUPP;
1164         }
1165
1166         rc = yaml_parser_set_input_netlink(&reply, sk, false);
1167         if (rc == 0)
1168                 goto parser_error;
1169
1170         /* Create Netlink emitter to send request to kernel */
1171         yaml_emitter_initialize(&request);
1172         rc = yaml_emitter_set_output_netlink(&request, sk,
1173                                              LNET_SELFTEST_GENL_NAME,
1174                                              LNET_SELFTEST_GENL_VERSION,
1175                                              LNET_SELFTEST_CMD_GROUPS,
1176                                              nlflags);
1177         if (rc == 0)
1178                 goto emitter_error;
1179
1180         yaml_emitter_open(&request);
1181         yaml_document_start_event_initialize(&event, NULL, NULL, NULL, 0);
1182         rc = yaml_emitter_emit(&request, &event);
1183         if (rc == 0)
1184                 goto emitter_error;
1185
1186         yaml_mapping_start_event_initialize(&event, NULL,
1187                                             (yaml_char_t *)YAML_MAP_TAG,
1188                                             1, YAML_BLOCK_MAPPING_STYLE);
1189         rc = yaml_emitter_emit(&request, &event);
1190         if (rc == 0)
1191                 goto emitter_error;
1192
1193         yaml_scalar_event_initialize(&event, NULL,
1194                                      (yaml_char_t *)YAML_STR_TAG,
1195                                      (yaml_char_t *)"groups",
1196                                      strlen("groups"), 1, 0,
1197                                      YAML_PLAIN_SCALAR_STYLE);
1198         rc = yaml_emitter_emit(&request, &event);
1199         if (rc == 0)
1200                 goto emitter_error;
1201
1202         if (name) {
1203                 yaml_sequence_start_event_initialize(&event, NULL,
1204                                                      (yaml_char_t *)YAML_SEQ_TAG,
1205                                                      1, YAML_BLOCK_SEQUENCE_STYLE);
1206                 rc = yaml_emitter_emit(&request, &event);
1207                 if (rc == 0)
1208                         goto emitter_error;
1209
1210                 yaml_mapping_start_event_initialize(&event, NULL,
1211                                                     (yaml_char_t *)YAML_MAP_TAG,
1212                                                     1, YAML_BLOCK_MAPPING_STYLE);
1213                 rc = yaml_emitter_emit(&request, &event);
1214                 if (rc == 0)
1215                         goto emitter_error;
1216
1217                 yaml_scalar_event_initialize(&event, NULL,
1218                                              (yaml_char_t *)YAML_STR_TAG,
1219                                              (yaml_char_t *)name,
1220                                              strlen(name), 1, 0,
1221                                              YAML_PLAIN_SCALAR_STYLE);
1222                 rc = yaml_emitter_emit(&request, &event);
1223                 if (rc == 0)
1224                         goto emitter_error;
1225
1226                 if (states) {
1227                         int max = ffs(LST_NODE_UNKNOWN) + 1, i;
1228                         char *state;
1229
1230                         yaml_sequence_start_event_initialize(&event, NULL,
1231                                                              (yaml_char_t *)YAML_SEQ_TAG,
1232                                                              1, YAML_BLOCK_SEQUENCE_STYLE);
1233                         rc = yaml_emitter_emit(&request, &event);
1234                         if (rc == 0)
1235                                 goto emitter_error;
1236
1237
1238                         for (i = 0; i < max; i++) {
1239                                 int mask = states & (1 << i);
1240
1241                                 state = lst_node_state2str(mask);
1242                                 if (mask != LST_NODE_UNKNOWN && strcmp(state, "Unknown") == 0)
1243                                         continue;
1244
1245                                 yaml_mapping_start_event_initialize(&event, NULL,
1246                                                                     (yaml_char_t *)YAML_MAP_TAG,
1247                                                                     1, YAML_BLOCK_MAPPING_STYLE);
1248                                 rc = yaml_emitter_emit(&request, &event);
1249                                 if (rc == 0)
1250                                         goto emitter_error;
1251
1252                                 yaml_scalar_event_initialize(&event, NULL,
1253                                                              (yaml_char_t *)YAML_STR_TAG,
1254                                                              (yaml_char_t *)"status",
1255                                                              strlen("status"), 1, 0,
1256                                                              YAML_PLAIN_SCALAR_STYLE);
1257                                 rc = yaml_emitter_emit(&request, &event);
1258                                 if (rc == 0)
1259                                         goto emitter_error;
1260
1261                                 yaml_scalar_event_initialize(&event, NULL,
1262                                                              (yaml_char_t *)YAML_STR_TAG,
1263                                                              (yaml_char_t *)state,
1264                                                              strlen(state), 1, 0,
1265                                                              YAML_PLAIN_SCALAR_STYLE);
1266                                 rc = yaml_emitter_emit(&request, &event);
1267                                 if (rc == 0)
1268                                         goto emitter_error;
1269
1270                                 yaml_mapping_end_event_initialize(&event);
1271                                 rc = yaml_emitter_emit(&request, &event);
1272                                 if (rc == 0)
1273                                         goto emitter_error;
1274                         }
1275
1276                         yaml_sequence_end_event_initialize(&event);
1277                         rc = yaml_emitter_emit(&request, &event);
1278                         if (rc == 0)
1279                                 goto emitter_error;
1280                 } else {
1281                         yaml_scalar_event_initialize(&event, NULL,
1282                                                      (yaml_char_t *)YAML_STR_TAG,
1283                                                      (yaml_char_t *)"",
1284                                                      strlen(""), 1, 0,
1285                                                      YAML_PLAIN_SCALAR_STYLE);
1286                         rc = yaml_emitter_emit(&request, &event);
1287                         if (rc == 0)
1288                                 goto emitter_error;
1289                 }
1290
1291                 yaml_mapping_end_event_initialize(&event);
1292                 rc = yaml_emitter_emit(&request, &event);
1293                 if (rc == 0)
1294                         goto emitter_error;
1295
1296                 yaml_sequence_end_event_initialize(&event);
1297                 rc = yaml_emitter_emit(&request, &event);
1298                 if (rc == 0)
1299                         goto emitter_error;
1300         } else {
1301                 yaml_scalar_event_initialize(&event, NULL,
1302                                              (yaml_char_t *)YAML_STR_TAG,
1303                                              (yaml_char_t *)"",
1304                                              strlen(""), 1, 0,
1305                                              YAML_PLAIN_SCALAR_STYLE);
1306                 rc = yaml_emitter_emit(&request, &event);
1307                 if (rc == 0)
1308                         goto emitter_error;
1309         }
1310         yaml_mapping_end_event_initialize(&event);
1311         rc = yaml_emitter_emit(&request, &event);
1312         if (rc == 0)
1313                 goto emitter_error;
1314
1315         yaml_document_end_event_initialize(&event, 0);
1316         rc = yaml_emitter_emit(&request, &event);
1317         if (rc == 0)
1318                 goto emitter_error;
1319
1320         rc = yaml_emitter_close(&request);
1321         if (rc == 0) {
1322 emitter_error:
1323                 yaml_emitter_log_error(&request, stderr);
1324                 yaml_emitter_delete(&request);
1325                 rc = -EINVAL;
1326                 goto parser_error;
1327         }
1328         yaml_emitter_delete(&request);
1329
1330         /* display output */
1331         if (nlflags == NLM_F_DUMP)
1332                 rc = lst_yaml_display_groups(&reply, name, states, print);
1333 parser_error:
1334         if (rc == 0)
1335                 yaml_parser_log_error(&reply, stderr, NULL);
1336         yaml_parser_delete(&reply);
1337         nl_socket_free(sk);
1338
1339         if (print)
1340                 rc = rc == 1 ? 0 : -EINVAL;
1341
1342         return rc;
1343 }
1344
1345 static int
1346 lst_ping_ioctl(char *str, int type, int timeout,
1347                int count, struct lnet_process_id *ids, struct list_head *head)
1348 {
1349         struct lstio_debug_args args = { 0 };
1350
1351         args.lstio_dbg_key     = session_key;
1352         args.lstio_dbg_type    = type;
1353         args.lstio_dbg_flags   = 0;
1354         args.lstio_dbg_timeout = timeout;
1355         args.lstio_dbg_nmlen   = (str == NULL) ? 0: strlen(str);
1356         args.lstio_dbg_namep   = str;
1357         args.lstio_dbg_count   = count;
1358         args.lstio_dbg_idsp    = ids;
1359         args.lstio_dbg_resultp = head;
1360
1361         return lst_ioctl (LSTIO_DEBUG, &args, sizeof(args));
1362 }
1363
1364 static int
1365 lst_get_node_count(int type, char *str, int *countp,
1366                    struct lnet_process_id **idspp)
1367 {
1368         char                    buf[LST_NAME_SIZE];
1369         struct lstcon_test_batch_ent ent;
1370         struct lstcon_ndlist_ent    *entp = &ent.tbe_cli_nle;
1371         struct lst_sid sid;
1372         unsigned                feats;
1373         int                     key;
1374         int                     rc;
1375
1376         switch (type) {
1377         case LST_OPC_SESSION:
1378                 rc = lst_session_info_ioctl(buf, LST_NAME_SIZE,
1379                                             &key, &feats, &sid, entp);
1380                 break;
1381
1382         case LST_OPC_BATCHSRV:
1383                 entp = &ent.tbe_srv_nle;
1384         case LST_OPC_BATCHCLI:
1385                 rc = lst_info_batch_ioctl(str, 0, 0, &ent, NULL, NULL, NULL);
1386                 break;
1387
1388         case LST_OPC_GROUP:
1389                 rc = lst_yaml_groups(NLM_F_DUMP, NULL, 0, false);
1390                 if (rc == -EOPNOTSUPP) {
1391                         rc = lst_info_group_ioctl(str, entp, NULL, NULL, NULL);
1392                 } else if (rc > 0) {
1393                         entp->nle_nnode = rc;
1394                         rc = 0;
1395                 }
1396                 break;
1397
1398         case LST_OPC_NODES:
1399                 rc = lst_parse_nids(str, &entp->nle_nnode, idspp) < 0 ? -1 : 0;
1400                 break;
1401
1402         default:
1403                 rc = -1;
1404                 break;
1405         }
1406
1407         if (rc == 0)
1408                 *countp = entp->nle_nnode;
1409
1410         return rc;
1411 }
1412
1413 static int
1414 jt_lst_ping(int argc,  char **argv)
1415 {
1416         struct list_head   head;
1417         struct lnet_process_id *ids = NULL;
1418         struct lstcon_rpc_ent  *ent = NULL;
1419         char              *str = NULL;
1420         int                optidx  = 0;
1421         int                server  = 0;
1422         int                timeout = 5;
1423         int                count   = 0;
1424         int                type    = 0;
1425         int                rc      = 0;
1426         int                c;
1427
1428         static const struct option ping_opts[] = {
1429                 { .name = "session", .has_arg = no_argument,       .val = 's' },
1430                 { .name = "server",  .has_arg = no_argument,       .val = 'v' },
1431                 { .name = "batch",   .has_arg = required_argument, .val = 'b' },
1432                 { .name = "group",   .has_arg = required_argument, .val = 'g' },
1433                 { .name = "nodes",   .has_arg = required_argument, .val = 'n' },
1434                 { .name = "timeout", .has_arg = required_argument, .val = 't' },
1435                 { .name = NULL, } };
1436
1437         if (session_key == 0) {
1438                 fprintf(stderr,
1439                         "Can't find env LST_SESSION or value is not valid\n");
1440                 return -1;
1441         }
1442
1443         while (1) {
1444
1445                 c = getopt_long(argc, argv, "g:b:n:t:sv",
1446                                 ping_opts, &optidx);
1447
1448                 if (c == -1)
1449                         break;
1450
1451                 switch (c) {
1452                 case 's':
1453                         type = LST_OPC_SESSION;
1454                         break;
1455
1456                 case 'g':
1457                         type = LST_OPC_GROUP;
1458                         str = optarg;
1459                         break;
1460
1461                 case 'b':
1462                         type = LST_OPC_BATCHCLI;
1463                         str = optarg;
1464                         break;
1465
1466                 case 'n':
1467                         type = LST_OPC_NODES;
1468                         str = optarg;
1469                         break;
1470
1471                 case 't':
1472                         timeout = atoi(optarg);
1473                         break;
1474
1475                 case 'v':
1476                         server = 1;
1477                         break;
1478
1479                 default:
1480                         lst_print_usage(argv[0]);
1481                         return -1;
1482                 }
1483         }
1484
1485         if (type == 0 || timeout <= 0 || optind != argc) {
1486                 lst_print_usage(argv[0]);
1487                 return -1;
1488         }
1489
1490         if (type == LST_OPC_BATCHCLI && server)
1491                 type = LST_OPC_BATCHSRV;
1492
1493         rc = lst_get_node_count(type, str, &count, &ids);
1494         if (rc < 0) {
1495                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
1496                         (str == NULL) ? "session" : str, strerror(errno));
1497                 return -1;
1498         }
1499
1500         INIT_LIST_HEAD(&head);
1501
1502         rc = lst_alloc_rpcent(&head, count, LST_NAME_SIZE);
1503         if (rc != 0) {
1504                 fprintf(stderr, "Out of memory\n");
1505                 goto out;
1506         }
1507
1508         if (count == 0) {
1509                 fprintf(stdout, "Target %s is empty\n",
1510                         (str == NULL) ? "session" : str);
1511                 goto out;
1512         }
1513
1514         rc = lst_ping_ioctl(str, type, timeout, count, ids, &head);
1515         if (rc == -1) { /* local failure */
1516                 lst_print_error("debug", "Failed to ping %s: %s\n",
1517                                 (str == NULL) ? "session" : str,
1518                                 strerror(errno));
1519                 rc = -1;
1520                 goto out;
1521         }
1522
1523         /* ignore RPC errors and framwork errors */
1524         list_for_each_entry(ent, &head, rpe_link) {
1525                 fprintf(stdout, "\t%s: %s [session: %s id: %s]\n",
1526                         libcfs_id2str(ent->rpe_peer),
1527                         lst_node_state2str(ent->rpe_state),
1528                         (ent->rpe_state == LST_NODE_ACTIVE ||
1529                          ent->rpe_state == LST_NODE_BUSY) ?
1530                                 (ent->rpe_rpc_errno == 0 ?
1531                                         &ent->rpe_payload[0] : "Unknown") :
1532                                 "<NULL>", libcfs_nid2str(ent->rpe_sid.ses_nid));
1533         }
1534
1535 out:
1536         lst_free_rpcent(&head);
1537
1538         if (ids != NULL)
1539                 free(ids);
1540
1541         return rc;
1542
1543 }
1544
1545 static int
1546 lst_add_nodes_ioctl(char *name, int count, struct lnet_process_id *ids,
1547                     unsigned *featp, struct list_head *resultp)
1548 {
1549         struct lstio_group_nodes_args args = { 0 };
1550
1551         args.lstio_grp_key     = session_key;
1552         args.lstio_grp_nmlen   = strlen(name);
1553         args.lstio_grp_namep   = name;
1554         args.lstio_grp_count   = count;
1555         args.lstio_grp_featp   = featp;
1556         args.lstio_grp_idsp    = ids;
1557         args.lstio_grp_resultp = resultp;
1558
1559         return lst_ioctl(LSTIO_NODES_ADD, &args, sizeof(args));
1560 }
1561
1562 static int
1563 lst_del_group_ioctl(char *name)
1564 {
1565         struct lstio_group_del_args args = { 0 };
1566
1567         args.lstio_grp_key   = session_key;
1568         args.lstio_grp_nmlen = strlen(name);
1569         args.lstio_grp_namep = name;
1570
1571         return lst_ioctl(LSTIO_GROUP_DEL, &args, sizeof(args));
1572 }
1573
1574 static int
1575 lst_del_group(char *grp_name)
1576 {
1577         int     rc;
1578
1579         rc = lst_del_group_ioctl(grp_name);
1580         if (rc == 0) {
1581                 fprintf(stdout, "Group is deleted\n");
1582                 return 0;
1583         }
1584
1585         if (rc == -1) {
1586                 lst_print_error("group", "Failed to delete group: %s\n",
1587                                 strerror(errno));
1588                 return rc;
1589         }
1590
1591         fprintf(stderr, "Group is deleted with some errors\n");
1592
1593         if (trans_stat.trs_rpc_errno != 0) {
1594                 fprintf(stderr,
1595                         "[RPC] Failed to send %d end session RPCs: %s\n",
1596                         lstcon_rpc_stat_failure(&trans_stat, 0),
1597                         strerror(trans_stat.trs_rpc_errno));
1598         }
1599
1600         if (trans_stat.trs_fwk_errno != 0) {
1601                 fprintf(stderr,
1602                         "[FWK] Failed to end session on %d nodes: %s\n",
1603                 lstcon_sesop_stat_failure(&trans_stat, 0),
1604                 strerror(trans_stat.trs_fwk_errno));
1605         }
1606
1607         return -1;
1608 }
1609
1610 static int
1611 lst_add_group_ioctl(char *name)
1612 {
1613         struct lstio_group_add_args args = { 0 };
1614
1615         args.lstio_grp_key     =  session_key;
1616         args.lstio_grp_nmlen   =  strlen(name);
1617         args.lstio_grp_namep   =  name;
1618
1619         return lst_ioctl(LSTIO_GROUP_ADD, &args, sizeof(args));
1620 }
1621
1622 static int
1623 jt_lst_add_group(int argc, char **argv)
1624 {
1625         struct list_head   head;
1626         struct lnet_process_id *ids;
1627         char              *name;
1628         unsigned           feats = session_features;
1629         int                count;
1630         int                rc;
1631         int                i;
1632         bool               nodes_added = false;
1633
1634         if (session_key == 0) {
1635                 fprintf(stderr,
1636                         "Can't find env LST_SESSION or value is not valid\n");
1637                 return -1;
1638         }
1639
1640         if (argc < 3) {
1641                 lst_print_usage(argv[0]);
1642                 return -1;
1643         }
1644
1645         name = argv[1];
1646         if (strlen(name) >= LST_NAME_SIZE) {
1647                 fprintf(stderr, "Name length is limited to %d\n",
1648                         LST_NAME_SIZE - 1);
1649                 return -1;
1650         }
1651
1652         rc = lst_add_group_ioctl(name);
1653         if (rc != 0) {
1654                 lst_print_error("group", "Failed to add group %s: %s\n",
1655                                 name, strerror(errno));
1656                 return -1;
1657         }
1658
1659         INIT_LIST_HEAD(&head);
1660
1661         for (i = 2; i < argc; i++) {
1662                 /* parse address list */
1663                 rc = lst_parse_nids(argv[i], &count, &ids);
1664                 if (rc < 0) {
1665                         fprintf(stderr, "Ignore invalid id list %s\n",
1666                                 argv[i]);
1667                         continue;
1668                 }
1669
1670                 if (count == 0)
1671                         continue;
1672
1673                 rc = lst_alloc_rpcent(&head, count, 0);
1674                 if (rc != 0) {
1675                         fprintf(stderr, "Out of memory\n");
1676                         free(ids);
1677                         rc = -1;
1678                         goto failed;
1679                 }
1680
1681                 rc = lst_add_nodes_ioctl(name, count, ids, &feats, &head);
1682
1683                 free(ids);
1684
1685                 if (rc != 0)
1686                         goto failed;
1687
1688                 fprintf(stdout, "%s are added to session\n", argv[i]);
1689
1690                 nodes_added = true;
1691
1692                 if ((feats & session_features) != session_features) {
1693                         fprintf(stdout,
1694                                 "Warning, this session will run with "
1695                                 "compatible mode because some test nodes "
1696                                 "might not understand these features: %x\n",
1697                                 (~feats & session_features));
1698                 }
1699
1700                 lst_free_rpcent(&head);
1701         }
1702
1703         if (!nodes_added) {
1704                 /*
1705                  * The selftest kernel module expects that a group should
1706                  * have at least one node, since it doesn't make sense for
1707                  * an empty group to be added to a test.
1708                  */
1709                 fprintf(stderr,
1710                         "No nodes added successfully, deleting group %s\n",
1711                         name);
1712                 rc = lst_del_group(name);
1713                 if (rc != 0) {
1714                         fprintf(stderr,
1715                                 "Failed to delete group %s."
1716                                 "  Group is empty.\n", name);
1717                 }
1718         }
1719
1720         return rc;
1721
1722 failed:
1723         if (rc == -1) {
1724                 lst_print_error("group", "Failed to add nodes %s: %s\n",
1725                                 argv[i], strerror(errno));
1726
1727         } else {
1728                 if (trans_stat.trs_fwk_errno == EPROTO) {
1729                         fprintf(stderr,
1730                                 "test nodes might have different LST "
1731                                 "features, please disable some features by "
1732                                 "setting LST_FEATURES\n");
1733                 }
1734
1735                 lst_print_transerr(&head, "create session");
1736         }
1737
1738         lst_free_rpcent(&head);
1739
1740         if (!nodes_added) {
1741                 fprintf(stderr,
1742                         "No nodes added successfully, deleting group %s\n",
1743                         name);
1744                 if (lst_del_group(name) != 0) {
1745                         fprintf(stderr,
1746                                 "Failed to delete group %s."
1747                                 "  Group is empty.\n", name);
1748                 }
1749         }
1750
1751         return rc;
1752 }
1753
1754 static int
1755 jt_lst_del_group(int argc, char **argv)
1756 {
1757         int     rc;
1758
1759         if (session_key == 0) {
1760                 fprintf(stderr,
1761                         "Can't find env LST_SESSION or value is not valid\n");
1762                 return -1;
1763         }
1764
1765         if (argc != 2) {
1766                 lst_print_usage(argv[0]);
1767                 return -1;
1768         }
1769
1770         rc = lst_del_group(argv[1]);
1771
1772         return rc;
1773 }
1774
1775 static int
1776 lst_update_group_ioctl(int opc, char *name, int clean, int count,
1777                        struct lnet_process_id *ids, struct list_head *resultp)
1778 {
1779         struct lstio_group_update_args args = { 0 };
1780
1781         args.lstio_grp_key      = session_key;
1782         args.lstio_grp_opc      = opc;
1783         args.lstio_grp_args     = clean;
1784         args.lstio_grp_nmlen    = strlen(name);
1785         args.lstio_grp_namep    = name;
1786         args.lstio_grp_count    = count;
1787         args.lstio_grp_idsp     = ids;
1788         args.lstio_grp_resultp  = resultp;
1789
1790         return lst_ioctl(LSTIO_GROUP_UPDATE, &args, sizeof(args));
1791 }
1792
1793 static int
1794 jt_lst_update_group(int argc, char **argv)
1795 {
1796         struct list_head   head;
1797         struct lnet_process_id *ids = NULL;
1798         char              *str = NULL;
1799         char              *grp = NULL;
1800         int                optidx = 0;
1801         int                count = 0;
1802         int                clean = 0;
1803         int                opc = 0;
1804         int                rc;
1805         int                c;
1806
1807         static const struct option update_group_opts[] = {
1808                 { .name = "refresh", .has_arg = no_argument,       .val = 'f' },
1809                 { .name = "clean",   .has_arg = required_argument, .val = 'c' },
1810                 { .name = "remove",  .has_arg = required_argument, .val = 'r' },
1811                 { .name = NULL } };
1812
1813         if (session_key == 0) {
1814                 fprintf(stderr,
1815                         "Can't find env LST_SESSION or value is not valid\n");
1816                 return -1;
1817         }
1818
1819         while (1) {
1820                 c = getopt_long(argc, argv, "fc:r:",
1821                                 update_group_opts, &optidx);
1822
1823                 /* Detect the end of the options. */
1824                 if (c == -1)
1825                         break;
1826
1827                 switch (c) {
1828                 case 'f':
1829                         if (opc != 0) {
1830                                 lst_print_usage(argv[0]);
1831                                 return -1;
1832                         }
1833                         opc = LST_GROUP_REFRESH;
1834                         break;
1835
1836                 case 'r':
1837                         if (opc != 0) {
1838                                 lst_print_usage(argv[0]);
1839                                 return -1;
1840                         }
1841                         opc = LST_GROUP_RMND;
1842                         str = optarg;
1843                         break;
1844
1845                 case 'c':
1846                         clean = lst_node_str2state(optarg);
1847                         if (opc != 0 || clean <= 0) {
1848                                 lst_print_usage(argv[0]);
1849                                 return -1;
1850                         }
1851                         opc = LST_GROUP_CLEAN;
1852                         break;
1853
1854                 default:
1855                         lst_print_usage(argv[0]);
1856                         return -1;
1857                 }
1858         }
1859
1860         /* no OPC or group is specified */
1861         if (opc == 0 || optind != argc - 1) {
1862                 lst_print_usage(argv[0]);
1863                 return -1;
1864         }
1865
1866         grp = argv[optind];
1867
1868         INIT_LIST_HEAD(&head);
1869
1870         if (opc == LST_GROUP_RMND || opc == LST_GROUP_REFRESH) {
1871                 rc = lst_get_node_count(opc == LST_GROUP_RMND ? LST_OPC_NODES :
1872                                                                 LST_OPC_GROUP,
1873                                         opc == LST_GROUP_RMND ? str : grp,
1874                                         &count, &ids);
1875
1876                 if (rc != 0) {
1877                         fprintf(stderr, "Can't get count of nodes from %s: %s\n",
1878                                 opc == LST_GROUP_RMND ? str : grp,
1879                                 strerror(errno));
1880                         return -1;
1881                 }
1882
1883                 rc = lst_alloc_rpcent(&head, count, 0);
1884                 if (rc != 0) {
1885                         fprintf(stderr, "Out of memory\n");
1886                         free(ids);
1887                         return -1;
1888                 }
1889
1890         }
1891
1892         rc = lst_update_group_ioctl(opc, grp, clean, count, ids, &head);
1893
1894         if (ids != NULL)
1895                 free(ids);
1896
1897         if (rc == 0) {
1898                 lst_free_rpcent(&head);
1899                 return 0;
1900         }
1901
1902         if (rc == -1) {
1903                 lst_free_rpcent(&head);
1904                 lst_print_error("group", "Failed to update group: %s\n",
1905                                 strerror(errno));
1906                 return rc;
1907         }
1908
1909         lst_print_transerr(&head, "Updating group");
1910
1911         lst_free_rpcent(&head);
1912
1913         return rc;
1914 }
1915
1916 static int
1917 lst_list_group_ioctl(int len, char *name, int idx)
1918 {
1919         struct lstio_group_list_args args = { 0 };
1920
1921         args.lstio_grp_key   = session_key;
1922         args.lstio_grp_idx   = idx;
1923         args.lstio_grp_nmlen = len;
1924         args.lstio_grp_namep = name;
1925
1926         return lst_ioctl(LSTIO_GROUP_LIST, &args, sizeof(args));
1927 }
1928
1929 static int
1930 lst_info_group_ioctl(char *name, struct lstcon_ndlist_ent *gent,
1931                      int *idx, int *count, struct lstcon_node_ent *dents)
1932 {
1933         struct lstio_group_info_args args = { 0 };
1934
1935         args.lstio_grp_key    = session_key;
1936         args.lstio_grp_nmlen  = strlen(name);
1937         args.lstio_grp_namep  = name;
1938         args.lstio_grp_entp   = gent;
1939         args.lstio_grp_idxp   = idx;
1940         args.lstio_grp_ndentp = count;
1941         args.lstio_grp_dentsp = dents;
1942
1943         return lst_ioctl(LSTIO_GROUP_INFO, &args, sizeof(args));
1944 }
1945
1946 static int
1947 lst_list_group_all(void)
1948 {
1949         char  name[LST_NAME_SIZE];
1950         int   rc;
1951         int   i;
1952
1953         /* no group is specified, list name of all groups */
1954         for (i = 0; ; i++) {
1955                 rc = lst_list_group_ioctl(LST_NAME_SIZE, name, i);
1956                 if (rc == 0) {
1957                         fprintf(stdout, "%d) %s\n", i + 1, name);
1958                         continue;
1959                 }
1960
1961                 if (errno == ENOENT)
1962                         break;
1963
1964                 lst_print_error("group", "Failed to list group: %s\n",
1965                                 strerror(errno));
1966                 return -1;
1967         }
1968
1969         fprintf(stdout, "Total %d groups\n", i);
1970
1971         return 0;
1972 }
1973
1974 static int
1975 jt_lst_list_group(int argc, char **argv)
1976 {
1977         struct lstcon_ndlist_ent gent;
1978         struct lstcon_node_ent   *dents;
1979         int states = 0;
1980         int optidx  = 0;
1981         int verbose = 0;
1982         int active  = 0;
1983         int busy    = 0;
1984         int down    = 0;
1985         int unknown = 0;
1986         int all     = 0;
1987         int count;
1988         int index;
1989         int i;
1990         int j;
1991         int c;
1992         int rc      = 0;
1993         static const struct option list_group_opts[] = {
1994                 { .name = "active",  .has_arg = no_argument, .val = 'a' },
1995                 { .name = "busy",    .has_arg = no_argument, .val = 'b' },
1996                 { .name = "down",    .has_arg = no_argument, .val = 'd' },
1997                 { .name = "unknown", .has_arg = no_argument, .val = 'u' },
1998                 { .name = "all",     .has_arg = no_argument, .val = 'l' },
1999                 { .name = NULL, } };
2000
2001         if (session_key == 0) {
2002                 fprintf(stderr,
2003                         "Can't find env LST_SESSION or value is not valid\n");
2004                 return -1;
2005         }
2006
2007         while (1) {
2008                 c = getopt_long(argc, argv, "abdul",
2009                                 list_group_opts, &optidx);
2010
2011                 if (c == -1)
2012                         break;
2013
2014                 switch (c) {
2015                 case 'a':
2016                         verbose = active = 1;
2017                         states |= LST_NODE_ACTIVE;
2018                         all = 0;
2019                         break;
2020                 case 'b':
2021                         verbose = busy = 1;
2022                         states |= LST_NODE_BUSY;
2023                         all = 0;
2024                         break;
2025                 case 'd':
2026                         verbose = down = 1;
2027                         states |= LST_NODE_DOWN;
2028                         all = 0;
2029                         break;
2030                 case 'u':
2031                         verbose = unknown = 1;
2032                         states |= LST_NODE_UNKNOWN;
2033                         all = 0;
2034                         break;
2035                 case 'l':
2036                         states |= LST_NODE_ACTIVE | LST_NODE_BUSY |
2037                                   LST_NODE_DOWN | LST_NODE_UNKNOWN;
2038                         verbose = all = 1;
2039                         break;
2040                 default:
2041                         lst_print_usage(argv[0]);
2042                         return -1;
2043                 }
2044         }
2045
2046         if (optind == argc) {
2047                 rc = lst_yaml_groups(NLM_F_DUMP, NULL, 0, true);
2048                 if (rc <= 0) {
2049                         if (rc == -EOPNOTSUPP)
2050                                 goto old_api;
2051                         return rc;
2052                 }
2053         } else {
2054                 for (i = optind; i < argc; i++) {
2055                         rc = lst_yaml_groups(NLM_F_DUMP, argv[i], states,
2056                                              i == optind ? true : false);
2057                         if (rc < 0) {
2058                                 if (rc == -EOPNOTSUPP)
2059                                         goto old_api;
2060                                 return rc;
2061                         }
2062                 }
2063                 return 0;
2064         }
2065 old_api:
2066         if (optind == argc) {
2067                 /* no group is specified, list name of all groups */
2068                 rc = lst_list_group_all();
2069
2070                 return rc;
2071         }
2072
2073         if (!verbose)
2074                 fprintf(stdout, LST_NODES_TITLE);
2075
2076         /* list nodes in specified groups */
2077         for (i = optind; i < argc; i++) {
2078                 rc = lst_info_group_ioctl(argv[i], &gent, NULL, NULL, NULL);
2079                 if (rc != 0) {
2080                         if (errno == ENOENT) {
2081                                 rc = 0;
2082                                 break;
2083                         }
2084
2085                         lst_print_error("group", "Failed to list group: %s\n",
2086                                         strerror(errno));
2087                         break;
2088                 }
2089
2090                 if (!verbose) {
2091                         fprintf(stdout, "\t%d\t%d\t%d\t%d\t%d\t%s\n",
2092                                 gent.nle_nactive, gent.nle_nbusy,
2093                                 gent.nle_ndown, gent.nle_nunknown,
2094                                 gent.nle_nnode, argv[i]);
2095                         continue;
2096                 }
2097
2098                 fprintf(stdout, "Group [ %s ]\n", argv[i]);
2099
2100                 if (gent.nle_nnode == 0) {
2101                         fprintf(stdout, "No nodes found [ %s ]\n", argv[i]);
2102                         continue;
2103                 }
2104
2105                 count = gent.nle_nnode;
2106
2107                 dents = malloc(count * sizeof(struct lstcon_node_ent));
2108                 if (dents == NULL) {
2109                         fprintf(stderr, "Failed to malloc: %s\n",
2110                                 strerror(errno));
2111                         return -1;
2112                 }
2113
2114                 index = 0;
2115                 rc = lst_info_group_ioctl(argv[i], &gent, &index, &count, dents);
2116                 if (rc != 0) {
2117                         lst_print_error("group", "Failed to list group: %s\n",
2118                                         strerror(errno));
2119                         free(dents);
2120                         return -1;
2121                 }
2122
2123                 for (j = 0, c = 0; j < count; j++) {
2124                         if (all ||
2125                             ((active  &&  dents[j].nde_state == LST_NODE_ACTIVE) ||
2126                              (busy    &&  dents[j].nde_state == LST_NODE_BUSY)   ||
2127                              (down    &&  dents[j].nde_state == LST_NODE_DOWN)   ||
2128                              (unknown &&  dents[j].nde_state == LST_NODE_UNKNOWN))) {
2129
2130                                 fprintf(stdout, "\t%s: %s\n",
2131                                         libcfs_id2str(dents[j].nde_id),
2132                                         lst_node_state2str(dents[j].nde_state));
2133                                 c++;
2134                         }
2135                 }
2136
2137                 fprintf(stdout, "Total %d nodes [ %s ]\n", c, argv[i]);
2138
2139                 free(dents);
2140         }
2141
2142         return rc;
2143 }
2144
2145 static int
2146 lst_stat_ioctl(char *name, int count, struct lnet_process_id *idsp,
2147                int timeout, struct list_head *resultp)
2148 {
2149         struct lstio_stat_args args = { 0 };
2150
2151         args.lstio_sta_key     = session_key;
2152         args.lstio_sta_timeout = timeout;
2153         args.lstio_sta_nmlen   = strlen(name);
2154         args.lstio_sta_namep   = name;
2155         args.lstio_sta_count   = count;
2156         args.lstio_sta_idsp    = idsp;
2157         args.lstio_sta_resultp = resultp;
2158
2159         return lst_ioctl(LSTIO_STAT_QUERY, &args, sizeof(args));
2160 }
2161
2162 typedef struct {
2163         struct list_head              srp_link;
2164         int                     srp_count;
2165         char                   *srp_name;
2166         struct lnet_process_id      *srp_ids;
2167         struct list_head              srp_result[2];
2168 } lst_stat_req_param_t;
2169
2170 static void
2171 lst_stat_req_param_free(lst_stat_req_param_t *srp)
2172 {
2173         int     i;
2174
2175         for (i = 0; i < 2; i++)
2176                 lst_free_rpcent(&srp->srp_result[i]);
2177
2178         if (srp->srp_ids != NULL)
2179                 free(srp->srp_ids);
2180
2181         free(srp);
2182 }
2183
2184 static int
2185 lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp, int save_old)
2186 {
2187         lst_stat_req_param_t *srp = NULL;
2188         int                   count = save_old ? 2 : 1;
2189         int                   rc;
2190         int                   i;
2191
2192         srp = malloc(sizeof(*srp));
2193         if (srp == NULL)
2194                 return -ENOMEM;
2195
2196         memset(srp, 0, sizeof(*srp));
2197         INIT_LIST_HEAD(&srp->srp_result[0]);
2198         INIT_LIST_HEAD(&srp->srp_result[1]);
2199
2200         rc = lst_get_node_count(LST_OPC_GROUP, name,
2201                                 &srp->srp_count, NULL);
2202         if (rc != 0 && errno == ENOENT) {
2203                 rc = lst_get_node_count(LST_OPC_NODES, name,
2204                                         &srp->srp_count, &srp->srp_ids);
2205         }
2206
2207         if (rc != 0) {
2208                 fprintf(stderr,
2209                         "Failed to get count of nodes from %s: %s\n",
2210                         name, strerror(errno));
2211                 lst_stat_req_param_free(srp);
2212
2213                 return rc;
2214         }
2215
2216         srp->srp_name = name;
2217
2218         for (i = 0; i < count; i++) {
2219                 rc = lst_alloc_rpcent(&srp->srp_result[i], srp->srp_count,
2220                                       sizeof(struct sfw_counters)  +
2221                                       sizeof(struct srpc_counters) +
2222                                       sizeof(struct lnet_counters_common));
2223                 if (rc != 0) {
2224                         fprintf(stderr, "Out of memory\n");
2225                         break;
2226                 }
2227         }
2228
2229         if (rc == 0) {
2230                 *srpp = srp;
2231                 return 0;
2232         }
2233
2234         lst_stat_req_param_free(srp);
2235
2236         return rc;
2237 }
2238
2239 typedef struct {
2240         /* TODO */
2241         int foo;
2242 } lst_srpc_stat_result;
2243
2244 #define LST_LNET_AVG    0
2245 #define LST_LNET_MIN    1
2246 #define LST_LNET_MAX    2
2247
2248 typedef struct {
2249         float           lnet_avg_sndrate;
2250         float           lnet_min_sndrate;
2251         float           lnet_max_sndrate;
2252         float           lnet_total_sndrate;
2253
2254         float           lnet_avg_rcvrate;
2255         float           lnet_min_rcvrate;
2256         float           lnet_max_rcvrate;
2257         float           lnet_total_rcvrate;
2258
2259         float           lnet_avg_sndperf;
2260         float           lnet_min_sndperf;
2261         float           lnet_max_sndperf;
2262         float           lnet_total_sndperf;
2263
2264         float           lnet_avg_rcvperf;
2265         float           lnet_min_rcvperf;
2266         float           lnet_max_rcvperf;
2267         float           lnet_total_rcvperf;
2268
2269         int             lnet_stat_count;
2270 } lst_lnet_stat_result_t;
2271
2272 lst_lnet_stat_result_t lnet_stat_result;
2273
2274 static float
2275 lst_lnet_stat_value(int bw, int send, int off)
2276 {
2277         float  *p;
2278
2279         p = bw ? &lnet_stat_result.lnet_avg_sndperf :
2280                  &lnet_stat_result.lnet_avg_sndrate;
2281
2282         if (!send)
2283                 p += 4;
2284
2285         p += off;
2286
2287         return *p;
2288 }
2289
2290 static void
2291 lst_cal_lnet_stat(float delta, struct lnet_counters_common *lnet_new,
2292                   struct lnet_counters_common *lnet_old, int mbs)
2293 {
2294         float perf;
2295         float rate;
2296         unsigned int unit_divisor;
2297
2298         unit_divisor = (mbs) ? (1000 * 1000) : (1024 * 1024);
2299         perf = (float)(lnet_new->lcc_send_length -
2300                        lnet_old->lcc_send_length) / unit_divisor / delta;
2301         lnet_stat_result.lnet_total_sndperf += perf;
2302
2303         if (lnet_stat_result.lnet_min_sndperf > perf ||
2304             lnet_stat_result.lnet_min_sndperf == 0)
2305                 lnet_stat_result.lnet_min_sndperf = perf;
2306
2307         if (lnet_stat_result.lnet_max_sndperf < perf)
2308                 lnet_stat_result.lnet_max_sndperf = perf;
2309
2310         perf = (float)(lnet_new->lcc_recv_length -
2311                        lnet_old->lcc_recv_length) / unit_divisor / delta;
2312         lnet_stat_result.lnet_total_rcvperf += perf;
2313
2314         if (lnet_stat_result.lnet_min_rcvperf > perf ||
2315             lnet_stat_result.lnet_min_rcvperf == 0)
2316                 lnet_stat_result.lnet_min_rcvperf = perf;
2317
2318         if (lnet_stat_result.lnet_max_rcvperf < perf)
2319                 lnet_stat_result.lnet_max_rcvperf = perf;
2320
2321         rate = (lnet_new->lcc_send_count - lnet_old->lcc_send_count) / delta;
2322         lnet_stat_result.lnet_total_sndrate += rate;
2323
2324         if (lnet_stat_result.lnet_min_sndrate > rate ||
2325             lnet_stat_result.lnet_min_sndrate == 0)
2326                 lnet_stat_result.lnet_min_sndrate = rate;
2327
2328         if (lnet_stat_result.lnet_max_sndrate < rate)
2329                 lnet_stat_result.lnet_max_sndrate = rate;
2330
2331         rate = (lnet_new->lcc_recv_count - lnet_old->lcc_recv_count) / delta;
2332         lnet_stat_result.lnet_total_rcvrate += rate;
2333
2334         if (lnet_stat_result.lnet_min_rcvrate > rate ||
2335             lnet_stat_result.lnet_min_rcvrate == 0)
2336                 lnet_stat_result.lnet_min_rcvrate = rate;
2337
2338         if (lnet_stat_result.lnet_max_rcvrate < rate)
2339                 lnet_stat_result.lnet_max_rcvrate = rate;
2340
2341         lnet_stat_result.lnet_stat_count++;
2342
2343         lnet_stat_result.lnet_avg_sndrate = lnet_stat_result.lnet_total_sndrate /
2344                                             lnet_stat_result.lnet_stat_count;
2345         lnet_stat_result.lnet_avg_rcvrate = lnet_stat_result.lnet_total_rcvrate /
2346                                             lnet_stat_result.lnet_stat_count;
2347
2348         lnet_stat_result.lnet_avg_sndperf = lnet_stat_result.lnet_total_sndperf /
2349                                             lnet_stat_result.lnet_stat_count;
2350         lnet_stat_result.lnet_avg_rcvperf = lnet_stat_result.lnet_total_rcvperf /
2351                                             lnet_stat_result.lnet_stat_count;
2352 }
2353
2354 static void
2355 lst_print_lnet_stat(char *name, int bwrt, int rdwr, int type, int mbs)
2356 {
2357         int     start1 = 0;
2358         int     end1   = 1;
2359         int     start2 = 0;
2360         int     end2   = 1;
2361         int     i;
2362         int     j;
2363         char   *units;
2364
2365         if (lnet_stat_result.lnet_stat_count == 0)
2366                 return;
2367
2368         units = (mbs) ? "MB/s  " : "MiB/s ";
2369
2370         if (bwrt == 1) /* bw only */
2371                 start1 = 1;
2372
2373         if (bwrt == 2) /* rates only */
2374                 end1 = 0;
2375
2376         if (rdwr == 1) /* recv only */
2377                 start2 = 1;
2378
2379         if (rdwr == 2) /* send only */
2380                 end2 = 0;
2381
2382         for (i = start1; i <= end1; i++) {
2383                 fprintf(stdout, "[LNet %s of %s]\n",
2384                         i == 0 ? "Rates" : "Bandwidth", name);
2385
2386                 for (j = start2; j <= end2; j++) {
2387                         fprintf(stdout, "[%c] ", j == 0 ? 'R' : 'W');
2388
2389                         if ((type & 1) != 0) {
2390                                 fprintf(stdout, i == 0 ? "Avg: %-8.0f RPC/s " :
2391                                                          "Avg: %-8.2f %s",
2392                                         lst_lnet_stat_value(i, j, 0), units);
2393                         }
2394
2395                         if ((type & 2) != 0) {
2396                                 fprintf(stdout, i == 0 ? "Min: %-8.0f RPC/s " :
2397                                                          "Min: %-8.2f %s",
2398                                         lst_lnet_stat_value(i, j, 1), units);
2399                         }
2400
2401                         if ((type & 4) != 0) {
2402                                 fprintf(stdout, i == 0 ? "Max: %-8.0f RPC/s" :
2403                                                          "Max: %-8.2f %s",
2404                                         lst_lnet_stat_value(i, j, 2), units);
2405                         }
2406
2407                         fprintf(stdout, "\n");
2408                 }
2409         }
2410 }
2411
2412 static void
2413 lst_print_stat(char *name, struct list_head *resultp,
2414                int idx, int lnet, int bwrt, int rdwr, int type,
2415                int mbs)
2416 {
2417         struct list_head tmp[2];
2418         struct lstcon_rpc_ent *new;
2419         struct lstcon_rpc_ent *old;
2420         struct sfw_counters *sfwk_new;
2421         struct sfw_counters *sfwk_old;
2422         struct srpc_counters *srpc_new;
2423         struct srpc_counters *srpc_old;
2424         struct lnet_counters_common *lnet_new;
2425         struct lnet_counters_common *lnet_old;
2426         float delta;
2427         int errcount = 0;
2428
2429         INIT_LIST_HEAD(&tmp[0]);
2430         INIT_LIST_HEAD(&tmp[1]);
2431
2432         memset(&lnet_stat_result, 0, sizeof(lnet_stat_result));
2433
2434         while (!list_empty(&resultp[idx])) {
2435                 if (list_empty(&resultp[1 - idx])) {
2436                         fprintf(stderr, "Group is changed, re-run stat\n");
2437                         break;
2438                 }
2439
2440                 new = list_first_entry(&resultp[idx], struct lstcon_rpc_ent,
2441                                        rpe_link);
2442                 old = list_first_entry(&resultp[1 - idx], struct lstcon_rpc_ent,
2443                                        rpe_link);
2444
2445                 /* first time get stats result, can't calculate diff */
2446                 if (new->rpe_peer.nid == LNET_NID_ANY)
2447                         break;
2448
2449                 if (new->rpe_peer.nid != old->rpe_peer.nid ||
2450                     new->rpe_peer.pid != old->rpe_peer.pid) {
2451                         /* Something wrong. i.e, somebody change the group */
2452                         break;
2453                 }
2454
2455                 list_move_tail(&new->rpe_link, &tmp[idx]);
2456
2457                 list_move_tail(&old->rpe_link, &tmp[1 - idx]);
2458
2459                 if (new->rpe_rpc_errno != 0 || new->rpe_fwk_errno != 0 ||
2460                     old->rpe_rpc_errno != 0 || old->rpe_fwk_errno != 0) {
2461                         errcount++;
2462                         continue;
2463                 }
2464
2465                 sfwk_new = (struct sfw_counters *)&new->rpe_payload[0];
2466                 sfwk_old = (struct sfw_counters *)&old->rpe_payload[0];
2467
2468                 srpc_new = (struct srpc_counters *)((char *)sfwk_new +
2469                                                     sizeof(*sfwk_new));
2470                 srpc_old = (struct srpc_counters *)((char *)sfwk_old +
2471                                                     sizeof(*sfwk_old));
2472
2473                 lnet_new = (struct lnet_counters_common *)((char *)srpc_new +
2474                                                            sizeof(*srpc_new));
2475                 lnet_old = (struct lnet_counters_common *)((char *)srpc_old +
2476                                                            sizeof(*srpc_old));
2477
2478                 /* Prior to version 2.3, the running_ms was a counter for
2479                  * the number of running tests. Since 2.3, running_ms is
2480                  * changed to hold the millisecond since the start of
2481                  * the work item. The rpe_stamp field was formerly used,
2482                  * but is no longer. In 2.12 rpe_stamp was changed to
2483                  * struct timespec64 and has nanosecond resolution, in
2484                  * case it is needed in the future.
2485                  */
2486                 delta = (float)(sfwk_new->running_ms -
2487                                 sfwk_old->running_ms) / 1000;
2488
2489                 if (!lnet) /* TODO */
2490                         continue;
2491
2492                 lst_cal_lnet_stat(delta, lnet_new, lnet_old, mbs);
2493         }
2494
2495         list_splice(&tmp[idx], &resultp[idx]);
2496         list_splice(&tmp[1 - idx], &resultp[1 - idx]);
2497
2498         if (errcount > 0)
2499                 fprintf(stdout, "Failed to stat on %d nodes\n", errcount);
2500
2501         if (!lnet)  /* TODO */
2502                 return;
2503
2504         lst_print_lnet_stat(name, bwrt, rdwr, type, mbs);
2505 }
2506
2507 static int
2508 jt_lst_stat(int argc, char **argv)
2509 {
2510         struct list_head        head;
2511         lst_stat_req_param_t *srp;
2512         time_t                last    = 0;
2513         int                   optidx  = 0;
2514         int                   timeout = 5; /* default timeout, 5 sec */
2515         int                   delay   = 5; /* default delay, 5 sec */
2516         int                   count   = -1; /* run forever */
2517         int                   lnet    = 1; /* lnet stat by default */
2518         int                   bwrt    = 0;
2519         int                   rdwr    = 0;
2520         int                   type    = -1;
2521         int                   idx     = 0;
2522         int                   rc;
2523         int                   c;
2524         int                   mbs     = 0; /* report as MB/s */
2525
2526         static const struct option stat_opts[] = {
2527                 { .name = "timeout", .has_arg = required_argument, .val = 't' },
2528                 { .name = "delay",   .has_arg = required_argument, .val = 'd' },
2529                 { .name = "count",   .has_arg = required_argument, .val = 'o' },
2530                 { .name = "lnet",    .has_arg = no_argument,       .val = 'l' },
2531                 { .name = "rpc",     .has_arg = no_argument,       .val = 'c' },
2532                 { .name = "bw",      .has_arg = no_argument,       .val = 'b' },
2533                 { .name = "rate",    .has_arg = no_argument,       .val = 'a' },
2534                 { .name = "read",    .has_arg = no_argument,       .val = 'r' },
2535                 { .name = "write",   .has_arg = no_argument,       .val = 'w' },
2536                 { .name = "avg",     .has_arg = no_argument,       .val = 'g' },
2537                 { .name = "min",     .has_arg = no_argument,       .val = 'n' },
2538                 { .name = "max",     .has_arg = no_argument,       .val = 'x' },
2539                 { .name = "mbs",     .has_arg = no_argument,       .val = 'm' },
2540                 { .name = NULL } };
2541
2542         if (session_key == 0) {
2543                 fprintf(stderr,
2544                         "Can't find env LST_SESSION or value is not valid\n");
2545                 return -1;
2546         }
2547
2548         while (1) {
2549                 c = getopt_long(argc, argv, "t:d:lcbarwgnxm", stat_opts,
2550                                 &optidx);
2551
2552                 if (c == -1)
2553                         break;
2554
2555                 switch (c) {
2556                 case 't':
2557                         timeout = atoi(optarg);
2558                         break;
2559                 case 'd':
2560                         delay = atoi(optarg);
2561                         break;
2562                 case 'o':
2563                         count = atoi(optarg);
2564                         break;
2565                 case 'l':
2566                         lnet = 1;
2567                         break;
2568                 case 'c':
2569                         lnet = 0;
2570                         break;
2571                 case 'b':
2572                         bwrt |= 1;
2573                         break;
2574                 case 'a':
2575                         bwrt |= 2;
2576                         break;
2577                 case 'r':
2578                         rdwr |= 1;
2579                         break;
2580                 case 'w':
2581                         rdwr |= 2;
2582                         break;
2583                 case 'g':
2584                         if (type == -1) {
2585                                 type = 1;
2586                                 break;
2587                         }
2588                         type |= 1;
2589                         break;
2590                 case 'n':
2591                         if (type == -1) {
2592                                 type = 2;
2593                                 break;
2594                         }
2595                         type |= 2;
2596                         break;
2597                 case 'x':
2598                         if (type == -1) {
2599                                 type = 4;
2600                                 break;
2601                         }
2602                         type |= 4;
2603                         break;
2604                 case 'm':
2605                         mbs = 1;
2606                         break;
2607
2608                 default:
2609                         lst_print_usage(argv[0]);
2610                         return -1;
2611                 }
2612         }
2613
2614         if (optind == argc) {
2615                 lst_print_usage(argv[0]);
2616                 return -1;
2617         }
2618
2619         if (timeout <= 0 || delay <= 0) {
2620                 fprintf(stderr, "Invalid timeout or delay value\n");
2621                 return -1;
2622         }
2623
2624         if (count < -1) {
2625             fprintf(stderr, "Invalid count value\n");
2626             return -1;
2627         }
2628
2629         /* extra count to get first data point */
2630         if (count != -1)
2631             count++;
2632
2633         INIT_LIST_HEAD(&head);
2634
2635         while (optind < argc) {
2636                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 1);
2637                 if (rc != 0)
2638                         goto out;
2639
2640                 list_add_tail(&srp->srp_link, &head);
2641         }
2642
2643         do {
2644                 time_t  now = time(NULL);
2645
2646                 if (now - last < delay) {
2647                         sleep(delay - now + last);
2648                         time(&now);
2649                 }
2650                 last = now;
2651
2652                 list_for_each_entry(srp, &head, srp_link) {
2653                         rc = lst_stat_ioctl(srp->srp_name,
2654                                             srp->srp_count, srp->srp_ids,
2655                                             timeout, &srp->srp_result[idx]);
2656                         if (rc == -1) {
2657                                 lst_print_error("stat", "Failed to stat %s: %s\n",
2658                                                 srp->srp_name, strerror(errno));
2659                                 goto out;
2660                         }
2661
2662                         lst_print_stat(srp->srp_name, srp->srp_result,
2663                                        idx, lnet, bwrt, rdwr, type, mbs);
2664
2665                         lst_reset_rpcent(&srp->srp_result[1 - idx]);
2666                 }
2667
2668                 idx = 1 - idx;
2669
2670                 if (count > 0)
2671                         count--;
2672         } while (count == -1 || count > 0);
2673
2674 out:
2675         while (!list_empty(&head)) {
2676                 srp = list_first_entry(&head, lst_stat_req_param_t, srp_link);
2677
2678                 list_del(&srp->srp_link);
2679                 lst_stat_req_param_free(srp);
2680         }
2681
2682         return rc;
2683 }
2684
2685 static int
2686 jt_lst_show_error(int argc, char **argv)
2687 {
2688         struct list_head       head;
2689         lst_stat_req_param_t  *srp;
2690         struct lstcon_rpc_ent *ent;
2691         struct sfw_counters   *sfwk;
2692         struct srpc_counters  *srpc;
2693         int                    show_rpc = 1;
2694         int                    optidx = 0;
2695         int                    rc = 0;
2696         int                    ecount;
2697         int                    c;
2698
2699         static const struct option show_error_opts[] = {
2700                 { .name = "session", .has_arg = no_argument, .val = 's' },
2701                 { .name = NULL, } };
2702
2703         if (session_key == 0) {
2704                 fprintf(stderr,
2705                         "Can't find env LST_SESSION or value is not valid\n");
2706                 return -1;
2707         }
2708
2709         while (1) {
2710                 c = getopt_long(argc, argv, "s", show_error_opts, &optidx);
2711
2712                 if (c == -1)
2713                         break;
2714
2715                 switch (c) {
2716                 case 's':
2717                         show_rpc  = 0;
2718                         break;
2719
2720                 default:
2721                         lst_print_usage(argv[0]);
2722                         return -1;
2723                 }
2724         }
2725
2726         if (optind == argc) {
2727                 lst_print_usage(argv[0]);
2728                 return -1;
2729         }
2730
2731         INIT_LIST_HEAD(&head);
2732
2733         while (optind < argc) {
2734                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 0);
2735                 if (rc != 0)
2736                         goto out;
2737
2738                 list_add_tail(&srp->srp_link, &head);
2739         }
2740
2741         list_for_each_entry(srp, &head, srp_link) {
2742                 rc = lst_stat_ioctl(srp->srp_name, srp->srp_count,
2743                                     srp->srp_ids, 10, &srp->srp_result[0]);
2744
2745                 if (rc == -1) {
2746                         lst_print_error(srp->srp_name, "Failed to show errors of %s: %s\n",
2747                                         srp->srp_name, strerror(errno));
2748                         goto out;
2749                 }
2750
2751                 fprintf(stdout, "%s:\n", srp->srp_name);
2752
2753                 ecount = 0;
2754
2755                 list_for_each_entry(ent, &srp->srp_result[0], rpe_link) {
2756                         if (ent->rpe_rpc_errno != 0) {
2757                                 ecount ++;
2758                                 fprintf(stderr, "RPC failure, can't show error on %s\n",
2759                                         libcfs_id2str(ent->rpe_peer));
2760                                 continue;
2761                         }
2762
2763                         if (ent->rpe_fwk_errno != 0) {
2764                                 ecount ++;
2765                                 fprintf(stderr, "Framework failure, can't show error on %s\n",
2766                                         libcfs_id2str(ent->rpe_peer));
2767                                 continue;
2768                         }
2769
2770                         sfwk = (struct sfw_counters *)&ent->rpe_payload[0];
2771                         srpc = (struct srpc_counters *)((char *)sfwk + sizeof(*sfwk));
2772
2773                         if (srpc->errors == 0 &&
2774                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2775                                 continue;
2776
2777                         if (!show_rpc  &&
2778                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2779                                 continue;
2780
2781                         ecount ++;
2782
2783                         fprintf(stderr, "%s: [Session %d brw errors, %d ping errors]%c",
2784                                 libcfs_id2str(ent->rpe_peer),
2785                                 sfwk->brw_errors, sfwk->ping_errors,
2786                                 show_rpc  ? ' ' : '\n');
2787
2788                         if (!show_rpc)
2789                                 continue;
2790
2791                         fprintf(stderr, "[RPC: %d errors, %d dropped, %d expired]\n",
2792                                 srpc->errors, srpc->rpcs_dropped, srpc->rpcs_expired);
2793                 }
2794
2795                 fprintf(stdout, "Total %d error nodes in %s\n", ecount, srp->srp_name);
2796         }
2797 out:
2798         while (!list_empty(&head)) {
2799                 srp = list_first_entry(&head, lst_stat_req_param_t, srp_link);
2800
2801                 list_del(&srp->srp_link);
2802                 lst_stat_req_param_free(srp);
2803         }
2804
2805         return rc;
2806 }
2807
2808 static int
2809 lst_add_batch_ioctl(char *name)
2810 {
2811         struct lstio_batch_add_args args = { 0 };
2812
2813         args.lstio_bat_key   = session_key;
2814         args.lstio_bat_nmlen = strlen(name);
2815         args.lstio_bat_namep = name;
2816
2817         return lst_ioctl (LSTIO_BATCH_ADD, &args, sizeof(args));
2818 }
2819
2820 static int
2821 jt_lst_add_batch(int argc, char **argv)
2822 {
2823         char   *name;
2824         int     rc;
2825
2826         if (session_key == 0) {
2827                 fprintf(stderr,
2828                         "Can't find env LST_SESSION or value is not valid\n");
2829                 return -1;
2830         }
2831
2832         if (argc != 2) {
2833                 lst_print_usage(argv[0]);
2834                 return -1;
2835         }
2836
2837         name = argv[1];
2838         if (strlen(name) >= LST_NAME_SIZE) {
2839                 fprintf(stderr, "Name length is limited to %d\n",
2840                         LST_NAME_SIZE - 1);
2841                 return -1;
2842         }
2843
2844         rc = lst_add_batch_ioctl(name);
2845         if (rc == 0)
2846                 return 0;
2847
2848         lst_print_error("batch", "Failed to create batch: %s\n",
2849                         strerror(errno));
2850
2851         return -1;
2852 }
2853
2854 static int
2855 lst_start_batch_ioctl(char *name, int timeout, struct list_head *resultp)
2856 {
2857         struct lstio_batch_run_args args = { 0 };
2858
2859         args.lstio_bat_key     = session_key;
2860         args.lstio_bat_timeout = timeout;
2861         args.lstio_bat_nmlen   = strlen(name);
2862         args.lstio_bat_namep   = name;
2863         args.lstio_bat_resultp = resultp;
2864
2865         return lst_ioctl(LSTIO_BATCH_START, &args, sizeof(args));
2866 }
2867
2868 static int
2869 jt_lst_start_batch(int argc, char **argv)
2870 {
2871         struct list_head  head;
2872         char             *batch;
2873         int               optidx = 0;
2874         int               timeout = 0;
2875         int               count = 0;
2876         int               rc;
2877         int               c;
2878
2879         static const struct option start_batch_opts[] = {
2880                 { .name = "timeout", .has_arg = required_argument, .val = 't' },
2881                 { .name = NULL } };
2882
2883         if (session_key == 0) {
2884                 fprintf(stderr,
2885                         "Can't find env LST_SESSION or value is not valid\n");
2886                 return -1;
2887         }
2888
2889         while (1) {
2890                 c = getopt_long(argc, argv, "t:",
2891                                 start_batch_opts, &optidx);
2892
2893                 /* Detect the end of the options. */
2894                 if (c == -1)
2895                         break;
2896
2897                 switch (c) {
2898                 case 't':
2899                         timeout = atoi(optarg);
2900                         break;
2901                 default:
2902                         lst_print_usage(argv[0]);
2903                         return -1;
2904                 }
2905         }
2906
2907         if (optind == argc) {
2908                 batch = LST_DEFAULT_BATCH;
2909
2910         } else if (optind == argc - 1) {
2911                 batch = argv[optind];
2912
2913         } else {
2914                 lst_print_usage(argv[0]);
2915                 return -1;
2916         }
2917
2918         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2919         if (rc != 0) {
2920                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2921                         batch, strerror(errno));
2922                 return -1;
2923         }
2924
2925         INIT_LIST_HEAD(&head);
2926
2927         rc = lst_alloc_rpcent(&head, count, 0);
2928         if (rc != 0) {
2929                 fprintf(stderr, "Out of memory\n");
2930                 return -1;
2931         }
2932
2933         rc = lst_start_batch_ioctl(batch, timeout, &head);
2934
2935         if (rc == 0) {
2936                 fprintf(stdout, "%s is running now\n", batch);
2937                 lst_free_rpcent(&head);
2938                 return 0;
2939         }
2940
2941         if (rc == -1) {
2942                 lst_print_error("batch", "Failed to start batch: %s\n",
2943                                 strerror(errno));
2944                 lst_free_rpcent(&head);
2945                 return rc;
2946         }
2947
2948         lst_print_transerr(&head, "Run batch");
2949
2950         lst_free_rpcent(&head);
2951
2952         return rc;
2953 }
2954
2955 static int
2956 lst_stop_batch_ioctl(char *name, int force, struct list_head *resultp)
2957 {
2958         struct lstio_batch_stop_args args = { 0 };
2959
2960         args.lstio_bat_key     = session_key;
2961         args.lstio_bat_force   = force;
2962         args.lstio_bat_nmlen   = strlen(name);
2963         args.lstio_bat_namep   = name;
2964         args.lstio_bat_resultp = resultp;
2965
2966         return lst_ioctl(LSTIO_BATCH_STOP, &args, sizeof(args));
2967 }
2968
2969 static int
2970 jt_lst_stop_batch(int argc, char **argv)
2971 {
2972         struct list_head  head;
2973         char             *batch;
2974         int               force = 0;
2975         int               optidx;
2976         int               count;
2977         int               rc;
2978         int               c;
2979
2980         static const struct option stop_batch_opts[] = {
2981                 { .name = "force", .has_arg = no_argument, .val = 'f' },
2982                 { .name = NULL } };
2983
2984         if (session_key == 0) {
2985                 fprintf(stderr,
2986                         "Can't find env LST_SESSION or value is not valid\n");
2987                 return -1;
2988         }
2989
2990         while (1) {
2991                 c = getopt_long(argc, argv, "f",
2992                                 stop_batch_opts, &optidx);
2993
2994                 /* Detect the end of the options. */
2995                 if (c == -1)
2996                         break;
2997
2998                 switch (c) {
2999                 case 'f':
3000                         force = 1;
3001                         break;
3002                 default:
3003                         lst_print_usage(argv[0]);
3004                         return -1;
3005                 }
3006         }
3007
3008         if (optind == argc) {
3009                 batch = LST_DEFAULT_BATCH;
3010
3011         } else if (optind == argc - 1) {
3012                 batch = argv[optind];
3013
3014         } else {
3015                 lst_print_usage(argv[0]);
3016                 return -1;
3017         }
3018
3019         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
3020         if (rc != 0) {
3021                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
3022                         batch, strerror(errno));
3023                 return -1;
3024         }
3025
3026         INIT_LIST_HEAD(&head);
3027
3028         rc = lst_alloc_rpcent(&head, count, 0);
3029         if (rc != 0) {
3030                 fprintf(stderr, "Out of memory\n");
3031                 return -1;
3032         }
3033
3034         rc = lst_stop_batch_ioctl(batch, force, &head);
3035         if (rc != 0)
3036                 goto out;
3037
3038         while (1) {
3039                 lst_reset_rpcent(&head);
3040
3041                 rc = lst_query_batch_ioctl(batch, 0, 0, 30, &head);
3042                 if (rc != 0)
3043                         goto out;
3044
3045                 if (lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0 &&
3046                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0)
3047                         break;
3048
3049                 fprintf(stdout, "%d batch in stopping\n",
3050                         lstcon_tsbqry_stat_run(&trans_stat, 0));
3051                 sleep(1);
3052         }
3053
3054         fprintf(stdout, "Batch is stopped\n");
3055         lst_free_rpcent(&head);
3056
3057         return 0;
3058 out:
3059         if (rc == -1) {
3060                 lst_print_error("batch", "Failed to stop batch: %s\n",
3061                                 strerror(errno));
3062                 lst_free_rpcent(&head);
3063                 return -1;
3064         }
3065
3066         lst_print_transerr(&head, "stop batch");
3067
3068         lst_free_rpcent(&head);
3069
3070         return rc;
3071 }
3072
3073 static int
3074 lst_list_batch_ioctl(int len, char *name, int index)
3075 {
3076         struct lstio_batch_list_args args = { 0 };
3077
3078         args.lstio_bat_key   = session_key;
3079         args.lstio_bat_idx   = index;
3080         args.lstio_bat_nmlen = len;
3081         args.lstio_bat_namep = name;
3082
3083         return lst_ioctl(LSTIO_BATCH_LIST, &args, sizeof(args));
3084 }
3085
3086 static int
3087 lst_info_batch_ioctl(char *batch, int test, int server,
3088                      struct lstcon_test_batch_ent *entp, int *idxp,
3089                      int *ndentp, struct lstcon_node_ent *dentsp)
3090 {
3091         struct lstio_batch_info_args args = { 0 };
3092
3093         args.lstio_bat_key     = session_key;
3094         args.lstio_bat_nmlen   = strlen(batch);
3095         args.lstio_bat_namep   = batch;
3096         args.lstio_bat_server  = server;
3097         args.lstio_bat_testidx = test;
3098         args.lstio_bat_entp    = entp;
3099         args.lstio_bat_idxp    = idxp;
3100         args.lstio_bat_ndentp  = ndentp;
3101         args.lstio_bat_dentsp  = dentsp;
3102
3103         return lst_ioctl(LSTIO_BATCH_INFO, &args, sizeof(args));
3104 }
3105
3106 static int
3107 lst_list_batch_all(void)
3108 {
3109         char name[LST_NAME_SIZE];
3110         int  rc;
3111         int  i;
3112
3113         for (i = 0; ; i++) {
3114                 rc = lst_list_batch_ioctl(LST_NAME_SIZE, name, i);
3115                 if (rc == 0) {
3116                         fprintf(stdout, "%d) %s\n", i + 1, name);
3117                         continue;
3118                 }
3119
3120                 if (errno == ENOENT)
3121                         break;
3122
3123                 lst_print_error("batch", "Failed to list batch: %s\n",
3124                                 strerror(errno));
3125                 return rc;
3126         }
3127
3128         fprintf(stdout, "Total %d batches\n", i);
3129
3130         return 0;
3131 }
3132
3133 static int
3134 lst_list_tsb_nodes(char *batch, int test, int server,
3135                    int count, int active, int invalid)
3136 {
3137         struct lstcon_node_ent *dents;
3138         int                index = 0;
3139         int                rc;
3140         int                c;
3141         int                i;
3142
3143         if (count == 0)
3144                 return 0;
3145
3146         /* verbose list, show nodes in batch or test */
3147         dents = malloc(count * sizeof(struct lstcon_node_ent));
3148         if (dents == NULL) {
3149                 fprintf(stdout, "Can't allocate memory\n");
3150                 return -1;
3151         }
3152
3153         rc = lst_info_batch_ioctl(batch, test, server,
3154                                   NULL, &index, &count, dents);
3155         if (rc != 0) {
3156                 free(dents);
3157                 lst_print_error((test > 0) ? "test" : "batch",
3158                                 (test > 0) ? "Failed to query test: %s\n" :
3159                                              "Failed to query batch: %s\n",
3160                                 strerror(errno));
3161                 return -1;
3162         }
3163
3164         for (i = 0, c = 0; i < count; i++) {
3165                 if ((!active  && dents[i].nde_state == LST_NODE_ACTIVE) ||
3166                     (!invalid && (dents[i].nde_state == LST_NODE_BUSY  ||
3167                                   dents[i].nde_state == LST_NODE_DOWN  ||
3168                                   dents[i].nde_state == LST_NODE_UNKNOWN)))
3169                         continue;
3170
3171                 fprintf(stdout, "\t%s: %s\n",
3172                         libcfs_id2str(dents[i].nde_id),
3173                         lst_node_state2str(dents[i].nde_state));
3174                 c++;
3175         }
3176
3177         fprintf(stdout, "Total %d nodes\n", c);
3178         free(dents);
3179
3180         return 0;
3181 }
3182
3183 static int
3184 jt_lst_list_batch(int argc, char **argv)
3185 {
3186         struct lstcon_test_batch_ent ent;
3187         char *batch   = NULL;
3188         int   optidx  = 0;
3189         int   verbose = 0; /* list nodes in batch or test */
3190         int   invalid = 0;
3191         int   active  = 0;
3192         int   server  = 0;
3193         int   ntest   = 0;
3194         int   test    = 0;
3195         int   c       = 0;
3196         int   rc;
3197
3198         static const struct option list_batch_opts[] = {
3199                 { .name = "test",    .has_arg = required_argument, .val = 't' },
3200                 { .name = "invalid", .has_arg = no_argument,       .val = 'i' },
3201                 { .name = "active",  .has_arg = no_argument,       .val = 'a' },
3202                 { .name = "all",     .has_arg = no_argument,       .val = 'l' },
3203                 { .name = "server",  .has_arg = no_argument,       .val = 's' },
3204                 { .name = NULL, } };
3205
3206         if (session_key == 0) {
3207                 fprintf(stderr,
3208                         "Can't find env LST_SESSION or value is not valid\n");
3209                 return -1;
3210         }
3211
3212         while (1) {
3213                 c = getopt_long(argc, argv, "ailst:",
3214                                 list_batch_opts, &optidx);
3215
3216                 if (c == -1)
3217                         break;
3218
3219                 switch (c) {
3220                 case 'a':
3221                         verbose = active = 1;
3222                         break;
3223                 case 'i':
3224                         verbose = invalid = 1;
3225                         break;
3226                 case 'l':
3227                         verbose = active = invalid = 1;
3228                         break;
3229                 case 's':
3230                         server = 1;
3231                         break;
3232                 case 't':
3233                         test = atoi(optarg);
3234                         ntest = 1;
3235                         break;
3236                 default:
3237                         lst_print_usage(argv[0]);
3238                         return -1;
3239                 }
3240         }
3241
3242         if (optind == argc) {
3243                 /* list all batches */
3244                 rc = lst_list_batch_all();
3245                 return rc;
3246         }
3247
3248         if (ntest == 1 && test <= 0) {
3249                 fprintf(stderr, "Invalid test id, test id starts from 1\n");
3250                 return -1;
3251         }
3252
3253         if (optind != argc - 1) {
3254                 lst_print_usage(argv[0]);
3255                 return -1;
3256         }
3257
3258         batch = argv[optind];
3259
3260 loop:
3261         /* show detail of specified batch or test */
3262         rc = lst_info_batch_ioctl(batch, test, server,
3263                                   &ent, NULL, NULL, NULL);
3264         if (rc != 0) {
3265                 lst_print_error((test > 0) ? "test" : "batch",
3266                                 (test > 0) ? "Failed to query test: %s\n" :
3267                                              "Failed to query batch: %s\n",
3268                                 strerror(errno));
3269                 return -1;
3270         }
3271
3272         if (verbose) {
3273                 /* list nodes in test or batch */
3274                 rc = lst_list_tsb_nodes(batch, test, server,
3275                                         server ? ent.tbe_srv_nle.nle_nnode :
3276                                                  ent.tbe_cli_nle.nle_nnode,
3277                                         active, invalid);
3278                 return rc;
3279         }
3280
3281         /* only show number of hosts in batch or test */
3282         if (test == 0) {
3283                 fprintf(stdout, "Batch: %s Tests: %d State: %d\n",
3284                         batch, ent.u.tbe_batch.bae_ntest,
3285                         ent.u.tbe_batch.bae_state);
3286                 ntest = ent.u.tbe_batch.bae_ntest;
3287                 test = 1; /* starting from test 1 */
3288
3289         } else {
3290                 fprintf(stdout,
3291                         "\tTest %d(%s) (loop: %d, concurrency: %d)\n",
3292                         test, lst_test_type2name(ent.u.tbe_test.tse_type),
3293                         ent.u.tbe_test.tse_loop,
3294                         ent.u.tbe_test.tse_concur);
3295                 ntest --;
3296                 test ++;
3297         }
3298
3299         fprintf(stdout, LST_NODES_TITLE);
3300         fprintf(stdout, "client\t%d\t%d\t%d\t%d\t%d\n"
3301                         "server\t%d\t%d\t%d\t%d\t%d\n",
3302                 ent.tbe_cli_nle.nle_nactive,
3303                 ent.tbe_cli_nle.nle_nbusy,
3304                 ent.tbe_cli_nle.nle_ndown,
3305                 ent.tbe_cli_nle.nle_nunknown,
3306                 ent.tbe_cli_nle.nle_nnode,
3307                 ent.tbe_srv_nle.nle_nactive,
3308                 ent.tbe_srv_nle.nle_nbusy,
3309                 ent.tbe_srv_nle.nle_ndown,
3310                 ent.tbe_srv_nle.nle_nunknown,
3311                 ent.tbe_srv_nle.nle_nnode);
3312
3313         if (ntest != 0)
3314                 goto loop;
3315
3316         return 0;
3317 }
3318
3319 static int
3320 lst_query_batch_ioctl(char *batch, int test, int server,
3321                       int timeout, struct list_head *head)
3322 {
3323         struct lstio_batch_query_args args = { 0 };
3324
3325         args.lstio_bat_key     = session_key;
3326         args.lstio_bat_testidx = test;
3327         args.lstio_bat_client  = !(server);
3328         args.lstio_bat_timeout = timeout;
3329         args.lstio_bat_nmlen   = strlen(batch);
3330         args.lstio_bat_namep   = batch;
3331         args.lstio_bat_resultp = head;
3332
3333         return lst_ioctl(LSTIO_BATCH_QUERY, &args, sizeof(args));
3334 }
3335
3336 static void
3337 lst_print_tsb_verbose(struct list_head *head,
3338                       int active, int idle, int error)
3339 {
3340         struct lstcon_rpc_ent *ent;
3341
3342         list_for_each_entry(ent, head, rpe_link) {
3343                 if (ent->rpe_priv[0] == 0 && active)
3344                         continue;
3345
3346                 if (ent->rpe_priv[0] != 0 && idle)
3347                         continue;
3348
3349                 if (ent->rpe_fwk_errno == 0 && error)
3350                         continue;
3351
3352                 fprintf(stdout, "%s [%s]: %s\n",
3353                         libcfs_id2str(ent->rpe_peer),
3354                         lst_node_state2str(ent->rpe_state),
3355                         ent->rpe_rpc_errno != 0 ?
3356                                 strerror(ent->rpe_rpc_errno) :
3357                                 (ent->rpe_priv[0] > 0 ? "Running" : "Idle"));
3358         }
3359 }
3360
3361 static int
3362 jt_lst_query_batch(int argc, char **argv)
3363 {
3364         struct lstcon_test_batch_ent ent;
3365         struct list_head head;
3366         char   *batch   = NULL;
3367         time_t  last    = 0;
3368         int     optidx  = 0;
3369         int     verbose = 0;
3370         int     server  = 0;
3371         int     timeout = 5; /* default 5 seconds */
3372         int     delay   = 5; /* default 5 seconds */
3373         int     loop    = 1; /* default 1 loop */
3374         int     active  = 0;
3375         int     error   = 0;
3376         int     idle    = 0;
3377         int     count   = 0;
3378         int     test    = 0;
3379         int     rc      = 0;
3380         int     c       = 0;
3381         int     i;
3382
3383         static const struct option query_batch_opts[] = {
3384                 { .name = "timeout", .has_arg = required_argument, .val = 'o' },
3385                 { .name = "delay",   .has_arg = required_argument, .val = 'd' },
3386                 { .name = "loop",    .has_arg = required_argument, .val = 'c' },
3387                 { .name = "test",    .has_arg = required_argument, .val = 't' },
3388                 { .name = "server",  .has_arg = no_argument,       .val = 's' },
3389                 { .name = "active",  .has_arg = no_argument,       .val = 'a' },
3390                 { .name = "idle",    .has_arg = no_argument,       .val = 'i' },
3391                 { .name = "error",   .has_arg = no_argument,       .val = 'e' },
3392                 { .name = "all",     .has_arg = no_argument,       .val = 'l' },
3393                 { .name = NULL, } };
3394
3395         if (session_key == 0) {
3396                 fprintf(stderr,
3397                         "Can't find env LST_SESSION or value is not valid\n");
3398                 return -1;
3399         }
3400
3401         while (1) {
3402                 c = getopt_long(argc, argv, "o:d:c:t:saiel",
3403                                 query_batch_opts, &optidx);
3404
3405                 /* Detect the end of the options. */
3406                 if (c == -1)
3407                         break;
3408
3409                 switch (c) {
3410                 case 'o':
3411                         timeout = atoi(optarg);
3412                         break;
3413                 case 'd':
3414                         delay = atoi(optarg);
3415                         break;
3416                 case 'c':
3417                         loop = atoi(optarg);
3418                         break;
3419                 case 't':
3420                         test = atoi(optarg);
3421                         break;
3422                 case 's':
3423                         server = 1;
3424                         break;
3425                 case 'a':
3426                         active = verbose = 1;
3427                         break;
3428                 case 'i':
3429                         idle = verbose = 1;
3430                         break;
3431                 case 'e':
3432                         error = verbose = 1;
3433                         break;
3434                 case 'l':
3435                         verbose = 1;
3436                         break;
3437                 default:
3438                         lst_print_usage(argv[0]);
3439                         return -1;
3440                 }
3441         }
3442
3443         if (test < 0 || timeout <= 0 || delay <= 0 || loop <= 0) {
3444                 lst_print_usage(argv[0]);
3445                 return -1;
3446         }
3447
3448         if (optind == argc) {
3449                 batch = LST_DEFAULT_BATCH;
3450
3451         } else if (optind == argc - 1) {
3452                 batch = argv[optind];
3453
3454         } else {
3455                 lst_print_usage(argv[0]);
3456                 return -1;
3457         }
3458
3459
3460         INIT_LIST_HEAD(&head);
3461
3462         if (verbose) {
3463                 rc = lst_info_batch_ioctl(batch, test, server,
3464                                           &ent, NULL, NULL, NULL);
3465                 if (rc != 0) {
3466                         fprintf(stderr, "Failed to query %s [%d]: %s\n",
3467                                 batch, test, strerror(errno));
3468                         return -1;
3469                 }
3470
3471                 count = server ? ent.tbe_srv_nle.nle_nnode :
3472                                  ent.tbe_cli_nle.nle_nnode;
3473                 if (count == 0) {
3474                         fprintf(stdout, "Batch or test is empty\n");
3475                         return 0;
3476                 }
3477         }
3478
3479         rc = lst_alloc_rpcent(&head, count, 0);
3480         if (rc != 0) {
3481                 fprintf(stderr, "Out of memory\n");
3482                 return rc;
3483         }
3484
3485         for (i = 0; i < loop; i++) {
3486                 time_t  now = time(NULL);
3487
3488                 if (now - last < delay) {
3489                         sleep(delay - now + last);
3490                         time(&now);
3491                 }
3492
3493                 last = now;
3494
3495                 rc = lst_query_batch_ioctl(batch, test,
3496                                            server, timeout, &head);
3497                 if (rc == -1) {
3498                         fprintf(stderr, "Failed to query batch: %s\n",
3499                                 strerror(errno));
3500                         break;
3501                 }
3502
3503                 if (verbose) {
3504                         /* Verbose mode */
3505                         lst_print_tsb_verbose(&head, active, idle, error);
3506                         continue;
3507                 }
3508
3509                 fprintf(stdout, "%s [%d] ", batch, test);
3510
3511                 if (lstcon_rpc_stat_failure(&trans_stat, 0) != 0) {
3512                         fprintf(stdout, "%d of %d nodes are unknown, ",
3513                                 lstcon_rpc_stat_failure(&trans_stat, 0),
3514                                 lstcon_rpc_stat_total(&trans_stat, 0));
3515                 }
3516
3517                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
3518                     lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0  &&
3519                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
3520                         fprintf(stdout, "is stopped\n");
3521                         continue;
3522                 }
3523
3524                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
3525                     lstcon_tsbqry_stat_idle(&trans_stat, 0) == 0 &&
3526                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
3527                         fprintf(stdout, "is running\n");
3528                         continue;
3529                 }
3530
3531                 fprintf(stdout, "stopped: %d , running: %d, failed: %d\n",
3532                                 lstcon_tsbqry_stat_idle(&trans_stat, 0),
3533                                 lstcon_tsbqry_stat_run(&trans_stat, 0),
3534                                 lstcon_tsbqry_stat_failure(&trans_stat, 0));
3535         }
3536
3537         lst_free_rpcent(&head);
3538
3539         return rc;
3540 }
3541
3542 static int
3543 lst_parse_distribute(char *dstr, int *dist, int *span)
3544 {
3545         *dist = atoi(dstr);
3546         if (*dist <= 0)
3547                 return -1;
3548
3549         dstr = strchr(dstr, ':');
3550         if (dstr == NULL)
3551                 return -1;
3552
3553         *span = atoi(dstr + 1);
3554         if (*span <= 0)
3555                 return -1;
3556
3557         return 0;
3558 }
3559
3560 static int
3561 lst_get_bulk_param(int argc, char **argv, struct lst_test_bulk_param *bulk)
3562 {
3563         char   *tok = NULL;
3564         char   *end = NULL;
3565         int     rc  = 0;
3566         int     i   = 0;
3567
3568         bulk->blk_size  = 4096;
3569         bulk->blk_opc   = LST_BRW_READ;
3570         bulk->blk_flags = LST_BRW_CHECK_NONE;
3571         bulk->blk_srv_off = bulk->blk_cli_off = 0;
3572
3573         while (i < argc) {
3574                 if (strcasestr(argv[i], "check=") == argv[i] ||
3575                     strcasestr(argv[i], "c=") == argv[i]) {
3576                         tok = strchr(argv[i], '=') + 1;
3577
3578                         if (strcasecmp(tok, "full") == 0) {
3579                                 bulk->blk_flags = LST_BRW_CHECK_FULL;
3580                         } else if (strcasecmp(tok, "simple") == 0) {
3581                                 bulk->blk_flags = LST_BRW_CHECK_SIMPLE;
3582                         } else {
3583                                 fprintf(stderr, "Unknow flag %s\n", tok);
3584                                 return -1;
3585                         }
3586
3587                 } else if (strcasestr(argv[i], "size=") == argv[i] ||
3588                            strcasestr(argv[i], "s=") == argv[i]) {
3589                         tok = strchr(argv[i], '=') + 1;
3590
3591                         bulk->blk_size = strtol(tok, &end, 0);
3592                         if (bulk->blk_size <= 0) {
3593                                 fprintf(stderr, "Invalid size %s\n", tok);
3594                                 return -1;
3595                         }
3596
3597                         if (end == NULL)
3598                                 return 0;
3599
3600                         if (*end == 'k' || *end == 'K')
3601                                 bulk->blk_size *= 1024;
3602                         else if (*end == 'm' || *end == 'M')
3603                                 bulk->blk_size *= 1024 * 1024;
3604
3605                         if (bulk->blk_size > LNET_MTU) {
3606                                 fprintf(stderr, "Size exceed limitation: %d bytes\n",
3607                                         bulk->blk_size);
3608                                 return -1;
3609                         }
3610
3611                 } else if (strcasestr(argv[i], "off=") == argv[i]) {
3612                         int     off;
3613
3614                         tok = strchr(argv[i], '=') + 1;
3615
3616                         off = strtol(tok, &end, 0);
3617                         /* NB: align with sizeof(__u64) to simplify page
3618                          * checking implementation */
3619                         if (off < 0 || off % sizeof(__u64) != 0) {
3620                                 fprintf(stderr,
3621                                         "Invalid offset %s/%d, it should be "
3622                                         "postive value and multiple of %d\n",
3623                                         tok, off, (int)sizeof(__u64));
3624                                 return -1;
3625                         }
3626
3627                         /* NB: blk_srv_off is reserved so far */
3628                         bulk->blk_cli_off = bulk->blk_srv_off = off;
3629                         if (end == NULL)
3630                                 return 0;
3631
3632                 } else if (strcasecmp(argv[i], "read") == 0 ||
3633                            strcasecmp(argv[i], "r") == 0) {
3634                         bulk->blk_opc = LST_BRW_READ;
3635
3636                 } else if (strcasecmp(argv[i], "write") == 0 ||
3637                            strcasecmp(argv[i], "w") == 0) {
3638                         bulk->blk_opc = LST_BRW_WRITE;
3639
3640                 } else {
3641                         fprintf(stderr, "Unknow parameter: %s\n", argv[i]);
3642                         return -1;
3643                 }
3644
3645                 i++;
3646         }
3647
3648         return rc;
3649 }
3650
3651 static int
3652 lst_get_test_param(char *test, int argc, char **argv, void **param, int *plen)
3653 {
3654         struct lst_test_bulk_param *bulk = NULL;
3655         struct lst_test_ping_param *ping = NULL;
3656         int                    type;
3657
3658         type = lst_test_name2type(test);
3659         if (type < 0) {
3660                 fprintf(stderr, "Unknow test name %s\n", test);
3661                 return -1;
3662         }
3663
3664         switch (type) {
3665         case LST_TEST_PING:
3666                 /* unused but needs for kernel part */
3667                 ping = malloc(sizeof(*ping));
3668                 if (ping == NULL) {
3669                         fprintf(stderr, "Out of memory\n");
3670                         return -1;
3671                 }
3672                 memset(ping, 0, sizeof(*ping));
3673
3674                 *param = ping;
3675                 *plen  = sizeof(*ping);
3676
3677                 break;
3678
3679         case LST_TEST_BULK:
3680                 bulk = malloc(sizeof(*bulk));
3681                 if (bulk == NULL) {
3682                         fprintf(stderr, "Out of memory\n");
3683                         return -1;
3684                 }
3685
3686                 memset(bulk, 0, sizeof(*bulk));
3687
3688                 if (lst_get_bulk_param(argc, argv, bulk) != 0) {
3689                         free(bulk);
3690                         return -1;
3691                 }
3692
3693                 *param = bulk;
3694                 *plen  = sizeof(*bulk);
3695
3696                 break;
3697
3698         default:
3699                 break;
3700         }
3701
3702         /* TODO: parse more parameter */
3703         return type;
3704 }
3705
3706 static int
3707 lst_add_test_ioctl(char *batch, int type, int loop, int concur,
3708                    int dist, int span, char *sgrp, char *dgrp,
3709                    void *param, int plen, int *retp, struct list_head *resultp)
3710 {
3711         struct lstio_test_args args = { 0 };
3712
3713         args.lstio_tes_key        = session_key;
3714         args.lstio_tes_bat_nmlen  = strlen(batch);
3715         args.lstio_tes_bat_name   = batch;
3716         args.lstio_tes_type       = type;
3717         args.lstio_tes_oneside    = 0;
3718         args.lstio_tes_loop       = loop;
3719         args.lstio_tes_concur     = concur;
3720         args.lstio_tes_dist       = dist;
3721         args.lstio_tes_span       = span;
3722         args.lstio_tes_sgrp_nmlen = strlen(sgrp);
3723         args.lstio_tes_sgrp_name  = sgrp;
3724         args.lstio_tes_dgrp_nmlen = strlen(dgrp);
3725         args.lstio_tes_dgrp_name  = dgrp;
3726         args.lstio_tes_param_len  = plen;
3727         args.lstio_tes_param      = param;
3728         args.lstio_tes_retp       = retp;
3729         args.lstio_tes_resultp    = resultp;
3730
3731         return lst_ioctl(LSTIO_TEST_ADD, &args, sizeof(args));
3732 }
3733
3734 static int
3735 jt_lst_add_test(int argc, char **argv)
3736 {
3737         struct list_head head;
3738         char *batch  = NULL;
3739         char *test   = NULL;
3740         char *dstr   = NULL;
3741         char *from   = NULL;
3742         char *to     = NULL;
3743         void *param  = NULL;
3744         int   optidx = 0;
3745         int   concur = 1;
3746         int   loop   = -1;
3747         int   dist   = 1;
3748         int   span   = 1;
3749         int   plen   = 0;
3750         int   fcount = 0;
3751         int   tcount = 0;
3752         int   ret    = 0;
3753         int   type;
3754         int   rc;
3755         int   c;
3756
3757         static const struct option add_test_opts[] = {
3758         { .name = "batch",       .has_arg = required_argument, .val = 'b' },
3759         { .name = "concurrency", .has_arg = required_argument, .val = 'c' },
3760         { .name = "distribute",  .has_arg = required_argument, .val = 'd' },
3761         { .name = "from",        .has_arg = required_argument, .val = 'f' },
3762         { .name = "to",          .has_arg = required_argument, .val = 't' },
3763         { .name = "loop",        .has_arg = required_argument, .val = 'l' },
3764         { .name = NULL } };
3765
3766         if (session_key == 0) {
3767                 fprintf(stderr,
3768                         "Can't find env LST_SESSION or value is not valid\n");
3769                 return -1;
3770         }
3771
3772         while (1) {
3773                 c = getopt_long(argc, argv, "b:c:d:f:l:t:",
3774                                 add_test_opts, &optidx);
3775
3776                 /* Detect the end of the options. */
3777                 if (c == -1)
3778                         break;
3779
3780                 switch (c) {
3781                 case 'b':
3782                         batch = optarg;
3783                         break;
3784                 case 'c':
3785                         concur = atoi(optarg);
3786                         break;
3787                 case 'd':
3788                         dstr = optarg;
3789                         break;
3790                 case 'f':
3791                         from = optarg;
3792                         break;
3793                 case 'l':
3794                         loop = atoi(optarg);
3795                         break;
3796                 case 't':
3797                         to = optarg;
3798                         break;
3799                 default:
3800                         lst_print_usage(argv[0]);
3801                         return -1;
3802                 }
3803         }
3804
3805         if (optind == argc || from == NULL || to == NULL) {
3806                 lst_print_usage(argv[0]);
3807                 return -1;
3808         }
3809
3810         if (concur <= 0 || concur > LST_MAX_CONCUR) {
3811                 fprintf(stderr, "Invalid concurrency of test: %d\n", concur);
3812                 return -1;
3813         }
3814
3815         if (batch == NULL)
3816                 batch = LST_DEFAULT_BATCH;
3817
3818         if (dstr != NULL) {
3819                 rc = lst_parse_distribute(dstr, &dist, &span);
3820                 if (rc != 0) {
3821                         fprintf(stderr, "Invalid distribution: %s\n", dstr);
3822                         return -1;
3823                 }
3824         }
3825
3826         test = argv[optind++];
3827
3828         argc -= optind;
3829         argv += optind;
3830
3831         type = lst_get_test_param(test, argc, argv, &param, &plen);
3832         if (type < 0) {
3833                 fprintf(stderr, "Failed to add test (%s)\n", test);
3834                 return -1;
3835         }
3836
3837         INIT_LIST_HEAD(&head);
3838
3839         rc = lst_get_node_count(LST_OPC_GROUP, from, &fcount, NULL);
3840         if (rc != 0) {
3841                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3842                         from, strerror(errno));
3843                 goto out;
3844         }
3845
3846         rc = lst_get_node_count(LST_OPC_GROUP, to, &tcount, NULL);
3847         if (rc != 0) {
3848                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3849                         to, strerror(errno));
3850                 goto out;
3851         }
3852
3853         rc = lst_alloc_rpcent(&head, fcount > tcount ? fcount : tcount, 0);
3854         if (rc != 0) {
3855                 fprintf(stderr, "Out of memory\n");
3856                 goto out;
3857         }
3858
3859         rc = lst_add_test_ioctl(batch, type, loop, concur,
3860                                 dist, span, from, to, param, plen, &ret, &head);
3861
3862         if (rc == 0) {
3863                 fprintf(stdout, "Test was added successfully\n");
3864                 if (ret != 0) {
3865                         fprintf(stdout, "Server group contains userland test "
3866                                 "nodes, old version of tcplnd can't accept "
3867                                 "connection request\n");
3868                 }
3869
3870                 goto out;
3871         }
3872
3873         if (rc == -1) {
3874                 lst_print_error("test", "Failed to add test: %s\n",
3875                                 strerror(errno));
3876                 goto out;
3877         }
3878
3879         lst_print_transerr(&head, "add test");
3880 out:
3881         lst_free_rpcent(&head);
3882
3883         if (param != NULL)
3884                 free(param);
3885
3886         return rc;
3887 }
3888
3889 static command_t lst_cmdlist[] = {
3890         {"new_session",         jt_lst_new_session,     NULL,
3891          "Usage: lst new_session [--timeout TIME] [--force] [NAME]"                     },
3892         {"end_session",         jt_lst_end_session,     NULL,
3893          "Usage: lst end_session"                                                       },
3894         {"show_session",        jt_lst_show_session,    NULL,
3895          "Usage: lst show_session"                                                      },
3896         {"ping",                jt_lst_ping ,           NULL,
3897          "Usage: lst ping  [--group NAME] [--batch NAME] [--session] [--nodes IDS]"     },
3898         {"add_group",           jt_lst_add_group,       NULL,
3899          "Usage: lst group NAME IDs [IDs]..."                                           },
3900         {"del_group",           jt_lst_del_group,       NULL,
3901          "Usage: lst del_group NAME"                                                    },
3902         {"update_group",        jt_lst_update_group,    NULL,
3903          "Usage: lst update_group NAME [--clean] [--refresh] [--remove IDs]"            },
3904         {"list_group",          jt_lst_list_group,      NULL,
3905           "Usage: lst list_group [--active] [--busy] [--down] [--unknown] GROUP ..."    },
3906         {"stat",                jt_lst_stat,            NULL,
3907          "Usage: lst stat [--bw] [--rate] [--read] [--write] [--max] [--min] [--avg] "
3908          " [--mbs] [--timeout #] [--delay #] [--count #] GROUP [GROUP]"                 },
3909         {"show_error",          jt_lst_show_error,      NULL,
3910          "Usage: lst show_error NAME | IDS ..."                                         },
3911         {"add_batch",           jt_lst_add_batch,       NULL,
3912          "Usage: lst add_batch NAME"                                                    },
3913         {"run",                 jt_lst_start_batch,     NULL,
3914          "Usage: lst run [--timeout TIME] [NAME]"                                       },
3915         {"stop",                jt_lst_stop_batch,      NULL,
3916          "Usage: lst stop [--force] BATCH_NAME"                                         },
3917         {"list_batch",          jt_lst_list_batch,      NULL,
3918          "Usage: lst list_batch NAME [--test ID] [--server]"                            },
3919         {"query",               jt_lst_query_batch,     NULL,
3920          "Usage: lst query [--test ID] [--server] [--timeout TIME] NAME"                },
3921         {"add_test",            jt_lst_add_test,        NULL,
3922          "Usage: lst add_test [--batch BATCH] [--loop #] [--concurrency #] "
3923          " [--distribute #:#] [--from GROUP] [--to GROUP] TEST..."                      },
3924         {0,                     0,                      0,      NULL                    }
3925 };
3926
3927 static int
3928 lst_initialize(void)
3929 {
3930         char   *key;
3931         char   *feats;
3932
3933         feats = getenv("LST_FEATURES");
3934         if (feats != NULL)
3935                 session_features = strtol(feats, NULL, 16);
3936
3937         if ((session_features & ~LST_FEATS_MASK) != 0) {
3938                 fprintf(stderr,
3939                         "Unsupported session features %x, "
3940                         "only support these features so far: %x\n",
3941                         (session_features & ~LST_FEATS_MASK), LST_FEATS_MASK);
3942                 return -1;
3943         }
3944
3945         key = getenv("LST_SESSION");
3946
3947         if (key == NULL) {
3948                 session_key = 0;
3949                 return 0;
3950         }
3951
3952         session_key = atoi(key);
3953
3954         return 0;
3955 }
3956
3957 int main(int argc, char **argv)
3958 {
3959         int rc = 0;
3960
3961         setlinebuf(stdout);
3962
3963         rc = lst_initialize();
3964         if (rc < 0)
3965                 goto errorout;
3966
3967         rc = lustre_lnet_config_lib_init();
3968         if (rc < 0)
3969                 goto errorout;
3970
3971         rc = cfs_parser(argc, argv, lst_cmdlist);
3972
3973 errorout:
3974         return rc;
3975 }