lnet/utils/lst.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2012, 2017, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  *
  31  * lnet/selftest/conctl.c
  32  *
  33  * Author: Liang Zhen <liangzhen@clusterfs.com>
  34  */
  35 #include <errno.h>
  36 #include <getopt.h>
  37 #include <inttypes.h>
  38 #include <pwd.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <stdio.h>
  42 #include <stdlib.h>
  43 #include <string.h>
  44 #include <sys/ioctl.h>
  45 #include <time.h>
  46 #include <linux/types.h>
  47
  48 #include <libcfs/util/list.h>
  49 #include <libcfs/util/ioctl.h>
  50 #include <libcfs/util/parser.h>
  51 #include <linux/lnet/lnetctl.h>
  52 #include <linux/lnet/lnetst.h>
  53 #include <linux/lnet/nidstr.h>
  54 #include "lnetconfig/liblnetconfig.h"
  55
  56 static int lst_info_batch_ioctl(char *batch, int test, int server,
  57                        struct lstcon_test_batch_ent *entp, int *idxp,
  58                        int *ndentp, struct lstcon_node_ent *dentsp);
  59 static int lst_info_group_ioctl(char *name, struct lstcon_ndlist_ent *gent,
  60                         int *idx, int *count, struct lstcon_node_ent *dents);
  61 static int lst_query_batch_ioctl(char *batch, int test, int server,
  62                          int timeout, struct list_head *head);
  63
  64 struct lst_sid LST_INVALID_SID = { .ses_nid = LNET_NID_ANY, .ses_stamp = -1 };
  65 static unsigned int session_key;
  66
  67 /* All nodes running 2.6.50 or later understand feature LST_FEAT_BULK_LEN */
  68 static unsigned int session_features = LST_FEATS_MASK;
  69 static struct lstcon_trans_stat trans_stat;
  70
  71 typedef struct list_string {
  72         struct list_string *lstr_next;
  73         int                 lstr_sz;
  74         char                lstr_str[0];
  75 } lstr_t;
  76
  77 #ifndef offsetof
  78 # define offsetof(typ,memb)     ((unsigned long)((char *)&(((typ *)0)->memb)))
  79 #endif
  80
  81 static int alloc_count = 0;
  82 static int alloc_nob   = 0;
  83
  84 static lstr_t *
  85 alloc_lstr(int sz)
  86 {
  87         lstr_t  *lstr = malloc(offsetof(lstr_t, lstr_str[sz]));
  88
  89         if (lstr == NULL) {
  90                 fprintf(stderr, "Can't allocate lstr\n");
  91                 abort();
  92         }
  93
  94         alloc_nob += sz;
  95         alloc_count++;
  96
  97         lstr->lstr_str[0] = 0;
  98         lstr->lstr_sz = sz;
  99         return lstr;
 100 }
 101
 102 static void
 103 free_lstr(lstr_t *lstr)
 104 {
 105         alloc_count--;
 106         alloc_nob -= lstr->lstr_sz;
 107         free(lstr);
 108 }
 109
 110 static void
 111 new_lstrs(lstr_t **list, char *prefix, char *postfix,
 112           int lo, int hi, int stride)
 113 {
 114         int    n1 = strlen(prefix);
 115         int    n2 = strlen(postfix);
 116         int    sz = n1 + 20 + n2 + 1;
 117
 118         do {
 119                 lstr_t *n = alloc_lstr(sz);
 120
 121                 snprintf(n->lstr_str, sz - 1, "%s%u%s",
 122                          prefix, lo, postfix);
 123
 124                 n->lstr_next = *list;
 125                 *list = n;
 126
 127                 lo += stride;
 128         } while (lo <= hi);
 129 }
 130
 131 static int
 132 expand_lstr(lstr_t **list, lstr_t *l)
 133 {
 134         int          nob = strlen(l->lstr_str);
 135         char        *b1;
 136         char        *b2;
 137         char        *expr;
 138         char        *sep;
 139         int          x;
 140         int          y;
 141         int          z;
 142         int          n;
 143
 144         b1 = strchr(l->lstr_str, '[');
 145         if (b1 == NULL) {
 146                 l->lstr_next = *list;
 147                 *list = l;
 148                 return 0;
 149         }
 150
 151         b2 = strchr(b1, ']');
 152         if (b2 == NULL || b2 == b1 + 1)
 153                 return -1;
 154
 155         *b1++ = 0;
 156         *b2++ = 0;
 157         expr = b1;
 158         do {
 159
 160                 sep = strchr(expr, ',');
 161                 if (sep != NULL)
 162                         *sep++ = 0;
 163
 164                 nob = strlen(expr);
 165                 n = nob;
 166                 if (sscanf(expr, "%u%n", &x, &n) >= 1 && n == nob) {
 167                         /* simple number */
 168                         new_lstrs(list, l->lstr_str, b2, x, x, 1);
 169                         continue;
 170                 }
 171
 172                 n = nob;
 173                 if (sscanf(expr, "%u-%u%n", &x, &y, &n) >= 2 && n == nob &&
 174                     x < y) {
 175                         /* simple range */
 176                         new_lstrs(list, l->lstr_str, b2, x, y, 1);
 177                         continue;
 178                 }
 179
 180                 n = nob;
 181                 if (sscanf(expr, "%u-%u/%u%n", &x, &y, &z, &n) >= 3 && n == nob &&
 182                     x < y) {
 183                         /* strided range */
 184                         new_lstrs(list, l->lstr_str, b2, x, y, z);
 185                         continue;
 186                 }
 187
 188                 /* syntax error */
 189                 return -1;
 190         } while ((expr = sep) != NULL);
 191
 192         free_lstr(l);
 193
 194         return 1;
 195 }
 196
 197 static int
 198 expand_strs(char *str, lstr_t **head)
 199 {
 200         lstr_t  *list = NULL;
 201         lstr_t  *nlist;
 202         lstr_t  *l;
 203         int      rc = 0;
 204         int      expanded;
 205
 206         l = alloc_lstr(strlen(str) + 1);
 207         memcpy(l->lstr_str, str, strlen(str) + 1);
 208         l->lstr_next = NULL;
 209         list = l;
 210
 211         do {
 212                 expanded = 0;
 213                 nlist = NULL;
 214
 215                 while ((l = list) != NULL) {
 216                         list = l->lstr_next;
 217
 218                         rc = expand_lstr(&nlist, l);
 219                         if (rc < 0) {
 220                                 fprintf(stderr, "Syntax error in \"%s\"\n", str);
 221                                 free_lstr(l);
 222                                 break;
 223                         }
 224
 225                         expanded |= rc > 0;
 226                 }
 227
 228                 /* re-order onto 'list' */
 229                 while ((l = nlist) != NULL) {
 230                         nlist = l->lstr_next;
 231                         l->lstr_next = list;
 232                         list = l;
 233                 }
 234
 235         } while (expanded && rc > 0);
 236
 237         if (rc >= 0) {
 238                 *head = list;
 239                 return 0;
 240         }
 241
 242         while ((l = list) != NULL) {
 243                 list = l->lstr_next;
 244
 245                 free_lstr(l);
 246         }
 247         return rc;
 248 }
 249
 250 static int
 251 lst_parse_nids(char *str, int *countp, struct lnet_process_id **idspp)
 252 {
 253         lstr_t  *head = NULL;
 254         lstr_t  *l;
 255         int      c = 0;
 256         int      i;
 257         int      rc;
 258
 259         rc = expand_strs(str, &head);
 260         if (rc != 0)
 261                 goto out;
 262
 263         l = head;
 264         while (l != NULL) {
 265                 l = l->lstr_next;
 266                 c++;
 267         }
 268
 269         *idspp = malloc(c * sizeof(struct lnet_process_id));
 270         if (*idspp == NULL) {
 271                 fprintf(stderr, "Out of memory\n");
 272                 rc = -1;
 273         }
 274
 275         *countp = c;
 276 out:
 277         i = 0;
 278         while ((l = head) != NULL) {
 279                 head = l->lstr_next;
 280
 281                 if (rc == 0) {
 282                         (*idspp)[i].nid = libcfs_str2nid(l->lstr_str);
 283                         if ((*idspp)[i].nid == LNET_NID_ANY) {
 284                                 fprintf(stderr, "Invalid nid: %s\n",
 285                                         l->lstr_str);
 286                                 rc = -1;
 287                         }
 288
 289                         (*idspp)[i].pid = LNET_PID_LUSTRE;
 290                         i++;
 291                 }
 292
 293                 free_lstr(l);
 294         }
 295
 296         if (rc == 0)
 297                 return 0;
 298
 299         free(*idspp);
 300         *idspp = NULL;
 301
 302         return rc;
 303 }
 304
 305 static char *
 306 lst_node_state2str(int state)
 307 {
 308         if (state == LST_NODE_ACTIVE)
 309                 return "Active";
 310         if (state == LST_NODE_BUSY)
 311                 return "Busy";
 312         if (state == LST_NODE_DOWN)
 313                 return "Down";
 314
 315         return "Unknown";
 316 }
 317
 318 static int
 319 lst_node_str2state(char *str)
 320 {
 321         if (strcasecmp(str, "active") == 0)
 322                 return LST_NODE_ACTIVE;
 323         if (strcasecmp(str, "busy") == 0)
 324                 return LST_NODE_BUSY;
 325         if (strcasecmp(str, "down") == 0)
 326                 return LST_NODE_DOWN;
 327         if (strcasecmp(str, "unknown") == 0)
 328                 return LST_NODE_UNKNOWN;
 329         if (strcasecmp(str, "invalid") == 0)
 330                 return (LST_NODE_UNKNOWN | LST_NODE_DOWN | LST_NODE_BUSY);
 331
 332         return -1;
 333 }
 334
 335 static char *
 336 lst_test_type2name(int type)
 337 {
 338         if (type == LST_TEST_PING)
 339                 return "ping";
 340         if (type == LST_TEST_BULK)
 341                 return "brw";
 342
 343         return "unknown";
 344 }
 345
 346 static int
 347 lst_test_name2type(char *name)
 348 {
 349         if (strcasecmp(name, "ping") == 0)
 350                 return LST_TEST_PING;
 351         if (strcasecmp(name, "brw") == 0)
 352                 return LST_TEST_BULK;
 353
 354         return -1;
 355 }
 356
 357 static void
 358 lst_print_usage(char *cmd)
 359 {
 360         char *argv[] = { "help", cmd };
 361
 362         cfs_parser(2, argv, NULL);
 363 }
 364
 365 static void
 366 lst_print_error(char *sub, const char *def_format, ...)
 367 {
 368         va_list ap;
 369
 370         /* local error returned from kernel */
 371         switch (errno) {
 372         case ESRCH:
 373                 fprintf(stderr, "No session exists\n");
 374                 return;
 375         case ESHUTDOWN:
 376                 fprintf(stderr, "Session is shutting down\n");
 377                 return;
 378         case EACCES:
 379                 fprintf(stderr, "Unmatched session key or not root\n");
 380                 return;
 381         case ENOENT:
 382                 fprintf(stderr, "Can't find %s in current session\n", sub);
 383                 return;
 384         case EINVAL:
 385                 fprintf(stderr, "Invalid parameters list in command line\n");
 386                 return;
 387         case EFAULT:
 388                 fprintf(stderr, "Bad parameter address\n");
 389                 return;
 390         case EEXIST:
 391                 fprintf(stderr, "%s already exists\n", sub);
 392                 return;
 393         default:
 394                 va_start(ap, def_format);
 395                 vfprintf(stderr, def_format, ap);
 396                 va_end(ap);
 397
 398                 return;
 399         }
 400 }
 401
 402 static void
 403 lst_free_rpcent(struct list_head *head)
 404 {
 405         struct lstcon_rpc_ent *ent;
 406
 407         while (!list_empty(head)) {
 408                 ent = list_first_entry(head, struct lstcon_rpc_ent, rpe_link);
 409
 410                 list_del(&ent->rpe_link);
 411                 free(ent);
 412         }
 413 }
 414
 415 static void
 416 lst_reset_rpcent(struct list_head *head)
 417 {
 418         struct lstcon_rpc_ent *ent;
 419
 420         list_for_each_entry(ent, head, rpe_link) {
 421                 ent->rpe_sid       = LST_INVALID_SID;
 422                 ent->rpe_peer.nid  = LNET_NID_ANY;
 423                 ent->rpe_peer.pid  = LNET_PID_ANY;
 424                 ent->rpe_rpc_errno = ent->rpe_fwk_errno = 0;
 425         }
 426 }
 427
 428 static int
 429 lst_alloc_rpcent(struct list_head *head, int count, int offset)
 430 {
 431         struct lstcon_rpc_ent *ent;
 432         int               i;
 433
 434         for (i = 0; i < count; i++) {
 435                 ent = malloc(offsetof(struct lstcon_rpc_ent, rpe_payload[offset]));
 436                 if (ent == NULL) {
 437                         lst_free_rpcent(head);
 438                         return -1;
 439                 }
 440
 441                 memset(ent, 0, offsetof(struct lstcon_rpc_ent, rpe_payload[offset]));
 442
 443                 ent->rpe_sid      = LST_INVALID_SID;
 444                 ent->rpe_peer.nid = LNET_NID_ANY;
 445                 ent->rpe_peer.pid = LNET_PID_ANY;
 446                 list_add(&ent->rpe_link, head);
 447         }
 448
 449         return 0;
 450 }
 451
 452 static void
 453 lst_print_transerr(struct list_head *head, char *optstr)
 454 {
 455         struct lstcon_rpc_ent *ent;
 456
 457         list_for_each_entry(ent, head, rpe_link) {
 458                 if (ent->rpe_rpc_errno == 0 && ent->rpe_fwk_errno == 0)
 459                         continue;
 460
 461                 if (ent->rpe_rpc_errno != 0) {
 462                         fprintf(stderr, "%s RPC failed on %s: %s\n",
 463                                 optstr, libcfs_id2str(ent->rpe_peer),
 464                                 strerror(ent->rpe_rpc_errno));
 465                         continue;
 466                 }
 467
 468                 fprintf(stderr, "operation %s failed on %s: %s\n",
 469                         optstr, libcfs_id2str(ent->rpe_peer),
 470                         strerror(ent->rpe_fwk_errno));
 471         }
 472 }
 473
 474 static int
 475 lst_ioctl(unsigned int opc, void *buf, int len)
 476 {
 477         struct libcfs_ioctl_data data;
 478         int    rc;
 479
 480         LIBCFS_IOC_INIT (data);
 481         data.ioc_u32[0]  = opc;
 482         data.ioc_plen1   = len;
 483         data.ioc_pbuf1   = (char *)buf;
 484         data.ioc_plen2   = sizeof(trans_stat);
 485         data.ioc_pbuf2   = (char *)&trans_stat;
 486
 487         memset(&trans_stat, 0, sizeof(trans_stat));
 488
 489         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNETST, &data);
 490
 491         /* local error, no valid RPC result */
 492         if (rc != 0)
 493                 return -1;
 494
 495         /* RPC error */
 496         if (trans_stat.trs_rpc_errno != 0)
 497                 return -2;
 498
 499         /* Framework error */
 500         if (trans_stat.trs_fwk_errno != 0)
 501                 return -3;
 502
 503         return 0;
 504 }
 505
 506 static int lst_yaml_session(const char *label, const char *timeout, int nlflags,
 507                             const char *errmsg)
 508 {
 509         struct lstcon_ndlist_ent ndinfo = { };
 510         struct lst_sid sid = LST_INVALID_SID;
 511         /* nlflags being zero means we are destroying the session.
 512          * No parsing of reply needed.
 513          */
 514         bool done = nlflags ? false : true;
 515         char nid[LNET_NIDSTR_SIZE];
 516         char name[LST_NAME_SIZE];
 517         unsigned int key = 0;
 518         yaml_emitter_t request;
 519         yaml_parser_t reply;
 520         yaml_event_t event;
 521         struct nl_sock *sk;
 522         int rc;
 523
 524         sk = nl_socket_alloc();
 525         if (!sk)
 526                 return -1;
 527
 528         /* Note: NL_AUTO_PID == zero which we use by default for the
 529          * session_key when creating a new session. This is considered
 530          * an invalid key so we need to get the real session key from
 531          * the yaml parser yet to be created. If the user did request
 532          * a specific session key then set the socket's port id to this
 533          * value.
 534          */
 535         if (session_key)
 536                 nl_socket_set_local_port(sk, session_key);
 537
 538         /* Setup reply parser to recieve Netlink packets */
 539         rc = yaml_parser_initialize(&reply);
 540         if (rc == 0) {
 541                 nl_socket_free(sk);
 542                 return -1;
 543         }
 544
 545         rc = yaml_parser_set_input_netlink(&reply, sk, false);
 546         if (rc == 0)
 547                 goto parser_error;
 548
 549         /* Create Netlink emitter to send request to kernel */
 550         yaml_emitter_initialize(&request);
 551         rc = yaml_emitter_set_output_netlink(&request, sk,
 552                                              LNET_SELFTEST_GENL_NAME,
 553                                              LNET_SELFTEST_GENL_VERSION,
 554                                              LNET_SELFTEST_CMD_SESSIONS,
 555                                              nlflags);
 556         if (rc == 0)
 557                 goto emitter_error;
 558
 559         yaml_emitter_open(&request);
 560         yaml_document_start_event_initialize(&event, NULL, NULL, NULL, 0);
 561         rc = yaml_emitter_emit(&request, &event);
 562         if (rc == 0)
 563                 goto emitter_error;
 564
 565         yaml_mapping_start_event_initialize(&event, NULL,
 566                                             (yaml_char_t *)YAML_MAP_TAG,
 567                                             1, YAML_BLOCK_MAPPING_STYLE);
 568         rc = yaml_emitter_emit(&request, &event);
 569         if (rc == 0)
 570                 goto emitter_error;
 571
 572         yaml_scalar_event_initialize(&event, NULL,
 573                                      (yaml_char_t *)YAML_STR_TAG,
 574                                      (yaml_char_t *)"sessions",
 575                                      strlen("sessions"), 1, 0,
 576                                      YAML_PLAIN_SCALAR_STYLE);
 577         rc = yaml_emitter_emit(&request, &event);
 578         if (rc == 0)
 579                 goto emitter_error;
 580
 581         if (!label) {
 582                 yaml_scalar_event_initialize(&event, NULL,
 583                                              (yaml_char_t *)YAML_STR_TAG,
 584                                              (yaml_char_t *)"",
 585                                              strlen(""), 1, 0,
 586                                              YAML_PLAIN_SCALAR_STYLE);
 587                 rc = yaml_emitter_emit(&request, &event);
 588                 if (rc == 0)
 589                         goto emitter_error;
 590                 goto skip_params;
 591         }
 592
 593         /* sessions: { name: 'name', timeout: 300 }
 594          * or
 595          * sessions:
 596          *   name: 'name'
 597          *   timeout: 300
 598          */
 599         yaml_mapping_start_event_initialize(&event, NULL,
 600                                             (yaml_char_t *)YAML_MAP_TAG,
 601                                             1, YAML_FLOW_MAPPING_STYLE);
 602         rc = yaml_emitter_emit(&request, &event);
 603         if (rc == 0)
 604                 goto emitter_error;
 605
 606         yaml_scalar_event_initialize(&event, NULL,
 607                                      (yaml_char_t *)YAML_STR_TAG,
 608                                      (yaml_char_t *)"name",
 609                                      strlen("name"), 1, 0,
 610                                      YAML_PLAIN_SCALAR_STYLE);
 611         rc = yaml_emitter_emit(&request, &event);
 612         if (rc == 0)
 613                 goto emitter_error;
 614
 615         yaml_scalar_event_initialize(&event, NULL,
 616                                      (yaml_char_t *)YAML_STR_TAG,
 617                                      (yaml_char_t *)label,
 618                                      strlen(label), 1, 0,
 619                                      YAML_PLAIN_SCALAR_STYLE);
 620         rc = yaml_emitter_emit(&request, &event);
 621         if (rc == 0)
 622                 goto emitter_error;
 623
 624         if (timeout) {
 625                 yaml_scalar_event_initialize(&event, NULL,
 626                                              (yaml_char_t *)YAML_STR_TAG,
 627                                              (yaml_char_t *)"timeout",
 628                                              strlen("timeout"), 1, 0,
 629                                              YAML_PLAIN_SCALAR_STYLE);
 630                 rc = yaml_emitter_emit(&request, &event);
 631                 if (rc == 0)
 632                         goto emitter_error;
 633
 634                 yaml_scalar_event_initialize(&event, NULL,
 635                                              (yaml_char_t *)YAML_STR_TAG,
 636                                              (yaml_char_t *)timeout,
 637                                              strlen(timeout), 1, 0,
 638                                              YAML_PLAIN_SCALAR_STYLE);
 639                 rc = yaml_emitter_emit(&request, &event);
 640                 if (rc == 0)
 641                         goto emitter_error;
 642         }
 643
 644         yaml_mapping_end_event_initialize(&event);
 645         rc = yaml_emitter_emit(&request, &event);
 646         if (rc == 0)
 647                 goto emitter_error;
 648
 649 skip_params:
 650         yaml_mapping_end_event_initialize(&event);
 651         rc = yaml_emitter_emit(&request, &event);
 652         if (rc == 0)
 653                 goto emitter_error;
 654
 655         yaml_document_end_event_initialize(&event, 0);
 656         rc = yaml_emitter_emit(&request, &event);
 657         if (rc == 0)
 658                 goto emitter_error;
 659
 660         rc = yaml_emitter_close(&request);
 661         if (rc == 0) {
 662 emitter_error:
 663                 yaml_emitter_log_error(&request, stderr);
 664                 yaml_emitter_delete(&request);
 665                 errmsg = NULL;
 666                 goto parser_error;
 667         }
 668         yaml_emitter_delete(&request);
 669
 670         while (!done) {
 671                 rc = yaml_parser_parse(&reply, &event);
 672                 if (rc == 0)
 673                         goto parser_error;
 674
 675                 if (event.type == YAML_SCALAR_EVENT) {
 676                         char *tmp, *endp = NULL;
 677
 678                         if (strcmp((char *)event.data.scalar.value,
 679                                    "name") == 0) {
 680                                 yaml_event_delete(&event);
 681                                 rc = yaml_parser_parse(&reply, &event);
 682                                 if (rc == 0)
 683                                         goto parser_error;
 684
 685                                 strncpy(name, (char *)event.data.scalar.value,
 686                                         sizeof(name) - 1);
 687                         }
 688
 689                         if (strcmp((char *)event.data.scalar.value,
 690                                    "key") == 0) {
 691                                 yaml_event_delete(&event);
 692                                 rc = yaml_parser_parse(&reply, &event);
 693                                 if (rc == 0)
 694                                         goto parser_error;
 695
 696                                 tmp = (char *)event.data.scalar.value;
 697                                 key = strtoul(tmp, &endp, 10);
 698                                 if (endp == tmp)
 699                                         goto parser_error;
 700                         }
 701
 702                         if (strcmp((char *)event.data.scalar.value,
 703                                    "timestamp") == 0) {
 704                                 yaml_event_delete(&event);
 705                                 rc = yaml_parser_parse(&reply, &event);
 706                                 if (rc == 0)
 707                                         goto parser_error;
 708
 709                                 tmp = (char *)event.data.scalar.value;
 710                                 sid.ses_stamp = strtoll(tmp, &endp, 10);
 711                                 if (endp == tmp)
 712                                         goto parser_error;
 713                         }
 714
 715                         if (strcmp((char *)event.data.scalar.value,
 716                                    "nid") == 0) {
 717                                 yaml_event_delete(&event);
 718                                 rc = yaml_parser_parse(&reply, &event);
 719                                 if (rc == 0)
 720                                         goto parser_error;
 721
 722                                 strncpy(nid, (char *)event.data.scalar.value,
 723                                         sizeof(nid) - 1);
 724                         }
 725
 726                         if (strcmp((char *)event.data.scalar.value,
 727                                    "nodes") == 0) {
 728                                 yaml_event_delete(&event);
 729                                 rc = yaml_parser_parse(&reply, &event);
 730                                 if (rc == 0)
 731                                         goto parser_error;
 732
 733                                 tmp = (char *)event.data.scalar.value;
 734                                 ndinfo.nle_nnode = strtoul(tmp, &endp, 10);
 735                                 if (endp == tmp)
 736                                         goto parser_error;
 737                         }
 738                 }
 739
 740                 done = (event.type == YAML_STREAM_END_EVENT);
 741
 742                 yaml_event_delete(&event);
 743         }
 744
 745         if (nlflags & NLM_F_CREATE) {
 746                 session_features = yaml_parser_get_reader_proto_version(&reply);
 747                 session_key = key;
 748         }
 749 parser_error:
 750         if (rc == 0 && errmsg)
 751                 yaml_parser_log_error(&reply, stderr, errmsg);
 752         yaml_parser_delete(&reply);
 753         nl_socket_free(sk);
 754
 755         if (((nlflags & NLM_F_DUMP) == NLM_F_DUMP) && rc != 0) {
 756                 fprintf(stdout,
 757                         "%s ID: %ju@%s, KEY: %u FEATURES: %x NODES: %d\n",
 758                         name, (uintmax_t)sid.ses_stamp, nid,
 759                         key, session_features, ndinfo.nle_nnode);
 760         }
 761
 762         return rc == 0 ? -1 : 0;
 763 }
 764
 765 static int
 766 lst_new_session_ioctl(char *name, int timeout, int force, struct lst_sid *sid)
 767 {
 768         struct lstio_session_new_args args = { 0 };
 769
 770         args.lstio_ses_key     = session_key;
 771         args.lstio_ses_timeout = timeout;
 772         args.lstio_ses_force   = force;
 773         args.lstio_ses_idp     = sid;
 774         args.lstio_ses_feats   = session_features;
 775         args.lstio_ses_nmlen   = strlen(name);
 776         args.lstio_ses_namep   = name;
 777
 778         return lst_ioctl (LSTIO_SESSION_NEW, &args, sizeof(args));
 779 }
 780
 781 static int
 782 jt_lst_new_session(int argc, char **argv)
 783 {
 784         char  buf[LST_NAME_SIZE * 2 + 1];
 785         char *name, *timeout_s = NULL;
 786         int nlflags = NLM_F_CREATE;
 787         struct lst_sid session_id;
 788         int   optidx = 0;
 789         int   timeout = 300;
 790         int   force = 0;
 791         int   c;
 792         int   rc;
 793
 794         static const struct option session_opts[] = {
 795                 { .name = "timeout", .has_arg = required_argument, .val = 't' },
 796                 { .name = "force",   .has_arg = no_argument,       .val = 'f' },
 797                 { .name = NULL } };
 798
 799         while (1) {
 800                 c = getopt_long(argc, argv, "ft:",
 801                                 session_opts, &optidx);
 802                 if (c == -1)
 803                         break;
 804
 805                 switch (c) {
 806                 case 'f':
 807                         nlflags |= NLM_F_REPLACE;
 808                         force = 1;
 809                         break;
 810                 case 't':
 811                         timeout_s = optarg;
 812                         break;
 813                 default:
 814                         lst_print_usage(argv[0]);
 815                         return -1;
 816                 }
 817         }
 818
 819         if (timeout_s) {
 820                 timeout = atoi(timeout_s);
 821                 if (timeout <= 0) {
 822                         fprintf(stderr, "Invalid timeout value\n");
 823                         return -1;
 824                 }
 825         }
 826
 827         if (optind == argc - 1) {
 828                 name = argv[optind ++];
 829                 if (strlen(name) >= LST_NAME_SIZE) {
 830                         fprintf(stderr, "Name size is limited to %d\n",
 831                                 LST_NAME_SIZE - 1);
 832                         return -1;
 833                 }
 834         } else if (optind == argc) {
 835                 char           user[LST_NAME_SIZE];
 836                 char           host[LST_NAME_SIZE];
 837                 struct passwd *pw = getpwuid(getuid());
 838
 839                 if (pw == NULL)
 840                         snprintf(user, sizeof(user), "%d", (int)getuid());
 841                 else
 842                         snprintf(user, sizeof(user), "%s", pw->pw_name);
 843
 844                 rc = gethostname(host, sizeof(host));
 845                 if (rc != 0)
 846                         snprintf(host, sizeof(host), "unknown_host");
 847
 848                 snprintf(buf, sizeof(buf), "%s@%s", user, host);
 849                 name = buf;
 850         } else {
 851                 lst_print_usage(argv[0]);
 852                 return -1;
 853         }
 854
 855         rc = lst_yaml_session(name, timeout_s, nlflags, "new session");
 856         if (rc == 0)
 857                 goto success;
 858
 859         if (session_key == 0) {
 860                 fprintf(stderr,
 861                         "Can't find env LST_SESSION or value is not valid\n");
 862                 return -1;
 863         }
 864
 865         rc = lst_new_session_ioctl(name, timeout, force, &session_id);
 866         if (rc != 0) {
 867                 lst_print_error("session", "Failed to create session: %s\n",
 868                                 strerror(errno));
 869                 return rc;
 870         }
 871 success:
 872         fprintf(stdout, "SESSION: %s FEATURES: %x TIMEOUT: %d FORCE: %s\n",
 873                 name, session_features, timeout, force ? "Yes" : "No");
 874         return 0;
 875 }
 876
 877 static int
 878 lst_session_info_ioctl(char *name, int len, int *key, unsigned *featp,
 879                        struct lst_sid *sid, struct lstcon_ndlist_ent *ndinfo)
 880 {
 881         struct lstio_session_info_args args = { 0 };
 882
 883         args.lstio_ses_idp     = sid;
 884         args.lstio_ses_keyp    = key;
 885         args.lstio_ses_featp   = featp;
 886         args.lstio_ses_ndinfo  = ndinfo;
 887         args.lstio_ses_nmlen   = len;
 888         args.lstio_ses_namep   = name;
 889
 890         return lst_ioctl(LSTIO_SESSION_INFO, &args, sizeof(args));
 891 }
 892
 893 static int
 894 jt_lst_show_session(int argc, char **argv)
 895 {
 896         struct lstcon_ndlist_ent ndinfo;
 897         struct lst_sid sid;
 898         char name[LST_NAME_SIZE];
 899         unsigned int feats;
 900         int key;
 901         int rc;
 902
 903         rc = lst_yaml_session(NULL, NULL, NLM_F_DUMP, "show session");
 904         if (rc == 0)
 905                 return 0;
 906
 907         rc = lst_session_info_ioctl(name, sizeof(name), &key,
 908                                     &feats, &sid, &ndinfo);
 909
 910         if (rc != 0) {
 911                 lst_print_error("session", "Failed to show session: %s\n",
 912                                 strerror(errno));
 913                 return -1;
 914         }
 915
 916         fprintf(stdout, "%s ID: %ju@%s, KEY: %d FEATURES: %x NODES: %d\n",
 917                 name, (uintmax_t)sid.ses_stamp, libcfs_nid2str(sid.ses_nid),
 918                 key, feats, ndinfo.nle_nnode);
 919
 920         return 0;
 921 }
 922
 923 static int
 924 lst_end_session_ioctl(void)
 925 {
 926         struct lstio_session_end_args args = { 0 };
 927
 928         args.lstio_ses_key = session_key;
 929         return lst_ioctl(LSTIO_SESSION_END, &args, sizeof(args));
 930 }
 931
 932 static int
 933 jt_lst_end_session(int argc, char **argv)
 934 {
 935         int             rc;
 936
 937         if (session_key == 0) {
 938                 fprintf(stderr,
 939                         "Can't find env LST_SESSION or value is not valid\n");
 940                 return -1;
 941         }
 942
 943         rc = lst_yaml_session(NULL, NULL, 0, "end session");
 944         if (rc == 0)
 945                 goto finish;
 946
 947         rc = lst_end_session_ioctl();
 948
 949         if (rc == 0) {
 950                 fprintf(stdout, "session is ended\n");
 951                 return 0;
 952         }
 953
 954         if (rc == -1) {
 955                 lst_print_error("session", "Failed to end session: %s\n",
 956                                 strerror(errno));
 957                 return rc;
 958         }
 959 finish:
 960         if (trans_stat.trs_rpc_errno != 0) {
 961                 fprintf(stderr,
 962                         "[RPC] Failed to send %d session RPCs: %s\n",
 963                         lstcon_rpc_stat_failure(&trans_stat, 0),
 964                         strerror(trans_stat.trs_rpc_errno));
 965         }
 966
 967         if (trans_stat.trs_fwk_errno != 0) {
 968                 fprintf(stderr,
 969                         "[FWK] Failed to end session on %d nodes: %s\n",
 970                         lstcon_sesop_stat_failure(&trans_stat, 0),
 971                         strerror(trans_stat.trs_fwk_errno));
 972         }
 973
 974         return rc;
 975 }
 976
 977 #define LST_NODES_TITLE "\tACTIVE\tBUSY\tDOWN\tUNKNOWN\tTOTAL\n"
 978
 979 static int lst_yaml_display_groups(yaml_parser_t *reply, char *group,
 980                                    int states, bool print)
 981 {
 982         yaml_event_t event;
 983         int rc;
 984
 985         if (states) {
 986                 char name[LST_NAME_SIZE] = {};
 987                 bool done = false;
 988                 int count = 0;
 989
 990                 if (print)
 991                         fprintf(stdout, LST_NODES_TITLE);
 992
 993                 while (!done) {
 994                         rc = yaml_parser_parse(reply, &event);
 995                         if (rc == 0)
 996                                 goto parser_error;
 997
 998                         if (event.type == YAML_MAPPING_START_EVENT) {
 999                                 char *value = NULL;
1000                                 yaml_event_t next;
1001
1002                                 rc = yaml_parser_parse(reply, &next);
1003                                 if (rc == 0)
1004                                         goto parser_error;
1005
1006                                 if (next.type != YAML_SCALAR_EVENT) {
1007                                         yaml_event_delete(&next);
1008                                         continue;
1009                                 }
1010
1011                                 value = (char *)next.data.scalar.value;
1012                                 if (strcmp(value, "groups") == 0) {
1013                                         yaml_event_delete(&next);
1014                                 } else if (strcmp(value, "nid") == 0) {
1015                                         yaml_event_t next;
1016                                         char *tmp;
1017
1018                                         rc = yaml_parser_parse(reply, &next);
1019                                         if (rc == 0)
1020                                                 goto parser_error;
1021
1022                                         fprintf(stdout, "\t%s: ",
1023                                                 (char *)next.data.scalar.value);
1024
1025                                         rc = yaml_parser_parse(reply, &next);
1026                                         if (rc == 0)
1027                                                 goto parser_error;
1028
1029                                         tmp = (char *)next.data.scalar.value;
1030                                         if (strcmp(tmp, "status") == 0) {
1031                                                 yaml_event_t state;
1032
1033                                                 rc = yaml_parser_parse(reply, &state);
1034                                                 if (rc == 0)
1035                                                         goto parser_error;
1036
1037                                                 fprintf(stdout, "%s\n",
1038                                                         (char *)state.data.scalar.value);
1039                                                 count++;
1040                                         }
1041                                 } else {
1042                                         yaml_event_t next;
1043
1044                                         rc = yaml_parser_parse(reply, &next);
1045                                         if (rc == 0)
1046                                                 goto parser_error;
1047
1048                                         strncpy(name, value, sizeof(name) - 1);
1049                                         fprintf(stdout, "Group [ %s ]\n", name);
1050                                         count = 0;
1051
1052                                         if (next.type != YAML_SEQUENCE_START_EVENT) {
1053                                                 fprintf(stdout,
1054                                                         "No nodes found [ %s ]\n",
1055                                                         name);
1056                                         }
1057                                 }
1058                         } else if (event.type == YAML_SEQUENCE_END_EVENT &&
1059                                    count) {
1060                                 fprintf(stdout, "Total %d nodes [ %s ]\n",
1061                                         count, name);
1062                                 count = 0;
1063                         }
1064
1065                         done = (event.type == YAML_STREAM_END_EVENT);
1066                         yaml_event_delete(&event);
1067                 }
1068         } else if (group) {
1069                 int active = 0, busy = 0, down = 0, unknown = 0;
1070                 char group[LST_NAME_SIZE];
1071                 bool done = false;
1072
1073                 while (!done) {
1074                         rc = yaml_parser_parse(reply, &event);
1075                         if (rc == 0)
1076                                 goto parser_error;
1077
1078                         if (event.type == YAML_SCALAR_EVENT) {
1079                                 char *value = (char *)event.data.scalar.value;
1080                                 yaml_event_t next;
1081
1082                                 value = (char *)event.data.scalar.value;
1083                                 if (strcmp(value, "groups") == 0) {
1084                                         yaml_event_delete(&event);
1085                                         continue;
1086                                 }
1087
1088                                 rc = yaml_parser_parse(reply, &next);
1089                                 if (rc == 0)
1090                                         goto parser_error;
1091
1092                                 if (next.type == YAML_SCALAR_EVENT) {
1093                                         int status;
1094
1095                                         status = lst_node_str2state((char *)next.data.scalar.value);
1096                                         switch (status) {
1097                                         case LST_NODE_ACTIVE:
1098                                                 active++;
1099                                                 break;
1100                                         case LST_NODE_BUSY:
1101                                                 busy++;
1102                                                 break;
1103                                         case LST_NODE_DOWN:
1104                                                 down++;
1105                                                 break;
1106                                         case LST_NODE_UNKNOWN:
1107                                                 unknown++;
1108                                         default:
1109                                                 break;
1110                                         }
1111                                 } else if (next.type == YAML_SEQUENCE_START_EVENT) {
1112                                         strncpy(group, value, sizeof(group) - 1);
1113                                         active = 0;
1114                                         busy = 0;
1115                                         down = 0;
1116                                         unknown = 0;
1117                                 }
1118                                 yaml_event_delete(&next);
1119                         } else if (event.type == YAML_SEQUENCE_END_EVENT) {
1120                                 if (strlen(group)) {
1121                                         fprintf(stdout, "\t%d\t%d\t%d\t%d\t%d\t%s\n",
1122                                                 active, busy, down, unknown,
1123                                                 active + busy + down + unknown, group);
1124                                 }
1125                                 memset(group, 0, sizeof(group));
1126                         }
1127                         done = (event.type == YAML_STREAM_END_EVENT);
1128                         yaml_event_delete(&event);
1129                 }
1130         } else {
1131                 bool done = false;
1132                 unsigned int i = 1;
1133
1134                 while (!done) {
1135                         rc = yaml_parser_parse(reply, &event);
1136                         if (rc == 0)
1137                                 goto parser_error;
1138
1139                         if (event.type == YAML_SCALAR_EVENT) {
1140                                 char *value;
1141
1142                                 value = (char *)event.data.scalar.value;
1143                                 if (strlen(value) &&
1144                                     strcmp(value, "groups") != 0) {
1145                                         if (print)
1146                                                 fprintf(stdout, "%d) %s\n",
1147                                                         i, value);
1148                                         i++;
1149                                 }
1150                         }
1151                         done = (event.type == YAML_STREAM_END_EVENT);
1152
1153                         yaml_event_delete(&event);
1154                 }
1155                 if (print)
1156                         fprintf(stdout, "Total %d groups\n", i - 1);
1157                 else
1158                         rc = i - 1;
1159         }
1160 parser_error:
1161         return rc;
1162 }
1163
1164 #define LST_NODES_TITLE "\tACTIVE\tBUSY\tDOWN\tUNKNOWN\tTOTAL\n"
1165
1166 static int lst_yaml_groups(int nlflags, char *name, int states, bool print)
1167 {
1168         yaml_emitter_t request;
1169         yaml_parser_t reply;
1170         yaml_event_t event;
1171         struct nl_sock *sk;
1172         int rc;
1173
1174         sk = nl_socket_alloc();
1175         if (!sk)
1176                 return -EOPNOTSUPP;
1177
1178         /* Setup reply parser to recieve Netlink packets */
1179         rc = yaml_parser_initialize(&reply);
1180         if (rc == 0) {
1181                 nl_socket_free(sk);
1182                 return -EOPNOTSUPP;
1183         }
1184
1185         rc = yaml_parser_set_input_netlink(&reply, sk, false);
1186         if (rc == 0)
1187                 goto parser_error;
1188
1189         /* Create Netlink emitter to send request to kernel */
1190         yaml_emitter_initialize(&request);
1191         rc = yaml_emitter_set_output_netlink(&request, sk,
1192                                              LNET_SELFTEST_GENL_NAME,
1193                                              LNET_SELFTEST_GENL_VERSION,
1194                                              LNET_SELFTEST_CMD_GROUPS,
1195                                              nlflags);
1196         if (rc == 0)
1197                 goto emitter_error;
1198
1199         yaml_emitter_open(&request);
1200         yaml_document_start_event_initialize(&event, NULL, NULL, NULL, 0);
1201         rc = yaml_emitter_emit(&request, &event);
1202         if (rc == 0)
1203                 goto emitter_error;
1204
1205         yaml_mapping_start_event_initialize(&event, NULL,
1206                                             (yaml_char_t *)YAML_MAP_TAG,
1207                                             1, YAML_BLOCK_MAPPING_STYLE);
1208         rc = yaml_emitter_emit(&request, &event);
1209         if (rc == 0)
1210                 goto emitter_error;
1211
1212         yaml_scalar_event_initialize(&event, NULL,
1213                                      (yaml_char_t *)YAML_STR_TAG,
1214                                      (yaml_char_t *)"groups",
1215                                      strlen("groups"), 1, 0,
1216                                      YAML_PLAIN_SCALAR_STYLE);
1217         rc = yaml_emitter_emit(&request, &event);
1218         if (rc == 0)
1219                 goto emitter_error;
1220
1221         if (name) {
1222                 yaml_sequence_start_event_initialize(&event, NULL,
1223                                                      (yaml_char_t *)YAML_SEQ_TAG,
1224                                                      1, YAML_BLOCK_SEQUENCE_STYLE);
1225                 rc = yaml_emitter_emit(&request, &event);
1226                 if (rc == 0)
1227                         goto emitter_error;
1228
1229                 yaml_mapping_start_event_initialize(&event, NULL,
1230                                                     (yaml_char_t *)YAML_MAP_TAG,
1231                                                     1, YAML_BLOCK_MAPPING_STYLE);
1232                 rc = yaml_emitter_emit(&request, &event);
1233                 if (rc == 0)
1234                         goto emitter_error;
1235
1236                 yaml_scalar_event_initialize(&event, NULL,
1237                                              (yaml_char_t *)YAML_STR_TAG,
1238                                              (yaml_char_t *)name,
1239                                              strlen(name), 1, 0,
1240                                              YAML_PLAIN_SCALAR_STYLE);
1241                 rc = yaml_emitter_emit(&request, &event);
1242                 if (rc == 0)
1243                         goto emitter_error;
1244
1245                 if (states) {
1246                         int max = ffs(LST_NODE_UNKNOWN) + 1, i;
1247                         char *state;
1248
1249                         yaml_sequence_start_event_initialize(&event, NULL,
1250                                                              (yaml_char_t *)YAML_SEQ_TAG,
1251                                                              1, YAML_BLOCK_SEQUENCE_STYLE);
1252                         rc = yaml_emitter_emit(&request, &event);
1253                         if (rc == 0)
1254                                 goto emitter_error;
1255
1256
1257                         for (i = 0; i < max; i++) {
1258                                 int mask = states & (1 << i);
1259
1260                                 state = lst_node_state2str(mask);
1261                                 if (mask != LST_NODE_UNKNOWN && strcmp(state, "Unknown") == 0)
1262                                         continue;
1263
1264                                 yaml_mapping_start_event_initialize(&event, NULL,
1265                                                                     (yaml_char_t *)YAML_MAP_TAG,
1266                                                                     1, YAML_BLOCK_MAPPING_STYLE);
1267                                 rc = yaml_emitter_emit(&request, &event);
1268                                 if (rc == 0)
1269                                         goto emitter_error;
1270
1271                                 yaml_scalar_event_initialize(&event, NULL,
1272                                                              (yaml_char_t *)YAML_STR_TAG,
1273                                                              (yaml_char_t *)"status",
1274                                                              strlen("status"), 1, 0,
1275                                                              YAML_PLAIN_SCALAR_STYLE);
1276                                 rc = yaml_emitter_emit(&request, &event);
1277                                 if (rc == 0)
1278                                         goto emitter_error;
1279
1280                                 yaml_scalar_event_initialize(&event, NULL,
1281                                                              (yaml_char_t *)YAML_STR_TAG,
1282                                                              (yaml_char_t *)state,
1283                                                              strlen(state), 1, 0,
1284                                                              YAML_PLAIN_SCALAR_STYLE);
1285                                 rc = yaml_emitter_emit(&request, &event);
1286                                 if (rc == 0)
1287                                         goto emitter_error;
1288
1289                                 yaml_mapping_end_event_initialize(&event);
1290                                 rc = yaml_emitter_emit(&request, &event);
1291                                 if (rc == 0)
1292                                         goto emitter_error;
1293                         }
1294
1295                         yaml_sequence_end_event_initialize(&event);
1296                         rc = yaml_emitter_emit(&request, &event);
1297                         if (rc == 0)
1298                                 goto emitter_error;
1299                 } else {
1300                         yaml_scalar_event_initialize(&event, NULL,
1301                                                      (yaml_char_t *)YAML_STR_TAG,
1302                                                      (yaml_char_t *)"",
1303                                                      strlen(""), 1, 0,
1304                                                      YAML_PLAIN_SCALAR_STYLE);
1305                         rc = yaml_emitter_emit(&request, &event);
1306                         if (rc == 0)
1307                                 goto emitter_error;
1308                 }
1309
1310                 yaml_mapping_end_event_initialize(&event);
1311                 rc = yaml_emitter_emit(&request, &event);
1312                 if (rc == 0)
1313                         goto emitter_error;
1314
1315                 yaml_sequence_end_event_initialize(&event);
1316                 rc = yaml_emitter_emit(&request, &event);
1317                 if (rc == 0)
1318                         goto emitter_error;
1319         } else {
1320                 yaml_scalar_event_initialize(&event, NULL,
1321                                              (yaml_char_t *)YAML_STR_TAG,
1322                                              (yaml_char_t *)"",
1323                                              strlen(""), 1, 0,
1324                                              YAML_PLAIN_SCALAR_STYLE);
1325                 rc = yaml_emitter_emit(&request, &event);
1326                 if (rc == 0)
1327                         goto emitter_error;
1328         }
1329         yaml_mapping_end_event_initialize(&event);
1330         rc = yaml_emitter_emit(&request, &event);
1331         if (rc == 0)
1332                 goto emitter_error;
1333
1334         yaml_document_end_event_initialize(&event, 0);
1335         rc = yaml_emitter_emit(&request, &event);
1336         if (rc == 0)
1337                 goto emitter_error;
1338
1339         rc = yaml_emitter_close(&request);
1340         if (rc == 0) {
1341 emitter_error:
1342                 yaml_emitter_log_error(&request, stderr);
1343                 yaml_emitter_delete(&request);
1344                 rc = -EINVAL;
1345                 goto parser_error;
1346         }
1347         yaml_emitter_delete(&request);
1348
1349         /* display output */
1350         if (nlflags == NLM_F_DUMP)
1351                 rc = lst_yaml_display_groups(&reply, name, states, print);
1352 parser_error:
1353         if (rc == 0)
1354                 yaml_parser_log_error(&reply, stderr, NULL);
1355         yaml_parser_delete(&reply);
1356         nl_socket_free(sk);
1357
1358         if (print)
1359                 rc = rc == 1 ? 0 : -EINVAL;
1360
1361         return rc;
1362 }
1363
1364 static int
1365 lst_ping_ioctl(char *str, int type, int timeout,
1366                int count, struct lnet_process_id *ids, struct list_head *head)
1367 {
1368         struct lstio_debug_args args = { 0 };
1369
1370         args.lstio_dbg_key     = session_key;
1371         args.lstio_dbg_type    = type;
1372         args.lstio_dbg_flags   = 0;
1373         args.lstio_dbg_timeout = timeout;
1374         args.lstio_dbg_nmlen   = (str == NULL) ? 0: strlen(str);
1375         args.lstio_dbg_namep   = str;
1376         args.lstio_dbg_count   = count;
1377         args.lstio_dbg_idsp    = ids;
1378         args.lstio_dbg_resultp = head;
1379
1380         return lst_ioctl (LSTIO_DEBUG, &args, sizeof(args));
1381 }
1382
1383 static int
1384 lst_get_node_count(int type, char *str, int *countp,
1385                    struct lnet_process_id **idspp)
1386 {
1387         char                    buf[LST_NAME_SIZE];
1388         struct lstcon_test_batch_ent ent;
1389         struct lstcon_ndlist_ent    *entp = &ent.tbe_cli_nle;
1390         struct lst_sid sid;
1391         unsigned                feats;
1392         int                     key;
1393         int                     rc;
1394
1395         switch (type) {
1396         case LST_OPC_SESSION:
1397                 rc = lst_session_info_ioctl(buf, LST_NAME_SIZE,
1398                                             &key, &feats, &sid, entp);
1399                 break;
1400
1401         case LST_OPC_BATCHSRV:
1402                 entp = &ent.tbe_srv_nle;
1403         case LST_OPC_BATCHCLI:
1404                 rc = lst_info_batch_ioctl(str, 0, 0, &ent, NULL, NULL, NULL);
1405                 break;
1406
1407         case LST_OPC_GROUP:
1408                 rc = lst_yaml_groups(NLM_F_DUMP, NULL, 0, false);
1409                 if (rc == -EOPNOTSUPP) {
1410                         rc = lst_info_group_ioctl(str, entp, NULL, NULL, NULL);
1411                 } else if (rc > 0) {
1412                         entp->nle_nnode = rc;
1413                         rc = 0;
1414                 }
1415                 break;
1416
1417         case LST_OPC_NODES:
1418                 rc = lst_parse_nids(str, &entp->nle_nnode, idspp) < 0 ? -1 : 0;
1419                 break;
1420
1421         default:
1422                 rc = -1;
1423                 break;
1424         }
1425
1426         if (rc == 0)
1427                 *countp = entp->nle_nnode;
1428
1429         return rc;
1430 }
1431
1432 static int
1433 jt_lst_ping(int argc,  char **argv)
1434 {
1435         struct list_head   head;
1436         struct lnet_process_id *ids = NULL;
1437         struct lstcon_rpc_ent  *ent = NULL;
1438         char              *str = NULL;
1439         int                optidx  = 0;
1440         int                server  = 0;
1441         int                timeout = 5;
1442         int                count   = 0;
1443         int                type    = 0;
1444         int                rc      = 0;
1445         int                c;
1446
1447         static const struct option ping_opts[] = {
1448                 { .name = "session", .has_arg = no_argument,       .val = 's' },
1449                 { .name = "server",  .has_arg = no_argument,       .val = 'v' },
1450                 { .name = "batch",   .has_arg = required_argument, .val = 'b' },
1451                 { .name = "group",   .has_arg = required_argument, .val = 'g' },
1452                 { .name = "nodes",   .has_arg = required_argument, .val = 'n' },
1453                 { .name = "timeout", .has_arg = required_argument, .val = 't' },
1454                 { .name = NULL, } };
1455
1456         if (session_key == 0) {
1457                 fprintf(stderr,
1458                         "Can't find env LST_SESSION or value is not valid\n");
1459                 return -1;
1460         }
1461
1462         while (1) {
1463
1464                 c = getopt_long(argc, argv, "g:b:n:t:sv",
1465                                 ping_opts, &optidx);
1466
1467                 if (c == -1)
1468                         break;
1469
1470                 switch (c) {
1471                 case 's':
1472                         type = LST_OPC_SESSION;
1473                         break;
1474
1475                 case 'g':
1476                         type = LST_OPC_GROUP;
1477                         str = optarg;
1478                         break;
1479
1480                 case 'b':
1481                         type = LST_OPC_BATCHCLI;
1482                         str = optarg;
1483                         break;
1484
1485                 case 'n':
1486                         type = LST_OPC_NODES;
1487                         str = optarg;
1488                         break;
1489
1490                 case 't':
1491                         timeout = atoi(optarg);
1492                         break;
1493
1494                 case 'v':
1495                         server = 1;
1496                         break;
1497
1498                 default:
1499                         lst_print_usage(argv[0]);
1500                         return -1;
1501                 }
1502         }
1503
1504         if (type == 0 || timeout <= 0 || optind != argc) {
1505                 lst_print_usage(argv[0]);
1506                 return -1;
1507         }
1508
1509         if (type == LST_OPC_BATCHCLI && server)
1510                 type = LST_OPC_BATCHSRV;
1511
1512         rc = lst_get_node_count(type, str, &count, &ids);
1513         if (rc < 0) {
1514                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
1515                         (str == NULL) ? "session" : str, strerror(errno));
1516                 return -1;
1517         }
1518
1519         INIT_LIST_HEAD(&head);
1520
1521         rc = lst_alloc_rpcent(&head, count, LST_NAME_SIZE);
1522         if (rc != 0) {
1523                 fprintf(stderr, "Out of memory\n");
1524                 goto out;
1525         }
1526
1527         if (count == 0) {
1528                 fprintf(stdout, "Target %s is empty\n",
1529                         (str == NULL) ? "session" : str);
1530                 goto out;
1531         }
1532
1533         rc = lst_ping_ioctl(str, type, timeout, count, ids, &head);
1534         if (rc == -1) { /* local failure */
1535                 lst_print_error("debug", "Failed to ping %s: %s\n",
1536                                 (str == NULL) ? "session" : str,
1537                                 strerror(errno));
1538                 rc = -1;
1539                 goto out;
1540         }
1541
1542         /* ignore RPC errors and framwork errors */
1543         list_for_each_entry(ent, &head, rpe_link) {
1544                 fprintf(stdout, "\t%s: %s [session: %s id: %s]\n",
1545                         libcfs_id2str(ent->rpe_peer),
1546                         lst_node_state2str(ent->rpe_state),
1547                         (ent->rpe_state == LST_NODE_ACTIVE ||
1548                          ent->rpe_state == LST_NODE_BUSY) ?
1549                                 (ent->rpe_rpc_errno == 0 ?
1550                                         &ent->rpe_payload[0] : "Unknown") :
1551                                 "<NULL>", libcfs_nid2str(ent->rpe_sid.ses_nid));
1552         }
1553
1554 out:
1555         lst_free_rpcent(&head);
1556
1557         if (ids != NULL)
1558                 free(ids);
1559
1560         return rc;
1561
1562 }
1563
1564 static int
1565 lst_add_nodes_ioctl(char *name, int count, struct lnet_process_id *ids,
1566                     unsigned *featp, struct list_head *resultp)
1567 {
1568         struct lstio_group_nodes_args args = { 0 };
1569
1570         args.lstio_grp_key     = session_key;
1571         args.lstio_grp_nmlen   = strlen(name);
1572         args.lstio_grp_namep   = name;
1573         args.lstio_grp_count   = count;
1574         args.lstio_grp_featp   = featp;
1575         args.lstio_grp_idsp    = ids;
1576         args.lstio_grp_resultp = resultp;
1577
1578         return lst_ioctl(LSTIO_NODES_ADD, &args, sizeof(args));
1579 }
1580
1581 static int
1582 lst_del_group_ioctl(char *name)
1583 {
1584         struct lstio_group_del_args args = { 0 };
1585
1586         args.lstio_grp_key   = session_key;
1587         args.lstio_grp_nmlen = strlen(name);
1588         args.lstio_grp_namep = name;
1589
1590         return lst_ioctl(LSTIO_GROUP_DEL, &args, sizeof(args));
1591 }
1592
1593 static int
1594 lst_del_group(char *grp_name)
1595 {
1596         int     rc;
1597
1598         rc = lst_del_group_ioctl(grp_name);
1599         if (rc == 0) {
1600                 fprintf(stdout, "Group is deleted\n");
1601                 return 0;
1602         }
1603
1604         if (rc == -1) {
1605                 lst_print_error("group", "Failed to delete group: %s\n",
1606                                 strerror(errno));
1607                 return rc;
1608         }
1609
1610         fprintf(stderr, "Group is deleted with some errors\n");
1611
1612         if (trans_stat.trs_rpc_errno != 0) {
1613                 fprintf(stderr,
1614                         "[RPC] Failed to send %d end session RPCs: %s\n",
1615                         lstcon_rpc_stat_failure(&trans_stat, 0),
1616                         strerror(trans_stat.trs_rpc_errno));
1617         }
1618
1619         if (trans_stat.trs_fwk_errno != 0) {
1620                 fprintf(stderr,
1621                         "[FWK] Failed to end session on %d nodes: %s\n",
1622                 lstcon_sesop_stat_failure(&trans_stat, 0),
1623                 strerror(trans_stat.trs_fwk_errno));
1624         }
1625
1626         return -1;
1627 }
1628
1629 static int
1630 lst_add_group_ioctl(char *name)
1631 {
1632         struct lstio_group_add_args args = { 0 };
1633
1634         args.lstio_grp_key     =  session_key;
1635         args.lstio_grp_nmlen   =  strlen(name);
1636         args.lstio_grp_namep   =  name;
1637
1638         return lst_ioctl(LSTIO_GROUP_ADD, &args, sizeof(args));
1639 }
1640
1641 static int
1642 jt_lst_add_group(int argc, char **argv)
1643 {
1644         struct list_head   head;
1645         struct lnet_process_id *ids;
1646         char              *name;
1647         unsigned           feats = session_features;
1648         int                count;
1649         int                rc;
1650         int                i;
1651         bool               nodes_added = false;
1652
1653         if (session_key == 0) {
1654                 fprintf(stderr,
1655                         "Can't find env LST_SESSION or value is not valid\n");
1656                 return -1;
1657         }
1658
1659         if (argc < 3) {
1660                 lst_print_usage(argv[0]);
1661                 return -1;
1662         }
1663
1664         name = argv[1];
1665         if (strlen(name) >= LST_NAME_SIZE) {
1666                 fprintf(stderr, "Name length is limited to %d\n",
1667                         LST_NAME_SIZE - 1);
1668                 return -1;
1669         }
1670
1671         rc = lst_add_group_ioctl(name);
1672         if (rc != 0) {
1673                 lst_print_error("group", "Failed to add group %s: %s\n",
1674                                 name, strerror(errno));
1675                 return -1;
1676         }
1677
1678         INIT_LIST_HEAD(&head);
1679
1680         for (i = 2; i < argc; i++) {
1681                 /* parse address list */
1682                 rc = lst_parse_nids(argv[i], &count, &ids);
1683                 if (rc < 0) {
1684                         fprintf(stderr, "Ignore invalid id list %s\n",
1685                                 argv[i]);
1686                         continue;
1687                 }
1688
1689                 if (count == 0)
1690                         continue;
1691
1692                 rc = lst_alloc_rpcent(&head, count, 0);
1693                 if (rc != 0) {
1694                         fprintf(stderr, "Out of memory\n");
1695                         free(ids);
1696                         rc = -1;
1697                         goto failed;
1698                 }
1699
1700                 rc = lst_add_nodes_ioctl(name, count, ids, &feats, &head);
1701
1702                 free(ids);
1703
1704                 if (rc != 0)
1705                         goto failed;
1706
1707                 fprintf(stdout, "%s are added to session\n", argv[i]);
1708
1709                 nodes_added = true;
1710
1711                 if ((feats & session_features) != session_features) {
1712                         fprintf(stdout,
1713                                 "Warning, this session will run with "
1714                                 "compatible mode because some test nodes "
1715                                 "might not understand these features: %x\n",
1716                                 (~feats & session_features));
1717                 }
1718
1719                 lst_free_rpcent(&head);
1720         }
1721
1722         if (!nodes_added) {
1723                 /*
1724                  * The selftest kernel module expects that a group should
1725                  * have at least one node, since it doesn't make sense for
1726                  * an empty group to be added to a test.
1727                  */
1728                 fprintf(stderr,
1729                         "No nodes added successfully, deleting group %s\n",
1730                         name);
1731                 rc = lst_del_group(name);
1732                 if (rc != 0) {
1733                         fprintf(stderr,
1734                                 "Failed to delete group %s."
1735                                 "  Group is empty.\n", name);
1736                 }
1737         }
1738
1739         return rc;
1740
1741 failed:
1742         if (rc == -1) {
1743                 lst_print_error("group", "Failed to add nodes %s: %s\n",
1744                                 argv[i], strerror(errno));
1745
1746         } else {
1747                 if (trans_stat.trs_fwk_errno == EPROTO) {
1748                         fprintf(stderr,
1749                                 "test nodes might have different LST "
1750                                 "features, please disable some features by "
1751                                 "setting LST_FEATURES\n");
1752                 }
1753
1754                 lst_print_transerr(&head, "create session");
1755         }
1756
1757         lst_free_rpcent(&head);
1758
1759         if (!nodes_added) {
1760                 fprintf(stderr,
1761                         "No nodes added successfully, deleting group %s\n",
1762                         name);
1763                 if (lst_del_group(name) != 0) {
1764                         fprintf(stderr,
1765                                 "Failed to delete group %s."
1766                                 "  Group is empty.\n", name);
1767                 }
1768         }
1769
1770         return rc;
1771 }
1772
1773 static int
1774 jt_lst_del_group(int argc, char **argv)
1775 {
1776         int     rc;
1777
1778         if (session_key == 0) {
1779                 fprintf(stderr,
1780                         "Can't find env LST_SESSION or value is not valid\n");
1781                 return -1;
1782         }
1783
1784         if (argc != 2) {
1785                 lst_print_usage(argv[0]);
1786                 return -1;
1787         }
1788
1789         rc = lst_del_group(argv[1]);
1790
1791         return rc;
1792 }
1793
1794 static int
1795 lst_update_group_ioctl(int opc, char *name, int clean, int count,
1796                        struct lnet_process_id *ids, struct list_head *resultp)
1797 {
1798         struct lstio_group_update_args args = { 0 };
1799
1800         args.lstio_grp_key      = session_key;
1801         args.lstio_grp_opc      = opc;
1802         args.lstio_grp_args     = clean;
1803         args.lstio_grp_nmlen    = strlen(name);
1804         args.lstio_grp_namep    = name;
1805         args.lstio_grp_count    = count;
1806         args.lstio_grp_idsp     = ids;
1807         args.lstio_grp_resultp  = resultp;
1808
1809         return lst_ioctl(LSTIO_GROUP_UPDATE, &args, sizeof(args));
1810 }
1811
1812 static int
1813 jt_lst_update_group(int argc, char **argv)
1814 {
1815         struct list_head   head;
1816         struct lnet_process_id *ids = NULL;
1817         char              *str = NULL;
1818         char              *grp = NULL;
1819         int                optidx = 0;
1820         int                count = 0;
1821         int                clean = 0;
1822         int                opc = 0;
1823         int                rc;
1824         int                c;
1825
1826         static const struct option update_group_opts[] = {
1827                 { .name = "refresh", .has_arg = no_argument,       .val = 'f' },
1828                 { .name = "clean",   .has_arg = required_argument, .val = 'c' },
1829                 { .name = "remove",  .has_arg = required_argument, .val = 'r' },
1830                 { .name = NULL } };
1831
1832         if (session_key == 0) {
1833                 fprintf(stderr,
1834                         "Can't find env LST_SESSION or value is not valid\n");
1835                 return -1;
1836         }
1837
1838         while (1) {
1839                 c = getopt_long(argc, argv, "fc:r:",
1840                                 update_group_opts, &optidx);
1841
1842                 /* Detect the end of the options. */
1843                 if (c == -1)
1844                         break;
1845
1846                 switch (c) {
1847                 case 'f':
1848                         if (opc != 0) {
1849                                 lst_print_usage(argv[0]);
1850                                 return -1;
1851                         }
1852                         opc = LST_GROUP_REFRESH;
1853                         break;
1854
1855                 case 'r':
1856                         if (opc != 0) {
1857                                 lst_print_usage(argv[0]);
1858                                 return -1;
1859                         }
1860                         opc = LST_GROUP_RMND;
1861                         str = optarg;
1862                         break;
1863
1864                 case 'c':
1865                         clean = lst_node_str2state(optarg);
1866                         if (opc != 0 || clean <= 0) {
1867                                 lst_print_usage(argv[0]);
1868                                 return -1;
1869                         }
1870                         opc = LST_GROUP_CLEAN;
1871                         break;
1872
1873                 default:
1874                         lst_print_usage(argv[0]);
1875                         return -1;
1876                 }
1877         }
1878
1879         /* no OPC or group is specified */
1880         if (opc == 0 || optind != argc - 1) {
1881                 lst_print_usage(argv[0]);
1882                 return -1;
1883         }
1884
1885         grp = argv[optind];
1886
1887         INIT_LIST_HEAD(&head);
1888
1889         if (opc == LST_GROUP_RMND || opc == LST_GROUP_REFRESH) {
1890                 rc = lst_get_node_count(opc == LST_GROUP_RMND ? LST_OPC_NODES :
1891                                                                 LST_OPC_GROUP,
1892                                         opc == LST_GROUP_RMND ? str : grp,
1893                                         &count, &ids);
1894
1895                 if (rc != 0) {
1896                         fprintf(stderr, "Can't get count of nodes from %s: %s\n",
1897                                 opc == LST_GROUP_RMND ? str : grp,
1898                                 strerror(errno));
1899                         return -1;
1900                 }
1901
1902                 rc = lst_alloc_rpcent(&head, count, 0);
1903                 if (rc != 0) {
1904                         fprintf(stderr, "Out of memory\n");
1905                         free(ids);
1906                         return -1;
1907                 }
1908
1909         }
1910
1911         rc = lst_update_group_ioctl(opc, grp, clean, count, ids, &head);
1912
1913         if (ids != NULL)
1914                 free(ids);
1915
1916         if (rc == 0) {
1917                 lst_free_rpcent(&head);
1918                 return 0;
1919         }
1920
1921         if (rc == -1) {
1922                 lst_free_rpcent(&head);
1923                 lst_print_error("group", "Failed to update group: %s\n",
1924                                 strerror(errno));
1925                 return rc;
1926         }
1927
1928         lst_print_transerr(&head, "Updating group");
1929
1930         lst_free_rpcent(&head);
1931
1932         return rc;
1933 }
1934
1935 static int
1936 lst_list_group_ioctl(int len, char *name, int idx)
1937 {
1938         struct lstio_group_list_args args = { 0 };
1939
1940         args.lstio_grp_key   = session_key;
1941         args.lstio_grp_idx   = idx;
1942         args.lstio_grp_nmlen = len;
1943         args.lstio_grp_namep = name;
1944
1945         return lst_ioctl(LSTIO_GROUP_LIST, &args, sizeof(args));
1946 }
1947
1948 static int
1949 lst_info_group_ioctl(char *name, struct lstcon_ndlist_ent *gent,
1950                      int *idx, int *count, struct lstcon_node_ent *dents)
1951 {
1952         struct lstio_group_info_args args = { 0 };
1953
1954         args.lstio_grp_key    = session_key;
1955         args.lstio_grp_nmlen  = strlen(name);
1956         args.lstio_grp_namep  = name;
1957         args.lstio_grp_entp   = gent;
1958         args.lstio_grp_idxp   = idx;
1959         args.lstio_grp_ndentp = count;
1960         args.lstio_grp_dentsp = dents;
1961
1962         return lst_ioctl(LSTIO_GROUP_INFO, &args, sizeof(args));
1963 }
1964
1965 static int
1966 lst_list_group_all(void)
1967 {
1968         char  name[LST_NAME_SIZE];
1969         int   rc;
1970         int   i;
1971
1972         /* no group is specified, list name of all groups */
1973         for (i = 0; ; i++) {
1974                 rc = lst_list_group_ioctl(LST_NAME_SIZE, name, i);
1975                 if (rc == 0) {
1976                         fprintf(stdout, "%d) %s\n", i + 1, name);
1977                         continue;
1978                 }
1979
1980                 if (errno == ENOENT)
1981                         break;
1982
1983                 lst_print_error("group", "Failed to list group: %s\n",
1984                                 strerror(errno));
1985                 return -1;
1986         }
1987
1988         fprintf(stdout, "Total %d groups\n", i);
1989
1990         return 0;
1991 }
1992
1993 static int
1994 jt_lst_list_group(int argc, char **argv)
1995 {
1996         struct lstcon_ndlist_ent gent;
1997         struct lstcon_node_ent   *dents;
1998         int states = 0;
1999         int optidx  = 0;
2000         int verbose = 0;
2001         int active  = 0;
2002         int busy    = 0;
2003         int down    = 0;
2004         int unknown = 0;
2005         int all     = 0;
2006         int count;
2007         int index;
2008         int i;
2009         int j;
2010         int c;
2011         int rc      = 0;
2012         static const struct option list_group_opts[] = {
2013                 { .name = "active",  .has_arg = no_argument, .val = 'a' },
2014                 { .name = "busy",    .has_arg = no_argument, .val = 'b' },
2015                 { .name = "down",    .has_arg = no_argument, .val = 'd' },
2016                 { .name = "unknown", .has_arg = no_argument, .val = 'u' },
2017                 { .name = "all",     .has_arg = no_argument, .val = 'l' },
2018                 { .name = NULL, } };
2019
2020         if (session_key == 0) {
2021                 fprintf(stderr,
2022                         "Can't find env LST_SESSION or value is not valid\n");
2023                 return -1;
2024         }
2025
2026         while (1) {
2027                 c = getopt_long(argc, argv, "abdul",
2028                                 list_group_opts, &optidx);
2029
2030                 if (c == -1)
2031                         break;
2032
2033                 switch (c) {
2034                 case 'a':
2035                         verbose = active = 1;
2036                         states |= LST_NODE_ACTIVE;
2037                         all = 0;
2038                         break;
2039                 case 'b':
2040                         verbose = busy = 1;
2041                         states |= LST_NODE_BUSY;
2042                         all = 0;
2043                         break;
2044                 case 'd':
2045                         verbose = down = 1;
2046                         states |= LST_NODE_DOWN;
2047                         all = 0;
2048                         break;
2049                 case 'u':
2050                         verbose = unknown = 1;
2051                         states |= LST_NODE_UNKNOWN;
2052                         all = 0;
2053                         break;
2054                 case 'l':
2055                         states |= LST_NODE_ACTIVE | LST_NODE_BUSY |
2056                                   LST_NODE_DOWN | LST_NODE_UNKNOWN;
2057                         verbose = all = 1;
2058                         break;
2059                 default:
2060                         lst_print_usage(argv[0]);
2061                         return -1;
2062                 }
2063         }
2064
2065         if (optind == argc) {
2066                 rc = lst_yaml_groups(NLM_F_DUMP, NULL, 0, true);
2067                 if (rc <= 0) {
2068                         if (rc == -EOPNOTSUPP)
2069                                 goto old_api;
2070                         return rc;
2071                 }
2072         } else {
2073                 for (i = optind; i < argc; i++) {
2074                         rc = lst_yaml_groups(NLM_F_DUMP, argv[i], states,
2075                                              i == optind ? true : false);
2076                         if (rc < 0) {
2077                                 if (rc == -EOPNOTSUPP)
2078                                         goto old_api;
2079                                 return rc;
2080                         }
2081                 }
2082                 return 0;
2083         }
2084 old_api:
2085         if (optind == argc) {
2086                 /* no group is specified, list name of all groups */
2087                 rc = lst_list_group_all();
2088
2089                 return rc;
2090         }
2091
2092         if (!verbose)
2093                 fprintf(stdout, LST_NODES_TITLE);
2094
2095         /* list nodes in specified groups */
2096         for (i = optind; i < argc; i++) {
2097                 rc = lst_info_group_ioctl(argv[i], &gent, NULL, NULL, NULL);
2098                 if (rc != 0) {
2099                         if (errno == ENOENT) {
2100                                 rc = 0;
2101                                 break;
2102                         }
2103
2104                         lst_print_error("group", "Failed to list group\n",
2105                                         strerror(errno));
2106                         break;
2107                 }
2108
2109                 if (!verbose) {
2110                         fprintf(stdout, "\t%d\t%d\t%d\t%d\t%d\t%s\n",
2111                                 gent.nle_nactive, gent.nle_nbusy,
2112                                 gent.nle_ndown, gent.nle_nunknown,
2113                                 gent.nle_nnode, argv[i]);
2114                         continue;
2115                 }
2116
2117                 fprintf(stdout, "Group [ %s ]\n", argv[i]);
2118
2119                 if (gent.nle_nnode == 0) {
2120                         fprintf(stdout, "No nodes found [ %s ]\n", argv[i]);
2121                         continue;
2122                 }
2123
2124                 count = gent.nle_nnode;
2125
2126                 dents = malloc(count * sizeof(struct lstcon_node_ent));
2127                 if (dents == NULL) {
2128                         fprintf(stderr, "Failed to malloc: %s\n",
2129                                 strerror(errno));
2130                         return -1;
2131                 }
2132
2133                 index = 0;
2134                 rc = lst_info_group_ioctl(argv[i], &gent, &index, &count, dents);
2135                 if (rc != 0) {
2136                         lst_print_error("group", "Failed to list group: %s\n",
2137                                         strerror(errno));
2138                         free(dents);
2139                         return -1;
2140                 }
2141
2142                 for (j = 0, c = 0; j < count; j++) {
2143                         if (all ||
2144                             ((active  &&  dents[j].nde_state == LST_NODE_ACTIVE) ||
2145                              (busy    &&  dents[j].nde_state == LST_NODE_BUSY)   ||
2146                              (down    &&  dents[j].nde_state == LST_NODE_DOWN)   ||
2147                              (unknown &&  dents[j].nde_state == LST_NODE_UNKNOWN))) {
2148
2149                                 fprintf(stdout, "\t%s: %s\n",
2150                                         libcfs_id2str(dents[j].nde_id),
2151                                         lst_node_state2str(dents[j].nde_state));
2152                                 c++;
2153                         }
2154                 }
2155
2156                 fprintf(stdout, "Total %d nodes [ %s ]\n", c, argv[i]);
2157
2158                 free(dents);
2159         }
2160
2161         return rc;
2162 }
2163
2164 static int
2165 lst_stat_ioctl(char *name, int count, struct lnet_process_id *idsp,
2166                int timeout, struct list_head *resultp)
2167 {
2168         struct lstio_stat_args args = { 0 };
2169
2170         args.lstio_sta_key     = session_key;
2171         args.lstio_sta_timeout = timeout;
2172         args.lstio_sta_nmlen   = strlen(name);
2173         args.lstio_sta_namep   = name;
2174         args.lstio_sta_count   = count;
2175         args.lstio_sta_idsp    = idsp;
2176         args.lstio_sta_resultp = resultp;
2177
2178         return lst_ioctl(LSTIO_STAT_QUERY, &args, sizeof(args));
2179 }
2180
2181 typedef struct {
2182         struct list_head              srp_link;
2183         int                     srp_count;
2184         char                   *srp_name;
2185         struct lnet_process_id      *srp_ids;
2186         struct list_head              srp_result[2];
2187 } lst_stat_req_param_t;
2188
2189 static void
2190 lst_stat_req_param_free(lst_stat_req_param_t *srp)
2191 {
2192         int     i;
2193
2194         for (i = 0; i < 2; i++)
2195                 lst_free_rpcent(&srp->srp_result[i]);
2196
2197         if (srp->srp_ids != NULL)
2198                 free(srp->srp_ids);
2199
2200         free(srp);
2201 }
2202
2203 static int
2204 lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp, int save_old)
2205 {
2206         lst_stat_req_param_t *srp = NULL;
2207         int                   count = save_old ? 2 : 1;
2208         int                   rc;
2209         int                   i;
2210
2211         srp = malloc(sizeof(*srp));
2212         if (srp == NULL)
2213                 return -ENOMEM;
2214
2215         memset(srp, 0, sizeof(*srp));
2216         INIT_LIST_HEAD(&srp->srp_result[0]);
2217         INIT_LIST_HEAD(&srp->srp_result[1]);
2218
2219         rc = lst_get_node_count(LST_OPC_GROUP, name,
2220                                 &srp->srp_count, NULL);
2221         if (rc != 0 && errno == ENOENT) {
2222                 rc = lst_get_node_count(LST_OPC_NODES, name,
2223                                         &srp->srp_count, &srp->srp_ids);
2224         }
2225
2226         if (rc != 0) {
2227                 fprintf(stderr,
2228                         "Failed to get count of nodes from %s: %s\n",
2229                         name, strerror(errno));
2230                 lst_stat_req_param_free(srp);
2231
2232                 return rc;
2233         }
2234
2235         srp->srp_name = name;
2236
2237         for (i = 0; i < count; i++) {
2238                 rc = lst_alloc_rpcent(&srp->srp_result[i], srp->srp_count,
2239                                       sizeof(struct sfw_counters)  +
2240                                       sizeof(struct srpc_counters) +
2241                                       sizeof(struct lnet_counters_common));
2242                 if (rc != 0) {
2243                         fprintf(stderr, "Out of memory\n");
2244                         break;
2245                 }
2246         }
2247
2248         if (rc == 0) {
2249                 *srpp = srp;
2250                 return 0;
2251         }
2252
2253         lst_stat_req_param_free(srp);
2254
2255         return rc;
2256 }
2257
2258 typedef struct {
2259         /* TODO */
2260         int foo;
2261 } lst_srpc_stat_result;
2262
2263 #define LST_LNET_AVG    0
2264 #define LST_LNET_MIN    1
2265 #define LST_LNET_MAX    2
2266
2267 typedef struct {
2268         float           lnet_avg_sndrate;
2269         float           lnet_min_sndrate;
2270         float           lnet_max_sndrate;
2271         float           lnet_total_sndrate;
2272
2273         float           lnet_avg_rcvrate;
2274         float           lnet_min_rcvrate;
2275         float           lnet_max_rcvrate;
2276         float           lnet_total_rcvrate;
2277
2278         float           lnet_avg_sndperf;
2279         float           lnet_min_sndperf;
2280         float           lnet_max_sndperf;
2281         float           lnet_total_sndperf;
2282
2283         float           lnet_avg_rcvperf;
2284         float           lnet_min_rcvperf;
2285         float           lnet_max_rcvperf;
2286         float           lnet_total_rcvperf;
2287
2288         int             lnet_stat_count;
2289 } lst_lnet_stat_result_t;
2290
2291 lst_lnet_stat_result_t lnet_stat_result;
2292
2293 static float
2294 lst_lnet_stat_value(int bw, int send, int off)
2295 {
2296         float  *p;
2297
2298         p = bw ? &lnet_stat_result.lnet_avg_sndperf :
2299                  &lnet_stat_result.lnet_avg_sndrate;
2300
2301         if (!send)
2302                 p += 4;
2303
2304         p += off;
2305
2306         return *p;
2307 }
2308
2309 static void
2310 lst_cal_lnet_stat(float delta, struct lnet_counters_common *lnet_new,
2311                   struct lnet_counters_common *lnet_old, int mbs)
2312 {
2313         float perf;
2314         float rate;
2315         unsigned int unit_divisor;
2316
2317         unit_divisor = (mbs) ? (1000 * 1000) : (1024 * 1024);
2318         perf = (float)(lnet_new->lcc_send_length -
2319                        lnet_old->lcc_send_length) / unit_divisor / delta;
2320         lnet_stat_result.lnet_total_sndperf += perf;
2321
2322         if (lnet_stat_result.lnet_min_sndperf > perf ||
2323             lnet_stat_result.lnet_min_sndperf == 0)
2324                 lnet_stat_result.lnet_min_sndperf = perf;
2325
2326         if (lnet_stat_result.lnet_max_sndperf < perf)
2327                 lnet_stat_result.lnet_max_sndperf = perf;
2328
2329         perf = (float)(lnet_new->lcc_recv_length -
2330                        lnet_old->lcc_recv_length) / unit_divisor / delta;
2331         lnet_stat_result.lnet_total_rcvperf += perf;
2332
2333         if (lnet_stat_result.lnet_min_rcvperf > perf ||
2334             lnet_stat_result.lnet_min_rcvperf == 0)
2335                 lnet_stat_result.lnet_min_rcvperf = perf;
2336
2337         if (lnet_stat_result.lnet_max_rcvperf < perf)
2338                 lnet_stat_result.lnet_max_rcvperf = perf;
2339
2340         rate = (lnet_new->lcc_send_count - lnet_old->lcc_send_count) / delta;
2341         lnet_stat_result.lnet_total_sndrate += rate;
2342
2343         if (lnet_stat_result.lnet_min_sndrate > rate ||
2344             lnet_stat_result.lnet_min_sndrate == 0)
2345                 lnet_stat_result.lnet_min_sndrate = rate;
2346
2347         if (lnet_stat_result.lnet_max_sndrate < rate)
2348                 lnet_stat_result.lnet_max_sndrate = rate;
2349
2350         rate = (lnet_new->lcc_recv_count - lnet_old->lcc_recv_count) / delta;
2351         lnet_stat_result.lnet_total_rcvrate += rate;
2352
2353         if (lnet_stat_result.lnet_min_rcvrate > rate ||
2354             lnet_stat_result.lnet_min_rcvrate == 0)
2355                 lnet_stat_result.lnet_min_rcvrate = rate;
2356
2357         if (lnet_stat_result.lnet_max_rcvrate < rate)
2358                 lnet_stat_result.lnet_max_rcvrate = rate;
2359
2360         lnet_stat_result.lnet_stat_count++;
2361
2362         lnet_stat_result.lnet_avg_sndrate = lnet_stat_result.lnet_total_sndrate /
2363                                             lnet_stat_result.lnet_stat_count;
2364         lnet_stat_result.lnet_avg_rcvrate = lnet_stat_result.lnet_total_rcvrate /
2365                                             lnet_stat_result.lnet_stat_count;
2366
2367         lnet_stat_result.lnet_avg_sndperf = lnet_stat_result.lnet_total_sndperf /
2368                                             lnet_stat_result.lnet_stat_count;
2369         lnet_stat_result.lnet_avg_rcvperf = lnet_stat_result.lnet_total_rcvperf /
2370                                             lnet_stat_result.lnet_stat_count;
2371 }
2372
2373 static void
2374 lst_print_lnet_stat(char *name, int bwrt, int rdwr, int type, int mbs)
2375 {
2376         int     start1 = 0;
2377         int     end1   = 1;
2378         int     start2 = 0;
2379         int     end2   = 1;
2380         int     i;
2381         int     j;
2382         char   *units;
2383
2384         if (lnet_stat_result.lnet_stat_count == 0)
2385                 return;
2386
2387         units = (mbs) ? "MB/s  " : "MiB/s ";
2388
2389         if (bwrt == 1) /* bw only */
2390                 start1 = 1;
2391
2392         if (bwrt == 2) /* rates only */
2393                 end1 = 0;
2394
2395         if (rdwr == 1) /* recv only */
2396                 start2 = 1;
2397
2398         if (rdwr == 2) /* send only */
2399                 end2 = 0;
2400
2401         for (i = start1; i <= end1; i++) {
2402                 fprintf(stdout, "[LNet %s of %s]\n",
2403                         i == 0 ? "Rates" : "Bandwidth", name);
2404
2405                 for (j = start2; j <= end2; j++) {
2406                         fprintf(stdout, "[%c] ", j == 0 ? 'R' : 'W');
2407
2408                         if ((type & 1) != 0) {
2409                                 fprintf(stdout, i == 0 ? "Avg: %-8.0f RPC/s " :
2410                                                          "Avg: %-8.2f %s",
2411                                         lst_lnet_stat_value(i, j, 0), units);
2412                         }
2413
2414                         if ((type & 2) != 0) {
2415                                 fprintf(stdout, i == 0 ? "Min: %-8.0f RPC/s " :
2416                                                          "Min: %-8.2f %s",
2417                                         lst_lnet_stat_value(i, j, 1), units);
2418                         }
2419
2420                         if ((type & 4) != 0) {
2421                                 fprintf(stdout, i == 0 ? "Max: %-8.0f RPC/s" :
2422                                                          "Max: %-8.2f %s",
2423                                         lst_lnet_stat_value(i, j, 2), units);
2424                         }
2425
2426                         fprintf(stdout, "\n");
2427                 }
2428         }
2429 }
2430
2431 static void
2432 lst_print_stat(char *name, struct list_head *resultp,
2433                int idx, int lnet, int bwrt, int rdwr, int type,
2434                int mbs)
2435 {
2436         struct list_head tmp[2];
2437         struct lstcon_rpc_ent *new;
2438         struct lstcon_rpc_ent *old;
2439         struct sfw_counters *sfwk_new;
2440         struct sfw_counters *sfwk_old;
2441         struct srpc_counters *srpc_new;
2442         struct srpc_counters *srpc_old;
2443         struct lnet_counters_common *lnet_new;
2444         struct lnet_counters_common *lnet_old;
2445         float delta;
2446         int errcount = 0;
2447
2448         INIT_LIST_HEAD(&tmp[0]);
2449         INIT_LIST_HEAD(&tmp[1]);
2450
2451         memset(&lnet_stat_result, 0, sizeof(lnet_stat_result));
2452
2453         while (!list_empty(&resultp[idx])) {
2454                 if (list_empty(&resultp[1 - idx])) {
2455                         fprintf(stderr, "Group is changed, re-run stat\n");
2456                         break;
2457                 }
2458
2459                 new = list_first_entry(&resultp[idx], struct lstcon_rpc_ent,
2460                                        rpe_link);
2461                 old = list_first_entry(&resultp[1 - idx], struct lstcon_rpc_ent,
2462                                        rpe_link);
2463
2464                 /* first time get stats result, can't calculate diff */
2465                 if (new->rpe_peer.nid == LNET_NID_ANY)
2466                         break;
2467
2468                 if (new->rpe_peer.nid != old->rpe_peer.nid ||
2469                     new->rpe_peer.pid != old->rpe_peer.pid) {
2470                         /* Something wrong. i.e, somebody change the group */
2471                         break;
2472                 }
2473
2474                 list_move_tail(&new->rpe_link, &tmp[idx]);
2475
2476                 list_move_tail(&old->rpe_link, &tmp[1 - idx]);
2477
2478                 if (new->rpe_rpc_errno != 0 || new->rpe_fwk_errno != 0 ||
2479                     old->rpe_rpc_errno != 0 || old->rpe_fwk_errno != 0) {
2480                         errcount++;
2481                         continue;
2482                 }
2483
2484                 sfwk_new = (struct sfw_counters *)&new->rpe_payload[0];
2485                 sfwk_old = (struct sfw_counters *)&old->rpe_payload[0];
2486
2487                 srpc_new = (struct srpc_counters *)((char *)sfwk_new +
2488                                                     sizeof(*sfwk_new));
2489                 srpc_old = (struct srpc_counters *)((char *)sfwk_old +
2490                                                     sizeof(*sfwk_old));
2491
2492                 lnet_new = (struct lnet_counters_common *)((char *)srpc_new +
2493                                                            sizeof(*srpc_new));
2494                 lnet_old = (struct lnet_counters_common *)((char *)srpc_old +
2495                                                            sizeof(*srpc_old));
2496
2497                 /* Prior to version 2.3, the running_ms was a counter for
2498                  * the number of running tests. Since 2.3, running_ms is
2499                  * changed to hold the millisecond since the start of
2500                  * the work item. The rpe_stamp field was formerly used,
2501                  * but is no longer. In 2.12 rpe_stamp was changed to
2502                  * struct timespec64 and has nanosecond resolution, in
2503                  * case it is needed in the future.
2504                  */
2505                 delta = (float)(sfwk_new->running_ms -
2506                                 sfwk_old->running_ms) / 1000;
2507
2508                 if (!lnet) /* TODO */
2509                         continue;
2510
2511                 lst_cal_lnet_stat(delta, lnet_new, lnet_old, mbs);
2512         }
2513
2514         list_splice(&tmp[idx], &resultp[idx]);
2515         list_splice(&tmp[1 - idx], &resultp[1 - idx]);
2516
2517         if (errcount > 0)
2518                 fprintf(stdout, "Failed to stat on %d nodes\n", errcount);
2519
2520         if (!lnet)  /* TODO */
2521                 return;
2522
2523         lst_print_lnet_stat(name, bwrt, rdwr, type, mbs);
2524 }
2525
2526 static int
2527 jt_lst_stat(int argc, char **argv)
2528 {
2529         struct list_head        head;
2530         lst_stat_req_param_t *srp;
2531         time_t                last    = 0;
2532         int                   optidx  = 0;
2533         int                   timeout = 5; /* default timeout, 5 sec */
2534         int                   delay   = 5; /* default delay, 5 sec */
2535         int                   count   = -1; /* run forever */
2536         int                   lnet    = 1; /* lnet stat by default */
2537         int                   bwrt    = 0;
2538         int                   rdwr    = 0;
2539         int                   type    = -1;
2540         int                   idx     = 0;
2541         int                   rc;
2542         int                   c;
2543         int                   mbs     = 0; /* report as MB/s */
2544
2545         static const struct option stat_opts[] = {
2546                 { .name = "timeout", .has_arg = required_argument, .val = 't' },
2547                 { .name = "delay",   .has_arg = required_argument, .val = 'd' },
2548                 { .name = "count",   .has_arg = required_argument, .val = 'o' },
2549                 { .name = "lnet",    .has_arg = no_argument,       .val = 'l' },
2550                 { .name = "rpc",     .has_arg = no_argument,       .val = 'c' },
2551                 { .name = "bw",      .has_arg = no_argument,       .val = 'b' },
2552                 { .name = "rate",    .has_arg = no_argument,       .val = 'a' },
2553                 { .name = "read",    .has_arg = no_argument,       .val = 'r' },
2554                 { .name = "write",   .has_arg = no_argument,       .val = 'w' },
2555                 { .name = "avg",     .has_arg = no_argument,       .val = 'g' },
2556                 { .name = "min",     .has_arg = no_argument,       .val = 'n' },
2557                 { .name = "max",     .has_arg = no_argument,       .val = 'x' },
2558                 { .name = "mbs",     .has_arg = no_argument,       .val = 'm' },
2559                 { .name = NULL } };
2560
2561         if (session_key == 0) {
2562                 fprintf(stderr,
2563                         "Can't find env LST_SESSION or value is not valid\n");
2564                 return -1;
2565         }
2566
2567         while (1) {
2568                 c = getopt_long(argc, argv, "t:d:lcbarwgnxm", stat_opts,
2569                                 &optidx);
2570
2571                 if (c == -1)
2572                         break;
2573
2574                 switch (c) {
2575                 case 't':
2576                         timeout = atoi(optarg);
2577                         break;
2578                 case 'd':
2579                         delay = atoi(optarg);
2580                         break;
2581                 case 'o':
2582                         count = atoi(optarg);
2583                         break;
2584                 case 'l':
2585                         lnet = 1;
2586                         break;
2587                 case 'c':
2588                         lnet = 0;
2589                         break;
2590                 case 'b':
2591                         bwrt |= 1;
2592                         break;
2593                 case 'a':
2594                         bwrt |= 2;
2595                         break;
2596                 case 'r':
2597                         rdwr |= 1;
2598                         break;
2599                 case 'w':
2600                         rdwr |= 2;
2601                         break;
2602                 case 'g':
2603                         if (type == -1) {
2604                                 type = 1;
2605                                 break;
2606                         }
2607                         type |= 1;
2608                         break;
2609                 case 'n':
2610                         if (type == -1) {
2611                                 type = 2;
2612                                 break;
2613                         }
2614                         type |= 2;
2615                         break;
2616                 case 'x':
2617                         if (type == -1) {
2618                                 type = 4;
2619                                 break;
2620                         }
2621                         type |= 4;
2622                         break;
2623                 case 'm':
2624                         mbs = 1;
2625                         break;
2626
2627                 default:
2628                         lst_print_usage(argv[0]);
2629                         return -1;
2630                 }
2631         }
2632
2633         if (optind == argc) {
2634                 lst_print_usage(argv[0]);
2635                 return -1;
2636         }
2637
2638         if (timeout <= 0 || delay <= 0) {
2639                 fprintf(stderr, "Invalid timeout or delay value\n");
2640                 return -1;
2641         }
2642
2643         if (count < -1) {
2644             fprintf(stderr, "Invalid count value\n");
2645             return -1;
2646         }
2647
2648         /* extra count to get first data point */
2649         if (count != -1)
2650             count++;
2651
2652         INIT_LIST_HEAD(&head);
2653
2654         while (optind < argc) {
2655                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 1);
2656                 if (rc != 0)
2657                         goto out;
2658
2659                 list_add_tail(&srp->srp_link, &head);
2660         }
2661
2662         do {
2663                 time_t  now = time(NULL);
2664
2665                 if (now - last < delay) {
2666                         sleep(delay - now + last);
2667                         time(&now);
2668                 }
2669                 last = now;
2670
2671                 list_for_each_entry(srp, &head, srp_link) {
2672                         rc = lst_stat_ioctl(srp->srp_name,
2673                                             srp->srp_count, srp->srp_ids,
2674                                             timeout, &srp->srp_result[idx]);
2675                         if (rc == -1) {
2676                                 lst_print_error("stat", "Failed to stat %s: %s\n",
2677                                                 srp->srp_name, strerror(errno));
2678                                 goto out;
2679                         }
2680
2681                         lst_print_stat(srp->srp_name, srp->srp_result,
2682                                        idx, lnet, bwrt, rdwr, type, mbs);
2683
2684                         lst_reset_rpcent(&srp->srp_result[1 - idx]);
2685                 }
2686
2687                 idx = 1 - idx;
2688
2689                 if (count > 0)
2690                         count--;
2691         } while (count == -1 || count > 0);
2692
2693 out:
2694         while (!list_empty(&head)) {
2695                 srp = list_first_entry(&head, lst_stat_req_param_t, srp_link);
2696
2697                 list_del(&srp->srp_link);
2698                 lst_stat_req_param_free(srp);
2699         }
2700
2701         return rc;
2702 }
2703
2704 static int
2705 jt_lst_show_error(int argc, char **argv)
2706 {
2707         struct list_head       head;
2708         lst_stat_req_param_t  *srp;
2709         struct lstcon_rpc_ent *ent;
2710         struct sfw_counters   *sfwk;
2711         struct srpc_counters  *srpc;
2712         int                    show_rpc = 1;
2713         int                    optidx = 0;
2714         int                    rc = 0;
2715         int                    ecount;
2716         int                    c;
2717
2718         static const struct option show_error_opts[] = {
2719                 { .name = "session", .has_arg = no_argument, .val = 's' },
2720                 { .name = NULL, } };
2721
2722         if (session_key == 0) {
2723                 fprintf(stderr,
2724                         "Can't find env LST_SESSION or value is not valid\n");
2725                 return -1;
2726         }
2727
2728         while (1) {
2729                 c = getopt_long(argc, argv, "s", show_error_opts, &optidx);
2730
2731                 if (c == -1)
2732                         break;
2733
2734                 switch (c) {
2735                 case 's':
2736                         show_rpc  = 0;
2737                         break;
2738
2739                 default:
2740                         lst_print_usage(argv[0]);
2741                         return -1;
2742                 }
2743         }
2744
2745         if (optind == argc) {
2746                 lst_print_usage(argv[0]);
2747                 return -1;
2748         }
2749
2750         INIT_LIST_HEAD(&head);
2751
2752         while (optind < argc) {
2753                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 0);
2754                 if (rc != 0)
2755                         goto out;
2756
2757                 list_add_tail(&srp->srp_link, &head);
2758         }
2759
2760         list_for_each_entry(srp, &head, srp_link) {
2761                 rc = lst_stat_ioctl(srp->srp_name, srp->srp_count,
2762                                     srp->srp_ids, 10, &srp->srp_result[0]);
2763
2764                 if (rc == -1) {
2765                         lst_print_error(srp->srp_name, "Failed to show errors of %s: %s\n",
2766                                         srp->srp_name, strerror(errno));
2767                         goto out;
2768                 }
2769
2770                 fprintf(stdout, "%s:\n", srp->srp_name);
2771
2772                 ecount = 0;
2773
2774                 list_for_each_entry(ent, &srp->srp_result[0], rpe_link) {
2775                         if (ent->rpe_rpc_errno != 0) {
2776                                 ecount ++;
2777                                 fprintf(stderr, "RPC failure, can't show error on %s\n",
2778                                         libcfs_id2str(ent->rpe_peer));
2779                                 continue;
2780                         }
2781
2782                         if (ent->rpe_fwk_errno != 0) {
2783                                 ecount ++;
2784                                 fprintf(stderr, "Framework failure, can't show error on %s\n",
2785                                         libcfs_id2str(ent->rpe_peer));
2786                                 continue;
2787                         }
2788
2789                         sfwk = (struct sfw_counters *)&ent->rpe_payload[0];
2790                         srpc = (struct srpc_counters *)((char *)sfwk + sizeof(*sfwk));
2791
2792                         if (srpc->errors == 0 &&
2793                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2794                                 continue;
2795
2796                         if (!show_rpc  &&
2797                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2798                                 continue;
2799
2800                         ecount ++;
2801
2802                         fprintf(stderr, "%s: [Session %d brw errors, %d ping errors]%c",
2803                                 libcfs_id2str(ent->rpe_peer),
2804                                 sfwk->brw_errors, sfwk->ping_errors,
2805                                 show_rpc  ? ' ' : '\n');
2806
2807                         if (!show_rpc)
2808                                 continue;
2809
2810                         fprintf(stderr, "[RPC: %d errors, %d dropped, %d expired]\n",
2811                                 srpc->errors, srpc->rpcs_dropped, srpc->rpcs_expired);
2812                 }
2813
2814                 fprintf(stdout, "Total %d error nodes in %s\n", ecount, srp->srp_name);
2815         }
2816 out:
2817         while (!list_empty(&head)) {
2818                 srp = list_first_entry(&head, lst_stat_req_param_t, srp_link);
2819
2820                 list_del(&srp->srp_link);
2821                 lst_stat_req_param_free(srp);
2822         }
2823
2824         return rc;
2825 }
2826
2827 static int
2828 lst_add_batch_ioctl(char *name)
2829 {
2830         struct lstio_batch_add_args args = { 0 };
2831
2832         args.lstio_bat_key   = session_key;
2833         args.lstio_bat_nmlen = strlen(name);
2834         args.lstio_bat_namep = name;
2835
2836         return lst_ioctl (LSTIO_BATCH_ADD, &args, sizeof(args));
2837 }
2838
2839 static int
2840 jt_lst_add_batch(int argc, char **argv)
2841 {
2842         char   *name;
2843         int     rc;
2844
2845         if (session_key == 0) {
2846                 fprintf(stderr,
2847                         "Can't find env LST_SESSION or value is not valid\n");
2848                 return -1;
2849         }
2850
2851         if (argc != 2) {
2852                 lst_print_usage(argv[0]);
2853                 return -1;
2854         }
2855
2856         name = argv[1];
2857         if (strlen(name) >= LST_NAME_SIZE) {
2858                 fprintf(stderr, "Name length is limited to %d\n",
2859                         LST_NAME_SIZE - 1);
2860                 return -1;
2861         }
2862
2863         rc = lst_add_batch_ioctl(name);
2864         if (rc == 0)
2865                 return 0;
2866
2867         lst_print_error("batch", "Failed to create batch: %s\n",
2868                         strerror(errno));
2869
2870         return -1;
2871 }
2872
2873 static int
2874 lst_start_batch_ioctl(char *name, int timeout, struct list_head *resultp)
2875 {
2876         struct lstio_batch_run_args args = { 0 };
2877
2878         args.lstio_bat_key     = session_key;
2879         args.lstio_bat_timeout = timeout;
2880         args.lstio_bat_nmlen   = strlen(name);
2881         args.lstio_bat_namep   = name;
2882         args.lstio_bat_resultp = resultp;
2883
2884         return lst_ioctl(LSTIO_BATCH_START, &args, sizeof(args));
2885 }
2886
2887 static int
2888 jt_lst_start_batch(int argc, char **argv)
2889 {
2890         struct list_head  head;
2891         char             *batch;
2892         int               optidx = 0;
2893         int               timeout = 0;
2894         int               count = 0;
2895         int               rc;
2896         int               c;
2897
2898         static const struct option start_batch_opts[] = {
2899                 { .name = "timeout", .has_arg = required_argument, .val = 't' },
2900                 { .name = NULL } };
2901
2902         if (session_key == 0) {
2903                 fprintf(stderr,
2904                         "Can't find env LST_SESSION or value is not valid\n");
2905                 return -1;
2906         }
2907
2908         while (1) {
2909                 c = getopt_long(argc, argv, "t:",
2910                                 start_batch_opts, &optidx);
2911
2912                 /* Detect the end of the options. */
2913                 if (c == -1)
2914                         break;
2915
2916                 switch (c) {
2917                 case 't':
2918                         timeout = atoi(optarg);
2919                         break;
2920                 default:
2921                         lst_print_usage(argv[0]);
2922                         return -1;
2923                 }
2924         }
2925
2926         if (optind == argc) {
2927                 batch = LST_DEFAULT_BATCH;
2928
2929         } else if (optind == argc - 1) {
2930                 batch = argv[optind];
2931
2932         } else {
2933                 lst_print_usage(argv[0]);
2934                 return -1;
2935         }
2936
2937         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2938         if (rc != 0) {
2939                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2940                         batch, strerror(errno));
2941                 return -1;
2942         }
2943
2944         INIT_LIST_HEAD(&head);
2945
2946         rc = lst_alloc_rpcent(&head, count, 0);
2947         if (rc != 0) {
2948                 fprintf(stderr, "Out of memory\n");
2949                 return -1;
2950         }
2951
2952         rc = lst_start_batch_ioctl(batch, timeout, &head);
2953
2954         if (rc == 0) {
2955                 fprintf(stdout, "%s is running now\n", batch);
2956                 lst_free_rpcent(&head);
2957                 return 0;
2958         }
2959
2960         if (rc == -1) {
2961                 lst_print_error("batch", "Failed to start batch: %s\n",
2962                                 strerror(errno));
2963                 lst_free_rpcent(&head);
2964                 return rc;
2965         }
2966
2967         lst_print_transerr(&head, "Run batch");
2968
2969         lst_free_rpcent(&head);
2970
2971         return rc;
2972 }
2973
2974 static int
2975 lst_stop_batch_ioctl(char *name, int force, struct list_head *resultp)
2976 {
2977         struct lstio_batch_stop_args args = { 0 };
2978
2979         args.lstio_bat_key     = session_key;
2980         args.lstio_bat_force   = force;
2981         args.lstio_bat_nmlen   = strlen(name);
2982         args.lstio_bat_namep   = name;
2983         args.lstio_bat_resultp = resultp;
2984
2985         return lst_ioctl(LSTIO_BATCH_STOP, &args, sizeof(args));
2986 }
2987
2988 static int
2989 jt_lst_stop_batch(int argc, char **argv)
2990 {
2991         struct list_head  head;
2992         char             *batch;
2993         int               force = 0;
2994         int               optidx;
2995         int               count;
2996         int               rc;
2997         int               c;
2998
2999         static const struct option stop_batch_opts[] = {
3000                 { .name = "force", .has_arg = no_argument, .val = 'f' },
3001                 { .name = NULL } };
3002
3003         if (session_key == 0) {
3004                 fprintf(stderr,
3005                         "Can't find env LST_SESSION or value is not valid\n");
3006                 return -1;
3007         }
3008
3009         while (1) {
3010                 c = getopt_long(argc, argv, "f",
3011                                 stop_batch_opts, &optidx);
3012
3013                 /* Detect the end of the options. */
3014                 if (c == -1)
3015                         break;
3016
3017                 switch (c) {
3018                 case 'f':
3019                         force = 1;
3020                         break;
3021                 default:
3022                         lst_print_usage(argv[0]);
3023                         return -1;
3024                 }
3025         }
3026
3027         if (optind == argc) {
3028                 batch = LST_DEFAULT_BATCH;
3029
3030         } else if (optind == argc - 1) {
3031                 batch = argv[optind];
3032
3033         } else {
3034                 lst_print_usage(argv[0]);
3035                 return -1;
3036         }
3037
3038         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
3039         if (rc != 0) {
3040                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
3041                         batch, strerror(errno));
3042                 return -1;
3043         }
3044
3045         INIT_LIST_HEAD(&head);
3046
3047         rc = lst_alloc_rpcent(&head, count, 0);
3048         if (rc != 0) {
3049                 fprintf(stderr, "Out of memory\n");
3050                 return -1;
3051         }
3052
3053         rc = lst_stop_batch_ioctl(batch, force, &head);
3054         if (rc != 0)
3055                 goto out;
3056
3057         while (1) {
3058                 lst_reset_rpcent(&head);
3059
3060                 rc = lst_query_batch_ioctl(batch, 0, 0, 30, &head);
3061                 if (rc != 0)
3062                         goto out;
3063
3064                 if (lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0 &&
3065                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0)
3066                         break;
3067
3068                 fprintf(stdout, "%d batch in stopping\n",
3069                         lstcon_tsbqry_stat_run(&trans_stat, 0));
3070                 sleep(1);
3071         }
3072
3073         fprintf(stdout, "Batch is stopped\n");
3074         lst_free_rpcent(&head);
3075
3076         return 0;
3077 out:
3078         if (rc == -1) {
3079                 lst_print_error("batch", "Failed to stop batch: %s\n",
3080                                 strerror(errno));
3081                 lst_free_rpcent(&head);
3082                 return -1;
3083         }
3084
3085         lst_print_transerr(&head, "stop batch");
3086
3087         lst_free_rpcent(&head);
3088
3089         return rc;
3090 }
3091
3092 static int
3093 lst_list_batch_ioctl(int len, char *name, int index)
3094 {
3095         struct lstio_batch_list_args args = { 0 };
3096
3097         args.lstio_bat_key   = session_key;
3098         args.lstio_bat_idx   = index;
3099         args.lstio_bat_nmlen = len;
3100         args.lstio_bat_namep = name;
3101
3102         return lst_ioctl(LSTIO_BATCH_LIST, &args, sizeof(args));
3103 }
3104
3105 static int
3106 lst_info_batch_ioctl(char *batch, int test, int server,
3107                      struct lstcon_test_batch_ent *entp, int *idxp,
3108                      int *ndentp, struct lstcon_node_ent *dentsp)
3109 {
3110         struct lstio_batch_info_args args = { 0 };
3111
3112         args.lstio_bat_key     = session_key;
3113         args.lstio_bat_nmlen   = strlen(batch);
3114         args.lstio_bat_namep   = batch;
3115         args.lstio_bat_server  = server;
3116         args.lstio_bat_testidx = test;
3117         args.lstio_bat_entp    = entp;
3118         args.lstio_bat_idxp    = idxp;
3119         args.lstio_bat_ndentp  = ndentp;
3120         args.lstio_bat_dentsp  = dentsp;
3121
3122         return lst_ioctl(LSTIO_BATCH_INFO, &args, sizeof(args));
3123 }
3124
3125 static int
3126 lst_list_batch_all(void)
3127 {
3128         char name[LST_NAME_SIZE];
3129         int  rc;
3130         int  i;
3131
3132         for (i = 0; ; i++) {
3133                 rc = lst_list_batch_ioctl(LST_NAME_SIZE, name, i);
3134                 if (rc == 0) {
3135                         fprintf(stdout, "%d) %s\n", i + 1, name);
3136                         continue;
3137                 }
3138
3139                 if (errno == ENOENT)
3140                         break;
3141
3142                 lst_print_error("batch", "Failed to list batch: %s\n",
3143                                 strerror(errno));
3144                 return rc;
3145         }
3146
3147         fprintf(stdout, "Total %d batches\n", i);
3148
3149         return 0;
3150 }
3151
3152 static int
3153 lst_list_tsb_nodes(char *batch, int test, int server,
3154                    int count, int active, int invalid)
3155 {
3156         struct lstcon_node_ent *dents;
3157         int                index = 0;
3158         int                rc;
3159         int                c;
3160         int                i;
3161
3162         if (count == 0)
3163                 return 0;
3164
3165         /* verbose list, show nodes in batch or test */
3166         dents = malloc(count * sizeof(struct lstcon_node_ent));
3167         if (dents == NULL) {
3168                 fprintf(stdout, "Can't allocate memory\n");
3169                 return -1;
3170         }
3171
3172         rc = lst_info_batch_ioctl(batch, test, server,
3173                                   NULL, &index, &count, dents);
3174         if (rc != 0) {
3175                 free(dents);
3176                 lst_print_error((test > 0) ? "test" : "batch",
3177                                 (test > 0) ? "Failed to query test: %s\n" :
3178                                              "Failed to query batch: %s\n",
3179                                 strerror(errno));
3180                 return -1;
3181         }
3182
3183         for (i = 0, c = 0; i < count; i++) {
3184                 if ((!active  && dents[i].nde_state == LST_NODE_ACTIVE) ||
3185                     (!invalid && (dents[i].nde_state == LST_NODE_BUSY  ||
3186                                   dents[i].nde_state == LST_NODE_DOWN  ||
3187                                   dents[i].nde_state == LST_NODE_UNKNOWN)))
3188                         continue;
3189
3190                 fprintf(stdout, "\t%s: %s\n",
3191                         libcfs_id2str(dents[i].nde_id),
3192                         lst_node_state2str(dents[i].nde_state));
3193                 c++;
3194         }
3195
3196         fprintf(stdout, "Total %d nodes\n", c);
3197         free(dents);
3198
3199         return 0;
3200 }
3201
3202 static int
3203 jt_lst_list_batch(int argc, char **argv)
3204 {
3205         struct lstcon_test_batch_ent ent;
3206         char *batch   = NULL;
3207         int   optidx  = 0;
3208         int   verbose = 0; /* list nodes in batch or test */
3209         int   invalid = 0;
3210         int   active  = 0;
3211         int   server  = 0;
3212         int   ntest   = 0;
3213         int   test    = 0;
3214         int   c       = 0;
3215         int   rc;
3216
3217         static const struct option list_batch_opts[] = {
3218                 { .name = "test",    .has_arg = required_argument, .val = 't' },
3219                 { .name = "invalid", .has_arg = no_argument,       .val = 'i' },
3220                 { .name = "active",  .has_arg = no_argument,       .val = 'a' },
3221                 { .name = "all",     .has_arg = no_argument,       .val = 'l' },
3222                 { .name = "server",  .has_arg = no_argument,       .val = 's' },
3223                 { .name = NULL, } };
3224
3225         if (session_key == 0) {
3226                 fprintf(stderr,
3227                         "Can't find env LST_SESSION or value is not valid\n");
3228                 return -1;
3229         }
3230
3231         while (1) {
3232                 c = getopt_long(argc, argv, "ailst:",
3233                                 list_batch_opts, &optidx);
3234
3235                 if (c == -1)
3236                         break;
3237
3238                 switch (c) {
3239                 case 'a':
3240                         verbose = active = 1;
3241                         break;
3242                 case 'i':
3243                         verbose = invalid = 1;
3244                         break;
3245                 case 'l':
3246                         verbose = active = invalid = 1;
3247                         break;
3248                 case 's':
3249                         server = 1;
3250                         break;
3251                 case 't':
3252                         test = atoi(optarg);
3253                         ntest = 1;
3254                         break;
3255                 default:
3256                         lst_print_usage(argv[0]);
3257                         return -1;
3258                 }
3259         }
3260
3261         if (optind == argc) {
3262                 /* list all batches */
3263                 rc = lst_list_batch_all();
3264                 return rc;
3265         }
3266
3267         if (ntest == 1 && test <= 0) {
3268                 fprintf(stderr, "Invalid test id, test id starts from 1\n");
3269                 return -1;
3270         }
3271
3272         if (optind != argc - 1) {
3273                 lst_print_usage(argv[0]);
3274                 return -1;
3275         }
3276
3277         batch = argv[optind];
3278
3279 loop:
3280         /* show detail of specified batch or test */
3281         rc = lst_info_batch_ioctl(batch, test, server,
3282                                   &ent, NULL, NULL, NULL);
3283         if (rc != 0) {
3284                 lst_print_error((test > 0) ? "test" : "batch",
3285                                 (test > 0) ? "Failed to query test: %s\n" :
3286                                              "Failed to query batch: %s\n",
3287                                 strerror(errno));
3288                 return -1;
3289         }
3290
3291         if (verbose) {
3292                 /* list nodes in test or batch */
3293                 rc = lst_list_tsb_nodes(batch, test, server,
3294                                         server ? ent.tbe_srv_nle.nle_nnode :
3295                                                  ent.tbe_cli_nle.nle_nnode,
3296                                         active, invalid);
3297                 return rc;
3298         }
3299
3300         /* only show number of hosts in batch or test */
3301         if (test == 0) {
3302                 fprintf(stdout, "Batch: %s Tests: %d State: %d\n",
3303                         batch, ent.u.tbe_batch.bae_ntest,
3304                         ent.u.tbe_batch.bae_state);
3305                 ntest = ent.u.tbe_batch.bae_ntest;
3306                 test = 1; /* starting from test 1 */
3307
3308         } else {
3309                 fprintf(stdout,
3310                         "\tTest %d(%s) (loop: %d, concurrency: %d)\n",
3311                         test, lst_test_type2name(ent.u.tbe_test.tse_type),
3312                         ent.u.tbe_test.tse_loop,
3313                         ent.u.tbe_test.tse_concur);
3314                 ntest --;
3315                 test ++;
3316         }
3317
3318         fprintf(stdout, LST_NODES_TITLE);
3319         fprintf(stdout, "client\t%d\t%d\t%d\t%d\t%d\n"
3320                         "server\t%d\t%d\t%d\t%d\t%d\n",
3321                 ent.tbe_cli_nle.nle_nactive,
3322                 ent.tbe_cli_nle.nle_nbusy,
3323                 ent.tbe_cli_nle.nle_ndown,
3324                 ent.tbe_cli_nle.nle_nunknown,
3325                 ent.tbe_cli_nle.nle_nnode,
3326                 ent.tbe_srv_nle.nle_nactive,
3327                 ent.tbe_srv_nle.nle_nbusy,
3328                 ent.tbe_srv_nle.nle_ndown,
3329                 ent.tbe_srv_nle.nle_nunknown,
3330                 ent.tbe_srv_nle.nle_nnode);
3331
3332         if (ntest != 0)
3333                 goto loop;
3334
3335         return 0;
3336 }
3337
3338 static int
3339 lst_query_batch_ioctl(char *batch, int test, int server,
3340                       int timeout, struct list_head *head)
3341 {
3342         struct lstio_batch_query_args args = { 0 };
3343
3344         args.lstio_bat_key     = session_key;
3345         args.lstio_bat_testidx = test;
3346         args.lstio_bat_client  = !(server);
3347         args.lstio_bat_timeout = timeout;
3348         args.lstio_bat_nmlen   = strlen(batch);
3349         args.lstio_bat_namep   = batch;
3350         args.lstio_bat_resultp = head;
3351
3352         return lst_ioctl(LSTIO_BATCH_QUERY, &args, sizeof(args));
3353 }
3354
3355 static void
3356 lst_print_tsb_verbose(struct list_head *head,
3357                       int active, int idle, int error)
3358 {
3359         struct lstcon_rpc_ent *ent;
3360
3361         list_for_each_entry(ent, head, rpe_link) {
3362                 if (ent->rpe_priv[0] == 0 && active)
3363                         continue;
3364
3365                 if (ent->rpe_priv[0] != 0 && idle)
3366                         continue;
3367
3368                 if (ent->rpe_fwk_errno == 0 && error)
3369                         continue;
3370
3371                 fprintf(stdout, "%s [%s]: %s\n",
3372                         libcfs_id2str(ent->rpe_peer),
3373                         lst_node_state2str(ent->rpe_state),
3374                         ent->rpe_rpc_errno != 0 ?
3375                                 strerror(ent->rpe_rpc_errno) :
3376                                 (ent->rpe_priv[0] > 0 ? "Running" : "Idle"));
3377         }
3378 }
3379
3380 static int
3381 jt_lst_query_batch(int argc, char **argv)
3382 {
3383         struct lstcon_test_batch_ent ent;
3384         struct list_head head;
3385         char   *batch   = NULL;
3386         time_t  last    = 0;
3387         int     optidx  = 0;
3388         int     verbose = 0;
3389         int     server  = 0;
3390         int     timeout = 5; /* default 5 seconds */
3391         int     delay   = 5; /* default 5 seconds */
3392         int     loop    = 1; /* default 1 loop */
3393         int     active  = 0;
3394         int     error   = 0;
3395         int     idle    = 0;
3396         int     count   = 0;
3397         int     test    = 0;
3398         int     rc      = 0;
3399         int     c       = 0;
3400         int     i;
3401
3402         static const struct option query_batch_opts[] = {
3403                 { .name = "timeout", .has_arg = required_argument, .val = 'o' },
3404                 { .name = "delay",   .has_arg = required_argument, .val = 'd' },
3405                 { .name = "loop",    .has_arg = required_argument, .val = 'c' },
3406                 { .name = "test",    .has_arg = required_argument, .val = 't' },
3407                 { .name = "server",  .has_arg = no_argument,       .val = 's' },
3408                 { .name = "active",  .has_arg = no_argument,       .val = 'a' },
3409                 { .name = "idle",    .has_arg = no_argument,       .val = 'i' },
3410                 { .name = "error",   .has_arg = no_argument,       .val = 'e' },
3411                 { .name = "all",     .has_arg = no_argument,       .val = 'l' },
3412                 { .name = NULL, } };
3413
3414         if (session_key == 0) {
3415                 fprintf(stderr,
3416                         "Can't find env LST_SESSION or value is not valid\n");
3417                 return -1;
3418         }
3419
3420         while (1) {
3421                 c = getopt_long(argc, argv, "o:d:c:t:saiel",
3422                                 query_batch_opts, &optidx);
3423
3424                 /* Detect the end of the options. */
3425                 if (c == -1)
3426                         break;
3427
3428                 switch (c) {
3429                 case 'o':
3430                         timeout = atoi(optarg);
3431                         break;
3432                 case 'd':
3433                         delay = atoi(optarg);
3434                         break;
3435                 case 'c':
3436                         loop = atoi(optarg);
3437                         break;
3438                 case 't':
3439                         test = atoi(optarg);
3440                         break;
3441                 case 's':
3442                         server = 1;
3443                         break;
3444                 case 'a':
3445                         active = verbose = 1;
3446                         break;
3447                 case 'i':
3448                         idle = verbose = 1;
3449                         break;
3450                 case 'e':
3451                         error = verbose = 1;
3452                         break;
3453                 case 'l':
3454                         verbose = 1;
3455                         break;
3456                 default:
3457                         lst_print_usage(argv[0]);
3458                         return -1;
3459                 }
3460         }
3461
3462         if (test < 0 || timeout <= 0 || delay <= 0 || loop <= 0) {
3463                 lst_print_usage(argv[0]);
3464                 return -1;
3465         }
3466
3467         if (optind == argc) {
3468                 batch = LST_DEFAULT_BATCH;
3469
3470         } else if (optind == argc - 1) {
3471                 batch = argv[optind];
3472
3473         } else {
3474                 lst_print_usage(argv[0]);
3475                 return -1;
3476         }
3477
3478
3479         INIT_LIST_HEAD(&head);
3480
3481         if (verbose) {
3482                 rc = lst_info_batch_ioctl(batch, test, server,
3483                                           &ent, NULL, NULL, NULL);
3484                 if (rc != 0) {
3485                         fprintf(stderr, "Failed to query %s [%d]: %s\n",
3486                                 batch, test, strerror(errno));
3487                         return -1;
3488                 }
3489
3490                 count = server ? ent.tbe_srv_nle.nle_nnode :
3491                                  ent.tbe_cli_nle.nle_nnode;
3492                 if (count == 0) {
3493                         fprintf(stdout, "Batch or test is empty\n");
3494                         return 0;
3495                 }
3496         }
3497
3498         rc = lst_alloc_rpcent(&head, count, 0);
3499         if (rc != 0) {
3500                 fprintf(stderr, "Out of memory\n");
3501                 return rc;
3502         }
3503
3504         for (i = 0; i < loop; i++) {
3505                 time_t  now = time(NULL);
3506
3507                 if (now - last < delay) {
3508                         sleep(delay - now + last);
3509                         time(&now);
3510                 }
3511
3512                 last = now;
3513
3514                 rc = lst_query_batch_ioctl(batch, test,
3515                                            server, timeout, &head);
3516                 if (rc == -1) {
3517                         fprintf(stderr, "Failed to query batch: %s\n",
3518                                 strerror(errno));
3519                         break;
3520                 }
3521
3522                 if (verbose) {
3523                         /* Verbose mode */
3524                         lst_print_tsb_verbose(&head, active, idle, error);
3525                         continue;
3526                 }
3527
3528                 fprintf(stdout, "%s [%d] ", batch, test);
3529
3530                 if (lstcon_rpc_stat_failure(&trans_stat, 0) != 0) {
3531                         fprintf(stdout, "%d of %d nodes are unknown, ",
3532                                 lstcon_rpc_stat_failure(&trans_stat, 0),
3533                                 lstcon_rpc_stat_total(&trans_stat, 0));
3534                 }
3535
3536                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
3537                     lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0  &&
3538                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
3539                         fprintf(stdout, "is stopped\n");
3540                         continue;
3541                 }
3542
3543                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
3544                     lstcon_tsbqry_stat_idle(&trans_stat, 0) == 0 &&
3545                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
3546                         fprintf(stdout, "is running\n");
3547                         continue;
3548                 }
3549
3550                 fprintf(stdout, "stopped: %d , running: %d, failed: %d\n",
3551                                 lstcon_tsbqry_stat_idle(&trans_stat, 0),
3552                                 lstcon_tsbqry_stat_run(&trans_stat, 0),
3553                                 lstcon_tsbqry_stat_failure(&trans_stat, 0));
3554         }
3555
3556         lst_free_rpcent(&head);
3557
3558         return rc;
3559 }
3560
3561 static int
3562 lst_parse_distribute(char *dstr, int *dist, int *span)
3563 {
3564         *dist = atoi(dstr);
3565         if (*dist <= 0)
3566                 return -1;
3567
3568         dstr = strchr(dstr, ':');
3569         if (dstr == NULL)
3570                 return -1;
3571
3572         *span = atoi(dstr + 1);
3573         if (*span <= 0)
3574                 return -1;
3575
3576         return 0;
3577 }
3578
3579 static int
3580 lst_get_bulk_param(int argc, char **argv, struct lst_test_bulk_param *bulk)
3581 {
3582         char   *tok = NULL;
3583         char   *end = NULL;
3584         int     rc  = 0;
3585         int     i   = 0;
3586
3587         bulk->blk_size  = 4096;
3588         bulk->blk_opc   = LST_BRW_READ;
3589         bulk->blk_flags = LST_BRW_CHECK_NONE;
3590         bulk->blk_srv_off = bulk->blk_cli_off = 0;
3591
3592         while (i < argc) {
3593                 if (strcasestr(argv[i], "check=") == argv[i] ||
3594                     strcasestr(argv[i], "c=") == argv[i]) {
3595                         tok = strchr(argv[i], '=') + 1;
3596
3597                         if (strcasecmp(tok, "full") == 0) {
3598                                 bulk->blk_flags = LST_BRW_CHECK_FULL;
3599                         } else if (strcasecmp(tok, "simple") == 0) {
3600                                 bulk->blk_flags = LST_BRW_CHECK_SIMPLE;
3601                         } else {
3602                                 fprintf(stderr, "Unknow flag %s\n", tok);
3603                                 return -1;
3604                         }
3605
3606                 } else if (strcasestr(argv[i], "size=") == argv[i] ||
3607                            strcasestr(argv[i], "s=") == argv[i]) {
3608                         tok = strchr(argv[i], '=') + 1;
3609
3610                         bulk->blk_size = strtol(tok, &end, 0);
3611                         if (bulk->blk_size <= 0) {
3612                                 fprintf(stderr, "Invalid size %s\n", tok);
3613                                 return -1;
3614                         }
3615
3616                         if (end == NULL)
3617                                 return 0;
3618
3619                         if (*end == 'k' || *end == 'K')
3620                                 bulk->blk_size *= 1024;
3621                         else if (*end == 'm' || *end == 'M')
3622                                 bulk->blk_size *= 1024 * 1024;
3623
3624                         if (bulk->blk_size > LNET_MTU) {
3625                                 fprintf(stderr, "Size exceed limitation: %d bytes\n",
3626                                         bulk->blk_size);
3627                                 return -1;
3628                         }
3629
3630                 } else if (strcasestr(argv[i], "off=") == argv[i]) {
3631                         int     off;
3632
3633                         tok = strchr(argv[i], '=') + 1;
3634
3635                         off = strtol(tok, &end, 0);
3636                         /* NB: align with sizeof(__u64) to simplify page
3637                          * checking implementation */
3638                         if (off < 0 || off % sizeof(__u64) != 0) {
3639                                 fprintf(stderr,
3640                                         "Invalid offset %s/%d, it should be "
3641                                         "postive value and multiple of %d\n",
3642                                         tok, off, (int)sizeof(__u64));
3643                                 return -1;
3644                         }
3645
3646                         /* NB: blk_srv_off is reserved so far */
3647                         bulk->blk_cli_off = bulk->blk_srv_off = off;
3648                         if (end == NULL)
3649                                 return 0;
3650
3651                 } else if (strcasecmp(argv[i], "read") == 0 ||
3652                            strcasecmp(argv[i], "r") == 0) {
3653                         bulk->blk_opc = LST_BRW_READ;
3654
3655                 } else if (strcasecmp(argv[i], "write") == 0 ||
3656                            strcasecmp(argv[i], "w") == 0) {
3657                         bulk->blk_opc = LST_BRW_WRITE;
3658
3659                 } else {
3660                         fprintf(stderr, "Unknow parameter: %s\n", argv[i]);
3661                         return -1;
3662                 }
3663
3664                 i++;
3665         }
3666
3667         return rc;
3668 }
3669
3670 static int
3671 lst_get_test_param(char *test, int argc, char **argv, void **param, int *plen)
3672 {
3673         struct lst_test_bulk_param *bulk = NULL;
3674         struct lst_test_ping_param *ping = NULL;
3675         int                    type;
3676
3677         type = lst_test_name2type(test);
3678         if (type < 0) {
3679                 fprintf(stderr, "Unknow test name %s\n", test);
3680                 return -1;
3681         }
3682
3683         switch (type) {
3684         case LST_TEST_PING:
3685                 /* unused but needs for kernel part */
3686                 ping = malloc(sizeof(*ping));
3687                 if (ping == NULL) {
3688                         fprintf(stderr, "Out of memory\n");
3689                         return -1;
3690                 }
3691                 memset(ping, 0, sizeof(*ping));
3692
3693                 *param = ping;
3694                 *plen  = sizeof(*ping);
3695
3696                 break;
3697
3698         case LST_TEST_BULK:
3699                 bulk = malloc(sizeof(*bulk));
3700                 if (bulk == NULL) {
3701                         fprintf(stderr, "Out of memory\n");
3702                         return -1;
3703                 }
3704
3705                 memset(bulk, 0, sizeof(*bulk));
3706
3707                 if (lst_get_bulk_param(argc, argv, bulk) != 0) {
3708                         free(bulk);
3709                         return -1;
3710                 }
3711
3712                 *param = bulk;
3713                 *plen  = sizeof(*bulk);
3714
3715                 break;
3716
3717         default:
3718                 break;
3719         }
3720
3721         /* TODO: parse more parameter */
3722         return type;
3723 }
3724
3725 static int
3726 lst_add_test_ioctl(char *batch, int type, int loop, int concur,
3727                    int dist, int span, char *sgrp, char *dgrp,
3728                    void *param, int plen, int *retp, struct list_head *resultp)
3729 {
3730         struct lstio_test_args args = { 0 };
3731
3732         args.lstio_tes_key        = session_key;
3733         args.lstio_tes_bat_nmlen  = strlen(batch);
3734         args.lstio_tes_bat_name   = batch;
3735         args.lstio_tes_type       = type;
3736         args.lstio_tes_oneside    = 0;
3737         args.lstio_tes_loop       = loop;
3738         args.lstio_tes_concur     = concur;
3739         args.lstio_tes_dist       = dist;
3740         args.lstio_tes_span       = span;
3741         args.lstio_tes_sgrp_nmlen = strlen(sgrp);
3742         args.lstio_tes_sgrp_name  = sgrp;
3743         args.lstio_tes_dgrp_nmlen = strlen(dgrp);
3744         args.lstio_tes_dgrp_name  = dgrp;
3745         args.lstio_tes_param_len  = plen;
3746         args.lstio_tes_param      = param;
3747         args.lstio_tes_retp       = retp;
3748         args.lstio_tes_resultp    = resultp;
3749
3750         return lst_ioctl(LSTIO_TEST_ADD, &args, sizeof(args));
3751 }
3752
3753 static int
3754 jt_lst_add_test(int argc, char **argv)
3755 {
3756         struct list_head head;
3757         char *batch  = NULL;
3758         char *test   = NULL;
3759         char *dstr   = NULL;
3760         char *from   = NULL;
3761         char *to     = NULL;
3762         void *param  = NULL;
3763         int   optidx = 0;
3764         int   concur = 1;
3765         int   loop   = -1;
3766         int   dist   = 1;
3767         int   span   = 1;
3768         int   plen   = 0;
3769         int   fcount = 0;
3770         int   tcount = 0;
3771         int   ret    = 0;
3772         int   type;
3773         int   rc;
3774         int   c;
3775
3776         static const struct option add_test_opts[] = {
3777         { .name = "batch",       .has_arg = required_argument, .val = 'b' },
3778         { .name = "concurrency", .has_arg = required_argument, .val = 'c' },
3779         { .name = "distribute",  .has_arg = required_argument, .val = 'd' },
3780         { .name = "from",        .has_arg = required_argument, .val = 'f' },
3781         { .name = "to",          .has_arg = required_argument, .val = 't' },
3782         { .name = "loop",        .has_arg = required_argument, .val = 'l' },
3783         { .name = NULL } };
3784
3785         if (session_key == 0) {
3786                 fprintf(stderr,
3787                         "Can't find env LST_SESSION or value is not valid\n");
3788                 return -1;
3789         }
3790
3791         while (1) {
3792                 c = getopt_long(argc, argv, "b:c:d:f:l:t:",
3793                                 add_test_opts, &optidx);
3794
3795                 /* Detect the end of the options. */
3796                 if (c == -1)
3797                         break;
3798
3799                 switch (c) {
3800                 case 'b':
3801                         batch = optarg;
3802                         break;
3803                 case 'c':
3804                         concur = atoi(optarg);
3805                         break;
3806                 case 'd':
3807                         dstr = optarg;
3808                         break;
3809                 case 'f':
3810                         from = optarg;
3811                         break;
3812                 case 'l':
3813                         loop = atoi(optarg);
3814                         break;
3815                 case 't':
3816                         to = optarg;
3817                         break;
3818                 default:
3819                         lst_print_usage(argv[0]);
3820                         return -1;
3821                 }
3822         }
3823
3824         if (optind == argc || from == NULL || to == NULL) {
3825                 lst_print_usage(argv[0]);
3826                 return -1;
3827         }
3828
3829         if (concur <= 0 || concur > LST_MAX_CONCUR) {
3830                 fprintf(stderr, "Invalid concurrency of test: %d\n", concur);
3831                 return -1;
3832         }
3833
3834         if (batch == NULL)
3835                 batch = LST_DEFAULT_BATCH;
3836
3837         if (dstr != NULL) {
3838                 rc = lst_parse_distribute(dstr, &dist, &span);
3839                 if (rc != 0) {
3840                         fprintf(stderr, "Invalid distribution: %s\n", dstr);
3841                         return -1;
3842                 }
3843         }
3844
3845         test = argv[optind++];
3846
3847         argc -= optind;
3848         argv += optind;
3849
3850         type = lst_get_test_param(test, argc, argv, &param, &plen);
3851         if (type < 0) {
3852                 fprintf(stderr, "Failed to add test (%s)\n", test);
3853                 return -1;
3854         }
3855
3856         INIT_LIST_HEAD(&head);
3857
3858         rc = lst_get_node_count(LST_OPC_GROUP, from, &fcount, NULL);
3859         if (rc != 0) {
3860                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3861                         from, strerror(errno));
3862                 goto out;
3863         }
3864
3865         rc = lst_get_node_count(LST_OPC_GROUP, to, &tcount, NULL);
3866         if (rc != 0) {
3867                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3868                         to, strerror(errno));
3869                 goto out;
3870         }
3871
3872         rc = lst_alloc_rpcent(&head, fcount > tcount ? fcount : tcount, 0);
3873         if (rc != 0) {
3874                 fprintf(stderr, "Out of memory\n");
3875                 goto out;
3876         }
3877
3878         rc = lst_add_test_ioctl(batch, type, loop, concur,
3879                                 dist, span, from, to, param, plen, &ret, &head);
3880
3881         if (rc == 0) {
3882                 fprintf(stdout, "Test was added successfully\n");
3883                 if (ret != 0) {
3884                         fprintf(stdout, "Server group contains userland test "
3885                                 "nodes, old version of tcplnd can't accept "
3886                                 "connection request\n");
3887                 }
3888
3889                 goto out;
3890         }
3891
3892         if (rc == -1) {
3893                 lst_print_error("test", "Failed to add test: %s\n",
3894                                 strerror(errno));
3895                 goto out;
3896         }
3897
3898         lst_print_transerr(&head, "add test");
3899 out:
3900         lst_free_rpcent(&head);
3901
3902         if (param != NULL)
3903                 free(param);
3904
3905         return rc;
3906 }
3907
3908 static command_t lst_cmdlist[] = {
3909         {"new_session",         jt_lst_new_session,     NULL,
3910          "Usage: lst new_session [--timeout TIME] [--force] [NAME]"                     },
3911         {"end_session",         jt_lst_end_session,     NULL,
3912          "Usage: lst end_session"                                                       },
3913         {"show_session",        jt_lst_show_session,    NULL,
3914          "Usage: lst show_session"                                                      },
3915         {"ping",                jt_lst_ping ,           NULL,
3916          "Usage: lst ping  [--group NAME] [--batch NAME] [--session] [--nodes IDS]"     },
3917         {"add_group",           jt_lst_add_group,       NULL,
3918          "Usage: lst group NAME IDs [IDs]..."                                           },
3919         {"del_group",           jt_lst_del_group,       NULL,
3920          "Usage: lst del_group NAME"                                                    },
3921         {"update_group",        jt_lst_update_group,    NULL,
3922          "Usage: lst update_group NAME [--clean] [--refresh] [--remove IDs]"            },
3923         {"list_group",          jt_lst_list_group,      NULL,
3924           "Usage: lst list_group [--active] [--busy] [--down] [--unknown] GROUP ..."    },
3925         {"stat",                jt_lst_stat,            NULL,
3926          "Usage: lst stat [--bw] [--rate] [--read] [--write] [--max] [--min] [--avg] "
3927          " [--mbs] [--timeout #] [--delay #] [--count #] GROUP [GROUP]"                 },
3928         {"show_error",          jt_lst_show_error,      NULL,
3929          "Usage: lst show_error NAME | IDS ..."                                         },
3930         {"add_batch",           jt_lst_add_batch,       NULL,
3931          "Usage: lst add_batch NAME"                                                    },
3932         {"run",                 jt_lst_start_batch,     NULL,
3933          "Usage: lst run [--timeout TIME] [NAME]"                                       },
3934         {"stop",                jt_lst_stop_batch,      NULL,
3935          "Usage: lst stop [--force] BATCH_NAME"                                         },
3936         {"list_batch",          jt_lst_list_batch,      NULL,
3937          "Usage: lst list_batch NAME [--test ID] [--server]"                            },
3938         {"query",               jt_lst_query_batch,     NULL,
3939          "Usage: lst query [--test ID] [--server] [--timeout TIME] NAME"                },
3940         {"add_test",            jt_lst_add_test,        NULL,
3941          "Usage: lst add_test [--batch BATCH] [--loop #] [--concurrency #] "
3942          " [--distribute #:#] [--from GROUP] [--to GROUP] TEST..."                      },
3943         {0,                     0,                      0,      NULL                    }
3944 };
3945
3946 static int
3947 lst_initialize(void)
3948 {
3949         char   *key;
3950         char   *feats;
3951
3952         feats = getenv("LST_FEATURES");
3953         if (feats != NULL)
3954                 session_features = strtol(feats, NULL, 16);
3955
3956         if ((session_features & ~LST_FEATS_MASK) != 0) {
3957                 fprintf(stderr,
3958                         "Unsupported session features %x, "
3959                         "only support these features so far: %x\n",
3960                         (session_features & ~LST_FEATS_MASK), LST_FEATS_MASK);
3961                 return -1;
3962         }
3963
3964         key = getenv("LST_SESSION");
3965
3966         if (key == NULL) {
3967                 session_key = 0;
3968                 return 0;
3969         }
3970
3971         session_key = atoi(key);
3972
3973         return 0;
3974 }
3975
3976 int main(int argc, char **argv)
3977 {
3978         int rc = 0;
3979
3980         setlinebuf(stdout);
3981
3982         rc = lst_initialize();
3983         if (rc < 0)
3984                 goto errorout;
3985
3986         rc = lustre_lnet_config_lib_init();
3987         if (rc < 0)
3988                 goto errorout;
3989
3990         rc = cfs_parser(argc, argv, lst_cmdlist);
3991
3992 errorout:
3993         return rc;
3994 }