lnet/utils/lst.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2012, 2017, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  *
  31  * lnet/selftest/conctl.c
  32  *
  33  * Author: Liang Zhen <liangzhen@clusterfs.com>
  34  */
  35 #include <errno.h>
  36 #include <getopt.h>
  37 #include <inttypes.h>
  38 #include <pwd.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <stdio.h>
  42 #include <stdlib.h>
  43 #include <string.h>
  44 #include <sys/ioctl.h>
  45 #include <time.h>
  46 #include <linux/types.h>
  47
  48 #include <libcfs/util/list.h>
  49 #include <libcfs/util/ioctl.h>
  50 #include <libcfs/util/parser.h>
  51 #include <linux/lnet/lnetctl.h>
  52 #include <linux/lnet/lnetst.h>
  53 #include <linux/lnet/nidstr.h>
  54 #include "lnetconfig/liblnetconfig.h"
  55
  56 struct lst_sid LST_INVALID_SID = { .ses_nid = LNET_NID_ANY, .ses_stamp = -1 };
  57 static unsigned int session_key;
  58 static int lst_list_commands(int argc, char **argv);
  59
  60 /* All nodes running 2.6.50 or later understand feature LST_FEAT_BULK_LEN */
  61 static unsigned int session_features = LST_FEATS_MASK;
  62 static struct lstcon_trans_stat trans_stat;
  63
  64 typedef struct list_string {
  65         struct list_string *lstr_next;
  66         int                 lstr_sz;
  67         char                lstr_str[0];
  68 } lstr_t;
  69
  70 #ifndef offsetof
  71 # define offsetof(typ,memb)     ((unsigned long)((char *)&(((typ *)0)->memb)))
  72 #endif
  73
  74 static int alloc_count = 0;
  75 static int alloc_nob   = 0;
  76
  77 lstr_t *
  78 alloc_lstr(int sz)
  79 {
  80         lstr_t  *lstr = malloc(offsetof(lstr_t, lstr_str[sz]));
  81
  82         if (lstr == NULL) {
  83                 fprintf(stderr, "Can't allocate lstr\n");
  84                 abort();
  85         }
  86
  87         alloc_nob += sz;
  88         alloc_count++;
  89
  90         lstr->lstr_str[0] = 0;
  91         lstr->lstr_sz = sz;
  92         return lstr;
  93 }
  94
  95 void
  96 free_lstr(lstr_t *lstr)
  97 {
  98         alloc_count--;
  99         alloc_nob -= lstr->lstr_sz;
 100         free(lstr);
 101 }
 102
 103 void
 104 free_lstrs(lstr_t **list)
 105 {
 106         lstr_t   *lstr;
 107
 108         while ((lstr = *list) != NULL) {
 109                 *list = lstr->lstr_next;
 110                 free_lstr(lstr);
 111         }
 112 }
 113
 114 void
 115 new_lstrs(lstr_t **list, char *prefix, char *postfix,
 116           int lo, int hi, int stride)
 117 {
 118         int    n1 = strlen(prefix);
 119         int    n2 = strlen(postfix);
 120         int    sz = n1 + 20 + n2 + 1;
 121
 122         do {
 123                 lstr_t *n = alloc_lstr(sz);
 124
 125                 snprintf(n->lstr_str, sz - 1, "%s%u%s",
 126                          prefix, lo, postfix);
 127
 128                 n->lstr_next = *list;
 129                 *list = n;
 130
 131                 lo += stride;
 132         } while (lo <= hi);
 133 }
 134
 135 int
 136 expand_lstr(lstr_t **list, lstr_t *l)
 137 {
 138         int          nob = strlen(l->lstr_str);
 139         char        *b1;
 140         char        *b2;
 141         char        *expr;
 142         char        *sep;
 143         int          x;
 144         int          y;
 145         int          z;
 146         int          n;
 147
 148         b1 = strchr(l->lstr_str, '[');
 149         if (b1 == NULL) {
 150                 l->lstr_next = *list;
 151                 *list = l;
 152                 return 0;
 153         }
 154
 155         b2 = strchr(b1, ']');
 156         if (b2 == NULL || b2 == b1 + 1)
 157                 return -1;
 158
 159         *b1++ = 0;
 160         *b2++ = 0;
 161         expr = b1;
 162         do {
 163
 164                 sep = strchr(expr, ',');
 165                 if (sep != NULL)
 166                         *sep++ = 0;
 167
 168                 nob = strlen(expr);
 169                 n = nob;
 170                 if (sscanf(expr, "%u%n", &x, &n) >= 1 && n == nob) {
 171                         /* simple number */
 172                         new_lstrs(list, l->lstr_str, b2, x, x, 1);
 173                         continue;
 174                 }
 175
 176                 n = nob;
 177                 if (sscanf(expr, "%u-%u%n", &x, &y, &n) >= 2 && n == nob &&
 178                     x < y) {
 179                         /* simple range */
 180                         new_lstrs(list, l->lstr_str, b2, x, y, 1);
 181                         continue;
 182                 }
 183
 184                 n = nob;
 185                 if (sscanf(expr, "%u-%u/%u%n", &x, &y, &z, &n) >= 3 && n == nob &&
 186                     x < y) {
 187                         /* strided range */
 188                         new_lstrs(list, l->lstr_str, b2, x, y, z);
 189                         continue;
 190                 }
 191
 192                 /* syntax error */
 193                 return -1;
 194         } while ((expr = sep) != NULL);
 195
 196         free_lstr(l);
 197
 198         return 1;
 199 }
 200
 201 int
 202 expand_strs(char *str, lstr_t **head)
 203 {
 204         lstr_t  *list = NULL;
 205         lstr_t  *nlist;
 206         lstr_t  *l;
 207         int      rc = 0;
 208         int      expanded;
 209
 210         l = alloc_lstr(strlen(str) + 1);
 211         memcpy(l->lstr_str, str, strlen(str) + 1);
 212         l->lstr_next = NULL;
 213         list = l;
 214
 215         do {
 216                 expanded = 0;
 217                 nlist = NULL;
 218
 219                 while ((l = list) != NULL) {
 220                         list = l->lstr_next;
 221
 222                         rc = expand_lstr(&nlist, l);
 223                         if (rc < 0) {
 224                                 fprintf(stderr, "Syntax error in \"%s\"\n", str);
 225                                 free_lstr(l);
 226                                 break;
 227                         }
 228
 229                         expanded |= rc > 0;
 230                 }
 231
 232                 /* re-order onto 'list' */
 233                 while ((l = nlist) != NULL) {
 234                         nlist = l->lstr_next;
 235                         l->lstr_next = list;
 236                         list = l;
 237                 }
 238
 239         } while (expanded && rc > 0);
 240
 241         if (rc >= 0) {
 242                 *head = list;
 243                 return 0;
 244         }
 245
 246         while ((l = list) != NULL) {
 247                 list = l->lstr_next;
 248
 249                 free_lstr(l);
 250         }
 251         return rc;
 252 }
 253
 254 int
 255 lst_parse_nids(char *str, int *countp, struct lnet_process_id **idspp)
 256 {
 257         lstr_t  *head = NULL;
 258         lstr_t  *l;
 259         int      c = 0;
 260         int      i;
 261         int      rc;
 262
 263         rc = expand_strs(str, &head);
 264         if (rc != 0)
 265                 goto out;
 266
 267         l = head;
 268         while (l != NULL) {
 269                 l = l->lstr_next;
 270                 c++;
 271         }
 272
 273         *idspp = malloc(c * sizeof(struct lnet_process_id));
 274         if (*idspp == NULL) {
 275                 fprintf(stderr, "Out of memory\n");
 276                 rc = -1;
 277         }
 278
 279         *countp = c;
 280 out:
 281         i = 0;
 282         while ((l = head) != NULL) {
 283                 head = l->lstr_next;
 284
 285                 if (rc == 0) {
 286                         (*idspp)[i].nid = libcfs_str2nid(l->lstr_str);
 287                         if ((*idspp)[i].nid == LNET_NID_ANY) {
 288                                 fprintf(stderr, "Invalid nid: %s\n",
 289                                         l->lstr_str);
 290                                 rc = -1;
 291                         }
 292
 293                         (*idspp)[i].pid = LNET_PID_LUSTRE;
 294                         i++;
 295                 }
 296
 297                 free_lstr(l);
 298         }
 299
 300         if (rc == 0)
 301                 return 0;
 302
 303         free(*idspp);
 304         *idspp = NULL;
 305
 306         return rc;
 307 }
 308
 309 char *
 310 lst_node_state2str(int state)
 311 {
 312         if (state == LST_NODE_ACTIVE)
 313                 return "Active";
 314         if (state == LST_NODE_BUSY)
 315                 return "Busy";
 316         if (state == LST_NODE_DOWN)
 317                 return "Down";
 318
 319         return "Unknown";
 320 }
 321
 322 int
 323 lst_node_str2state(char *str)
 324 {
 325         if (strcasecmp(str, "active") == 0)
 326                 return LST_NODE_ACTIVE;
 327         if (strcasecmp(str, "busy") == 0)
 328                 return LST_NODE_BUSY;
 329         if (strcasecmp(str, "down") == 0)
 330                 return LST_NODE_DOWN;
 331         if (strcasecmp(str, "unknown") == 0)
 332                 return LST_NODE_UNKNOWN;
 333         if (strcasecmp(str, "invalid") == 0)
 334                 return (LST_NODE_UNKNOWN | LST_NODE_DOWN | LST_NODE_BUSY);
 335
 336         return -1;
 337 }
 338
 339 char *
 340 lst_test_type2name(int type)
 341 {
 342         if (type == LST_TEST_PING)
 343                 return "ping";
 344         if (type == LST_TEST_BULK)
 345                 return "brw";
 346
 347         return "unknown";
 348 }
 349
 350 int
 351 lst_test_name2type(char *name)
 352 {
 353         if (strcasecmp(name, "ping") == 0)
 354                 return LST_TEST_PING;
 355         if (strcasecmp(name, "brw") == 0)
 356                 return LST_TEST_BULK;
 357
 358         return -1;
 359 }
 360
 361 void
 362 lst_print_usage(char *cmd)
 363 {
 364         Parser_printhelp(cmd);
 365 }
 366
 367 void
 368 lst_print_error(char *sub, const char *def_format, ...)
 369 {
 370         va_list ap;
 371
 372         /* local error returned from kernel */
 373         switch (errno) {
 374         case ESRCH:
 375                 fprintf(stderr, "No session exists\n");
 376                 return;
 377         case ESHUTDOWN:
 378                 fprintf(stderr, "Session is shutting down\n");
 379                 return;
 380         case EACCES:
 381                 fprintf(stderr, "Unmatched session key or not root\n");
 382                 return;
 383         case ENOENT:
 384                 fprintf(stderr, "Can't find %s in current session\n", sub);
 385                 return;
 386         case EINVAL:
 387                 fprintf(stderr, "Invalid parameters list in command line\n");
 388                 return;
 389         case EFAULT:
 390                 fprintf(stderr, "Bad parameter address\n");
 391                 return;
 392         case EEXIST:
 393                 fprintf(stderr, "%s already exists\n", sub);
 394                 return;
 395         default:
 396                 va_start(ap, def_format);
 397                 vfprintf(stderr, def_format, ap);
 398                 va_end(ap);
 399
 400                 return;
 401         }
 402 }
 403
 404 void
 405 lst_free_rpcent(struct list_head *head)
 406 {
 407         struct lstcon_rpc_ent *ent;
 408
 409         while (!list_empty(head)) {
 410                 ent = list_entry(head->next, struct lstcon_rpc_ent, rpe_link);
 411
 412                 list_del(&ent->rpe_link);
 413                 free(ent);
 414         }
 415 }
 416
 417 void
 418 lst_reset_rpcent(struct list_head *head)
 419 {
 420         struct lstcon_rpc_ent *ent;
 421
 422         list_for_each_entry(ent, head, rpe_link) {
 423                 ent->rpe_sid       = LST_INVALID_SID;
 424                 ent->rpe_peer.nid  = LNET_NID_ANY;
 425                 ent->rpe_peer.pid  = LNET_PID_ANY;
 426                 ent->rpe_rpc_errno = ent->rpe_fwk_errno = 0;
 427         }
 428 }
 429
 430 int
 431 lst_alloc_rpcent(struct list_head *head, int count, int offset)
 432 {
 433         struct lstcon_rpc_ent *ent;
 434         int               i;
 435
 436         for (i = 0; i < count; i++) {
 437                 ent = malloc(offsetof(struct lstcon_rpc_ent, rpe_payload[offset]));
 438                 if (ent == NULL) {
 439                         lst_free_rpcent(head);
 440                         return -1;
 441                 }
 442
 443                 memset(ent, 0, offsetof(struct lstcon_rpc_ent, rpe_payload[offset]));
 444
 445                 ent->rpe_sid      = LST_INVALID_SID;
 446                 ent->rpe_peer.nid = LNET_NID_ANY;
 447                 ent->rpe_peer.pid = LNET_PID_ANY;
 448                 list_add(&ent->rpe_link, head);
 449         }
 450
 451         return 0;
 452 }
 453
 454 void
 455 lst_print_transerr(struct list_head *head, char *optstr)
 456 {
 457         struct lstcon_rpc_ent *ent;
 458
 459         list_for_each_entry(ent, head, rpe_link) {
 460                 if (ent->rpe_rpc_errno == 0 && ent->rpe_fwk_errno == 0)
 461                         continue;
 462
 463                 if (ent->rpe_rpc_errno != 0) {
 464                         fprintf(stderr, "%s RPC failed on %s: %s\n",
 465                                 optstr, libcfs_id2str(ent->rpe_peer),
 466                                 strerror(ent->rpe_rpc_errno));
 467                         continue;
 468                 }
 469
 470                 fprintf(stderr, "operation %s failed on %s: %s\n",
 471                         optstr, libcfs_id2str(ent->rpe_peer),
 472                         strerror(ent->rpe_fwk_errno));
 473         }
 474 }
 475
 476 int lst_info_batch_ioctl(char *batch, int test, int server,
 477                         struct lstcon_test_batch_ent *entp, int *idxp,
 478                         int *ndentp, struct lstcon_node_ent *dentsp);
 479
 480 int lst_info_group_ioctl(char *name, struct lstcon_ndlist_ent *gent,
 481                          int *idx, int *count, struct lstcon_node_ent *dents);
 482
 483 int lst_query_batch_ioctl(char *batch, int test, int server,
 484                           int timeout, struct list_head *head);
 485
 486 int
 487 lst_ioctl(unsigned int opc, void *buf, int len)
 488 {
 489         struct libcfs_ioctl_data data;
 490         int    rc;
 491
 492         LIBCFS_IOC_INIT (data);
 493         data.ioc_u32[0]  = opc;
 494         data.ioc_plen1   = len;
 495         data.ioc_pbuf1   = (char *)buf;
 496         data.ioc_plen2   = sizeof(trans_stat);
 497         data.ioc_pbuf2   = (char *)&trans_stat;
 498
 499         memset(&trans_stat, 0, sizeof(trans_stat));
 500
 501         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNETST, &data);
 502
 503         /* local error, no valid RPC result */
 504         if (rc != 0)
 505                 return -1;
 506
 507         /* RPC error */
 508         if (trans_stat.trs_rpc_errno != 0)
 509                 return -2;
 510
 511         /* Framework error */
 512         if (trans_stat.trs_fwk_errno != 0)
 513                 return -3;
 514
 515         return 0;
 516 }
 517
 518 int lst_yaml_session(const char *label, const char *timeout, int nlflags,
 519                      const char *errmsg)
 520 {
 521         struct lstcon_ndlist_ent ndinfo = { };
 522         struct lst_sid sid = LST_INVALID_SID;
 523         /* nlflags being zero means we are destroying the session.
 524          * No parsing of reply needed.
 525          */
 526         bool done = nlflags ? false : true;
 527         char nid[LNET_NIDSTR_SIZE];
 528         char name[LST_NAME_SIZE];
 529         unsigned int key = 0;
 530         yaml_emitter_t request;
 531         yaml_parser_t reply;
 532         yaml_event_t event;
 533         struct nl_sock *sk;
 534         int rc;
 535
 536         sk = nl_socket_alloc();
 537         if (!sk)
 538                 return -1;
 539
 540         /* Note: NL_AUTO_PID == zero which we use by default for the
 541          * session_key when creating a new session. This is considered
 542          * an invalid key so we need to get the real session key from
 543          * the yaml parser yet to be created. If the user did request
 544          * a specific session key then set the socket's port id to this
 545          * value.
 546          */
 547         if (session_key)
 548                 nl_socket_set_local_port(sk, session_key);
 549
 550         /* Setup reply parser to recieve Netlink packets */
 551         rc = yaml_parser_initialize(&reply);
 552         if (rc == 0) {
 553                 nl_socket_free(sk);
 554                 return -1;
 555         }
 556
 557         rc = yaml_parser_set_input_netlink(&reply, sk, false);
 558         if (rc == 0)
 559                 goto parser_error;
 560
 561         /* Create Netlink emitter to send request to kernel */
 562         yaml_emitter_initialize(&request);
 563         rc = yaml_emitter_set_output_netlink(&request, sk,
 564                                              LNET_SELFTEST_GENL_NAME,
 565                                              LNET_SELFTEST_GENL_VERSION,
 566                                              LNET_SELFTEST_CMD_SESSIONS,
 567                                              nlflags);
 568         if (rc == 0)
 569                 goto emitter_error;
 570
 571         yaml_emitter_open(&request);
 572         yaml_document_start_event_initialize(&event, NULL, NULL, NULL, 0);
 573         rc = yaml_emitter_emit(&request, &event);
 574         if (rc == 0)
 575                 goto emitter_error;
 576
 577         yaml_mapping_start_event_initialize(&event, NULL,
 578                                             (yaml_char_t *)YAML_MAP_TAG,
 579                                             1, YAML_BLOCK_MAPPING_STYLE);
 580         rc = yaml_emitter_emit(&request, &event);
 581         if (rc == 0)
 582                 goto emitter_error;
 583
 584         yaml_scalar_event_initialize(&event, NULL,
 585                                      (yaml_char_t *)YAML_STR_TAG,
 586                                      (yaml_char_t *)"sessions",
 587                                      strlen("sessions"), 1, 0,
 588                                      YAML_PLAIN_SCALAR_STYLE);
 589         rc = yaml_emitter_emit(&request, &event);
 590         if (rc == 0)
 591                 goto emitter_error;
 592
 593         if (!label) {
 594                 yaml_scalar_event_initialize(&event, NULL,
 595                                              (yaml_char_t *)YAML_STR_TAG,
 596                                              (yaml_char_t *)"",
 597                                              strlen(""), 1, 0,
 598                                              YAML_PLAIN_SCALAR_STYLE);
 599                 rc = yaml_emitter_emit(&request, &event);
 600                 if (rc == 0)
 601                         goto emitter_error;
 602                 goto skip_params;
 603         }
 604
 605         /* sessions: { name: 'name', timeout: 300 }
 606          * or
 607          * sessions:
 608          *   name: 'name'
 609          *   timeout: 300
 610          */
 611         yaml_mapping_start_event_initialize(&event, NULL,
 612                                             (yaml_char_t *)YAML_MAP_TAG,
 613                                             1, YAML_FLOW_MAPPING_STYLE);
 614         rc = yaml_emitter_emit(&request, &event);
 615         if (rc == 0)
 616                 goto emitter_error;
 617
 618         yaml_scalar_event_initialize(&event, NULL,
 619                                      (yaml_char_t *)YAML_STR_TAG,
 620                                      (yaml_char_t *)"name",
 621                                      strlen("name"), 1, 0,
 622                                      YAML_PLAIN_SCALAR_STYLE);
 623         rc = yaml_emitter_emit(&request, &event);
 624         if (rc == 0)
 625                 goto emitter_error;
 626
 627         yaml_scalar_event_initialize(&event, NULL,
 628                                      (yaml_char_t *)YAML_STR_TAG,
 629                                      (yaml_char_t *)label,
 630                                      strlen(label), 1, 0,
 631                                      YAML_PLAIN_SCALAR_STYLE);
 632         rc = yaml_emitter_emit(&request, &event);
 633         if (rc == 0)
 634                 goto emitter_error;
 635
 636         if (timeout) {
 637                 yaml_scalar_event_initialize(&event, NULL,
 638                                              (yaml_char_t *)YAML_STR_TAG,
 639                                              (yaml_char_t *)"timeout",
 640                                              strlen("timeout"), 1, 0,
 641                                              YAML_PLAIN_SCALAR_STYLE);
 642                 rc = yaml_emitter_emit(&request, &event);
 643                 if (rc == 0)
 644                         goto emitter_error;
 645
 646                 yaml_scalar_event_initialize(&event, NULL,
 647                                              (yaml_char_t *)YAML_STR_TAG,
 648                                              (yaml_char_t *)timeout,
 649                                              strlen(timeout), 1, 0,
 650                                              YAML_PLAIN_SCALAR_STYLE);
 651                 rc = yaml_emitter_emit(&request, &event);
 652                 if (rc == 0)
 653                         goto emitter_error;
 654         }
 655
 656         yaml_mapping_end_event_initialize(&event);
 657         rc = yaml_emitter_emit(&request, &event);
 658         if (rc == 0)
 659                 goto emitter_error;
 660
 661 skip_params:
 662         yaml_mapping_end_event_initialize(&event);
 663         rc = yaml_emitter_emit(&request, &event);
 664         if (rc == 0)
 665                 goto emitter_error;
 666
 667         yaml_document_end_event_initialize(&event, 0);
 668         rc = yaml_emitter_emit(&request, &event);
 669         if (rc == 0)
 670                 goto emitter_error;
 671
 672         rc = yaml_emitter_close(&request);
 673         if (rc == 0) {
 674 emitter_error:
 675                 yaml_emitter_log_error(&request, stderr);
 676                 yaml_emitter_delete(&request);
 677                 errmsg = NULL;
 678                 goto parser_error;
 679         }
 680         yaml_emitter_delete(&request);
 681
 682         while (!done) {
 683                 rc = yaml_parser_parse(&reply, &event);
 684                 if (rc == 0)
 685                         goto parser_error;
 686
 687                 if (event.type == YAML_SCALAR_EVENT) {
 688                         char *tmp, *endp = NULL;
 689
 690                         if (strcmp((char *)event.data.scalar.value,
 691                                    "name") == 0) {
 692                                 yaml_event_delete(&event);
 693                                 rc = yaml_parser_parse(&reply, &event);
 694                                 if (rc == 0)
 695                                         goto parser_error;
 696
 697                                 strncpy(name, (char *)event.data.scalar.value,
 698                                         sizeof(name) - 1);
 699                         }
 700
 701                         if (strcmp((char *)event.data.scalar.value,
 702                                    "key") == 0) {
 703                                 yaml_event_delete(&event);
 704                                 rc = yaml_parser_parse(&reply, &event);
 705                                 if (rc == 0)
 706                                         goto parser_error;
 707
 708                                 tmp = (char *)event.data.scalar.value;
 709                                 key = strtoul(tmp, &endp, 10);
 710                                 if (endp == tmp)
 711                                         goto parser_error;
 712                         }
 713
 714                         if (strcmp((char *)event.data.scalar.value,
 715                                    "timestamp") == 0) {
 716                                 yaml_event_delete(&event);
 717                                 rc = yaml_parser_parse(&reply, &event);
 718                                 if (rc == 0)
 719                                         goto parser_error;
 720
 721                                 tmp = (char *)event.data.scalar.value;
 722                                 sid.ses_stamp = strtoll(tmp, &endp, 10);
 723                                 if (endp == tmp)
 724                                         goto parser_error;
 725                         }
 726
 727                         if (strcmp((char *)event.data.scalar.value,
 728                                    "nid") == 0) {
 729                                 yaml_event_delete(&event);
 730                                 rc = yaml_parser_parse(&reply, &event);
 731                                 if (rc == 0)
 732                                         goto parser_error;
 733
 734                                 strncpy(nid, (char *)event.data.scalar.value,
 735                                         sizeof(nid) - 1);
 736                         }
 737
 738                         if (strcmp((char *)event.data.scalar.value,
 739                                    "nodes") == 0) {
 740                                 yaml_event_delete(&event);
 741                                 rc = yaml_parser_parse(&reply, &event);
 742                                 if (rc == 0)
 743                                         goto parser_error;
 744
 745                                 tmp = (char *)event.data.scalar.value;
 746                                 ndinfo.nle_nnode = strtoul(tmp, &endp, 10);
 747                                 if (endp == tmp)
 748                                         goto parser_error;
 749                         }
 750                 }
 751
 752                 done = (event.type == YAML_STREAM_END_EVENT);
 753
 754                 yaml_event_delete(&event);
 755         }
 756
 757         if (nlflags & NLM_F_CREATE) {
 758                 session_features = yaml_parser_get_reader_proto_version(&reply);
 759                 session_key = key;
 760         }
 761 parser_error:
 762         if (rc == 0 && errmsg)
 763                 yaml_parser_log_error(&reply, stderr, errmsg);
 764         yaml_parser_delete(&reply);
 765         nl_socket_free(sk);
 766
 767         if (((nlflags & NLM_F_DUMP) == NLM_F_DUMP) && rc != 0) {
 768                 fprintf(stdout,
 769                         "%s ID: %ju@%s, KEY: %u FEATURES: %x NODES: %d\n",
 770                         name, (uintmax_t)sid.ses_stamp, nid,
 771                         key, session_features, ndinfo.nle_nnode);
 772         }
 773
 774         return rc == 0 ? -1 : 0;
 775 }
 776
 777 int
 778 lst_new_session_ioctl(char *name, int timeout, int force, struct lst_sid *sid)
 779 {
 780         struct lstio_session_new_args args = { 0 };
 781
 782         args.lstio_ses_key     = session_key;
 783         args.lstio_ses_timeout = timeout;
 784         args.lstio_ses_force   = force;
 785         args.lstio_ses_idp     = sid;
 786         args.lstio_ses_feats   = session_features;
 787         args.lstio_ses_nmlen   = strlen(name);
 788         args.lstio_ses_namep   = name;
 789
 790         return lst_ioctl (LSTIO_SESSION_NEW, &args, sizeof(args));
 791 }
 792
 793 int
 794 jt_lst_new_session(int argc, char **argv)
 795 {
 796         char  buf[LST_NAME_SIZE * 2 + 1];
 797         char *name, *timeout_s = NULL;
 798         int nlflags = NLM_F_CREATE;
 799         struct lst_sid session_id;
 800         int   optidx = 0;
 801         int   timeout = 300;
 802         int   force = 0;
 803         int   c;
 804         int   rc;
 805
 806         static const struct option session_opts[] = {
 807                 { .name = "timeout", .has_arg = required_argument, .val = 't' },
 808                 { .name = "force",   .has_arg = no_argument,       .val = 'f' },
 809                 { .name = NULL } };
 810
 811         while (1) {
 812                 c = getopt_long(argc, argv, "ft:",
 813                                 session_opts, &optidx);
 814                 if (c == -1)
 815                         break;
 816
 817                 switch (c) {
 818                 case 'f':
 819                         nlflags |= NLM_F_REPLACE;
 820                         force = 1;
 821                         break;
 822                 case 't':
 823                         timeout_s = optarg;
 824                         break;
 825                 default:
 826                         lst_print_usage(argv[0]);
 827                         return -1;
 828                 }
 829         }
 830
 831         if (timeout_s) {
 832                 timeout = atoi(timeout_s);
 833                 if (timeout <= 0) {
 834                         fprintf(stderr, "Invalid timeout value\n");
 835                         return -1;
 836                 }
 837         }
 838
 839         if (optind == argc - 1) {
 840                 name = argv[optind ++];
 841                 if (strlen(name) >= LST_NAME_SIZE) {
 842                         fprintf(stderr, "Name size is limited to %d\n",
 843                                 LST_NAME_SIZE - 1);
 844                         return -1;
 845                 }
 846         } else if (optind == argc) {
 847                 char           user[LST_NAME_SIZE];
 848                 char           host[LST_NAME_SIZE];
 849                 struct passwd *pw = getpwuid(getuid());
 850
 851                 if (pw == NULL)
 852                         snprintf(user, sizeof(user), "%d", (int)getuid());
 853                 else
 854                         snprintf(user, sizeof(user), "%s", pw->pw_name);
 855
 856                 rc = gethostname(host, sizeof(host));
 857                 if (rc != 0)
 858                         snprintf(host, sizeof(host), "unknown_host");
 859
 860                 snprintf(buf, sizeof(buf), "%s@%s", user, host);
 861                 name = buf;
 862         } else {
 863                 lst_print_usage(argv[0]);
 864                 return -1;
 865         }
 866
 867         rc = lst_yaml_session(name, timeout_s, nlflags, "new session");
 868         if (rc == 0)
 869                 goto success;
 870
 871         if (session_key == 0) {
 872                 fprintf(stderr,
 873                         "Can't find env LST_SESSION or value is not valid\n");
 874                 return -1;
 875         }
 876
 877         rc = lst_new_session_ioctl(name, timeout, force, &session_id);
 878         if (rc != 0) {
 879                 lst_print_error("session", "Failed to create session: %s\n",
 880                                 strerror(errno));
 881                 return rc;
 882         }
 883 success:
 884         fprintf(stdout, "SESSION: %s FEATURES: %x TIMEOUT: %d FORCE: %s\n",
 885                 name, session_features, timeout, force ? "Yes" : "No");
 886         return 0;
 887 }
 888
 889 int
 890 lst_session_info_ioctl(char *name, int len, int *key, unsigned *featp,
 891                        struct lst_sid *sid, struct lstcon_ndlist_ent *ndinfo)
 892 {
 893         struct lstio_session_info_args args = { 0 };
 894
 895         args.lstio_ses_idp     = sid;
 896         args.lstio_ses_keyp    = key;
 897         args.lstio_ses_featp   = featp;
 898         args.lstio_ses_ndinfo  = ndinfo;
 899         args.lstio_ses_nmlen   = len;
 900         args.lstio_ses_namep   = name;
 901
 902         return lst_ioctl(LSTIO_SESSION_INFO, &args, sizeof(args));
 903 }
 904
 905 int
 906 jt_lst_show_session(int argc, char **argv)
 907 {
 908         struct lstcon_ndlist_ent ndinfo;
 909         struct lst_sid sid;
 910         char name[LST_NAME_SIZE];
 911         unsigned int feats;
 912         int key;
 913         int rc;
 914
 915         rc = lst_yaml_session(NULL, NULL, NLM_F_DUMP, "show session");
 916         if (rc == 0)
 917                 return 0;
 918
 919         rc = lst_session_info_ioctl(name, sizeof(name), &key,
 920                                     &feats, &sid, &ndinfo);
 921
 922         if (rc != 0) {
 923                 lst_print_error("session", "Failed to show session: %s\n",
 924                                 strerror(errno));
 925                 return -1;
 926         }
 927
 928         fprintf(stdout, "%s ID: %ju@%s, KEY: %d FEATURES: %x NODES: %d\n",
 929                 name, (uintmax_t)sid.ses_stamp, libcfs_nid2str(sid.ses_nid),
 930                 key, feats, ndinfo.nle_nnode);
 931
 932         return 0;
 933 }
 934
 935 int
 936 lst_end_session_ioctl(void)
 937 {
 938         struct lstio_session_end_args args = { 0 };
 939
 940         args.lstio_ses_key = session_key;
 941         return lst_ioctl(LSTIO_SESSION_END, &args, sizeof(args));
 942 }
 943
 944 int
 945 jt_lst_end_session(int argc, char **argv)
 946 {
 947         int             rc;
 948
 949         if (session_key == 0) {
 950                 fprintf(stderr,
 951                         "Can't find env LST_SESSION or value is not valid\n");
 952                 return -1;
 953         }
 954
 955         rc = lst_yaml_session(NULL, NULL, 0, "end session");
 956         if (rc == 0)
 957                 goto finish;
 958
 959         rc = lst_end_session_ioctl();
 960
 961         if (rc == 0) {
 962                 fprintf(stdout, "session is ended\n");
 963                 return 0;
 964         }
 965
 966         if (rc == -1) {
 967                 lst_print_error("session", "Failed to end session: %s\n",
 968                                 strerror(errno));
 969                 return rc;
 970         }
 971 finish:
 972         if (trans_stat.trs_rpc_errno != 0) {
 973                 fprintf(stderr,
 974                         "[RPC] Failed to send %d session RPCs: %s\n",
 975                         lstcon_rpc_stat_failure(&trans_stat, 0),
 976                         strerror(trans_stat.trs_rpc_errno));
 977         }
 978
 979         if (trans_stat.trs_fwk_errno != 0) {
 980                 fprintf(stderr,
 981                         "[FWK] Failed to end session on %d nodes: %s\n",
 982                         lstcon_sesop_stat_failure(&trans_stat, 0),
 983                         strerror(trans_stat.trs_fwk_errno));
 984         }
 985
 986         return rc;
 987 }
 988
 989 #define LST_NODES_TITLE "\tACTIVE\tBUSY\tDOWN\tUNKNOWN\tTOTAL\n"
 990
 991 static int lst_yaml_display_groups(yaml_parser_t *reply, char *group,
 992                                    int states, bool print)
 993 {
 994         yaml_event_t event;
 995         int rc;
 996
 997         if (states) {
 998                 char name[LST_NAME_SIZE] = {};
 999                 bool done = false;
1000                 int count = 0;
1001
1002                 if (print)
1003                         fprintf(stdout, LST_NODES_TITLE);
1004
1005                 while (!done) {
1006                         rc = yaml_parser_parse(reply, &event);
1007                         if (rc == 0)
1008                                 goto parser_error;
1009
1010                         if (event.type == YAML_MAPPING_START_EVENT) {
1011                                 char *value = NULL;
1012                                 yaml_event_t next;
1013
1014                                 rc = yaml_parser_parse(reply, &next);
1015                                 if (rc == 0)
1016                                         goto parser_error;
1017
1018                                 if (next.type != YAML_SCALAR_EVENT) {
1019                                         yaml_event_delete(&next);
1020                                         continue;
1021                                 }
1022
1023                                 value = (char *)next.data.scalar.value;
1024                                 if (strcmp(value, "groups") == 0) {
1025                                         yaml_event_delete(&next);
1026                                 } else if (strcmp(value, "nid") == 0) {
1027                                         yaml_event_t next;
1028                                         char *tmp;
1029
1030                                         rc = yaml_parser_parse(reply, &next);
1031                                         if (rc == 0)
1032                                                 goto parser_error;
1033
1034                                         fprintf(stdout, "\t%s: ",
1035                                                 (char *)next.data.scalar.value);
1036
1037                                         rc = yaml_parser_parse(reply, &next);
1038                                         if (rc == 0)
1039                                                 goto parser_error;
1040
1041                                         tmp = (char *)next.data.scalar.value;
1042                                         if (strcmp(tmp, "status") == 0) {
1043                                                 yaml_event_t state;
1044
1045                                                 rc = yaml_parser_parse(reply, &state);
1046                                                 if (rc == 0)
1047                                                         goto parser_error;
1048
1049                                                 fprintf(stdout, "%s\n",
1050                                                         (char *)state.data.scalar.value);
1051                                                 count++;
1052                                         }
1053                                 } else {
1054                                         yaml_event_t next;
1055
1056                                         rc = yaml_parser_parse(reply, &next);
1057                                         if (rc == 0)
1058                                                 goto parser_error;
1059
1060                                         strncpy(name, value, sizeof(name) - 1);
1061                                         fprintf(stdout, "Group [ %s ]\n", name);
1062                                         count = 0;
1063
1064                                         if (next.type != YAML_SEQUENCE_START_EVENT) {
1065                                                 fprintf(stdout,
1066                                                         "No nodes found [ %s ]\n",
1067                                                         name);
1068                                         }
1069                                 }
1070                         } else if (event.type == YAML_SEQUENCE_END_EVENT &&
1071                                    count) {
1072                                 fprintf(stdout, "Total %d nodes [ %s ]\n",
1073                                         count, name);
1074                                 count = 0;
1075                         }
1076
1077                         done = (event.type == YAML_STREAM_END_EVENT);
1078                         yaml_event_delete(&event);
1079                 }
1080         } else if (group) {
1081                 int active = 0, busy = 0, down = 0, unknown = 0;
1082                 char group[LST_NAME_SIZE];
1083                 bool done = false;
1084
1085                 while (!done) {
1086                         rc = yaml_parser_parse(reply, &event);
1087                         if (rc == 0)
1088                                 goto parser_error;
1089
1090                         if (event.type == YAML_SCALAR_EVENT) {
1091                                 char *value = (char *)event.data.scalar.value;
1092                                 yaml_event_t next;
1093
1094                                 value = (char *)event.data.scalar.value;
1095                                 if (strcmp(value, "groups") == 0) {
1096                                         yaml_event_delete(&event);
1097                                         continue;
1098                                 }
1099
1100                                 rc = yaml_parser_parse(reply, &next);
1101                                 if (rc == 0)
1102                                         goto parser_error;
1103
1104                                 if (next.type == YAML_SCALAR_EVENT) {
1105                                         int status;
1106
1107                                         status = lst_node_str2state((char *)next.data.scalar.value);
1108                                         switch (status) {
1109                                         case LST_NODE_ACTIVE:
1110                                                 active++;
1111                                                 break;
1112                                         case LST_NODE_BUSY:
1113                                                 busy++;
1114                                                 break;
1115                                         case LST_NODE_DOWN:
1116                                                 down++;
1117                                                 break;
1118                                         case LST_NODE_UNKNOWN:
1119                                                 unknown++;
1120                                         default:
1121                                                 break;
1122                                         }
1123                                 } else if (next.type == YAML_SEQUENCE_START_EVENT) {
1124                                         strncpy(group, value, sizeof(group) - 1);
1125                                         active = 0;
1126                                         busy = 0;
1127                                         down = 0;
1128                                         unknown = 0;
1129                                 }
1130                                 yaml_event_delete(&next);
1131                         } else if (event.type == YAML_SEQUENCE_END_EVENT) {
1132                                 if (strlen(group)) {
1133                                         fprintf(stdout, "\t%d\t%d\t%d\t%d\t%d\t%s\n",
1134                                                 active, busy, down, unknown,
1135                                                 active + busy + down + unknown, group);
1136                                 }
1137                                 memset(group, 0, sizeof(group));
1138                         }
1139                         done = (event.type == YAML_STREAM_END_EVENT);
1140                         yaml_event_delete(&event);
1141                 }
1142         } else {
1143                 bool done = false;
1144                 unsigned int i = 1;
1145
1146                 while (!done) {
1147                         rc = yaml_parser_parse(reply, &event);
1148                         if (rc == 0)
1149                                 goto parser_error;
1150
1151                         if (event.type == YAML_SCALAR_EVENT) {
1152                                 char *value;
1153
1154                                 value = (char *)event.data.scalar.value;
1155                                 if (strlen(value) &&
1156                                     strcmp(value, "groups") != 0) {
1157                                         if (print)
1158                                                 fprintf(stdout, "%d) %s\n",
1159                                                         i, value);
1160                                         i++;
1161                                 }
1162                         }
1163                         done = (event.type == YAML_STREAM_END_EVENT);
1164
1165                         yaml_event_delete(&event);
1166                 }
1167                 if (print)
1168                         fprintf(stdout, "Total %d groups\n", i - 1);
1169                 else
1170                         rc = i - 1;
1171         }
1172 parser_error:
1173         return rc;
1174 }
1175
1176 #define LST_NODES_TITLE "\tACTIVE\tBUSY\tDOWN\tUNKNOWN\tTOTAL\n"
1177
1178 static int lst_yaml_groups(int nlflags, char *name, int states, bool print)
1179 {
1180         yaml_emitter_t request;
1181         yaml_parser_t reply;
1182         yaml_event_t event;
1183         struct nl_sock *sk;
1184         int rc;
1185
1186         sk = nl_socket_alloc();
1187         if (!sk)
1188                 return -EOPNOTSUPP;
1189
1190         /* Setup reply parser to recieve Netlink packets */
1191         rc = yaml_parser_initialize(&reply);
1192         if (rc == 0) {
1193                 nl_socket_free(sk);
1194                 return -EOPNOTSUPP;
1195         }
1196
1197         rc = yaml_parser_set_input_netlink(&reply, sk, false);
1198         if (rc == 0)
1199                 goto parser_error;
1200
1201         /* Create Netlink emitter to send request to kernel */
1202         yaml_emitter_initialize(&request);
1203         rc = yaml_emitter_set_output_netlink(&request, sk,
1204                                              LNET_SELFTEST_GENL_NAME,
1205                                              LNET_SELFTEST_GENL_VERSION,
1206                                              LNET_SELFTEST_CMD_GROUPS,
1207                                              nlflags);
1208         if (rc == 0)
1209                 goto emitter_error;
1210
1211         yaml_emitter_open(&request);
1212         yaml_document_start_event_initialize(&event, NULL, NULL, NULL, 0);
1213         rc = yaml_emitter_emit(&request, &event);
1214         if (rc == 0)
1215                 goto emitter_error;
1216
1217         yaml_mapping_start_event_initialize(&event, NULL,
1218                                             (yaml_char_t *)YAML_MAP_TAG,
1219                                             1, YAML_BLOCK_MAPPING_STYLE);
1220         rc = yaml_emitter_emit(&request, &event);
1221         if (rc == 0)
1222                 goto emitter_error;
1223
1224         yaml_scalar_event_initialize(&event, NULL,
1225                                      (yaml_char_t *)YAML_STR_TAG,
1226                                      (yaml_char_t *)"groups",
1227                                      strlen("groups"), 1, 0,
1228                                      YAML_PLAIN_SCALAR_STYLE);
1229         rc = yaml_emitter_emit(&request, &event);
1230         if (rc == 0)
1231                 goto emitter_error;
1232
1233         if (name) {
1234                 yaml_sequence_start_event_initialize(&event, NULL,
1235                                                      (yaml_char_t *)YAML_SEQ_TAG,
1236                                                      1, YAML_BLOCK_SEQUENCE_STYLE);
1237                 rc = yaml_emitter_emit(&request, &event);
1238                 if (rc == 0)
1239                         goto emitter_error;
1240
1241                 yaml_mapping_start_event_initialize(&event, NULL,
1242                                                     (yaml_char_t *)YAML_MAP_TAG,
1243                                                     1, YAML_BLOCK_MAPPING_STYLE);
1244                 rc = yaml_emitter_emit(&request, &event);
1245                 if (rc == 0)
1246                         goto emitter_error;
1247
1248                 yaml_scalar_event_initialize(&event, NULL,
1249                                              (yaml_char_t *)YAML_STR_TAG,
1250                                              (yaml_char_t *)name,
1251                                              strlen(name), 1, 0,
1252                                              YAML_PLAIN_SCALAR_STYLE);
1253                 rc = yaml_emitter_emit(&request, &event);
1254                 if (rc == 0)
1255                         goto emitter_error;
1256
1257                 if (states) {
1258                         int max = ffs(LST_NODE_UNKNOWN) + 1, i;
1259                         char *state;
1260
1261                         yaml_sequence_start_event_initialize(&event, NULL,
1262                                                              (yaml_char_t *)YAML_SEQ_TAG,
1263                                                              1, YAML_BLOCK_SEQUENCE_STYLE);
1264                         rc = yaml_emitter_emit(&request, &event);
1265                         if (rc == 0)
1266                                 goto emitter_error;
1267
1268
1269                         for (i = 0; i < max; i++) {
1270                                 int mask = states & (1 << i);
1271
1272                                 state = lst_node_state2str(mask);
1273                                 if (mask != LST_NODE_UNKNOWN && strcmp(state, "Unknown") == 0)
1274                                         continue;
1275
1276                                 yaml_mapping_start_event_initialize(&event, NULL,
1277                                                                     (yaml_char_t *)YAML_MAP_TAG,
1278                                                                     1, YAML_BLOCK_MAPPING_STYLE);
1279                                 rc = yaml_emitter_emit(&request, &event);
1280                                 if (rc == 0)
1281                                         goto emitter_error;
1282
1283                                 yaml_scalar_event_initialize(&event, NULL,
1284                                                              (yaml_char_t *)YAML_STR_TAG,
1285                                                              (yaml_char_t *)"status",
1286                                                              strlen("status"), 1, 0,
1287                                                              YAML_PLAIN_SCALAR_STYLE);
1288                                 rc = yaml_emitter_emit(&request, &event);
1289                                 if (rc == 0)
1290                                         goto emitter_error;
1291
1292                                 yaml_scalar_event_initialize(&event, NULL,
1293                                                              (yaml_char_t *)YAML_STR_TAG,
1294                                                              (yaml_char_t *)state,
1295                                                              strlen(state), 1, 0,
1296                                                              YAML_PLAIN_SCALAR_STYLE);
1297                                 rc = yaml_emitter_emit(&request, &event);
1298                                 if (rc == 0)
1299                                         goto emitter_error;
1300
1301                                 yaml_mapping_end_event_initialize(&event);
1302                                 rc = yaml_emitter_emit(&request, &event);
1303                                 if (rc == 0)
1304                                         goto emitter_error;
1305                         }
1306
1307                         yaml_sequence_end_event_initialize(&event);
1308                         rc = yaml_emitter_emit(&request, &event);
1309                         if (rc == 0)
1310                                 goto emitter_error;
1311                 } else {
1312                         yaml_scalar_event_initialize(&event, NULL,
1313                                                      (yaml_char_t *)YAML_STR_TAG,
1314                                                      (yaml_char_t *)"",
1315                                                      strlen(""), 1, 0,
1316                                                      YAML_PLAIN_SCALAR_STYLE);
1317                         rc = yaml_emitter_emit(&request, &event);
1318                         if (rc == 0)
1319                                 goto emitter_error;
1320                 }
1321
1322                 yaml_mapping_end_event_initialize(&event);
1323                 rc = yaml_emitter_emit(&request, &event);
1324                 if (rc == 0)
1325                         goto emitter_error;
1326
1327                 yaml_sequence_end_event_initialize(&event);
1328                 rc = yaml_emitter_emit(&request, &event);
1329                 if (rc == 0)
1330                         goto emitter_error;
1331         } else {
1332                 yaml_scalar_event_initialize(&event, NULL,
1333                                              (yaml_char_t *)YAML_STR_TAG,
1334                                              (yaml_char_t *)"",
1335                                              strlen(""), 1, 0,
1336                                              YAML_PLAIN_SCALAR_STYLE);
1337                 rc = yaml_emitter_emit(&request, &event);
1338                 if (rc == 0)
1339                         goto emitter_error;
1340         }
1341         yaml_mapping_end_event_initialize(&event);
1342         rc = yaml_emitter_emit(&request, &event);
1343         if (rc == 0)
1344                 goto emitter_error;
1345
1346         yaml_document_end_event_initialize(&event, 0);
1347         rc = yaml_emitter_emit(&request, &event);
1348         if (rc == 0)
1349                 goto emitter_error;
1350
1351         rc = yaml_emitter_close(&request);
1352         if (rc == 0) {
1353 emitter_error:
1354                 yaml_emitter_log_error(&request, stderr);
1355                 yaml_emitter_delete(&request);
1356                 rc = -EINVAL;
1357                 goto parser_error;
1358         }
1359         yaml_emitter_delete(&request);
1360
1361         /* display output */
1362         if (nlflags == NLM_F_DUMP)
1363                 rc = lst_yaml_display_groups(&reply, name, states, print);
1364 parser_error:
1365         if (rc == 0)
1366                 yaml_parser_log_error(&reply, stderr, NULL);
1367         yaml_parser_delete(&reply);
1368         nl_socket_free(sk);
1369
1370         if (print)
1371                 rc = rc == 1 ? 0 : -EINVAL;
1372
1373         return rc;
1374 }
1375
1376 int
1377 lst_ping_ioctl(char *str, int type, int timeout,
1378                int count, struct lnet_process_id *ids, struct list_head *head)
1379 {
1380         struct lstio_debug_args args = { 0 };
1381
1382         args.lstio_dbg_key     = session_key;
1383         args.lstio_dbg_type    = type;
1384         args.lstio_dbg_flags   = 0;
1385         args.lstio_dbg_timeout = timeout;
1386         args.lstio_dbg_nmlen   = (str == NULL) ? 0: strlen(str);
1387         args.lstio_dbg_namep   = str;
1388         args.lstio_dbg_count   = count;
1389         args.lstio_dbg_idsp    = ids;
1390         args.lstio_dbg_resultp = head;
1391
1392         return lst_ioctl (LSTIO_DEBUG, &args, sizeof(args));
1393 }
1394
1395 int
1396 lst_get_node_count(int type, char *str, int *countp,
1397                    struct lnet_process_id **idspp)
1398 {
1399         char                    buf[LST_NAME_SIZE];
1400         struct lstcon_test_batch_ent ent;
1401         struct lstcon_ndlist_ent    *entp = &ent.tbe_cli_nle;
1402         struct lst_sid sid;
1403         unsigned                feats;
1404         int                     key;
1405         int                     rc;
1406
1407         switch (type) {
1408         case LST_OPC_SESSION:
1409                 rc = lst_session_info_ioctl(buf, LST_NAME_SIZE,
1410                                             &key, &feats, &sid, entp);
1411                 break;
1412
1413         case LST_OPC_BATCHSRV:
1414                 entp = &ent.tbe_srv_nle;
1415         case LST_OPC_BATCHCLI:
1416                 rc = lst_info_batch_ioctl(str, 0, 0, &ent, NULL, NULL, NULL);
1417                 break;
1418
1419         case LST_OPC_GROUP:
1420                 rc = lst_yaml_groups(NLM_F_DUMP, NULL, 0, false);
1421                 if (rc == -EOPNOTSUPP) {
1422                         rc = lst_info_group_ioctl(str, entp, NULL, NULL, NULL);
1423                 } else if (rc > 0) {
1424                         entp->nle_nnode = rc;
1425                         rc = 0;
1426                 }
1427                 break;
1428
1429         case LST_OPC_NODES:
1430                 rc = lst_parse_nids(str, &entp->nle_nnode, idspp) < 0 ? -1 : 0;
1431                 break;
1432
1433         default:
1434                 rc = -1;
1435                 break;
1436         }
1437
1438         if (rc == 0)
1439                 *countp = entp->nle_nnode;
1440
1441         return rc;
1442 }
1443
1444 int
1445 jt_lst_ping(int argc,  char **argv)
1446 {
1447         struct list_head   head;
1448         struct lnet_process_id *ids = NULL;
1449         struct lstcon_rpc_ent  *ent = NULL;
1450         char              *str = NULL;
1451         int                optidx  = 0;
1452         int                server  = 0;
1453         int                timeout = 5;
1454         int                count   = 0;
1455         int                type    = 0;
1456         int                rc      = 0;
1457         int                c;
1458
1459         static const struct option ping_opts[] = {
1460                 { .name = "session", .has_arg = no_argument,       .val = 's' },
1461                 { .name = "server",  .has_arg = no_argument,       .val = 'v' },
1462                 { .name = "batch",   .has_arg = required_argument, .val = 'b' },
1463                 { .name = "group",   .has_arg = required_argument, .val = 'g' },
1464                 { .name = "nodes",   .has_arg = required_argument, .val = 'n' },
1465                 { .name = "timeout", .has_arg = required_argument, .val = 't' },
1466                 { .name = NULL, } };
1467
1468         if (session_key == 0) {
1469                 fprintf(stderr,
1470                         "Can't find env LST_SESSION or value is not valid\n");
1471                 return -1;
1472         }
1473
1474         while (1) {
1475
1476                 c = getopt_long(argc, argv, "g:b:n:t:sv",
1477                                 ping_opts, &optidx);
1478
1479                 if (c == -1)
1480                         break;
1481
1482                 switch (c) {
1483                 case 's':
1484                         type = LST_OPC_SESSION;
1485                         break;
1486
1487                 case 'g':
1488                         type = LST_OPC_GROUP;
1489                         str = optarg;
1490                         break;
1491
1492                 case 'b':
1493                         type = LST_OPC_BATCHCLI;
1494                         str = optarg;
1495                         break;
1496
1497                 case 'n':
1498                         type = LST_OPC_NODES;
1499                         str = optarg;
1500                         break;
1501
1502                 case 't':
1503                         timeout = atoi(optarg);
1504                         break;
1505
1506                 case 'v':
1507                         server = 1;
1508                         break;
1509
1510                 default:
1511                         lst_print_usage(argv[0]);
1512                         return -1;
1513                 }
1514         }
1515
1516         if (type == 0 || timeout <= 0 || optind != argc) {
1517                 lst_print_usage(argv[0]);
1518                 return -1;
1519         }
1520
1521         if (type == LST_OPC_BATCHCLI && server)
1522                 type = LST_OPC_BATCHSRV;
1523
1524         rc = lst_get_node_count(type, str, &count, &ids);
1525         if (rc < 0) {
1526                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
1527                         (str == NULL) ? "session" : str, strerror(errno));
1528                 return -1;
1529         }
1530
1531         INIT_LIST_HEAD(&head);
1532
1533         rc = lst_alloc_rpcent(&head, count, LST_NAME_SIZE);
1534         if (rc != 0) {
1535                 fprintf(stderr, "Out of memory\n");
1536                 goto out;
1537         }
1538
1539         if (count == 0) {
1540                 fprintf(stdout, "Target %s is empty\n",
1541                         (str == NULL) ? "session" : str);
1542                 goto out;
1543         }
1544
1545         rc = lst_ping_ioctl(str, type, timeout, count, ids, &head);
1546         if (rc == -1) { /* local failure */
1547                 lst_print_error("debug", "Failed to ping %s: %s\n",
1548                                 (str == NULL) ? "session" : str,
1549                                 strerror(errno));
1550                 rc = -1;
1551                 goto out;
1552         }
1553
1554         /* ignore RPC errors and framwork errors */
1555         list_for_each_entry(ent, &head, rpe_link) {
1556                 fprintf(stdout, "\t%s: %s [session: %s id: %s]\n",
1557                         libcfs_id2str(ent->rpe_peer),
1558                         lst_node_state2str(ent->rpe_state),
1559                         (ent->rpe_state == LST_NODE_ACTIVE ||
1560                          ent->rpe_state == LST_NODE_BUSY) ?
1561                                 (ent->rpe_rpc_errno == 0 ?
1562                                         &ent->rpe_payload[0] : "Unknown") :
1563                                 "<NULL>", libcfs_nid2str(ent->rpe_sid.ses_nid));
1564         }
1565
1566 out:
1567         lst_free_rpcent(&head);
1568
1569         if (ids != NULL)
1570                 free(ids);
1571
1572         return rc;
1573
1574 }
1575
1576 int
1577 lst_add_nodes_ioctl(char *name, int count, struct lnet_process_id *ids,
1578                     unsigned *featp, struct list_head *resultp)
1579 {
1580         struct lstio_group_nodes_args args = { 0 };
1581
1582         args.lstio_grp_key     = session_key;
1583         args.lstio_grp_nmlen   = strlen(name);
1584         args.lstio_grp_namep   = name;
1585         args.lstio_grp_count   = count;
1586         args.lstio_grp_featp   = featp;
1587         args.lstio_grp_idsp    = ids;
1588         args.lstio_grp_resultp = resultp;
1589
1590         return lst_ioctl(LSTIO_NODES_ADD, &args, sizeof(args));
1591 }
1592
1593 int
1594 lst_del_group_ioctl(char *name)
1595 {
1596         struct lstio_group_del_args args = { 0 };
1597
1598         args.lstio_grp_key   = session_key;
1599         args.lstio_grp_nmlen = strlen(name);
1600         args.lstio_grp_namep = name;
1601
1602         return lst_ioctl(LSTIO_GROUP_DEL, &args, sizeof(args));
1603 }
1604
1605 int
1606 lst_del_group(char *grp_name)
1607 {
1608         int     rc;
1609
1610         rc = lst_del_group_ioctl(grp_name);
1611         if (rc == 0) {
1612                 fprintf(stdout, "Group is deleted\n");
1613                 return 0;
1614         }
1615
1616         if (rc == -1) {
1617                 lst_print_error("group", "Failed to delete group: %s\n",
1618                                 strerror(errno));
1619                 return rc;
1620         }
1621
1622         fprintf(stderr, "Group is deleted with some errors\n");
1623
1624         if (trans_stat.trs_rpc_errno != 0) {
1625                 fprintf(stderr,
1626                         "[RPC] Failed to send %d end session RPCs: %s\n",
1627                         lstcon_rpc_stat_failure(&trans_stat, 0),
1628                         strerror(trans_stat.trs_rpc_errno));
1629         }
1630
1631         if (trans_stat.trs_fwk_errno != 0) {
1632                 fprintf(stderr,
1633                         "[FWK] Failed to end session on %d nodes: %s\n",
1634                 lstcon_sesop_stat_failure(&trans_stat, 0),
1635                 strerror(trans_stat.trs_fwk_errno));
1636         }
1637
1638         return -1;
1639 }
1640
1641 int
1642 lst_add_group_ioctl(char *name)
1643 {
1644         struct lstio_group_add_args args = { 0 };
1645
1646         args.lstio_grp_key     =  session_key;
1647         args.lstio_grp_nmlen   =  strlen(name);
1648         args.lstio_grp_namep   =  name;
1649
1650         return lst_ioctl(LSTIO_GROUP_ADD, &args, sizeof(args));
1651 }
1652
1653 int
1654 jt_lst_add_group(int argc, char **argv)
1655 {
1656         struct list_head   head;
1657         struct lnet_process_id *ids;
1658         char              *name;
1659         unsigned           feats = session_features;
1660         int                count;
1661         int                rc;
1662         int                i;
1663         bool               nodes_added = false;
1664
1665         if (session_key == 0) {
1666                 fprintf(stderr,
1667                         "Can't find env LST_SESSION or value is not valid\n");
1668                 return -1;
1669         }
1670
1671         if (argc < 3) {
1672                 lst_print_usage(argv[0]);
1673                 return -1;
1674         }
1675
1676         name = argv[1];
1677         if (strlen(name) >= LST_NAME_SIZE) {
1678                 fprintf(stderr, "Name length is limited to %d\n",
1679                         LST_NAME_SIZE - 1);
1680                 return -1;
1681         }
1682
1683         rc = lst_add_group_ioctl(name);
1684         if (rc != 0) {
1685                 lst_print_error("group", "Failed to add group %s: %s\n",
1686                                 name, strerror(errno));
1687                 return -1;
1688         }
1689
1690         INIT_LIST_HEAD(&head);
1691
1692         for (i = 2; i < argc; i++) {
1693                 /* parse address list */
1694                 rc = lst_parse_nids(argv[i], &count, &ids);
1695                 if (rc < 0) {
1696                         fprintf(stderr, "Ignore invalid id list %s\n",
1697                                 argv[i]);
1698                         continue;
1699                 }
1700
1701                 if (count == 0)
1702                         continue;
1703
1704                 rc = lst_alloc_rpcent(&head, count, 0);
1705                 if (rc != 0) {
1706                         fprintf(stderr, "Out of memory\n");
1707                         free(ids);
1708                         rc = -1;
1709                         goto failed;
1710                 }
1711
1712                 rc = lst_add_nodes_ioctl(name, count, ids, &feats, &head);
1713
1714                 free(ids);
1715
1716                 if (rc != 0)
1717                         goto failed;
1718
1719                 fprintf(stdout, "%s are added to session\n", argv[i]);
1720
1721                 nodes_added = true;
1722
1723                 if ((feats & session_features) != session_features) {
1724                         fprintf(stdout,
1725                                 "Warning, this session will run with "
1726                                 "compatible mode because some test nodes "
1727                                 "might not understand these features: %x\n",
1728                                 (~feats & session_features));
1729                 }
1730
1731                 lst_free_rpcent(&head);
1732         }
1733
1734         if (!nodes_added) {
1735                 /*
1736                  * The selftest kernel module expects that a group should
1737                  * have at least one node, since it doesn't make sense for
1738                  * an empty group to be added to a test.
1739                  */
1740                 fprintf(stderr,
1741                         "No nodes added successfully, deleting group %s\n",
1742                         name);
1743                 rc = lst_del_group(name);
1744                 if (rc != 0) {
1745                         fprintf(stderr,
1746                                 "Failed to delete group %s."
1747                                 "  Group is empty.\n", name);
1748                 }
1749         }
1750
1751         return rc;
1752
1753 failed:
1754         if (rc == -1) {
1755                 lst_print_error("group", "Failed to add nodes %s: %s\n",
1756                                 argv[i], strerror(errno));
1757
1758         } else {
1759                 if (trans_stat.trs_fwk_errno == EPROTO) {
1760                         fprintf(stderr,
1761                                 "test nodes might have different LST "
1762                                 "features, please disable some features by "
1763                                 "setting LST_FEATURES\n");
1764                 }
1765
1766                 lst_print_transerr(&head, "create session");
1767         }
1768
1769         lst_free_rpcent(&head);
1770
1771         if (!nodes_added) {
1772                 fprintf(stderr,
1773                         "No nodes added successfully, deleting group %s\n",
1774                         name);
1775                 if (lst_del_group(name) != 0) {
1776                         fprintf(stderr,
1777                                 "Failed to delete group %s."
1778                                 "  Group is empty.\n", name);
1779                 }
1780         }
1781
1782         return rc;
1783 }
1784
1785 int
1786 jt_lst_del_group(int argc, char **argv)
1787 {
1788         int     rc;
1789
1790         if (session_key == 0) {
1791                 fprintf(stderr,
1792                         "Can't find env LST_SESSION or value is not valid\n");
1793                 return -1;
1794         }
1795
1796         if (argc != 2) {
1797                 lst_print_usage(argv[0]);
1798                 return -1;
1799         }
1800
1801         rc = lst_del_group(argv[1]);
1802
1803         return rc;
1804 }
1805
1806 int
1807 lst_update_group_ioctl(int opc, char *name, int clean, int count,
1808                        struct lnet_process_id *ids, struct list_head *resultp)
1809 {
1810         struct lstio_group_update_args args = { 0 };
1811
1812         args.lstio_grp_key      = session_key;
1813         args.lstio_grp_opc      = opc;
1814         args.lstio_grp_args     = clean;
1815         args.lstio_grp_nmlen    = strlen(name);
1816         args.lstio_grp_namep    = name;
1817         args.lstio_grp_count    = count;
1818         args.lstio_grp_idsp     = ids;
1819         args.lstio_grp_resultp  = resultp;
1820
1821         return lst_ioctl(LSTIO_GROUP_UPDATE, &args, sizeof(args));
1822 }
1823
1824 int
1825 jt_lst_update_group(int argc, char **argv)
1826 {
1827         struct list_head   head;
1828         struct lnet_process_id *ids = NULL;
1829         char              *str = NULL;
1830         char              *grp = NULL;
1831         int                optidx = 0;
1832         int                count = 0;
1833         int                clean = 0;
1834         int                opc = 0;
1835         int                rc;
1836         int                c;
1837
1838         static const struct option update_group_opts[] = {
1839                 { .name = "refresh", .has_arg = no_argument,       .val = 'f' },
1840                 { .name = "clean",   .has_arg = required_argument, .val = 'c' },
1841                 { .name = "remove",  .has_arg = required_argument, .val = 'r' },
1842                 { .name = NULL } };
1843
1844         if (session_key == 0) {
1845                 fprintf(stderr,
1846                         "Can't find env LST_SESSION or value is not valid\n");
1847                 return -1;
1848         }
1849
1850         while (1) {
1851                 c = getopt_long(argc, argv, "fc:r:",
1852                                 update_group_opts, &optidx);
1853
1854                 /* Detect the end of the options. */
1855                 if (c == -1)
1856                         break;
1857
1858                 switch (c) {
1859                 case 'f':
1860                         if (opc != 0) {
1861                                 lst_print_usage(argv[0]);
1862                                 return -1;
1863                         }
1864                         opc = LST_GROUP_REFRESH;
1865                         break;
1866
1867                 case 'r':
1868                         if (opc != 0) {
1869                                 lst_print_usage(argv[0]);
1870                                 return -1;
1871                         }
1872                         opc = LST_GROUP_RMND;
1873                         str = optarg;
1874                         break;
1875
1876                 case 'c':
1877                         clean = lst_node_str2state(optarg);
1878                         if (opc != 0 || clean <= 0) {
1879                                 lst_print_usage(argv[0]);
1880                                 return -1;
1881                         }
1882                         opc = LST_GROUP_CLEAN;
1883                         break;
1884
1885                 default:
1886                         lst_print_usage(argv[0]);
1887                         return -1;
1888                 }
1889         }
1890
1891         /* no OPC or group is specified */
1892         if (opc == 0 || optind != argc - 1) {
1893                 lst_print_usage(argv[0]);
1894                 return -1;
1895         }
1896
1897         grp = argv[optind];
1898
1899         INIT_LIST_HEAD(&head);
1900
1901         if (opc == LST_GROUP_RMND || opc == LST_GROUP_REFRESH) {
1902                 rc = lst_get_node_count(opc == LST_GROUP_RMND ? LST_OPC_NODES :
1903                                                                 LST_OPC_GROUP,
1904                                         opc == LST_GROUP_RMND ? str : grp,
1905                                         &count, &ids);
1906
1907                 if (rc != 0) {
1908                         fprintf(stderr, "Can't get count of nodes from %s: %s\n",
1909                                 opc == LST_GROUP_RMND ? str : grp,
1910                                 strerror(errno));
1911                         return -1;
1912                 }
1913
1914                 rc = lst_alloc_rpcent(&head, count, 0);
1915                 if (rc != 0) {
1916                         fprintf(stderr, "Out of memory\n");
1917                         free(ids);
1918                         return -1;
1919                 }
1920
1921         }
1922
1923         rc = lst_update_group_ioctl(opc, grp, clean, count, ids, &head);
1924
1925         if (ids != NULL)
1926                 free(ids);
1927
1928         if (rc == 0) {
1929                 lst_free_rpcent(&head);
1930                 return 0;
1931         }
1932
1933         if (rc == -1) {
1934                 lst_free_rpcent(&head);
1935                 lst_print_error("group", "Failed to update group: %s\n",
1936                                 strerror(errno));
1937                 return rc;
1938         }
1939
1940         lst_print_transerr(&head, "Updating group");
1941
1942         lst_free_rpcent(&head);
1943
1944         return rc;
1945 }
1946
1947 int
1948 lst_list_group_ioctl(int len, char *name, int idx)
1949 {
1950         struct lstio_group_list_args args = { 0 };
1951
1952         args.lstio_grp_key   = session_key;
1953         args.lstio_grp_idx   = idx;
1954         args.lstio_grp_nmlen = len;
1955         args.lstio_grp_namep = name;
1956
1957         return lst_ioctl(LSTIO_GROUP_LIST, &args, sizeof(args));
1958 }
1959
1960 int
1961 lst_info_group_ioctl(char *name, struct lstcon_ndlist_ent *gent,
1962                      int *idx, int *count, struct lstcon_node_ent *dents)
1963 {
1964         struct lstio_group_info_args args = { 0 };
1965
1966         args.lstio_grp_key    = session_key;
1967         args.lstio_grp_nmlen  = strlen(name);
1968         args.lstio_grp_namep  = name;
1969         args.lstio_grp_entp   = gent;
1970         args.lstio_grp_idxp   = idx;
1971         args.lstio_grp_ndentp = count;
1972         args.lstio_grp_dentsp = dents;
1973
1974         return lst_ioctl(LSTIO_GROUP_INFO, &args, sizeof(args));
1975 }
1976
1977 int
1978 lst_list_group_all(void)
1979 {
1980         char  name[LST_NAME_SIZE];
1981         int   rc;
1982         int   i;
1983
1984         /* no group is specified, list name of all groups */
1985         for (i = 0; ; i++) {
1986                 rc = lst_list_group_ioctl(LST_NAME_SIZE, name, i);
1987                 if (rc == 0) {
1988                         fprintf(stdout, "%d) %s\n", i + 1, name);
1989                         continue;
1990                 }
1991
1992                 if (errno == ENOENT)
1993                         break;
1994
1995                 lst_print_error("group", "Failed to list group: %s\n",
1996                                 strerror(errno));
1997                 return -1;
1998         }
1999
2000         fprintf(stdout, "Total %d groups\n", i);
2001
2002         return 0;
2003 }
2004
2005 int
2006 jt_lst_list_group(int argc, char **argv)
2007 {
2008         struct lstcon_ndlist_ent gent;
2009         struct lstcon_node_ent   *dents;
2010         int states = 0;
2011         int optidx  = 0;
2012         int verbose = 0;
2013         int active  = 0;
2014         int busy    = 0;
2015         int down    = 0;
2016         int unknown = 0;
2017         int all     = 0;
2018         int count;
2019         int index;
2020         int i;
2021         int j;
2022         int c;
2023         int rc      = 0;
2024         static const struct option list_group_opts[] = {
2025                 { .name = "active",  .has_arg = no_argument, .val = 'a' },
2026                 { .name = "busy",    .has_arg = no_argument, .val = 'b' },
2027                 { .name = "down",    .has_arg = no_argument, .val = 'd' },
2028                 { .name = "unknown", .has_arg = no_argument, .val = 'u' },
2029                 { .name = "all",     .has_arg = no_argument, .val = 'l' },
2030                 { .name = NULL, } };
2031
2032         if (session_key == 0) {
2033                 fprintf(stderr,
2034                         "Can't find env LST_SESSION or value is not valid\n");
2035                 return -1;
2036         }
2037
2038         while (1) {
2039                 c = getopt_long(argc, argv, "abdul",
2040                                 list_group_opts, &optidx);
2041
2042                 if (c == -1)
2043                         break;
2044
2045                 switch (c) {
2046                 case 'a':
2047                         verbose = active = 1;
2048                         states |= LST_NODE_ACTIVE;
2049                         all = 0;
2050                         break;
2051                 case 'b':
2052                         verbose = busy = 1;
2053                         states |= LST_NODE_BUSY;
2054                         all = 0;
2055                         break;
2056                 case 'd':
2057                         verbose = down = 1;
2058                         states |= LST_NODE_DOWN;
2059                         all = 0;
2060                         break;
2061                 case 'u':
2062                         verbose = unknown = 1;
2063                         states |= LST_NODE_UNKNOWN;
2064                         all = 0;
2065                         break;
2066                 case 'l':
2067                         states |= LST_NODE_ACTIVE | LST_NODE_BUSY |
2068                                   LST_NODE_DOWN | LST_NODE_UNKNOWN;
2069                         verbose = all = 1;
2070                         break;
2071                 default:
2072                         lst_print_usage(argv[0]);
2073                         return -1;
2074                 }
2075         }
2076
2077         if (optind == argc) {
2078                 rc = lst_yaml_groups(NLM_F_DUMP, NULL, 0, true);
2079                 if (rc <= 0) {
2080                         if (rc == -EOPNOTSUPP)
2081                                 goto old_api;
2082                         return rc;
2083                 }
2084         } else {
2085                 for (i = optind; i < argc; i++) {
2086                         rc = lst_yaml_groups(NLM_F_DUMP, argv[i], states,
2087                                              i == optind ? true : false);
2088                         if (rc < 0) {
2089                                 if (rc == -EOPNOTSUPP)
2090                                         goto old_api;
2091                                 return rc;
2092                         }
2093                 }
2094                 return 0;
2095         }
2096 old_api:
2097         if (optind == argc) {
2098                 /* no group is specified, list name of all groups */
2099                 rc = lst_list_group_all();
2100
2101                 return rc;
2102         }
2103
2104         if (!verbose)
2105                 fprintf(stdout, LST_NODES_TITLE);
2106
2107         /* list nodes in specified groups */
2108         for (i = optind; i < argc; i++) {
2109                 rc = lst_info_group_ioctl(argv[i], &gent, NULL, NULL, NULL);
2110                 if (rc != 0) {
2111                         if (errno == ENOENT) {
2112                                 rc = 0;
2113                                 break;
2114                         }
2115
2116                         lst_print_error("group", "Failed to list group\n",
2117                                         strerror(errno));
2118                         break;
2119                 }
2120
2121                 if (!verbose) {
2122                         fprintf(stdout, "\t%d\t%d\t%d\t%d\t%d\t%s\n",
2123                                 gent.nle_nactive, gent.nle_nbusy,
2124                                 gent.nle_ndown, gent.nle_nunknown,
2125                                 gent.nle_nnode, argv[i]);
2126                         continue;
2127                 }
2128
2129                 fprintf(stdout, "Group [ %s ]\n", argv[i]);
2130
2131                 if (gent.nle_nnode == 0) {
2132                         fprintf(stdout, "No nodes found [ %s ]\n", argv[i]);
2133                         continue;
2134                 }
2135
2136                 count = gent.nle_nnode;
2137
2138                 dents = malloc(count * sizeof(struct lstcon_node_ent));
2139                 if (dents == NULL) {
2140                         fprintf(stderr, "Failed to malloc: %s\n",
2141                                 strerror(errno));
2142                         return -1;
2143                 }
2144
2145                 index = 0;
2146                 rc = lst_info_group_ioctl(argv[i], &gent, &index, &count, dents);
2147                 if (rc != 0) {
2148                         lst_print_error("group", "Failed to list group: %s\n",
2149                                         strerror(errno));
2150                         free(dents);
2151                         return -1;
2152                 }
2153
2154                 for (j = 0, c = 0; j < count; j++) {
2155                         if (all ||
2156                             ((active  &&  dents[j].nde_state == LST_NODE_ACTIVE) ||
2157                              (busy    &&  dents[j].nde_state == LST_NODE_BUSY)   ||
2158                              (down    &&  dents[j].nde_state == LST_NODE_DOWN)   ||
2159                              (unknown &&  dents[j].nde_state == LST_NODE_UNKNOWN))) {
2160
2161                                 fprintf(stdout, "\t%s: %s\n",
2162                                         libcfs_id2str(dents[j].nde_id),
2163                                         lst_node_state2str(dents[j].nde_state));
2164                                 c++;
2165                         }
2166                 }
2167
2168                 fprintf(stdout, "Total %d nodes [ %s ]\n", c, argv[i]);
2169
2170                 free(dents);
2171         }
2172
2173         return rc;
2174 }
2175
2176 int
2177 lst_stat_ioctl(char *name, int count, struct lnet_process_id *idsp,
2178                int timeout, struct list_head *resultp)
2179 {
2180         struct lstio_stat_args args = { 0 };
2181
2182         args.lstio_sta_key     = session_key;
2183         args.lstio_sta_timeout = timeout;
2184         args.lstio_sta_nmlen   = strlen(name);
2185         args.lstio_sta_namep   = name;
2186         args.lstio_sta_count   = count;
2187         args.lstio_sta_idsp    = idsp;
2188         args.lstio_sta_resultp = resultp;
2189
2190         return lst_ioctl(LSTIO_STAT_QUERY, &args, sizeof(args));
2191 }
2192
2193 typedef struct {
2194         struct list_head              srp_link;
2195         int                     srp_count;
2196         char                   *srp_name;
2197         struct lnet_process_id      *srp_ids;
2198         struct list_head              srp_result[2];
2199 } lst_stat_req_param_t;
2200
2201 static void
2202 lst_stat_req_param_free(lst_stat_req_param_t *srp)
2203 {
2204         int     i;
2205
2206         for (i = 0; i < 2; i++)
2207                 lst_free_rpcent(&srp->srp_result[i]);
2208
2209         if (srp->srp_ids != NULL)
2210                 free(srp->srp_ids);
2211
2212         free(srp);
2213 }
2214
2215 static int
2216 lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp, int save_old)
2217 {
2218         lst_stat_req_param_t *srp = NULL;
2219         int                   count = save_old ? 2 : 1;
2220         int                   rc;
2221         int                   i;
2222
2223         srp = malloc(sizeof(*srp));
2224         if (srp == NULL)
2225                 return -ENOMEM;
2226
2227         memset(srp, 0, sizeof(*srp));
2228         INIT_LIST_HEAD(&srp->srp_result[0]);
2229         INIT_LIST_HEAD(&srp->srp_result[1]);
2230
2231         rc = lst_get_node_count(LST_OPC_GROUP, name,
2232                                 &srp->srp_count, NULL);
2233         if (rc != 0 && errno == ENOENT) {
2234                 rc = lst_get_node_count(LST_OPC_NODES, name,
2235                                         &srp->srp_count, &srp->srp_ids);
2236         }
2237
2238         if (rc != 0) {
2239                 fprintf(stderr,
2240                         "Failed to get count of nodes from %s: %s\n",
2241                         name, strerror(errno));
2242                 lst_stat_req_param_free(srp);
2243
2244                 return rc;
2245         }
2246
2247         srp->srp_name = name;
2248
2249         for (i = 0; i < count; i++) {
2250                 rc = lst_alloc_rpcent(&srp->srp_result[i], srp->srp_count,
2251                                       sizeof(struct sfw_counters)  +
2252                                       sizeof(struct srpc_counters) +
2253                                       sizeof(struct lnet_counters_common));
2254                 if (rc != 0) {
2255                         fprintf(stderr, "Out of memory\n");
2256                         break;
2257                 }
2258         }
2259
2260         if (rc == 0) {
2261                 *srpp = srp;
2262                 return 0;
2263         }
2264
2265         lst_stat_req_param_free(srp);
2266
2267         return rc;
2268 }
2269
2270 typedef struct {
2271         /* TODO */
2272         int foo;
2273 } lst_srpc_stat_result;
2274
2275 #define LST_LNET_AVG    0
2276 #define LST_LNET_MIN    1
2277 #define LST_LNET_MAX    2
2278
2279 typedef struct {
2280         float           lnet_avg_sndrate;
2281         float           lnet_min_sndrate;
2282         float           lnet_max_sndrate;
2283         float           lnet_total_sndrate;
2284
2285         float           lnet_avg_rcvrate;
2286         float           lnet_min_rcvrate;
2287         float           lnet_max_rcvrate;
2288         float           lnet_total_rcvrate;
2289
2290         float           lnet_avg_sndperf;
2291         float           lnet_min_sndperf;
2292         float           lnet_max_sndperf;
2293         float           lnet_total_sndperf;
2294
2295         float           lnet_avg_rcvperf;
2296         float           lnet_min_rcvperf;
2297         float           lnet_max_rcvperf;
2298         float           lnet_total_rcvperf;
2299
2300         int             lnet_stat_count;
2301 } lst_lnet_stat_result_t;
2302
2303 lst_lnet_stat_result_t lnet_stat_result;
2304
2305 static float
2306 lst_lnet_stat_value(int bw, int send, int off)
2307 {
2308         float  *p;
2309
2310         p = bw ? &lnet_stat_result.lnet_avg_sndperf :
2311                  &lnet_stat_result.lnet_avg_sndrate;
2312
2313         if (!send)
2314                 p += 4;
2315
2316         p += off;
2317
2318         return *p;
2319 }
2320
2321 static void
2322 lst_cal_lnet_stat(float delta, struct lnet_counters_common *lnet_new,
2323                   struct lnet_counters_common *lnet_old, int mbs)
2324 {
2325         float perf;
2326         float rate;
2327         unsigned int unit_divisor;
2328
2329         unit_divisor = (mbs) ? (1000 * 1000) : (1024 * 1024);
2330         perf = (float)(lnet_new->lcc_send_length -
2331                        lnet_old->lcc_send_length) / unit_divisor / delta;
2332         lnet_stat_result.lnet_total_sndperf += perf;
2333
2334         if (lnet_stat_result.lnet_min_sndperf > perf ||
2335             lnet_stat_result.lnet_min_sndperf == 0)
2336                 lnet_stat_result.lnet_min_sndperf = perf;
2337
2338         if (lnet_stat_result.lnet_max_sndperf < perf)
2339                 lnet_stat_result.lnet_max_sndperf = perf;
2340
2341         perf = (float)(lnet_new->lcc_recv_length -
2342                        lnet_old->lcc_recv_length) / unit_divisor / delta;
2343         lnet_stat_result.lnet_total_rcvperf += perf;
2344
2345         if (lnet_stat_result.lnet_min_rcvperf > perf ||
2346             lnet_stat_result.lnet_min_rcvperf == 0)
2347                 lnet_stat_result.lnet_min_rcvperf = perf;
2348
2349         if (lnet_stat_result.lnet_max_rcvperf < perf)
2350                 lnet_stat_result.lnet_max_rcvperf = perf;
2351
2352         rate = (lnet_new->lcc_send_count - lnet_old->lcc_send_count) / delta;
2353         lnet_stat_result.lnet_total_sndrate += rate;
2354
2355         if (lnet_stat_result.lnet_min_sndrate > rate ||
2356             lnet_stat_result.lnet_min_sndrate == 0)
2357                 lnet_stat_result.lnet_min_sndrate = rate;
2358
2359         if (lnet_stat_result.lnet_max_sndrate < rate)
2360                 lnet_stat_result.lnet_max_sndrate = rate;
2361
2362         rate = (lnet_new->lcc_recv_count - lnet_old->lcc_recv_count) / delta;
2363         lnet_stat_result.lnet_total_rcvrate += rate;
2364
2365         if (lnet_stat_result.lnet_min_rcvrate > rate ||
2366             lnet_stat_result.lnet_min_rcvrate == 0)
2367                 lnet_stat_result.lnet_min_rcvrate = rate;
2368
2369         if (lnet_stat_result.lnet_max_rcvrate < rate)
2370                 lnet_stat_result.lnet_max_rcvrate = rate;
2371
2372         lnet_stat_result.lnet_stat_count++;
2373
2374         lnet_stat_result.lnet_avg_sndrate = lnet_stat_result.lnet_total_sndrate /
2375                                             lnet_stat_result.lnet_stat_count;
2376         lnet_stat_result.lnet_avg_rcvrate = lnet_stat_result.lnet_total_rcvrate /
2377                                             lnet_stat_result.lnet_stat_count;
2378
2379         lnet_stat_result.lnet_avg_sndperf = lnet_stat_result.lnet_total_sndperf /
2380                                             lnet_stat_result.lnet_stat_count;
2381         lnet_stat_result.lnet_avg_rcvperf = lnet_stat_result.lnet_total_rcvperf /
2382                                             lnet_stat_result.lnet_stat_count;
2383 }
2384
2385 static void
2386 lst_print_lnet_stat(char *name, int bwrt, int rdwr, int type, int mbs)
2387 {
2388         int     start1 = 0;
2389         int     end1   = 1;
2390         int     start2 = 0;
2391         int     end2   = 1;
2392         int     i;
2393         int     j;
2394         char   *units;
2395
2396         if (lnet_stat_result.lnet_stat_count == 0)
2397                 return;
2398
2399         units = (mbs) ? "MB/s  " : "MiB/s ";
2400
2401         if (bwrt == 1) /* bw only */
2402                 start1 = 1;
2403
2404         if (bwrt == 2) /* rates only */
2405                 end1 = 0;
2406
2407         if (rdwr == 1) /* recv only */
2408                 start2 = 1;
2409
2410         if (rdwr == 2) /* send only */
2411                 end2 = 0;
2412
2413         for (i = start1; i <= end1; i++) {
2414                 fprintf(stdout, "[LNet %s of %s]\n",
2415                         i == 0 ? "Rates" : "Bandwidth", name);
2416
2417                 for (j = start2; j <= end2; j++) {
2418                         fprintf(stdout, "[%c] ", j == 0 ? 'R' : 'W');
2419
2420                         if ((type & 1) != 0) {
2421                                 fprintf(stdout, i == 0 ? "Avg: %-8.0f RPC/s " :
2422                                                          "Avg: %-8.2f %s",
2423                                         lst_lnet_stat_value(i, j, 0), units);
2424                         }
2425
2426                         if ((type & 2) != 0) {
2427                                 fprintf(stdout, i == 0 ? "Min: %-8.0f RPC/s " :
2428                                                          "Min: %-8.2f %s",
2429                                         lst_lnet_stat_value(i, j, 1), units);
2430                         }
2431
2432                         if ((type & 4) != 0) {
2433                                 fprintf(stdout, i == 0 ? "Max: %-8.0f RPC/s" :
2434                                                          "Max: %-8.2f %s",
2435                                         lst_lnet_stat_value(i, j, 2), units);
2436                         }
2437
2438                         fprintf(stdout, "\n");
2439                 }
2440         }
2441 }
2442
2443 static void
2444 lst_print_stat(char *name, struct list_head *resultp,
2445                int idx, int lnet, int bwrt, int rdwr, int type,
2446                int mbs)
2447 {
2448         struct list_head tmp[2];
2449         struct lstcon_rpc_ent *new;
2450         struct lstcon_rpc_ent *old;
2451         struct sfw_counters *sfwk_new;
2452         struct sfw_counters *sfwk_old;
2453         struct srpc_counters *srpc_new;
2454         struct srpc_counters *srpc_old;
2455         struct lnet_counters_common *lnet_new;
2456         struct lnet_counters_common *lnet_old;
2457         float delta;
2458         int errcount = 0;
2459
2460         INIT_LIST_HEAD(&tmp[0]);
2461         INIT_LIST_HEAD(&tmp[1]);
2462
2463         memset(&lnet_stat_result, 0, sizeof(lnet_stat_result));
2464
2465         while (!list_empty(&resultp[idx])) {
2466                 if (list_empty(&resultp[1 - idx])) {
2467                         fprintf(stderr, "Group is changed, re-run stat\n");
2468                         break;
2469                 }
2470
2471                 new = list_entry(resultp[idx].next, struct lstcon_rpc_ent,
2472                                      rpe_link);
2473                 old = list_entry(resultp[1 - idx].next, struct lstcon_rpc_ent,
2474                                      rpe_link);
2475
2476                 /* first time get stats result, can't calculate diff */
2477                 if (new->rpe_peer.nid == LNET_NID_ANY)
2478                         break;
2479
2480                 if (new->rpe_peer.nid != old->rpe_peer.nid ||
2481                     new->rpe_peer.pid != old->rpe_peer.pid) {
2482                         /* Something wrong. i.e, somebody change the group */
2483                         break;
2484                 }
2485
2486                 list_move_tail(&new->rpe_link, &tmp[idx]);
2487
2488                 list_move_tail(&old->rpe_link, &tmp[1 - idx]);
2489
2490                 if (new->rpe_rpc_errno != 0 || new->rpe_fwk_errno != 0 ||
2491                     old->rpe_rpc_errno != 0 || old->rpe_fwk_errno != 0) {
2492                         errcount ++;
2493                         continue;
2494                 }
2495
2496                 sfwk_new = (struct sfw_counters *)&new->rpe_payload[0];
2497                 sfwk_old = (struct sfw_counters *)&old->rpe_payload[0];
2498
2499                 srpc_new = (struct srpc_counters *)((char *)sfwk_new +
2500                                                     sizeof(*sfwk_new));
2501                 srpc_old = (struct srpc_counters *)((char *)sfwk_old +
2502                                                     sizeof(*sfwk_old));
2503
2504                 lnet_new = (struct lnet_counters_common *)((char *)srpc_new +
2505                                                            sizeof(*srpc_new));
2506                 lnet_old = (struct lnet_counters_common *)((char *)srpc_old +
2507                                                            sizeof(*srpc_old));
2508
2509                 /* Prior to version 2.3, the running_ms was a counter for
2510                  * the number of running tests. Since 2.3, running_ms is
2511                  * changed to hold the millisecond since the start of
2512                  * the work item. The rpe_stamp field was formerly used,
2513                  * but is no longer. In 2.12 rpe_stamp was changed to
2514                  * struct timespec64 and has nanosecond resolution, in
2515                  * case it is needed in the future.
2516                  */
2517                 delta = (float)(sfwk_new->running_ms -
2518                                 sfwk_old->running_ms) / 1000;
2519
2520                 if (!lnet) /* TODO */
2521                         continue;
2522
2523                 lst_cal_lnet_stat(delta, lnet_new, lnet_old, mbs);
2524         }
2525
2526         list_splice(&tmp[idx], &resultp[idx]);
2527         list_splice(&tmp[1 - idx], &resultp[1 - idx]);
2528
2529         if (errcount > 0)
2530                 fprintf(stdout, "Failed to stat on %d nodes\n", errcount);
2531
2532         if (!lnet)  /* TODO */
2533                 return;
2534
2535         lst_print_lnet_stat(name, bwrt, rdwr, type, mbs);
2536 }
2537
2538 int
2539 jt_lst_stat(int argc, char **argv)
2540 {
2541         struct list_head        head;
2542         lst_stat_req_param_t *srp;
2543         time_t                last    = 0;
2544         int                   optidx  = 0;
2545         int                   timeout = 5; /* default timeout, 5 sec */
2546         int                   delay   = 5; /* default delay, 5 sec */
2547         int                   count   = -1; /* run forever */
2548         int                   lnet    = 1; /* lnet stat by default */
2549         int                   bwrt    = 0;
2550         int                   rdwr    = 0;
2551         int                   type    = -1;
2552         int                   idx     = 0;
2553         int                   rc;
2554         int                   c;
2555         int                   mbs     = 0; /* report as MB/s */
2556
2557         static const struct option stat_opts[] = {
2558                 { .name = "timeout", .has_arg = required_argument, .val = 't' },
2559                 { .name = "delay",   .has_arg = required_argument, .val = 'd' },
2560                 { .name = "count",   .has_arg = required_argument, .val = 'o' },
2561                 { .name = "lnet",    .has_arg = no_argument,       .val = 'l' },
2562                 { .name = "rpc",     .has_arg = no_argument,       .val = 'c' },
2563                 { .name = "bw",      .has_arg = no_argument,       .val = 'b' },
2564                 { .name = "rate",    .has_arg = no_argument,       .val = 'a' },
2565                 { .name = "read",    .has_arg = no_argument,       .val = 'r' },
2566                 { .name = "write",   .has_arg = no_argument,       .val = 'w' },
2567                 { .name = "avg",     .has_arg = no_argument,       .val = 'g' },
2568                 { .name = "min",     .has_arg = no_argument,       .val = 'n' },
2569                 { .name = "max",     .has_arg = no_argument,       .val = 'x' },
2570                 { .name = "mbs",     .has_arg = no_argument,       .val = 'm' },
2571                 { .name = NULL } };
2572
2573         if (session_key == 0) {
2574                 fprintf(stderr,
2575                         "Can't find env LST_SESSION or value is not valid\n");
2576                 return -1;
2577         }
2578
2579         while (1) {
2580                 c = getopt_long(argc, argv, "t:d:lcbarwgnxm", stat_opts,
2581                                 &optidx);
2582
2583                 if (c == -1)
2584                         break;
2585
2586                 switch (c) {
2587                 case 't':
2588                         timeout = atoi(optarg);
2589                         break;
2590                 case 'd':
2591                         delay = atoi(optarg);
2592                         break;
2593                 case 'o':
2594                         count = atoi(optarg);
2595                         break;
2596                 case 'l':
2597                         lnet = 1;
2598                         break;
2599                 case 'c':
2600                         lnet = 0;
2601                         break;
2602                 case 'b':
2603                         bwrt |= 1;
2604                         break;
2605                 case 'a':
2606                         bwrt |= 2;
2607                         break;
2608                 case 'r':
2609                         rdwr |= 1;
2610                         break;
2611                 case 'w':
2612                         rdwr |= 2;
2613                         break;
2614                 case 'g':
2615                         if (type == -1) {
2616                                 type = 1;
2617                                 break;
2618                         }
2619                         type |= 1;
2620                         break;
2621                 case 'n':
2622                         if (type == -1) {
2623                                 type = 2;
2624                                 break;
2625                         }
2626                         type |= 2;
2627                         break;
2628                 case 'x':
2629                         if (type == -1) {
2630                                 type = 4;
2631                                 break;
2632                         }
2633                         type |= 4;
2634                         break;
2635                 case 'm':
2636                         mbs = 1;
2637                         break;
2638
2639                 default:
2640                         lst_print_usage(argv[0]);
2641                         return -1;
2642                 }
2643         }
2644
2645         if (optind == argc) {
2646                 lst_print_usage(argv[0]);
2647                 return -1;
2648         }
2649
2650         if (timeout <= 0 || delay <= 0) {
2651                 fprintf(stderr, "Invalid timeout or delay value\n");
2652                 return -1;
2653         }
2654
2655         if (count < -1) {
2656             fprintf(stderr, "Invalid count value\n");
2657             return -1;
2658         }
2659
2660         /* extra count to get first data point */
2661         if (count != -1)
2662             count++;
2663
2664         INIT_LIST_HEAD(&head);
2665
2666         while (optind < argc) {
2667                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 1);
2668                 if (rc != 0)
2669                         goto out;
2670
2671                 list_add_tail(&srp->srp_link, &head);
2672         }
2673
2674         do {
2675                 time_t  now = time(NULL);
2676
2677                 if (now - last < delay) {
2678                         sleep(delay - now + last);
2679                         time(&now);
2680                 }
2681                 last = now;
2682
2683                 list_for_each_entry(srp, &head, srp_link) {
2684                         rc = lst_stat_ioctl(srp->srp_name,
2685                                             srp->srp_count, srp->srp_ids,
2686                                             timeout, &srp->srp_result[idx]);
2687                         if (rc == -1) {
2688                                 lst_print_error("stat", "Failed to stat %s: %s\n",
2689                                                 srp->srp_name, strerror(errno));
2690                                 goto out;
2691                         }
2692
2693                         lst_print_stat(srp->srp_name, srp->srp_result,
2694                                        idx, lnet, bwrt, rdwr, type, mbs);
2695
2696                         lst_reset_rpcent(&srp->srp_result[1 - idx]);
2697                 }
2698
2699                 idx = 1 - idx;
2700
2701                 if (count > 0)
2702                         count--;
2703         } while (count == -1 || count > 0);
2704
2705 out:
2706         while (!list_empty(&head)) {
2707                 srp = list_entry(head.next, lst_stat_req_param_t, srp_link);
2708
2709                 list_del(&srp->srp_link);
2710                 lst_stat_req_param_free(srp);
2711         }
2712
2713         return rc;
2714 }
2715
2716 int
2717 jt_lst_show_error(int argc, char **argv)
2718 {
2719         struct list_head       head;
2720         lst_stat_req_param_t  *srp;
2721         struct lstcon_rpc_ent *ent;
2722         struct sfw_counters   *sfwk;
2723         struct srpc_counters  *srpc;
2724         int                    show_rpc = 1;
2725         int                    optidx = 0;
2726         int                    rc = 0;
2727         int                    ecount;
2728         int                    c;
2729
2730         static const struct option show_error_opts[] = {
2731                 { .name = "session", .has_arg = no_argument, .val = 's' },
2732                 { .name = NULL, } };
2733
2734         if (session_key == 0) {
2735                 fprintf(stderr,
2736                         "Can't find env LST_SESSION or value is not valid\n");
2737                 return -1;
2738         }
2739
2740         while (1) {
2741                 c = getopt_long(argc, argv, "s", show_error_opts, &optidx);
2742
2743                 if (c == -1)
2744                         break;
2745
2746                 switch (c) {
2747                 case 's':
2748                         show_rpc  = 0;
2749                         break;
2750
2751                 default:
2752                         lst_print_usage(argv[0]);
2753                         return -1;
2754                 }
2755         }
2756
2757         if (optind == argc) {
2758                 lst_print_usage(argv[0]);
2759                 return -1;
2760         }
2761
2762         INIT_LIST_HEAD(&head);
2763
2764         while (optind < argc) {
2765                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 0);
2766                 if (rc != 0)
2767                         goto out;
2768
2769                 list_add_tail(&srp->srp_link, &head);
2770         }
2771
2772         list_for_each_entry(srp, &head, srp_link) {
2773                 rc = lst_stat_ioctl(srp->srp_name, srp->srp_count,
2774                                     srp->srp_ids, 10, &srp->srp_result[0]);
2775
2776                 if (rc == -1) {
2777                         lst_print_error(srp->srp_name, "Failed to show errors of %s: %s\n",
2778                                         srp->srp_name, strerror(errno));
2779                         goto out;
2780                 }
2781
2782                 fprintf(stdout, "%s:\n", srp->srp_name);
2783
2784                 ecount = 0;
2785
2786                 list_for_each_entry(ent, &srp->srp_result[0], rpe_link) {
2787                         if (ent->rpe_rpc_errno != 0) {
2788                                 ecount ++;
2789                                 fprintf(stderr, "RPC failure, can't show error on %s\n",
2790                                         libcfs_id2str(ent->rpe_peer));
2791                                 continue;
2792                         }
2793
2794                         if (ent->rpe_fwk_errno != 0) {
2795                                 ecount ++;
2796                                 fprintf(stderr, "Framework failure, can't show error on %s\n",
2797                                         libcfs_id2str(ent->rpe_peer));
2798                                 continue;
2799                         }
2800
2801                         sfwk = (struct sfw_counters *)&ent->rpe_payload[0];
2802                         srpc = (struct srpc_counters *)((char *)sfwk + sizeof(*sfwk));
2803
2804                         if (srpc->errors == 0 &&
2805                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2806                                 continue;
2807
2808                         if (!show_rpc  &&
2809                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2810                                 continue;
2811
2812                         ecount ++;
2813
2814                         fprintf(stderr, "%s: [Session %d brw errors, %d ping errors]%c",
2815                                 libcfs_id2str(ent->rpe_peer),
2816                                 sfwk->brw_errors, sfwk->ping_errors,
2817                                 show_rpc  ? ' ' : '\n');
2818
2819                         if (!show_rpc)
2820                                 continue;
2821
2822                         fprintf(stderr, "[RPC: %d errors, %d dropped, %d expired]\n",
2823                                 srpc->errors, srpc->rpcs_dropped, srpc->rpcs_expired);
2824                 }
2825
2826                 fprintf(stdout, "Total %d error nodes in %s\n", ecount, srp->srp_name);
2827         }
2828 out:
2829         while (!list_empty(&head)) {
2830                 srp = list_entry(head.next, lst_stat_req_param_t, srp_link);
2831
2832                 list_del(&srp->srp_link);
2833                 lst_stat_req_param_free(srp);
2834         }
2835
2836         return rc;
2837 }
2838
2839 int
2840 lst_add_batch_ioctl(char *name)
2841 {
2842         struct lstio_batch_add_args args = { 0 };
2843
2844         args.lstio_bat_key   = session_key;
2845         args.lstio_bat_nmlen = strlen(name);
2846         args.lstio_bat_namep = name;
2847
2848         return lst_ioctl (LSTIO_BATCH_ADD, &args, sizeof(args));
2849 }
2850
2851 int
2852 jt_lst_add_batch(int argc, char **argv)
2853 {
2854         char   *name;
2855         int     rc;
2856
2857         if (session_key == 0) {
2858                 fprintf(stderr,
2859                         "Can't find env LST_SESSION or value is not valid\n");
2860                 return -1;
2861         }
2862
2863         if (argc != 2) {
2864                 lst_print_usage(argv[0]);
2865                 return -1;
2866         }
2867
2868         name = argv[1];
2869         if (strlen(name) >= LST_NAME_SIZE) {
2870                 fprintf(stderr, "Name length is limited to %d\n",
2871                         LST_NAME_SIZE - 1);
2872                 return -1;
2873         }
2874
2875         rc = lst_add_batch_ioctl(name);
2876         if (rc == 0)
2877                 return 0;
2878
2879         lst_print_error("batch", "Failed to create batch: %s\n",
2880                         strerror(errno));
2881
2882         return -1;
2883 }
2884
2885 int
2886 lst_start_batch_ioctl(char *name, int timeout, struct list_head *resultp)
2887 {
2888         struct lstio_batch_run_args args = { 0 };
2889
2890         args.lstio_bat_key     = session_key;
2891         args.lstio_bat_timeout = timeout;
2892         args.lstio_bat_nmlen   = strlen(name);
2893         args.lstio_bat_namep   = name;
2894         args.lstio_bat_resultp = resultp;
2895
2896         return lst_ioctl(LSTIO_BATCH_START, &args, sizeof(args));
2897 }
2898
2899 int
2900 jt_lst_start_batch(int argc, char **argv)
2901 {
2902         struct list_head  head;
2903         char             *batch;
2904         int               optidx = 0;
2905         int               timeout = 0;
2906         int               count = 0;
2907         int               rc;
2908         int               c;
2909
2910         static const struct option start_batch_opts[] = {
2911                 { .name = "timeout", .has_arg = required_argument, .val = 't' },
2912                 { .name = NULL } };
2913
2914         if (session_key == 0) {
2915                 fprintf(stderr,
2916                         "Can't find env LST_SESSION or value is not valid\n");
2917                 return -1;
2918         }
2919
2920         while (1) {
2921                 c = getopt_long(argc, argv, "t:",
2922                                 start_batch_opts, &optidx);
2923
2924                 /* Detect the end of the options. */
2925                 if (c == -1)
2926                         break;
2927
2928                 switch (c) {
2929                 case 't':
2930                         timeout = atoi(optarg);
2931                         break;
2932                 default:
2933                         lst_print_usage(argv[0]);
2934                         return -1;
2935                 }
2936         }
2937
2938         if (optind == argc) {
2939                 batch = LST_DEFAULT_BATCH;
2940
2941         } else if (optind == argc - 1) {
2942                 batch = argv[optind];
2943
2944         } else {
2945                 lst_print_usage(argv[0]);
2946                 return -1;
2947         }
2948
2949         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2950         if (rc != 0) {
2951                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2952                         batch, strerror(errno));
2953                 return -1;
2954         }
2955
2956         INIT_LIST_HEAD(&head);
2957
2958         rc = lst_alloc_rpcent(&head, count, 0);
2959         if (rc != 0) {
2960                 fprintf(stderr, "Out of memory\n");
2961                 return -1;
2962         }
2963
2964         rc = lst_start_batch_ioctl(batch, timeout, &head);
2965
2966         if (rc == 0) {
2967                 fprintf(stdout, "%s is running now\n", batch);
2968                 lst_free_rpcent(&head);
2969                 return 0;
2970         }
2971
2972         if (rc == -1) {
2973                 lst_print_error("batch", "Failed to start batch: %s\n",
2974                                 strerror(errno));
2975                 lst_free_rpcent(&head);
2976                 return rc;
2977         }
2978
2979         lst_print_transerr(&head, "Run batch");
2980
2981         lst_free_rpcent(&head);
2982
2983         return rc;
2984 }
2985
2986 int
2987 lst_stop_batch_ioctl(char *name, int force, struct list_head *resultp)
2988 {
2989         struct lstio_batch_stop_args args = { 0 };
2990
2991         args.lstio_bat_key     = session_key;
2992         args.lstio_bat_force   = force;
2993         args.lstio_bat_nmlen   = strlen(name);
2994         args.lstio_bat_namep   = name;
2995         args.lstio_bat_resultp = resultp;
2996
2997         return lst_ioctl(LSTIO_BATCH_STOP, &args, sizeof(args));
2998 }
2999
3000 int
3001 jt_lst_stop_batch(int argc, char **argv)
3002 {
3003         struct list_head  head;
3004         char             *batch;
3005         int               force = 0;
3006         int               optidx;
3007         int               count;
3008         int               rc;
3009         int               c;
3010
3011         static const struct option stop_batch_opts[] = {
3012                 { .name = "force", .has_arg = no_argument, .val = 'f' },
3013                 { .name = NULL } };
3014
3015         if (session_key == 0) {
3016                 fprintf(stderr,
3017                         "Can't find env LST_SESSION or value is not valid\n");
3018                 return -1;
3019         }
3020
3021         while (1) {
3022                 c = getopt_long(argc, argv, "f",
3023                                 stop_batch_opts, &optidx);
3024
3025                 /* Detect the end of the options. */
3026                 if (c == -1)
3027                         break;
3028
3029                 switch (c) {
3030                 case 'f':
3031                         force = 1;
3032                         break;
3033                 default:
3034                         lst_print_usage(argv[0]);
3035                         return -1;
3036                 }
3037         }
3038
3039         if (optind == argc) {
3040                 batch = LST_DEFAULT_BATCH;
3041
3042         } else if (optind == argc - 1) {
3043                 batch = argv[optind];
3044
3045         } else {
3046                 lst_print_usage(argv[0]);
3047                 return -1;
3048         }
3049
3050         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
3051         if (rc != 0) {
3052                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
3053                         batch, strerror(errno));
3054                 return -1;
3055         }
3056
3057         INIT_LIST_HEAD(&head);
3058
3059         rc = lst_alloc_rpcent(&head, count, 0);
3060         if (rc != 0) {
3061                 fprintf(stderr, "Out of memory\n");
3062                 return -1;
3063         }
3064
3065         rc = lst_stop_batch_ioctl(batch, force, &head);
3066         if (rc != 0)
3067                 goto out;
3068
3069         while (1) {
3070                 lst_reset_rpcent(&head);
3071
3072                 rc = lst_query_batch_ioctl(batch, 0, 0, 30, &head);
3073                 if (rc != 0)
3074                         goto out;
3075
3076                 if (lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0 &&
3077                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0)
3078                         break;
3079
3080                 fprintf(stdout, "%d batch in stopping\n",
3081                         lstcon_tsbqry_stat_run(&trans_stat, 0));
3082                 sleep(1);
3083         }
3084
3085         fprintf(stdout, "Batch is stopped\n");
3086         lst_free_rpcent(&head);
3087
3088         return 0;
3089 out:
3090         if (rc == -1) {
3091                 lst_print_error("batch", "Failed to stop batch: %s\n",
3092                                 strerror(errno));
3093                 lst_free_rpcent(&head);
3094                 return -1;
3095         }
3096
3097         lst_print_transerr(&head, "stop batch");
3098
3099         lst_free_rpcent(&head);
3100
3101         return rc;
3102 }
3103
3104 int
3105 lst_list_batch_ioctl(int len, char *name, int index)
3106 {
3107         struct lstio_batch_list_args args = { 0 };
3108
3109         args.lstio_bat_key   = session_key;
3110         args.lstio_bat_idx   = index;
3111         args.lstio_bat_nmlen = len;
3112         args.lstio_bat_namep = name;
3113
3114         return lst_ioctl(LSTIO_BATCH_LIST, &args, sizeof(args));
3115 }
3116
3117 int
3118 lst_info_batch_ioctl(char *batch, int test, int server,
3119                      struct lstcon_test_batch_ent *entp, int *idxp,
3120                      int *ndentp, struct lstcon_node_ent *dentsp)
3121 {
3122         struct lstio_batch_info_args args = { 0 };
3123
3124         args.lstio_bat_key     = session_key;
3125         args.lstio_bat_nmlen   = strlen(batch);
3126         args.lstio_bat_namep   = batch;
3127         args.lstio_bat_server  = server;
3128         args.lstio_bat_testidx = test;
3129         args.lstio_bat_entp    = entp;
3130         args.lstio_bat_idxp    = idxp;
3131         args.lstio_bat_ndentp  = ndentp;
3132         args.lstio_bat_dentsp  = dentsp;
3133
3134         return lst_ioctl(LSTIO_BATCH_INFO, &args, sizeof(args));
3135 }
3136
3137 int
3138 lst_list_batch_all(void)
3139 {
3140         char name[LST_NAME_SIZE];
3141         int  rc;
3142         int  i;
3143
3144         for (i = 0; ; i++) {
3145                 rc = lst_list_batch_ioctl(LST_NAME_SIZE, name, i);
3146                 if (rc == 0) {
3147                         fprintf(stdout, "%d) %s\n", i + 1, name);
3148                         continue;
3149                 }
3150
3151                 if (errno == ENOENT)
3152                         break;
3153
3154                 lst_print_error("batch", "Failed to list batch: %s\n",
3155                                 strerror(errno));
3156                 return rc;
3157         }
3158
3159         fprintf(stdout, "Total %d batches\n", i);
3160
3161         return 0;
3162 }
3163
3164 int
3165 lst_list_tsb_nodes(char *batch, int test, int server,
3166                    int count, int active, int invalid)
3167 {
3168         struct lstcon_node_ent *dents;
3169         int                index = 0;
3170         int                rc;
3171         int                c;
3172         int                i;
3173
3174         if (count == 0)
3175                 return 0;
3176
3177         /* verbose list, show nodes in batch or test */
3178         dents = malloc(count * sizeof(struct lstcon_node_ent));
3179         if (dents == NULL) {
3180                 fprintf(stdout, "Can't allocate memory\n");
3181                 return -1;
3182         }
3183
3184         rc = lst_info_batch_ioctl(batch, test, server,
3185                                   NULL, &index, &count, dents);
3186         if (rc != 0) {
3187                 free(dents);
3188                 lst_print_error((test > 0) ? "test" : "batch",
3189                                 (test > 0) ? "Failed to query test: %s\n" :
3190                                              "Failed to query batch: %s\n",
3191                                 strerror(errno));
3192                 return -1;
3193         }
3194
3195         for (i = 0, c = 0; i < count; i++) {
3196                 if ((!active  && dents[i].nde_state == LST_NODE_ACTIVE) ||
3197                     (!invalid && (dents[i].nde_state == LST_NODE_BUSY  ||
3198                                   dents[i].nde_state == LST_NODE_DOWN  ||
3199                                   dents[i].nde_state == LST_NODE_UNKNOWN)))
3200                         continue;
3201
3202                 fprintf(stdout, "\t%s: %s\n",
3203                         libcfs_id2str(dents[i].nde_id),
3204                         lst_node_state2str(dents[i].nde_state));
3205                 c++;
3206         }
3207
3208         fprintf(stdout, "Total %d nodes\n", c);
3209         free(dents);
3210
3211         return 0;
3212 }
3213
3214 int
3215 jt_lst_list_batch(int argc, char **argv)
3216 {
3217         struct lstcon_test_batch_ent ent;
3218         char *batch   = NULL;
3219         int   optidx  = 0;
3220         int   verbose = 0; /* list nodes in batch or test */
3221         int   invalid = 0;
3222         int   active  = 0;
3223         int   server  = 0;
3224         int   ntest   = 0;
3225         int   test    = 0;
3226         int   c       = 0;
3227         int   rc;
3228
3229         static const struct option list_batch_opts[] = {
3230                 { .name = "test",    .has_arg = required_argument, .val = 't' },
3231                 { .name = "invalid", .has_arg = no_argument,       .val = 'i' },
3232                 { .name = "active",  .has_arg = no_argument,       .val = 'a' },
3233                 { .name = "all",     .has_arg = no_argument,       .val = 'l' },
3234                 { .name = "server",  .has_arg = no_argument,       .val = 's' },
3235                 { .name = NULL, } };
3236
3237         if (session_key == 0) {
3238                 fprintf(stderr,
3239                         "Can't find env LST_SESSION or value is not valid\n");
3240                 return -1;
3241         }
3242
3243         while (1) {
3244                 c = getopt_long(argc, argv, "ailst:",
3245                                 list_batch_opts, &optidx);
3246
3247                 if (c == -1)
3248                         break;
3249
3250                 switch (c) {
3251                 case 'a':
3252                         verbose = active = 1;
3253                         break;
3254                 case 'i':
3255                         verbose = invalid = 1;
3256                         break;
3257                 case 'l':
3258                         verbose = active = invalid = 1;
3259                         break;
3260                 case 's':
3261                         server = 1;
3262                         break;
3263                 case 't':
3264                         test = atoi(optarg);
3265                         ntest = 1;
3266                         break;
3267                 default:
3268                         lst_print_usage(argv[0]);
3269                         return -1;
3270                 }
3271         }
3272
3273         if (optind == argc) {
3274                 /* list all batches */
3275                 rc = lst_list_batch_all();
3276                 return rc;
3277         }
3278
3279         if (ntest == 1 && test <= 0) {
3280                 fprintf(stderr, "Invalid test id, test id starts from 1\n");
3281                 return -1;
3282         }
3283
3284         if (optind != argc - 1) {
3285                 lst_print_usage(argv[0]);
3286                 return -1;
3287         }
3288
3289         batch = argv[optind];
3290
3291 loop:
3292         /* show detail of specified batch or test */
3293         rc = lst_info_batch_ioctl(batch, test, server,
3294                                   &ent, NULL, NULL, NULL);
3295         if (rc != 0) {
3296                 lst_print_error((test > 0) ? "test" : "batch",
3297                                 (test > 0) ? "Failed to query test: %s\n" :
3298                                              "Failed to query batch: %s\n",
3299                                 strerror(errno));
3300                 return -1;
3301         }
3302
3303         if (verbose) {
3304                 /* list nodes in test or batch */
3305                 rc = lst_list_tsb_nodes(batch, test, server,
3306                                         server ? ent.tbe_srv_nle.nle_nnode :
3307                                                  ent.tbe_cli_nle.nle_nnode,
3308                                         active, invalid);
3309                 return rc;
3310         }
3311
3312         /* only show number of hosts in batch or test */
3313         if (test == 0) {
3314                 fprintf(stdout, "Batch: %s Tests: %d State: %d\n",
3315                         batch, ent.u.tbe_batch.bae_ntest,
3316                         ent.u.tbe_batch.bae_state);
3317                 ntest = ent.u.tbe_batch.bae_ntest;
3318                 test = 1; /* starting from test 1 */
3319
3320         } else {
3321                 fprintf(stdout,
3322                         "\tTest %d(%s) (loop: %d, concurrency: %d)\n",
3323                         test, lst_test_type2name(ent.u.tbe_test.tse_type),
3324                         ent.u.tbe_test.tse_loop,
3325                         ent.u.tbe_test.tse_concur);
3326                 ntest --;
3327                 test ++;
3328         }
3329
3330         fprintf(stdout, LST_NODES_TITLE);
3331         fprintf(stdout, "client\t%d\t%d\t%d\t%d\t%d\n"
3332                         "server\t%d\t%d\t%d\t%d\t%d\n",
3333                 ent.tbe_cli_nle.nle_nactive,
3334                 ent.tbe_cli_nle.nle_nbusy,
3335                 ent.tbe_cli_nle.nle_ndown,
3336                 ent.tbe_cli_nle.nle_nunknown,
3337                 ent.tbe_cli_nle.nle_nnode,
3338                 ent.tbe_srv_nle.nle_nactive,
3339                 ent.tbe_srv_nle.nle_nbusy,
3340                 ent.tbe_srv_nle.nle_ndown,
3341                 ent.tbe_srv_nle.nle_nunknown,
3342                 ent.tbe_srv_nle.nle_nnode);
3343
3344         if (ntest != 0)
3345                 goto loop;
3346
3347         return 0;
3348 }
3349
3350 int
3351 lst_query_batch_ioctl(char *batch, int test, int server,
3352                       int timeout, struct list_head *head)
3353 {
3354         struct lstio_batch_query_args args = { 0 };
3355
3356         args.lstio_bat_key     = session_key;
3357         args.lstio_bat_testidx = test;
3358         args.lstio_bat_client  = !(server);
3359         args.lstio_bat_timeout = timeout;
3360         args.lstio_bat_nmlen   = strlen(batch);
3361         args.lstio_bat_namep   = batch;
3362         args.lstio_bat_resultp = head;
3363
3364         return lst_ioctl(LSTIO_BATCH_QUERY, &args, sizeof(args));
3365 }
3366
3367 void
3368 lst_print_tsb_verbose(struct list_head *head,
3369                       int active, int idle, int error)
3370 {
3371         struct lstcon_rpc_ent *ent;
3372
3373         list_for_each_entry(ent, head, rpe_link) {
3374                 if (ent->rpe_priv[0] == 0 && active)
3375                         continue;
3376
3377                 if (ent->rpe_priv[0] != 0 && idle)
3378                         continue;
3379
3380                 if (ent->rpe_fwk_errno == 0 && error)
3381                         continue;
3382
3383                 fprintf(stdout, "%s [%s]: %s\n",
3384                         libcfs_id2str(ent->rpe_peer),
3385                         lst_node_state2str(ent->rpe_state),
3386                         ent->rpe_rpc_errno != 0 ?
3387                                 strerror(ent->rpe_rpc_errno) :
3388                                 (ent->rpe_priv[0] > 0 ? "Running" : "Idle"));
3389         }
3390 }
3391
3392 int
3393 jt_lst_query_batch(int argc, char **argv)
3394 {
3395         struct lstcon_test_batch_ent ent;
3396         struct list_head head;
3397         char   *batch   = NULL;
3398         time_t  last    = 0;
3399         int     optidx  = 0;
3400         int     verbose = 0;
3401         int     server  = 0;
3402         int     timeout = 5; /* default 5 seconds */
3403         int     delay   = 5; /* default 5 seconds */
3404         int     loop    = 1; /* default 1 loop */
3405         int     active  = 0;
3406         int     error   = 0;
3407         int     idle    = 0;
3408         int     count   = 0;
3409         int     test    = 0;
3410         int     rc      = 0;
3411         int     c       = 0;
3412         int     i;
3413
3414         static const struct option query_batch_opts[] = {
3415                 { .name = "timeout", .has_arg = required_argument, .val = 'o' },
3416                 { .name = "delay",   .has_arg = required_argument, .val = 'd' },
3417                 { .name = "loop",    .has_arg = required_argument, .val = 'c' },
3418                 { .name = "test",    .has_arg = required_argument, .val = 't' },
3419                 { .name = "server",  .has_arg = no_argument,       .val = 's' },
3420                 { .name = "active",  .has_arg = no_argument,       .val = 'a' },
3421                 { .name = "idle",    .has_arg = no_argument,       .val = 'i' },
3422                 { .name = "error",   .has_arg = no_argument,       .val = 'e' },
3423                 { .name = "all",     .has_arg = no_argument,       .val = 'l' },
3424                 { .name = NULL, } };
3425
3426         if (session_key == 0) {
3427                 fprintf(stderr,
3428                         "Can't find env LST_SESSION or value is not valid\n");
3429                 return -1;
3430         }
3431
3432         while (1) {
3433                 c = getopt_long(argc, argv, "o:d:c:t:saiel",
3434                                 query_batch_opts, &optidx);
3435
3436                 /* Detect the end of the options. */
3437                 if (c == -1)
3438                         break;
3439
3440                 switch (c) {
3441                 case 'o':
3442                         timeout = atoi(optarg);
3443                         break;
3444                 case 'd':
3445                         delay = atoi(optarg);
3446                         break;
3447                 case 'c':
3448                         loop = atoi(optarg);
3449                         break;
3450                 case 't':
3451                         test = atoi(optarg);
3452                         break;
3453                 case 's':
3454                         server = 1;
3455                         break;
3456                 case 'a':
3457                         active = verbose = 1;
3458                         break;
3459                 case 'i':
3460                         idle = verbose = 1;
3461                         break;
3462                 case 'e':
3463                         error = verbose = 1;
3464                         break;
3465                 case 'l':
3466                         verbose = 1;
3467                         break;
3468                 default:
3469                         lst_print_usage(argv[0]);
3470                         return -1;
3471                 }
3472         }
3473
3474         if (test < 0 || timeout <= 0 || delay <= 0 || loop <= 0) {
3475                 lst_print_usage(argv[0]);
3476                 return -1;
3477         }
3478
3479         if (optind == argc) {
3480                 batch = LST_DEFAULT_BATCH;
3481
3482         } else if (optind == argc - 1) {
3483                 batch = argv[optind];
3484
3485         } else {
3486                 lst_print_usage(argv[0]);
3487                 return -1;
3488         }
3489
3490
3491         INIT_LIST_HEAD(&head);
3492
3493         if (verbose) {
3494                 rc = lst_info_batch_ioctl(batch, test, server,
3495                                           &ent, NULL, NULL, NULL);
3496                 if (rc != 0) {
3497                         fprintf(stderr, "Failed to query %s [%d]: %s\n",
3498                                 batch, test, strerror(errno));
3499                         return -1;
3500                 }
3501
3502                 count = server ? ent.tbe_srv_nle.nle_nnode :
3503                                  ent.tbe_cli_nle.nle_nnode;
3504                 if (count == 0) {
3505                         fprintf(stdout, "Batch or test is empty\n");
3506                         return 0;
3507                 }
3508         }
3509
3510         rc = lst_alloc_rpcent(&head, count, 0);
3511         if (rc != 0) {
3512                 fprintf(stderr, "Out of memory\n");
3513                 return rc;
3514         }
3515
3516         for (i = 0; i < loop; i++) {
3517                 time_t  now = time(NULL);
3518
3519                 if (now - last < delay) {
3520                         sleep(delay - now + last);
3521                         time(&now);
3522                 }
3523
3524                 last = now;
3525
3526                 rc = lst_query_batch_ioctl(batch, test,
3527                                            server, timeout, &head);
3528                 if (rc == -1) {
3529                         fprintf(stderr, "Failed to query batch: %s\n",
3530                                 strerror(errno));
3531                         break;
3532                 }
3533
3534                 if (verbose) {
3535                         /* Verbose mode */
3536                         lst_print_tsb_verbose(&head, active, idle, error);
3537                         continue;
3538                 }
3539
3540                 fprintf(stdout, "%s [%d] ", batch, test);
3541
3542                 if (lstcon_rpc_stat_failure(&trans_stat, 0) != 0) {
3543                         fprintf(stdout, "%d of %d nodes are unknown, ",
3544                                 lstcon_rpc_stat_failure(&trans_stat, 0),
3545                                 lstcon_rpc_stat_total(&trans_stat, 0));
3546                 }
3547
3548                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
3549                     lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0  &&
3550                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
3551                         fprintf(stdout, "is stopped\n");
3552                         continue;
3553                 }
3554
3555                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
3556                     lstcon_tsbqry_stat_idle(&trans_stat, 0) == 0 &&
3557                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
3558                         fprintf(stdout, "is running\n");
3559                         continue;
3560                 }
3561
3562                 fprintf(stdout, "stopped: %d , running: %d, failed: %d\n",
3563                                 lstcon_tsbqry_stat_idle(&trans_stat, 0),
3564                                 lstcon_tsbqry_stat_run(&trans_stat, 0),
3565                                 lstcon_tsbqry_stat_failure(&trans_stat, 0));
3566         }
3567
3568         lst_free_rpcent(&head);
3569
3570         return rc;
3571 }
3572
3573 int
3574 lst_parse_distribute(char *dstr, int *dist, int *span)
3575 {
3576         *dist = atoi(dstr);
3577         if (*dist <= 0)
3578                 return -1;
3579
3580         dstr = strchr(dstr, ':');
3581         if (dstr == NULL)
3582                 return -1;
3583
3584         *span = atoi(dstr + 1);
3585         if (*span <= 0)
3586                 return -1;
3587
3588         return 0;
3589 }
3590
3591 int
3592 lst_get_bulk_param(int argc, char **argv, struct lst_test_bulk_param *bulk)
3593 {
3594         char   *tok = NULL;
3595         char   *end = NULL;
3596         int     rc  = 0;
3597         int     i   = 0;
3598
3599         bulk->blk_size  = 4096;
3600         bulk->blk_opc   = LST_BRW_READ;
3601         bulk->blk_flags = LST_BRW_CHECK_NONE;
3602         bulk->blk_srv_off = bulk->blk_cli_off = 0;
3603
3604         while (i < argc) {
3605                 if (strcasestr(argv[i], "check=") == argv[i] ||
3606                     strcasestr(argv[i], "c=") == argv[i]) {
3607                         tok = strchr(argv[i], '=') + 1;
3608
3609                         if (strcasecmp(tok, "full") == 0) {
3610                                 bulk->blk_flags = LST_BRW_CHECK_FULL;
3611                         } else if (strcasecmp(tok, "simple") == 0) {
3612                                 bulk->blk_flags = LST_BRW_CHECK_SIMPLE;
3613                         } else {
3614                                 fprintf(stderr, "Unknow flag %s\n", tok);
3615                                 return -1;
3616                         }
3617
3618                 } else if (strcasestr(argv[i], "size=") == argv[i] ||
3619                            strcasestr(argv[i], "s=") == argv[i]) {
3620                         tok = strchr(argv[i], '=') + 1;
3621
3622                         bulk->blk_size = strtol(tok, &end, 0);
3623                         if (bulk->blk_size <= 0) {
3624                                 fprintf(stderr, "Invalid size %s\n", tok);
3625                                 return -1;
3626                         }
3627
3628                         if (end == NULL)
3629                                 return 0;
3630
3631                         if (*end == 'k' || *end == 'K')
3632                                 bulk->blk_size *= 1024;
3633                         else if (*end == 'm' || *end == 'M')
3634                                 bulk->blk_size *= 1024 * 1024;
3635
3636                         if (bulk->blk_size > LNET_MTU) {
3637                                 fprintf(stderr, "Size exceed limitation: %d bytes\n",
3638                                         bulk->blk_size);
3639                                 return -1;
3640                         }
3641
3642                 } else if (strcasestr(argv[i], "off=") == argv[i]) {
3643                         int     off;
3644
3645                         tok = strchr(argv[i], '=') + 1;
3646
3647                         off = strtol(tok, &end, 0);
3648                         /* NB: align with sizeof(__u64) to simplify page
3649                          * checking implementation */
3650                         if (off < 0 || off % sizeof(__u64) != 0) {
3651                                 fprintf(stderr,
3652                                         "Invalid offset %s/%d, it should be "
3653                                         "postive value and multiple of %d\n",
3654                                         tok, off, (int)sizeof(__u64));
3655                                 return -1;
3656                         }
3657
3658                         /* NB: blk_srv_off is reserved so far */
3659                         bulk->blk_cli_off = bulk->blk_srv_off = off;
3660                         if (end == NULL)
3661                                 return 0;
3662
3663                 } else if (strcasecmp(argv[i], "read") == 0 ||
3664                            strcasecmp(argv[i], "r") == 0) {
3665                         bulk->blk_opc = LST_BRW_READ;
3666
3667                 } else if (strcasecmp(argv[i], "write") == 0 ||
3668                            strcasecmp(argv[i], "w") == 0) {
3669                         bulk->blk_opc = LST_BRW_WRITE;
3670
3671                 } else {
3672                         fprintf(stderr, "Unknow parameter: %s\n", argv[i]);
3673                         return -1;
3674                 }
3675
3676                 i++;
3677         }
3678
3679         return rc;
3680 }
3681
3682 int
3683 lst_get_test_param(char *test, int argc, char **argv, void **param, int *plen)
3684 {
3685         struct lst_test_bulk_param *bulk = NULL;
3686         struct lst_test_ping_param *ping = NULL;
3687         int                    type;
3688
3689         type = lst_test_name2type(test);
3690         if (type < 0) {
3691                 fprintf(stderr, "Unknow test name %s\n", test);
3692                 return -1;
3693         }
3694
3695         switch (type) {
3696         case LST_TEST_PING:
3697                 /* unused but needs for kernel part */
3698                 ping = malloc(sizeof(*ping));
3699                 if (ping == NULL) {
3700                         fprintf(stderr, "Out of memory\n");
3701                         return -1;
3702                 }
3703                 memset(ping, 0, sizeof(*ping));
3704
3705                 *param = ping;
3706                 *plen  = sizeof(*ping);
3707
3708                 break;
3709
3710         case LST_TEST_BULK:
3711                 bulk = malloc(sizeof(*bulk));
3712                 if (bulk == NULL) {
3713                         fprintf(stderr, "Out of memory\n");
3714                         return -1;
3715                 }
3716
3717                 memset(bulk, 0, sizeof(*bulk));
3718
3719                 if (lst_get_bulk_param(argc, argv, bulk) != 0) {
3720                         free(bulk);
3721                         return -1;
3722                 }
3723
3724                 *param = bulk;
3725                 *plen  = sizeof(*bulk);
3726
3727                 break;
3728
3729         default:
3730                 break;
3731         }
3732
3733         /* TODO: parse more parameter */
3734         return type;
3735 }
3736
3737 int
3738 lst_add_test_ioctl(char *batch, int type, int loop, int concur,
3739                    int dist, int span, char *sgrp, char *dgrp,
3740                    void *param, int plen, int *retp, struct list_head *resultp)
3741 {
3742         struct lstio_test_args args = { 0 };
3743
3744         args.lstio_tes_key        = session_key;
3745         args.lstio_tes_bat_nmlen  = strlen(batch);
3746         args.lstio_tes_bat_name   = batch;
3747         args.lstio_tes_type       = type;
3748         args.lstio_tes_oneside    = 0;
3749         args.lstio_tes_loop       = loop;
3750         args.lstio_tes_concur     = concur;
3751         args.lstio_tes_dist       = dist;
3752         args.lstio_tes_span       = span;
3753         args.lstio_tes_sgrp_nmlen = strlen(sgrp);
3754         args.lstio_tes_sgrp_name  = sgrp;
3755         args.lstio_tes_dgrp_nmlen = strlen(dgrp);
3756         args.lstio_tes_dgrp_name  = dgrp;
3757         args.lstio_tes_param_len  = plen;
3758         args.lstio_tes_param      = param;
3759         args.lstio_tes_retp       = retp;
3760         args.lstio_tes_resultp    = resultp;
3761
3762         return lst_ioctl(LSTIO_TEST_ADD, &args, sizeof(args));
3763 }
3764
3765 int
3766 jt_lst_add_test(int argc, char **argv)
3767 {
3768         struct list_head head;
3769         char *batch  = NULL;
3770         char *test   = NULL;
3771         char *dstr   = NULL;
3772         char *from   = NULL;
3773         char *to     = NULL;
3774         void *param  = NULL;
3775         int   optidx = 0;
3776         int   concur = 1;
3777         int   loop   = -1;
3778         int   dist   = 1;
3779         int   span   = 1;
3780         int   plen   = 0;
3781         int   fcount = 0;
3782         int   tcount = 0;
3783         int   ret    = 0;
3784         int   type;
3785         int   rc;
3786         int   c;
3787
3788         static const struct option add_test_opts[] = {
3789         { .name = "batch",       .has_arg = required_argument, .val = 'b' },
3790         { .name = "concurrency", .has_arg = required_argument, .val = 'c' },
3791         { .name = "distribute",  .has_arg = required_argument, .val = 'd' },
3792         { .name = "from",        .has_arg = required_argument, .val = 'f' },
3793         { .name = "to",          .has_arg = required_argument, .val = 't' },
3794         { .name = "loop",        .has_arg = required_argument, .val = 'l' },
3795         { .name = NULL } };
3796
3797         if (session_key == 0) {
3798                 fprintf(stderr,
3799                         "Can't find env LST_SESSION or value is not valid\n");
3800                 return -1;
3801         }
3802
3803         while (1) {
3804                 c = getopt_long(argc, argv, "b:c:d:f:l:t:",
3805                                 add_test_opts, &optidx);
3806
3807                 /* Detect the end of the options. */
3808                 if (c == -1)
3809                         break;
3810
3811                 switch (c) {
3812                 case 'b':
3813                         batch = optarg;
3814                         break;
3815                 case 'c':
3816                         concur = atoi(optarg);
3817                         break;
3818                 case 'd':
3819                         dstr = optarg;
3820                         break;
3821                 case 'f':
3822                         from = optarg;
3823                         break;
3824                 case 'l':
3825                         loop = atoi(optarg);
3826                         break;
3827                 case 't':
3828                         to = optarg;
3829                         break;
3830                 default:
3831                         lst_print_usage(argv[0]);
3832                         return -1;
3833                 }
3834         }
3835
3836         if (optind == argc || from == NULL || to == NULL) {
3837                 lst_print_usage(argv[0]);
3838                 return -1;
3839         }
3840
3841         if (concur <= 0 || concur > LST_MAX_CONCUR) {
3842                 fprintf(stderr, "Invalid concurrency of test: %d\n", concur);
3843                 return -1;
3844         }
3845
3846         if (batch == NULL)
3847                 batch = LST_DEFAULT_BATCH;
3848
3849         if (dstr != NULL) {
3850                 rc = lst_parse_distribute(dstr, &dist, &span);
3851                 if (rc != 0) {
3852                         fprintf(stderr, "Invalid distribution: %s\n", dstr);
3853                         return -1;
3854                 }
3855         }
3856
3857         test = argv[optind++];
3858
3859         argc -= optind;
3860         argv += optind;
3861
3862         type = lst_get_test_param(test, argc, argv, &param, &plen);
3863         if (type < 0) {
3864                 fprintf(stderr, "Failed to add test (%s)\n", test);
3865                 return -1;
3866         }
3867
3868         INIT_LIST_HEAD(&head);
3869
3870         rc = lst_get_node_count(LST_OPC_GROUP, from, &fcount, NULL);
3871         if (rc != 0) {
3872                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3873                         from, strerror(errno));
3874                 goto out;
3875         }
3876
3877         rc = lst_get_node_count(LST_OPC_GROUP, to, &tcount, NULL);
3878         if (rc != 0) {
3879                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3880                         to, strerror(errno));
3881                 goto out;
3882         }
3883
3884         rc = lst_alloc_rpcent(&head, fcount > tcount ? fcount : tcount, 0);
3885         if (rc != 0) {
3886                 fprintf(stderr, "Out of memory\n");
3887                 goto out;
3888         }
3889
3890         rc = lst_add_test_ioctl(batch, type, loop, concur,
3891                                 dist, span, from, to, param, plen, &ret, &head);
3892
3893         if (rc == 0) {
3894                 fprintf(stdout, "Test was added successfully\n");
3895                 if (ret != 0) {
3896                         fprintf(stdout, "Server group contains userland test "
3897                                 "nodes, old version of tcplnd can't accept "
3898                                 "connection request\n");
3899                 }
3900
3901                 goto out;
3902         }
3903
3904         if (rc == -1) {
3905                 lst_print_error("test", "Failed to add test: %s\n",
3906                                 strerror(errno));
3907                 goto out;
3908         }
3909
3910         lst_print_transerr(&head, "add test");
3911 out:
3912         lst_free_rpcent(&head);
3913
3914         if (param != NULL)
3915                 free(param);
3916
3917         return rc;
3918 }
3919
3920 static command_t lst_cmdlist[] = {
3921         {"new_session",         jt_lst_new_session,     NULL,
3922          "Usage: lst new_session [--timeout TIME] [--force] [NAME]"                     },
3923         {"end_session",         jt_lst_end_session,     NULL,
3924          "Usage: lst end_session"                                                       },
3925         {"show_session",        jt_lst_show_session,    NULL,
3926          "Usage: lst show_session"                                                      },
3927         {"ping",                jt_lst_ping ,           NULL,
3928          "Usage: lst ping  [--group NAME] [--batch NAME] [--session] [--nodes IDS]"     },
3929         {"add_group",           jt_lst_add_group,       NULL,
3930          "Usage: lst group NAME IDs [IDs]..."                                           },
3931         {"del_group",           jt_lst_del_group,       NULL,
3932          "Usage: lst del_group NAME"                                                    },
3933         {"update_group",        jt_lst_update_group,    NULL,
3934          "Usage: lst update_group NAME [--clean] [--refresh] [--remove IDs]"            },
3935         {"list_group",          jt_lst_list_group,      NULL,
3936           "Usage: lst list_group [--active] [--busy] [--down] [--unknown] GROUP ..."    },
3937         {"stat",                jt_lst_stat,            NULL,
3938          "Usage: lst stat [--bw] [--rate] [--read] [--write] [--max] [--min] [--avg] "
3939          " [--mbs] [--timeout #] [--delay #] [--count #] GROUP [GROUP]"                 },
3940         {"show_error",          jt_lst_show_error,      NULL,
3941          "Usage: lst show_error NAME | IDS ..."                                         },
3942         {"add_batch",           jt_lst_add_batch,       NULL,
3943          "Usage: lst add_batch NAME"                                                    },
3944         {"run",                 jt_lst_start_batch,     NULL,
3945          "Usage: lst run [--timeout TIME] [NAME]"                                       },
3946         {"stop",                jt_lst_stop_batch,      NULL,
3947          "Usage: lst stop [--force] BATCH_NAME"                                         },
3948         {"list_batch",          jt_lst_list_batch,      NULL,
3949          "Usage: lst list_batch NAME [--test ID] [--server]"                            },
3950         {"query",               jt_lst_query_batch,     NULL,
3951          "Usage: lst query [--test ID] [--server] [--timeout TIME] NAME"                },
3952         {"add_test",            jt_lst_add_test,        NULL,
3953          "Usage: lst add_test [--batch BATCH] [--loop #] [--concurrency #] "
3954          " [--distribute #:#] [--from GROUP] [--to GROUP] TEST..."                      },
3955         {"help",                Parser_help,            0,     "help"                   },
3956         {"--list-commands",     lst_list_commands,      0,     "list commands"          },
3957         {0,                     0,                      0,      NULL                    }
3958 };
3959
3960 int
3961 lst_initialize(void)
3962 {
3963         char   *key;
3964         char   *feats;
3965
3966         feats = getenv("LST_FEATURES");
3967         if (feats != NULL)
3968                 session_features = strtol(feats, NULL, 16);
3969
3970         if ((session_features & ~LST_FEATS_MASK) != 0) {
3971                 fprintf(stderr,
3972                         "Unsupported session features %x, "
3973                         "only support these features so far: %x\n",
3974                         (session_features & ~LST_FEATS_MASK), LST_FEATS_MASK);
3975                 return -1;
3976         }
3977
3978         key = getenv("LST_SESSION");
3979
3980         if (key == NULL) {
3981                 session_key = 0;
3982                 return 0;
3983         }
3984
3985         session_key = atoi(key);
3986
3987         return 0;
3988 }
3989
3990 static int lst_list_commands(int argc, char **argv)
3991 {
3992         char buffer[81] = ""; /* 80 printable chars + terminating NUL */
3993
3994         Parser_list_commands(lst_cmdlist, buffer, sizeof(buffer), NULL, 0, 4);
3995
3996         return 0;
3997 }
3998
3999 int
4000 main(int argc, char **argv)
4001 {
4002         int rc = 0;
4003
4004         setlinebuf(stdout);
4005
4006         rc = lst_initialize();
4007         if (rc < 0)
4008                 goto errorout;
4009
4010         rc = lustre_lnet_config_lib_init();
4011         if (rc < 0)
4012                 goto errorout;
4013
4014         Parser_init("lst > ", lst_cmdlist);
4015
4016         if (argc != 1)  {
4017                 rc = Parser_execarg(argc - 1, argv + 1, lst_cmdlist);
4018                 goto errorout;
4019         }
4020
4021         Parser_commands();
4022
4023 errorout:
4024         return rc;
4025 }