lnet/utils/lst.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * GPL HEADER START
   5  *
   6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 only,
  10  * as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * General Public License version 2 for more details (a copy is included
  16  * in the LICENSE file that accompanied this code).
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * version 2 along with this program; If not, see
  20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  21  *
  22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  23  * CA 95054 USA or visit www.sun.com if you need additional information or
  24  * have any questions.
  25  *
  26  * GPL HEADER END
  27  */
  28 /*
  29  * Copyright  2008 Sun Microsystems, Inc. All rights reserved
  30  * Use is subject to license terms.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * lnet/selftest/conctl.c
  37  *
  38  * Author: Liang Zhen <liangzhen@clusterfs.com>
  39  */
  40
  41 #define _GNU_SOURCE
  42
  43 #include <stdio.h>
  44 #include <stdlib.h>
  45 #include <string.h>
  46 #include <getopt.h>
  47 #include <errno.h>
  48 #include <pwd.h>
  49 #include <lnet/lnetctl.h>
  50 #include <lnet/lnetst.h>
  51 #include <libcfs/libcfsutil.h>
  52
  53 static command_t           lst_cmdlist[];
  54 static lst_sid_t           session_id;
  55 static int                 session_key;
  56 static lstcon_trans_stat_t trans_stat;
  57
  58 typedef struct list_string {
  59         struct list_string *lstr_next;
  60         int                 lstr_sz;
  61         char                lstr_str[0];
  62 } lstr_t;
  63
  64 #ifndef offsetof
  65 # define offsetof(typ,memb)     ((unsigned long)((char *)&(((typ *)0)->memb)))
  66 #endif
  67
  68 static int alloc_count = 0;
  69 static int alloc_nob   = 0;
  70
  71 lstr_t *
  72 alloc_lstr(int sz)
  73 {
  74         lstr_t  *lstr = malloc(offsetof(lstr_t, lstr_str[sz]));
  75
  76         if (lstr == NULL) {
  77                 fprintf(stderr, "Can't allocate lstr\n");
  78                 abort();
  79         }
  80
  81         alloc_nob += sz;
  82         alloc_count++;
  83
  84         lstr->lstr_str[0] = 0;
  85         lstr->lstr_sz = sz;
  86         return lstr;
  87 }
  88
  89 void
  90 free_lstr(lstr_t *lstr)
  91 {
  92         alloc_count--;
  93         alloc_nob -= lstr->lstr_sz;
  94         free(lstr);
  95 }
  96
  97 void
  98 free_lstrs(lstr_t **list)
  99 {
 100         lstr_t   *lstr;
 101
 102         while ((lstr = *list) != NULL) {
 103                 *list = lstr->lstr_next;
 104                 free_lstr(lstr);
 105         }
 106 }
 107
 108 void
 109 new_lstrs(lstr_t **list, char *prefix, char *postfix,
 110           int lo, int hi, int stride)
 111 {
 112         int    n1 = strlen(prefix);
 113         int    n2 = strlen(postfix);
 114         int    sz = n1 + 20 + n2 + 1;
 115
 116         do {
 117                 lstr_t *n = alloc_lstr(sz);
 118
 119                 snprintf(n->lstr_str, sz - 1, "%s%u%s",
 120                          prefix, lo, postfix);
 121
 122                 n->lstr_next = *list;
 123                 *list = n;
 124
 125                 lo += stride;
 126         } while (lo <= hi);
 127 }
 128
 129 int
 130 expand_lstr(lstr_t **list, lstr_t *l)
 131 {
 132         int          nob = strlen(l->lstr_str);
 133         char        *b1;
 134         char        *b2;
 135         char        *expr;
 136         char        *sep;
 137         int          x;
 138         int          y;
 139         int          z;
 140         int          n;
 141
 142         b1 = strchr(l->lstr_str, '[');
 143         if (b1 == NULL) {
 144                 l->lstr_next = *list;
 145                 *list = l;
 146                 return 0;
 147         }
 148
 149         b2 = strchr(b1, ']');
 150         if (b2 == NULL || b2 == b1 + 1)
 151                 return -1;
 152
 153         *b1++ = 0;
 154         *b2++ = 0;
 155         expr = b1;
 156         do {
 157
 158                 sep = strchr(expr, ',');
 159                 if (sep != NULL)
 160                         *sep++ = 0;
 161
 162                 nob = strlen(expr);
 163                 n = nob;
 164                 if (sscanf(expr, "%u%n", &x, &n) >= 1 && n == nob) {
 165                         /* simple number */
 166                         new_lstrs(list, l->lstr_str, b2, x, x, 1);
 167                         continue;
 168                 }
 169
 170                 n = nob;
 171                 if (sscanf(expr, "%u-%u%n", &x, &y, &n) >= 2 && n == nob &&
 172                     x < y) {
 173                         /* simple range */
 174                         new_lstrs(list, l->lstr_str, b2, x, y, 1);
 175                         continue;
 176                 }
 177
 178                 n = nob;
 179                 if (sscanf(expr, "%u-%u/%u%n", &x, &y, &z, &n) >= 3 && n == nob &&
 180                     x < y) {
 181                         /* strided range */
 182                         new_lstrs(list, l->lstr_str, b2, x, y, z);
 183                         continue;
 184                 }
 185
 186                 /* syntax error */
 187                 return -1;
 188         } while ((expr = sep) != NULL);
 189
 190         free_lstr(l);
 191
 192         return 1;
 193 }
 194
 195 int
 196 expand_strs(char *str, lstr_t **head)
 197 {
 198         lstr_t  *list = NULL;
 199         lstr_t  *nlist;
 200         lstr_t  *l;
 201         int      rc = 0;
 202         int      expanded;
 203
 204         l = alloc_lstr(strlen(str) + 1);
 205         memcpy(l->lstr_str, str, strlen(str) + 1);
 206         l->lstr_next = NULL;
 207         list = l;
 208
 209         do {
 210                 expanded = 0;
 211                 nlist = NULL;
 212
 213                 while ((l = list) != NULL) {
 214                         list = l->lstr_next;
 215
 216                         rc = expand_lstr(&nlist, l);
 217                         if (rc < 0) {
 218                                 fprintf(stderr, "Syntax error in \"%s\"\n", str);
 219                                 free_lstr(l);
 220                                 break;
 221                         }
 222
 223                         expanded |= rc > 0;
 224                 }
 225
 226                 /* re-order onto 'list' */
 227                 while ((l = nlist) != NULL) {
 228                         nlist = l->lstr_next;
 229                         l->lstr_next = list;
 230                         list = l;
 231                 }
 232
 233         } while (expanded && rc > 0);
 234
 235         if (rc >= 0) {
 236                 *head = list;
 237                 return 0;
 238         }
 239
 240         while ((l = list) != NULL) {
 241                 list = l->lstr_next;
 242
 243                 free_lstr(l);
 244         }
 245         return rc;
 246 }
 247
 248 int
 249 lst_parse_nids(char *str, int *countp, lnet_process_id_t **idspp)
 250 {
 251         lstr_t  *head = NULL;
 252         lstr_t  *l;
 253         int      c = 0;
 254         int      i;
 255         int      rc;
 256
 257         rc = expand_strs(str, &head);
 258         if (rc != 0)
 259                 goto out;
 260
 261         l = head;
 262         while (l != NULL) {
 263                 l = l->lstr_next;
 264                 c++;
 265         }
 266
 267         *idspp = malloc(c * sizeof(lnet_process_id_t));
 268         if (*idspp == NULL) {
 269                 fprintf(stderr, "Out of memory\n");
 270                 rc = -1;
 271         }
 272
 273         *countp = c;
 274 out:
 275         i = 0;
 276         while ((l = head) != NULL) {
 277                 head = l->lstr_next;
 278
 279                 if (rc == 0) {
 280                         (*idspp)[i].nid = libcfs_str2nid(l->lstr_str);
 281                         if ((*idspp)[i].nid == LNET_NID_ANY) {
 282                                 fprintf(stderr, "Invalid nid: %s\n",
 283                                         l->lstr_str);
 284                                 rc = -1;
 285                         }
 286
 287                         (*idspp)[i].pid = LUSTRE_LNET_PID;
 288                         i++;
 289                 }
 290
 291                 free_lstr(l);
 292         }
 293
 294         if (rc == 0)
 295                 return 0;
 296
 297         free(*idspp);
 298         *idspp = NULL;
 299
 300         return rc;
 301 }
 302
 303 char *
 304 lst_node_state2str(int state)
 305 {
 306         if (state == LST_NODE_ACTIVE)
 307                 return "Active";
 308         if (state == LST_NODE_BUSY)
 309                 return "Busy";
 310         if (state == LST_NODE_DOWN)
 311                 return "Down";
 312
 313         return "Unknown";
 314 }
 315
 316 int
 317 lst_node_str2state(char *str)
 318 {
 319         if (strcasecmp(str, "active") == 0)
 320                 return LST_NODE_ACTIVE;
 321         if (strcasecmp(str, "busy") == 0)
 322                 return LST_NODE_BUSY;
 323         if (strcasecmp(str, "down") == 0)
 324                 return LST_NODE_DOWN;
 325         if (strcasecmp(str, "unknown") == 0)
 326                 return LST_NODE_UNKNOWN;
 327         if (strcasecmp(str, "invalid") == 0)
 328                 return (LST_NODE_UNKNOWN | LST_NODE_DOWN | LST_NODE_BUSY);
 329
 330         return -1;
 331 }
 332
 333 char *
 334 lst_test_type2name(int type)
 335 {
 336         if (type == LST_TEST_PING)
 337                 return "ping";
 338         if (type == LST_TEST_BULK)
 339                 return "brw";
 340
 341         return "unknown";
 342 }
 343
 344 int
 345 lst_test_name2type(char *name)
 346 {
 347         if (strcasecmp(name, "ping") == 0)
 348                 return LST_TEST_PING;
 349         if (strcasecmp(name, "brw") == 0)
 350                 return LST_TEST_BULK;
 351
 352         return -1;
 353 }
 354
 355 void
 356 lst_print_usage(char *cmd)
 357 {
 358         Parser_printhelp(cmd);
 359 }
 360
 361 void
 362 lst_print_error(char *sub, const char *def_format, ...)
 363 {
 364         va_list ap;
 365
 366         /* local error returned from kernel */
 367         switch (errno) {
 368         case ESRCH:
 369                 fprintf(stderr, "No session exists\n");
 370                 return;
 371         case ESHUTDOWN:
 372                 fprintf(stderr, "Session is shutting down\n");
 373                 return;
 374         case EACCES:
 375                 fprintf(stderr, "Unmatched session key or not root\n");
 376                 return;
 377         case ENOENT:
 378                 fprintf(stderr, "Can't find %s in current session\n", sub);
 379                 return;
 380         case EINVAL:
 381                 fprintf(stderr, "Invalid parameters list in command line\n");
 382                 return;
 383         case EFAULT:
 384                 fprintf(stderr, "Bad parameter address\n");
 385                 return;
 386         case EEXIST:
 387                 fprintf(stderr, "%s already exists\n", sub);
 388                 return;
 389         default:
 390                 va_start(ap, def_format);
 391                 vfprintf(stderr, def_format, ap);
 392                 va_end(ap);
 393
 394                 return;
 395         }
 396 }
 397
 398 void
 399 lst_free_rpcent(struct list_head *head)
 400 {
 401         lstcon_rpc_ent_t *ent;
 402
 403         while (!list_empty(head)) {
 404                 ent = list_entry(head->next, lstcon_rpc_ent_t, rpe_link);
 405
 406                 list_del(&ent->rpe_link);
 407                 free(ent);
 408         }
 409 }
 410
 411 void
 412 lst_reset_rpcent(struct list_head *head)
 413 {
 414         lstcon_rpc_ent_t *ent;
 415
 416         list_for_each_entry(ent, head, rpe_link) {
 417                 ent->rpe_sid      = LST_INVALID_SID;
 418                 ent->rpe_peer.nid = LNET_NID_ANY;
 419                 ent->rpe_peer.pid = LNET_PID_ANY;
 420                 ent->rpe_rpc_errno = ent->rpe_fwk_errno = 0;
 421         }
 422 }
 423
 424 int
 425 lst_alloc_rpcent(struct list_head *head, int count, int offset)
 426 {
 427         lstcon_rpc_ent_t *ent;
 428         int               i;
 429
 430         for (i = 0; i < count; i++) {
 431                 ent = malloc(offsetof(lstcon_rpc_ent_t, rpe_payload[offset]));
 432                 if (ent == NULL) {
 433                         lst_free_rpcent(head);
 434                         return -1;
 435                 }
 436
 437                 memset(ent, 0, offsetof(lstcon_rpc_ent_t, rpe_payload[offset]));
 438
 439                 ent->rpe_sid      = LST_INVALID_SID;
 440                 ent->rpe_peer.nid = LNET_NID_ANY;
 441                 ent->rpe_peer.pid = LNET_PID_ANY;
 442                 list_add(&ent->rpe_link, head);
 443         }
 444
 445         return 0;
 446 }
 447
 448 void
 449 lst_print_transerr(struct list_head *head, char *optstr)
 450 {
 451         lstcon_rpc_ent_t  *ent;
 452
 453         list_for_each_entry(ent, head, rpe_link) {
 454                 if (ent->rpe_rpc_errno == 0 && ent->rpe_fwk_errno == 0)
 455                         continue;
 456
 457                 if (ent->rpe_rpc_errno != 0) {
 458                         fprintf(stderr, "%s RPC failed on %s: %s\n",
 459                                 optstr, libcfs_id2str(ent->rpe_peer),
 460                                 strerror(ent->rpe_rpc_errno));
 461                         continue;
 462                 }
 463
 464                 fprintf(stderr, "%s failed on %s: %s\n",
 465                         optstr, libcfs_id2str(ent->rpe_peer),
 466                         strerror(ent->rpe_fwk_errno));
 467         }
 468 }
 469
 470 int lst_info_batch_ioctl(char *batch, int test, int server,
 471                         lstcon_test_batch_ent_t *entp, int *idxp,
 472                         int *ndentp, lstcon_node_ent_t *dentsp);
 473
 474 int lst_info_group_ioctl(char *name, lstcon_ndlist_ent_t *gent,
 475                          int *idx, int *count, lstcon_node_ent_t *dents);
 476
 477 int lst_query_batch_ioctl(char *batch, int test, int server,
 478                           int timeout, struct list_head *head);
 479
 480 int
 481 lst_ioctl(unsigned int opc, void *buf, int len)
 482 {
 483         struct libcfs_ioctl_data data;
 484         int    rc;
 485
 486         LIBCFS_IOC_INIT (data);
 487         data.ioc_u32[0]  = opc;
 488         data.ioc_plen1   = len;
 489         data.ioc_pbuf1   = (char *)buf;
 490         data.ioc_plen2   = sizeof(trans_stat);
 491         data.ioc_pbuf2   = (char *)&trans_stat;
 492
 493         memset(&trans_stat, 0, sizeof(trans_stat));
 494
 495         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNETST, &data);
 496
 497         /* local error, no valid RPC result */
 498         if (rc != 0)
 499                 return -1;
 500
 501         /* RPC error */
 502         if (trans_stat.trs_rpc_errno != 0)
 503                 return -2;
 504
 505         /* Framework error */
 506         if (trans_stat.trs_fwk_errno != 0)
 507                 return -3;
 508
 509         return 0;
 510 }
 511
 512 int
 513 lst_new_session_ioctl (char *name, int timeout, int force, lst_sid_t *sid)
 514 {
 515         lstio_session_new_args_t        args = {
 516                 .lstio_ses_key          = session_key,
 517                 .lstio_ses_timeout      = timeout,
 518                 .lstio_ses_force        = force,
 519                 .lstio_ses_idp          = sid,
 520                 .lstio_ses_namep        = name,
 521                 .lstio_ses_nmlen        = strlen(name),
 522         };
 523
 524         return lst_ioctl (LSTIO_SESSION_NEW, &args, sizeof(args));
 525 }
 526
 527 int
 528 jt_lst_new_session(int argc,  char **argv)
 529 {
 530         char  buf[LST_NAME_SIZE];
 531         char *name;
 532         int   optidx  = 0;
 533         int   timeout = 300;
 534         int   force   = 0;
 535         int   c;
 536         int   rc;
 537
 538         static struct option session_opts[] =
 539         {
 540                 {"timeout", required_argument,  0, 't' },
 541                 {"force",   no_argument,        0, 'f' },
 542                 {0,         0,                  0,  0  }
 543         };
 544
 545         if (session_key == 0) {
 546                 fprintf(stderr,
 547                         "Can't find env LST_SESSION or value is not valid\n");
 548                 return -1;
 549         }
 550
 551         while (1) {
 552
 553                 c = getopt_long(argc, argv, "ft:",
 554                                 session_opts, &optidx);
 555
 556                 if (c == -1)
 557                         break;
 558
 559                 switch (c) {
 560                 case 'f':
 561                         force = 1;
 562                         break;
 563                 case 't':
 564                         timeout = atoi(optarg);
 565                         break;
 566                 default:
 567                         lst_print_usage(argv[0]);
 568                         return -1;
 569                 }
 570         }
 571
 572         if (timeout <= 0) {
 573                 fprintf(stderr, "Invalid timeout value\n");
 574                 return -1;
 575         }
 576
 577         if (optind == argc - 1) {
 578                 name = argv[optind ++];
 579                 if (strlen(name) >= LST_NAME_SIZE) {
 580                         fprintf(stderr, "Name size is limited to %d\n",
 581                                 LST_NAME_SIZE - 1);
 582                         return -1;
 583                 }
 584
 585         } else if (optind == argc) {
 586                 char           user[LST_NAME_SIZE];
 587                 char           host[LST_NAME_SIZE];
 588                 struct passwd *pw = getpwuid(getuid());
 589
 590                 if (pw == NULL)
 591                         snprintf(user, sizeof(user), "%d", (int)getuid());
 592                 else
 593                         snprintf(user, sizeof(user), "%s", pw->pw_name);
 594
 595                 rc = gethostname(host, sizeof(host));
 596                 if (rc != 0)
 597                         snprintf(host, sizeof(host), "unknown_host");
 598
 599                 snprintf(buf, LST_NAME_SIZE, "%s@%s", user, host);
 600                 name = buf;
 601
 602         } else {
 603                 lst_print_usage(argv[0]);
 604                 return -1;
 605         }
 606
 607         rc = lst_new_session_ioctl(name, timeout, force, &session_id);
 608
 609         if (rc != 0) {
 610                 lst_print_error("session", "Failed to create session: %s\n",
 611                                 strerror(errno));
 612                 return rc;
 613         }
 614
 615         fprintf(stdout, "SESSION: %s TIMEOUT: %d FORCE: %s\n",
 616                 name, timeout, force ? "Yes": "No");
 617
 618         return rc;
 619 }
 620
 621 int
 622 lst_session_info_ioctl(char *name, int len, int *key,
 623                        lst_sid_t *sid, lstcon_ndlist_ent_t *ndinfo)
 624 {
 625         lstio_session_info_args_t args = {
 626                 .lstio_ses_keyp         = key,
 627                 .lstio_ses_idp          = sid,
 628                 .lstio_ses_ndinfo       = ndinfo,
 629                 .lstio_ses_nmlen        = len,
 630                 .lstio_ses_namep        = name,
 631         };
 632
 633         return lst_ioctl(LSTIO_SESSION_INFO, &args, sizeof(args));
 634 }
 635
 636 int
 637 jt_lst_show_session(int argc, char **argv)
 638 {
 639         lstcon_ndlist_ent_t ndinfo;
 640         lst_sid_t           sid;
 641         char                name[LST_NAME_SIZE];
 642         int                 key;
 643         int                 rc;
 644
 645         rc = lst_session_info_ioctl(name, LST_NAME_SIZE, &key, &sid, &ndinfo);
 646
 647         if (rc != 0) {
 648                 lst_print_error("session", "Failed to show session: %s\n",
 649                                 strerror(errno));
 650                 return -1;
 651         }
 652
 653         fprintf(stdout, "%s ID: %Lu@%s, KEY: %d NODES: %d\n",
 654                 name, sid.ses_stamp, libcfs_nid2str(sid.ses_nid),
 655                 key, ndinfo.nle_nnode);
 656
 657         return 0;
 658 }
 659
 660 int
 661 lst_end_session_ioctl(void)
 662 {
 663         lstio_session_end_args_t args = {
 664                 .lstio_ses_key           = session_key,
 665         };
 666
 667         return lst_ioctl (LSTIO_SESSION_END, &args, sizeof(args));
 668 }
 669
 670 int
 671 jt_lst_end_session(int argc, char **argv)
 672 {
 673         int             rc;
 674
 675         if (session_key == 0) {
 676                 fprintf(stderr,
 677                         "Can't find env LST_SESSION or value is not valid\n");
 678                 return -1;
 679         }
 680
 681         rc = lst_end_session_ioctl();
 682
 683         if (rc == 0) {
 684                 fprintf(stdout, "session is ended\n");
 685                 return 0;
 686         }
 687
 688         if (rc == -1) {
 689                 lst_print_error("session", "Failed to end session: %s\n",
 690                                 strerror(errno));
 691                 return rc;
 692         }
 693
 694         if (trans_stat.trs_rpc_errno != 0) {
 695                 fprintf(stderr,
 696                         "[RPC] Failed to send %d session RPCs: %s\n",
 697                         lstcon_rpc_stat_failure(&trans_stat, 0),
 698                         strerror(trans_stat.trs_rpc_errno));
 699         }
 700
 701         if (trans_stat.trs_fwk_errno != 0) {
 702                 fprintf(stderr,
 703                         "[FWK] Failed to end session on %d nodes: %s\n",
 704                         lstcon_sesop_stat_failure(&trans_stat, 0),
 705                         strerror(trans_stat.trs_fwk_errno));
 706         }
 707
 708         return rc;
 709 }
 710
 711 int
 712 lst_ping_ioctl(char *str, int type, int timeout,
 713                int count, lnet_process_id_t *ids, struct list_head *head)
 714 {
 715         lstio_debug_args_t args = {
 716                 .lstio_dbg_key          = session_key,
 717                 .lstio_dbg_type         = type,
 718                 .lstio_dbg_flags        = 0,
 719                 .lstio_dbg_timeout      = timeout,
 720                 .lstio_dbg_nmlen        = (str == NULL) ? 0: strlen(str),
 721                 .lstio_dbg_namep        = str,
 722                 .lstio_dbg_count        = count,
 723                 .lstio_dbg_idsp         = ids,
 724                 .lstio_dbg_resultp      = head,
 725         };
 726
 727         return lst_ioctl (LSTIO_DEBUG, &args, sizeof(args));
 728 }
 729
 730 int
 731 lst_get_node_count(int type, char *str, int *countp, lnet_process_id_t **idspp)
 732 {
 733         char                    buf[LST_NAME_SIZE];
 734         lstcon_test_batch_ent_t ent;
 735         lstcon_ndlist_ent_t    *entp = &ent.tbe_cli_nle;
 736         lst_sid_t               sid;
 737         int                     key;
 738         int                     rc;
 739
 740         switch (type) {
 741         case LST_OPC_SESSION:
 742                 rc = lst_session_info_ioctl(buf, LST_NAME_SIZE,
 743                                             &key, &sid, entp);
 744                 break;
 745
 746         case LST_OPC_BATCHSRV:
 747                 entp = &ent.tbe_srv_nle;
 748         case LST_OPC_BATCHCLI:
 749                 rc = lst_info_batch_ioctl(str, 0, 0, &ent, NULL, NULL, NULL);
 750                 break;
 751
 752         case LST_OPC_GROUP:
 753                 rc = lst_info_group_ioctl(str, entp, NULL, NULL, NULL);
 754                 break;
 755
 756         case LST_OPC_NODES:
 757                 rc = lst_parse_nids(str, &entp->nle_nnode, idspp) < 0 ? -1 : 0;
 758                 break;
 759
 760         default:
 761                 rc = -1;
 762                 break;
 763         }
 764
 765         if (rc == 0)
 766                 *countp = entp->nle_nnode;
 767
 768         return rc;
 769 }
 770
 771 int
 772 jt_lst_ping(int argc,  char **argv)
 773 {
 774         struct list_head   head;
 775         lnet_process_id_t *ids = NULL;
 776         lstcon_rpc_ent_t  *ent = NULL;
 777         char              *str = NULL;
 778         int                optidx  = 0;
 779         int                server  = 0;
 780         int                timeout = 5;
 781         int                count   = 0;
 782         int                type    = 0;
 783         int                rc      = 0;
 784         int                c;
 785
 786         static struct option ping_opts[] =
 787         {
 788                 {"session", no_argument,       0, 's' },
 789                 {"server",  no_argument,       0, 'v' },
 790                 {"batch",   required_argument, 0, 'b' },
 791                 {"group",   required_argument, 0, 'g' },
 792                 {"nodes",   required_argument, 0, 'n' },
 793                 {"timeout", required_argument, 0, 't' },
 794                 {0,         0,                 0,  0  }
 795         };
 796
 797         if (session_key == 0) {
 798                 fprintf(stderr,
 799                         "Can't find env LST_SESSION or value is not valid\n");
 800                 return -1;
 801         }
 802
 803         while (1) {
 804
 805                 c = getopt_long(argc, argv, "g:b:n:t:sv",
 806                                 ping_opts, &optidx);
 807
 808                 if (c == -1)
 809                         break;
 810
 811                 switch (c) {
 812                 case 's':
 813                         type = LST_OPC_SESSION;
 814                         break;
 815
 816                 case 'g':
 817                         type = LST_OPC_GROUP;
 818                         str = optarg;
 819                         break;
 820
 821                 case 'b':
 822                         type = LST_OPC_BATCHCLI;
 823                         str = optarg;
 824                         break;
 825
 826                 case 'n':
 827                         type = LST_OPC_NODES;
 828                         str = optarg;
 829                         break;
 830
 831                 case 't':
 832                         timeout = atoi(optarg);
 833                         break;
 834
 835                 case 'v':
 836                         server = 1;
 837                         break;
 838
 839                 default:
 840                         lst_print_usage(argv[0]);
 841                         return -1;
 842                 }
 843         }
 844
 845         if (type == 0 || timeout <= 0 || optind != argc) {
 846                 lst_print_usage(argv[0]);
 847                 return -1;
 848         }
 849
 850         if (type == LST_OPC_BATCHCLI && server)
 851                 type = LST_OPC_BATCHSRV;
 852
 853         rc = lst_get_node_count(type, str, &count, &ids);
 854         if (rc < 0) {
 855                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
 856                         (str == NULL) ? "session" : str, strerror(errno));
 857                 return -1;
 858         }
 859
 860         CFS_INIT_LIST_HEAD(&head);
 861
 862         rc = lst_alloc_rpcent(&head, count, LST_NAME_SIZE);
 863         if (rc != 0) {
 864                 fprintf(stderr, "Out of memory\n");
 865                 goto out;
 866         }
 867
 868         if (count == 0) {
 869                 fprintf(stdout, "Target %s is empty\n",
 870                         (str == NULL) ? "session" : str);
 871                 goto out;
 872         }
 873
 874         rc = lst_ping_ioctl(str, type, timeout, count, ids, &head);
 875         if (rc == -1) { /* local failure */
 876                 lst_print_error("debug", "Failed to ping %s: %s\n",
 877                                 (str == NULL) ? "session" : str,
 878                                 strerror(errno));
 879                 rc = -1;
 880                 goto out;
 881         }
 882
 883         /* ignore RPC errors and framwork errors */
 884         list_for_each_entry(ent, &head, rpe_link) {
 885                 fprintf(stdout, "\t%s: %s [session: %s id: %s]\n",
 886                         libcfs_id2str(ent->rpe_peer),
 887                         lst_node_state2str(ent->rpe_state),
 888                         (ent->rpe_state == LST_NODE_ACTIVE ||
 889                          ent->rpe_state == LST_NODE_BUSY)?
 890                                  (ent->rpe_rpc_errno == 0 ?
 891                                          &ent->rpe_payload[0] : "Unknown") :
 892                                  "<NULL>", libcfs_nid2str(ent->rpe_sid.ses_nid));
 893         }
 894
 895 out:
 896         lst_free_rpcent(&head);
 897
 898         if (ids != NULL)
 899                 free(ids);
 900
 901         return rc;
 902
 903 }
 904
 905 int
 906 lst_add_nodes_ioctl (char *name, int count, lnet_process_id_t *ids,
 907                      struct list_head *resultp)
 908 {
 909         lstio_group_nodes_args_t        args = {
 910                 .lstio_grp_key          = session_key,
 911                 .lstio_grp_nmlen        = strlen(name),
 912                 .lstio_grp_namep        = name,
 913                 .lstio_grp_count        = count,
 914                 .lstio_grp_idsp         = ids,
 915                 .lstio_grp_resultp      = resultp,
 916         };
 917
 918         return lst_ioctl(LSTIO_NODES_ADD, &args, sizeof(args));
 919 }
 920
 921 int
 922 lst_add_group_ioctl (char *name)
 923 {
 924         lstio_group_add_args_t  args = {
 925                 .lstio_grp_key          = session_key,
 926                 .lstio_grp_nmlen        = strlen(name),
 927                 .lstio_grp_namep        = name,
 928         };
 929
 930         return lst_ioctl(LSTIO_GROUP_ADD, &args, sizeof(args));
 931 }
 932
 933 int
 934 jt_lst_add_group(int argc, char **argv)
 935 {
 936         struct list_head   head;
 937         lnet_process_id_t *ids;
 938         char              *name;
 939         int                count;
 940         int                rc;
 941         int                i;
 942
 943         if (session_key == 0) {
 944                 fprintf(stderr,
 945                         "Can't find env LST_SESSION or value is not valid\n");
 946                 return -1;
 947         }
 948
 949         if (argc < 3) {
 950                 lst_print_usage(argv[0]);
 951                 return -1;
 952         }
 953
 954         name = argv[1];
 955         if (strlen(name) >= LST_NAME_SIZE) {
 956                 fprintf(stderr, "Name length is limited to %d\n",
 957                         LST_NAME_SIZE - 1);
 958                 return -1;
 959         }
 960
 961         rc = lst_add_group_ioctl(name);
 962         if (rc != 0) {
 963                 lst_print_error("group", "Failed to add group %s: %s\n",
 964                                 name, strerror(errno));
 965                 return -1;
 966         }
 967
 968         CFS_INIT_LIST_HEAD(&head);
 969
 970         for (i = 2; i < argc; i++) {
 971                 /* parse address list */
 972                 rc = lst_parse_nids(argv[i], &count, &ids);
 973                 if (rc < 0) {
 974                         fprintf(stderr, "Ignore invalid id list %s\n",
 975                                 argv[i]);
 976                         continue;
 977                 }
 978
 979                 if (count == 0)
 980                         continue;
 981
 982                 rc = lst_alloc_rpcent(&head, count, 0);
 983                 if (rc != 0) {
 984                         fprintf(stderr, "Out of memory\n");
 985                         break;
 986                 }
 987
 988                 rc = lst_add_nodes_ioctl(name, count, ids, &head);
 989
 990                 free(ids);
 991
 992                 if (rc == 0) {
 993                         lst_free_rpcent(&head);
 994                         fprintf(stderr, "%s are added to session\n", argv[i]);
 995                         continue;
 996                 }
 997
 998                 if (rc == -1) {
 999                         lst_free_rpcent(&head);
1000                         lst_print_error("group", "Failed to add nodes %s: %s\n",
1001                                         argv[i], strerror(errno));
1002                         break;
1003                 }
1004
1005                 lst_print_transerr(&head, "create session");
1006                 lst_free_rpcent(&head);
1007         }
1008
1009         return rc;
1010 }
1011
1012 int
1013 lst_del_group_ioctl (char *name)
1014 {
1015         lstio_group_del_args_t  args = {
1016                 .lstio_grp_key          = session_key,
1017                 .lstio_grp_nmlen        = strlen(name),
1018                 .lstio_grp_namep        = name,
1019         };
1020
1021         return lst_ioctl(LSTIO_GROUP_DEL, &args, sizeof(args));
1022 }
1023
1024 int
1025 jt_lst_del_group(int argc, char **argv)
1026 {
1027         int     rc;
1028
1029         if (session_key == 0) {
1030                 fprintf(stderr,
1031                         "Can't find env LST_SESSION or value is not valid\n");
1032                 return -1;
1033         }
1034
1035         if (argc != 2) {
1036                 lst_print_usage(argv[0]);
1037                 return -1;
1038         }
1039
1040         rc = lst_del_group_ioctl(argv[1]);
1041         if (rc == 0) {
1042                 fprintf(stdout, "Group is deleted\n");
1043                 return 0;
1044         }
1045
1046         if (rc == -1) {
1047                 lst_print_error("group", "Failed to delete group: %s\n",
1048                                 strerror(errno));
1049                 return rc;
1050         }
1051
1052         fprintf(stderr, "Group is deleted with some errors\n");
1053
1054         if (trans_stat.trs_rpc_errno != 0) {
1055                 fprintf(stderr, "[RPC] Failed to send %d end session RPCs: %s\n",
1056                         lstcon_rpc_stat_failure(&trans_stat, 0),
1057                         strerror(trans_stat.trs_rpc_errno));
1058         }
1059
1060         if (trans_stat.trs_fwk_errno != 0) {
1061                 fprintf(stderr,
1062                         "[FWK] Failed to end session on %d nodes: %s\n",
1063                         lstcon_sesop_stat_failure(&trans_stat, 0),
1064                         strerror(trans_stat.trs_fwk_errno));
1065         }
1066
1067         return -1;
1068 }
1069
1070 int
1071 lst_update_group_ioctl(int opc, char *name, int clean, int count,
1072                        lnet_process_id_t *ids, struct list_head *resultp)
1073 {
1074         lstio_group_update_args_t  args = {
1075                 .lstio_grp_key          = session_key,
1076                 .lstio_grp_opc          = opc,
1077                 .lstio_grp_args         = clean,
1078                 .lstio_grp_nmlen        = strlen(name),
1079                 .lstio_grp_namep        = name,
1080                 .lstio_grp_count        = count,
1081                 .lstio_grp_idsp         = ids,
1082                 .lstio_grp_resultp      = resultp,
1083         };
1084
1085         return lst_ioctl(LSTIO_GROUP_UPDATE, &args, sizeof(args));
1086 }
1087
1088 int
1089 jt_lst_update_group(int argc, char **argv)
1090 {
1091         struct list_head   head;
1092         lnet_process_id_t *ids = NULL;
1093         char              *str = NULL;
1094         char              *grp = NULL;
1095         int                optidx = 0;
1096         int                count = 0;
1097         int                clean = 0;
1098         int                opc = 0;
1099         int                rc;
1100         int                c;
1101
1102         static struct option update_group_opts[] =
1103         {
1104                 {"refresh", no_argument,       0, 'f' },
1105                 {"clean",   required_argument, 0, 'c' },
1106                 {"remove",  required_argument, 0, 'r' },
1107                 {0,         0,                 0,  0  }
1108         };
1109
1110         if (session_key == 0) {
1111                 fprintf(stderr,
1112                         "Can't find env LST_SESSION or value is not valid\n");
1113                 return -1;
1114         }
1115
1116         while (1) {
1117                 c = getopt_long(argc, argv, "fc:r:",
1118                                 update_group_opts, &optidx);
1119
1120                 /* Detect the end of the options. */
1121                 if (c == -1)
1122                         break;
1123
1124                 switch (c) {
1125                 case 'f':
1126                         if (opc != 0) {
1127                                 lst_print_usage(argv[0]);
1128                                 return -1;
1129                         }
1130                         opc = LST_GROUP_REFRESH;
1131                         break;
1132
1133                 case 'r':
1134                         if (opc != 0) {
1135                                 lst_print_usage(argv[0]);
1136                                 return -1;
1137                         }
1138                         opc = LST_GROUP_RMND;
1139                         str = optarg;
1140                         break;
1141
1142                 case 'c':
1143                         clean = lst_node_str2state(optarg);
1144                         if (opc != 0 || clean <= 0) {
1145                                 lst_print_usage(argv[0]);
1146                                 return -1;
1147                         }
1148                         opc = LST_GROUP_CLEAN;
1149                         break;
1150
1151                 default:
1152                         lst_print_usage(argv[0]);
1153                         return -1;
1154                 }
1155         }
1156
1157         /* no OPC or group is specified */
1158         if (opc == 0 || optind != argc - 1) {
1159                 lst_print_usage(argv[0]);
1160                 return -1;
1161         }
1162
1163         grp = argv[optind];
1164
1165         CFS_INIT_LIST_HEAD(&head);
1166
1167         if (opc == LST_GROUP_RMND || opc == LST_GROUP_REFRESH) {
1168                 rc = lst_get_node_count(opc == LST_GROUP_RMND ? LST_OPC_NODES :
1169                                                                 LST_OPC_GROUP,
1170                                         opc == LST_GROUP_RMND ? str : grp,
1171                                         &count, &ids);
1172
1173                 if (rc != 0) {
1174                         fprintf(stderr, "Can't get count of nodes from %s: %s\n",
1175                                 opc == LST_GROUP_RMND ? str : grp,
1176                                 strerror(errno));
1177                         return -1;
1178                 }
1179
1180                 rc = lst_alloc_rpcent(&head, count, 0);
1181                 if (rc != 0) {
1182                         fprintf(stderr, "Out of memory\n");
1183                         free(ids);
1184                         return -1;
1185                 }
1186
1187         }
1188
1189         rc = lst_update_group_ioctl(opc, grp, clean, count, ids, &head);
1190
1191         if (ids != NULL)
1192                 free(ids);
1193
1194         if (rc == 0) {
1195                 lst_free_rpcent(&head);
1196                 return 0;
1197         }
1198
1199         if (rc == -1) {
1200                 lst_free_rpcent(&head);
1201                 lst_print_error("group", "Failed to update group: %s\n",
1202                                 strerror(errno));
1203                 return rc;
1204         }
1205
1206         lst_print_transerr(&head, "Updating group");
1207
1208         lst_free_rpcent(&head);
1209
1210         return rc;
1211 }
1212
1213 int
1214 lst_list_group_ioctl(int len, char *name, int idx)
1215 {
1216         lstio_group_list_args_t         args = {
1217                 .lstio_grp_key          = session_key,
1218                 .lstio_grp_idx          = idx,
1219                 .lstio_grp_nmlen        = len,
1220                 .lstio_grp_namep        = name,
1221         };
1222
1223         return lst_ioctl(LSTIO_GROUP_LIST, &args, sizeof(args));
1224 }
1225
1226 int
1227 lst_info_group_ioctl(char *name, lstcon_ndlist_ent_t *gent,
1228                      int *idx, int *count, lstcon_node_ent_t *dents)
1229 {
1230         lstio_group_info_args_t         args = {
1231                 .lstio_grp_key          = session_key,
1232                 .lstio_grp_nmlen        = strlen(name),
1233                 .lstio_grp_namep        = name,
1234                 .lstio_grp_entp         = gent,
1235                 .lstio_grp_idxp         = idx,
1236                 .lstio_grp_ndentp       = count,
1237                 .lstio_grp_dentsp       = dents,
1238         };
1239
1240         return lst_ioctl(LSTIO_GROUP_INFO, &args, sizeof(args));
1241 }
1242
1243 int
1244 lst_list_group_all(void)
1245 {
1246         char  name[LST_NAME_SIZE];
1247         int   rc;
1248         int   i;
1249
1250         /* no group is specified, list name of all groups */
1251         for (i = 0; ; i++) {
1252                 rc = lst_list_group_ioctl(LST_NAME_SIZE, name, i);
1253                 if (rc == 0) {
1254                         fprintf(stdout, "%d) %s\n", i + 1, name);
1255                         continue;
1256                 }
1257
1258                 if (errno == ENOENT)
1259                         break;
1260
1261                 lst_print_error("group", "Failed to list group: %s\n",
1262                                 strerror(errno));
1263                 return -1;
1264         }
1265
1266         fprintf(stdout, "Total %d groups\n", i);
1267
1268         return 0;
1269 }
1270
1271 #define LST_NODES_TITLE "\tACTIVE\tBUSY\tDOWN\tUNKNOWN\tTOTAL\n"
1272
1273 int
1274 jt_lst_list_group(int argc, char **argv)
1275 {
1276         lstcon_ndlist_ent_t  gent;
1277         lstcon_node_ent_t   *dents;
1278         int               optidx  = 0;
1279         int               verbose = 0;
1280         int               active  = 0;
1281         int               busy    = 0;
1282         int               down    = 0;
1283         int               unknown = 0;
1284         int               all     = 0;
1285         int               count;
1286         int               index;
1287         int               i;
1288         int               j;
1289         int               c;
1290         int               rc = 0;
1291
1292         static struct option list_group_opts[] =
1293         {
1294                 {"active",  no_argument, 0, 'a' },
1295                 {"busy",    no_argument, 0, 'b' },
1296                 {"down",    no_argument, 0, 'd' },
1297                 {"unknown", no_argument, 0, 'u' },
1298                 {"all",     no_argument, 0, 'l' },
1299                 {0,         0,           0,  0  }
1300         };
1301
1302         if (session_key == 0) {
1303                 fprintf(stderr,
1304                         "Can't find env LST_SESSION or value is not valid\n");
1305                 return -1;
1306         }
1307
1308         while (1) {
1309                 c = getopt_long(argc, argv, "abdul",
1310                                 list_group_opts, &optidx);
1311
1312                 if (c == -1)
1313                         break;
1314
1315                 switch (c) {
1316                 case 'a':
1317                         verbose = active = 1;
1318                         all = 0;
1319                         break;
1320                 case 'b':
1321                         verbose = busy = 1;
1322                         all = 0;
1323                         break;
1324                 case 'd':
1325                         verbose = down = 1;
1326                         all = 0;
1327                         break;
1328                 case 'u':
1329                         verbose = unknown = 1;
1330                         all = 0;
1331                         break;
1332                 case 'l':
1333                         verbose = all = 1;
1334                         break;
1335                 default:
1336                         lst_print_usage(argv[0]);
1337                         return -1;
1338                 }
1339         }
1340
1341         if (optind == argc) {
1342                 /* no group is specified, list name of all groups */
1343                 rc = lst_list_group_all();
1344
1345                 return rc;
1346         }
1347
1348         if (!verbose)
1349                 fprintf(stdout, LST_NODES_TITLE);
1350
1351         /* list nodes in specified groups */
1352         for (i = optind; i < argc; i++) {
1353                 rc = lst_info_group_ioctl(argv[i], &gent, NULL, NULL, NULL);
1354                 if (rc != 0) {
1355                         if (errno == ENOENT) {
1356                                 rc = 0;
1357                                 break;
1358                         }
1359
1360                         lst_print_error("group", "Failed to list group\n",
1361                                         strerror(errno));
1362                         break;
1363                 }
1364
1365                 if (!verbose) {
1366                         fprintf(stdout, "\t%d\t%d\t%d\t%d\t%d\t%s\n",
1367                                 gent.nle_nactive, gent.nle_nbusy,
1368                                 gent.nle_ndown, gent.nle_nunknown,
1369                                 gent.nle_nnode, argv[i]);
1370                         continue;
1371                 }
1372
1373                 fprintf(stdout, "Group [ %s ]\n", argv[i]);
1374
1375                 if (gent.nle_nnode == 0) {
1376                         fprintf(stdout, "No nodes found [ %s ]\n", argv[i]);
1377                         continue;
1378                 }
1379
1380                 count = gent.nle_nnode;
1381
1382                 dents = malloc(count * sizeof(lstcon_node_ent_t));
1383                 if (dents == NULL) {
1384                         fprintf(stderr, "Failed to malloc: %s\n",
1385                                 strerror(errno));
1386                         return -1;
1387                 }
1388
1389                 index = 0;
1390                 rc = lst_info_group_ioctl(argv[i], &gent, &index, &count, dents);
1391                 if (rc != 0) {
1392                         lst_print_error("group", "Failed to list group: %s\n",
1393                                         strerror(errno));
1394                         free(dents);
1395                         return -1;
1396                 }
1397
1398                 for (j = 0, c = 0; j < count; j++) {
1399                         if (all ||
1400                             ((active  &&  dents[j].nde_state == LST_NODE_ACTIVE) ||
1401                              (busy    &&  dents[j].nde_state == LST_NODE_BUSY)   ||
1402                              (down    &&  dents[j].nde_state == LST_NODE_DOWN)   ||
1403                              (unknown &&  dents[j].nde_state == LST_NODE_UNKNOWN))) {
1404
1405                                 fprintf(stdout, "\t%s: %s\n",
1406                                         libcfs_id2str(dents[j].nde_id),
1407                                         lst_node_state2str(dents[j].nde_state));
1408                                 c++;
1409                         }
1410                 }
1411
1412                 fprintf(stdout, "Total %d nodes [ %s ]\n", c, argv[i]);
1413
1414                 free(dents);
1415         }
1416
1417         return rc;
1418 }
1419
1420 int
1421 lst_stat_ioctl (char *name, int count, lnet_process_id_t *idsp,
1422                 int timeout, struct list_head *resultp)
1423 {
1424         lstio_stat_args_t  args = {
1425                 .lstio_sta_key           = session_key,
1426                 .lstio_sta_timeout       = timeout,
1427                 .lstio_sta_nmlen         = strlen(name),
1428                 .lstio_sta_namep         = name,
1429                 .lstio_sta_count         = count,
1430                 .lstio_sta_idsp          = idsp,
1431                 .lstio_sta_resultp       = resultp,
1432         };
1433
1434         return lst_ioctl (LSTIO_STAT_QUERY, &args, sizeof(args));
1435 }
1436
1437 typedef struct {
1438         struct list_head        srp_link;
1439         int                     srp_count;
1440         char                   *srp_name;
1441         lnet_process_id_t      *srp_ids;
1442         struct list_head        srp_result[2];
1443 } lst_stat_req_param_t;
1444
1445 static void
1446 lst_stat_req_param_free(lst_stat_req_param_t *srp)
1447 {
1448         int     i;
1449
1450         for (i = 0; i < 2; i++)
1451                 lst_free_rpcent(&srp->srp_result[i]);
1452
1453         if (srp->srp_ids != NULL)
1454                 free(srp->srp_ids);
1455
1456         free(srp);
1457 }
1458
1459 static int
1460 lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp, int save_old)
1461 {
1462         lst_stat_req_param_t *srp = NULL;
1463         int                   count = save_old ? 2 : 1;
1464         int                   rc;
1465         int                   i;
1466
1467         srp = malloc(sizeof(*srp));
1468         if (srp == NULL)
1469                 return -ENOMEM;
1470
1471         memset(srp, 0, sizeof(*srp));
1472         CFS_INIT_LIST_HEAD(&srp->srp_result[0]);
1473         CFS_INIT_LIST_HEAD(&srp->srp_result[1]);
1474
1475         rc = lst_get_node_count(LST_OPC_GROUP, name,
1476                                 &srp->srp_count, NULL);
1477         if (rc != 0 && errno == ENOENT) {
1478                 rc = lst_get_node_count(LST_OPC_NODES, name,
1479                                         &srp->srp_count, &srp->srp_ids);
1480         }
1481
1482         if (rc != 0) {
1483                 fprintf(stderr,
1484                         "Failed to get count of nodes from %s: %s\n",
1485                         name, strerror(errno));
1486                 lst_stat_req_param_free(srp);
1487
1488                 return rc;
1489         }
1490
1491         srp->srp_name = name;
1492
1493         for (i = 0; i < count; i++) {
1494                 rc = lst_alloc_rpcent(&srp->srp_result[i], srp->srp_count,
1495                                       sizeof(sfw_counters_t)  +
1496                                       sizeof(srpc_counters_t) +
1497                                       sizeof(lnet_counters_t));
1498                 if (rc != 0) {
1499                         fprintf(stderr, "Out of memory\n");
1500                         break;
1501                 }
1502         }
1503
1504         if (rc == 0) {
1505                 *srpp = srp;
1506                 return 0;
1507         }
1508
1509         lst_stat_req_param_free(srp);
1510
1511         return rc;
1512 }
1513
1514 typedef struct {
1515         /* TODO */
1516 } lst_srpc_stat_result;
1517
1518 #define LST_LNET_AVG    0
1519 #define LST_LNET_MIN    1
1520 #define LST_LNET_MAX    2
1521
1522 typedef struct {
1523         float           lnet_avg_sndrate;
1524         float           lnet_min_sndrate;
1525         float           lnet_max_sndrate;
1526         float           lnet_total_sndrate;
1527
1528         float           lnet_avg_rcvrate;
1529         float           lnet_min_rcvrate;
1530         float           lnet_max_rcvrate;
1531         float           lnet_total_rcvrate;
1532
1533         float           lnet_avg_sndperf;
1534         float           lnet_min_sndperf;
1535         float           lnet_max_sndperf;
1536         float           lnet_total_sndperf;
1537
1538         float           lnet_avg_rcvperf;
1539         float           lnet_min_rcvperf;
1540         float           lnet_max_rcvperf;
1541         float           lnet_total_rcvperf;
1542
1543         int             lnet_stat_count;
1544 } lst_lnet_stat_result_t;
1545
1546 lst_lnet_stat_result_t lnet_stat_result;
1547
1548 static float
1549 lst_lnet_stat_value(int bw, int send, int off)
1550 {
1551         float  *p;
1552
1553         p = bw ? &lnet_stat_result.lnet_avg_sndperf :
1554                  &lnet_stat_result.lnet_avg_sndrate;
1555
1556         if (!send)
1557                 p += 4;
1558
1559         p += off;
1560
1561         return *p;
1562 }
1563
1564 static void
1565 lst_timeval_diff(struct timeval *tv1,
1566                  struct timeval *tv2, struct timeval *df)
1567 {
1568         if (tv1->tv_usec >= tv2->tv_usec) {
1569                 df->tv_sec  = tv1->tv_sec - tv2->tv_sec;
1570                 df->tv_usec = tv1->tv_usec - tv2->tv_usec;
1571                 return;
1572         }
1573
1574         df->tv_sec  = tv1->tv_sec - 1 - tv2->tv_sec;
1575         df->tv_usec = tv1->tv_sec + 1000000 - tv2->tv_usec;
1576
1577         return;
1578 }
1579
1580 void
1581 lst_cal_lnet_stat(float delta, lnet_counters_t *lnet_new,
1582                   lnet_counters_t *lnet_old)
1583 {
1584         float perf;
1585         float rate;
1586
1587         perf = (float)(lnet_new->send_length -
1588                        lnet_old->send_length) / (1024 * 1024) / delta;
1589         lnet_stat_result.lnet_total_sndperf += perf;
1590
1591         if (lnet_stat_result.lnet_min_sndperf > perf ||
1592             lnet_stat_result.lnet_min_sndperf == 0)
1593                 lnet_stat_result.lnet_min_sndperf = perf;
1594
1595         if (lnet_stat_result.lnet_max_sndperf < perf)
1596                 lnet_stat_result.lnet_max_sndperf = perf;
1597
1598         perf = (float)(lnet_new->recv_length -
1599                        lnet_old->recv_length) / (1024 * 1024) / delta;
1600         lnet_stat_result.lnet_total_rcvperf += perf;
1601
1602         if (lnet_stat_result.lnet_min_rcvperf > perf ||
1603             lnet_stat_result.lnet_min_rcvperf == 0)
1604                 lnet_stat_result.lnet_min_rcvperf = perf;
1605
1606         if (lnet_stat_result.lnet_max_rcvperf < perf)
1607                 lnet_stat_result.lnet_max_rcvperf = perf;
1608
1609         rate = (lnet_new->send_count - lnet_old->send_count) / delta;
1610         lnet_stat_result.lnet_total_sndrate += rate;
1611
1612         if (lnet_stat_result.lnet_min_sndrate > rate ||
1613             lnet_stat_result.lnet_min_sndrate == 0)
1614                 lnet_stat_result.lnet_min_sndrate = rate;
1615
1616         if (lnet_stat_result.lnet_max_sndrate < rate)
1617                 lnet_stat_result.lnet_max_sndrate = rate;
1618
1619         rate = (lnet_new->recv_count - lnet_old->recv_count) / delta;
1620         lnet_stat_result.lnet_total_rcvrate += rate;
1621
1622         if (lnet_stat_result.lnet_min_rcvrate > rate ||
1623             lnet_stat_result.lnet_min_rcvrate == 0)
1624                 lnet_stat_result.lnet_min_rcvrate = rate;
1625
1626         if (lnet_stat_result.lnet_max_rcvrate < rate)
1627                 lnet_stat_result.lnet_max_rcvrate = rate;
1628
1629         lnet_stat_result.lnet_stat_count ++;
1630
1631         lnet_stat_result.lnet_avg_sndrate = lnet_stat_result.lnet_total_sndrate /
1632                                             lnet_stat_result.lnet_stat_count;
1633         lnet_stat_result.lnet_avg_rcvrate = lnet_stat_result.lnet_total_rcvrate /
1634                                             lnet_stat_result.lnet_stat_count;
1635
1636         lnet_stat_result.lnet_avg_sndperf = lnet_stat_result.lnet_total_sndperf /
1637                                             lnet_stat_result.lnet_stat_count;
1638         lnet_stat_result.lnet_avg_rcvperf = lnet_stat_result.lnet_total_rcvperf /
1639                                             lnet_stat_result.lnet_stat_count;
1640
1641 }
1642
1643 void
1644 lst_print_lnet_stat(char *name, int bwrt, int rdwr, int type)
1645 {
1646         int     start1 = 0;
1647         int     end1   = 1;
1648         int     start2 = 0;
1649         int     end2   = 1;
1650         int     i;
1651         int     j;
1652
1653         if (lnet_stat_result.lnet_stat_count == 0)
1654                 return;
1655
1656         if (bwrt == 1) /* bw only */
1657                 start1 = 1;
1658
1659         if (bwrt == 2) /* rates only */
1660                 end1 = 0;
1661
1662         if (rdwr == 1) /* recv only */
1663                 start2 = 1;
1664
1665         if (rdwr == 2) /* send only */
1666                 end2 = 0;
1667
1668         for (i = start1; i <= end1; i++) {
1669                 fprintf(stdout, "[LNet %s of %s]\n",
1670                         i == 0 ? "Rates" : "Bandwidth", name);
1671
1672                 for (j = start2; j <= end2; j++) {
1673                         fprintf(stdout, "[%c] ", j == 0 ? 'R' : 'W');
1674
1675                         if ((type & 1) != 0) {
1676                                 fprintf(stdout, i == 0 ? "Avg: %-8.0f RPC/s " :
1677                                                          "Avg: %-8.2f MB/s  ",
1678                                         lst_lnet_stat_value(i, j, 0));
1679                         }
1680
1681                         if ((type & 2) != 0) {
1682                                 fprintf(stdout, i == 0 ? "Min: %-8.0f RPC/s " :
1683                                                          "Min: %-8.2f MB/s  ",
1684                                         lst_lnet_stat_value(i, j, 1));
1685                         }
1686
1687                         if ((type & 4) != 0) {
1688                                 fprintf(stdout, i == 0 ? "Max: %-8.0f RPC/s" :
1689                                                          "Max: %-8.2f MB/s",
1690                                         lst_lnet_stat_value(i, j, 2));
1691                         }
1692
1693                         fprintf(stdout, "\n");
1694                 }
1695         }
1696 }
1697
1698 void
1699 lst_print_stat(char *name, struct list_head *resultp,
1700                int idx, int lnet, int bwrt, int rdwr, int type)
1701 {
1702         struct list_head  tmp[2];
1703         lstcon_rpc_ent_t *new;
1704         lstcon_rpc_ent_t *old;
1705         sfw_counters_t   *sfwk_new;
1706         sfw_counters_t   *sfwk_old;
1707         srpc_counters_t  *srpc_new;
1708         srpc_counters_t  *srpc_old;
1709         lnet_counters_t  *lnet_new;
1710         lnet_counters_t  *lnet_old;
1711         struct timeval    tv;
1712         float             delta;
1713         int               errcount = 0;
1714
1715         CFS_INIT_LIST_HEAD(&tmp[0]);
1716         CFS_INIT_LIST_HEAD(&tmp[1]);
1717
1718         memset(&lnet_stat_result, 0, sizeof(lnet_stat_result));
1719
1720         while (!list_empty(&resultp[idx])) {
1721                 if (list_empty(&resultp[1 - idx])) {
1722                         fprintf(stderr, "Group is changed, re-run stat\n");
1723                         break;
1724                 }
1725
1726                 new = list_entry(resultp[idx].next, lstcon_rpc_ent_t, rpe_link);
1727                 old = list_entry(resultp[1 - idx].next, lstcon_rpc_ent_t, rpe_link);
1728
1729                 /* first time get stats result, can't calculate diff */
1730                 if (new->rpe_peer.nid == LNET_NID_ANY)
1731                         break;
1732
1733                 if (new->rpe_peer.nid != old->rpe_peer.nid ||
1734                     new->rpe_peer.pid != old->rpe_peer.pid) {
1735                         /* Something wrong. i.e, somebody change the group */
1736                         break;
1737                 }
1738
1739                 list_del(&new->rpe_link);
1740                 list_add_tail(&new->rpe_link, &tmp[idx]);
1741
1742                 list_del(&old->rpe_link);
1743                 list_add_tail(&old->rpe_link, &tmp[1 - idx]);
1744
1745                 if (new->rpe_rpc_errno != 0 || new->rpe_fwk_errno != 0 ||
1746                     old->rpe_rpc_errno != 0 || old->rpe_fwk_errno != 0) {
1747                         errcount ++;
1748                         continue;
1749                 }
1750
1751                 sfwk_new = (sfw_counters_t *)&new->rpe_payload[0];
1752                 sfwk_old = (sfw_counters_t *)&old->rpe_payload[0];
1753
1754                 srpc_new = (srpc_counters_t *)((char *)sfwk_new + sizeof(*sfwk_new));
1755                 srpc_old = (srpc_counters_t *)((char *)sfwk_old + sizeof(*sfwk_old));
1756
1757                 lnet_new = (lnet_counters_t *)((char *)srpc_new + sizeof(*srpc_new));
1758                 lnet_old = (lnet_counters_t *)((char *)srpc_old + sizeof(*srpc_old));
1759
1760                 lst_timeval_diff(&new->rpe_stamp, &old->rpe_stamp, &tv);
1761
1762                 delta = tv.tv_sec + (float)tv.tv_usec/1000000;
1763
1764                 if (!lnet) /* TODO */
1765                         continue;
1766
1767                 lst_cal_lnet_stat(delta, lnet_new, lnet_old);
1768         }
1769
1770         list_splice(&tmp[idx], &resultp[idx]);
1771         list_splice(&tmp[1 - idx], &resultp[1 - idx]);
1772
1773         if (errcount > 0)
1774                 fprintf(stdout, "Failed to stat on %d nodes\n", errcount);
1775
1776         if (!lnet)  /* TODO */
1777                 return;
1778
1779         lst_print_lnet_stat(name, bwrt, rdwr, type);
1780 }
1781
1782 int
1783 jt_lst_stat(int argc, char **argv)
1784 {
1785         struct list_head      head;
1786         lst_stat_req_param_t *srp;
1787         time_t                last    = 0;
1788         int                   optidx  = 0;
1789         int                   timeout = 5; /* default timeout, 5 sec */
1790         int                   delay   = 5; /* default delay, 5 sec */
1791         int                   lnet    = 1; /* lnet stat by default */
1792         int                   bwrt    = 0;
1793         int                   rdwr    = 0;
1794         int                   type    = -1;
1795         int                   idx     = 0;
1796         int                   rc;
1797         int                   c;
1798
1799         static struct option stat_opts[] =
1800         {
1801                 {"timeout", required_argument, 0, 't' },
1802                 {"delay"  , required_argument, 0, 'd' },
1803                 {"lnet"   , no_argument,       0, 'l' },
1804                 {"rpc"    , no_argument,       0, 'c' },
1805                 {"bw"     , no_argument,       0, 'b' },
1806                 {"rate"   , no_argument,       0, 'a' },
1807                 {"read"   , no_argument,       0, 'r' },
1808                 {"write"  , no_argument,       0, 'w' },
1809                 {"avg"    , no_argument,       0, 'g' },
1810                 {"min"    , no_argument,       0, 'n' },
1811                 {"max"    , no_argument,       0, 'x' },
1812                 {0,         0,                 0,  0  }
1813         };
1814
1815         if (session_key == 0) {
1816                 fprintf(stderr,
1817                         "Can't find env LST_SESSION or value is not valid\n");
1818                 return -1;
1819         }
1820
1821         while (1) {
1822                 c = getopt_long(argc, argv, "t:d:lcbarwgnx", stat_opts, &optidx);
1823
1824                 if (c == -1)
1825                         break;
1826
1827                 switch (c) {
1828                 case 't':
1829                         timeout = atoi(optarg);
1830                         break;
1831                 case 'd':
1832                         delay = atoi(optarg);
1833                         break;
1834                 case 'l':
1835                         lnet = 1;
1836                         break;
1837                 case 'c':
1838                         lnet = 0;
1839                         break;
1840                 case 'b':
1841                         bwrt |= 1;
1842                         break;
1843                 case 'a':
1844                         bwrt |= 2;
1845                         break;
1846                 case 'r':
1847                         rdwr |= 1;
1848                         break;
1849                 case 'w':
1850                         rdwr |= 2;
1851                         break;
1852                 case 'g':
1853                         if (type == -1) {
1854                                 type = 1;
1855                                 break;
1856                         }
1857                         type |= 1;
1858                         break;
1859                 case 'n':
1860                         if (type == -1) {
1861                                 type = 2;
1862                                 break;
1863                         }
1864                         type |= 2;
1865                         break;
1866                 case 'x':
1867                         if (type == -1) {
1868                                 type = 4;
1869                                 break;
1870                         }
1871                         type |= 4;
1872                         break;
1873                 default:
1874                         lst_print_usage(argv[0]);
1875                         return -1;
1876                 }
1877         }
1878
1879         if (optind == argc) {
1880                 lst_print_usage(argv[0]);
1881                 return -1;
1882         }
1883
1884         if (timeout <= 0 || delay <= 0) {
1885                 fprintf(stderr, "Invalid timeout or delay value\n");
1886                 return -1;
1887         }
1888
1889         CFS_INIT_LIST_HEAD(&head);
1890
1891         while (optind < argc) {
1892                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 1);
1893                 if (rc != 0)
1894                         goto out;
1895
1896                 list_add_tail(&srp->srp_link, &head);
1897         }
1898
1899         while (1) {
1900                 time_t  now = time(NULL);
1901
1902                 if (now - last < delay) {
1903                         sleep(delay - now + last);
1904                         time(&now);
1905                 }
1906
1907                 last = now;
1908
1909                 list_for_each_entry(srp, &head, srp_link) {
1910                         rc = lst_stat_ioctl(srp->srp_name,
1911                                             srp->srp_count, srp->srp_ids,
1912                                             timeout, &srp->srp_result[idx]);
1913                         if (rc == -1) {
1914                                 lst_print_error("stat", "Failed to stat %s: %s\n",
1915                                                 srp->srp_name, strerror(errno));
1916                                 goto out;
1917                         }
1918
1919                         lst_print_stat(srp->srp_name, srp->srp_result,
1920                                        idx, lnet, bwrt, rdwr, type);
1921
1922                         lst_reset_rpcent(&srp->srp_result[1 - idx]);
1923                 }
1924
1925                 idx = 1 - idx;
1926         }
1927
1928 out:
1929         while (!list_empty(&head)) {
1930                 srp = list_entry(head.next, lst_stat_req_param_t, srp_link);
1931
1932                 list_del(&srp->srp_link);
1933                 lst_stat_req_param_free(srp);
1934         }
1935
1936         return rc;
1937 }
1938
1939 int
1940 jt_lst_show_error(int argc, char **argv)
1941 {
1942         struct list_head      head;
1943         lst_stat_req_param_t *srp;
1944         lstcon_rpc_ent_t     *ent;
1945         sfw_counters_t       *sfwk;
1946         srpc_counters_t      *srpc;
1947         lnet_counters_t      *lnet;
1948         int                   show_rpc = 1;
1949         int                   optidx   = 0;
1950         int                   rc       = 0;
1951         int                   ecount;
1952         int                   c;
1953
1954         static struct option  show_error_opts[] =
1955         {
1956                 {"session", no_argument,       0, 's' },
1957                 {0,         0,                 0,  0  }
1958         };
1959
1960         if (session_key == 0) {
1961                 fprintf(stderr,
1962                         "Can't find env LST_SESSION or value is not valid\n");
1963                 return -1;
1964         }
1965
1966         while (1) {
1967                 c = getopt_long(argc, argv, "s", show_error_opts, &optidx);
1968
1969                 if (c == -1)
1970                         break;
1971
1972                 switch (c) {
1973                 case 's':
1974                         show_rpc  = 0;
1975                         break;
1976
1977                 default:
1978                         lst_print_usage(argv[0]);
1979                         return -1;
1980                 }
1981         }
1982
1983         if (optind == argc) {
1984                 lst_print_usage(argv[0]);
1985                 return -1;
1986         }
1987
1988         CFS_INIT_LIST_HEAD(&head);
1989
1990         while (optind < argc) {
1991                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 0);
1992                 if (rc != 0)
1993                         goto out;
1994
1995                 list_add_tail(&srp->srp_link, &head);
1996         }
1997
1998         list_for_each_entry(srp, &head, srp_link) {
1999                 rc = lst_stat_ioctl(srp->srp_name, srp->srp_count,
2000                                     srp->srp_ids, 5, &srp->srp_result[0]);
2001
2002                 if (rc == -1) {
2003                         lst_print_error(srp->srp_name, "Failed to show errors of %s: %s\n",
2004                                         srp->srp_name, strerror(errno));
2005                         goto out;
2006                 }
2007
2008                 fprintf(stdout, "%s:\n", srp->srp_name);
2009
2010                 ecount = 0;
2011
2012                 list_for_each_entry(ent, &srp->srp_result[0], rpe_link) {
2013                         if (ent->rpe_rpc_errno != 0) {
2014                                 ecount ++;
2015                                 fprintf(stderr, "RPC failure, can't show error on %s\n",
2016                                         libcfs_id2str(ent->rpe_peer));
2017                                 continue;
2018                         }
2019
2020                         if (ent->rpe_fwk_errno != 0) {
2021                                 ecount ++;
2022                                 fprintf(stderr, "Framework failure, can't show error on %s\n",
2023                                         libcfs_id2str(ent->rpe_peer));
2024                                 continue;
2025                         }
2026
2027                         sfwk = (sfw_counters_t *)&ent->rpe_payload[0];
2028                         srpc = (srpc_counters_t *)((char *)sfwk + sizeof(*sfwk));
2029                         lnet = (lnet_counters_t *)((char *)srpc + sizeof(*srpc));
2030
2031                         if (srpc->errors == 0 &&
2032                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2033                                 continue;
2034
2035                         if (!show_rpc  &&
2036                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2037                                 continue;
2038
2039                         ecount ++;
2040
2041                         fprintf(stderr, "%s: [Session %d brw errors, %d ping errors]%c",
2042                                 libcfs_id2str(ent->rpe_peer),
2043                                 sfwk->brw_errors, sfwk->ping_errors,
2044                                 show_rpc  ? ' ' : '\n');
2045
2046                         if (!show_rpc)
2047                                 continue;
2048
2049                         fprintf(stderr, "[RPC: %d errors, %d dropped, %d expired]\n",
2050                                 srpc->errors, srpc->rpcs_dropped, srpc->rpcs_expired);
2051                 }
2052
2053                 fprintf(stdout, "Total %d error nodes in %s\n", ecount, srp->srp_name);
2054         }
2055 out:
2056         while (!list_empty(&head)) {
2057                 srp = list_entry(head.next, lst_stat_req_param_t, srp_link);
2058
2059                 list_del(&srp->srp_link);
2060                 lst_stat_req_param_free(srp);
2061         }
2062
2063         return rc;
2064 }
2065
2066 int
2067 lst_add_batch_ioctl (char *name)
2068 {
2069         lstio_batch_add_args_t  args = {
2070                 .lstio_bat_key           = session_key,
2071                 .lstio_bat_nmlen         = strlen(name),
2072                 .lstio_bat_namep         = name,
2073         };
2074
2075         return lst_ioctl (LSTIO_BATCH_ADD, &args, sizeof(args));
2076 }
2077
2078 int
2079 jt_lst_add_batch(int argc, char **argv)
2080 {
2081         char   *name;
2082         int     rc;
2083
2084         if (session_key == 0) {
2085                 fprintf(stderr,
2086                         "Can't find env LST_SESSION or value is not valid\n");
2087                 return -1;
2088         }
2089
2090         if (argc != 2) {
2091                 lst_print_usage(argv[0]);
2092                 return -1;
2093         }
2094
2095         name = argv[1];
2096         if (strlen(name) >= LST_NAME_SIZE) {
2097                 fprintf(stderr, "Name length is limited to %d\n",
2098                         LST_NAME_SIZE - 1);
2099                 return -1;
2100         }
2101
2102         rc = lst_add_batch_ioctl(name);
2103         if (rc == 0)
2104                 return 0;
2105
2106         lst_print_error("batch", "Failed to create batch: %s\n",
2107                         strerror(errno));
2108
2109         return -1;
2110 }
2111
2112 int
2113 lst_start_batch_ioctl (char *name, int timeout, struct list_head *resultp)
2114 {
2115         lstio_batch_run_args_t   args = {
2116                 .lstio_bat_key          = session_key,
2117                 .lstio_bat_timeout      = timeout,
2118                 .lstio_bat_nmlen        = strlen(name),
2119                 .lstio_bat_namep        = name,
2120                 .lstio_bat_resultp      = resultp,
2121         };
2122
2123         return lst_ioctl(LSTIO_BATCH_START, &args, sizeof(args));
2124 }
2125
2126 int
2127 jt_lst_start_batch(int argc, char **argv)
2128 {
2129         struct list_head  head;
2130         char             *batch;
2131         int               optidx  = 0;
2132         int               timeout = 0;
2133         int               count = 0;
2134         int               rc;
2135         int               c;
2136
2137         static struct option start_batch_opts[] =
2138         {
2139                 {"timeout", required_argument, 0, 't' },
2140                 {0,         0,                 0,  0  }
2141         };
2142
2143         if (session_key == 0) {
2144                 fprintf(stderr,
2145                         "Can't find env LST_SESSION or value is not valid\n");
2146                 return -1;
2147         }
2148
2149         while (1) {
2150                 c = getopt_long(argc, argv, "t:",
2151                                 start_batch_opts, &optidx);
2152
2153                 /* Detect the end of the options. */
2154                 if (c == -1)
2155                         break;
2156
2157                 switch (c) {
2158                 case 't':
2159                         timeout = atoi(optarg);
2160                         break;
2161                 default:
2162                         lst_print_usage(argv[0]);
2163                         return -1;
2164                 }
2165         }
2166
2167         if (optind == argc) {
2168                 batch = LST_DEFAULT_BATCH;
2169
2170         } else if (optind == argc - 1) {
2171                 batch = argv[optind];
2172
2173         } else {
2174                 lst_print_usage(argv[0]);
2175                 return -1;
2176         }
2177
2178         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2179         if (rc != 0) {
2180                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2181                         batch, strerror(errno));
2182                 return -1;
2183         }
2184
2185         CFS_INIT_LIST_HEAD(&head);
2186
2187         rc = lst_alloc_rpcent(&head, count, 0);
2188         if (rc != 0) {
2189                 fprintf(stderr, "Out of memory\n");
2190                 return -1;
2191         }
2192
2193         rc = lst_start_batch_ioctl(batch, timeout, &head);
2194
2195         if (rc == 0) {
2196                 fprintf(stdout, "%s is running now\n", batch);
2197                 lst_free_rpcent(&head);
2198                 return 0;
2199         }
2200
2201         if (rc == -1) {
2202                 lst_print_error("batch", "Failed to start batch: %s\n",
2203                                 strerror(errno));
2204                 lst_free_rpcent(&head);
2205                 return rc;
2206         }
2207
2208         lst_print_transerr(&head, "Run batch");
2209
2210         lst_free_rpcent(&head);
2211
2212         return rc;
2213 }
2214
2215 int
2216 lst_stop_batch_ioctl(char *name, int force, struct list_head *resultp)
2217 {
2218         lstio_batch_stop_args_t   args = {
2219                 .lstio_bat_key          = session_key,
2220                 .lstio_bat_force        = force,
2221                 .lstio_bat_nmlen        = strlen(name),
2222                 .lstio_bat_namep        = name,
2223                 .lstio_bat_resultp      = resultp,
2224         };
2225
2226         return lst_ioctl(LSTIO_BATCH_STOP, &args, sizeof(args));
2227 }
2228
2229 int
2230 jt_lst_stop_batch(int argc, char **argv)
2231 {
2232         struct list_head  head;
2233         char             *batch;
2234         int               force = 0;
2235         int               optidx;
2236         int               count;
2237         int               rc;
2238         int               c;
2239
2240         static struct option stop_batch_opts[] =
2241         {
2242                 {"force",   no_argument,   0, 'f' },
2243                 {0,         0,             0,  0  }
2244         };
2245
2246         if (session_key == 0) {
2247                 fprintf(stderr,
2248                         "Can't find env LST_SESSION or value is not valid\n");
2249                 return -1;
2250         }
2251
2252         while (1) {
2253                 c = getopt_long(argc, argv, "f",
2254                                 stop_batch_opts, &optidx);
2255
2256                 /* Detect the end of the options. */
2257                 if (c == -1)
2258                         break;
2259
2260                 switch (c) {
2261                 case 'f':
2262                         force = 1;
2263                         break;
2264                 default:
2265                         lst_print_usage(argv[0]);
2266                         return -1;
2267                 }
2268         }
2269
2270         if (optind == argc) {
2271                 batch = LST_DEFAULT_BATCH;
2272
2273         } else if (optind == argc - 1) {
2274                 batch = argv[optind];
2275
2276         } else {
2277                 lst_print_usage(argv[0]);
2278                 return -1;
2279         }
2280
2281         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2282         if (rc != 0) {
2283                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2284                         batch, strerror(errno));
2285                 return -1;
2286         }
2287
2288         CFS_INIT_LIST_HEAD(&head);
2289
2290         rc = lst_alloc_rpcent(&head, count, 0);
2291         if (rc != 0) {
2292                 fprintf(stderr, "Out of memory\n");
2293                 return -1;
2294         }
2295
2296         rc = lst_stop_batch_ioctl(batch, force, &head);
2297         if (rc != 0)
2298                 goto out;
2299
2300         while (1) {
2301                 lst_reset_rpcent(&head);
2302
2303                 rc = lst_query_batch_ioctl(batch, 0, 0, 30, &head);
2304                 if (rc != 0)
2305                         goto out;
2306
2307                 if (lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0 &&
2308                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0)
2309                         break;
2310
2311                 fprintf(stdout, "%d batch in stopping\n",
2312                         lstcon_tsbqry_stat_run(&trans_stat, 0));
2313                 sleep(1);
2314         }
2315
2316         fprintf(stdout, "Batch is stopped\n");
2317         lst_free_rpcent(&head);
2318
2319         return 0;
2320 out:
2321         if (rc == -1) {
2322                 lst_print_error("batch", "Failed to stop batch: %s\n",
2323                                 strerror(errno));
2324                 lst_free_rpcent(&head);
2325                 return -1;
2326         }
2327
2328         lst_print_transerr(&head, "stop batch");
2329
2330         lst_free_rpcent(&head);
2331
2332         return rc;
2333 }
2334
2335 int
2336 lst_list_batch_ioctl(int len, char *name, int index)
2337 {
2338         lstio_batch_list_args_t         args = {
2339                 .lstio_bat_key          = session_key,
2340                 .lstio_bat_idx          = index,
2341                 .lstio_bat_nmlen        = len,
2342                 .lstio_bat_namep        = name,
2343         };
2344
2345         return lst_ioctl(LSTIO_BATCH_LIST, &args, sizeof(args));
2346 }
2347
2348 int
2349 lst_info_batch_ioctl(char *batch, int test, int server,
2350                      lstcon_test_batch_ent_t *entp, int *idxp,
2351                      int *ndentp, lstcon_node_ent_t *dentsp)
2352 {
2353         lstio_batch_info_args_t         args = {
2354                 .lstio_bat_key          = session_key,
2355                 .lstio_bat_nmlen        = strlen(batch),
2356                 .lstio_bat_namep        = batch,
2357                 .lstio_bat_server       = server,
2358                 .lstio_bat_testidx      = test,
2359                 .lstio_bat_entp         = entp,
2360                 .lstio_bat_idxp         = idxp,
2361                 .lstio_bat_ndentp       = ndentp,
2362                 .lstio_bat_dentsp       = dentsp,
2363         };
2364
2365         return lst_ioctl(LSTIO_BATCH_INFO, &args, sizeof(args));
2366 }
2367
2368 int
2369 lst_list_batch_all(void)
2370 {
2371         char name[LST_NAME_SIZE];
2372         int  rc;
2373         int  i;
2374
2375         for (i = 0; ; i++) {
2376                 rc = lst_list_batch_ioctl(LST_NAME_SIZE, name, i);
2377                 if (rc == 0) {
2378                         fprintf(stdout, "%d) %s\n", i + 1, name);
2379                         continue;
2380                 }
2381
2382                 if (errno == ENOENT)
2383                         break;
2384
2385                 lst_print_error("batch", "Failed to list batch: %s\n",
2386                                 strerror(errno));
2387                 return rc;
2388         }
2389
2390         fprintf(stdout, "Total %d batches\n", i);
2391
2392         return 0;
2393 }
2394
2395 int
2396 lst_list_tsb_nodes(char *batch, int test, int server,
2397                    int count, int active, int invalid)
2398 {
2399         lstcon_node_ent_t *dents;
2400         int                index = 0;
2401         int                rc;
2402         int                c;
2403         int                i;
2404
2405         if (count == 0)
2406                 return 0;
2407
2408         /* verbose list, show nodes in batch or test */
2409         dents = malloc(count * sizeof(lstcon_node_ent_t));
2410         if (dents == NULL) {
2411                 fprintf(stdout, "Can't allocate memory\n");
2412                 return -1;
2413         }
2414
2415         rc = lst_info_batch_ioctl(batch, test, server,
2416                                   NULL, &index, &count, dents);
2417         if (rc != 0) {
2418                 free(dents);
2419                 lst_print_error((test > 0) ? "test" : "batch",
2420                                 (test > 0) ? "Failed to query test: %s\n" :
2421                                              "Failed to query batch: %s\n",
2422                                 strerror(errno));
2423                 return -1;
2424         }
2425
2426         for (i = 0, c = 0; i < count; i++) {
2427                 if ((!active  && dents[i].nde_state == LST_NODE_ACTIVE) ||
2428                     (!invalid && (dents[i].nde_state == LST_NODE_BUSY  ||
2429                                   dents[i].nde_state == LST_NODE_DOWN  ||
2430                                   dents[i].nde_state == LST_NODE_UNKNOWN)))
2431                         continue;
2432
2433                 fprintf(stdout, "\t%s: %s\n",
2434                         libcfs_id2str(dents[i].nde_id),
2435                         lst_node_state2str(dents[i].nde_state));
2436                 c++;
2437         }
2438
2439         fprintf(stdout, "Total %d nodes\n", c);
2440         free(dents);
2441
2442         return 0;
2443 }
2444
2445 int
2446 jt_lst_list_batch(int argc, char **argv)
2447 {
2448         lstcon_test_batch_ent_t ent;
2449         char                *batch   = NULL;
2450         int                  optidx  = 0;
2451         int                  verbose = 0; /* list nodes in batch or test */
2452         int                  invalid = 0;
2453         int                  active  = 0;
2454         int                  server  = 0;
2455         int                  ntest   = 0;
2456         int                  test    = 0;
2457         int                  c       = 0;
2458         int                  rc;
2459
2460         static struct option list_batch_opts[] =
2461         {
2462                 {"test",    required_argument, 0, 't' },
2463                 {"invalid", no_argument,       0, 'i' },
2464                 {"active",  no_argument,       0, 'a' },
2465                 {"all",     no_argument,       0, 'l' },
2466                 {"server",  no_argument,       0, 's' },
2467                 {0,         0,                 0,  0  }
2468         };
2469
2470         if (session_key == 0) {
2471                 fprintf(stderr,
2472                         "Can't find env LST_SESSION or value is not valid\n");
2473                 return -1;
2474         }
2475
2476         while (1) {
2477                 c = getopt_long(argc, argv, "ailst:",
2478                                 list_batch_opts, &optidx);
2479
2480                 if (c == -1)
2481                         break;
2482
2483                 switch (c) {
2484                 case 'a':
2485                         verbose = active = 1;
2486                         break;
2487                 case 'i':
2488                         verbose = invalid = 1;
2489                         break;
2490                 case 'l':
2491                         verbose = active = invalid = 1;
2492                         break;
2493                 case 's':
2494                         server = 1;
2495                         break;
2496                 case 't':
2497                         test = atoi(optarg);
2498                         ntest = 1;
2499                         break;
2500                 default:
2501                         lst_print_usage(argv[0]);
2502                         return -1;
2503                 }
2504         }
2505
2506         if (optind == argc) {
2507                 /* list all batches */
2508                 rc = lst_list_batch_all();
2509                 return rc;
2510         }
2511
2512         if (ntest == 1 && test <= 0) {
2513                 fprintf(stderr, "Invalid test id, test id starts from 1\n");
2514                 return -1;
2515         }
2516
2517         if (optind != argc - 1) {
2518                 lst_print_usage(argv[0]);
2519                 return -1;
2520         }
2521
2522         batch = argv[optind];
2523
2524 loop:
2525         /* show detail of specified batch or test */
2526         rc = lst_info_batch_ioctl(batch, test, server,
2527                                   &ent, NULL, NULL, NULL);
2528         if (rc != 0) {
2529                 lst_print_error((test > 0) ? "test" : "batch",
2530                                 (test > 0) ? "Failed to query test: %s\n" :
2531                                              "Failed to query batch: %s\n",
2532                                 strerror(errno));
2533                 return -1;
2534         }
2535
2536         if (verbose) {
2537                 /* list nodes in test or batch */
2538                 rc = lst_list_tsb_nodes(batch, test, server,
2539                                         server ? ent.tbe_srv_nle.nle_nnode :
2540                                                  ent.tbe_cli_nle.nle_nnode,
2541                                         active, invalid);
2542                 return rc;
2543         }
2544
2545         /* only show number of hosts in batch or test */
2546         if (test == 0) {
2547                 fprintf(stdout, "Batch: %s Tests: %d State: %d\n",
2548                         batch, ent.u.tbe_batch.bae_ntest,
2549                         ent.u.tbe_batch.bae_state);
2550                 ntest = ent.u.tbe_batch.bae_ntest;
2551                 test = 1; /* starting from test 1 */
2552
2553         } else {
2554                 fprintf(stdout,
2555                         "\tTest %d(%s) (loop: %d, concurrency: %d)\n",
2556                         test, lst_test_type2name(ent.u.tbe_test.tse_type),
2557                         ent.u.tbe_test.tse_loop,
2558                         ent.u.tbe_test.tse_concur);
2559                 ntest --;
2560                 test ++;
2561         }
2562
2563         fprintf(stdout, LST_NODES_TITLE);
2564         fprintf(stdout, "client\t%d\t%d\t%d\t%d\t%d\n"
2565                         "server\t%d\t%d\t%d\t%d\t%d\n",
2566                 ent.tbe_cli_nle.nle_nactive,
2567                 ent.tbe_cli_nle.nle_nbusy,
2568                 ent.tbe_cli_nle.nle_ndown,
2569                 ent.tbe_cli_nle.nle_nunknown,
2570                 ent.tbe_cli_nle.nle_nnode,
2571                 ent.tbe_srv_nle.nle_nactive,
2572                 ent.tbe_srv_nle.nle_nbusy,
2573                 ent.tbe_srv_nle.nle_ndown,
2574                 ent.tbe_srv_nle.nle_nunknown,
2575                 ent.tbe_srv_nle.nle_nnode);
2576
2577         if (ntest != 0)
2578                 goto loop;
2579
2580         return 0;
2581 }
2582
2583 int
2584 lst_query_batch_ioctl(char *batch, int test, int server,
2585                       int timeout, struct list_head *head)
2586 {
2587         lstio_batch_query_args_t args = {
2588                 .lstio_bat_key     = session_key,
2589                 .lstio_bat_testidx = test,
2590                 .lstio_bat_client  = !(server),
2591                 .lstio_bat_timeout = timeout,
2592                 .lstio_bat_nmlen   = strlen(batch),
2593                 .lstio_bat_namep   = batch,
2594                 .lstio_bat_resultp = head,
2595         };
2596
2597         return lst_ioctl(LSTIO_BATCH_QUERY, &args, sizeof(args));
2598 }
2599
2600 void
2601 lst_print_tsb_verbose(struct list_head *head,
2602                       int active, int idle, int error)
2603 {
2604         lstcon_rpc_ent_t *ent;
2605
2606         list_for_each_entry(ent, head, rpe_link) {
2607                 if (ent->rpe_priv[0] == 0 && active)
2608                         continue;
2609
2610                 if (ent->rpe_priv[0] != 0 && idle)
2611                         continue;
2612
2613                 if (ent->rpe_fwk_errno == 0 && error)
2614                         continue;
2615
2616                 fprintf(stdout, "%s [%s]: %s\n",
2617                         libcfs_id2str(ent->rpe_peer),
2618                         lst_node_state2str(ent->rpe_state),
2619                         ent->rpe_rpc_errno != 0 ?
2620                                 strerror(ent->rpe_rpc_errno) :
2621                                 (ent->rpe_priv[0] > 0 ? "Running" : "Idle"));
2622         }
2623 }
2624
2625 int
2626 jt_lst_query_batch(int argc, char **argv)
2627 {
2628         lstcon_test_batch_ent_t ent;
2629         struct list_head     head;
2630         char                *batch   = NULL;
2631         time_t               last    = 0;
2632         int                  optidx  = 0;
2633         int                  verbose = 0;
2634         int                  server  = 0;
2635         int                  timeout = 5; /* default 5 seconds */
2636         int                  delay   = 5; /* default 5 seconds */
2637         int                  loop    = 1; /* default 1 loop */
2638         int                  active  = 0;
2639         int                  error   = 0;
2640         int                  idle    = 0;
2641         int                  count   = 0;
2642         int                  test    = 0;
2643         int                  rc      = 0;
2644         int                  c       = 0;
2645         int                  i;
2646
2647         static struct option query_batch_opts[] =
2648         {
2649                 {"timeout", required_argument, 0, 'o' },
2650                 {"delay",   required_argument, 0, 'd' },
2651                 {"loop",    required_argument, 0, 'c' },
2652                 {"test",    required_argument, 0, 't' },
2653                 {"server",  no_argument,       0, 's' },
2654                 {"active",  no_argument,       0, 'a' },
2655                 {"idle",    no_argument,       0, 'i' },
2656                 {"error",   no_argument,       0, 'e' },
2657                 {"all",     no_argument,       0, 'l' },
2658                 {0,         0,                 0,  0  }
2659         };
2660
2661         if (session_key == 0) {
2662                 fprintf(stderr,
2663                         "Can't find env LST_SESSION or value is not valid\n");
2664                 return -1;
2665         }
2666
2667         while (1) {
2668                 c = getopt_long(argc, argv, "o:d:c:t:saiel",
2669                                 query_batch_opts, &optidx);
2670
2671                 /* Detect the end of the options. */
2672                 if (c == -1)
2673                         break;
2674
2675                 switch (c) {
2676                 case 'o':
2677                         timeout = atoi(optarg);
2678                         break;
2679                 case 'd':
2680                         delay = atoi(optarg);
2681                         break;
2682                 case 'c':
2683                         loop = atoi(optarg);
2684                         break;
2685                 case 't':
2686                         test = atoi(optarg);
2687                         break;
2688                 case 's':
2689                         server = 1;
2690                         break;
2691                 case 'a':
2692                         active = verbose = 1;
2693                         break;
2694                 case 'i':
2695                         idle = verbose = 1;
2696                         break;
2697                 case 'e':
2698                         error = verbose = 1;
2699                         break;
2700                 case 'l':
2701                         verbose = 1;
2702                         break;
2703                 default:
2704                         lst_print_usage(argv[0]);
2705                         return -1;
2706                 }
2707         }
2708
2709         if (test < 0 || timeout <= 0 || delay <= 0 || loop <= 0) {
2710                 lst_print_usage(argv[0]);
2711                 return -1;
2712         }
2713
2714         if (optind == argc) {
2715                 batch = LST_DEFAULT_BATCH;
2716
2717         } else if (optind == argc - 1) {
2718                 batch = argv[optind];
2719
2720         } else {
2721                 lst_print_usage(argv[0]);
2722                 return -1;
2723         }
2724
2725
2726         CFS_INIT_LIST_HEAD(&head);
2727
2728         if (verbose) {
2729                 rc = lst_info_batch_ioctl(batch, test, server,
2730                                           &ent, NULL, NULL, NULL);
2731                 if (rc != 0) {
2732                         fprintf(stderr, "Failed to query %s [%d]: %s\n",
2733                                 batch, test, strerror(errno));
2734                         return -1;
2735                 }
2736
2737                 count = server ? ent.tbe_srv_nle.nle_nnode :
2738                                  ent.tbe_cli_nle.nle_nnode;
2739                 if (count == 0) {
2740                         fprintf(stdout, "Batch or test is empty\n");
2741                         return 0;
2742                 }
2743         }
2744
2745         rc = lst_alloc_rpcent(&head, count, 0);
2746         if (rc != 0) {
2747                 fprintf(stderr, "Out of memory\n");
2748                 return rc;
2749         }
2750
2751         for (i = 0; i < loop; i++) {
2752                 time_t  now = time(NULL);
2753
2754                 if (now - last < delay) {
2755                         sleep(delay - now + last);
2756                         time(&now);
2757                 }
2758
2759                 last = now;
2760
2761                 rc = lst_query_batch_ioctl(batch, test,
2762                                            server, timeout, &head);
2763                 if (rc == -1) {
2764                         fprintf(stderr, "Failed to query batch: %s\n",
2765                                 strerror(errno));
2766                         break;
2767                 }
2768
2769                 if (verbose) {
2770                         /* Verbose mode */
2771                         lst_print_tsb_verbose(&head, active, idle, error);
2772                         continue;
2773                 }
2774
2775                 fprintf(stdout, "%s [%d] ", batch, test);
2776
2777                 if (lstcon_rpc_stat_failure(&trans_stat, 0) != 0) {
2778                         fprintf(stdout, "%d of %d nodes are unknown, ",
2779                                 lstcon_rpc_stat_failure(&trans_stat, 0),
2780                                 lstcon_rpc_stat_total(&trans_stat, 0));
2781                 }
2782
2783                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
2784                     lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0  &&
2785                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
2786                         fprintf(stdout, "is stopped\n");
2787                         continue;
2788                 }
2789
2790                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
2791                     lstcon_tsbqry_stat_idle(&trans_stat, 0) == 0 &&
2792                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
2793                         fprintf(stdout, "is running\n");
2794                         continue;
2795                 }
2796
2797                 fprintf(stdout, "stopped: %d , running: %d, failed: %d\n",
2798                                 lstcon_tsbqry_stat_idle(&trans_stat, 0),
2799                                 lstcon_tsbqry_stat_run(&trans_stat, 0),
2800                                 lstcon_tsbqry_stat_failure(&trans_stat, 0));
2801         }
2802
2803         lst_free_rpcent(&head);
2804
2805         return rc;
2806 }
2807
2808 int
2809 lst_parse_distribute(char *dstr, int *dist, int *span)
2810 {
2811         *dist = atoi(dstr);
2812         if (*dist <= 0)
2813                 return -1;
2814
2815         dstr = strchr(dstr, ':');
2816         if (dstr == NULL)
2817                 return -1;
2818
2819         *span = atoi(dstr + 1);
2820         if (*span <= 0)
2821                 return -1;
2822
2823         return 0;
2824 }
2825
2826 int
2827 lst_get_bulk_param(int argc, char **argv, lst_test_bulk_param_t *bulk)
2828 {
2829         char   *tok = NULL;
2830         char   *end = NULL;
2831         int     rc  = 0;
2832         int     i   = 0;
2833
2834         bulk->blk_size  = 4096;
2835         bulk->blk_opc   = LST_BRW_READ;
2836         bulk->blk_flags = LST_BRW_CHECK_NONE;
2837
2838         while (i < argc) {
2839                 if (strcasestr(argv[i], "check=") == argv[i] ||
2840                     strcasestr(argv[i], "c=") == argv[i]) {
2841                         tok = strchr(argv[i], '=') + 1;
2842
2843                         if (strcasecmp(tok, "full") == 0) {
2844                                 bulk->blk_flags = LST_BRW_CHECK_FULL;
2845                         } else if (strcasecmp(tok, "simple") == 0) {
2846                                 bulk->blk_flags = LST_BRW_CHECK_SIMPLE;
2847                         } else {
2848                                 fprintf(stderr, "Unknow flag %s\n", tok);
2849                                 return -1;
2850                         }
2851
2852                 } else if (strcasestr(argv[i], "size=") == argv[i] ||
2853                          strcasestr(argv[i], "s=") == argv[i]) {
2854                         tok = strchr(argv[i], '=') + 1;
2855
2856                         bulk->blk_size = strtol(tok, &end, 0);
2857                         if (bulk->blk_size <= 0) {
2858                                 fprintf(stderr, "Invalid size %s\n", tok);
2859                                 return -1;
2860                         }
2861
2862                         if (end == NULL)
2863                                 return 0;
2864
2865                         if (*end == 'k' || *end == 'K')
2866                                 bulk->blk_size *= 1024;
2867                         else if (*end == 'm' || *end == 'M')
2868                                 bulk->blk_size *= 1024 * 1024;
2869
2870                         if (bulk->blk_size > CFS_PAGE_SIZE * LNET_MAX_IOV) {
2871                                 fprintf(stderr, "Size exceed limitation: %d bytes\n",
2872                                         bulk->blk_size);
2873                                 return -1;
2874                         }
2875
2876                 } else if (strcasecmp(argv[i], "read") == 0 ||
2877                            strcasecmp(argv[i], "r") == 0) {
2878                         bulk->blk_opc = LST_BRW_READ;
2879
2880                 } else if (strcasecmp(argv[i], "write") == 0 ||
2881                            strcasecmp(argv[i], "w") == 0) {
2882                         bulk->blk_opc = LST_BRW_WRITE;
2883
2884                 } else {
2885                         fprintf(stderr, "Unknow parameter: %s\n", argv[i]);
2886                         return -1;
2887                 }
2888
2889                 i++;
2890         }
2891
2892         return rc;
2893 }
2894
2895 int
2896 lst_get_test_param(char *test, int argc, char **argv, void **param, int *plen)
2897 {
2898         lst_test_bulk_param_t *bulk = NULL;
2899         int                    type;
2900
2901         type = lst_test_name2type(test);
2902         if (type < 0) {
2903                 fprintf(stderr, "Unknow test name %s\n", test);
2904                 return -1;
2905         }
2906
2907         switch (type) {
2908         case LST_TEST_PING:
2909                 break;
2910
2911         case LST_TEST_BULK:
2912                 bulk = malloc(sizeof(*bulk));
2913                 if (bulk == NULL) {
2914                         fprintf(stderr, "Out of memory\n");
2915                         return -1;
2916                 }
2917
2918                 memset(bulk, 0, sizeof(*bulk));
2919
2920                 if (lst_get_bulk_param(argc, argv, bulk) != 0) {
2921                         free(bulk);
2922                         return -1;
2923                 }
2924
2925                 *param = bulk;
2926                 *plen  = sizeof(*bulk);
2927
2928                 break;
2929
2930         default:
2931                 break;
2932         }
2933
2934         /* TODO: parse more parameter */
2935         return type;
2936 }
2937
2938 int
2939 lst_add_test_ioctl(char *batch, int type, int loop, int concur,
2940                    int dist, int span, char *sgrp, char *dgrp,
2941                    void *param, int plen, int *retp, struct list_head *resultp)
2942 {
2943         lstio_test_args_t args = {
2944                 .lstio_tes_key          = session_key,
2945                 .lstio_tes_bat_nmlen    = strlen(batch),
2946                 .lstio_tes_bat_name     = batch,
2947                 .lstio_tes_type         = type,
2948                 .lstio_tes_loop         = loop,
2949                 .lstio_tes_concur       = concur,
2950                 .lstio_tes_dist         = dist,
2951                 .lstio_tes_span         = span,
2952                 .lstio_tes_sgrp_nmlen   = strlen(sgrp),
2953                 .lstio_tes_sgrp_name    = sgrp,
2954                 .lstio_tes_dgrp_nmlen   = strlen(dgrp),
2955                 .lstio_tes_dgrp_name    = dgrp,
2956                 .lstio_tes_param_len    = plen,
2957                 .lstio_tes_param        = param,
2958                 .lstio_tes_retp         = retp,
2959                 .lstio_tes_resultp      = resultp,
2960         };
2961
2962         return lst_ioctl(LSTIO_TEST_ADD, &args, sizeof(args));
2963 }
2964
2965 int
2966 jt_lst_add_test(int argc, char **argv)
2967 {
2968         struct list_head  head;
2969         char             *batch  = NULL;
2970         char             *test   = NULL;
2971         char             *dstr   = NULL;
2972         char             *from   = NULL;
2973         char             *to     = NULL;
2974         void             *param  = NULL;
2975         int               optidx = 0;
2976         int               concur = 1;
2977         int               loop   = -1;
2978         int               dist   = 1;
2979         int               span   = 1;
2980         int               plen   = 0;
2981         int               fcount = 0;
2982         int               tcount = 0;
2983         int               ret    = 0;
2984         int               type;
2985         int               rc;
2986         int               c;
2987
2988         static struct option add_test_opts[] =
2989         {
2990                 {"batch",       required_argument, 0, 'b' },
2991                 {"concurrency", required_argument, 0, 'c' },
2992                 {"distribute",  required_argument, 0, 'd' },
2993                 {"from",        required_argument, 0, 'f' },
2994                 {"to",          required_argument, 0, 't' },
2995                 {"loop",        required_argument, 0, 'l' },
2996                 {0,             0,                 0,  0  }
2997         };
2998
2999         if (session_key == 0) {
3000                 fprintf(stderr,
3001                         "Can't find env LST_SESSION or value is not valid\n");
3002                 return -1;
3003         }
3004
3005         while (1) {
3006                 c = getopt_long(argc, argv, "b:c:d:f:l:t:",
3007                                 add_test_opts, &optidx);
3008
3009                 /* Detect the end of the options. */
3010                 if (c == -1)
3011                         break;
3012
3013                 switch (c) {
3014                 case 'b':
3015                         batch = optarg;
3016                         break;
3017                 case 'c':
3018                         concur = atoi(optarg);
3019                         break;
3020                 case 'd':
3021                         dstr = optarg;
3022                         break;
3023                 case 'f':
3024                         from = optarg;
3025                         break;
3026                 case 'l':
3027                         loop = atoi(optarg);
3028                         break;
3029                 case 't':
3030                         to = optarg;
3031                         break;
3032                 default:
3033                         lst_print_usage(argv[0]);
3034                         return -1;
3035                 }
3036         }
3037
3038         if (optind == argc || from == NULL || to == NULL) {
3039                 lst_print_usage(argv[0]);
3040                 return -1;
3041         }
3042
3043         if (concur <= 0 || concur > LST_MAX_CONCUR) {
3044                 fprintf(stderr, "Invalid concurrency of test: %d\n", concur);
3045                 return -1;
3046         }
3047
3048         if (batch == NULL)
3049                 batch = LST_DEFAULT_BATCH;
3050
3051         if (dstr != NULL) {
3052                 rc = lst_parse_distribute(dstr, &dist, &span);
3053                 if (rc != 0) {
3054                         fprintf(stderr, "Invalid distribution: %s\n", dstr);
3055                         return -1;
3056                 }
3057         }
3058
3059         test = argv[optind++];
3060
3061         argc -= optind;
3062         argv += optind;
3063
3064         type = lst_get_test_param(test, argc, argv, &param, &plen);
3065         if (type < 0) {
3066                 fprintf(stderr, "Failed to add test (%s)\n", test);
3067                 return -1;
3068         }
3069
3070         CFS_INIT_LIST_HEAD(&head);
3071
3072         rc = lst_get_node_count(LST_OPC_GROUP, from, &fcount, NULL);
3073         if (rc != 0) {
3074                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3075                         from, strerror(errno));
3076                 goto out;
3077         }
3078
3079         rc = lst_get_node_count(LST_OPC_GROUP, to, &tcount, NULL);
3080         if (rc != 0) {
3081                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3082                         to, strerror(errno));
3083                 goto out;
3084         }
3085
3086         rc = lst_alloc_rpcent(&head, fcount > tcount ? fcount : tcount, 0);
3087         if (rc != 0) {
3088                 fprintf(stderr, "Out of memory\n");
3089                 goto out;
3090         }
3091
3092         rc = lst_add_test_ioctl(batch, type, loop, concur,
3093                                 dist, span, from, to, param, plen, &ret, &head);
3094
3095         if (rc == 0) {
3096                 fprintf(stdout, "Test was added successfully\n");
3097                 if (ret != 0) {
3098                         fprintf(stdout, "Server group contains userland test "
3099                                 "nodes, old version of tcplnd can't accept "
3100                                 "connection request\n");
3101                 }
3102
3103                 goto out;
3104         }
3105
3106         if (rc == -1) {
3107                 lst_print_error("test", "Failed to add test: %s\n",
3108                                 strerror(errno));
3109                 goto out;
3110         }
3111
3112         lst_print_transerr(&head, "add test");
3113 out:
3114         lst_free_rpcent(&head);
3115
3116         if (param != NULL)
3117                 free(param);
3118
3119         return rc;
3120 }
3121
3122 static command_t lst_cmdlist[] = {
3123         {"new_session",         jt_lst_new_session,     NULL,
3124          "Usage: lst new_session [--timeout TIME] [--force] [NAME]"                     },
3125         {"end_session",         jt_lst_end_session,     NULL,
3126          "Usage: lst end_session"                                                       },
3127         {"show_session",        jt_lst_show_session,    NULL,
3128          "Usage: lst show_session"                                                      },
3129         {"ping",                jt_lst_ping ,           NULL,
3130          "Usage: lst ping  [--group NAME] [--batch NAME] [--session] [--nodes IDS]"     },
3131         {"add_group",           jt_lst_add_group,       NULL,
3132          "Usage: lst group NAME IDs [IDs]..."                                           },
3133         {"del_group",           jt_lst_del_group,       NULL,
3134          "Usage: lst del_group NAME"                                                    },
3135         {"update_group",        jt_lst_update_group,    NULL,
3136          "Usage: lst update_group NAME [--clean] [--refresh] [--remove IDs]"            },
3137         {"list_group",          jt_lst_list_group,      NULL,
3138           "Usage: lst list_group [--active] [--busy] [--down] [--unknown] GROUP ..."    },
3139         {"stat",                jt_lst_stat,            NULL,
3140          "Usage: lst stat [--bw] [--rate] [--read] [--write] [--max] [--min] [--avg] "
3141          " [--timeout #] [--delay #] GROUP [GROUP]"                                     },
3142         {"show_error",          jt_lst_show_error,      NULL,
3143          "Usage: lst show_error NAME | IDS ..."                                         },
3144         {"add_batch",           jt_lst_add_batch,       NULL,
3145          "Usage: lst add_batch NAME"                                                    },
3146         {"run",                 jt_lst_start_batch,     NULL,
3147          "Usage: lst run [--timeout TIME] [NAME]"                                       },
3148         {"stop",                jt_lst_stop_batch,      NULL,
3149          "Usage: lst stop [--force] BATCH_NAME"                                         },
3150         {"list_batch",          jt_lst_list_batch,      NULL,
3151          "Usage: lst list_batch NAME [--test ID] [--server]"                            },
3152         {"query",               jt_lst_query_batch,     NULL,
3153          "Usage: lst query [--test ID] [--server] [--timeout TIME] NAME"                },
3154         {"add_test",            jt_lst_add_test,        NULL,
3155          "Usage: lst add_test [--batch BATCH] [--loop #] [--concurrency #] "
3156          " [--distribute #:#] [--from GROUP] [--to GROUP] TEST..."                      },
3157         {"help",                Parser_help,            0,     "help"                   },
3158         {0,                     0,                      0,      NULL                    }
3159 };
3160
3161 int
3162 lst_initialize(void)
3163 {
3164         char   *key;
3165
3166         key = getenv("LST_SESSION");
3167
3168         if (key == NULL) {
3169                 session_key = 0;
3170                 return 0;
3171         }
3172
3173         session_key = atoi(key);
3174
3175         return 0;
3176 }
3177
3178 int
3179 main(int argc, char **argv)
3180 {
3181         setlinebuf(stdout);
3182
3183         if (lst_initialize() < 0)
3184                 exit(0);
3185
3186         if (ptl_initialize(argc, argv) < 0)
3187                 exit(0);
3188
3189         Parser_init("lst > ", lst_cmdlist);
3190
3191         if (argc != 1)
3192                 return Parser_execarg(argc - 1, argv + 1, lst_cmdlist);
3193
3194         Parser_commands();
3195
3196         return 0;
3197 }