lnet/utils/lst.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * GPL HEADER START
   5  *
   6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 only,
  10  * as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * General Public License version 2 for more details (a copy is included
  16  * in the LICENSE file that accompanied this code).
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * version 2 along with this program; If not, see
  20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  21  *
  22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  23  * CA 95054 USA or visit www.sun.com if you need additional information or
  24  * have any questions.
  25  *
  26  * GPL HEADER END
  27  */
  28 /*
  29  * Copyright  2008 Sun Microsystems, Inc. All rights reserved
  30  * Use is subject to license terms.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * lnet/selftest/conctl.c
  37  *
  38  * Author: Liang Zhen <liangzhen@clusterfs.com>
  39  */
  40
  41 #define _GNU_SOURCE
  42
  43 #include <stdio.h>
  44 #include <stdlib.h>
  45 #include <string.h>
  46 #include <getopt.h>
  47 #include <errno.h>
  48 #include <pwd.h>
  49 #include <lnet/lnetctl.h>
  50 #include <lnet/lnetst.h>
  51 #include "parser.h"
  52
  53 static command_t           lst_cmdlist[];
  54 static lst_sid_t           session_id;
  55 static int                 session_key;
  56 static lstcon_trans_stat_t trans_stat;
  57
  58 typedef struct list_string {
  59         struct list_string *lstr_next;
  60         int                 lstr_sz;
  61         char                lstr_str[0];
  62 } lstr_t;
  63
  64 #define offsetof(typ,memb)     ((unsigned long)((char *)&(((typ *)0)->memb)))
  65
  66 static int alloc_count = 0;
  67 static int alloc_nob   = 0;
  68
  69 lstr_t *
  70 alloc_lstr(int sz)
  71 {
  72         lstr_t  *lstr = malloc(offsetof(lstr_t, lstr_str[sz]));
  73
  74         if (lstr == NULL) {
  75                 fprintf(stderr, "Can't allocate lstr\n");
  76                 abort();
  77         }
  78
  79         alloc_nob += sz;
  80         alloc_count++;
  81
  82         lstr->lstr_str[0] = 0;
  83         lstr->lstr_sz = sz;
  84         return lstr;
  85 }
  86
  87 void
  88 free_lstr(lstr_t *lstr)
  89 {
  90         alloc_count--;
  91         alloc_nob -= lstr->lstr_sz;
  92         free(lstr);
  93 }
  94
  95 void
  96 free_lstrs(lstr_t **list)
  97 {
  98         lstr_t   *lstr;
  99
 100         while ((lstr = *list) != NULL) {
 101                 *list = lstr->lstr_next;
 102                 free_lstr(lstr);
 103         }
 104 }
 105
 106 void
 107 new_lstrs(lstr_t **list, char *prefix, char *postfix,
 108           int lo, int hi, int stride)
 109 {
 110         int    n1 = strlen(prefix);
 111         int    n2 = strlen(postfix);
 112         int    sz = n1 + 20 + n2 + 1;
 113
 114         do {
 115                 lstr_t *n = alloc_lstr(sz);
 116
 117                 snprintf(n->lstr_str, sz - 1, "%s%u%s",
 118                          prefix, lo, postfix);
 119
 120                 n->lstr_next = *list;
 121                 *list = n;
 122
 123                 lo += stride;
 124         } while (lo <= hi);
 125 }
 126
 127 int
 128 expand_lstr(lstr_t **list, lstr_t *l)
 129 {
 130         int          nob = strlen(l->lstr_str);
 131         char        *b1;
 132         char        *b2;
 133         char        *expr;
 134         char        *sep;
 135         int          x;
 136         int          y;
 137         int          z;
 138         int          n;
 139
 140         b1 = strchr(l->lstr_str, '[');
 141         if (b1 == NULL) {
 142                 l->lstr_next = *list;
 143                 *list = l;
 144                 return 0;
 145         }
 146
 147         b2 = strchr(b1, ']');
 148         if (b2 == NULL || b2 == b1 + 1)
 149                 return -1;
 150
 151         *b1++ = 0;
 152         *b2++ = 0;
 153         expr = b1;
 154         do {
 155
 156                 sep = strchr(expr, ',');
 157                 if (sep != NULL)
 158                         *sep++ = 0;
 159
 160                 nob = strlen(expr);
 161                 n = nob;
 162                 if (sscanf(expr, "%u%n", &x, &n) >= 1 && n == nob) {
 163                         /* simple number */
 164                         new_lstrs(list, l->lstr_str, b2, x, x, 1);
 165                         continue;
 166                 }
 167
 168                 n = nob;
 169                 if (sscanf(expr, "%u-%u%n", &x, &y, &n) >= 2 && n == nob &&
 170                     x < y) {
 171                         /* simple range */
 172                         new_lstrs(list, l->lstr_str, b2, x, y, 1);
 173                         continue;
 174                 }
 175
 176                 n = nob;
 177                 if (sscanf(expr, "%u-%u/%u%n", &x, &y, &z, &n) >= 3 && n == nob &&
 178                     x < y) {
 179                         /* strided range */
 180                         new_lstrs(list, l->lstr_str, b2, x, y, z);
 181                         continue;
 182                 }
 183
 184                 /* syntax error */
 185                 return -1;
 186         } while ((expr = sep) != NULL);
 187
 188         free_lstr(l);
 189
 190         return 1;
 191 }
 192
 193 int
 194 expand_strs(char *str, lstr_t **head)
 195 {
 196         lstr_t  *list = NULL;
 197         lstr_t  *nlist;
 198         lstr_t  *l;
 199         int      rc = 0;
 200         int      expanded;
 201
 202         l = alloc_lstr(strlen(str) + 1);
 203         memcpy(l->lstr_str, str, strlen(str) + 1);
 204         l->lstr_next = NULL;
 205         list = l;
 206
 207         do {
 208                 expanded = 0;
 209                 nlist = NULL;
 210
 211                 while ((l = list) != NULL) {
 212                         list = l->lstr_next;
 213
 214                         rc = expand_lstr(&nlist, l);
 215                         if (rc < 0) {
 216                                 fprintf(stderr, "Syntax error in \"%s\"\n", str);
 217                                 free_lstr(l);
 218                                 break;
 219                         }
 220
 221                         expanded |= rc > 0;
 222                 }
 223
 224                 /* re-order onto 'list' */
 225                 while ((l = nlist) != NULL) {
 226                         nlist = l->lstr_next;
 227                         l->lstr_next = list;
 228                         list = l;
 229                 }
 230
 231         } while (expanded && rc > 0);
 232
 233         if (rc >= 0) {
 234                 *head = list;
 235                 return 0;
 236         }
 237
 238         while ((l = list) != NULL) {
 239                 list = l->lstr_next;
 240
 241                 free_lstr(l);
 242         }
 243         return rc;
 244 }
 245
 246 int
 247 lst_parse_nids(char *str, int *countp, lnet_process_id_t **idspp)
 248 {
 249         lstr_t  *head = NULL;
 250         lstr_t  *l;
 251         int      c = 0;
 252         int      i;
 253         int      rc;
 254
 255         rc = expand_strs(str, &head);
 256         if (rc != 0)
 257                 goto out;
 258
 259         l = head;
 260         while (l != NULL) {
 261                 l = l->lstr_next;
 262                 c++;
 263         }
 264
 265         *idspp = malloc(c * sizeof(lnet_process_id_t));
 266         if (*idspp == NULL) {
 267                 fprintf(stderr, "Out of memory\n");
 268                 rc = -1;
 269         }
 270
 271         *countp = c;
 272 out:
 273         i = 0;
 274         while ((l = head) != NULL) {
 275                 head = l->lstr_next;
 276
 277                 if (rc == 0) {
 278                         (*idspp)[i].nid = libcfs_str2nid(l->lstr_str);
 279                         if ((*idspp)[i].nid == LNET_NID_ANY) {
 280                                 fprintf(stderr, "Invalid nid: %s\n",
 281                                         l->lstr_str);
 282                                 rc = -1;
 283                         }
 284
 285                         (*idspp)[i].pid = LUSTRE_LNET_PID;
 286                         i++;
 287                 }
 288
 289                 free_lstr(l);
 290         }
 291
 292         if (rc == 0)
 293                 return 0;
 294
 295         free(*idspp);
 296         *idspp = NULL;
 297
 298         return rc;
 299 }
 300
 301 char *
 302 lst_node_state2str(int state)
 303 {
 304         if (state == LST_NODE_ACTIVE)
 305                 return "Active";
 306         if (state == LST_NODE_BUSY)
 307                 return "Busy";
 308         if (state == LST_NODE_DOWN)
 309                 return "Down";
 310
 311         return "Unknown";
 312 }
 313
 314 int
 315 lst_node_str2state(char *str)
 316 {
 317         if (strcasecmp(str, "active") == 0)
 318                 return LST_NODE_ACTIVE;
 319         if (strcasecmp(str, "busy") == 0)
 320                 return LST_NODE_BUSY;
 321         if (strcasecmp(str, "down") == 0)
 322                 return LST_NODE_DOWN;
 323         if (strcasecmp(str, "unknown") == 0)
 324                 return LST_NODE_UNKNOWN;
 325         if (strcasecmp(str, "invalid") == 0)
 326                 return (LST_NODE_UNKNOWN | LST_NODE_DOWN | LST_NODE_BUSY);
 327
 328         return -1;
 329 }
 330
 331 char *
 332 lst_test_type2name(int type)
 333 {
 334         if (type == LST_TEST_PING)
 335                 return "ping";
 336         if (type == LST_TEST_BULK)
 337                 return "brw";
 338
 339         return "unknown";
 340 }
 341
 342 int
 343 lst_test_name2type(char *name)
 344 {
 345         if (strcasecmp(name, "ping") == 0)
 346                 return LST_TEST_PING;
 347         if (strcasecmp(name, "brw") == 0)
 348                 return LST_TEST_BULK;
 349
 350         return -1;
 351 }
 352
 353 void
 354 lst_print_usage(char *cmd)
 355 {
 356         Parser_printhelp(cmd);
 357 }
 358
 359 void
 360 lst_print_error(char *sub, const char *def_format, ...)
 361 {
 362         va_list ap;
 363
 364         /* local error returned from kernel */
 365         switch (errno) {
 366         case ESRCH:
 367                 fprintf(stderr, "No session exists\n");
 368                 return;
 369         case ESHUTDOWN:
 370                 fprintf(stderr, "Session is shutting down\n");
 371                 return;
 372         case EACCES:
 373                 fprintf(stderr, "Unmatched session key or not root\n");
 374                 return;
 375         case ENOENT:
 376                 fprintf(stderr, "Can't find %s in current session\n", sub);
 377                 return;
 378         case EINVAL:
 379                 fprintf(stderr, "Invalid parameters list in command line\n");
 380                 return;
 381         case EFAULT:
 382                 fprintf(stderr, "Bad parameter address\n");
 383                 return;
 384         case EEXIST:
 385                 fprintf(stderr, "%s already exists\n", sub);
 386                 return;
 387         default:
 388                 va_start(ap, def_format);
 389                 vfprintf(stderr, def_format, ap);
 390                 va_end(ap);
 391
 392                 return;
 393         }
 394 }
 395
 396 void
 397 lst_free_rpcent(struct list_head *head)
 398 {
 399         lstcon_rpc_ent_t *ent;
 400
 401         while (!list_empty(head)) {
 402                 ent = list_entry(head->next, lstcon_rpc_ent_t, rpe_link);
 403
 404                 list_del(&ent->rpe_link);
 405                 free(ent);
 406         }
 407 }
 408
 409 void
 410 lst_reset_rpcent(struct list_head *head)
 411 {
 412         lstcon_rpc_ent_t *ent;
 413
 414         list_for_each_entry(ent, head, rpe_link) {
 415                 ent->rpe_sid      = LST_INVALID_SID;
 416                 ent->rpe_peer.nid = LNET_NID_ANY;
 417                 ent->rpe_peer.pid = LNET_PID_ANY;
 418                 ent->rpe_rpc_errno = ent->rpe_fwk_errno = 0;
 419         }
 420 }
 421
 422 int
 423 lst_alloc_rpcent(struct list_head *head, int count, int offset)
 424 {
 425         lstcon_rpc_ent_t *ent;
 426         int               i;
 427
 428         for (i = 0; i < count; i++) {
 429                 ent = malloc(offsetof(lstcon_rpc_ent_t, rpe_payload[offset]));
 430                 if (ent == NULL) {
 431                         lst_free_rpcent(head);
 432                         return -1;
 433                 }
 434
 435                 memset(ent, 0, offsetof(lstcon_rpc_ent_t, rpe_payload[offset]));
 436
 437                 ent->rpe_sid      = LST_INVALID_SID;
 438                 ent->rpe_peer.nid = LNET_NID_ANY;
 439                 ent->rpe_peer.pid = LNET_PID_ANY;
 440                 list_add(&ent->rpe_link, head);
 441         }
 442
 443         return 0;
 444 }
 445
 446 void
 447 lst_print_transerr(struct list_head *head, char *optstr)
 448 {
 449         lstcon_rpc_ent_t  *ent;
 450
 451         list_for_each_entry(ent, head, rpe_link) {
 452                 if (ent->rpe_rpc_errno == 0 && ent->rpe_fwk_errno == 0)
 453                         continue;
 454
 455                 if (ent->rpe_rpc_errno != 0) {
 456                         fprintf(stderr, "%s RPC failed on %s: %s\n",
 457                                 optstr, libcfs_id2str(ent->rpe_peer),
 458                                 strerror(ent->rpe_rpc_errno));
 459                         continue;
 460                 }
 461
 462                 fprintf(stderr, "%s failed on %s: %s\n",
 463                         optstr, libcfs_id2str(ent->rpe_peer),
 464                         strerror(ent->rpe_fwk_errno));
 465         }
 466 }
 467
 468 int lst_info_batch_ioctl(char *batch, int test, int server,
 469                         lstcon_test_batch_ent_t *entp, int *idxp,
 470                         int *ndentp, lstcon_node_ent_t *dentsp);
 471
 472 int lst_info_group_ioctl(char *name, lstcon_ndlist_ent_t *gent,
 473                          int *idx, int *count, lstcon_node_ent_t *dents);
 474
 475 int lst_query_batch_ioctl(char *batch, int test, int server,
 476                           int timeout, struct list_head *head);
 477
 478 int
 479 lst_ioctl(unsigned int opc, void *buf, int len)
 480 {
 481         struct libcfs_ioctl_data data;
 482         int    rc;
 483
 484         LIBCFS_IOC_INIT (data);
 485         data.ioc_u32[0]  = opc;
 486         data.ioc_plen1   = len;
 487         data.ioc_pbuf1   = (char *)buf;
 488         data.ioc_plen2   = sizeof(trans_stat);
 489         data.ioc_pbuf2   = (char *)&trans_stat;
 490
 491         memset(&trans_stat, 0, sizeof(trans_stat));
 492
 493         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNETST, &data);
 494
 495         /* local error, no valid RPC result */
 496         if (rc != 0)
 497                 return -1;
 498
 499         /* RPC error */
 500         if (trans_stat.trs_rpc_errno != 0)
 501                 return -2;
 502
 503         /* Framework error */
 504         if (trans_stat.trs_fwk_errno != 0)
 505                 return -3;
 506
 507         return 0;
 508 }
 509
 510 int
 511 lst_new_session_ioctl (char *name, int timeout, int force, lst_sid_t *sid)
 512 {
 513         lstio_session_new_args_t        args = {
 514                 .lstio_ses_key          = session_key,
 515                 .lstio_ses_timeout      = timeout,
 516                 .lstio_ses_force        = force,
 517                 .lstio_ses_idp          = sid,
 518                 .lstio_ses_namep        = name,
 519                 .lstio_ses_nmlen        = strlen(name),
 520         };
 521
 522         return lst_ioctl (LSTIO_SESSION_NEW, &args, sizeof(args));
 523 }
 524
 525 int
 526 jt_lst_new_session(int argc,  char **argv)
 527 {
 528         char  buf[LST_NAME_SIZE];
 529         char *name;
 530         int   optidx  = 0;
 531         int   timeout = 300;
 532         int   force   = 0;
 533         int   c;
 534         int   rc;
 535
 536         static struct option session_opts[] =
 537         {
 538                 {"timeout", required_argument,  0, 't' },
 539                 {"force",   no_argument,        0, 'f' },
 540                 {0,         0,                  0,  0  }
 541         };
 542
 543         if (session_key == 0) {
 544                 fprintf(stderr,
 545                         "Can't find env LST_SESSION or value is not valid\n");
 546                 return -1;
 547         }
 548
 549         while (1) {
 550
 551                 c = getopt_long(argc, argv, "ft:",
 552                                 session_opts, &optidx);
 553
 554                 if (c == -1)
 555                         break;
 556
 557                 switch (c) {
 558                 case 'f':
 559                         force = 1;
 560                         break;
 561                 case 't':
 562                         timeout = atoi(optarg);
 563                         break;
 564                 default:
 565                         lst_print_usage(argv[0]);
 566                         return -1;
 567                 }
 568         }
 569
 570         if (timeout <= 0) {
 571                 fprintf(stderr, "Invalid timeout value\n");
 572                 return -1;
 573         }
 574
 575         if (optind == argc - 1) {
 576                 name = argv[optind ++];
 577                 if (strlen(name) >= LST_NAME_SIZE) {
 578                         fprintf(stderr, "Name size is limited to %d\n",
 579                                 LST_NAME_SIZE - 1);
 580                         return -1;
 581                 }
 582
 583         } else if (optind == argc) {
 584                 char           user[LST_NAME_SIZE];
 585                 char           host[LST_NAME_SIZE];
 586                 struct passwd *pw = getpwuid(getuid());
 587
 588                 if (pw == NULL)
 589                         snprintf(user, sizeof(user), "%d", (int)getuid());
 590                 else
 591                         snprintf(user, sizeof(user), "%s", pw->pw_name);
 592
 593                 rc = gethostname(host, sizeof(host));
 594                 if (rc != 0)
 595                         snprintf(host, sizeof(host), "unknown_host");
 596
 597                 snprintf(buf, LST_NAME_SIZE, "%s@%s", user, host);
 598                 name = buf;
 599
 600         } else {
 601                 lst_print_usage(argv[0]);
 602                 return -1;
 603         }
 604
 605         rc = lst_new_session_ioctl(name, timeout, force, &session_id);
 606
 607         if (rc != 0) {
 608                 lst_print_error("session", "Failed to create session: %s\n",
 609                                 strerror(errno));
 610                 return rc;
 611         }
 612
 613         fprintf(stdout, "SESSION: %s TIMEOUT: %d FORCE: %s\n",
 614                 name, timeout, force ? "Yes": "No");
 615
 616         return rc;
 617 }
 618
 619 int
 620 lst_session_info_ioctl(char *name, int len, int *key,
 621                        lst_sid_t *sid, lstcon_ndlist_ent_t *ndinfo)
 622 {
 623         lstio_session_info_args_t args = {
 624                 .lstio_ses_keyp         = key,
 625                 .lstio_ses_idp          = sid,
 626                 .lstio_ses_ndinfo       = ndinfo,
 627                 .lstio_ses_nmlen        = len,
 628                 .lstio_ses_namep        = name,
 629         };
 630
 631         return lst_ioctl(LSTIO_SESSION_INFO, &args, sizeof(args));
 632 }
 633
 634 int
 635 jt_lst_show_session(int argc, char **argv)
 636 {
 637         lstcon_ndlist_ent_t ndinfo;
 638         lst_sid_t           sid;
 639         char                name[LST_NAME_SIZE];
 640         int                 key;
 641         int                 rc;
 642
 643         rc = lst_session_info_ioctl(name, LST_NAME_SIZE, &key, &sid, &ndinfo);
 644
 645         if (rc != 0) {
 646                 lst_print_error("session", "Failed to show session: %s\n",
 647                                 strerror(errno));
 648                 return -1;
 649         }
 650
 651         fprintf(stdout, "%s ID: %Lu@%s, KEY: %d NODES: %d\n",
 652                 name, sid.ses_stamp, libcfs_nid2str(sid.ses_nid),
 653                 key, ndinfo.nle_nnode);
 654
 655         return 0;
 656 }
 657
 658 int
 659 lst_end_session_ioctl(void)
 660 {
 661         lstio_session_end_args_t args = {
 662                 .lstio_ses_key           = session_key,
 663         };
 664
 665         return lst_ioctl (LSTIO_SESSION_END, &args, sizeof(args));
 666 }
 667
 668 int
 669 jt_lst_end_session(int argc, char **argv)
 670 {
 671         int             rc;
 672
 673         if (session_key == 0) {
 674                 fprintf(stderr,
 675                         "Can't find env LST_SESSION or value is not valid\n");
 676                 return -1;
 677         }
 678
 679         rc = lst_end_session_ioctl();
 680
 681         if (rc == 0) {
 682                 fprintf(stdout, "session is ended\n");
 683                 return 0;
 684         }
 685
 686         if (rc == -1) {
 687                 lst_print_error("session", "Failed to end session: %s\n",
 688                                 strerror(errno));
 689                 return rc;
 690         }
 691
 692         if (trans_stat.trs_rpc_errno != 0) {
 693                 fprintf(stderr,
 694                         "[RPC] Failed to send %d session RPCs: %s\n",
 695                         lstcon_rpc_stat_failure(&trans_stat, 0),
 696                         strerror(trans_stat.trs_rpc_errno));
 697         }
 698
 699         if (trans_stat.trs_fwk_errno != 0) {
 700                 fprintf(stderr,
 701                         "[FWK] Failed to end session on %d nodes: %s\n",
 702                         lstcon_sesop_stat_failure(&trans_stat, 0),
 703                         strerror(trans_stat.trs_fwk_errno));
 704         }
 705
 706         return rc;
 707 }
 708
 709 int
 710 lst_ping_ioctl(char *str, int type, int timeout,
 711                int count, lnet_process_id_t *ids, struct list_head *head)
 712 {
 713         lstio_debug_args_t args = {
 714                 .lstio_dbg_key          = session_key,
 715                 .lstio_dbg_type         = type,
 716                 .lstio_dbg_flags        = 0,
 717                 .lstio_dbg_timeout      = timeout,
 718                 .lstio_dbg_nmlen        = (str == NULL) ? 0: strlen(str),
 719                 .lstio_dbg_namep        = str,
 720                 .lstio_dbg_count        = count,
 721                 .lstio_dbg_idsp         = ids,
 722                 .lstio_dbg_resultp      = head,
 723         };
 724
 725         return lst_ioctl (LSTIO_DEBUG, &args, sizeof(args));
 726 }
 727
 728 int
 729 lst_get_node_count(int type, char *str, int *countp, lnet_process_id_t **idspp)
 730 {
 731         char                    buf[LST_NAME_SIZE];
 732         lstcon_test_batch_ent_t ent;
 733         lstcon_ndlist_ent_t    *entp = &ent.tbe_cli_nle;
 734         lst_sid_t               sid;
 735         int                     key;
 736         int                     rc;
 737
 738         switch (type) {
 739         case LST_OPC_SESSION:
 740                 rc = lst_session_info_ioctl(buf, LST_NAME_SIZE,
 741                                             &key, &sid, entp);
 742                 break;
 743
 744         case LST_OPC_BATCHSRV:
 745                 entp = &ent.tbe_srv_nle;
 746         case LST_OPC_BATCHCLI:
 747                 rc = lst_info_batch_ioctl(str, 0, 0, &ent, NULL, NULL, NULL);
 748                 break;
 749
 750         case LST_OPC_GROUP:
 751                 rc = lst_info_group_ioctl(str, entp, NULL, NULL, NULL);
 752                 break;
 753
 754         case LST_OPC_NODES:
 755                 rc = lst_parse_nids(str, &entp->nle_nnode, idspp) < 0 ? -1 : 0;
 756                 break;
 757
 758         default:
 759                 rc = -1;
 760                 break;
 761         }
 762
 763         if (rc == 0)
 764                 *countp = entp->nle_nnode;
 765
 766         return rc;
 767 }
 768
 769 int
 770 jt_lst_ping(int argc,  char **argv)
 771 {
 772         struct list_head   head;
 773         lnet_process_id_t *ids = NULL;
 774         lstcon_rpc_ent_t  *ent = NULL;
 775         char              *str = NULL;
 776         int                optidx  = 0;
 777         int                server  = 0;
 778         int                timeout = 5;
 779         int                count   = 0;
 780         int                type    = 0;
 781         int                rc      = 0;
 782         int                c;
 783
 784         static struct option ping_opts[] =
 785         {
 786                 {"session", no_argument,       0, 's' },
 787                 {"server",  no_argument,       0, 'v' },
 788                 {"batch",   required_argument, 0, 'b' },
 789                 {"group",   required_argument, 0, 'g' },
 790                 {"nodes",   required_argument, 0, 'n' },
 791                 {"timeout", required_argument, 0, 't' },
 792                 {0,         0,                 0,  0  }
 793         };
 794
 795         if (session_key == 0) {
 796                 fprintf(stderr,
 797                         "Can't find env LST_SESSION or value is not valid\n");
 798                 return -1;
 799         }
 800
 801         while (1) {
 802
 803                 c = getopt_long(argc, argv, "g:b:n:t:sv",
 804                                 ping_opts, &optidx);
 805
 806                 if (c == -1)
 807                         break;
 808
 809                 switch (c) {
 810                 case 's':
 811                         type = LST_OPC_SESSION;
 812                         break;
 813
 814                 case 'g':
 815                         type = LST_OPC_GROUP;
 816                         str = optarg;
 817                         break;
 818
 819                 case 'b':
 820                         type = LST_OPC_BATCHCLI;
 821                         str = optarg;
 822                         break;
 823
 824                 case 'n':
 825                         type = LST_OPC_NODES;
 826                         str = optarg;
 827                         break;
 828
 829                 case 't':
 830                         timeout = atoi(optarg);
 831                         break;
 832
 833                 case 'v':
 834                         server = 1;
 835                         break;
 836
 837                 default:
 838                         lst_print_usage(argv[0]);
 839                         return -1;
 840                 }
 841         }
 842
 843         if (type == 0 || timeout <= 0 || optind != argc) {
 844                 lst_print_usage(argv[0]);
 845                 return -1;
 846         }
 847
 848         if (type == LST_OPC_BATCHCLI && server)
 849                 type = LST_OPC_BATCHSRV;
 850
 851         rc = lst_get_node_count(type, str, &count, &ids);
 852         if (rc < 0) {
 853                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
 854                         (str == NULL) ? "session" : str, strerror(errno));
 855                 return -1;
 856         }
 857
 858         CFS_INIT_LIST_HEAD(&head);
 859
 860         rc = lst_alloc_rpcent(&head, count, LST_NAME_SIZE);
 861         if (rc != 0) {
 862                 fprintf(stderr, "Out of memory\n");
 863                 goto out;
 864         }
 865
 866         if (count == 0) {
 867                 fprintf(stdout, "Target %s is empty\n",
 868                         (str == NULL) ? "session" : str);
 869                 goto out;
 870         }
 871
 872         rc = lst_ping_ioctl(str, type, timeout, count, ids, &head);
 873         if (rc == -1) { /* local failure */
 874                 lst_print_error("debug", "Failed to ping %s: %s\n",
 875                                 (str == NULL) ? "session" : str,
 876                                 strerror(errno));
 877                 rc = -1;
 878                 goto out;
 879         }
 880
 881         /* ignore RPC errors and framwork errors */
 882         list_for_each_entry(ent, &head, rpe_link) {
 883                 fprintf(stdout, "\t%s: %s [session: %s id: %s]\n",
 884                         libcfs_id2str(ent->rpe_peer),
 885                         lst_node_state2str(ent->rpe_state),
 886                         (ent->rpe_state == LST_NODE_ACTIVE ||
 887                          ent->rpe_state == LST_NODE_BUSY)?
 888                                  (ent->rpe_rpc_errno == 0 ?
 889                                          &ent->rpe_payload[0] : "Unknown") :
 890                                  "<NULL>", libcfs_nid2str(ent->rpe_sid.ses_nid));
 891         }
 892
 893 out:
 894         lst_free_rpcent(&head);
 895
 896         if (ids != NULL)
 897                 free(ids);
 898
 899         return rc;
 900
 901 }
 902
 903 int
 904 lst_add_nodes_ioctl (char *name, int count, lnet_process_id_t *ids,
 905                      struct list_head *resultp)
 906 {
 907         lstio_group_nodes_args_t        args = {
 908                 .lstio_grp_key          = session_key,
 909                 .lstio_grp_nmlen        = strlen(name),
 910                 .lstio_grp_namep        = name,
 911                 .lstio_grp_count        = count,
 912                 .lstio_grp_idsp         = ids,
 913                 .lstio_grp_resultp      = resultp,
 914         };
 915
 916         return lst_ioctl(LSTIO_NODES_ADD, &args, sizeof(args));
 917 }
 918
 919 int
 920 lst_add_group_ioctl (char *name)
 921 {
 922         lstio_group_add_args_t  args = {
 923                 .lstio_grp_key          = session_key,
 924                 .lstio_grp_nmlen        = strlen(name),
 925                 .lstio_grp_namep        = name,
 926         };
 927
 928         return lst_ioctl(LSTIO_GROUP_ADD, &args, sizeof(args));
 929 }
 930
 931 int
 932 jt_lst_add_group(int argc, char **argv)
 933 {
 934         struct list_head   head;
 935         lnet_process_id_t *ids;
 936         char              *name;
 937         int                count;
 938         int                rc;
 939         int                i;
 940
 941         if (session_key == 0) {
 942                 fprintf(stderr,
 943                         "Can't find env LST_SESSION or value is not valid\n");
 944                 return -1;
 945         }
 946
 947         if (argc < 3) {
 948                 lst_print_usage(argv[0]);
 949                 return -1;
 950         }
 951
 952         name = argv[1];
 953         if (strlen(name) >= LST_NAME_SIZE) {
 954                 fprintf(stderr, "Name length is limited to %d\n",
 955                         LST_NAME_SIZE - 1);
 956                 return -1;
 957         }
 958
 959         rc = lst_add_group_ioctl(name);
 960         if (rc != 0) {
 961                 lst_print_error("group", "Failed to add group %s: %s\n",
 962                                 name, strerror(errno));
 963                 return -1;
 964         }
 965
 966         CFS_INIT_LIST_HEAD(&head);
 967
 968         for (i = 2; i < argc; i++) {
 969                 /* parse address list */
 970                 rc = lst_parse_nids(argv[i], &count, &ids);
 971                 if (rc < 0) {
 972                         fprintf(stderr, "Ignore invalid id list %s\n",
 973                                 argv[i]);
 974                         continue;
 975                 }
 976
 977                 if (count == 0)
 978                         continue;
 979
 980                 rc = lst_alloc_rpcent(&head, count, 0);
 981                 if (rc != 0) {
 982                         fprintf(stderr, "Out of memory\n");
 983                         break;
 984                 }
 985
 986                 rc = lst_add_nodes_ioctl(name, count, ids, &head);
 987
 988                 free(ids);
 989
 990                 if (rc == 0) {
 991                         lst_free_rpcent(&head);
 992                         fprintf(stderr, "%s are added to session\n", argv[i]);
 993                         continue;
 994                 }
 995
 996                 if (rc == -1) {
 997                         lst_free_rpcent(&head);
 998                         lst_print_error("group", "Failed to add nodes %s: %s\n",
 999                                         argv[i], strerror(errno));
1000                         break;
1001                 }
1002
1003                 lst_print_transerr(&head, "create session");
1004                 lst_free_rpcent(&head);
1005         }
1006
1007         return rc;
1008 }
1009
1010 int
1011 lst_del_group_ioctl (char *name)
1012 {
1013         lstio_group_del_args_t  args = {
1014                 .lstio_grp_key          = session_key,
1015                 .lstio_grp_nmlen        = strlen(name),
1016                 .lstio_grp_namep        = name,
1017         };
1018
1019         return lst_ioctl(LSTIO_GROUP_DEL, &args, sizeof(args));
1020 }
1021
1022 int
1023 jt_lst_del_group(int argc, char **argv)
1024 {
1025         int     rc;
1026
1027         if (session_key == 0) {
1028                 fprintf(stderr,
1029                         "Can't find env LST_SESSION or value is not valid\n");
1030                 return -1;
1031         }
1032
1033         if (argc != 2) {
1034                 lst_print_usage(argv[0]);
1035                 return -1;
1036         }
1037
1038         rc = lst_del_group_ioctl(argv[1]);
1039         if (rc == 0) {
1040                 fprintf(stdout, "Group is deleted\n");
1041                 return 0;
1042         }
1043
1044         if (rc == -1) {
1045                 lst_print_error("group", "Failed to delete group: %s\n",
1046                                 strerror(errno));
1047                 return rc;
1048         }
1049
1050         fprintf(stderr, "Group is deleted with some errors\n");
1051
1052         if (trans_stat.trs_rpc_errno != 0) {
1053                 fprintf(stderr, "[RPC] Failed to send %d end session RPCs: %s\n",
1054                         lstcon_rpc_stat_failure(&trans_stat, 0),
1055                         strerror(trans_stat.trs_rpc_errno));
1056         }
1057
1058         if (trans_stat.trs_fwk_errno != 0) {
1059                 fprintf(stderr,
1060                         "[FWK] Failed to end session on %d nodes: %s\n",
1061                         lstcon_sesop_stat_failure(&trans_stat, 0),
1062                         strerror(trans_stat.trs_fwk_errno));
1063         }
1064
1065         return -1;
1066 }
1067
1068 int
1069 lst_update_group_ioctl(int opc, char *name, int clean, int count,
1070                        lnet_process_id_t *ids, struct list_head *resultp)
1071 {
1072         lstio_group_update_args_t  args = {
1073                 .lstio_grp_key          = session_key,
1074                 .lstio_grp_opc          = opc,
1075                 .lstio_grp_args         = clean,
1076                 .lstio_grp_nmlen        = strlen(name),
1077                 .lstio_grp_namep        = name,
1078                 .lstio_grp_count        = count,
1079                 .lstio_grp_idsp         = ids,
1080                 .lstio_grp_resultp      = resultp,
1081         };
1082
1083         return lst_ioctl(LSTIO_GROUP_UPDATE, &args, sizeof(args));
1084 }
1085
1086 int
1087 jt_lst_update_group(int argc, char **argv)
1088 {
1089         struct list_head   head;
1090         lnet_process_id_t *ids = NULL;
1091         char              *str = NULL;
1092         char              *grp = NULL;
1093         int                optidx = 0;
1094         int                count = 0;
1095         int                clean = 0;
1096         int                opc = 0;
1097         int                rc;
1098         int                c;
1099
1100         static struct option update_group_opts[] =
1101         {
1102                 {"refresh", no_argument,       0, 'f' },
1103                 {"clean",   required_argument, 0, 'c' },
1104                 {"remove",  required_argument, 0, 'r' },
1105                 {0,         0,                 0,  0  }
1106         };
1107
1108         if (session_key == 0) {
1109                 fprintf(stderr,
1110                         "Can't find env LST_SESSION or value is not valid\n");
1111                 return -1;
1112         }
1113
1114         while (1) {
1115                 c = getopt_long(argc, argv, "fc:r:",
1116                                 update_group_opts, &optidx);
1117
1118                 /* Detect the end of the options. */
1119                 if (c == -1)
1120                         break;
1121
1122                 switch (c) {
1123                 case 'f':
1124                         if (opc != 0) {
1125                                 lst_print_usage(argv[0]);
1126                                 return -1;
1127                         }
1128                         opc = LST_GROUP_REFRESH;
1129                         break;
1130
1131                 case 'r':
1132                         if (opc != 0) {
1133                                 lst_print_usage(argv[0]);
1134                                 return -1;
1135                         }
1136                         opc = LST_GROUP_RMND;
1137                         str = optarg;
1138                         break;
1139
1140                 case 'c':
1141                         clean = lst_node_str2state(optarg);
1142                         if (opc != 0 || clean <= 0) {
1143                                 lst_print_usage(argv[0]);
1144                                 return -1;
1145                         }
1146                         opc = LST_GROUP_CLEAN;
1147                         break;
1148
1149                 default:
1150                         lst_print_usage(argv[0]);
1151                         return -1;
1152                 }
1153         }
1154
1155         /* no OPC or group is specified */
1156         if (opc == 0 || optind != argc - 1) {
1157                 lst_print_usage(argv[0]);
1158                 return -1;
1159         }
1160
1161         grp = argv[optind];
1162
1163         CFS_INIT_LIST_HEAD(&head);
1164
1165         if (opc == LST_GROUP_RMND || opc == LST_GROUP_REFRESH) {
1166                 rc = lst_get_node_count(opc == LST_GROUP_RMND ? LST_OPC_NODES :
1167                                                                 LST_OPC_GROUP,
1168                                         opc == LST_GROUP_RMND ? str : grp,
1169                                         &count, &ids);
1170
1171                 if (rc != 0) {
1172                         fprintf(stderr, "Can't get count of nodes from %s: %s\n",
1173                                 opc == LST_GROUP_RMND ? str : grp,
1174                                 strerror(errno));
1175                         return -1;
1176                 }
1177
1178                 rc = lst_alloc_rpcent(&head, count, 0);
1179                 if (rc != 0) {
1180                         fprintf(stderr, "Out of memory\n");
1181                         free(ids);
1182                         return -1;
1183                 }
1184
1185         }
1186
1187         rc = lst_update_group_ioctl(opc, grp, clean, count, ids, &head);
1188
1189         if (ids != NULL)
1190                 free(ids);
1191
1192         if (rc == 0) {
1193                 lst_free_rpcent(&head);
1194                 return 0;
1195         }
1196
1197         if (rc == -1) {
1198                 lst_free_rpcent(&head);
1199                 lst_print_error("group", "Failed to update group: %s\n",
1200                                 strerror(errno));
1201                 return rc;
1202         }
1203
1204         lst_print_transerr(&head, "Updating group");
1205
1206         lst_free_rpcent(&head);
1207
1208         return rc;
1209 }
1210
1211 int
1212 lst_list_group_ioctl(int len, char *name, int idx)
1213 {
1214         lstio_group_list_args_t         args = {
1215                 .lstio_grp_key          = session_key,
1216                 .lstio_grp_idx          = idx,
1217                 .lstio_grp_nmlen        = len,
1218                 .lstio_grp_namep        = name,
1219         };
1220
1221         return lst_ioctl(LSTIO_GROUP_LIST, &args, sizeof(args));
1222 }
1223
1224 int
1225 lst_info_group_ioctl(char *name, lstcon_ndlist_ent_t *gent,
1226                      int *idx, int *count, lstcon_node_ent_t *dents)
1227 {
1228         lstio_group_info_args_t         args = {
1229                 .lstio_grp_key          = session_key,
1230                 .lstio_grp_nmlen        = strlen(name),
1231                 .lstio_grp_namep        = name,
1232                 .lstio_grp_entp         = gent,
1233                 .lstio_grp_idxp         = idx,
1234                 .lstio_grp_ndentp       = count,
1235                 .lstio_grp_dentsp       = dents,
1236         };
1237
1238         return lst_ioctl(LSTIO_GROUP_INFO, &args, sizeof(args));
1239 }
1240
1241 int
1242 lst_list_group_all(void)
1243 {
1244         char  name[LST_NAME_SIZE];
1245         int   rc;
1246         int   i;
1247
1248         /* no group is specified, list name of all groups */
1249         for (i = 0; ; i++) {
1250                 rc = lst_list_group_ioctl(LST_NAME_SIZE, name, i);
1251                 if (rc == 0) {
1252                         fprintf(stdout, "%d) %s\n", i + 1, name);
1253                         continue;
1254                 }
1255
1256                 if (errno == ENOENT)
1257                         break;
1258
1259                 lst_print_error("group", "Failed to list group: %s\n",
1260                                 strerror(errno));
1261                 return -1;
1262         }
1263
1264         fprintf(stdout, "Total %d groups\n", i);
1265
1266         return 0;
1267 }
1268
1269 #define LST_NODES_TITLE "\tACTIVE\tBUSY\tDOWN\tUNKNOWN\tTOTAL\n"
1270
1271 int
1272 jt_lst_list_group(int argc, char **argv)
1273 {
1274         lstcon_ndlist_ent_t  gent;
1275         lstcon_node_ent_t   *dents;
1276         int               optidx  = 0;
1277         int               verbose = 0;
1278         int               active  = 0;
1279         int               busy    = 0;
1280         int               down    = 0;
1281         int               unknown = 0;
1282         int               all     = 0;
1283         int               count;
1284         int               index;
1285         int               i;
1286         int               j;
1287         int               c;
1288         int               rc = 0;
1289
1290         static struct option list_group_opts[] =
1291         {
1292                 {"active",  no_argument, 0, 'a' },
1293                 {"busy",    no_argument, 0, 'b' },
1294                 {"down",    no_argument, 0, 'd' },
1295                 {"unknown", no_argument, 0, 'u' },
1296                 {"all",     no_argument, 0, 'l' },
1297                 {0,         0,           0,  0  }
1298         };
1299
1300         if (session_key == 0) {
1301                 fprintf(stderr,
1302                         "Can't find env LST_SESSION or value is not valid\n");
1303                 return -1;
1304         }
1305
1306         while (1) {
1307                 c = getopt_long(argc, argv, "abdul",
1308                                 list_group_opts, &optidx);
1309
1310                 if (c == -1)
1311                         break;
1312
1313                 switch (c) {
1314                 case 'a':
1315                         verbose = active = 1;
1316                         all = 0;
1317                         break;
1318                 case 'b':
1319                         verbose = busy = 1;
1320                         all = 0;
1321                         break;
1322                 case 'd':
1323                         verbose = down = 1;
1324                         all = 0;
1325                         break;
1326                 case 'u':
1327                         verbose = unknown = 1;
1328                         all = 0;
1329                         break;
1330                 case 'l':
1331                         verbose = all = 1;
1332                         break;
1333                 default:
1334                         lst_print_usage(argv[0]);
1335                         return -1;
1336                 }
1337         }
1338
1339         if (optind == argc) {
1340                 /* no group is specified, list name of all groups */
1341                 rc = lst_list_group_all();
1342
1343                 return rc;
1344         }
1345
1346         if (!verbose)
1347                 fprintf(stdout, LST_NODES_TITLE);
1348
1349         /* list nodes in specified groups */
1350         for (i = optind; i < argc; i++) {
1351                 rc = lst_info_group_ioctl(argv[i], &gent, NULL, NULL, NULL);
1352                 if (rc != 0) {
1353                         if (errno == ENOENT) {
1354                                 rc = 0;
1355                                 break;
1356                         }
1357
1358                         lst_print_error("group", "Failed to list group\n",
1359                                         strerror(errno));
1360                         break;
1361                 }
1362
1363                 if (!verbose) {
1364                         fprintf(stdout, "\t%d\t%d\t%d\t%d\t%d\t%s\n",
1365                                 gent.nle_nactive, gent.nle_nbusy,
1366                                 gent.nle_ndown, gent.nle_nunknown,
1367                                 gent.nle_nnode, argv[i]);
1368                         continue;
1369                 }
1370
1371                 fprintf(stdout, "Group [ %s ]\n", argv[i]);
1372
1373                 if (gent.nle_nnode == 0) {
1374                         fprintf(stdout, "No nodes found [ %s ]\n", argv[i]);
1375                         continue;
1376                 }
1377
1378                 count = gent.nle_nnode;
1379
1380                 dents = malloc(count * sizeof(lstcon_node_ent_t));
1381                 if (dents == NULL) {
1382                         fprintf(stderr, "Failed to malloc: %s\n",
1383                                 strerror(errno));
1384                         return -1;
1385                 }
1386
1387                 index = 0;
1388                 rc = lst_info_group_ioctl(argv[i], &gent, &index, &count, dents);
1389                 if (rc != 0) {
1390                         lst_print_error("group", "Failed to list group: %s\n",
1391                                         strerror(errno));
1392                         free(dents);
1393                         return -1;
1394                 }
1395
1396                 for (j = 0, c = 0; j < count; j++) {
1397                         if (all ||
1398                             ((active  &&  dents[j].nde_state == LST_NODE_ACTIVE) ||
1399                              (busy    &&  dents[j].nde_state == LST_NODE_BUSY)   ||
1400                              (down    &&  dents[j].nde_state == LST_NODE_DOWN)   ||
1401                              (unknown &&  dents[j].nde_state == LST_NODE_UNKNOWN))) {
1402
1403                                 fprintf(stdout, "\t%s: %s\n",
1404                                         libcfs_id2str(dents[j].nde_id),
1405                                         lst_node_state2str(dents[j].nde_state));
1406                                 c++;
1407                         }
1408                 }
1409
1410                 fprintf(stdout, "Total %d nodes [ %s ]\n", c, argv[i]);
1411
1412                 free(dents);
1413         }
1414
1415         return rc;
1416 }
1417
1418 int
1419 lst_stat_ioctl (char *name, int count, lnet_process_id_t *idsp,
1420                 int timeout, struct list_head *resultp)
1421 {
1422         lstio_stat_args_t  args = {
1423                 .lstio_sta_key           = session_key,
1424                 .lstio_sta_timeout       = timeout,
1425                 .lstio_sta_nmlen         = strlen(name),
1426                 .lstio_sta_namep         = name,
1427                 .lstio_sta_count         = count,
1428                 .lstio_sta_idsp          = idsp,
1429                 .lstio_sta_resultp       = resultp,
1430         };
1431
1432         return lst_ioctl (LSTIO_STAT_QUERY, &args, sizeof(args));
1433 }
1434
1435 typedef struct {
1436         struct list_head        srp_link;
1437         int                     srp_count;
1438         char                   *srp_name;
1439         lnet_process_id_t      *srp_ids;
1440         struct list_head        srp_result[2];
1441 } lst_stat_req_param_t;
1442
1443 static void
1444 lst_stat_req_param_free(lst_stat_req_param_t *srp)
1445 {
1446         int     i;
1447
1448         for (i = 0; i < 2; i++)
1449                 lst_free_rpcent(&srp->srp_result[i]);
1450
1451         if (srp->srp_ids != NULL)
1452                 free(srp->srp_ids);
1453
1454         free(srp);
1455 }
1456
1457 static int
1458 lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp, int save_old)
1459 {
1460         lst_stat_req_param_t *srp = NULL;
1461         int                   count = save_old ? 2 : 1;
1462         int                   rc;
1463         int                   i;
1464
1465         srp = malloc(sizeof(*srp));
1466         if (srp == NULL)
1467                 return -ENOMEM;
1468
1469         memset(srp, 0, sizeof(*srp));
1470         CFS_INIT_LIST_HEAD(&srp->srp_result[0]);
1471         CFS_INIT_LIST_HEAD(&srp->srp_result[1]);
1472
1473         rc = lst_get_node_count(LST_OPC_GROUP, name,
1474                                 &srp->srp_count, NULL);
1475         if (rc != 0 && errno == ENOENT) {
1476                 rc = lst_get_node_count(LST_OPC_NODES, name,
1477                                         &srp->srp_count, &srp->srp_ids);
1478         }
1479
1480         if (rc != 0) {
1481                 fprintf(stderr,
1482                         "Failed to get count of nodes from %s: %s\n",
1483                         name, strerror(errno));
1484                 lst_stat_req_param_free(srp);
1485
1486                 return rc;
1487         }
1488
1489         srp->srp_name = name;
1490
1491         for (i = 0; i < count; i++) {
1492                 rc = lst_alloc_rpcent(&srp->srp_result[i], srp->srp_count,
1493                                       sizeof(sfw_counters_t)  +
1494                                       sizeof(srpc_counters_t) +
1495                                       sizeof(lnet_counters_t));
1496                 if (rc != 0) {
1497                         fprintf(stderr, "Out of memory\n");
1498                         break;
1499                 }
1500         }
1501
1502         if (rc == 0) {
1503                 *srpp = srp;
1504                 return 0;
1505         }
1506
1507         lst_stat_req_param_free(srp);
1508
1509         return rc;
1510 }
1511
1512 typedef struct {
1513         /* TODO */
1514 } lst_srpc_stat_result;
1515
1516 #define LST_LNET_AVG    0
1517 #define LST_LNET_MIN    1
1518 #define LST_LNET_MAX    2
1519
1520 typedef struct {
1521         float           lnet_avg_sndrate;
1522         float           lnet_min_sndrate;
1523         float           lnet_max_sndrate;
1524         float           lnet_total_sndrate;
1525
1526         float           lnet_avg_rcvrate;
1527         float           lnet_min_rcvrate;
1528         float           lnet_max_rcvrate;
1529         float           lnet_total_rcvrate;
1530
1531         float           lnet_avg_sndperf;
1532         float           lnet_min_sndperf;
1533         float           lnet_max_sndperf;
1534         float           lnet_total_sndperf;
1535
1536         float           lnet_avg_rcvperf;
1537         float           lnet_min_rcvperf;
1538         float           lnet_max_rcvperf;
1539         float           lnet_total_rcvperf;
1540
1541         int             lnet_stat_count;
1542 } lst_lnet_stat_result_t;
1543
1544 lst_lnet_stat_result_t lnet_stat_result;
1545
1546 static float
1547 lst_lnet_stat_value(int bw, int send, int off)
1548 {
1549         float  *p;
1550
1551         p = bw ? &lnet_stat_result.lnet_avg_sndperf :
1552                  &lnet_stat_result.lnet_avg_sndrate;
1553
1554         if (!send)
1555                 p += 4;
1556
1557         p += off;
1558
1559         return *p;
1560 }
1561
1562 static void
1563 lst_timeval_diff(struct timeval *tv1,
1564                  struct timeval *tv2, struct timeval *df)
1565 {
1566         if (tv1->tv_usec >= tv2->tv_usec) {
1567                 df->tv_sec  = tv1->tv_sec - tv2->tv_sec;
1568                 df->tv_usec = tv1->tv_usec - tv2->tv_usec;
1569                 return;
1570         }
1571
1572         df->tv_sec  = tv1->tv_sec - 1 - tv2->tv_sec;
1573         df->tv_usec = tv1->tv_sec + 1000000 - tv2->tv_usec;
1574
1575         return;
1576 }
1577
1578 void
1579 lst_cal_lnet_stat(float delta, lnet_counters_t *lnet_new,
1580                   lnet_counters_t *lnet_old)
1581 {
1582         float perf;
1583         float rate;
1584
1585         perf = (float)(lnet_new->send_length -
1586                        lnet_old->send_length) / (1024 * 1024) / delta;
1587         lnet_stat_result.lnet_total_sndperf += perf;
1588
1589         if (lnet_stat_result.lnet_min_sndperf > perf ||
1590             lnet_stat_result.lnet_min_sndperf == 0)
1591                 lnet_stat_result.lnet_min_sndperf = perf;
1592
1593         if (lnet_stat_result.lnet_max_sndperf < perf)
1594                 lnet_stat_result.lnet_max_sndperf = perf;
1595
1596         perf = (float)(lnet_new->recv_length -
1597                        lnet_old->recv_length) / (1024 * 1024) / delta;
1598         lnet_stat_result.lnet_total_rcvperf += perf;
1599
1600         if (lnet_stat_result.lnet_min_rcvperf > perf ||
1601             lnet_stat_result.lnet_min_rcvperf == 0)
1602                 lnet_stat_result.lnet_min_rcvperf = perf;
1603
1604         if (lnet_stat_result.lnet_max_rcvperf < perf)
1605                 lnet_stat_result.lnet_max_rcvperf = perf;
1606
1607         rate = (lnet_new->send_count - lnet_old->send_count) / delta;
1608         lnet_stat_result.lnet_total_sndrate += rate;
1609
1610         if (lnet_stat_result.lnet_min_sndrate > rate ||
1611             lnet_stat_result.lnet_min_sndrate == 0)
1612                 lnet_stat_result.lnet_min_sndrate = rate;
1613
1614         if (lnet_stat_result.lnet_max_sndrate < rate)
1615                 lnet_stat_result.lnet_max_sndrate = rate;
1616
1617         rate = (lnet_new->recv_count - lnet_old->recv_count) / delta;
1618         lnet_stat_result.lnet_total_rcvrate += rate;
1619
1620         if (lnet_stat_result.lnet_min_rcvrate > rate ||
1621             lnet_stat_result.lnet_min_rcvrate == 0)
1622                 lnet_stat_result.lnet_min_rcvrate = rate;
1623
1624         if (lnet_stat_result.lnet_max_rcvrate < rate)
1625                 lnet_stat_result.lnet_max_rcvrate = rate;
1626
1627         lnet_stat_result.lnet_stat_count ++;
1628
1629         lnet_stat_result.lnet_avg_sndrate = lnet_stat_result.lnet_total_sndrate /
1630                                             lnet_stat_result.lnet_stat_count;
1631         lnet_stat_result.lnet_avg_rcvrate = lnet_stat_result.lnet_total_rcvrate /
1632                                             lnet_stat_result.lnet_stat_count;
1633
1634         lnet_stat_result.lnet_avg_sndperf = lnet_stat_result.lnet_total_sndperf /
1635                                             lnet_stat_result.lnet_stat_count;
1636         lnet_stat_result.lnet_avg_rcvperf = lnet_stat_result.lnet_total_rcvperf /
1637                                             lnet_stat_result.lnet_stat_count;
1638
1639 }
1640
1641 void
1642 lst_print_lnet_stat(char *name, int bwrt, int rdwr, int type)
1643 {
1644         int     start1 = 0;
1645         int     end1   = 1;
1646         int     start2 = 0;
1647         int     end2   = 1;
1648         int     i;
1649         int     j;
1650
1651         if (lnet_stat_result.lnet_stat_count == 0)
1652                 return;
1653
1654         if (bwrt == 1) /* bw only */
1655                 start1 = 1;
1656
1657         if (bwrt == 2) /* rates only */
1658                 end1 = 0;
1659
1660         if (rdwr == 1) /* recv only */
1661                 start2 = 1;
1662
1663         if (rdwr == 2) /* send only */
1664                 end2 = 0;
1665
1666         for (i = start1; i <= end1; i++) {
1667                 fprintf(stdout, "[LNet %s of %s]\n",
1668                         i == 0 ? "Rates" : "Bandwidth", name);
1669
1670                 for (j = start2; j <= end2; j++) {
1671                         fprintf(stdout, "[%c] ", j == 0 ? 'R' : 'W');
1672
1673                         if ((type & 1) != 0) {
1674                                 fprintf(stdout, i == 0 ? "Avg: %-8.0f RPC/s " :
1675                                                          "Avg: %-8.2f MB/s  ",
1676                                         lst_lnet_stat_value(i, j, 0));
1677                         }
1678
1679                         if ((type & 2) != 0) {
1680                                 fprintf(stdout, i == 0 ? "Min: %-8.0f RPC/s " :
1681                                                          "Min: %-8.2f MB/s  ",
1682                                         lst_lnet_stat_value(i, j, 1));
1683                         }
1684
1685                         if ((type & 4) != 0) {
1686                                 fprintf(stdout, i == 0 ? "Max: %-8.0f RPC/s" :
1687                                                          "Max: %-8.2f MB/s",
1688                                         lst_lnet_stat_value(i, j, 2));
1689                         }
1690
1691                         fprintf(stdout, "\n");
1692                 }
1693         }
1694 }
1695
1696 void
1697 lst_print_stat(char *name, struct list_head *resultp,
1698                int idx, int lnet, int bwrt, int rdwr, int type)
1699 {
1700         struct list_head  tmp[2];
1701         lstcon_rpc_ent_t *new;
1702         lstcon_rpc_ent_t *old;
1703         sfw_counters_t   *sfwk_new;
1704         sfw_counters_t   *sfwk_old;
1705         srpc_counters_t  *srpc_new;
1706         srpc_counters_t  *srpc_old;
1707         lnet_counters_t  *lnet_new;
1708         lnet_counters_t  *lnet_old;
1709         struct timeval    tv;
1710         float             delta;
1711         int               errcount = 0;
1712
1713         CFS_INIT_LIST_HEAD(&tmp[0]);
1714         CFS_INIT_LIST_HEAD(&tmp[1]);
1715
1716         memset(&lnet_stat_result, 0, sizeof(lnet_stat_result));
1717
1718         while (!list_empty(&resultp[idx])) {
1719                 if (list_empty(&resultp[1 - idx])) {
1720                         fprintf(stderr, "Group is changed, re-run stat\n");
1721                         break;
1722                 }
1723
1724                 new = list_entry(resultp[idx].next, lstcon_rpc_ent_t, rpe_link);
1725                 old = list_entry(resultp[1 - idx].next, lstcon_rpc_ent_t, rpe_link);
1726
1727                 /* first time get stats result, can't calculate diff */
1728                 if (new->rpe_peer.nid == LNET_NID_ANY)
1729                         break;
1730
1731                 if (new->rpe_peer.nid != old->rpe_peer.nid ||
1732                     new->rpe_peer.pid != old->rpe_peer.pid) {
1733                         /* Something wrong. i.e, somebody change the group */
1734                         break;
1735                 }
1736
1737                 list_del(&new->rpe_link);
1738                 list_add_tail(&new->rpe_link, &tmp[idx]);
1739
1740                 list_del(&old->rpe_link);
1741                 list_add_tail(&old->rpe_link, &tmp[1 - idx]);
1742
1743                 if (new->rpe_rpc_errno != 0 || new->rpe_fwk_errno != 0 ||
1744                     old->rpe_rpc_errno != 0 || old->rpe_fwk_errno != 0) {
1745                         errcount ++;
1746                         continue;
1747                 }
1748
1749                 sfwk_new = (sfw_counters_t *)&new->rpe_payload[0];
1750                 sfwk_old = (sfw_counters_t *)&old->rpe_payload[0];
1751
1752                 srpc_new = (srpc_counters_t *)((char *)sfwk_new + sizeof(*sfwk_new));
1753                 srpc_old = (srpc_counters_t *)((char *)sfwk_old + sizeof(*sfwk_old));
1754
1755                 lnet_new = (lnet_counters_t *)((char *)srpc_new + sizeof(*srpc_new));
1756                 lnet_old = (lnet_counters_t *)((char *)srpc_old + sizeof(*srpc_old));
1757
1758                 lst_timeval_diff(&new->rpe_stamp, &old->rpe_stamp, &tv);
1759
1760                 delta = tv.tv_sec + (float)tv.tv_usec/1000000;
1761
1762                 if (!lnet) /* TODO */
1763                         continue;
1764
1765                 lst_cal_lnet_stat(delta, lnet_new, lnet_old);
1766         }
1767
1768         list_splice(&tmp[idx], &resultp[idx]);
1769         list_splice(&tmp[1 - idx], &resultp[1 - idx]);
1770
1771         if (errcount > 0)
1772                 fprintf(stdout, "Failed to stat on %d nodes\n", errcount);
1773
1774         if (!lnet)  /* TODO */
1775                 return;
1776
1777         lst_print_lnet_stat(name, bwrt, rdwr, type);
1778 }
1779
1780 int
1781 jt_lst_stat(int argc, char **argv)
1782 {
1783         struct list_head      head;
1784         lst_stat_req_param_t *srp;
1785         time_t                last    = 0;
1786         int                   optidx  = 0;
1787         int                   timeout = 5; /* default timeout, 5 sec */
1788         int                   delay   = 5; /* default delay, 5 sec */
1789         int                   lnet    = 1; /* lnet stat by default */
1790         int                   bwrt    = 0;
1791         int                   rdwr    = 0;
1792         int                   type    = -1;
1793         int                   idx     = 0;
1794         int                   rc;
1795         int                   c;
1796
1797         static struct option stat_opts[] =
1798         {
1799                 {"timeout", required_argument, 0, 't' },
1800                 {"delay"  , required_argument, 0, 'd' },
1801                 {"lnet"   , no_argument,       0, 'l' },
1802                 {"rpc"    , no_argument,       0, 'c' },
1803                 {"bw"     , no_argument,       0, 'b' },
1804                 {"rate"   , no_argument,       0, 'a' },
1805                 {"read"   , no_argument,       0, 'r' },
1806                 {"write"  , no_argument,       0, 'w' },
1807                 {"avg"    , no_argument,       0, 'g' },
1808                 {"min"    , no_argument,       0, 'n' },
1809                 {"max"    , no_argument,       0, 'x' },
1810                 {0,         0,                 0,  0  }
1811         };
1812
1813         if (session_key == 0) {
1814                 fprintf(stderr,
1815                         "Can't find env LST_SESSION or value is not valid\n");
1816                 return -1;
1817         }
1818
1819         while (1) {
1820                 c = getopt_long(argc, argv, "t:d:lcbarwgnx", stat_opts, &optidx);
1821
1822                 if (c == -1)
1823                         break;
1824
1825                 switch (c) {
1826                 case 't':
1827                         timeout = atoi(optarg);
1828                         break;
1829                 case 'd':
1830                         delay = atoi(optarg);
1831                         break;
1832                 case 'l':
1833                         lnet = 1;
1834                         break;
1835                 case 'c':
1836                         lnet = 0;
1837                         break;
1838                 case 'b':
1839                         bwrt |= 1;
1840                         break;
1841                 case 'a':
1842                         bwrt |= 2;
1843                         break;
1844                 case 'r':
1845                         rdwr |= 1;
1846                         break;
1847                 case 'w':
1848                         rdwr |= 2;
1849                         break;
1850                 case 'g':
1851                         if (type == -1) {
1852                                 type = 1;
1853                                 break;
1854                         }
1855                         type |= 1;
1856                         break;
1857                 case 'n':
1858                         if (type == -1) {
1859                                 type = 2;
1860                                 break;
1861                         }
1862                         type |= 2;
1863                         break;
1864                 case 'x':
1865                         if (type == -1) {
1866                                 type = 4;
1867                                 break;
1868                         }
1869                         type |= 4;
1870                         break;
1871                 default:
1872                         lst_print_usage(argv[0]);
1873                         return -1;
1874                 }
1875         }
1876
1877         if (optind == argc) {
1878                 lst_print_usage(argv[0]);
1879                 return -1;
1880         }
1881
1882         if (timeout <= 0 || delay <= 0) {
1883                 fprintf(stderr, "Invalid timeout or delay value\n");
1884                 return -1;
1885         }
1886
1887         CFS_INIT_LIST_HEAD(&head);
1888
1889         while (optind < argc) {
1890                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 1);
1891                 if (rc != 0)
1892                         goto out;
1893
1894                 list_add_tail(&srp->srp_link, &head);
1895         }
1896
1897         while (1) {
1898                 time_t  now = time(NULL);
1899
1900                 if (now - last < delay) {
1901                         sleep(delay - now + last);
1902                         time(&now);
1903                 }
1904
1905                 last = now;
1906
1907                 list_for_each_entry(srp, &head, srp_link) {
1908                         rc = lst_stat_ioctl(srp->srp_name,
1909                                             srp->srp_count, srp->srp_ids,
1910                                             timeout, &srp->srp_result[idx]);
1911                         if (rc == -1) {
1912                                 lst_print_error("stat", "Failed to stat %s: %s\n",
1913                                                 srp->srp_name, strerror(errno));
1914                                 goto out;
1915                         }
1916
1917                         lst_print_stat(srp->srp_name, srp->srp_result,
1918                                        idx, lnet, bwrt, rdwr, type);
1919
1920                         lst_reset_rpcent(&srp->srp_result[1 - idx]);
1921                 }
1922
1923                 idx = 1 - idx;
1924         }
1925
1926 out:
1927         while (!list_empty(&head)) {
1928                 srp = list_entry(head.next, lst_stat_req_param_t, srp_link);
1929
1930                 list_del(&srp->srp_link);
1931                 lst_stat_req_param_free(srp);
1932         }
1933
1934         return rc;
1935 }
1936
1937 int
1938 jt_lst_show_error(int argc, char **argv)
1939 {
1940         struct list_head      head;
1941         lst_stat_req_param_t *srp;
1942         lstcon_rpc_ent_t     *ent;
1943         sfw_counters_t       *sfwk;
1944         srpc_counters_t      *srpc;
1945         lnet_counters_t      *lnet;
1946         int                   show_rpc = 1;
1947         int                   optidx   = 0;
1948         int                   rc       = 0;
1949         int                   ecount;
1950         int                   c;
1951
1952         static struct option  show_error_opts[] =
1953         {
1954                 {"session", no_argument,       0, 's' },
1955                 {0,         0,                 0,  0  }
1956         };
1957
1958         if (session_key == 0) {
1959                 fprintf(stderr,
1960                         "Can't find env LST_SESSION or value is not valid\n");
1961                 return -1;
1962         }
1963
1964         while (1) {
1965                 c = getopt_long(argc, argv, "s", show_error_opts, &optidx);
1966
1967                 if (c == -1)
1968                         break;
1969
1970                 switch (c) {
1971                 case 's':
1972                         show_rpc  = 0;
1973                         break;
1974
1975                 default:
1976                         lst_print_usage(argv[0]);
1977                         return -1;
1978                 }
1979         }
1980
1981         if (optind == argc) {
1982                 lst_print_usage(argv[0]);
1983                 return -1;
1984         }
1985
1986         CFS_INIT_LIST_HEAD(&head);
1987
1988         while (optind < argc) {
1989                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 0);
1990                 if (rc != 0)
1991                         goto out;
1992
1993                 list_add_tail(&srp->srp_link, &head);
1994         }
1995
1996         list_for_each_entry(srp, &head, srp_link) {
1997                 rc = lst_stat_ioctl(srp->srp_name, srp->srp_count,
1998                                     srp->srp_ids, 5, &srp->srp_result[0]);
1999
2000                 if (rc == -1) {
2001                         lst_print_error(srp->srp_name, "Failed to show errors of %s: %s\n",
2002                                         srp->srp_name, strerror(errno));
2003                         goto out;
2004                 }
2005
2006                 fprintf(stdout, "%s:\n", srp->srp_name);
2007
2008                 ecount = 0;
2009
2010                 list_for_each_entry(ent, &srp->srp_result[0], rpe_link) {
2011                         if (ent->rpe_rpc_errno != 0) {
2012                                 ecount ++;
2013                                 fprintf(stderr, "RPC failure, can't show error on %s\n",
2014                                         libcfs_id2str(ent->rpe_peer));
2015                                 continue;
2016                         }
2017
2018                         if (ent->rpe_fwk_errno != 0) {
2019                                 ecount ++;
2020                                 fprintf(stderr, "Framework failure, can't show error on %s\n",
2021                                         libcfs_id2str(ent->rpe_peer));
2022                                 continue;
2023                         }
2024
2025                         sfwk = (sfw_counters_t *)&ent->rpe_payload[0];
2026                         srpc = (srpc_counters_t *)((char *)sfwk + sizeof(*sfwk));
2027                         lnet = (lnet_counters_t *)((char *)srpc + sizeof(*srpc));
2028
2029                         if (srpc->errors == 0 &&
2030                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2031                                 continue;
2032
2033                         if (!show_rpc  &&
2034                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2035                                 continue;
2036
2037                         ecount ++;
2038
2039                         fprintf(stderr, "%s: [Session %d brw errors, %d ping errors]%c",
2040                                 libcfs_id2str(ent->rpe_peer),
2041                                 sfwk->brw_errors, sfwk->ping_errors,
2042                                 show_rpc  ? ' ' : '\n');
2043
2044                         if (!show_rpc)
2045                                 continue;
2046
2047                         fprintf(stderr, "[RPC: %d errors, %d dropped, %d expired]\n",
2048                                 srpc->errors, srpc->rpcs_dropped, srpc->rpcs_expired);
2049                 }
2050
2051                 fprintf(stdout, "Total %d error nodes in %s\n", ecount, srp->srp_name);
2052         }
2053 out:
2054         while (!list_empty(&head)) {
2055                 srp = list_entry(head.next, lst_stat_req_param_t, srp_link);
2056
2057                 list_del(&srp->srp_link);
2058                 lst_stat_req_param_free(srp);
2059         }
2060
2061         return rc;
2062 }
2063
2064 int
2065 lst_add_batch_ioctl (char *name)
2066 {
2067         lstio_batch_add_args_t  args = {
2068                 .lstio_bat_key           = session_key,
2069                 .lstio_bat_nmlen         = strlen(name),
2070                 .lstio_bat_namep         = name,
2071         };
2072
2073         return lst_ioctl (LSTIO_BATCH_ADD, &args, sizeof(args));
2074 }
2075
2076 int
2077 jt_lst_add_batch(int argc, char **argv)
2078 {
2079         char   *name;
2080         int     rc;
2081
2082         if (session_key == 0) {
2083                 fprintf(stderr,
2084                         "Can't find env LST_SESSION or value is not valid\n");
2085                 return -1;
2086         }
2087
2088         if (argc != 2) {
2089                 lst_print_usage(argv[0]);
2090                 return -1;
2091         }
2092
2093         name = argv[1];
2094         if (strlen(name) >= LST_NAME_SIZE) {
2095                 fprintf(stderr, "Name length is limited to %d\n",
2096                         LST_NAME_SIZE - 1);
2097                 return -1;
2098         }
2099
2100         rc = lst_add_batch_ioctl(name);
2101         if (rc == 0)
2102                 return 0;
2103
2104         lst_print_error("batch", "Failed to create batch: %s\n",
2105                         strerror(errno));
2106
2107         return -1;
2108 }
2109
2110 int
2111 lst_start_batch_ioctl (char *name, int timeout, struct list_head *resultp)
2112 {
2113         lstio_batch_run_args_t   args = {
2114                 .lstio_bat_key          = session_key,
2115                 .lstio_bat_timeout      = timeout,
2116                 .lstio_bat_nmlen        = strlen(name),
2117                 .lstio_bat_namep        = name,
2118                 .lstio_bat_resultp      = resultp,
2119         };
2120
2121         return lst_ioctl(LSTIO_BATCH_START, &args, sizeof(args));
2122 }
2123
2124 int
2125 jt_lst_start_batch(int argc, char **argv)
2126 {
2127         struct list_head  head;
2128         char             *batch;
2129         int               optidx  = 0;
2130         int               timeout = 0;
2131         int               count = 0;
2132         int               rc;
2133         int               c;
2134
2135         static struct option start_batch_opts[] =
2136         {
2137                 {"timeout", required_argument, 0, 't' },
2138                 {0,         0,                 0,  0  }
2139         };
2140
2141         if (session_key == 0) {
2142                 fprintf(stderr,
2143                         "Can't find env LST_SESSION or value is not valid\n");
2144                 return -1;
2145         }
2146
2147         while (1) {
2148                 c = getopt_long(argc, argv, "t:",
2149                                 start_batch_opts, &optidx);
2150
2151                 /* Detect the end of the options. */
2152                 if (c == -1)
2153                         break;
2154
2155                 switch (c) {
2156                 case 't':
2157                         timeout = atoi(optarg);
2158                         break;
2159                 default:
2160                         lst_print_usage(argv[0]);
2161                         return -1;
2162                 }
2163         }
2164
2165         if (optind == argc) {
2166                 batch = LST_DEFAULT_BATCH;
2167
2168         } else if (optind == argc - 1) {
2169                 batch = argv[optind];
2170
2171         } else {
2172                 lst_print_usage(argv[0]);
2173                 return -1;
2174         }
2175
2176         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2177         if (rc != 0) {
2178                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2179                         batch, strerror(errno));
2180                 return -1;
2181         }
2182
2183         CFS_INIT_LIST_HEAD(&head);
2184
2185         rc = lst_alloc_rpcent(&head, count, 0);
2186         if (rc != 0) {
2187                 fprintf(stderr, "Out of memory\n");
2188                 return -1;
2189         }
2190
2191         rc = lst_start_batch_ioctl(batch, timeout, &head);
2192
2193         if (rc == 0) {
2194                 fprintf(stdout, "%s is running now\n", batch);
2195                 lst_free_rpcent(&head);
2196                 return 0;
2197         }
2198
2199         if (rc == -1) {
2200                 lst_print_error("batch", "Failed to start batch: %s\n",
2201                                 strerror(errno));
2202                 lst_free_rpcent(&head);
2203                 return rc;
2204         }
2205
2206         lst_print_transerr(&head, "Run batch");
2207
2208         lst_free_rpcent(&head);
2209
2210         return rc;
2211 }
2212
2213 int
2214 lst_stop_batch_ioctl(char *name, int force, struct list_head *resultp)
2215 {
2216         lstio_batch_stop_args_t   args = {
2217                 .lstio_bat_key          = session_key,
2218                 .lstio_bat_force        = force,
2219                 .lstio_bat_nmlen        = strlen(name),
2220                 .lstio_bat_namep        = name,
2221                 .lstio_bat_resultp      = resultp,
2222         };
2223
2224         return lst_ioctl(LSTIO_BATCH_STOP, &args, sizeof(args));
2225 }
2226
2227 int
2228 jt_lst_stop_batch(int argc, char **argv)
2229 {
2230         struct list_head  head;
2231         char             *batch;
2232         int               force = 0;
2233         int               optidx;
2234         int               count;
2235         int               rc;
2236         int               c;
2237
2238         static struct option stop_batch_opts[] =
2239         {
2240                 {"force",   no_argument,   0, 'f' },
2241                 {0,         0,             0,  0  }
2242         };
2243
2244         if (session_key == 0) {
2245                 fprintf(stderr,
2246                         "Can't find env LST_SESSION or value is not valid\n");
2247                 return -1;
2248         }
2249
2250         while (1) {
2251                 c = getopt_long(argc, argv, "f",
2252                                 stop_batch_opts, &optidx);
2253
2254                 /* Detect the end of the options. */
2255                 if (c == -1)
2256                         break;
2257
2258                 switch (c) {
2259                 case 'f':
2260                         force = 1;
2261                         break;
2262                 default:
2263                         lst_print_usage(argv[0]);
2264                         return -1;
2265                 }
2266         }
2267
2268         if (optind == argc) {
2269                 batch = LST_DEFAULT_BATCH;
2270
2271         } else if (optind == argc - 1) {
2272                 batch = argv[optind];
2273
2274         } else {
2275                 lst_print_usage(argv[0]);
2276                 return -1;
2277         }
2278
2279         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2280         if (rc != 0) {
2281                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2282                         batch, strerror(errno));
2283                 return -1;
2284         }
2285
2286         CFS_INIT_LIST_HEAD(&head);
2287
2288         rc = lst_alloc_rpcent(&head, count, 0);
2289         if (rc != 0) {
2290                 fprintf(stderr, "Out of memory\n");
2291                 return -1;
2292         }
2293
2294         rc = lst_stop_batch_ioctl(batch, force, &head);
2295         if (rc != 0)
2296                 goto out;
2297
2298         while (1) {
2299                 lst_reset_rpcent(&head);
2300
2301                 rc = lst_query_batch_ioctl(batch, 0, 0, 30, &head);
2302                 if (rc != 0)
2303                         goto out;
2304
2305                 if (lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0 &&
2306                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0)
2307                         break;
2308
2309                 fprintf(stdout, "%d batch in stopping\n",
2310                         lstcon_tsbqry_stat_run(&trans_stat, 0));
2311                 sleep(1);
2312         }
2313
2314         fprintf(stdout, "Batch is stopped\n");
2315         lst_free_rpcent(&head);
2316
2317         return 0;
2318 out:
2319         if (rc == -1) {
2320                 lst_print_error("batch", "Failed to stop batch: %s\n",
2321                                 strerror(errno));
2322                 lst_free_rpcent(&head);
2323                 return -1;
2324         }
2325
2326         lst_print_transerr(&head, "stop batch");
2327
2328         lst_free_rpcent(&head);
2329
2330         return rc;
2331 }
2332
2333 int
2334 lst_list_batch_ioctl(int len, char *name, int index)
2335 {
2336         lstio_batch_list_args_t         args = {
2337                 .lstio_bat_key          = session_key,
2338                 .lstio_bat_idx          = index,
2339                 .lstio_bat_nmlen        = len,
2340                 .lstio_bat_namep        = name,
2341         };
2342
2343         return lst_ioctl(LSTIO_BATCH_LIST, &args, sizeof(args));
2344 }
2345
2346 int
2347 lst_info_batch_ioctl(char *batch, int test, int server,
2348                      lstcon_test_batch_ent_t *entp, int *idxp,
2349                      int *ndentp, lstcon_node_ent_t *dentsp)
2350 {
2351         lstio_batch_info_args_t         args = {
2352                 .lstio_bat_key          = session_key,
2353                 .lstio_bat_nmlen        = strlen(batch),
2354                 .lstio_bat_namep        = batch,
2355                 .lstio_bat_server       = server,
2356                 .lstio_bat_testidx      = test,
2357                 .lstio_bat_entp         = entp,
2358                 .lstio_bat_idxp         = idxp,
2359                 .lstio_bat_ndentp       = ndentp,
2360                 .lstio_bat_dentsp       = dentsp,
2361         };
2362
2363         return lst_ioctl(LSTIO_BATCH_INFO, &args, sizeof(args));
2364 }
2365
2366 int
2367 lst_list_batch_all(void)
2368 {
2369         char name[LST_NAME_SIZE];
2370         int  rc;
2371         int  i;
2372
2373         for (i = 0; ; i++) {
2374                 rc = lst_list_batch_ioctl(LST_NAME_SIZE, name, i);
2375                 if (rc == 0) {
2376                         fprintf(stdout, "%d) %s\n", i + 1, name);
2377                         continue;
2378                 }
2379
2380                 if (errno == ENOENT)
2381                         break;
2382
2383                 lst_print_error("batch", "Failed to list batch: %s\n",
2384                                 strerror(errno));
2385                 return rc;
2386         }
2387
2388         fprintf(stdout, "Total %d batches\n", i);
2389
2390         return 0;
2391 }
2392
2393 int
2394 lst_list_tsb_nodes(char *batch, int test, int server,
2395                    int count, int active, int invalid)
2396 {
2397         lstcon_node_ent_t *dents;
2398         int                index = 0;
2399         int                rc;
2400         int                c;
2401         int                i;
2402
2403         if (count == 0)
2404                 return 0;
2405
2406         /* verbose list, show nodes in batch or test */
2407         dents = malloc(count * sizeof(lstcon_node_ent_t));
2408         if (dents == NULL) {
2409                 fprintf(stdout, "Can't allocate memory\n");
2410                 return -1;
2411         }
2412
2413         rc = lst_info_batch_ioctl(batch, test, server,
2414                                   NULL, &index, &count, dents);
2415         if (rc != 0) {
2416                 free(dents);
2417                 lst_print_error((test > 0) ? "test" : "batch",
2418                                 (test > 0) ? "Failed to query test: %s\n" :
2419                                              "Failed to query batch: %s\n",
2420                                 strerror(errno));
2421                 return -1;
2422         }
2423
2424         for (i = 0, c = 0; i < count; i++) {
2425                 if ((!active  && dents[i].nde_state == LST_NODE_ACTIVE) ||
2426                     (!invalid && (dents[i].nde_state == LST_NODE_BUSY  ||
2427                                   dents[i].nde_state == LST_NODE_DOWN  ||
2428                                   dents[i].nde_state == LST_NODE_UNKNOWN)))
2429                         continue;
2430
2431                 fprintf(stdout, "\t%s: %s\n",
2432                         libcfs_id2str(dents[i].nde_id),
2433                         lst_node_state2str(dents[i].nde_state));
2434                 c++;
2435         }
2436
2437         fprintf(stdout, "Total %d nodes\n", c);
2438         free(dents);
2439
2440         return 0;
2441 }
2442
2443 int
2444 jt_lst_list_batch(int argc, char **argv)
2445 {
2446         lstcon_test_batch_ent_t ent;
2447         char                *batch   = NULL;
2448         int                  optidx  = 0;
2449         int                  verbose = 0; /* list nodes in batch or test */
2450         int                  invalid = 0;
2451         int                  active  = 0;
2452         int                  server  = 0;
2453         int                  ntest   = 0;
2454         int                  test    = 0;
2455         int                  c       = 0;
2456         int                  rc;
2457
2458         static struct option list_batch_opts[] =
2459         {
2460                 {"test",    required_argument, 0, 't' },
2461                 {"invalid", no_argument,       0, 'i' },
2462                 {"active",  no_argument,       0, 'a' },
2463                 {"all",     no_argument,       0, 'l' },
2464                 {"server",  no_argument,       0, 's' },
2465                 {0,         0,                 0,  0  }
2466         };
2467
2468         if (session_key == 0) {
2469                 fprintf(stderr,
2470                         "Can't find env LST_SESSION or value is not valid\n");
2471                 return -1;
2472         }
2473
2474         while (1) {
2475                 c = getopt_long(argc, argv, "ailst:",
2476                                 list_batch_opts, &optidx);
2477
2478                 if (c == -1)
2479                         break;
2480
2481                 switch (c) {
2482                 case 'a':
2483                         verbose = active = 1;
2484                         break;
2485                 case 'i':
2486                         verbose = invalid = 1;
2487                         break;
2488                 case 'l':
2489                         verbose = active = invalid = 1;
2490                         break;
2491                 case 's':
2492                         server = 1;
2493                         break;
2494                 case 't':
2495                         test = atoi(optarg);
2496                         ntest = 1;
2497                         break;
2498                 default:
2499                         lst_print_usage(argv[0]);
2500                         return -1;
2501                 }
2502         }
2503
2504         if (optind == argc) {
2505                 /* list all batches */
2506                 rc = lst_list_batch_all();
2507                 return rc;
2508         }
2509
2510         if (ntest == 1 && test <= 0) {
2511                 fprintf(stderr, "Invalid test id, test id starts from 1\n");
2512                 return -1;
2513         }
2514
2515         if (optind != argc - 1) {
2516                 lst_print_usage(argv[0]);
2517                 return -1;
2518         }
2519
2520         batch = argv[optind];
2521
2522 loop:
2523         /* show detail of specified batch or test */
2524         rc = lst_info_batch_ioctl(batch, test, server,
2525                                   &ent, NULL, NULL, NULL);
2526         if (rc != 0) {
2527                 lst_print_error((test > 0) ? "test" : "batch",
2528                                 (test > 0) ? "Failed to query test: %s\n" :
2529                                              "Failed to query batch: %s\n",
2530                                 strerror(errno));
2531                 return -1;
2532         }
2533
2534         if (verbose) {
2535                 /* list nodes in test or batch */
2536                 rc = lst_list_tsb_nodes(batch, test, server,
2537                                         server ? ent.tbe_srv_nle.nle_nnode :
2538                                                  ent.tbe_cli_nle.nle_nnode,
2539                                         active, invalid);
2540                 return rc;
2541         }
2542
2543         /* only show number of hosts in batch or test */
2544         if (test == 0) {
2545                 fprintf(stdout, "Batch: %s Tests: %d State: %d\n",
2546                         batch, ent.u.tbe_batch.bae_ntest,
2547                         ent.u.tbe_batch.bae_state);
2548                 ntest = ent.u.tbe_batch.bae_ntest;
2549                 test = 1; /* starting from test 1 */
2550
2551         } else {
2552                 fprintf(stdout,
2553                         "\tTest %d(%s) (loop: %d, concurrency: %d)\n",
2554                         test, lst_test_type2name(ent.u.tbe_test.tse_type),
2555                         ent.u.tbe_test.tse_loop,
2556                         ent.u.tbe_test.tse_concur);
2557                 ntest --;
2558                 test ++;
2559         }
2560
2561         fprintf(stdout, LST_NODES_TITLE);
2562         fprintf(stdout, "client\t%d\t%d\t%d\t%d\t%d\n"
2563                         "server\t%d\t%d\t%d\t%d\t%d\n",
2564                 ent.tbe_cli_nle.nle_nactive,
2565                 ent.tbe_cli_nle.nle_nbusy,
2566                 ent.tbe_cli_nle.nle_ndown,
2567                 ent.tbe_cli_nle.nle_nunknown,
2568                 ent.tbe_cli_nle.nle_nnode,
2569                 ent.tbe_srv_nle.nle_nactive,
2570                 ent.tbe_srv_nle.nle_nbusy,
2571                 ent.tbe_srv_nle.nle_ndown,
2572                 ent.tbe_srv_nle.nle_nunknown,
2573                 ent.tbe_srv_nle.nle_nnode);
2574
2575         if (ntest != 0)
2576                 goto loop;
2577
2578         return 0;
2579 }
2580
2581 int
2582 lst_query_batch_ioctl(char *batch, int test, int server,
2583                       int timeout, struct list_head *head)
2584 {
2585         lstio_batch_query_args_t args = {
2586                 .lstio_bat_key     = session_key,
2587                 .lstio_bat_testidx = test,
2588                 .lstio_bat_client  = !(server),
2589                 .lstio_bat_timeout = timeout,
2590                 .lstio_bat_nmlen   = strlen(batch),
2591                 .lstio_bat_namep   = batch,
2592                 .lstio_bat_resultp = head,
2593         };
2594
2595         return lst_ioctl(LSTIO_BATCH_QUERY, &args, sizeof(args));
2596 }
2597
2598 void
2599 lst_print_tsb_verbose(struct list_head *head,
2600                       int active, int idle, int error)
2601 {
2602         lstcon_rpc_ent_t *ent;
2603
2604         list_for_each_entry(ent, head, rpe_link) {
2605                 if (ent->rpe_priv[0] == 0 && active)
2606                         continue;
2607
2608                 if (ent->rpe_priv[0] != 0 && idle)
2609                         continue;
2610
2611                 if (ent->rpe_fwk_errno == 0 && error)
2612                         continue;
2613
2614                 fprintf(stdout, "%s [%s]: %s\n",
2615                         libcfs_id2str(ent->rpe_peer),
2616                         lst_node_state2str(ent->rpe_state),
2617                         ent->rpe_rpc_errno != 0 ?
2618                                 strerror(ent->rpe_rpc_errno) :
2619                                 (ent->rpe_priv[0] > 0 ? "Running" : "Idle"));
2620         }
2621 }
2622
2623 int
2624 jt_lst_query_batch(int argc, char **argv)
2625 {
2626         lstcon_test_batch_ent_t ent;
2627         struct list_head     head;
2628         char                *batch   = NULL;
2629         time_t               last    = 0;
2630         int                  optidx  = 0;
2631         int                  verbose = 0;
2632         int                  server  = 0;
2633         int                  timeout = 5; /* default 5 seconds */
2634         int                  delay   = 5; /* default 5 seconds */
2635         int                  loop    = 1; /* default 1 loop */
2636         int                  active  = 0;
2637         int                  error   = 0;
2638         int                  idle    = 0;
2639         int                  count   = 0;
2640         int                  test    = 0;
2641         int                  rc      = 0;
2642         int                  c       = 0;
2643         int                  i;
2644
2645         static struct option query_batch_opts[] =
2646         {
2647                 {"timeout", required_argument, 0, 'o' },
2648                 {"delay",   required_argument, 0, 'd' },
2649                 {"loop",    required_argument, 0, 'c' },
2650                 {"test",    required_argument, 0, 't' },
2651                 {"server",  no_argument,       0, 's' },
2652                 {"active",  no_argument,       0, 'a' },
2653                 {"idle",    no_argument,       0, 'i' },
2654                 {"error",   no_argument,       0, 'e' },
2655                 {"all",     no_argument,       0, 'l' },
2656                 {0,         0,                 0,  0  }
2657         };
2658
2659         if (session_key == 0) {
2660                 fprintf(stderr,
2661                         "Can't find env LST_SESSION or value is not valid\n");
2662                 return -1;
2663         }
2664
2665         while (1) {
2666                 c = getopt_long(argc, argv, "o:d:c:t:saiel",
2667                                 query_batch_opts, &optidx);
2668
2669                 /* Detect the end of the options. */
2670                 if (c == -1)
2671                         break;
2672
2673                 switch (c) {
2674                 case 'o':
2675                         timeout = atoi(optarg);
2676                         break;
2677                 case 'd':
2678                         delay = atoi(optarg);
2679                         break;
2680                 case 'c':
2681                         loop = atoi(optarg);
2682                         break;
2683                 case 't':
2684                         test = atoi(optarg);
2685                         break;
2686                 case 's':
2687                         server = 1;
2688                         break;
2689                 case 'a':
2690                         active = verbose = 1;
2691                         break;
2692                 case 'i':
2693                         idle = verbose = 1;
2694                         break;
2695                 case 'e':
2696                         error = verbose = 1;
2697                         break;
2698                 case 'l':
2699                         verbose = 1;
2700                         break;
2701                 default:
2702                         lst_print_usage(argv[0]);
2703                         return -1;
2704                 }
2705         }
2706
2707         if (test < 0 || timeout <= 0 || delay <= 0 || loop <= 0) {
2708                 lst_print_usage(argv[0]);
2709                 return -1;
2710         }
2711
2712         if (optind == argc) {
2713                 batch = LST_DEFAULT_BATCH;
2714
2715         } else if (optind == argc - 1) {
2716                 batch = argv[optind];
2717
2718         } else {
2719                 lst_print_usage(argv[0]);
2720                 return -1;
2721         }
2722
2723
2724         CFS_INIT_LIST_HEAD(&head);
2725
2726         if (verbose) {
2727                 rc = lst_info_batch_ioctl(batch, test, server,
2728                                           &ent, NULL, NULL, NULL);
2729                 if (rc != 0) {
2730                         fprintf(stderr, "Failed to query %s [%d]: %s\n",
2731                                 batch, test, strerror(errno));
2732                         return -1;
2733                 }
2734
2735                 count = server ? ent.tbe_srv_nle.nle_nnode :
2736                                  ent.tbe_cli_nle.nle_nnode;
2737                 if (count == 0) {
2738                         fprintf(stdout, "Batch or test is empty\n");
2739                         return 0;
2740                 }
2741         }
2742
2743         rc = lst_alloc_rpcent(&head, count, 0);
2744         if (rc != 0) {
2745                 fprintf(stderr, "Out of memory\n");
2746                 return rc;
2747         }
2748
2749         for (i = 0; i < loop; i++) {
2750                 time_t  now = time(NULL);
2751
2752                 if (now - last < delay) {
2753                         sleep(delay - now + last);
2754                         time(&now);
2755                 }
2756
2757                 last = now;
2758
2759                 rc = lst_query_batch_ioctl(batch, test,
2760                                            server, timeout, &head);
2761                 if (rc == -1) {
2762                         fprintf(stderr, "Failed to query batch: %s\n",
2763                                 strerror(errno));
2764                         break;
2765                 }
2766
2767                 if (verbose) {
2768                         /* Verbose mode */
2769                         lst_print_tsb_verbose(&head, active, idle, error);
2770                         continue;
2771                 }
2772
2773                 fprintf(stdout, "%s [%d] ", batch, test);
2774
2775                 if (lstcon_rpc_stat_failure(&trans_stat, 0) != 0) {
2776                         fprintf(stdout, "%d of %d nodes are unknown, ",
2777                                 lstcon_rpc_stat_failure(&trans_stat, 0),
2778                                 lstcon_rpc_stat_total(&trans_stat, 0));
2779                 }
2780
2781                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
2782                     lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0  &&
2783                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
2784                         fprintf(stdout, "is stopped\n");
2785                         continue;
2786                 }
2787
2788                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
2789                     lstcon_tsbqry_stat_idle(&trans_stat, 0) == 0 &&
2790                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
2791                         fprintf(stdout, "is running\n");
2792                         continue;
2793                 }
2794
2795                 fprintf(stdout, "stopped: %d , running: %d, failed: %d\n",
2796                                 lstcon_tsbqry_stat_idle(&trans_stat, 0),
2797                                 lstcon_tsbqry_stat_run(&trans_stat, 0),
2798                                 lstcon_tsbqry_stat_failure(&trans_stat, 0));
2799         }
2800
2801         lst_free_rpcent(&head);
2802
2803         return rc;
2804 }
2805
2806 int
2807 lst_parse_distribute(char *dstr, int *dist, int *span)
2808 {
2809         *dist = atoi(dstr);
2810         if (*dist <= 0)
2811                 return -1;
2812
2813         dstr = strchr(dstr, ':');
2814         if (dstr == NULL)
2815                 return -1;
2816
2817         *span = atoi(dstr + 1);
2818         if (*span <= 0)
2819                 return -1;
2820
2821         return 0;
2822 }
2823
2824 int
2825 lst_get_bulk_param(int argc, char **argv, lst_test_bulk_param_t *bulk)
2826 {
2827         char   *tok = NULL;
2828         char   *end = NULL;
2829         int     rc  = 0;
2830         int     i   = 0;
2831
2832         bulk->blk_size  = 4096;
2833         bulk->blk_opc   = LST_BRW_READ;
2834         bulk->blk_flags = LST_BRW_CHECK_NONE;
2835
2836         while (i < argc) {
2837                 if (strcasestr(argv[i], "check=") == argv[i] ||
2838                     strcasestr(argv[i], "c=") == argv[i]) {
2839                         tok = strchr(argv[i], '=') + 1;
2840
2841                         if (strcasecmp(tok, "full") == 0) {
2842                                 bulk->blk_flags = LST_BRW_CHECK_FULL;
2843                         } else if (strcasecmp(tok, "simple") == 0) {
2844                                 bulk->blk_flags = LST_BRW_CHECK_SIMPLE;
2845                         } else {
2846                                 fprintf(stderr, "Unknow flag %s\n", tok);
2847                                 return -1;
2848                         }
2849
2850                 } else if (strcasestr(argv[i], "size=") == argv[i] ||
2851                          strcasestr(argv[i], "s=") == argv[i]) {
2852                         tok = strchr(argv[i], '=') + 1;
2853
2854                         bulk->blk_size = strtol(tok, &end, 0);
2855                         if (bulk->blk_size <= 0) {
2856                                 fprintf(stderr, "Invalid size %s\n", tok);
2857                                 return -1;
2858                         }
2859
2860                         if (end == NULL)
2861                                 return 0;
2862
2863                         if (*end == 'k' || *end == 'K')
2864                                 bulk->blk_size *= 1024;
2865                         else if (*end == 'm' || *end == 'M')
2866                                 bulk->blk_size *= 1024 * 1024;
2867
2868                         if (bulk->blk_size > CFS_PAGE_SIZE * LNET_MAX_IOV) {
2869                                 fprintf(stderr, "Size exceed limitation: %d bytes\n",
2870                                         bulk->blk_size);
2871                                 return -1;
2872                         }
2873
2874                 } else if (strcasecmp(argv[i], "read") == 0 ||
2875                            strcasecmp(argv[i], "r") == 0) {
2876                         bulk->blk_opc = LST_BRW_READ;
2877
2878                 } else if (strcasecmp(argv[i], "write") == 0 ||
2879                            strcasecmp(argv[i], "w") == 0) {
2880                         bulk->blk_opc = LST_BRW_WRITE;
2881
2882                 } else {
2883                         fprintf(stderr, "Unknow parameter: %s\n", argv[i]);
2884                         return -1;
2885                 }
2886
2887                 i++;
2888         }
2889
2890         return rc;
2891 }
2892
2893 int
2894 lst_get_test_param(char *test, int argc, char **argv, void **param, int *plen)
2895 {
2896         lst_test_bulk_param_t *bulk = NULL;
2897         int                    type;
2898
2899         type = lst_test_name2type(test);
2900         if (type < 0) {
2901                 fprintf(stderr, "Unknow test name %s\n", test);
2902                 return -1;
2903         }
2904
2905         switch (type) {
2906         case LST_TEST_PING:
2907                 break;
2908
2909         case LST_TEST_BULK:
2910                 bulk = malloc(sizeof(*bulk));
2911                 if (bulk == NULL) {
2912                         fprintf(stderr, "Out of memory\n");
2913                         return -1;
2914                 }
2915
2916                 memset(bulk, 0, sizeof(*bulk));
2917
2918                 if (lst_get_bulk_param(argc, argv, bulk) != 0) {
2919                         free(bulk);
2920                         return -1;
2921                 }
2922
2923                 *param = bulk;
2924                 *plen  = sizeof(*bulk);
2925
2926                 break;
2927
2928         default:
2929                 break;
2930         }
2931
2932         /* TODO: parse more parameter */
2933         return type;
2934 }
2935
2936 int
2937 lst_add_test_ioctl(char *batch, int type, int loop, int concur,
2938                    int dist, int span, char *sgrp, char *dgrp,
2939                    void *param, int plen, int *retp, struct list_head *resultp)
2940 {
2941         lstio_test_args_t args = {
2942                 .lstio_tes_key          = session_key,
2943                 .lstio_tes_bat_nmlen    = strlen(batch),
2944                 .lstio_tes_bat_name     = batch,
2945                 .lstio_tes_type         = type,
2946                 .lstio_tes_loop         = loop,
2947                 .lstio_tes_concur       = concur,
2948                 .lstio_tes_dist         = dist,
2949                 .lstio_tes_span         = span,
2950                 .lstio_tes_sgrp_nmlen   = strlen(sgrp),
2951                 .lstio_tes_sgrp_name    = sgrp,
2952                 .lstio_tes_dgrp_nmlen   = strlen(dgrp),
2953                 .lstio_tes_dgrp_name    = dgrp,
2954                 .lstio_tes_param_len    = plen,
2955                 .lstio_tes_param        = param,
2956                 .lstio_tes_retp         = retp,
2957                 .lstio_tes_resultp      = resultp,
2958         };
2959
2960         return lst_ioctl(LSTIO_TEST_ADD, &args, sizeof(args));
2961 }
2962
2963 int
2964 jt_lst_add_test(int argc, char **argv)
2965 {
2966         struct list_head  head;
2967         char             *batch  = NULL;
2968         char             *test   = NULL;
2969         char             *dstr   = NULL;
2970         char             *from   = NULL;
2971         char             *to     = NULL;
2972         void             *param  = NULL;
2973         int               optidx = 0;
2974         int               concur = 1;
2975         int               loop   = -1;
2976         int               dist   = 1;
2977         int               span   = 1;
2978         int               plen   = 0;
2979         int               fcount = 0;
2980         int               tcount = 0;
2981         int               ret    = 0;
2982         int               type;
2983         int               rc;
2984         int               c;
2985
2986         static struct option add_test_opts[] =
2987         {
2988                 {"batch",       required_argument, 0, 'b' },
2989                 {"concurrency", required_argument, 0, 'c' },
2990                 {"distribute",  required_argument, 0, 'd' },
2991                 {"from",        required_argument, 0, 'f' },
2992                 {"to",          required_argument, 0, 't' },
2993                 {"loop",        required_argument, 0, 'l' },
2994                 {0,             0,                 0,  0  }
2995         };
2996
2997         if (session_key == 0) {
2998                 fprintf(stderr,
2999                         "Can't find env LST_SESSION or value is not valid\n");
3000                 return -1;
3001         }
3002
3003         while (1) {
3004                 c = getopt_long(argc, argv, "b:c:d:f:l:t:",
3005                                 add_test_opts, &optidx);
3006
3007                 /* Detect the end of the options. */
3008                 if (c == -1)
3009                         break;
3010
3011                 switch (c) {
3012                 case 'b':
3013                         batch = optarg;
3014                         break;
3015                 case 'c':
3016                         concur = atoi(optarg);
3017                         break;
3018                 case 'd':
3019                         dstr = optarg;
3020                         break;
3021                 case 'f':
3022                         from = optarg;
3023                         break;
3024                 case 'l':
3025                         loop = atoi(optarg);
3026                         break;
3027                 case 't':
3028                         to = optarg;
3029                         break;
3030                 default:
3031                         lst_print_usage(argv[0]);
3032                         return -1;
3033                 }
3034         }
3035
3036         if (optind == argc || from == NULL || to == NULL) {
3037                 lst_print_usage(argv[0]);
3038                 return -1;
3039         }
3040
3041         if (concur <= 0 || concur > LST_MAX_CONCUR) {
3042                 fprintf(stderr, "Invalid concurrency of test: %d\n", concur);
3043                 return -1;
3044         }
3045
3046         if (batch == NULL)
3047                 batch = LST_DEFAULT_BATCH;
3048
3049         if (dstr != NULL) {
3050                 rc = lst_parse_distribute(dstr, &dist, &span);
3051                 if (rc != 0) {
3052                         fprintf(stderr, "Invalid distribution: %s\n", dstr);
3053                         return -1;
3054                 }
3055         }
3056
3057         test = argv[optind++];
3058
3059         argc -= optind;
3060         argv += optind;
3061
3062         type = lst_get_test_param(test, argc, argv, &param, &plen);
3063         if (type < 0) {
3064                 fprintf(stderr, "Failed to add test (%s)\n", test);
3065                 return -1;
3066         }
3067
3068         CFS_INIT_LIST_HEAD(&head);
3069
3070         rc = lst_get_node_count(LST_OPC_GROUP, from, &fcount, NULL);
3071         if (rc != 0) {
3072                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3073                         from, strerror(errno));
3074                 goto out;
3075         }
3076
3077         rc = lst_get_node_count(LST_OPC_GROUP, to, &tcount, NULL);
3078         if (rc != 0) {
3079                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3080                         to, strerror(errno));
3081                 goto out;
3082         }
3083
3084         rc = lst_alloc_rpcent(&head, fcount > tcount ? fcount : tcount, 0);
3085         if (rc != 0) {
3086                 fprintf(stderr, "Out of memory\n");
3087                 goto out;
3088         }
3089
3090         rc = lst_add_test_ioctl(batch, type, loop, concur,
3091                                 dist, span, from, to, param, plen, &ret, &head);
3092
3093         if (rc == 0) {
3094                 fprintf(stdout, "Test was added successfully\n");
3095                 if (ret != 0) {
3096                         fprintf(stdout, "Server group contains userland test "
3097                                 "nodes, old version of tcplnd can't accept "
3098                                 "connection request\n");
3099                 }
3100
3101                 goto out;
3102         }
3103
3104         if (rc == -1) {
3105                 lst_print_error("test", "Failed to add test: %s\n",
3106                                 strerror(errno));
3107                 goto out;
3108         }
3109
3110         lst_print_transerr(&head, "add test");
3111 out:
3112         lst_free_rpcent(&head);
3113
3114         if (param != NULL)
3115                 free(param);
3116
3117         return rc;
3118 }
3119
3120 static command_t lst_cmdlist[] = {
3121         {"new_session",         jt_lst_new_session,     NULL,
3122          "Usage: lst new_session [--timeout TIME] [--force] [NAME]"                     },
3123         {"end_session",         jt_lst_end_session,     NULL,
3124          "Usage: lst end_session"                                                       },
3125         {"show_session",        jt_lst_show_session,    NULL,
3126          "Usage: lst show_session"                                                      },
3127         {"ping",                jt_lst_ping ,           NULL,
3128          "Usage: lst ping  [--group NAME] [--batch NAME] [--session] [--nodes IDS]"     },
3129         {"add_group",           jt_lst_add_group,       NULL,
3130          "Usage: lst group NAME IDs [IDs]..."                                           },
3131         {"del_group",           jt_lst_del_group,       NULL,
3132          "Usage: lst del_group NAME"                                                    },
3133         {"update_group",        jt_lst_update_group,    NULL,
3134          "Usage: lst update_group NAME [--clean] [--refresh] [--remove IDs]"            },
3135         {"list_group",          jt_lst_list_group,      NULL,
3136           "Usage: lst list_group [--active] [--busy] [--down] [--unknown] GROUP ..."    },
3137         {"stat",                jt_lst_stat,            NULL,
3138          "Usage: lst stat [--bw] [--rate] [--read] [--write] [--max] [--min] [--avg] "
3139          " [--timeout #] [--delay #] GROUP [GROUP]"                                     },
3140         {"show_error",          jt_lst_show_error,      NULL,
3141          "Usage: lst show_error NAME | IDS ..."                                         },
3142         {"add_batch",           jt_lst_add_batch,       NULL,
3143          "Usage: lst add_batch NAME"                                                    },
3144         {"run",                 jt_lst_start_batch,     NULL,
3145          "Usage: lst run [--timeout TIME] [NAME]"                                       },
3146         {"stop",                jt_lst_stop_batch,      NULL,
3147          "Usage: lst stop [--force] BATCH_NAME"                                         },
3148         {"list_batch",          jt_lst_list_batch,      NULL,
3149          "Usage: lst list_batch NAME [--test ID] [--server]"                            },
3150         {"query",               jt_lst_query_batch,     NULL,
3151          "Usage: lst query [--test ID] [--server] [--timeout TIME] NAME"                },
3152         {"add_test",            jt_lst_add_test,        NULL,
3153          "Usage: lst add_test [--batch BATCH] [--loop #] [--concurrency #] "
3154          " [--distribute #:#] [--from GROUP] [--to GROUP] TEST..."                      },
3155         {"help",                Parser_help,            0,     "help"                   },
3156         {0,                     0,                      0,      NULL                    }
3157 };
3158
3159 int
3160 lst_initialize(void)
3161 {
3162         char   *key;
3163
3164         key = getenv("LST_SESSION");
3165
3166         if (key == NULL) {
3167                 session_key = 0;
3168                 return 0;
3169         }
3170
3171         session_key = atoi(key);
3172
3173         return 0;
3174 }
3175
3176 int
3177 main(int argc, char **argv)
3178 {
3179         setlinebuf(stdout);
3180
3181         if (lst_initialize() < 0)
3182                 exit(0);
3183
3184         if (ptl_initialize(argc, argv) < 0)
3185                 exit(0);
3186
3187         Parser_init("lst > ", lst_cmdlist);
3188
3189         if (argc != 1)
3190                 return Parser_execarg(argc - 1, argv + 1, lst_cmdlist);
3191
3192         Parser_commands();
3193
3194         return 0;
3195 }