lnet/utils/lst.c

   1 /*
   2  * -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   3  * vim:expandtab:shiftwidth=8:tabstop=8:
   4  *
   5  * GPL HEADER START
   6  *
   7  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 only,
  11  * as published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful, but
  14  * WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * General Public License version 2 for more details (a copy is included
  17  * in the LICENSE file that accompanied this code).
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * version 2 along with this program; If not, see [sun.com URL with a
  21  * copy of GPLv2].
  22  *
  23  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  24  * CA 95054 USA or visit www.sun.com if you need additional information or
  25  * have any questions.
  26  *
  27  * GPL HEADER END
  28  */
  29 /*
  30  * Copyright  2008 Sun Microsystems, Inc. All rights reserved
  31  * Use is subject to license terms.
  32  */
  33 /*
  34  * This file is part of Lustre, http://www.lustre.org/
  35  * Lustre is a trademark of Sun Microsystems, Inc.
  36  *
  37  * lnet/selftest/conctl.c
  38  *
  39  * Author: Liang Zhen <liangzhen@clusterfs.com>
  40  */
  41
  42 #define _GNU_SOURCE
  43
  44 #include <stdio.h>
  45 #include <stdlib.h>
  46 #include <string.h>
  47 #include <getopt.h>
  48 #include <errno.h>
  49 #include <pwd.h>
  50 #include <lnet/lnetctl.h>
  51 #include <lnet/lnetst.h>
  52 #include <libcfs/libcfsutil.h>
  53
  54 static command_t           lst_cmdlist[];
  55 static lst_sid_t           session_id;
  56 static int                 session_key;
  57 static lstcon_trans_stat_t trans_stat;
  58
  59 typedef struct list_string {
  60         struct list_string *lstr_next;
  61         int                 lstr_sz;
  62         char                lstr_str[0];
  63 } lstr_t;
  64
  65 #define offsetof(typ,memb)     ((unsigned long)((char *)&(((typ *)0)->memb)))
  66
  67 static int alloc_count = 0;
  68 static int alloc_nob   = 0;
  69
  70 lstr_t *
  71 alloc_lstr(int sz)
  72 {
  73         lstr_t  *lstr = malloc(offsetof(lstr_t, lstr_str[sz]));
  74
  75         if (lstr == NULL) {
  76                 fprintf(stderr, "Can't allocate lstr\n");
  77                 abort();
  78         }
  79
  80         alloc_nob += sz;
  81         alloc_count++;
  82
  83         lstr->lstr_str[0] = 0;
  84         lstr->lstr_sz = sz;
  85         return lstr;
  86 }
  87
  88 void
  89 free_lstr(lstr_t *lstr)
  90 {
  91         alloc_count--;
  92         alloc_nob -= lstr->lstr_sz;
  93         free(lstr);
  94 }
  95
  96 void
  97 free_lstrs(lstr_t **list)
  98 {
  99         lstr_t   *lstr;
 100
 101         while ((lstr = *list) != NULL) {
 102                 *list = lstr->lstr_next;
 103                 free_lstr(lstr);
 104         }
 105 }
 106
 107 void
 108 new_lstrs(lstr_t **list, char *prefix, char *postfix,
 109           int lo, int hi, int stride)
 110 {
 111         int    n1 = strlen(prefix);
 112         int    n2 = strlen(postfix);
 113         int    sz = n1 + 20 + n2 + 1;
 114
 115         do {
 116                 lstr_t *n = alloc_lstr(sz);
 117
 118                 snprintf(n->lstr_str, sz - 1, "%s%u%s",
 119                          prefix, lo, postfix);
 120
 121                 n->lstr_next = *list;
 122                 *list = n;
 123
 124                 lo += stride;
 125         } while (lo <= hi);
 126 }
 127
 128 int
 129 expand_lstr(lstr_t **list, lstr_t *l)
 130 {
 131         int          nob = strlen(l->lstr_str);
 132         char        *b1;
 133         char        *b2;
 134         char        *expr;
 135         char        *sep;
 136         int          x;
 137         int          y;
 138         int          z;
 139         int          n;
 140
 141         b1 = strchr(l->lstr_str, '[');
 142         if (b1 == NULL) {
 143                 l->lstr_next = *list;
 144                 *list = l;
 145                 return 0;
 146         }
 147
 148         b2 = strchr(b1, ']');
 149         if (b2 == NULL || b2 == b1 + 1)
 150                 return -1;
 151
 152         *b1++ = 0;
 153         *b2++ = 0;
 154         expr = b1;
 155         do {
 156
 157                 sep = strchr(expr, ',');
 158                 if (sep != NULL)
 159                         *sep++ = 0;
 160
 161                 nob = strlen(expr);
 162                 n = nob;
 163                 if (sscanf(expr, "%u%n", &x, &n) >= 1 && n == nob) {
 164                         /* simple number */
 165                         new_lstrs(list, l->lstr_str, b2, x, x, 1);
 166                         continue;
 167                 }
 168
 169                 n = nob;
 170                 if (sscanf(expr, "%u-%u%n", &x, &y, &n) >= 2 && n == nob &&
 171                     x < y) {
 172                         /* simple range */
 173                         new_lstrs(list, l->lstr_str, b2, x, y, 1);
 174                         continue;
 175                 }
 176
 177                 n = nob;
 178                 if (sscanf(expr, "%u-%u/%u%n", &x, &y, &z, &n) >= 3 && n == nob &&
 179                     x < y) {
 180                         /* strided range */
 181                         new_lstrs(list, l->lstr_str, b2, x, y, z);
 182                         continue;
 183                 }
 184
 185                 /* syntax error */
 186                 return -1;
 187         } while ((expr = sep) != NULL);
 188
 189         free_lstr(l);
 190
 191         return 1;
 192 }
 193
 194 int
 195 expand_strs(char *str, lstr_t **head)
 196 {
 197         lstr_t  *list = NULL;
 198         lstr_t  *nlist;
 199         lstr_t  *l;
 200         int      rc = 0;
 201         int      expanded;
 202
 203         l = alloc_lstr(strlen(str) + 1);
 204         memcpy(l->lstr_str, str, strlen(str) + 1);
 205         l->lstr_next = NULL;
 206         list = l;
 207
 208         do {
 209                 expanded = 0;
 210                 nlist = NULL;
 211
 212                 while ((l = list) != NULL) {
 213                         list = l->lstr_next;
 214
 215                         rc = expand_lstr(&nlist, l);
 216                         if (rc < 0) {
 217                                 fprintf(stderr, "Syntax error in \"%s\"\n", str);
 218                                 free_lstr(l);
 219                                 break;
 220                         }
 221
 222                         expanded |= rc > 0;
 223                 }
 224
 225                 /* re-order onto 'list' */
 226                 while ((l = nlist) != NULL) {
 227                         nlist = l->lstr_next;
 228                         l->lstr_next = list;
 229                         list = l;
 230                 }
 231
 232         } while (expanded && rc > 0);
 233
 234         if (rc >= 0) {
 235                 *head = list;
 236                 return 0;
 237         }
 238
 239         while ((l = list) != NULL) {
 240                 list = l->lstr_next;
 241
 242                 free_lstr(l);
 243         }
 244         return rc;
 245 }
 246
 247 int
 248 lst_parse_nids(char *str, int *countp, lnet_process_id_t **idspp)
 249 {
 250         lstr_t  *head = NULL;
 251         lstr_t  *l;
 252         int      c = 0;
 253         int      i;
 254         int      rc;
 255
 256         rc = expand_strs(str, &head);
 257         if (rc != 0)
 258                 goto out;
 259
 260         l = head;
 261         while (l != NULL) {
 262                 l = l->lstr_next;
 263                 c++;
 264         }
 265
 266         *idspp = malloc(c * sizeof(lnet_process_id_t));
 267         if (*idspp == NULL) {
 268                 fprintf(stderr, "Out of memory\n");
 269                 rc = -1;
 270         }
 271
 272         *countp = c;
 273 out:
 274         i = 0;
 275         while ((l = head) != NULL) {
 276                 head = l->lstr_next;
 277
 278                 if (rc == 0) {
 279                         (*idspp)[i].nid = libcfs_str2nid(l->lstr_str);
 280                         if ((*idspp)[i].nid == LNET_NID_ANY) {
 281                                 fprintf(stderr, "Invalid nid: %s\n",
 282                                         l->lstr_str);
 283                                 rc = -1;
 284                         }
 285
 286                         (*idspp)[i].pid = LUSTRE_LNET_PID;
 287                         i++;
 288                 }
 289
 290                 free_lstr(l);
 291         }
 292
 293         if (rc == 0)
 294                 return 0;
 295
 296         free(*idspp);
 297         *idspp = NULL;
 298
 299         return rc;
 300 }
 301
 302 char *
 303 lst_node_state2str(int state)
 304 {
 305         if (state == LST_NODE_ACTIVE)
 306                 return "Active";
 307         if (state == LST_NODE_BUSY)
 308                 return "Busy";
 309         if (state == LST_NODE_DOWN)
 310                 return "Down";
 311
 312         return "Unknown";
 313 }
 314
 315 int
 316 lst_node_str2state(char *str)
 317 {
 318         if (strcasecmp(str, "active") == 0)
 319                 return LST_NODE_ACTIVE;
 320         if (strcasecmp(str, "busy") == 0)
 321                 return LST_NODE_BUSY;
 322         if (strcasecmp(str, "down") == 0)
 323                 return LST_NODE_DOWN;
 324         if (strcasecmp(str, "unknown") == 0)
 325                 return LST_NODE_UNKNOWN;
 326         if (strcasecmp(str, "invalid") == 0)
 327                 return (LST_NODE_UNKNOWN | LST_NODE_DOWN | LST_NODE_BUSY);
 328
 329         return -1;
 330 }
 331
 332 char *
 333 lst_test_type2name(int type)
 334 {
 335         if (type == LST_TEST_PING)
 336                 return "ping";
 337         if (type == LST_TEST_BULK)
 338                 return "brw";
 339
 340         return "unknown";
 341 }
 342
 343 int
 344 lst_test_name2type(char *name)
 345 {
 346         if (strcasecmp(name, "ping") == 0)
 347                 return LST_TEST_PING;
 348         if (strcasecmp(name, "brw") == 0)
 349                 return LST_TEST_BULK;
 350
 351         return -1;
 352 }
 353
 354 void
 355 lst_print_usage(char *cmd)
 356 {
 357         Parser_printhelp(cmd);
 358 }
 359
 360 void
 361 lst_print_error(char *sub, const char *def_format, ...)
 362 {
 363         va_list ap;
 364
 365         /* local error returned from kernel */
 366         switch (errno) {
 367         case ESRCH:
 368                 fprintf(stderr, "No session exists\n");
 369                 return;
 370         case ESHUTDOWN:
 371                 fprintf(stderr, "Session is shutting down\n");
 372                 return;
 373         case EACCES:
 374                 fprintf(stderr, "Unmatched session key or not root\n");
 375                 return;
 376         case ENOENT:
 377                 fprintf(stderr, "Can't find %s in current session\n", sub);
 378                 return;
 379         case EINVAL:
 380                 fprintf(stderr, "Invalid parameters list in command line\n");
 381                 return;
 382         case EFAULT:
 383                 fprintf(stderr, "Bad parameter address\n");
 384                 return;
 385         case EEXIST:
 386                 fprintf(stderr, "%s already exists\n", sub);
 387                 return;
 388         default:
 389                 va_start(ap, def_format);
 390                 vfprintf(stderr, def_format, ap);
 391                 va_end(ap);
 392
 393                 return;
 394         }
 395 }
 396
 397 void
 398 lst_free_rpcent(struct list_head *head)
 399 {
 400         lstcon_rpc_ent_t *ent;
 401
 402         while (!list_empty(head)) {
 403                 ent = list_entry(head->next, lstcon_rpc_ent_t, rpe_link);
 404
 405                 list_del(&ent->rpe_link);
 406                 free(ent);
 407         }
 408 }
 409
 410 void
 411 lst_reset_rpcent(struct list_head *head)
 412 {
 413         lstcon_rpc_ent_t *ent;
 414
 415         list_for_each_entry(ent, head, rpe_link) {
 416                 ent->rpe_sid      = LST_INVALID_SID;
 417                 ent->rpe_peer.nid = LNET_NID_ANY;
 418                 ent->rpe_peer.pid = LNET_PID_ANY;
 419                 ent->rpe_rpc_errno = ent->rpe_fwk_errno = 0;
 420         }
 421 }
 422
 423 int
 424 lst_alloc_rpcent(struct list_head *head, int count, int offset)
 425 {
 426         lstcon_rpc_ent_t *ent;
 427         int               i;
 428
 429         for (i = 0; i < count; i++) {
 430                 ent = malloc(offsetof(lstcon_rpc_ent_t, rpe_payload[offset]));
 431                 if (ent == NULL) {
 432                         lst_free_rpcent(head);
 433                         return -1;
 434                 }
 435
 436                 memset(ent, 0, offsetof(lstcon_rpc_ent_t, rpe_payload[offset]));
 437
 438                 ent->rpe_sid      = LST_INVALID_SID;
 439                 ent->rpe_peer.nid = LNET_NID_ANY;
 440                 ent->rpe_peer.pid = LNET_PID_ANY;
 441                 list_add(&ent->rpe_link, head);
 442         }
 443
 444         return 0;
 445 }
 446
 447 void
 448 lst_print_transerr(struct list_head *head, char *optstr)
 449 {
 450         lstcon_rpc_ent_t  *ent;
 451
 452         list_for_each_entry(ent, head, rpe_link) {
 453                 if (ent->rpe_rpc_errno == 0 && ent->rpe_fwk_errno == 0)
 454                         continue;
 455
 456                 if (ent->rpe_rpc_errno != 0) {
 457                         fprintf(stderr, "%s RPC failed on %s: %s\n",
 458                                 optstr, libcfs_id2str(ent->rpe_peer),
 459                                 strerror(ent->rpe_rpc_errno));
 460                         continue;
 461                 }
 462
 463                 fprintf(stderr, "%s failed on %s: %s\n",
 464                         optstr, libcfs_id2str(ent->rpe_peer),
 465                         strerror(ent->rpe_fwk_errno));
 466         }
 467 }
 468
 469 int lst_info_batch_ioctl(char *batch, int test, int server,
 470                         lstcon_test_batch_ent_t *entp, int *idxp,
 471                         int *ndentp, lstcon_node_ent_t *dentsp);
 472
 473 int lst_info_group_ioctl(char *name, lstcon_ndlist_ent_t *gent,
 474                          int *idx, int *count, lstcon_node_ent_t *dents);
 475
 476 int lst_query_batch_ioctl(char *batch, int test, int server,
 477                           int timeout, struct list_head *head);
 478
 479 int
 480 lst_ioctl(unsigned int opc, void *buf, int len)
 481 {
 482         struct libcfs_ioctl_data data;
 483         int    rc;
 484
 485         LIBCFS_IOC_INIT (data);
 486         data.ioc_u32[0]  = opc;
 487         data.ioc_plen1   = len;
 488         data.ioc_pbuf1   = (char *)buf;
 489         data.ioc_plen2   = sizeof(trans_stat);
 490         data.ioc_pbuf2   = (char *)&trans_stat;
 491
 492         memset(&trans_stat, 0, sizeof(trans_stat));
 493
 494         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNETST, &data);
 495
 496         /* local error, no valid RPC result */
 497         if (rc != 0)
 498                 return -1;
 499
 500         /* RPC error */
 501         if (trans_stat.trs_rpc_errno != 0)
 502                 return -2;
 503
 504         /* Framework error */
 505         if (trans_stat.trs_fwk_errno != 0)
 506                 return -3;
 507
 508         return 0;
 509 }
 510
 511 int
 512 lst_new_session_ioctl (char *name, int timeout, int force, lst_sid_t *sid)
 513 {
 514         lstio_session_new_args_t        args = {
 515                 .lstio_ses_key          = session_key,
 516                 .lstio_ses_timeout      = timeout,
 517                 .lstio_ses_force        = force,
 518                 .lstio_ses_idp          = sid,
 519                 .lstio_ses_namep        = name,
 520                 .lstio_ses_nmlen        = strlen(name),
 521         };
 522
 523         return lst_ioctl (LSTIO_SESSION_NEW, &args, sizeof(args));
 524 }
 525
 526 int
 527 jt_lst_new_session(int argc,  char **argv)
 528 {
 529         char  buf[LST_NAME_SIZE];
 530         char *name;
 531         int   optidx  = 0;
 532         int   timeout = 300;
 533         int   force   = 0;
 534         int   c;
 535         int   rc;
 536
 537         static struct option session_opts[] =
 538         {
 539                 {"timeout", required_argument,  0, 't' },
 540                 {"force",   no_argument,        0, 'f' },
 541                 {0,         0,                  0,  0  }
 542         };
 543
 544         if (session_key == 0) {
 545                 fprintf(stderr,
 546                         "Can't find env LST_SESSION or value is not valid\n");
 547                 return -1;
 548         }
 549
 550         while (1) {
 551
 552                 c = getopt_long(argc, argv, "ft:",
 553                                 session_opts, &optidx);
 554
 555                 if (c == -1)
 556                         break;
 557
 558                 switch (c) {
 559                 case 'f':
 560                         force = 1;
 561                         break;
 562                 case 't':
 563                         timeout = atoi(optarg);
 564                         break;
 565                 default:
 566                         lst_print_usage(argv[0]);
 567                         return -1;
 568                 }
 569         }
 570
 571         if (timeout <= 0) {
 572                 fprintf(stderr, "Invalid timeout value\n");
 573                 return -1;
 574         }
 575
 576         if (optind == argc - 1) {
 577                 name = argv[optind ++];
 578                 if (strlen(name) >= LST_NAME_SIZE) {
 579                         fprintf(stderr, "Name size is limited to %d\n",
 580                                 LST_NAME_SIZE - 1);
 581                         return -1;
 582                 }
 583
 584         } else if (optind == argc) {
 585                 char           user[LST_NAME_SIZE];
 586                 char           host[LST_NAME_SIZE];
 587                 struct passwd *pw = getpwuid(getuid());
 588
 589                 if (pw == NULL)
 590                         snprintf(user, sizeof(user), "%d", (int)getuid());
 591                 else
 592                         snprintf(user, sizeof(user), "%s", pw->pw_name);
 593
 594                 rc = gethostname(host, sizeof(host));
 595                 if (rc != 0)
 596                         snprintf(host, sizeof(host), "unknown_host");
 597
 598                 snprintf(buf, LST_NAME_SIZE, "%s@%s", user, host);
 599                 name = buf;
 600
 601         } else {
 602                 lst_print_usage(argv[0]);
 603                 return -1;
 604         }
 605
 606         rc = lst_new_session_ioctl(name, timeout, force, &session_id);
 607
 608         if (rc != 0) {
 609                 lst_print_error("session", "Failed to create session: %s\n",
 610                                 strerror(errno));
 611                 return rc;
 612         }
 613
 614         fprintf(stdout, "SESSION: %s TIMEOUT: %d FORCE: %s\n",
 615                 name, timeout, force ? "Yes": "No");
 616
 617         return rc;
 618 }
 619
 620 int
 621 lst_session_info_ioctl(char *name, int len, int *key,
 622                        lst_sid_t *sid, lstcon_ndlist_ent_t *ndinfo)
 623 {
 624         lstio_session_info_args_t args = {
 625                 .lstio_ses_keyp         = key,
 626                 .lstio_ses_idp          = sid,
 627                 .lstio_ses_ndinfo       = ndinfo,
 628                 .lstio_ses_nmlen        = len,
 629                 .lstio_ses_namep        = name,
 630         };
 631
 632         return lst_ioctl(LSTIO_SESSION_INFO, &args, sizeof(args));
 633 }
 634
 635 int
 636 jt_lst_show_session(int argc, char **argv)
 637 {
 638         lstcon_ndlist_ent_t ndinfo;
 639         lst_sid_t           sid;
 640         char                name[LST_NAME_SIZE];
 641         int                 key;
 642         int                 rc;
 643
 644         rc = lst_session_info_ioctl(name, LST_NAME_SIZE, &key, &sid, &ndinfo);
 645
 646         if (rc != 0) {
 647                 lst_print_error("session", "Failed to show session: %s\n",
 648                                 strerror(errno));
 649                 return -1;
 650         }
 651
 652         fprintf(stdout, "%s ID: %Lu@%s, KEY: %d NODES: %d\n",
 653                 name, sid.ses_stamp, libcfs_nid2str(sid.ses_nid),
 654                 key, ndinfo.nle_nnode);
 655
 656         return 0;
 657 }
 658
 659 int
 660 lst_end_session_ioctl(void)
 661 {
 662         lstio_session_end_args_t args = {
 663                 .lstio_ses_key           = session_key,
 664         };
 665
 666         return lst_ioctl (LSTIO_SESSION_END, &args, sizeof(args));
 667 }
 668
 669 int
 670 jt_lst_end_session(int argc, char **argv)
 671 {
 672         int             rc;
 673
 674         if (session_key == 0) {
 675                 fprintf(stderr,
 676                         "Can't find env LST_SESSION or value is not valid\n");
 677                 return -1;
 678         }
 679
 680         rc = lst_end_session_ioctl();
 681
 682         if (rc == 0) {
 683                 fprintf(stdout, "session is ended\n");
 684                 return 0;
 685         }
 686
 687         if (rc == -1) {
 688                 lst_print_error("session", "Failed to end session: %s\n",
 689                                 strerror(errno));
 690                 return rc;
 691         }
 692
 693         if (trans_stat.trs_rpc_errno != 0) {
 694                 fprintf(stderr,
 695                         "[RPC] Failed to send %d session RPCs: %s\n",
 696                         lstcon_rpc_stat_failure(&trans_stat, 0),
 697                         strerror(trans_stat.trs_rpc_errno));
 698         }
 699
 700         if (trans_stat.trs_fwk_errno != 0) {
 701                 fprintf(stderr,
 702                         "[FWK] Failed to end session on %d nodes: %s\n",
 703                         lstcon_sesop_stat_failure(&trans_stat, 0),
 704                         strerror(trans_stat.trs_fwk_errno));
 705         }
 706
 707         return rc;
 708 }
 709
 710 int
 711 lst_ping_ioctl(char *str, int type, int timeout,
 712                int count, lnet_process_id_t *ids, struct list_head *head)
 713 {
 714         lstio_debug_args_t args = {
 715                 .lstio_dbg_key          = session_key,
 716                 .lstio_dbg_type         = type,
 717                 .lstio_dbg_flags        = 0,
 718                 .lstio_dbg_timeout      = timeout,
 719                 .lstio_dbg_nmlen        = (str == NULL) ? 0: strlen(str),
 720                 .lstio_dbg_namep        = str,
 721                 .lstio_dbg_count        = count,
 722                 .lstio_dbg_idsp         = ids,
 723                 .lstio_dbg_resultp      = head,
 724         };
 725
 726         return lst_ioctl (LSTIO_DEBUG, &args, sizeof(args));
 727 }
 728
 729 int
 730 lst_get_node_count(int type, char *str, int *countp, lnet_process_id_t **idspp)
 731 {
 732         char                    buf[LST_NAME_SIZE];
 733         lstcon_test_batch_ent_t ent;
 734         lstcon_ndlist_ent_t    *entp = &ent.tbe_cli_nle;
 735         lst_sid_t               sid;
 736         int                     key;
 737         int                     rc;
 738
 739         switch (type) {
 740         case LST_OPC_SESSION:
 741                 rc = lst_session_info_ioctl(buf, LST_NAME_SIZE,
 742                                             &key, &sid, entp);
 743                 break;
 744
 745         case LST_OPC_BATCHSRV:
 746                 entp = &ent.tbe_srv_nle;
 747         case LST_OPC_BATCHCLI:
 748                 rc = lst_info_batch_ioctl(str, 0, 0, &ent, NULL, NULL, NULL);
 749                 break;
 750
 751         case LST_OPC_GROUP:
 752                 rc = lst_info_group_ioctl(str, entp, NULL, NULL, NULL);
 753                 break;
 754
 755         case LST_OPC_NODES:
 756                 rc = lst_parse_nids(str, &entp->nle_nnode, idspp) < 0 ? -1 : 0;
 757                 break;
 758
 759         default:
 760                 rc = -1;
 761                 break;
 762         }
 763
 764         if (rc == 0)
 765                 *countp = entp->nle_nnode;
 766
 767         return rc;
 768 }
 769
 770 int
 771 jt_lst_ping(int argc,  char **argv)
 772 {
 773         struct list_head   head;
 774         lnet_process_id_t *ids = NULL;
 775         lstcon_rpc_ent_t  *ent = NULL;
 776         char              *str = NULL;
 777         int                optidx  = 0;
 778         int                server  = 0;
 779         int                timeout = 5;
 780         int                count   = 0;
 781         int                type    = 0;
 782         int                rc      = 0;
 783         int                c;
 784
 785         static struct option ping_opts[] =
 786         {
 787                 {"session", no_argument,       0, 's' },
 788                 {"server",  no_argument,       0, 'v' },
 789                 {"batch",   required_argument, 0, 'b' },
 790                 {"group",   required_argument, 0, 'g' },
 791                 {"nodes",   required_argument, 0, 'n' },
 792                 {"timeout", required_argument, 0, 't' },
 793                 {0,         0,                 0,  0  }
 794         };
 795
 796         if (session_key == 0) {
 797                 fprintf(stderr,
 798                         "Can't find env LST_SESSION or value is not valid\n");
 799                 return -1;
 800         }
 801
 802         while (1) {
 803
 804                 c = getopt_long(argc, argv, "g:b:n:t:sv",
 805                                 ping_opts, &optidx);
 806
 807                 if (c == -1)
 808                         break;
 809
 810                 switch (c) {
 811                 case 's':
 812                         type = LST_OPC_SESSION;
 813                         break;
 814
 815                 case 'g':
 816                         type = LST_OPC_GROUP;
 817                         str = optarg;
 818                         break;
 819
 820                 case 'b':
 821                         type = LST_OPC_BATCHCLI;
 822                         str = optarg;
 823                         break;
 824
 825                 case 'n':
 826                         type = LST_OPC_NODES;
 827                         str = optarg;
 828                         break;
 829
 830                 case 't':
 831                         timeout = atoi(optarg);
 832                         break;
 833
 834                 case 'v':
 835                         server = 1;
 836                         break;
 837
 838                 default:
 839                         lst_print_usage(argv[0]);
 840                         return -1;
 841                 }
 842         }
 843
 844         if (type == 0 || timeout <= 0 || optind != argc) {
 845                 lst_print_usage(argv[0]);
 846                 return -1;
 847         }
 848
 849         if (type == LST_OPC_BATCHCLI && server)
 850                 type = LST_OPC_BATCHSRV;
 851
 852         rc = lst_get_node_count(type, str, &count, &ids);
 853         if (rc < 0) {
 854                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
 855                         (str == NULL) ? "session" : str, strerror(errno));
 856                 return -1;
 857         }
 858
 859         CFS_INIT_LIST_HEAD(&head);
 860
 861         rc = lst_alloc_rpcent(&head, count, LST_NAME_SIZE);
 862         if (rc != 0) {
 863                 fprintf(stderr, "Out of memory\n");
 864                 goto out;
 865         }
 866
 867         if (count == 0) {
 868                 fprintf(stdout, "Target %s is empty\n",
 869                         (str == NULL) ? "session" : str);
 870                 goto out;
 871         }
 872
 873         rc = lst_ping_ioctl(str, type, timeout, count, ids, &head);
 874         if (rc == -1) { /* local failure */
 875                 lst_print_error("debug", "Failed to ping %s: %s\n",
 876                                 (str == NULL) ? "session" : str,
 877                                 strerror(errno));
 878                 rc = -1;
 879                 goto out;
 880         }
 881
 882         /* ignore RPC errors and framwork errors */
 883         list_for_each_entry(ent, &head, rpe_link) {
 884                 fprintf(stdout, "\t%s: %s [session: %s id: %s]\n",
 885                         libcfs_id2str(ent->rpe_peer),
 886                         lst_node_state2str(ent->rpe_state),
 887                         (ent->rpe_state == LST_NODE_ACTIVE ||
 888                          ent->rpe_state == LST_NODE_BUSY)?
 889                                  (ent->rpe_rpc_errno == 0 ?
 890                                          &ent->rpe_payload[0] : "Unknown") :
 891                                  "<NULL>", libcfs_nid2str(ent->rpe_sid.ses_nid));
 892         }
 893
 894 out:
 895         lst_free_rpcent(&head);
 896
 897         if (ids != NULL)
 898                 free(ids);
 899
 900         return rc;
 901
 902 }
 903
 904 int
 905 lst_add_nodes_ioctl (char *name, int count, lnet_process_id_t *ids,
 906                      struct list_head *resultp)
 907 {
 908         lstio_group_nodes_args_t        args = {
 909                 .lstio_grp_key          = session_key,
 910                 .lstio_grp_nmlen        = strlen(name),
 911                 .lstio_grp_namep        = name,
 912                 .lstio_grp_count        = count,
 913                 .lstio_grp_idsp         = ids,
 914                 .lstio_grp_resultp      = resultp,
 915         };
 916
 917         return lst_ioctl(LSTIO_NODES_ADD, &args, sizeof(args));
 918 }
 919
 920 int
 921 lst_add_group_ioctl (char *name)
 922 {
 923         lstio_group_add_args_t  args = {
 924                 .lstio_grp_key          = session_key,
 925                 .lstio_grp_nmlen        = strlen(name),
 926                 .lstio_grp_namep        = name,
 927         };
 928
 929         return lst_ioctl(LSTIO_GROUP_ADD, &args, sizeof(args));
 930 }
 931
 932 int
 933 jt_lst_add_group(int argc, char **argv)
 934 {
 935         struct list_head   head;
 936         lnet_process_id_t *ids;
 937         char              *name;
 938         int                count;
 939         int                rc;
 940         int                i;
 941
 942         if (session_key == 0) {
 943                 fprintf(stderr,
 944                         "Can't find env LST_SESSION or value is not valid\n");
 945                 return -1;
 946         }
 947
 948         if (argc < 3) {
 949                 lst_print_usage(argv[0]);
 950                 return -1;
 951         }
 952
 953         name = argv[1];
 954         if (strlen(name) >= LST_NAME_SIZE) {
 955                 fprintf(stderr, "Name length is limited to %d\n",
 956                         LST_NAME_SIZE - 1);
 957                 return -1;
 958         }
 959
 960         rc = lst_add_group_ioctl(name);
 961         if (rc != 0) {
 962                 lst_print_error("group", "Failed to add group %s: %s\n",
 963                                 name, strerror(errno));
 964                 return -1;
 965         }
 966
 967         CFS_INIT_LIST_HEAD(&head);
 968
 969         for (i = 2; i < argc; i++) {
 970                 /* parse address list */
 971                 rc = lst_parse_nids(argv[i], &count, &ids);
 972                 if (rc < 0) {
 973                         fprintf(stderr, "Ignore invalid id list %s\n",
 974                                 argv[i]);
 975                         continue;
 976                 }
 977
 978                 if (count == 0)
 979                         continue;
 980
 981                 rc = lst_alloc_rpcent(&head, count, 0);
 982                 if (rc != 0) {
 983                         fprintf(stderr, "Out of memory\n");
 984                         break;
 985                 }
 986
 987                 rc = lst_add_nodes_ioctl(name, count, ids, &head);
 988
 989                 free(ids);
 990
 991                 if (rc == 0) {
 992                         lst_free_rpcent(&head);
 993                         fprintf(stderr, "%s are added to session\n", argv[i]);
 994                         continue;
 995                 }
 996
 997                 if (rc == -1) {
 998                         lst_free_rpcent(&head);
 999                         lst_print_error("group", "Failed to add nodes %s: %s\n",
1000                                         argv[i], strerror(errno));
1001                         break;
1002                 }
1003
1004                 lst_print_transerr(&head, "create session");
1005                 lst_free_rpcent(&head);
1006         }
1007
1008         return rc;
1009 }
1010
1011 int
1012 lst_del_group_ioctl (char *name)
1013 {
1014         lstio_group_del_args_t  args = {
1015                 .lstio_grp_key          = session_key,
1016                 .lstio_grp_nmlen        = strlen(name),
1017                 .lstio_grp_namep        = name,
1018         };
1019
1020         return lst_ioctl(LSTIO_GROUP_DEL, &args, sizeof(args));
1021 }
1022
1023 int
1024 jt_lst_del_group(int argc, char **argv)
1025 {
1026         int     rc;
1027
1028         if (session_key == 0) {
1029                 fprintf(stderr,
1030                         "Can't find env LST_SESSION or value is not valid\n");
1031                 return -1;
1032         }
1033
1034         if (argc != 2) {
1035                 lst_print_usage(argv[0]);
1036                 return -1;
1037         }
1038
1039         rc = lst_del_group_ioctl(argv[1]);
1040         if (rc == 0) {
1041                 fprintf(stdout, "Group is deleted\n");
1042                 return 0;
1043         }
1044
1045         if (rc == -1) {
1046                 lst_print_error("group", "Failed to delete group: %s\n",
1047                                 strerror(errno));
1048                 return rc;
1049         }
1050
1051         fprintf(stderr, "Group is deleted with some errors\n");
1052
1053         if (trans_stat.trs_rpc_errno != 0) {
1054                 fprintf(stderr, "[RPC] Failed to send %d end session RPCs: %s\n",
1055                         lstcon_rpc_stat_failure(&trans_stat, 0),
1056                         strerror(trans_stat.trs_rpc_errno));
1057         }
1058
1059         if (trans_stat.trs_fwk_errno != 0) {
1060                 fprintf(stderr,
1061                         "[FWK] Failed to end session on %d nodes: %s\n",
1062                         lstcon_sesop_stat_failure(&trans_stat, 0),
1063                         strerror(trans_stat.trs_fwk_errno));
1064         }
1065
1066         return -1;
1067 }
1068
1069 int
1070 lst_update_group_ioctl(int opc, char *name, int clean, int count,
1071                        lnet_process_id_t *ids, struct list_head *resultp)
1072 {
1073         lstio_group_update_args_t  args = {
1074                 .lstio_grp_key          = session_key,
1075                 .lstio_grp_opc          = opc,
1076                 .lstio_grp_args         = clean,
1077                 .lstio_grp_nmlen        = strlen(name),
1078                 .lstio_grp_namep        = name,
1079                 .lstio_grp_count        = count,
1080                 .lstio_grp_idsp         = ids,
1081                 .lstio_grp_resultp      = resultp,
1082         };
1083
1084         return lst_ioctl(LSTIO_GROUP_UPDATE, &args, sizeof(args));
1085 }
1086
1087 int
1088 jt_lst_update_group(int argc, char **argv)
1089 {
1090         struct list_head   head;
1091         lnet_process_id_t *ids = NULL;
1092         char              *str = NULL;
1093         char              *grp = NULL;
1094         int                optidx = 0;
1095         int                count = 0;
1096         int                clean = 0;
1097         int                opc = 0;
1098         int                rc;
1099         int                c;
1100
1101         static struct option update_group_opts[] =
1102         {
1103                 {"refresh", no_argument,       0, 'f' },
1104                 {"clean",   required_argument, 0, 'c' },
1105                 {"remove",  required_argument, 0, 'r' },
1106                 {0,         0,                 0,  0  }
1107         };
1108
1109         if (session_key == 0) {
1110                 fprintf(stderr,
1111                         "Can't find env LST_SESSION or value is not valid\n");
1112                 return -1;
1113         }
1114
1115         while (1) {
1116                 c = getopt_long(argc, argv, "fc:r:",
1117                                 update_group_opts, &optidx);
1118
1119                 /* Detect the end of the options. */
1120                 if (c == -1)
1121                         break;
1122
1123                 switch (c) {
1124                 case 'f':
1125                         if (opc != 0) {
1126                                 lst_print_usage(argv[0]);
1127                                 return -1;
1128                         }
1129                         opc = LST_GROUP_REFRESH;
1130                         break;
1131
1132                 case 'r':
1133                         if (opc != 0) {
1134                                 lst_print_usage(argv[0]);
1135                                 return -1;
1136                         }
1137                         opc = LST_GROUP_RMND;
1138                         str = optarg;
1139                         break;
1140
1141                 case 'c':
1142                         clean = lst_node_str2state(optarg);
1143                         if (opc != 0 || clean <= 0) {
1144                                 lst_print_usage(argv[0]);
1145                                 return -1;
1146                         }
1147                         opc = LST_GROUP_CLEAN;
1148                         break;
1149
1150                 default:
1151                         lst_print_usage(argv[0]);
1152                         return -1;
1153                 }
1154         }
1155
1156         /* no OPC or group is specified */
1157         if (opc == 0 || optind != argc - 1) {
1158                 lst_print_usage(argv[0]);
1159                 return -1;
1160         }
1161
1162         grp = argv[optind];
1163
1164         CFS_INIT_LIST_HEAD(&head);
1165
1166         if (opc == LST_GROUP_RMND || opc == LST_GROUP_REFRESH) {
1167                 rc = lst_get_node_count(opc == LST_GROUP_RMND ? LST_OPC_NODES :
1168                                                                 LST_OPC_GROUP,
1169                                         opc == LST_GROUP_RMND ? str : grp,
1170                                         &count, &ids);
1171
1172                 if (rc != 0) {
1173                         fprintf(stderr, "Can't get count of nodes from %s: %s\n",
1174                                 opc == LST_GROUP_RMND ? str : grp,
1175                                 strerror(errno));
1176                         return -1;
1177                 }
1178
1179                 rc = lst_alloc_rpcent(&head, count, 0);
1180                 if (rc != 0) {
1181                         fprintf(stderr, "Out of memory\n");
1182                         free(ids);
1183                         return -1;
1184                 }
1185
1186         }
1187
1188         rc = lst_update_group_ioctl(opc, grp, clean, count, ids, &head);
1189
1190         if (ids != NULL)
1191                 free(ids);
1192
1193         if (rc == 0) {
1194                 lst_free_rpcent(&head);
1195                 return 0;
1196         }
1197
1198         if (rc == -1) {
1199                 lst_free_rpcent(&head);
1200                 lst_print_error("group", "Failed to update group: %s\n",
1201                                 strerror(errno));
1202                 return rc;
1203         }
1204
1205         lst_print_transerr(&head, "Updating group");
1206
1207         lst_free_rpcent(&head);
1208
1209         return rc;
1210 }
1211
1212 int
1213 lst_list_group_ioctl(int len, char *name, int idx)
1214 {
1215         lstio_group_list_args_t         args = {
1216                 .lstio_grp_key          = session_key,
1217                 .lstio_grp_idx          = idx,
1218                 .lstio_grp_nmlen        = len,
1219                 .lstio_grp_namep        = name,
1220         };
1221
1222         return lst_ioctl(LSTIO_GROUP_LIST, &args, sizeof(args));
1223 }
1224
1225 int
1226 lst_info_group_ioctl(char *name, lstcon_ndlist_ent_t *gent,
1227                      int *idx, int *count, lstcon_node_ent_t *dents)
1228 {
1229         lstio_group_info_args_t         args = {
1230                 .lstio_grp_key          = session_key,
1231                 .lstio_grp_nmlen        = strlen(name),
1232                 .lstio_grp_namep        = name,
1233                 .lstio_grp_entp         = gent,
1234                 .lstio_grp_idxp         = idx,
1235                 .lstio_grp_ndentp       = count,
1236                 .lstio_grp_dentsp       = dents,
1237         };
1238
1239         return lst_ioctl(LSTIO_GROUP_INFO, &args, sizeof(args));
1240 }
1241
1242 int
1243 lst_list_group_all(void)
1244 {
1245         char  name[LST_NAME_SIZE];
1246         int   rc;
1247         int   i;
1248
1249         /* no group is specified, list name of all groups */
1250         for (i = 0; ; i++) {
1251                 rc = lst_list_group_ioctl(LST_NAME_SIZE, name, i);
1252                 if (rc == 0) {
1253                         fprintf(stdout, "%d) %s\n", i + 1, name);
1254                         continue;
1255                 }
1256
1257                 if (errno == ENOENT)
1258                         break;
1259
1260                 lst_print_error("group", "Failed to list group: %s\n",
1261                                 strerror(errno));
1262                 return -1;
1263         }
1264
1265         fprintf(stdout, "Total %d groups\n", i);
1266
1267         return 0;
1268 }
1269
1270 #define LST_NODES_TITLE "\tACTIVE\tBUSY\tDOWN\tUNKNOWN\tTOTAL\n"
1271
1272 int
1273 jt_lst_list_group(int argc, char **argv)
1274 {
1275         lstcon_ndlist_ent_t  gent;
1276         lstcon_node_ent_t   *dents;
1277         int               optidx  = 0;
1278         int               verbose = 0;
1279         int               active  = 0;
1280         int               busy    = 0;
1281         int               down    = 0;
1282         int               unknown = 0;
1283         int               all     = 0;
1284         int               count;
1285         int               index;
1286         int               i;
1287         int               j;
1288         int               c;
1289         int               rc = 0;
1290
1291         static struct option list_group_opts[] =
1292         {
1293                 {"active",  no_argument, 0, 'a' },
1294                 {"busy",    no_argument, 0, 'b' },
1295                 {"down",    no_argument, 0, 'd' },
1296                 {"unknown", no_argument, 0, 'u' },
1297                 {"all",     no_argument, 0, 'l' },
1298                 {0,         0,           0,  0  }
1299         };
1300
1301         if (session_key == 0) {
1302                 fprintf(stderr,
1303                         "Can't find env LST_SESSION or value is not valid\n");
1304                 return -1;
1305         }
1306
1307         while (1) {
1308                 c = getopt_long(argc, argv, "abdul",
1309                                 list_group_opts, &optidx);
1310
1311                 if (c == -1)
1312                         break;
1313
1314                 switch (c) {
1315                 case 'a':
1316                         verbose = active = 1;
1317                         all = 0;
1318                         break;
1319                 case 'b':
1320                         verbose = busy = 1;
1321                         all = 0;
1322                         break;
1323                 case 'd':
1324                         verbose = down = 1;
1325                         all = 0;
1326                         break;
1327                 case 'u':
1328                         verbose = unknown = 1;
1329                         all = 0;
1330                         break;
1331                 case 'l':
1332                         verbose = all = 1;
1333                         break;
1334                 default:
1335                         lst_print_usage(argv[0]);
1336                         return -1;
1337                 }
1338         }
1339
1340         if (optind == argc) {
1341                 /* no group is specified, list name of all groups */
1342                 rc = lst_list_group_all();
1343
1344                 return rc;
1345         }
1346
1347         if (!verbose)
1348                 fprintf(stdout, LST_NODES_TITLE);
1349
1350         /* list nodes in specified groups */
1351         for (i = optind; i < argc; i++) {
1352                 rc = lst_info_group_ioctl(argv[i], &gent, NULL, NULL, NULL);
1353                 if (rc != 0) {
1354                         if (errno == ENOENT) {
1355                                 rc = 0;
1356                                 break;
1357                         }
1358
1359                         lst_print_error("group", "Failed to list group\n",
1360                                         strerror(errno));
1361                         break;
1362                 }
1363
1364                 if (!verbose) {
1365                         fprintf(stdout, "\t%d\t%d\t%d\t%d\t%d\t%s\n",
1366                                 gent.nle_nactive, gent.nle_nbusy,
1367                                 gent.nle_ndown, gent.nle_nunknown,
1368                                 gent.nle_nnode, argv[i]);
1369                         continue;
1370                 }
1371
1372                 fprintf(stdout, "Group [ %s ]\n", argv[i]);
1373
1374                 if (gent.nle_nnode == 0) {
1375                         fprintf(stdout, "No nodes found [ %s ]\n", argv[i]);
1376                         continue;
1377                 }
1378
1379                 count = gent.nle_nnode;
1380
1381                 dents = malloc(count * sizeof(lstcon_node_ent_t));
1382                 if (dents == NULL) {
1383                         fprintf(stderr, "Failed to malloc: %s\n",
1384                                 strerror(errno));
1385                         return -1;
1386                 }
1387
1388                 index = 0;
1389                 rc = lst_info_group_ioctl(argv[i], &gent, &index, &count, dents);
1390                 if (rc != 0) {
1391                         lst_print_error("group", "Failed to list group: %s\n",
1392                                         strerror(errno));
1393                         free(dents);
1394                         return -1;
1395                 }
1396
1397                 for (j = 0, c = 0; j < count; j++) {
1398                         if (all ||
1399                             ((active  &&  dents[j].nde_state == LST_NODE_ACTIVE) ||
1400                              (busy    &&  dents[j].nde_state == LST_NODE_BUSY)   ||
1401                              (down    &&  dents[j].nde_state == LST_NODE_DOWN)   ||
1402                              (unknown &&  dents[j].nde_state == LST_NODE_UNKNOWN))) {
1403
1404                                 fprintf(stdout, "\t%s: %s\n",
1405                                         libcfs_id2str(dents[j].nde_id),
1406                                         lst_node_state2str(dents[j].nde_state));
1407                                 c++;
1408                         }
1409                 }
1410
1411                 fprintf(stdout, "Total %d nodes [ %s ]\n", c, argv[i]);
1412
1413                 free(dents);
1414         }
1415
1416         return rc;
1417 }
1418
1419 int
1420 lst_stat_ioctl (char *name, int count, lnet_process_id_t *idsp,
1421                 int timeout, struct list_head *resultp)
1422 {
1423         lstio_stat_args_t  args = {
1424                 .lstio_sta_key           = session_key,
1425                 .lstio_sta_timeout       = timeout,
1426                 .lstio_sta_nmlen         = strlen(name),
1427                 .lstio_sta_namep         = name,
1428                 .lstio_sta_count         = count,
1429                 .lstio_sta_idsp          = idsp,
1430                 .lstio_sta_resultp       = resultp,
1431         };
1432
1433         return lst_ioctl (LSTIO_STAT_QUERY, &args, sizeof(args));
1434 }
1435
1436 typedef struct {
1437         struct list_head        srp_link;
1438         int                     srp_count;
1439         char                   *srp_name;
1440         lnet_process_id_t      *srp_ids;
1441         struct list_head        srp_result[2];
1442 } lst_stat_req_param_t;
1443
1444 static void
1445 lst_stat_req_param_free(lst_stat_req_param_t *srp)
1446 {
1447         int     i;
1448
1449         for (i = 0; i < 2; i++)
1450                 lst_free_rpcent(&srp->srp_result[i]);
1451
1452         if (srp->srp_ids != NULL)
1453                 free(srp->srp_ids);
1454
1455         free(srp);
1456 }
1457
1458 static int
1459 lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp, int save_old)
1460 {
1461         lst_stat_req_param_t *srp = NULL;
1462         int                   count = save_old ? 2 : 1;
1463         int                   rc;
1464         int                   i;
1465
1466         srp = malloc(sizeof(*srp));
1467         if (srp == NULL)
1468                 return -ENOMEM;
1469
1470         memset(srp, 0, sizeof(*srp));
1471         CFS_INIT_LIST_HEAD(&srp->srp_result[0]);
1472         CFS_INIT_LIST_HEAD(&srp->srp_result[1]);
1473
1474         rc = lst_get_node_count(LST_OPC_GROUP, name,
1475                                 &srp->srp_count, NULL);
1476         if (rc != 0 && errno == ENOENT) {
1477                 rc = lst_get_node_count(LST_OPC_NODES, name,
1478                                         &srp->srp_count, &srp->srp_ids);
1479         }
1480
1481         if (rc != 0) {
1482                 fprintf(stderr,
1483                         "Failed to get count of nodes from %s: %s\n",
1484                         name, strerror(errno));
1485                 lst_stat_req_param_free(srp);
1486
1487                 return rc;
1488         }
1489
1490         srp->srp_name = name;
1491
1492         for (i = 0; i < count; i++) {
1493                 rc = lst_alloc_rpcent(&srp->srp_result[i], srp->srp_count,
1494                                       sizeof(sfw_counters_t)  +
1495                                       sizeof(srpc_counters_t) +
1496                                       sizeof(lnet_counters_t));
1497                 if (rc != 0) {
1498                         fprintf(stderr, "Out of memory\n");
1499                         break;
1500                 }
1501         }
1502
1503         if (rc == 0) {
1504                 *srpp = srp;
1505                 return 0;
1506         }
1507
1508         lst_stat_req_param_free(srp);
1509
1510         return rc;
1511 }
1512
1513 typedef struct {
1514         /* TODO */
1515 } lst_srpc_stat_result;
1516
1517 #define LST_LNET_AVG    0
1518 #define LST_LNET_MIN    1
1519 #define LST_LNET_MAX    2
1520
1521 typedef struct {
1522         float           lnet_avg_sndrate;
1523         float           lnet_min_sndrate;
1524         float           lnet_max_sndrate;
1525         float           lnet_total_sndrate;
1526
1527         float           lnet_avg_rcvrate;
1528         float           lnet_min_rcvrate;
1529         float           lnet_max_rcvrate;
1530         float           lnet_total_rcvrate;
1531
1532         float           lnet_avg_sndperf;
1533         float           lnet_min_sndperf;
1534         float           lnet_max_sndperf;
1535         float           lnet_total_sndperf;
1536
1537         float           lnet_avg_rcvperf;
1538         float           lnet_min_rcvperf;
1539         float           lnet_max_rcvperf;
1540         float           lnet_total_rcvperf;
1541
1542         int             lnet_stat_count;
1543 } lst_lnet_stat_result_t;
1544
1545 lst_lnet_stat_result_t lnet_stat_result;
1546
1547 static float
1548 lst_lnet_stat_value(int bw, int send, int off)
1549 {
1550         float  *p;
1551
1552         p = bw ? &lnet_stat_result.lnet_avg_sndperf :
1553                  &lnet_stat_result.lnet_avg_sndrate;
1554
1555         if (!send)
1556                 p += 4;
1557
1558         p += off;
1559
1560         return *p;
1561 }
1562
1563 static void
1564 lst_timeval_diff(struct timeval *tv1,
1565                  struct timeval *tv2, struct timeval *df)
1566 {
1567         if (tv1->tv_usec >= tv2->tv_usec) {
1568                 df->tv_sec  = tv1->tv_sec - tv2->tv_sec;
1569                 df->tv_usec = tv1->tv_usec - tv2->tv_usec;
1570                 return;
1571         }
1572
1573         df->tv_sec  = tv1->tv_sec - 1 - tv2->tv_sec;
1574         df->tv_usec = tv1->tv_sec + 1000000 - tv2->tv_usec;
1575
1576         return;
1577 }
1578
1579 void
1580 lst_cal_lnet_stat(float delta, lnet_counters_t *lnet_new,
1581                   lnet_counters_t *lnet_old)
1582 {
1583         float perf;
1584         float rate;
1585
1586         perf = (float)(lnet_new->send_length -
1587                        lnet_old->send_length) / (1024 * 1024) / delta;
1588         lnet_stat_result.lnet_total_sndperf += perf;
1589
1590         if (lnet_stat_result.lnet_min_sndperf > perf ||
1591             lnet_stat_result.lnet_min_sndperf == 0)
1592                 lnet_stat_result.lnet_min_sndperf = perf;
1593
1594         if (lnet_stat_result.lnet_max_sndperf < perf)
1595                 lnet_stat_result.lnet_max_sndperf = perf;
1596
1597         perf = (float)(lnet_new->recv_length -
1598                        lnet_old->recv_length) / (1024 * 1024) / delta;
1599         lnet_stat_result.lnet_total_rcvperf += perf;
1600
1601         if (lnet_stat_result.lnet_min_rcvperf > perf ||
1602             lnet_stat_result.lnet_min_rcvperf == 0)
1603                 lnet_stat_result.lnet_min_rcvperf = perf;
1604
1605         if (lnet_stat_result.lnet_max_rcvperf < perf)
1606                 lnet_stat_result.lnet_max_rcvperf = perf;
1607
1608         rate = (lnet_new->send_count - lnet_old->send_count) / delta;
1609         lnet_stat_result.lnet_total_sndrate += rate;
1610
1611         if (lnet_stat_result.lnet_min_sndrate > rate ||
1612             lnet_stat_result.lnet_min_sndrate == 0)
1613                 lnet_stat_result.lnet_min_sndrate = rate;
1614
1615         if (lnet_stat_result.lnet_max_sndrate < rate)
1616                 lnet_stat_result.lnet_max_sndrate = rate;
1617
1618         rate = (lnet_new->recv_count - lnet_old->recv_count) / delta;
1619         lnet_stat_result.lnet_total_rcvrate += rate;
1620
1621         if (lnet_stat_result.lnet_min_rcvrate > rate ||
1622             lnet_stat_result.lnet_min_rcvrate == 0)
1623                 lnet_stat_result.lnet_min_rcvrate = rate;
1624
1625         if (lnet_stat_result.lnet_max_rcvrate < rate)
1626                 lnet_stat_result.lnet_max_rcvrate = rate;
1627
1628         lnet_stat_result.lnet_stat_count ++;
1629
1630         lnet_stat_result.lnet_avg_sndrate = lnet_stat_result.lnet_total_sndrate /
1631                                             lnet_stat_result.lnet_stat_count;
1632         lnet_stat_result.lnet_avg_rcvrate = lnet_stat_result.lnet_total_rcvrate /
1633                                             lnet_stat_result.lnet_stat_count;
1634
1635         lnet_stat_result.lnet_avg_sndperf = lnet_stat_result.lnet_total_sndperf /
1636                                             lnet_stat_result.lnet_stat_count;
1637         lnet_stat_result.lnet_avg_rcvperf = lnet_stat_result.lnet_total_rcvperf /
1638                                             lnet_stat_result.lnet_stat_count;
1639
1640 }
1641
1642 void
1643 lst_print_lnet_stat(char *name, int bwrt, int rdwr, int type)
1644 {
1645         int     start1 = 0;
1646         int     end1   = 1;
1647         int     start2 = 0;
1648         int     end2   = 1;
1649         int     i;
1650         int     j;
1651
1652         if (lnet_stat_result.lnet_stat_count == 0)
1653                 return;
1654
1655         if (bwrt == 1) /* bw only */
1656                 start1 = 1;
1657
1658         if (bwrt == 2) /* rates only */
1659                 end1 = 0;
1660
1661         if (rdwr == 1) /* recv only */
1662                 start2 = 1;
1663
1664         if (rdwr == 2) /* send only */
1665                 end2 = 0;
1666
1667         for (i = start1; i <= end1; i++) {
1668                 fprintf(stdout, "[LNet %s of %s]\n",
1669                         i == 0 ? "Rates" : "Bandwidth", name);
1670
1671                 for (j = start2; j <= end2; j++) {
1672                         fprintf(stdout, "[%c] ", j == 0 ? 'R' : 'W');
1673
1674                         if ((type & 1) != 0) {
1675                                 fprintf(stdout, i == 0 ? "Avg: %-8.0f RPC/s " :
1676                                                          "Avg: %-8.2f MB/s  ",
1677                                         lst_lnet_stat_value(i, j, 0));
1678                         }
1679
1680                         if ((type & 2) != 0) {
1681                                 fprintf(stdout, i == 0 ? "Min: %-8.0f RPC/s " :
1682                                                          "Min: %-8.2f MB/s  ",
1683                                         lst_lnet_stat_value(i, j, 1));
1684                         }
1685
1686                         if ((type & 4) != 0) {
1687                                 fprintf(stdout, i == 0 ? "Max: %-8.0f RPC/s" :
1688                                                          "Max: %-8.2f MB/s",
1689                                         lst_lnet_stat_value(i, j, 2));
1690                         }
1691
1692                         fprintf(stdout, "\n");
1693                 }
1694         }
1695 }
1696
1697 void
1698 lst_print_stat(char *name, struct list_head *resultp,
1699                int idx, int lnet, int bwrt, int rdwr, int type)
1700 {
1701         struct list_head  tmp[2];
1702         lstcon_rpc_ent_t *new;
1703         lstcon_rpc_ent_t *old;
1704         sfw_counters_t   *sfwk_new;
1705         sfw_counters_t   *sfwk_old;
1706         srpc_counters_t  *srpc_new;
1707         srpc_counters_t  *srpc_old;
1708         lnet_counters_t  *lnet_new;
1709         lnet_counters_t  *lnet_old;
1710         struct timeval    tv;
1711         float             delta;
1712         int               errcount = 0;
1713
1714         CFS_INIT_LIST_HEAD(&tmp[0]);
1715         CFS_INIT_LIST_HEAD(&tmp[1]);
1716
1717         memset(&lnet_stat_result, 0, sizeof(lnet_stat_result));
1718
1719         while (!list_empty(&resultp[idx])) {
1720                 if (list_empty(&resultp[1 - idx])) {
1721                         fprintf(stderr, "Group is changed, re-run stat\n");
1722                         break;
1723                 }
1724
1725                 new = list_entry(resultp[idx].next, lstcon_rpc_ent_t, rpe_link);
1726                 old = list_entry(resultp[1 - idx].next, lstcon_rpc_ent_t, rpe_link);
1727
1728                 /* first time get stats result, can't calculate diff */
1729                 if (new->rpe_peer.nid == LNET_NID_ANY)
1730                         break;
1731
1732                 if (new->rpe_peer.nid != old->rpe_peer.nid ||
1733                     new->rpe_peer.pid != old->rpe_peer.pid) {
1734                         /* Something wrong. i.e, somebody change the group */
1735                         break;
1736                 }
1737
1738                 list_del(&new->rpe_link);
1739                 list_add_tail(&new->rpe_link, &tmp[idx]);
1740
1741                 list_del(&old->rpe_link);
1742                 list_add_tail(&old->rpe_link, &tmp[1 - idx]);
1743
1744                 if (new->rpe_rpc_errno != 0 || new->rpe_fwk_errno != 0 ||
1745                     old->rpe_rpc_errno != 0 || old->rpe_fwk_errno != 0) {
1746                         errcount ++;
1747                         continue;
1748                 }
1749
1750                 sfwk_new = (sfw_counters_t *)&new->rpe_payload[0];
1751                 sfwk_old = (sfw_counters_t *)&old->rpe_payload[0];
1752
1753                 srpc_new = (srpc_counters_t *)((char *)sfwk_new + sizeof(*sfwk_new));
1754                 srpc_old = (srpc_counters_t *)((char *)sfwk_old + sizeof(*sfwk_old));
1755
1756                 lnet_new = (lnet_counters_t *)((char *)srpc_new + sizeof(*srpc_new));
1757                 lnet_old = (lnet_counters_t *)((char *)srpc_old + sizeof(*srpc_old));
1758
1759                 lst_timeval_diff(&new->rpe_stamp, &old->rpe_stamp, &tv);
1760
1761                 delta = tv.tv_sec + (float)tv.tv_usec/1000000;
1762
1763                 if (!lnet) /* TODO */
1764                         continue;
1765
1766                 lst_cal_lnet_stat(delta, lnet_new, lnet_old);
1767         }
1768
1769         list_splice(&tmp[idx], &resultp[idx]);
1770         list_splice(&tmp[1 - idx], &resultp[1 - idx]);
1771
1772         if (errcount > 0)
1773                 fprintf(stdout, "Failed to stat on %d nodes\n", errcount);
1774
1775         if (!lnet)  /* TODO */
1776                 return;
1777
1778         lst_print_lnet_stat(name, bwrt, rdwr, type);
1779 }
1780
1781 int
1782 jt_lst_stat(int argc, char **argv)
1783 {
1784         struct list_head      head;
1785         lst_stat_req_param_t *srp;
1786         time_t                last    = 0;
1787         int                   optidx  = 0;
1788         int                   timeout = 5; /* default timeout, 5 sec */
1789         int                   delay   = 5; /* default delay, 5 sec */
1790         int                   lnet    = 1; /* lnet stat by default */
1791         int                   bwrt    = 0;
1792         int                   rdwr    = 0;
1793         int                   type    = -1;
1794         int                   idx     = 0;
1795         int                   rc;
1796         int                   c;
1797
1798         static struct option stat_opts[] =
1799         {
1800                 {"timeout", required_argument, 0, 't' },
1801                 {"delay"  , required_argument, 0, 'd' },
1802                 {"lnet"   , no_argument,       0, 'l' },
1803                 {"rpc"    , no_argument,       0, 'c' },
1804                 {"bw"     , no_argument,       0, 'b' },
1805                 {"rate"   , no_argument,       0, 'a' },
1806                 {"read"   , no_argument,       0, 'r' },
1807                 {"write"  , no_argument,       0, 'w' },
1808                 {"avg"    , no_argument,       0, 'g' },
1809                 {"min"    , no_argument,       0, 'n' },
1810                 {"max"    , no_argument,       0, 'x' },
1811                 {0,         0,                 0,  0  }
1812         };
1813
1814         if (session_key == 0) {
1815                 fprintf(stderr,
1816                         "Can't find env LST_SESSION or value is not valid\n");
1817                 return -1;
1818         }
1819
1820         while (1) {
1821                 c = getopt_long(argc, argv, "t:d:lcbarwgnx", stat_opts, &optidx);
1822
1823                 if (c == -1)
1824                         break;
1825
1826                 switch (c) {
1827                 case 't':
1828                         timeout = atoi(optarg);
1829                         break;
1830                 case 'd':
1831                         delay = atoi(optarg);
1832                         break;
1833                 case 'l':
1834                         lnet = 1;
1835                         break;
1836                 case 'c':
1837                         lnet = 0;
1838                         break;
1839                 case 'b':
1840                         bwrt |= 1;
1841                         break;
1842                 case 'a':
1843                         bwrt |= 2;
1844                         break;
1845                 case 'r':
1846                         rdwr |= 1;
1847                         break;
1848                 case 'w':
1849                         rdwr |= 2;
1850                         break;
1851                 case 'g':
1852                         if (type == -1) {
1853                                 type = 1;
1854                                 break;
1855                         }
1856                         type |= 1;
1857                         break;
1858                 case 'n':
1859                         if (type == -1) {
1860                                 type = 2;
1861                                 break;
1862                         }
1863                         type |= 2;
1864                         break;
1865                 case 'x':
1866                         if (type == -1) {
1867                                 type = 4;
1868                                 break;
1869                         }
1870                         type |= 4;
1871                         break;
1872                 default:
1873                         lst_print_usage(argv[0]);
1874                         return -1;
1875                 }
1876         }
1877
1878         if (optind == argc) {
1879                 lst_print_usage(argv[0]);
1880                 return -1;
1881         }
1882
1883         if (timeout <= 0 || delay <= 0) {
1884                 fprintf(stderr, "Invalid timeout or delay value\n");
1885                 return -1;
1886         }
1887
1888         CFS_INIT_LIST_HEAD(&head);
1889
1890         while (optind < argc) {
1891                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 1);
1892                 if (rc != 0)
1893                         goto out;
1894
1895                 list_add_tail(&srp->srp_link, &head);
1896         }
1897
1898         while (1) {
1899                 time_t  now = time(NULL);
1900
1901                 if (now - last < delay) {
1902                         sleep(delay - now + last);
1903                         time(&now);
1904                 }
1905
1906                 last = now;
1907
1908                 list_for_each_entry(srp, &head, srp_link) {
1909                         rc = lst_stat_ioctl(srp->srp_name,
1910                                             srp->srp_count, srp->srp_ids,
1911                                             timeout, &srp->srp_result[idx]);
1912                         if (rc == -1) {
1913                                 lst_print_error("stat", "Failed to stat %s: %s\n",
1914                                                 srp->srp_name, strerror(errno));
1915                                 goto out;
1916                         }
1917
1918                         lst_print_stat(srp->srp_name, srp->srp_result,
1919                                        idx, lnet, bwrt, rdwr, type);
1920
1921                         lst_reset_rpcent(&srp->srp_result[1 - idx]);
1922                 }
1923
1924                 idx = 1 - idx;
1925         }
1926
1927 out:
1928         while (!list_empty(&head)) {
1929                 srp = list_entry(head.next, lst_stat_req_param_t, srp_link);
1930
1931                 list_del(&srp->srp_link);
1932                 lst_stat_req_param_free(srp);
1933         }
1934
1935         return rc;
1936 }
1937
1938 int
1939 jt_lst_show_error(int argc, char **argv)
1940 {
1941         struct list_head      head;
1942         lst_stat_req_param_t *srp;
1943         lstcon_rpc_ent_t     *ent;
1944         sfw_counters_t       *sfwk;
1945         srpc_counters_t      *srpc;
1946         lnet_counters_t      *lnet;
1947         int                   show_rpc = 1;
1948         int                   optidx   = 0;
1949         int                   rc       = 0;
1950         int                   ecount;
1951         int                   c;
1952
1953         static struct option  show_error_opts[] =
1954         {
1955                 {"session", no_argument,       0, 's' },
1956                 {0,         0,                 0,  0  }
1957         };
1958
1959         if (session_key == 0) {
1960                 fprintf(stderr,
1961                         "Can't find env LST_SESSION or value is not valid\n");
1962                 return -1;
1963         }
1964
1965         while (1) {
1966                 c = getopt_long(argc, argv, "s", show_error_opts, &optidx);
1967
1968                 if (c == -1)
1969                         break;
1970
1971                 switch (c) {
1972                 case 's':
1973                         show_rpc  = 0;
1974                         break;
1975
1976                 default:
1977                         lst_print_usage(argv[0]);
1978                         return -1;
1979                 }
1980         }
1981
1982         if (optind == argc) {
1983                 lst_print_usage(argv[0]);
1984                 return -1;
1985         }
1986
1987         CFS_INIT_LIST_HEAD(&head);
1988
1989         while (optind < argc) {
1990                 rc = lst_stat_req_param_alloc(argv[optind++], &srp, 0);
1991                 if (rc != 0)
1992                         goto out;
1993
1994                 list_add_tail(&srp->srp_link, &head);
1995         }
1996
1997         list_for_each_entry(srp, &head, srp_link) {
1998                 rc = lst_stat_ioctl(srp->srp_name, srp->srp_count,
1999                                     srp->srp_ids, 5, &srp->srp_result[0]);
2000
2001                 if (rc == -1) {
2002                         lst_print_error(srp->srp_name, "Failed to show errors of %s: %s\n",
2003                                         srp->srp_name, strerror(errno));
2004                         goto out;
2005                 }
2006
2007                 fprintf(stdout, "%s:\n", srp->srp_name);
2008
2009                 ecount = 0;
2010
2011                 list_for_each_entry(ent, &srp->srp_result[0], rpe_link) {
2012                         if (ent->rpe_rpc_errno != 0) {
2013                                 ecount ++;
2014                                 fprintf(stderr, "RPC failure, can't show error on %s\n",
2015                                         libcfs_id2str(ent->rpe_peer));
2016                                 continue;
2017                         }
2018
2019                         if (ent->rpe_fwk_errno != 0) {
2020                                 ecount ++;
2021                                 fprintf(stderr, "Framework failure, can't show error on %s\n",
2022                                         libcfs_id2str(ent->rpe_peer));
2023                                 continue;
2024                         }
2025
2026                         sfwk = (sfw_counters_t *)&ent->rpe_payload[0];
2027                         srpc = (srpc_counters_t *)((char *)sfwk + sizeof(*sfwk));
2028                         lnet = (lnet_counters_t *)((char *)srpc + sizeof(*srpc));
2029
2030                         if (srpc->errors == 0 &&
2031                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2032                                 continue;
2033
2034                         if (!show_rpc  &&
2035                             sfwk->brw_errors == 0 && sfwk->ping_errors == 0)
2036                                 continue;
2037
2038                         ecount ++;
2039
2040                         fprintf(stderr, "%s: [Session %d brw errors, %d ping errors]%c",
2041                                 libcfs_id2str(ent->rpe_peer),
2042                                 sfwk->brw_errors, sfwk->ping_errors,
2043                                 show_rpc  ? ' ' : '\n');
2044
2045                         if (!show_rpc)
2046                                 continue;
2047
2048                         fprintf(stderr, "[RPC: %d errors, %d dropped, %d expired]\n",
2049                                 srpc->errors, srpc->rpcs_dropped, srpc->rpcs_expired);
2050                 }
2051
2052                 fprintf(stdout, "Total %d error nodes in %s\n", ecount, srp->srp_name);
2053         }
2054 out:
2055         while (!list_empty(&head)) {
2056                 srp = list_entry(head.next, lst_stat_req_param_t, srp_link);
2057
2058                 list_del(&srp->srp_link);
2059                 lst_stat_req_param_free(srp);
2060         }
2061
2062         return rc;
2063 }
2064
2065 int
2066 lst_add_batch_ioctl (char *name)
2067 {
2068         lstio_batch_add_args_t  args = {
2069                 .lstio_bat_key           = session_key,
2070                 .lstio_bat_nmlen         = strlen(name),
2071                 .lstio_bat_namep         = name,
2072         };
2073
2074         return lst_ioctl (LSTIO_BATCH_ADD, &args, sizeof(args));
2075 }
2076
2077 int
2078 jt_lst_add_batch(int argc, char **argv)
2079 {
2080         char   *name;
2081         int     rc;
2082
2083         if (session_key == 0) {
2084                 fprintf(stderr,
2085                         "Can't find env LST_SESSION or value is not valid\n");
2086                 return -1;
2087         }
2088
2089         if (argc != 2) {
2090                 lst_print_usage(argv[0]);
2091                 return -1;
2092         }
2093
2094         name = argv[1];
2095         if (strlen(name) >= LST_NAME_SIZE) {
2096                 fprintf(stderr, "Name length is limited to %d\n",
2097                         LST_NAME_SIZE - 1);
2098                 return -1;
2099         }
2100
2101         rc = lst_add_batch_ioctl(name);
2102         if (rc == 0)
2103                 return 0;
2104
2105         lst_print_error("batch", "Failed to create batch: %s\n",
2106                         strerror(errno));
2107
2108         return -1;
2109 }
2110
2111 int
2112 lst_start_batch_ioctl (char *name, int timeout, struct list_head *resultp)
2113 {
2114         lstio_batch_run_args_t   args = {
2115                 .lstio_bat_key          = session_key,
2116                 .lstio_bat_timeout      = timeout,
2117                 .lstio_bat_nmlen        = strlen(name),
2118                 .lstio_bat_namep        = name,
2119                 .lstio_bat_resultp      = resultp,
2120         };
2121
2122         return lst_ioctl(LSTIO_BATCH_START, &args, sizeof(args));
2123 }
2124
2125 int
2126 jt_lst_start_batch(int argc, char **argv)
2127 {
2128         struct list_head  head;
2129         char             *batch;
2130         int               optidx  = 0;
2131         int               timeout = 0;
2132         int               count = 0;
2133         int               rc;
2134         int               c;
2135
2136         static struct option start_batch_opts[] =
2137         {
2138                 {"timeout", required_argument, 0, 't' },
2139                 {0,         0,                 0,  0  }
2140         };
2141
2142         if (session_key == 0) {
2143                 fprintf(stderr,
2144                         "Can't find env LST_SESSION or value is not valid\n");
2145                 return -1;
2146         }
2147
2148         while (1) {
2149                 c = getopt_long(argc, argv, "t:",
2150                                 start_batch_opts, &optidx);
2151
2152                 /* Detect the end of the options. */
2153                 if (c == -1)
2154                         break;
2155
2156                 switch (c) {
2157                 case 't':
2158                         timeout = atoi(optarg);
2159                         break;
2160                 default:
2161                         lst_print_usage(argv[0]);
2162                         return -1;
2163                 }
2164         }
2165
2166         if (optind == argc) {
2167                 batch = LST_DEFAULT_BATCH;
2168
2169         } else if (optind == argc - 1) {
2170                 batch = argv[optind];
2171
2172         } else {
2173                 lst_print_usage(argv[0]);
2174                 return -1;
2175         }
2176
2177         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2178         if (rc != 0) {
2179                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2180                         batch, strerror(errno));
2181                 return -1;
2182         }
2183
2184         CFS_INIT_LIST_HEAD(&head);
2185
2186         rc = lst_alloc_rpcent(&head, count, 0);
2187         if (rc != 0) {
2188                 fprintf(stderr, "Out of memory\n");
2189                 return -1;
2190         }
2191
2192         rc = lst_start_batch_ioctl(batch, timeout, &head);
2193
2194         if (rc == 0) {
2195                 fprintf(stdout, "%s is running now\n", batch);
2196                 lst_free_rpcent(&head);
2197                 return 0;
2198         }
2199
2200         if (rc == -1) {
2201                 lst_print_error("batch", "Failed to start batch: %s\n",
2202                                 strerror(errno));
2203                 lst_free_rpcent(&head);
2204                 return rc;
2205         }
2206
2207         lst_print_transerr(&head, "Run batch");
2208
2209         lst_free_rpcent(&head);
2210
2211         return rc;
2212 }
2213
2214 int
2215 lst_stop_batch_ioctl(char *name, int force, struct list_head *resultp)
2216 {
2217         lstio_batch_stop_args_t   args = {
2218                 .lstio_bat_key          = session_key,
2219                 .lstio_bat_force        = force,
2220                 .lstio_bat_nmlen        = strlen(name),
2221                 .lstio_bat_namep        = name,
2222                 .lstio_bat_resultp      = resultp,
2223         };
2224
2225         return lst_ioctl(LSTIO_BATCH_STOP, &args, sizeof(args));
2226 }
2227
2228 int
2229 jt_lst_stop_batch(int argc, char **argv)
2230 {
2231         struct list_head  head;
2232         char             *batch;
2233         int               force = 0;
2234         int               optidx;
2235         int               count;
2236         int               rc;
2237         int               c;
2238
2239         static struct option stop_batch_opts[] =
2240         {
2241                 {"force",   no_argument,   0, 'f' },
2242                 {0,         0,             0,  0  }
2243         };
2244
2245         if (session_key == 0) {
2246                 fprintf(stderr,
2247                         "Can't find env LST_SESSION or value is not valid\n");
2248                 return -1;
2249         }
2250
2251         while (1) {
2252                 c = getopt_long(argc, argv, "f",
2253                                 stop_batch_opts, &optidx);
2254
2255                 /* Detect the end of the options. */
2256                 if (c == -1)
2257                         break;
2258
2259                 switch (c) {
2260                 case 'f':
2261                         force = 1;
2262                         break;
2263                 default:
2264                         lst_print_usage(argv[0]);
2265                         return -1;
2266                 }
2267         }
2268
2269         if (optind == argc) {
2270                 batch = LST_DEFAULT_BATCH;
2271
2272         } else if (optind == argc - 1) {
2273                 batch = argv[optind];
2274
2275         } else {
2276                 lst_print_usage(argv[0]);
2277                 return -1;
2278         }
2279
2280         rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL);
2281         if (rc != 0) {
2282                 fprintf(stderr, "Failed to get count of nodes from %s: %s\n",
2283                         batch, strerror(errno));
2284                 return -1;
2285         }
2286
2287         CFS_INIT_LIST_HEAD(&head);
2288
2289         rc = lst_alloc_rpcent(&head, count, 0);
2290         if (rc != 0) {
2291                 fprintf(stderr, "Out of memory\n");
2292                 return -1;
2293         }
2294
2295         rc = lst_stop_batch_ioctl(batch, force, &head);
2296         if (rc != 0)
2297                 goto out;
2298
2299         while (1) {
2300                 lst_reset_rpcent(&head);
2301
2302                 rc = lst_query_batch_ioctl(batch, 0, 0, 30, &head);
2303                 if (rc != 0)
2304                         goto out;
2305
2306                 if (lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0 &&
2307                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0)
2308                         break;
2309
2310                 fprintf(stdout, "%d batch in stopping\n",
2311                         lstcon_tsbqry_stat_run(&trans_stat, 0));
2312                 sleep(1);
2313         }
2314
2315         fprintf(stdout, "Batch is stopped\n");
2316         lst_free_rpcent(&head);
2317
2318         return 0;
2319 out:
2320         if (rc == -1) {
2321                 lst_print_error("batch", "Failed to stop batch: %s\n",
2322                                 strerror(errno));
2323                 lst_free_rpcent(&head);
2324                 return -1;
2325         }
2326
2327         lst_print_transerr(&head, "stop batch");
2328
2329         lst_free_rpcent(&head);
2330
2331         return rc;
2332 }
2333
2334 int
2335 lst_list_batch_ioctl(int len, char *name, int index)
2336 {
2337         lstio_batch_list_args_t         args = {
2338                 .lstio_bat_key          = session_key,
2339                 .lstio_bat_idx          = index,
2340                 .lstio_bat_nmlen        = len,
2341                 .lstio_bat_namep        = name,
2342         };
2343
2344         return lst_ioctl(LSTIO_BATCH_LIST, &args, sizeof(args));
2345 }
2346
2347 int
2348 lst_info_batch_ioctl(char *batch, int test, int server,
2349                      lstcon_test_batch_ent_t *entp, int *idxp,
2350                      int *ndentp, lstcon_node_ent_t *dentsp)
2351 {
2352         lstio_batch_info_args_t         args = {
2353                 .lstio_bat_key          = session_key,
2354                 .lstio_bat_nmlen        = strlen(batch),
2355                 .lstio_bat_namep        = batch,
2356                 .lstio_bat_server       = server,
2357                 .lstio_bat_testidx      = test,
2358                 .lstio_bat_entp         = entp,
2359                 .lstio_bat_idxp         = idxp,
2360                 .lstio_bat_ndentp       = ndentp,
2361                 .lstio_bat_dentsp       = dentsp,
2362         };
2363
2364         return lst_ioctl(LSTIO_BATCH_INFO, &args, sizeof(args));
2365 }
2366
2367 int
2368 lst_list_batch_all(void)
2369 {
2370         char name[LST_NAME_SIZE];
2371         int  rc;
2372         int  i;
2373
2374         for (i = 0; ; i++) {
2375                 rc = lst_list_batch_ioctl(LST_NAME_SIZE, name, i);
2376                 if (rc == 0) {
2377                         fprintf(stdout, "%d) %s\n", i + 1, name);
2378                         continue;
2379                 }
2380
2381                 if (errno == ENOENT)
2382                         break;
2383
2384                 lst_print_error("batch", "Failed to list batch: %s\n",
2385                                 strerror(errno));
2386                 return rc;
2387         }
2388
2389         fprintf(stdout, "Total %d batches\n", i);
2390
2391         return 0;
2392 }
2393
2394 int
2395 lst_list_tsb_nodes(char *batch, int test, int server,
2396                    int count, int active, int invalid)
2397 {
2398         lstcon_node_ent_t *dents;
2399         int                index = 0;
2400         int                rc;
2401         int                c;
2402         int                i;
2403
2404         if (count == 0)
2405                 return 0;
2406
2407         /* verbose list, show nodes in batch or test */
2408         dents = malloc(count * sizeof(lstcon_node_ent_t));
2409         if (dents == NULL) {
2410                 fprintf(stdout, "Can't allocate memory\n");
2411                 return -1;
2412         }
2413
2414         rc = lst_info_batch_ioctl(batch, test, server,
2415                                   NULL, &index, &count, dents);
2416         if (rc != 0) {
2417                 free(dents);
2418                 lst_print_error((test > 0) ? "test" : "batch",
2419                                 (test > 0) ? "Failed to query test: %s\n" :
2420                                              "Failed to query batch: %s\n",
2421                                 strerror(errno));
2422                 return -1;
2423         }
2424
2425         for (i = 0, c = 0; i < count; i++) {
2426                 if ((!active  && dents[i].nde_state == LST_NODE_ACTIVE) ||
2427                     (!invalid && (dents[i].nde_state == LST_NODE_BUSY  ||
2428                                   dents[i].nde_state == LST_NODE_DOWN  ||
2429                                   dents[i].nde_state == LST_NODE_UNKNOWN)))
2430                         continue;
2431
2432                 fprintf(stdout, "\t%s: %s\n",
2433                         libcfs_id2str(dents[i].nde_id),
2434                         lst_node_state2str(dents[i].nde_state));
2435                 c++;
2436         }
2437
2438         fprintf(stdout, "Total %d nodes\n", c);
2439         free(dents);
2440
2441         return 0;
2442 }
2443
2444 int
2445 jt_lst_list_batch(int argc, char **argv)
2446 {
2447         lstcon_test_batch_ent_t ent;
2448         char                *batch   = NULL;
2449         int                  optidx  = 0;
2450         int                  verbose = 0; /* list nodes in batch or test */
2451         int                  invalid = 0;
2452         int                  active  = 0;
2453         int                  server  = 0;
2454         int                  ntest   = 0;
2455         int                  test    = 0;
2456         int                  c       = 0;
2457         int                  rc;
2458
2459         static struct option list_batch_opts[] =
2460         {
2461                 {"test",    required_argument, 0, 't' },
2462                 {"invalid", no_argument,       0, 'i' },
2463                 {"active",  no_argument,       0, 'a' },
2464                 {"all",     no_argument,       0, 'l' },
2465                 {"server",  no_argument,       0, 's' },
2466                 {0,         0,                 0,  0  }
2467         };
2468
2469         if (session_key == 0) {
2470                 fprintf(stderr,
2471                         "Can't find env LST_SESSION or value is not valid\n");
2472                 return -1;
2473         }
2474
2475         while (1) {
2476                 c = getopt_long(argc, argv, "ailst:",
2477                                 list_batch_opts, &optidx);
2478
2479                 if (c == -1)
2480                         break;
2481
2482                 switch (c) {
2483                 case 'a':
2484                         verbose = active = 1;
2485                         break;
2486                 case 'i':
2487                         verbose = invalid = 1;
2488                         break;
2489                 case 'l':
2490                         verbose = active = invalid = 1;
2491                         break;
2492                 case 's':
2493                         server = 1;
2494                         break;
2495                 case 't':
2496                         test = atoi(optarg);
2497                         ntest = 1;
2498                         break;
2499                 default:
2500                         lst_print_usage(argv[0]);
2501                         return -1;
2502                 }
2503         }
2504
2505         if (optind == argc) {
2506                 /* list all batches */
2507                 rc = lst_list_batch_all();
2508                 return rc;
2509         }
2510
2511         if (ntest == 1 && test <= 0) {
2512                 fprintf(stderr, "Invalid test id, test id starts from 1\n");
2513                 return -1;
2514         }
2515
2516         if (optind != argc - 1) {
2517                 lst_print_usage(argv[0]);
2518                 return -1;
2519         }
2520
2521         batch = argv[optind];
2522
2523 loop:
2524         /* show detail of specified batch or test */
2525         rc = lst_info_batch_ioctl(batch, test, server,
2526                                   &ent, NULL, NULL, NULL);
2527         if (rc != 0) {
2528                 lst_print_error((test > 0) ? "test" : "batch",
2529                                 (test > 0) ? "Failed to query test: %s\n" :
2530                                              "Failed to query batch: %s\n",
2531                                 strerror(errno));
2532                 return -1;
2533         }
2534
2535         if (verbose) {
2536                 /* list nodes in test or batch */
2537                 rc = lst_list_tsb_nodes(batch, test, server,
2538                                         server ? ent.tbe_srv_nle.nle_nnode :
2539                                                  ent.tbe_cli_nle.nle_nnode,
2540                                         active, invalid);
2541                 return rc;
2542         }
2543
2544         /* only show number of hosts in batch or test */
2545         if (test == 0) {
2546                 fprintf(stdout, "Batch: %s Tests: %d State: %d\n",
2547                         batch, ent.u.tbe_batch.bae_ntest,
2548                         ent.u.tbe_batch.bae_state);
2549                 ntest = ent.u.tbe_batch.bae_ntest;
2550                 test = 1; /* starting from test 1 */
2551
2552         } else {
2553                 fprintf(stdout,
2554                         "\tTest %d(%s) (loop: %d, concurrency: %d)\n",
2555                         test, lst_test_type2name(ent.u.tbe_test.tse_type),
2556                         ent.u.tbe_test.tse_loop,
2557                         ent.u.tbe_test.tse_concur);
2558                 ntest --;
2559                 test ++;
2560         }
2561
2562         fprintf(stdout, LST_NODES_TITLE);
2563         fprintf(stdout, "client\t%d\t%d\t%d\t%d\t%d\n"
2564                         "server\t%d\t%d\t%d\t%d\t%d\n",
2565                 ent.tbe_cli_nle.nle_nactive,
2566                 ent.tbe_cli_nle.nle_nbusy,
2567                 ent.tbe_cli_nle.nle_ndown,
2568                 ent.tbe_cli_nle.nle_nunknown,
2569                 ent.tbe_cli_nle.nle_nnode,
2570                 ent.tbe_srv_nle.nle_nactive,
2571                 ent.tbe_srv_nle.nle_nbusy,
2572                 ent.tbe_srv_nle.nle_ndown,
2573                 ent.tbe_srv_nle.nle_nunknown,
2574                 ent.tbe_srv_nle.nle_nnode);
2575
2576         if (ntest != 0)
2577                 goto loop;
2578
2579         return 0;
2580 }
2581
2582 int
2583 lst_query_batch_ioctl(char *batch, int test, int server,
2584                       int timeout, struct list_head *head)
2585 {
2586         lstio_batch_query_args_t args = {
2587                 .lstio_bat_key     = session_key,
2588                 .lstio_bat_testidx = test,
2589                 .lstio_bat_client  = !(server),
2590                 .lstio_bat_timeout = timeout,
2591                 .lstio_bat_nmlen   = strlen(batch),
2592                 .lstio_bat_namep   = batch,
2593                 .lstio_bat_resultp = head,
2594         };
2595
2596         return lst_ioctl(LSTIO_BATCH_QUERY, &args, sizeof(args));
2597 }
2598
2599 void
2600 lst_print_tsb_verbose(struct list_head *head,
2601                       int active, int idle, int error)
2602 {
2603         lstcon_rpc_ent_t *ent;
2604
2605         list_for_each_entry(ent, head, rpe_link) {
2606                 if (ent->rpe_priv[0] == 0 && active)
2607                         continue;
2608
2609                 if (ent->rpe_priv[0] != 0 && idle)
2610                         continue;
2611
2612                 if (ent->rpe_fwk_errno == 0 && error)
2613                         continue;
2614
2615                 fprintf(stdout, "%s [%s]: %s\n",
2616                         libcfs_id2str(ent->rpe_peer),
2617                         lst_node_state2str(ent->rpe_state),
2618                         ent->rpe_rpc_errno != 0 ?
2619                                 strerror(ent->rpe_rpc_errno) :
2620                                 (ent->rpe_priv[0] > 0 ? "Running" : "Idle"));
2621         }
2622 }
2623
2624 int
2625 jt_lst_query_batch(int argc, char **argv)
2626 {
2627         lstcon_test_batch_ent_t ent;
2628         struct list_head     head;
2629         char                *batch   = NULL;
2630         time_t               last    = 0;
2631         int                  optidx  = 0;
2632         int                  verbose = 0;
2633         int                  server  = 0;
2634         int                  timeout = 5; /* default 5 seconds */
2635         int                  delay   = 5; /* default 5 seconds */
2636         int                  loop    = 1; /* default 1 loop */
2637         int                  active  = 0;
2638         int                  error   = 0;
2639         int                  idle    = 0;
2640         int                  count   = 0;
2641         int                  test    = 0;
2642         int                  rc      = 0;
2643         int                  c       = 0;
2644         int                  i;
2645
2646         static struct option query_batch_opts[] =
2647         {
2648                 {"timeout", required_argument, 0, 'o' },
2649                 {"delay",   required_argument, 0, 'd' },
2650                 {"loop",    required_argument, 0, 'c' },
2651                 {"test",    required_argument, 0, 't' },
2652                 {"server",  no_argument,       0, 's' },
2653                 {"active",  no_argument,       0, 'a' },
2654                 {"idle",    no_argument,       0, 'i' },
2655                 {"error",   no_argument,       0, 'e' },
2656                 {"all",     no_argument,       0, 'l' },
2657                 {0,         0,                 0,  0  }
2658         };
2659
2660         if (session_key == 0) {
2661                 fprintf(stderr,
2662                         "Can't find env LST_SESSION or value is not valid\n");
2663                 return -1;
2664         }
2665
2666         while (1) {
2667                 c = getopt_long(argc, argv, "o:d:c:t:saiel",
2668                                 query_batch_opts, &optidx);
2669
2670                 /* Detect the end of the options. */
2671                 if (c == -1)
2672                         break;
2673
2674                 switch (c) {
2675                 case 'o':
2676                         timeout = atoi(optarg);
2677                         break;
2678                 case 'd':
2679                         delay = atoi(optarg);
2680                         break;
2681                 case 'c':
2682                         loop = atoi(optarg);
2683                         break;
2684                 case 't':
2685                         test = atoi(optarg);
2686                         break;
2687                 case 's':
2688                         server = 1;
2689                         break;
2690                 case 'a':
2691                         active = verbose = 1;
2692                         break;
2693                 case 'i':
2694                         idle = verbose = 1;
2695                         break;
2696                 case 'e':
2697                         error = verbose = 1;
2698                         break;
2699                 case 'l':
2700                         verbose = 1;
2701                         break;
2702                 default:
2703                         lst_print_usage(argv[0]);
2704                         return -1;
2705                 }
2706         }
2707
2708         if (test < 0 || timeout <= 0 || delay <= 0 || loop <= 0) {
2709                 lst_print_usage(argv[0]);
2710                 return -1;
2711         }
2712
2713         if (optind == argc) {
2714                 batch = LST_DEFAULT_BATCH;
2715
2716         } else if (optind == argc - 1) {
2717                 batch = argv[optind];
2718
2719         } else {
2720                 lst_print_usage(argv[0]);
2721                 return -1;
2722         }
2723
2724
2725         CFS_INIT_LIST_HEAD(&head);
2726
2727         if (verbose) {
2728                 rc = lst_info_batch_ioctl(batch, test, server,
2729                                           &ent, NULL, NULL, NULL);
2730                 if (rc != 0) {
2731                         fprintf(stderr, "Failed to query %s [%d]: %s\n",
2732                                 batch, test, strerror(errno));
2733                         return -1;
2734                 }
2735
2736                 count = server ? ent.tbe_srv_nle.nle_nnode :
2737                                  ent.tbe_cli_nle.nle_nnode;
2738                 if (count == 0) {
2739                         fprintf(stdout, "Batch or test is empty\n");
2740                         return 0;
2741                 }
2742         }
2743
2744         rc = lst_alloc_rpcent(&head, count, 0);
2745         if (rc != 0) {
2746                 fprintf(stderr, "Out of memory\n");
2747                 return rc;
2748         }
2749
2750         for (i = 0; i < loop; i++) {
2751                 time_t  now = time(NULL);
2752
2753                 if (now - last < delay) {
2754                         sleep(delay - now + last);
2755                         time(&now);
2756                 }
2757
2758                 last = now;
2759
2760                 rc = lst_query_batch_ioctl(batch, test,
2761                                            server, timeout, &head);
2762                 if (rc == -1) {
2763                         fprintf(stderr, "Failed to query batch: %s\n",
2764                                 strerror(errno));
2765                         break;
2766                 }
2767
2768                 if (verbose) {
2769                         /* Verbose mode */
2770                         lst_print_tsb_verbose(&head, active, idle, error);
2771                         continue;
2772                 }
2773
2774                 fprintf(stdout, "%s [%d] ", batch, test);
2775
2776                 if (lstcon_rpc_stat_failure(&trans_stat, 0) != 0) {
2777                         fprintf(stdout, "%d of %d nodes are unknown, ",
2778                                 lstcon_rpc_stat_failure(&trans_stat, 0),
2779                                 lstcon_rpc_stat_total(&trans_stat, 0));
2780                 }
2781
2782                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
2783                     lstcon_tsbqry_stat_run(&trans_stat, 0)  == 0  &&
2784                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
2785                         fprintf(stdout, "is stopped\n");
2786                         continue;
2787                 }
2788
2789                 if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 &&
2790                     lstcon_tsbqry_stat_idle(&trans_stat, 0) == 0 &&
2791                     lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) {
2792                         fprintf(stdout, "is running\n");
2793                         continue;
2794                 }
2795
2796                 fprintf(stdout, "stopped: %d , running: %d, failed: %d\n",
2797                                 lstcon_tsbqry_stat_idle(&trans_stat, 0),
2798                                 lstcon_tsbqry_stat_run(&trans_stat, 0),
2799                                 lstcon_tsbqry_stat_failure(&trans_stat, 0));
2800         }
2801
2802         lst_free_rpcent(&head);
2803
2804         return rc;
2805 }
2806
2807 int
2808 lst_parse_distribute(char *dstr, int *dist, int *span)
2809 {
2810         *dist = atoi(dstr);
2811         if (*dist <= 0)
2812                 return -1;
2813
2814         dstr = strchr(dstr, ':');
2815         if (dstr == NULL)
2816                 return -1;
2817
2818         *span = atoi(dstr + 1);
2819         if (*span <= 0)
2820                 return -1;
2821
2822         return 0;
2823 }
2824
2825 int
2826 lst_get_bulk_param(int argc, char **argv, lst_test_bulk_param_t *bulk)
2827 {
2828         char   *tok = NULL;
2829         char   *end = NULL;
2830         int     rc  = 0;
2831         int     i   = 0;
2832
2833         bulk->blk_size  = 4096;
2834         bulk->blk_opc   = LST_BRW_READ;
2835         bulk->blk_flags = LST_BRW_CHECK_NONE;
2836
2837         while (i < argc) {
2838                 if (strcasestr(argv[i], "check=") == argv[i] ||
2839                     strcasestr(argv[i], "c=") == argv[i]) {
2840                         tok = strchr(argv[i], '=') + 1;
2841
2842                         if (strcasecmp(tok, "full") == 0) {
2843                                 bulk->blk_flags = LST_BRW_CHECK_FULL;
2844                         } else if (strcasecmp(tok, "simple") == 0) {
2845                                 bulk->blk_flags = LST_BRW_CHECK_SIMPLE;
2846                         } else {
2847                                 fprintf(stderr, "Unknow flag %s\n", tok);
2848                                 return -1;
2849                         }
2850
2851                 } else if (strcasestr(argv[i], "size=") == argv[i] ||
2852                          strcasestr(argv[i], "s=") == argv[i]) {
2853                         tok = strchr(argv[i], '=') + 1;
2854
2855                         bulk->blk_size = strtol(tok, &end, 0);
2856                         if (bulk->blk_size <= 0) {
2857                                 fprintf(stderr, "Invalid size %s\n", tok);
2858                                 return -1;
2859                         }
2860
2861                         if (end == NULL)
2862                                 return 0;
2863
2864                         if (*end == 'k' || *end == 'K')
2865                                 bulk->blk_size *= 1024;
2866                         else if (*end == 'm' || *end == 'M')
2867                                 bulk->blk_size *= 1024 * 1024;
2868
2869                         if (bulk->blk_size > CFS_PAGE_SIZE * LNET_MAX_IOV) {
2870                                 fprintf(stderr, "Size exceed limitation: %d bytes\n",
2871                                         bulk->blk_size);
2872                                 return -1;
2873                         }
2874
2875                 } else if (strcasecmp(argv[i], "read") == 0 ||
2876                            strcasecmp(argv[i], "r") == 0) {
2877                         bulk->blk_opc = LST_BRW_READ;
2878
2879                 } else if (strcasecmp(argv[i], "write") == 0 ||
2880                            strcasecmp(argv[i], "w") == 0) {
2881                         bulk->blk_opc = LST_BRW_WRITE;
2882
2883                 } else {
2884                         fprintf(stderr, "Unknow parameter: %s\n", argv[i]);
2885                         return -1;
2886                 }
2887
2888                 i++;
2889         }
2890
2891         return rc;
2892 }
2893
2894 int
2895 lst_get_test_param(char *test, int argc, char **argv, void **param, int *plen)
2896 {
2897         lst_test_bulk_param_t *bulk = NULL;
2898         int                    type;
2899
2900         type = lst_test_name2type(test);
2901         if (type < 0) {
2902                 fprintf(stderr, "Unknow test name %s\n", test);
2903                 return -1;
2904         }
2905
2906         switch (type) {
2907         case LST_TEST_PING:
2908                 break;
2909
2910         case LST_TEST_BULK:
2911                 bulk = malloc(sizeof(*bulk));
2912                 if (bulk == NULL) {
2913                         fprintf(stderr, "Out of memory\n");
2914                         return -1;
2915                 }
2916
2917                 memset(bulk, 0, sizeof(*bulk));
2918
2919                 if (lst_get_bulk_param(argc, argv, bulk) != 0) {
2920                         free(bulk);
2921                         return -1;
2922                 }
2923
2924                 *param = bulk;
2925                 *plen  = sizeof(*bulk);
2926
2927                 break;
2928
2929         default:
2930                 break;
2931         }
2932
2933         /* TODO: parse more parameter */
2934         return type;
2935 }
2936
2937 int
2938 lst_add_test_ioctl(char *batch, int type, int loop, int concur,
2939                    int dist, int span, char *sgrp, char *dgrp,
2940                    void *param, int plen, int *retp, struct list_head *resultp)
2941 {
2942         lstio_test_args_t args = {
2943                 .lstio_tes_key          = session_key,
2944                 .lstio_tes_bat_nmlen    = strlen(batch),
2945                 .lstio_tes_bat_name     = batch,
2946                 .lstio_tes_type         = type,
2947                 .lstio_tes_loop         = loop,
2948                 .lstio_tes_concur       = concur,
2949                 .lstio_tes_dist         = dist,
2950                 .lstio_tes_span         = span,
2951                 .lstio_tes_sgrp_nmlen   = strlen(sgrp),
2952                 .lstio_tes_sgrp_name    = sgrp,
2953                 .lstio_tes_dgrp_nmlen   = strlen(dgrp),
2954                 .lstio_tes_dgrp_name    = dgrp,
2955                 .lstio_tes_param_len    = plen,
2956                 .lstio_tes_param        = param,
2957                 .lstio_tes_retp         = retp,
2958                 .lstio_tes_resultp      = resultp,
2959         };
2960
2961         return lst_ioctl(LSTIO_TEST_ADD, &args, sizeof(args));
2962 }
2963
2964 int
2965 jt_lst_add_test(int argc, char **argv)
2966 {
2967         struct list_head  head;
2968         char             *batch  = NULL;
2969         char             *test   = NULL;
2970         char             *dstr   = NULL;
2971         char             *from   = NULL;
2972         char             *to     = NULL;
2973         void             *param  = NULL;
2974         int               optidx = 0;
2975         int               concur = 1;
2976         int               loop   = -1;
2977         int               dist   = 1;
2978         int               span   = 1;
2979         int               plen   = 0;
2980         int               fcount = 0;
2981         int               tcount = 0;
2982         int               ret    = 0;
2983         int               type;
2984         int               rc;
2985         int               c;
2986
2987         static struct option add_test_opts[] =
2988         {
2989                 {"batch",       required_argument, 0, 'b' },
2990                 {"concurrency", required_argument, 0, 'c' },
2991                 {"distribute",  required_argument, 0, 'd' },
2992                 {"from",        required_argument, 0, 'f' },
2993                 {"to",          required_argument, 0, 't' },
2994                 {"loop",        required_argument, 0, 'l' },
2995                 {0,             0,                 0,  0  }
2996         };
2997
2998         if (session_key == 0) {
2999                 fprintf(stderr,
3000                         "Can't find env LST_SESSION or value is not valid\n");
3001                 return -1;
3002         }
3003
3004         while (1) {
3005                 c = getopt_long(argc, argv, "b:c:d:f:l:t:",
3006                                 add_test_opts, &optidx);
3007
3008                 /* Detect the end of the options. */
3009                 if (c == -1)
3010                         break;
3011
3012                 switch (c) {
3013                 case 'b':
3014                         batch = optarg;
3015                         break;
3016                 case 'c':
3017                         concur = atoi(optarg);
3018                         break;
3019                 case 'd':
3020                         dstr = optarg;
3021                         break;
3022                 case 'f':
3023                         from = optarg;
3024                         break;
3025                 case 'l':
3026                         loop = atoi(optarg);
3027                         break;
3028                 case 't':
3029                         to = optarg;
3030                         break;
3031                 default:
3032                         lst_print_usage(argv[0]);
3033                         return -1;
3034                 }
3035         }
3036
3037         if (optind == argc || from == NULL || to == NULL) {
3038                 lst_print_usage(argv[0]);
3039                 return -1;
3040         }
3041
3042         if (concur <= 0 || concur > LST_MAX_CONCUR) {
3043                 fprintf(stderr, "Invalid concurrency of test: %d\n", concur);
3044                 return -1;
3045         }
3046
3047         if (batch == NULL)
3048                 batch = LST_DEFAULT_BATCH;
3049
3050         if (dstr != NULL) {
3051                 rc = lst_parse_distribute(dstr, &dist, &span);
3052                 if (rc != 0) {
3053                         fprintf(stderr, "Invalid distribution: %s\n", dstr);
3054                         return -1;
3055                 }
3056         }
3057
3058         test = argv[optind++];
3059
3060         argc -= optind;
3061         argv += optind;
3062
3063         type = lst_get_test_param(test, argc, argv, &param, &plen);
3064         if (type < 0) {
3065                 fprintf(stderr, "Failed to add test (%s)\n", test);
3066                 return -1;
3067         }
3068
3069         CFS_INIT_LIST_HEAD(&head);
3070
3071         rc = lst_get_node_count(LST_OPC_GROUP, from, &fcount, NULL);
3072         if (rc != 0) {
3073                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3074                         from, strerror(errno));
3075                 goto out;
3076         }
3077
3078         rc = lst_get_node_count(LST_OPC_GROUP, to, &tcount, NULL);
3079         if (rc != 0) {
3080                 fprintf(stderr, "Can't get count of nodes from %s: %s\n",
3081                         to, strerror(errno));
3082                 goto out;
3083         }
3084
3085         rc = lst_alloc_rpcent(&head, fcount > tcount ? fcount : tcount, 0);
3086         if (rc != 0) {
3087                 fprintf(stderr, "Out of memory\n");
3088                 goto out;
3089         }
3090
3091         rc = lst_add_test_ioctl(batch, type, loop, concur,
3092                                 dist, span, from, to, param, plen, &ret, &head);
3093
3094         if (rc == 0) {
3095                 fprintf(stdout, "Test was added successfully\n");
3096                 if (ret != 0) {
3097                         fprintf(stdout, "Server group contains userland test "
3098                                 "nodes, old version of tcplnd can't accept "
3099                                 "connection request\n");
3100                 }
3101
3102                 goto out;
3103         }
3104
3105         if (rc == -1) {
3106                 lst_print_error("test", "Failed to add test: %s\n",
3107                                 strerror(errno));
3108                 goto out;
3109         }
3110
3111         lst_print_transerr(&head, "add test");
3112 out:
3113         lst_free_rpcent(&head);
3114
3115         if (param != NULL)
3116                 free(param);
3117
3118         return rc;
3119 }
3120
3121 static command_t lst_cmdlist[] = {
3122         {"new_session",         jt_lst_new_session,     NULL,
3123          "Usage: lst new_session [--timeout TIME] [--force] [NAME]"                     },
3124         {"end_session",         jt_lst_end_session,     NULL,
3125          "Usage: lst end_session"                                                       },
3126         {"show_session",        jt_lst_show_session,    NULL,
3127          "Usage: lst show_session"                                                      },
3128         {"ping",                jt_lst_ping ,           NULL,
3129          "Usage: lst ping  [--group NAME] [--batch NAME] [--session] [--nodes IDS]"     },
3130         {"add_group",           jt_lst_add_group,       NULL,
3131          "Usage: lst group NAME IDs [IDs]..."                                           },
3132         {"del_group",           jt_lst_del_group,       NULL,
3133          "Usage: lst del_group NAME"                                                    },
3134         {"update_group",        jt_lst_update_group,    NULL,
3135          "Usage: lst update_group NAME [--clean] [--refresh] [--remove IDs]"            },
3136         {"list_group",          jt_lst_list_group,      NULL,
3137           "Usage: lst list_group [--active] [--busy] [--down] [--unknown] GROUP ..."    },
3138         {"stat",                jt_lst_stat,            NULL,
3139          "Usage: lst stat [--bw] [--rate] [--read] [--write] [--max] [--min] [--avg] "
3140          " [--timeout #] [--delay #] GROUP [GROUP]"                                     },
3141         {"show_error",          jt_lst_show_error,      NULL,
3142          "Usage: lst show_error NAME | IDS ..."                                         },
3143         {"add_batch",           jt_lst_add_batch,       NULL,
3144          "Usage: lst add_batch NAME"                                                    },
3145         {"run",                 jt_lst_start_batch,     NULL,
3146          "Usage: lst run [--timeout TIME] [NAME]"                                       },
3147         {"stop",                jt_lst_stop_batch,      NULL,
3148          "Usage: lst stop [--force] BATCH_NAME"                                         },
3149         {"list_batch",          jt_lst_list_batch,      NULL,
3150          "Usage: lst list_batch NAME [--test ID] [--server]"                            },
3151         {"query",               jt_lst_query_batch,     NULL,
3152          "Usage: lst query [--test ID] [--server] [--timeout TIME] NAME"                },
3153         {"add_test",            jt_lst_add_test,        NULL,
3154          "Usage: lst add_test [--batch BATCH] [--loop #] [--concurrency #] "
3155          " [--distribute #:#] [--from GROUP] [--to GROUP] TEST..."                      },
3156         {"help",                Parser_help,            0,     "help"                   },
3157         {0,                     0,                      0,      NULL                    }
3158 };
3159
3160 int
3161 lst_initialize(void)
3162 {
3163         char   *key;
3164
3165         key = getenv("LST_SESSION");
3166
3167         if (key == NULL) {
3168                 session_key = 0;
3169                 return 0;
3170         }
3171
3172         session_key = atoi(key);
3173
3174         return 0;
3175 }
3176
3177 int
3178 main(int argc, char **argv)
3179 {
3180         setlinebuf(stdout);
3181
3182         if (lst_initialize() < 0)
3183                 exit(0);
3184
3185         if (ptl_initialize(argc, argv) < 0)
3186                 exit(0);
3187
3188         Parser_init("lst > ", lst_cmdlist);
3189
3190         if (argc != 1)
3191                 return Parser_execarg(argc - 1, argv + 1, lst_cmdlist);
3192
3193         Parser_commands();
3194
3195         return 0;
3196 }