Whamcloud - gitweb
LU-6068 misc: update old URLs to hpdd.intel.com
[fs/lustre-release.git] / lnet / utils / portals.c
index 110716c..bd670ce 100644 (file)
@@ -1,7 +1,9 @@
 /*
  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  *
- *   This file is part of Portals, http://www.sf.net/projects/lustre/
+ * Copyright (c) 2013, 2014, Intel Corporation.
+ *
+ *   This file is part of Lustre, https://wiki.hpdd.intel.com/
  *
  *   Portals is free software; you can redistribute it and/or
  *   modify it under the terms of version 2 of the GNU General Public
  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  *
  */
-
-#include <libcfs/libcfsutil.h>
-#include <lnet/api-support.h>
+#include <errno.h>
+#include <getopt.h>
+#include <limits.h>
+#include <netdb.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/time.h>
+#include <time.h>
+#include <linux/types.h>
+#include <libcfs/libcfs.h>
+#include <libcfs/util/string.h>
+#include <libcfs/util/ioctl.h>
 #include <lnet/lnetctl.h>
 #include <lnet/socklnd.h>
+#include <lnet/lnet.h>
 
 unsigned int libcfs_debug;
 unsigned int libcfs_printk = D_CANTMASK;
@@ -29,6 +44,10 @@ unsigned int libcfs_printk = D_CANTMASK;
 static int   g_net_set;
 static __u32 g_net;
 
+#define IOC_BUF_SIZE   8192
+static char local_buf[IOC_BUF_SIZE];
+static char *ioc_buf = local_buf;
+
 /* Convert a string boolean to an int; "enable" -> 1 */
 int
 lnet_parse_bool (int *b, char *str)
@@ -152,8 +171,7 @@ ptl_ipaddr_2_str(__u32 ipaddr, char *str, size_t strsize, int lookup)
                 net_ip = htonl (ipaddr);
                 he = gethostbyaddr (&net_ip, sizeof (net_ip), AF_INET);
                 if (he != NULL) {
-                       strncpy(str, he->h_name, strsize - 1);
-                       str[strsize - 1] = '\0';
+                       strlcpy(str, he->h_name, strsize);
                         return (str);
                 }
         }
@@ -566,8 +584,7 @@ jt_ptl_print_peers (int argc, char **argv)
         int                      index;
         int                      rc;
 
-       if (!g_net_is_compatible (argv[0], SOCKLND, RALND, MXLND,
-                                 O2IBLND, GNILND, 0))
+       if (!g_net_is_compatible (argv[0], SOCKLND, O2IBLND, GNILND, 0))
                 return -1;
 
         for (index = 0;;index++) {
@@ -583,7 +600,7 @@ jt_ptl_print_peers (int argc, char **argv)
                         id.nid = data.ioc_nid;
                         id.pid = data.ioc_u32[4];
                         printf ("%-20s [%d]%s->%s:%d #%d\n",
-                                libcfs_id2str(id), 
+                               libcfs_id2str(id),
                                 data.ioc_count, /* persistence */
                                /* my ip */
                                ptl_ipaddr_2_str(data.ioc_u32[2], buffer[0],
@@ -593,14 +610,6 @@ jt_ptl_print_peers (int argc, char **argv)
                                                 sizeof(buffer[1]), 1),
                                 data.ioc_u32[1], /* peer port */
                                 data.ioc_u32[3]); /* conn_count */
-                } else if (g_net_is_compatible(NULL, RALND, 0)) {
-                        printf ("%-20s [%d]@%s:%d\n",
-                                libcfs_nid2str(data.ioc_nid), /* peer nid */
-                                data.ioc_count,   /* peer persistence */
-                               /* peer ip */
-                               ptl_ipaddr_2_str(data.ioc_u32[0], buffer[1],
-                                                sizeof(buffer[1]), 1),
-                                data.ioc_u32[1]); /* peer port */
                } else if (g_net_is_compatible(NULL, GNILND, 0)) {
                        int disconn = data.ioc_flags >> 16;
                        char *state;
@@ -648,8 +657,7 @@ jt_ptl_add_peer (int argc, char **argv)
         int                      port = 0;
         int                      rc;
 
-       if (!g_net_is_compatible (argv[0], SOCKLND, RALND,
-                                 GNILND, 0))
+       if (!g_net_is_compatible(argv[0], SOCKLND, GNILND, 0))
                 return -1;
 
         if (argc != 4) {
@@ -699,8 +707,7 @@ jt_ptl_del_peer (int argc, char **argv)
         __u32                    ip = 0;
         int                      rc;
 
-       if (!g_net_is_compatible (argv[0], SOCKLND, RALND, MXLND,
-                                 O2IBLND, GNILND, 0))
+       if (!g_net_is_compatible(argv[0], SOCKLND, O2IBLND, GNILND, 0))
                 return -1;
 
         if (g_net_is_compatible(NULL, SOCKLND, 0)) {
@@ -754,8 +761,7 @@ jt_ptl_print_connections (int argc, char **argv)
         int                      index;
         int                      rc;
 
-       if (!g_net_is_compatible (argv[0], SOCKLND, RALND, MXLND, O2IBLND,
-                                 GNILND, 0))
+       if (!g_net_is_compatible(argv[0], SOCKLND, O2IBLND, GNILND, 0))
                 return -1;
 
         for (index = 0; ; index++) {
@@ -787,10 +793,6 @@ jt_ptl_print_connections (int argc, char **argv)
                                 data.ioc_count, /* tx buffer size */
                                 data.ioc_u32[5], /* rx buffer size */
                                 data.ioc_flags ? "nagle" : "nonagle");
-                } else if (g_net_is_compatible (NULL, RALND, 0)) {
-                        printf ("%-20s [%d]\n",
-                                libcfs_nid2str(data.ioc_nid),
-                                data.ioc_u32[0] /* device id */);
                 } else if (g_net_is_compatible (NULL, O2IBLND, 0)) {
                         printf ("%s mtu %d\n",
                                 libcfs_nid2str(data.ioc_nid),
@@ -828,8 +830,7 @@ int jt_ptl_disconnect(int argc, char **argv)
                 return 0;
         }
 
-       if (!g_net_is_compatible (NULL, SOCKLND, RALND, MXLND, O2IBLND,
-                                 GNILND, 0))
+       if (!g_net_is_compatible(NULL, SOCKLND, O2IBLND, GNILND, 0))
                 return 0;
 
         if (argc >= 2 &&
@@ -894,49 +895,6 @@ int jt_ptl_push_connection (int argc, char **argv)
         return 0;
 }
 
-int
-jt_ptl_print_active_txs (int argc, char **argv)
-{
-        struct libcfs_ioctl_data data;
-        int                      index;
-        int                      rc;
-
-        if (!g_net_is_compatible (argv[0], QSWLND, 0))
-                return -1;
-
-        for (index = 0;;index++) {
-                LIBCFS_IOC_INIT(data);
-                data.ioc_net   = g_net;
-                data.ioc_count = index;
-
-                rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_GET_TXDESC, &data);
-                if (rc != 0)
-                        break;
-
-                printf ("type %u payload %6d to %s via %s by pid %6d: "
-                        "%s, %s, state %d\n",
-                        data.ioc_u32[0],
-                        data.ioc_count,
-                        libcfs_nid2str(data.ioc_nid),
-                        libcfs_nid2str(data.ioc_u64[0]),
-                        data.ioc_u32[1],
-                        (data.ioc_flags & 1) ? "delayed" : "immediate",
-                        (data.ioc_flags & 2) ? "nblk"    : "normal",
-                        data.ioc_flags >> 2);
-        }
-
-        if (index == 0) {
-                if (errno == ENOENT) {
-                        printf ("<no active descs>\n");
-                } else {
-                        fprintf(stderr, "Error getting active transmits list: "
-                                "%s: check dmesg.\n",
-                                strerror(errno));
-                }
-        }
-        return 0;
-}
-
 int jt_ptl_ping(int argc, char **argv)
 {
         int                      rc;
@@ -1049,7 +1007,7 @@ jt_ptl_fail_nid (int argc, char **argv)
 {
         int                      rc;
         lnet_nid_t               nid;
-        unsigned int             threshold;
+       int                      threshold;
         struct libcfs_ioctl_data data;
 
         if (argc < 2 || argc > 3)
@@ -1066,7 +1024,7 @@ jt_ptl_fail_nid (int argc, char **argv)
 
         if (argc < 3) {
                 threshold = LNET_MD_THRESH_INF;
-        } else if (sscanf (argv[2], "%i", &threshold) != 1) {
+       } else if (sscanf(argv[2], "%i", &threshold) != 1) {
                 fprintf (stderr, "Can't parse count \"%s\"\n", argv[2]);
                 return (-1);
         }
@@ -1088,12 +1046,12 @@ jt_ptl_fail_nid (int argc, char **argv)
 int
 jt_ptl_add_route (int argc, char **argv)
 {
-        struct libcfs_ioctl_data data;
-        lnet_nid_t               gateway_nid;
-        unsigned int             hops = 1;
+       struct lnet_ioctl_config_data data;
+       lnet_nid_t               gateway_nid;
+       unsigned int             hops = 1;
        unsigned int             priority = 0;
-        char                    *end;
-        int                      rc;
+       char                    *end;
+       int                      rc;
 
        if (argc < 2 || argc > 4) {
                fprintf(stderr, "usage: %s gateway [hopcount [priority]]\n",
@@ -1101,14 +1059,14 @@ jt_ptl_add_route (int argc, char **argv)
                return -1;
        }
 
-        if (!g_net_is_set(argv[0]))
-                return (-1);
+       if (g_net_is_set(argv[0]) == 0)
+               return -1;
 
-        gateway_nid = libcfs_str2nid(argv[1]);
-        if (gateway_nid == LNET_NID_ANY) {
-                fprintf (stderr, "Can't parse gateway NID \"%s\"\n", argv[1]);
-                return (-1);
-        }
+       gateway_nid = libcfs_str2nid(argv[1]);
+       if (gateway_nid == LNET_NID_ANY) {
+               fprintf(stderr, "Can't parse gateway NID \"%s\"\n", argv[1]);
+               return -1;
+       }
 
        if (argc > 2) {
                hops = strtoul(argv[2], &end, 0);
@@ -1128,51 +1086,52 @@ jt_ptl_add_route (int argc, char **argv)
                }
        }
 
-       LIBCFS_IOC_INIT(data);
-       data.ioc_net = g_net;
-       data.ioc_count = hops;
-       data.ioc_nid = gateway_nid;
-       data.ioc_priority = priority;
+       LIBCFS_IOC_INIT_V2(data, cfg_hdr);
+       data.cfg_net = g_net;
+       data.cfg_config_u.cfg_route.rtr_hop = hops;
+       data.cfg_nid = gateway_nid;
+       data.cfg_config_u.cfg_route.rtr_priority = priority;
 
-        rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_ADD_ROUTE, &data);
-        if (rc != 0) {
-                fprintf (stderr, "IOC_LIBCFS_ADD_ROUTE failed: %s\n", strerror (errno));
-                return (-1);
-        }
+       rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_ADD_ROUTE, &data);
+       if (rc != 0) {
+               fprintf(stderr, "IOC_LIBCFS_ADD_ROUTE failed: %s\n",
+                       strerror(errno));
+               return -1;
+       }
 
-        return (0);
+       return 0;
 }
 
 int
 jt_ptl_del_route (int argc, char **argv)
 {
-        struct libcfs_ioctl_data data;
-        lnet_nid_t               nid;
-        int                      rc;
+       struct lnet_ioctl_config_data data;
+       lnet_nid_t               nid;
+       int                      rc;
 
-        if (argc != 2) {
-                fprintf (stderr, "usage: %s gatewayNID\n", argv[0]);
-                return (0);
-        }
+       if (argc != 2) {
+               fprintf(stderr, "usage: %s gatewayNID\n", argv[0]);
+               return 0;
+       }
 
-        if (!libcfs_str2anynid(&nid, argv[1])) {
-                fprintf (stderr, "Can't parse gateway NID "
-                         "\"%s\"\n", argv[1]);
-                return -1;
-        }
+       if (libcfs_str2anynid(&nid, argv[1]) == 0) {
+               fprintf(stderr, "Can't parse gateway NID "
+                       "\"%s\"\n", argv[1]);
+               return -1;
+       }
 
-        LIBCFS_IOC_INIT(data);
-        data.ioc_net = g_net_set ? g_net : LNET_NIDNET(LNET_NID_ANY);
-        data.ioc_nid = nid;
+       LIBCFS_IOC_INIT_V2(data, cfg_hdr);
+       data.cfg_net = g_net_set ? g_net : LNET_NIDNET(LNET_NID_ANY);
+       data.cfg_nid = nid;
 
-        rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_DEL_ROUTE, &data);
-        if (rc != 0) {
-                fprintf (stderr, "IOC_LIBCFS_DEL_ROUTE (%s) failed: %s\n",
-                         libcfs_nid2str(nid), strerror (errno));
-                return (-1);
-        }
+       rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_DEL_ROUTE, &data);
+       if (rc != 0) {
+               fprintf(stderr, "IOC_LIBCFS_DEL_ROUTE (%s) failed: %s\n",
+                       libcfs_nid2str(nid), strerror(errno));
+               return -1;
+       }
 
-        return (0);
+       return 0;
 }
 
 int
@@ -1237,40 +1196,454 @@ jt_ptl_notify_router (int argc, char **argv)
 int
 jt_ptl_print_routes (int argc, char **argv)
 {
-        struct libcfs_ioctl_data  data;
-        int                       rc;
-        int                       index;
-        __u32                     net;
-        lnet_nid_t                nid;
-        unsigned int              hops;
-        int                       alive;
+       struct lnet_ioctl_config_data  data;
+       int                       rc;
+       int                       index;
+       __u32                     net;
+       lnet_nid_t                nid;
+       unsigned int              hops;
+       int                       alive;
        unsigned int              pri;
 
-        for (index = 0;;index++)
-        {
-                LIBCFS_IOC_INIT(data);
-                data.ioc_count = index;
+       for (index = 0; ; index++) {
+               LIBCFS_IOC_INIT_V2(data, cfg_hdr);
+               data.cfg_count = index;
 
-                rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_GET_ROUTE, &data);
-                if (rc != 0)
-                        break;
+               rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_GET_ROUTE, &data);
+               if (rc != 0)
+                       break;
 
-                net     = data.ioc_net;
-                hops    = data.ioc_count;
-                nid     = data.ioc_nid;
-                alive   = data.ioc_flags;
-               pri     = data.ioc_priority;
+               net     = data.cfg_net;
+               hops    = data.cfg_config_u.cfg_route.rtr_hop;
+               nid     = data.cfg_nid;
+               alive   = data.cfg_config_u.cfg_route.rtr_flags;
+               pri     = data.cfg_config_u.cfg_route.rtr_priority;
 
                printf("net %18s hops %u gw %32s %s pri %u\n",
                       libcfs_net2str(net), hops,
                       libcfs_nid2str(nid), alive ? "up" : "down", pri);
-        }
+       }
 
-        if (errno != ENOENT)
-                fprintf(stderr, "Error getting routes: %s: check dmesg.\n",
-                        strerror(errno));
+       if (errno != ENOENT)
+               fprintf(stderr, "Error getting routes: %s: check dmesg.\n",
+                       strerror(errno));
 
-        return (0);
+       return 0;
+}
+
+static int
+fault_attr_nid_parse(char *str, lnet_nid_t *nid_p)
+{
+       lnet_nid_t nid;
+       __u32      net;
+       int        rc = 0;
+
+       /* NB: can't support range ipaddress except * and *@net */
+       if (strlen(str) > 2 && str[0] == '*' && str[1] == '@') {
+               net = libcfs_str2net(str + 2);
+               if (net == LNET_NIDNET(LNET_NID_ANY))
+                       goto failed;
+
+               nid = LNET_MKNID(net, LNET_NIDADDR(LNET_NID_ANY));
+       } else {
+               rc = libcfs_str2anynid(&nid, str);
+               if (!rc)
+                       goto failed;
+       }
+
+       *nid_p = nid;
+       return 0;
+failed:
+       fprintf(stderr, "Invalid NID : %s\n", str);
+       return -1;
+}
+
+static int
+fault_attr_msg_parse(char *msg_str, __u32 *mask_p)
+{
+       if (!strcasecmp(msg_str, "put")) {
+               *mask_p |= LNET_PUT_BIT;
+               return 0;
+
+       } else if (!strcasecmp(msg_str, "ack")) {
+               *mask_p |= LNET_ACK_BIT;
+               return 0;
+
+       } else if (!strcasecmp(msg_str, "get")) {
+               *mask_p |= LNET_GET_BIT;
+               return 0;
+
+       } else if (!strcasecmp(msg_str, "reply")) {
+               *mask_p |= LNET_REPLY_BIT;
+               return 0;
+       }
+
+       fprintf(stderr, "unknown message type %s\n", msg_str);
+       return -1;
+}
+
+static int
+fault_attr_ptl_parse(char *ptl_str, __u64 *mask_p)
+{
+       unsigned long rc = strtoul(optarg, NULL, 0);
+
+       if (rc >= 64) {
+               fprintf(stderr, "invalid portal: %lu\n", rc);
+               return -1;
+       }
+
+       *mask_p |= (1ULL << rc);
+       return 0;
+}
+
+static int
+fault_simul_rule_add(__u32 opc, char *name, int argc, char **argv)
+{
+       struct libcfs_ioctl_data  data = {{0}};
+       struct lnet_fault_attr    attr;
+       char                     *optstr;
+       int                       rc;
+
+       static struct option opts[] = {
+               {"source",      required_argument,      0,      's'},
+               {"dest",        required_argument,      0,      'd'},
+               {"rate",        required_argument,      0,      'r'},
+               {"interval",    required_argument,      0,      'i'},
+               {"latency",     required_argument,      0,      'l'},
+               {"portal",      required_argument,      0,      'p'},
+               {"message",     required_argument,      0,      'm'},
+               {0, 0, 0, 0}
+       };
+
+       if (argc == 1) {
+               fprintf(stderr, "Failed, please provide source, destination "
+                               "and rate of rule\n");
+               return -1;
+       }
+
+       optstr = opc == LNET_CTL_DROP_ADD ? "s:d:r:i:p:m:" : "s:d:r:l:p:m:";
+       memset(&attr, 0, sizeof(attr));
+       while (1) {
+               char c = getopt_long(argc, argv, optstr, opts, NULL);
+
+               if (c == -1)
+                       break;
+
+               switch (c) {
+               case 's': /* source NID/NET */
+                       rc = fault_attr_nid_parse(optarg, &attr.fa_src);
+                       if (rc != 0)
+                               goto getopt_failed;
+                       break;
+
+               case 'd': /* dest NID/NET */
+                       rc = fault_attr_nid_parse(optarg, &attr.fa_dst);
+                       if (rc != 0)
+                               goto getopt_failed;
+                       break;
+
+               case 'r': /* drop rate */
+                       if (opc == LNET_CTL_DROP_ADD)
+                               attr.u.drop.da_rate = strtoul(optarg, NULL, 0);
+                       else
+                               attr.u.delay.la_rate = strtoul(optarg, NULL, 0);
+                       break;
+
+               case 'i': /* time interval (# seconds) for message drop */
+                       if (opc == LNET_CTL_DROP_ADD)
+                               attr.u.drop.da_interval = strtoul(optarg,
+                                                                 NULL, 0);
+                       else
+                               attr.u.delay.la_interval = strtoul(optarg,
+                                                                  NULL, 0);
+                       break;
+
+               case 'l': /* seconds to wait before activating rule */
+                       attr.u.delay.la_latency = strtoul(optarg, NULL, 0);
+                       break;
+
+               case 'p': /* portal to filter */
+                       rc = fault_attr_ptl_parse(optarg, &attr.fa_ptl_mask);
+                       if (rc != 0)
+                               goto getopt_failed;
+                       break;
+
+               case 'm': /* message types to filter */
+                       rc = fault_attr_msg_parse(optarg, &attr.fa_msg_mask);
+                       if (rc != 0)
+                               goto getopt_failed;
+                       break;
+
+               default:
+                       fprintf(stderr, "error: %s: option '%s' "
+                               "unrecognized\n", argv[0], argv[optind - 1]);
+                       goto getopt_failed;
+               }
+       }
+       optind = 1;
+
+       if (opc == LNET_CTL_DROP_ADD) {
+               /* NB: drop rate and interval are exclusive to each other */
+               if (!((attr.u.drop.da_rate == 0) ^
+                     (attr.u.drop.da_interval == 0))) {
+                       fprintf(stderr,
+                               "please provide either drop rate or interval "
+                               "but not both at the same time.\n");
+                       return -1;
+               }
+       } else if (opc == LNET_CTL_DELAY_ADD) {
+               if (!((attr.u.delay.la_rate == 0) ^
+                     (attr.u.delay.la_interval == 0))) {
+                       fprintf(stderr,
+                               "please provide either delay rate or interval "
+                               "but not both at the same time.\n");
+                       return -1;
+               }
+
+               if (attr.u.delay.la_latency == 0) {
+                       fprintf(stderr, "latency cannot be zero\n");
+                       return -1;
+               }
+       }
+
+       if (attr.fa_src == 0 || attr.fa_dst == 0) {
+               fprintf(stderr, "Please provide both source and destination "
+                               "of %s rule\n", name);
+               return -1;
+       }
+
+       data.ioc_flags = opc;
+       data.ioc_inllen1 = sizeof(attr);
+       data.ioc_inlbuf1 = (char *)&attr;
+       if (libcfs_ioctl_pack(&data, &ioc_buf, IOC_BUF_SIZE) != 0) {
+               fprintf(stderr, "libcfs_ioctl_pack failed\n");
+               return -1;
+       }
+
+       rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNET_FAULT, ioc_buf);
+       if (rc != 0) {
+               fprintf(stderr, "add %s rule %s->%s failed: %s\n",
+                       name, libcfs_nid2str(attr.fa_src),
+                       libcfs_nid2str(attr.fa_dst), strerror(errno));
+               return -1;
+       }
+
+       printf("Added %s rule %s->%s (1/%d)\n",
+              name, libcfs_nid2str(attr.fa_src), libcfs_nid2str(attr.fa_dst),
+              opc == LNET_CTL_DROP_ADD ?
+              attr.u.drop.da_rate : attr.u.delay.la_rate);
+       return 0;
+
+getopt_failed:
+       optind = 1;
+       return -1;
+}
+
+int
+jt_ptl_drop_add(int argc, char **argv)
+{
+       return fault_simul_rule_add(LNET_CTL_DROP_ADD, "drop", argc, argv);
+}
+
+int
+jt_ptl_delay_add(int argc, char **argv)
+{
+       return fault_simul_rule_add(LNET_CTL_DELAY_ADD, "delay", argc, argv);
+}
+
+static int
+fault_simul_rule_del(__u32 opc, char *name, int argc, char **argv)
+{
+       struct libcfs_ioctl_data data = {{0}};
+       struct lnet_fault_attr   attr;
+       bool                     all = false;
+       int                      rc;
+
+       static struct option opts[] = {
+               {"source",      required_argument,      0,      's'},
+               {"dest",        required_argument,      0,      'd'},
+               {"all",         no_argument,            0,      'a'},
+               {0, 0, 0, 0}
+       };
+
+       if (argc == 1) {
+               fprintf(stderr, "Failed, please provide source and "
+                               "destination of rule\n");
+               return -1;
+       }
+
+       memset(&attr, 0, sizeof(attr));
+       while (1) {
+               char c = getopt_long(argc, argv, "s:d:a", opts, NULL);
+
+               if (c == -1 || all)
+                       break;
+
+               switch (c) {
+               case 's':
+                       rc = fault_attr_nid_parse(optarg, &attr.fa_src);
+                       if (rc != 0)
+                               goto getopt_failed;
+                       break;
+               case 'd':
+                       rc = fault_attr_nid_parse(optarg, &attr.fa_dst);
+                       if (rc != 0)
+                               goto getopt_failed;
+                       break;
+               case 'a':
+                       attr.fa_src = attr.fa_dst = 0;
+                       all = true;
+                       break;
+               default:
+                       fprintf(stderr, "error: %s: option '%s' "
+                               "unrecognized\n", argv[0], argv[optind - 1]);
+                       goto getopt_failed;
+               }
+       }
+       optind = 1;
+
+       data.ioc_flags = opc;
+       data.ioc_inllen1 = sizeof(attr);
+       data.ioc_inlbuf1 = (char *)&attr;
+       if (libcfs_ioctl_pack(&data, &ioc_buf, IOC_BUF_SIZE) != 0) {
+               fprintf(stderr, "libcfs_ioctl_pack failed\n");
+               return -1;
+       }
+
+       rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNET_FAULT, ioc_buf);
+       if (rc != 0) {
+               fprintf(stderr, "remove %s rule %s->%s failed: %s\n", name,
+                       all ? "all" : libcfs_nid2str(attr.fa_src),
+                       all ? "all" : libcfs_nid2str(attr.fa_dst),
+                       strerror(errno));
+               return -1;
+       }
+
+       libcfs_ioctl_unpack(&data, ioc_buf);
+       printf("Removed %d %s rules\n", data.ioc_count, name);
+       return 0;
+
+getopt_failed:
+       optind = 1;
+       return -1;
+}
+
+int
+jt_ptl_drop_del(int argc, char **argv)
+{
+       return fault_simul_rule_del(LNET_CTL_DROP_DEL, "drop", argc, argv);
+}
+
+int
+jt_ptl_delay_del(int argc, char **argv)
+{
+       return fault_simul_rule_del(LNET_CTL_DELAY_DEL, "delay", argc, argv);
+}
+
+static int
+fault_simul_rule_reset(__u32 opc, char *name, int argc, char **argv)
+{
+       struct libcfs_ioctl_data   data = {{0}};
+       int                        rc;
+
+       LIBCFS_IOC_INIT(data);
+       data.ioc_flags = opc;
+
+       rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNET_FAULT, &data);
+       if (rc != 0) {
+               fprintf(stderr, "failed to reset %s stats: %s\n",
+                       name, strerror(errno));
+               return -1;
+       }
+       return 0;
+}
+
+int
+jt_ptl_drop_reset(int argc, char **argv)
+{
+       return fault_simul_rule_reset(LNET_CTL_DROP_RESET, "drop", argc, argv);
+}
+
+int
+jt_ptl_delay_reset(int argc, char **argv)
+{
+       return fault_simul_rule_reset(LNET_CTL_DELAY_RESET, "delay",
+                                     argc, argv);
+}
+
+static int
+fault_simul_rule_list(__u32 opc, char *name, int argc, char **argv)
+{
+       struct libcfs_ioctl_data data = {{0}};
+       struct lnet_fault_attr   attr;
+       struct lnet_fault_stat   stat;
+       int                      pos;
+
+       printf("LNet %s rules:\n", name);
+       for (pos = 0;; pos++) {
+               int             rc;
+
+               memset(&attr, 0, sizeof(attr));
+               memset(&stat, 0, sizeof(stat));
+
+               data.ioc_count = pos;
+               data.ioc_flags = opc;
+               data.ioc_inllen1 = sizeof(attr);
+               data.ioc_inlbuf1 = (char *)&attr;
+               data.ioc_inllen2 = sizeof(stat);
+               data.ioc_inlbuf2 = (char *)&stat;
+               if (libcfs_ioctl_pack(&data, &ioc_buf, IOC_BUF_SIZE) != 0) {
+                       fprintf(stderr, "libcfs_ioctl_pack failed\n");
+                       return -1;
+               }
+
+               rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNET_FAULT, ioc_buf);
+               if (rc != 0)
+                       break;
+
+               libcfs_ioctl_unpack(&data, ioc_buf);
+
+               if (opc == LNET_CTL_DROP_LIST) {
+                       printf("%s->%s (1/%d | %d) ptl "LPX64", msg %x, "
+                              LPU64"/"LPU64", PUT "LPU64", ACK "LPU64", GET "
+                              LPU64", REP "LPU64"\n",
+                              libcfs_nid2str(attr.fa_src),
+                              libcfs_nid2str(attr.fa_dst),
+                              attr.u.drop.da_rate, attr.u.drop.da_interval,
+                              attr.fa_ptl_mask, attr.fa_msg_mask,
+                              stat.u.drop.ds_dropped, stat.fs_count,
+                              stat.fs_put, stat.fs_ack,
+                              stat.fs_get, stat.fs_reply);
+
+               } else if (opc == LNET_CTL_DELAY_LIST) {
+                       printf("%s->%s (1/%d | %d, latency %d) ptl "LPX64
+                              ", msg %x, "LPU64"/"LPU64", PUT "LPU64
+                              ", ACK "LPU64", GET "LPU64", REP "LPU64"\n",
+                              libcfs_nid2str(attr.fa_src),
+                              libcfs_nid2str(attr.fa_dst),
+                              attr.u.delay.la_rate, attr.u.delay.la_interval,
+                              attr.u.delay.la_latency,
+                              attr.fa_ptl_mask, attr.fa_msg_mask,
+                              stat.u.delay.ls_delayed, stat.fs_count,
+                              stat.fs_put, stat.fs_ack, stat.fs_get,
+                              stat.fs_reply);
+               }
+       }
+       printf("found total %d\n", pos);
+
+       return 0;
+}
+
+int
+jt_ptl_drop_list(int argc, char **argv)
+{
+       return fault_simul_rule_list(LNET_CTL_DROP_LIST, "drop", argc, argv);
+}
+
+int
+jt_ptl_delay_list(int argc, char **argv)
+{
+       return fault_simul_rule_list(LNET_CTL_DELAY_LIST, "delay", argc, argv);
 }
 
 double