#define IOC_LIBCFS_PING _IOWR('e', 61, IOCTL_LIBCFS_TYPE)
#define IOC_LIBCFS_DEBUG_PEER _IOWR('e', 62, IOCTL_LIBCFS_TYPE)
#define IOC_LIBCFS_LNETST _IOWR('e', 63, IOCTL_LIBCFS_TYPE)
+#define IOC_LIBCFS_LNET_FAULT _IOWR('e', 64, IOCTL_LIBCFS_TYPE)
/* lnd ioctls */
#define IOC_LIBCFS_REGISTER_MYNID _IOWR('e', 70, IOCTL_LIBCFS_TYPE)
#define IOC_LIBCFS_CLOSE_CONNECTION _IOWR('e', 71, IOCTL_LIBCFS_TYPE)
void lnet_print_hdr (lnet_hdr_t * hdr);
int lnet_fail_nid(lnet_nid_t nid, unsigned int threshold);
+/** \addtogroup lnet_fault_simulation @{ */
+
+int lnet_fault_ctl(int cmd, struct libcfs_ioctl_data *data);
+int lnet_fault_init(void);
+void lnet_fault_fini(void);
+
+bool lnet_drop_rule_match(lnet_hdr_t *hdr);
+
+/** @} lnet_fault_simulation */
+
void lnet_counters_get(lnet_counters_t *counters);
void lnet_counters_reset(void);
#include <libcfs/libcfs.h>
#include <libcfs/list.h>
#include <lnet/types.h>
+#include <lnet/lnetctl.h>
#define WIRE_ATTR __attribute__((packed))
struct lnet_peer_table **ln_peer_tables;
/* failure simulation */
struct list_head ln_test_peers;
+ struct list_head ln_drop_rules;
struct list_head ln_nis; /* LND instances */
/* NIs bond on specific CPT(s) */
* along with Portals; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*
- * header for libptlctl.a
+ * header for lnet ioctl
*/
#ifndef _PTLCTL_H_
#define _PTLCTL_H_
#include <libcfs/libcfs.h>
#include <lnet/types.h>
+/** \addtogroup lnet_fault_simulation
+ * @{ */
+
+enum {
+ LNET_CTL_DROP_ADD,
+ LNET_CTL_DROP_DEL,
+ LNET_CTL_DROP_RESET,
+ LNET_CTL_DROP_LIST,
+};
+
+#define LNET_ACK_BIT (1 << 0)
+#define LNET_PUT_BIT (1 << 1)
+#define LNET_GET_BIT (1 << 2)
+#define LNET_REPLY_BIT (1 << 3)
+
+/** ioctl parameter for LNet fault simulation */
+struct lnet_fault_attr {
+ /**
+ * source NID of drop rule
+ * LNET_NID_ANY is wildcard for all sources
+ * 255.255.255.255@net is wildcard for all addresses from @net
+ */
+ lnet_nid_t fa_src;
+ /** destination NID of drop rule, see \a dr_src for details */
+ lnet_nid_t fa_dst;
+ /**
+ * Portal mask to drop, -1 means all portals, for example:
+ * fa_ptl_mask = (1 << _LDLM_CB_REQUEST_PORTAL ) |
+ * (1 << LDLM_CANCEL_REQUEST_PORTAL)
+ *
+ * If it is non-zero then only PUT and GET will be filtered, otherwise
+ * there is no portal filter, all matched messages will be checked.
+ */
+ __u64 fa_ptl_mask;
+ /**
+ * message types to drop, for example:
+ * dra_type = LNET_DROP_ACK_BIT | LNET_DROP_PUT_BIT
+ *
+ * If it is non-zero then only specified message types are filtered,
+ * otherwise all message types will be checked.
+ */
+ __u32 fa_msg_mask;
+ union {
+ /** message drop simulation */
+ struct {
+ /** drop rate of this rule */
+ __u32 da_rate;
+ /**
+ * time interval of message drop, it is exclusive
+ * with da_rate
+ */
+ __u32 da_interval;
+ } drop;
+ /** TODO: add more */
+ __u64 space[8];
+ } u;
+
+};
+
+/** fault simluation stats */
+struct lnet_fault_stat {
+ /** total # matched messages */
+ __u64 fs_count;
+ /** # dropped LNET_MSG_PUT by this rule */
+ __u64 fs_put;
+ /** # dropped LNET_MSG_ACK by this rule */
+ __u64 fs_ack;
+ /** # dropped LNET_MSG_GET by this rule */
+ __u64 fs_get;
+ /** # dropped LNET_MSG_REPLY by this rule */
+ __u64 fs_reply;
+ union {
+ struct {
+ /** total # dropped messages */
+ __u64 ds_dropped;
+ } drop;
+ /** TODO: add more */
+ __u64 space[8];
+ } u;
+};
+
+/** @} lnet_fault_simulation */
+
#define LNET_DEV_ID 0
#define LNET_DEV_PATH "/dev/lnet"
#define LNET_DEV_MAJOR 10
int jt_ptl_fail_nid (int argc, char **argv);
int jt_ptl_testprotocompat(int argc, char **argv);
int jt_ptl_memhog(int argc, char **argv);
+int jt_ptl_drop_add(int argc, char **argv);
+int jt_ptl_drop_del(int argc, char **argv);
+int jt_ptl_drop_reset(int argc, char **argv);
+int jt_ptl_drop_list(int argc, char **argv);
int dbg_initialize(int argc, char **argv);
int jt_dbg_filter(int argc, char **argv);
lnet-objs := api-ni.o config.o
lnet-objs += lib-me.o lib-msg.o lib-eq.o lib-md.o lib-ptl.o
lnet-objs += lib-move.o module.o lo.o
-lnet-objs += router.o router_proc.o acceptor.o peer.o
+lnet-objs += router.o router_proc.o acceptor.o peer.o net_fault.o
default: all
INIT_LIST_HEAD(&the_lnet.ln_nis_cpt);
INIT_LIST_HEAD(&the_lnet.ln_nis_zombie);
INIT_LIST_HEAD(&the_lnet.ln_routers);
+ INIT_LIST_HEAD(&the_lnet.ln_drop_rules);
rc = lnet_create_remote_nets_table();
if (rc != 0)
if (rc != 0)
goto failed4;
+ lnet_fault_init();
lnet_proc_init();
LNET_MUTEX_UNLOCK(&the_lnet.ln_api_mutex);
if (the_lnet.ln_refcount != 1) {
the_lnet.ln_refcount--;
} else {
- LASSERT (!the_lnet.ln_niinit_self);
+ LASSERT(!the_lnet.ln_niinit_self);
+
+ lnet_fault_fini();
lnet_proc_fini();
lnet_router_checker_stop();
lnet_net_unlock(LNET_LOCK_EX);
return 0;
+ case IOC_LIBCFS_LNET_FAULT:
+ return lnet_fault_ctl(data->ioc_flags, data);
+
case IOC_LIBCFS_PING:
id.nid = data->ioc_nid;
id.pid = data->ioc_u32[0];
lib-me.c lib-msg.c lib-eq.c \
lib-md.c lib-ptl.c lib-move.c lo.c \
router.c router_proc.c \
- acceptor.c peer.c
+ acceptor.c peer.c net_fault.c
if LIBLUSTRE
goto drop;
}
+ if (!list_empty(&the_lnet.ln_drop_rules) &&
+ lnet_drop_rule_match(hdr)) {
+ CDEBUG(D_NET, "%s, src %s, dst %s: Dropping %s to simulate"
+ "silent message loss\n",
+ libcfs_nid2str(from_nid), libcfs_nid2str(src_nid),
+ libcfs_nid2str(dest_nid), lnet_msgtyp2str(type));
+ goto drop;
+ }
+
+
msg = lnet_msg_alloc();
if (msg == NULL) {
CERROR("%s, src %s: Dropping %s (out of memory)\n",
--- /dev/null
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 021110-1307, USA
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2014, Intel Corporation.
+ */
+/*
+ * This file is part of Lustre, http://www.lustre.org/
+ * Lustre is a trademark of Sun Microsystems, Inc.
+ *
+ * lnet/lnet/net_fault.c
+ *
+ * Lustre network fault simulation
+ *
+ * Author: liang.zhen@intel.com
+ */
+
+#define DEBUG_SUBSYSTEM S_LNET
+
+#include <lnet/lib-lnet.h>
+#include <lnet/lnetctl.h>
+
+#define LNET_MSG_MASK (LNET_PUT_BIT | LNET_ACK_BIT | \
+ LNET_GET_BIT | LNET_REPLY_BIT)
+
+struct lnet_drop_rule {
+ /** link chain on the_lnet.ln_drop_rules */
+ struct list_head dr_link;
+ /** attributes of this rule */
+ struct lnet_fault_attr dr_attr;
+ /** lock to protect \a dr_drop_at and \a dr_stat */
+ spinlock_t dr_lock;
+ /**
+ * the message sequence to drop, which means message is dropped when
+ * dr_stat.drs_count == dr_drop_at
+ */
+ unsigned long dr_drop_at;
+ /**
+ * seconds to drop the next message, it's exclusive with dr_drop_at
+ */
+ cfs_time_t dr_drop_time;
+ /** baseline to caculate dr_drop_time */
+ cfs_time_t dr_time_base;
+ /** statistic of dropped messages */
+ struct lnet_fault_stat dr_stat;
+};
+
+static bool
+lnet_fault_nid_match(lnet_nid_t nid, lnet_nid_t msg_nid)
+{
+ if (nid == msg_nid || nid == LNET_NID_ANY)
+ return true;
+
+ if (LNET_NIDNET(nid) != LNET_NIDNET(msg_nid))
+ return false;
+
+ /* 255.255.255.255@net is wildcard for all addresses in a network */
+ return LNET_NIDADDR(nid) == LNET_NIDADDR(LNET_NID_ANY);
+}
+
+static bool
+lnet_fault_attr_match(struct lnet_fault_attr *attr, lnet_nid_t src,
+ lnet_nid_t dst, unsigned int type, unsigned int portal)
+{
+ if (!lnet_fault_nid_match(attr->fa_src, src) ||
+ !lnet_fault_nid_match(attr->fa_dst, dst))
+ return false;
+
+ if (!(attr->fa_msg_mask & (1 << type)))
+ return false;
+
+ /* NB: ACK and REPLY have no portal, but they should have been
+ * rejected by message mask */
+ if (attr->fa_ptl_mask != 0 && /* has portal filter */
+ !(attr->fa_ptl_mask & (1ULL << portal)))
+ return false;
+
+ return true;
+}
+
+static int
+lnet_fault_attr_validate(struct lnet_fault_attr *attr)
+{
+ if (attr->fa_msg_mask == 0)
+ attr->fa_msg_mask = LNET_MSG_MASK; /* all message types */
+
+ if (attr->fa_ptl_mask == 0) /* no portal filter */
+ return 0;
+
+ /* NB: only PUT and GET can be filtered if portal filter has been set */
+ attr->fa_msg_mask &= LNET_GET_BIT | LNET_PUT_BIT;
+ if (attr->fa_msg_mask == 0) {
+ CDEBUG(D_NET, "can't find valid message type bits %x\n",
+ attr->fa_msg_mask);
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static void
+lnet_fault_stat_inc(struct lnet_fault_stat *stat, unsigned int type)
+{
+ /* NB: fs_counter is NOT updated by this function */
+ switch (type) {
+ case LNET_MSG_PUT:
+ stat->fs_put++;
+ return;
+ case LNET_MSG_ACK:
+ stat->fs_ack++;
+ return;
+ case LNET_MSG_GET:
+ stat->fs_get++;
+ return;
+ case LNET_MSG_REPLY:
+ stat->fs_reply++;
+ return;
+ }
+}
+
+/**
+ * Add a new drop rule to LNet
+ * There is no check for duplicated drop rule, all rules will be checked for
+ * incoming message.
+ */
+static int
+lnet_drop_rule_add(struct lnet_fault_attr *attr)
+{
+ struct lnet_drop_rule *rule;
+ ENTRY;
+
+ if ((attr->u.drop.da_rate == 0) == (attr->u.drop.da_interval == 0)) {
+ CDEBUG(D_NET, "invalid rate %d or interval %d\n",
+ attr->u.drop.da_rate, attr->u.drop.da_interval);
+ RETURN(-EINVAL);
+ }
+
+ if (lnet_fault_attr_validate(attr) != 0)
+ RETURN(-EINVAL);
+
+ CFS_ALLOC_PTR(rule);
+ if (rule == NULL)
+ RETURN(-ENOMEM);
+
+ spin_lock_init(&rule->dr_lock);
+
+ rule->dr_attr = *attr;
+ if (attr->u.drop.da_interval != 0) {
+ rule->dr_time_base = cfs_time_shift(attr->u.drop.da_interval);
+ rule->dr_drop_time = cfs_time_shift(cfs_rand() %
+ attr->u.drop.da_interval);
+ } else {
+ rule->dr_drop_at = cfs_rand() % attr->u.drop.da_rate;
+ }
+
+ lnet_net_lock(LNET_LOCK_EX);
+ list_add(&rule->dr_link, &the_lnet.ln_drop_rules);
+ lnet_net_unlock(LNET_LOCK_EX);
+
+ CDEBUG(D_NET, "Added drop rule: src %s, dst %s, rate %d, interval %d\n",
+ libcfs_nid2str(attr->fa_src), libcfs_nid2str(attr->fa_src),
+ attr->u.drop.da_rate, attr->u.drop.da_interval);
+ RETURN(0);
+}
+
+/**
+ * Remove matched drop rules from lnet, all rules that can match \a src and
+ * \a dst will be removed.
+ * If \a src is zero, then all rules have \a dst as destination will be remove
+ * If \a dst is zero, then all rules have \a src as source will be removed
+ * If both of them are zero, all rules will be removed
+ */
+static int
+lnet_drop_rule_del(lnet_nid_t src, lnet_nid_t dst)
+{
+ struct lnet_drop_rule *rule;
+ struct lnet_drop_rule *tmp;
+ struct list_head zombies;
+ int n = 0;
+ ENTRY;
+
+ INIT_LIST_HEAD(&zombies);
+
+ lnet_net_lock(LNET_LOCK_EX);
+ list_for_each_entry_safe(rule, tmp, &the_lnet.ln_drop_rules, dr_link) {
+ if (rule->dr_attr.fa_src != src && src != 0)
+ continue;
+
+ if (rule->dr_attr.fa_dst != dst && dst != 0)
+ continue;
+
+ list_move(&rule->dr_link, &zombies);
+ }
+ lnet_net_unlock(LNET_LOCK_EX);
+
+ list_for_each_entry_safe(rule, tmp, &zombies, dr_link) {
+ CDEBUG(D_NET, "Remove drop rule: src %s->dst: %s (1/%d, %d)\n",
+ libcfs_nid2str(rule->dr_attr.fa_src),
+ libcfs_nid2str(rule->dr_attr.fa_dst),
+ rule->dr_attr.u.drop.da_rate,
+ rule->dr_attr.u.drop.da_interval);
+
+ list_del(&rule->dr_link);
+ CFS_FREE_PTR(rule);
+ n++;
+ }
+
+ RETURN(n);
+}
+
+/**
+ * List drop rule at position of \a pos
+ */
+static int
+lnet_drop_rule_list(int pos, struct lnet_fault_attr *attr,
+ struct lnet_fault_stat *stat)
+{
+ struct lnet_drop_rule *rule;
+ int cpt;
+ int i = 0;
+ int rc = -ENOENT;
+ ENTRY;
+
+ cpt = lnet_net_lock_current();
+ list_for_each_entry(rule, &the_lnet.ln_drop_rules, dr_link) {
+ if (i++ < pos)
+ continue;
+
+ spin_lock(&rule->dr_lock);
+ *attr = rule->dr_attr;
+ *stat = rule->dr_stat;
+ spin_unlock(&rule->dr_lock);
+ rc = 0;
+ break;
+ }
+
+ lnet_net_unlock(cpt);
+ RETURN(rc);
+}
+
+/**
+ * reset counters for all drop rules
+ */
+static void
+lnet_drop_rule_reset(void)
+{
+ struct lnet_drop_rule *rule;
+ int cpt;
+ ENTRY;
+
+ cpt = lnet_net_lock_current();
+
+ list_for_each_entry(rule, &the_lnet.ln_drop_rules, dr_link) {
+ struct lnet_fault_attr *attr = &rule->dr_attr;
+
+ spin_lock(&rule->dr_lock);
+
+ memset(&rule->dr_stat, 0, sizeof(rule->dr_stat));
+ if (attr->u.drop.da_rate != 0) {
+ rule->dr_drop_at = cfs_rand() % attr->u.drop.da_rate;
+ } else {
+ rule->dr_drop_time = cfs_time_shift(cfs_rand() %
+ attr->u.drop.da_interval);
+ rule->dr_time_base = cfs_time_shift(attr->u.drop.
+ da_interval);
+ }
+ spin_unlock(&rule->dr_lock);
+ }
+
+ lnet_net_unlock(cpt);
+ EXIT;
+}
+
+/**
+ * check source/destination NID, portal, message type and drop rate,
+ * decide whether should drop this message or not
+ */
+static bool
+drop_rule_match(struct lnet_drop_rule *rule, lnet_nid_t src,
+ lnet_nid_t dst, unsigned int type, unsigned int portal)
+{
+ struct lnet_fault_attr *attr = &rule->dr_attr;
+ bool drop;
+
+ if (!lnet_fault_attr_match(attr, src, dst, type, portal))
+ return false;
+
+ /* match this rule, check drop rate now */
+ spin_lock(&rule->dr_lock);
+ if (rule->dr_drop_time != 0) { /* time based drop */
+ cfs_time_t now = cfs_time_current();
+
+ rule->dr_stat.fs_count++;
+ drop = cfs_time_aftereq(now, rule->dr_drop_time);
+ if (drop) {
+ if (cfs_time_after(now, rule->dr_time_base))
+ rule->dr_time_base = now;
+
+ rule->dr_drop_time = rule->dr_time_base +
+ cfs_time_seconds(cfs_rand() %
+ attr->u.drop.da_interval);
+ rule->dr_time_base += cfs_time_seconds(attr->u.drop.
+ da_interval);
+
+ CDEBUG(D_NET, "Drop Rule %s->%s: next drop : "
+ CFS_TIME_T"\n",
+ libcfs_nid2str(attr->fa_src),
+ libcfs_nid2str(attr->fa_dst),
+ rule->dr_drop_time);
+ }
+
+ } else { /* rate based drop */
+ drop = rule->dr_stat.fs_count++ == rule->dr_drop_at;
+
+ if (rule->dr_stat.fs_count % attr->u.drop.da_rate == 0) {
+ rule->dr_drop_at = rule->dr_stat.fs_count +
+ cfs_rand() % attr->u.drop.da_rate;
+ CDEBUG(D_NET, "Drop Rule %s->%s: next drop: %lu\n",
+ libcfs_nid2str(attr->fa_src),
+ libcfs_nid2str(attr->fa_dst), rule->dr_drop_at);
+ }
+ }
+
+ if (drop) { /* drop this message, update counters */
+ lnet_fault_stat_inc(&rule->dr_stat, type);
+ rule->dr_stat.u.drop.ds_dropped++;
+ }
+
+ spin_unlock(&rule->dr_lock);
+ return drop;
+}
+
+/**
+ * Check if message from \a src to \a dst can match any existed drop rule
+ */
+bool
+lnet_drop_rule_match(lnet_hdr_t *hdr)
+{
+ struct lnet_drop_rule *rule;
+ lnet_nid_t src = le64_to_cpu(hdr->src_nid);
+ lnet_nid_t dst = le64_to_cpu(hdr->dest_nid);
+ unsigned int typ = le32_to_cpu(hdr->type);
+ unsigned int ptl = -1;
+ bool drop = false;
+ int cpt;
+
+ /* NB: if Portal is specified, then only PUT and GET will be
+ * filtered by drop rule */
+ if (typ == LNET_MSG_PUT)
+ ptl = le32_to_cpu(hdr->msg.put.ptl_index);
+ else if (typ == LNET_MSG_GET)
+ ptl = le32_to_cpu(hdr->msg.get.ptl_index);
+
+ cpt = lnet_net_lock_current();
+ list_for_each_entry(rule, &the_lnet.ln_drop_rules, dr_link) {
+ drop = drop_rule_match(rule, src, dst, typ, ptl);
+ if (drop)
+ break;
+ }
+
+ lnet_net_unlock(cpt);
+ return drop;
+}
+
+int
+lnet_fault_ctl(int opc, struct libcfs_ioctl_data *data)
+{
+ struct lnet_fault_attr *attr;
+ struct lnet_fault_stat *stat;
+
+ attr = (struct lnet_fault_attr *)data->ioc_inlbuf1;
+
+ switch (opc) {
+ default:
+ return -EINVAL;
+
+ case LNET_CTL_DROP_ADD:
+ if (attr == NULL)
+ return -EINVAL;
+
+ return lnet_drop_rule_add(attr);
+
+ case LNET_CTL_DROP_DEL:
+ if (attr == NULL)
+ return -EINVAL;
+
+ data->ioc_count = lnet_drop_rule_del(attr->fa_src,
+ attr->fa_dst);
+ return 0;
+
+ case LNET_CTL_DROP_RESET:
+ lnet_drop_rule_reset();
+ return 0;
+
+ case LNET_CTL_DROP_LIST:
+ stat = (struct lnet_fault_stat *)data->ioc_inlbuf2;
+ if (attr == NULL || stat == NULL)
+ return -EINVAL;
+
+ return lnet_drop_rule_list(data->ioc_count, attr, stat);
+ }
+}
+
+int
+lnet_fault_init(void)
+{
+ CLASSERT(LNET_PUT_BIT == 1 << LNET_MSG_PUT);
+ CLASSERT(LNET_ACK_BIT == 1 << LNET_MSG_ACK);
+ CLASSERT(LNET_GET_BIT == 1 << LNET_MSG_GET);
+ CLASSERT(LNET_REPLY_BIT == 1 << LNET_MSG_REPLY);
+
+ return 0;
+}
+
+void
+lnet_fault_fini(void)
+{
+ lnet_drop_rule_del(0, 0);
+
+ LASSERT(list_empty(&the_lnet.ln_drop_rules));
+}
static int g_net_set;
static __u32 g_net;
+#define IOC_BUF_SIZE 8192
+static char local_buf[IOC_BUF_SIZE];
+static char *ioc_buf = local_buf;
+
/* Convert a string boolean to an int; "enable" -> 1 */
int
lnet_parse_bool (int *b, char *str)
return 0;
}
+static int
+fault_attr_nid_parse(char *str, lnet_nid_t *nid_p)
+{
+ lnet_nid_t nid;
+ __u32 net;
+ int rc = 0;
+
+ /* NB: can't support range ipaddress except * and *@net */
+ if (strlen(str) > 2 && str[0] == '*' && str[1] == '@') {
+ net = libcfs_str2net(str + 2);
+ if (net == LNET_NIDNET(LNET_NID_ANY))
+ goto failed;
+
+ nid = LNET_MKNID(net, LNET_NIDADDR(LNET_NID_ANY));
+ } else {
+ rc = libcfs_str2anynid(&nid, str);
+ if (!rc)
+ goto failed;
+ }
+
+ *nid_p = nid;
+ return 0;
+failed:
+ fprintf(stderr, "Invalid NID : %s\n", str);
+ return -1;
+}
+
+static int
+fault_attr_msg_parse(char *msg_str, __u32 *mask_p)
+{
+ if (!strcasecmp(msg_str, "put")) {
+ *mask_p |= LNET_PUT_BIT;
+ return 0;
+
+ } else if (!strcasecmp(msg_str, "ack")) {
+ *mask_p |= LNET_ACK_BIT;
+ return 0;
+
+ } else if (!strcasecmp(msg_str, "get")) {
+ *mask_p |= LNET_GET_BIT;
+ return 0;
+
+ } else if (!strcasecmp(msg_str, "reply")) {
+ *mask_p |= LNET_REPLY_BIT;
+ return 0;
+ }
+
+ fprintf(stderr, "unknown message type %s\n", msg_str);
+ return -1;
+}
+
+static int
+fault_attr_ptl_parse(char *ptl_str, __u64 *mask_p)
+{
+ unsigned long rc = strtoul(optarg, NULL, 0);
+
+ if (rc >= 64) {
+ fprintf(stderr, "invalid portal: %lu\n", rc);
+ return -1;
+ }
+
+ *mask_p |= (1ULL << rc);
+ return 0;
+}
+
+static int
+fault_simul_rule_add(__u32 opc, char *name, int argc, char **argv)
+{
+ struct libcfs_ioctl_data data = {{0}};
+ struct lnet_fault_attr attr;
+ char *optstr;
+ int rc;
+
+ static struct option opts[] = {
+ {"source", required_argument, 0, 's'},
+ {"dest", required_argument, 0, 'd'},
+ {"rate", required_argument, 0, 'r'},
+ {"interval", required_argument, 0, 'i'},
+ {"portal", required_argument, 0, 'p'},
+ {"message", required_argument, 0, 'm'},
+ {0, 0, 0, 0}
+ };
+
+ if (argc == 1) {
+ fprintf(stderr, "Failed, please provide source, destination "
+ "and rate of rule\n");
+ return -1;
+ }
+
+ optstr = "s:d:r:i:p:m:";
+ memset(&attr, 0, sizeof(attr));
+ while (1) {
+ char c = getopt_long(argc, argv, optstr, opts, NULL);
+
+ if (c == -1)
+ break;
+
+ switch (c) {
+ case 's': /* source NID/NET */
+ rc = fault_attr_nid_parse(optarg, &attr.fa_src);
+ if (rc != 0)
+ goto getopt_failed;
+ break;
+
+ case 'd': /* dest NID/NET */
+ rc = fault_attr_nid_parse(optarg, &attr.fa_dst);
+ if (rc != 0)
+ goto getopt_failed;
+ break;
+
+ case 'r': /* drop rate */
+ attr.u.drop.da_rate = strtoul(optarg, NULL, 0);
+ break;
+
+ case 'i': /* time interval (# seconds) for message drop */
+ attr.u.drop.da_interval = strtoul(optarg, NULL, 0);
+ break;
+
+ case 'p': /* portal to filter */
+ rc = fault_attr_ptl_parse(optarg, &attr.fa_ptl_mask);
+ if (rc != 0)
+ goto getopt_failed;
+ break;
+
+ case 'm': /* message types to filter */
+ rc = fault_attr_msg_parse(optarg, &attr.fa_msg_mask);
+ if (rc != 0)
+ goto getopt_failed;
+ break;
+
+ default:
+ fprintf(stderr, "error: %s: option '%s' "
+ "unrecognized\n", argv[0], argv[optind - 1]);
+ goto getopt_failed;
+ }
+ }
+ optind = 1;
+
+ if ((attr.u.drop.da_rate == 0) == (attr.u.drop.da_interval == 0)) {
+ fprintf(stderr, "please provide drop rate or interval\n");
+ return -1;
+ }
+
+ if (attr.fa_src == 0 || attr.fa_dst == 0) {
+ fprintf(stderr, "Please provide both source and destination "
+ "of %s rule\n", name);
+ return -1;
+ }
+
+ data.ioc_flags = opc;
+ data.ioc_inllen1 = sizeof(attr);
+ data.ioc_inlbuf1 = (char *)&attr;
+ if (libcfs_ioctl_pack(&data, &ioc_buf, IOC_BUF_SIZE) != 0) {
+ fprintf(stderr, "libcfs_ioctl_pack failed\n");
+ return -1;
+ }
+
+ rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNET_FAULT, ioc_buf);
+ if (rc != 0) {
+ fprintf(stderr, "add %s rule %s->%s failed: %s\n",
+ name, libcfs_nid2str(attr.fa_src),
+ libcfs_nid2str(attr.fa_dst), strerror(errno));
+ return -1;
+ }
+
+ printf("Added %s rule %s->%s (1/%d)\n",
+ name, libcfs_nid2str(attr.fa_src),
+ libcfs_nid2str(attr.fa_dst), attr.u.drop.da_rate);
+ return 0;
+
+getopt_failed:
+ optind = 1;
+ return -1;
+}
+
+int
+jt_ptl_drop_add(int argc, char **argv)
+{
+ return fault_simul_rule_add(LNET_CTL_DROP_ADD, "drop", argc, argv);
+}
+
+static int
+fault_simul_rule_del(__u32 opc, char *name, int argc, char **argv)
+{
+ struct libcfs_ioctl_data data = {{0}};
+ struct lnet_fault_attr attr;
+ bool all = false;
+ int rc;
+
+ static struct option opts[] = {
+ {"source", required_argument, 0, 's'},
+ {"dest", required_argument, 0, 'd'},
+ {"all", no_argument, 0, 'a'},
+ {0, 0, 0, 0}
+ };
+
+ if (argc == 1) {
+ fprintf(stderr, "Failed, please provide source and "
+ "destination of rule\n");
+ return -1;
+ }
+
+ memset(&attr, 0, sizeof(attr));
+ while (1) {
+ char c = getopt_long(argc, argv, "s:d:a", opts, NULL);
+
+ if (c == -1 || all)
+ break;
+
+ switch (c) {
+ case 's':
+ rc = fault_attr_nid_parse(optarg, &attr.fa_src);
+ if (rc != 0)
+ goto getopt_failed;
+ break;
+ case 'd':
+ rc = fault_attr_nid_parse(optarg, &attr.fa_dst);
+ if (rc != 0)
+ goto getopt_failed;
+ break;
+ case 'a':
+ attr.fa_src = attr.fa_dst = 0;
+ all = true;
+ break;
+ default:
+ fprintf(stderr, "error: %s: option '%s' "
+ "unrecognized\n", argv[0], argv[optind - 1]);
+ goto getopt_failed;
+ }
+ }
+ optind = 1;
+
+ data.ioc_flags = opc;
+ data.ioc_inllen1 = sizeof(attr);
+ data.ioc_inlbuf1 = (char *)&attr;
+ if (libcfs_ioctl_pack(&data, &ioc_buf, IOC_BUF_SIZE) != 0) {
+ fprintf(stderr, "libcfs_ioctl_pack failed\n");
+ return -1;
+ }
+
+ rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNET_FAULT, ioc_buf);
+ if (rc != 0) {
+ fprintf(stderr, "remove %s rule %s->%s failed: %s\n", name,
+ all ? "all" : libcfs_nid2str(attr.fa_src),
+ all ? "all" : libcfs_nid2str(attr.fa_dst),
+ strerror(errno));
+ return -1;
+ }
+
+ libcfs_ioctl_unpack(&data, ioc_buf);
+ printf("Removed %d %s rules\n", data.ioc_count, name);
+ return 0;
+
+getopt_failed:
+ optind = 1;
+ return -1;
+}
+
+int
+jt_ptl_drop_del(int argc, char **argv)
+{
+ return fault_simul_rule_del(LNET_CTL_DROP_DEL, "drop", argc, argv);
+}
+
+static int
+fault_simul_rule_reset(__u32 opc, char *name, int argc, char **argv)
+{
+ struct libcfs_ioctl_data data = {{0}};
+ int rc;
+
+ LIBCFS_IOC_INIT(data);
+ data.ioc_flags = opc;
+
+ rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNET_FAULT, &data);
+ if (rc != 0) {
+ fprintf(stderr, "failed to reset %s stats: %s\n",
+ name, strerror(errno));
+ return -1;
+ }
+ return 0;
+}
+
+int
+jt_ptl_drop_reset(int argc, char **argv)
+{
+ return fault_simul_rule_reset(LNET_CTL_DROP_RESET, "drop", argc, argv);
+}
+
+static int
+fault_simul_rule_list(__u32 opc, char *name, int argc, char **argv)
+{
+ struct libcfs_ioctl_data data = {{0}};
+ struct lnet_fault_attr attr;
+ struct lnet_fault_stat stat;
+ int pos;
+
+ printf("LNet %s rules:\n", name);
+ for (pos = 0;; pos++) {
+ int rc;
+
+ memset(&attr, 0, sizeof(attr));
+ memset(&stat, 0, sizeof(stat));
+
+ data.ioc_count = pos;
+ data.ioc_flags = opc;
+ data.ioc_inllen1 = sizeof(attr);
+ data.ioc_inlbuf1 = (char *)&attr;
+ data.ioc_inllen2 = sizeof(stat);
+ data.ioc_inlbuf2 = (char *)&stat;
+ if (libcfs_ioctl_pack(&data, &ioc_buf, IOC_BUF_SIZE) != 0) {
+ fprintf(stderr, "libcfs_ioctl_pack failed\n");
+ return -1;
+ }
+
+ rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNET_FAULT, ioc_buf);
+ if (rc != 0)
+ break;
+
+ libcfs_ioctl_unpack(&data, ioc_buf);
+
+ if (opc == LNET_CTL_DROP_LIST) {
+ printf("%s->%s (1/%d | %d) ptl "LPX64", msg %x, "
+ LPU64"/"LPU64", PUT "LPU64", ACK "LPU64", GET "
+ LPU64", REP "LPU64"\n",
+ libcfs_nid2str(attr.fa_src),
+ libcfs_nid2str(attr.fa_dst),
+ attr.u.drop.da_rate, attr.u.drop.da_interval,
+ attr.fa_ptl_mask, attr.fa_msg_mask,
+ stat.u.drop.ds_dropped, stat.fs_count,
+ stat.fs_put, stat.fs_ack,
+ stat.fs_get, stat.fs_reply);
+ }
+ }
+ printf("found total %d\n", pos);
+
+ return 0;
+}
+
+int
+jt_ptl_drop_list(int argc, char **argv)
+{
+ return fault_simul_rule_list(LNET_CTL_DROP_LIST, "drop", argc, argv);
+}
+
double
get_cycles_per_usec ()
{
"usage: show_route"},
{"ping", jt_ptl_ping, 0, "Check LNET connectivity\n"
"usage: ping nid [timeout] [pid]"},
+ {"net_drop_add", jt_ptl_drop_add, 0, "Add LNet drop rule\n"
+ "usage: net_drop_add <-s | --source NID>\n"
+ " <-d | --dest NID>\n"
+ " <<-r | --rate DROP_RATE> |\n"
+ " <-i | --interval SECONDS>>\n"
+ " [<-p | --portal> PORTAL...]\n"
+ " [<-m | --message> <PUT|ACK|GET|REPLY>...]\n"},
+ {"net_drop_del", jt_ptl_drop_del, 0, "remove LNet drop rule\n"
+ "usage: net_drop_del <[-a | --all] |\n"
+ " <-s | --source NID>\n"
+ " <-d | --dest NID>>\n"},
+ {"net_drop_reset", jt_ptl_drop_reset, 0, "reset drop rule stats\n"
+ "usage: net_drop_reset"},
+ {"net_drop_list", jt_ptl_drop_list, 0, "list LNet drop rules\n"
+ "usage: net_drop_list"},
/* Device selection commands */
{"==== obd device selection ====", jt_noop, 0, "device selection"},