From a3488d2f33e3ba4ae1d9c127132a6b75b5bb0112 Mon Sep 17 00:00:00 2001 From: Liang Zhen Date: Mon, 4 Aug 2014 17:16:56 +0800 Subject: [PATCH] LU-5435 lnet: LNet drop rule implementation This is implementation of LNet Drop Rule, which can randomly drop LNet messages at specified rate. LNet Drop Rule can only be applied to receive side of message. User can add drop_rule either on end point of cluster (client/server) or on LNet routers. Here are lctl command to control LNet Drop Rules: - net_drop_add -s SRC_NID -d DEST_NID --rate VALUE drop 1/@VALUE of messages from @SRC_NID to @DEST_NID - net_drop_del -s SRC_NID -d DEST_NID remove all drop rules from @SRC_NID to @DEST_NID - net_drop_list list all drop rules on current node Examples: - lctl net_drop_add -s *@o2ib0 -d 192.168.1.102@tcp 1000 add new drop rule, it will drop 1/1000 messages from network o2ib0 to 192.168.1.102@tcp - lctl net_drop_add -s 10.8.6.123@o2ib1 -d * 500 add new drop rule, it will drop 1/500 messages from 10.8.6.123@o2ib1 to all nodes Signed-off-by: Liang Zhen Change-Id: Ib4cc384a8a191e88bd5f293cf1a5d7f524970d01 Reviewed-on: http://review.whamcloud.com/11314 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Bobi Jam Reviewed-by: Amir Shehata Reviewed-by: Johann Lombardi Reviewed-by: Oleg Drokin --- libcfs/include/libcfs/libcfs_ioctl.h | 1 + lnet/include/lnet/lib-lnet.h | 10 + lnet/include/lnet/lib-types.h | 2 + lnet/include/lnet/lnetctl.h | 89 ++++++- lnet/lnet/Makefile.in | 2 +- lnet/lnet/api-ni.c | 9 +- lnet/lnet/autoMakefile.am | 2 +- lnet/lnet/lib-move.c | 10 + lnet/lnet/net_fault.c | 439 +++++++++++++++++++++++++++++++++++ lnet/utils/portals.c | 348 +++++++++++++++++++++++++++ lustre/utils/lctl.c | 15 ++ 11 files changed, 923 insertions(+), 4 deletions(-) create mode 100644 lnet/lnet/net_fault.c diff --git a/libcfs/include/libcfs/libcfs_ioctl.h b/libcfs/include/libcfs/libcfs_ioctl.h index eab645c..d895926 100644 --- a/libcfs/include/libcfs/libcfs_ioctl.h +++ b/libcfs/include/libcfs/libcfs_ioctl.h @@ -139,6 +139,7 @@ struct libcfs_ioctl_handler { #define IOC_LIBCFS_PING _IOWR('e', 61, IOCTL_LIBCFS_TYPE) #define IOC_LIBCFS_DEBUG_PEER _IOWR('e', 62, IOCTL_LIBCFS_TYPE) #define IOC_LIBCFS_LNETST _IOWR('e', 63, IOCTL_LIBCFS_TYPE) +#define IOC_LIBCFS_LNET_FAULT _IOWR('e', 64, IOCTL_LIBCFS_TYPE) /* lnd ioctls */ #define IOC_LIBCFS_REGISTER_MYNID _IOWR('e', 70, IOCTL_LIBCFS_TYPE) #define IOC_LIBCFS_CLOSE_CONNECTION _IOWR('e', 71, IOCTL_LIBCFS_TYPE) diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index 8c09022..817963c 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -853,6 +853,16 @@ char *lnet_msgtyp2str (int type); void lnet_print_hdr (lnet_hdr_t * hdr); int lnet_fail_nid(lnet_nid_t nid, unsigned int threshold); +/** \addtogroup lnet_fault_simulation @{ */ + +int lnet_fault_ctl(int cmd, struct libcfs_ioctl_data *data); +int lnet_fault_init(void); +void lnet_fault_fini(void); + +bool lnet_drop_rule_match(lnet_hdr_t *hdr); + +/** @} lnet_fault_simulation */ + void lnet_counters_get(lnet_counters_t *counters); void lnet_counters_reset(void); diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h index a6e92bf..8f14572 100644 --- a/lnet/include/lnet/lib-types.h +++ b/lnet/include/lnet/lib-types.h @@ -47,6 +47,7 @@ #include #include #include +#include #define WIRE_ATTR __attribute__((packed)) @@ -778,6 +779,7 @@ typedef struct struct lnet_peer_table **ln_peer_tables; /* failure simulation */ struct list_head ln_test_peers; + struct list_head ln_drop_rules; struct list_head ln_nis; /* LND instances */ /* NIs bond on specific CPT(s) */ diff --git a/lnet/include/lnet/lnetctl.h b/lnet/include/lnet/lnetctl.h index 5368530..23560e6 100644 --- a/lnet/include/lnet/lnetctl.h +++ b/lnet/include/lnet/lnetctl.h @@ -14,7 +14,7 @@ * along with Portals; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. * - * header for libptlctl.a + * header for lnet ioctl */ #ifndef _PTLCTL_H_ #define _PTLCTL_H_ @@ -22,6 +22,89 @@ #include #include +/** \addtogroup lnet_fault_simulation + * @{ */ + +enum { + LNET_CTL_DROP_ADD, + LNET_CTL_DROP_DEL, + LNET_CTL_DROP_RESET, + LNET_CTL_DROP_LIST, +}; + +#define LNET_ACK_BIT (1 << 0) +#define LNET_PUT_BIT (1 << 1) +#define LNET_GET_BIT (1 << 2) +#define LNET_REPLY_BIT (1 << 3) + +/** ioctl parameter for LNet fault simulation */ +struct lnet_fault_attr { + /** + * source NID of drop rule + * LNET_NID_ANY is wildcard for all sources + * 255.255.255.255@net is wildcard for all addresses from @net + */ + lnet_nid_t fa_src; + /** destination NID of drop rule, see \a dr_src for details */ + lnet_nid_t fa_dst; + /** + * Portal mask to drop, -1 means all portals, for example: + * fa_ptl_mask = (1 << _LDLM_CB_REQUEST_PORTAL ) | + * (1 << LDLM_CANCEL_REQUEST_PORTAL) + * + * If it is non-zero then only PUT and GET will be filtered, otherwise + * there is no portal filter, all matched messages will be checked. + */ + __u64 fa_ptl_mask; + /** + * message types to drop, for example: + * dra_type = LNET_DROP_ACK_BIT | LNET_DROP_PUT_BIT + * + * If it is non-zero then only specified message types are filtered, + * otherwise all message types will be checked. + */ + __u32 fa_msg_mask; + union { + /** message drop simulation */ + struct { + /** drop rate of this rule */ + __u32 da_rate; + /** + * time interval of message drop, it is exclusive + * with da_rate + */ + __u32 da_interval; + } drop; + /** TODO: add more */ + __u64 space[8]; + } u; + +}; + +/** fault simluation stats */ +struct lnet_fault_stat { + /** total # matched messages */ + __u64 fs_count; + /** # dropped LNET_MSG_PUT by this rule */ + __u64 fs_put; + /** # dropped LNET_MSG_ACK by this rule */ + __u64 fs_ack; + /** # dropped LNET_MSG_GET by this rule */ + __u64 fs_get; + /** # dropped LNET_MSG_REPLY by this rule */ + __u64 fs_reply; + union { + struct { + /** total # dropped messages */ + __u64 ds_dropped; + } drop; + /** TODO: add more */ + __u64 space[8]; + } u; +}; + +/** @} lnet_fault_simulation */ + #define LNET_DEV_ID 0 #define LNET_DEV_PATH "/dev/lnet" #define LNET_DEV_MAJOR 10 @@ -63,6 +146,10 @@ int jt_ptl_print_routes (int argc, char **argv); int jt_ptl_fail_nid (int argc, char **argv); int jt_ptl_testprotocompat(int argc, char **argv); int jt_ptl_memhog(int argc, char **argv); +int jt_ptl_drop_add(int argc, char **argv); +int jt_ptl_drop_del(int argc, char **argv); +int jt_ptl_drop_reset(int argc, char **argv); +int jt_ptl_drop_list(int argc, char **argv); int dbg_initialize(int argc, char **argv); int jt_dbg_filter(int argc, char **argv); diff --git a/lnet/lnet/Makefile.in b/lnet/lnet/Makefile.in index 8187b17..a1f7168 100644 --- a/lnet/lnet/Makefile.in +++ b/lnet/lnet/Makefile.in @@ -3,7 +3,7 @@ MODULES := lnet lnet-objs := api-ni.o config.o lnet-objs += lib-me.o lib-msg.o lib-eq.o lib-md.o lib-ptl.o lnet-objs += lib-move.o module.o lo.o -lnet-objs += router.o router_proc.o acceptor.o peer.o +lnet-objs += router.o router_proc.o acceptor.o peer.o net_fault.o default: all diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index bb08436..5ecf560 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -756,6 +756,7 @@ lnet_prepare(lnet_pid_t requested_pid) INIT_LIST_HEAD(&the_lnet.ln_nis_cpt); INIT_LIST_HEAD(&the_lnet.ln_nis_zombie); INIT_LIST_HEAD(&the_lnet.ln_routers); + INIT_LIST_HEAD(&the_lnet.ln_drop_rules); rc = lnet_create_remote_nets_table(); if (rc != 0) @@ -1830,6 +1831,7 @@ LNetNIInit(lnet_pid_t requested_pid) if (rc != 0) goto failed4; + lnet_fault_init(); lnet_proc_init(); LNET_MUTEX_UNLOCK(&the_lnet.ln_api_mutex); @@ -1881,7 +1883,9 @@ LNetNIFini() if (the_lnet.ln_refcount != 1) { the_lnet.ln_refcount--; } else { - LASSERT (!the_lnet.ln_niinit_self); + LASSERT(!the_lnet.ln_niinit_self); + + lnet_fault_fini(); lnet_proc_fini(); lnet_router_checker_stop(); @@ -2218,6 +2222,9 @@ LNetCtl(unsigned int cmd, void *arg) lnet_net_unlock(LNET_LOCK_EX); return 0; + case IOC_LIBCFS_LNET_FAULT: + return lnet_fault_ctl(data->ioc_flags, data); + case IOC_LIBCFS_PING: id.nid = data->ioc_nid; id.pid = data->ioc_u32[0]; diff --git a/lnet/lnet/autoMakefile.am b/lnet/lnet/autoMakefile.am index 49dc222..424c660 100644 --- a/lnet/lnet/autoMakefile.am +++ b/lnet/lnet/autoMakefile.am @@ -2,7 +2,7 @@ my_sources = api-ni.c config.c \ lib-me.c lib-msg.c lib-eq.c \ lib-md.c lib-ptl.c lib-move.c lo.c \ router.c router_proc.c \ - acceptor.c peer.c + acceptor.c peer.c net_fault.c if LIBLUSTRE diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index f576b42..00f6614 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -1976,6 +1976,16 @@ lnet_parse(lnet_ni_t *ni, lnet_hdr_t *hdr, lnet_nid_t from_nid, goto drop; } + if (!list_empty(&the_lnet.ln_drop_rules) && + lnet_drop_rule_match(hdr)) { + CDEBUG(D_NET, "%s, src %s, dst %s: Dropping %s to simulate" + "silent message loss\n", + libcfs_nid2str(from_nid), libcfs_nid2str(src_nid), + libcfs_nid2str(dest_nid), lnet_msgtyp2str(type)); + goto drop; + } + + msg = lnet_msg_alloc(); if (msg == NULL) { CERROR("%s, src %s: Dropping %s (out of memory)\n", diff --git a/lnet/lnet/net_fault.c b/lnet/lnet/net_fault.c new file mode 100644 index 0000000..1810121 --- /dev/null +++ b/lnet/lnet/net_fault.c @@ -0,0 +1,439 @@ +/* + * GPL HEADER START + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 021110-1307, USA + * + * GPL HEADER END + */ +/* + * Copyright (c) 2014, Intel Corporation. + */ +/* + * This file is part of Lustre, http://www.lustre.org/ + * Lustre is a trademark of Sun Microsystems, Inc. + * + * lnet/lnet/net_fault.c + * + * Lustre network fault simulation + * + * Author: liang.zhen@intel.com + */ + +#define DEBUG_SUBSYSTEM S_LNET + +#include +#include + +#define LNET_MSG_MASK (LNET_PUT_BIT | LNET_ACK_BIT | \ + LNET_GET_BIT | LNET_REPLY_BIT) + +struct lnet_drop_rule { + /** link chain on the_lnet.ln_drop_rules */ + struct list_head dr_link; + /** attributes of this rule */ + struct lnet_fault_attr dr_attr; + /** lock to protect \a dr_drop_at and \a dr_stat */ + spinlock_t dr_lock; + /** + * the message sequence to drop, which means message is dropped when + * dr_stat.drs_count == dr_drop_at + */ + unsigned long dr_drop_at; + /** + * seconds to drop the next message, it's exclusive with dr_drop_at + */ + cfs_time_t dr_drop_time; + /** baseline to caculate dr_drop_time */ + cfs_time_t dr_time_base; + /** statistic of dropped messages */ + struct lnet_fault_stat dr_stat; +}; + +static bool +lnet_fault_nid_match(lnet_nid_t nid, lnet_nid_t msg_nid) +{ + if (nid == msg_nid || nid == LNET_NID_ANY) + return true; + + if (LNET_NIDNET(nid) != LNET_NIDNET(msg_nid)) + return false; + + /* 255.255.255.255@net is wildcard for all addresses in a network */ + return LNET_NIDADDR(nid) == LNET_NIDADDR(LNET_NID_ANY); +} + +static bool +lnet_fault_attr_match(struct lnet_fault_attr *attr, lnet_nid_t src, + lnet_nid_t dst, unsigned int type, unsigned int portal) +{ + if (!lnet_fault_nid_match(attr->fa_src, src) || + !lnet_fault_nid_match(attr->fa_dst, dst)) + return false; + + if (!(attr->fa_msg_mask & (1 << type))) + return false; + + /* NB: ACK and REPLY have no portal, but they should have been + * rejected by message mask */ + if (attr->fa_ptl_mask != 0 && /* has portal filter */ + !(attr->fa_ptl_mask & (1ULL << portal))) + return false; + + return true; +} + +static int +lnet_fault_attr_validate(struct lnet_fault_attr *attr) +{ + if (attr->fa_msg_mask == 0) + attr->fa_msg_mask = LNET_MSG_MASK; /* all message types */ + + if (attr->fa_ptl_mask == 0) /* no portal filter */ + return 0; + + /* NB: only PUT and GET can be filtered if portal filter has been set */ + attr->fa_msg_mask &= LNET_GET_BIT | LNET_PUT_BIT; + if (attr->fa_msg_mask == 0) { + CDEBUG(D_NET, "can't find valid message type bits %x\n", + attr->fa_msg_mask); + return -EINVAL; + } + return 0; +} + +static void +lnet_fault_stat_inc(struct lnet_fault_stat *stat, unsigned int type) +{ + /* NB: fs_counter is NOT updated by this function */ + switch (type) { + case LNET_MSG_PUT: + stat->fs_put++; + return; + case LNET_MSG_ACK: + stat->fs_ack++; + return; + case LNET_MSG_GET: + stat->fs_get++; + return; + case LNET_MSG_REPLY: + stat->fs_reply++; + return; + } +} + +/** + * Add a new drop rule to LNet + * There is no check for duplicated drop rule, all rules will be checked for + * incoming message. + */ +static int +lnet_drop_rule_add(struct lnet_fault_attr *attr) +{ + struct lnet_drop_rule *rule; + ENTRY; + + if ((attr->u.drop.da_rate == 0) == (attr->u.drop.da_interval == 0)) { + CDEBUG(D_NET, "invalid rate %d or interval %d\n", + attr->u.drop.da_rate, attr->u.drop.da_interval); + RETURN(-EINVAL); + } + + if (lnet_fault_attr_validate(attr) != 0) + RETURN(-EINVAL); + + CFS_ALLOC_PTR(rule); + if (rule == NULL) + RETURN(-ENOMEM); + + spin_lock_init(&rule->dr_lock); + + rule->dr_attr = *attr; + if (attr->u.drop.da_interval != 0) { + rule->dr_time_base = cfs_time_shift(attr->u.drop.da_interval); + rule->dr_drop_time = cfs_time_shift(cfs_rand() % + attr->u.drop.da_interval); + } else { + rule->dr_drop_at = cfs_rand() % attr->u.drop.da_rate; + } + + lnet_net_lock(LNET_LOCK_EX); + list_add(&rule->dr_link, &the_lnet.ln_drop_rules); + lnet_net_unlock(LNET_LOCK_EX); + + CDEBUG(D_NET, "Added drop rule: src %s, dst %s, rate %d, interval %d\n", + libcfs_nid2str(attr->fa_src), libcfs_nid2str(attr->fa_src), + attr->u.drop.da_rate, attr->u.drop.da_interval); + RETURN(0); +} + +/** + * Remove matched drop rules from lnet, all rules that can match \a src and + * \a dst will be removed. + * If \a src is zero, then all rules have \a dst as destination will be remove + * If \a dst is zero, then all rules have \a src as source will be removed + * If both of them are zero, all rules will be removed + */ +static int +lnet_drop_rule_del(lnet_nid_t src, lnet_nid_t dst) +{ + struct lnet_drop_rule *rule; + struct lnet_drop_rule *tmp; + struct list_head zombies; + int n = 0; + ENTRY; + + INIT_LIST_HEAD(&zombies); + + lnet_net_lock(LNET_LOCK_EX); + list_for_each_entry_safe(rule, tmp, &the_lnet.ln_drop_rules, dr_link) { + if (rule->dr_attr.fa_src != src && src != 0) + continue; + + if (rule->dr_attr.fa_dst != dst && dst != 0) + continue; + + list_move(&rule->dr_link, &zombies); + } + lnet_net_unlock(LNET_LOCK_EX); + + list_for_each_entry_safe(rule, tmp, &zombies, dr_link) { + CDEBUG(D_NET, "Remove drop rule: src %s->dst: %s (1/%d, %d)\n", + libcfs_nid2str(rule->dr_attr.fa_src), + libcfs_nid2str(rule->dr_attr.fa_dst), + rule->dr_attr.u.drop.da_rate, + rule->dr_attr.u.drop.da_interval); + + list_del(&rule->dr_link); + CFS_FREE_PTR(rule); + n++; + } + + RETURN(n); +} + +/** + * List drop rule at position of \a pos + */ +static int +lnet_drop_rule_list(int pos, struct lnet_fault_attr *attr, + struct lnet_fault_stat *stat) +{ + struct lnet_drop_rule *rule; + int cpt; + int i = 0; + int rc = -ENOENT; + ENTRY; + + cpt = lnet_net_lock_current(); + list_for_each_entry(rule, &the_lnet.ln_drop_rules, dr_link) { + if (i++ < pos) + continue; + + spin_lock(&rule->dr_lock); + *attr = rule->dr_attr; + *stat = rule->dr_stat; + spin_unlock(&rule->dr_lock); + rc = 0; + break; + } + + lnet_net_unlock(cpt); + RETURN(rc); +} + +/** + * reset counters for all drop rules + */ +static void +lnet_drop_rule_reset(void) +{ + struct lnet_drop_rule *rule; + int cpt; + ENTRY; + + cpt = lnet_net_lock_current(); + + list_for_each_entry(rule, &the_lnet.ln_drop_rules, dr_link) { + struct lnet_fault_attr *attr = &rule->dr_attr; + + spin_lock(&rule->dr_lock); + + memset(&rule->dr_stat, 0, sizeof(rule->dr_stat)); + if (attr->u.drop.da_rate != 0) { + rule->dr_drop_at = cfs_rand() % attr->u.drop.da_rate; + } else { + rule->dr_drop_time = cfs_time_shift(cfs_rand() % + attr->u.drop.da_interval); + rule->dr_time_base = cfs_time_shift(attr->u.drop. + da_interval); + } + spin_unlock(&rule->dr_lock); + } + + lnet_net_unlock(cpt); + EXIT; +} + +/** + * check source/destination NID, portal, message type and drop rate, + * decide whether should drop this message or not + */ +static bool +drop_rule_match(struct lnet_drop_rule *rule, lnet_nid_t src, + lnet_nid_t dst, unsigned int type, unsigned int portal) +{ + struct lnet_fault_attr *attr = &rule->dr_attr; + bool drop; + + if (!lnet_fault_attr_match(attr, src, dst, type, portal)) + return false; + + /* match this rule, check drop rate now */ + spin_lock(&rule->dr_lock); + if (rule->dr_drop_time != 0) { /* time based drop */ + cfs_time_t now = cfs_time_current(); + + rule->dr_stat.fs_count++; + drop = cfs_time_aftereq(now, rule->dr_drop_time); + if (drop) { + if (cfs_time_after(now, rule->dr_time_base)) + rule->dr_time_base = now; + + rule->dr_drop_time = rule->dr_time_base + + cfs_time_seconds(cfs_rand() % + attr->u.drop.da_interval); + rule->dr_time_base += cfs_time_seconds(attr->u.drop. + da_interval); + + CDEBUG(D_NET, "Drop Rule %s->%s: next drop : " + CFS_TIME_T"\n", + libcfs_nid2str(attr->fa_src), + libcfs_nid2str(attr->fa_dst), + rule->dr_drop_time); + } + + } else { /* rate based drop */ + drop = rule->dr_stat.fs_count++ == rule->dr_drop_at; + + if (rule->dr_stat.fs_count % attr->u.drop.da_rate == 0) { + rule->dr_drop_at = rule->dr_stat.fs_count + + cfs_rand() % attr->u.drop.da_rate; + CDEBUG(D_NET, "Drop Rule %s->%s: next drop: %lu\n", + libcfs_nid2str(attr->fa_src), + libcfs_nid2str(attr->fa_dst), rule->dr_drop_at); + } + } + + if (drop) { /* drop this message, update counters */ + lnet_fault_stat_inc(&rule->dr_stat, type); + rule->dr_stat.u.drop.ds_dropped++; + } + + spin_unlock(&rule->dr_lock); + return drop; +} + +/** + * Check if message from \a src to \a dst can match any existed drop rule + */ +bool +lnet_drop_rule_match(lnet_hdr_t *hdr) +{ + struct lnet_drop_rule *rule; + lnet_nid_t src = le64_to_cpu(hdr->src_nid); + lnet_nid_t dst = le64_to_cpu(hdr->dest_nid); + unsigned int typ = le32_to_cpu(hdr->type); + unsigned int ptl = -1; + bool drop = false; + int cpt; + + /* NB: if Portal is specified, then only PUT and GET will be + * filtered by drop rule */ + if (typ == LNET_MSG_PUT) + ptl = le32_to_cpu(hdr->msg.put.ptl_index); + else if (typ == LNET_MSG_GET) + ptl = le32_to_cpu(hdr->msg.get.ptl_index); + + cpt = lnet_net_lock_current(); + list_for_each_entry(rule, &the_lnet.ln_drop_rules, dr_link) { + drop = drop_rule_match(rule, src, dst, typ, ptl); + if (drop) + break; + } + + lnet_net_unlock(cpt); + return drop; +} + +int +lnet_fault_ctl(int opc, struct libcfs_ioctl_data *data) +{ + struct lnet_fault_attr *attr; + struct lnet_fault_stat *stat; + + attr = (struct lnet_fault_attr *)data->ioc_inlbuf1; + + switch (opc) { + default: + return -EINVAL; + + case LNET_CTL_DROP_ADD: + if (attr == NULL) + return -EINVAL; + + return lnet_drop_rule_add(attr); + + case LNET_CTL_DROP_DEL: + if (attr == NULL) + return -EINVAL; + + data->ioc_count = lnet_drop_rule_del(attr->fa_src, + attr->fa_dst); + return 0; + + case LNET_CTL_DROP_RESET: + lnet_drop_rule_reset(); + return 0; + + case LNET_CTL_DROP_LIST: + stat = (struct lnet_fault_stat *)data->ioc_inlbuf2; + if (attr == NULL || stat == NULL) + return -EINVAL; + + return lnet_drop_rule_list(data->ioc_count, attr, stat); + } +} + +int +lnet_fault_init(void) +{ + CLASSERT(LNET_PUT_BIT == 1 << LNET_MSG_PUT); + CLASSERT(LNET_ACK_BIT == 1 << LNET_MSG_ACK); + CLASSERT(LNET_GET_BIT == 1 << LNET_MSG_GET); + CLASSERT(LNET_REPLY_BIT == 1 << LNET_MSG_REPLY); + + return 0; +} + +void +lnet_fault_fini(void) +{ + lnet_drop_rule_del(0, 0); + + LASSERT(list_empty(&the_lnet.ln_drop_rules)); +} diff --git a/lnet/utils/portals.c b/lnet/utils/portals.c index fcfb91b..8f1b2c5 100644 --- a/lnet/utils/portals.c +++ b/lnet/utils/portals.c @@ -30,6 +30,10 @@ unsigned int libcfs_printk = D_CANTMASK; static int g_net_set; static __u32 g_net; +#define IOC_BUF_SIZE 8192 +static char local_buf[IOC_BUF_SIZE]; +static char *ioc_buf = local_buf; + /* Convert a string boolean to an int; "enable" -> 1 */ int lnet_parse_bool (int *b, char *str) @@ -1273,6 +1277,350 @@ jt_ptl_print_routes (int argc, char **argv) return 0; } +static int +fault_attr_nid_parse(char *str, lnet_nid_t *nid_p) +{ + lnet_nid_t nid; + __u32 net; + int rc = 0; + + /* NB: can't support range ipaddress except * and *@net */ + if (strlen(str) > 2 && str[0] == '*' && str[1] == '@') { + net = libcfs_str2net(str + 2); + if (net == LNET_NIDNET(LNET_NID_ANY)) + goto failed; + + nid = LNET_MKNID(net, LNET_NIDADDR(LNET_NID_ANY)); + } else { + rc = libcfs_str2anynid(&nid, str); + if (!rc) + goto failed; + } + + *nid_p = nid; + return 0; +failed: + fprintf(stderr, "Invalid NID : %s\n", str); + return -1; +} + +static int +fault_attr_msg_parse(char *msg_str, __u32 *mask_p) +{ + if (!strcasecmp(msg_str, "put")) { + *mask_p |= LNET_PUT_BIT; + return 0; + + } else if (!strcasecmp(msg_str, "ack")) { + *mask_p |= LNET_ACK_BIT; + return 0; + + } else if (!strcasecmp(msg_str, "get")) { + *mask_p |= LNET_GET_BIT; + return 0; + + } else if (!strcasecmp(msg_str, "reply")) { + *mask_p |= LNET_REPLY_BIT; + return 0; + } + + fprintf(stderr, "unknown message type %s\n", msg_str); + return -1; +} + +static int +fault_attr_ptl_parse(char *ptl_str, __u64 *mask_p) +{ + unsigned long rc = strtoul(optarg, NULL, 0); + + if (rc >= 64) { + fprintf(stderr, "invalid portal: %lu\n", rc); + return -1; + } + + *mask_p |= (1ULL << rc); + return 0; +} + +static int +fault_simul_rule_add(__u32 opc, char *name, int argc, char **argv) +{ + struct libcfs_ioctl_data data = {{0}}; + struct lnet_fault_attr attr; + char *optstr; + int rc; + + static struct option opts[] = { + {"source", required_argument, 0, 's'}, + {"dest", required_argument, 0, 'd'}, + {"rate", required_argument, 0, 'r'}, + {"interval", required_argument, 0, 'i'}, + {"portal", required_argument, 0, 'p'}, + {"message", required_argument, 0, 'm'}, + {0, 0, 0, 0} + }; + + if (argc == 1) { + fprintf(stderr, "Failed, please provide source, destination " + "and rate of rule\n"); + return -1; + } + + optstr = "s:d:r:i:p:m:"; + memset(&attr, 0, sizeof(attr)); + while (1) { + char c = getopt_long(argc, argv, optstr, opts, NULL); + + if (c == -1) + break; + + switch (c) { + case 's': /* source NID/NET */ + rc = fault_attr_nid_parse(optarg, &attr.fa_src); + if (rc != 0) + goto getopt_failed; + break; + + case 'd': /* dest NID/NET */ + rc = fault_attr_nid_parse(optarg, &attr.fa_dst); + if (rc != 0) + goto getopt_failed; + break; + + case 'r': /* drop rate */ + attr.u.drop.da_rate = strtoul(optarg, NULL, 0); + break; + + case 'i': /* time interval (# seconds) for message drop */ + attr.u.drop.da_interval = strtoul(optarg, NULL, 0); + break; + + case 'p': /* portal to filter */ + rc = fault_attr_ptl_parse(optarg, &attr.fa_ptl_mask); + if (rc != 0) + goto getopt_failed; + break; + + case 'm': /* message types to filter */ + rc = fault_attr_msg_parse(optarg, &attr.fa_msg_mask); + if (rc != 0) + goto getopt_failed; + break; + + default: + fprintf(stderr, "error: %s: option '%s' " + "unrecognized\n", argv[0], argv[optind - 1]); + goto getopt_failed; + } + } + optind = 1; + + if ((attr.u.drop.da_rate == 0) == (attr.u.drop.da_interval == 0)) { + fprintf(stderr, "please provide drop rate or interval\n"); + return -1; + } + + if (attr.fa_src == 0 || attr.fa_dst == 0) { + fprintf(stderr, "Please provide both source and destination " + "of %s rule\n", name); + return -1; + } + + data.ioc_flags = opc; + data.ioc_inllen1 = sizeof(attr); + data.ioc_inlbuf1 = (char *)&attr; + if (libcfs_ioctl_pack(&data, &ioc_buf, IOC_BUF_SIZE) != 0) { + fprintf(stderr, "libcfs_ioctl_pack failed\n"); + return -1; + } + + rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNET_FAULT, ioc_buf); + if (rc != 0) { + fprintf(stderr, "add %s rule %s->%s failed: %s\n", + name, libcfs_nid2str(attr.fa_src), + libcfs_nid2str(attr.fa_dst), strerror(errno)); + return -1; + } + + printf("Added %s rule %s->%s (1/%d)\n", + name, libcfs_nid2str(attr.fa_src), + libcfs_nid2str(attr.fa_dst), attr.u.drop.da_rate); + return 0; + +getopt_failed: + optind = 1; + return -1; +} + +int +jt_ptl_drop_add(int argc, char **argv) +{ + return fault_simul_rule_add(LNET_CTL_DROP_ADD, "drop", argc, argv); +} + +static int +fault_simul_rule_del(__u32 opc, char *name, int argc, char **argv) +{ + struct libcfs_ioctl_data data = {{0}}; + struct lnet_fault_attr attr; + bool all = false; + int rc; + + static struct option opts[] = { + {"source", required_argument, 0, 's'}, + {"dest", required_argument, 0, 'd'}, + {"all", no_argument, 0, 'a'}, + {0, 0, 0, 0} + }; + + if (argc == 1) { + fprintf(stderr, "Failed, please provide source and " + "destination of rule\n"); + return -1; + } + + memset(&attr, 0, sizeof(attr)); + while (1) { + char c = getopt_long(argc, argv, "s:d:a", opts, NULL); + + if (c == -1 || all) + break; + + switch (c) { + case 's': + rc = fault_attr_nid_parse(optarg, &attr.fa_src); + if (rc != 0) + goto getopt_failed; + break; + case 'd': + rc = fault_attr_nid_parse(optarg, &attr.fa_dst); + if (rc != 0) + goto getopt_failed; + break; + case 'a': + attr.fa_src = attr.fa_dst = 0; + all = true; + break; + default: + fprintf(stderr, "error: %s: option '%s' " + "unrecognized\n", argv[0], argv[optind - 1]); + goto getopt_failed; + } + } + optind = 1; + + data.ioc_flags = opc; + data.ioc_inllen1 = sizeof(attr); + data.ioc_inlbuf1 = (char *)&attr; + if (libcfs_ioctl_pack(&data, &ioc_buf, IOC_BUF_SIZE) != 0) { + fprintf(stderr, "libcfs_ioctl_pack failed\n"); + return -1; + } + + rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNET_FAULT, ioc_buf); + if (rc != 0) { + fprintf(stderr, "remove %s rule %s->%s failed: %s\n", name, + all ? "all" : libcfs_nid2str(attr.fa_src), + all ? "all" : libcfs_nid2str(attr.fa_dst), + strerror(errno)); + return -1; + } + + libcfs_ioctl_unpack(&data, ioc_buf); + printf("Removed %d %s rules\n", data.ioc_count, name); + return 0; + +getopt_failed: + optind = 1; + return -1; +} + +int +jt_ptl_drop_del(int argc, char **argv) +{ + return fault_simul_rule_del(LNET_CTL_DROP_DEL, "drop", argc, argv); +} + +static int +fault_simul_rule_reset(__u32 opc, char *name, int argc, char **argv) +{ + struct libcfs_ioctl_data data = {{0}}; + int rc; + + LIBCFS_IOC_INIT(data); + data.ioc_flags = opc; + + rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNET_FAULT, &data); + if (rc != 0) { + fprintf(stderr, "failed to reset %s stats: %s\n", + name, strerror(errno)); + return -1; + } + return 0; +} + +int +jt_ptl_drop_reset(int argc, char **argv) +{ + return fault_simul_rule_reset(LNET_CTL_DROP_RESET, "drop", argc, argv); +} + +static int +fault_simul_rule_list(__u32 opc, char *name, int argc, char **argv) +{ + struct libcfs_ioctl_data data = {{0}}; + struct lnet_fault_attr attr; + struct lnet_fault_stat stat; + int pos; + + printf("LNet %s rules:\n", name); + for (pos = 0;; pos++) { + int rc; + + memset(&attr, 0, sizeof(attr)); + memset(&stat, 0, sizeof(stat)); + + data.ioc_count = pos; + data.ioc_flags = opc; + data.ioc_inllen1 = sizeof(attr); + data.ioc_inlbuf1 = (char *)&attr; + data.ioc_inllen2 = sizeof(stat); + data.ioc_inlbuf2 = (char *)&stat; + if (libcfs_ioctl_pack(&data, &ioc_buf, IOC_BUF_SIZE) != 0) { + fprintf(stderr, "libcfs_ioctl_pack failed\n"); + return -1; + } + + rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNET_FAULT, ioc_buf); + if (rc != 0) + break; + + libcfs_ioctl_unpack(&data, ioc_buf); + + if (opc == LNET_CTL_DROP_LIST) { + printf("%s->%s (1/%d | %d) ptl "LPX64", msg %x, " + LPU64"/"LPU64", PUT "LPU64", ACK "LPU64", GET " + LPU64", REP "LPU64"\n", + libcfs_nid2str(attr.fa_src), + libcfs_nid2str(attr.fa_dst), + attr.u.drop.da_rate, attr.u.drop.da_interval, + attr.fa_ptl_mask, attr.fa_msg_mask, + stat.u.drop.ds_dropped, stat.fs_count, + stat.fs_put, stat.fs_ack, + stat.fs_get, stat.fs_reply); + } + } + printf("found total %d\n", pos); + + return 0; +} + +int +jt_ptl_drop_list(int argc, char **argv) +{ + return fault_simul_rule_list(LNET_CTL_DROP_LIST, "drop", argc, argv); +} + double get_cycles_per_usec () { diff --git a/lustre/utils/lctl.c b/lustre/utils/lctl.c index 579e704..6f50454 100644 --- a/lustre/utils/lctl.c +++ b/lustre/utils/lctl.c @@ -112,6 +112,21 @@ command_t cmdlist[] = { "usage: show_route"}, {"ping", jt_ptl_ping, 0, "Check LNET connectivity\n" "usage: ping nid [timeout] [pid]"}, + {"net_drop_add", jt_ptl_drop_add, 0, "Add LNet drop rule\n" + "usage: net_drop_add <-s | --source NID>\n" + " <-d | --dest NID>\n" + " <<-r | --rate DROP_RATE> |\n" + " <-i | --interval SECONDS>>\n" + " [<-p | --portal> PORTAL...]\n" + " [<-m | --message> ...]\n"}, + {"net_drop_del", jt_ptl_drop_del, 0, "remove LNet drop rule\n" + "usage: net_drop_del <[-a | --all] |\n" + " <-s | --source NID>\n" + " <-d | --dest NID>>\n"}, + {"net_drop_reset", jt_ptl_drop_reset, 0, "reset drop rule stats\n" + "usage: net_drop_reset"}, + {"net_drop_list", jt_ptl_drop_list, 0, "list LNet drop rules\n" + "usage: net_drop_list"}, /* Device selection commands */ {"==== obd device selection ====", jt_noop, 0, "device selection"}, -- 1.8.3.1