X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lnet%2Flnet%2Fnet_fault.c;h=efe69d92dd1d869a1be3c00d8cee4a5d534e5b53;hp=04c98d5f40d11cd203737b9bec1ba82ad75fff8d;hb=9cc0a5ff5fc8f45aa60cd0407ae9893d5c116ccd;hpb=5c17777d97bd20cde68771c6186320b5eae90e62

diff --git a/lnet/lnet/net_fault.c b/lnet/lnet/net_fault.c
index 04c98d5..efe69d9 100644
--- a/lnet/lnet/net_fault.c
+++ b/lnet/lnet/net_fault.c
@@ -25,7 +25,6 @@
  */
 /*
  * This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Sun Microsystems, Inc.
  *
  * lnet/lnet/net_fault.c
  *
@@ -36,6 +35,7 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
+#include <linux/random.h>
 #include <lnet/lib-lnet.h>
 #include <uapi/linux/lnet/lnetctl.h>
 
@@ -79,13 +79,15 @@ lnet_fault_nid_match(lnet_nid_t nid, lnet_nid_t msg_nid)
 
 static bool
 lnet_fault_attr_match(struct lnet_fault_attr *attr, lnet_nid_t src,
-		      lnet_nid_t dst, unsigned int type, unsigned int portal)
+		      lnet_nid_t local_nid, lnet_nid_t dst,
+		      unsigned int type, unsigned int portal)
 {
 	if (!lnet_fault_nid_match(attr->fa_src, src) ||
-	    !lnet_fault_nid_match(attr->fa_dst, dst))
+	    !lnet_fault_nid_match(attr->fa_dst, dst) ||
+	    !lnet_fault_nid_match(attr->fa_local_nid, local_nid))
 		return false;
 
-	if (!(attr->fa_msg_mask & (1 << type)))
+	if (!(attr->fa_msg_mask & BIT(type)))
 		return false;
 
 	/* NB: ACK and REPLY have no portal, but they should have been
@@ -172,9 +174,9 @@ lnet_drop_rule_add(struct lnet_fault_attr *attr)
 	if (attr->u.drop.da_interval != 0) {
 		rule->dr_time_base = ktime_get_seconds() + attr->u.drop.da_interval;
 		rule->dr_drop_time = ktime_get_seconds() +
-				     cfs_rand() % attr->u.drop.da_interval;
+				     prandom_u32_max(attr->u.drop.da_interval);
 	} else {
-		rule->dr_drop_at = cfs_rand() % attr->u.drop.da_rate;
+		rule->dr_drop_at = prandom_u32_max(attr->u.drop.da_rate);
 	}
 
 	lnet_net_lock(LNET_LOCK_EX);
@@ -199,12 +201,10 @@ lnet_drop_rule_del(lnet_nid_t src, lnet_nid_t dst)
 {
 	struct lnet_drop_rule *rule;
 	struct lnet_drop_rule *tmp;
-	struct list_head       zombies;
-	int		       n = 0;
+	LIST_HEAD(zombies);
+	int n = 0;
 	ENTRY;
 
-	INIT_LIST_HEAD(&zombies);
-
 	lnet_net_lock(LNET_LOCK_EX);
 	list_for_each_entry_safe(rule, tmp, &the_lnet.ln_drop_rules, dr_link) {
 		if (rule->dr_attr.fa_src != src && src != 0)
@@ -281,10 +281,10 @@ lnet_drop_rule_reset(void)
 
 		memset(&rule->dr_stat, 0, sizeof(rule->dr_stat));
 		if (attr->u.drop.da_rate != 0) {
-			rule->dr_drop_at = cfs_rand() % attr->u.drop.da_rate;
+			rule->dr_drop_at = prandom_u32_max(attr->u.drop.da_rate);
 		} else {
 			rule->dr_drop_time = ktime_get_seconds() +
-					     cfs_rand() % attr->u.drop.da_interval;
+					     prandom_u32_max(attr->u.drop.da_interval);
 			rule->dr_time_base = ktime_get_seconds() + attr->u.drop.da_interval;
 		}
 		spin_unlock(&rule->dr_lock);
@@ -297,15 +297,13 @@ lnet_drop_rule_reset(void)
 static void
 lnet_fault_match_health(enum lnet_msg_hstatus *hstatus, __u32 mask)
 {
-	unsigned int random;
 	int choice;
 	int delta;
 	int best_delta;
 	int i;
 
 	/* assign a random failure */
-	random = cfs_rand();
-	choice = random % (LNET_MSG_STATUS_END - LNET_MSG_STATUS_OK);
+	choice = prandom_u32_max(LNET_MSG_STATUS_END - LNET_MSG_STATUS_OK);
 	if (choice == 0)
 		choice++;
 
@@ -314,7 +312,7 @@ lnet_fault_match_health(enum lnet_msg_hstatus *hstatus, __u32 mask)
 		return;
 	}
 
-	if (mask & (1 << choice)) {
+	if (mask & BIT(choice)) {
 		*hstatus = choice;
 		return;
 	}
@@ -323,7 +321,7 @@ lnet_fault_match_health(enum lnet_msg_hstatus *hstatus, __u32 mask)
 	i = HSTATUS_END;
 	best_delta = HSTATUS_END;
 	while (i > 0) {
-		if (mask & (1 << i)) {
+		if (mask & BIT(i)) {
 			delta = choice - i;
 			if (delta < 0)
 				delta *= -1;
@@ -344,15 +342,22 @@ lnet_fault_match_health(enum lnet_msg_hstatus *hstatus, __u32 mask)
  */
 static bool
 drop_rule_match(struct lnet_drop_rule *rule, lnet_nid_t src,
-		lnet_nid_t dst, unsigned int type, unsigned int portal,
+		lnet_nid_t local_nid, lnet_nid_t dst,
+		unsigned int type, unsigned int portal,
 		enum lnet_msg_hstatus *hstatus)
 {
 	struct lnet_fault_attr	*attr = &rule->dr_attr;
 	bool			 drop;
 
-	if (!lnet_fault_attr_match(attr, src, dst, type, portal))
+	if (!lnet_fault_attr_match(attr, src, local_nid, dst, type, portal))
 		return false;
 
+	if (attr->u.drop.da_drop_all) {
+		CDEBUG(D_NET, "set to drop all messages\n");
+		drop = true;
+		goto drop_matched;
+	}
+
 	/*
 	 * if we're trying to match a health status error but it hasn't
 	 * been set in the rule, then don't match
@@ -364,7 +369,7 @@ drop_rule_match(struct lnet_drop_rule *rule, lnet_nid_t src,
 	/* match this rule, check drop rate now */
 	spin_lock(&rule->dr_lock);
 	if (attr->u.drop.da_random) {
-		int value = cfs_rand() % attr->u.drop.da_interval;
+		int value = prandom_u32_max(attr->u.drop.da_interval);
 		if (value >= (attr->u.drop.da_interval / 2))
 			drop = true;
 		else
@@ -379,7 +384,7 @@ drop_rule_match(struct lnet_drop_rule *rule, lnet_nid_t src,
 				rule->dr_time_base = now;
 
 			rule->dr_drop_time = rule->dr_time_base +
-					     cfs_rand() % attr->u.drop.da_interval;
+					     prandom_u32_max(attr->u.drop.da_interval);
 			rule->dr_time_base += attr->u.drop.da_interval;
 
 			CDEBUG(D_NET, "Drop Rule %s->%s: next drop : %lld\n",
@@ -395,13 +400,15 @@ drop_rule_match(struct lnet_drop_rule *rule, lnet_nid_t src,
 		count = rule->dr_stat.fs_count;
 		if (do_div(count, attr->u.drop.da_rate) == 0) {
 			rule->dr_drop_at = rule->dr_stat.fs_count +
-					   cfs_rand() % attr->u.drop.da_rate;
+					   prandom_u32_max(attr->u.drop.da_rate);
 			CDEBUG(D_NET, "Drop Rule %s->%s: next drop: %lu\n",
 			       libcfs_nid2str(attr->fa_src),
 			       libcfs_nid2str(attr->fa_dst), rule->dr_drop_at);
 		}
 	}
 
+drop_matched:
+
 	if (drop) { /* drop this message, update counters */
 		if (hstatus)
 			lnet_fault_match_health(hstatus,
@@ -418,7 +425,9 @@ drop_rule_match(struct lnet_drop_rule *rule, lnet_nid_t src,
  * Check if message from \a src to \a dst can match any existed drop rule
  */
 bool
-lnet_drop_rule_match(struct lnet_hdr *hdr, enum lnet_msg_hstatus *hstatus)
+lnet_drop_rule_match(struct lnet_hdr *hdr,
+		     lnet_nid_t local_nid,
+		     enum lnet_msg_hstatus *hstatus)
 {
 	lnet_nid_t src = le64_to_cpu(hdr->src_nid);
 	lnet_nid_t dst = le64_to_cpu(hdr->dest_nid);
@@ -437,7 +446,7 @@ lnet_drop_rule_match(struct lnet_hdr *hdr, enum lnet_msg_hstatus *hstatus)
 
 	cpt = lnet_net_lock_current();
 	list_for_each_entry(rule, &the_lnet.ln_drop_rules, dr_link) {
-		drop = drop_rule_match(rule, src, dst, typ, ptl,
+		drop = drop_rule_match(rule, src, local_nid, dst, typ, ptl,
 				       hstatus);
 		if (drop)
 			break;
@@ -475,8 +484,8 @@ struct lnet_delay_rule {
 	time64_t		dl_delay_time;
 	/** baseline to caculate dl_delay_time */
 	time64_t		dl_time_base;
-	/** jiffies to send the next delayed message */
-	unsigned long		dl_msg_send;
+	/** seconds until we send the next delayed message */
+	time64_t		dl_msg_send;
 	/** delayed message list */
 	struct list_head	dl_msg_list;
 	/** statistic of delayed messages */
@@ -525,17 +534,17 @@ delay_rule_match(struct lnet_delay_rule *rule, lnet_nid_t src,
 		lnet_nid_t dst, unsigned int type, unsigned int portal,
 		struct lnet_msg *msg)
 {
-	struct lnet_fault_attr	*attr = &rule->dl_attr;
-	bool			 delay;
+	struct lnet_fault_attr *attr = &rule->dl_attr;
+	bool delay;
+	time64_t now = ktime_get_seconds();
 
-	if (!lnet_fault_attr_match(attr, src, dst, type, portal))
+	if (!lnet_fault_attr_match(attr, src, LNET_NID_ANY,
+				   dst, type, portal))
 		return false;
 
 	/* match this rule, check delay rate now */
 	spin_lock(&rule->dl_lock);
 	if (rule->dl_delay_time != 0) { /* time based delay */
-		time64_t now = ktime_get_seconds();
-
 		rule->dl_stat.fs_count++;
 		delay = now >= rule->dl_delay_time;
 		if (delay) {
@@ -543,7 +552,7 @@ delay_rule_match(struct lnet_delay_rule *rule, lnet_nid_t src,
 				rule->dl_time_base = now;
 
 			rule->dl_delay_time = rule->dl_time_base +
-					      cfs_rand() % attr->u.delay.la_interval;
+					      prandom_u32_max(attr->u.delay.la_interval);
 			rule->dl_time_base += attr->u.delay.la_interval;
 
 			CDEBUG(D_NET, "Delay Rule %s->%s: next delay : %lld\n",
@@ -560,7 +569,7 @@ delay_rule_match(struct lnet_delay_rule *rule, lnet_nid_t src,
 		count = rule->dl_stat.fs_count;
 		if (do_div(count, attr->u.delay.la_rate) == 0) {
 			rule->dl_delay_at = rule->dl_stat.fs_count +
-					    cfs_rand() % attr->u.delay.la_rate;
+					    prandom_u32_max(attr->u.delay.la_rate);
 			CDEBUG(D_NET, "Delay Rule %s->%s: next delay: %lu\n",
 			       libcfs_nid2str(attr->fa_src),
 			       libcfs_nid2str(attr->fa_dst), rule->dl_delay_at);
@@ -577,10 +586,11 @@ delay_rule_match(struct lnet_delay_rule *rule, lnet_nid_t src,
 	rule->dl_stat.u.delay.ls_delayed++;
 
 	list_add_tail(&msg->msg_list, &rule->dl_msg_list);
-	msg->msg_delay_send = ktime_get_seconds() + attr->u.delay.la_latency;
+	msg->msg_delay_send = now + attr->u.delay.la_latency;
 	if (rule->dl_msg_send == -1) {
 		rule->dl_msg_send = msg->msg_delay_send;
-		mod_timer(&rule->dl_timer, rule->dl_msg_send);
+		mod_timer(&rule->dl_timer,
+			  jiffies + cfs_time_seconds(attr->u.delay.la_latency));
 	}
 
 	spin_unlock(&rule->dl_lock);
@@ -626,7 +636,7 @@ delayed_msg_check(struct lnet_delay_rule *rule, bool all,
 	struct lnet_msg *tmp;
 	time64_t now = ktime_get_seconds();
 
-	if (!all && cfs_time_seconds(rule->dl_msg_send) > now)
+	if (!all && rule->dl_msg_send > now)
 		return;
 
 	spin_lock(&rule->dl_lock);
@@ -648,7 +658,9 @@ delayed_msg_check(struct lnet_delay_rule *rule, bool all,
 		msg = list_entry(rule->dl_msg_list.next,
 				 struct lnet_msg, msg_list);
 		rule->dl_msg_send = msg->msg_delay_send;
-		mod_timer(&rule->dl_timer, rule->dl_msg_send);
+		mod_timer(&rule->dl_timer,
+			  jiffies +
+			  cfs_time_seconds(msg->msg_delay_send - now));
 	}
 	spin_unlock(&rule->dl_lock);
 }
@@ -664,6 +676,20 @@ delayed_msg_process(struct list_head *msg_list, bool drop)
 		int		rc;
 
 		msg = list_entry(msg_list->next, struct lnet_msg, msg_list);
+
+		if (msg->msg_sending) {
+			/* Delayed send */
+			list_del_init(&msg->msg_list);
+			ni = msg->msg_txni;
+			CDEBUG(D_NET, "TRACE: msg %p %s -> %s : %s\n", msg,
+			       libcfs_nidstr(&ni->ni_nid),
+			       libcfs_nidstr(&msg->msg_txpeer->lpni_nid),
+			       lnet_msgtyp2str(msg->msg_type));
+			lnet_ni_send(ni, msg);
+			continue;
+		}
+
+		/* Delayed receive */
 		LASSERT(msg->msg_rxpeer != NULL);
 		LASSERT(msg->msg_rxni != NULL);
 
@@ -688,6 +714,7 @@ delayed_msg_process(struct list_head *msg_list, bool drop)
 			case LNET_CREDIT_OK:
 				lnet_ni_recv(ni, msg->msg_private, msg, 0,
 					     0, msg->msg_len, msg->msg_len);
+				/* fallthrough */
 			case LNET_CREDIT_WAIT:
 				continue;
 			default: /* failures */
@@ -708,10 +735,9 @@ delayed_msg_process(struct list_head *msg_list, bool drop)
 void
 lnet_delay_rule_check(void)
 {
-	struct lnet_delay_rule	*rule;
-	struct list_head	 msgs;
+	struct lnet_delay_rule *rule;
+	LIST_HEAD(msgs);
 
-	INIT_LIST_HEAD(&msgs);
 	while (1) {
 		if (list_empty(&delay_dd.dd_sched_rules))
 			break;
@@ -835,9 +861,9 @@ lnet_delay_rule_add(struct lnet_fault_attr *attr)
 		rule->dl_time_base = ktime_get_seconds() +
 				     attr->u.delay.la_interval;
 		rule->dl_delay_time = ktime_get_seconds() +
-				      cfs_rand() % attr->u.delay.la_interval;
+				      prandom_u32_max(attr->u.delay.la_interval);
 	} else {
-		rule->dl_delay_at = cfs_rand() % attr->u.delay.la_rate;
+		rule->dl_delay_at = prandom_u32_max(attr->u.delay.la_rate);
 	}
 
 	rule->dl_msg_send = -1;
@@ -873,16 +899,13 @@ int
 lnet_delay_rule_del(lnet_nid_t src, lnet_nid_t dst, bool shutdown)
 {
 	struct lnet_delay_rule *rule;
-	struct lnet_delay_rule	*tmp;
-	struct list_head	rule_list;
-	struct list_head	msg_list;
-	int			n = 0;
-	bool			cleanup;
+	struct lnet_delay_rule *tmp;
+	LIST_HEAD(rule_list);
+	LIST_HEAD(msg_list);
+	int n = 0;
+	bool cleanup;
 	ENTRY;
 
-	INIT_LIST_HEAD(&rule_list);
-	INIT_LIST_HEAD(&msg_list);
-
 	if (shutdown)
 		src = dst = 0;
 
@@ -984,10 +1007,10 @@ lnet_delay_rule_reset(void)
 
 		memset(&rule->dl_stat, 0, sizeof(rule->dl_stat));
 		if (attr->u.delay.la_rate != 0) {
-			rule->dl_delay_at = cfs_rand() % attr->u.delay.la_rate;
+			rule->dl_delay_at = prandom_u32_max(attr->u.delay.la_rate);
 		} else {
 			rule->dl_delay_time = ktime_get_seconds() +
-					      cfs_rand() % attr->u.delay.la_interval;
+					      prandom_u32_max(attr->u.delay.la_interval);
 			rule->dl_time_base = ktime_get_seconds() +
 					     attr->u.delay.la_interval;
 		}
@@ -1065,10 +1088,10 @@ lnet_fault_ctl(int opc, struct libcfs_ioctl_data *data)
 int
 lnet_fault_init(void)
 {
-	CLASSERT(LNET_PUT_BIT == 1 << LNET_MSG_PUT);
-	CLASSERT(LNET_ACK_BIT == 1 << LNET_MSG_ACK);
-	CLASSERT(LNET_GET_BIT == 1 << LNET_MSG_GET);
-	CLASSERT(LNET_REPLY_BIT == 1 << LNET_MSG_REPLY);
+	BUILD_BUG_ON(LNET_PUT_BIT != BIT(LNET_MSG_PUT));
+	BUILD_BUG_ON(LNET_ACK_BIT != BIT(LNET_MSG_ACK));
+	BUILD_BUG_ON(LNET_GET_BIT != BIT(LNET_MSG_GET));
+	BUILD_BUG_ON(LNET_REPLY_BIT != BIT(LNET_MSG_REPLY));
 
 	mutex_init(&delay_dd.dd_mutex);
 	spin_lock_init(&delay_dd.dd_lock);