LU-14627 lnet: Allow delayed sends

The net_delay_add has some code related to delaying sends, but it
isn't fully implemented. Modify lnet_post_send_locked() to check
whether the message being sent matches a rule and should be delayed.

Fix some bugs with how the delay timers were set and checked.

Lustre-change: https://review.whamcloud.com/43416
Lustre-commit: ab14f3bc852e708100d21770c00235f95841708a

HPE-bug-id: LUS-7651
Test-Parameters: trivial
Signed-off-by: Chris Horn <chris.horn@hpe.com>
Change-Id: Icbd9ee81d2ff0162a01a4187807ea2114a42276d
Reviewed-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Reviewed-by: James Simmons <jsimmons@infradead.org>
Signed-off-by: Minh Diep <mdiep@whamcloud.com>
Reviewed-on: https://review.whamcloud.com/43959
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h
index 61815a6..048531b 100644 (file)
--- a/lnet/include/lnet/lib-lnet.h
+++ b/lnet/include/lnet/lib-lnet.h
@@ -700,6 +700,7 @@ void lnet_recv(struct lnet_ni *ni, void *private, struct lnet_msg *msg,
 void lnet_ni_recv(struct lnet_ni *ni, void *private, struct lnet_msg *msg,
                  int delayed, unsigned int offset,
                  unsigned int mlen, unsigned int rlen);
+void lnet_ni_send(struct lnet_ni *ni, struct lnet_msg *msg);
 
 struct lnet_msg *lnet_create_reply_msg(struct lnet_ni *ni,
                                       struct lnet_msg *get_msg);

diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c
index b1a2f01..b178fd0 100644 (file)
--- a/lnet/lnet/lib-move.c
+++ b/lnet/lnet/lib-move.c
@@ -753,7 +753,7 @@ lnet_prep_send(struct lnet_msg *msg, int type, struct lnet_process_id target,
        msg->msg_hdr.payload_length = cpu_to_le32(len);
 }
 
-static void
+void
 lnet_ni_send(struct lnet_ni *ni, struct lnet_msg *msg)
 {
        void *priv = msg->msg_private;
@@ -1024,6 +1024,12 @@ lnet_post_send_locked(struct lnet_msg *msg, int do_send)
                }
        }
 
+       if (unlikely(!list_empty(&the_lnet.ln_delay_rules)) &&
+           lnet_delay_rule_match_locked(&msg->msg_hdr, msg)) {
+               msg->msg_tx_delayed = 1;
+               return LNET_CREDIT_WAIT;
+       }
+
        /* unset the tx_delay flag as we're going to send it now */
        msg->msg_tx_delayed = 0;
 

diff --git a/lnet/lnet/net_fault.c b/lnet/lnet/net_fault.c
index 04c98d5..6b1469e 100644 (file)
--- a/lnet/lnet/net_fault.c
+++ b/lnet/lnet/net_fault.c
@@ -525,8 +525,9 @@ delay_rule_match(struct lnet_delay_rule *rule, lnet_nid_t src,
                lnet_nid_t dst, unsigned int type, unsigned int portal,
                struct lnet_msg *msg)
 {
-       struct lnet_fault_attr  *attr = &rule->dl_attr;
-       bool                     delay;
+       struct lnet_fault_attr *attr = &rule->dl_attr;
+       bool delay;
+       time64_t now = ktime_get_seconds();
 
        if (!lnet_fault_attr_match(attr, src, dst, type, portal))
                return false;
@@ -534,8 +535,6 @@ delay_rule_match(struct lnet_delay_rule *rule, lnet_nid_t src,
        /* match this rule, check delay rate now */
        spin_lock(&rule->dl_lock);
        if (rule->dl_delay_time != 0) { /* time based delay */
-               time64_t now = ktime_get_seconds();
-
                rule->dl_stat.fs_count++;
                delay = now >= rule->dl_delay_time;
                if (delay) {
@@ -577,10 +576,11 @@ delay_rule_match(struct lnet_delay_rule *rule, lnet_nid_t src,
        rule->dl_stat.u.delay.ls_delayed++;
 
        list_add_tail(&msg->msg_list, &rule->dl_msg_list);
-       msg->msg_delay_send = ktime_get_seconds() + attr->u.delay.la_latency;
+       msg->msg_delay_send = now + attr->u.delay.la_latency;
        if (rule->dl_msg_send == -1) {
                rule->dl_msg_send = msg->msg_delay_send;
-               mod_timer(&rule->dl_timer, rule->dl_msg_send);
+               mod_timer(&rule->dl_timer,
+                         jiffies + cfs_time_seconds(attr->u.delay.la_latency));
        }
 
        spin_unlock(&rule->dl_lock);
@@ -626,7 +626,7 @@ delayed_msg_check(struct lnet_delay_rule *rule, bool all,
        struct lnet_msg *tmp;
        time64_t now = ktime_get_seconds();
 
-       if (!all && cfs_time_seconds(rule->dl_msg_send) > now)
+       if (!all && rule->dl_msg_send > now)
                return;
 
        spin_lock(&rule->dl_lock);
@@ -648,7 +648,9 @@ delayed_msg_check(struct lnet_delay_rule *rule, bool all,
                msg = list_entry(rule->dl_msg_list.next,
                                 struct lnet_msg, msg_list);
                rule->dl_msg_send = msg->msg_delay_send;
-               mod_timer(&rule->dl_timer, rule->dl_msg_send);
+               mod_timer(&rule->dl_timer,
+                         jiffies +
+                         cfs_time_seconds(msg->msg_delay_send - now));
        }
        spin_unlock(&rule->dl_lock);
 }
@@ -664,6 +666,20 @@ delayed_msg_process(struct list_head *msg_list, bool drop)
                int             rc;
 
                msg = list_entry(msg_list->next, struct lnet_msg, msg_list);
+
+               if (msg->msg_sending) {
+                       /* Delayed send */
+                       list_del_init(&msg->msg_list);
+                       ni = msg->msg_txni;
+                       CDEBUG(D_NET, "TRACE: msg %p %s -> %s : %s\n", msg,
+                              libcfs_nid2str(ni->ni_nid),
+                              libcfs_nid2str(msg->msg_txpeer->lpni_nid),
+                              lnet_msgtyp2str(msg->msg_type));
+                       lnet_ni_send(ni, msg);
+                       continue;
+               }
+
+               /* Delayed receive */
                LASSERT(msg->msg_rxpeer != NULL);
                LASSERT(msg->msg_rxni != NULL);