LU-13569 lnet: Age peer NI out of recovery

No longer send recovery pings to a peer NI that has been in recovery
for the recovery time limit. A peer NI will become eligible for
recovery again once we receive a message from it.

The existing lpni_last_alive field is utilized for this new purpose.

A check for NULL lpni is removed from
lnet_handle_remote_failure_locked() because all callers of that
function already ensure the lpni is non-NULL.

lnet_peer_ni_add_to_recoveryq_locked() now takes the recovery queue
as an argument rather than using the_lnet.ln_mt_peerNIRecovq. This
allows the function to be used by lnet_recover_peer_nis().
lnet_peer_ni_add_to_recoveryq_locked() is also modified to take a ref
on the peer NI if it is added to the recovery queue. Previously, it
was the responsibility of callers to take this ref.

Test-Parameters: trivial
HPE-bug-id: LUS-9109
Signed-off-by: Chris Horn <chris.horn@hpe.com>
Change-Id: Ib4676540ac4bb040690a4fb047236c54eea0e752
Reviewed-on: https://review.whamcloud.com/39718
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Neil Brown <neilb@suse.de>
Reviewed-by: Alexander Boyko <alexander.boyko@hpe.com>
Reviewed-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h
index 9bcf31c..bb5eed0 100644 (file)
--- a/lnet/include/lnet/lib-lnet.h
+++ b/lnet/include/lnet/lib-lnet.h
@@ -556,7 +556,9 @@ extern int libcfs_ioctl_getdata(struct libcfs_ioctl_hdr **hdr_pp,
 extern int lnet_get_peer_list(__u32 *countp, __u32 *sizep,
                              struct lnet_process_id __user *ids);
 extern void lnet_peer_ni_set_healthv(lnet_nid_t nid, int value, bool all);
 extern int lnet_get_peer_list(__u32 *countp, __u32 *sizep,
                              struct lnet_process_id __user *ids);
 extern void lnet_peer_ni_set_healthv(lnet_nid_t nid, int value, bool all);

-extern void lnet_peer_ni_add_to_recoveryq_locked(struct lnet_peer_ni *lpni);
+extern void lnet_peer_ni_add_to_recoveryq_locked(struct lnet_peer_ni *lpni,
+                                                struct list_head *queue,
+                                                time64_t now);

 extern int lnet_peer_add_pref_nid(struct lnet_peer_ni *lpni, lnet_nid_t nid);
 extern void lnet_peer_clr_pref_nids(struct lnet_peer_ni *lpni);
 extern int lnet_peer_del_pref_nid(struct lnet_peer_ni *lpni, lnet_nid_t nid);
 extern int lnet_peer_add_pref_nid(struct lnet_peer_ni *lpni, lnet_nid_t nid);
 extern void lnet_peer_clr_pref_nids(struct lnet_peer_ni *lpni);
 extern int lnet_peer_del_pref_nid(struct lnet_peer_ni *lpni, lnet_nid_t nid);

diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c
index 73b4636..5987b1b 100644 (file)
--- a/lnet/lnet/lib-move.c
+++ b/lnet/lnet/lib-move.c
@@ -3610,6 +3610,7 @@ lnet_recover_peer_nis(void)
        lnet_nid_t nid;
        int healthv;
        int rc;
        lnet_nid_t nid;
        int healthv;
        int rc;

+       time64_t now;

 
        /*
         * Always use cpt 0 for locking across all interactions with
 
        /*
         * Always use cpt 0 for locking across all interactions with


@@ -3620,6 +3621,8 @@ lnet_recover_peer_nis(void)
                         &local_queue);
        lnet_net_unlock(0);
 
                         &local_queue);
        lnet_net_unlock(0);
 

+       now = ktime_get_seconds();
+

        list_for_each_entry_safe(lpni, tmp, &local_queue,
                                 lpni_recovery) {
                /*
        list_for_each_entry_safe(lpni, tmp, &local_queue,
                                 lpni_recovery) {
                /*


@@ -3700,29 +3703,21 @@ lnet_recover_peer_nis(void)
                        }
 
                        lpni->lpni_recovery_ping_mdh = mdh;
                        }
 
                        lpni->lpni_recovery_ping_mdh = mdh;

-                       /*
-                        * While we're unlocked the lpni could've been
-                        * readded on the recovery queue. In this case we
-                        * don't need to add it to the local queue, since
-                        * it's already on there and the thread that added
-                        * it would've incremented the refcount on the
-                        * peer, which means we need to decref the refcount
-                        * that was implicitly grabbed by find_peer_ni_locked.
-                        * Otherwise, if the lpni is still not on
-                        * the recovery queue, then we'll add it to the
-                        * processed list.
-                        */
-                       if (list_empty(&lpni->lpni_recovery))
-                               list_add_tail(&lpni->lpni_recovery, &processed_list);
-                       else
-                               lnet_peer_ni_decref_locked(lpni);
-                       lnet_net_unlock(0);

 
 

-                       spin_lock(&lpni->lpni_lock);
-                       if (rc)
+                       lnet_peer_ni_add_to_recoveryq_locked(lpni,
+                                                            &processed_list,
+                                                            now);
+                       if (rc) {
+                               spin_lock(&lpni->lpni_lock);

                                lpni->lpni_state &= ~LNET_PEER_NI_RECOVERY_PENDING;
                                lpni->lpni_state &= ~LNET_PEER_NI_RECOVERY_PENDING;

-               }
-               spin_unlock(&lpni->lpni_lock);
+                               spin_unlock(&lpni->lpni_lock);
+                       }
+
+                       /* Drop the ref taken by lnet_find_peer_ni_locked() */
+                       lnet_peer_ni_decref_locked(lpni);
+                       lnet_net_unlock(0);
+               } else
+                       spin_unlock(&lpni->lpni_lock);

        }
 
        list_splice_init(&processed_list, &local_queue);
        }
 
        list_splice_init(&processed_list, &local_queue);


@@ -4650,8 +4645,7 @@ lnet_parse(struct lnet_ni *ni, struct lnet_hdr *hdr, lnet_nid_t from_nid,
                }
        }
 
                }
        }
 

-       if (the_lnet.ln_routing)
-               lpni->lpni_last_alive = ktime_get_seconds();
+       lpni->lpni_last_alive = ktime_get_seconds();

 
        msg->msg_rxpeer = lpni;
        msg->msg_rxni = ni;
 
        msg->msg_rxpeer = lpni;
        msg->msg_rxni = ni;

diff --git a/lnet/lnet/lib-msg.c b/lnet/lnet/lib-msg.c
index 0722bde..94f2a78 100644 (file)
--- a/lnet/lnet/lib-msg.c
+++ b/lnet/lnet/lib-msg.c
@@ -484,6 +484,7 @@ lnet_handle_local_failure(struct lnet_ni *local_ni)
        lnet_net_unlock(0);
 }
 
        lnet_net_unlock(0);
 }
 

+/* must hold net_lock/0 */

 void
 lnet_handle_remote_failure_locked(struct lnet_peer_ni *lpni)
 {
 void
 lnet_handle_remote_failure_locked(struct lnet_peer_ni *lpni)
 {


@@ -491,14 +492,6 @@ lnet_handle_remote_failure_locked(struct lnet_peer_ni *lpni)
        __u32 lp_sensitivity;
 
        /*
        __u32 lp_sensitivity;
 
        /*

-        * NO-OP if:
-        * 1. lpni could be NULL if we're in the LOLND case
-        * 2. this is a recovery message
-        */
-       if (!lpni)
-               return;
-
-       /*

         * If there is a health sensitivity in the peer then use that
         * instead of the globally set one.
         */
         * If there is a health sensitivity in the peer then use that
         * instead of the globally set one.
         */


@@ -518,7 +511,9 @@ lnet_handle_remote_failure_locked(struct lnet_peer_ni *lpni)
         * value will not be reduced. In this case, there is no reason to
         * invoke recovery
         */
         * value will not be reduced. In this case, there is no reason to
         * invoke recovery
         */

-       lnet_peer_ni_add_to_recoveryq_locked(lpni);
+       lnet_peer_ni_add_to_recoveryq_locked(lpni,
+                                            &the_lnet.ln_mt_peerNIRecovq,
+                                            ktime_get_seconds());

 }
 
 static void
 }
 
 static void


@@ -901,6 +896,15 @@ lnet_health_check(struct lnet_msg *msg)
                                lnet_inc_lpni_healthv_locked(lpni,
                                        (sensitivity) ? sensitivity :
                                        lnet_health_sensitivity);
                                lnet_inc_lpni_healthv_locked(lpni,
                                        (sensitivity) ? sensitivity :
                                        lnet_health_sensitivity);

+                               /* This peer NI may have previously aged out
+                                * of recovery. Now that we've received a
+                                * message from it, we can continue recovery
+                                * if its health value is still below the
+                                * maximum.
+                                */
+                               lnet_peer_ni_add_to_recoveryq_locked(lpni,
+                                               &the_lnet.ln_mt_peerNIRecovq,
+                                               ktime_get_seconds());

                        }
                        lnet_net_unlock(0);
                }
                        }
                        lnet_net_unlock(0);
                }

diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c
index 6ec793b..a1d2552 100644 (file)
--- a/lnet/lnet/peer.c
+++ b/lnet/lnet/peer.c
@@ -4001,21 +4001,38 @@ out:
        return rc;
 }
 
        return rc;
 }
 

+/* must hold net_lock/0 */

 void
 void

-lnet_peer_ni_add_to_recoveryq_locked(struct lnet_peer_ni *lpni)
+lnet_peer_ni_add_to_recoveryq_locked(struct lnet_peer_ni *lpni,
+                                    struct list_head *recovery_queue,
+                                    time64_t now)

 {
        /* the mt could've shutdown and cleaned up the queues */
        if (the_lnet.ln_mt_state != LNET_MT_STATE_RUNNING)
                return;
 
 {
        /* the mt could've shutdown and cleaned up the queues */
        if (the_lnet.ln_mt_state != LNET_MT_STATE_RUNNING)
                return;
 

-       if (list_empty(&lpni->lpni_recovery) &&
-           atomic_read(&lpni->lpni_healthv) < LNET_MAX_HEALTH_VALUE) {
-               CDEBUG(D_NET, "lpni %s added to recovery queue. Health = %d\n",
-                       libcfs_nid2str(lpni->lpni_nid),
-                       atomic_read(&lpni->lpni_healthv));
-               list_add_tail(&lpni->lpni_recovery, &the_lnet.ln_mt_peerNIRecovq);
-               lnet_peer_ni_addref_locked(lpni);
+       if (!list_empty(&lpni->lpni_recovery))
+               return;
+
+       if (atomic_read(&lpni->lpni_healthv) == LNET_MAX_HEALTH_VALUE)
+               return;
+
+       if (now > lpni->lpni_last_alive + lnet_recovery_limit) {
+               CDEBUG(D_NET, "lpni %s aged out last alive %lld\n",
+                      libcfs_nid2str(lpni->lpni_nid),
+                      lpni->lpni_last_alive);
+               return;

        }
        }

+
+       /* This peer NI is going on the recovery queue, so take a ref on it */
+       lnet_peer_ni_addref_locked(lpni);
+
+       CDEBUG(D_NET, "%s added to recovery queue. last alive: %lld health: %d\n",
+              libcfs_nid2str(lpni->lpni_nid),
+              lpni->lpni_last_alive,
+              atomic_read(&lpni->lpni_healthv));
+
+       list_add_tail(&lpni->lpni_recovery, recovery_queue);

 }
 
 /* Call with the ln_api_mutex held */
 }
 
 /* Call with the ln_api_mutex held */


@@ -4028,10 +4045,13 @@ lnet_peer_ni_set_healthv(lnet_nid_t nid, int value, bool all)
        struct lnet_peer_ni *lpni;
        int lncpt;
        int cpt;
        struct lnet_peer_ni *lpni;
        int lncpt;
        int cpt;

+       time64_t now;

 
        if (the_lnet.ln_state != LNET_STATE_RUNNING)
                return;
 
 
        if (the_lnet.ln_state != LNET_STATE_RUNNING)
                return;
 

+       now = ktime_get_seconds();
+

        if (!all) {
                lnet_net_lock(LNET_LOCK_EX);
                lpni = lnet_find_peer_ni_locked(nid);
        if (!all) {
                lnet_net_lock(LNET_LOCK_EX);
                lpni = lnet_find_peer_ni_locked(nid);


@@ -4040,7 +4060,8 @@ lnet_peer_ni_set_healthv(lnet_nid_t nid, int value, bool all)
                        return;
                }
                atomic_set(&lpni->lpni_healthv, value);
                        return;
                }
                atomic_set(&lpni->lpni_healthv, value);

-               lnet_peer_ni_add_to_recoveryq_locked(lpni);
+               lnet_peer_ni_add_to_recoveryq_locked(lpni,
+                                            &the_lnet.ln_mt_peerNIRecovq, now);

                lnet_peer_ni_decref_locked(lpni);
                lnet_net_unlock(LNET_LOCK_EX);
                return;
                lnet_peer_ni_decref_locked(lpni);
                lnet_net_unlock(LNET_LOCK_EX);
                return;


@@ -4049,8 +4070,8 @@ lnet_peer_ni_set_healthv(lnet_nid_t nid, int value, bool all)
        lncpt = cfs_percpt_number(the_lnet.ln_peer_tables);
 
        /*
        lncpt = cfs_percpt_number(the_lnet.ln_peer_tables);
 
        /*

-        * Walk all the peers and reset the healhv for each one to the
-        * maximum value.
+        * Walk all the peers and reset the health value for each one to the
+        * specified value.

         */
        lnet_net_lock(LNET_LOCK_EX);
        for (cpt = 0; cpt < lncpt; cpt++) {
         */
        lnet_net_lock(LNET_LOCK_EX);
        for (cpt = 0; cpt < lncpt; cpt++) {


@@ -4060,7 +4081,8 @@ lnet_peer_ni_set_healthv(lnet_nid_t nid, int value, bool all)
                                list_for_each_entry(lpni, &lpn->lpn_peer_nis,
                                                    lpni_peer_nis) {
                                        atomic_set(&lpni->lpni_healthv, value);
                                list_for_each_entry(lpni, &lpn->lpn_peer_nis,
                                                    lpni_peer_nis) {
                                        atomic_set(&lpni->lpni_healthv, value);

-                                       lnet_peer_ni_add_to_recoveryq_locked(lpni);
+                                       lnet_peer_ni_add_to_recoveryq_locked(lpni,
+                                            &the_lnet.ln_mt_peerNIRecovq, now);

                                }
                        }
                }
                                }
                        }
                }