LU-11299 lnet: net aliveness

If a router is discovered on any interface on the network, then
update the network last alive time and the NI's status to UP.
If a router isn't discovered on any interface on a network,
then change the status of all the interfaces on that network to down.

Test-Parameters: forbuildonly
Signed-off-by: Amir Shehata <ashehata@whamcloud.com>
Change-Id: I1d67eb4b3284ccb8306ad4c877a2fcbdf4958d8c
Reviewed-on: https://review.whamcloud.com/34510
Reviewed-by: Olaf Weber <olaf.weber@hpe.com>
Tested-by: Jenkins

diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h
index 6121592..c959e5c 100644 (file)
--- a/lnet/include/lnet/lib-types.h
+++ b/lnet/include/lnet/lib-types.h
@@ -415,6 +415,12 @@ struct lnet_net {
 
        /* network state */
        enum lnet_net_state     net_state;
+
+       /* when I was last alive */
+       time64_t                net_last_alive;
+
+       /* protects access to net_last_alive */
+       spinlock_t              net_lock;
 };
 
 struct lnet_ni {
@@ -450,9 +456,6 @@ struct lnet_ni {
        /* percpt reference count */
        int                     **ni_refs;
 
-       /* when I was last alive */
-       time64_t                ni_last_alive;
-
        /* pointer to parent network */
        struct lnet_net         *ni_net;
 

diff --git a/lnet/lnet/config.c b/lnet/lnet/config.c
index 75e6e39..15f80bd 100644 (file)
--- a/lnet/lnet/config.c
+++ b/lnet/lnet/config.c
@@ -377,8 +377,10 @@ lnet_net_alloc(__u32 net_id, struct list_head *net_list)
        INIT_LIST_HEAD(&net->net_ni_list);
        INIT_LIST_HEAD(&net->net_ni_added);
        INIT_LIST_HEAD(&net->net_ni_zombie);
+       spin_lock_init(&net->net_lock);
 
        net->net_id = net_id;
+       net->net_last_alive = ktime_get_real_seconds();
        net->net_state = LNET_NET_STATE_INIT;
 
        /* initialize global paramters to undefiend */
@@ -483,7 +485,6 @@ lnet_ni_alloc_common(struct lnet_net *net, char *iface)
        else
                ni->ni_net_ns = NULL;
 
-       ni->ni_last_alive = ktime_get_real_seconds();
        ni->ni_state = LNET_NI_STATE_INIT;
        list_add_tail(&ni->ni_netlist, &net->net_ni_added);
 

diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c
index b2d3c6a..70ece19 100644 (file)
--- a/lnet/lnet/lib-move.c
+++ b/lnet/lnet/lib-move.c
@@ -4174,10 +4174,11 @@ lnet_parse(struct lnet_ni *ni, struct lnet_hdr *hdr, lnet_nid_t from_nid,
        }
 
        if (the_lnet.ln_routing &&
-           ni->ni_last_alive != ktime_get_real_seconds()) {
-               /* NB: so far here is the only place to set NI status to "up */
+           ni->ni_net->net_last_alive != ktime_get_real_seconds()) {
                lnet_ni_lock(ni);
-               ni->ni_last_alive = ktime_get_real_seconds();
+               spin_lock(&ni->ni_net->net_lock);
+               ni->ni_net->net_last_alive = ktime_get_real_seconds();
+               spin_unlock(&ni->ni_net->net_lock);
                if (ni->ni_status != NULL &&
                    ni->ni_status->ns_status == LNET_NI_STATUS_DOWN) {
                        ni->ni_status->ns_status = LNET_NI_STATUS_UP;

diff --git a/lnet/lnet/router.c b/lnet/lnet/router.c
index c8deecb..b0ed4be 100644 (file)
--- a/lnet/lnet/router.c
+++ b/lnet/lnet/router.c
@@ -777,10 +777,29 @@ lnet_wait_known_routerstate(void)
        }
 }
 
+static inline bool
+lnet_net_set_status_locked(struct lnet_net *net, __u32 status)
+{
+       struct lnet_ni *ni;
+       bool update = false;
+
+       list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
+               lnet_ni_lock(ni);
+               if (ni->ni_status &&
+                   ni->ni_status->ns_status != status) {
+                   ni->ni_status->ns_status = status;
+                   update = true;
+               }
+               lnet_ni_unlock(ni);
+       }
+
+       return update;
+}
+
 static bool
 lnet_update_ni_status_locked(void)
 {
-       struct lnet_ni *ni = NULL;
+       struct lnet_net *net;
        bool push = false;
        time64_t now;
        time64_t timeout;
@@ -790,31 +809,27 @@ lnet_update_ni_status_locked(void)
        timeout = router_ping_timeout + alive_router_check_interval;
 
        now = ktime_get_real_seconds();
-       while ((ni = lnet_get_next_ni_locked(NULL, ni))) {
-               if (ni->ni_net->net_lnd->lnd_type == LOLND)
+       list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
+               if (net->net_lnd->lnd_type == LOLND)
                        continue;
 
-               if (now < ni->ni_last_alive + timeout)
+               if (now < net->net_last_alive + timeout)
                        continue;
 
-               lnet_ni_lock(ni);
+               spin_lock(&net->net_lock);
                /* re-check with lock */
-               if (now < ni->ni_last_alive + timeout) {
-                       lnet_ni_unlock(ni);
+               if (now < net->net_last_alive + timeout) {
+                       spin_unlock(&net->net_lock);
                        continue;
                }
+               spin_unlock(&net->net_lock);
 
-               LASSERT(ni->ni_status != NULL);
-
-               if (ni->ni_status->ns_status != LNET_NI_STATUS_DOWN) {
-                       CDEBUG(D_NET, "NI(%s:%lld) status changed to down\n",
-                              libcfs_nid2str(ni->ni_nid), timeout);
-                       /* NB: so far, this is the only place to set
-                        * NI status to "down" */
-                       ni->ni_status->ns_status = LNET_NI_STATUS_DOWN;
-                       push = true;
-               }
-               lnet_ni_unlock(ni);
+               /*
+                * if the net didn't receive any traffic for past the
+                * timeout on any of its constituent NIs, then mark all
+                * the NIs down.
+                */
+               push = lnet_net_set_status_locked(net, LNET_NI_STATUS_DOWN);
        }
 
        return push;

diff --git a/lnet/lnet/router_proc.c b/lnet/lnet/router_proc.c
index 8d6b040..2c831a7 100644 (file)
--- a/lnet/lnet/router_proc.c
+++ b/lnet/lnet/router_proc.c
@@ -689,7 +689,7 @@ proc_lnet_nis(struct ctl_table *table, int write, void __user *buffer,
                        int j;
 
                        if (the_lnet.ln_routing)
-                               last_alive = now - ni->ni_last_alive;
+                               last_alive = now - ni->ni_net->net_last_alive;
 
                        /* @lo forever alive */
                        if (ni->ni_net->net_lnd->lnd_type == LOLND)