LU-13477 lnet: Force full discovery cycle

There are scenarios where there could be a discrepancy between
cached peer information and reality. In these cases what could
end-up happening is incomplete interface information might be
cached because one side determined that the peer didn't require
a PUSH. This will lead to undesired MR behavior, where not all
the interfaces are used for a period of time.

Therefore, it is safer to always force a full discovery cycle:
GET/PUSH to ensure both sides are up-to-date.

In the NMR case, when discovery is turned off, make sure to flag
discovery as complete to avoid stalling the state machine.

Signed-off-by: Amir Shehata <ashehata@whamcloud.com>
Change-Id: Ie49ad11e8ff874206baa268a4ef2d58ebb536ed5
Reviewed-on: https://review.whamcloud.com/38322
Tested-by: jenkins <devops@whamcloud.com>
Reviewed-by: Chris Horn <chris.horn@hpe.com>
Reviewed-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c
index f444911..a86ffe8 100644 (file)
--- a/lnet/lnet/peer.c
+++ b/lnet/lnet/peer.c
@@ -1193,6 +1193,11 @@ LNetPrimaryNID(lnet_nid_t nid)
        lp = lpni->lpni_peer_net->lpn_peer;
 
        while (!lnet_peer_is_uptodate(lp)) {
        lp = lpni->lpni_peer_net->lpn_peer;
 
        while (!lnet_peer_is_uptodate(lp)) {

+               spin_lock(&lp->lp_lock);
+               /* force a full discovery cycle */
+               lp->lp_state |= LNET_PEER_FORCE_PING | LNET_PEER_FORCE_PUSH;
+               spin_unlock(&lp->lp_lock);
+

                rc = lnet_discover_peer_locked(lpni, cpt, true);
                if (rc)
                        goto out_decref;
                rc = lnet_discover_peer_locked(lpni, cpt, true);
                if (rc)
                        goto out_decref;


@@ -3064,6 +3069,21 @@ __must_hold(&lp->lp_lock)
        return rc ? rc : LNET_REDISCOVER_PEER;
 }
 
        return rc ? rc : LNET_REDISCOVER_PEER;
 }
 

+/*
+ * Mark the peer as discovered.
+ */
+static int lnet_peer_discovered(struct lnet_peer *lp)
+__must_hold(&lp->lp_lock)
+{
+       lp->lp_state |= LNET_PEER_DISCOVERED;
+       lp->lp_state &= ~(LNET_PEER_DISCOVERING |
+                         LNET_PEER_REDISCOVER);
+
+       CDEBUG(D_NET, "peer %s\n", libcfs_nid2str(lp->lp_primary_nid));
+
+       return 0;
+}
+

 /* Active side of push. */
 static int lnet_peer_send_push(struct lnet_peer *lp)
 __must_hold(&lp->lp_lock)
 /* Active side of push. */
 static int lnet_peer_send_push(struct lnet_peer *lp)
 __must_hold(&lp->lp_lock)


@@ -3077,6 +3097,12 @@ __must_hold(&lp->lp_lock)
        /* Don't push to a non-multi-rail peer. */
        if (!(lp->lp_state & LNET_PEER_MULTI_RAIL)) {
                lp->lp_state &= ~LNET_PEER_FORCE_PUSH;
        /* Don't push to a non-multi-rail peer. */
        if (!(lp->lp_state & LNET_PEER_MULTI_RAIL)) {
                lp->lp_state &= ~LNET_PEER_FORCE_PUSH;

+               /* if peer's NIDs are uptodate then peer is discovered */
+               if (lp->lp_state & LNET_PEER_NIDS_UPTODATE) {
+                       rc = lnet_peer_discovered(lp);
+                       return rc;
+               }
+

                return 0;
        }
 
                return 0;
        }
 


@@ -3170,22 +3196,6 @@ static void lnet_peer_discovery_error(struct lnet_peer *lp, int error)
 }
 
 /*
 }
 
 /*

- * Mark the peer as discovered.
- */
-static int lnet_peer_discovered(struct lnet_peer *lp)
-__must_hold(&lp->lp_lock)
-{
-       lp->lp_state |= LNET_PEER_DISCOVERED;
-       lp->lp_state &= ~(LNET_PEER_DISCOVERING |
-                         LNET_PEER_REDISCOVER);
-
-       CDEBUG(D_NET, "peer %s\n", libcfs_nid2str(lp->lp_primary_nid));
-
-       return 0;
-}
-
-
-/*

  * Discovering this peer is taking too long. Cancel any Ping or Push
  * that discovery is waiting on by unlinking the relevant MDs. The
  * lnet_discovery_event_handler() will proceed from here and complete
  * Discovering this peer is taking too long. Cancel any Ping or Push
  * that discovery is waiting on by unlinking the relevant MDs. The
  * lnet_discovery_event_handler() will proceed from here and complete

diff --git a/lnet/lnet/router.c b/lnet/lnet/router.c
index 9d7a839..3e90c30 100644 (file)
--- a/lnet/lnet/router.c
+++ b/lnet/lnet/router.c
@@ -1193,9 +1193,10 @@ rescan:
                        spin_unlock(&rtr->lp_lock);
                        continue;
                }
                        spin_unlock(&rtr->lp_lock);
                        continue;
                }

-               /* make sure we actively discover the router */
+               /* make sure we fully discover the router */

                rtr->lp_state &= ~LNET_PEER_NIDS_UPTODATE;
                rtr->lp_state &= ~LNET_PEER_NIDS_UPTODATE;

-               rtr->lp_state |= LNET_PEER_RTR_DISCOVERY;
+               rtr->lp_state |= LNET_PEER_FORCE_PING | LNET_PEER_FORCE_PUSH |
+                       LNET_PEER_RTR_DISCOVERY;

                spin_unlock(&rtr->lp_lock);
 
                /* find the peer_ni associated with the primary NID */
                spin_unlock(&rtr->lp_lock);
 
                /* find the peer_ni associated with the primary NID */