LU-14660 lnet: Fix destination NID for discovery PUSH

If we're sending a discovery PUSH after receiving a discovery
REPLY then we want to send via the same NID that the reply was
sent to. This introduces a challenge in selecting an appropriate
destination NID for the PUSH because lnet_select_pathway() will not
run the MR selection algorithm for choosing a peer NI if the source
NI has been specified.

It is reasonable to assume that the NID used by the message
originator in sending the REPLY is a suitable destination for the
discovery PUSH. Thus, we record this NID in the same location we
currently record the lp_disc_src_nid, and use it when sending the
PUSH. With this change, the only other user of lnet_peer_select_nid()
is lnet_peer_send_ping(). In the ping case we do not set a source NID,
so lnet_select_pathway() is free to choose any peer NI. So this change
allows us to get rid of lnet_peer_select_nid() altogether.

Alternatively, we would need to reproduce a lot of the path selection
algorithm inside lnet_peer_select_nid() in order to avoid sending to
unhealthy NIDs. It seems undesirable and unnecessary to duplicate that
logic.

Test-Parameters: trivial
HPE-bug-id: LUS-9333
Signed-off-by: Chris Horn <chris.horn@hpe.com>
Change-Id: I47ef856075f049d71c395565974204b8f6fa9003
Reviewed-on: https://review.whamcloud.com/43507
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Reviewed-by: Alexander Boyko <alexander.boyko@hpe.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h
index ed52e0a..87867ad 100644 (file)
--- a/lnet/include/lnet/lib-types.h
+++ b/lnet/include/lnet/lib-types.h
@@ -652,6 +652,8 @@ struct lnet_peer {
 
        /* source NID to use during discovery */
        lnet_nid_t              lp_disc_src_nid;
+       /* destination NID to use during discovery */
+       lnet_nid_t              lp_disc_dst_nid;
 
        /* net to perform discovery on */
        __u32                   lp_disc_net_id;

diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c
index d70415e..155419d 100644 (file)
--- a/lnet/lnet/peer.c
+++ b/lnet/lnet/peer.c
@@ -263,6 +263,7 @@ lnet_peer_alloc(lnet_nid_t nid)
        spin_lock_init(&lp->lp_lock);
        lp->lp_primary_nid = nid;
        lp->lp_disc_src_nid = LNET_NID_ANY;
+       lp->lp_disc_dst_nid = LNET_NID_ANY;
        if (lnet_peers_start_down())
                lp->lp_alive = false;
        else
@@ -2531,6 +2532,7 @@ lnet_discovery_event_reply(struct lnet_peer *lp, struct lnet_event *ev)
        spin_lock(&lp->lp_lock);
 
        lp->lp_disc_src_nid = ev->target.nid;
+       lp->lp_disc_dst_nid = ev->source.nid;
 
        /*
         * If some kind of error happened the contents of message
@@ -3237,8 +3239,10 @@ __must_hold(&lp->lp_lock)
                         * received by lp, we need to set the discovery source
                         * NID for new_lp to the NID stored in lp.
                         */
-                       if (lp->lp_disc_src_nid != LNET_NID_ANY)
+                       if (lp->lp_disc_src_nid != LNET_NID_ANY) {
                                new_lp->lp_disc_src_nid = lp->lp_disc_src_nid;
+                               new_lp->lp_disc_dst_nid = lp->lp_disc_dst_nid;
+                       }
                        spin_unlock(&new_lp->lp_lock);
                        spin_unlock(&lp->lp_lock);
 
@@ -3288,41 +3292,10 @@ __must_hold(&lp->lp_lock)
        return rc ? rc : LNET_REDISCOVER_PEER;
 }
 
-/*
- * Select NID to send a Ping or Push to.
- */
-static lnet_nid_t lnet_peer_select_nid(struct lnet_peer *lp)
-{
-       struct lnet_peer_ni *lpni;
-
-       /* Look for a direct-connected NID for this peer. */
-       lpni = NULL;
-       while ((lpni = lnet_get_next_peer_ni_locked(lp, NULL, lpni)) != NULL) {
-               if (!lnet_get_net_locked(lpni->lpni_peer_net->lpn_net_id))
-                       continue;
-               break;
-       }
-       if (lpni)
-               return lpni->lpni_nid;
-
-       /* Look for a routed-connected NID for this peer. */
-       lpni = NULL;
-       while ((lpni = lnet_get_next_peer_ni_locked(lp, NULL, lpni)) != NULL) {
-               if (!lnet_find_rnet_locked(lpni->lpni_peer_net->lpn_net_id))
-                       continue;
-               break;
-       }
-       if (lpni)
-               return lpni->lpni_nid;
-
-       return LNET_NID_ANY;
-}
-
 /* Active side of ping. */
 static int lnet_peer_send_ping(struct lnet_peer *lp)
 __must_hold(&lp->lp_lock)
 {
-       lnet_nid_t pnid;
        int nnis;
        int rc;
        int cpt;
@@ -3334,12 +3307,11 @@ __must_hold(&lp->lp_lock)
        cpt = lnet_net_lock_current();
        /* Refcount for MD. */
        lnet_peer_addref_locked(lp);
-       pnid = lnet_peer_select_nid(lp);
        lnet_net_unlock(cpt);
 
        nnis = max(lp->lp_data_nnis, LNET_INTERFACES_MIN);
 
-       rc = lnet_send_ping(pnid, &lp->lp_ping_mdh, nnis, lp,
+       rc = lnet_send_ping(lp->lp_primary_nid, &lp->lp_ping_mdh, nnis, lp,
                            the_lnet.ln_dc_handler, false);
 
        /*
@@ -3464,18 +3436,17 @@ __must_hold(&lp->lp_lock)
                CERROR("Can't bind push source MD: %d\n", rc);
                goto fail_error;
        }
+
        cpt = lnet_net_lock_current();
        /* Refcount for MD. */
        lnet_peer_addref_locked(lp);
        id.pid = LNET_PID_LUSTRE;
-       id.nid = lnet_peer_select_nid(lp);
+       if (lp->lp_disc_dst_nid != LNET_NID_ANY)
+               id.nid = lp->lp_disc_dst_nid;
+       else
+               id.nid = lp->lp_primary_nid;
        lnet_net_unlock(cpt);
 
-       if (id.nid == LNET_NID_ANY) {
-               rc = -EHOSTUNREACH;
-               goto fail_unlink;
-       }
-
        rc = LNetPut(lp->lp_disc_src_nid, lp->lp_push_mdh,
                     LNET_ACK_REQ, id, LNET_RESERVED_PORTAL,
                     LNET_PROTO_PING_MATCHBITS, 0, 0);
@@ -3487,6 +3458,7 @@ __must_hold(&lp->lp_lock)
         * scratch
         */
        lp->lp_disc_src_nid = LNET_NID_ANY;
+       lp->lp_disc_dst_nid = LNET_NID_ANY;
 
        if (rc)
                goto fail_unlink;