From dce2f7d1987711dfdced903b13e67091cffe9628 Mon Sep 17 00:00:00 2001 From: Chris Horn Date: Fri, 29 Jan 2021 17:08:08 +0300 Subject: [PATCH] LU-14660 lnet: Fix destination NID for discovery PUSH If we're sending a discovery PUSH after receiving a discovery REPLY then we want to send via the same NID that the reply was sent to. This introduces a challenge in selecting an appropriate destination NID for the PUSH because lnet_select_pathway() will not run the MR selection algorithm for choosing a peer NI if the source NI has been specified. It is reasonable to assume that the NID used by the message originator in sending the REPLY is a suitable destination for the discovery PUSH. Thus, we record this NID in the same location we currently record the lp_disc_src_nid, and use it when sending the PUSH. With this change, the only other user of lnet_peer_select_nid() is lnet_peer_send_ping(). In the ping case we do not set a source NID, so lnet_select_pathway() is free to choose any peer NI. So this change allows us to get rid of lnet_peer_select_nid() altogether. Alternatively, we would need to reproduce a lot of the path selection algorithm inside lnet_peer_select_nid() in order to avoid sending to unhealthy NIDs. It seems undesirable and unnecessary to duplicate that logic. Test-Parameters: trivial HPE-bug-id: LUS-9333 Signed-off-by: Chris Horn Change-Id: I47ef856075f049d71c395565974204b8f6fa9003 Reviewed-on: https://review.whamcloud.com/43507 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Serguei Smirnov Reviewed-by: Alexander Boyko Reviewed-by: Oleg Drokin --- lnet/include/lnet/lib-types.h | 2 ++ lnet/lnet/peer.c | 52 ++++++++++--------------------------------- 2 files changed, 14 insertions(+), 40 deletions(-) diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h index ed52e0a..87867ad 100644 --- a/lnet/include/lnet/lib-types.h +++ b/lnet/include/lnet/lib-types.h @@ -652,6 +652,8 @@ struct lnet_peer { /* source NID to use during discovery */ lnet_nid_t lp_disc_src_nid; + /* destination NID to use during discovery */ + lnet_nid_t lp_disc_dst_nid; /* net to perform discovery on */ __u32 lp_disc_net_id; diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c index d70415e..155419d 100644 --- a/lnet/lnet/peer.c +++ b/lnet/lnet/peer.c @@ -263,6 +263,7 @@ lnet_peer_alloc(lnet_nid_t nid) spin_lock_init(&lp->lp_lock); lp->lp_primary_nid = nid; lp->lp_disc_src_nid = LNET_NID_ANY; + lp->lp_disc_dst_nid = LNET_NID_ANY; if (lnet_peers_start_down()) lp->lp_alive = false; else @@ -2531,6 +2532,7 @@ lnet_discovery_event_reply(struct lnet_peer *lp, struct lnet_event *ev) spin_lock(&lp->lp_lock); lp->lp_disc_src_nid = ev->target.nid; + lp->lp_disc_dst_nid = ev->source.nid; /* * If some kind of error happened the contents of message @@ -3237,8 +3239,10 @@ __must_hold(&lp->lp_lock) * received by lp, we need to set the discovery source * NID for new_lp to the NID stored in lp. */ - if (lp->lp_disc_src_nid != LNET_NID_ANY) + if (lp->lp_disc_src_nid != LNET_NID_ANY) { new_lp->lp_disc_src_nid = lp->lp_disc_src_nid; + new_lp->lp_disc_dst_nid = lp->lp_disc_dst_nid; + } spin_unlock(&new_lp->lp_lock); spin_unlock(&lp->lp_lock); @@ -3288,41 +3292,10 @@ __must_hold(&lp->lp_lock) return rc ? rc : LNET_REDISCOVER_PEER; } -/* - * Select NID to send a Ping or Push to. - */ -static lnet_nid_t lnet_peer_select_nid(struct lnet_peer *lp) -{ - struct lnet_peer_ni *lpni; - - /* Look for a direct-connected NID for this peer. */ - lpni = NULL; - while ((lpni = lnet_get_next_peer_ni_locked(lp, NULL, lpni)) != NULL) { - if (!lnet_get_net_locked(lpni->lpni_peer_net->lpn_net_id)) - continue; - break; - } - if (lpni) - return lpni->lpni_nid; - - /* Look for a routed-connected NID for this peer. */ - lpni = NULL; - while ((lpni = lnet_get_next_peer_ni_locked(lp, NULL, lpni)) != NULL) { - if (!lnet_find_rnet_locked(lpni->lpni_peer_net->lpn_net_id)) - continue; - break; - } - if (lpni) - return lpni->lpni_nid; - - return LNET_NID_ANY; -} - /* Active side of ping. */ static int lnet_peer_send_ping(struct lnet_peer *lp) __must_hold(&lp->lp_lock) { - lnet_nid_t pnid; int nnis; int rc; int cpt; @@ -3334,12 +3307,11 @@ __must_hold(&lp->lp_lock) cpt = lnet_net_lock_current(); /* Refcount for MD. */ lnet_peer_addref_locked(lp); - pnid = lnet_peer_select_nid(lp); lnet_net_unlock(cpt); nnis = max(lp->lp_data_nnis, LNET_INTERFACES_MIN); - rc = lnet_send_ping(pnid, &lp->lp_ping_mdh, nnis, lp, + rc = lnet_send_ping(lp->lp_primary_nid, &lp->lp_ping_mdh, nnis, lp, the_lnet.ln_dc_handler, false); /* @@ -3464,18 +3436,17 @@ __must_hold(&lp->lp_lock) CERROR("Can't bind push source MD: %d\n", rc); goto fail_error; } + cpt = lnet_net_lock_current(); /* Refcount for MD. */ lnet_peer_addref_locked(lp); id.pid = LNET_PID_LUSTRE; - id.nid = lnet_peer_select_nid(lp); + if (lp->lp_disc_dst_nid != LNET_NID_ANY) + id.nid = lp->lp_disc_dst_nid; + else + id.nid = lp->lp_primary_nid; lnet_net_unlock(cpt); - if (id.nid == LNET_NID_ANY) { - rc = -EHOSTUNREACH; - goto fail_unlink; - } - rc = LNetPut(lp->lp_disc_src_nid, lp->lp_push_mdh, LNET_ACK_REQ, id, LNET_RESERVED_PORTAL, LNET_PROTO_PING_MATCHBITS, 0, 0); @@ -3487,6 +3458,7 @@ __must_hold(&lp->lp_lock) * scratch */ lp->lp_disc_src_nid = LNET_NID_ANY; + lp->lp_disc_dst_nid = LNET_NID_ANY; if (rc) goto fail_unlink; -- 1.8.3.1