From 9eb9474c41c823c70f34e6bb102a8861ca21a3d1 Mon Sep 17 00:00:00 2001 From: Chris Horn Date: Fri, 11 Dec 2020 12:04:32 -0600 Subject: [PATCH] LU-13806 lnet: Ensure proper peer, peer NI, peer net hierarchy The MR design dictates that the peer nets and peer NIs are ordered such that the peer net and peer NI for a peer's primary NID appears first, followed by other peer NIs in the primary NID's peer net, followed by other peer nets/NIs. This ordering is broken and it can result in tripping an assertion if the primary NID of a peer is deleted. Modify lnet_peer_attach_peer_ni() to check whether the NI being attached is the peer's primary, and place it, and its associated peer net, appropriately. Modify lnet_peer_set_primary_nid() so that it updates the lp_primary_nid before calling lnet_peer_add_nid() so that lnet_peer_attach_peer_ni() can detect the situation where the primary is changing and act appropriately. Finally, modify lnet_peer_merge_data() to enforce the hierarchy after it has finished merging the contents of the ping buffer. This ensures we maintain the correct hierarchy in certain edge cases where we've needed to reconcile two peers. e.g. if a peer adds a new interface, the discovery push may arrive from that new interface which will result in a second peer object being created which will need to be reconciled with the original peer object. HPE-bug-id: LUS-9630 Signed-off-by: Chris Horn Change-Id: I8397a24ba1ba0bba33846e7e97b8d60a8f26a1be Reviewed-on: https://review.whamcloud.com/40985 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Serguei Smirnov Reviewed-by: James Simmons Reviewed-by: Oleg Drokin --- lnet/lnet/peer.c | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c index c15ae26..20ae558 100644 --- a/lnet/lnet/peer.c +++ b/lnet/lnet/peer.c @@ -1438,7 +1438,10 @@ lnet_peer_attach_peer_ni(struct lnet_peer *lp, /* Add peer_ni to peer_net */ lpni->lpni_peer_net = lpn; - list_add_tail(&lpni->lpni_peer_nis, &lpn->lpn_peer_nis); + if (lp->lp_primary_nid == lpni->lpni_nid) + list_add(&lpni->lpni_peer_nis, &lpn->lpn_peer_nis); + else + list_add_tail(&lpni->lpni_peer_nis, &lpn->lpn_peer_nis); lnet_update_peer_net_healthv(lpni); lnet_peer_net_addref_locked(lpn); @@ -1446,7 +1449,10 @@ lnet_peer_attach_peer_ni(struct lnet_peer *lp, if (!lpn->lpn_peer) { new_lpn = true; lpn->lpn_peer = lp; - list_add_tail(&lpn->lpn_peer_nets, &lp->lp_peer_nets); + if (lp->lp_primary_nid == lpni->lpni_nid) + list_add(&lpn->lpn_peer_nets, &lp->lp_peer_nets); + else + list_add_tail(&lpn->lpn_peer_nets, &lp->lp_peer_nets); lnet_peer_addref_locked(lp); } @@ -1687,10 +1693,14 @@ lnet_peer_set_primary_nid(struct lnet_peer *lp, lnet_nid_t nid, unsigned flags) if (lp->lp_primary_nid == nid) goto out; + + lp->lp_primary_nid = nid; + rc = lnet_peer_add_nid(lp, nid, flags); - if (rc) + if (rc) { + lp->lp_primary_nid = old; goto out; - lp->lp_primary_nid = nid; + } out: CDEBUG(D_NET, "peer %s NID %s: %d\n", libcfs_nid2str(old), libcfs_nid2str(nid), rc); @@ -2792,6 +2802,7 @@ static void lnet_discovery_event_handler(struct lnet_event *event) static int lnet_peer_merge_data(struct lnet_peer *lp, struct lnet_ping_buffer *pbuf) { + struct lnet_peer_net *lpn; struct lnet_peer_ni *lpni; lnet_nid_t *curnis = NULL; struct lnet_ni_status *addnis = NULL; @@ -2920,6 +2931,28 @@ static int lnet_peer_merge_data(struct lnet_peer *lp, goto out; } } + + /* The peer net for the primary NID should be the first entry in the + * peer's lp_peer_nets list, and the peer NI for the primary NID should + * be the first entry in its peer net's lpn_peer_nis list. + */ + lpni = lnet_find_peer_ni_locked(pbuf->pb_info.pi_ni[1].ns_nid); + if (!lpni) { + CERROR("Internal error: Failed to lookup peer NI for primary NID: %s\n", + libcfs_nid2str(pbuf->pb_info.pi_ni[1].ns_nid)); + goto out; + } + + lnet_peer_ni_decref_locked(lpni); + + lpn = lpni->lpni_peer_net; + if (lpn->lpn_peer_nets.prev != &lp->lp_peer_nets) + list_move(&lpn->lpn_peer_nets, &lp->lp_peer_nets); + + if (lpni->lpni_peer_nis.prev != &lpni->lpni_peer_net->lpn_peer_nis) + list_move(&lpni->lpni_peer_nis, + &lpni->lpni_peer_net->lpn_peer_nis); + /* * Errors other than -ENOMEM are due to peers having been * configured with DLC. Ignore these because DLC overrides -- 1.8.3.1