From 1a2db3e14b7886f6207fd17bc8ab0d1142535709 Mon Sep 17 00:00:00 2001 From: Serguei Smirnov Date: Tue, 4 Apr 2023 14:02:51 -0700 Subject: [PATCH] EX-7251 lnet: fix locking multiple NIDs of the MR peer If LNetPrimaryNID is called on multiple NIDs of the same node, as a result of peer discovery it is possible that the discovered peer is found to contain a NID which is locked as primary by a different existing peer record. In this case it is safe to delete one of the peer records, but the NID which got locked the earliest should be kept as primary. This allows for the first NID listed in the mount command's comma-separated list to stay primary as intended for the purpose of communicating with Lustre even if peer discovery succeeded using a different NID of MR peer. Lustre-change: https://review.whamcloud.com/50530 Lustre-commit: TBD (47df7c726987d49e92b7145c128414daa413835f) Fixes: aacb16191a ("LU-14668 lnet: Lock primary NID logic") Signed-off-by: Serguei Smirnov Change-Id: Iec9f8b70053fe24cddee552358500dfad0234b7f Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/50533 Reviewed-by: Andreas Dilger Reviewed-by: Cyril Bordage Reviewed-by: Amir Shehata Tested-by: jenkins Tested-by: Shuichi Ihara Tested-by: Maloo --- lnet/include/lnet/lib-types.h | 3 +++ lnet/lnet/peer.c | 37 +++++++++++++++++++++++++++++-------- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h index 51c795c..f597a52 100644 --- a/lnet/include/lnet/lib-types.h +++ b/lnet/include/lnet/lib-types.h @@ -702,6 +702,9 @@ struct lnet_peer { /* cached peer aliveness */ bool lp_alive; + + /* timestamp of primary nid lock */ + __u64 lp_prim_lock_ts; }; /* diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c index 6d5cc87..31b7f86 100644 --- a/lnet/lnet/peer.c +++ b/lnet/lnet/peer.c @@ -1316,6 +1316,7 @@ LNetPrimaryNID(lnet_nid_t nid) /* force a full discovery cycle */ lp->lp_state |= LNET_PEER_FORCE_PING | LNET_PEER_FORCE_PUSH | LNET_PEER_LOCK_PRIMARY; + lp->lp_prim_lock_ts = ktime_get_real_ns(); spin_unlock(&lp->lp_lock); /* start discovery in the background. Messages to that @@ -1440,8 +1441,10 @@ lnet_peer_attach_peer_ni(struct lnet_peer *lp, lnet_peer_clr_non_mr_pref_nids(lp); } } - if (flags & LNET_PEER_LOCK_PRIMARY) + if (flags & LNET_PEER_LOCK_PRIMARY) { lp->lp_state |= LNET_PEER_LOCK_PRIMARY; + lp->lp_prim_lock_ts = ktime_get_real_ns(); + } spin_unlock(&lp->lp_lock); lp->lp_nnis++; @@ -1604,15 +1607,16 @@ lnet_peer_add_nid(struct lnet_peer *lp, lnet_nid_t nid, unsigned flags) lpni->lpni_peer_net->lpn_peer; int rtr_refcount = lp2->lp_rtr_refcount; - /* If the new peer that this NID belongs to is - * a primary NID for another peer which we're - * suppose to preserve the Primary for then we - * don't want to mess with it. But the - * configuration is wrong at this point, so we - * should flag both of these peers as in a bad + /* If there's another peer that this NID belongs to + * and the primary NID for another peer is locked, + * then, unless it is the only NID, we don't want + * to mess with it. + * But the configuration is wrong at this point, + * so we should flag both of these peers as in a bad * state */ - if (lp2->lp_state & LNET_PEER_LOCK_PRIMARY) { + if (lp2->lp_state & LNET_PEER_LOCK_PRIMARY && + lp2->lp_nnis > 1) { spin_lock(&lp->lp_lock); lp->lp_state |= LNET_PEER_BAD_CONFIG; spin_unlock(&lp->lp_lock); @@ -1621,6 +1625,23 @@ lnet_peer_add_nid(struct lnet_peer *lp, lnet_nid_t nid, unsigned flags) spin_unlock(&lp2->lp_lock); goto out_free_lpni; } + + /* If both peers have their primary NIDs locked, + * the NID which got locked the earliest should be + * kept as primary. In case if the peers were + * created with LNetPrimaryNID, this allows the + * first listed NID to stay primary as intended + * for the purpose of communicating with Lustre + * even if peer discovery succeeded using + * a different NID of MR peer. + */ + if (lp2->lp_state & LNET_PEER_LOCK_PRIMARY && + lp->lp_state & LNET_PEER_LOCK_PRIMARY && + lp2->lp_prim_lock_ts < lp->lp_prim_lock_ts) { + lp->lp_prim_lock_ts = lp2->lp_prim_lock_ts; + lp->lp_primary_nid = nid; + } + /* * if we're trying to delete a router it means * we're moving this peer NI to a new peer so must -- 1.8.3.1