LU-14668 lnet: Peers added via kernel API should be permanent

The LNetAddPeer() API allows Lustre to predefine the Peer for LNet.
Originally these peers would be temporary and potentially re-created
via discovery. Instead, let's make these peers permanent. This allows
Lustre to dictate the primary NID of the peer. LNet makes sure this
primary NID is not changed afterwards.

Lustre-change: https://review.whamcloud.com/43788
Lustre-commit: 41733dadd8ad0e87e44dd19e25e576e90484cb9b

Test-Parameters: trivial
Signed-off-by: Amir Shehata <ashehata@whamcloud.com>
Signed-off-by: Chris Horn <chris.horn@hpe.com>
Change-Id: I3f54c04719c9e0374176682af08183f0c93ef737
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51131
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Frank Sehr <fsehr@whamcloud.com>
Reviewed-by: Cyril Bordage <cbordage@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h
index 223c6d3..a1261d1 100644 (file)
--- a/lnet/include/lnet/lib-lnet.h
+++ b/lnet/include/lnet/lib-lnet.h
@@ -1011,8 +1011,8 @@ void lnet_peer_clr_pref_rtrs(struct lnet_peer_ni *lpni);
 int lnet_peer_add_pref_rtr(struct lnet_peer_ni *lpni, struct lnet_nid *nid);
 int lnet_peer_ni_set_non_mr_pref_nid(struct lnet_peer_ni *lpni,
                                     struct lnet_nid *nid);
-int lnet_add_peer_ni(lnet_nid_t key_nid, lnet_nid_t nid, bool mr, bool temp);
 int lnet_del_peer_ni(lnet_nid_t key_nid, lnet_nid_t nid);
+int lnet_user_add_peer_ni(lnet_nid_t prim_nid, struct lnet_nid *nid, bool mr);
 int lnet_get_peer_info(struct lnet_ioctl_peer_cfg *cfg, void __user *bulk);
 int lnet_get_peer_ni_info(__u32 peer_index, __u64 *nid,
                          char alivness[LNET_MAX_STR_LEN],

diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c
index b99c85b..08c28d5 100644 (file)
--- a/lnet/lnet/api-ni.c
+++ b/lnet/lnet/api-ni.c
@@ -4204,9 +4204,10 @@ LNetCtl(unsigned int cmd, void *arg)
                        return -EINVAL;
 
                mutex_lock(&the_lnet.ln_api_mutex);
-               rc = lnet_add_peer_ni(cfg->prcfg_prim_nid,
-                                     cfg->prcfg_cfg_nid,
-                                     cfg->prcfg_mr, false);
+               lnet_nid4_to_nid(cfg->prcfg_cfg_nid, &nid);
+               rc = lnet_user_add_peer_ni(cfg->prcfg_prim_nid,
+                                          &nid,
+                                          cfg->prcfg_mr);
                mutex_unlock(&the_lnet.ln_api_mutex);
                return rc;
        }

diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c
index 01eb9aa..040fa36 100644 (file)
--- a/lnet/lnet/peer.c
+++ b/lnet/lnet/peer.c
@@ -47,6 +47,8 @@
 #define LNET_REDISCOVER_PEER   (1)
 
 static int lnet_peer_queue_for_discovery(struct lnet_peer *lp);
+static int lnet_add_peer_ni(lnet_nid_t prim_nid, lnet_nid_t nid, bool mr,
+                           unsigned int flags);
 
 static void
 lnet_peer_remove_from_remote_list(struct lnet_peer_ni *lpni)
@@ -1428,7 +1430,8 @@ LNetAddPeer(lnet_nid_t *nids, __u32 num_nids)
 
                if (!pnid) {
                        pnid = nids[i];
-                       rc = lnet_add_peer_ni(pnid, LNET_NID_ANY, mr, true);
+                       rc = lnet_add_peer_ni(pnid, LNET_NID_ANY, mr,
+                                             LNET_PEER_LOCK_PRIMARY);
                        if (rc == -EALREADY) {
                                struct lnet_peer *lp;
 
@@ -1443,9 +1446,11 @@ LNetAddPeer(lnet_nid_t *nids, __u32 num_nids)
                                lnet_peer_decref_locked(lp);
                        }
                } else if (lnet_peer_discovery_disabled) {
-                       rc = lnet_add_peer_ni(nids[i], LNET_NID_ANY, mr, true);
+                       rc = lnet_add_peer_ni(nids[i], LNET_NID_ANY, mr,
+                                             LNET_PEER_LOCK_PRIMARY);
                } else {
-                       rc = lnet_add_peer_ni(pnid, nids[i], mr, true);
+                       rc = lnet_add_peer_ni(pnid, nids[i], mr,
+                                             LNET_PEER_LOCK_PRIMARY);
                }
 
                if (rc && rc != -EEXIST)
@@ -1977,20 +1982,18 @@ out:
  * The caller must hold ln_api_mutex. This prevents the peer from
  * being created/modified/deleted by a different thread.
  */
-int
-lnet_add_peer_ni(lnet_nid_t prim_nid, lnet_nid_t nid, bool mr, bool temp)
+static int
+lnet_add_peer_ni(lnet_nid_t prim_nid, lnet_nid_t nid, bool mr,
+                unsigned int flags)
+__must_hold(&the_lnet.ln_api_mutex)
 {
        struct lnet_peer *lp = NULL;
        struct lnet_peer_ni *lpni;
-       unsigned int flags = 0;
 
        /* The prim_nid must always be specified */
        if (prim_nid == LNET_NID_ANY)
                return -EINVAL;
 
-       if (!temp)
-               flags = LNET_PEER_CONFIGURED;
-
        if (mr)
                flags |= LNET_PEER_MULTI_RAIL;
 
@@ -2008,13 +2011,6 @@ lnet_add_peer_ni(lnet_nid_t prim_nid, lnet_nid_t nid, bool mr, bool temp)
        lnet_peer_ni_decref_locked(lpni);
        lp = lpni->lpni_peer_net->lpn_peer;
 
-       /* Peer must have been configured. */
-       if (!temp && !(lp->lp_state & LNET_PEER_CONFIGURED)) {
-               CDEBUG(D_NET, "peer %s was not configured\n",
-                      libcfs_nid2str(prim_nid));
-               return -ENOENT;
-       }
-
        /* Primary NID must match */
        if (lnet_nid_to_nid4(&lp->lp_primary_nid) != prim_nid) {
                CDEBUG(D_NET, "prim_nid %s is not primary for peer %s\n",
@@ -2030,9 +2026,24 @@ lnet_add_peer_ni(lnet_nid_t prim_nid, lnet_nid_t nid, bool mr, bool temp)
                return -EPERM;
        }
 
+       if ((flags & LNET_PEER_LOCK_PRIMARY) &&
+           (lnet_peer_is_uptodate(lp) &&
+            (lp->lp_state & LNET_PEER_LOCK_PRIMARY))) {
+               CDEBUG(D_NET,
+                      "Don't add temporary peer NI for uptodate peer %s\n",
+                      libcfs_nidstr(&lp->lp_primary_nid));
+               return -EINVAL;
+       }
+
        return lnet_peer_add_nid(lp, nid, flags);
 }
 
+int lnet_user_add_peer_ni(lnet_nid_t prim_nid, struct lnet_nid *nid, bool mr)
+{
+       return lnet_add_peer_ni(prim_nid, lnet_nid_to_nid4(nid), mr,
+                               LNET_PEER_CONFIGURED);
+}
+
 /*
  * Implementation of IOC_LIBCFS_DEL_PEER_NI.
  *