LU-9480 lnet: introduce LNET_PEER_MULTI_RAIL flag bit

Add lp_state as a flag word to lnet_peer, and add lp_lock
to protect it. This lock needs to be taken whenever the
field is updated, because setting or clearing a bit is
a read-modify-write cycle.

The lp_multi_rail is removed, its function is replaced by
the new LNET_PEER_MULTI_RAIL flag bit.

The helper lnet_peer_is_multi_rail() tests the bit.

Test-Parameters: trivial
Signed-off-by: Olaf Weber <olaf@sgi.com>
Change-Id: I15034be7670bcb18460dc709accf675711a48113
Reviewed-on: https://review.whamcloud.com/25781
Reviewed-by: Olaf Weber <olaf.weber@hpe.com>
Reviewed-by: Amir Shehata <amir.shehata@intel.com>
Tested-by: Amir Shehata <amir.shehata@intel.com>

diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h
index 65bf460..8062d0e 100644 (file)
--- a/lnet/include/lnet/lib-lnet.h
+++ b/lnet/include/lnet/lib-lnet.h
@@ -912,4 +912,12 @@ lnet_peer_set_alive(struct lnet_peer_ni *lp)
                lnet_notify_locked(lp, 0, 1, lp->lpni_last_alive);
 }
 
+static inline bool
+lnet_peer_is_multi_rail(struct lnet_peer *lp)
+{
+       if (lp->lp_state & LNET_PEER_MULTI_RAIL)
+               return true;
+       return false;
+}
+
 #endif

diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h
index fbd4bc9..82d33c3 100644 (file)
--- a/lnet/include/lnet/lib-types.h
+++ b/lnet/include/lnet/lib-types.h
@@ -491,6 +491,8 @@ struct lnet_peer_ni {
        atomic_t                lpni_refcount;
        /* CPT this peer attached on */
        int                     lpni_cpt;
+       /* state flags -- protected by lpni_lock */
+       unsigned                lpni_state;
        /* # refs from lnet_route_t::lr_gateway */
        int                     lpni_rtr_refcount;
        /* sequence number used to round robin over peer nis within a net */
@@ -521,10 +523,15 @@ struct lnet_peer {
        /* primary NID of the peer */
        lnet_nid_t              lp_primary_nid;
 
-       /* peer is Multi-Rail enabled peer */
-       bool                    lp_multi_rail;
+       /* lock protecting peer state flags */
+       spinlock_t              lp_lock;
+
+       /* peer state flags */
+       unsigned                lp_state;
 };
 
+#define LNET_PEER_MULTI_RAIL   (1 << 0)
+
 struct lnet_peer_net {
        /* chain on peer block */
        struct list_head        lpn_on_peer_list;

diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c
index 2779eea..3fc8bdb 100644 (file)
--- a/lnet/lnet/lib-move.c
+++ b/lnet/lnet/lib-move.c
@@ -1462,8 +1462,8 @@ again:
                return -EHOSTUNREACH;
        }
 
-       if (!peer->lp_multi_rail && lnet_get_num_peer_nis(peer) > 1) {
-               lnet_net_unlock(cpt);
+       if (!lnet_peer_is_multi_rail(peer) &&
+            lnet_get_num_peer_nis(peer) > 1) {
                CERROR("peer %s is declared to be non MR capable, "
                       "yet configured with more than one NID\n",
                       libcfs_nid2str(dst_nid));
@@ -1489,7 +1489,7 @@ again:
 
        if (msg->msg_type == LNET_MSG_REPLY ||
            msg->msg_type == LNET_MSG_ACK ||
-           !peer->lp_multi_rail ||
+           !lnet_peer_is_multi_rail(peer) ||
            best_ni) {
                /*
                 * for replies we want to respond on the same peer_ni we
@@ -1535,7 +1535,7 @@ again:
                                * if the router is not multi-rail then use the best_gw
                                * found to send the message to
                                */
-                               if (!peer->lp_multi_rail)
+                               if (!lnet_peer_is_multi_rail(peer))
                                        best_lpni = best_gw;
                                else
                                        best_lpni = NULL;
@@ -1557,7 +1557,7 @@ again:
         * if the peer is not MR capable, then we should always send to it
         * using the first NI in the NET we determined.
         */
-       if (!peer->lp_multi_rail) {
+       if (!lnet_peer_is_multi_rail(peer)) {
                if (!best_lpni) {
                        lnet_net_unlock(cpt);
                        CERROR("no route to %s\n",

diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c
index 4310f8c..e11a028 100644 (file)
--- a/lnet/lnet/peer.c
+++ b/lnet/lnet/peer.c
@@ -216,6 +216,7 @@ lnet_peer_alloc(lnet_nid_t nid)
 
        INIT_LIST_HEAD(&lp->lp_on_lnet_peer_list);
        INIT_LIST_HEAD(&lp->lp_peer_nets);
+       spin_lock_init(&lp->lp_lock);
        lp->lp_primary_nid = nid;
 
        /* TODO: update flags */
@@ -792,13 +793,15 @@ lnet_peer_add(lnet_nid_t nid, bool mr)
         *
         * TODO: update flags if necessary
         */
-       if (mr && !lp->lp_multi_rail) {
-               lp->lp_multi_rail = true;
-       } else if (!mr && lp->lp_multi_rail) {
+       spin_lock(&lp->lp_lock);
+       if (mr && !(lp->lp_state & LNET_PEER_MULTI_RAIL)) {
+               lp->lp_state |= LNET_PEER_MULTI_RAIL;
+       } else if (!mr && (lp->lp_state & LNET_PEER_MULTI_RAIL)) {
                /* The mr state is sticky. */
-               CDEBUG(D_NET, "Cannot clear multi-flag from peer %s\n",
+               CDEBUG(D_NET, "Cannot clear multi-rail flag from peer %s\n",
                       libcfs_nid2str(nid));
        }
+       spin_unlock(&lp->lp_lock);
 
        return 0;
 }
@@ -811,15 +814,18 @@ lnet_peer_add_nid(struct lnet_peer *lp, lnet_nid_t nid, bool mr)
        LASSERT(lp);
        LASSERT(nid != LNET_NID_ANY);
 
-       if (!mr && !lp->lp_multi_rail) {
+       spin_lock(&lp->lp_lock);
+       if (!mr && !(lp->lp_state & LNET_PEER_MULTI_RAIL)) {
+               spin_unlock(&lp->lp_lock);
                CERROR("Cannot add nid %s to non-multi-rail peer %s\n",
                       libcfs_nid2str(nid),
                       libcfs_nid2str(lp->lp_primary_nid));
                return -EPERM;
        }
 
-       if (!lp->lp_multi_rail)
-               lp->lp_multi_rail = true;
+       if (!(lp->lp_state & LNET_PEER_MULTI_RAIL))
+               lp->lp_state |= LNET_PEER_MULTI_RAIL;
+       spin_unlock(&lp->lp_lock);
 
        lpni = lnet_find_peer_ni_locked(nid);
        if (!lpni)
@@ -1178,7 +1184,7 @@ int lnet_get_peer_info(__u32 idx, lnet_nid_t *primary_nid, lnet_nid_t *nid,
                return -ENOENT;
 
        *primary_nid = lp->lp_primary_nid;
-       *mr = lp->lp_multi_rail;
+       *mr = lnet_peer_is_multi_rail(lp);
        *nid = lpni->lpni_nid;
        snprintf(ni_info.cr_aliveness, LNET_MAX_STR_LEN, "NA");
        if (lnet_isrouter(lpni) ||