LU-13806 lnet: Ensure proper peer, peer NI, peer net hierarchy

The MR design dictates that the peer nets and peer NIs are ordered
such that the peer net and peer NI for a peer's primary NID appears
first, followed by other peer NIs in the primary NID's peer net,
followed by other peer nets/NIs. This ordering is broken and it can
result in tripping an assertion if the primary NID of a peer is
deleted. Modify lnet_peer_attach_peer_ni() to check whether the
NI being attached is the peer's primary, and place it, and its
associated peer net, appropriately.

Modify lnet_peer_set_primary_nid() so that it updates the
lp_primary_nid before calling lnet_peer_add_nid() so that
lnet_peer_attach_peer_ni() can detect the situation where the
primary is changing and act appropriately.

Finally, modify lnet_peer_merge_data() to enforce the hierarchy
after it has finished merging the contents of the ping buffer. This
ensures we maintain the correct hierarchy in certain edge cases where
we've needed to reconcile two peers. e.g. if a peer adds a new
interface, the discovery push may arrive from that new interface
which will result in a second peer object being created which will
need to be reconciled with the original peer object.

HPE-bug-id: LUS-9630
Signed-off-by: Chris Horn <chris.horn@hpe.com>
Change-Id: I8397a24ba1ba0bba33846e7e97b8d60a8f26a1be
Reviewed-on: https://review.whamcloud.com/40985
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Reviewed-by: James Simmons <jsimmons@infradead.org>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c
index c15ae26..20ae558 100644 (file)
--- a/lnet/lnet/peer.c
+++ b/lnet/lnet/peer.c
@@ -1438,7 +1438,10 @@ lnet_peer_attach_peer_ni(struct lnet_peer *lp,
 
        /* Add peer_ni to peer_net */
        lpni->lpni_peer_net = lpn;
 
        /* Add peer_ni to peer_net */
        lpni->lpni_peer_net = lpn;

-       list_add_tail(&lpni->lpni_peer_nis, &lpn->lpn_peer_nis);
+       if (lp->lp_primary_nid == lpni->lpni_nid)
+               list_add(&lpni->lpni_peer_nis, &lpn->lpn_peer_nis);
+       else
+               list_add_tail(&lpni->lpni_peer_nis, &lpn->lpn_peer_nis);

        lnet_update_peer_net_healthv(lpni);
        lnet_peer_net_addref_locked(lpn);
 
        lnet_update_peer_net_healthv(lpni);
        lnet_peer_net_addref_locked(lpn);
 


@@ -1446,7 +1449,10 @@ lnet_peer_attach_peer_ni(struct lnet_peer *lp,
        if (!lpn->lpn_peer) {
                new_lpn = true;
                lpn->lpn_peer = lp;
        if (!lpn->lpn_peer) {
                new_lpn = true;
                lpn->lpn_peer = lp;

-               list_add_tail(&lpn->lpn_peer_nets, &lp->lp_peer_nets);
+               if (lp->lp_primary_nid == lpni->lpni_nid)
+                       list_add(&lpn->lpn_peer_nets, &lp->lp_peer_nets);
+               else
+                       list_add_tail(&lpn->lpn_peer_nets, &lp->lp_peer_nets);

                lnet_peer_addref_locked(lp);
        }
 
                lnet_peer_addref_locked(lp);
        }
 


@@ -1687,10 +1693,14 @@ lnet_peer_set_primary_nid(struct lnet_peer *lp, lnet_nid_t nid, unsigned flags)
 
        if (lp->lp_primary_nid == nid)
                goto out;
 
        if (lp->lp_primary_nid == nid)
                goto out;

+
+       lp->lp_primary_nid = nid;
+

        rc = lnet_peer_add_nid(lp, nid, flags);
        rc = lnet_peer_add_nid(lp, nid, flags);

-       if (rc)
+       if (rc) {
+               lp->lp_primary_nid = old;

                goto out;
                goto out;

-       lp->lp_primary_nid = nid;
+       }

 out:
        CDEBUG(D_NET, "peer %s NID %s: %d\n",
               libcfs_nid2str(old), libcfs_nid2str(nid), rc);
 out:
        CDEBUG(D_NET, "peer %s NID %s: %d\n",
               libcfs_nid2str(old), libcfs_nid2str(nid), rc);


@@ -2792,6 +2802,7 @@ static void lnet_discovery_event_handler(struct lnet_event *event)
 static int lnet_peer_merge_data(struct lnet_peer *lp,
                                struct lnet_ping_buffer *pbuf)
 {
 static int lnet_peer_merge_data(struct lnet_peer *lp,
                                struct lnet_ping_buffer *pbuf)
 {

+       struct lnet_peer_net *lpn;

        struct lnet_peer_ni *lpni;
        lnet_nid_t *curnis = NULL;
        struct lnet_ni_status *addnis = NULL;
        struct lnet_peer_ni *lpni;
        lnet_nid_t *curnis = NULL;
        struct lnet_ni_status *addnis = NULL;


@@ -2920,6 +2931,28 @@ static int lnet_peer_merge_data(struct lnet_peer *lp,
                                goto out;
                }
        }
                                goto out;
                }
        }

+
+       /* The peer net for the primary NID should be the first entry in the
+        * peer's lp_peer_nets list, and the peer NI for the primary NID should
+        * be the first entry in its peer net's lpn_peer_nis list.
+        */
+       lpni = lnet_find_peer_ni_locked(pbuf->pb_info.pi_ni[1].ns_nid);
+       if (!lpni) {
+               CERROR("Internal error: Failed to lookup peer NI for primary NID: %s\n",
+                      libcfs_nid2str(pbuf->pb_info.pi_ni[1].ns_nid));
+               goto out;
+       }
+
+       lnet_peer_ni_decref_locked(lpni);
+
+       lpn = lpni->lpni_peer_net;
+       if (lpn->lpn_peer_nets.prev != &lp->lp_peer_nets)
+               list_move(&lpn->lpn_peer_nets, &lp->lp_peer_nets);
+
+       if (lpni->lpni_peer_nis.prev != &lpni->lpni_peer_net->lpn_peer_nis)
+               list_move(&lpni->lpni_peer_nis,
+                         &lpni->lpni_peer_net->lpn_peer_nis);
+

        /*
         * Errors other than -ENOMEM are due to peers having been
         * configured with DLC. Ignore these because DLC overrides
        /*
         * Errors other than -ENOMEM are due to peers having been
         * configured with DLC. Ignore these because DLC overrides