LU-13502 lnet: Ensure LNet pings and pushes are always tracked

[fs/lustre-release.git] / lnet / lnet / peer.c

diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c
index 37a76c8..20f78fb 100644 (file)
--- a/lnet/lnet/peer.c
+++ b/lnet/lnet/peer.c
@@ -258,6 +258,7 @@ lnet_peer_alloc(lnet_nid_t nid)
        init_waitqueue_head(&lp->lp_dc_waitq);
        spin_lock_init(&lp->lp_lock);
        lp->lp_primary_nid = nid;
+       lp->lp_disc_src_nid = LNET_NID_ANY;
        if (lnet_peers_start_down())
                lp->lp_alive = false;
        else
@@ -277,7 +278,7 @@ lnet_peer_alloc(lnet_nid_t nid)
         * to ever use a different interface when sending messages to
         * myself.
         */
-       if (LNET_NETTYP(LNET_NIDNET(nid)) == LOLND)
+       if (nid == LNET_NID_LO_0)
                lp->lp_state = LNET_PEER_NO_DISCOVERY;
        lp->lp_cpt = lnet_nid_cpt_hash(nid, LNET_CPT_NUMBER);
 
@@ -1180,6 +1181,9 @@ LNetPrimaryNID(lnet_nid_t nid)
        int rc = 0;
        int cpt;
 
+       if (nid == LNET_NID_LO_0)
+               return LNET_NID_LO_0;
+
        cpt = lnet_net_lock_current();
        lpni = lnet_nid2peerni_locked(nid, LNET_NID_ANY, cpt);
        if (IS_ERR(lpni)) {
@@ -1189,6 +1193,11 @@ LNetPrimaryNID(lnet_nid_t nid)
        lp = lpni->lpni_peer_net->lpn_peer;
 
        while (!lnet_peer_is_uptodate(lp)) {
+               spin_lock(&lp->lp_lock);
+               /* force a full discovery cycle */
+               lp->lp_state |= LNET_PEER_FORCE_PING | LNET_PEER_FORCE_PUSH;
+               spin_unlock(&lp->lp_lock);
+
                rc = lnet_discover_peer_locked(lpni, cpt, true);
                if (rc)
                        goto out_decref;
@@ -2028,13 +2037,17 @@ void lnet_peer_push_event(struct lnet_event *ev)
                CDEBUG(D_NET, "Peer %s has discovery disabled\n",
                       libcfs_nid2str(lp->lp_primary_nid));
                /*
-                * If the peer is going from discovery enabled to
-                * discovery disabled, we need to reflect that in our
-                * representation of the peer.
+                * Mark the peer for deletion if we already know about it
+                * and it's going from discovery set to no discovery set
                 */
                if (!(lp->lp_state & (LNET_PEER_NO_DISCOVERY |
-                                     LNET_PEER_DISCOVERING)))
+                                     LNET_PEER_DISCOVERING)) &&
+                    lp->lp_state & LNET_PEER_DISCOVERED) {
+                       CDEBUG(D_NET, "Marking %s:0x%x for deletion\n",
+                              libcfs_nid2str(lp->lp_primary_nid),
+                              lp->lp_state);
                        lp->lp_state |= LNET_PEER_MARK_DELETION;
+               }
                lp->lp_state |= LNET_PEER_NO_DISCOVERY;
        } else if (lp->lp_state & LNET_PEER_NO_DISCOVERY) {
                CDEBUG(D_NET, "Peer %s has discovery enabled\n",
@@ -2288,6 +2301,8 @@ lnet_discovery_event_reply(struct lnet_peer *lp, struct lnet_event *ev)
 
        spin_lock(&lp->lp_lock);
 
+       lp->lp_disc_src_nid = ev->target.nid;
+
        /*
         * If some kind of error happened the contents of message
         * cannot be used. Set PING_FAILED to trigger a retry.
@@ -2632,7 +2647,7 @@ static int lnet_peer_merge_data(struct lnet_peer *lp,
         * present in curnis[] then this peer is for this node.
         */
        for (i = 0; i < ncurnis; i++) {
-               if (LNET_NETTYP(LNET_NIDNET(curnis[i])) == LOLND)
+               if (curnis[i] == LNET_NID_LO_0)
                        continue;
                for (j = 1; j < pbuf->pb_info.pi_nnis; j++) {
                        if (curnis[i] == pbuf->pb_info.pi_ni[j].ns_nid) {
@@ -2855,7 +2870,7 @@ __must_hold(&lp->lp_lock)
        if (pbuf->pb_info.pi_nnis <= 1)
                goto out;
        nid = pbuf->pb_info.pi_ni[1].ns_nid;
-       if (LNET_NETTYP(LNET_NIDNET(lp->lp_primary_nid)) == LOLND) {
+       if (lp->lp_primary_nid == LNET_NID_LO_0) {
                rc = lnet_peer_set_primary_nid(lp, nid, flags);
                if (!rc)
                        rc = lnet_peer_merge_data(lp, pbuf);
@@ -3054,6 +3069,21 @@ __must_hold(&lp->lp_lock)
        return rc ? rc : LNET_REDISCOVER_PEER;
 }
 
+/*
+ * Mark the peer as discovered.
+ */
+static int lnet_peer_discovered(struct lnet_peer *lp)
+__must_hold(&lp->lp_lock)
+{
+       lp->lp_state |= LNET_PEER_DISCOVERED;
+       lp->lp_state &= ~(LNET_PEER_DISCOVERING |
+                         LNET_PEER_REDISCOVER);
+
+       CDEBUG(D_NET, "peer %s\n", libcfs_nid2str(lp->lp_primary_nid));
+
+       return 0;
+}
+
 /* Active side of push. */
 static int lnet_peer_send_push(struct lnet_peer *lp)
 __must_hold(&lp->lp_lock)
@@ -3067,6 +3097,12 @@ __must_hold(&lp->lp_lock)
        /* Don't push to a non-multi-rail peer. */
        if (!(lp->lp_state & LNET_PEER_MULTI_RAIL)) {
                lp->lp_state &= ~LNET_PEER_FORCE_PUSH;
+               /* if peer's NIDs are uptodate then peer is discovered */
+               if (lp->lp_state & LNET_PEER_NIDS_UPTODATE) {
+                       rc = lnet_peer_discovered(lp);
+                       return rc;
+               }
+
                return 0;
        }
 
@@ -3084,11 +3120,11 @@ __must_hold(&lp->lp_lock)
        md.length    = LNET_PING_INFO_SIZE(pbuf->pb_nnis);
        md.threshold = 2; /* Put/Ack */
        md.max_size  = 0;
-       md.options   = 0;
+       md.options   = LNET_MD_TRACK_RESPONSE;
        md.handler   = the_lnet.ln_dc_handler;
        md.user_ptr  = lp;
 
-       rc = LNetMDBind(md, LNET_UNLINK, &lp->lp_push_mdh);
+       rc = LNetMDBind(&md, LNET_UNLINK, &lp->lp_push_mdh);
        if (rc) {
                lnet_ping_buffer_decref(pbuf);
                CERROR("Can't bind push source MD: %d\n", rc);
@@ -3106,10 +3142,18 @@ __must_hold(&lp->lp_lock)
                goto fail_unlink;
        }
 
-       rc = LNetPut(LNET_NID_ANY, lp->lp_push_mdh,
+       rc = LNetPut(lp->lp_disc_src_nid, lp->lp_push_mdh,
                     LNET_ACK_REQ, id, LNET_RESERVED_PORTAL,
                     LNET_PROTO_PING_MATCHBITS, 0, 0);
 
+       /*
+        * reset the discovery nid. There is no need to restrict sending
+        * from that source, if we call lnet_push_update_to_peers(). It'll
+        * get set to a specific NID, if we initiate discovery from the
+        * scratch
+        */
+       lp->lp_disc_src_nid = LNET_NID_ANY;
+
        if (rc)
                goto fail_unlink;
 
@@ -3152,22 +3196,6 @@ static void lnet_peer_discovery_error(struct lnet_peer *lp, int error)
 }
 
 /*
- * Mark the peer as discovered.
- */
-static int lnet_peer_discovered(struct lnet_peer *lp)
-__must_hold(&lp->lp_lock)
-{
-       lp->lp_state |= LNET_PEER_DISCOVERED;
-       lp->lp_state &= ~(LNET_PEER_DISCOVERING |
-                         LNET_PEER_REDISCOVER);
-
-       CDEBUG(D_NET, "peer %s\n", libcfs_nid2str(lp->lp_primary_nid));
-
-       return 0;
-}
-
-
-/*
  * Discovering this peer is taking too long. Cancel any Ping or Push
  * that discovery is waiting on by unlinking the relevant MDs. The
  * lnet_discovery_event_handler() will proceed from here and complete