LU-13277 lnet: Discovery thread can deadlock on shutdown

[fs/lustre-release.git] / lnet / lnet / peer.c

diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c
index 6336410..fe4f028 100644 (file)
--- a/lnet/lnet/peer.c
+++ b/lnet/lnet/peer.c
@@ -104,7 +104,7 @@ lnet_peer_tables_destroy(void)
                for (j = 0; j < LNET_PEER_HASH_SIZE; j++)
                        LASSERT(list_empty(&hash[j]));
 
-               LIBCFS_FREE(hash, LNET_PEER_HASH_SIZE * sizeof(*hash));
+               CFS_FREE_PTR_ARRAY(hash, LNET_PEER_HASH_SIZE);
        }
 
        cfs_percpt_free(the_lnet.ln_peer_tables);
@@ -587,8 +587,7 @@ lnet_peer_ni_finalize_wait(struct lnet_peer_table *ptable)
                               "Waiting for %d zombies on peer table\n",
                               ptable->pt_zombies);
                }
-               set_current_state(TASK_UNINTERRUPTIBLE);
-               schedule_timeout(cfs_time_seconds(1) >> 1);
+               schedule_timeout_uninterruptible(cfs_time_seconds(1) >> 1);
                spin_lock(&ptable->pt_zombie_lock);
        }
        spin_unlock(&ptable->pt_zombie_lock);
@@ -1050,7 +1049,7 @@ lnet_peer_add_pref_nid(struct lnet_peer_ni *lpni, lnet_nid_t nid)
 
        if (oldnids) {
                size = sizeof(*nids) * (lpni->lpni_pref_nnids - 1);
-               LIBCFS_FREE(oldnids, sizeof(*oldnids) * size);
+               CFS_FREE_PTR_ARRAY(oldnids, size);
        }
 out:
        if (rc == -EEXIST && (lpni->lpni_state & LNET_PEER_NI_NON_MR_PREF)) {
@@ -1130,7 +1129,7 @@ lnet_peer_del_pref_nid(struct lnet_peer_ni *lpni, lnet_nid_t nid)
 
        if (oldnids) {
                size = sizeof(*nids) * (lpni->lpni_pref_nnids + 1);
-               LIBCFS_FREE(oldnids, sizeof(*oldnids) * size);
+               CFS_FREE_PTR_ARRAY(oldnids, size);
        }
 out:
        CDEBUG(D_NET, "peer %s nid %s: %d\n",
@@ -1276,6 +1275,7 @@ lnet_peer_attach_peer_ni(struct lnet_peer *lp,
        /* Add peer_ni to peer_net */
        lpni->lpni_peer_net = lpn;
        list_add_tail(&lpni->lpni_peer_nis, &lpn->lpn_peer_nis);
+       lnet_update_peer_net_healthv(lpni);
        lnet_peer_net_addref_locked(lpn);
 
        /* Add peer_net to peer */
@@ -1720,10 +1720,9 @@ lnet_destroy_peer_ni_locked(struct lnet_peer_ni *lpni)
        ptable->pt_zombies--;
        spin_unlock(&ptable->pt_zombie_lock);
 
-       if (lpni->lpni_pref_nnids > 1) {
-               LIBCFS_FREE(lpni->lpni_pref.nids,
-                       sizeof(*lpni->lpni_pref.nids) * lpni->lpni_pref_nnids);
-       }
+       if (lpni->lpni_pref_nnids > 1)
+               CFS_FREE_PTR_ARRAY(lpni->lpni_pref.nids, lpni->lpni_pref_nnids);
+
        LIBCFS_FREE(lpni, sizeof(*lpni));
 
        lnet_peer_net_decref_locked(lpn);
@@ -2583,10 +2582,10 @@ static int lnet_peer_merge_data(struct lnet_peer *lp,
                lp->lp_state &= ~LNET_PEER_ROUTER_ENABLED;
        spin_unlock(&lp->lp_lock);
 
-       nnis = MAX(lp->lp_nnis, pbuf->pb_info.pi_nnis);
-       LIBCFS_ALLOC(curnis, nnis * sizeof(*curnis));
-       LIBCFS_ALLOC(addnis, nnis * sizeof(*addnis));
-       LIBCFS_ALLOC(delnis, nnis * sizeof(*delnis));
+       nnis = max_t(int, lp->lp_nnis, pbuf->pb_info.pi_nnis);
+       CFS_ALLOC_PTR_ARRAY(curnis, nnis);
+       CFS_ALLOC_PTR_ARRAY(addnis, nnis);
+       CFS_ALLOC_PTR_ARRAY(delnis, nnis);
        if (!curnis || !addnis || !delnis) {
                rc = -ENOMEM;
                goto out;
@@ -2690,9 +2689,9 @@ static int lnet_peer_merge_data(struct lnet_peer *lp,
         */
        rc = 0;
 out:
-       LIBCFS_FREE(curnis, nnis * sizeof(*curnis));
-       LIBCFS_FREE(addnis, nnis * sizeof(*addnis));
-       LIBCFS_FREE(delnis, nnis * sizeof(*delnis));
+       CFS_FREE_PTR_ARRAY(curnis, nnis);
+       CFS_FREE_PTR_ARRAY(addnis, nnis);
+       CFS_FREE_PTR_ARRAY(delnis, nnis);
        lnet_ping_buffer_decref(pbuf);
        CDEBUG(D_NET, "peer %s (%p): %d\n", libcfs_nid2str(lp->lp_primary_nid), lp, rc);
 
@@ -2977,10 +2976,10 @@ __must_hold(&lp->lp_lock)
        pnid = lnet_peer_select_nid(lp);
        lnet_net_unlock(cpt);
 
-       nnis = MAX(lp->lp_data_nnis, LNET_INTERFACES_MIN);
+       nnis = max(lp->lp_data_nnis, LNET_INTERFACES_MIN);
 
        rc = lnet_send_ping(pnid, &lp->lp_ping_mdh, nnis, lp,
-                           the_lnet.ln_dc_eqh, false);
+                           the_lnet.ln_dc_eq, false);
 
        /*
         * if LNetMDBind in lnet_send_ping fails we need to decrement the
@@ -3072,7 +3071,7 @@ __must_hold(&lp->lp_lock)
        md.threshold = 2; /* Put/Ack */
        md.max_size  = 0;
        md.options   = 0;
-       md.eq_handle = the_lnet.ln_dc_eqh;
+       md.eq_handle = the_lnet.ln_dc_eq;
        md.user_ptr  = lp;
 
        rc = LNetMDBind(md, LNET_UNLINK, &lp->lp_push_mdh);
@@ -3281,7 +3280,6 @@ static int lnet_peer_discovery(void *arg)
        wait_for_completion(&the_lnet.ln_started);
 
        CDEBUG(D_NET, "started\n");
-       cfs_block_allsigs();
 
        for (;;) {
                if (lnet_peer_discovery_wait_for_work())
@@ -3293,8 +3291,10 @@ static int lnet_peer_discovery(void *arg)
                        lnet_push_target_resize();
 
                lnet_net_lock(LNET_LOCK_EX);
-               if (the_lnet.ln_dc_state == LNET_DC_STATE_STOPPING)
+               if (the_lnet.ln_dc_state == LNET_DC_STATE_STOPPING) {
+                       lnet_net_unlock(LNET_LOCK_EX);
                        break;
+               }
 
                /*
                 * Process all incoming discovery work requests.  When
@@ -3389,7 +3389,7 @@ static int lnet_peer_discovery(void *arg)
 
        /* Queue cleanup 2: wait for the expired queue to clear. */
        while (!list_empty(&the_lnet.ln_dc_expired))
-               schedule_timeout(cfs_time_seconds(1));
+               schedule_timeout_uninterruptible(cfs_time_seconds(1));
 
        /* Queue cleanup 3: clear the request queue. */
        lnet_net_lock(LNET_LOCK_EX);
@@ -3401,8 +3401,8 @@ static int lnet_peer_discovery(void *arg)
        }
        lnet_net_unlock(LNET_LOCK_EX);
 
-       LNetEQFree(the_lnet.ln_dc_eqh);
-       LNetInvalidateEQHandle(&the_lnet.ln_dc_eqh);
+       LNetEQFree(the_lnet.ln_dc_eq);
+       the_lnet.ln_dc_eq = NULL;
 
        the_lnet.ln_dc_state = LNET_DC_STATE_SHUTDOWN;
        wake_up(&the_lnet.ln_dc_waitq);
@@ -3416,13 +3416,14 @@ static int lnet_peer_discovery(void *arg)
 int lnet_peer_discovery_start(void)
 {
        struct task_struct *task;
-       int rc;
+       int rc = 0;
 
        if (the_lnet.ln_dc_state != LNET_DC_STATE_SHUTDOWN)
                return -EALREADY;
 
-       rc = LNetEQAlloc(0, lnet_discovery_event_handler, &the_lnet.ln_dc_eqh);
-       if (rc != 0) {
+       the_lnet.ln_dc_eq = LNetEQAlloc(lnet_discovery_event_handler);
+       if (IS_ERR(the_lnet.ln_dc_eq)) {
+               rc = PTR_ERR(the_lnet.ln_dc_eq);
                CERROR("Can't allocate discovery EQ: %d\n", rc);
                return rc;
        }
@@ -3433,8 +3434,8 @@ int lnet_peer_discovery_start(void)
                rc = PTR_ERR(task);
                CERROR("Can't start peer discovery thread: %d\n", rc);
 
-               LNetEQFree(the_lnet.ln_dc_eqh);
-               LNetInvalidateEQHandle(&the_lnet.ln_dc_eqh);
+               LNetEQFree(the_lnet.ln_dc_eq);
+               the_lnet.ln_dc_eq = NULL;
 
                the_lnet.ln_dc_state = LNET_DC_STATE_SHUTDOWN;
        }
@@ -3701,7 +3702,7 @@ lnet_peer_ni_add_to_recoveryq_locked(struct lnet_peer_ni *lpni)
 
        if (list_empty(&lpni->lpni_recovery) &&
            atomic_read(&lpni->lpni_healthv) < LNET_MAX_HEALTH_VALUE) {
-               CERROR("lpni %s added to recovery queue. Health = %d\n",
+               CDEBUG(D_NET, "lpni %s added to recovery queue. Health = %d\n",
                        libcfs_nid2str(lpni->lpni_nid),
                        atomic_read(&lpni->lpni_healthv));
                list_add_tail(&lpni->lpni_recovery, &the_lnet.ln_mt_peerNIRecovq);