From: Chris Horn Date: Wed, 17 Aug 2022 18:11:58 +0000 (-0600) Subject: LU-14499 lnet: Revert "LU-13368 lnet: discard the callback" X-Git-Tag: 2.16.0-RC1~7 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=refs%2Fchanges%2F37%2F41937%2F9;p=fs%2Flustre-release.git LU-14499 lnet: Revert "LU-13368 lnet: discard the callback" The changes introduced by LU-13368 have been shown to cause the o2iblnd shutdown procedure to hang on lustre_rmmod as it infinitely waits for peers to disconnect. Those changes were also shown to cause a memory leak. Revert them. This reverts commit babf0232273467b7199ec9a7c36047b1968913df. HPE-bug-id: LUS-11089 Signed-off-by: Serguei Smirnov Signed-off-by: Chris Horn Change-Id: I489ae4af445b18df852ec35adc958c4fac33de09 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/41937 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Frank Sehr Reviewed-by: Oleg Drokin --- diff --git a/lnet/include/lnet/api.h b/lnet/include/lnet/api.h index eacf730..6c280d1 100644 --- a/lnet/include/lnet/api.h +++ b/lnet/include/lnet/api.h @@ -104,8 +104,7 @@ int LNetMDBind(const struct lnet_md *md_in, enum lnet_unlink unlink_in, struct lnet_handle_md *md_handle_out); -int __LNetMDUnlink(struct lnet_handle_md md_in, bool discard); -#define LNetMDUnlink(handle) __LNetMDUnlink(handle, false) +int LNetMDUnlink(struct lnet_handle_md md_in); void lnet_assert_handler_unused(lnet_handler_t handler); /** @} lnet_md */ diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index 81bf473..eba26b6 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -772,7 +772,6 @@ void lnet_set_reply_msg_len(struct lnet_ni *ni, struct lnet_msg *msg, void lnet_detach_rsp_tracker(struct lnet_libmd *md, int cpt); void lnet_clean_zombie_rstqs(void); -bool lnet_md_discarded(struct lnet_libmd *md); void lnet_finalize(struct lnet_msg *msg, int rc); bool lnet_send_error_simulation(struct lnet_msg *msg, enum lnet_msg_hstatus *hstatus); diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h index 376fde6..9396b88 100644 --- a/lnet/include/lnet/lib-types.h +++ b/lnet/include/lnet/lib-types.h @@ -213,7 +213,6 @@ struct lnet_libmd { * call. */ #define LNET_MD_FLAG_HANDLING BIT(3) -#define LNET_MD_FLAG_DISCARD BIT(4) #define LNET_MD_FLAG_GPU BIT(5) /**< Special mapping needs */ static inline bool lnet_md_is_gpu(struct lnet_libmd *md) diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c index 29993c6..c45c6fa 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.c +++ b/lnet/klnds/o2iblnd/o2iblnd.c @@ -3579,7 +3579,6 @@ kiblnd_base_startup(struct net *ns) spin_lock_init(&kiblnd_data.kib_connd_lock); INIT_LIST_HEAD(&kiblnd_data.kib_connd_conns); - INIT_LIST_HEAD(&kiblnd_data.kib_connd_waits); INIT_LIST_HEAD(&kiblnd_data.kib_connd_zombies); INIT_LIST_HEAD(&kiblnd_data.kib_reconn_list); INIT_LIST_HEAD(&kiblnd_data.kib_reconn_wait); diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h index 92c95ea..c091050 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.h +++ b/lnet/klnds/o2iblnd/o2iblnd.h @@ -455,8 +455,6 @@ struct kib_data { struct list_head kib_reconn_list; /* peers wait for reconnection */ struct list_head kib_reconn_wait; - /* connections wait for completion */ - struct list_head kib_connd_waits; /* * The second that peers are pulled out from \a kib_reconn_wait * for reconnection. @@ -595,8 +593,6 @@ struct kib_conn { __u16 ibc_queue_depth; /* connections max frags */ __u16 ibc_max_frags; - /* count of timeout txs waiting on cq */ - __u16 ibc_waits; /* receive buffers owned */ unsigned int ibc_nrx:16; /* scheduled for attention */ diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c index a1c5e3b..b28b446 100644 --- a/lnet/klnds/o2iblnd/o2iblnd_cb.c +++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c @@ -2280,10 +2280,6 @@ kiblnd_abort_txs(struct kib_conn *conn, struct list_head *txs) if (tx->tx_sending == 0) { tx->tx_queued = 0; list_move(&tx->tx_list, &zombies); - } else { - /* keep tx until cq destroy */ - list_move(&tx->tx_list, &conn->ibc_zombie_txs); - conn->ibc_waits++; } } @@ -2298,31 +2294,6 @@ kiblnd_abort_txs(struct kib_conn *conn, struct list_head *txs) kiblnd_txlist_done(&zombies, -ECONNABORTED, LNET_MSG_STATUS_OK); } -static bool -kiblnd_tx_may_discard(struct kib_conn *conn) -{ - bool rc = false; - struct kib_tx *nxt; - struct kib_tx *tx; - - spin_lock(&conn->ibc_lock); - - list_for_each_entry_safe(tx, nxt, &conn->ibc_zombie_txs, tx_list) { - if (tx->tx_sending > 0 && tx->tx_lntmsg[0] && - lnet_md_discarded(tx->tx_lntmsg[0]->msg_md)) { - tx->tx_sending--; - if (tx->tx_sending == 0) { - kiblnd_conn_decref(tx->tx_conn); - tx->tx_conn = NULL; - rc = true; - } - } - } - - spin_unlock(&conn->ibc_lock); - return rc; -} - static void kiblnd_finalise_conn(struct kib_conn *conn) { @@ -3593,9 +3564,8 @@ kiblnd_check_txs_locked(struct kib_conn *conn, struct list_head *txs) } if (ktime_compare(ktime_get(), tx->tx_deadline) >= 0) { - CERROR("Timed out tx: %s(WSQ:%d%d%d), %lld seconds\n", + CERROR("Timed out tx: %s, %lld seconds\n", kiblnd_queue2str(conn, txs), - tx->tx_waiting, tx->tx_sending, tx->tx_queued, kiblnd_timeout() + ktime_ms_delta(ktime_get(), tx->tx_deadline) / MSEC_PER_SEC); @@ -3827,23 +3797,15 @@ kiblnd_connd(void *arg) conn = list_first_entry_or_null(&kiblnd_data.kib_connd_conns, struct kib_conn, ibc_list); if (conn) { - int wait; - list_del(&conn->ibc_list); spin_unlock_irqrestore(lock, flags); dropped_lock = true; kiblnd_disconnect_conn(conn); - wait = conn->ibc_waits; - if (wait == 0) /* keep ref for connd_wait, see below */ - kiblnd_conn_decref(conn); + kiblnd_conn_decref(conn); spin_lock_irqsave(lock, flags); - - if (wait) - list_add_tail(&conn->ibc_list, - &kiblnd_data.kib_connd_waits); } while (reconn < KIB_RECONN_BREAK) { @@ -3872,22 +3834,6 @@ kiblnd_connd(void *arg) spin_lock_irqsave(lock, flags); } - conn = list_first_entry_or_null(&kiblnd_data.kib_connd_waits, - struct kib_conn, ibc_list); - if (conn) { - list_del(&conn->ibc_list); - spin_unlock_irqrestore(lock, flags); - - dropped_lock = kiblnd_tx_may_discard(conn); - if (dropped_lock) - kiblnd_conn_decref(conn); - - spin_lock_irqsave(lock, flags); - if (!dropped_lock) - list_add_tail(&conn->ibc_list, - &kiblnd_data.kib_connd_waits); - } - /* careful with the jiffy wrap... */ timeout = (int)(deadline - jiffies); if (timeout <= 0) { diff --git a/lnet/lnet/lib-md.c b/lnet/lnet/lib-md.c index d153003..4238b03 100644 --- a/lnet/lnet/lib-md.c +++ b/lnet/lnet/lib-md.c @@ -442,7 +442,7 @@ EXPORT_SYMBOL(LNetMDBind); * \retval -ENOENT If \a mdh does not point to a valid MD object. */ int -__LNetMDUnlink(struct lnet_handle_md mdh, bool discard) +LNetMDUnlink(struct lnet_handle_md mdh) { struct lnet_event ev; struct lnet_libmd *md = NULL; @@ -476,9 +476,6 @@ __LNetMDUnlink(struct lnet_handle_md mdh, bool discard) handler = md->md_handler; } - if (discard) - md->md_flags |= LNET_MD_FLAG_DISCARD; - if (md->md_rspt_ptr != NULL) lnet_detach_rsp_tracker(md, cpt); @@ -491,22 +488,4 @@ __LNetMDUnlink(struct lnet_handle_md mdh, bool discard) return 0; } -EXPORT_SYMBOL(__LNetMDUnlink); - -bool -lnet_md_discarded(struct lnet_libmd *md) -{ - bool rc; - int cpt; - - if (md == NULL) - return false; - - cpt = lnet_cpt_of_cookie(md->md_lh.lh_cookie); - lnet_res_lock(cpt); - rc = md->md_flags & LNET_MD_FLAG_DISCARD; - lnet_res_unlock(cpt); - - return rc; -} -EXPORT_SYMBOL(lnet_md_discarded); +EXPORT_SYMBOL(LNetMDUnlink); diff --git a/lustre/include/lustre_net.h b/lustre/include/lustre_net.h index ced8f0a..046fadf 100644 --- a/lustre/include/lustre_net.h +++ b/lustre/include/lustre_net.h @@ -2466,10 +2466,8 @@ ptlrpc_client_recv(struct ptlrpc_request *req) return req->rq_receiving_reply; } -#define ptlrpc_cli_wait_unlink(req) __ptlrpc_cli_wait_unlink(req, NULL) - static inline int -__ptlrpc_cli_wait_unlink(struct ptlrpc_request *req, bool *discard) +ptlrpc_client_recv_or_unlink(struct ptlrpc_request *req) { int rc; @@ -2483,15 +2481,6 @@ __ptlrpc_cli_wait_unlink(struct ptlrpc_request *req, bool *discard) return 1; } - if (discard) { - *discard = false; - if (req->rq_reply_unlinked && req->rq_req_unlinked == 0) { - *discard = true; - spin_unlock(&req->rq_lock); - return 1; /* Should call again after LNetMDUnlink */ - } - } - rc = !req->rq_req_unlinked || !req->rq_reply_unlinked || req->rq_receiving_reply; spin_unlock(&req->rq_lock); diff --git a/lustre/ptlrpc/client.c b/lustre/ptlrpc/client.c index 23f41db..3795720 100644 --- a/lustre/ptlrpc/client.c +++ b/lustre/ptlrpc/client.c @@ -1946,7 +1946,7 @@ int ptlrpc_check_set(const struct lu_env *env, struct ptlrpc_request_set *set) * not corrupt any data. */ if (req->rq_phase == RQ_PHASE_UNREG_RPC && - ptlrpc_cli_wait_unlink(req)) + ptlrpc_client_recv_or_unlink(req)) continue; if (req->rq_phase == RQ_PHASE_UNREG_BULK && ptlrpc_client_bulk_active(req)) @@ -1984,7 +1984,7 @@ int ptlrpc_check_set(const struct lu_env *env, struct ptlrpc_request_set *set) /* * Check if we still need to wait for unlink. */ - if (ptlrpc_cli_wait_unlink(req) || + if (ptlrpc_client_recv_or_unlink(req) || ptlrpc_client_bulk_active(req)) continue; /* If there is no need to resend, fail it now. */ @@ -2825,7 +2825,6 @@ EXPORT_SYMBOL(ptlrpc_req_xid); */ static int ptlrpc_unregister_reply(struct ptlrpc_request *request, int async) { - bool discard = false; /* * Might sleep. */ @@ -2840,18 +2839,15 @@ static int ptlrpc_unregister_reply(struct ptlrpc_request *request, int async) /* * Nothing left to do. */ - if (!__ptlrpc_cli_wait_unlink(request, &discard)) + if (!ptlrpc_client_recv_or_unlink(request)) RETURN(1); LNetMDUnlink(request->rq_reply_md_h); - if (discard) /* Discard the request-out callback */ - __LNetMDUnlink(request->rq_req_md_h, discard); - /* * Let's check it once again. */ - if (!ptlrpc_cli_wait_unlink(request)) + if (!ptlrpc_client_recv_or_unlink(request)) RETURN(1); /* Move to "Unregistering" phase as reply was not unlinked yet. */ @@ -2880,7 +2876,7 @@ static int ptlrpc_unregister_reply(struct ptlrpc_request *request, int async) while (seconds > 0 && wait_event_idle_timeout( *wq, - !ptlrpc_cli_wait_unlink(request), + !ptlrpc_client_recv_or_unlink(request), cfs_time_seconds(1)) == 0) seconds -= 1; if (seconds > 0) { diff --git a/lustre/ptlrpc/niobuf.c b/lustre/ptlrpc/niobuf.c index 0d5687f..740592b 100644 --- a/lustre/ptlrpc/niobuf.c +++ b/lustre/ptlrpc/niobuf.c @@ -104,15 +104,12 @@ static int ptl_send_buf(struct lnet_handle_md *mdh, void *base, int len, RETURN(0); } -#define mdunlink_iterate_helper(mds, count) \ - __mdunlink_iterate_helper(mds, count, false) -static void __mdunlink_iterate_helper(struct lnet_handle_md *bd_mds, - int count, bool discard) +static void mdunlink_iterate_helper(struct lnet_handle_md *bd_mds, int count) { int i; for (i = 0; i < count; i++) - __LNetMDUnlink(bd_mds[i], discard); + LNetMDUnlink(bd_mds[i]); } #ifdef HAVE_SERVER_SUPPORT @@ -289,7 +286,7 @@ void ptlrpc_abort_bulk(struct ptlrpc_bulk_desc *desc) * but we must still wait_event_idle_timeout() in this case, to give * us a chance to run server_bulk_callback() */ - __mdunlink_iterate_helper(desc->bd_mds, desc->bd_md_max_brw, true); + mdunlink_iterate_helper(desc->bd_mds, desc->bd_md_max_brw); for (;;) { /* Network access will complete in finite time but the HUGE