From a12e5832df9338edbd37f239ce77b487933a1007 Mon Sep 17 00:00:00 2001 From: Chris Horn Date: Mon, 27 Jul 2015 09:59:57 -0400 Subject: [PATCH] LU-6261 gnilnd: kgnilnd_check_rdma_cq race in error path. It is possible for multiple threads to both be processing a transaction error that causes the connection to close. If the close path stalls after changing the conn state to CLOSING, the tx_done path will put a tx in purgatory since the conn state is not ESTABLISHED anymore. When the close path continues, it expects to find that the conn->gnc_mdd_list is empty and assert if it isn't empty. Change the error code for the tx_done path so that we don't put this tx in purgatory since we are closing the connection anyway. Signed-off-by: Chris Horn Change-Id: I4458522f16508eb53d380f62320c65c7bf84657a Reviewed-on: http://review.whamcloud.com/15439 Tested-by: Jenkins Reviewed-by: James Shimek Reviewed-by: James Simmons Reviewed-by: Doug Oucharek Tested-by: Maloo Reviewed-by: Oleg Drokin --- lnet/klnds/gnilnd/gnilnd.c | 4 ++++ lnet/klnds/gnilnd/gnilnd.h | 2 ++ lnet/klnds/gnilnd/gnilnd_api_wrap.h | 1 + lnet/klnds/gnilnd/gnilnd_cb.c | 8 +++++++- 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lnet/klnds/gnilnd/gnilnd.c b/lnet/klnds/gnilnd/gnilnd.c index e110c27..be104c3 100644 --- a/lnet/klnds/gnilnd/gnilnd.c +++ b/lnet/klnds/gnilnd/gnilnd.c @@ -671,6 +671,10 @@ kgnilnd_close_conn_locked(kgn_conn_t *conn, int error) conn->gnc_state = GNILND_CONN_CLOSING; } + if (CFS_FAIL_CHECK(CFS_FAIL_GNI_RDMA_CQ_ERROR)) { + msleep_interruptible(MSEC_PER_SEC); + } + /* leave on peer->gnp_conns to make sure we don't let the reaper * or others try to unlink this peer until the conn is fully * processed for closing */ diff --git a/lnet/klnds/gnilnd/gnilnd.h b/lnet/klnds/gnilnd/gnilnd.h index 0b599ae..785142c 100644 --- a/lnet/klnds/gnilnd/gnilnd.h +++ b/lnet/klnds/gnilnd/gnilnd.h @@ -139,6 +139,8 @@ /* Max number of connections to keep in purgatory per peer */ #define GNILND_PURGATORY_MAX 5 +/* Closing, don't put in purgatory */ +#define GNILND_NOPURG 222 /* payload size to add to the base mailbox size * This is subtracting 2 from the concurrent_sends as 4 messages are included in the size diff --git a/lnet/klnds/gnilnd/gnilnd_api_wrap.h b/lnet/klnds/gnilnd/gnilnd_api_wrap.h index 15be388..1584e6d 100644 --- a/lnet/klnds/gnilnd/gnilnd_api_wrap.h +++ b/lnet/klnds/gnilnd/gnilnd_api_wrap.h @@ -102,6 +102,7 @@ #define CFS_FAIL_GNI_SCHED_DEADLINE 0xf052 #define CFS_FAIL_GNI_DGRAM_DEADLINE 0xf053 #define CFS_FAIL_GNI_DGRAM_DROP_TX 0xf054 +#define CFS_FAIL_GNI_RDMA_CQ_ERROR 0xf055 /* helper macros */ extern void diff --git a/lnet/klnds/gnilnd/gnilnd_cb.c b/lnet/klnds/gnilnd/gnilnd_cb.c index de84a5b..b8f56f5 100644 --- a/lnet/klnds/gnilnd/gnilnd_cb.c +++ b/lnet/klnds/gnilnd/gnilnd_cb.c @@ -1164,6 +1164,7 @@ kgnilnd_unmap_buffer(kgn_tx_t *tx, int error) * verified peer notification - the theory is that * a TX error can be communicated in all other cases */ if (tx->tx_conn->gnc_state != GNILND_CONN_ESTABLISHED && + error != -GNILND_NOPURG && kgnilnd_check_purgatory_conn(tx->tx_conn)) { kgnilnd_add_purgatory_tx(tx); @@ -3246,6 +3247,11 @@ kgnilnd_check_rdma_cq(kgn_device_t *dev) spin_unlock(&conn->gnc_list_lock); kgnilnd_conn_mutex_unlock(&conn->gnc_rdma_mutex); + if (CFS_FAIL_CHECK(CFS_FAIL_GNI_RDMA_CQ_ERROR)) { + event_data = 1LL << 48; + rc = 1; + } + if (likely(desc->status == GNI_RC_SUCCESS) && rc == 0) { atomic_inc(&dev->gnd_rdma_ntx); atomic64_add(tx->tx_nob, &dev->gnd_rdma_txbytes); @@ -3300,7 +3306,7 @@ kgnilnd_check_rdma_cq(kgn_device_t *dev) -EFAULT, rcookie, tx->tx_msg.gnm_srcnid); - kgnilnd_tx_done(tx, -EFAULT); + kgnilnd_tx_done(tx, -GNILND_NOPURG); kgnilnd_close_conn(conn, -ECOMM); } -- 1.8.3.1