LU-6261 gnilnd: kgnilnd_check_rdma_cq race in error path.

It is possible for multiple threads to both be processing a
transaction error that causes the connection to close.
If the close path stalls after changing the conn state to CLOSING,
the tx_done path will put a tx in purgatory since the conn state is
not ESTABLISHED anymore. When the close path continues, it expects to
find that the conn->gnc_mdd_list is empty and assert if it isn't
empty.
Change the error code for the tx_done path so that we don't put this
tx in purgatory since we are closing the connection anyway.

Signed-off-by: Chris Horn <hornc@cray.com>
Change-Id: I4458522f16508eb53d380f62320c65c7bf84657a
Reviewed-on: http://review.whamcloud.com/15439
Tested-by: Jenkins
Reviewed-by: James Shimek <jshimek@cray.com>
Reviewed-by: James Simmons <uja.ornl@yahoo.com>
Reviewed-by: Doug Oucharek <doug.s.oucharek@intel.com>
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lnet/klnds/gnilnd/gnilnd.c b/lnet/klnds/gnilnd/gnilnd.c
index e110c27..be104c3 100644 (file)
--- a/lnet/klnds/gnilnd/gnilnd.c
+++ b/lnet/klnds/gnilnd/gnilnd.c
@@ -671,6 +671,10 @@ kgnilnd_close_conn_locked(kgn_conn_t *conn, int error)
                conn->gnc_state = GNILND_CONN_CLOSING;
        }
 
+       if (CFS_FAIL_CHECK(CFS_FAIL_GNI_RDMA_CQ_ERROR)) {
+               msleep_interruptible(MSEC_PER_SEC);
+       }
+
        /* leave on peer->gnp_conns to make sure we don't let the reaper
         * or others try to unlink this peer until the conn is fully
         * processed for closing */

diff --git a/lnet/klnds/gnilnd/gnilnd.h b/lnet/klnds/gnilnd/gnilnd.h
index 0b599ae..785142c 100644 (file)
--- a/lnet/klnds/gnilnd/gnilnd.h
+++ b/lnet/klnds/gnilnd/gnilnd.h
@@ -139,6 +139,8 @@
 
 /* Max number of connections to keep in purgatory per peer */
 #define GNILND_PURGATORY_MAX     5
+/* Closing, don't put in purgatory */
+#define GNILND_NOPURG             222
 
 /* payload size to add to the base mailbox size
  * This is subtracting 2 from the concurrent_sends as 4 messages are included in the size

diff --git a/lnet/klnds/gnilnd/gnilnd_api_wrap.h b/lnet/klnds/gnilnd/gnilnd_api_wrap.h
index 15be388..1584e6d 100644 (file)
--- a/lnet/klnds/gnilnd/gnilnd_api_wrap.h
+++ b/lnet/klnds/gnilnd/gnilnd_api_wrap.h
@@ -102,6 +102,7 @@
 #define CFS_FAIL_GNI_SCHED_DEADLINE    0xf052
 #define CFS_FAIL_GNI_DGRAM_DEADLINE    0xf053
 #define CFS_FAIL_GNI_DGRAM_DROP_TX     0xf054
+#define CFS_FAIL_GNI_RDMA_CQ_ERROR     0xf055
 
 /* helper macros */
 extern void

diff --git a/lnet/klnds/gnilnd/gnilnd_cb.c b/lnet/klnds/gnilnd/gnilnd_cb.c
index de84a5b..b8f56f5 100644 (file)
--- a/lnet/klnds/gnilnd/gnilnd_cb.c
+++ b/lnet/klnds/gnilnd/gnilnd_cb.c
@@ -1164,6 +1164,7 @@ kgnilnd_unmap_buffer(kgn_tx_t *tx, int error)
                 * verified peer notification  - the theory is that
                 * a TX error can be communicated in all other cases */
                if (tx->tx_conn->gnc_state != GNILND_CONN_ESTABLISHED &&
+                   error != -GNILND_NOPURG &&
                    kgnilnd_check_purgatory_conn(tx->tx_conn)) {
                        kgnilnd_add_purgatory_tx(tx);
 
@@ -3246,6 +3247,11 @@ kgnilnd_check_rdma_cq(kgn_device_t *dev)
                spin_unlock(&conn->gnc_list_lock);
                kgnilnd_conn_mutex_unlock(&conn->gnc_rdma_mutex);
 
+               if (CFS_FAIL_CHECK(CFS_FAIL_GNI_RDMA_CQ_ERROR)) {
+                       event_data = 1LL << 48;
+                       rc = 1;
+               }
+
                if (likely(desc->status == GNI_RC_SUCCESS) && rc == 0) {
                        atomic_inc(&dev->gnd_rdma_ntx);
                        atomic64_add(tx->tx_nob, &dev->gnd_rdma_txbytes);
@@ -3300,7 +3306,7 @@ kgnilnd_check_rdma_cq(kgn_device_t *dev)
                                         -EFAULT,
                                         rcookie,
                                         tx->tx_msg.gnm_srcnid);
-                       kgnilnd_tx_done(tx, -EFAULT);
+                       kgnilnd_tx_done(tx, -GNILND_NOPURG);
                        kgnilnd_close_conn(conn, -ECOMM);
                }