From: Liang Zhen <Zhen.Liang@sun.com>
Date: Tue, 17 Aug 2010 06:26:17 +0000 (+0400)
Subject: b20288 fix a deadlock in kiblnd_check_conns i=isaac i=maxim
X-Git-Tag: 2.0.51.0~78
X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=ac54ab5516bc3c1fb995f8189234123cbbac61e7;ds=sidebyside

b20288 fix a deadlock in kiblnd_check_conns i=isaac i=maxim
---

diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h
index 9122f2a..f696ead 100644
--- a/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/lnet/klnds/o2iblnd/o2iblnd.h
@@ -540,6 +540,7 @@ typedef struct kib_conn
         int                  ibc_credits;       /* # credits I have */
         int                  ibc_outstanding_credits; /* # credits to return */
         int                  ibc_reserved_credits;/* # ACK/DONE msg credits */
+        int                  ibc_retry_noop;    /* need to retry returning credits */
         int                  ibc_comms_error;   /* set on comms error */
         int                  ibc_nrx:16;        /* receive buffers owned */
         int                  ibc_scheduled:1;   /* scheduled for attention */
diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c
index a05f13e..6c58461 100644
--- a/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -910,6 +910,7 @@ kiblnd_check_sends (kib_conn_t *conn)
                 conn->ibc_reserved_credits--;
         }
 
+        conn->ibc_retry_noop = 0;
         if (kiblnd_send_noop(conn)) {
                 cfs_spin_unlock(&conn->ibc_lock);
 
@@ -920,6 +921,8 @@ kiblnd_check_sends (kib_conn_t *conn)
                 cfs_spin_lock(&conn->ibc_lock);
                 if (tx != NULL)
                         kiblnd_queue_tx_locked(tx, conn);
+                else if (kiblnd_send_noop(conn)) /* still... */
+                        conn->ibc_retry_noop = 1;
         }
 
         kiblnd_conn_addref(conn); /* 1 ref for me.... (see b21911) */
@@ -2874,14 +2877,11 @@ kiblnd_cm_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event)
         }
 }
 
-int
-kiblnd_check_txs (kib_conn_t *conn, cfs_list_t *txs)
+static int
+kiblnd_check_txs_locked (kib_conn_t *conn, cfs_list_t *txs)
 {
         kib_tx_t          *tx;
         cfs_list_t        *ttmp;
-        int                timed_out = 0;
-
-        cfs_spin_lock(&conn->ibc_lock);
 
         cfs_list_for_each (ttmp, txs) {
                 tx = cfs_list_entry (ttmp, kib_tx_t, tx_list);
@@ -2894,25 +2894,23 @@ kiblnd_check_txs (kib_conn_t *conn, cfs_list_t *txs)
                 }
 
                 if (cfs_time_aftereq (jiffies, tx->tx_deadline)) {
-                        timed_out = 1;
                         CERROR("Timed out tx: %s, %lu seconds\n",
                                kiblnd_queue2str(conn, txs),
                                cfs_duration_sec(jiffies - tx->tx_deadline));
-                        break;
+                        return 1;
                 }
         }
 
-        cfs_spin_unlock(&conn->ibc_lock);
-        return timed_out;
+        return 0;
 }
 
-int
-kiblnd_conn_timed_out (kib_conn_t *conn)
+static int
+kiblnd_conn_timed_out_locked (kib_conn_t *conn)
 {
-        return  kiblnd_check_txs(conn, &conn->ibc_tx_queue) ||
-                kiblnd_check_txs(conn, &conn->ibc_tx_queue_rsrvd) ||
-                kiblnd_check_txs(conn, &conn->ibc_tx_queue_nocred) ||
-                kiblnd_check_txs(conn, &conn->ibc_active_txs);
+        return  kiblnd_check_txs_locked(conn, &conn->ibc_tx_queue) ||
+                kiblnd_check_txs_locked(conn, &conn->ibc_tx_queue_rsrvd) ||
+                kiblnd_check_txs_locked(conn, &conn->ibc_tx_queue_nocred) ||
+                kiblnd_check_txs_locked(conn, &conn->ibc_active_txs);
 }
 
 void
@@ -2935,17 +2933,34 @@ kiblnd_check_conns (int idx)
                 peer = cfs_list_entry (ptmp, kib_peer_t, ibp_list);
 
                 cfs_list_for_each (ctmp, &peer->ibp_conns) {
+                        int     check_sends;
+                        int     timedout;
+
                         conn = cfs_list_entry (ctmp, kib_conn_t, ibc_list);
 
                         LASSERT (conn->ibc_state == IBLND_CONN_ESTABLISHED);
 
-                        /* In case we have enough credits to return via a
-                         * NOOP, but there were no non-blocking tx descs
-                         * free to do it last time... */
-                        kiblnd_check_sends(conn);
+                        cfs_spin_lock(&conn->ibc_lock);
 
-                        if (!kiblnd_conn_timed_out(conn))
+                        check_sends = conn->ibc_retry_noop;
+                        timedout    = kiblnd_conn_timed_out_locked(conn);
+                        if (!check_sends && !timedout) {
+                                cfs_spin_unlock(&conn->ibc_lock);
                                 continue;
+                        }
+
+                        if (timedout) {
+                                CERROR("Timed out RDMA with %s (%lu): "
+                                       "c: %u, oc: %u, rc: %u\n",
+                                       libcfs_nid2str(peer->ibp_nid),
+                                       cfs_duration_sec(cfs_time_current() -
+                                                        peer->ibp_last_alive),
+                                       conn->ibc_credits,
+                                       conn->ibc_outstanding_credits,
+                                       conn->ibc_reserved_credits);
+                        }
+
+                        cfs_spin_unlock(&conn->ibc_lock);
 
                         /* Handle timeout by closing the whole connection.  We
                          * can only be sure RDMA activity has ceased once the
@@ -2956,12 +2971,15 @@ kiblnd_check_conns (int idx)
                         cfs_read_unlock_irqrestore(&kiblnd_data.kib_global_lock,
                                                    flags);
 
-                        CERROR("Timed out RDMA with %s (%lu)\n",
-                               libcfs_nid2str(peer->ibp_nid),
-                               cfs_duration_sec(cfs_time_current() -
-                                                peer->ibp_last_alive));
+                        if (timedout) {
+                                kiblnd_close_conn(conn, -ETIMEDOUT);
+                        } else {
+                                /* In case we have enough credits to return via a
+                                 * NOOP, but there were no non-blocking tx descs
+                                 * free to do it last time... */
+                                kiblnd_check_sends(conn);
+                        }
 
-                        kiblnd_close_conn(conn, -ETIMEDOUT);
                         kiblnd_conn_decref(conn); /* ...until here */
 
                         /* start again now I've dropped the lock */