From 53d94b01a7c97ec962a38dae638364f89c83a1b3 Mon Sep 17 00:00:00 2001
From: maxim <maxim>
Date: Wed, 15 Apr 2009 20:24:27 +0000
Subject: [PATCH] b=18844 i=isaac Landing a patch fixing deadlock in usocklnd
 on b1_x. The patch also includes:  - minor code cleanup suggested by Isaac in
 comment #6 of bug #18844  - minor one-line fix for handling EINTR error of
 poll(2) - inspected by Shadow.

---
 lnet/ChangeLog                |  8 ++++++++
 lnet/ulnds/socklnd/conn.c     | 24 +++++++++++++++++++++---
 lnet/ulnds/socklnd/handlers.c |  3 ++-
 lnet/ulnds/socklnd/poll.c     |  2 +-
 4 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/lnet/ChangeLog b/lnet/ChangeLog
index 6c1e9a0..4376f05 100644
--- a/lnet/ChangeLog
+++ b/lnet/ChangeLog
@@ -17,6 +17,14 @@ Bugzilla   :
 Description: 
 Details    : 
 
+Severity   : normal
+Bugzilla   : 18844
+Description: Fixing deadlock in usocklnd
+Details    : A deadlock was possible in usocklnd due to race condition while
+             tearing connection down. The problem resulted from erroneous
+             assumption that lnet_finalize() could have been called holding
+             some lnd-level locks.
+
 Severity   : major
 Bugzilla   : 13621, 15983
 Description: Protocol V2 of o2iblnd
diff --git a/lnet/ulnds/socklnd/conn.c b/lnet/ulnds/socklnd/conn.c
index a4d89ae..d7e5969 100644
--- a/lnet/ulnds/socklnd/conn.c
+++ b/lnet/ulnds/socklnd/conn.c
@@ -128,6 +128,8 @@ usocklnd_tear_peer_conn(usock_conn_t *conn)
         lnet_process_id_t id;
         int               decref_flag  = 0;
         int               killall_flag = 0;
+        void             *rx_lnetmsg   = NULL; 
+        CFS_LIST_HEAD    (zombie_txs);
         
         if (peer == NULL) /* nothing to tear */
                 return;
@@ -142,11 +144,12 @@ usocklnd_tear_peer_conn(usock_conn_t *conn)
                 if (conn->uc_rx_state == UC_RX_LNET_PAYLOAD) {
                         /* change state not to finalize twice */
                         conn->uc_rx_state = UC_RX_KSM_HEADER;
-                        lnet_finalize(peer->up_ni, conn->uc_rx_lnetmsg, -EIO);                        
+                        /* stash lnetmsg while holding locks */
+                        rx_lnetmsg = conn->uc_rx_lnetmsg;
                 }
                 
-                usocklnd_destroy_txlist(peer->up_ni,
-                                        &conn->uc_tx_list);
+                /* we cannot finilize txs right now (bug #18844) */
+                list_splice_init(&conn->uc_tx_list, &zombie_txs);
 
                 peer->up_conns[idx] = NULL;
                 conn->uc_peer = NULL;
@@ -154,6 +157,9 @@ usocklnd_tear_peer_conn(usock_conn_t *conn)
 
                 if(conn->uc_errored && !peer->up_errored)
                         peer->up_errored = killall_flag = 1;
+
+                /* prevent queueing new txs to this conn */
+                conn->uc_errored = 1;
         }
         
         pthread_mutex_unlock(&conn->uc_lock);
@@ -166,6 +172,11 @@ usocklnd_tear_peer_conn(usock_conn_t *conn)
         if (!decref_flag)
                 return;
 
+        if (rx_lnetmsg != NULL)
+                lnet_finalize(ni, rx_lnetmsg, -EIO);
+        
+        usocklnd_destroy_txlist(ni, &zombie_txs);
+
         usocklnd_conn_decref(conn);
         usocklnd_peer_decref(peer);
 
@@ -852,6 +863,13 @@ usocklnd_find_or_create_conn(usock_peer_t *peer, int type,
 
         LASSERT(tx == NULL || zc_ack == NULL);
         if (tx != NULL) {
+                /* usocklnd_tear_peer_conn() could signal us stop queueing */
+                if (conn->uc_errored) {
+                        rc = -EIO;
+                        pthread_mutex_unlock(&conn->uc_lock);
+                        goto find_or_create_conn_failed;
+                }
+
                 usocklnd_enqueue_tx(conn, tx, send_immediately);
         } else {
                 rc = usocklnd_enqueue_zcack(conn, zc_ack);        
diff --git a/lnet/ulnds/socklnd/handlers.c b/lnet/ulnds/socklnd/handlers.c
index 32ef7d1..91fe30c 100644
--- a/lnet/ulnds/socklnd/handlers.c
+++ b/lnet/ulnds/socklnd/handlers.c
@@ -999,7 +999,8 @@ usocklnd_read_data(usock_conn_t *conn)
                 
                 nob = libcfs_sock_readv(conn->uc_fd, conn->uc_rx_iov, conn->uc_rx_niov);                
                 if (nob <= 0) {/* read nothing or error */
-                        conn->uc_errored = 1;
+                        if (nob < 0)
+                                conn->uc_errored = 1;
                         return nob;
                 }
                 
diff --git a/lnet/ulnds/socklnd/poll.c b/lnet/ulnds/socklnd/poll.c
index 3388d85..1c16b5a 100644
--- a/lnet/ulnds/socklnd/poll.c
+++ b/lnet/ulnds/socklnd/poll.c
@@ -113,7 +113,7 @@ usocklnd_poll_thread(void *arg)
                           pt_data->upt_nfds,
                           usock_tuns.ut_poll_timeout * 1000);
 
-                if (rc < 0) {
+                if (rc < 0 && errno != EINTR) {
                         CERROR("Cannot poll(2): errno=%d\n", errno);
                         break;
                 }
-- 
1.8.3.1