LU-5678 o2iblnd: connection refcount fix for kiblnd_post_rx

kiblnd_post_rx() can't refer to rx::rx_conn anymore after
ib_post_recv() because this rx can be polled out by another thread
which may drop this rx and destroy rx::rx_conn.

This patch fixes this issue by taking an extra refcount on connection
before calling ib_post_recv().

Signed-off-by: Liang Zhen <liang.zhen@intel.com>
Change-Id: I100f186ea372b1e8aa46eb18a5e29b31b2cdd9a7
Reviewed-on: http://review.whamcloud.com/12852
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Isaac Huang <he.huang@intel.com>
Reviewed-by: Amir Shehata <amir.shehata@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c
index 8d98e9e..cb7ee4e 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -189,24 +189,28 @@ kiblnd_post_rx (kib_rx_t *rx, int credit)
 
         rx->rx_nob = -1;                        /* flag posted */
 
-        rc = ib_post_recv(conn->ibc_cmid->qp, &rx->rx_wrq, &bad_wrq);
-        if (rc != 0) {
-                CERROR("Can't post rx for %s: %d, bad_wrq: %p\n",
-                       libcfs_nid2str(conn->ibc_peer->ibp_nid), rc, bad_wrq);
-                rx->rx_nob = 0;
-        }
+       /* NB: need an extra reference after ib_post_recv because we don't
+        * own this rx (and rx::rx_conn) anymore, LU-5678.
+        */
+       kiblnd_conn_addref(conn);
+       rc = ib_post_recv(conn->ibc_cmid->qp, &rx->rx_wrq, &bad_wrq);
+       if (unlikely(rc != 0)) {
+               CERROR("Can't post rx for %s: %d, bad_wrq: %p\n",
+                      libcfs_nid2str(conn->ibc_peer->ibp_nid), rc, bad_wrq);
+               rx->rx_nob = 0;
+       }
 
-        if (conn->ibc_state < IBLND_CONN_ESTABLISHED) /* Initial post */
-                return rc;
+       if (conn->ibc_state < IBLND_CONN_ESTABLISHED) /* Initial post */
+               goto out;
 
-        if (rc != 0) {
-                kiblnd_close_conn(conn, rc);
-                kiblnd_drop_rx(rx);             /* No more posts for this rx */
-                return rc;
-        }
+       if (unlikely(rc != 0)) {
+               kiblnd_close_conn(conn, rc);
+               kiblnd_drop_rx(rx);     /* No more posts for this rx */
+               goto out;
+       }
 
-        if (credit == IBLND_POSTRX_NO_CREDIT)
-                return 0;
+       if (credit == IBLND_POSTRX_NO_CREDIT)
+               goto out;
 
        spin_lock(&conn->ibc_lock);
        if (credit == IBLND_POSTRX_PEER_CREDIT)
@@ -216,7 +220,9 @@ kiblnd_post_rx (kib_rx_t *rx, int credit)
        spin_unlock(&conn->ibc_lock);
 
        kiblnd_check_sends(conn);
-       return 0;
+out:
+       kiblnd_conn_decref(conn);
+       return rc;
 }
 
 static kib_tx_t *