LU-11702 o2iblnd: ibc_rxs is created and freed with different size

kiblnd_create_conn()) alloc '(conn->ibc_rxs)': 26832 at ffffc90012e69000
kiblnd_destroy_conn()) kfreed 'conn->ibc_rxs': 4576 at ffffc90012e69000

The size changed by kiblnd_create_conn() :
"peer 172.18.2.3@o2ib - queue depth reduced from 128 to 21"

Based on size LIBCFS_FREE() decides whether to use kfree or vfree
and accounts memory usage.

Allocate ibc_rxs after rdma_create_qp()

Change-Id: I1fb1516bd5427e0c959ce2e71bb248d727bb3c49
Cray-bug-id: LUS-6339
Signed-off-by: Andriy Skulysh <c17819@cray.com>
Reviewed-by: Andrew Perepechko <c17827@cray.com>
Reviewed-by: Chris Horn <hornc@cray.com>
Reviewed-on: https://review.whamcloud.com/33721
Tested-by: Jenkins
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index c3aaea7..0ccc0e9 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -813,6 +813,8 @@ kiblnd_create_conn(struct kib_peer_ni *peer_ni, struct rdma_cm_id *cmid,
        conn->ibc_cmid = cmid;
        conn->ibc_max_frags = peer_ni->ibp_max_frags;
        conn->ibc_queue_depth = peer_ni->ibp_queue_depth;
+       conn->ibc_rxs = NULL;
+       conn->ibc_rx_pages = NULL;
 
        INIT_LIST_HEAD(&conn->ibc_early_rxs);
        INIT_LIST_HEAD(&conn->ibc_tx_noops);
@@ -857,20 +859,6 @@ kiblnd_create_conn(struct kib_peer_ni *peer_ni, struct rdma_cm_id *cmid,
 
        write_unlock_irqrestore(glock, flags);
 
-       LIBCFS_CPT_ALLOC(conn->ibc_rxs, lnet_cpt_table(), cpt,
-                        IBLND_RX_MSGS(conn) * sizeof(struct kib_rx));
-       if (conn->ibc_rxs == NULL) {
-               CERROR("Cannot allocate RX buffers\n");
-               goto failed_2;
-       }
-
-       rc = kiblnd_alloc_pages(&conn->ibc_rx_pages, cpt,
-                               IBLND_RX_MSG_PAGES(conn));
-       if (rc != 0)
-               goto failed_2;
-
-       kiblnd_map_rx_descs(conn);
-
 #ifdef HAVE_IB_CQ_INIT_ATTR
        cq_attr.cqe = IBLND_CQ_ENTRIES(conn);
        cq_attr.comp_vector = kiblnd_get_completion_vector(conn, cpt);
@@ -942,6 +930,20 @@ kiblnd_create_conn(struct kib_peer_ni *peer_ni, struct rdma_cm_id *cmid,
                      peer_ni->ibp_queue_depth,
                      conn->ibc_queue_depth);
 
+       LIBCFS_CPT_ALLOC(conn->ibc_rxs, lnet_cpt_table(), cpt,
+                        IBLND_RX_MSGS(conn) * sizeof(struct kib_rx));
+       if (conn->ibc_rxs == NULL) {
+               CERROR("Cannot allocate RX buffers\n");
+               goto failed_2;
+       }
+
+       rc = kiblnd_alloc_pages(&conn->ibc_rx_pages, cpt,
+                               IBLND_RX_MSG_PAGES(conn));
+       if (rc != 0)
+               goto failed_2;
+
+       kiblnd_map_rx_descs(conn);
+
        LIBCFS_FREE(init_qp_attr, sizeof(*init_qp_attr));
 
        /* 1 ref for caller and each rxmsg */