From 277a6faa5b1684b9462956a2801c1e0d80c61d53 Mon Sep 17 00:00:00 2001 From: Andriy Skulysh Date: Mon, 6 Aug 2018 19:31:05 +0300 Subject: [PATCH] LU-11702 o2iblnd: ibc_rxs is created and freed with different size kiblnd_create_conn()) alloc '(conn->ibc_rxs)': 26832 at ffffc90012e69000 kiblnd_destroy_conn()) kfreed 'conn->ibc_rxs': 4576 at ffffc90012e69000 The size changed by kiblnd_create_conn() : "peer 172.18.2.3@o2ib - queue depth reduced from 128 to 21" Based on size LIBCFS_FREE() decides whether to use kfree or vfree and accounts memory usage. Allocate ibc_rxs after rdma_create_qp() Change-Id: I1fb1516bd5427e0c959ce2e71bb248d727bb3c49 Cray-bug-id: LUS-6339 Signed-off-by: Andriy Skulysh Reviewed-by: Andrew Perepechko Reviewed-by: Chris Horn Reviewed-on: https://review.whamcloud.com/33721 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Oleg Drokin --- lnet/klnds/o2iblnd/o2iblnd.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c index c3aaea7..0ccc0e9 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.c +++ b/lnet/klnds/o2iblnd/o2iblnd.c @@ -813,6 +813,8 @@ kiblnd_create_conn(struct kib_peer_ni *peer_ni, struct rdma_cm_id *cmid, conn->ibc_cmid = cmid; conn->ibc_max_frags = peer_ni->ibp_max_frags; conn->ibc_queue_depth = peer_ni->ibp_queue_depth; + conn->ibc_rxs = NULL; + conn->ibc_rx_pages = NULL; INIT_LIST_HEAD(&conn->ibc_early_rxs); INIT_LIST_HEAD(&conn->ibc_tx_noops); @@ -857,20 +859,6 @@ kiblnd_create_conn(struct kib_peer_ni *peer_ni, struct rdma_cm_id *cmid, write_unlock_irqrestore(glock, flags); - LIBCFS_CPT_ALLOC(conn->ibc_rxs, lnet_cpt_table(), cpt, - IBLND_RX_MSGS(conn) * sizeof(struct kib_rx)); - if (conn->ibc_rxs == NULL) { - CERROR("Cannot allocate RX buffers\n"); - goto failed_2; - } - - rc = kiblnd_alloc_pages(&conn->ibc_rx_pages, cpt, - IBLND_RX_MSG_PAGES(conn)); - if (rc != 0) - goto failed_2; - - kiblnd_map_rx_descs(conn); - #ifdef HAVE_IB_CQ_INIT_ATTR cq_attr.cqe = IBLND_CQ_ENTRIES(conn); cq_attr.comp_vector = kiblnd_get_completion_vector(conn, cpt); @@ -942,6 +930,20 @@ kiblnd_create_conn(struct kib_peer_ni *peer_ni, struct rdma_cm_id *cmid, peer_ni->ibp_queue_depth, conn->ibc_queue_depth); + LIBCFS_CPT_ALLOC(conn->ibc_rxs, lnet_cpt_table(), cpt, + IBLND_RX_MSGS(conn) * sizeof(struct kib_rx)); + if (conn->ibc_rxs == NULL) { + CERROR("Cannot allocate RX buffers\n"); + goto failed_2; + } + + rc = kiblnd_alloc_pages(&conn->ibc_rx_pages, cpt, + IBLND_RX_MSG_PAGES(conn)); + if (rc != 0) + goto failed_2; + + kiblnd_map_rx_descs(conn); + LIBCFS_FREE(init_qp_attr, sizeof(*init_qp_attr)); /* 1 ref for caller and each rxmsg */ -- 1.8.3.1