From 7954a520428002de8629916d4ae7b6660818dc6c Mon Sep 17 00:00:00 2001 From: "John L. Hammond" Date: Mon, 2 Oct 2017 16:23:15 -0500 Subject: [PATCH] LU-9983 ko2iblnd: allow for discontiguous fragments In the IOVEC case the buffers passed to the LND may not span complete pages, therefore the RDMA descriptor needs to describe all the buffers. Moreover for the FMR case, the addresses that get set in the RDMA descriptor need to be relative addresses. This issue was exposed after: LU-9026 o2iblnd: Adapt to the removal of ib_get_dma_mr() Fastreg still expects only one fragment with the total nob. Otherwise there is a dump_cqe error from MLX5 Test-Parameters: trivial Signed-off-by: John L. Hammond Signed-off-by: Amir Shehata Change-Id: Ie1cf52677f65af83357a3dd25fd1a45f3466a96e Reviewed-on: https://review.whamcloud.com/29290 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Oleg Drokin --- lnet/klnds/o2iblnd/o2iblnd_cb.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c index d2a922d..9699fa7 100644 --- a/lnet/klnds/o2iblnd/o2iblnd_cb.c +++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c @@ -546,7 +546,8 @@ kiblnd_fmr_map_tx(kib_net_t *net, kib_tx_t *tx, kib_rdma_desc_t *rd, __u32 nob) kib_fmr_poolset_t *fps; int cpt; int rc; - bool is_fastreg = 0; + int i; + bool is_fastreg = 0; LASSERT(tx->tx_pool != NULL); LASSERT(tx->tx_pool->tpo_pool.po_owner != NULL); @@ -564,10 +565,15 @@ kiblnd_fmr_map_tx(kib_net_t *net, kib_tx_t *tx, kib_rdma_desc_t *rd, __u32 nob) /* If rd is not tx_rd, it's going to get sent to a peer_ni, who will need * the rkey */ rd->rd_key = tx->fmr.fmr_key; - if (!is_fastreg) - rd->rd_frags[0].rf_addr &= ~hdev->ibh_page_mask; - rd->rd_frags[0].rf_nob = nob; - rd->rd_nfrags = 1; + if (!is_fastreg) { + for (i = 0; i < rd->rd_nfrags; i++) { + rd->rd_frags[i].rf_addr &= ~hdev->ibh_page_mask; + rd->rd_frags[i].rf_addr += i << hdev->ibh_page_shift; + } + } else { + rd->rd_frags[0].rf_nob = nob; + rd->rd_nfrags = 1; + } return 0; } -- 1.8.3.1