LU-9983 ko2iblnd: allow for discontiguous fragments

In the IOVEC case the buffers passed to the LND may not span
complete pages, therefore the RDMA descriptor needs to describe
all the buffers.  Moreover for the FMR case, the addresses that get
set in the RDMA descriptor need to be relative addresses. This
issue was exposed after:
LU-9026 o2iblnd: Adapt to the removal of ib_get_dma_mr()

Fastreg still expects only one fragment with the total nob.
Otherwise there is a dump_cqe error from MLX5

Test-Parameters: trivial
Signed-off-by: John L. Hammond <john.hammond@intel.com>
Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Change-Id: Ie1cf52677f65af83357a3dd25fd1a45f3466a96e
Reviewed-on: https://review.whamcloud.com/29290
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c
index d2a922d..9699fa7 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -546,7 +546,8 @@ kiblnd_fmr_map_tx(kib_net_t *net, kib_tx_t *tx, kib_rdma_desc_t *rd, __u32 nob)
        kib_fmr_poolset_t       *fps;
        int                     cpt;
        int                     rc;
-       bool                    is_fastreg = 0;
+       int i;
+       bool is_fastreg = 0;
 
        LASSERT(tx->tx_pool != NULL);
        LASSERT(tx->tx_pool->tpo_pool.po_owner != NULL);
@@ -564,10 +565,15 @@ kiblnd_fmr_map_tx(kib_net_t *net, kib_tx_t *tx, kib_rdma_desc_t *rd, __u32 nob)
        /* If rd is not tx_rd, it's going to get sent to a peer_ni, who will need
         * the rkey */
        rd->rd_key = tx->fmr.fmr_key;
-       if (!is_fastreg)
-               rd->rd_frags[0].rf_addr &= ~hdev->ibh_page_mask;
-       rd->rd_frags[0].rf_nob   = nob;
-       rd->rd_nfrags = 1;
+       if (!is_fastreg) {
+               for (i = 0; i < rd->rd_nfrags; i++) {
+                       rd->rd_frags[i].rf_addr &= ~hdev->ibh_page_mask;
+                       rd->rd_frags[i].rf_addr += i << hdev->ibh_page_shift;
+               }
+       } else {
+               rd->rd_frags[0].rf_nob = nob;
+               rd->rd_nfrags = 1;
+       }
 
        return 0;
 }