LU-10394 lnd: default to using MEM_REG

There is a performance drop when using IB_MR_TYPE_SG_GAPS. To
mitigate this, we added a module parameter, use_fastreg_gaps, which
defaults to 0. When allocating the memory region if this parameter
is set to 1 and the hw has gaps support then use it and output
a warning that performance may drop. Otherwise always use
IB_MR_TYPE_MEM_REG.

Test-Parameters: trivial
Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Change-Id: I08a8b72756b9b5b5bcb391bf3e979f6d28eb5cbb
Reviewed-on: https://review.whamcloud.com/30749
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: James Simmons <uja.ornl@yahoo.com>
Reviewed-by: Doug Oucharek <dougso@me.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index 798132d..fc1ce47 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -1592,14 +1592,17 @@ static int kiblnd_alloc_freg_pool(kib_fmr_poolset_t *fps, kib_fmr_pool_t *fpo,
                 */
                frd->frd_mr = ib_alloc_mr(fpo->fpo_hdev->ibh_pd,
 #ifdef IB_MR_TYPE_SG_GAPS
-                                         (dev_caps &
-                                               IBLND_DEV_CAPS_FASTREG_GAPS_SUPPORT) ?
+                                         ((*kiblnd_tunables.kib_use_fastreg_gaps == 1) &&
+                                          (dev_caps & IBLND_DEV_CAPS_FASTREG_GAPS_SUPPORT)) ?
                                                IB_MR_TYPE_SG_GAPS :
                                                IB_MR_TYPE_MEM_REG,
 #else
                                                IB_MR_TYPE_MEM_REG,
 #endif
                                          LNET_MAX_PAYLOAD/PAGE_SIZE);
+               if ((*kiblnd_tunables.kib_use_fastreg_gaps == 1) &&
+                   (dev_caps & IBLND_DEV_CAPS_FASTREG_GAPS_SUPPORT))
+                       CWARN("using IB_MR_TYPE_SG_GAPS, expect a performance drop\n");
 #endif
                if (IS_ERR(frd->frd_mr)) {
                        rc = PTR_ERR(frd->frd_mr);

diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h
index 7d2e6e0..ccc8596 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/lnet/klnds/o2iblnd/o2iblnd.h
@@ -112,6 +112,7 @@ typedef struct
        /* # threads on each CPT */
        int              *kib_nscheds;
        int              *kib_wrq_sge;          /* # sg elements per wrq */
+       int              *kib_use_fastreg_gaps; /* enable discontiguous fastreg fragment support */
 } kib_tunables_t;
 
 extern kib_tunables_t  kiblnd_tunables;

diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c
index faa3170..b96131a 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -567,7 +567,8 @@ kiblnd_fmr_map_tx(kib_net_t *net, kib_tx_t *tx, kib_rdma_desc_t *rd, __u32 nob)
        if ((dev->ibd_dev_caps & IBLND_DEV_CAPS_FASTREG_ENABLED) &&
            !(dev->ibd_dev_caps & IBLND_DEV_CAPS_FASTREG_GAPS_SUPPORT) &&
            tx->tx_gaps) {
-               CERROR("Using FastReg with no GAPS support, but tx has gaps\n");
+               CERROR("Using FastReg with no GAPS support, but tx has gaps. "
+                      "Try setting use_fastreg_gaps to 1\n");
                return -EPROTONOSUPPORT;
        }
 

diff --git a/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
index 85c66fd..2224b04 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd_modparams.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
@@ -110,6 +110,10 @@ static int concurrent_sends;
 module_param(concurrent_sends, int, 0444);
 MODULE_PARM_DESC(concurrent_sends, "send work-queue sizing (obsolete)");
 
+static int use_fastreg_gaps;
+module_param(use_fastreg_gaps, int, 0444);
+MODULE_PARM_DESC(use_fastreg_gaps, "Enable discontiguous fastreg fragment support. Expect performance drop");
+
 /*
  * map_on_demand is a flag used to determine if we can use FMR or FastReg.
  * This is applicable for kernels which support global memory regions. For
@@ -196,6 +200,7 @@ kib_tunables_t kiblnd_tunables = {
        .kib_use_priv_port          = &use_privileged_port,
        .kib_nscheds                = &nscheds,
        .kib_wrq_sge                = &wrq_sge,
+       .kib_use_fastreg_gaps       = &use_fastreg_gaps,
 };
 
 static struct lnet_ioctl_config_o2iblnd_tunables default_tunables;