From 27116ee18c04057936837f6f0aef3f4c09c21d78 Mon Sep 17 00:00:00 2001 From: Amir Shehata Date: Fri, 5 Jan 2018 11:50:33 -0800 Subject: [PATCH] LU-10394 lnd: default to using MEM_REG There is a performance drop when using IB_MR_TYPE_SG_GAPS. To mitigate this, we added a module parameter, use_fastreg_gaps, which defaults to 0. When allocating the memory region if this parameter is set to 1 and the hw has gaps support then use it and output a warning that performance may drop. Otherwise always use IB_MR_TYPE_MEM_REG. Test-Parameters: trivial Signed-off-by: Amir Shehata Change-Id: I08a8b72756b9b5b5bcb391bf3e979f6d28eb5cbb Reviewed-on: https://review.whamcloud.com/30749 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: James Simmons Reviewed-by: Doug Oucharek Reviewed-by: Oleg Drokin --- lnet/klnds/o2iblnd/o2iblnd.c | 7 +++++-- lnet/klnds/o2iblnd/o2iblnd.h | 1 + lnet/klnds/o2iblnd/o2iblnd_cb.c | 3 ++- lnet/klnds/o2iblnd/o2iblnd_modparams.c | 5 +++++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c index 798132d..fc1ce47 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.c +++ b/lnet/klnds/o2iblnd/o2iblnd.c @@ -1592,14 +1592,17 @@ static int kiblnd_alloc_freg_pool(kib_fmr_poolset_t *fps, kib_fmr_pool_t *fpo, */ frd->frd_mr = ib_alloc_mr(fpo->fpo_hdev->ibh_pd, #ifdef IB_MR_TYPE_SG_GAPS - (dev_caps & - IBLND_DEV_CAPS_FASTREG_GAPS_SUPPORT) ? + ((*kiblnd_tunables.kib_use_fastreg_gaps == 1) && + (dev_caps & IBLND_DEV_CAPS_FASTREG_GAPS_SUPPORT)) ? IB_MR_TYPE_SG_GAPS : IB_MR_TYPE_MEM_REG, #else IB_MR_TYPE_MEM_REG, #endif LNET_MAX_PAYLOAD/PAGE_SIZE); + if ((*kiblnd_tunables.kib_use_fastreg_gaps == 1) && + (dev_caps & IBLND_DEV_CAPS_FASTREG_GAPS_SUPPORT)) + CWARN("using IB_MR_TYPE_SG_GAPS, expect a performance drop\n"); #endif if (IS_ERR(frd->frd_mr)) { rc = PTR_ERR(frd->frd_mr); diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h index 7d2e6e0..ccc8596 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.h +++ b/lnet/klnds/o2iblnd/o2iblnd.h @@ -112,6 +112,7 @@ typedef struct /* # threads on each CPT */ int *kib_nscheds; int *kib_wrq_sge; /* # sg elements per wrq */ + int *kib_use_fastreg_gaps; /* enable discontiguous fastreg fragment support */ } kib_tunables_t; extern kib_tunables_t kiblnd_tunables; diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c index faa3170..b96131a 100644 --- a/lnet/klnds/o2iblnd/o2iblnd_cb.c +++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c @@ -567,7 +567,8 @@ kiblnd_fmr_map_tx(kib_net_t *net, kib_tx_t *tx, kib_rdma_desc_t *rd, __u32 nob) if ((dev->ibd_dev_caps & IBLND_DEV_CAPS_FASTREG_ENABLED) && !(dev->ibd_dev_caps & IBLND_DEV_CAPS_FASTREG_GAPS_SUPPORT) && tx->tx_gaps) { - CERROR("Using FastReg with no GAPS support, but tx has gaps\n"); + CERROR("Using FastReg with no GAPS support, but tx has gaps. " + "Try setting use_fastreg_gaps to 1\n"); return -EPROTONOSUPPORT; } diff --git a/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/lnet/klnds/o2iblnd/o2iblnd_modparams.c index 85c66fd..2224b04 100644 --- a/lnet/klnds/o2iblnd/o2iblnd_modparams.c +++ b/lnet/klnds/o2iblnd/o2iblnd_modparams.c @@ -110,6 +110,10 @@ static int concurrent_sends; module_param(concurrent_sends, int, 0444); MODULE_PARM_DESC(concurrent_sends, "send work-queue sizing (obsolete)"); +static int use_fastreg_gaps; +module_param(use_fastreg_gaps, int, 0444); +MODULE_PARM_DESC(use_fastreg_gaps, "Enable discontiguous fastreg fragment support. Expect performance drop"); + /* * map_on_demand is a flag used to determine if we can use FMR or FastReg. * This is applicable for kernels which support global memory regions. For @@ -196,6 +200,7 @@ kib_tunables_t kiblnd_tunables = { .kib_use_priv_port = &use_privileged_port, .kib_nscheds = &nscheds, .kib_wrq_sge = &wrq_sge, + .kib_use_fastreg_gaps = &use_fastreg_gaps, }; static struct lnet_ioctl_config_o2iblnd_tunables default_tunables; -- 1.8.3.1