From 8d70042d785e08bb59059ed3ffb0b2f5eb0e3dad Mon Sep 17 00:00:00 2001 From: Amir Shehata Date: Tue, 23 May 2017 16:52:37 -0700 Subject: [PATCH] LU-9536 lnet: set LND tunalbes properly Make sure to set all NIs to the proper LND tunables specified. Add ntx tunable to dynamic configuration. This way all tunables required to tune OPA performance can be configured via lnetctl, allowing the ability to tune OPA network and IB network differently Test-Parameters: trivial Signed-off-by: Amir Shehata Change-Id: I015f3f959bd46784d4607bd4259b4640303dc362 Reviewed-on: https://review.whamcloud.com/27263 Tested-by: Jenkins Reviewed-by: Doug Oucharek Tested-by: Maloo Reviewed-by: Sonia Sharma Reviewed-by: Olaf Weber Reviewed-by: Oleg Drokin --- lnet/include/lnet/lib-dlc.h | 2 +- lnet/klnds/o2iblnd/o2iblnd.c | 14 +++++++++----- lnet/klnds/o2iblnd/o2iblnd_modparams.c | 4 +++- lnet/utils/lnetconfig/liblnetconfig.c | 7 +++---- lnet/utils/lnetconfig/liblnetconfig_lnd.c | 9 ++++++++- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/lnet/include/lnet/lib-dlc.h b/lnet/include/lnet/lib-dlc.h index 48662b9..4141f7c 100644 --- a/lnet/include/lnet/lib-dlc.h +++ b/lnet/include/lnet/lib-dlc.h @@ -66,7 +66,7 @@ struct lnet_ioctl_config_o2iblnd_tunables { __u32 lnd_fmr_flush_trigger; __u32 lnd_fmr_cache; __u16 lnd_conns_per_peer; - __u16 pad; + __u16 lnd_ntx; }; struct lnet_lnd_tunables { diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c index 12dade5..298cd9f 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.c +++ b/lnet/klnds/o2iblnd/o2iblnd.c @@ -2305,9 +2305,13 @@ out: LIBCFS_FREE(tpo, sizeof(kib_tx_pool_t)); } -static int kiblnd_tx_pool_size(int ncpts) +static int kiblnd_tx_pool_size(struct lnet_ni *ni, int ncpts) { - int ntx = *kiblnd_tunables.kib_ntx / ncpts; + struct lnet_ioctl_config_o2iblnd_tunables *tunables; + int ntx; + + tunables = &ni->ni_lnd_tunables.lnd_tun_u.lnd_o2ib; + ntx = tunables->lnd_ntx / ncpts; return max(IBLND_TX_POOL, ntx); } @@ -2464,10 +2468,10 @@ kiblnd_net_init_pools(kib_net_t *net, struct lnet_ni *ni, __u32 *cpts, read_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags); #endif - if (tunables->lnd_fmr_pool_size < *kiblnd_tunables.kib_ntx / 4) { + if (tunables->lnd_fmr_pool_size < tunables->lnd_ntx / 4) { CERROR("Can't set fmr pool size (%d) < ntx / 4(%d)\n", tunables->lnd_fmr_pool_size, - *kiblnd_tunables.kib_ntx / 4); + tunables->lnd_ntx / 4); rc = -EINVAL; goto failed; } @@ -2516,7 +2520,7 @@ kiblnd_net_init_pools(kib_net_t *net, struct lnet_ni *ni, __u32 *cpts, cpt = (cpts == NULL) ? i : cpts[i]; rc = kiblnd_init_poolset(&net->ibn_tx_ps[cpt]->tps_poolset, cpt, net, "TX", - kiblnd_tx_pool_size(ncpts), + kiblnd_tx_pool_size(ni, ncpts), kiblnd_create_tx_pool, kiblnd_destroy_tx_pool, kiblnd_tx_init, NULL); diff --git a/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/lnet/klnds/o2iblnd/o2iblnd_modparams.c index 3ca0689..72cb50e 100644 --- a/lnet/klnds/o2iblnd/o2iblnd_modparams.c +++ b/lnet/klnds/o2iblnd/o2iblnd_modparams.c @@ -162,7 +162,6 @@ kib_tunables_t kiblnd_tunables = { .kib_cksum = &cksum, .kib_timeout = &timeout, .kib_keepalive = &keepalive, - .kib_ntx = &ntx, .kib_default_ipif = &ipif_name, .kib_retry_count = &retry_count, .kib_rnr_retry_count = &rnr_retry_count, @@ -290,6 +289,8 @@ kiblnd_tunables_setup(struct lnet_ni *ni) tunables->lnd_fmr_flush_trigger = fmr_flush_trigger; if (!tunables->lnd_fmr_cache) tunables->lnd_fmr_cache = fmr_cache; + if (!tunables->lnd_ntx) + tunables->lnd_ntx = ntx; if (!tunables->lnd_conns_per_peer) { tunables->lnd_conns_per_peer = (conns_per_peer) ? conns_per_peer : 1; @@ -308,6 +309,7 @@ kiblnd_tunables_init(void) default_tunables.lnd_fmr_pool_size = fmr_pool_size; default_tunables.lnd_fmr_flush_trigger = fmr_flush_trigger; default_tunables.lnd_fmr_cache = fmr_cache; + default_tunables.lnd_ntx = ntx; default_tunables.lnd_conns_per_peer = conns_per_peer; return 0; } diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c index 35192e4..2630d7e 100644 --- a/lnet/utils/lnetconfig/liblnetconfig.c +++ b/lnet/utils/lnetconfig/liblnetconfig.c @@ -1153,7 +1153,7 @@ lustre_lnet_ioctl_config_ni(struct list_head *intf_list, list_for_each_entry(intf_descr, intf_list, intf_on_network) { - if (i == 0 && tunables != NULL) + if (tunables != NULL) len = sizeof(struct lnet_ioctl_config_ni) + sizeof(struct lnet_ioctl_config_lnd_tunables); else @@ -1163,7 +1163,7 @@ lustre_lnet_ioctl_config_ni(struct list_head *intf_list, if (!data) return LUSTRE_CFG_RC_OUT_OF_MEM; conf = (struct lnet_ioctl_config_ni*) data; - if (i == 0 && tunables != NULL) + if (tunables != NULL) tun = (struct lnet_ioctl_config_lnd_tunables*) conf->lic_bulk; @@ -1197,8 +1197,7 @@ lustre_lnet_ioctl_config_ni(struct list_head *intf_list, conf->lic_ncpts = count; - if (i == 0 && tunables != NULL) - /* TODO put in the LND tunables */ + if (tunables != NULL) memcpy(tun, tunables, sizeof(*tunables)); rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_ADD_LOCAL_NI, data); diff --git a/lnet/utils/lnetconfig/liblnetconfig_lnd.c b/lnet/utils/lnetconfig/liblnetconfig_lnd.c index ae5d7706..5aa8787 100644 --- a/lnet/utils/lnetconfig/liblnetconfig_lnd.c +++ b/lnet/utils/lnetconfig/liblnetconfig_lnd.c @@ -62,6 +62,10 @@ lustre_o2iblnd_show_tun(struct cYAML *lndparams, lnd_cfg->lnd_fmr_cache) == NULL) return LUSTRE_CFG_RC_OUT_OF_MEM; + if (cYAML_create_number(lndparams, "ntx", + lnd_cfg->lnd_ntx) == NULL) + return LUSTRE_CFG_RC_OUT_OF_MEM; + if (cYAML_create_number(lndparams, "conns_per_peer", lnd_cfg->lnd_conns_per_peer) == NULL) return LUSTRE_CFG_RC_OUT_OF_MEM; @@ -123,7 +127,7 @@ yaml_extract_o2ib_tun(struct cYAML *tree, struct cYAML *map_on_demand = NULL, *concurrent_sends = NULL; struct cYAML *fmr_pool_size = NULL, *fmr_cache = NULL; struct cYAML *fmr_flush_trigger = NULL, *lndparams = NULL; - struct cYAML *conns_per_peer = NULL; + struct cYAML *conns_per_peer = NULL, *ntx = NULL; lndparams = cYAML_get_object_item(tree, "lnd tunables"); if (!lndparams) @@ -150,6 +154,9 @@ yaml_extract_o2ib_tun(struct cYAML *tree, lnd_cfg->lnd_fmr_cache = (fmr_cache) ? fmr_cache->cy_valueint : 0; + ntx = cYAML_get_object_item(lndparams, "ntx"); + lnd_cfg->lnd_ntx = (ntx) ? ntx->cy_valueint : 0; + conns_per_peer = cYAML_get_object_item(lndparams, "conns_per_peer"); lnd_cfg->lnd_conns_per_peer = (conns_per_peer) ? conns_per_peer->cy_valueint : 1; -- 1.8.3.1