LU-9536 lnet: set LND tunalbes properly

Make sure to set all NIs to the proper LND tunables
specified.

Add ntx tunable to dynamic configuration. This way all
tunables required to tune OPA performance can be configured
via lnetctl, allowing the ability to tune OPA network and IB
network differently

Test-Parameters: trivial
Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Change-Id: I015f3f959bd46784d4607bd4259b4640303dc362
Reviewed-on: https://review.whamcloud.com/27263
Tested-by: Jenkins
Reviewed-by: Doug Oucharek <doug.s.oucharek@intel.com>
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Sonia Sharma <sonia.sharma@intel.com>
Reviewed-by: Olaf Weber <olaf.weber@hpe.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lnet/include/lnet/lib-dlc.h b/lnet/include/lnet/lib-dlc.h
index 48662b9..4141f7c 100644 (file)
--- a/lnet/include/lnet/lib-dlc.h
+++ b/lnet/include/lnet/lib-dlc.h
@@ -66,7 +66,7 @@ struct lnet_ioctl_config_o2iblnd_tunables {
        __u32 lnd_fmr_flush_trigger;
        __u32 lnd_fmr_cache;
        __u16 lnd_conns_per_peer;
-       __u16 pad;
+       __u16 lnd_ntx;
 };
 
 struct lnet_lnd_tunables {

diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index 12dade5..298cd9f 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -2305,9 +2305,13 @@ out:
         LIBCFS_FREE(tpo, sizeof(kib_tx_pool_t));
 }
 
-static int kiblnd_tx_pool_size(int ncpts)
+static int kiblnd_tx_pool_size(struct lnet_ni *ni, int ncpts)
 {
-       int ntx = *kiblnd_tunables.kib_ntx / ncpts;
+       struct lnet_ioctl_config_o2iblnd_tunables *tunables;
+       int ntx;
+
+       tunables = &ni->ni_lnd_tunables.lnd_tun_u.lnd_o2ib;
+       ntx = tunables->lnd_ntx / ncpts;
 
        return max(IBLND_TX_POOL, ntx);
 }
@@ -2464,10 +2468,10 @@ kiblnd_net_init_pools(kib_net_t *net, struct lnet_ni *ni, __u32 *cpts,
        read_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
 #endif
 
-       if (tunables->lnd_fmr_pool_size < *kiblnd_tunables.kib_ntx / 4) {
+       if (tunables->lnd_fmr_pool_size < tunables->lnd_ntx / 4) {
                CERROR("Can't set fmr pool size (%d) < ntx / 4(%d)\n",
                       tunables->lnd_fmr_pool_size,
-                      *kiblnd_tunables.kib_ntx / 4);
+                      tunables->lnd_ntx / 4);
                rc = -EINVAL;
                goto failed;
        }
@@ -2516,7 +2520,7 @@ kiblnd_net_init_pools(kib_net_t *net, struct lnet_ni *ni, __u32 *cpts,
                cpt = (cpts == NULL) ? i : cpts[i];
                rc = kiblnd_init_poolset(&net->ibn_tx_ps[cpt]->tps_poolset,
                                         cpt, net, "TX",
-                                        kiblnd_tx_pool_size(ncpts),
+                                        kiblnd_tx_pool_size(ni, ncpts),
                                         kiblnd_create_tx_pool,
                                         kiblnd_destroy_tx_pool,
                                         kiblnd_tx_init, NULL);

diff --git a/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
index 3ca0689..72cb50e 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd_modparams.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
@@ -162,7 +162,6 @@ kib_tunables_t kiblnd_tunables = {
         .kib_cksum                  = &cksum,
         .kib_timeout                = &timeout,
         .kib_keepalive              = &keepalive,
-        .kib_ntx                    = &ntx,
         .kib_default_ipif           = &ipif_name,
         .kib_retry_count            = &retry_count,
         .kib_rnr_retry_count        = &rnr_retry_count,
@@ -290,6 +289,8 @@ kiblnd_tunables_setup(struct lnet_ni *ni)
                tunables->lnd_fmr_flush_trigger = fmr_flush_trigger;
        if (!tunables->lnd_fmr_cache)
                tunables->lnd_fmr_cache = fmr_cache;
+       if (!tunables->lnd_ntx)
+               tunables->lnd_ntx = ntx;
        if (!tunables->lnd_conns_per_peer) {
                tunables->lnd_conns_per_peer = (conns_per_peer) ?
                        conns_per_peer : 1;
@@ -308,6 +309,7 @@ kiblnd_tunables_init(void)
        default_tunables.lnd_fmr_pool_size = fmr_pool_size;
        default_tunables.lnd_fmr_flush_trigger = fmr_flush_trigger;
        default_tunables.lnd_fmr_cache = fmr_cache;
+       default_tunables.lnd_ntx = ntx;
        default_tunables.lnd_conns_per_peer = conns_per_peer;
        return 0;
 }

diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c
index 35192e4..2630d7e 100644 (file)
--- a/lnet/utils/lnetconfig/liblnetconfig.c
+++ b/lnet/utils/lnetconfig/liblnetconfig.c
@@ -1153,7 +1153,7 @@ lustre_lnet_ioctl_config_ni(struct list_head *intf_list,
 
        list_for_each_entry(intf_descr, intf_list,
                            intf_on_network) {
-               if (i == 0 && tunables != NULL)
+               if (tunables != NULL)
                        len = sizeof(struct lnet_ioctl_config_ni) +
                              sizeof(struct lnet_ioctl_config_lnd_tunables);
                else
@@ -1163,7 +1163,7 @@ lustre_lnet_ioctl_config_ni(struct list_head *intf_list,
                if (!data)
                        return LUSTRE_CFG_RC_OUT_OF_MEM;
                conf = (struct lnet_ioctl_config_ni*) data;
-               if (i == 0 && tunables != NULL)
+               if (tunables != NULL)
                        tun = (struct lnet_ioctl_config_lnd_tunables*)
                                conf->lic_bulk;
 
@@ -1197,8 +1197,7 @@ lustre_lnet_ioctl_config_ni(struct list_head *intf_list,
 
                conf->lic_ncpts = count;
 
-               if (i == 0 && tunables != NULL)
-                       /* TODO put in the LND tunables */
+               if (tunables != NULL)
                        memcpy(tun, tunables, sizeof(*tunables));
 
                rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_ADD_LOCAL_NI, data);

diff --git a/lnet/utils/lnetconfig/liblnetconfig_lnd.c b/lnet/utils/lnetconfig/liblnetconfig_lnd.c
index ae5d770..5aa8787 100644 (file)
--- a/lnet/utils/lnetconfig/liblnetconfig_lnd.c
+++ b/lnet/utils/lnetconfig/liblnetconfig_lnd.c
@@ -62,6 +62,10 @@ lustre_o2iblnd_show_tun(struct cYAML *lndparams,
                                lnd_cfg->lnd_fmr_cache) == NULL)
                return LUSTRE_CFG_RC_OUT_OF_MEM;
 
+       if (cYAML_create_number(lndparams, "ntx",
+                               lnd_cfg->lnd_ntx) == NULL)
+               return LUSTRE_CFG_RC_OUT_OF_MEM;
+
        if (cYAML_create_number(lndparams, "conns_per_peer",
                                lnd_cfg->lnd_conns_per_peer) == NULL)
                return LUSTRE_CFG_RC_OUT_OF_MEM;
@@ -123,7 +127,7 @@ yaml_extract_o2ib_tun(struct cYAML *tree,
        struct cYAML *map_on_demand = NULL, *concurrent_sends = NULL;
        struct cYAML *fmr_pool_size = NULL, *fmr_cache = NULL;
        struct cYAML *fmr_flush_trigger = NULL, *lndparams = NULL;
-       struct cYAML *conns_per_peer = NULL;
+       struct cYAML *conns_per_peer = NULL, *ntx = NULL;
 
        lndparams = cYAML_get_object_item(tree, "lnd tunables");
        if (!lndparams)
@@ -150,6 +154,9 @@ yaml_extract_o2ib_tun(struct cYAML *tree,
        lnd_cfg->lnd_fmr_cache =
                (fmr_cache) ? fmr_cache->cy_valueint : 0;
 
+       ntx = cYAML_get_object_item(lndparams, "ntx");
+       lnd_cfg->lnd_ntx = (ntx) ? ntx->cy_valueint : 0;
+
        conns_per_peer = cYAML_get_object_item(lndparams, "conns_per_peer");
        lnd_cfg->lnd_conns_per_peer =
                (conns_per_peer) ? conns_per_peer->cy_valueint : 1;