LU-1164 o2iblnd: param to tune number of kib scheduler threads

This patch gives the ability to control the number of kib scheduler
threads launched by the ko2iblnd module, via a kernel module option.

Indeed, we can have some situations where the default threads number
is not appropriate. The default value is to create as many threads as
the number of CPU cores.

On certain platforms, we have noticed a performance penalty
when running too many kib scheduler threads.

Signed-off-by: Sebastien Buisson <sebastien.buisson@bull.net>
Change-Id: I2f7baa2f9bc9350502d1a0738c8e72777b57fa57
Reviewed-on: http://review.whamcloud.com/3047
Tested-by: Hudson
Reviewed-by: Liang Zhen <liang@whamcloud.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Doug Oucharek <doug@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index 5eed508..c657e22 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -2791,7 +2791,7 @@ kiblnd_base_startup (void)
         kiblnd_data.kib_init = IBLND_INIT_DATA;
         /*****************************************************/
 
-        for (i = 0; i < IBLND_N_SCHED; i++) {
+        for (i = 0; i < *kiblnd_tunables.kib_schedulers; i++) {
                 rc = kiblnd_thread_start(kiblnd_scheduler, (void *)((long)i));
                 if (rc != 0) {
                         CERROR("Can't spawn o2iblnd scheduler[%d]: %d\n",

diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h
index c0ca1b6..4e29ffb 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/lnet/klnds/o2iblnd/o2iblnd.h
@@ -127,6 +127,7 @@ typedef struct
 #endif
         int              *kib_require_priv_port;/* accept only privileged ports */
         int              *kib_use_priv_port;    /* use privileged port for active connect */
+        int              *kib_schedulers;       /* schedulers */
 } kib_tunables_t;
 
 extern kib_tunables_t  kiblnd_tunables;

diff --git a/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
index 52ce830..d5f83b0 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd_modparams.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
@@ -138,6 +138,10 @@ static int use_privileged_port = 1;
 CFS_MODULE_PARM(use_privileged_port, "i", int, 0644,
                 "use privileged port when initiating connection");
 
+static int schedulers; /* initialized to zero by compiler */
+CFS_MODULE_PARM(schedulers, "i", int, 0444,
+                "Schedulers");
+
 kib_tunables_t kiblnd_tunables = {
         .kib_dev_failover           = &dev_failover,
         .kib_service                = &service,
@@ -161,7 +165,8 @@ kib_tunables_t kiblnd_tunables = {
         .kib_fmr_cache              = &fmr_cache,
         .kib_pmr_pool_size          = &pmr_pool_size,
         .kib_require_priv_port      = &require_privileged_port,
-        .kib_use_priv_port          = &use_privileged_port
+        .kib_use_priv_port          = &use_privileged_port,
+        .kib_schedulers             = &schedulers
 };
 
 #if defined(CONFIG_SYSCTL) && !CFS_SYSFS_MODULE_PARM
@@ -191,7 +196,8 @@ enum {
         O2IBLND_FMR_FLUSH_TRIGGER,
         O2IBLND_FMR_CACHE,
         O2IBLND_PMR_POOL_SIZE,
-        O2IBLND_DEV_FAILOVER
+        O2IBLND_DEV_FAILOVER,
+        O2IBLND_SCHEDULERS
 };
 #else
 
@@ -216,6 +222,7 @@ enum {
 #define O2IBLND_FMR_CACHE        CTL_UNNUMBERED
 #define O2IBLND_PMR_POOL_SIZE    CTL_UNNUMBERED
 #define O2IBLND_DEV_FAILOVER     CTL_UNNUMBERED
+#define O2IBLND_SCHEDULERS       CTL_UNNUMBERED
 
 #endif
 
@@ -389,6 +396,14 @@ static cfs_sysctl_table_t kiblnd_ctl_table[] = {
                 .mode     = 0444,
                 .proc_handler = &proc_dointvec
         },
+        {
+                .ctl_name = O2IBLND_SCHEDULERS,
+                .procname = "schedulers",
+                .data     = &schedulers,
+                .maxlen   = sizeof(int),
+                .mode     = 0444,
+                .proc_handler = &proc_dointvec
+        },
         {0}
 };
 
@@ -496,6 +511,14 @@ kiblnd_tunables_init (void)
                       *kiblnd_tunables.kib_concurrent_sends, *kiblnd_tunables.kib_peertxcredits);
         }
 
+        if (*kiblnd_tunables.kib_schedulers <= 0) {
+                *kiblnd_tunables.kib_schedulers = IBLND_N_SCHED;
+        } else if (*kiblnd_tunables.kib_schedulers > IBLND_N_SCHED) {
+                CWARN("Number of schedulers %d exceeds machine capacity, lowering to %d .\n",
+                      *kiblnd_tunables.kib_schedulers, IBLND_N_SCHED);
+                *kiblnd_tunables.kib_schedulers = IBLND_N_SCHED;
+        }
+
         kiblnd_sysctl_init();
         return 0;
 }