LU-56 o2iblnd: CPT affinity o2iblnd

[fs/lustre-release.git] / lnet / klnds / o2iblnd / o2iblnd_modparams.c

diff --git a/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
index 1b96ff3..c654ab6 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd_modparams.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
@@ -50,11 +50,19 @@ static int timeout = 50;
 CFS_MODULE_PARM(timeout, "i", int, 0644,
                 "timeout (seconds)");
 
-static int ntx = 256;
+/* Number of threads in each scheduler pool which is percpt,
+ * we will estimate reasonable value based on CPUs if it's set to zero. */
+static int nscheds;
+CFS_MODULE_PARM(nscheds, "i", int, 0444,
+               "number of threads in each scheduler pool");
+
+/* NB: this value is shared by all CPTs, it can grow at runtime */
+static int ntx = 512;
 CFS_MODULE_PARM(ntx, "i", int, 0444,
-                "# of message descriptors");
+               "# of message descriptors allocated for each pool");
 
-static int credits = 64;
+/* NB: this value is shared by all CPTs */
+static int credits = 256;
 CFS_MODULE_PARM(credits, "i", int, 0444,
                 "# concurrent sends");
 
@@ -102,21 +110,24 @@ static int map_on_demand = 0;
 CFS_MODULE_PARM(map_on_demand, "i", int, 0444,
                 "map on demand");
 
+/* NB: this value is shared by all CPTs, it can grow at runtime */
 static int fmr_pool_size = 512;
 CFS_MODULE_PARM(fmr_pool_size, "i", int, 0444,
-                "size of the fmr pool (>= ntx / 4)");
+               "size of fmr pool on each CPT (>= ntx / 4)");
 
+/* NB: this value is shared by all CPTs, it can grow at runtime */
 static int fmr_flush_trigger = 384;
 CFS_MODULE_PARM(fmr_flush_trigger, "i", int, 0444,
-                "# dirty FMRs that triggers pool flush");
+               "# dirty FMRs that triggers pool flush");
 
 static int fmr_cache = 1;
 CFS_MODULE_PARM(fmr_cache, "i", int, 0444,
-                "non-zero to enable FMR caching");
+               "non-zero to enable FMR caching");
 
+/* NB: this value is shared by all CPTs, it can grow at runtime */
 static int pmr_pool_size = 512;
 CFS_MODULE_PARM(pmr_pool_size, "i", int, 0444,
-                "size of the MR cache pmr pool");
+               "size of MR cache pmr pool on each CPT");
 
 /*
  * 0: disable failover
@@ -159,7 +170,8 @@ kib_tunables_t kiblnd_tunables = {
         .kib_fmr_cache              = &fmr_cache,
         .kib_pmr_pool_size          = &pmr_pool_size,
         .kib_require_priv_port      = &require_privileged_port,
-        .kib_use_priv_port          = &use_privileged_port
+       .kib_use_priv_port          = &use_privileged_port,
+       .kib_nscheds                = &nscheds
 };
 
 #if defined(CONFIG_SYSCTL) && !CFS_SYSFS_MODULE_PARM