LU-16994 kfilnd: Add work queue parameters

Added kfilnd work queue parameters to allow tuning.

The wq_high_priority parameter enables the work queue to run
as high priority. Default is enabled.

The wq_cpu_intensive parameter enables the work queue to run
as cpu intensive. Default is disabled.

The wq_max_active parameter sets the max in-flight work items
of the work queue. Default is 512.

The prov_cpu_exclusive parameter enables reserving one of a
CPT's CPUs for the exlusive use of the kfabric provider.
Default is disabled.

Test-Parameters: trivial
HPE-bug-id: LUS-11605
Signed-off-by: Ron Gredvig <ron.gredvig@hpe.com>
Change-Id: Ic4db95787a864efca3ea1234953ce3ea828f3594
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51786
Reviewed-by: Ian Ziemba <ian.ziemba@hpe.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>

diff --git a/lnet/klnds/kfilnd/kfilnd.c b/lnet/klnds/kfilnd/kfilnd.c
index 5854057..3bd70c7 100644 (file)
--- a/lnet/klnds/kfilnd/kfilnd.c
+++ b/lnet/klnds/kfilnd/kfilnd.c
@@ -34,14 +34,6 @@
 #include "kfilnd_tn.h"
 #include "kfilnd_dev.h"
 
-/* These are temp constants to get stuff to compile */
-#define KFILND_DEFAULT_DEVICE "eth0"
-
-/* Some constants which should be turned into tunables */
-#define KFILND_MAX_WORKER_THREADS 4
-#define KFILND_MAX_EVENT_QUEUE 100
-
-#define KFILND_WQ_FLAGS (WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_SYSFS)
 struct workqueue_struct *kfilnd_wq;
 struct dentry *kfilnd_debug_dir;
 
@@ -485,6 +477,7 @@ static void __exit kfilnd_exit(void)
 static int __init kfilnd_init(void)
 {
        int rc;
+       unsigned int flags;
 
        kfilnd_debug_dir = debugfs_create_dir("kfilnd", NULL);
 
@@ -499,8 +492,13 @@ static int __init kfilnd_init(void)
                goto err;
        }
 
-       kfilnd_wq = alloc_workqueue("kfilnd_wq", KFILND_WQ_FLAGS,
-                                   WQ_MAX_ACTIVE);
+       flags = WQ_MEM_RECLAIM | WQ_SYSFS;
+       if (wq_cpu_intensive)
+               flags = flags | WQ_CPU_INTENSIVE;
+       if (wq_high_priority)
+               flags = flags | WQ_HIGHPRI;
+
+       kfilnd_wq = alloc_workqueue("kfilnd_wq", flags, wq_max_active);
        if (!kfilnd_wq) {
                rc = -ENOMEM;
                CERROR("Failed to allocated kfilnd work queue\n");

diff --git a/lnet/klnds/kfilnd/kfilnd.h b/lnet/klnds/kfilnd/kfilnd.h
index cead25b..edd99c0 100644 (file)
--- a/lnet/klnds/kfilnd/kfilnd.h
+++ b/lnet/klnds/kfilnd/kfilnd.h
@@ -158,6 +158,10 @@ extern unsigned int rx_cq_scale_factor;
 extern unsigned int tx_cq_scale_factor;
 extern unsigned int eq_size;
 extern unsigned int immediate_rx_buf_count;
+extern unsigned int prov_cpu_exclusive;
+extern unsigned int wq_high_priority;
+extern unsigned int wq_cpu_intensive;
+extern unsigned int wq_max_active;
 
 int kfilnd_tunables_setup(struct lnet_ni *ni);
 int kfilnd_tunables_init(void);

diff --git a/lnet/klnds/kfilnd/kfilnd_cq.c b/lnet/klnds/kfilnd/kfilnd_cq.c
index 5b5678e..fd651dc 100644 (file)
--- a/lnet/klnds/kfilnd/kfilnd_cq.c
+++ b/lnet/klnds/kfilnd/kfilnd_cq.c
@@ -188,8 +188,14 @@ static void kfilnd_cq_completion(struct kfid_cq *cq, void *context)
        struct kfilnd_cq *kfilnd_cq = context;
        struct kfilnd_cq_work *cq_work;
        unsigned int i;
-
-       for (i = 0; i < kfilnd_cq->cq_work_count; i++) {
+       unsigned int start_count;
+
+       /* kcxi provider queues on signaling vector (index 0 cpu),
+        * optionally don't queue kfilnd on that cpu
+        */
+       start_count = kfilnd_cq->cq_work_count > 1 &&
+                       prov_cpu_exclusive ? 1 : 0;
+       for (i = start_count; i < kfilnd_cq->cq_work_count; i++) {
                cq_work = &kfilnd_cq->cq_works[i];
                queue_work_on(cq_work->work_cpu, kfilnd_wq, &cq_work->work);
        }

diff --git a/lnet/klnds/kfilnd/kfilnd_modparams.c b/lnet/klnds/kfilnd/kfilnd_modparams.c
index 22c84c9..41d31d5 100644 (file)
--- a/lnet/klnds/kfilnd/kfilnd_modparams.c
+++ b/lnet/klnds/kfilnd/kfilnd_modparams.c
@@ -65,6 +65,26 @@ module_param(immediate_rx_buf_count, uint, 0444);
 MODULE_PARM_DESC(immediate_rx_buf_count,
                 "Number of immediate multi-receive buffers posted per CPT");
 
+unsigned int prov_cpu_exclusive;
+module_param(prov_cpu_exclusive, uint, 0644);
+MODULE_PARM_DESC(prov_cpu_exclusive,
+                "Enables kfabric provider exclusive use of CPT's base CPU. Disabled by default. Set > 0 to enable.");
+
+unsigned int wq_high_priority = 1;
+module_param(wq_high_priority, uint, 0444);
+MODULE_PARM_DESC(wq_high_priority,
+                "Enables work queue to run at high priority. Enabled by default. Set > 0 to enable.");
+
+unsigned int wq_cpu_intensive;
+module_param(wq_cpu_intensive, uint, 0444);
+MODULE_PARM_DESC(wq_cpu_intensive,
+                "Marks work queue as CPU intensive. Disabled by default. Set > 0 to enable.");
+
+unsigned int wq_max_active = 512;
+module_param(wq_max_active, uint, 0444);
+MODULE_PARM_DESC(wq_max_active,
+                "Max work queue work items active per CPU. Default is 512. Valid values 0 to 512.");
+
 /* Common LND network tunables. */
 static int credits = 256;
 module_param(credits, int, 0444);
@@ -220,6 +240,9 @@ int kfilnd_tunables_init(void)
                return -EINVAL;
        }
 
+       if (wq_max_active > WQ_MAX_ACTIVE)
+               wq_max_active = WQ_MAX_ACTIVE;
+
        if (auth_key < 1) {
                CERROR("Authorization key cannot be less than 1");
                return -EINVAL;