LU-13258 ptlrpc: bind pinger workqueue to CPT set

Lustre uses a pinger to determine if nodes are available which is
done using a workqueue. The workqueue can currently run on any
core. This has performance impact on applications but this can be
avoided by binding the workqueues to the cores Lustre as set aside
for its self. Additionally create a proper kernel config option
for enabling the pinger.

Change-Id: I6fefb0f1fd266ef4ac7016f092fe9e7908f010e9
Signed-off-by: James Simmons <jsimmons@infradead.org>
Reviewed-on: https://review.whamcloud.com/38091
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Shaun Tancheff <shaun.tancheff@hpe.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/libcfs/autoconf/lustre-libcfs.m4 b/libcfs/autoconf/lustre-libcfs.m4
index 5992d2a..8db36ae 100644 (file)
--- a/libcfs/autoconf/lustre-libcfs.m4
+++ b/libcfs/autoconf/lustre-libcfs.m4
@@ -813,7 +813,7 @@ cpu_read_lock, [
        cpus_read_lock();
        cpus_read_unlock();
 ],[
-       AC_DEFINE(HAVE_CPUS_READ_LOCK, 1, ['cpu_read_lock' exist])
+       AC_DEFINE(HAVE_CPUS_READ_LOCK, 1, ['cpus_read_lock' exist])
 ])
 ]) # LIBCFS_CPUS_READ_LOCK
 

diff --git a/lustre/autoconf/lustre-core.m4 b/lustre/autoconf/lustre-core.m4
index 02db2a7..3ab5094 100644 (file)
--- a/lustre/autoconf/lustre-core.m4
+++ b/lustre/autoconf/lustre-core.m4
@@ -102,7 +102,7 @@ AC_ARG_ENABLE([pinger],
        [], [enable_pinger="yes"])
 AC_MSG_RESULT([$enable_pinger])
 AS_IF([test "x$enable_pinger" != xno],
-       [AC_DEFINE(ENABLE_PINGER, 1,[Use the Pinger])])
+       [AC_DEFINE(CONFIG_LUSTRE_PINGER, 1,[Use the Pinger])])
 ]) # LC_CONFIG_PINGER
 
 #

diff --git a/lustre/obdclass/obd_sysfs.c b/lustre/obdclass/obd_sysfs.c
index 6493ad9..84bc676 100644 (file)
--- a/lustre/obdclass/obd_sysfs.c
+++ b/lustre/obdclass/obd_sysfs.c
@@ -214,7 +214,7 @@ static ssize_t version_show(struct kobject *kobj, struct attribute *attr,
 static ssize_t pinger_show(struct kobject *kobj, struct attribute *attr,
                           char *buf)
 {
-#ifdef ENABLE_PINGER
+#ifdef CONFIG_LUSTRE_PINGER
        const char *state = "on";
 #else
        const char *state = "off";

diff --git a/lustre/ptlrpc/import.c b/lustre/ptlrpc/import.c
index 4fd5193..321385c 100644 (file)
--- a/lustre/ptlrpc/import.c
+++ b/lustre/ptlrpc/import.c
@@ -466,7 +466,7 @@ void ptlrpc_fail_import(struct obd_import *imp, __u32 conn_cnt)
 
 int ptlrpc_reconnect_import(struct obd_import *imp)
 {
-#ifdef ENABLE_PINGER
+#ifdef CONFIG_LUSTRE_PINGER
        long timeout_jiffies = cfs_time_seconds(obd_timeout);
        int rc;
 

diff --git a/lustre/ptlrpc/pinger.c b/lustre/ptlrpc/pinger.c
index 481b73b..2a80c45 100644 (file)
--- a/lustre/ptlrpc/pinger.c
+++ b/lustre/ptlrpc/pinger.c
@@ -148,7 +148,7 @@ static int ptlrpc_ping(struct obd_import *imp)
 
 static void ptlrpc_update_next_ping(struct obd_import *imp, int soon)
 {
-#ifdef ENABLE_PINGER
+#ifdef CONFIG_LUSTRE_PINGER
        time64_t time = soon ? PING_INTERVAL_SHORT : PING_INTERVAL;
 
        if (imp->imp_state == LUSTRE_IMP_DISCON) {
@@ -158,7 +158,7 @@ static void ptlrpc_update_next_ping(struct obd_import *imp, int soon)
                time = min(time, dtime);
        }
        imp->imp_next_ping = ktime_get_seconds() + time;
-#endif /* ENABLE_PINGER */
+#endif /* CONFIG_LUSTRE_PINGER */
 }
 
 void ptlrpc_ping_import_soon(struct obd_import *imp)
@@ -323,16 +323,28 @@ static void ptlrpc_pinger_main(struct work_struct *ws)
 
 int ptlrpc_start_pinger(void)
 {
-#ifdef ENABLE_PINGER
+#ifdef CONFIG_LUSTRE_PINGER
+       struct workqueue_attrs attrs = { };
+       cpumask_var_t *mask;
+
        if (pinger_wq)
                return -EALREADY;
 
-       pinger_wq = alloc_workqueue("ptlrpc_pinger", 0, 1);
+       pinger_wq = alloc_workqueue("ptlrpc_pinger", WQ_UNBOUND, 1);
        if (!pinger_wq) {
                CERROR("cannot start pinger workqueue\n");
                return -ENOMEM;
        }
 
+       mask = cfs_cpt_cpumask(cfs_cpt_tab, CFS_CPT_ANY);
+       if (mask && alloc_cpumask_var(&attrs.cpumask, GFP_KERNEL)) {
+               cpumask_copy(attrs.cpumask, *mask);
+               cpus_read_lock();
+               cfs_apply_workqueue_attrs(pinger_wq, &attrs);
+               cpus_read_unlock();
+               free_cpumask_var(attrs.cpumask);
+       }
+
        queue_delayed_work(pinger_wq, &ping_work, 0);
 
        if (suppress_pings)
@@ -345,7 +357,7 @@ int ptlrpc_pinger_remove_timeouts(void);
 
 int ptlrpc_stop_pinger(void)
 {
-#ifdef ENABLE_PINGER
+#ifdef CONFIG_LUSTRE_PINGER
        if (!pinger_wq)
                return -EALREADY;
 
@@ -540,7 +552,7 @@ int ptlrpc_pinger_remove_timeouts(void)
 
 void ptlrpc_pinger_wake_up(void)
 {
-#ifdef ENABLE_PINGER
+#ifdef CONFIG_LUSTRE_PINGER
        mod_delayed_work(pinger_wq, &ping_work, 0);
 #endif
 }