From 8a43201a2f35f6210d7feb7fd02b79233dd6da6b Mon Sep 17 00:00:00 2001 From: James Simmons Date: Fri, 27 Mar 2020 12:26:30 -0400 Subject: [PATCH] LU-13258 ptlrpc: bind pinger workqueue to CPT set Lustre uses a pinger to determine if nodes are available which is done using a workqueue. The workqueue can currently run on any core. This has performance impact on applications but this can be avoided by binding the workqueues to the cores Lustre as set aside for its self. Additionally create a proper kernel config option for enabling the pinger. Change-Id: I6fefb0f1fd266ef4ac7016f092fe9e7908f010e9 Signed-off-by: James Simmons Reviewed-on: https://review.whamcloud.com/38091 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Shaun Tancheff Reviewed-by: Oleg Drokin --- libcfs/autoconf/lustre-libcfs.m4 | 2 +- lustre/autoconf/lustre-core.m4 | 2 +- lustre/obdclass/obd_sysfs.c | 2 +- lustre/ptlrpc/import.c | 2 +- lustre/ptlrpc/pinger.c | 24 ++++++++++++++++++------ 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/libcfs/autoconf/lustre-libcfs.m4 b/libcfs/autoconf/lustre-libcfs.m4 index 5992d2a..8db36ae 100644 --- a/libcfs/autoconf/lustre-libcfs.m4 +++ b/libcfs/autoconf/lustre-libcfs.m4 @@ -813,7 +813,7 @@ cpu_read_lock, [ cpus_read_lock(); cpus_read_unlock(); ],[ - AC_DEFINE(HAVE_CPUS_READ_LOCK, 1, ['cpu_read_lock' exist]) + AC_DEFINE(HAVE_CPUS_READ_LOCK, 1, ['cpus_read_lock' exist]) ]) ]) # LIBCFS_CPUS_READ_LOCK diff --git a/lustre/autoconf/lustre-core.m4 b/lustre/autoconf/lustre-core.m4 index 02db2a7..3ab5094 100644 --- a/lustre/autoconf/lustre-core.m4 +++ b/lustre/autoconf/lustre-core.m4 @@ -102,7 +102,7 @@ AC_ARG_ENABLE([pinger], [], [enable_pinger="yes"]) AC_MSG_RESULT([$enable_pinger]) AS_IF([test "x$enable_pinger" != xno], - [AC_DEFINE(ENABLE_PINGER, 1,[Use the Pinger])]) + [AC_DEFINE(CONFIG_LUSTRE_PINGER, 1,[Use the Pinger])]) ]) # LC_CONFIG_PINGER # diff --git a/lustre/obdclass/obd_sysfs.c b/lustre/obdclass/obd_sysfs.c index 6493ad9..84bc676 100644 --- a/lustre/obdclass/obd_sysfs.c +++ b/lustre/obdclass/obd_sysfs.c @@ -214,7 +214,7 @@ static ssize_t version_show(struct kobject *kobj, struct attribute *attr, static ssize_t pinger_show(struct kobject *kobj, struct attribute *attr, char *buf) { -#ifdef ENABLE_PINGER +#ifdef CONFIG_LUSTRE_PINGER const char *state = "on"; #else const char *state = "off"; diff --git a/lustre/ptlrpc/import.c b/lustre/ptlrpc/import.c index 4fd5193..321385c 100644 --- a/lustre/ptlrpc/import.c +++ b/lustre/ptlrpc/import.c @@ -466,7 +466,7 @@ void ptlrpc_fail_import(struct obd_import *imp, __u32 conn_cnt) int ptlrpc_reconnect_import(struct obd_import *imp) { -#ifdef ENABLE_PINGER +#ifdef CONFIG_LUSTRE_PINGER long timeout_jiffies = cfs_time_seconds(obd_timeout); int rc; diff --git a/lustre/ptlrpc/pinger.c b/lustre/ptlrpc/pinger.c index 481b73b..2a80c45 100644 --- a/lustre/ptlrpc/pinger.c +++ b/lustre/ptlrpc/pinger.c @@ -148,7 +148,7 @@ static int ptlrpc_ping(struct obd_import *imp) static void ptlrpc_update_next_ping(struct obd_import *imp, int soon) { -#ifdef ENABLE_PINGER +#ifdef CONFIG_LUSTRE_PINGER time64_t time = soon ? PING_INTERVAL_SHORT : PING_INTERVAL; if (imp->imp_state == LUSTRE_IMP_DISCON) { @@ -158,7 +158,7 @@ static void ptlrpc_update_next_ping(struct obd_import *imp, int soon) time = min(time, dtime); } imp->imp_next_ping = ktime_get_seconds() + time; -#endif /* ENABLE_PINGER */ +#endif /* CONFIG_LUSTRE_PINGER */ } void ptlrpc_ping_import_soon(struct obd_import *imp) @@ -323,16 +323,28 @@ static void ptlrpc_pinger_main(struct work_struct *ws) int ptlrpc_start_pinger(void) { -#ifdef ENABLE_PINGER +#ifdef CONFIG_LUSTRE_PINGER + struct workqueue_attrs attrs = { }; + cpumask_var_t *mask; + if (pinger_wq) return -EALREADY; - pinger_wq = alloc_workqueue("ptlrpc_pinger", 0, 1); + pinger_wq = alloc_workqueue("ptlrpc_pinger", WQ_UNBOUND, 1); if (!pinger_wq) { CERROR("cannot start pinger workqueue\n"); return -ENOMEM; } + mask = cfs_cpt_cpumask(cfs_cpt_tab, CFS_CPT_ANY); + if (mask && alloc_cpumask_var(&attrs.cpumask, GFP_KERNEL)) { + cpumask_copy(attrs.cpumask, *mask); + cpus_read_lock(); + cfs_apply_workqueue_attrs(pinger_wq, &attrs); + cpus_read_unlock(); + free_cpumask_var(attrs.cpumask); + } + queue_delayed_work(pinger_wq, &ping_work, 0); if (suppress_pings) @@ -345,7 +357,7 @@ int ptlrpc_pinger_remove_timeouts(void); int ptlrpc_stop_pinger(void) { -#ifdef ENABLE_PINGER +#ifdef CONFIG_LUSTRE_PINGER if (!pinger_wq) return -EALREADY; @@ -540,7 +552,7 @@ int ptlrpc_pinger_remove_timeouts(void) void ptlrpc_pinger_wake_up(void) { -#ifdef ENABLE_PINGER +#ifdef CONFIG_LUSTRE_PINGER mod_delayed_work(pinger_wq, &ping_work, 0); #endif } -- 1.8.3.1