From a8b207bf39c0f2b298b57272162d0352e84a3912 Mon Sep 17 00:00:00 2001 From: Timothy Day Date: Mon, 16 Jun 2025 16:33:45 +0000 Subject: [PATCH] LU-18687 compat: move wait/wait_bit to lustre_compat Migrate the backported waiting code to lustre_compat. Eventually, all of the Lustre/LNet compatability code will live in lustre_compat - maintaining a clear separation from the functional code in Lustre and LNet. Test-Parameters: trivial Signed-off-by: Timothy Day Change-Id: I9ffaabf7d4665abb002f11599f993e776e7a38b6 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/59812 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Oleg Drokin Reviewed-by: James Simmons Reviewed-by: Shaun Tancheff --- .../lustre_compat/linux/wait.h | 131 +------------------- include/lustre_compat/linux/wait_bit.h | 132 +++++++++++++++++++++ libcfs/include/libcfs/libcfs.h | 3 +- libcfs/include/libcfs/linux/Makefile.am | 2 +- libcfs/libcfs/Makefile.in | 6 +- libcfs/libcfs/linux/Makefile.am | 3 +- libcfs/libcfs/linux/linux-prim.c | 3 +- lustre_compat/kernel/sched/Makefile | 7 ++ .../kernel/sched/wait.c | 72 +---------- lustre_compat/kernel/sched/wait_bit.c | 75 ++++++++++++ 10 files changed, 230 insertions(+), 204 deletions(-) rename libcfs/include/libcfs/linux/linux-wait.h => include/lustre_compat/linux/wait.h (79%) create mode 100644 include/lustre_compat/linux/wait_bit.h create mode 100644 lustre_compat/kernel/sched/Makefile rename libcfs/libcfs/linux/linux-wait.c => lustre_compat/kernel/sched/wait.c (67%) create mode 100644 lustre_compat/kernel/sched/wait_bit.c diff --git a/libcfs/include/libcfs/linux/linux-wait.h b/include/lustre_compat/linux/wait.h similarity index 79% rename from libcfs/include/libcfs/linux/linux-wait.h rename to include/lustre_compat/linux/wait.h index 36dc30f..904b335 100644 --- a/libcfs/include/libcfs/linux/linux-wait.h +++ b/include/lustre_compat/linux/wait.h @@ -1,16 +1,10 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __LIBCFS_LINUX_WAIT_BIT_H -#define __LIBCFS_LINUX_WAIT_BIT_H +#ifndef __LIBCFS_LINUX_WAIT_H +#define __LIBCFS_LINUX_WAIT_H /* Make sure we can see if we have TASK_NOLOAD */ #include -/* - * Linux wait-bit related types and methods: - */ -#ifdef HAVE_WAIT_BIT_HEADER_H -#include -#endif #include #ifndef HAVE_WAIT_QUEUE_ENTRY @@ -21,18 +15,6 @@ #define __add_wait_queue_entry_tail __add_wait_queue_tail #endif -#ifndef HAVE_WAIT_BIT_HEADER_H -struct wait_bit_queue_entry { - struct wait_bit_key key; - wait_queue_entry_t wq_entry; -}; - -#define ___wait_is_interruptible(state) \ - (!__builtin_constant_p(state) || \ - state == TASK_INTERRUPTIBLE || state == TASK_KILLABLE) \ - -#endif /* ! HAVE_WAIT_BIT_HEADER_H */ - #ifndef HAVE_PREPARE_TO_WAIT_EVENT extern long prepare_to_wait_event(wait_queue_head_t *wq_head, wait_queue_entry_t *wq_entry, int state); @@ -50,113 +32,6 @@ extern long prepare_to_wait_event(wait_queue_head_t *wq_head, __cond || !__ret; \ }) -#ifndef HAVE_CLEAR_AND_WAKE_UP_BIT -/** - * clear_and_wake_up_bit - clear a bit and wake up anyone waiting on that bit - * - * @bit: the bit of the word being waited on - * @word: the word being waited on, a kernel virtual address - * - * You can use this helper if bitflags are manipulated atomically rather than - * non-atomically under a lock. - */ -static inline void clear_and_wake_up_bit(int bit, void *word) -{ - clear_bit_unlock(bit, word); - /* See wake_up_bit() for which memory barrier you need to use. */ - smp_mb__after_atomic(); - wake_up_bit(word, bit); -} -#endif /* ! HAVE_CLEAR_AND_WAKE_UP_BIT */ - -#ifndef HAVE_WAIT_VAR_EVENT -extern void __init wait_bit_init(void); -extern void init_wait_var_entry(struct wait_bit_queue_entry *wbq_entry, - void *var, int flags); -extern void wake_up_var(void *var); -extern wait_queue_head_t *__var_waitqueue(void *p); - -#define ___wait_var_event(var, condition, state, exclusive, ret, cmd) \ -({ \ - __label__ __out; \ - wait_queue_head_t *__wq_head = __var_waitqueue(var); \ - struct wait_bit_queue_entry __wbq_entry; \ - long __ret = ret; /* explicit shadow */ \ - \ - init_wait_var_entry(&__wbq_entry, var, \ - exclusive ? WQ_FLAG_EXCLUSIVE : 0); \ - for (;;) { \ - long __int = prepare_to_wait_event(__wq_head, \ - &__wbq_entry.wq_entry, \ - state); \ - if (condition) \ - break; \ - \ - if (___wait_is_interruptible(state) && __int) { \ - __ret = __int; \ - goto __out; \ - } \ - \ - cmd; \ - } \ - finish_wait(__wq_head, &__wbq_entry.wq_entry); \ -__out: __ret; \ -}) - -#define __wait_var_event(var, condition) \ - ___wait_var_event(var, condition, TASK_UNINTERRUPTIBLE, 0, 0, \ - schedule()) - -#define wait_var_event(var, condition) \ -do { \ - might_sleep(); \ - if (condition) \ - break; \ - __wait_var_event(var, condition); \ -} while (0) - -#define __wait_var_event_killable(var, condition) \ - ___wait_var_event(var, condition, TASK_KILLABLE, 0, 0, \ - schedule()) - -#define wait_var_event_killable(var, condition) \ -({ \ - int __ret = 0; \ - might_sleep(); \ - if (!(condition)) \ - __ret = __wait_var_event_killable(var, condition); \ - __ret; \ -}) - -#define __wait_var_event_timeout(var, condition, timeout) \ - ___wait_var_event(var, ___wait_cond_timeout1(condition), \ - TASK_UNINTERRUPTIBLE, 0, timeout, \ - __ret = schedule_timeout(__ret)) - -#define wait_var_event_timeout(var, condition, timeout) \ -({ \ - long __ret = timeout; \ - might_sleep(); \ - if (!___wait_cond_timeout1(condition)) \ - __ret = __wait_var_event_timeout(var, condition, timeout); \ - __ret; \ -}) -#else /* !HAVE_WAIT_VAR_EVENT */ -/* linux-3.10.0-1062.el7 defines wait_var_event_timeout() using - * __wait_cond_timeout(), but doesn't define __wait_cond_timeout !!! - */ -# ifndef __wait_cond_timeout -# define ___wait_cond_timeout(condition) \ -({ \ - bool __cond = (condition); \ - if (__cond && !__ret) \ - __ret = 1; \ - __cond || !__ret; \ -}) -# endif /* __wait_cond_timeout */ - -#endif /* ! HAVE_WAIT_VAR_EVENT */ - /* * prepare_to_wait_event() does not support an exclusive * lifo wait. @@ -584,4 +459,4 @@ int woken_wake_function(wait_queue_entry_t *wait, unsigned int mode, int sync, void *key); #endif /* HAVE_WAIT_WOKEN */ -#endif /* __LICBFS_LINUX_WAIT_BIT_H */ +#endif /* __LICBFS_LINUX_WAIT_H */ diff --git a/include/lustre_compat/linux/wait_bit.h b/include/lustre_compat/linux/wait_bit.h new file mode 100644 index 0000000..2b3726f --- /dev/null +++ b/include/lustre_compat/linux/wait_bit.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __LIBCFS_LINUX_WAIT_BIT_H +#define __LIBCFS_LINUX_WAIT_BIT_H + +/* Make sure we can see if we have TASK_NOLOAD */ +#include +#ifdef HAVE_WAIT_BIT_HEADER_H +#include +#endif + +#include + +#ifndef HAVE_WAIT_BIT_HEADER_H +struct wait_bit_queue_entry { + struct wait_bit_key key; + wait_queue_entry_t wq_entry; +}; + +#define ___wait_is_interruptible(state) \ + (!__builtin_constant_p(state) || \ + state == TASK_INTERRUPTIBLE || state == TASK_KILLABLE) \ + +#endif /* ! HAVE_WAIT_BIT_HEADER_H */ + +#ifndef HAVE_CLEAR_AND_WAKE_UP_BIT +/** + * clear_and_wake_up_bit - clear a bit and wake up anyone waiting on that bit + * + * @bit: the bit of the word being waited on + * @word: the word being waited on, a kernel virtual address + * + * You can use this helper if bitflags are manipulated atomically rather than + * non-atomically under a lock. + */ +static inline void clear_and_wake_up_bit(int bit, void *word) +{ + clear_bit_unlock(bit, word); + /* See wake_up_bit() for which memory barrier you need to use. */ + smp_mb__after_atomic(); + wake_up_bit(word, bit); +} +#endif /* ! HAVE_CLEAR_AND_WAKE_UP_BIT */ + +#ifndef HAVE_WAIT_VAR_EVENT +extern void __init wait_bit_init(void); +extern void init_wait_var_entry(struct wait_bit_queue_entry *wbq_entry, + void *var, int flags); +extern void wake_up_var(void *var); +extern wait_queue_head_t *__var_waitqueue(void *p); + +#define ___wait_var_event(var, condition, state, exclusive, ret, cmd) \ +({ \ + __label__ __out; \ + wait_queue_head_t *__wq_head = __var_waitqueue(var); \ + struct wait_bit_queue_entry __wbq_entry; \ + long __ret = ret; /* explicit shadow */ \ + \ + init_wait_var_entry(&__wbq_entry, var, \ + exclusive ? WQ_FLAG_EXCLUSIVE : 0); \ + for (;;) { \ + long __int = prepare_to_wait_event(__wq_head, \ + &__wbq_entry.wq_entry, \ + state); \ + if (condition) \ + break; \ + \ + if (___wait_is_interruptible(state) && __int) { \ + __ret = __int; \ + goto __out; \ + } \ + \ + cmd; \ + } \ + finish_wait(__wq_head, &__wbq_entry.wq_entry); \ +__out: __ret; \ +}) + +#define __wait_var_event(var, condition) \ + ___wait_var_event(var, condition, TASK_UNINTERRUPTIBLE, 0, 0, \ + schedule()) + +#define wait_var_event(var, condition) \ +do { \ + might_sleep(); \ + if (condition) \ + break; \ + __wait_var_event(var, condition); \ +} while (0) + +#define __wait_var_event_killable(var, condition) \ + ___wait_var_event(var, condition, TASK_KILLABLE, 0, 0, \ + schedule()) + +#define wait_var_event_killable(var, condition) \ +({ \ + int __ret = 0; \ + might_sleep(); \ + if (!(condition)) \ + __ret = __wait_var_event_killable(var, condition); \ + __ret; \ +}) + +#define __wait_var_event_timeout(var, condition, timeout) \ + ___wait_var_event(var, ___wait_cond_timeout1(condition), \ + TASK_UNINTERRUPTIBLE, 0, timeout, \ + __ret = schedule_timeout(__ret)) + +#define wait_var_event_timeout(var, condition, timeout) \ +({ \ + long __ret = timeout; \ + might_sleep(); \ + if (!___wait_cond_timeout1(condition)) \ + __ret = __wait_var_event_timeout(var, condition, timeout); \ + __ret; \ +}) +#else /* !HAVE_WAIT_VAR_EVENT */ +/* linux-3.10.0-1062.el7 defines wait_var_event_timeout() using + * __wait_cond_timeout(), but doesn't define __wait_cond_timeout !!! + */ +# ifndef __wait_cond_timeout +# define ___wait_cond_timeout(condition) \ +({ \ + bool __cond = (condition); \ + if (__cond && !__ret) \ + __ret = 1; \ + __cond || !__ret; \ +}) +# endif /* __wait_cond_timeout */ + +#endif /* ! HAVE_WAIT_VAR_EVENT */ +#endif /* __LICBFS_LINUX_WAIT_BIT_H */ diff --git a/libcfs/include/libcfs/libcfs.h b/libcfs/include/libcfs/libcfs.h index c475c06..ed5a0b7 100644 --- a/libcfs/include/libcfs/libcfs.h +++ b/libcfs/include/libcfs/libcfs.h @@ -25,7 +25,8 @@ #include #include #include -#include +#include +#include #include #include diff --git a/libcfs/include/libcfs/linux/Makefile.am b/libcfs/include/libcfs/linux/Makefile.am index 5d012f2..ce57e1c 100644 --- a/libcfs/include/libcfs/linux/Makefile.am +++ b/libcfs/include/libcfs/linux/Makefile.am @@ -5,5 +5,5 @@ # EXTRA_DIST = linux-misc.h linux-fs.h linux-mem.h linux-time.h linux-cpu.h \ - linux-wait.h linux-net.h \ + linux-net.h \ refcount.h processor.h linux-fortify-string.h diff --git a/libcfs/libcfs/Makefile.in b/libcfs/libcfs/Makefile.in index 3692050..7cd5df1 100644 --- a/libcfs/libcfs/Makefile.in +++ b/libcfs/libcfs/Makefile.in @@ -11,6 +11,10 @@ libcfs_dir := $(dir $(lastword $(MAKEFILE_LIST))) libcfs-compat-objs := +COMPAT_SCHED := @top_srcdir@/lustre_compat/kernel/sched/ +include $(libcfs_dir)/../../lustre_compat/kernel/sched/Makefile +libcfs-compat-objs += $(patsubst %,$(COMPAT_SCHED)%,$(sched_objs)) + COMPAT_MM := @top_srcdir@/lustre_compat/mm/ include $(libcfs_dir)/../../lustre_compat/mm/Makefile libcfs-compat-objs += $(patsubst %,$(COMPAT_MM)%,$(mm_objs)) @@ -19,8 +23,8 @@ COMPAT_LIB := @top_srcdir@/lustre_compat/lib/ include $(libcfs_dir)/../../lustre_compat/lib/Makefile libcfs-compat-objs += $(patsubst %,$(COMPAT_LIB)%,$(lib_objs)) + libcfs-linux-objs := linux-prim.o -libcfs-linux-objs += linux-wait.o EXTRA_DIST = $(libcfs-compat-objs:.o=.c) diff --git a/libcfs/libcfs/linux/Makefile.am b/libcfs/libcfs/linux/Makefile.am index a605d49..baff597 100644 --- a/libcfs/libcfs/linux/Makefile.am +++ b/libcfs/libcfs/linux/Makefile.am @@ -4,5 +4,4 @@ # This file is part of Lustre, http://www.lustre.org/ # -EXTRA_DIST = linux-prim.c \ - linux-wait.c +EXTRA_DIST = linux-prim.c diff --git a/libcfs/libcfs/linux/linux-prim.c b/libcfs/libcfs/linux/linux-prim.c index bbb24c7..6338214 100644 --- a/libcfs/libcfs/linux/linux-prim.c +++ b/libcfs/libcfs/linux/linux-prim.c @@ -30,10 +30,11 @@ #include #include -#include #include #include #include +#include +#include #include #ifndef HAVE_KTIME_GET_TS64 diff --git a/lustre_compat/kernel/sched/Makefile b/lustre_compat/kernel/sched/Makefile new file mode 100644 index 0000000..0400562 --- /dev/null +++ b/lustre_compat/kernel/sched/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 + +# +# This file is part of Lustre, http://www.lustre.org/ +# + +sched_objs := wait.o wait_bit.o diff --git a/libcfs/libcfs/linux/linux-wait.c b/lustre_compat/kernel/sched/wait.c similarity index 67% rename from libcfs/libcfs/linux/linux-wait.c rename to lustre_compat/kernel/sched/wait.c index c5763e0..1488493 100644 --- a/libcfs/libcfs/linux/linux-wait.c +++ b/lustre_compat/kernel/sched/wait.c @@ -1,17 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only -/* - * The implementation of the wait_bit*() and related waiting APIs: - */ - #include #ifdef HAVE_SCHED_HEADERS #include #endif -#include +#include +#include #ifndef HAVE_PREPARE_TO_WAIT_EVENT - long prepare_to_wait_event(wait_queue_head_t *wq_head, wait_queue_entry_t *wq_entry, int state) { @@ -50,70 +46,6 @@ long prepare_to_wait_event(wait_queue_head_t *wq_head, EXPORT_SYMBOL(prepare_to_wait_event); #endif /* !HAVE_PREPARE_TO_WAIT_EVENT */ -#ifndef HAVE_WAIT_VAR_EVENT - -#define WAIT_TABLE_BITS 8 -#define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS) - -static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned; - -wait_queue_head_t *__var_waitqueue(void *p) -{ - return bit_wait_table + hash_ptr(p, WAIT_TABLE_BITS); -} -EXPORT_SYMBOL(__var_waitqueue); - -static int -var_wake_function(wait_queue_entry_t *wq_entry, unsigned int mode, - int sync, void *arg) -{ - struct wait_bit_key *key = arg; - struct wait_bit_queue_entry *wbq_entry = - container_of(wq_entry, struct wait_bit_queue_entry, wq_entry); - - if (wbq_entry->key.flags != key->flags || - wbq_entry->key.bit_nr != key->bit_nr) - return 0; - - return autoremove_wake_function(wq_entry, mode, sync, key); -} - -void init_wait_var_entry(struct wait_bit_queue_entry *wbq_entry, void *var, - int flags) -{ - *wbq_entry = (struct wait_bit_queue_entry){ - .key = { - .flags = (var), - .bit_nr = -1, - }, - .wq_entry = { - .private = current, - .func = var_wake_function, -#ifdef HAVE_WAIT_QUEUE_ENTRY_LIST - .entry = LIST_HEAD_INIT(wbq_entry->wq_entry.entry), -#else - .task_list = LIST_HEAD_INIT(wbq_entry->wq_entry.task_list), -#endif - }, - }; -} -EXPORT_SYMBOL(init_wait_var_entry); - -void wake_up_var(void *var) -{ - __wake_up_bit(__var_waitqueue(var), var, -1); -} -EXPORT_SYMBOL(wake_up_var); - -void __init wait_bit_init(void) -{ - int i; - - for (i = 0; i < WAIT_TABLE_SIZE; i++) - init_waitqueue_head(bit_wait_table + i); -} -#endif /* ! HAVE_WAIT_VAR_EVENT */ - #ifndef HAVE_WAIT_WOKEN /* * DEFINE_WAIT_FUNC(wait, woken_wake_func); diff --git a/lustre_compat/kernel/sched/wait_bit.c b/lustre_compat/kernel/sched/wait_bit.c new file mode 100644 index 0000000..bf499d1 --- /dev/null +++ b/lustre_compat/kernel/sched/wait_bit.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* + * The implementation of the wait_bit*() and related waiting APIs: + */ + +#include +#ifdef HAVE_SCHED_HEADERS +#include +#endif +#include + +#ifndef HAVE_WAIT_VAR_EVENT + +#define WAIT_TABLE_BITS 8 +#define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS) + +static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned; + +wait_queue_head_t *__var_waitqueue(void *p) +{ + return bit_wait_table + hash_ptr(p, WAIT_TABLE_BITS); +} +EXPORT_SYMBOL(__var_waitqueue); + +static int +var_wake_function(wait_queue_entry_t *wq_entry, unsigned int mode, + int sync, void *arg) +{ + struct wait_bit_key *key = arg; + struct wait_bit_queue_entry *wbq_entry = + container_of(wq_entry, struct wait_bit_queue_entry, wq_entry); + + if (wbq_entry->key.flags != key->flags || + wbq_entry->key.bit_nr != key->bit_nr) + return 0; + + return autoremove_wake_function(wq_entry, mode, sync, key); +} + +void init_wait_var_entry(struct wait_bit_queue_entry *wbq_entry, void *var, + int flags) +{ + *wbq_entry = (struct wait_bit_queue_entry){ + .key = { + .flags = (var), + .bit_nr = -1, + }, + .wq_entry = { + .private = current, + .func = var_wake_function, +#ifdef HAVE_WAIT_QUEUE_ENTRY_LIST + .entry = LIST_HEAD_INIT(wbq_entry->wq_entry.entry), +#else + .task_list = LIST_HEAD_INIT(wbq_entry->wq_entry.task_list), +#endif + }, + }; +} +EXPORT_SYMBOL(init_wait_var_entry); + +void wake_up_var(void *var) +{ + __wake_up_bit(__var_waitqueue(var), var, -1); +} +EXPORT_SYMBOL(wake_up_var); + +void __init wait_bit_init(void) +{ + int i; + + for (i = 0; i < WAIT_TABLE_SIZE; i++) + init_waitqueue_head(bit_wait_table + i); +} +#endif /* ! HAVE_WAIT_VAR_EVENT */ -- 1.8.3.1