2 * The implementation of the wait_bit*() and related waiting APIs:
4 #include <libcfs/linux/linux-hash.h>
5 #include <linux/sched.h>
6 #ifdef HAVE_SCHED_HEADERS
7 #include <linux/sched/signal.h>
9 #include <libcfs/linux/linux-wait.h>
11 #ifndef HAVE_PREPARE_TO_WAIT_EVENT
13 long prepare_to_wait_event(wait_queue_head_t *wq_head,
14 wait_queue_entry_t *wq_entry, int state)
19 spin_lock_irqsave(&wq_head->lock, flags);
20 if (unlikely(signal_pending_state(state, current))) {
22 * Exclusive waiter must not fail if it was selected by wakeup,
23 * it should "consume" the condition we were waiting for.
25 * The caller will recheck the condition and return success if
26 * we were already woken up, we can not miss the event because
27 * wakeup locks/unlocks the same wq_head->lock.
29 * But we need to ensure that set-condition + wakeup after that
30 * can't see us, it should wake up another exclusive waiter if
33 list_del_init(&wq_entry->task_list);
36 if (list_empty(&wq_entry->task_list)) {
37 if (wq_entry->flags & WQ_FLAG_EXCLUSIVE)
38 __add_wait_queue_entry_tail(wq_head, wq_entry);
40 __add_wait_queue(wq_head, wq_entry);
42 set_current_state(state);
44 spin_unlock_irqrestore(&wq_head->lock, flags);
48 EXPORT_SYMBOL(prepare_to_wait_event);
49 #endif /* !HAVE_PREPARE_TO_WAIT_EVENT */
51 #ifndef HAVE_WAIT_VAR_EVENT
53 #define WAIT_TABLE_BITS 8
54 #define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS)
56 static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned;
58 wait_queue_head_t *__var_waitqueue(void *p)
60 return bit_wait_table + hash_ptr(p, WAIT_TABLE_BITS);
62 EXPORT_SYMBOL(__var_waitqueue);
65 var_wake_function(wait_queue_entry_t *wq_entry, unsigned int mode,
68 struct wait_bit_key *key = arg;
69 struct wait_bit_queue_entry *wbq_entry =
70 container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
72 if (wbq_entry->key.flags != key->flags ||
73 wbq_entry->key.bit_nr != key->bit_nr)
76 return autoremove_wake_function(wq_entry, mode, sync, key);
79 void init_wait_var_entry(struct wait_bit_queue_entry *wbq_entry, void *var,
82 *wbq_entry = (struct wait_bit_queue_entry){
89 .func = var_wake_function,
90 #ifdef HAVE_WAIT_QUEUE_ENTRY_LIST
91 .entry = LIST_HEAD_INIT(wbq_entry->wq_entry.entry),
93 .task_list = LIST_HEAD_INIT(wbq_entry->wq_entry.task_list),
98 EXPORT_SYMBOL(init_wait_var_entry);
100 void wake_up_var(void *var)
102 __wake_up_bit(__var_waitqueue(var), var, -1);
104 EXPORT_SYMBOL(wake_up_var);
106 void __init wait_bit_init(void)
110 for (i = 0; i < WAIT_TABLE_SIZE; i++)
111 init_waitqueue_head(bit_wait_table + i);
113 #endif /* ! HAVE_WAIT_VAR_EVENT */
115 #ifndef HAVE_WAIT_WOKEN
117 * DEFINE_WAIT_FUNC(wait, woken_wake_func);
119 * add_wait_queue(&wq_head, &wait);
124 * // in wait_woken() // in woken_wake_function()
126 * p->state = mode; wq_entry->flags |= WQ_FLAG_WOKEN;
127 * smp_mb(); // A try_to_wake_up():
128 * if (!(wq_entry->flags & WQ_FLAG_WOKEN)) <full barrier>
129 * schedule() if (p->state & mode)
130 * p->state = TASK_RUNNING; p->state = TASK_RUNNING;
131 * wq_entry->flags &= ~WQ_FLAG_WOKEN; ~~~~~~~~~~~~~~~~~~
132 * smp_mb(); // B condition = true;
134 * remove_wait_queue(&wq_head, &wait); wq_entry->flags |= WQ_FLAG_WOKEN;
136 long wait_woken(struct wait_queue_entry *wq_entry, unsigned int mode,
140 * The below executes an smp_mb(), which matches with the full barrier
141 * executed by the try_to_wake_up() in woken_wake_function() such that
142 * either we see the store to wq_entry->flags in woken_wake_function()
143 * or woken_wake_function() sees our store to current->state.
145 set_current_state(mode); /* A */
146 if (!(wq_entry->flags & WQ_FLAG_WOKEN))
147 timeout = schedule_timeout(timeout);
148 __set_current_state(TASK_RUNNING);
151 * The below executes an smp_mb(), which matches with the smp_mb() (C)
152 * in woken_wake_function() such that either we see the wait condition
153 * being true or the store to wq_entry->flags in woken_wake_function()
154 * follows ours in the coherence order.
156 smp_store_mb(wq_entry->flags, wq_entry->flags & ~WQ_FLAG_WOKEN); /* B */
160 EXPORT_SYMBOL(wait_woken);
162 int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode,
165 /* Pairs with the smp_store_mb() in wait_woken(). */
167 wq_entry->flags |= WQ_FLAG_WOKEN;
169 return default_wake_function(wq_entry, mode, sync, key);
171 EXPORT_SYMBOL(woken_wake_function);
172 #endif /* HAVE_WAIT_WOKEN */