libcfs/libcfs/linux/linux-wait.c

   1 /*
   2  * The implementation of the wait_bit*() and related waiting APIs:
   3  */
   4 #include <libcfs/linux/linux-hash.h>
   5 #include <linux/sched.h>
   6 #ifdef HAVE_SCHED_HEADERS
   7 #include <linux/sched/signal.h>
   8 #endif
   9 #include <libcfs/linux/linux-wait.h>
  10
  11 #ifndef HAVE_PREPARE_TO_WAIT_EVENT
  12
  13 long prepare_to_wait_event(wait_queue_head_t *wq_head,
  14                            wait_queue_entry_t *wq_entry, int state)
  15 {
  16         unsigned long flags;
  17         long ret = 0;
  18
  19         spin_lock_irqsave(&wq_head->lock, flags);
  20         if (unlikely(signal_pending_state(state, current))) {
  21                 /*
  22                  * Exclusive waiter must not fail if it was selected by wakeup,
  23                  * it should "consume" the condition we were waiting for.
  24                  *
  25                  * The caller will recheck the condition and return success if
  26                  * we were already woken up, we can not miss the event because
  27                  * wakeup locks/unlocks the same wq_head->lock.
  28                  *
  29                  * But we need to ensure that set-condition + wakeup after that
  30                  * can't see us, it should wake up another exclusive waiter if
  31                  * we fail.
  32                  */
  33                 list_del_init(&wq_entry->task_list);
  34                 ret = -ERESTARTSYS;
  35         } else {
  36                 if (list_empty(&wq_entry->task_list)) {
  37                         if (wq_entry->flags & WQ_FLAG_EXCLUSIVE)
  38                                 __add_wait_queue_entry_tail(wq_head, wq_entry);
  39                         else
  40                                 __add_wait_queue(wq_head, wq_entry);
  41                 }
  42                 set_current_state(state);
  43         }
  44         spin_unlock_irqrestore(&wq_head->lock, flags);
  45
  46         return ret;
  47 }
  48 EXPORT_SYMBOL(prepare_to_wait_event);
  49 #endif /* !HAVE_PREPARE_TO_WAIT_EVENT */
  50
  51 #ifndef HAVE_WAIT_VAR_EVENT
  52
  53 #define WAIT_TABLE_BITS 8
  54 #define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS)
  55
  56 static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned;
  57
  58 wait_queue_head_t *__var_waitqueue(void *p)
  59 {
  60         return bit_wait_table + hash_ptr(p, WAIT_TABLE_BITS);
  61 }
  62 EXPORT_SYMBOL(__var_waitqueue);
  63
  64 static int
  65 var_wake_function(wait_queue_entry_t *wq_entry, unsigned int mode,
  66                   int sync, void *arg)
  67 {
  68         struct wait_bit_key *key = arg;
  69         struct wait_bit_queue_entry *wbq_entry =
  70                 container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
  71
  72         if (wbq_entry->key.flags != key->flags ||
  73             wbq_entry->key.bit_nr != key->bit_nr)
  74                 return 0;
  75
  76         return autoremove_wake_function(wq_entry, mode, sync, key);
  77 }
  78
  79 void init_wait_var_entry(struct wait_bit_queue_entry *wbq_entry, void *var,
  80                          int flags)
  81 {
  82         *wbq_entry = (struct wait_bit_queue_entry){
  83                 .key = {
  84                         .flags  = (var),
  85                         .bit_nr = -1,
  86                 },
  87                 .wq_entry = {
  88                         .private = current,
  89                         .func = var_wake_function,
  90 #ifdef HAVE_WAIT_QUEUE_ENTRY_LIST
  91                         .entry = LIST_HEAD_INIT(wbq_entry->wq_entry.entry),
  92 #else
  93                         .task_list = LIST_HEAD_INIT(wbq_entry->wq_entry.task_list),
  94 #endif
  95                 },
  96         };
  97 }
  98 EXPORT_SYMBOL(init_wait_var_entry);
  99
 100 void wake_up_var(void *var)
 101 {
 102         __wake_up_bit(__var_waitqueue(var), var, -1);
 103 }
 104 EXPORT_SYMBOL(wake_up_var);
 105
 106 void __init wait_bit_init(void)
 107 {
 108         int i;
 109
 110         for (i = 0; i < WAIT_TABLE_SIZE; i++)
 111                 init_waitqueue_head(bit_wait_table + i);
 112 }
 113 #endif /* ! HAVE_WAIT_VAR_EVENT */
 114
 115 #ifndef HAVE_WAIT_WOKEN
 116 /*
 117  * DEFINE_WAIT_FUNC(wait, woken_wake_func);
 118  *
 119  * add_wait_queue(&wq_head, &wait);
 120  * for (;;) {
 121  *     if (condition)
 122  *         break;
 123  *
 124  *     // in wait_woken()                       // in woken_wake_function()
 125  *
 126  *     p->state = mode;                         wq_entry->flags |= WQ_FLAG_WOKEN;
 127  *     smp_mb(); // A                           try_to_wake_up():
 128  *     if (!(wq_entry->flags & WQ_FLAG_WOKEN))     <full barrier>
 129  *         schedule()                              if (p->state & mode)
 130  *     p->state = TASK_RUNNING;                       p->state = TASK_RUNNING;
 131  *     wq_entry->flags &= ~WQ_FLAG_WOKEN;       ~~~~~~~~~~~~~~~~~~
 132  *     smp_mb(); // B                           condition = true;
 133  * }                                            smp_mb(); // C
 134  * remove_wait_queue(&wq_head, &wait);          wq_entry->flags |= WQ_FLAG_WOKEN;
 135  */
 136 long wait_woken(struct wait_queue_entry *wq_entry, unsigned int mode,
 137                 long timeout)
 138 {
 139         /*
 140          * The below executes an smp_mb(), which matches with the full barrier
 141          * executed by the try_to_wake_up() in woken_wake_function() such that
 142          * either we see the store to wq_entry->flags in woken_wake_function()
 143          * or woken_wake_function() sees our store to current->state.
 144          */
 145         set_current_state(mode); /* A */
 146         if (!(wq_entry->flags & WQ_FLAG_WOKEN))
 147                 timeout = schedule_timeout(timeout);
 148         __set_current_state(TASK_RUNNING);
 149
 150         /*
 151          * The below executes an smp_mb(), which matches with the smp_mb() (C)
 152          * in woken_wake_function() such that either we see the wait condition
 153          * being true or the store to wq_entry->flags in woken_wake_function()
 154          * follows ours in the coherence order.
 155          */
 156         smp_store_mb(wq_entry->flags, wq_entry->flags & ~WQ_FLAG_WOKEN); /* B */
 157
 158         return timeout;
 159 }
 160 EXPORT_SYMBOL(wait_woken);
 161
 162 int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode,
 163                         int sync, void *key)
 164 {
 165         /* Pairs with the smp_store_mb() in wait_woken(). */
 166         smp_mb(); /* C */
 167         wq_entry->flags |= WQ_FLAG_WOKEN;
 168
 169         return default_wake_function(wq_entry, mode, sync, key);
 170 }
 171 EXPORT_SYMBOL(woken_wake_function);
 172 #endif /* HAVE_WAIT_WOKEN */