include/linux/wait.h

   1 #ifndef _LINUX_WAIT_H
   2 #define _LINUX_WAIT_H
   3 /*
   4  * Linux wait queue related types and methods
   5  */
   6 #include <linux/list.h>
   7 #include <linux/stddef.h>
   8 #include <linux/spinlock.h>
   9 #include <asm/current.h>
  10 #include <uapi/linux/wait.h>
  11
  12 typedef struct __wait_queue wait_queue_t;
  13 typedef int (*wait_queue_func_t)(wait_queue_t *wait, unsigned mode, int flags, void *key);
  14 int default_wake_function(wait_queue_t *wait, unsigned mode, int flags, void *key);
  15
  16 /* __wait_queue::flags */
  17 #define WQ_FLAG_EXCLUSIVE       0x01
  18 #define WQ_FLAG_WOKEN           0x02
  19
  20 struct __wait_queue {
  21         unsigned int            flags;
  22         void                    *private;
  23         wait_queue_func_t       func;
  24         struct list_head        task_list;
  25 };
  26
  27 struct wait_bit_key {
  28         void                    *flags;
  29         int                     bit_nr;
  30 #define WAIT_ATOMIC_T_BIT_NR    -1
  31         unsigned long           timeout;
  32 };
  33
  34 struct wait_bit_queue {
  35         struct wait_bit_key     key;
  36         wait_queue_t            wait;
  37 };
  38
  39 struct __wait_queue_head {
  40         spinlock_t              lock;
  41         struct list_head        task_list;
  42 };
  43 typedef struct __wait_queue_head wait_queue_head_t;
  44
  45 struct task_struct;
  46
  47 /*
  48  * Macros for declaration and initialisaton of the datatypes
  49  */
  50
  51 #define __WAITQUEUE_INITIALIZER(name, tsk) {                            \
  52         .private        = tsk,                                          \
  53         .func           = default_wake_function,                        \
  54         .task_list      = { NULL, NULL } }
  55
  56 #define DECLARE_WAITQUEUE(name, tsk)                                    \
  57         wait_queue_t name = __WAITQUEUE_INITIALIZER(name, tsk)
  58
  59 #define __WAIT_QUEUE_HEAD_INITIALIZER(name) {                           \
  60         .lock           = __SPIN_LOCK_UNLOCKED(name.lock),              \
  61         .task_list      = { &(name).task_list, &(name).task_list } }
  62
  63 #define DECLARE_WAIT_QUEUE_HEAD(name) \
  64         wait_queue_head_t name = __WAIT_QUEUE_HEAD_INITIALIZER(name)
  65
  66 #define __WAIT_BIT_KEY_INITIALIZER(word, bit)                           \
  67         { .flags = word, .bit_nr = bit, }
  68
  69 #define __WAIT_ATOMIC_T_KEY_INITIALIZER(p)                              \
  70         { .flags = p, .bit_nr = WAIT_ATOMIC_T_BIT_NR, }
  71
  72 extern void __init_waitqueue_head(wait_queue_head_t *q, const char *name, struct lock_class_key *);
  73
  74 #define init_waitqueue_head(q)                          \
  75         do {                                            \
  76                 static struct lock_class_key __key;     \
  77                                                         \
  78                 __init_waitqueue_head((q), #q, &__key); \
  79         } while (0)
  80
  81 #ifdef CONFIG_LOCKDEP
  82 # define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
  83         ({ init_waitqueue_head(&name); name; })
  84 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
  85         wait_queue_head_t name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
  86 #else
  87 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
  88 #endif
  89
  90 static inline void init_waitqueue_entry(wait_queue_t *q, struct task_struct *p)
  91 {
  92         q->flags        = 0;
  93         q->private      = p;
  94         q->func         = default_wake_function;
  95 }
  96
  97 static inline void
  98 init_waitqueue_func_entry(wait_queue_t *q, wait_queue_func_t func)
  99 {
 100         q->flags        = 0;
 101         q->private      = NULL;
 102         q->func         = func;
 103 }
 104
 105 static inline int waitqueue_active(wait_queue_head_t *q)
 106 {
 107         return !list_empty(&q->task_list);
 108 }
 109
 110 extern void add_wait_queue(wait_queue_head_t *q, wait_queue_t *wait);
 111 extern void add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait);
 112 extern void remove_wait_queue(wait_queue_head_t *q, wait_queue_t *wait);
 113
 114 static inline void __add_wait_queue(wait_queue_head_t *head, wait_queue_t *new)
 115 {
 116         list_add(&new->task_list, &head->task_list);
 117 }
 118
 119 /*
 120  * Used for wake-one threads:
 121  */
 122 static inline void
 123 __add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait)
 124 {
 125         wait->flags |= WQ_FLAG_EXCLUSIVE;
 126         __add_wait_queue(q, wait);
 127 }
 128
 129 static inline void __add_wait_queue_tail(wait_queue_head_t *head,
 130                                          wait_queue_t *new)
 131 {
 132         list_add_tail(&new->task_list, &head->task_list);
 133 }
 134
 135 static inline void
 136 __add_wait_queue_tail_exclusive(wait_queue_head_t *q, wait_queue_t *wait)
 137 {
 138         wait->flags |= WQ_FLAG_EXCLUSIVE;
 139         __add_wait_queue_tail(q, wait);
 140 }
 141
 142 static inline void
 143 __remove_wait_queue(wait_queue_head_t *head, wait_queue_t *old)
 144 {
 145         list_del(&old->task_list);
 146 }
 147
 148 typedef int wait_bit_action_f(struct wait_bit_key *);
 149 void __wake_up(wait_queue_head_t *q, unsigned int mode, int nr, void *key);
 150 void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key);
 151 void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode, int nr, void *key);
 152 void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr);
 153 void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr);
 154 void __wake_up_bit(wait_queue_head_t *, void *, int);
 155 int __wait_on_bit(wait_queue_head_t *, struct wait_bit_queue *, wait_bit_action_f *, unsigned);
 156 int __wait_on_bit_lock(wait_queue_head_t *, struct wait_bit_queue *, wait_bit_action_f *, unsigned);
 157 void wake_up_bit(void *, int);
 158 void wake_up_atomic_t(atomic_t *);
 159 int out_of_line_wait_on_bit(void *, int, wait_bit_action_f *, unsigned);
 160 int out_of_line_wait_on_bit_timeout(void *, int, wait_bit_action_f *, unsigned, unsigned long);
 161 int out_of_line_wait_on_bit_lock(void *, int, wait_bit_action_f *, unsigned);
 162 int out_of_line_wait_on_atomic_t(atomic_t *, int (*)(atomic_t *), unsigned);
 163 wait_queue_head_t *bit_waitqueue(void *, int);
 164
 165 #define wake_up(x)                      __wake_up(x, TASK_NORMAL, 1, NULL)
 166 #define wake_up_nr(x, nr)               __wake_up(x, TASK_NORMAL, nr, NULL)
 167 #define wake_up_all(x)                  __wake_up(x, TASK_NORMAL, 0, NULL)
 168 #define wake_up_locked(x)               __wake_up_locked((x), TASK_NORMAL, 1)
 169 #define wake_up_all_locked(x)           __wake_up_locked((x), TASK_NORMAL, 0)
 170
 171 #define wake_up_interruptible(x)        __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
 172 #define wake_up_interruptible_nr(x, nr) __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
 173 #define wake_up_interruptible_all(x)    __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
 174 #define wake_up_interruptible_sync(x)   __wake_up_sync((x), TASK_INTERRUPTIBLE, 1)
 175
 176 /*
 177  * Wakeup macros to be used to report events to the targets.
 178  */
 179 #define wake_up_poll(x, m)                                              \
 180         __wake_up(x, TASK_NORMAL, 1, (void *) (m))
 181 #define wake_up_locked_poll(x, m)                                       \
 182         __wake_up_locked_key((x), TASK_NORMAL, (void *) (m))
 183 #define wake_up_interruptible_poll(x, m)                                \
 184         __wake_up(x, TASK_INTERRUPTIBLE, 1, (void *) (m))
 185 #define wake_up_interruptible_sync_poll(x, m)                           \
 186         __wake_up_sync_key((x), TASK_INTERRUPTIBLE, 1, (void *) (m))
 187
 188 #define ___wait_cond_timeout(condition)                                 \
 189 ({                                                                      \
 190         bool __cond = (condition);                                      \
 191         if (__cond && !__ret)                                           \
 192                 __ret = 1;                                              \
 193         __cond || !__ret;                                               \
 194 })
 195
 196 #define ___wait_is_interruptible(state)                                 \
 197         (!__builtin_constant_p(state) ||                                \
 198                 state == TASK_INTERRUPTIBLE || state == TASK_KILLABLE)  \
 199
 200 /*
 201  * The below macro ___wait_event() has an explicit shadow of the __ret
 202  * variable when used from the wait_event_*() macros.
 203  *
 204  * This is so that both can use the ___wait_cond_timeout() construct
 205  * to wrap the condition.
 206  *
 207  * The type inconsistency of the wait_event_*() __ret variable is also
 208  * on purpose; we use long where we can return timeout values and int
 209  * otherwise.
 210  */
 211
 212 #define ___wait_event(wq, condition, state, exclusive, ret, cmd)        \
 213 ({                                                                      \
 214         __label__ __out;                                                \
 215         wait_queue_t __wait;                                            \
 216         long __ret = ret;       /* explicit shadow */                   \
 217                                                                         \
 218         INIT_LIST_HEAD(&__wait.task_list);                              \
 219         if (exclusive)                                                  \
 220                 __wait.flags = WQ_FLAG_EXCLUSIVE;                       \
 221         else                                                            \
 222                 __wait.flags = 0;                                       \
 223                                                                         \
 224         for (;;) {                                                      \
 225                 long __int = prepare_to_wait_event(&wq, &__wait, state);\
 226                                                                         \
 227                 if (condition)                                          \
 228                         break;                                          \
 229                                                                         \
 230                 if (___wait_is_interruptible(state) && __int) {         \
 231                         __ret = __int;                                  \
 232                         if (exclusive) {                                \
 233                                 abort_exclusive_wait(&wq, &__wait,      \
 234                                                      state, NULL);      \
 235                                 goto __out;                             \
 236                         }                                               \
 237                         break;                                          \
 238                 }                                                       \
 239                                                                         \
 240                 cmd;                                                    \
 241         }                                                               \
 242         finish_wait(&wq, &__wait);                                      \
 243 __out:  __ret;                                                          \
 244 })
 245
 246 #define __wait_event(wq, condition)                                     \
 247         (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0,  \
 248                             schedule())
 249
 250 /**
 251  * wait_event - sleep until a condition gets true
 252  * @wq: the waitqueue to wait on
 253  * @condition: a C expression for the event to wait for
 254  *
 255  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 256  * @condition evaluates to true. The @condition is checked each time
 257  * the waitqueue @wq is woken up.
 258  *
 259  * wake_up() has to be called after changing any variable that could
 260  * change the result of the wait condition.
 261  */
 262 #define wait_event(wq, condition)                                       \
 263 do {                                                                    \
 264         might_sleep();                                                  \
 265         if (condition)                                                  \
 266                 break;                                                  \
 267         __wait_event(wq, condition);                                    \
 268 } while (0)
 269
 270 #define __io_wait_event(wq, condition)                                  \
 271         (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0,  \
 272                             io_schedule())
 273
 274 /*
 275  * io_wait_event() -- like wait_event() but with io_schedule()
 276  */
 277 #define io_wait_event(wq, condition)                                    \
 278 do {                                                                    \
 279         might_sleep();                                                  \
 280         if (condition)                                                  \
 281                 break;                                                  \
 282         __io_wait_event(wq, condition);                                 \
 283 } while (0)
 284
 285 #define __wait_event_freezable(wq, condition)                           \
 286         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 0, 0,          \
 287                             schedule(); try_to_freeze())
 288
 289 /**
 290  * wait_event - sleep (or freeze) until a condition gets true
 291  * @wq: the waitqueue to wait on
 292  * @condition: a C expression for the event to wait for
 293  *
 294  * The process is put to sleep (TASK_INTERRUPTIBLE -- so as not to contribute
 295  * to system load) until the @condition evaluates to true. The
 296  * @condition is checked each time the waitqueue @wq is woken up.
 297  *
 298  * wake_up() has to be called after changing any variable that could
 299  * change the result of the wait condition.
 300  */
 301 #define wait_event_freezable(wq, condition)                             \
 302 ({                                                                      \
 303         int __ret = 0;                                                  \
 304         might_sleep();                                                  \
 305         if (!(condition))                                               \
 306                 __ret = __wait_event_freezable(wq, condition);          \
 307         __ret;                                                          \
 308 })
 309
 310 #define __wait_event_timeout(wq, condition, timeout)                    \
 311         ___wait_event(wq, ___wait_cond_timeout(condition),              \
 312                       TASK_UNINTERRUPTIBLE, 0, timeout,                 \
 313                       __ret = schedule_timeout(__ret))
 314
 315 /**
 316  * wait_event_timeout - sleep until a condition gets true or a timeout elapses
 317  * @wq: the waitqueue to wait on
 318  * @condition: a C expression for the event to wait for
 319  * @timeout: timeout, in jiffies
 320  *
 321  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 322  * @condition evaluates to true. The @condition is checked each time
 323  * the waitqueue @wq is woken up.
 324  *
 325  * wake_up() has to be called after changing any variable that could
 326  * change the result of the wait condition.
 327  *
 328  * Returns:
 329  * 0 if the @condition evaluated to %false after the @timeout elapsed,
 330  * 1 if the @condition evaluated to %true after the @timeout elapsed,
 331  * or the remaining jiffies (at least 1) if the @condition evaluated
 332  * to %true before the @timeout elapsed.
 333  */
 334 #define wait_event_timeout(wq, condition, timeout)                      \
 335 ({                                                                      \
 336         long __ret = timeout;                                           \
 337         might_sleep();                                                  \
 338         if (!___wait_cond_timeout(condition))                           \
 339                 __ret = __wait_event_timeout(wq, condition, timeout);   \
 340         __ret;                                                          \
 341 })
 342
 343 #define __wait_event_freezable_timeout(wq, condition, timeout)          \
 344         ___wait_event(wq, ___wait_cond_timeout(condition),              \
 345                       TASK_INTERRUPTIBLE, 0, timeout,                   \
 346                       __ret = schedule_timeout(__ret); try_to_freeze())
 347
 348 /*
 349  * like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid
 350  * increasing load and is freezable.
 351  */
 352 #define wait_event_freezable_timeout(wq, condition, timeout)            \
 353 ({                                                                      \
 354         long __ret = timeout;                                           \
 355         might_sleep();                                                  \
 356         if (!___wait_cond_timeout(condition))                           \
 357                 __ret = __wait_event_freezable_timeout(wq, condition, timeout); \
 358         __ret;                                                          \
 359 })
 360
 361 #define __wait_event_cmd(wq, condition, cmd1, cmd2)                     \
 362         (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0,  \
 363                             cmd1; schedule(); cmd2)
 364
 365 /**
 366  * wait_event_cmd - sleep until a condition gets true
 367  * @wq: the waitqueue to wait on
 368  * @condition: a C expression for the event to wait for
 369  * @cmd1: the command will be executed before sleep
 370  * @cmd2: the command will be executed after sleep
 371  *
 372  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 373  * @condition evaluates to true. The @condition is checked each time
 374  * the waitqueue @wq is woken up.
 375  *
 376  * wake_up() has to be called after changing any variable that could
 377  * change the result of the wait condition.
 378  */
 379 #define wait_event_cmd(wq, condition, cmd1, cmd2)                       \
 380 do {                                                                    \
 381         if (condition)                                                  \
 382                 break;                                                  \
 383         __wait_event_cmd(wq, condition, cmd1, cmd2);                    \
 384 } while (0)
 385
 386 #define __wait_event_interruptible(wq, condition)                       \
 387         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 0, 0,          \
 388                       schedule())
 389
 390 /**
 391  * wait_event_interruptible - sleep until a condition gets true
 392  * @wq: the waitqueue to wait on
 393  * @condition: a C expression for the event to wait for
 394  *
 395  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 396  * @condition evaluates to true or a signal is received.
 397  * The @condition is checked each time the waitqueue @wq is woken up.
 398  *
 399  * wake_up() has to be called after changing any variable that could
 400  * change the result of the wait condition.
 401  *
 402  * The function will return -ERESTARTSYS if it was interrupted by a
 403  * signal and 0 if @condition evaluated to true.
 404  */
 405 #define wait_event_interruptible(wq, condition)                         \
 406 ({                                                                      \
 407         int __ret = 0;                                                  \
 408         might_sleep();                                                  \
 409         if (!(condition))                                               \
 410                 __ret = __wait_event_interruptible(wq, condition);      \
 411         __ret;                                                          \
 412 })
 413
 414 #define __wait_event_interruptible_timeout(wq, condition, timeout)      \
 415         ___wait_event(wq, ___wait_cond_timeout(condition),              \
 416                       TASK_INTERRUPTIBLE, 0, timeout,                   \
 417                       __ret = schedule_timeout(__ret))
 418
 419 /**
 420  * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
 421  * @wq: the waitqueue to wait on
 422  * @condition: a C expression for the event to wait for
 423  * @timeout: timeout, in jiffies
 424  *
 425  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 426  * @condition evaluates to true or a signal is received.
 427  * The @condition is checked each time the waitqueue @wq is woken up.
 428  *
 429  * wake_up() has to be called after changing any variable that could
 430  * change the result of the wait condition.
 431  *
 432  * Returns:
 433  * 0 if the @condition evaluated to %false after the @timeout elapsed,
 434  * 1 if the @condition evaluated to %true after the @timeout elapsed,
 435  * the remaining jiffies (at least 1) if the @condition evaluated
 436  * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
 437  * interrupted by a signal.
 438  */
 439 #define wait_event_interruptible_timeout(wq, condition, timeout)        \
 440 ({                                                                      \
 441         long __ret = timeout;                                           \
 442         might_sleep();                                                  \
 443         if (!___wait_cond_timeout(condition))                           \
 444                 __ret = __wait_event_interruptible_timeout(wq,          \
 445                                                 condition, timeout);    \
 446         __ret;                                                          \
 447 })
 448
 449 #define __wait_event_hrtimeout(wq, condition, timeout, state)           \
 450 ({                                                                      \
 451         int __ret = 0;                                                  \
 452         struct hrtimer_sleeper __t;                                     \
 453                                                                         \
 454         hrtimer_init_on_stack(&__t.timer, CLOCK_MONOTONIC,              \
 455                               HRTIMER_MODE_REL);                        \
 456         hrtimer_init_sleeper(&__t, current);                            \
 457         if ((timeout).tv64 != KTIME_MAX)                                \
 458                 hrtimer_start_range_ns(&__t.timer, timeout,             \
 459                                        current->timer_slack_ns,         \
 460                                        HRTIMER_MODE_REL);               \
 461                                                                         \
 462         __ret = ___wait_event(wq, condition, state, 0, 0,               \
 463                 if (!__t.task) {                                        \
 464                         __ret = -ETIME;                                 \
 465                         break;                                          \
 466                 }                                                       \
 467                 schedule());                                            \
 468                                                                         \
 469         hrtimer_cancel(&__t.timer);                                     \
 470         destroy_hrtimer_on_stack(&__t.timer);                           \
 471         __ret;                                                          \
 472 })
 473
 474 /**
 475  * wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
 476  * @wq: the waitqueue to wait on
 477  * @condition: a C expression for the event to wait for
 478  * @timeout: timeout, as a ktime_t
 479  *
 480  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 481  * @condition evaluates to true or a signal is received.
 482  * The @condition is checked each time the waitqueue @wq is woken up.
 483  *
 484  * wake_up() has to be called after changing any variable that could
 485  * change the result of the wait condition.
 486  *
 487  * The function returns 0 if @condition became true, or -ETIME if the timeout
 488  * elapsed.
 489  */
 490 #define wait_event_hrtimeout(wq, condition, timeout)                    \
 491 ({                                                                      \
 492         int __ret = 0;                                                  \
 493         might_sleep();                                                  \
 494         if (!(condition))                                               \
 495                 __ret = __wait_event_hrtimeout(wq, condition, timeout,  \
 496                                                TASK_UNINTERRUPTIBLE);   \
 497         __ret;                                                          \
 498 })
 499
 500 /**
 501  * wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
 502  * @wq: the waitqueue to wait on
 503  * @condition: a C expression for the event to wait for
 504  * @timeout: timeout, as a ktime_t
 505  *
 506  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 507  * @condition evaluates to true or a signal is received.
 508  * The @condition is checked each time the waitqueue @wq is woken up.
 509  *
 510  * wake_up() has to be called after changing any variable that could
 511  * change the result of the wait condition.
 512  *
 513  * The function returns 0 if @condition became true, -ERESTARTSYS if it was
 514  * interrupted by a signal, or -ETIME if the timeout elapsed.
 515  */
 516 #define wait_event_interruptible_hrtimeout(wq, condition, timeout)      \
 517 ({                                                                      \
 518         long __ret = 0;                                                 \
 519         might_sleep();                                                  \
 520         if (!(condition))                                               \
 521                 __ret = __wait_event_hrtimeout(wq, condition, timeout,  \
 522                                                TASK_INTERRUPTIBLE);     \
 523         __ret;                                                          \
 524 })
 525
 526 #define __wait_event_interruptible_exclusive(wq, condition)             \
 527         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0,          \
 528                       schedule())
 529
 530 #define wait_event_interruptible_exclusive(wq, condition)               \
 531 ({                                                                      \
 532         int __ret = 0;                                                  \
 533         might_sleep();                                                  \
 534         if (!(condition))                                               \
 535                 __ret = __wait_event_interruptible_exclusive(wq, condition);\
 536         __ret;                                                          \
 537 })
 538
 539
 540 #define __wait_event_freezable_exclusive(wq, condition)                 \
 541         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0,          \
 542                         schedule(); try_to_freeze())
 543
 544 #define wait_event_freezable_exclusive(wq, condition)                   \
 545 ({                                                                      \
 546         int __ret = 0;                                                  \
 547         might_sleep();                                                  \
 548         if (!(condition))                                               \
 549                 __ret = __wait_event_freezable_exclusive(wq, condition);\
 550         __ret;                                                          \
 551 })
 552
 553
 554 #define __wait_event_interruptible_locked(wq, condition, exclusive, irq) \
 555 ({                                                                      \
 556         int __ret = 0;                                                  \
 557         DEFINE_WAIT(__wait);                                            \
 558         if (exclusive)                                                  \
 559                 __wait.flags |= WQ_FLAG_EXCLUSIVE;                      \
 560         do {                                                            \
 561                 if (likely(list_empty(&__wait.task_list)))              \
 562                         __add_wait_queue_tail(&(wq), &__wait);          \
 563                 set_current_state(TASK_INTERRUPTIBLE);                  \
 564                 if (signal_pending(current)) {                          \
 565                         __ret = -ERESTARTSYS;                           \
 566                         break;                                          \
 567                 }                                                       \
 568                 if (irq)                                                \
 569                         spin_unlock_irq(&(wq).lock);                    \
 570                 else                                                    \
 571                         spin_unlock(&(wq).lock);                        \
 572                 schedule();                                             \
 573                 if (irq)                                                \
 574                         spin_lock_irq(&(wq).lock);                      \
 575                 else                                                    \
 576                         spin_lock(&(wq).lock);                          \
 577         } while (!(condition));                                         \
 578         __remove_wait_queue(&(wq), &__wait);                            \
 579         __set_current_state(TASK_RUNNING);                              \
 580         __ret;                                                          \
 581 })
 582
 583
 584 /**
 585  * wait_event_interruptible_locked - sleep until a condition gets true
 586  * @wq: the waitqueue to wait on
 587  * @condition: a C expression for the event to wait for
 588  *
 589  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 590  * @condition evaluates to true or a signal is received.
 591  * The @condition is checked each time the waitqueue @wq is woken up.
 592  *
 593  * It must be called with wq.lock being held.  This spinlock is
 594  * unlocked while sleeping but @condition testing is done while lock
 595  * is held and when this macro exits the lock is held.
 596  *
 597  * The lock is locked/unlocked using spin_lock()/spin_unlock()
 598  * functions which must match the way they are locked/unlocked outside
 599  * of this macro.
 600  *
 601  * wake_up_locked() has to be called after changing any variable that could
 602  * change the result of the wait condition.
 603  *
 604  * The function will return -ERESTARTSYS if it was interrupted by a
 605  * signal and 0 if @condition evaluated to true.
 606  */
 607 #define wait_event_interruptible_locked(wq, condition)                  \
 608         ((condition)                                                    \
 609          ? 0 : __wait_event_interruptible_locked(wq, condition, 0, 0))
 610
 611 /**
 612  * wait_event_interruptible_locked_irq - sleep until a condition gets true
 613  * @wq: the waitqueue to wait on
 614  * @condition: a C expression for the event to wait for
 615  *
 616  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 617  * @condition evaluates to true or a signal is received.
 618  * The @condition is checked each time the waitqueue @wq is woken up.
 619  *
 620  * It must be called with wq.lock being held.  This spinlock is
 621  * unlocked while sleeping but @condition testing is done while lock
 622  * is held and when this macro exits the lock is held.
 623  *
 624  * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
 625  * functions which must match the way they are locked/unlocked outside
 626  * of this macro.
 627  *
 628  * wake_up_locked() has to be called after changing any variable that could
 629  * change the result of the wait condition.
 630  *
 631  * The function will return -ERESTARTSYS if it was interrupted by a
 632  * signal and 0 if @condition evaluated to true.
 633  */
 634 #define wait_event_interruptible_locked_irq(wq, condition)              \
 635         ((condition)                                                    \
 636          ? 0 : __wait_event_interruptible_locked(wq, condition, 0, 1))
 637
 638 /**
 639  * wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
 640  * @wq: the waitqueue to wait on
 641  * @condition: a C expression for the event to wait for
 642  *
 643  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 644  * @condition evaluates to true or a signal is received.
 645  * The @condition is checked each time the waitqueue @wq is woken up.
 646  *
 647  * It must be called with wq.lock being held.  This spinlock is
 648  * unlocked while sleeping but @condition testing is done while lock
 649  * is held and when this macro exits the lock is held.
 650  *
 651  * The lock is locked/unlocked using spin_lock()/spin_unlock()
 652  * functions which must match the way they are locked/unlocked outside
 653  * of this macro.
 654  *
 655  * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 656  * set thus when other process waits process on the list if this
 657  * process is awaken further processes are not considered.
 658  *
 659  * wake_up_locked() has to be called after changing any variable that could
 660  * change the result of the wait condition.
 661  *
 662  * The function will return -ERESTARTSYS if it was interrupted by a
 663  * signal and 0 if @condition evaluated to true.
 664  */
 665 #define wait_event_interruptible_exclusive_locked(wq, condition)        \
 666         ((condition)                                                    \
 667          ? 0 : __wait_event_interruptible_locked(wq, condition, 1, 0))
 668
 669 /**
 670  * wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
 671  * @wq: the waitqueue to wait on
 672  * @condition: a C expression for the event to wait for
 673  *
 674  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 675  * @condition evaluates to true or a signal is received.
 676  * The @condition is checked each time the waitqueue @wq is woken up.
 677  *
 678  * It must be called with wq.lock being held.  This spinlock is
 679  * unlocked while sleeping but @condition testing is done while lock
 680  * is held and when this macro exits the lock is held.
 681  *
 682  * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
 683  * functions which must match the way they are locked/unlocked outside
 684  * of this macro.
 685  *
 686  * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 687  * set thus when other process waits process on the list if this
 688  * process is awaken further processes are not considered.
 689  *
 690  * wake_up_locked() has to be called after changing any variable that could
 691  * change the result of the wait condition.
 692  *
 693  * The function will return -ERESTARTSYS if it was interrupted by a
 694  * signal and 0 if @condition evaluated to true.
 695  */
 696 #define wait_event_interruptible_exclusive_locked_irq(wq, condition)    \
 697         ((condition)                                                    \
 698          ? 0 : __wait_event_interruptible_locked(wq, condition, 1, 1))
 699
 700
 701 #define __wait_event_killable(wq, condition)                            \
 702         ___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())
 703
 704 /**
 705  * wait_event_killable - sleep until a condition gets true
 706  * @wq: the waitqueue to wait on
 707  * @condition: a C expression for the event to wait for
 708  *
 709  * The process is put to sleep (TASK_KILLABLE) until the
 710  * @condition evaluates to true or a signal is received.
 711  * The @condition is checked each time the waitqueue @wq is woken up.
 712  *
 713  * wake_up() has to be called after changing any variable that could
 714  * change the result of the wait condition.
 715  *
 716  * The function will return -ERESTARTSYS if it was interrupted by a
 717  * signal and 0 if @condition evaluated to true.
 718  */
 719 #define wait_event_killable(wq, condition)                              \
 720 ({                                                                      \
 721         int __ret = 0;                                                  \
 722         might_sleep();                                                  \
 723         if (!(condition))                                               \
 724                 __ret = __wait_event_killable(wq, condition);           \
 725         __ret;                                                          \
 726 })
 727
 728
 729 #define __wait_event_lock_irq(wq, condition, lock, cmd)                 \
 730         (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0,  \
 731                             spin_unlock_irq(&lock);                     \
 732                             cmd;                                        \
 733                             schedule();                                 \
 734                             spin_lock_irq(&lock))
 735
 736 /**
 737  * wait_event_lock_irq_cmd - sleep until a condition gets true. The
 738  *                           condition is checked under the lock. This
 739  *                           is expected to be called with the lock
 740  *                           taken.
 741  * @wq: the waitqueue to wait on
 742  * @condition: a C expression for the event to wait for
 743  * @lock: a locked spinlock_t, which will be released before cmd
 744  *        and schedule() and reacquired afterwards.
 745  * @cmd: a command which is invoked outside the critical section before
 746  *       sleep
 747  *
 748  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 749  * @condition evaluates to true. The @condition is checked each time
 750  * the waitqueue @wq is woken up.
 751  *
 752  * wake_up() has to be called after changing any variable that could
 753  * change the result of the wait condition.
 754  *
 755  * This is supposed to be called while holding the lock. The lock is
 756  * dropped before invoking the cmd and going to sleep and is reacquired
 757  * afterwards.
 758  */
 759 #define wait_event_lock_irq_cmd(wq, condition, lock, cmd)               \
 760 do {                                                                    \
 761         if (condition)                                                  \
 762                 break;                                                  \
 763         __wait_event_lock_irq(wq, condition, lock, cmd);                \
 764 } while (0)
 765
 766 /**
 767  * wait_event_lock_irq - sleep until a condition gets true. The
 768  *                       condition is checked under the lock. This
 769  *                       is expected to be called with the lock
 770  *                       taken.
 771  * @wq: the waitqueue to wait on
 772  * @condition: a C expression for the event to wait for
 773  * @lock: a locked spinlock_t, which will be released before schedule()
 774  *        and reacquired afterwards.
 775  *
 776  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 777  * @condition evaluates to true. The @condition is checked each time
 778  * the waitqueue @wq is woken up.
 779  *
 780  * wake_up() has to be called after changing any variable that could
 781  * change the result of the wait condition.
 782  *
 783  * This is supposed to be called while holding the lock. The lock is
 784  * dropped before going to sleep and is reacquired afterwards.
 785  */
 786 #define wait_event_lock_irq(wq, condition, lock)                        \
 787 do {                                                                    \
 788         if (condition)                                                  \
 789                 break;                                                  \
 790         __wait_event_lock_irq(wq, condition, lock, );                   \
 791 } while (0)
 792
 793
 794 #define __wait_event_interruptible_lock_irq(wq, condition, lock, cmd)   \
 795         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 0, 0,          \
 796                       spin_unlock_irq(&lock);                           \
 797                       cmd;                                              \
 798                       schedule();                                       \
 799                       spin_lock_irq(&lock))
 800
 801 /**
 802  * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
 803  *              The condition is checked under the lock. This is expected to
 804  *              be called with the lock taken.
 805  * @wq: the waitqueue to wait on
 806  * @condition: a C expression for the event to wait for
 807  * @lock: a locked spinlock_t, which will be released before cmd and
 808  *        schedule() and reacquired afterwards.
 809  * @cmd: a command which is invoked outside the critical section before
 810  *       sleep
 811  *
 812  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 813  * @condition evaluates to true or a signal is received. The @condition is
 814  * checked each time the waitqueue @wq is woken up.
 815  *
 816  * wake_up() has to be called after changing any variable that could
 817  * change the result of the wait condition.
 818  *
 819  * This is supposed to be called while holding the lock. The lock is
 820  * dropped before invoking the cmd and going to sleep and is reacquired
 821  * afterwards.
 822  *
 823  * The macro will return -ERESTARTSYS if it was interrupted by a signal
 824  * and 0 if @condition evaluated to true.
 825  */
 826 #define wait_event_interruptible_lock_irq_cmd(wq, condition, lock, cmd) \
 827 ({                                                                      \
 828         int __ret = 0;                                                  \
 829         if (!(condition))                                               \
 830                 __ret = __wait_event_interruptible_lock_irq(wq,         \
 831                                                 condition, lock, cmd);  \
 832         __ret;                                                          \
 833 })
 834
 835 /**
 836  * wait_event_interruptible_lock_irq - sleep until a condition gets true.
 837  *              The condition is checked under the lock. This is expected
 838  *              to be called with the lock taken.
 839  * @wq: the waitqueue to wait on
 840  * @condition: a C expression for the event to wait for
 841  * @lock: a locked spinlock_t, which will be released before schedule()
 842  *        and reacquired afterwards.
 843  *
 844  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 845  * @condition evaluates to true or signal is received. The @condition is
 846  * checked each time the waitqueue @wq is woken up.
 847  *
 848  * wake_up() has to be called after changing any variable that could
 849  * change the result of the wait condition.
 850  *
 851  * This is supposed to be called while holding the lock. The lock is
 852  * dropped before going to sleep and is reacquired afterwards.
 853  *
 854  * The macro will return -ERESTARTSYS if it was interrupted by a signal
 855  * and 0 if @condition evaluated to true.
 856  */
 857 #define wait_event_interruptible_lock_irq(wq, condition, lock)          \
 858 ({                                                                      \
 859         int __ret = 0;                                                  \
 860         if (!(condition))                                               \
 861                 __ret = __wait_event_interruptible_lock_irq(wq,         \
 862                                                 condition, lock,);      \
 863         __ret;                                                          \
 864 })
 865
 866 #define __wait_event_interruptible_lock_irq_timeout(wq, condition,      \
 867                                                     lock, timeout)      \
 868         ___wait_event(wq, ___wait_cond_timeout(condition),              \
 869                       TASK_INTERRUPTIBLE, 0, timeout,                   \
 870                       spin_unlock_irq(&lock);                           \
 871                       __ret = schedule_timeout(__ret);                  \
 872                       spin_lock_irq(&lock));
 873
 874 /**
 875  * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
 876  *              true or a timeout elapses. The condition is checked under
 877  *              the lock. This is expected to be called with the lock taken.
 878  * @wq: the waitqueue to wait on
 879  * @condition: a C expression for the event to wait for
 880  * @lock: a locked spinlock_t, which will be released before schedule()
 881  *        and reacquired afterwards.
 882  * @timeout: timeout, in jiffies
 883  *
 884  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 885  * @condition evaluates to true or signal is received. The @condition is
 886  * checked each time the waitqueue @wq is woken up.
 887  *
 888  * wake_up() has to be called after changing any variable that could
 889  * change the result of the wait condition.
 890  *
 891  * This is supposed to be called while holding the lock. The lock is
 892  * dropped before going to sleep and is reacquired afterwards.
 893  *
 894  * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
 895  * was interrupted by a signal, and the remaining jiffies otherwise
 896  * if the condition evaluated to true before the timeout elapsed.
 897  */
 898 #define wait_event_interruptible_lock_irq_timeout(wq, condition, lock,  \
 899                                                   timeout)              \
 900 ({                                                                      \
 901         long __ret = timeout;                                           \
 902         if (!___wait_cond_timeout(condition))                           \
 903                 __ret = __wait_event_interruptible_lock_irq_timeout(    \
 904                                         wq, condition, lock, timeout);  \
 905         __ret;                                                          \
 906 })
 907
 908 /*
 909  * Waitqueues which are removed from the waitqueue_head at wakeup time
 910  */
 911 void prepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state);
 912 void prepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state);
 913 long prepare_to_wait_event(wait_queue_head_t *q, wait_queue_t *wait, int state);
 914 void finish_wait(wait_queue_head_t *q, wait_queue_t *wait);
 915 void abort_exclusive_wait(wait_queue_head_t *q, wait_queue_t *wait, unsigned int mode, void *key);
 916 long wait_woken(wait_queue_t *wait, unsigned mode, long timeout);
 917 int woken_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
 918 int autoremove_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
 919 int wake_bit_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
 920
 921 #define DEFINE_WAIT_FUNC(name, function)                                \
 922         wait_queue_t name = {                                           \
 923                 .private        = current,                              \
 924                 .func           = function,                             \
 925                 .task_list      = LIST_HEAD_INIT((name).task_list),     \
 926         }
 927
 928 #define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)
 929
 930 #define DEFINE_WAIT_BIT(name, word, bit)                                \
 931         struct wait_bit_queue name = {                                  \
 932                 .key = __WAIT_BIT_KEY_INITIALIZER(word, bit),           \
 933                 .wait   = {                                             \
 934                         .private        = current,                      \
 935                         .func           = wake_bit_function,            \
 936                         .task_list      =                               \
 937                                 LIST_HEAD_INIT((name).wait.task_list),  \
 938                 },                                                      \
 939         }
 940
 941 #define init_wait(wait)                                                 \
 942         do {                                                            \
 943                 (wait)->private = current;                              \
 944                 (wait)->func = autoremove_wake_function;                \
 945                 INIT_LIST_HEAD(&(wait)->task_list);                     \
 946                 (wait)->flags = 0;                                      \
 947         } while (0)
 948
 949
 950 extern int bit_wait(struct wait_bit_key *);
 951 extern int bit_wait_io(struct wait_bit_key *);
 952 extern int bit_wait_timeout(struct wait_bit_key *);
 953 extern int bit_wait_io_timeout(struct wait_bit_key *);
 954
 955 /**
 956  * wait_on_bit - wait for a bit to be cleared
 957  * @word: the word being waited on, a kernel virtual address
 958  * @bit: the bit of the word being waited on
 959  * @mode: the task state to sleep in
 960  *
 961  * There is a standard hashed waitqueue table for generic use. This
 962  * is the part of the hashtable's accessor API that waits on a bit.
 963  * For instance, if one were to have waiters on a bitflag, one would
 964  * call wait_on_bit() in threads waiting for the bit to clear.
 965  * One uses wait_on_bit() where one is waiting for the bit to clear,
 966  * but has no intention of setting it.
 967  * Returned value will be zero if the bit was cleared, or non-zero
 968  * if the process received a signal and the mode permitted wakeup
 969  * on that signal.
 970  */
 971 static inline int
 972 wait_on_bit(void *word, int bit, unsigned mode)
 973 {
 974         might_sleep();
 975         if (!test_bit(bit, word))
 976                 return 0;
 977         return out_of_line_wait_on_bit(word, bit,
 978                                        bit_wait,
 979                                        mode);
 980 }
 981
 982 /**
 983  * wait_on_bit_io - wait for a bit to be cleared
 984  * @word: the word being waited on, a kernel virtual address
 985  * @bit: the bit of the word being waited on
 986  * @mode: the task state to sleep in
 987  *
 988  * Use the standard hashed waitqueue table to wait for a bit
 989  * to be cleared.  This is similar to wait_on_bit(), but calls
 990  * io_schedule() instead of schedule() for the actual waiting.
 991  *
 992  * Returned value will be zero if the bit was cleared, or non-zero
 993  * if the process received a signal and the mode permitted wakeup
 994  * on that signal.
 995  */
 996 static inline int
 997 wait_on_bit_io(void *word, int bit, unsigned mode)
 998 {
 999         might_sleep();
1000         if (!test_bit(bit, word))
1001                 return 0;
1002         return out_of_line_wait_on_bit(word, bit,
1003                                        bit_wait_io,
1004                                        mode);
1005 }
1006
1007 /**
1008  * wait_on_bit_timeout - wait for a bit to be cleared or a timeout elapses
1009  * @word: the word being waited on, a kernel virtual address
1010  * @bit: the bit of the word being waited on
1011  * @mode: the task state to sleep in
1012  * @timeout: timeout, in jiffies
1013  *
1014  * Use the standard hashed waitqueue table to wait for a bit
1015  * to be cleared. This is similar to wait_on_bit(), except also takes a
1016  * timeout parameter.
1017  *
1018  * Returned value will be zero if the bit was cleared before the
1019  * @timeout elapsed, or non-zero if the @timeout elapsed or process
1020  * received a signal and the mode permitted wakeup on that signal.
1021  */
1022 static inline int
1023 wait_on_bit_timeout(void *word, int bit, unsigned mode, unsigned long timeout)
1024 {
1025         might_sleep();
1026         if (!test_bit(bit, word))
1027                 return 0;
1028         return out_of_line_wait_on_bit_timeout(word, bit,
1029                                                bit_wait_timeout,
1030                                                mode, timeout);
1031 }
1032
1033 /**
1034  * wait_on_bit_action - wait for a bit to be cleared
1035  * @word: the word being waited on, a kernel virtual address
1036  * @bit: the bit of the word being waited on
1037  * @action: the function used to sleep, which may take special actions
1038  * @mode: the task state to sleep in
1039  *
1040  * Use the standard hashed waitqueue table to wait for a bit
1041  * to be cleared, and allow the waiting action to be specified.
1042  * This is like wait_on_bit() but allows fine control of how the waiting
1043  * is done.
1044  *
1045  * Returned value will be zero if the bit was cleared, or non-zero
1046  * if the process received a signal and the mode permitted wakeup
1047  * on that signal.
1048  */
1049 static inline int
1050 wait_on_bit_action(void *word, int bit, wait_bit_action_f *action, unsigned mode)
1051 {
1052         might_sleep();
1053         if (!test_bit(bit, word))
1054                 return 0;
1055         return out_of_line_wait_on_bit(word, bit, action, mode);
1056 }
1057
1058 /**
1059  * wait_on_bit_lock - wait for a bit to be cleared, when wanting to set it
1060  * @word: the word being waited on, a kernel virtual address
1061  * @bit: the bit of the word being waited on
1062  * @mode: the task state to sleep in
1063  *
1064  * There is a standard hashed waitqueue table for generic use. This
1065  * is the part of the hashtable's accessor API that waits on a bit
1066  * when one intends to set it, for instance, trying to lock bitflags.
1067  * For instance, if one were to have waiters trying to set bitflag
1068  * and waiting for it to clear before setting it, one would call
1069  * wait_on_bit() in threads waiting to be able to set the bit.
1070  * One uses wait_on_bit_lock() where one is waiting for the bit to
1071  * clear with the intention of setting it, and when done, clearing it.
1072  *
1073  * Returns zero if the bit was (eventually) found to be clear and was
1074  * set.  Returns non-zero if a signal was delivered to the process and
1075  * the @mode allows that signal to wake the process.
1076  */
1077 static inline int
1078 wait_on_bit_lock(void *word, int bit, unsigned mode)
1079 {
1080         might_sleep();
1081         if (!test_and_set_bit(bit, word))
1082                 return 0;
1083         return out_of_line_wait_on_bit_lock(word, bit, bit_wait, mode);
1084 }
1085
1086 /**
1087  * wait_on_bit_lock_io - wait for a bit to be cleared, when wanting to set it
1088  * @word: the word being waited on, a kernel virtual address
1089  * @bit: the bit of the word being waited on
1090  * @mode: the task state to sleep in
1091  *
1092  * Use the standard hashed waitqueue table to wait for a bit
1093  * to be cleared and then to atomically set it.  This is similar
1094  * to wait_on_bit(), but calls io_schedule() instead of schedule()
1095  * for the actual waiting.
1096  *
1097  * Returns zero if the bit was (eventually) found to be clear and was
1098  * set.  Returns non-zero if a signal was delivered to the process and
1099  * the @mode allows that signal to wake the process.
1100  */
1101 static inline int
1102 wait_on_bit_lock_io(void *word, int bit, unsigned mode)
1103 {
1104         might_sleep();
1105         if (!test_and_set_bit(bit, word))
1106                 return 0;
1107         return out_of_line_wait_on_bit_lock(word, bit, bit_wait_io, mode);
1108 }
1109
1110 /**
1111  * wait_on_bit_lock_action - wait for a bit to be cleared, when wanting to set it
1112  * @word: the word being waited on, a kernel virtual address
1113  * @bit: the bit of the word being waited on
1114  * @action: the function used to sleep, which may take special actions
1115  * @mode: the task state to sleep in
1116  *
1117  * Use the standard hashed waitqueue table to wait for a bit
1118  * to be cleared and then to set it, and allow the waiting action
1119  * to be specified.
1120  * This is like wait_on_bit() but allows fine control of how the waiting
1121  * is done.
1122  *
1123  * Returns zero if the bit was (eventually) found to be clear and was
1124  * set.  Returns non-zero if a signal was delivered to the process and
1125  * the @mode allows that signal to wake the process.
1126  */
1127 static inline int
1128 wait_on_bit_lock_action(void *word, int bit, wait_bit_action_f *action, unsigned mode)
1129 {
1130         might_sleep();
1131         if (!test_and_set_bit(bit, word))
1132                 return 0;
1133         return out_of_line_wait_on_bit_lock(word, bit, action, mode);
1134 }
1135
1136 /**
1137  * wait_on_atomic_t - Wait for an atomic_t to become 0
1138  * @val: The atomic value being waited on, a kernel virtual address
1139  * @action: the function used to sleep, which may take special actions
1140  * @mode: the task state to sleep in
1141  *
1142  * Wait for an atomic_t to become 0.  We abuse the bit-wait waitqueue table for
1143  * the purpose of getting a waitqueue, but we set the key to a bit number
1144  * outside of the target 'word'.
1145  */
1146 static inline
1147 int wait_on_atomic_t(atomic_t *val, int (*action)(atomic_t *), unsigned mode)
1148 {
1149         might_sleep();
1150         if (atomic_read(val) == 0)
1151                 return 0;
1152         return out_of_line_wait_on_atomic_t(val, action, mode);
1153 }
1154
1155 #endif /* _LINUX_WAIT_H */