1 // SPDX-License-Identifier: GPL-2.0-or-later
2
3 #include <linux/sched/task.h>
4 #include <linux/sched/signal.h>
5 #include <linux/freezer.h>
6
7 #include "futex.h"
8 #include <trace/hooks/futex.h>
9
10 /*
11 * READ this before attempting to hack on futexes!
12 *
13 * Basic futex operation and ordering guarantees
14 * =============================================
15 *
16 * The waiter reads the futex value in user space and calls
17 * futex_wait(). This function computes the hash bucket and acquires
18 * the hash bucket lock. After that it reads the futex user space value
19 * again and verifies that the data has not changed. If it has not changed
20 * it enqueues itself into the hash bucket, releases the hash bucket lock
21 * and schedules.
22 *
23 * The waker side modifies the user space value of the futex and calls
24 * futex_wake(). This function computes the hash bucket and acquires the
25 * hash bucket lock. Then it looks for waiters on that futex in the hash
26 * bucket and wakes them.
27 *
28 * In futex wake up scenarios where no tasks are blocked on a futex, taking
29 * the hb spinlock can be avoided and simply return. In order for this
30 * optimization to work, ordering guarantees must exist so that the waiter
31 * being added to the list is acknowledged when the list is concurrently being
32 * checked by the waker, avoiding scenarios like the following:
33 *
34 * CPU 0 CPU 1
35 * val = *futex;
36 * sys_futex(WAIT, futex, val);
37 * futex_wait(futex, val);
38 * uval = *futex;
39 * *futex = newval;
40 * sys_futex(WAKE, futex);
41 * futex_wake(futex);
42 * if (queue_empty())
43 * return;
44 * if (uval == val)
45 * lock(hash_bucket(futex));
46 * queue();
47 * unlock(hash_bucket(futex));
48 * schedule();
49 *
50 * This would cause the waiter on CPU 0 to wait forever because it
51 * missed the transition of the user space value from val to newval
52 * and the waker did not find the waiter in the hash bucket queue.
53 *
54 * The correct serialization ensures that a waiter either observes
55 * the changed user space value before blocking or is woken by a
56 * concurrent waker:
57 *
58 * CPU 0 CPU 1
59 * val = *futex;
60 * sys_futex(WAIT, futex, val);
61 * futex_wait(futex, val);
62 *
63 * waiters++; (a)
64 * smp_mb(); (A) <-- paired with -.
65 * |
66 * lock(hash_bucket(futex)); |
67 * |
68 * uval = *futex; |
69 * | *futex = newval;
70 * | sys_futex(WAKE, futex);
71 * | futex_wake(futex);
72 * |
73 * `--------> smp_mb(); (B)
74 * if (uval == val)
75 * queue();
76 * unlock(hash_bucket(futex));
77 * schedule(); if (waiters)
78 * lock(hash_bucket(futex));
79 * else wake_waiters(futex);
80 * waiters--; (b) unlock(hash_bucket(futex));
81 *
82 * Where (A) orders the waiters increment and the futex value read through
83 * atomic operations (see futex_hb_waiters_inc) and where (B) orders the write
84 * to futex and the waiters read (see futex_hb_waiters_pending()).
85 *
86 * This yields the following case (where X:=waiters, Y:=futex):
87 *
88 * X = Y = 0
89 *
90 * w[X]=1 w[Y]=1
91 * MB MB
92 * r[Y]=y r[X]=x
93 *
94 * Which guarantees that x==0 && y==0 is impossible; which translates back into
95 * the guarantee that we cannot both miss the futex variable change and the
96 * enqueue.
97 *
98 * Note that a new waiter is accounted for in (a) even when it is possible that
99 * the wait call can return error, in which case we backtrack from it in (b).
100 * Refer to the comment in futex_q_lock().
101 *
102 * Similarly, in order to account for waiters being requeued on another
103 * address we always increment the waiters for the destination bucket before
104 * acquiring the lock. It then decrements them again after releasing it -
105 * the code that actually moves the futex(es) between hash buckets (requeue_futex)
106 * will do the additional required waiter count housekeeping. This is done for
107 * double_lock_hb() and double_unlock_hb(), respectively.
108 */
109
110 /*
111 * The hash bucket lock must be held when this is called.
112 * Afterwards, the futex_q must not be accessed. Callers
113 * must ensure to later call wake_up_q() for the actual
114 * wakeups to occur.
115 */
futex_wake_mark(struct wake_q_head * wake_q,struct futex_q * q)116 void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q)
117 {
118 struct task_struct *p = q->task;
119
120 if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n"))
121 return;
122
123 get_task_struct(p);
124 __futex_unqueue(q);
125 /*
126 * The waiting task can free the futex_q as soon as q->lock_ptr = NULL
127 * is written, without taking any locks. This is possible in the event
128 * of a spurious wakeup, for example. A memory barrier is required here
129 * to prevent the following store to lock_ptr from getting ahead of the
130 * plist_del in __futex_unqueue().
131 */
132 smp_store_release(&q->lock_ptr, NULL);
133
134 /*
135 * Queue the task for later wakeup for after we've released
136 * the hb->lock.
137 */
138 wake_q_add_safe(wake_q, p);
139 }
140
141 /*
142 * Wake up waiters matching bitset queued on this futex (uaddr).
143 */
futex_wake(u32 __user * uaddr,unsigned int flags,int nr_wake,u32 bitset)144 int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset)
145 {
146 struct futex_hash_bucket *hb;
147 struct futex_q *this, *next;
148 union futex_key key = FUTEX_KEY_INIT;
149 int ret;
150 int target_nr;
151 DEFINE_WAKE_Q(wake_q);
152
153 if (!bitset)
154 return -EINVAL;
155
156 ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, FUTEX_READ);
157 if (unlikely(ret != 0))
158 return ret;
159
160 hb = futex_hash(&key);
161
162 /* Make sure we really have tasks to wakeup */
163 if (!futex_hb_waiters_pending(hb))
164 return ret;
165
166 spin_lock(&hb->lock);
167
168 trace_android_vh_futex_wake_traverse_plist(&hb->chain, &target_nr, key, bitset);
169 plist_for_each_entry_safe(this, next, &hb->chain, list) {
170 if (futex_match (&this->key, &key)) {
171 if (this->pi_state || this->rt_waiter) {
172 ret = -EINVAL;
173 break;
174 }
175
176 /* Check if one of the bits is set in both bitsets */
177 if (!(this->bitset & bitset))
178 continue;
179
180 trace_android_vh_futex_wake_this(ret, nr_wake, target_nr, this->task);
181 futex_wake_mark(&wake_q, this);
182 if (++ret >= nr_wake)
183 break;
184 }
185 }
186
187 spin_unlock(&hb->lock);
188 wake_up_q(&wake_q);
189 trace_android_vh_futex_wake_up_q_finish(nr_wake, target_nr);
190 return ret;
191 }
192
futex_atomic_op_inuser(unsigned int encoded_op,u32 __user * uaddr)193 static int futex_atomic_op_inuser(unsigned int encoded_op, u32 __user *uaddr)
194 {
195 unsigned int op = (encoded_op & 0x70000000) >> 28;
196 unsigned int cmp = (encoded_op & 0x0f000000) >> 24;
197 int oparg = sign_extend32((encoded_op & 0x00fff000) >> 12, 11);
198 int cmparg = sign_extend32(encoded_op & 0x00000fff, 11);
199 int oldval, ret;
200
201 if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28)) {
202 if (oparg < 0 || oparg > 31) {
203 char comm[sizeof(current->comm)];
204 /*
205 * kill this print and return -EINVAL when userspace
206 * is sane again
207 */
208 pr_info_ratelimited("futex_wake_op: %s tries to shift op by %d; fix this program\n",
209 get_task_comm(comm, current), oparg);
210 oparg &= 31;
211 }
212 oparg = 1 << oparg;
213 }
214
215 pagefault_disable();
216 ret = arch_futex_atomic_op_inuser(op, oparg, &oldval, uaddr);
217 pagefault_enable();
218 if (ret)
219 return ret;
220
221 switch (cmp) {
222 case FUTEX_OP_CMP_EQ:
223 return oldval == cmparg;
224 case FUTEX_OP_CMP_NE:
225 return oldval != cmparg;
226 case FUTEX_OP_CMP_LT:
227 return oldval < cmparg;
228 case FUTEX_OP_CMP_GE:
229 return oldval >= cmparg;
230 case FUTEX_OP_CMP_LE:
231 return oldval <= cmparg;
232 case FUTEX_OP_CMP_GT:
233 return oldval > cmparg;
234 default:
235 return -ENOSYS;
236 }
237 }
238
239 /*
240 * Wake up all waiters hashed on the physical page that is mapped
241 * to this virtual address:
242 */
futex_wake_op(u32 __user * uaddr1,unsigned int flags,u32 __user * uaddr2,int nr_wake,int nr_wake2,int op)243 int futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2,
244 int nr_wake, int nr_wake2, int op)
245 {
246 union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
247 struct futex_hash_bucket *hb1, *hb2;
248 struct futex_q *this, *next;
249 int ret, op_ret;
250 DEFINE_WAKE_Q(wake_q);
251
252 retry:
253 ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, FUTEX_READ);
254 if (unlikely(ret != 0))
255 return ret;
256 ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, FUTEX_WRITE);
257 if (unlikely(ret != 0))
258 return ret;
259
260 hb1 = futex_hash(&key1);
261 hb2 = futex_hash(&key2);
262
263 retry_private:
264 double_lock_hb(hb1, hb2);
265 op_ret = futex_atomic_op_inuser(op, uaddr2);
266 if (unlikely(op_ret < 0)) {
267 double_unlock_hb(hb1, hb2);
268
269 if (!IS_ENABLED(CONFIG_MMU) ||
270 unlikely(op_ret != -EFAULT && op_ret != -EAGAIN)) {
271 /*
272 * we don't get EFAULT from MMU faults if we don't have
273 * an MMU, but we might get them from range checking
274 */
275 ret = op_ret;
276 return ret;
277 }
278
279 if (op_ret == -EFAULT) {
280 ret = fault_in_user_writeable(uaddr2);
281 if (ret)
282 return ret;
283 }
284
285 cond_resched();
286 if (!(flags & FLAGS_SHARED))
287 goto retry_private;
288 goto retry;
289 }
290
291 plist_for_each_entry_safe(this, next, &hb1->chain, list) {
292 if (futex_match (&this->key, &key1)) {
293 if (this->pi_state || this->rt_waiter) {
294 ret = -EINVAL;
295 goto out_unlock;
296 }
297 futex_wake_mark(&wake_q, this);
298 if (++ret >= nr_wake)
299 break;
300 }
301 }
302
303 if (op_ret > 0) {
304 op_ret = 0;
305 plist_for_each_entry_safe(this, next, &hb2->chain, list) {
306 if (futex_match (&this->key, &key2)) {
307 if (this->pi_state || this->rt_waiter) {
308 ret = -EINVAL;
309 goto out_unlock;
310 }
311 futex_wake_mark(&wake_q, this);
312 if (++op_ret >= nr_wake2)
313 break;
314 }
315 }
316 ret += op_ret;
317 }
318
319 out_unlock:
320 double_unlock_hb(hb1, hb2);
321 wake_up_q(&wake_q);
322 return ret;
323 }
324
325 static long futex_wait_restart(struct restart_block *restart);
326
327 /**
328 * futex_wait_queue() - futex_queue() and wait for wakeup, timeout, or signal
329 * @hb: the futex hash bucket, must be locked by the caller
330 * @q: the futex_q to queue up on
331 * @timeout: the prepared hrtimer_sleeper, or null for no timeout
332 */
futex_wait_queue(struct futex_hash_bucket * hb,struct futex_q * q,struct hrtimer_sleeper * timeout)333 void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q,
334 struct hrtimer_sleeper *timeout)
335 {
336 /*
337 * The task state is guaranteed to be set before another task can
338 * wake it. set_current_state() is implemented using smp_store_mb() and
339 * futex_queue() calls spin_unlock() upon completion, both serializing
340 * access to the hash list and forcing another memory barrier.
341 */
342 set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
343 futex_queue(q, hb);
344
345 /* Arm the timer */
346 if (timeout)
347 hrtimer_sleeper_start_expires(timeout, HRTIMER_MODE_ABS);
348
349 /*
350 * If we have been removed from the hash list, then another task
351 * has tried to wake us, and we can skip the call to schedule().
352 */
353 if (likely(!plist_node_empty(&q->list))) {
354 /*
355 * If the timer has already expired, current will already be
356 * flagged for rescheduling. Only call schedule if there
357 * is no timeout, or if it has yet to expire.
358 */
359 if (!timeout || timeout->task) {
360 trace_android_vh_futex_sleep_start(current);
361 schedule();
362 }
363 }
364 __set_current_state(TASK_RUNNING);
365 }
366
367 /**
368 * unqueue_multiple - Remove various futexes from their hash bucket
369 * @v: The list of futexes to unqueue
370 * @count: Number of futexes in the list
371 *
372 * Helper to unqueue a list of futexes. This can't fail.
373 *
374 * Return:
375 * - >=0 - Index of the last futex that was awoken;
376 * - -1 - No futex was awoken
377 */
unqueue_multiple(struct futex_vector * v,int count)378 static int unqueue_multiple(struct futex_vector *v, int count)
379 {
380 int ret = -1, i;
381
382 for (i = 0; i < count; i++) {
383 if (!futex_unqueue(&v[i].q))
384 ret = i;
385 }
386
387 return ret;
388 }
389
390 /**
391 * futex_wait_multiple_setup - Prepare to wait and enqueue multiple futexes
392 * @vs: The futex list to wait on
393 * @count: The size of the list
394 * @woken: Index of the last woken futex, if any. Used to notify the
395 * caller that it can return this index to userspace (return parameter)
396 *
397 * Prepare multiple futexes in a single step and enqueue them. This may fail if
398 * the futex list is invalid or if any futex was already awoken. On success the
399 * task is ready to interruptible sleep.
400 *
401 * Return:
402 * - 1 - One of the futexes was woken by another thread
403 * - 0 - Success
404 * - <0 - -EFAULT, -EWOULDBLOCK or -EINVAL
405 */
futex_wait_multiple_setup(struct futex_vector * vs,int count,int * woken)406 static int futex_wait_multiple_setup(struct futex_vector *vs, int count, int *woken)
407 {
408 struct futex_hash_bucket *hb;
409 bool retry = false;
410 int ret, i;
411 u32 uval;
412
413 /*
414 * Enqueuing multiple futexes is tricky, because we need to enqueue
415 * each futex on the list before dealing with the next one to avoid
416 * deadlocking on the hash bucket. But, before enqueuing, we need to
417 * make sure that current->state is TASK_INTERRUPTIBLE, so we don't
418 * lose any wake events, which cannot be done before the get_futex_key
419 * of the next key, because it calls get_user_pages, which can sleep.
420 * Thus, we fetch the list of futexes keys in two steps, by first
421 * pinning all the memory keys in the futex key, and only then we read
422 * each key and queue the corresponding futex.
423 *
424 * Private futexes doesn't need to recalculate hash in retry, so skip
425 * get_futex_key() when retrying.
426 */
427 retry:
428 for (i = 0; i < count; i++) {
429 if ((vs[i].w.flags & FUTEX_PRIVATE_FLAG) && retry)
430 continue;
431
432 ret = get_futex_key(u64_to_user_ptr(vs[i].w.uaddr),
433 !(vs[i].w.flags & FUTEX_PRIVATE_FLAG),
434 &vs[i].q.key, FUTEX_READ);
435
436 if (unlikely(ret))
437 return ret;
438 }
439
440 set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
441
442 for (i = 0; i < count; i++) {
443 u32 __user *uaddr = (u32 __user *)(unsigned long)vs[i].w.uaddr;
444 struct futex_q *q = &vs[i].q;
445 u32 val = (u32)vs[i].w.val;
446
447 hb = futex_q_lock(q);
448 ret = futex_get_value_locked(&uval, uaddr);
449
450 if (!ret && uval == val) {
451 /*
452 * The bucket lock can't be held while dealing with the
453 * next futex. Queue each futex at this moment so hb can
454 * be unlocked.
455 */
456 futex_queue(q, hb);
457 continue;
458 }
459
460 futex_q_unlock(hb);
461 __set_current_state(TASK_RUNNING);
462
463 /*
464 * Even if something went wrong, if we find out that a futex
465 * was woken, we don't return error and return this index to
466 * userspace
467 */
468 *woken = unqueue_multiple(vs, i);
469 if (*woken >= 0)
470 return 1;
471
472 if (ret) {
473 /*
474 * If we need to handle a page fault, we need to do so
475 * without any lock and any enqueued futex (otherwise
476 * we could lose some wakeup). So we do it here, after
477 * undoing all the work done so far. In success, we
478 * retry all the work.
479 */
480 if (get_user(uval, uaddr))
481 return -EFAULT;
482
483 retry = true;
484 goto retry;
485 }
486
487 if (uval != val)
488 return -EWOULDBLOCK;
489 }
490
491 return 0;
492 }
493
494 /**
495 * futex_sleep_multiple - Check sleeping conditions and sleep
496 * @vs: List of futexes to wait for
497 * @count: Length of vs
498 * @to: Timeout
499 *
500 * Sleep if and only if the timeout hasn't expired and no futex on the list has
501 * been woken up.
502 */
futex_sleep_multiple(struct futex_vector * vs,unsigned int count,struct hrtimer_sleeper * to)503 static void futex_sleep_multiple(struct futex_vector *vs, unsigned int count,
504 struct hrtimer_sleeper *to)
505 {
506 if (to && !to->task)
507 return;
508
509 for (; count; count--, vs++) {
510 if (!READ_ONCE(vs->q.lock_ptr))
511 return;
512 }
513
514 schedule();
515 }
516
517 /**
518 * futex_wait_multiple - Prepare to wait on and enqueue several futexes
519 * @vs: The list of futexes to wait on
520 * @count: The number of objects
521 * @to: Timeout before giving up and returning to userspace
522 *
523 * Entry point for the FUTEX_WAIT_MULTIPLE futex operation, this function
524 * sleeps on a group of futexes and returns on the first futex that is
525 * wake, or after the timeout has elapsed.
526 *
527 * Return:
528 * - >=0 - Hint to the futex that was awoken
529 * - <0 - On error
530 */
futex_wait_multiple(struct futex_vector * vs,unsigned int count,struct hrtimer_sleeper * to)531 int futex_wait_multiple(struct futex_vector *vs, unsigned int count,
532 struct hrtimer_sleeper *to)
533 {
534 int ret, hint = 0;
535
536 if (to)
537 hrtimer_sleeper_start_expires(to, HRTIMER_MODE_ABS);
538
539 while (1) {
540 ret = futex_wait_multiple_setup(vs, count, &hint);
541 if (ret) {
542 if (ret > 0) {
543 /* A futex was woken during setup */
544 ret = hint;
545 }
546 return ret;
547 }
548
549 futex_sleep_multiple(vs, count, to);
550
551 __set_current_state(TASK_RUNNING);
552
553 ret = unqueue_multiple(vs, count);
554 if (ret >= 0)
555 return ret;
556
557 if (to && !to->task)
558 return -ETIMEDOUT;
559 else if (signal_pending(current))
560 return -ERESTARTSYS;
561 /*
562 * The final case is a spurious wakeup, for
563 * which just retry.
564 */
565 }
566 }
567
568 /**
569 * futex_wait_setup() - Prepare to wait on a futex
570 * @uaddr: the futex userspace address
571 * @val: the expected value
572 * @flags: futex flags (FLAGS_SHARED, etc.)
573 * @q: the associated futex_q
574 * @hb: storage for hash_bucket pointer to be returned to caller
575 *
576 * Setup the futex_q and locate the hash_bucket. Get the futex value and
577 * compare it with the expected value. Handle atomic faults internally.
578 * Return with the hb lock held on success, and unlocked on failure.
579 *
580 * Return:
581 * - 0 - uaddr contains val and hb has been locked;
582 * - <1 - -EFAULT or -EWOULDBLOCK (uaddr does not contain val) and hb is unlocked
583 */
futex_wait_setup(u32 __user * uaddr,u32 val,unsigned int flags,struct futex_q * q,struct futex_hash_bucket ** hb)584 int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
585 struct futex_q *q, struct futex_hash_bucket **hb)
586 {
587 u32 uval;
588 int ret;
589
590 /*
591 * Access the page AFTER the hash-bucket is locked.
592 * Order is important:
593 *
594 * Userspace waiter: val = var; if (cond(val)) futex_wait(&var, val);
595 * Userspace waker: if (cond(var)) { var = new; futex_wake(&var); }
596 *
597 * The basic logical guarantee of a futex is that it blocks ONLY
598 * if cond(var) is known to be true at the time of blocking, for
599 * any cond. If we locked the hash-bucket after testing *uaddr, that
600 * would open a race condition where we could block indefinitely with
601 * cond(var) false, which would violate the guarantee.
602 *
603 * On the other hand, we insert q and release the hash-bucket only
604 * after testing *uaddr. This guarantees that futex_wait() will NOT
605 * absorb a wakeup if *uaddr does not match the desired values
606 * while the syscall executes.
607 */
608 retry:
609 ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, FUTEX_READ);
610 if (unlikely(ret != 0))
611 return ret;
612
613 retry_private:
614 *hb = futex_q_lock(q);
615
616 ret = futex_get_value_locked(&uval, uaddr);
617
618 if (ret) {
619 futex_q_unlock(*hb);
620
621 ret = get_user(uval, uaddr);
622 if (ret)
623 return ret;
624
625 if (!(flags & FLAGS_SHARED))
626 goto retry_private;
627
628 goto retry;
629 }
630
631 if (uval != val) {
632 futex_q_unlock(*hb);
633 ret = -EWOULDBLOCK;
634 }
635
636 return ret;
637 }
638
futex_wait(u32 __user * uaddr,unsigned int flags,u32 val,ktime_t * abs_time,u32 bitset)639 int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val, ktime_t *abs_time, u32 bitset)
640 {
641 struct hrtimer_sleeper timeout, *to;
642 struct restart_block *restart;
643 struct futex_hash_bucket *hb;
644 struct futex_q q = futex_q_init;
645 int ret;
646
647 if (!bitset)
648 return -EINVAL;
649 q.bitset = bitset;
650 trace_android_vh_futex_wait_start(flags, bitset);
651
652 to = futex_setup_timer(abs_time, &timeout, flags,
653 current->timer_slack_ns);
654 retry:
655 /*
656 * Prepare to wait on uaddr. On success, it holds hb->lock and q
657 * is initialized.
658 */
659 ret = futex_wait_setup(uaddr, val, flags, &q, &hb);
660 if (ret)
661 goto out;
662
663 /* futex_queue and wait for wakeup, timeout, or a signal. */
664 futex_wait_queue(hb, &q, to);
665
666 /* If we were woken (and unqueued), we succeeded, whatever. */
667 ret = 0;
668 if (!futex_unqueue(&q))
669 goto out;
670 ret = -ETIMEDOUT;
671 if (to && !to->task)
672 goto out;
673
674 /*
675 * We expect signal_pending(current), but we might be the
676 * victim of a spurious wakeup as well.
677 */
678 if (!signal_pending(current))
679 goto retry;
680
681 ret = -ERESTARTSYS;
682 if (!abs_time)
683 goto out;
684
685 restart = ¤t->restart_block;
686 restart->futex.uaddr = uaddr;
687 restart->futex.val = val;
688 restart->futex.time = *abs_time;
689 restart->futex.bitset = bitset;
690 restart->futex.flags = flags | FLAGS_HAS_TIMEOUT;
691
692 ret = set_restart_fn(restart, futex_wait_restart);
693
694 out:
695 if (to) {
696 hrtimer_cancel(&to->timer);
697 destroy_hrtimer_on_stack(&to->timer);
698 }
699 trace_android_vh_futex_wait_end(flags, bitset);
700 return ret;
701 }
702
futex_wait_restart(struct restart_block * restart)703 static long futex_wait_restart(struct restart_block *restart)
704 {
705 u32 __user *uaddr = restart->futex.uaddr;
706 ktime_t t, *tp = NULL;
707
708 if (restart->futex.flags & FLAGS_HAS_TIMEOUT) {
709 t = restart->futex.time;
710 tp = &t;
711 }
712 restart->fn = do_no_restart_syscall;
713
714 return (long)futex_wait(uaddr, restart->futex.flags,
715 restart->futex.val, tp, restart->futex.bitset);
716 }
717
718