1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _FUTEX_H
3 #define _FUTEX_H
4
5 #include <linux/futex.h>
6 #include <linux/rtmutex.h>
7 #include <linux/sched/wake_q.h>
8 #include <linux/compat.h>
9
10 #ifdef CONFIG_PREEMPT_RT
11 #include <linux/rcuwait.h>
12 #endif
13
14 #include <asm/futex.h>
15
16 /*
17 * Futex flags used to encode options to functions and preserve them across
18 * restarts.
19 */
20 #define FLAGS_SIZE_8 0x0000
21 #define FLAGS_SIZE_16 0x0001
22 #define FLAGS_SIZE_32 0x0002
23 #define FLAGS_SIZE_64 0x0003
24
25 #define FLAGS_SIZE_MASK 0x0003
26
27 #ifdef CONFIG_MMU
28 # define FLAGS_SHARED 0x0010
29 #else
30 /*
31 * NOMMU does not have per process address space. Let the compiler optimize
32 * code away.
33 */
34 # define FLAGS_SHARED 0x0000
35 #endif
36 #define FLAGS_CLOCKRT 0x0020
37 #define FLAGS_HAS_TIMEOUT 0x0040
38 #define FLAGS_NUMA 0x0080
39 #define FLAGS_STRICT 0x0100
40
41 /* FUTEX_ to FLAGS_ */
futex_to_flags(unsigned int op)42 static inline unsigned int futex_to_flags(unsigned int op)
43 {
44 unsigned int flags = FLAGS_SIZE_32;
45
46 if (!(op & FUTEX_PRIVATE_FLAG))
47 flags |= FLAGS_SHARED;
48
49 if (op & FUTEX_CLOCK_REALTIME)
50 flags |= FLAGS_CLOCKRT;
51
52 return flags;
53 }
54
55 #define FUTEX2_VALID_MASK (FUTEX2_SIZE_MASK | FUTEX2_PRIVATE)
56
57 /* FUTEX2_ to FLAGS_ */
futex2_to_flags(unsigned int flags2)58 static inline unsigned int futex2_to_flags(unsigned int flags2)
59 {
60 unsigned int flags = flags2 & FUTEX2_SIZE_MASK;
61
62 if (!(flags2 & FUTEX2_PRIVATE))
63 flags |= FLAGS_SHARED;
64
65 if (flags2 & FUTEX2_NUMA)
66 flags |= FLAGS_NUMA;
67
68 return flags;
69 }
70
futex_size(unsigned int flags)71 static inline unsigned int futex_size(unsigned int flags)
72 {
73 return 1 << (flags & FLAGS_SIZE_MASK);
74 }
75
futex_flags_valid(unsigned int flags)76 static inline bool futex_flags_valid(unsigned int flags)
77 {
78 /* Only 64bit futexes for 64bit code */
79 if (!IS_ENABLED(CONFIG_64BIT) || in_compat_syscall()) {
80 if ((flags & FLAGS_SIZE_MASK) == FLAGS_SIZE_64)
81 return false;
82 }
83
84 /* Only 32bit futexes are implemented -- for now */
85 if ((flags & FLAGS_SIZE_MASK) != FLAGS_SIZE_32)
86 return false;
87
88 return true;
89 }
90
futex_validate_input(unsigned int flags,u64 val)91 static inline bool futex_validate_input(unsigned int flags, u64 val)
92 {
93 int bits = 8 * futex_size(flags);
94
95 if (bits < 64 && (val >> bits))
96 return false;
97
98 return true;
99 }
100
101 #ifdef CONFIG_FAIL_FUTEX
102 extern bool should_fail_futex(bool fshared);
103 #else
should_fail_futex(bool fshared)104 static inline bool should_fail_futex(bool fshared)
105 {
106 return false;
107 }
108 #endif
109
110 /*
111 * Hash buckets are shared by all the futex_keys that hash to the same
112 * location. Each key may have multiple futex_q structures, one for each task
113 * waiting on a futex.
114 */
115 struct futex_hash_bucket {
116 atomic_t waiters;
117 spinlock_t lock;
118 struct plist_head chain;
119 } ____cacheline_aligned_in_smp;
120
121 /*
122 * Priority Inheritance state:
123 */
124 struct futex_pi_state {
125 /*
126 * list of 'owned' pi_state instances - these have to be
127 * cleaned up in do_exit() if the task exits prematurely:
128 */
129 struct list_head list;
130
131 /*
132 * The PI object:
133 */
134 struct rt_mutex_base pi_mutex;
135
136 struct task_struct *owner;
137 refcount_t refcount;
138
139 union futex_key key;
140 } __randomize_layout;
141
142 struct futex_q;
143 typedef void (futex_wake_fn)(struct wake_q_head *wake_q, struct futex_q *q);
144
145 /**
146 * struct futex_q - The hashed futex queue entry, one per waiting task
147 * @list: priority-sorted list of tasks waiting on this futex
148 * @task: the task waiting on the futex
149 * @lock_ptr: the hash bucket lock
150 * @wake: the wake handler for this queue
151 * @wake_data: data associated with the wake handler
152 * @key: the key the futex is hashed on
153 * @pi_state: optional priority inheritance state
154 * @rt_waiter: rt_waiter storage for use with requeue_pi
155 * @requeue_pi_key: the requeue_pi target futex key
156 * @bitset: bitset for the optional bitmasked wakeup
157 * @requeue_state: State field for futex_requeue_pi()
158 * @requeue_wait: RCU wait for futex_requeue_pi() (RT only)
159 *
160 * We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so
161 * we can wake only the relevant ones (hashed queues may be shared).
162 *
163 * A futex_q has a woken state, just like tasks have TASK_RUNNING.
164 * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0.
165 * The order of wakeup is always to make the first condition true, then
166 * the second.
167 *
168 * PI futexes are typically woken before they are removed from the hash list via
169 * the rt_mutex code. See futex_unqueue_pi().
170 */
171 struct futex_q {
172 struct plist_node list;
173
174 struct task_struct *task;
175 spinlock_t *lock_ptr;
176 futex_wake_fn *wake;
177 void *wake_data;
178 union futex_key key;
179 struct futex_pi_state *pi_state;
180 struct rt_mutex_waiter *rt_waiter;
181 union futex_key *requeue_pi_key;
182 u32 bitset;
183 atomic_t requeue_state;
184 #ifdef CONFIG_PREEMPT_RT
185 struct rcuwait requeue_wait;
186 #endif
187 } __randomize_layout;
188
189 extern const struct futex_q futex_q_init;
190
191 enum futex_access {
192 FUTEX_READ,
193 FUTEX_WRITE
194 };
195
196 extern int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key,
197 enum futex_access rw);
198
199 extern struct hrtimer_sleeper *
200 futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout,
201 int flags, u64 range_ns);
202
203 extern struct futex_hash_bucket *futex_hash(union futex_key *key);
204
205 /**
206 * futex_match - Check whether two futex keys are equal
207 * @key1: Pointer to key1
208 * @key2: Pointer to key2
209 *
210 * Return 1 if two futex_keys are equal, 0 otherwise.
211 */
futex_match(union futex_key * key1,union futex_key * key2)212 static inline int futex_match(union futex_key *key1, union futex_key *key2)
213 {
214 return (key1 && key2
215 && key1->both.word == key2->both.word
216 && key1->both.ptr == key2->both.ptr
217 && key1->both.offset == key2->both.offset);
218 }
219
220 extern int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
221 struct futex_q *q, struct futex_hash_bucket **hb);
222 extern void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q,
223 struct hrtimer_sleeper *timeout);
224 extern bool __futex_wake_mark(struct futex_q *q);
225 extern void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q);
226
227 extern int fault_in_user_writeable(u32 __user *uaddr);
228 extern int futex_cmpxchg_value_locked(u32 *curval, u32 __user *uaddr, u32 uval, u32 newval);
229 extern int futex_get_value_locked(u32 *dest, u32 __user *from);
230 extern struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb, union futex_key *key);
231
232 extern void __futex_unqueue(struct futex_q *q);
233 extern void __futex_queue(struct futex_q *q, struct futex_hash_bucket *hb,
234 struct task_struct *task);
235 extern int futex_unqueue(struct futex_q *q);
236
237 /**
238 * futex_queue() - Enqueue the futex_q on the futex_hash_bucket
239 * @q: The futex_q to enqueue
240 * @hb: The destination hash bucket
241 * @task: Task queueing this futex
242 *
243 * The hb->lock must be held by the caller, and is released here. A call to
244 * futex_queue() is typically paired with exactly one call to futex_unqueue(). The
245 * exceptions involve the PI related operations, which may use futex_unqueue_pi()
246 * or nothing if the unqueue is done as part of the wake process and the unqueue
247 * state is implicit in the state of woken task (see futex_wait_requeue_pi() for
248 * an example).
249 *
250 * Note that @task may be NULL, for async usage of futexes.
251 */
futex_queue(struct futex_q * q,struct futex_hash_bucket * hb,struct task_struct * task)252 static inline void futex_queue(struct futex_q *q, struct futex_hash_bucket *hb,
253 struct task_struct *task)
254 __releases(&hb->lock)
255 {
256 __futex_queue(q, hb, task);
257 spin_unlock(&hb->lock);
258 }
259
260 extern void futex_unqueue_pi(struct futex_q *q);
261
262 extern void wait_for_owner_exiting(int ret, struct task_struct *exiting);
263
264 /*
265 * Reflects a new waiter being added to the waitqueue.
266 */
futex_hb_waiters_inc(struct futex_hash_bucket * hb)267 static inline void futex_hb_waiters_inc(struct futex_hash_bucket *hb)
268 {
269 #ifdef CONFIG_SMP
270 atomic_inc(&hb->waiters);
271 /*
272 * Full barrier (A), see the ordering comment above.
273 */
274 smp_mb__after_atomic();
275 #endif
276 }
277
278 /*
279 * Reflects a waiter being removed from the waitqueue by wakeup
280 * paths.
281 */
futex_hb_waiters_dec(struct futex_hash_bucket * hb)282 static inline void futex_hb_waiters_dec(struct futex_hash_bucket *hb)
283 {
284 #ifdef CONFIG_SMP
285 atomic_dec(&hb->waiters);
286 #endif
287 }
288
futex_hb_waiters_pending(struct futex_hash_bucket * hb)289 static inline int futex_hb_waiters_pending(struct futex_hash_bucket *hb)
290 {
291 #ifdef CONFIG_SMP
292 /*
293 * Full barrier (B), see the ordering comment above.
294 */
295 smp_mb();
296 return atomic_read(&hb->waiters);
297 #else
298 return 1;
299 #endif
300 }
301
302 extern struct futex_hash_bucket *futex_q_lock(struct futex_q *q);
303 extern void futex_q_unlock(struct futex_hash_bucket *hb);
304
305
306 extern int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,
307 union futex_key *key,
308 struct futex_pi_state **ps,
309 struct task_struct *task,
310 struct task_struct **exiting,
311 int set_waiters);
312
313 extern int refill_pi_state_cache(void);
314 extern void get_pi_state(struct futex_pi_state *pi_state);
315 extern void put_pi_state(struct futex_pi_state *pi_state);
316 extern int fixup_pi_owner(u32 __user *uaddr, struct futex_q *q, int locked);
317
318 /*
319 * Express the locking dependencies for lockdep:
320 */
321 static inline void
double_lock_hb(struct futex_hash_bucket * hb1,struct futex_hash_bucket * hb2)322 double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
323 {
324 if (hb1 > hb2)
325 swap(hb1, hb2);
326
327 spin_lock(&hb1->lock);
328 if (hb1 != hb2)
329 spin_lock_nested(&hb2->lock, SINGLE_DEPTH_NESTING);
330 }
331
332 static inline void
double_unlock_hb(struct futex_hash_bucket * hb1,struct futex_hash_bucket * hb2)333 double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
334 {
335 spin_unlock(&hb1->lock);
336 if (hb1 != hb2)
337 spin_unlock(&hb2->lock);
338 }
339
340 /* syscalls */
341
342 extern int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, u32
343 val, ktime_t *abs_time, u32 bitset, u32 __user
344 *uaddr2);
345
346 extern int futex_requeue(u32 __user *uaddr1, unsigned int flags1,
347 u32 __user *uaddr2, unsigned int flags2,
348 int nr_wake, int nr_requeue,
349 u32 *cmpval, int requeue_pi);
350
351 extern int __futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
352 struct hrtimer_sleeper *to, u32 bitset);
353
354 extern int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
355 ktime_t *abs_time, u32 bitset);
356
357 /**
358 * struct futex_vector - Auxiliary struct for futex_waitv()
359 * @w: Userspace provided data
360 * @q: Kernel side data
361 *
362 * Struct used to build an array with all data need for futex_waitv()
363 */
364 struct futex_vector {
365 struct futex_waitv w;
366 struct futex_q q;
367 };
368
369 extern int futex_parse_waitv(struct futex_vector *futexv,
370 struct futex_waitv __user *uwaitv,
371 unsigned int nr_futexes, futex_wake_fn *wake,
372 void *wake_data);
373
374 extern int futex_wait_multiple_setup(struct futex_vector *vs, int count,
375 int *woken);
376
377 extern int futex_unqueue_multiple(struct futex_vector *v, int count);
378
379 extern int futex_wait_multiple(struct futex_vector *vs, unsigned int count,
380 struct hrtimer_sleeper *to);
381
382 extern int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset);
383
384 extern int futex_wake_op(u32 __user *uaddr1, unsigned int flags,
385 u32 __user *uaddr2, int nr_wake, int nr_wake2, int op);
386
387 extern int futex_unlock_pi(u32 __user *uaddr, unsigned int flags);
388
389 extern int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int trylock);
390
391 #endif /* _FUTEX_H */
392