1 /* SPDX-License-Identifier: MIT */
2 /*
3 * Description: exercise futex wait/wake/waitv
4 *
5 */
6 #include <stdio.h>
7 #include <unistd.h>
8 #include <stdlib.h>
9 #include <pthread.h>
10 #include <errno.h>
11 #include <linux/futex.h>
12
13 #include "liburing.h"
14 #include "helpers.h"
15
16 #define LOOPS 500
17 #define NFUTEX 8
18
19 #ifndef FUTEX2_SIZE_U8
20 #define FUTEX2_SIZE_U8 0x00
21 #define FUTEX2_SIZE_U16 0x01
22 #define FUTEX2_SIZE_U32 0x02
23 #define FUTEX2_SIZE_U64 0x03
24 #define FUTEX2_NUMA 0x04
25 /* 0x08 */
26 /* 0x10 */
27 /* 0x20 */
28 /* 0x40 */
29 #define FUTEX2_PRIVATE FUTEX_PRIVATE_FLAG
30
31 #define FUTEX2_SIZE_MASK 0x03
32 #endif
33
34 static int no_futex;
35
fwake(void * data)36 static void *fwake(void *data)
37 {
38 unsigned int *futex = data;
39 struct io_uring_sqe *sqe;
40 struct io_uring_cqe *cqe;
41 struct io_uring ring;
42 int ret;
43
44 ret = io_uring_queue_init(1, &ring, 0);
45 if (ret) {
46 fprintf(stderr, "queue init: %d\n", ret);
47 return NULL;
48 }
49
50 *futex = 1;
51 sqe = io_uring_get_sqe(&ring);
52 io_uring_prep_futex_wake(sqe, futex, 1, FUTEX_BITSET_MATCH_ANY,
53 FUTEX2_SIZE_U32, 0);
54 sqe->user_data = 3;
55
56 io_uring_submit(&ring);
57
58 ret = io_uring_wait_cqe(&ring, &cqe);
59 if (ret) {
60 fprintf(stderr, "wait: %d\n", ret);
61 return NULL;
62 }
63 io_uring_cqe_seen(&ring, cqe);
64 io_uring_queue_exit(&ring);
65 return NULL;
66 }
67
__test(struct io_uring * ring,int vectored,int async,int async_cancel)68 static int __test(struct io_uring *ring, int vectored, int async,
69 int async_cancel)
70 {
71 struct io_uring_sqe *sqe;
72 struct io_uring_cqe *cqe;
73 struct futex_waitv fw[NFUTEX];
74 unsigned int *futex;
75 pthread_t threads[NFUTEX];
76 void *tret;
77 int ret, i, nfutex;
78
79 nfutex = NFUTEX;
80 if (!vectored)
81 nfutex = 1;
82
83 futex = calloc(nfutex, sizeof(*futex));
84 for (i = 0; i < nfutex; i++) {
85 fw[i].val = 0;
86 fw[i].uaddr = (unsigned long) &futex[i];
87 fw[i].flags = FUTEX2_SIZE_U32;
88 fw[i].__reserved = 0;
89 }
90
91 sqe = io_uring_get_sqe(ring);
92 if (vectored)
93 io_uring_prep_futex_waitv(sqe, fw, nfutex, 0);
94 else
95 io_uring_prep_futex_wait(sqe, futex, 0, FUTEX_BITSET_MATCH_ANY,
96 FUTEX2_SIZE_U32, 0);
97 if (async)
98 sqe->flags |= IOSQE_ASYNC;
99 sqe->user_data = 1;
100
101 io_uring_submit(ring);
102
103 for (i = 0; i < nfutex; i++)
104 pthread_create(&threads[i], NULL, fwake, &futex[i]);
105
106 sqe = io_uring_get_sqe(ring);
107 io_uring_prep_cancel64(sqe, 1, 0);
108 if (async_cancel)
109 sqe->flags |= IOSQE_ASYNC;
110 sqe->user_data = 2;
111
112 io_uring_submit(ring);
113
114 for (i = 0; i < 2; i++) {
115 ret = io_uring_wait_cqe(ring, &cqe);
116 if (ret) {
117 fprintf(stderr, "parent wait %d\n", ret);
118 return 1;
119 }
120
121 if (cqe->res == -EINVAL || cqe->res == -EOPNOTSUPP) {
122 no_futex = 1;
123 return 0;
124 }
125 io_uring_cqe_seen(ring, cqe);
126 }
127
128 ret = io_uring_peek_cqe(ring, &cqe);
129 if (!ret) {
130 fprintf(stderr, "peek found cqe!\n");
131 return 1;
132 }
133
134 for (i = 0; i < nfutex; i++)
135 pthread_join(threads[i], &tret);
136
137 return 0;
138 }
139
test(int flags,int vectored)140 static int test(int flags, int vectored)
141 {
142 struct io_uring ring;
143 int ret, i;
144
145 ret = io_uring_queue_init(8, &ring, flags);
146 if (ret)
147 return ret;
148
149 for (i = 0; i < LOOPS; i++) {
150 int async_cancel = (!i % 2);
151 int async_wait = !(i % 3);
152 ret = __test(&ring, vectored, async_wait, async_cancel);
153 if (ret) {
154 fprintf(stderr, "flags=%x, failed=%d\n", flags, i);
155 break;
156 }
157 if (no_futex)
158 break;
159 }
160
161 io_uring_queue_exit(&ring);
162 return ret;
163 }
164
test_order(int vectored,int async)165 static int test_order(int vectored, int async)
166 {
167 struct io_uring_sqe *sqe;
168 struct io_uring_cqe *cqe;
169 struct futex_waitv fw;
170 struct io_uring ring;
171 unsigned int *futex;
172 int ret, i;
173
174 ret = io_uring_queue_init(8, &ring, 0);
175 if (ret)
176 return ret;
177
178 futex = malloc(sizeof(*futex));
179 *futex = 0;
180
181 fw.val = 0;
182 fw.uaddr = (unsigned long) futex;
183 fw.flags = FUTEX2_SIZE_U32;
184 fw.__reserved = 0;
185
186 /*
187 * Submit two futex waits
188 */
189 sqe = io_uring_get_sqe(&ring);
190 if (!vectored)
191 io_uring_prep_futex_wait(sqe, futex, 0, FUTEX_BITSET_MATCH_ANY,
192 FUTEX2_SIZE_U32, 0);
193 else
194 io_uring_prep_futex_waitv(sqe, &fw, 1, 0);
195 sqe->user_data = 1;
196
197 sqe = io_uring_get_sqe(&ring);
198 if (!vectored)
199 io_uring_prep_futex_wait(sqe, futex, 0, FUTEX_BITSET_MATCH_ANY,
200 FUTEX2_SIZE_U32, 0);
201 else
202 io_uring_prep_futex_waitv(sqe, &fw, 1, 0);
203 sqe->user_data = 2;
204
205 io_uring_submit(&ring);
206
207 /*
208 * Now submit wake for just one futex
209 */
210 *futex = 1;
211 sqe = io_uring_get_sqe(&ring);
212 io_uring_prep_futex_wake(sqe, futex, 1, FUTEX_BITSET_MATCH_ANY,
213 FUTEX2_SIZE_U32, 0);
214 sqe->user_data = 100;
215 if (async)
216 sqe->flags |= IOSQE_ASYNC;
217
218 io_uring_submit(&ring);
219
220 /*
221 * We expect to find completions for the first futex wait, and
222 * the futex wake. We should not see the last futex wait.
223 */
224 for (i = 0; i < 2; i++) {
225 ret = io_uring_wait_cqe(&ring, &cqe);
226 if (ret) {
227 fprintf(stderr, "wait %d\n", ret);
228 return 1;
229 }
230 if (cqe->user_data == 1 || cqe->user_data == 100) {
231 io_uring_cqe_seen(&ring, cqe);
232 continue;
233 }
234 fprintf(stderr, "unexpected cqe %lu, res %d\n", (unsigned long) cqe->user_data, cqe->res);
235 return 1;
236 }
237
238 ret = io_uring_peek_cqe(&ring, &cqe);
239 if (ret != -EAGAIN) {
240 fprintf(stderr, "Unexpected cqe available: %d\n", cqe->res);
241 return 1;
242 }
243
244 io_uring_queue_exit(&ring);
245 return 0;
246 }
247
test_multi_wake(int vectored)248 static int test_multi_wake(int vectored)
249 {
250 struct io_uring_sqe *sqe;
251 struct io_uring_cqe *cqe;
252 struct futex_waitv fw;
253 struct io_uring ring;
254 unsigned int *futex;
255 int ret, i;
256
257 ret = io_uring_queue_init(8, &ring, 0);
258 if (ret)
259 return ret;
260
261 futex = malloc(sizeof(*futex));
262 *futex = 0;
263
264 fw.val = 0;
265 fw.uaddr = (unsigned long) futex;
266 fw.flags = FUTEX2_SIZE_U32;
267 fw.__reserved = 0;
268
269 /*
270 * Submit two futex waits
271 */
272 sqe = io_uring_get_sqe(&ring);
273 if (!vectored)
274 io_uring_prep_futex_wait(sqe, futex, 0, FUTEX_BITSET_MATCH_ANY,
275 FUTEX2_SIZE_U32, 0);
276 else
277 io_uring_prep_futex_waitv(sqe, &fw, 1, 0);
278 sqe->user_data = 1;
279
280 sqe = io_uring_get_sqe(&ring);
281 if (!vectored)
282 io_uring_prep_futex_wait(sqe, futex, 0, FUTEX_BITSET_MATCH_ANY,
283 FUTEX2_SIZE_U32, 0);
284 else
285 io_uring_prep_futex_waitv(sqe, &fw, 1, 0);
286 sqe->user_data = 2;
287
288 io_uring_submit(&ring);
289
290 /*
291 * Now submit wake for both futexes
292 */
293 *futex = 1;
294 sqe = io_uring_get_sqe(&ring);
295 io_uring_prep_futex_wake(sqe, futex, 2, FUTEX_BITSET_MATCH_ANY,
296 FUTEX2_SIZE_U32, 0);
297 sqe->user_data = 100;
298
299 io_uring_submit(&ring);
300
301 /*
302 * We expect to find completions for the both futex waits, and
303 * the futex wake.
304 */
305 for (i = 0; i < 3; i++) {
306 ret = io_uring_wait_cqe(&ring, &cqe);
307 if (ret) {
308 fprintf(stderr, "wait %d\n", ret);
309 return 1;
310 }
311 if (cqe->res < 0) {
312 fprintf(stderr, "cqe error %d\n", cqe->res);
313 return 1;
314 }
315 io_uring_cqe_seen(&ring, cqe);
316 }
317
318 ret = io_uring_peek_cqe(&ring, &cqe);
319 if (!ret) {
320 fprintf(stderr, "peek found cqe!\n");
321 return 1;
322 }
323
324 io_uring_queue_exit(&ring);
325 return 0;
326 }
327
328 /*
329 * Test that waking 0 futexes returns 0
330 */
test_wake_zero(void)331 static int test_wake_zero(void)
332 {
333 struct io_uring_sqe *sqe;
334 struct io_uring_cqe *cqe;
335 struct io_uring ring;
336 unsigned int *futex;
337 int ret;
338
339 ret = io_uring_queue_init(8, &ring, 0);
340 if (ret)
341 return ret;
342
343 futex = malloc(sizeof(*futex));
344 *futex = 0;
345
346 sqe = io_uring_get_sqe(&ring);
347 sqe->user_data = 1;
348 io_uring_prep_futex_wait(sqe, futex, 0, FUTEX_BITSET_MATCH_ANY,
349 FUTEX2_SIZE_U32, 0);
350
351 io_uring_submit(&ring);
352
353 sqe = io_uring_get_sqe(&ring);
354 sqe->user_data = 2;
355 io_uring_prep_futex_wake(sqe, futex, 0, FUTEX_BITSET_MATCH_ANY,
356 FUTEX2_SIZE_U32, 0);
357
358 io_uring_submit(&ring);
359
360 ret = io_uring_wait_cqe(&ring, &cqe);
361
362 /*
363 * Should get zero res and it should be the wake
364 */
365 if (cqe->res || cqe->user_data != 2) {
366 fprintf(stderr, "cqe res %d, data %ld\n", cqe->res, (long) cqe->user_data);
367 return 1;
368 }
369 io_uring_cqe_seen(&ring, cqe);
370
371 /*
372 * Should not have the wait complete
373 */
374 ret = io_uring_peek_cqe(&ring, &cqe);
375 if (!ret) {
376 fprintf(stderr, "peek found cqe!\n");
377 return 1;
378 }
379
380 io_uring_queue_exit(&ring);
381 return 0;
382 }
383
384 /*
385 * Test invalid wait/wake/waitv flags
386 */
test_invalid(void)387 static int test_invalid(void)
388 {
389 struct io_uring_sqe *sqe;
390 struct io_uring_cqe *cqe;
391 struct futex_waitv fw;
392 struct io_uring ring;
393 unsigned int *futex;
394 int ret;
395
396 ret = io_uring_queue_init(8, &ring, 0);
397 if (ret)
398 return ret;
399
400 futex = malloc(sizeof(*futex));
401 *futex = 0;
402
403 sqe = io_uring_get_sqe(&ring);
404 sqe->user_data = 1;
405 io_uring_prep_futex_wait(sqe, futex, 0, FUTEX_BITSET_MATCH_ANY, 0x1000,
406 0);
407
408 io_uring_submit(&ring);
409
410 ret = io_uring_wait_cqe(&ring, &cqe);
411
412 /*
413 * Should get zero res and it should be the wake
414 */
415 if (cqe->res != -EINVAL) {
416 fprintf(stderr, "wait cqe res %d\n", cqe->res);
417 return 1;
418 }
419 io_uring_cqe_seen(&ring, cqe);
420
421 sqe = io_uring_get_sqe(&ring);
422 sqe->user_data = 1;
423 io_uring_prep_futex_wake(sqe, futex, 0, FUTEX_BITSET_MATCH_ANY, 0x1000,
424 0);
425
426 io_uring_submit(&ring);
427
428 ret = io_uring_wait_cqe(&ring, &cqe);
429
430 /*
431 * Should get zero res and it should be the wake
432 */
433 if (cqe->res != -EINVAL) {
434 fprintf(stderr, "wake cqe res %d\n", cqe->res);
435 return 1;
436 }
437 io_uring_cqe_seen(&ring, cqe);
438
439 fw.val = 0;
440 fw.uaddr = (unsigned long) futex;
441 fw.flags = FUTEX2_SIZE_U32 | 0x1000;
442 fw.__reserved = 0;
443
444 sqe = io_uring_get_sqe(&ring);
445 sqe->user_data = 1;
446 io_uring_prep_futex_waitv(sqe, &fw, 1, 0);
447
448 io_uring_submit(&ring);
449
450 ret = io_uring_wait_cqe(&ring, &cqe);
451
452 /*
453 * Should get zero res and it should be the wake
454 */
455 if (cqe->res != -EINVAL) {
456 fprintf(stderr, "waitv cqe res %d\n", cqe->res);
457 return 1;
458 }
459 io_uring_cqe_seen(&ring, cqe);
460
461 io_uring_queue_exit(&ring);
462 return 0;
463 }
464
main(int argc,char * argv[])465 int main(int argc, char *argv[])
466 {
467 int ret;
468
469 if (argc > 1)
470 return T_EXIT_SKIP;
471
472 ret = test(0, 0);
473 if (ret) {
474 fprintf(stderr, "test 0 0 failed\n");
475 return T_EXIT_FAIL;
476 }
477 if (no_futex)
478 return T_EXIT_SKIP;
479
480 ret = test(0, 1);
481 if (ret) {
482 fprintf(stderr, "test 0 1 failed\n");
483 return T_EXIT_FAIL;
484 }
485
486 ret = test_wake_zero();
487 if (ret) {
488 fprintf(stderr, "wake 0 failed\n");
489 return T_EXIT_FAIL;
490 }
491
492 ret = test_invalid();
493 if (ret) {
494 fprintf(stderr, "test invalid failed\n");
495 return T_EXIT_FAIL;
496 }
497
498 ret = test(IORING_SETUP_SQPOLL, 0);
499 if (ret) {
500 fprintf(stderr, "test sqpoll 0 failed\n");
501 return T_EXIT_FAIL;
502 }
503
504 ret = test(IORING_SETUP_SQPOLL, 1);
505 if (ret) {
506 fprintf(stderr, "test sqpoll 1 failed\n");
507 return T_EXIT_FAIL;
508 }
509
510 ret = test(IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN, 0);
511 if (ret) {
512 fprintf(stderr, "test single coop 0 failed\n");
513 return T_EXIT_FAIL;
514 }
515
516 ret = test(IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN, 1);
517 if (ret) {
518 fprintf(stderr, "test single coop 1 failed\n");
519 return T_EXIT_FAIL;
520 }
521
522 ret = test(IORING_SETUP_COOP_TASKRUN, 0);
523 if (ret) {
524 fprintf(stderr, "test taskrun 0 failed\n");
525 return T_EXIT_FAIL;
526 }
527
528 ret = test(IORING_SETUP_COOP_TASKRUN, 1);
529 if (ret) {
530 fprintf(stderr, "test taskrun 1 failed\n");
531 return T_EXIT_FAIL;
532 }
533
534 ret = test_order(0, 0);
535 if (ret) {
536 fprintf(stderr, "test_order 0 0 failed\n");
537 return T_EXIT_FAIL;
538 }
539
540 ret = test_order(1, 0);
541 if (ret) {
542 fprintf(stderr, "test_order 1 0 failed\n");
543 return T_EXIT_FAIL;
544 }
545
546 ret = test_order(0, 1);
547 if (ret) {
548 fprintf(stderr, "test_order 0 1 failed\n");
549 return T_EXIT_FAIL;
550 }
551
552 ret = test_order(1, 1);
553 if (ret) {
554 fprintf(stderr, "test_order 1 1 failed\n");
555 return T_EXIT_FAIL;
556 }
557
558 ret = test_multi_wake(0);
559 if (ret) {
560 fprintf(stderr, "multi_wake 0 failed\n");
561 return T_EXIT_FAIL;
562 }
563
564 ret = test_multi_wake(1);
565 if (ret) {
566 fprintf(stderr, "multi_wake 1 failed\n");
567 return T_EXIT_FAIL;
568 }
569
570 return T_EXIT_PASS;
571 }
572