1 /* SPDX-License-Identifier: MIT */
2 /*
3 * Description: run various shared buffer ring sanity checks
4 *
5 */
6 #include <errno.h>
7 #include <stdio.h>
8 #include <unistd.h>
9 #include <stdlib.h>
10 #include <string.h>
11 #include <fcntl.h>
12 #include <sys/mman.h>
13
14 #include "liburing.h"
15 #include "helpers.h"
16
17 static int no_buf_ring;
18 static int pagesize;
19
20 /* test trying to register classic group when ring group exists */
test_mixed_reg2(int bgid)21 static int test_mixed_reg2(int bgid)
22 {
23 struct io_uring_buf_ring *br;
24 struct io_uring_sqe *sqe;
25 struct io_uring_cqe *cqe;
26 struct io_uring ring;
27 void *bufs;
28 int ret;
29
30 ret = t_create_ring(1, &ring, 0);
31 if (ret == T_SETUP_SKIP)
32 return 0;
33 else if (ret != T_SETUP_OK)
34 return 1;
35
36 br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret);
37 if (!br) {
38 fprintf(stderr, "Buffer ring register failed %d\n", ret);
39 return 1;
40 }
41
42 /* provide classic buffers, group 1 */
43 bufs = malloc(8 * 1024);
44 sqe = io_uring_get_sqe(&ring);
45 io_uring_prep_provide_buffers(sqe, bufs, 1024, 8, bgid, 0);
46 io_uring_submit(&ring);
47 ret = io_uring_wait_cqe(&ring, &cqe);
48 if (ret) {
49 fprintf(stderr, "wait_cqe %d\n", ret);
50 return 1;
51 }
52 if (cqe->res != -EEXIST && cqe->res != -EINVAL) {
53 fprintf(stderr, "cqe res %d\n", cqe->res);
54 return 1;
55 }
56 io_uring_cqe_seen(&ring, cqe);
57
58 io_uring_free_buf_ring(&ring, br, 32, bgid);
59 io_uring_queue_exit(&ring);
60 free(bufs);
61 return 0;
62 }
63
64 /* test trying to register ring group when classic group exists */
test_mixed_reg(int bgid)65 static int test_mixed_reg(int bgid)
66 {
67 struct io_uring_buf_ring *br;
68 struct io_uring_sqe *sqe;
69 struct io_uring_cqe *cqe;
70 struct io_uring ring;
71 void *bufs;
72 int ret;
73
74 ret = t_create_ring(1, &ring, 0);
75 if (ret == T_SETUP_SKIP)
76 return 0;
77 else if (ret != T_SETUP_OK)
78 return 1;
79
80 /* provide classic buffers, group 1 */
81 bufs = malloc(8 * 1024);
82 sqe = io_uring_get_sqe(&ring);
83 io_uring_prep_provide_buffers(sqe, bufs, 1024, 8, bgid, 0);
84 io_uring_submit(&ring);
85 ret = io_uring_wait_cqe(&ring, &cqe);
86 if (ret) {
87 fprintf(stderr, "wait_cqe %d\n", ret);
88 return 1;
89 }
90 if (cqe->res) {
91 fprintf(stderr, "cqe res %d\n", cqe->res);
92 return 1;
93 }
94 io_uring_cqe_seen(&ring, cqe);
95
96 br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret);
97 if (br) {
98 fprintf(stderr, "Buffer ring setup succeeded unexpectedly %d\n", ret);
99 return 1;
100 }
101
102 io_uring_queue_exit(&ring);
103 free(bufs);
104 return 0;
105 }
106
test_double_reg_unreg(int bgid)107 static int test_double_reg_unreg(int bgid)
108 {
109 struct io_uring_buf_reg reg = { };
110 struct io_uring_buf_ring *br;
111 struct io_uring ring;
112 int ret;
113
114 ret = t_create_ring(1, &ring, 0);
115 if (ret == T_SETUP_SKIP)
116 return 0;
117 else if (ret != T_SETUP_OK)
118 return 1;
119
120 br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret);
121 if (!br) {
122 fprintf(stderr, "Buffer ring register failed %d\n", ret);
123 return 1;
124 }
125
126 /* check that 2nd register with same bgid fails */
127 reg.ring_addr = (unsigned long) br;
128 reg.ring_entries = 32;
129 reg.bgid = bgid;
130
131 ret = io_uring_register_buf_ring(&ring, ®, 0);
132 if (ret != -EEXIST) {
133 fprintf(stderr, "Buffer ring register failed %d\n", ret);
134 return 1;
135 }
136
137 ret = io_uring_free_buf_ring(&ring, br, 32, bgid);
138 if (ret) {
139 fprintf(stderr, "Buffer ring register failed %d\n", ret);
140 return 1;
141 }
142
143 ret = io_uring_unregister_buf_ring(&ring, bgid);
144 if (ret != -EINVAL && ret != -ENOENT) {
145 fprintf(stderr, "Buffer ring register failed %d\n", ret);
146 return 1;
147 }
148
149 io_uring_queue_exit(&ring);
150 return 0;
151 }
152
test_reg_unreg(int bgid)153 static int test_reg_unreg(int bgid)
154 {
155 struct io_uring_buf_ring *br;
156 struct io_uring ring;
157 int ret;
158
159 ret = t_create_ring(1, &ring, 0);
160 if (ret == T_SETUP_SKIP)
161 return 0;
162 else if (ret != T_SETUP_OK)
163 return 1;
164
165 br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret);
166 if (!br) {
167 if (ret == -EINVAL) {
168 no_buf_ring = 1;
169 return 0;
170 }
171 fprintf(stderr, "Buffer ring register failed %d\n", ret);
172 return 1;
173 }
174
175 ret = io_uring_free_buf_ring(&ring, br, 32, bgid);
176 if (ret) {
177 fprintf(stderr, "Buffer ring unregister failed %d\n", ret);
178 return 1;
179 }
180
181 io_uring_queue_exit(&ring);
182 return 0;
183 }
184
test_bad_reg(int bgid)185 static int test_bad_reg(int bgid)
186 {
187 struct io_uring ring;
188 int ret;
189 struct io_uring_buf_reg reg = { };
190
191 ret = t_create_ring(1, &ring, 0);
192 if (ret == T_SETUP_SKIP)
193 return 0;
194 else if (ret != T_SETUP_OK)
195 return 1;
196
197 reg.ring_addr = 4096;
198 reg.ring_entries = 32;
199 reg.bgid = bgid;
200
201 ret = io_uring_register_buf_ring(&ring, ®, 0);
202 if (!ret)
203 fprintf(stderr, "Buffer ring register worked unexpectedly\n");
204
205 io_uring_queue_exit(&ring);
206 return !ret;
207 }
208
test_full_page_reg(int bgid)209 static int test_full_page_reg(int bgid)
210 {
211 #if defined(__hppa__)
212 return T_EXIT_SKIP;
213 #else
214 struct io_uring ring;
215 int ret;
216 void *ptr;
217 struct io_uring_buf_reg reg = { };
218 int entries = pagesize / sizeof(struct io_uring_buf);
219
220 ret = io_uring_queue_init(1, &ring, 0);
221 if (ret) {
222 fprintf(stderr, "queue init failed %d\n", ret);
223 return T_EXIT_FAIL;
224 }
225
226 ret = posix_memalign(&ptr, pagesize, pagesize * 2);
227 if (ret) {
228 fprintf(stderr, "posix_memalign failed %d\n", ret);
229 goto err;
230 }
231
232 ret = mprotect(ptr + pagesize, pagesize, PROT_NONE);
233 if (ret) {
234 fprintf(stderr, "mprotect failed %d\n", errno);
235 goto err1;
236 }
237
238 reg.ring_addr = (unsigned long) ptr;
239 reg.ring_entries = entries;
240 reg.bgid = bgid;
241
242 ret = io_uring_register_buf_ring(&ring, ®, 0);
243 if (ret)
244 fprintf(stderr, "register buf ring failed %d\n", ret);
245
246 if (mprotect(ptr + pagesize, pagesize, PROT_READ | PROT_WRITE))
247 fprintf(stderr, "reverting mprotect failed %d\n", errno);
248
249 err1:
250 free(ptr);
251 err:
252 io_uring_queue_exit(&ring);
253 return ret ? T_EXIT_FAIL : T_EXIT_PASS;
254 #endif
255 }
256
test_one_read(int fd,int bgid,struct io_uring * ring)257 static int test_one_read(int fd, int bgid, struct io_uring *ring)
258 {
259 int ret;
260 struct io_uring_cqe *cqe;
261 struct io_uring_sqe *sqe;
262
263 sqe = io_uring_get_sqe(ring);
264 if (!sqe) {
265 fprintf(stderr, "get sqe failed\n");
266 return -1;
267 }
268
269 io_uring_prep_read(sqe, fd, NULL, 1, 0);
270 sqe->flags |= IOSQE_BUFFER_SELECT;
271 sqe->buf_group = bgid;
272 ret = io_uring_submit(ring);
273 if (ret <= 0) {
274 fprintf(stderr, "sqe submit failed: %d\n", ret);
275 return -1;
276 }
277
278 ret = io_uring_wait_cqe(ring, &cqe);
279 if (ret < 0) {
280 fprintf(stderr, "wait completion %d\n", ret);
281 return -1;
282 }
283 ret = cqe->res;
284 io_uring_cqe_seen(ring, cqe);
285
286 if (ret == -ENOBUFS)
287 return ret;
288
289 if (ret != 1) {
290 fprintf(stderr, "read result %d\n", ret);
291 return -1;
292 }
293
294 return cqe->flags >> 16;
295 }
296
test_running(int bgid,int entries,int loops,int use_mmap)297 static int test_running(int bgid, int entries, int loops, int use_mmap)
298 {
299 int ring_mask = io_uring_buf_ring_mask(entries);
300 struct io_uring_buf_ring *br;
301 int ret, loop, idx, read_fd;
302 struct io_uring ring;
303 char buffer[8];
304 bool *buffers;
305
306 ret = t_create_ring(1, &ring, 0);
307 if (ret == T_SETUP_SKIP)
308 return T_EXIT_SKIP;
309 else if (ret != T_SETUP_OK)
310 return T_EXIT_FAIL;
311
312 if (!use_mmap) {
313 br = io_uring_setup_buf_ring(&ring, entries, bgid, 0, &ret);
314 if (!br) {
315 /* by now should have checked if this is supported or not */
316 fprintf(stderr, "Buffer ring register failed %d\n", ret);
317 return T_EXIT_FAIL;
318 }
319 } else {
320 struct io_uring_buf_reg reg = {
321 .ring_entries = entries,
322 .bgid = bgid,
323 .flags = IOU_PBUF_RING_MMAP,
324 };
325 size_t ring_size;
326 off_t off;
327
328 ret = io_uring_register_buf_ring(&ring, ®, 0);
329 if (ret) {
330 if (ret == -EINVAL)
331 return T_EXIT_SKIP;
332 fprintf(stderr, "mmap ring register failed %d\n", ret);
333 return T_EXIT_FAIL;
334 }
335
336 off = IORING_OFF_PBUF_RING |
337 (unsigned long long) bgid << IORING_OFF_PBUF_SHIFT;
338 ring_size = sizeof(struct io_uring_buf) * entries;
339 br = mmap(NULL, ring_size, PROT_READ | PROT_WRITE,
340 MAP_SHARED | MAP_POPULATE, ring.ring_fd, off);
341 if (br == MAP_FAILED) {
342 perror("mmap");
343 return T_EXIT_FAIL;
344 }
345 }
346
347 buffers = malloc(sizeof(bool) * entries);
348 if (!buffers)
349 return T_EXIT_SKIP;
350
351 read_fd = open("/dev/zero", O_RDONLY);
352 if (read_fd < 0)
353 return T_EXIT_SKIP;
354
355 for (loop = 0; loop < loops; loop++) {
356 memset(buffers, 0, sizeof(bool) * entries);
357 for (idx = 0; idx < entries; idx++)
358 io_uring_buf_ring_add(br, buffer, sizeof(buffer), idx, ring_mask, idx);
359 io_uring_buf_ring_advance(br, entries);
360
361 for (idx = 0; idx < entries; idx++) {
362 memset(buffer, 1, sizeof(buffer));
363 ret = test_one_read(read_fd, bgid, &ring);
364 if (ret < 0) {
365 fprintf(stderr, "bad run %d/%d = %d\n", loop, idx, ret);
366 return T_EXIT_FAIL;
367 }
368 if (buffers[ret]) {
369 fprintf(stderr, "reused buffer %d/%d = %d!\n", loop, idx, ret);
370 return T_EXIT_FAIL;
371 }
372 if (buffer[0] != 0) {
373 fprintf(stderr, "unexpected read %d %d/%d = %d!\n",
374 (int)buffer[0], loop, idx, ret);
375 return T_EXIT_FAIL;
376 }
377 if (buffer[1] != 1) {
378 fprintf(stderr, "unexpected spilled read %d %d/%d = %d!\n",
379 (int)buffer[1], loop, idx, ret);
380 return T_EXIT_FAIL;
381 }
382 buffers[ret] = true;
383 }
384 ret = test_one_read(read_fd, bgid, &ring);
385 if (ret != -ENOBUFS) {
386 fprintf(stderr, "expected enobufs run %d = %d\n", loop, ret);
387 return T_EXIT_FAIL;
388 }
389
390 }
391
392 ret = io_uring_unregister_buf_ring(&ring, bgid);
393 if (ret) {
394 fprintf(stderr, "Buffer ring register failed %d\n", ret);
395 return T_EXIT_FAIL;
396 }
397
398 close(read_fd);
399 io_uring_queue_exit(&ring);
400 free(buffers);
401 return T_EXIT_PASS;
402 }
403
main(int argc,char * argv[])404 int main(int argc, char *argv[])
405 {
406 int bgids[] = { 1, 127, -1 };
407 int entries[] = {1, 32768, 4096, -1 };
408 int ret, i;
409
410 if (argc > 1)
411 return T_EXIT_SKIP;
412
413 pagesize = getpagesize();
414
415 for (i = 0; bgids[i] != -1; i++) {
416 ret = test_reg_unreg(bgids[i]);
417 if (ret) {
418 fprintf(stderr, "test_reg_unreg failed\n");
419 return T_EXIT_FAIL;
420 }
421 if (no_buf_ring)
422 break;
423
424 ret = test_bad_reg(bgids[i]);
425 if (ret) {
426 fprintf(stderr, "test_bad_reg failed\n");
427 return T_EXIT_FAIL;
428 }
429
430 ret = test_double_reg_unreg(bgids[i]);
431 if (ret) {
432 fprintf(stderr, "test_double_reg_unreg failed\n");
433 return T_EXIT_FAIL;
434 }
435
436 ret = test_mixed_reg(bgids[i]);
437 if (ret) {
438 fprintf(stderr, "test_mixed_reg failed\n");
439 return T_EXIT_FAIL;
440 }
441
442 ret = test_mixed_reg2(bgids[i]);
443 if (ret) {
444 fprintf(stderr, "test_mixed_reg2 failed\n");
445 return T_EXIT_FAIL;
446 }
447
448 ret = test_full_page_reg(bgids[i]);
449 if (ret == T_EXIT_FAIL) {
450 fprintf(stderr, "test_full_page_reg failed\n");
451 return T_EXIT_FAIL;
452 }
453 }
454
455 for (i = 0; !no_buf_ring && entries[i] != -1; i++) {
456 ret = test_running(2, entries[i], 3, 0);
457 if (ret) {
458 fprintf(stderr, "test_running(%d) failed\n", entries[i]);
459 return T_EXIT_FAIL;
460 }
461 }
462
463 for (i = 0; !no_buf_ring && entries[i] != -1; i++) {
464 ret = test_running(2, entries[i], 3, 1);
465 if (ret == T_EXIT_SKIP) {
466 break;
467 } else if (ret != T_EXIT_PASS) {
468 fprintf(stderr, "test_running(%d) mmap failed\n", entries[i]);
469 return T_EXIT_FAIL;
470 }
471 }
472
473
474 return T_EXIT_PASS;
475 }
476