1 /* SPDX-License-Identifier: MIT */
2 /*
3 * Description: basic read/write tests with polled IO
4 */
5 #include <errno.h>
6 #include <stdio.h>
7 #include <unistd.h>
8 #include <stdlib.h>
9 #include <string.h>
10 #include <fcntl.h>
11 #include <sys/types.h>
12 #include <sys/poll.h>
13 #include <sys/eventfd.h>
14 #include <sys/resource.h>
15 #include "helpers.h"
16 #include "liburing.h"
17 #include "../src/syscall.h"
18
19 #define FILE_SIZE (128 * 1024)
20 #define BS 4096
21 #define BUFFERS (FILE_SIZE / BS)
22
23 static struct iovec *vecs;
24 static int no_buf_select;
25 static int no_iopoll;
26
provide_buffers(struct io_uring * ring)27 static int provide_buffers(struct io_uring *ring)
28 {
29 struct io_uring_sqe *sqe;
30 struct io_uring_cqe *cqe;
31 int ret, i;
32
33 for (i = 0; i < BUFFERS; i++) {
34 sqe = io_uring_get_sqe(ring);
35 io_uring_prep_provide_buffers(sqe, vecs[i].iov_base,
36 vecs[i].iov_len, 1, 1, i);
37 }
38
39 ret = io_uring_submit(ring);
40 if (ret != BUFFERS) {
41 fprintf(stderr, "submit: %d\n", ret);
42 return 1;
43 }
44
45 for (i = 0; i < BUFFERS; i++) {
46 ret = io_uring_wait_cqe(ring, &cqe);
47 if (cqe->res < 0) {
48 fprintf(stderr, "cqe->res=%d\n", cqe->res);
49 return 1;
50 }
51 io_uring_cqe_seen(ring, cqe);
52 }
53
54 return 0;
55 }
56
__test_io(const char * file,struct io_uring * ring,int write,int sqthread,int fixed,int buf_select)57 static int __test_io(const char *file, struct io_uring *ring, int write, int sqthread,
58 int fixed, int buf_select)
59 {
60 struct io_uring_sqe *sqe;
61 struct io_uring_cqe *cqe;
62 int open_flags;
63 int i, fd, ret;
64 off_t offset;
65
66 if (buf_select && write)
67 write = 0;
68 if (buf_select && fixed)
69 fixed = 0;
70
71 if (buf_select && provide_buffers(ring))
72 return 1;
73
74 if (write)
75 open_flags = O_WRONLY;
76 else
77 open_flags = O_RDONLY;
78 open_flags |= O_DIRECT;
79
80 fd = open(file, open_flags);
81 if (fd < 0) {
82 perror("file open");
83 goto err;
84 }
85
86 if (fixed) {
87 ret = io_uring_register_buffers(ring, vecs, BUFFERS);
88 if (ret) {
89 fprintf(stderr, "buffer reg failed: %d\n", ret);
90 goto err;
91 }
92 }
93 if (sqthread) {
94 ret = io_uring_register_files(ring, &fd, 1);
95 if (ret) {
96 fprintf(stderr, "file reg failed: %d\n", ret);
97 goto err;
98 }
99 }
100
101 offset = 0;
102 for (i = 0; i < BUFFERS; i++) {
103 sqe = io_uring_get_sqe(ring);
104 if (!sqe) {
105 fprintf(stderr, "sqe get failed\n");
106 goto err;
107 }
108 offset = BS * (rand() % BUFFERS);
109 if (write) {
110 int do_fixed = fixed;
111 int use_fd = fd;
112
113 if (sqthread)
114 use_fd = 0;
115 if (fixed && (i & 1))
116 do_fixed = 0;
117 if (do_fixed) {
118 io_uring_prep_write_fixed(sqe, use_fd, vecs[i].iov_base,
119 vecs[i].iov_len,
120 offset, i);
121 } else {
122 io_uring_prep_writev(sqe, use_fd, &vecs[i], 1,
123 offset);
124 }
125 } else {
126 int do_fixed = fixed;
127 int use_fd = fd;
128
129 if (sqthread)
130 use_fd = 0;
131 if (fixed && (i & 1))
132 do_fixed = 0;
133 if (do_fixed) {
134 io_uring_prep_read_fixed(sqe, use_fd, vecs[i].iov_base,
135 vecs[i].iov_len,
136 offset, i);
137 } else {
138 io_uring_prep_readv(sqe, use_fd, &vecs[i], 1,
139 offset);
140 }
141
142 }
143 if (sqthread)
144 sqe->flags |= IOSQE_FIXED_FILE;
145 if (buf_select) {
146 sqe->flags |= IOSQE_BUFFER_SELECT;
147 sqe->buf_group = buf_select;
148 sqe->user_data = i;
149 }
150 }
151
152 ret = io_uring_submit(ring);
153 if (ret != BUFFERS) {
154 fprintf(stderr, "submit got %d, wanted %d\n", ret, BUFFERS);
155 goto err;
156 }
157
158 for (i = 0; i < BUFFERS; i++) {
159 ret = io_uring_wait_cqe(ring, &cqe);
160 if (ret) {
161 fprintf(stderr, "wait_cqe=%d\n", ret);
162 goto err;
163 } else if (cqe->res == -EOPNOTSUPP) {
164 fprintf(stdout, "File/device/fs doesn't support polled IO\n");
165 no_iopoll = 1;
166 goto out;
167 } else if (cqe->res != BS) {
168 fprintf(stderr, "cqe res %d, wanted %d\n", cqe->res, BS);
169 goto err;
170 }
171 io_uring_cqe_seen(ring, cqe);
172 }
173
174 if (fixed) {
175 ret = io_uring_unregister_buffers(ring);
176 if (ret) {
177 fprintf(stderr, "buffer unreg failed: %d\n", ret);
178 goto err;
179 }
180 }
181 if (sqthread) {
182 ret = io_uring_unregister_files(ring);
183 if (ret) {
184 fprintf(stderr, "file unreg failed: %d\n", ret);
185 goto err;
186 }
187 }
188
189 out:
190 close(fd);
191 return 0;
192 err:
193 if (fd != -1)
194 close(fd);
195 return 1;
196 }
197
198 extern int __io_uring_flush_sq(struct io_uring *ring);
199
200 /*
201 * if we are polling io_uring_submit needs to always enter the
202 * kernel to fetch events
203 */
test_io_uring_submit_enters(const char * file)204 static int test_io_uring_submit_enters(const char *file)
205 {
206 struct io_uring ring;
207 int fd, i, ret, ring_flags, open_flags;
208 unsigned head;
209 struct io_uring_cqe *cqe;
210
211 if (no_iopoll)
212 return 0;
213
214 ring_flags = IORING_SETUP_IOPOLL;
215 ret = io_uring_queue_init(64, &ring, ring_flags);
216 if (ret) {
217 fprintf(stderr, "ring create failed: %d\n", ret);
218 return 1;
219 }
220
221 open_flags = O_WRONLY | O_DIRECT;
222 fd = open(file, open_flags);
223 if (fd < 0) {
224 perror("file open");
225 goto err;
226 }
227
228 for (i = 0; i < BUFFERS; i++) {
229 struct io_uring_sqe *sqe;
230 off_t offset = BS * (rand() % BUFFERS);
231
232 sqe = io_uring_get_sqe(&ring);
233 io_uring_prep_writev(sqe, fd, &vecs[i], 1, offset);
234 sqe->user_data = 1;
235 }
236
237 /* submit manually to avoid adding IORING_ENTER_GETEVENTS */
238 ret = __sys_io_uring_enter(ring.ring_fd, __io_uring_flush_sq(&ring), 0,
239 0, NULL);
240 if (ret < 0)
241 goto err;
242
243 for (i = 0; i < 500; i++) {
244 ret = io_uring_submit(&ring);
245 if (ret != 0) {
246 fprintf(stderr, "still had %d sqes to submit, this is unexpected", ret);
247 goto err;
248 }
249
250 io_uring_for_each_cqe(&ring, head, cqe) {
251 /* runs after test_io so should not have happened */
252 if (cqe->res == -EOPNOTSUPP) {
253 fprintf(stdout, "File/device/fs doesn't support polled IO\n");
254 goto err;
255 }
256 goto ok;
257 }
258 usleep(10000);
259 }
260 err:
261 ret = 1;
262 if (fd != -1)
263 close(fd);
264
265 ok:
266 io_uring_queue_exit(&ring);
267 return ret;
268 }
269
test_io(const char * file,int write,int sqthread,int fixed,int buf_select)270 static int test_io(const char *file, int write, int sqthread, int fixed,
271 int buf_select)
272 {
273 struct io_uring ring;
274 int ret, ring_flags;
275
276 if (no_iopoll)
277 return 0;
278
279 ring_flags = IORING_SETUP_IOPOLL;
280 if (sqthread) {
281 static int warned;
282
283 if (geteuid()) {
284 if (!warned)
285 fprintf(stdout, "SQPOLL requires root, skipping\n");
286 warned = 1;
287 return 0;
288 }
289 }
290
291 ret = io_uring_queue_init(64, &ring, ring_flags);
292 if (ret) {
293 fprintf(stderr, "ring create failed: %d\n", ret);
294 return 1;
295 }
296
297 ret = __test_io(file, &ring, write, sqthread, fixed, buf_select);
298
299 io_uring_queue_exit(&ring);
300 return ret;
301 }
302
probe_buf_select(void)303 static int probe_buf_select(void)
304 {
305 struct io_uring_probe *p;
306 struct io_uring ring;
307 int ret;
308
309 ret = io_uring_queue_init(1, &ring, 0);
310 if (ret) {
311 fprintf(stderr, "ring create failed: %d\n", ret);
312 return 1;
313 }
314
315 p = io_uring_get_probe_ring(&ring);
316 if (!p || !io_uring_opcode_supported(p, IORING_OP_PROVIDE_BUFFERS)) {
317 no_buf_select = 1;
318 fprintf(stdout, "Buffer select not supported, skipping\n");
319 return 0;
320 }
321 free(p);
322 return 0;
323 }
324
main(int argc,char * argv[])325 int main(int argc, char *argv[])
326 {
327 int i, ret, nr;
328 char *fname;
329
330 if (probe_buf_select())
331 return 1;
332
333 if (argc > 1) {
334 fname = argv[1];
335 } else {
336 fname = ".iopoll-rw";
337 t_create_file(fname, FILE_SIZE);
338 }
339
340 vecs = t_create_buffers(BUFFERS, BS);
341
342 nr = 16;
343 if (no_buf_select)
344 nr = 8;
345 for (i = 0; i < nr; i++) {
346 int v1, v2, v3, v4;
347
348 v1 = (i & 1) != 0;
349 v2 = (i & 2) != 0;
350 v3 = (i & 4) != 0;
351 v4 = (i & 8) != 0;
352 ret = test_io(fname, v1, v2, v3, v4);
353 if (ret) {
354 fprintf(stderr, "test_io failed %d/%d/%d/%d\n", v1, v2, v3, v4);
355 goto err;
356 }
357 if (no_iopoll)
358 break;
359 }
360
361 ret = test_io_uring_submit_enters(fname);
362 if (ret) {
363 fprintf(stderr, "test_io_uring_submit_enters failed\n");
364 goto err;
365 }
366
367 if (fname != argv[1])
368 unlink(fname);
369 return 0;
370 err:
371 if (fname != argv[1])
372 unlink(fname);
373 return 1;
374 }
375