1 /* SPDX-License-Identifier: MIT */
2 /*
3 * Description: run various reads tests, verifying data
4 *
5 */
6 #include <errno.h>
7 #include <stdio.h>
8 #include <unistd.h>
9 #include <stdlib.h>
10 #include <string.h>
11 #include <fcntl.h>
12 #include <assert.h>
13 #include <sys/ioctl.h>
14 #include <sys/stat.h>
15 #include <linux/fs.h>
16
17 #include "helpers.h"
18 #include "liburing.h"
19
20 #define FSIZE 128*1024*1024
21 #define CHUNK_SIZE 131072
22 #define PUNCH_SIZE 32768
23
24 /*
25 * 8 because it fits within the on-stack iov, 16 because it's larger than 8
26 */
27 #define MIN_VECS 8
28 #define MAX_VECS 16
29
30 /*
31 * Can be anything, let's just do something for a bit of parallelism
32 */
33 #define READ_BATCH 16
34
verify_buf_sync(void * buf,size_t size,bool registered)35 static void verify_buf_sync(void *buf, size_t size, bool registered)
36 {
37 #if defined(__hppa__)
38 if (registered) {
39 unsigned long off = (unsigned long) buf & 4095;
40 unsigned long p = (unsigned long) buf & ~4095;
41 int i;
42
43 size += off;
44 for (i = 0; i < size; i += 32)
45 asm volatile("fdc 0(%0)" : : "r" (p + i));
46 }
47 #endif
48 }
49
50 /*
51 * Each offset in the file has the offset / sizeof(int) stored for every
52 * sizeof(int) address.
53 */
verify_buf(void * buf,size_t size,off_t off,bool registered)54 static int verify_buf(void *buf, size_t size, off_t off, bool registered)
55 {
56 int i, u_in_buf = size / sizeof(unsigned int);
57 unsigned int *ptr;
58
59 verify_buf_sync(buf, size, registered);
60
61 off /= sizeof(unsigned int);
62 ptr = buf;
63 for (i = 0; i < u_in_buf; i++) {
64 if (off != *ptr) {
65 fprintf(stderr, "Found %u, wanted %llu\n", *ptr,
66 (unsigned long long) off);
67 return 1;
68 }
69 ptr++;
70 off++;
71 }
72
73 return 0;
74 }
75
test_truncate(struct io_uring * ring,const char * fname,int buffered,int vectored,int provide_buf)76 static int test_truncate(struct io_uring *ring, const char *fname, int buffered,
77 int vectored, int provide_buf)
78 {
79 struct io_uring_cqe *cqe;
80 struct io_uring_sqe *sqe;
81 struct iovec vec;
82 struct stat sb;
83 off_t punch_off, off, file_size;
84 void *buf = NULL;
85 int u_in_buf, i, ret, fd, first_pass = 1;
86 unsigned int *ptr;
87
88 if (buffered)
89 fd = open(fname, O_RDWR);
90 else
91 fd = open(fname, O_DIRECT | O_RDWR);
92 if (fd < 0) {
93 if (!buffered && errno == EINVAL)
94 return T_EXIT_SKIP;
95 perror("open");
96 return 1;
97 }
98
99 if (fstat(fd, &sb) < 0) {
100 perror("stat");
101 close(fd);
102 return 1;
103 }
104
105 if (S_ISREG(sb.st_mode)) {
106 file_size = sb.st_size;
107 } else if (S_ISBLK(sb.st_mode)) {
108 unsigned long long bytes;
109
110 if (ioctl(fd, BLKGETSIZE64, &bytes) < 0) {
111 perror("ioctl");
112 close(fd);
113 return 1;
114 }
115 file_size = bytes;
116 } else {
117 goto out;
118 }
119
120 if (file_size < CHUNK_SIZE)
121 goto out;
122
123 t_posix_memalign(&buf, 4096, CHUNK_SIZE);
124
125 off = file_size - (CHUNK_SIZE / 2);
126 punch_off = off + CHUNK_SIZE / 4;
127
128 u_in_buf = CHUNK_SIZE / sizeof(unsigned int);
129 ptr = buf;
130 for (i = 0; i < u_in_buf; i++) {
131 *ptr = i;
132 ptr++;
133 }
134 ret = pwrite(fd, buf, CHUNK_SIZE / 2, off);
135 if (ret < 0) {
136 perror("pwrite");
137 goto err;
138 } else if (ret != CHUNK_SIZE / 2)
139 goto out;
140
141 again:
142 /*
143 * Read in last bit of file so it's known cached, then remove half of that
144 * last bit so we get a short read that needs retry
145 */
146 ret = pread(fd, buf, CHUNK_SIZE / 2, off);
147 if (ret < 0) {
148 perror("pread");
149 goto err;
150 } else if (ret != CHUNK_SIZE / 2)
151 goto out;
152
153 if (posix_fadvise(fd, punch_off, CHUNK_SIZE / 4, POSIX_FADV_DONTNEED) < 0) {
154 perror("posix_fadivse");
155 goto err;
156 }
157
158 if (provide_buf) {
159 sqe = io_uring_get_sqe(ring);
160 io_uring_prep_provide_buffers(sqe, buf, CHUNK_SIZE, 1, 0, 0);
161 ret = io_uring_submit(ring);
162 if (ret != 1) {
163 fprintf(stderr, "submit failed %d\n", ret);
164 goto err;
165 }
166 ret = io_uring_wait_cqe(ring, &cqe);
167 if (ret < 0) {
168 fprintf(stderr, "wait completion %d\n", ret);
169 goto err;
170 }
171 ret = cqe->res;
172 io_uring_cqe_seen(ring, cqe);
173 if (ret) {
174 fprintf(stderr, "Provide buffer failed %d\n", ret);
175 goto err;
176 }
177 }
178
179 sqe = io_uring_get_sqe(ring);
180 if (!sqe) {
181 fprintf(stderr, "get sqe failed\n");
182 goto err;
183 }
184
185 if (vectored) {
186 assert(!provide_buf);
187 vec.iov_base = buf;
188 vec.iov_len = CHUNK_SIZE;
189 io_uring_prep_readv(sqe, fd, &vec, 1, off);
190 } else {
191 if (provide_buf) {
192 io_uring_prep_read(sqe, fd, NULL, CHUNK_SIZE, off);
193 sqe->flags |= IOSQE_BUFFER_SELECT;
194 } else {
195 io_uring_prep_read(sqe, fd, buf, CHUNK_SIZE, off);
196 }
197 }
198 memset(buf, 0, CHUNK_SIZE);
199
200 ret = io_uring_submit(ring);
201 if (ret != 1) {
202 fprintf(stderr, "Submit failed %d\n", ret);
203 goto err;
204 }
205
206 ret = io_uring_wait_cqe(ring, &cqe);
207 if (ret < 0) {
208 fprintf(stderr, "wait completion %d\n", ret);
209 goto err;
210 }
211
212 ret = cqe->res;
213 io_uring_cqe_seen(ring, cqe);
214 if (ret != CHUNK_SIZE / 2) {
215 fprintf(stderr, "Unexpected truncated read %d\n", ret);
216 goto err;
217 }
218
219 if (verify_buf(buf, CHUNK_SIZE / 2, 0, false))
220 goto err;
221
222 /*
223 * Repeat, but punch first part instead of last
224 */
225 if (first_pass) {
226 punch_off = file_size - CHUNK_SIZE / 4;
227 first_pass = 0;
228 goto again;
229 }
230
231 out:
232 free(buf);
233 close(fd);
234 return 0;
235 err:
236 free(buf);
237 close(fd);
238 return 1;
239 }
240
241 enum {
242 PUNCH_NONE,
243 PUNCH_FRONT,
244 PUNCH_MIDDLE,
245 PUNCH_END,
246 };
247
248 /*
249 * For each chunk in file, DONTNEED a start, end, or middle segment of it.
250 * We enter here with the file fully cached every time, either freshly
251 * written or after other reads. This forces (at least) the buffered reads
252 * to be handled incrementally, exercising that path.
253 */
do_punch(int fd)254 static int do_punch(int fd)
255 {
256 off_t offset = 0;
257 int punch_type;
258
259 while (offset + CHUNK_SIZE <= FSIZE) {
260 off_t punch_off;
261
262 punch_type = rand() % (PUNCH_END + 1);
263 switch (punch_type) {
264 default:
265 case PUNCH_NONE:
266 punch_off = -1; /* gcc... */
267 break;
268 case PUNCH_FRONT:
269 punch_off = offset;
270 break;
271 case PUNCH_MIDDLE:
272 punch_off = offset + PUNCH_SIZE;
273 break;
274 case PUNCH_END:
275 punch_off = offset + CHUNK_SIZE - PUNCH_SIZE;
276 break;
277 }
278
279 offset += CHUNK_SIZE;
280 if (punch_type == PUNCH_NONE)
281 continue;
282 if (posix_fadvise(fd, punch_off, PUNCH_SIZE, POSIX_FADV_DONTNEED) < 0) {
283 perror("posix_fadivse");
284 return 1;
285 }
286 }
287
288 return 0;
289 }
290
provide_buffers(struct io_uring * ring,void ** buf)291 static int provide_buffers(struct io_uring *ring, void **buf)
292 {
293 struct io_uring_cqe *cqe;
294 struct io_uring_sqe *sqe;
295 int i, ret;
296
297 /* real use case would have one buffer chopped up, but... */
298 for (i = 0; i < READ_BATCH; i++) {
299 sqe = io_uring_get_sqe(ring);
300 io_uring_prep_provide_buffers(sqe, buf[i], CHUNK_SIZE, 1, 0, i);
301 }
302
303 ret = io_uring_submit(ring);
304 if (ret != READ_BATCH) {
305 fprintf(stderr, "Submit failed %d\n", ret);
306 return 1;
307 }
308
309 for (i = 0; i < READ_BATCH; i++) {
310 ret = io_uring_wait_cqe(ring, &cqe);
311 if (ret) {
312 fprintf(stderr, "wait cqe %d\n", ret);
313 return 1;
314 }
315 if (cqe->res < 0) {
316 fprintf(stderr, "cqe res provide %d\n", cqe->res);
317 return 1;
318 }
319 io_uring_cqe_seen(ring, cqe);
320 }
321
322 return 0;
323 }
324
test(struct io_uring * ring,const char * fname,int buffered,int vectored,int small_vecs,int registered,int provide)325 static int test(struct io_uring *ring, const char *fname, int buffered,
326 int vectored, int small_vecs, int registered, int provide)
327 {
328 struct iovec vecs[READ_BATCH][MAX_VECS];
329 struct io_uring_cqe *cqe;
330 struct io_uring_sqe *sqe;
331 void *buf[READ_BATCH];
332 int ret, fd, flags;
333 int i, j, nr_vecs;
334 off_t off, voff;
335 size_t left;
336
337 if (registered) {
338 assert(!provide);
339 assert(!vectored && !small_vecs);
340 }
341 if (provide) {
342 assert(!registered);
343 assert(!vectored && !small_vecs);
344 }
345
346 flags = O_RDONLY;
347 if (!buffered)
348 flags |= O_DIRECT;
349 fd = open(fname, flags);
350 if (fd < 0) {
351 if (errno == EINVAL || errno == EPERM || errno == EACCES)
352 return T_EXIT_SKIP;
353 perror("open");
354 return 1;
355 }
356
357 if (do_punch(fd))
358 return 1;
359
360 if (vectored) {
361 if (small_vecs)
362 nr_vecs = MIN_VECS;
363 else
364 nr_vecs = MAX_VECS;
365
366 for (j = 0; j < READ_BATCH; j++) {
367 for (i = 0; i < nr_vecs; i++) {
368 void *ptr;
369
370 t_posix_memalign(&ptr, 4096, CHUNK_SIZE / nr_vecs);
371 vecs[j][i].iov_base = ptr;
372 vecs[j][i].iov_len = CHUNK_SIZE / nr_vecs;
373 }
374 }
375 } else {
376 for (j = 0; j < READ_BATCH; j++)
377 t_posix_memalign(&buf[j], 4096, CHUNK_SIZE);
378 nr_vecs = 0;
379 }
380
381 if (registered) {
382 struct iovec v[READ_BATCH];
383
384 for (i = 0; i < READ_BATCH; i++) {
385 v[i].iov_base = buf[i];
386 v[i].iov_len = CHUNK_SIZE;
387 }
388 ret = t_register_buffers(ring, v, READ_BATCH);
389 if (ret) {
390 if (ret == T_SETUP_SKIP) {
391 ret = 0;
392 goto free_bufs;
393 }
394 goto err;
395 }
396 }
397
398 i = 0;
399 left = FSIZE;
400 off = 0;
401 while (left) {
402 int pending = 0;
403
404 if (provide && provide_buffers(ring, buf))
405 goto err;
406
407 for (i = 0; i < READ_BATCH; i++) {
408 size_t this = left;
409
410 if (this > CHUNK_SIZE)
411 this = CHUNK_SIZE;
412
413 sqe = io_uring_get_sqe(ring);
414 if (!sqe) {
415 fprintf(stderr, "get sqe failed\n");
416 goto err;
417 }
418
419 if (vectored) {
420 io_uring_prep_readv(sqe, fd, vecs[i], nr_vecs, off);
421 } else {
422 if (registered) {
423 io_uring_prep_read_fixed(sqe, fd, buf[i], this, off, i);
424 } else if (provide) {
425 io_uring_prep_read(sqe, fd, NULL, this, off);
426 sqe->flags |= IOSQE_BUFFER_SELECT;
427 } else {
428 io_uring_prep_read(sqe, fd, buf[i], this, off);
429 }
430 }
431 sqe->user_data = ((uint64_t)off << 32) | i;
432 off += this;
433 left -= this;
434 pending++;
435 if (!left)
436 break;
437 }
438
439 ret = io_uring_submit(ring);
440 if (ret != pending) {
441 fprintf(stderr, "sqe submit failed: %d\n", ret);
442 goto err;
443 }
444
445 for (i = 0; i < pending; i++) {
446 int index;
447
448 ret = io_uring_wait_cqe(ring, &cqe);
449 if (ret < 0) {
450 fprintf(stderr, "wait completion %d\n", ret);
451 goto err;
452 }
453 if (cqe->res < 0) {
454 fprintf(stderr, "bad read %d, read %d\n", cqe->res, i);
455 goto err;
456 }
457 if (cqe->res < CHUNK_SIZE) {
458 fprintf(stderr, "short read %d, read %d\n", cqe->res, i);
459 goto err;
460 }
461 if (cqe->flags & IORING_CQE_F_BUFFER)
462 index = cqe->flags >> 16;
463 else
464 index = cqe->user_data & 0xffffffff;
465 voff = cqe->user_data >> 32;
466 io_uring_cqe_seen(ring, cqe);
467 if (vectored) {
468 for (j = 0; j < nr_vecs; j++) {
469 void *buf = vecs[index][j].iov_base;
470 size_t len = vecs[index][j].iov_len;
471
472 if (verify_buf(buf, len, voff, registered))
473 goto err;
474 voff += len;
475 }
476 } else {
477 if (verify_buf(buf[index], CHUNK_SIZE, voff, registered))
478 goto err;
479 }
480 }
481 }
482
483 ret = 0;
484 done:
485 if (registered)
486 io_uring_unregister_buffers(ring);
487 free_bufs:
488 if (vectored) {
489 for (j = 0; j < READ_BATCH; j++)
490 for (i = 0; i < nr_vecs; i++)
491 free(vecs[j][i].iov_base);
492 } else {
493 for (j = 0; j < READ_BATCH; j++)
494 free(buf[j]);
495 }
496 close(fd);
497 return ret;
498 err:
499 ret = 1;
500 goto done;
501 }
502
fill_pattern(const char * fname)503 static int fill_pattern(const char *fname)
504 {
505 size_t left = FSIZE;
506 unsigned int val, *ptr;
507 void *buf;
508 int fd, i;
509
510 fd = open(fname, O_WRONLY);
511 if (fd < 0) {
512 if (errno == EPERM || errno == EACCES)
513 return T_EXIT_SKIP;
514 perror("open");
515 return 1;
516 }
517
518 val = 0;
519 buf = t_malloc(4096);
520 while (left) {
521 int u_in_buf = 4096 / sizeof(val);
522 size_t this = left;
523
524 if (this > 4096)
525 this = 4096;
526 ptr = buf;
527 for (i = 0; i < u_in_buf; i++) {
528 *ptr = val;
529 val++;
530 ptr++;
531 }
532 if (write(fd, buf, 4096) != 4096)
533 return 1;
534 left -= 4096;
535 }
536
537 fsync(fd);
538 close(fd);
539 free(buf);
540 return 0;
541 }
542
main(int argc,char * argv[])543 int main(int argc, char *argv[])
544 {
545 struct io_uring ring;
546 const char *fname;
547 char buf[32];
548 int ret;
549
550 srand(getpid());
551
552 if (argc > 1) {
553 fname = argv[1];
554 } else {
555 sprintf(buf, ".file-verify.%d", getpid());
556 fname = buf;
557 t_create_file(fname, FSIZE);
558 }
559
560 ret = io_uring_queue_init(READ_BATCH, &ring, 0);
561 if (ret) {
562 fprintf(stderr, "ring setup failed: %d\n", ret);
563 goto err;
564 }
565
566 ret = fill_pattern(fname);
567 if (ret == T_EXIT_SKIP)
568 return T_EXIT_SKIP;
569 else if (ret)
570 goto err;
571
572 ret = test(&ring, fname, 1, 0, 0, 0, 0);
573 if (ret == T_EXIT_SKIP)
574 return T_EXIT_SKIP;
575 if (ret) {
576 fprintf(stderr, "Buffered novec test failed\n");
577 goto err;
578 }
579 ret = test(&ring, fname, 1, 0, 0, 1, 0);
580 if (ret == T_EXIT_FAIL) {
581 fprintf(stderr, "Buffered novec reg test failed\n");
582 goto err;
583 }
584 ret = test(&ring, fname, 1, 0, 0, 0, 1);
585 if (ret == T_EXIT_FAIL) {
586 fprintf(stderr, "Buffered novec provide test failed\n");
587 goto err;
588 }
589 ret = test(&ring, fname, 1, 1, 0, 0, 0);
590 if (ret == T_EXIT_FAIL) {
591 fprintf(stderr, "Buffered vec test failed\n");
592 goto err;
593 }
594 ret = test(&ring, fname, 1, 1, 1, 0, 0);
595 if (ret == T_EXIT_FAIL) {
596 fprintf(stderr, "Buffered small vec test failed\n");
597 goto err;
598 }
599
600 ret = test(&ring, fname, 0, 0, 0, 0, 0);
601 if (ret == T_EXIT_FAIL) {
602 fprintf(stderr, "O_DIRECT novec test failed\n");
603 goto err;
604 }
605 ret = test(&ring, fname, 0, 0, 0, 1, 0);
606 if (ret == T_EXIT_FAIL) {
607 fprintf(stderr, "O_DIRECT novec reg test failed\n");
608 goto err;
609 }
610 ret = test(&ring, fname, 0, 0, 0, 0, 1);
611 if (ret == T_EXIT_FAIL) {
612 fprintf(stderr, "O_DIRECT novec provide test failed\n");
613 goto err;
614 }
615 ret = test(&ring, fname, 0, 1, 0, 0, 0);
616 if (ret == T_EXIT_FAIL) {
617 fprintf(stderr, "O_DIRECT vec test failed\n");
618 goto err;
619 }
620 ret = test(&ring, fname, 0, 1, 1, 0, 0);
621 if (ret == T_EXIT_FAIL) {
622 fprintf(stderr, "O_DIRECT small vec test failed\n");
623 goto err;
624 }
625
626 ret = test_truncate(&ring, fname, 1, 0, 0);
627 if (ret == T_EXIT_FAIL) {
628 fprintf(stderr, "Buffered end truncate read failed\n");
629 goto err;
630 }
631 ret = test_truncate(&ring, fname, 1, 1, 0);
632 if (ret == T_EXIT_FAIL) {
633 fprintf(stderr, "Buffered end truncate vec read failed\n");
634 goto err;
635 }
636 ret = test_truncate(&ring, fname, 1, 0, 1);
637 if (ret == T_EXIT_FAIL) {
638 fprintf(stderr, "Buffered end truncate pbuf read failed\n");
639 goto err;
640 }
641
642 ret = test_truncate(&ring, fname, 0, 0, 0);
643 if (ret == T_EXIT_FAIL) {
644 fprintf(stderr, "O_DIRECT end truncate read failed\n");
645 goto err;
646 }
647 ret = test_truncate(&ring, fname, 0, 1, 0);
648 if (ret == T_EXIT_FAIL) {
649 fprintf(stderr, "O_DIRECT end truncate vec read failed\n");
650 goto err;
651 }
652 ret = test_truncate(&ring, fname, 0, 0, 1);
653 if (ret == T_EXIT_FAIL) {
654 fprintf(stderr, "O_DIRECT end truncate pbuf read failed\n");
655 goto err;
656 }
657
658 if (buf == fname)
659 unlink(fname);
660 return T_EXIT_PASS;
661 err:
662 if (buf == fname)
663 unlink(fname);
664 return T_EXIT_FAIL;
665 }
666