1 /* SPDX-License-Identifier: MIT */
2 /*
3 * Test fixed buffers consisting of hugepages.
4 */
5 #include <stdio.h>
6 #include <string.h>
7 #include <fcntl.h>
8 #include <stdlib.h>
9 #include <sys/mman.h>
10 #include <linux/mman.h>
11
12 #include "liburing.h"
13 #include "helpers.h"
14
15 /*
16 * Before testing
17 * echo (>=4) > /proc/sys/vm/nr_hugepages
18 * echo madvise > /sys/kernel/mm/transparent_hugepage/enabled
19 * echo always > /sys/kernel/mm/transparent_hugepage/hugepages-16kB/enabled
20 *
21 * Not 100% guaranteed to get THP-backed memory, but in general it does.
22 */
23 #define MTHP_16KB (16UL * 1024)
24 #define HUGEPAGE_SIZE (2UL * 1024 * 1024)
25 #define NR_BUFS 1
26 #define IN_FD "/dev/urandom"
27 #define OUT_FD "/dev/zero"
28
open_files(char * fname_in,int * fd_in,int * fd_out)29 static int open_files(char *fname_in, int *fd_in, int *fd_out)
30 {
31 *fd_in = open(fname_in, O_RDONLY, 0644);
32 if (*fd_in < 0) {
33 printf("open %s failed\n", fname_in);
34 return -1;
35 }
36
37 *fd_out = open(OUT_FD, O_RDWR, 0644);
38 if (*fd_out < 0) {
39 printf("open %s failed\n", OUT_FD);
40 return -1;
41 }
42
43 return 0;
44 }
45
unmap(struct iovec * iov,int nr_bufs,size_t offset)46 static void unmap(struct iovec *iov, int nr_bufs, size_t offset)
47 {
48 int i;
49
50 for (i = 0; i < nr_bufs; i++)
51 munmap(iov[i].iov_base - offset, iov[i].iov_len + offset);
52 }
53
mmap_hugebufs(struct iovec * iov,int nr_bufs,size_t buf_size,size_t offset)54 static int mmap_hugebufs(struct iovec *iov, int nr_bufs, size_t buf_size, size_t offset)
55 {
56 int i;
57
58 for (i = 0; i < nr_bufs; i++) {
59 void *base = NULL;
60
61 base = mmap(NULL, buf_size, PROT_READ | PROT_WRITE,
62 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
63 if (base == MAP_FAILED) {
64 printf("Unable to map hugetlb page. Try increasing the "
65 "value in /proc/sys/vm/nr_hugepages\n");
66 unmap(iov, i, offset);
67 return -1;
68 }
69
70 memset(base, 0, buf_size);
71 iov[i].iov_base = base + offset;
72 iov[i].iov_len = buf_size - offset;
73 }
74
75 return 0;
76 }
77
78 /* map a hugepage and smaller page to a contiguous memory */
mmap_mixture(struct iovec * iov,int nr_bufs,size_t buf_size,bool huge_on_left)79 static int mmap_mixture(struct iovec *iov, int nr_bufs, size_t buf_size, bool huge_on_left)
80 {
81 int i;
82 void *small_base = NULL, *huge_base = NULL, *start = NULL,
83 *huge_start = NULL, *small_start = NULL;
84 size_t small_size = buf_size - HUGEPAGE_SIZE;
85 size_t seg_size = ((buf_size / HUGEPAGE_SIZE) + 1) * HUGEPAGE_SIZE;
86
87 start = mmap(NULL, seg_size * nr_bufs, PROT_NONE,
88 MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
89 if (start == MAP_FAILED) {
90 printf("Unable to preserve the page mixture memory. "
91 "Try increasing the RLIMIT_MEMLOCK resource limit\n");
92 return -1;
93 }
94
95 for (i = 0; i < nr_bufs; i++) {
96 if (huge_on_left) {
97 huge_start = start;
98 small_start = start + HUGEPAGE_SIZE;
99 } else {
100 huge_start = start + HUGEPAGE_SIZE;
101 small_start = start + HUGEPAGE_SIZE - small_size;
102 }
103
104 huge_base = mmap(huge_start, HUGEPAGE_SIZE, PROT_READ | PROT_WRITE,
105 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_FIXED, -1, 0);
106 if (huge_base == MAP_FAILED) {
107 printf("Unable to map hugetlb page in the page mixture. "
108 "Try increasing the value in /proc/sys/vm/nr_hugepages\n");
109 unmap(iov, nr_bufs, 0);
110 return -1;
111 }
112
113 small_base = mmap(small_start, small_size, PROT_READ | PROT_WRITE,
114 MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
115 if (small_base == MAP_FAILED) {
116 printf("Unable to map small page in the page mixture. "
117 "Try increasing the RLIMIT_MEMLOCK resource limit\n");
118 unmap(iov, nr_bufs, 0);
119 return -1;
120 }
121
122 if (huge_on_left) {
123 iov[i].iov_base = huge_base;
124 memset(huge_base, 0, buf_size);
125 }
126 else {
127 iov[i].iov_base = small_base;
128 memset(small_base, 0, buf_size);
129 }
130 iov[i].iov_len = buf_size;
131 start += seg_size;
132 }
133
134 return 0;
135 }
136
free_bufs(struct iovec * iov,int nr_bufs,size_t offset)137 static void free_bufs(struct iovec *iov, int nr_bufs, size_t offset)
138 {
139 int i;
140
141 for (i = 0; i < nr_bufs; i++)
142 free(iov[i].iov_base - offset);
143 }
144
get_mthp_bufs(struct iovec * iov,int nr_bufs,size_t buf_size,size_t alignment,size_t offset)145 static int get_mthp_bufs(struct iovec *iov, int nr_bufs, size_t buf_size,
146 size_t alignment, size_t offset)
147 {
148 int i;
149
150 for (i = 0; i < nr_bufs; i++) {
151 void *base = NULL;
152
153 if (posix_memalign(&base, alignment, buf_size)) {
154 printf("Unable to allocate mthp pages. "
155 "Try increasing the RLIMIT_MEMLOCK resource limit\n");
156 free_bufs(iov, i, offset);
157 return -1;
158 }
159
160 memset(base, 0, buf_size);
161 iov[i].iov_base = base + offset;
162 iov[i].iov_len = buf_size - offset;
163 }
164
165 return 0;
166 }
167
do_read(struct io_uring * ring,int fd,struct iovec * iov,int nr_bufs)168 static int do_read(struct io_uring *ring, int fd, struct iovec *iov, int nr_bufs)
169 {
170 struct io_uring_sqe *sqe;
171 struct io_uring_cqe *cqe;
172 int i, ret;
173
174 for (i = 0; i < nr_bufs; i++) {
175 sqe = io_uring_get_sqe(ring);
176 if (!sqe) {
177 fprintf(stderr, "Could not get SQE.\n");
178 return -1;
179 }
180
181 io_uring_prep_read_fixed(sqe, fd, iov[i].iov_base, iov[i].iov_len, 0, i);
182 io_uring_submit(ring);
183
184 ret = io_uring_wait_cqe(ring, &cqe);
185 if (ret < 0) {
186 fprintf(stderr, "Error waiting for completion: %s\n", strerror(-ret));
187 return -1;
188 }
189
190 if (cqe->res < 0) {
191 fprintf(stderr, "Error in async read operation: %s\n", strerror(-cqe->res));
192 return -1;
193 }
194 if (cqe->res != iov[i].iov_len) {
195 fprintf(stderr, "cqe res: %d, expected: %lu\n", cqe->res, (unsigned long) iov[i].iov_len);
196 return -1;
197 }
198
199 io_uring_cqe_seen(ring, cqe);
200 }
201
202 return 0;
203 }
204
do_write(struct io_uring * ring,int fd,struct iovec * iov,int nr_bufs)205 static int do_write(struct io_uring *ring, int fd, struct iovec *iov, int nr_bufs)
206 {
207 struct io_uring_sqe *sqe;
208 struct io_uring_cqe *cqe;
209 int i, ret;
210
211 for (i = 0; i < nr_bufs; i++) {
212 sqe = io_uring_get_sqe(ring);
213 if (!sqe) {
214 fprintf(stderr, "Could not get SQE.\n");
215 return -1;
216 }
217
218 io_uring_prep_write_fixed(sqe, fd, iov[i].iov_base, iov[i].iov_len, 0, i);
219 io_uring_submit(ring);
220
221 ret = io_uring_wait_cqe(ring, &cqe);
222 if (ret < 0) {
223 fprintf(stderr, "Error waiting for completion: %s\n", strerror(-ret));
224 return -1;
225 }
226
227 if (cqe->res < 0) {
228 fprintf(stderr, "Error in async write operation: %s\n", strerror(-cqe->res));
229 return -1;
230 }
231 if (cqe->res != iov[i].iov_len) {
232 fprintf(stderr, "cqe res: %d, expected: %lu\n", cqe->res, (unsigned long) iov[i].iov_len);
233 return -1;
234 }
235
236 io_uring_cqe_seen(ring, cqe);
237 }
238
239 return 0;
240 }
241
register_submit(struct io_uring * ring,struct iovec * iov,int nr_bufs,int fd_in,int fd_out)242 static int register_submit(struct io_uring *ring, struct iovec *iov,
243 int nr_bufs, int fd_in, int fd_out)
244 {
245 int ret;
246
247 ret = io_uring_register_buffers(ring, iov, nr_bufs);
248 if (ret) {
249 if (ret != -ENOMEM)
250 fprintf(stderr, "Error registering buffers: %s\n", strerror(-ret));
251 return ret;
252 }
253
254 ret = do_read(ring, fd_in, iov, nr_bufs);
255 if (ret) {
256 fprintf(stderr, "Read test failed\n");
257 return ret;
258 }
259
260 ret = do_write(ring, fd_out, iov, nr_bufs);
261 if (ret) {
262 fprintf(stderr, "Write test failed\n");
263 return ret;
264 }
265
266 ret = io_uring_unregister_buffers(ring);
267 if (ret) {
268 fprintf(stderr, "Error unregistering buffers for one hugepage test: %s", strerror(-ret));
269 return ret;
270 }
271
272 return 0;
273 }
274
test_one_hugepage(struct io_uring * ring,int fd_in,int fd_out)275 static int test_one_hugepage(struct io_uring *ring, int fd_in, int fd_out)
276 {
277 struct iovec iov[NR_BUFS];
278 size_t buf_size = HUGEPAGE_SIZE;
279 int ret;
280
281 if (mmap_hugebufs(iov, NR_BUFS, buf_size, 0))
282 return T_EXIT_SKIP;
283
284 ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
285 unmap(iov, NR_BUFS, 0);
286 if (ret == -ENOMEM)
287 return T_EXIT_SKIP;
288 return ret ? T_EXIT_FAIL : T_EXIT_PASS;
289 }
290
test_multi_hugepages(struct io_uring * ring,int fd_in,int fd_out)291 static int test_multi_hugepages(struct io_uring *ring, int fd_in, int fd_out)
292 {
293 struct iovec iov[NR_BUFS];
294 size_t buf_size = 4 * HUGEPAGE_SIZE;
295 int ret;
296
297 if (mmap_hugebufs(iov, NR_BUFS, buf_size, 0))
298 return T_EXIT_SKIP;
299
300 ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
301 unmap(iov, NR_BUFS, 0);
302 if (ret == -ENOMEM)
303 return T_EXIT_SKIP;
304 return ret ? T_EXIT_FAIL : T_EXIT_PASS;
305 }
306
test_unaligned_hugepage(struct io_uring * ring,int fd_in,int fd_out)307 static int test_unaligned_hugepage(struct io_uring *ring, int fd_in, int fd_out)
308 {
309 struct iovec iov[NR_BUFS];
310 size_t buf_size = 3 * HUGEPAGE_SIZE;
311 size_t offset = 0x1234;
312 int ret;
313
314 if (mmap_hugebufs(iov, NR_BUFS, buf_size, offset))
315 return T_EXIT_SKIP;
316
317 ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
318 unmap(iov, NR_BUFS, offset);
319 if (ret == -ENOMEM)
320 return T_EXIT_SKIP;
321 return ret ? T_EXIT_FAIL : T_EXIT_PASS;
322 }
323
test_multi_unaligned_mthps(struct io_uring * ring,int fd_in,int fd_out)324 static int test_multi_unaligned_mthps(struct io_uring *ring, int fd_in, int fd_out)
325 {
326 struct iovec iov[NR_BUFS];
327 int ret;
328 size_t buf_size = 3 * MTHP_16KB;
329 size_t offset = 0x1234;
330
331 if (get_mthp_bufs(iov, NR_BUFS, buf_size, MTHP_16KB, offset))
332 return T_EXIT_SKIP;
333
334 ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
335 free_bufs(iov, NR_BUFS, offset);
336 if (ret == -ENOMEM)
337 return T_EXIT_SKIP;
338 return ret ? T_EXIT_FAIL : T_EXIT_PASS;
339 }
340
341 /* Should not coalesce */
test_page_mixture(struct io_uring * ring,int fd_in,int fd_out,int huge_on_left)342 static int test_page_mixture(struct io_uring *ring, int fd_in, int fd_out, int huge_on_left)
343 {
344 struct iovec iov[NR_BUFS];
345 size_t buf_size = HUGEPAGE_SIZE + MTHP_16KB;
346 int ret;
347
348 if (mmap_mixture(iov, NR_BUFS, buf_size, huge_on_left))
349 return T_EXIT_SKIP;
350
351 ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
352 unmap(iov, NR_BUFS, 0);
353 if (ret == -ENOMEM)
354 return T_EXIT_SKIP;
355 return ret ? T_EXIT_FAIL : T_EXIT_PASS;
356 }
357
main(int argc,char * argv[])358 int main(int argc, char *argv[])
359 {
360 struct io_uring ring;
361 int ret, fd_in, fd_out;
362 char *fname_in;
363
364 if (argc > 1)
365 fname_in = argv[1];
366 else
367 fname_in = IN_FD;
368
369 if (open_files(fname_in, &fd_in, &fd_out))
370 return T_EXIT_SKIP;
371
372 ret = t_create_ring(8, &ring, 0);
373 if (ret == T_SETUP_SKIP)
374 return T_EXIT_SKIP;
375 else if (ret < 0)
376 return T_EXIT_FAIL;
377
378 ret = test_one_hugepage(&ring, fd_in, fd_out);
379 if (ret != T_EXIT_PASS) {
380 if (ret != T_EXIT_SKIP)
381 fprintf(stderr, "Test one hugepage failed.\n");
382 return ret;
383 }
384
385 ret = test_multi_hugepages(&ring, fd_in, fd_out);
386 if (ret != T_EXIT_PASS) {
387 if (ret != T_EXIT_SKIP)
388 fprintf(stderr, "Test multi hugepages failed.\n");
389 return ret;
390 }
391
392 ret = test_unaligned_hugepage(&ring, fd_in, fd_out);
393 if (ret != T_EXIT_PASS) {
394 if (ret != T_EXIT_SKIP)
395 fprintf(stderr, "Test unaligned hugepage failed.\n");
396 return ret;
397 }
398
399 ret = test_multi_unaligned_mthps(&ring, fd_in, fd_out);
400 if (ret != T_EXIT_PASS) {
401 if (ret != T_EXIT_SKIP)
402 fprintf(stderr, "Test unaligned multi-size'd THPs failed.\n");
403 return ret;
404 }
405
406 ret = test_page_mixture(&ring, fd_in, fd_out, true);
407 if (ret != T_EXIT_PASS) {
408 if (ret != T_EXIT_SKIP)
409 fprintf(stderr, "Test huge small page mixture (start with huge) failed.\n");
410 return ret;
411 }
412
413 ret = test_page_mixture(&ring, fd_in, fd_out, false);
414 if (ret != T_EXIT_PASS) {
415 if (ret != T_EXIT_SKIP)
416 fprintf(stderr, "Test huge small page mixture (start with small) failed.\n");
417 return ret;
418 }
419
420 io_uring_queue_exit(&ring);
421 return T_EXIT_PASS;
422 }
423