1 /* SPDX-License-Identifier: MIT */
2 /*
3 * Test fixed buffers consisting of hugepages.
4 */
5 #include <stdio.h>
6 #include <string.h>
7 #include <fcntl.h>
8 #include <stdlib.h>
9 #include <sys/mman.h>
10 #include <linux/mman.h>
11
12 #include "liburing.h"
13 #include "helpers.h"
14
15 /*
16 * Before testing
17 * echo (>=4) > /proc/sys/vm/nr_hugepages
18 * echo madvise > /sys/kernel/mm/transparent_hugepage/enabled
19 * echo always > /sys/kernel/mm/transparent_hugepage/hugepages-16kB/enabled
20 *
21 * Not 100% guaranteed to get THP-backed memory, but in general it does.
22 */
23 #define MTHP_16KB (16UL * 1024)
24 #define HUGEPAGE_SIZE (2UL * 1024 * 1024)
25 #define NR_BUFS 1
26 #define IN_FD "/dev/urandom"
27 #define OUT_FD "/dev/zero"
28
open_files(char * fname_in,int * fd_in,int * fd_out)29 static int open_files(char *fname_in, int *fd_in, int *fd_out)
30 {
31 *fd_in = open(fname_in, O_RDONLY, 0644);
32 if (*fd_in < 0) {
33 printf("open %s failed\n", fname_in);
34 return -1;
35 }
36
37 *fd_out = open(OUT_FD, O_RDWR, 0644);
38 if (*fd_out < 0) {
39 printf("open %s failed\n", OUT_FD);
40 return -1;
41 }
42
43 return 0;
44 }
45
unmap(struct iovec * iov,int nr_bufs,size_t offset)46 static void unmap(struct iovec *iov, int nr_bufs, size_t offset)
47 {
48 int i;
49
50 for (i = 0; i < nr_bufs; i++)
51 munmap(iov[i].iov_base - offset, iov[i].iov_len + offset);
52 }
53
mmap_hugebufs(struct iovec * iov,int nr_bufs,size_t buf_size,size_t offset)54 static int mmap_hugebufs(struct iovec *iov, int nr_bufs, size_t buf_size, size_t offset)
55 {
56 int i;
57
58 for (i = 0; i < nr_bufs; i++) {
59 void *base = NULL;
60
61 base = mmap(NULL, buf_size, PROT_READ | PROT_WRITE,
62 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
63 if (base == MAP_FAILED) {
64 printf("Unable to map hugetlb page. Try increasing the "
65 "value in /proc/sys/vm/nr_hugepages\n");
66 unmap(iov, i, offset);
67 return -1;
68 }
69
70 memset(base, 0, buf_size);
71 iov[i].iov_base = base + offset;
72 iov[i].iov_len = buf_size - offset;
73 }
74
75 return 0;
76 }
77
78 /* map a hugepage and smaller page to a contiguous memory */
mmap_mixture(struct iovec * iov,int nr_bufs,size_t buf_size,bool huge_on_left)79 static int mmap_mixture(struct iovec *iov, int nr_bufs, size_t buf_size, bool huge_on_left)
80 {
81 int i;
82 void *small_base = NULL, *huge_base = NULL, *start = NULL,
83 *huge_start = NULL, *small_start = NULL;
84 size_t small_size = buf_size - HUGEPAGE_SIZE;
85 size_t seg_size = ((buf_size / HUGEPAGE_SIZE) + 1) * HUGEPAGE_SIZE;
86
87 start = mmap(NULL, seg_size * nr_bufs, PROT_NONE,
88 MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
89 if (start == MAP_FAILED) {
90 printf("Unable to preserve the page mixture memory. "
91 "Try increasing the RLIMIT_MEMLOCK resource limit\n");
92 return -1;
93 }
94
95 for (i = 0; i < nr_bufs; i++) {
96 if (huge_on_left) {
97 huge_start = start;
98 small_start = start + HUGEPAGE_SIZE;
99 } else {
100 huge_start = start + HUGEPAGE_SIZE;
101 small_start = start + HUGEPAGE_SIZE - small_size;
102 }
103
104 huge_base = mmap(huge_start, HUGEPAGE_SIZE, PROT_READ | PROT_WRITE,
105 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_FIXED, -1, 0);
106 if (huge_base == MAP_FAILED) {
107 printf("Unable to map hugetlb page in the page mixture. "
108 "Try increasing the value in /proc/sys/vm/nr_hugepages\n");
109 unmap(iov, nr_bufs, 0);
110 return -1;
111 }
112
113 small_base = mmap(small_start, small_size, PROT_READ | PROT_WRITE,
114 MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
115 if (small_base == MAP_FAILED) {
116 printf("Unable to map small page in the page mixture. "
117 "Try increasing the RLIMIT_MEMLOCK resource limit\n");
118 unmap(iov, nr_bufs, 0);
119 return -1;
120 }
121
122 if (huge_on_left) {
123 iov[i].iov_base = huge_base;
124 memset(huge_base, 0, buf_size);
125 }
126 else {
127 iov[i].iov_base = small_base;
128 memset(small_base, 0, buf_size);
129 }
130 iov[i].iov_len = buf_size;
131 start += seg_size;
132 }
133
134 return 0;
135 }
136
free_bufs(struct iovec * iov,int nr_bufs,size_t offset)137 static void free_bufs(struct iovec *iov, int nr_bufs, size_t offset)
138 {
139 int i;
140
141 for (i = 0; i < nr_bufs; i++)
142 free(iov[i].iov_base - offset);
143 }
144
get_mthp_bufs(struct iovec * iov,int nr_bufs,size_t buf_size,size_t alignment,size_t offset)145 static int get_mthp_bufs(struct iovec *iov, int nr_bufs, size_t buf_size,
146 size_t alignment, size_t offset)
147 {
148 int i;
149
150 for (i = 0; i < nr_bufs; i++) {
151 void *base = NULL;
152
153 if (posix_memalign(&base, alignment, buf_size)) {
154 printf("Unable to allocate mthp pages. "
155 "Try increasing the RLIMIT_MEMLOCK resource limit\n");
156 free_bufs(iov, i, offset);
157 return -1;
158 }
159
160 memset(base, 0, buf_size);
161 iov[i].iov_base = base + offset;
162 iov[i].iov_len = buf_size - offset;
163 }
164
165 return 0;
166 }
167
do_read(struct io_uring * ring,int fd,struct iovec * iov,int nr_bufs)168 static int do_read(struct io_uring *ring, int fd, struct iovec *iov, int nr_bufs)
169 {
170 struct io_uring_sqe *sqe;
171 struct io_uring_cqe *cqe;
172 int i, ret;
173
174 for (i = 0; i < nr_bufs; i++) {
175 sqe = io_uring_get_sqe(ring);
176 if (!sqe) {
177 fprintf(stderr, "Could not get SQE.\n");
178 return -1;
179 }
180
181 io_uring_prep_read_fixed(sqe, fd, iov[i].iov_base, iov[i].iov_len, 0, i);
182 io_uring_submit(ring);
183
184 ret = io_uring_wait_cqe(ring, &cqe);
185 if (ret < 0) {
186 fprintf(stderr, "Error waiting for completion: %s\n", strerror(-ret));
187 return -1;
188 }
189
190 if (cqe->res < 0) {
191 fprintf(stderr, "Error in async read operation: %s\n", strerror(-cqe->res));
192 return -1;
193 }
194 if (cqe->res != iov[i].iov_len) {
195 fprintf(stderr, "cqe res: %d, expected: %lu\n", cqe->res, (unsigned long) iov[i].iov_len);
196 return -1;
197 }
198
199 io_uring_cqe_seen(ring, cqe);
200 }
201
202 return 0;
203 }
204
do_write(struct io_uring * ring,int fd,struct iovec * iov,int nr_bufs)205 static int do_write(struct io_uring *ring, int fd, struct iovec *iov, int nr_bufs)
206 {
207 struct io_uring_sqe *sqe;
208 struct io_uring_cqe *cqe;
209 int i, ret;
210
211 for (i = 0; i < nr_bufs; i++) {
212 sqe = io_uring_get_sqe(ring);
213 if (!sqe) {
214 fprintf(stderr, "Could not get SQE.\n");
215 return -1;
216 }
217
218 io_uring_prep_write_fixed(sqe, fd, iov[i].iov_base, iov[i].iov_len, 0, i);
219 io_uring_submit(ring);
220
221 ret = io_uring_wait_cqe(ring, &cqe);
222 if (ret < 0) {
223 fprintf(stderr, "Error waiting for completion: %s\n", strerror(-ret));
224 return -1;
225 }
226
227 if (cqe->res < 0) {
228 fprintf(stderr, "Error in async write operation: %s\n", strerror(-cqe->res));
229 return -1;
230 }
231 if (cqe->res != iov[i].iov_len) {
232 fprintf(stderr, "cqe res: %d, expected: %lu\n", cqe->res, (unsigned long) iov[i].iov_len);
233 return -1;
234 }
235
236 io_uring_cqe_seen(ring, cqe);
237 }
238
239 return 0;
240 }
241
register_submit(struct io_uring * ring,struct iovec * iov,int nr_bufs,int fd_in,int fd_out)242 static int register_submit(struct io_uring *ring, struct iovec *iov,
243 int nr_bufs, int fd_in, int fd_out)
244 {
245 int ret;
246
247 ret = io_uring_register_buffers(ring, iov, nr_bufs);
248 if (ret) {
249 fprintf(stderr, "Error registering buffers: %s\n", strerror(-ret));
250 return ret;
251 }
252
253 ret = do_read(ring, fd_in, iov, nr_bufs);
254 if (ret) {
255 fprintf(stderr, "Read test failed\n");
256 return ret;
257 }
258
259 ret = do_write(ring, fd_out, iov, nr_bufs);
260 if (ret) {
261 fprintf(stderr, "Write test failed\n");
262 return ret;
263 }
264
265 ret = io_uring_unregister_buffers(ring);
266 if (ret) {
267 fprintf(stderr, "Error unregistering buffers for one hugepage test: %s", strerror(-ret));
268 return ret;
269 }
270
271 return 0;
272 }
273
test_one_hugepage(struct io_uring * ring,int fd_in,int fd_out)274 static int test_one_hugepage(struct io_uring *ring, int fd_in, int fd_out)
275 {
276 struct iovec iov[NR_BUFS];
277 size_t buf_size = HUGEPAGE_SIZE;
278 int ret;
279
280 if (mmap_hugebufs(iov, NR_BUFS, buf_size, 0))
281 return T_EXIT_SKIP;
282
283 ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
284 unmap(iov, NR_BUFS, 0);
285 return ret ? T_EXIT_FAIL : T_EXIT_PASS;
286 }
287
test_multi_hugepages(struct io_uring * ring,int fd_in,int fd_out)288 static int test_multi_hugepages(struct io_uring *ring, int fd_in, int fd_out)
289 {
290 struct iovec iov[NR_BUFS];
291 size_t buf_size = 4 * HUGEPAGE_SIZE;
292 int ret;
293
294 if (mmap_hugebufs(iov, NR_BUFS, buf_size, 0))
295 return T_EXIT_SKIP;
296
297 ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
298 unmap(iov, NR_BUFS, 0);
299 return ret ? T_EXIT_FAIL : T_EXIT_PASS;
300 }
301
test_unaligned_hugepage(struct io_uring * ring,int fd_in,int fd_out)302 static int test_unaligned_hugepage(struct io_uring *ring, int fd_in, int fd_out)
303 {
304 struct iovec iov[NR_BUFS];
305 size_t buf_size = 3 * HUGEPAGE_SIZE;
306 size_t offset = 0x1234;
307 int ret;
308
309 if (mmap_hugebufs(iov, NR_BUFS, buf_size, offset))
310 return T_EXIT_SKIP;
311
312 ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
313 unmap(iov, NR_BUFS, offset);
314 return ret ? T_EXIT_FAIL : T_EXIT_PASS;
315 }
316
test_multi_unaligned_mthps(struct io_uring * ring,int fd_in,int fd_out)317 static int test_multi_unaligned_mthps(struct io_uring *ring, int fd_in, int fd_out)
318 {
319 struct iovec iov[NR_BUFS];
320 int ret;
321 size_t buf_size = 3 * MTHP_16KB;
322 size_t offset = 0x1234;
323
324 if (get_mthp_bufs(iov, NR_BUFS, buf_size, MTHP_16KB, offset))
325 return T_EXIT_SKIP;
326
327 ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
328 free_bufs(iov, NR_BUFS, offset);
329 return ret ? T_EXIT_FAIL : T_EXIT_PASS;
330 }
331
332 /* Should not coalesce */
test_page_mixture(struct io_uring * ring,int fd_in,int fd_out,int huge_on_left)333 static int test_page_mixture(struct io_uring *ring, int fd_in, int fd_out, int huge_on_left)
334 {
335 struct iovec iov[NR_BUFS];
336 size_t buf_size = HUGEPAGE_SIZE + MTHP_16KB;
337 int ret;
338
339 if (mmap_mixture(iov, NR_BUFS, buf_size, huge_on_left))
340 return T_EXIT_SKIP;
341
342 ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
343 unmap(iov, NR_BUFS, 0);
344 return ret ? T_EXIT_FAIL : T_EXIT_PASS;
345 }
346
main(int argc,char * argv[])347 int main(int argc, char *argv[])
348 {
349 struct io_uring ring;
350 int ret, fd_in, fd_out;
351 char *fname_in;
352
353 if (argc > 1)
354 fname_in = argv[1];
355 else
356 fname_in = IN_FD;
357
358 if (open_files(fname_in, &fd_in, &fd_out))
359 return T_EXIT_SKIP;
360
361 ret = t_create_ring(8, &ring, 0);
362 if (ret == T_SETUP_SKIP)
363 return T_EXIT_SKIP;
364 else if (ret < 0)
365 return T_EXIT_FAIL;
366
367 ret = test_one_hugepage(&ring, fd_in, fd_out);
368 if (ret != T_EXIT_PASS) {
369 if (ret != T_EXIT_SKIP)
370 fprintf(stderr, "Test one hugepage failed.\n");
371 return ret;
372 }
373
374 ret = test_multi_hugepages(&ring, fd_in, fd_out);
375 if (ret != T_EXIT_PASS) {
376 if (ret != T_EXIT_SKIP)
377 fprintf(stderr, "Test multi hugepages failed.\n");
378 return ret;
379 }
380
381 ret = test_unaligned_hugepage(&ring, fd_in, fd_out);
382 if (ret != T_EXIT_PASS) {
383 if (ret != T_EXIT_SKIP)
384 fprintf(stderr, "Test unaligned hugepage failed.\n");
385 return ret;
386 }
387
388 ret = test_multi_unaligned_mthps(&ring, fd_in, fd_out);
389 if (ret != T_EXIT_PASS) {
390 if (ret != T_EXIT_SKIP)
391 fprintf(stderr, "Test unaligned multi-size'd THPs failed.\n");
392 return ret;
393 }
394
395 ret = test_page_mixture(&ring, fd_in, fd_out, true);
396 if (ret != T_EXIT_PASS) {
397 if (ret != T_EXIT_SKIP)
398 fprintf(stderr, "Test huge small page mixture (start with huge) failed.\n");
399 return ret;
400 }
401
402 ret = test_page_mixture(&ring, fd_in, fd_out, false);
403 if (ret != T_EXIT_PASS) {
404 if (ret != T_EXIT_SKIP)
405 fprintf(stderr, "Test huge small page mixture (start with small) failed.\n");
406 return ret;
407 }
408
409 io_uring_queue_exit(&ring);
410 return T_EXIT_PASS;
411 }
412