1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Stress userfaultfd syscall.
4 *
5 * Copyright (C) 2015 Red Hat, Inc.
6 *
7 * This test allocates two virtual areas and bounces the physical
8 * memory across the two virtual areas (from area_src to area_dst)
9 * using userfaultfd.
10 *
11 * There are three threads running per CPU:
12 *
13 * 1) one per-CPU thread takes a per-page pthread_mutex in a random
14 * page of the area_dst (while the physical page may still be in
15 * area_src), and increments a per-page counter in the same page,
16 * and checks its value against a verification region.
17 *
18 * 2) another per-CPU thread handles the userfaults generated by
19 * thread 1 above. userfaultfd blocking reads or poll() modes are
20 * exercised interleaved.
21 *
22 * 3) one last per-CPU thread transfers the memory in the background
23 * at maximum bandwidth (if not already transferred by thread
24 * 2). Each cpu thread takes cares of transferring a portion of the
25 * area.
26 *
27 * When all threads of type 3 completed the transfer, one bounce is
28 * complete. area_src and area_dst are then swapped. All threads are
29 * respawned and so the bounce is immediately restarted in the
30 * opposite direction.
31 *
32 * per-CPU threads 1 by triggering userfaults inside
33 * pthread_mutex_lock will also verify the atomicity of the memory
34 * transfer (UFFDIO_COPY).
35 */
36
37 #define _GNU_SOURCE
38 #include <stdio.h>
39 #include <errno.h>
40 #include <unistd.h>
41 #include <stdlib.h>
42 #include <sys/types.h>
43 #include <sys/stat.h>
44 #include <fcntl.h>
45 #include <time.h>
46 #include <signal.h>
47 #include <poll.h>
48 #include <string.h>
49 #include <linux/mman.h>
50 #include <sys/mman.h>
51 #include <sys/syscall.h>
52 #include <sys/ioctl.h>
53 #include <sys/wait.h>
54 #include <pthread.h>
55 #include <linux/userfaultfd.h>
56 #include <setjmp.h>
57 #include <stdbool.h>
58 #include <assert.h>
59 #include <inttypes.h>
60 #include <stdint.h>
61 #include <sys/random.h>
62
63 #include "../kselftest.h"
64
65 #ifdef __NR_userfaultfd
66
67 static unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size;
68
69 #define BOUNCE_RANDOM (1<<0)
70 #define BOUNCE_RACINGFAULTS (1<<1)
71 #define BOUNCE_VERIFY (1<<2)
72 #define BOUNCE_POLL (1<<3)
73 static int bounces;
74
75 #define TEST_ANON 1
76 #define TEST_HUGETLB 2
77 #define TEST_SHMEM 3
78 static int test_type;
79
80 /* exercise the test_uffdio_*_eexist every ALARM_INTERVAL_SECS */
81 #define ALARM_INTERVAL_SECS 10
82 static volatile bool test_uffdio_copy_eexist = true;
83 static volatile bool test_uffdio_zeropage_eexist = true;
84 /* Whether to test uffd write-protection */
85 static bool test_uffdio_wp = false;
86 /* Whether to test uffd minor faults */
87 static bool test_uffdio_minor = false;
88
89 static bool map_shared;
90 static int shm_fd;
91 static int huge_fd;
92 static char *huge_fd_off0;
93 static unsigned long long *count_verify;
94 static int uffd = -1;
95 static int uffd_flags, finished, *pipefd;
96 static volatile bool ready_for_fork;
97 static char *area_src, *area_src_alias, *area_dst, *area_dst_alias;
98 static char *zeropage;
99 pthread_attr_t attr;
100 pthread_key_t long_jmp_key;
101
102 /* Userfaultfd test statistics */
103 struct uffd_stats {
104 int cpu;
105 unsigned long missing_faults;
106 unsigned long wp_faults;
107 unsigned long minor_faults;
108 };
109
110 /* pthread_mutex_t starts at page offset 0 */
111 #define area_mutex(___area, ___nr) \
112 ((pthread_mutex_t *) ((___area) + (___nr)*page_size))
113 /*
114 * count is placed in the page after pthread_mutex_t naturally aligned
115 * to avoid non alignment faults on non-x86 archs.
116 */
117 #define area_count(___area, ___nr) \
118 ((volatile unsigned long long *) ((unsigned long) \
119 ((___area) + (___nr)*page_size + \
120 sizeof(pthread_mutex_t) + \
121 sizeof(unsigned long long) - 1) & \
122 ~(unsigned long)(sizeof(unsigned long long) \
123 - 1)))
124
125 const char *examples =
126 "# Run anonymous memory test on 100MiB region with 99999 bounces:\n"
127 "./userfaultfd anon 100 99999\n\n"
128 "# Run share memory test on 1GiB region with 99 bounces:\n"
129 "./userfaultfd shmem 1000 99\n\n"
130 "# Run hugetlb memory test on 256MiB region with 50 bounces (using /dev/hugepages/hugefile):\n"
131 "./userfaultfd hugetlb 256 50 /dev/hugepages/hugefile\n\n"
132 "# Run the same hugetlb test but using shmem:\n"
133 "./userfaultfd hugetlb_shared 256 50 /dev/hugepages/hugefile\n\n"
134 "# 10MiB-~6GiB 999 bounces anonymous test, "
135 "continue forever unless an error triggers\n"
136 "while ./userfaultfd anon $[RANDOM % 6000 + 10] 999; do true; done\n\n";
137
usage(void)138 static void usage(void)
139 {
140 fprintf(stderr, "\nUsage: ./userfaultfd <test type> <MiB> <bounces> "
141 "[hugetlbfs_file]\n\n");
142 fprintf(stderr, "Supported <test type>: anon, hugetlb, "
143 "hugetlb_shared, shmem\n\n");
144 fprintf(stderr, "Examples:\n\n");
145 fprintf(stderr, "%s", examples);
146 exit(1);
147 }
148
149 #define _err(fmt, ...) \
150 do { \
151 int ret = errno; \
152 fprintf(stderr, "ERROR: " fmt, ##__VA_ARGS__); \
153 fprintf(stderr, " (errno=%d, line=%d)\n", \
154 ret, __LINE__); \
155 } while (0)
156
157 #define err(fmt, ...) \
158 do { \
159 _err(fmt, ##__VA_ARGS__); \
160 exit(1); \
161 } while (0)
162
uffd_stats_reset(struct uffd_stats * uffd_stats,unsigned long n_cpus)163 static void uffd_stats_reset(struct uffd_stats *uffd_stats,
164 unsigned long n_cpus)
165 {
166 int i;
167
168 for (i = 0; i < n_cpus; i++) {
169 uffd_stats[i].cpu = i;
170 uffd_stats[i].missing_faults = 0;
171 uffd_stats[i].wp_faults = 0;
172 uffd_stats[i].minor_faults = 0;
173 }
174 }
175
uffd_stats_report(struct uffd_stats * stats,int n_cpus)176 static void uffd_stats_report(struct uffd_stats *stats, int n_cpus)
177 {
178 int i;
179 unsigned long long miss_total = 0, wp_total = 0, minor_total = 0;
180
181 for (i = 0; i < n_cpus; i++) {
182 miss_total += stats[i].missing_faults;
183 wp_total += stats[i].wp_faults;
184 minor_total += stats[i].minor_faults;
185 }
186
187 printf("userfaults: ");
188 if (miss_total) {
189 printf("%llu missing (", miss_total);
190 for (i = 0; i < n_cpus; i++)
191 printf("%lu+", stats[i].missing_faults);
192 printf("\b) ");
193 }
194 if (wp_total) {
195 printf("%llu wp (", wp_total);
196 for (i = 0; i < n_cpus; i++)
197 printf("%lu+", stats[i].wp_faults);
198 printf("\b) ");
199 }
200 if (minor_total) {
201 printf("%llu minor (", minor_total);
202 for (i = 0; i < n_cpus; i++)
203 printf("%lu+", stats[i].minor_faults);
204 printf("\b)");
205 }
206 printf("\n");
207 }
208
anon_release_pages(char * rel_area)209 static void anon_release_pages(char *rel_area)
210 {
211 if (madvise(rel_area, nr_pages * page_size, MADV_DONTNEED))
212 err("madvise(MADV_DONTNEED) failed");
213 }
214
anon_allocate_area(void ** alloc_area)215 static void anon_allocate_area(void **alloc_area)
216 {
217 *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
218 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
219 if (*alloc_area == MAP_FAILED)
220 err("mmap of anonymous memory failed");
221 }
222
noop_alias_mapping(__u64 * start,size_t len,unsigned long offset)223 static void noop_alias_mapping(__u64 *start, size_t len, unsigned long offset)
224 {
225 }
226
hugetlb_release_pages(char * rel_area)227 static void hugetlb_release_pages(char *rel_area)
228 {
229 if (fallocate(huge_fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
230 rel_area == huge_fd_off0 ? 0 : nr_pages * page_size,
231 nr_pages * page_size))
232 err("fallocate() failed");
233 }
234
hugetlb_allocate_area(void ** alloc_area)235 static void hugetlb_allocate_area(void **alloc_area)
236 {
237 void *area_alias = NULL;
238 char **alloc_area_alias;
239
240 *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
241 (map_shared ? MAP_SHARED : MAP_PRIVATE) |
242 MAP_HUGETLB |
243 (*alloc_area == area_src ? 0 : MAP_NORESERVE),
244 huge_fd, *alloc_area == area_src ? 0 :
245 nr_pages * page_size);
246 if (*alloc_area == MAP_FAILED)
247 err("mmap of hugetlbfs file failed");
248
249 if (map_shared) {
250 area_alias = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
251 MAP_SHARED | MAP_HUGETLB,
252 huge_fd, *alloc_area == area_src ? 0 :
253 nr_pages * page_size);
254 if (area_alias == MAP_FAILED)
255 err("mmap of hugetlb file alias failed");
256 }
257
258 if (*alloc_area == area_src) {
259 huge_fd_off0 = *alloc_area;
260 alloc_area_alias = &area_src_alias;
261 } else {
262 alloc_area_alias = &area_dst_alias;
263 }
264 if (area_alias)
265 *alloc_area_alias = area_alias;
266 }
267
hugetlb_alias_mapping(__u64 * start,size_t len,unsigned long offset)268 static void hugetlb_alias_mapping(__u64 *start, size_t len, unsigned long offset)
269 {
270 if (!map_shared)
271 return;
272 /*
273 * We can't zap just the pagetable with hugetlbfs because
274 * MADV_DONTEED won't work. So exercise -EEXIST on a alias
275 * mapping where the pagetables are not established initially,
276 * this way we'll exercise the -EEXEC at the fs level.
277 */
278 *start = (unsigned long) area_dst_alias + offset;
279 }
280
shmem_release_pages(char * rel_area)281 static void shmem_release_pages(char *rel_area)
282 {
283 if (madvise(rel_area, nr_pages * page_size, MADV_REMOVE))
284 err("madvise(MADV_REMOVE) failed");
285 }
286
shmem_allocate_area(void ** alloc_area)287 static void shmem_allocate_area(void **alloc_area)
288 {
289 void *area_alias = NULL;
290 bool is_src = alloc_area == (void **)&area_src;
291 unsigned long offset = is_src ? 0 : nr_pages * page_size;
292
293 *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
294 MAP_SHARED, shm_fd, offset);
295 if (*alloc_area == MAP_FAILED)
296 err("mmap of memfd failed");
297
298 area_alias = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
299 MAP_SHARED, shm_fd, offset);
300 if (area_alias == MAP_FAILED)
301 err("mmap of memfd alias failed");
302
303 if (is_src)
304 area_src_alias = area_alias;
305 else
306 area_dst_alias = area_alias;
307 }
308
shmem_alias_mapping(__u64 * start,size_t len,unsigned long offset)309 static void shmem_alias_mapping(__u64 *start, size_t len, unsigned long offset)
310 {
311 *start = (unsigned long)area_dst_alias + offset;
312 }
313
314 struct uffd_test_ops {
315 void (*allocate_area)(void **alloc_area);
316 void (*release_pages)(char *rel_area);
317 void (*alias_mapping)(__u64 *start, size_t len, unsigned long offset);
318 };
319
320 static struct uffd_test_ops anon_uffd_test_ops = {
321 .allocate_area = anon_allocate_area,
322 .release_pages = anon_release_pages,
323 .alias_mapping = noop_alias_mapping,
324 };
325
326 static struct uffd_test_ops shmem_uffd_test_ops = {
327 .allocate_area = shmem_allocate_area,
328 .release_pages = shmem_release_pages,
329 .alias_mapping = shmem_alias_mapping,
330 };
331
332 static struct uffd_test_ops hugetlb_uffd_test_ops = {
333 .allocate_area = hugetlb_allocate_area,
334 .release_pages = hugetlb_release_pages,
335 .alias_mapping = hugetlb_alias_mapping,
336 };
337
338 static struct uffd_test_ops *uffd_test_ops;
339
uffd_minor_feature(void)340 static inline uint64_t uffd_minor_feature(void)
341 {
342 if (test_type == TEST_HUGETLB && map_shared)
343 return UFFD_FEATURE_MINOR_HUGETLBFS;
344 else if (test_type == TEST_SHMEM)
345 return UFFD_FEATURE_MINOR_SHMEM;
346 else
347 return 0;
348 }
349
get_expected_ioctls(uint64_t mode)350 static uint64_t get_expected_ioctls(uint64_t mode)
351 {
352 uint64_t ioctls = UFFD_API_RANGE_IOCTLS;
353
354 if (test_type == TEST_HUGETLB)
355 ioctls &= ~(1 << _UFFDIO_ZEROPAGE);
356
357 if (!((mode & UFFDIO_REGISTER_MODE_WP) && test_uffdio_wp))
358 ioctls &= ~(1 << _UFFDIO_WRITEPROTECT);
359
360 if (!((mode & UFFDIO_REGISTER_MODE_MINOR) && test_uffdio_minor))
361 ioctls &= ~(1 << _UFFDIO_CONTINUE);
362
363 return ioctls;
364 }
365
assert_expected_ioctls_present(uint64_t mode,uint64_t ioctls)366 static void assert_expected_ioctls_present(uint64_t mode, uint64_t ioctls)
367 {
368 uint64_t expected = get_expected_ioctls(mode);
369 uint64_t actual = ioctls & expected;
370
371 if (actual != expected) {
372 err("missing ioctl(s): expected %"PRIx64" actual: %"PRIx64,
373 expected, actual);
374 }
375 }
376
userfaultfd_open(uint64_t * features)377 static void userfaultfd_open(uint64_t *features)
378 {
379 struct uffdio_api uffdio_api;
380
381 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY);
382 if (uffd < 0) {
383 if (errno == ENOSYS) {
384 printf("userfaultfd syscall not available in this kernel\n");
385 exit(KSFT_SKIP);
386 }
387 err("userfaultfd syscall failed with errno: %d\n", errno);
388 }
389 uffd_flags = fcntl(uffd, F_GETFD, NULL);
390
391 uffdio_api.api = UFFD_API;
392 uffdio_api.features = *features;
393 if (ioctl(uffd, UFFDIO_API, &uffdio_api))
394 err("UFFDIO_API failed.\nPlease make sure to "
395 "run with either root or ptrace capability.");
396 if (uffdio_api.api != UFFD_API)
397 err("UFFDIO_API error: %" PRIu64, (uint64_t)uffdio_api.api);
398
399 *features = uffdio_api.features;
400 }
401
munmap_area(void ** area)402 static inline void munmap_area(void **area)
403 {
404 if (*area)
405 if (munmap(*area, nr_pages * page_size))
406 err("munmap");
407
408 *area = NULL;
409 }
410
uffd_test_ctx_clear(void)411 static void uffd_test_ctx_clear(void)
412 {
413 size_t i;
414
415 if (pipefd) {
416 for (i = 0; i < nr_cpus * 2; ++i) {
417 if (close(pipefd[i]))
418 err("close pipefd");
419 }
420 free(pipefd);
421 pipefd = NULL;
422 }
423
424 if (count_verify) {
425 free(count_verify);
426 count_verify = NULL;
427 }
428
429 if (uffd != -1) {
430 if (close(uffd))
431 err("close uffd");
432 uffd = -1;
433 }
434
435 huge_fd_off0 = NULL;
436 munmap_area((void **)&area_src);
437 munmap_area((void **)&area_src_alias);
438 munmap_area((void **)&area_dst);
439 munmap_area((void **)&area_dst_alias);
440 }
441
uffd_test_ctx_init(uint64_t features)442 static void uffd_test_ctx_init(uint64_t features)
443 {
444 unsigned long nr, cpu;
445
446 uffd_test_ctx_clear();
447
448 uffd_test_ops->allocate_area((void **)&area_src);
449 uffd_test_ops->allocate_area((void **)&area_dst);
450
451 userfaultfd_open(&features);
452
453 count_verify = malloc(nr_pages * sizeof(unsigned long long));
454 if (!count_verify)
455 err("count_verify");
456
457 for (nr = 0; nr < nr_pages; nr++) {
458 *area_mutex(area_src, nr) =
459 (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER;
460 count_verify[nr] = *area_count(area_src, nr) = 1;
461 /*
462 * In the transition between 255 to 256, powerpc will
463 * read out of order in my_bcmp and see both bytes as
464 * zero, so leave a placeholder below always non-zero
465 * after the count, to avoid my_bcmp to trigger false
466 * positives.
467 */
468 *(area_count(area_src, nr) + 1) = 1;
469 }
470
471 /*
472 * After initialization of area_src, we must explicitly release pages
473 * for area_dst to make sure it's fully empty. Otherwise we could have
474 * some area_dst pages be errornously initialized with zero pages,
475 * hence we could hit memory corruption later in the test.
476 *
477 * One example is when THP is globally enabled, above allocate_area()
478 * calls could have the two areas merged into a single VMA (as they
479 * will have the same VMA flags so they're mergeable). When we
480 * initialize the area_src above, it's possible that some part of
481 * area_dst could have been faulted in via one huge THP that will be
482 * shared between area_src and area_dst. It could cause some of the
483 * area_dst won't be trapped by missing userfaults.
484 *
485 * This release_pages() will guarantee even if that happened, we'll
486 * proactively split the thp and drop any accidentally initialized
487 * pages within area_dst.
488 */
489 uffd_test_ops->release_pages(area_dst);
490
491 pipefd = malloc(sizeof(int) * nr_cpus * 2);
492 if (!pipefd)
493 err("pipefd");
494 for (cpu = 0; cpu < nr_cpus; cpu++)
495 if (pipe2(&pipefd[cpu * 2], O_CLOEXEC | O_NONBLOCK))
496 err("pipe");
497 }
498
my_bcmp(char * str1,char * str2,size_t n)499 static int my_bcmp(char *str1, char *str2, size_t n)
500 {
501 unsigned long i;
502 for (i = 0; i < n; i++)
503 if (str1[i] != str2[i])
504 return 1;
505 return 0;
506 }
507
wp_range(int ufd,__u64 start,__u64 len,bool wp)508 static void wp_range(int ufd, __u64 start, __u64 len, bool wp)
509 {
510 struct uffdio_writeprotect prms;
511
512 /* Write protection page faults */
513 prms.range.start = start;
514 prms.range.len = len;
515 /* Undo write-protect, do wakeup after that */
516 prms.mode = wp ? UFFDIO_WRITEPROTECT_MODE_WP : 0;
517
518 if (ioctl(ufd, UFFDIO_WRITEPROTECT, &prms))
519 err("clear WP failed: address=0x%"PRIx64, (uint64_t)start);
520 }
521
continue_range(int ufd,__u64 start,__u64 len)522 static void continue_range(int ufd, __u64 start, __u64 len)
523 {
524 struct uffdio_continue req;
525 int ret;
526
527 req.range.start = start;
528 req.range.len = len;
529 req.mode = 0;
530
531 if (ioctl(ufd, UFFDIO_CONTINUE, &req))
532 err("UFFDIO_CONTINUE failed for address 0x%" PRIx64,
533 (uint64_t)start);
534
535 /*
536 * Error handling within the kernel for continue is subtly different
537 * from copy or zeropage, so it may be a source of bugs. Trigger an
538 * error (-EEXIST) on purpose, to verify doing so doesn't cause a BUG.
539 */
540 req.mapped = 0;
541 ret = ioctl(ufd, UFFDIO_CONTINUE, &req);
542 if (ret >= 0 || req.mapped != -EEXIST)
543 err("failed to exercise UFFDIO_CONTINUE error handling, ret=%d, mapped=%" PRId64,
544 ret, (int64_t) req.mapped);
545 }
546
locking_thread(void * arg)547 static void *locking_thread(void *arg)
548 {
549 unsigned long cpu = (unsigned long) arg;
550 unsigned long page_nr = *(&(page_nr)); /* uninitialized warning */
551 unsigned long long count;
552
553 if (!(bounces & BOUNCE_RANDOM)) {
554 page_nr = -bounces;
555 if (!(bounces & BOUNCE_RACINGFAULTS))
556 page_nr += cpu * nr_pages_per_cpu;
557 }
558
559 while (!finished) {
560 if (bounces & BOUNCE_RANDOM) {
561 if (getrandom(&page_nr, sizeof(page_nr), 0) != sizeof(page_nr))
562 err("getrandom failed");
563 } else
564 page_nr += 1;
565 page_nr %= nr_pages;
566 pthread_mutex_lock(area_mutex(area_dst, page_nr));
567 count = *area_count(area_dst, page_nr);
568 if (count != count_verify[page_nr])
569 err("page_nr %lu memory corruption %llu %llu",
570 page_nr, count, count_verify[page_nr]);
571 count++;
572 *area_count(area_dst, page_nr) = count_verify[page_nr] = count;
573 pthread_mutex_unlock(area_mutex(area_dst, page_nr));
574 }
575
576 return NULL;
577 }
578
retry_copy_page(int ufd,struct uffdio_copy * uffdio_copy,unsigned long offset)579 static void retry_copy_page(int ufd, struct uffdio_copy *uffdio_copy,
580 unsigned long offset)
581 {
582 uffd_test_ops->alias_mapping(&uffdio_copy->dst,
583 uffdio_copy->len,
584 offset);
585 if (ioctl(ufd, UFFDIO_COPY, uffdio_copy)) {
586 /* real retval in ufdio_copy.copy */
587 if (uffdio_copy->copy != -EEXIST)
588 err("UFFDIO_COPY retry error: %"PRId64,
589 (int64_t)uffdio_copy->copy);
590 } else {
591 err("UFFDIO_COPY retry unexpected: %"PRId64,
592 (int64_t)uffdio_copy->copy);
593 }
594 }
595
wake_range(int ufd,unsigned long addr,unsigned long len)596 static void wake_range(int ufd, unsigned long addr, unsigned long len)
597 {
598 struct uffdio_range uffdio_wake;
599
600 uffdio_wake.start = addr;
601 uffdio_wake.len = len;
602
603 if (ioctl(ufd, UFFDIO_WAKE, &uffdio_wake))
604 fprintf(stderr, "error waking %lu\n",
605 addr), exit(1);
606 }
607
__copy_page(int ufd,unsigned long offset,bool retry)608 static int __copy_page(int ufd, unsigned long offset, bool retry)
609 {
610 struct uffdio_copy uffdio_copy;
611
612 if (offset >= nr_pages * page_size)
613 err("unexpected offset %lu\n", offset);
614 uffdio_copy.dst = (unsigned long) area_dst + offset;
615 uffdio_copy.src = (unsigned long) area_src + offset;
616 uffdio_copy.len = page_size;
617 if (test_uffdio_wp)
618 uffdio_copy.mode = UFFDIO_COPY_MODE_WP;
619 else
620 uffdio_copy.mode = 0;
621 uffdio_copy.copy = 0;
622 if (ioctl(ufd, UFFDIO_COPY, &uffdio_copy)) {
623 /* real retval in ufdio_copy.copy */
624 if (uffdio_copy.copy != -EEXIST)
625 err("UFFDIO_COPY error: %"PRId64,
626 (int64_t)uffdio_copy.copy);
627 wake_range(ufd, uffdio_copy.dst, page_size);
628 } else if (uffdio_copy.copy != page_size) {
629 err("UFFDIO_COPY error: %"PRId64, (int64_t)uffdio_copy.copy);
630 } else {
631 if (test_uffdio_copy_eexist && retry) {
632 test_uffdio_copy_eexist = false;
633 retry_copy_page(ufd, &uffdio_copy, offset);
634 }
635 return 1;
636 }
637 return 0;
638 }
639
copy_page_retry(int ufd,unsigned long offset)640 static int copy_page_retry(int ufd, unsigned long offset)
641 {
642 return __copy_page(ufd, offset, true);
643 }
644
copy_page(int ufd,unsigned long offset)645 static int copy_page(int ufd, unsigned long offset)
646 {
647 return __copy_page(ufd, offset, false);
648 }
649
uffd_read_msg(int ufd,struct uffd_msg * msg)650 static int uffd_read_msg(int ufd, struct uffd_msg *msg)
651 {
652 int ret = read(uffd, msg, sizeof(*msg));
653
654 if (ret != sizeof(*msg)) {
655 if (ret < 0) {
656 if (errno == EAGAIN || errno == EINTR)
657 return 1;
658 err("blocking read error");
659 } else {
660 err("short read");
661 }
662 }
663
664 return 0;
665 }
666
uffd_handle_page_fault(struct uffd_msg * msg,struct uffd_stats * stats)667 static void uffd_handle_page_fault(struct uffd_msg *msg,
668 struct uffd_stats *stats)
669 {
670 unsigned long offset;
671
672 if (msg->event != UFFD_EVENT_PAGEFAULT)
673 err("unexpected msg event %u", msg->event);
674
675 if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP) {
676 /* Write protect page faults */
677 wp_range(uffd, msg->arg.pagefault.address, page_size, false);
678 stats->wp_faults++;
679 } else if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_MINOR) {
680 uint8_t *area;
681 int b;
682
683 /*
684 * Minor page faults
685 *
686 * To prove we can modify the original range for testing
687 * purposes, we're going to bit flip this range before
688 * continuing.
689 *
690 * Note that this requires all minor page fault tests operate on
691 * area_dst (non-UFFD-registered) and area_dst_alias
692 * (UFFD-registered).
693 */
694
695 area = (uint8_t *)(area_dst +
696 ((char *)msg->arg.pagefault.address -
697 area_dst_alias));
698 for (b = 0; b < page_size; ++b)
699 area[b] = ~area[b];
700 continue_range(uffd, msg->arg.pagefault.address, page_size);
701 stats->minor_faults++;
702 } else {
703 /* Missing page faults */
704 if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE)
705 err("unexpected write fault");
706
707 offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;
708 offset &= ~(page_size-1);
709
710 if (copy_page(uffd, offset))
711 stats->missing_faults++;
712 }
713 }
714
uffd_poll_thread(void * arg)715 static void *uffd_poll_thread(void *arg)
716 {
717 struct uffd_stats *stats = (struct uffd_stats *)arg;
718 unsigned long cpu = stats->cpu;
719 struct pollfd pollfd[2];
720 struct uffd_msg msg;
721 struct uffdio_register uffd_reg;
722 int ret;
723 char tmp_chr;
724
725 pollfd[0].fd = uffd;
726 pollfd[0].events = POLLIN;
727 pollfd[1].fd = pipefd[cpu*2];
728 pollfd[1].events = POLLIN;
729
730 // Notify the main thread that it can now fork.
731 ready_for_fork = true;
732
733 for (;;) {
734 ret = poll(pollfd, 2, -1);
735 if (ret <= 0) {
736 if (errno == EINTR || errno == EAGAIN)
737 continue;
738 err("poll error: %d", ret);
739 }
740 if (pollfd[1].revents & POLLIN) {
741 if (read(pollfd[1].fd, &tmp_chr, 1) != 1)
742 err("read pipefd error");
743 break;
744 }
745 if (!(pollfd[0].revents & POLLIN))
746 err("pollfd[0].revents %d", pollfd[0].revents);
747 if (uffd_read_msg(uffd, &msg))
748 continue;
749 switch (msg.event) {
750 default:
751 err("unexpected msg event %u\n", msg.event);
752 break;
753 case UFFD_EVENT_PAGEFAULT:
754 uffd_handle_page_fault(&msg, stats);
755 break;
756 case UFFD_EVENT_FORK:
757 close(uffd);
758 uffd = msg.arg.fork.ufd;
759 pollfd[0].fd = uffd;
760 break;
761 case UFFD_EVENT_REMOVE:
762 uffd_reg.range.start = msg.arg.remove.start;
763 uffd_reg.range.len = msg.arg.remove.end -
764 msg.arg.remove.start;
765 if (ioctl(uffd, UFFDIO_UNREGISTER, &uffd_reg.range))
766 err("remove failure");
767 break;
768 case UFFD_EVENT_REMAP:
769 area_dst = (char *)(unsigned long)msg.arg.remap.to;
770 break;
771 }
772 }
773
774 return NULL;
775 }
776
777 pthread_mutex_t uffd_read_mutex = PTHREAD_MUTEX_INITIALIZER;
778
sigusr1_handler(int signum,siginfo_t * siginfo,void * ptr)779 static void sigusr1_handler(int signum, siginfo_t *siginfo, void *ptr)
780 {
781 jmp_buf *env;
782 env = pthread_getspecific(long_jmp_key);
783 longjmp(*env, 1);
784 }
785
uffd_read_thread(void * arg)786 static void *uffd_read_thread(void *arg)
787 {
788 struct uffd_stats *stats = (struct uffd_stats *)arg;
789 struct uffd_msg msg;
790 jmp_buf env;
791 int setjmp_ret;
792
793 pthread_setspecific(long_jmp_key, &env);
794
795 pthread_mutex_unlock(&uffd_read_mutex);
796
797 // One first return setjmp return 0. On second (fake) return from
798 // longjmp() it returns the provided value, which will be 1 in our case.
799 setjmp_ret = setjmp(env);
800 while (!setjmp_ret) {
801 if (uffd_read_msg(uffd, &msg))
802 continue;
803 uffd_handle_page_fault(&msg, stats);
804 }
805
806 return NULL;
807 }
808
background_thread(void * arg)809 static void *background_thread(void *arg)
810 {
811 unsigned long cpu = (unsigned long) arg;
812 unsigned long page_nr, start_nr, mid_nr, end_nr;
813
814 start_nr = cpu * nr_pages_per_cpu;
815 end_nr = (cpu+1) * nr_pages_per_cpu;
816 mid_nr = (start_nr + end_nr) / 2;
817
818 /* Copy the first half of the pages */
819 for (page_nr = start_nr; page_nr < mid_nr; page_nr++)
820 copy_page_retry(uffd, page_nr * page_size);
821
822 /*
823 * If we need to test uffd-wp, set it up now. Then we'll have
824 * at least the first half of the pages mapped already which
825 * can be write-protected for testing
826 */
827 if (test_uffdio_wp)
828 wp_range(uffd, (unsigned long)area_dst + start_nr * page_size,
829 nr_pages_per_cpu * page_size, true);
830
831 /*
832 * Continue the 2nd half of the page copying, handling write
833 * protection faults if any
834 */
835 for (page_nr = mid_nr; page_nr < end_nr; page_nr++)
836 copy_page_retry(uffd, page_nr * page_size);
837
838 return NULL;
839 }
840
stress(struct uffd_stats * uffd_stats)841 static int stress(struct uffd_stats *uffd_stats)
842 {
843 unsigned long cpu;
844 pthread_t locking_threads[nr_cpus];
845 pthread_t uffd_threads[nr_cpus];
846 pthread_t background_threads[nr_cpus];
847
848 finished = 0;
849 for (cpu = 0; cpu < nr_cpus; cpu++) {
850 if (pthread_create(&locking_threads[cpu], &attr,
851 locking_thread, (void *)cpu))
852 return 1;
853 if (bounces & BOUNCE_POLL) {
854 if (pthread_create(&uffd_threads[cpu], &attr,
855 uffd_poll_thread,
856 (void *)&uffd_stats[cpu]))
857 return 1;
858 } else {
859 if (pthread_create(&uffd_threads[cpu], &attr,
860 uffd_read_thread,
861 (void *)&uffd_stats[cpu]))
862 return 1;
863 pthread_mutex_lock(&uffd_read_mutex);
864 }
865 if (pthread_create(&background_threads[cpu], &attr,
866 background_thread, (void *)cpu))
867 return 1;
868 }
869 for (cpu = 0; cpu < nr_cpus; cpu++)
870 if (pthread_join(background_threads[cpu], NULL))
871 return 1;
872
873 /*
874 * Be strict and immediately zap area_src, the whole area has
875 * been transferred already by the background treads. The
876 * area_src could then be faulted in in a racy way by still
877 * running uffdio_threads reading zeropages after we zapped
878 * area_src (but they're guaranteed to get -EEXIST from
879 * UFFDIO_COPY without writing zero pages into area_dst
880 * because the background threads already completed).
881 */
882 uffd_test_ops->release_pages(area_src);
883
884 finished = 1;
885 for (cpu = 0; cpu < nr_cpus; cpu++)
886 if (pthread_join(locking_threads[cpu], NULL))
887 return 1;
888
889 for (cpu = 0; cpu < nr_cpus; cpu++) {
890 char c;
891 if (bounces & BOUNCE_POLL) {
892 if (write(pipefd[cpu*2+1], &c, 1) != 1)
893 err("pipefd write error");
894 if (pthread_join(uffd_threads[cpu],
895 (void *)&uffd_stats[cpu]))
896 return 1;
897 } else {
898 if (pthread_kill(uffd_threads[cpu], SIGUSR1))
899 return 1;
900 if (pthread_join(uffd_threads[cpu], NULL))
901 return 1;
902 }
903 }
904
905 return 0;
906 }
907
908 sigjmp_buf jbuf, *sigbuf;
909
sighndl(int sig,siginfo_t * siginfo,void * ptr)910 static void sighndl(int sig, siginfo_t *siginfo, void *ptr)
911 {
912 if (sig == SIGBUS) {
913 if (sigbuf)
914 siglongjmp(*sigbuf, 1);
915 abort();
916 }
917 }
918
919 /*
920 * For non-cooperative userfaultfd test we fork() a process that will
921 * generate pagefaults, will mremap the area monitored by the
922 * userfaultfd and at last this process will release the monitored
923 * area.
924 * For the anonymous and shared memory the area is divided into two
925 * parts, the first part is accessed before mremap, and the second
926 * part is accessed after mremap. Since hugetlbfs does not support
927 * mremap, the entire monitored area is accessed in a single pass for
928 * HUGETLB_TEST.
929 * The release of the pages currently generates event for shmem and
930 * anonymous memory (UFFD_EVENT_REMOVE), hence it is not checked
931 * for hugetlb.
932 * For signal test(UFFD_FEATURE_SIGBUS), signal_test = 1, we register
933 * monitored area, generate pagefaults and test that signal is delivered.
934 * Use UFFDIO_COPY to allocate missing page and retry. For signal_test = 2
935 * test robustness use case - we release monitored area, fork a process
936 * that will generate pagefaults and verify signal is generated.
937 * This also tests UFFD_FEATURE_EVENT_FORK event along with the signal
938 * feature. Using monitor thread, verify no userfault events are generated.
939 */
faulting_process(int signal_test)940 static int faulting_process(int signal_test)
941 {
942 unsigned long nr;
943 unsigned long long count;
944 unsigned long split_nr_pages;
945 unsigned long lastnr;
946 struct sigaction act;
947 volatile unsigned long signalled = 0;
948
949 if (test_type != TEST_HUGETLB)
950 split_nr_pages = (nr_pages + 1) / 2;
951 else
952 split_nr_pages = nr_pages;
953
954 if (signal_test) {
955 sigbuf = &jbuf;
956 memset(&act, 0, sizeof(act));
957 act.sa_sigaction = sighndl;
958 act.sa_flags = SA_SIGINFO;
959 if (sigaction(SIGBUS, &act, 0))
960 err("sigaction");
961 lastnr = (unsigned long)-1;
962 }
963
964 for (nr = 0; nr < split_nr_pages; nr++) {
965 volatile int steps = 1;
966 unsigned long offset = nr * page_size;
967
968 if (signal_test) {
969 if (sigsetjmp(*sigbuf, 1) != 0) {
970 if (steps == 1 && nr == lastnr)
971 err("Signal repeated");
972
973 lastnr = nr;
974 if (signal_test == 1) {
975 if (steps == 1) {
976 /* This is a MISSING request */
977 steps++;
978 if (copy_page(uffd, offset))
979 signalled++;
980 } else {
981 /* This is a WP request */
982 assert(steps == 2);
983 wp_range(uffd,
984 (__u64)area_dst +
985 offset,
986 page_size, false);
987 }
988 } else {
989 signalled++;
990 continue;
991 }
992 }
993 }
994
995 count = *area_count(area_dst, nr);
996 if (count != count_verify[nr])
997 err("nr %lu memory corruption %llu %llu\n",
998 nr, count, count_verify[nr]);
999 /*
1000 * Trigger write protection if there is by writing
1001 * the same value back.
1002 */
1003 *area_count(area_dst, nr) = count;
1004 }
1005
1006 if (signal_test)
1007 return signalled != split_nr_pages;
1008
1009 if (test_type == TEST_HUGETLB)
1010 return 0;
1011
1012 area_dst = mremap(area_dst, nr_pages * page_size, nr_pages * page_size,
1013 MREMAP_MAYMOVE | MREMAP_FIXED, area_src);
1014 if (area_dst == MAP_FAILED)
1015 err("mremap");
1016 /* Reset area_src since we just clobbered it */
1017 area_src = NULL;
1018
1019 for (; nr < nr_pages; nr++) {
1020 count = *area_count(area_dst, nr);
1021 if (count != count_verify[nr]) {
1022 err("nr %lu memory corruption %llu %llu\n",
1023 nr, count, count_verify[nr]);
1024 }
1025 /*
1026 * Trigger write protection if there is by writing
1027 * the same value back.
1028 */
1029 *area_count(area_dst, nr) = count;
1030 }
1031
1032 uffd_test_ops->release_pages(area_dst);
1033
1034 for (nr = 0; nr < nr_pages; nr++)
1035 if (my_bcmp(area_dst + nr * page_size, zeropage, page_size))
1036 err("nr %lu is not zero", nr);
1037
1038 return 0;
1039 }
1040
retry_uffdio_zeropage(int ufd,struct uffdio_zeropage * uffdio_zeropage,unsigned long offset)1041 static void retry_uffdio_zeropage(int ufd,
1042 struct uffdio_zeropage *uffdio_zeropage,
1043 unsigned long offset)
1044 {
1045 uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,
1046 uffdio_zeropage->range.len,
1047 offset);
1048 if (ioctl(ufd, UFFDIO_ZEROPAGE, uffdio_zeropage)) {
1049 if (uffdio_zeropage->zeropage != -EEXIST)
1050 err("UFFDIO_ZEROPAGE error: %"PRId64,
1051 (int64_t)uffdio_zeropage->zeropage);
1052 } else {
1053 err("UFFDIO_ZEROPAGE error: %"PRId64,
1054 (int64_t)uffdio_zeropage->zeropage);
1055 }
1056 }
1057
__uffdio_zeropage(int ufd,unsigned long offset,bool retry)1058 static int __uffdio_zeropage(int ufd, unsigned long offset, bool retry)
1059 {
1060 struct uffdio_zeropage uffdio_zeropage;
1061 int ret;
1062 bool has_zeropage = get_expected_ioctls(0) & (1 << _UFFDIO_ZEROPAGE);
1063 __s64 res;
1064
1065 if (offset >= nr_pages * page_size)
1066 err("unexpected offset %lu", offset);
1067 uffdio_zeropage.range.start = (unsigned long) area_dst + offset;
1068 uffdio_zeropage.range.len = page_size;
1069 uffdio_zeropage.mode = 0;
1070 ret = ioctl(ufd, UFFDIO_ZEROPAGE, &uffdio_zeropage);
1071 res = uffdio_zeropage.zeropage;
1072 if (ret) {
1073 /* real retval in ufdio_zeropage.zeropage */
1074 if (has_zeropage)
1075 err("UFFDIO_ZEROPAGE error: %"PRId64, (int64_t)res);
1076 else if (res != -EINVAL)
1077 err("UFFDIO_ZEROPAGE not -EINVAL");
1078 } else if (has_zeropage) {
1079 if (res != page_size) {
1080 err("UFFDIO_ZEROPAGE unexpected size");
1081 } else {
1082 if (test_uffdio_zeropage_eexist && retry) {
1083 test_uffdio_zeropage_eexist = false;
1084 retry_uffdio_zeropage(ufd, &uffdio_zeropage,
1085 offset);
1086 }
1087 return 1;
1088 }
1089 } else
1090 err("UFFDIO_ZEROPAGE succeeded");
1091
1092 return 0;
1093 }
1094
uffdio_zeropage(int ufd,unsigned long offset)1095 static int uffdio_zeropage(int ufd, unsigned long offset)
1096 {
1097 return __uffdio_zeropage(ufd, offset, false);
1098 }
1099
1100 /* exercise UFFDIO_ZEROPAGE */
userfaultfd_zeropage_test(void)1101 static int userfaultfd_zeropage_test(void)
1102 {
1103 struct uffdio_register uffdio_register;
1104
1105 printf("testing UFFDIO_ZEROPAGE: ");
1106 fflush(stdout);
1107
1108 uffd_test_ctx_init(0);
1109
1110 uffdio_register.range.start = (unsigned long) area_dst;
1111 uffdio_register.range.len = nr_pages * page_size;
1112 uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
1113 if (test_uffdio_wp)
1114 uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
1115 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
1116 err("register failure");
1117
1118 assert_expected_ioctls_present(
1119 uffdio_register.mode, uffdio_register.ioctls);
1120
1121 if (uffdio_zeropage(uffd, 0))
1122 if (my_bcmp(area_dst, zeropage, page_size))
1123 err("zeropage is not zero");
1124
1125 printf("done.\n");
1126 return 0;
1127 }
1128
userfaultfd_events_test(void)1129 static int userfaultfd_events_test(void)
1130 {
1131 struct uffdio_register uffdio_register;
1132 pthread_t uffd_mon;
1133 int err, features;
1134 pid_t pid;
1135 char c;
1136 struct uffd_stats stats = { 0 };
1137
1138 // All the syscalls below up to pthread_create will ensure that this
1139 // write is completed before, the uffd_thread sets it to true.
1140 ready_for_fork = false;
1141
1142 printf("testing events (fork, remap, remove): ");
1143 fflush(stdout);
1144
1145 features = UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_EVENT_REMAP |
1146 UFFD_FEATURE_EVENT_REMOVE;
1147 uffd_test_ctx_init(features);
1148
1149 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
1150
1151 uffdio_register.range.start = (unsigned long) area_dst;
1152 uffdio_register.range.len = nr_pages * page_size;
1153 uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
1154 if (test_uffdio_wp)
1155 uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
1156 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
1157 err("register failure");
1158
1159 assert_expected_ioctls_present(
1160 uffdio_register.mode, uffdio_register.ioctls);
1161
1162 if (pthread_create(&uffd_mon, &attr, uffd_poll_thread, &stats))
1163 err("uffd_poll_thread create");
1164
1165 // Wait for the poll_thread to start executing before forking. This is
1166 // required to avoid a deadlock, which can happen if poll_thread doesn't
1167 // start getting executed by the time fork is invoked.
1168 while (!ready_for_fork);
1169
1170 pid = fork();
1171 if (pid < 0)
1172 err("fork");
1173
1174 if (!pid)
1175 exit(faulting_process(0));
1176
1177 waitpid(pid, &err, 0);
1178 if (err)
1179 err("faulting process failed");
1180 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1181 err("pipe write");
1182 if (pthread_join(uffd_mon, NULL))
1183 return 1;
1184
1185 uffd_stats_report(&stats, 1);
1186
1187 return stats.missing_faults != nr_pages;
1188 }
1189
userfaultfd_sig_test(void)1190 static int userfaultfd_sig_test(void)
1191 {
1192 struct uffdio_register uffdio_register;
1193 unsigned long userfaults;
1194 pthread_t uffd_mon;
1195 int err, features;
1196 pid_t pid;
1197 char c;
1198 struct uffd_stats stats = { 0 };
1199
1200 // All the syscalls below up to pthread_create will ensure that this
1201 // write is completed before, the uffd_thread sets it to true.
1202 ready_for_fork = false;
1203
1204 printf("testing signal delivery: ");
1205 fflush(stdout);
1206
1207 features = UFFD_FEATURE_EVENT_FORK|UFFD_FEATURE_SIGBUS;
1208 uffd_test_ctx_init(features);
1209
1210 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
1211
1212 uffdio_register.range.start = (unsigned long) area_dst;
1213 uffdio_register.range.len = nr_pages * page_size;
1214 uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
1215 if (test_uffdio_wp)
1216 uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
1217 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
1218 err("register failure");
1219
1220 assert_expected_ioctls_present(
1221 uffdio_register.mode, uffdio_register.ioctls);
1222
1223 if (faulting_process(1))
1224 err("faulting process failed");
1225
1226 uffd_test_ops->release_pages(area_dst);
1227
1228 if (pthread_create(&uffd_mon, &attr, uffd_poll_thread, &stats))
1229 err("uffd_poll_thread create");
1230
1231 // Wait for the poll_thread to start executing before forking. This is
1232 // required to avoid a deadlock, which can happen if poll_thread doesn't
1233 // start getting executed by the time fork is invoked.
1234 while (!ready_for_fork);
1235
1236 pid = fork();
1237 if (pid < 0)
1238 err("fork");
1239
1240 if (!pid)
1241 exit(faulting_process(2));
1242
1243 waitpid(pid, &err, 0);
1244 if (err)
1245 err("faulting process failed");
1246 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1247 err("pipe write");
1248 if (pthread_join(uffd_mon, (void **)&userfaults))
1249 return 1;
1250
1251 printf("done.\n");
1252 if (userfaults)
1253 err("Signal test failed, userfaults: %ld", userfaults);
1254
1255 return userfaults != 0;
1256 }
1257
userfaultfd_minor_test(void)1258 static int userfaultfd_minor_test(void)
1259 {
1260 struct uffdio_register uffdio_register;
1261 unsigned long p;
1262 pthread_t uffd_mon;
1263 uint8_t expected_byte;
1264 void *expected_page;
1265 char c;
1266 struct uffd_stats stats = { 0 };
1267
1268 if (!test_uffdio_minor)
1269 return 0;
1270
1271 printf("testing minor faults: ");
1272 fflush(stdout);
1273
1274 uffd_test_ctx_init(uffd_minor_feature());
1275
1276 uffdio_register.range.start = (unsigned long)area_dst_alias;
1277 uffdio_register.range.len = nr_pages * page_size;
1278 uffdio_register.mode = UFFDIO_REGISTER_MODE_MINOR;
1279 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
1280 err("register failure");
1281
1282 assert_expected_ioctls_present(
1283 uffdio_register.mode, uffdio_register.ioctls);
1284
1285 /*
1286 * After registering with UFFD, populate the non-UFFD-registered side of
1287 * the shared mapping. This should *not* trigger any UFFD minor faults.
1288 */
1289 for (p = 0; p < nr_pages; ++p) {
1290 memset(area_dst + (p * page_size), p % ((uint8_t)-1),
1291 page_size);
1292 }
1293
1294 if (pthread_create(&uffd_mon, &attr, uffd_poll_thread, &stats))
1295 err("uffd_poll_thread create");
1296
1297 /*
1298 * Read each of the pages back using the UFFD-registered mapping. We
1299 * expect that the first time we touch a page, it will result in a minor
1300 * fault. uffd_poll_thread will resolve the fault by bit-flipping the
1301 * page's contents, and then issuing a CONTINUE ioctl.
1302 */
1303
1304 if (posix_memalign(&expected_page, page_size, page_size))
1305 err("out of memory");
1306
1307 for (p = 0; p < nr_pages; ++p) {
1308 expected_byte = ~((uint8_t)(p % ((uint8_t)-1)));
1309 memset(expected_page, expected_byte, page_size);
1310 if (my_bcmp(expected_page, area_dst_alias + (p * page_size),
1311 page_size))
1312 err("unexpected page contents after minor fault");
1313 }
1314
1315 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1316 err("pipe write");
1317 if (pthread_join(uffd_mon, NULL))
1318 return 1;
1319
1320 uffd_stats_report(&stats, 1);
1321
1322 return stats.missing_faults != 0 || stats.minor_faults != nr_pages;
1323 }
1324
1325 #define BIT_ULL(nr) (1ULL << (nr))
1326 #define PM_SOFT_DIRTY BIT_ULL(55)
1327 #define PM_MMAP_EXCLUSIVE BIT_ULL(56)
1328 #define PM_UFFD_WP BIT_ULL(57)
1329 #define PM_FILE BIT_ULL(61)
1330 #define PM_SWAP BIT_ULL(62)
1331 #define PM_PRESENT BIT_ULL(63)
1332
1333 /*
1334 * b/232026677
1335 * pagemap not compatible with < 5.14
1336 */
1337 #ifndef __ANDROID__
pagemap_open(void)1338 static int pagemap_open(void)
1339 {
1340 int fd = open("/proc/self/pagemap", O_RDONLY);
1341
1342 if (fd < 0)
1343 err("open pagemap");
1344
1345 return fd;
1346 }
1347
pagemap_read_vaddr(int fd,void * vaddr)1348 static uint64_t pagemap_read_vaddr(int fd, void *vaddr)
1349 {
1350 uint64_t value;
1351 int ret;
1352
1353 ret = pread(fd, &value, sizeof(uint64_t),
1354 ((uint64_t)vaddr >> 12) * sizeof(uint64_t));
1355 if (ret != sizeof(uint64_t))
1356 err("pread() on pagemap failed");
1357
1358 return value;
1359 }
1360
1361 /* This macro let __LINE__ works in err() */
1362 #define pagemap_check_wp(value, wp) do { \
1363 if (!!(value & PM_UFFD_WP) != wp) \
1364 err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \
1365 } while (0)
1366
pagemap_test_fork(bool present)1367 static int pagemap_test_fork(bool present)
1368 {
1369 pid_t child = fork();
1370 uint64_t value;
1371 int fd, result;
1372
1373 if (!child) {
1374 /* Open the pagemap fd of the child itself */
1375 fd = pagemap_open();
1376 value = pagemap_read_vaddr(fd, area_dst);
1377 /*
1378 * After fork() uffd-wp bit should be gone as long as we're
1379 * without UFFD_FEATURE_EVENT_FORK
1380 */
1381 pagemap_check_wp(value, false);
1382 /* Succeed */
1383 exit(0);
1384 }
1385 waitpid(child, &result, 0);
1386 return result;
1387 }
1388
userfaultfd_pagemap_test(unsigned int test_pgsize)1389 static void userfaultfd_pagemap_test(unsigned int test_pgsize)
1390 {
1391 struct uffdio_register uffdio_register;
1392 int pagemap_fd;
1393 uint64_t value;
1394
1395 /* Pagemap tests uffd-wp only */
1396 if (!test_uffdio_wp)
1397 return;
1398
1399 /* Not enough memory to test this page size */
1400 if (test_pgsize > nr_pages * page_size)
1401 return;
1402
1403 printf("testing uffd-wp with pagemap (pgsize=%u): ", test_pgsize);
1404 /* Flush so it doesn't flush twice in parent/child later */
1405 fflush(stdout);
1406
1407 uffd_test_ctx_init(0);
1408
1409 if (test_pgsize > page_size) {
1410 /* This is a thp test */
1411 if (madvise(area_dst, nr_pages * page_size, MADV_HUGEPAGE))
1412 err("madvise(MADV_HUGEPAGE) failed");
1413 } else if (test_pgsize == page_size) {
1414 /* This is normal page test; force no thp */
1415 if (madvise(area_dst, nr_pages * page_size, MADV_NOHUGEPAGE))
1416 err("madvise(MADV_NOHUGEPAGE) failed");
1417 }
1418
1419 uffdio_register.range.start = (unsigned long) area_dst;
1420 uffdio_register.range.len = nr_pages * page_size;
1421 uffdio_register.mode = UFFDIO_REGISTER_MODE_WP;
1422 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
1423 err("register failed");
1424
1425 pagemap_fd = pagemap_open();
1426
1427 /* Touch the page */
1428 *area_dst = 1;
1429 wp_range(uffd, (uint64_t)area_dst, test_pgsize, true);
1430 value = pagemap_read_vaddr(pagemap_fd, area_dst);
1431 pagemap_check_wp(value, true);
1432 /* Make sure uffd-wp bit dropped when fork */
1433 if (pagemap_test_fork(true))
1434 err("Detected stall uffd-wp bit in child");
1435
1436 /* Exclusive required or PAGEOUT won't work */
1437 if (!(value & PM_MMAP_EXCLUSIVE))
1438 err("multiple mapping detected: 0x%"PRIx64, value);
1439
1440 if (madvise(area_dst, test_pgsize, MADV_PAGEOUT))
1441 err("madvise(MADV_PAGEOUT) failed");
1442
1443 /* Uffd-wp should persist even swapped out */
1444 value = pagemap_read_vaddr(pagemap_fd, area_dst);
1445 pagemap_check_wp(value, true);
1446 /* Make sure uffd-wp bit dropped when fork */
1447 if (pagemap_test_fork(false))
1448 err("Detected stall uffd-wp bit in child");
1449
1450 /* Unprotect; this tests swap pte modifications */
1451 wp_range(uffd, (uint64_t)area_dst, page_size, false);
1452 value = pagemap_read_vaddr(pagemap_fd, area_dst);
1453 pagemap_check_wp(value, false);
1454
1455 /* Fault in the page from disk */
1456 *area_dst = 2;
1457 value = pagemap_read_vaddr(pagemap_fd, area_dst);
1458 pagemap_check_wp(value, false);
1459
1460 close(pagemap_fd);
1461 printf("done\n");
1462 }
1463 #endif
1464
userfaultfd_stress(void)1465 static int userfaultfd_stress(void)
1466 {
1467 void *area;
1468 char *tmp_area;
1469 unsigned long nr;
1470 struct uffdio_register uffdio_register;
1471 struct sigaction act;
1472 struct uffd_stats uffd_stats[nr_cpus];
1473
1474 uffd_test_ctx_init(0);
1475
1476 if (posix_memalign(&area, page_size, page_size))
1477 err("out of memory");
1478 zeropage = area;
1479 bzero(zeropage, page_size);
1480
1481 pthread_mutex_lock(&uffd_read_mutex);
1482
1483 pthread_attr_init(&attr);
1484 pthread_attr_setstacksize(&attr, 16*1024*1024);
1485
1486 // For handling thread termination of read thread in the absense of
1487 // pthread_cancel().
1488 pthread_key_create(&long_jmp_key, NULL);
1489 memset(&act, 0, sizeof(act));
1490 act.sa_sigaction = sigusr1_handler;
1491 act.sa_flags = SA_SIGINFO;
1492 if (sigaction(SIGUSR1, &act, 0)) {
1493 perror("sigaction");
1494 return 1;
1495 }
1496
1497 while (bounces--) {
1498 printf("bounces: %d, mode:", bounces);
1499 if (bounces & BOUNCE_RANDOM)
1500 printf(" rnd");
1501 if (bounces & BOUNCE_RACINGFAULTS)
1502 printf(" racing");
1503 if (bounces & BOUNCE_VERIFY)
1504 printf(" ver");
1505 if (bounces & BOUNCE_POLL)
1506 printf(" poll");
1507 else
1508 printf(" read");
1509 printf(", ");
1510 fflush(stdout);
1511
1512 if (bounces & BOUNCE_POLL)
1513 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
1514 else
1515 fcntl(uffd, F_SETFL, uffd_flags & ~O_NONBLOCK);
1516
1517 /* register */
1518 uffdio_register.range.start = (unsigned long) area_dst;
1519 uffdio_register.range.len = nr_pages * page_size;
1520 uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
1521 if (test_uffdio_wp)
1522 uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
1523 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
1524 err("register failure");
1525 assert_expected_ioctls_present(
1526 uffdio_register.mode, uffdio_register.ioctls);
1527
1528 if (area_dst_alias) {
1529 uffdio_register.range.start = (unsigned long)
1530 area_dst_alias;
1531 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
1532 err("register failure alias");
1533 }
1534
1535 /*
1536 * The madvise done previously isn't enough: some
1537 * uffd_thread could have read userfaults (one of
1538 * those already resolved by the background thread)
1539 * and it may be in the process of calling
1540 * UFFDIO_COPY. UFFDIO_COPY will read the zapped
1541 * area_src and it would map a zero page in it (of
1542 * course such a UFFDIO_COPY is perfectly safe as it'd
1543 * return -EEXIST). The problem comes at the next
1544 * bounce though: that racing UFFDIO_COPY would
1545 * generate zeropages in the area_src, so invalidating
1546 * the previous MADV_DONTNEED. Without this additional
1547 * MADV_DONTNEED those zeropages leftovers in the
1548 * area_src would lead to -EEXIST failure during the
1549 * next bounce, effectively leaving a zeropage in the
1550 * area_dst.
1551 *
1552 * Try to comment this out madvise to see the memory
1553 * corruption being caught pretty quick.
1554 *
1555 * khugepaged is also inhibited to collapse THP after
1556 * MADV_DONTNEED only after the UFFDIO_REGISTER, so it's
1557 * required to MADV_DONTNEED here.
1558 */
1559 uffd_test_ops->release_pages(area_dst);
1560
1561 uffd_stats_reset(uffd_stats, nr_cpus);
1562
1563 /* bounce pass */
1564 if (stress(uffd_stats))
1565 return 1;
1566
1567 /* Clear all the write protections if there is any */
1568 if (test_uffdio_wp)
1569 wp_range(uffd, (unsigned long)area_dst,
1570 nr_pages * page_size, false);
1571
1572 /* unregister */
1573 if (ioctl(uffd, UFFDIO_UNREGISTER, &uffdio_register.range))
1574 err("unregister failure");
1575 if (area_dst_alias) {
1576 uffdio_register.range.start = (unsigned long) area_dst;
1577 if (ioctl(uffd, UFFDIO_UNREGISTER,
1578 &uffdio_register.range))
1579 err("unregister failure alias");
1580 }
1581
1582 /* verification */
1583 if (bounces & BOUNCE_VERIFY)
1584 for (nr = 0; nr < nr_pages; nr++)
1585 if (*area_count(area_dst, nr) != count_verify[nr])
1586 err("error area_count %llu %llu %lu\n",
1587 *area_count(area_src, nr),
1588 count_verify[nr], nr);
1589
1590 /* prepare next bounce */
1591 tmp_area = area_src;
1592 area_src = area_dst;
1593 area_dst = tmp_area;
1594
1595 tmp_area = area_src_alias;
1596 area_src_alias = area_dst_alias;
1597 area_dst_alias = tmp_area;
1598
1599 uffd_stats_report(uffd_stats, nr_cpus);
1600 }
1601
1602 /*
1603 * b/232026677
1604 * pagemap not compatible with < 5.14
1605 */
1606 #ifndef __ANDROID__
1607 if (test_type == TEST_ANON) {
1608 /*
1609 * shmem/hugetlb won't be able to run since they have different
1610 * behavior on fork() (file-backed memory normally drops ptes
1611 * directly when fork), meanwhile the pagemap test will verify
1612 * pgtable entry of fork()ed child.
1613 */
1614 userfaultfd_pagemap_test(page_size);
1615 /*
1616 * Hard-code for x86_64 for now for 2M THP, as x86_64 is
1617 * currently the only one that supports uffd-wp
1618 */
1619 userfaultfd_pagemap_test(page_size * 512);
1620 }
1621 #endif
1622
1623 pthread_key_delete(long_jmp_key);
1624
1625 return userfaultfd_zeropage_test() || userfaultfd_sig_test()
1626 || userfaultfd_events_test() || userfaultfd_minor_test();
1627 }
1628
1629 /*
1630 * Copied from mlock2-tests.c
1631 */
default_huge_page_size(void)1632 unsigned long default_huge_page_size(void)
1633 {
1634 unsigned long hps = 0;
1635 char *line = NULL;
1636 size_t linelen = 0;
1637 FILE *f = fopen("/proc/meminfo", "r");
1638
1639 if (!f)
1640 return 0;
1641 while (getline(&line, &linelen, f) > 0) {
1642 if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) {
1643 hps <<= 10;
1644 break;
1645 }
1646 }
1647
1648 free(line);
1649 fclose(f);
1650 return hps;
1651 }
1652
set_test_type(const char * type)1653 static void set_test_type(const char *type)
1654 {
1655 /* b/234150821
1656 * UFFD_FEATURE_PAGEFAULT_FLAG_WP unsupported in kernel <5.7
1657 */
1658 #ifdef __ANDROID__
1659 uint64_t features = (
1660 UFFD_FEATURE_EVENT_FORK | \
1661 UFFD_FEATURE_EVENT_REMAP | \
1662 UFFD_FEATURE_EVENT_REMOVE | \
1663 UFFD_FEATURE_EVENT_UNMAP | \
1664 UFFD_FEATURE_MISSING_HUGETLBFS | \
1665 UFFD_FEATURE_MISSING_SHMEM | \
1666 UFFD_FEATURE_SIGBUS | \
1667 UFFD_FEATURE_THREAD_ID | \
1668 UFFD_FEATURE_MINOR_HUGETLBFS | \
1669 UFFD_FEATURE_MINOR_SHMEM);
1670 #else
1671 uint64_t features = UFFD_API_FEATURES;
1672 #endif
1673
1674 if (!strcmp(type, "anon")) {
1675 test_type = TEST_ANON;
1676 uffd_test_ops = &anon_uffd_test_ops;
1677 /* Only enable write-protect test for anonymous test */
1678 test_uffdio_wp = true;
1679 } else if (!strcmp(type, "hugetlb")) {
1680 test_type = TEST_HUGETLB;
1681 uffd_test_ops = &hugetlb_uffd_test_ops;
1682 } else if (!strcmp(type, "hugetlb_shared")) {
1683 map_shared = true;
1684 test_type = TEST_HUGETLB;
1685 uffd_test_ops = &hugetlb_uffd_test_ops;
1686 /* Minor faults require shared hugetlb; only enable here. */
1687 test_uffdio_minor = true;
1688 } else if (!strcmp(type, "shmem")) {
1689 map_shared = true;
1690 test_type = TEST_SHMEM;
1691 uffd_test_ops = &shmem_uffd_test_ops;
1692 test_uffdio_minor = true;
1693 } else {
1694 err("Unknown test type: %s", type);
1695 }
1696
1697 if (test_type == TEST_HUGETLB)
1698 page_size = default_huge_page_size();
1699 else
1700 page_size = sysconf(_SC_PAGE_SIZE);
1701
1702 if (!page_size)
1703 err("Unable to determine page size");
1704 if ((unsigned long) area_count(NULL, 0) + sizeof(unsigned long long) * 2
1705 > page_size)
1706 err("Impossible to run this test");
1707
1708 /*
1709 * Whether we can test certain features depends not just on test type,
1710 * but also on whether or not this particular kernel supports the
1711 * feature.
1712 */
1713
1714 userfaultfd_open(&features);
1715
1716 test_uffdio_wp = test_uffdio_wp &&
1717 (features & UFFD_FEATURE_PAGEFAULT_FLAG_WP);
1718 test_uffdio_minor = test_uffdio_minor &&
1719 (features & uffd_minor_feature());
1720
1721 close(uffd);
1722 uffd = -1;
1723 }
1724
sigalrm(int sig)1725 static void sigalrm(int sig)
1726 {
1727 if (sig != SIGALRM)
1728 abort();
1729 test_uffdio_copy_eexist = true;
1730 test_uffdio_zeropage_eexist = true;
1731 alarm(ALARM_INTERVAL_SECS);
1732 }
1733
main(int argc,char ** argv)1734 int main(int argc, char **argv)
1735 {
1736 char randstate[64];
1737 unsigned int seed;
1738
1739 if (argc < 4)
1740 usage();
1741
1742 if (signal(SIGALRM, sigalrm) == SIG_ERR)
1743 err("failed to arm SIGALRM");
1744 alarm(ALARM_INTERVAL_SECS);
1745
1746 set_test_type(argv[1]);
1747
1748 nr_cpus = sysconf(_SC_NPROCESSORS_ONLN);
1749 nr_pages_per_cpu = atol(argv[2]) * 1024*1024 / page_size /
1750 nr_cpus;
1751 if (!nr_pages_per_cpu) {
1752 _err("invalid MiB");
1753 usage();
1754 }
1755
1756 bounces = atoi(argv[3]);
1757 if (bounces <= 0) {
1758 _err("invalid bounces");
1759 usage();
1760 }
1761 nr_pages = nr_pages_per_cpu * nr_cpus;
1762
1763 if (test_type == TEST_HUGETLB) {
1764 if (argc < 5)
1765 usage();
1766 huge_fd = open(argv[4], O_CREAT | O_RDWR, 0755);
1767 if (huge_fd < 0)
1768 err("Open of %s failed", argv[4]);
1769 if (ftruncate(huge_fd, 0))
1770 err("ftruncate %s to size 0 failed", argv[4]);
1771 } else if (test_type == TEST_SHMEM) {
1772 shm_fd = memfd_create(argv[0], 0);
1773 if (shm_fd < 0)
1774 err("memfd_create");
1775 if (ftruncate(shm_fd, nr_pages * page_size * 2))
1776 err("ftruncate");
1777 if (fallocate(shm_fd,
1778 FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 0,
1779 nr_pages * page_size * 2))
1780 err("fallocate");
1781 }
1782
1783 seed = (unsigned int) time(NULL);
1784 bzero(&randstate, sizeof(randstate));
1785 if (!initstate(seed, randstate, sizeof(randstate)))
1786 fprintf(stderr, "srandom_r error\n"), exit(1);
1787
1788 printf("nr_pages: %lu, nr_pages_per_cpu: %lu\n",
1789 nr_pages, nr_pages_per_cpu);
1790 return userfaultfd_stress();
1791 }
1792
1793 #else /* __NR_userfaultfd */
1794
1795 #warning "missing __NR_userfaultfd definition"
1796
main(void)1797 int main(void)
1798 {
1799 printf("skip: Skipping userfaultfd test (missing __NR_userfaultfd)\n");
1800 return KSFT_SKIP;
1801 }
1802
1803 #endif /* __NR_userfaultfd */
1804