1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Stress userfaultfd syscall.
4 *
5 * Copyright (C) 2015 Red Hat, Inc.
6 *
7 * This test allocates two virtual areas and bounces the physical
8 * memory across the two virtual areas (from area_src to area_dst)
9 * using userfaultfd.
10 *
11 * There are three threads running per CPU:
12 *
13 * 1) one per-CPU thread takes a per-page pthread_mutex in a random
14 * page of the area_dst (while the physical page may still be in
15 * area_src), and increments a per-page counter in the same page,
16 * and checks its value against a verification region.
17 *
18 * 2) another per-CPU thread handles the userfaults generated by
19 * thread 1 above. userfaultfd blocking reads or poll() modes are
20 * exercised interleaved.
21 *
22 * 3) one last per-CPU thread transfers the memory in the background
23 * at maximum bandwidth (if not already transferred by thread
24 * 2). Each cpu thread takes cares of transferring a portion of the
25 * area.
26 *
27 * When all threads of type 3 completed the transfer, one bounce is
28 * complete. area_src and area_dst are then swapped. All threads are
29 * respawned and so the bounce is immediately restarted in the
30 * opposite direction.
31 *
32 * per-CPU threads 1 by triggering userfaults inside
33 * pthread_mutex_lock will also verify the atomicity of the memory
34 * transfer (UFFDIO_COPY).
35 */
36
37 #define _GNU_SOURCE
38 #include <stdio.h>
39 #include <errno.h>
40 #include <unistd.h>
41 #include <stdlib.h>
42 #include <sys/types.h>
43 #include <sys/stat.h>
44 #include <fcntl.h>
45 #include <time.h>
46 #include <signal.h>
47 #include <poll.h>
48 #include <string.h>
49 #include <linux/mman.h>
50 #include <sys/mman.h>
51 #include <sys/syscall.h>
52 #include <sys/ioctl.h>
53 #include <sys/wait.h>
54 #include <pthread.h>
55 #include <linux/userfaultfd.h>
56 #include <setjmp.h>
57 #include <stdbool.h>
58 #include <assert.h>
59
60 #include "../kselftest.h"
61
62 #ifdef __NR_userfaultfd
63
64 static unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size;
65
66 #define BOUNCE_RANDOM (1<<0)
67 #define BOUNCE_RACINGFAULTS (1<<1)
68 #define BOUNCE_VERIFY (1<<2)
69 #define BOUNCE_POLL (1<<3)
70 static int bounces;
71
72 #define TEST_ANON 1
73 #define TEST_HUGETLB 2
74 #define TEST_SHMEM 3
75 static int test_type;
76
77 /* exercise the test_uffdio_*_eexist every ALARM_INTERVAL_SECS */
78 #define ALARM_INTERVAL_SECS 10
79 static volatile bool test_uffdio_copy_eexist = true;
80 static volatile bool test_uffdio_zeropage_eexist = true;
81 /* Whether to test uffd write-protection */
82 static bool test_uffdio_wp = false;
83
84 static bool map_shared;
85 static int huge_fd;
86 static char *huge_fd_off0;
87 static unsigned long long *count_verify;
88 static int uffd, uffd_flags, finished, *pipefd;
89 static char *area_src, *area_src_alias, *area_dst, *area_dst_alias;
90 static char *zeropage;
91 pthread_attr_t attr;
92
93 /* Userfaultfd test statistics */
94 struct uffd_stats {
95 int cpu;
96 unsigned long missing_faults;
97 unsigned long wp_faults;
98 };
99
100 /* pthread_mutex_t starts at page offset 0 */
101 #define area_mutex(___area, ___nr) \
102 ((pthread_mutex_t *) ((___area) + (___nr)*page_size))
103 /*
104 * count is placed in the page after pthread_mutex_t naturally aligned
105 * to avoid non alignment faults on non-x86 archs.
106 */
107 #define area_count(___area, ___nr) \
108 ((volatile unsigned long long *) ((unsigned long) \
109 ((___area) + (___nr)*page_size + \
110 sizeof(pthread_mutex_t) + \
111 sizeof(unsigned long long) - 1) & \
112 ~(unsigned long)(sizeof(unsigned long long) \
113 - 1)))
114
115 const char *examples =
116 "# Run anonymous memory test on 100MiB region with 99999 bounces:\n"
117 "./userfaultfd anon 100 99999\n\n"
118 "# Run share memory test on 1GiB region with 99 bounces:\n"
119 "./userfaultfd shmem 1000 99\n\n"
120 "# Run hugetlb memory test on 256MiB region with 50 bounces (using /dev/hugepages/hugefile):\n"
121 "./userfaultfd hugetlb 256 50 /dev/hugepages/hugefile\n\n"
122 "# Run the same hugetlb test but using shmem:\n"
123 "./userfaultfd hugetlb_shared 256 50 /dev/hugepages/hugefile\n\n"
124 "# 10MiB-~6GiB 999 bounces anonymous test, "
125 "continue forever unless an error triggers\n"
126 "while ./userfaultfd anon $[RANDOM % 6000 + 10] 999; do true; done\n\n";
127
usage(void)128 static void usage(void)
129 {
130 fprintf(stderr, "\nUsage: ./userfaultfd <test type> <MiB> <bounces> "
131 "[hugetlbfs_file]\n\n");
132 fprintf(stderr, "Supported <test type>: anon, hugetlb, "
133 "hugetlb_shared, shmem\n\n");
134 fprintf(stderr, "Examples:\n\n");
135 fprintf(stderr, "%s", examples);
136 exit(1);
137 }
138
uffd_stats_reset(struct uffd_stats * uffd_stats,unsigned long n_cpus)139 static void uffd_stats_reset(struct uffd_stats *uffd_stats,
140 unsigned long n_cpus)
141 {
142 int i;
143
144 for (i = 0; i < n_cpus; i++) {
145 uffd_stats[i].cpu = i;
146 uffd_stats[i].missing_faults = 0;
147 uffd_stats[i].wp_faults = 0;
148 }
149 }
150
uffd_stats_report(struct uffd_stats * stats,int n_cpus)151 static void uffd_stats_report(struct uffd_stats *stats, int n_cpus)
152 {
153 int i;
154 unsigned long long miss_total = 0, wp_total = 0;
155
156 for (i = 0; i < n_cpus; i++) {
157 miss_total += stats[i].missing_faults;
158 wp_total += stats[i].wp_faults;
159 }
160
161 printf("userfaults: %llu missing (", miss_total);
162 for (i = 0; i < n_cpus; i++)
163 printf("%lu+", stats[i].missing_faults);
164 printf("\b), %llu wp (", wp_total);
165 for (i = 0; i < n_cpus; i++)
166 printf("%lu+", stats[i].wp_faults);
167 printf("\b)\n");
168 }
169
anon_release_pages(char * rel_area)170 static int anon_release_pages(char *rel_area)
171 {
172 int ret = 0;
173
174 if (madvise(rel_area, nr_pages * page_size, MADV_DONTNEED)) {
175 perror("madvise");
176 ret = 1;
177 }
178
179 return ret;
180 }
181
anon_allocate_area(void ** alloc_area)182 static void anon_allocate_area(void **alloc_area)
183 {
184 *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
185 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
186 if (*alloc_area == MAP_FAILED) {
187 fprintf(stderr, "mmap of anonymous memory failed");
188 *alloc_area = NULL;
189 }
190 }
191
noop_alias_mapping(__u64 * start,size_t len,unsigned long offset)192 static void noop_alias_mapping(__u64 *start, size_t len, unsigned long offset)
193 {
194 }
195
196 /* HugeTLB memory */
hugetlb_release_pages(char * rel_area)197 static int hugetlb_release_pages(char *rel_area)
198 {
199 int ret = 0;
200
201 if (fallocate(huge_fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
202 rel_area == huge_fd_off0 ? 0 :
203 nr_pages * page_size,
204 nr_pages * page_size)) {
205 perror("fallocate");
206 ret = 1;
207 }
208
209 return ret;
210 }
211
hugetlb_allocate_area(void ** alloc_area)212 static void hugetlb_allocate_area(void **alloc_area)
213 {
214 void *area_alias = NULL;
215 char **alloc_area_alias;
216
217 *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
218 (map_shared ? MAP_SHARED : MAP_PRIVATE) |
219 MAP_HUGETLB,
220 huge_fd, *alloc_area == area_src ? 0 :
221 nr_pages * page_size);
222 if (*alloc_area == MAP_FAILED) {
223 perror("mmap of hugetlbfs file failed");
224 goto fail;
225 }
226
227 if (map_shared) {
228 area_alias = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
229 MAP_SHARED | MAP_HUGETLB,
230 huge_fd, *alloc_area == area_src ? 0 :
231 nr_pages * page_size);
232 if (area_alias == MAP_FAILED) {
233 perror("mmap of hugetlb file alias failed");
234 goto fail_munmap;
235 }
236 }
237
238 if (*alloc_area == area_src) {
239 huge_fd_off0 = *alloc_area;
240 alloc_area_alias = &area_src_alias;
241 } else {
242 alloc_area_alias = &area_dst_alias;
243 }
244 if (area_alias)
245 *alloc_area_alias = area_alias;
246
247 return;
248
249 fail_munmap:
250 if (munmap(*alloc_area, nr_pages * page_size) < 0) {
251 perror("hugetlb munmap");
252 exit(1);
253 }
254 fail:
255 *alloc_area = NULL;
256 }
257
hugetlb_alias_mapping(__u64 * start,size_t len,unsigned long offset)258 static void hugetlb_alias_mapping(__u64 *start, size_t len, unsigned long offset)
259 {
260 if (!map_shared)
261 return;
262 /*
263 * We can't zap just the pagetable with hugetlbfs because
264 * MADV_DONTEED won't work. So exercise -EEXIST on a alias
265 * mapping where the pagetables are not established initially,
266 * this way we'll exercise the -EEXEC at the fs level.
267 */
268 *start = (unsigned long) area_dst_alias + offset;
269 }
270
271 /* Shared memory */
shmem_release_pages(char * rel_area)272 static int shmem_release_pages(char *rel_area)
273 {
274 int ret = 0;
275
276 if (madvise(rel_area, nr_pages * page_size, MADV_REMOVE)) {
277 perror("madvise");
278 ret = 1;
279 }
280
281 return ret;
282 }
283
shmem_allocate_area(void ** alloc_area)284 static void shmem_allocate_area(void **alloc_area)
285 {
286 *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
287 MAP_ANONYMOUS | MAP_SHARED, -1, 0);
288 if (*alloc_area == MAP_FAILED) {
289 fprintf(stderr, "shared memory mmap failed\n");
290 *alloc_area = NULL;
291 }
292 }
293
294 struct uffd_test_ops {
295 unsigned long expected_ioctls;
296 void (*allocate_area)(void **alloc_area);
297 int (*release_pages)(char *rel_area);
298 void (*alias_mapping)(__u64 *start, size_t len, unsigned long offset);
299 };
300
301 #define SHMEM_EXPECTED_IOCTLS ((1 << _UFFDIO_WAKE) | \
302 (1 << _UFFDIO_COPY) | \
303 (1 << _UFFDIO_ZEROPAGE))
304
305 #define ANON_EXPECTED_IOCTLS ((1 << _UFFDIO_WAKE) | \
306 (1 << _UFFDIO_COPY) | \
307 (1 << _UFFDIO_ZEROPAGE) | \
308 (1 << _UFFDIO_WRITEPROTECT))
309
310 static struct uffd_test_ops anon_uffd_test_ops = {
311 .expected_ioctls = ANON_EXPECTED_IOCTLS,
312 .allocate_area = anon_allocate_area,
313 .release_pages = anon_release_pages,
314 .alias_mapping = noop_alias_mapping,
315 };
316
317 static struct uffd_test_ops shmem_uffd_test_ops = {
318 .expected_ioctls = SHMEM_EXPECTED_IOCTLS,
319 .allocate_area = shmem_allocate_area,
320 .release_pages = shmem_release_pages,
321 .alias_mapping = noop_alias_mapping,
322 };
323
324 static struct uffd_test_ops hugetlb_uffd_test_ops = {
325 .expected_ioctls = UFFD_API_RANGE_IOCTLS_BASIC,
326 .allocate_area = hugetlb_allocate_area,
327 .release_pages = hugetlb_release_pages,
328 .alias_mapping = hugetlb_alias_mapping,
329 };
330
331 static struct uffd_test_ops *uffd_test_ops;
332
my_bcmp(char * str1,char * str2,size_t n)333 static int my_bcmp(char *str1, char *str2, size_t n)
334 {
335 unsigned long i;
336 for (i = 0; i < n; i++)
337 if (str1[i] != str2[i])
338 return 1;
339 return 0;
340 }
341
wp_range(int ufd,__u64 start,__u64 len,bool wp)342 static void wp_range(int ufd, __u64 start, __u64 len, bool wp)
343 {
344 struct uffdio_writeprotect prms = { 0 };
345
346 /* Write protection page faults */
347 prms.range.start = start;
348 prms.range.len = len;
349 /* Undo write-protect, do wakeup after that */
350 prms.mode = wp ? UFFDIO_WRITEPROTECT_MODE_WP : 0;
351
352 if (ioctl(ufd, UFFDIO_WRITEPROTECT, &prms)) {
353 fprintf(stderr, "clear WP failed for address 0x%Lx\n", start);
354 exit(1);
355 }
356 }
357
locking_thread(void * arg)358 static void *locking_thread(void *arg)
359 {
360 unsigned long cpu = (unsigned long) arg;
361 struct random_data rand;
362 unsigned long page_nr = *(&(page_nr)); /* uninitialized warning */
363 int32_t rand_nr;
364 unsigned long long count;
365 char randstate[64];
366 unsigned int seed;
367 time_t start;
368
369 if (bounces & BOUNCE_RANDOM) {
370 seed = (unsigned int) time(NULL) - bounces;
371 if (!(bounces & BOUNCE_RACINGFAULTS))
372 seed += cpu;
373 bzero(&rand, sizeof(rand));
374 bzero(&randstate, sizeof(randstate));
375 if (initstate_r(seed, randstate, sizeof(randstate), &rand)) {
376 fprintf(stderr, "srandom_r error\n");
377 exit(1);
378 }
379 } else {
380 page_nr = -bounces;
381 if (!(bounces & BOUNCE_RACINGFAULTS))
382 page_nr += cpu * nr_pages_per_cpu;
383 }
384
385 while (!finished) {
386 if (bounces & BOUNCE_RANDOM) {
387 if (random_r(&rand, &rand_nr)) {
388 fprintf(stderr, "random_r 1 error\n");
389 exit(1);
390 }
391 page_nr = rand_nr;
392 if (sizeof(page_nr) > sizeof(rand_nr)) {
393 if (random_r(&rand, &rand_nr)) {
394 fprintf(stderr, "random_r 2 error\n");
395 exit(1);
396 }
397 page_nr |= (((unsigned long) rand_nr) << 16) <<
398 16;
399 }
400 } else
401 page_nr += 1;
402 page_nr %= nr_pages;
403
404 start = time(NULL);
405 if (bounces & BOUNCE_VERIFY) {
406 count = *area_count(area_dst, page_nr);
407 if (!count) {
408 fprintf(stderr,
409 "page_nr %lu wrong count %Lu %Lu\n",
410 page_nr, count,
411 count_verify[page_nr]);
412 exit(1);
413 }
414
415
416 /*
417 * We can't use bcmp (or memcmp) because that
418 * returns 0 erroneously if the memory is
419 * changing under it (even if the end of the
420 * page is never changing and always
421 * different).
422 */
423 #if 1
424 if (!my_bcmp(area_dst + page_nr * page_size, zeropage,
425 page_size)) {
426 fprintf(stderr,
427 "my_bcmp page_nr %lu wrong count %Lu %Lu\n",
428 page_nr, count, count_verify[page_nr]);
429 exit(1);
430 }
431 #else
432 unsigned long loops;
433
434 loops = 0;
435 /* uncomment the below line to test with mutex */
436 /* pthread_mutex_lock(area_mutex(area_dst, page_nr)); */
437 while (!bcmp(area_dst + page_nr * page_size, zeropage,
438 page_size)) {
439 loops += 1;
440 if (loops > 10)
441 break;
442 }
443 /* uncomment below line to test with mutex */
444 /* pthread_mutex_unlock(area_mutex(area_dst, page_nr)); */
445 if (loops) {
446 fprintf(stderr,
447 "page_nr %lu all zero thread %lu %p %lu\n",
448 page_nr, cpu, area_dst + page_nr * page_size,
449 loops);
450 if (loops > 10)
451 exit(1);
452 }
453 #endif
454 }
455
456 pthread_mutex_lock(area_mutex(area_dst, page_nr));
457 count = *area_count(area_dst, page_nr);
458 if (count != count_verify[page_nr]) {
459 fprintf(stderr,
460 "page_nr %lu memory corruption %Lu %Lu\n",
461 page_nr, count,
462 count_verify[page_nr]); exit(1);
463 }
464 count++;
465 *area_count(area_dst, page_nr) = count_verify[page_nr] = count;
466 pthread_mutex_unlock(area_mutex(area_dst, page_nr));
467
468 if (time(NULL) - start > 1)
469 fprintf(stderr,
470 "userfault too slow %ld "
471 "possible false positive with overcommit\n",
472 time(NULL) - start);
473 }
474
475 return NULL;
476 }
477
retry_copy_page(int ufd,struct uffdio_copy * uffdio_copy,unsigned long offset)478 static void retry_copy_page(int ufd, struct uffdio_copy *uffdio_copy,
479 unsigned long offset)
480 {
481 uffd_test_ops->alias_mapping(&uffdio_copy->dst,
482 uffdio_copy->len,
483 offset);
484 if (ioctl(ufd, UFFDIO_COPY, uffdio_copy)) {
485 /* real retval in ufdio_copy.copy */
486 if (uffdio_copy->copy != -EEXIST) {
487 fprintf(stderr, "UFFDIO_COPY retry error %Ld\n",
488 uffdio_copy->copy);
489 exit(1);
490 }
491 } else {
492 fprintf(stderr, "UFFDIO_COPY retry unexpected %Ld\n",
493 uffdio_copy->copy); exit(1);
494 }
495 }
496
__copy_page(int ufd,unsigned long offset,bool retry)497 static int __copy_page(int ufd, unsigned long offset, bool retry)
498 {
499 struct uffdio_copy uffdio_copy;
500
501 if (offset >= nr_pages * page_size) {
502 fprintf(stderr, "unexpected offset %lu\n", offset);
503 exit(1);
504 }
505 uffdio_copy.dst = (unsigned long) area_dst + offset;
506 uffdio_copy.src = (unsigned long) area_src + offset;
507 uffdio_copy.len = page_size;
508 if (test_uffdio_wp)
509 uffdio_copy.mode = UFFDIO_COPY_MODE_WP;
510 else
511 uffdio_copy.mode = 0;
512 uffdio_copy.copy = 0;
513 if (ioctl(ufd, UFFDIO_COPY, &uffdio_copy)) {
514 /* real retval in ufdio_copy.copy */
515 if (uffdio_copy.copy != -EEXIST) {
516 fprintf(stderr, "UFFDIO_COPY error %Ld\n",
517 uffdio_copy.copy);
518 exit(1);
519 }
520 } else if (uffdio_copy.copy != page_size) {
521 fprintf(stderr, "UFFDIO_COPY unexpected copy %Ld\n",
522 uffdio_copy.copy); exit(1);
523 } else {
524 if (test_uffdio_copy_eexist && retry) {
525 test_uffdio_copy_eexist = false;
526 retry_copy_page(ufd, &uffdio_copy, offset);
527 }
528 return 1;
529 }
530 return 0;
531 }
532
copy_page_retry(int ufd,unsigned long offset)533 static int copy_page_retry(int ufd, unsigned long offset)
534 {
535 return __copy_page(ufd, offset, true);
536 }
537
copy_page(int ufd,unsigned long offset)538 static int copy_page(int ufd, unsigned long offset)
539 {
540 return __copy_page(ufd, offset, false);
541 }
542
uffd_read_msg(int ufd,struct uffd_msg * msg)543 static int uffd_read_msg(int ufd, struct uffd_msg *msg)
544 {
545 int ret = read(uffd, msg, sizeof(*msg));
546
547 if (ret != sizeof(*msg)) {
548 if (ret < 0) {
549 if (errno == EAGAIN)
550 return 1;
551 perror("blocking read error");
552 } else {
553 fprintf(stderr, "short read\n");
554 }
555 exit(1);
556 }
557
558 return 0;
559 }
560
uffd_handle_page_fault(struct uffd_msg * msg,struct uffd_stats * stats)561 static void uffd_handle_page_fault(struct uffd_msg *msg,
562 struct uffd_stats *stats)
563 {
564 unsigned long offset;
565
566 if (msg->event != UFFD_EVENT_PAGEFAULT) {
567 fprintf(stderr, "unexpected msg event %u\n", msg->event);
568 exit(1);
569 }
570
571 if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP) {
572 wp_range(uffd, msg->arg.pagefault.address, page_size, false);
573 stats->wp_faults++;
574 } else {
575 /* Missing page faults */
576 if (bounces & BOUNCE_VERIFY &&
577 msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE) {
578 fprintf(stderr, "unexpected write fault\n");
579 exit(1);
580 }
581
582 offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;
583 offset &= ~(page_size-1);
584
585 if (copy_page(uffd, offset))
586 stats->missing_faults++;
587 }
588 }
589
uffd_poll_thread(void * arg)590 static void *uffd_poll_thread(void *arg)
591 {
592 struct uffd_stats *stats = (struct uffd_stats *)arg;
593 unsigned long cpu = stats->cpu;
594 struct pollfd pollfd[2];
595 struct uffd_msg msg;
596 struct uffdio_register uffd_reg;
597 int ret;
598 char tmp_chr;
599
600 pollfd[0].fd = uffd;
601 pollfd[0].events = POLLIN;
602 pollfd[1].fd = pipefd[cpu*2];
603 pollfd[1].events = POLLIN;
604
605 for (;;) {
606 ret = poll(pollfd, 2, -1);
607 if (!ret) {
608 fprintf(stderr, "poll error %d\n", ret);
609 exit(1);
610 }
611 if (ret < 0) {
612 perror("poll");
613 exit(1);
614 }
615 if (pollfd[1].revents & POLLIN) {
616 if (read(pollfd[1].fd, &tmp_chr, 1) != 1) {
617 fprintf(stderr, "read pipefd error\n");
618 exit(1);
619 }
620 break;
621 }
622 if (!(pollfd[0].revents & POLLIN)) {
623 fprintf(stderr, "pollfd[0].revents %d\n",
624 pollfd[0].revents);
625 exit(1);
626 }
627 if (uffd_read_msg(uffd, &msg))
628 continue;
629 switch (msg.event) {
630 default:
631 fprintf(stderr, "unexpected msg event %u\n",
632 msg.event); exit(1);
633 break;
634 case UFFD_EVENT_PAGEFAULT:
635 uffd_handle_page_fault(&msg, stats);
636 break;
637 case UFFD_EVENT_FORK:
638 close(uffd);
639 uffd = msg.arg.fork.ufd;
640 pollfd[0].fd = uffd;
641 break;
642 case UFFD_EVENT_REMOVE:
643 uffd_reg.range.start = msg.arg.remove.start;
644 uffd_reg.range.len = msg.arg.remove.end -
645 msg.arg.remove.start;
646 if (ioctl(uffd, UFFDIO_UNREGISTER, &uffd_reg.range)) {
647 fprintf(stderr, "remove failure\n");
648 exit(1);
649 }
650 break;
651 case UFFD_EVENT_REMAP:
652 area_dst = (char *)(unsigned long)msg.arg.remap.to;
653 break;
654 }
655 }
656
657 return NULL;
658 }
659
660 pthread_mutex_t uffd_read_mutex = PTHREAD_MUTEX_INITIALIZER;
661
uffd_read_thread(void * arg)662 static void *uffd_read_thread(void *arg)
663 {
664 struct uffd_stats *stats = (struct uffd_stats *)arg;
665 struct uffd_msg msg;
666
667 pthread_mutex_unlock(&uffd_read_mutex);
668 /* from here cancellation is ok */
669
670 for (;;) {
671 if (uffd_read_msg(uffd, &msg))
672 continue;
673 uffd_handle_page_fault(&msg, stats);
674 }
675
676 return NULL;
677 }
678
background_thread(void * arg)679 static void *background_thread(void *arg)
680 {
681 unsigned long cpu = (unsigned long) arg;
682 unsigned long page_nr, start_nr, mid_nr, end_nr;
683
684 start_nr = cpu * nr_pages_per_cpu;
685 end_nr = (cpu+1) * nr_pages_per_cpu;
686 mid_nr = (start_nr + end_nr) / 2;
687
688 /* Copy the first half of the pages */
689 for (page_nr = start_nr; page_nr < mid_nr; page_nr++)
690 copy_page_retry(uffd, page_nr * page_size);
691
692 /*
693 * If we need to test uffd-wp, set it up now. Then we'll have
694 * at least the first half of the pages mapped already which
695 * can be write-protected for testing
696 */
697 if (test_uffdio_wp)
698 wp_range(uffd, (unsigned long)area_dst + start_nr * page_size,
699 nr_pages_per_cpu * page_size, true);
700
701 /*
702 * Continue the 2nd half of the page copying, handling write
703 * protection faults if any
704 */
705 for (page_nr = mid_nr; page_nr < end_nr; page_nr++)
706 copy_page_retry(uffd, page_nr * page_size);
707
708 return NULL;
709 }
710
stress(struct uffd_stats * uffd_stats)711 static int stress(struct uffd_stats *uffd_stats)
712 {
713 unsigned long cpu;
714 pthread_t locking_threads[nr_cpus];
715 pthread_t uffd_threads[nr_cpus];
716 pthread_t background_threads[nr_cpus];
717
718 finished = 0;
719 for (cpu = 0; cpu < nr_cpus; cpu++) {
720 if (pthread_create(&locking_threads[cpu], &attr,
721 locking_thread, (void *)cpu))
722 return 1;
723 if (bounces & BOUNCE_POLL) {
724 if (pthread_create(&uffd_threads[cpu], &attr,
725 uffd_poll_thread,
726 (void *)&uffd_stats[cpu]))
727 return 1;
728 } else {
729 if (pthread_create(&uffd_threads[cpu], &attr,
730 uffd_read_thread,
731 (void *)&uffd_stats[cpu]))
732 return 1;
733 pthread_mutex_lock(&uffd_read_mutex);
734 }
735 if (pthread_create(&background_threads[cpu], &attr,
736 background_thread, (void *)cpu))
737 return 1;
738 }
739 for (cpu = 0; cpu < nr_cpus; cpu++)
740 if (pthread_join(background_threads[cpu], NULL))
741 return 1;
742
743 /*
744 * Be strict and immediately zap area_src, the whole area has
745 * been transferred already by the background treads. The
746 * area_src could then be faulted in in a racy way by still
747 * running uffdio_threads reading zeropages after we zapped
748 * area_src (but they're guaranteed to get -EEXIST from
749 * UFFDIO_COPY without writing zero pages into area_dst
750 * because the background threads already completed).
751 */
752 if (uffd_test_ops->release_pages(area_src))
753 return 1;
754
755
756 finished = 1;
757 for (cpu = 0; cpu < nr_cpus; cpu++)
758 if (pthread_join(locking_threads[cpu], NULL))
759 return 1;
760
761 for (cpu = 0; cpu < nr_cpus; cpu++) {
762 char c;
763 if (bounces & BOUNCE_POLL) {
764 if (write(pipefd[cpu*2+1], &c, 1) != 1) {
765 fprintf(stderr, "pipefd write error\n");
766 return 1;
767 }
768 if (pthread_join(uffd_threads[cpu],
769 (void *)&uffd_stats[cpu]))
770 return 1;
771 } else {
772 if (pthread_cancel(uffd_threads[cpu]))
773 return 1;
774 if (pthread_join(uffd_threads[cpu], NULL))
775 return 1;
776 }
777 }
778
779 return 0;
780 }
781
userfaultfd_open(int features)782 static int userfaultfd_open(int features)
783 {
784 struct uffdio_api uffdio_api;
785
786 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
787 if (uffd < 0) {
788 fprintf(stderr,
789 "userfaultfd syscall not available in this kernel\n");
790 return 1;
791 }
792 uffd_flags = fcntl(uffd, F_GETFD, NULL);
793
794 uffdio_api.api = UFFD_API;
795 uffdio_api.features = features;
796 if (ioctl(uffd, UFFDIO_API, &uffdio_api)) {
797 fprintf(stderr, "UFFDIO_API\n");
798 return 1;
799 }
800 if (uffdio_api.api != UFFD_API) {
801 fprintf(stderr, "UFFDIO_API error %Lu\n", uffdio_api.api);
802 return 1;
803 }
804
805 return 0;
806 }
807
808 sigjmp_buf jbuf, *sigbuf;
809
sighndl(int sig,siginfo_t * siginfo,void * ptr)810 static void sighndl(int sig, siginfo_t *siginfo, void *ptr)
811 {
812 if (sig == SIGBUS) {
813 if (sigbuf)
814 siglongjmp(*sigbuf, 1);
815 abort();
816 }
817 }
818
819 /*
820 * For non-cooperative userfaultfd test we fork() a process that will
821 * generate pagefaults, will mremap the area monitored by the
822 * userfaultfd and at last this process will release the monitored
823 * area.
824 * For the anonymous and shared memory the area is divided into two
825 * parts, the first part is accessed before mremap, and the second
826 * part is accessed after mremap. Since hugetlbfs does not support
827 * mremap, the entire monitored area is accessed in a single pass for
828 * HUGETLB_TEST.
829 * The release of the pages currently generates event for shmem and
830 * anonymous memory (UFFD_EVENT_REMOVE), hence it is not checked
831 * for hugetlb.
832 * For signal test(UFFD_FEATURE_SIGBUS), signal_test = 1, we register
833 * monitored area, generate pagefaults and test that signal is delivered.
834 * Use UFFDIO_COPY to allocate missing page and retry. For signal_test = 2
835 * test robustness use case - we release monitored area, fork a process
836 * that will generate pagefaults and verify signal is generated.
837 * This also tests UFFD_FEATURE_EVENT_FORK event along with the signal
838 * feature. Using monitor thread, verify no userfault events are generated.
839 */
faulting_process(int signal_test)840 static int faulting_process(int signal_test)
841 {
842 unsigned long nr;
843 unsigned long long count;
844 unsigned long split_nr_pages;
845 unsigned long lastnr;
846 struct sigaction act;
847 unsigned long signalled = 0;
848
849 if (test_type != TEST_HUGETLB)
850 split_nr_pages = (nr_pages + 1) / 2;
851 else
852 split_nr_pages = nr_pages;
853
854 if (signal_test) {
855 sigbuf = &jbuf;
856 memset(&act, 0, sizeof(act));
857 act.sa_sigaction = sighndl;
858 act.sa_flags = SA_SIGINFO;
859 if (sigaction(SIGBUS, &act, 0)) {
860 perror("sigaction");
861 return 1;
862 }
863 lastnr = (unsigned long)-1;
864 }
865
866 for (nr = 0; nr < split_nr_pages; nr++) {
867 int steps = 1;
868 unsigned long offset = nr * page_size;
869
870 if (signal_test) {
871 if (sigsetjmp(*sigbuf, 1) != 0) {
872 if (steps == 1 && nr == lastnr) {
873 fprintf(stderr, "Signal repeated\n");
874 return 1;
875 }
876
877 lastnr = nr;
878 if (signal_test == 1) {
879 if (steps == 1) {
880 /* This is a MISSING request */
881 steps++;
882 if (copy_page(uffd, offset))
883 signalled++;
884 } else {
885 /* This is a WP request */
886 assert(steps == 2);
887 wp_range(uffd,
888 (__u64)area_dst +
889 offset,
890 page_size, false);
891 }
892 } else {
893 signalled++;
894 continue;
895 }
896 }
897 }
898
899 count = *area_count(area_dst, nr);
900 if (count != count_verify[nr]) {
901 fprintf(stderr,
902 "nr %lu memory corruption %Lu %Lu\n",
903 nr, count,
904 count_verify[nr]);
905 }
906 /*
907 * Trigger write protection if there is by writting
908 * the same value back.
909 */
910 *area_count(area_dst, nr) = count;
911 }
912
913 if (signal_test)
914 return signalled != split_nr_pages;
915
916 if (test_type == TEST_HUGETLB)
917 return 0;
918
919 area_dst = mremap(area_dst, nr_pages * page_size, nr_pages * page_size,
920 MREMAP_MAYMOVE | MREMAP_FIXED, area_src);
921 if (area_dst == MAP_FAILED) {
922 perror("mremap");
923 exit(1);
924 }
925
926 for (; nr < nr_pages; nr++) {
927 count = *area_count(area_dst, nr);
928 if (count != count_verify[nr]) {
929 fprintf(stderr,
930 "nr %lu memory corruption %Lu %Lu\n",
931 nr, count,
932 count_verify[nr]); exit(1);
933 }
934 /*
935 * Trigger write protection if there is by writting
936 * the same value back.
937 */
938 *area_count(area_dst, nr) = count;
939 }
940
941 if (uffd_test_ops->release_pages(area_dst))
942 return 1;
943
944 for (nr = 0; nr < nr_pages; nr++) {
945 if (my_bcmp(area_dst + nr * page_size, zeropage, page_size)) {
946 fprintf(stderr, "nr %lu is not zero\n", nr);
947 exit(1);
948 }
949 }
950
951 return 0;
952 }
953
retry_uffdio_zeropage(int ufd,struct uffdio_zeropage * uffdio_zeropage,unsigned long offset)954 static void retry_uffdio_zeropage(int ufd,
955 struct uffdio_zeropage *uffdio_zeropage,
956 unsigned long offset)
957 {
958 uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,
959 uffdio_zeropage->range.len,
960 offset);
961 if (ioctl(ufd, UFFDIO_ZEROPAGE, uffdio_zeropage)) {
962 if (uffdio_zeropage->zeropage != -EEXIST) {
963 fprintf(stderr, "UFFDIO_ZEROPAGE retry error %Ld\n",
964 uffdio_zeropage->zeropage);
965 exit(1);
966 }
967 } else {
968 fprintf(stderr, "UFFDIO_ZEROPAGE retry unexpected %Ld\n",
969 uffdio_zeropage->zeropage); exit(1);
970 }
971 }
972
__uffdio_zeropage(int ufd,unsigned long offset,bool retry)973 static int __uffdio_zeropage(int ufd, unsigned long offset, bool retry)
974 {
975 struct uffdio_zeropage uffdio_zeropage;
976 int ret;
977 unsigned long has_zeropage;
978
979 has_zeropage = uffd_test_ops->expected_ioctls & (1 << _UFFDIO_ZEROPAGE);
980
981 if (offset >= nr_pages * page_size) {
982 fprintf(stderr, "unexpected offset %lu\n", offset);
983 exit(1);
984 }
985 uffdio_zeropage.range.start = (unsigned long) area_dst + offset;
986 uffdio_zeropage.range.len = page_size;
987 uffdio_zeropage.mode = 0;
988 ret = ioctl(ufd, UFFDIO_ZEROPAGE, &uffdio_zeropage);
989 if (ret) {
990 /* real retval in ufdio_zeropage.zeropage */
991 if (has_zeropage) {
992 if (uffdio_zeropage.zeropage == -EEXIST) {
993 fprintf(stderr, "UFFDIO_ZEROPAGE -EEXIST\n");
994 exit(1);
995 } else {
996 fprintf(stderr, "UFFDIO_ZEROPAGE error %Ld\n",
997 uffdio_zeropage.zeropage);
998 exit(1);
999 }
1000 } else {
1001 if (uffdio_zeropage.zeropage != -EINVAL) {
1002 fprintf(stderr,
1003 "UFFDIO_ZEROPAGE not -EINVAL %Ld\n",
1004 uffdio_zeropage.zeropage);
1005 exit(1);
1006 }
1007 }
1008 } else if (has_zeropage) {
1009 if (uffdio_zeropage.zeropage != page_size) {
1010 fprintf(stderr, "UFFDIO_ZEROPAGE unexpected %Ld\n",
1011 uffdio_zeropage.zeropage); exit(1);
1012 } else {
1013 if (test_uffdio_zeropage_eexist && retry) {
1014 test_uffdio_zeropage_eexist = false;
1015 retry_uffdio_zeropage(ufd, &uffdio_zeropage,
1016 offset);
1017 }
1018 return 1;
1019 }
1020 } else {
1021 fprintf(stderr,
1022 "UFFDIO_ZEROPAGE succeeded %Ld\n",
1023 uffdio_zeropage.zeropage); exit(1);
1024 }
1025
1026 return 0;
1027 }
1028
uffdio_zeropage(int ufd,unsigned long offset)1029 static int uffdio_zeropage(int ufd, unsigned long offset)
1030 {
1031 return __uffdio_zeropage(ufd, offset, false);
1032 }
1033
1034 /* exercise UFFDIO_ZEROPAGE */
userfaultfd_zeropage_test(void)1035 static int userfaultfd_zeropage_test(void)
1036 {
1037 struct uffdio_register uffdio_register;
1038 unsigned long expected_ioctls;
1039
1040 printf("testing UFFDIO_ZEROPAGE: ");
1041 fflush(stdout);
1042
1043 if (uffd_test_ops->release_pages(area_dst))
1044 return 1;
1045
1046 if (userfaultfd_open(0) < 0)
1047 return 1;
1048 uffdio_register.range.start = (unsigned long) area_dst;
1049 uffdio_register.range.len = nr_pages * page_size;
1050 uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
1051 if (test_uffdio_wp)
1052 uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
1053 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) {
1054 fprintf(stderr, "register failure\n");
1055 exit(1);
1056 }
1057
1058 expected_ioctls = uffd_test_ops->expected_ioctls;
1059 if ((uffdio_register.ioctls & expected_ioctls) !=
1060 expected_ioctls) {
1061 fprintf(stderr,
1062 "unexpected missing ioctl for anon memory\n");
1063 exit(1);
1064 }
1065
1066 if (uffdio_zeropage(uffd, 0)) {
1067 if (my_bcmp(area_dst, zeropage, page_size)) {
1068 fprintf(stderr, "zeropage is not zero\n");
1069 exit(1);
1070 }
1071 }
1072
1073 close(uffd);
1074 printf("done.\n");
1075 return 0;
1076 }
1077
userfaultfd_events_test(void)1078 static int userfaultfd_events_test(void)
1079 {
1080 struct uffdio_register uffdio_register;
1081 unsigned long expected_ioctls;
1082 pthread_t uffd_mon;
1083 int err, features;
1084 pid_t pid;
1085 char c;
1086 struct uffd_stats stats = { 0 };
1087
1088 printf("testing events (fork, remap, remove): ");
1089 fflush(stdout);
1090
1091 if (uffd_test_ops->release_pages(area_dst))
1092 return 1;
1093
1094 features = UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_EVENT_REMAP |
1095 UFFD_FEATURE_EVENT_REMOVE;
1096 if (userfaultfd_open(features) < 0)
1097 return 1;
1098 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
1099
1100 uffdio_register.range.start = (unsigned long) area_dst;
1101 uffdio_register.range.len = nr_pages * page_size;
1102 uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
1103 if (test_uffdio_wp)
1104 uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
1105 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) {
1106 fprintf(stderr, "register failure\n");
1107 exit(1);
1108 }
1109
1110 expected_ioctls = uffd_test_ops->expected_ioctls;
1111 if ((uffdio_register.ioctls & expected_ioctls) != expected_ioctls) {
1112 fprintf(stderr, "unexpected missing ioctl for anon memory\n");
1113 exit(1);
1114 }
1115
1116 if (pthread_create(&uffd_mon, &attr, uffd_poll_thread, &stats)) {
1117 perror("uffd_poll_thread create");
1118 exit(1);
1119 }
1120
1121 pid = fork();
1122 if (pid < 0) {
1123 perror("fork");
1124 exit(1);
1125 }
1126
1127 if (!pid)
1128 return faulting_process(0);
1129
1130 waitpid(pid, &err, 0);
1131 if (err) {
1132 fprintf(stderr, "faulting process failed\n");
1133 exit(1);
1134 }
1135
1136 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) {
1137 perror("pipe write");
1138 exit(1);
1139 }
1140 if (pthread_join(uffd_mon, NULL))
1141 return 1;
1142
1143 close(uffd);
1144
1145 uffd_stats_report(&stats, 1);
1146
1147 return stats.missing_faults != nr_pages;
1148 }
1149
userfaultfd_sig_test(void)1150 static int userfaultfd_sig_test(void)
1151 {
1152 struct uffdio_register uffdio_register;
1153 unsigned long expected_ioctls;
1154 unsigned long userfaults;
1155 pthread_t uffd_mon;
1156 int err, features;
1157 pid_t pid;
1158 char c;
1159 struct uffd_stats stats = { 0 };
1160
1161 printf("testing signal delivery: ");
1162 fflush(stdout);
1163
1164 if (uffd_test_ops->release_pages(area_dst))
1165 return 1;
1166
1167 features = UFFD_FEATURE_EVENT_FORK|UFFD_FEATURE_SIGBUS;
1168 if (userfaultfd_open(features) < 0)
1169 return 1;
1170 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
1171
1172 uffdio_register.range.start = (unsigned long) area_dst;
1173 uffdio_register.range.len = nr_pages * page_size;
1174 uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
1175 if (test_uffdio_wp)
1176 uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
1177 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) {
1178 fprintf(stderr, "register failure\n");
1179 exit(1);
1180 }
1181
1182 expected_ioctls = uffd_test_ops->expected_ioctls;
1183 if ((uffdio_register.ioctls & expected_ioctls) != expected_ioctls) {
1184 fprintf(stderr, "unexpected missing ioctl for anon memory\n");
1185 exit(1);
1186 }
1187
1188 if (faulting_process(1)) {
1189 fprintf(stderr, "faulting process failed\n");
1190 exit(1);
1191 }
1192
1193 if (uffd_test_ops->release_pages(area_dst))
1194 return 1;
1195
1196 if (pthread_create(&uffd_mon, &attr, uffd_poll_thread, &stats)) {
1197 perror("uffd_poll_thread create");
1198 exit(1);
1199 }
1200
1201 pid = fork();
1202 if (pid < 0) {
1203 perror("fork");
1204 exit(1);
1205 }
1206
1207 if (!pid)
1208 exit(faulting_process(2));
1209
1210 waitpid(pid, &err, 0);
1211 if (err) {
1212 fprintf(stderr, "faulting process failed\n");
1213 exit(1);
1214 }
1215
1216 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) {
1217 perror("pipe write");
1218 exit(1);
1219 }
1220 if (pthread_join(uffd_mon, (void **)&userfaults))
1221 return 1;
1222
1223 printf("done.\n");
1224 if (userfaults)
1225 fprintf(stderr, "Signal test failed, userfaults: %ld\n",
1226 userfaults);
1227 close(uffd);
1228 return userfaults != 0;
1229 }
1230
userfaultfd_stress(void)1231 static int userfaultfd_stress(void)
1232 {
1233 void *area;
1234 char *tmp_area;
1235 unsigned long nr;
1236 struct uffdio_register uffdio_register;
1237 unsigned long cpu;
1238 int err;
1239 struct uffd_stats uffd_stats[nr_cpus];
1240
1241 uffd_test_ops->allocate_area((void **)&area_src);
1242 if (!area_src)
1243 return 1;
1244 uffd_test_ops->allocate_area((void **)&area_dst);
1245 if (!area_dst)
1246 return 1;
1247
1248 if (userfaultfd_open(0) < 0)
1249 return 1;
1250
1251 count_verify = malloc(nr_pages * sizeof(unsigned long long));
1252 if (!count_verify) {
1253 perror("count_verify");
1254 return 1;
1255 }
1256
1257 for (nr = 0; nr < nr_pages; nr++) {
1258 *area_mutex(area_src, nr) = (pthread_mutex_t)
1259 PTHREAD_MUTEX_INITIALIZER;
1260 count_verify[nr] = *area_count(area_src, nr) = 1;
1261 /*
1262 * In the transition between 255 to 256, powerpc will
1263 * read out of order in my_bcmp and see both bytes as
1264 * zero, so leave a placeholder below always non-zero
1265 * after the count, to avoid my_bcmp to trigger false
1266 * positives.
1267 */
1268 *(area_count(area_src, nr) + 1) = 1;
1269 }
1270
1271 pipefd = malloc(sizeof(int) * nr_cpus * 2);
1272 if (!pipefd) {
1273 perror("pipefd");
1274 return 1;
1275 }
1276 for (cpu = 0; cpu < nr_cpus; cpu++) {
1277 if (pipe2(&pipefd[cpu*2], O_CLOEXEC | O_NONBLOCK)) {
1278 perror("pipe");
1279 return 1;
1280 }
1281 }
1282
1283 if (posix_memalign(&area, page_size, page_size)) {
1284 fprintf(stderr, "out of memory\n");
1285 return 1;
1286 }
1287 zeropage = area;
1288 bzero(zeropage, page_size);
1289
1290 pthread_mutex_lock(&uffd_read_mutex);
1291
1292 pthread_attr_init(&attr);
1293 pthread_attr_setstacksize(&attr, 16*1024*1024);
1294
1295 err = 0;
1296 while (bounces--) {
1297 unsigned long expected_ioctls;
1298
1299 printf("bounces: %d, mode:", bounces);
1300 if (bounces & BOUNCE_RANDOM)
1301 printf(" rnd");
1302 if (bounces & BOUNCE_RACINGFAULTS)
1303 printf(" racing");
1304 if (bounces & BOUNCE_VERIFY)
1305 printf(" ver");
1306 if (bounces & BOUNCE_POLL)
1307 printf(" poll");
1308 printf(", ");
1309 fflush(stdout);
1310
1311 if (bounces & BOUNCE_POLL)
1312 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
1313 else
1314 fcntl(uffd, F_SETFL, uffd_flags & ~O_NONBLOCK);
1315
1316 /* register */
1317 uffdio_register.range.start = (unsigned long) area_dst;
1318 uffdio_register.range.len = nr_pages * page_size;
1319 uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
1320 if (test_uffdio_wp)
1321 uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
1322 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) {
1323 fprintf(stderr, "register failure\n");
1324 return 1;
1325 }
1326 expected_ioctls = uffd_test_ops->expected_ioctls;
1327 if ((uffdio_register.ioctls & expected_ioctls) !=
1328 expected_ioctls) {
1329 fprintf(stderr,
1330 "unexpected missing ioctl for anon memory\n");
1331 return 1;
1332 }
1333
1334 if (area_dst_alias) {
1335 uffdio_register.range.start = (unsigned long)
1336 area_dst_alias;
1337 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) {
1338 fprintf(stderr, "register failure alias\n");
1339 return 1;
1340 }
1341 }
1342
1343 /*
1344 * The madvise done previously isn't enough: some
1345 * uffd_thread could have read userfaults (one of
1346 * those already resolved by the background thread)
1347 * and it may be in the process of calling
1348 * UFFDIO_COPY. UFFDIO_COPY will read the zapped
1349 * area_src and it would map a zero page in it (of
1350 * course such a UFFDIO_COPY is perfectly safe as it'd
1351 * return -EEXIST). The problem comes at the next
1352 * bounce though: that racing UFFDIO_COPY would
1353 * generate zeropages in the area_src, so invalidating
1354 * the previous MADV_DONTNEED. Without this additional
1355 * MADV_DONTNEED those zeropages leftovers in the
1356 * area_src would lead to -EEXIST failure during the
1357 * next bounce, effectively leaving a zeropage in the
1358 * area_dst.
1359 *
1360 * Try to comment this out madvise to see the memory
1361 * corruption being caught pretty quick.
1362 *
1363 * khugepaged is also inhibited to collapse THP after
1364 * MADV_DONTNEED only after the UFFDIO_REGISTER, so it's
1365 * required to MADV_DONTNEED here.
1366 */
1367 if (uffd_test_ops->release_pages(area_dst))
1368 return 1;
1369
1370 uffd_stats_reset(uffd_stats, nr_cpus);
1371
1372 /* bounce pass */
1373 if (stress(uffd_stats))
1374 return 1;
1375
1376 /* Clear all the write protections if there is any */
1377 if (test_uffdio_wp)
1378 wp_range(uffd, (unsigned long)area_dst,
1379 nr_pages * page_size, false);
1380
1381 /* unregister */
1382 if (ioctl(uffd, UFFDIO_UNREGISTER, &uffdio_register.range)) {
1383 fprintf(stderr, "unregister failure\n");
1384 return 1;
1385 }
1386 if (area_dst_alias) {
1387 uffdio_register.range.start = (unsigned long) area_dst;
1388 if (ioctl(uffd, UFFDIO_UNREGISTER,
1389 &uffdio_register.range)) {
1390 fprintf(stderr, "unregister failure alias\n");
1391 return 1;
1392 }
1393 }
1394
1395 /* verification */
1396 if (bounces & BOUNCE_VERIFY) {
1397 for (nr = 0; nr < nr_pages; nr++) {
1398 if (*area_count(area_dst, nr) != count_verify[nr]) {
1399 fprintf(stderr,
1400 "error area_count %Lu %Lu %lu\n",
1401 *area_count(area_src, nr),
1402 count_verify[nr],
1403 nr);
1404 err = 1;
1405 bounces = 0;
1406 }
1407 }
1408 }
1409
1410 /* prepare next bounce */
1411 tmp_area = area_src;
1412 area_src = area_dst;
1413 area_dst = tmp_area;
1414
1415 tmp_area = area_src_alias;
1416 area_src_alias = area_dst_alias;
1417 area_dst_alias = tmp_area;
1418
1419 uffd_stats_report(uffd_stats, nr_cpus);
1420 }
1421
1422 if (err)
1423 return err;
1424
1425 close(uffd);
1426 return userfaultfd_zeropage_test() || userfaultfd_sig_test()
1427 || userfaultfd_events_test();
1428 }
1429
1430 /*
1431 * Copied from mlock2-tests.c
1432 */
default_huge_page_size(void)1433 unsigned long default_huge_page_size(void)
1434 {
1435 unsigned long hps = 0;
1436 char *line = NULL;
1437 size_t linelen = 0;
1438 FILE *f = fopen("/proc/meminfo", "r");
1439
1440 if (!f)
1441 return 0;
1442 while (getline(&line, &linelen, f) > 0) {
1443 if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) {
1444 hps <<= 10;
1445 break;
1446 }
1447 }
1448
1449 free(line);
1450 fclose(f);
1451 return hps;
1452 }
1453
set_test_type(const char * type)1454 static void set_test_type(const char *type)
1455 {
1456 if (!strcmp(type, "anon")) {
1457 test_type = TEST_ANON;
1458 uffd_test_ops = &anon_uffd_test_ops;
1459 /* Only enable write-protect test for anonymous test */
1460 test_uffdio_wp = true;
1461 } else if (!strcmp(type, "hugetlb")) {
1462 test_type = TEST_HUGETLB;
1463 uffd_test_ops = &hugetlb_uffd_test_ops;
1464 } else if (!strcmp(type, "hugetlb_shared")) {
1465 map_shared = true;
1466 test_type = TEST_HUGETLB;
1467 uffd_test_ops = &hugetlb_uffd_test_ops;
1468 } else if (!strcmp(type, "shmem")) {
1469 map_shared = true;
1470 test_type = TEST_SHMEM;
1471 uffd_test_ops = &shmem_uffd_test_ops;
1472 } else {
1473 fprintf(stderr, "Unknown test type: %s\n", type); exit(1);
1474 }
1475
1476 if (test_type == TEST_HUGETLB)
1477 page_size = default_huge_page_size();
1478 else
1479 page_size = sysconf(_SC_PAGE_SIZE);
1480
1481 if (!page_size) {
1482 fprintf(stderr, "Unable to determine page size\n");
1483 exit(2);
1484 }
1485 if ((unsigned long) area_count(NULL, 0) + sizeof(unsigned long long) * 2
1486 > page_size) {
1487 fprintf(stderr, "Impossible to run this test\n");
1488 exit(2);
1489 }
1490 }
1491
sigalrm(int sig)1492 static void sigalrm(int sig)
1493 {
1494 if (sig != SIGALRM)
1495 abort();
1496 test_uffdio_copy_eexist = true;
1497 test_uffdio_zeropage_eexist = true;
1498 alarm(ALARM_INTERVAL_SECS);
1499 }
1500
main(int argc,char ** argv)1501 int main(int argc, char **argv)
1502 {
1503 if (argc < 4)
1504 usage();
1505
1506 if (signal(SIGALRM, sigalrm) == SIG_ERR) {
1507 fprintf(stderr, "failed to arm SIGALRM");
1508 exit(1);
1509 }
1510 alarm(ALARM_INTERVAL_SECS);
1511
1512 set_test_type(argv[1]);
1513
1514 nr_cpus = sysconf(_SC_NPROCESSORS_ONLN);
1515 nr_pages_per_cpu = atol(argv[2]) * 1024*1024 / page_size /
1516 nr_cpus;
1517 if (!nr_pages_per_cpu) {
1518 fprintf(stderr, "invalid MiB\n");
1519 usage();
1520 }
1521
1522 bounces = atoi(argv[3]);
1523 if (bounces <= 0) {
1524 fprintf(stderr, "invalid bounces\n");
1525 usage();
1526 }
1527 nr_pages = nr_pages_per_cpu * nr_cpus;
1528
1529 if (test_type == TEST_HUGETLB) {
1530 if (argc < 5)
1531 usage();
1532 huge_fd = open(argv[4], O_CREAT | O_RDWR, 0755);
1533 if (huge_fd < 0) {
1534 fprintf(stderr, "Open of %s failed", argv[3]);
1535 perror("open");
1536 exit(1);
1537 }
1538 if (ftruncate(huge_fd, 0)) {
1539 fprintf(stderr, "ftruncate %s to size 0 failed", argv[3]);
1540 perror("ftruncate");
1541 exit(1);
1542 }
1543 }
1544 printf("nr_pages: %lu, nr_pages_per_cpu: %lu\n",
1545 nr_pages, nr_pages_per_cpu);
1546 return userfaultfd_stress();
1547 }
1548
1549 #else /* __NR_userfaultfd */
1550
1551 #warning "missing __NR_userfaultfd definition"
1552
main(void)1553 int main(void)
1554 {
1555 printf("skip: Skipping userfaultfd test (missing __NR_userfaultfd)\n");
1556 return KSFT_SKIP;
1557 }
1558
1559 #endif /* __NR_userfaultfd */
1560