1 // SPDX-License-Identifier: GPL-2.0-only
13  * 1) one per-CPU thread takes a per-page pthread_mutex in a random
15  *    area_src), and increments a per-page counter in the same page,
18  * 2) another per-CPU thread handles the userfaults generated by
22  * 3) one last per-CPU thread transfers the memory in the background
32  * per-CPU threads 1 by triggering userfaults inside
80 /* Whether to test uffd write-protection */
104  * to avoid non alignment faults on non-x86 archs.
110 				  sizeof(unsigned long long) - 1) &	\
112 						  -  1)))
123     "# 10MiB-~6GiB 999 bounces anonymous test, "
184 			   MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);  in anon_allocate_area()
263 	 * MADV_DONTEED won't work. So exercise -EEXIST on a alias  in hugetlb_alias_mapping()
265 	 * this way we'll exercise the -EEXEC at the fs level.  in hugetlb_alias_mapping()
286 			   MAP_ANONYMOUS | MAP_SHARED, -1, 0);  in shmem_allocate_area()
346 	prms.range.start = start;  in wp_range()
347 	prms.range.len = len;  in wp_range()
348 	/* Undo write-protect, do wakeup after that */  in wp_range()
369 		seed = (unsigned int) time(NULL) - bounces;  in locking_thread()
379 		page_nr = -bounces;  in locking_thread()
467 		if (time(NULL) - start > 1)  in locking_thread()
471 				time(NULL) - start);  in locking_thread()
480 	uffd_test_ops->alias_mapping(&uffdio_copy->dst,  in retry_copy_page()
481 				     uffdio_copy->len,  in retry_copy_page()
485 		if (uffdio_copy->copy != -EEXIST) {  in retry_copy_page()
487 				uffdio_copy->copy);  in retry_copy_page()
492 			uffdio_copy->copy); exit(1);  in retry_copy_page()
514 		if (uffdio_copy.copy != -EEXIST) {  in __copy_page()
565 	if (msg->event != UFFD_EVENT_PAGEFAULT) {  in uffd_handle_page_fault()
566 		fprintf(stderr, "unexpected msg event %u\n", msg->event);  in uffd_handle_page_fault()
570 	if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP) {  in uffd_handle_page_fault()
571 		wp_range(uffd, msg->arg.pagefault.address, page_size, false);  in uffd_handle_page_fault()
572 		stats->wp_faults++;  in uffd_handle_page_fault()
576 		    msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE) {  in uffd_handle_page_fault()
581 		offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;  in uffd_handle_page_fault()
582 		offset &= ~(page_size-1);  in uffd_handle_page_fault()
585 			stats->missing_faults++;  in uffd_handle_page_fault()
592 	unsigned long cpu = stats->cpu;  in uffd_poll_thread()
605 		ret = poll(pollfd, 2, -1);  in uffd_poll_thread()
637 			close(uffd);  in uffd_poll_thread()
642 			uffd_reg.range.start = msg.arg.remove.start;  in uffd_poll_thread()
643 			uffd_reg.range.len = msg.arg.remove.end -  in uffd_poll_thread()
645 			if (ioctl(uffd, UFFDIO_UNREGISTER, &uffd_reg.range)) {  in uffd_poll_thread()
692 	 * If we need to test uffd-wp, set it up now.  Then we'll have  in background_thread()
694 	 * can be write-protected for testing  in background_thread()
747 	 * area_src (but they're guaranteed to get -EEXIST from  in stress()
751 	if (uffd_test_ops->release_pages(area_src))  in stress()
819  * For non-cooperative userfaultfd test we fork() a process that will
834  * test robustness use case - we release monitored area, fork a process
862 		lastnr = (unsigned long)-1;  in faulting_process()
940 	if (uffd_test_ops->release_pages(area_dst))  in faulting_process()
957 	uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,  in retry_uffdio_zeropage()
958 				     uffdio_zeropage->range.len,  in retry_uffdio_zeropage()
961 		if (uffdio_zeropage->zeropage != -EEXIST) {  in retry_uffdio_zeropage()
963 				uffdio_zeropage->zeropage);  in retry_uffdio_zeropage()
968 			uffdio_zeropage->zeropage); exit(1);  in retry_uffdio_zeropage()
978 	has_zeropage = uffd_test_ops->expected_ioctls & (1 << _UFFDIO_ZEROPAGE);  in __uffdio_zeropage()
984 	uffdio_zeropage.range.start = (unsigned long) area_dst + offset;  in __uffdio_zeropage()
985 	uffdio_zeropage.range.len = page_size;  in __uffdio_zeropage()
991 			if (uffdio_zeropage.zeropage == -EEXIST) {  in __uffdio_zeropage()
992 				fprintf(stderr, "UFFDIO_ZEROPAGE -EEXIST\n");  in __uffdio_zeropage()
1000 			if (uffdio_zeropage.zeropage != -EINVAL) {  in __uffdio_zeropage()
1002 					"UFFDIO_ZEROPAGE not -EINVAL %Ld\n",  in __uffdio_zeropage()
1042 	if (uffd_test_ops->release_pages(area_dst))  in userfaultfd_zeropage_test()
1047 	uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_zeropage_test()
1048 	uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_zeropage_test()
1057 	expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_zeropage_test()
1072 	close(uffd);  in userfaultfd_zeropage_test()
1090 	if (uffd_test_ops->release_pages(area_dst))  in userfaultfd_events_test()
1099 	uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_events_test()
1100 	uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_events_test()
1109 	expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_events_test()
1142 	close(uffd);  in userfaultfd_events_test()
1163 	if (uffd_test_ops->release_pages(area_dst))  in userfaultfd_sig_test()
1171 	uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_sig_test()
1172 	uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_sig_test()
1181 	expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_sig_test()
1192 	if (uffd_test_ops->release_pages(area_dst))  in userfaultfd_sig_test()
1226 	close(uffd);  in userfaultfd_sig_test()
1240 	uffd_test_ops->allocate_area((void **)&area_src);  in userfaultfd_stress()
1243 	uffd_test_ops->allocate_area((void **)&area_dst);  in userfaultfd_stress()
1263 		 * zero, so leave a placeholder below always non-zero  in userfaultfd_stress()
1295 	while (bounces--) {  in userfaultfd_stress()
1316 		uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_stress()
1317 		uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_stress()
1325 		expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_stress()
1334 			uffdio_register.range.start = (unsigned long)  in userfaultfd_stress()
1350 		 * return -EEXIST). The problem comes at the next  in userfaultfd_stress()
1355 		 * area_src would lead to -EEXIST failure during the  in userfaultfd_stress()
1366 		if (uffd_test_ops->release_pages(area_dst))  in userfaultfd_stress()
1381 		if (ioctl(uffd, UFFDIO_UNREGISTER, &uffdio_register.range)) {  in userfaultfd_stress()
1386 			uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_stress()
1388 				  &uffdio_register.range)) {  in userfaultfd_stress()
1424 	close(uffd);  in userfaultfd_stress()
1430  * Copied from mlock2-tests.c
1458 		/* Only enable write-protect test for anonymous test */  in set_test_type()