1 // SPDX-License-Identifier: GPL-2.0-only
5  * Copyright (C) 2015-2023  Red Hat, Inc.
8 #include "uffd-common.h"
15 int uffd = -1, uffd_flags, finished, *pipefd, test_type;
28 	mem_fd = memfd_create("uffd-test", memfd_flags);  in uffd_mem_fd_create()
50 			   MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);  in anon_allocate_area()
53 		return -errno;  in anon_allocate_area()
87 		return -errno;  in hugetlb_allocate_area()
94 			return -errno;  in hugetlb_allocate_area()
105 	close(mem_fd);  in hugetlb_allocate_area()
144 		return -errno;  in shmem_allocate_area()
154 		return -errno;  in shmem_allocate_area()
164 	close(mem_fd);  in shmem_allocate_area()
241 		return -1;  in userfaultfd_open()
248 		return -1;  in userfaultfd_open()
271 			if (close(pipefd[i]))  in uffd_test_ctx_clear()
272 				err("close pipefd");  in uffd_test_ctx_clear()
283 	if (uffd != -1) {  in uffd_test_ctx_clear()
284 		if (close(uffd))  in uffd_test_ctx_clear()
285 			err("close uffd");  in uffd_test_ctx_clear()
286 		uffd = -1;  in uffd_test_ctx_clear()
303 	ret = uffd_test_ops->allocate_area((void **)&area_src, true);  in uffd_test_ctx_init()
304 	ret |= uffd_test_ops->allocate_area((void **)&area_dst, false);  in uffd_test_ctx_init()
329 		 * zero, so leave a placeholder below always non-zero  in uffd_test_ctx_init()
354 	uffd_test_ops->release_pages(area_dst);  in uffd_test_ctx_init()
371 	prms.range.start = start;  in wp_range()
372 	prms.range.len = len;  in wp_range()
373 	/* Undo write-protect, do wakeup after that */  in wp_range()
385 	req.range.start = start;  in continue_range()
386 	req.range.len = len;  in continue_range()
398 	 * error (-EEXIST) on purpose, to verify doing so doesn't cause a BUG.  in continue_range()
402 	if (ret >= 0 || req.mapped != -EEXIST)  in continue_range()
428 	if (msg->event != UFFD_EVENT_PAGEFAULT)  in uffd_handle_page_fault()
429 		err("unexpected msg event %u", msg->event);  in uffd_handle_page_fault()
431 	if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP) {  in uffd_handle_page_fault()
433 		wp_range(uffd, msg->arg.pagefault.address, page_size, false);  in uffd_handle_page_fault()
434 		args->wp_faults++;  in uffd_handle_page_fault()
435 	} else if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_MINOR) {  in uffd_handle_page_fault()
442 		 * To prove we can modify the original range for testing  in uffd_handle_page_fault()
443 		 * purposes, we're going to bit flip this range before  in uffd_handle_page_fault()
447 		 * area_dst (non-UFFD-registered) and area_dst_alias  in uffd_handle_page_fault()
448 		 * (UFFD-registered).  in uffd_handle_page_fault()
452 				   ((char *)msg->arg.pagefault.address -  in uffd_handle_page_fault()
456 		continue_range(uffd, msg->arg.pagefault.address, page_size,  in uffd_handle_page_fault()
457 			       args->apply_wp);  in uffd_handle_page_fault()
458 		args->minor_faults++;  in uffd_handle_page_fault()
474 		 * logically a pthread-compatible lib can implement the  in uffd_handle_page_fault()
481 		if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE)  in uffd_handle_page_fault()
484 		offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;  in uffd_handle_page_fault()
485 		offset &= ~(page_size-1);  in uffd_handle_page_fault()
487 		if (copy_page(uffd, offset, args->apply_wp))  in uffd_handle_page_fault()
488 			args->missing_faults++;  in uffd_handle_page_fault()
495 	unsigned long cpu = args->cpu;  in uffd_poll_thread()
502 	if (!args->handle_fault)  in uffd_poll_thread()
503 		args->handle_fault = uffd_handle_page_fault;  in uffd_poll_thread()
511 		ret = poll(pollfd, 2, -1);  in uffd_poll_thread()
533 			args->handle_fault(&msg, args);  in uffd_poll_thread()
536 			close(uffd);  in uffd_poll_thread()
541 			uffd_reg.range.start = msg.arg.remove.start;  in uffd_poll_thread()
542 			uffd_reg.range.len = msg.arg.remove.end -  in uffd_poll_thread()
544 			if (ioctl(uffd, UFFDIO_UNREGISTER, &uffd_reg.range))  in uffd_poll_thread()
560 	uffd_test_ops->alias_mapping(&uffdio_copy->dst,  in retry_copy_page()
561 				     uffdio_copy->len,  in retry_copy_page()
565 		if (uffdio_copy->copy != -EEXIST)  in retry_copy_page()
567 			    (int64_t)uffdio_copy->copy);  in retry_copy_page()
570 		    (int64_t)uffdio_copy->copy);  in retry_copy_page()
602 		if (uffdio_copy.copy != -EEXIST)  in __copy_page()
631 	close(fd);  in uffd_open_dev()
641 	return -1;  in uffd_open_sys()
665 		/* Maybe the kernel is older than user-only mode? */  in uffd_get_features()
672 		close(fd);  in uffd_get_features()
673 		return -errno;  in uffd_get_features()
677 	close(fd);  in uffd_get_features()