Lines Matching +full:cpu +full:- +full:offset
1 // SPDX-License-Identifier: GPL-2.0-only
11 * There are three threads running per CPU:
13 * 1) one per-CPU thread takes a per-page pthread_mutex in a random
15 * area_src), and increments a per-page counter in the same page,
18 * 2) another per-CPU thread handles the userfaults generated by
22 * 3) one last per-CPU thread transfers the memory in the background
24 * 2). Each cpu thread takes cares of transferring a portion of the
32 * per-CPU threads 1 by triggering userfaults inside
80 /* Whether to test uffd write-protection */
94 int cpu; member
99 /* pthread_mutex_t starts at page offset 0 */
104 * to avoid non alignment faults on non-x86 archs.
110 sizeof(unsigned long long) - 1) & \
112 - 1)))
123 "# 10MiB-~6GiB 999 bounces anonymous test, "
144 uffd_stats[i].cpu = i; in uffd_stats_reset()
184 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); in anon_allocate_area()
191 static void noop_alias_mapping(__u64 *start, size_t len, unsigned long offset) in noop_alias_mapping() argument
257 static void hugetlb_alias_mapping(__u64 *start, size_t len, unsigned long offset) in hugetlb_alias_mapping() argument
263 * MADV_DONTEED won't work. So exercise -EEXIST on a alias in hugetlb_alias_mapping()
265 * this way we'll exercise the -EEXEC at the fs level. in hugetlb_alias_mapping()
267 *start = (unsigned long) area_dst_alias + offset; in hugetlb_alias_mapping()
286 MAP_ANONYMOUS | MAP_SHARED, -1, 0); in shmem_allocate_area()
297 void (*alias_mapping)(__u64 *start, size_t len, unsigned long offset);
348 /* Undo write-protect, do wakeup after that */ in wp_range()
359 unsigned long cpu = (unsigned long) arg; in locking_thread() local
369 seed = (unsigned int) time(NULL) - bounces; in locking_thread()
371 seed += cpu; in locking_thread()
379 page_nr = -bounces; in locking_thread()
381 page_nr += cpu * nr_pages_per_cpu; in locking_thread()
447 page_nr, cpu, area_dst + page_nr * page_size, in locking_thread()
467 if (time(NULL) - start > 1) in locking_thread()
471 time(NULL) - start); in locking_thread()
478 unsigned long offset) in retry_copy_page() argument
480 uffd_test_ops->alias_mapping(&uffdio_copy->dst, in retry_copy_page()
481 uffdio_copy->len, in retry_copy_page()
482 offset); in retry_copy_page()
485 if (uffdio_copy->copy != -EEXIST) { in retry_copy_page()
487 uffdio_copy->copy); in retry_copy_page()
492 uffdio_copy->copy); exit(1); in retry_copy_page()
496 static int __copy_page(int ufd, unsigned long offset, bool retry) in __copy_page() argument
500 if (offset >= nr_pages * page_size) { in __copy_page()
501 fprintf(stderr, "unexpected offset %lu\n", offset); in __copy_page()
504 uffdio_copy.dst = (unsigned long) area_dst + offset; in __copy_page()
505 uffdio_copy.src = (unsigned long) area_src + offset; in __copy_page()
514 if (uffdio_copy.copy != -EEXIST) { in __copy_page()
525 retry_copy_page(ufd, &uffdio_copy, offset); in __copy_page()
532 static int copy_page_retry(int ufd, unsigned long offset) in copy_page_retry() argument
534 return __copy_page(ufd, offset, true); in copy_page_retry()
537 static int copy_page(int ufd, unsigned long offset) in copy_page() argument
539 return __copy_page(ufd, offset, false); in copy_page()
563 unsigned long offset; in uffd_handle_page_fault() local
565 if (msg->event != UFFD_EVENT_PAGEFAULT) { in uffd_handle_page_fault()
566 fprintf(stderr, "unexpected msg event %u\n", msg->event); in uffd_handle_page_fault()
570 if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP) { in uffd_handle_page_fault()
571 wp_range(uffd, msg->arg.pagefault.address, page_size, false); in uffd_handle_page_fault()
572 stats->wp_faults++; in uffd_handle_page_fault()
576 msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE) { in uffd_handle_page_fault()
581 offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst; in uffd_handle_page_fault()
582 offset &= ~(page_size-1); in uffd_handle_page_fault()
584 if (copy_page(uffd, offset)) in uffd_handle_page_fault()
585 stats->missing_faults++; in uffd_handle_page_fault()
592 unsigned long cpu = stats->cpu; in uffd_poll_thread() local
601 pollfd[1].fd = pipefd[cpu*2]; in uffd_poll_thread()
605 ret = poll(pollfd, 2, -1); in uffd_poll_thread()
643 uffd_reg.range.len = msg.arg.remove.end - in uffd_poll_thread()
680 unsigned long cpu = (unsigned long) arg; in background_thread() local
683 start_nr = cpu * nr_pages_per_cpu; in background_thread()
684 end_nr = (cpu+1) * nr_pages_per_cpu; in background_thread()
692 * If we need to test uffd-wp, set it up now. Then we'll have in background_thread()
694 * can be write-protected for testing in background_thread()
712 unsigned long cpu; in stress() local
718 for (cpu = 0; cpu < nr_cpus; cpu++) { in stress()
719 if (pthread_create(&locking_threads[cpu], &attr, in stress()
720 locking_thread, (void *)cpu)) in stress()
723 if (pthread_create(&uffd_threads[cpu], &attr, in stress()
725 (void *)&uffd_stats[cpu])) in stress()
728 if (pthread_create(&uffd_threads[cpu], &attr, in stress()
730 (void *)&uffd_stats[cpu])) in stress()
734 if (pthread_create(&background_threads[cpu], &attr, in stress()
735 background_thread, (void *)cpu)) in stress()
738 for (cpu = 0; cpu < nr_cpus; cpu++) in stress()
739 if (pthread_join(background_threads[cpu], NULL)) in stress()
747 * area_src (but they're guaranteed to get -EEXIST from in stress()
751 if (uffd_test_ops->release_pages(area_src)) in stress()
756 for (cpu = 0; cpu < nr_cpus; cpu++) in stress()
757 if (pthread_join(locking_threads[cpu], NULL)) in stress()
760 for (cpu = 0; cpu < nr_cpus; cpu++) { in stress()
763 if (write(pipefd[cpu*2+1], &c, 1) != 1) { in stress()
767 if (pthread_join(uffd_threads[cpu], in stress()
768 (void *)&uffd_stats[cpu])) in stress()
771 if (pthread_cancel(uffd_threads[cpu])) in stress()
773 if (pthread_join(uffd_threads[cpu], NULL)) in stress()
819 * For non-cooperative userfaultfd test we fork() a process that will
834 * test robustness use case - we release monitored area, fork a process
862 lastnr = (unsigned long)-1; in faulting_process()
867 unsigned long offset = nr * page_size; in faulting_process() local
881 if (copy_page(uffd, offset)) in faulting_process()
888 offset, in faulting_process()
940 if (uffd_test_ops->release_pages(area_dst)) in faulting_process()
955 unsigned long offset) in retry_uffdio_zeropage() argument
957 uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start, in retry_uffdio_zeropage()
958 uffdio_zeropage->range.len, in retry_uffdio_zeropage()
959 offset); in retry_uffdio_zeropage()
961 if (uffdio_zeropage->zeropage != -EEXIST) { in retry_uffdio_zeropage()
963 uffdio_zeropage->zeropage); in retry_uffdio_zeropage()
968 uffdio_zeropage->zeropage); exit(1); in retry_uffdio_zeropage()
972 static int __uffdio_zeropage(int ufd, unsigned long offset, bool retry) in __uffdio_zeropage() argument
978 has_zeropage = uffd_test_ops->expected_ioctls & (1 << _UFFDIO_ZEROPAGE); in __uffdio_zeropage()
980 if (offset >= nr_pages * page_size) { in __uffdio_zeropage()
981 fprintf(stderr, "unexpected offset %lu\n", offset); in __uffdio_zeropage()
984 uffdio_zeropage.range.start = (unsigned long) area_dst + offset; in __uffdio_zeropage()
991 if (uffdio_zeropage.zeropage == -EEXIST) { in __uffdio_zeropage()
992 fprintf(stderr, "UFFDIO_ZEROPAGE -EEXIST\n"); in __uffdio_zeropage()
1000 if (uffdio_zeropage.zeropage != -EINVAL) { in __uffdio_zeropage()
1002 "UFFDIO_ZEROPAGE not -EINVAL %Ld\n", in __uffdio_zeropage()
1015 offset); in __uffdio_zeropage()
1028 static int uffdio_zeropage(int ufd, unsigned long offset) in uffdio_zeropage() argument
1030 return __uffdio_zeropage(ufd, offset, false); in uffdio_zeropage()
1042 if (uffd_test_ops->release_pages(area_dst)) in userfaultfd_zeropage_test()
1057 expected_ioctls = uffd_test_ops->expected_ioctls; in userfaultfd_zeropage_test()
1090 if (uffd_test_ops->release_pages(area_dst)) in userfaultfd_events_test()
1109 expected_ioctls = uffd_test_ops->expected_ioctls; in userfaultfd_events_test()
1163 if (uffd_test_ops->release_pages(area_dst)) in userfaultfd_sig_test()
1181 expected_ioctls = uffd_test_ops->expected_ioctls; in userfaultfd_sig_test()
1192 if (uffd_test_ops->release_pages(area_dst)) in userfaultfd_sig_test()
1236 unsigned long cpu; in userfaultfd_stress() local
1240 uffd_test_ops->allocate_area((void **)&area_src); in userfaultfd_stress()
1243 uffd_test_ops->allocate_area((void **)&area_dst); in userfaultfd_stress()
1263 * zero, so leave a placeholder below always non-zero in userfaultfd_stress()
1275 for (cpu = 0; cpu < nr_cpus; cpu++) { in userfaultfd_stress()
1276 if (pipe2(&pipefd[cpu*2], O_CLOEXEC | O_NONBLOCK)) { in userfaultfd_stress()
1295 while (bounces--) { in userfaultfd_stress()
1325 expected_ioctls = uffd_test_ops->expected_ioctls; in userfaultfd_stress()
1350 * return -EEXIST). The problem comes at the next in userfaultfd_stress()
1355 * area_src would lead to -EEXIST failure during the in userfaultfd_stress()
1366 if (uffd_test_ops->release_pages(area_dst)) in userfaultfd_stress()
1430 * Copied from mlock2-tests.c
1458 /* Only enable write-protect test for anonymous test */ in set_test_type()