Lines Matching +full:cpu +full:- +full:offset
7 * the COPYING file in the top-level directory.
13 * There are three threads running per CPU:
15 * 1) one per-CPU thread takes a per-page pthread_mutex in a random
17 * area_src), and increments a per-page counter in the same page,
20 * 2) another per-CPU thread handles the userfaults generated by
24 * 3) one last per-CPU thread transfers the memory in the background
26 * 2). Each cpu thread takes cares of transferring a portion of the
34 * per-CPU threads 1 by triggering userfaults inside
47 * # 10MiB-~6GiB 999 bounces, continue forever unless an error triggers
103 /* pthread_mutex_t starts at page offset 0 */
108 * to avoid non alignment faults on non-x86 archs.
114 sizeof(unsigned long long) - 1) & \
116 - 1)))
138 static void noop_alias_mapping(__u64 *start, size_t len, unsigned long offset) in noop_alias_mapping() argument
195 static void hugetlb_alias_mapping(__u64 *start, size_t len, unsigned long offset) in hugetlb_alias_mapping() argument
201 * MADV_DONTEED won't work. So exercise -EEXIST on a alias in hugetlb_alias_mapping()
203 * this way we'll exercise the -EEXEC at the fs level. in hugetlb_alias_mapping()
205 *start = (unsigned long) area_dst_alias + offset; in hugetlb_alias_mapping()
224 MAP_ANONYMOUS | MAP_SHARED, -1, 0); in shmem_allocate_area()
235 void (*alias_mapping)(__u64 *start, size_t len, unsigned long offset);
276 unsigned long cpu = (unsigned long) arg; in locking_thread() local
286 seed = (unsigned int) time(NULL) - bounces; in locking_thread()
288 seed += cpu; in locking_thread()
294 page_nr = -bounces; in locking_thread()
296 page_nr += cpu * nr_pages_per_cpu; in locking_thread()
355 page_nr, cpu, area_dst + page_nr * page_size, in locking_thread()
375 if (time(NULL) - start > 1) in locking_thread()
379 time(NULL) - start); in locking_thread()
386 unsigned long offset) in retry_copy_page() argument
388 uffd_test_ops->alias_mapping(&uffdio_copy->dst, in retry_copy_page()
389 uffdio_copy->len, in retry_copy_page()
390 offset); in retry_copy_page()
393 if (uffdio_copy->copy != -EEXIST) in retry_copy_page()
395 uffdio_copy->copy), exit(1); in retry_copy_page()
398 uffdio_copy->copy), exit(1); in retry_copy_page()
402 static int __copy_page(int ufd, unsigned long offset, bool retry) in __copy_page() argument
406 if (offset >= nr_pages * page_size) in __copy_page()
407 fprintf(stderr, "unexpected offset %lu\n", in __copy_page()
408 offset), exit(1); in __copy_page()
409 uffdio_copy.dst = (unsigned long) area_dst + offset; in __copy_page()
410 uffdio_copy.src = (unsigned long) area_src + offset; in __copy_page()
416 if (uffdio_copy.copy != -EEXIST) in __copy_page()
425 retry_copy_page(ufd, &uffdio_copy, offset); in __copy_page()
432 static int copy_page_retry(int ufd, unsigned long offset) in copy_page_retry() argument
434 return __copy_page(ufd, offset, true); in copy_page_retry()
437 static int copy_page(int ufd, unsigned long offset) in copy_page() argument
439 return __copy_page(ufd, offset, false); in copy_page()
444 unsigned long cpu = (unsigned long) arg; in uffd_poll_thread() local
449 unsigned long offset; in uffd_poll_thread() local
455 pollfd[1].fd = pipefd[cpu*2]; in uffd_poll_thread()
459 ret = poll(pollfd, 2, -1); in uffd_poll_thread()
487 offset = (char *)(unsigned long)msg.arg.pagefault.address - in uffd_poll_thread()
489 offset &= ~(page_size-1); in uffd_poll_thread()
490 if (copy_page(uffd, offset)) in uffd_poll_thread()
500 uffd_reg.range.len = msg.arg.remove.end - in uffd_poll_thread()
519 unsigned long offset; in uffd_read_thread() local
542 offset = (char *)(unsigned long)msg.arg.pagefault.address - in uffd_read_thread()
544 offset &= ~(page_size-1); in uffd_read_thread()
545 if (copy_page(uffd, offset)) in uffd_read_thread()
553 unsigned long cpu = (unsigned long) arg; in background_thread() local
556 for (page_nr = cpu * nr_pages_per_cpu; in background_thread()
557 page_nr < (cpu+1) * nr_pages_per_cpu; in background_thread()
566 unsigned long cpu; in stress() local
573 for (cpu = 0; cpu < nr_cpus; cpu++) { in stress()
574 if (pthread_create(&locking_threads[cpu], &attr, in stress()
575 locking_thread, (void *)cpu)) in stress()
578 if (pthread_create(&uffd_threads[cpu], &attr, in stress()
579 uffd_poll_thread, (void *)cpu)) in stress()
582 if (pthread_create(&uffd_threads[cpu], &attr, in stress()
584 &_userfaults[cpu])) in stress()
588 if (pthread_create(&background_threads[cpu], &attr, in stress()
589 background_thread, (void *)cpu)) in stress()
592 for (cpu = 0; cpu < nr_cpus; cpu++) in stress()
593 if (pthread_join(background_threads[cpu], NULL)) in stress()
601 * area_src (but they're guaranteed to get -EEXIST from in stress()
605 if (uffd_test_ops->release_pages(area_src)) in stress()
608 for (cpu = 0; cpu < nr_cpus; cpu++) { in stress()
611 if (write(pipefd[cpu*2+1], &c, 1) != 1) { in stress()
615 if (pthread_join(uffd_threads[cpu], &_userfaults[cpu])) in stress()
618 if (pthread_cancel(uffd_threads[cpu])) in stress()
620 if (pthread_join(uffd_threads[cpu], NULL)) in stress()
626 for (cpu = 0; cpu < nr_cpus; cpu++) in stress()
627 if (pthread_join(locking_threads[cpu], NULL)) in stress()
671 * For non-cooperative userfaultfd test we fork() a process that will
686 * test robustness use case - we release monitored area, fork a process
714 lastnr = (unsigned long)-1; in faulting_process()
766 if (uffd_test_ops->release_pages(area_dst)) in faulting_process()
779 unsigned long offset) in retry_uffdio_zeropage() argument
781 uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start, in retry_uffdio_zeropage()
782 uffdio_zeropage->range.len, in retry_uffdio_zeropage()
783 offset); in retry_uffdio_zeropage()
785 if (uffdio_zeropage->zeropage != -EEXIST) in retry_uffdio_zeropage()
787 uffdio_zeropage->zeropage), exit(1); in retry_uffdio_zeropage()
790 uffdio_zeropage->zeropage), exit(1); in retry_uffdio_zeropage()
794 static int __uffdio_zeropage(int ufd, unsigned long offset, bool retry) in __uffdio_zeropage() argument
800 has_zeropage = uffd_test_ops->expected_ioctls & (1 << _UFFDIO_ZEROPAGE); in __uffdio_zeropage()
802 if (offset >= nr_pages * page_size) in __uffdio_zeropage()
803 fprintf(stderr, "unexpected offset %lu\n", in __uffdio_zeropage()
804 offset), exit(1); in __uffdio_zeropage()
805 uffdio_zeropage.range.start = (unsigned long) area_dst + offset; in __uffdio_zeropage()
812 if (uffdio_zeropage.zeropage == -EEXIST) in __uffdio_zeropage()
813 fprintf(stderr, "UFFDIO_ZEROPAGE -EEXIST\n"), in __uffdio_zeropage()
819 if (uffdio_zeropage.zeropage != -EINVAL) in __uffdio_zeropage()
821 "UFFDIO_ZEROPAGE not -EINVAL %Ld\n", in __uffdio_zeropage()
832 offset); in __uffdio_zeropage()
845 static int uffdio_zeropage(int ufd, unsigned long offset) in uffdio_zeropage() argument
847 return __uffdio_zeropage(ufd, offset, false); in uffdio_zeropage()
859 if (uffd_test_ops->release_pages(area_dst)) in userfaultfd_zeropage_test()
870 expected_ioctls = uffd_test_ops->expected_ioctls; in userfaultfd_zeropage_test()
900 if (uffd_test_ops->release_pages(area_dst)) in userfaultfd_events_test()
915 expected_ioctls = uffd_test_ops->expected_ioctls; in userfaultfd_events_test()
960 if (uffd_test_ops->release_pages(area_dst)) in userfaultfd_sig_test()
974 expected_ioctls = uffd_test_ops->expected_ioctls; in userfaultfd_sig_test()
984 if (uffd_test_ops->release_pages(area_dst)) in userfaultfd_sig_test()
1019 unsigned long cpu; in userfaultfd_stress() local
1023 uffd_test_ops->allocate_area((void **)&area_src); in userfaultfd_stress()
1026 uffd_test_ops->allocate_area((void **)&area_dst); in userfaultfd_stress()
1046 * zero, so leave a placeholder below always non-zero in userfaultfd_stress()
1058 for (cpu = 0; cpu < nr_cpus; cpu++) { in userfaultfd_stress()
1059 if (pipe2(&pipefd[cpu*2], O_CLOEXEC | O_NONBLOCK)) { in userfaultfd_stress()
1078 while (bounces--) { in userfaultfd_stress()
1106 expected_ioctls = uffd_test_ops->expected_ioctls; in userfaultfd_stress()
1131 * return -EEXIST). The problem comes at the next in userfaultfd_stress()
1136 * area_src would lead to -EEXIST failure during the in userfaultfd_stress()
1147 if (uffd_test_ops->release_pages(area_dst)) in userfaultfd_stress()
1193 for (cpu = 0; cpu < nr_cpus; cpu++) in userfaultfd_stress()
1194 printf(" %lu", userfaults[cpu]); in userfaultfd_stress()
1207 * Copied from mlock2-tests.c