1 // SPDX-License-Identifier: GPL-2.0
2 #include <string.h>
3 #include <fcntl.h>
4 #include <sys/ioctl.h>
5 #include <linux/userfaultfd.h>
6 #include <sys/syscall.h>
7 #include <unistd.h>
8 #include "../kselftest.h"
9 #include "vm_util.h"
10
11 #define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size"
12 #define SMAP_FILE_PATH "/proc/self/smaps"
13 #define MAX_LINE_LENGTH 500
14
pagemap_get_entry(int fd,char * start)15 uint64_t pagemap_get_entry(int fd, char *start)
16 {
17 const unsigned long pfn = (unsigned long)start / getpagesize();
18 uint64_t entry;
19 int ret;
20
21 ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry));
22 if (ret != sizeof(entry))
23 ksft_exit_fail_msg("reading pagemap failed\n");
24 return entry;
25 }
26
pagemap_is_softdirty(int fd,char * start)27 bool pagemap_is_softdirty(int fd, char *start)
28 {
29 uint64_t entry = pagemap_get_entry(fd, start);
30
31 // Check if dirty bit (55th bit) is set
32 return entry & 0x0080000000000000ull;
33 }
34
clear_softdirty(void)35 void clear_softdirty(void)
36 {
37 int ret;
38 const char *ctrl = "4";
39 int fd = open("/proc/self/clear_refs", O_WRONLY);
40
41 if (fd < 0)
42 ksft_exit_fail_msg("opening clear_refs failed\n");
43 ret = write(fd, ctrl, strlen(ctrl));
44 close(fd);
45 if (ret != strlen(ctrl))
46 ksft_exit_fail_msg("writing clear_refs failed\n");
47 }
48
check_for_pattern(FILE * fp,const char * pattern,char * buf,size_t len)49 bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len)
50 {
51 while (fgets(buf, len, fp)) {
52 if (!strncmp(buf, pattern, strlen(pattern)))
53 return true;
54 }
55 return false;
56 }
57
read_pmd_pagesize(void)58 uint64_t read_pmd_pagesize(void)
59 {
60 int fd;
61 char buf[20];
62 ssize_t num_read;
63
64 fd = open(PMD_SIZE_FILE_PATH, O_RDONLY);
65 if (fd == -1)
66 ksft_exit_fail_msg("Open hpage_pmd_size failed\n");
67
68 num_read = read(fd, buf, 19);
69 if (num_read < 1) {
70 close(fd);
71 ksft_exit_fail_msg("Read hpage_pmd_size failed\n");
72 }
73 buf[num_read] = '\0';
74 close(fd);
75
76 return strtoul(buf, NULL, 10);
77 }
78
__check_huge(void * addr,char * pattern,int nr_hpages,uint64_t hpage_size)79 bool __check_huge(void *addr, char *pattern, int nr_hpages,
80 uint64_t hpage_size)
81 {
82 uint64_t thp = -1;
83 int ret;
84 FILE *fp;
85 char buffer[MAX_LINE_LENGTH];
86 char addr_pattern[MAX_LINE_LENGTH];
87
88 ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-",
89 (unsigned long) addr);
90 if (ret >= MAX_LINE_LENGTH)
91 ksft_exit_fail_msg("%s: Pattern is too long\n", __func__);
92
93 fp = fopen(SMAP_FILE_PATH, "r");
94 if (!fp)
95 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH);
96
97 if (!check_for_pattern(fp, addr_pattern, buffer, sizeof(buffer)))
98 goto err_out;
99
100 /*
101 * Fetch the pattern in the same block and check the number of
102 * hugepages.
103 */
104 if (!check_for_pattern(fp, pattern, buffer, sizeof(buffer)))
105 goto err_out;
106
107 snprintf(addr_pattern, MAX_LINE_LENGTH, "%s%%9ld kB", pattern);
108
109 if (sscanf(buffer, addr_pattern, &thp) != 1)
110 ksft_exit_fail_msg("Reading smap error\n");
111
112 err_out:
113 fclose(fp);
114 return thp == (nr_hpages * (hpage_size >> 10));
115 }
116
check_huge_anon(void * addr,int nr_hpages,uint64_t hpage_size)117 bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size)
118 {
119 return __check_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size);
120 }
121
check_huge_file(void * addr,int nr_hpages,uint64_t hpage_size)122 bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size)
123 {
124 return __check_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size);
125 }
126
check_huge_shmem(void * addr,int nr_hpages,uint64_t hpage_size)127 bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size)
128 {
129 return __check_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size);
130 }
131
uffd_open_dev(unsigned int flags)132 int uffd_open_dev(unsigned int flags)
133 {
134 int fd, uffd;
135
136 fd = open("/dev/userfaultfd", O_RDWR | O_CLOEXEC);
137 if (fd < 0)
138 return fd;
139 uffd = ioctl(fd, USERFAULTFD_IOC_NEW, flags);
140 close(fd);
141
142 return uffd;
143 }
144
uffd_open_sys(unsigned int flags)145 int uffd_open_sys(unsigned int flags)
146 {
147 return syscall(__NR_userfaultfd, flags);
148 }
149
uffd_open(unsigned int flags)150 int uffd_open(unsigned int flags)
151 {
152 int uffd = uffd_open_sys(flags);
153
154 if (uffd < 0)
155 uffd = uffd_open_dev(flags);
156
157 return uffd;
158 }
159
uffd_get_features(uint64_t * features)160 int uffd_get_features(uint64_t *features)
161 {
162 struct uffdio_api uffdio_api = { .api = UFFD_API, .features = 0 };
163 /*
164 * This should by default work in most kernels; the feature list
165 * will be the same no matter what we pass in here.
166 */
167 int fd = uffd_open(UFFD_USER_MODE_ONLY);
168
169 if (fd < 0)
170 /* Maybe the kernel is older than user-only mode? */
171 fd = uffd_open(0);
172
173 if (fd < 0)
174 return fd;
175
176 if (ioctl(fd, UFFDIO_API, &uffdio_api)) {
177 close(fd);
178 return -errno;
179 }
180
181 *features = uffdio_api.features;
182 close(fd);
183
184 return 0;
185 }
186