1 // SPDX-License-Identifier: GPL-2.0
2 #include <string.h>
3 #include <fcntl.h>
4 #include <dirent.h>
5 #include <sys/ioctl.h>
6 #include <linux/userfaultfd.h>
7 #include <sys/syscall.h>
8 #include <unistd.h>
9 #include "../kselftest.h"
10 #include "vm_util.h"
11
12 #define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size"
13 #define SMAP_FILE_PATH "/proc/self/smaps"
14 #define MAX_LINE_LENGTH 500
15
16 unsigned int __page_size;
17 unsigned int __page_shift;
18
pagemap_get_entry(int fd,char * start)19 uint64_t pagemap_get_entry(int fd, char *start)
20 {
21 const unsigned long pfn = (unsigned long)start / getpagesize();
22 uint64_t entry;
23 int ret;
24
25 ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry));
26 if (ret != sizeof(entry))
27 ksft_exit_fail_msg("reading pagemap failed\n");
28 return entry;
29 }
30
pagemap_is_softdirty(int fd,char * start)31 bool pagemap_is_softdirty(int fd, char *start)
32 {
33 return pagemap_get_entry(fd, start) & PM_SOFT_DIRTY;
34 }
35
pagemap_is_swapped(int fd,char * start)36 bool pagemap_is_swapped(int fd, char *start)
37 {
38 return pagemap_get_entry(fd, start) & PM_SWAP;
39 }
40
pagemap_is_populated(int fd,char * start)41 bool pagemap_is_populated(int fd, char *start)
42 {
43 return pagemap_get_entry(fd, start) & (PM_PRESENT | PM_SWAP);
44 }
45
pagemap_get_pfn(int fd,char * start)46 unsigned long pagemap_get_pfn(int fd, char *start)
47 {
48 uint64_t entry = pagemap_get_entry(fd, start);
49
50 /* If present (63th bit), PFN is at bit 0 -- 54. */
51 if (entry & PM_PRESENT)
52 return entry & 0x007fffffffffffffull;
53 return -1ul;
54 }
55
clear_softdirty(void)56 void clear_softdirty(void)
57 {
58 int ret;
59 const char *ctrl = "4";
60 int fd = open("/proc/self/clear_refs", O_WRONLY);
61
62 if (fd < 0)
63 ksft_exit_fail_msg("opening clear_refs failed\n");
64 ret = write(fd, ctrl, strlen(ctrl));
65 close(fd);
66 if (ret != strlen(ctrl))
67 ksft_exit_fail_msg("writing clear_refs failed\n");
68 }
69
check_for_pattern(FILE * fp,const char * pattern,char * buf,size_t len)70 bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len)
71 {
72 while (fgets(buf, len, fp)) {
73 if (!strncmp(buf, pattern, strlen(pattern)))
74 return true;
75 }
76 return false;
77 }
78
read_pmd_pagesize(void)79 uint64_t read_pmd_pagesize(void)
80 {
81 int fd;
82 char buf[20];
83 ssize_t num_read;
84
85 fd = open(PMD_SIZE_FILE_PATH, O_RDONLY);
86 if (fd == -1)
87 return 0;
88
89 num_read = read(fd, buf, 19);
90 if (num_read < 1) {
91 close(fd);
92 return 0;
93 }
94 buf[num_read] = '\0';
95 close(fd);
96
97 return strtoul(buf, NULL, 10);
98 }
99
__check_huge(void * addr,char * pattern,int nr_hpages,uint64_t hpage_size)100 bool __check_huge(void *addr, char *pattern, int nr_hpages,
101 uint64_t hpage_size)
102 {
103 uint64_t thp = -1;
104 int ret;
105 FILE *fp;
106 char buffer[MAX_LINE_LENGTH];
107 char addr_pattern[MAX_LINE_LENGTH];
108
109 ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-",
110 (unsigned long) addr);
111 if (ret >= MAX_LINE_LENGTH)
112 ksft_exit_fail_msg("%s: Pattern is too long\n", __func__);
113
114 fp = fopen(SMAP_FILE_PATH, "r");
115 if (!fp)
116 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH);
117
118 if (!check_for_pattern(fp, addr_pattern, buffer, sizeof(buffer)))
119 goto err_out;
120
121 /*
122 * Fetch the pattern in the same block and check the number of
123 * hugepages.
124 */
125 if (!check_for_pattern(fp, pattern, buffer, sizeof(buffer)))
126 goto err_out;
127
128 snprintf(addr_pattern, MAX_LINE_LENGTH, "%s%%9ld kB", pattern);
129
130 if (sscanf(buffer, addr_pattern, &thp) != 1)
131 ksft_exit_fail_msg("Reading smap error\n");
132
133 err_out:
134 fclose(fp);
135 return thp == (nr_hpages * (hpage_size >> 10));
136 }
137
check_huge_anon(void * addr,int nr_hpages,uint64_t hpage_size)138 bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size)
139 {
140 return __check_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size);
141 }
142
check_huge_file(void * addr,int nr_hpages,uint64_t hpage_size)143 bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size)
144 {
145 return __check_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size);
146 }
147
check_huge_shmem(void * addr,int nr_hpages,uint64_t hpage_size)148 bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size)
149 {
150 return __check_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size);
151 }
152
allocate_transhuge(void * ptr,int pagemap_fd)153 int64_t allocate_transhuge(void *ptr, int pagemap_fd)
154 {
155 uint64_t ent[2];
156
157 /* drop pmd */
158 if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE,
159 MAP_FIXED | MAP_ANONYMOUS |
160 MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr)
161 errx(2, "mmap transhuge");
162
163 if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE))
164 err(2, "MADV_HUGEPAGE");
165
166 /* allocate transparent huge page */
167 *(volatile void **)ptr = ptr;
168
169 if (pread(pagemap_fd, ent, sizeof(ent),
170 (uintptr_t)ptr >> (pshift() - 3)) != sizeof(ent))
171 err(2, "read pagemap");
172
173 if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) &&
174 PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) &&
175 !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - pshift())) - 1)))
176 return PAGEMAP_PFN(ent[0]);
177
178 return -1;
179 }
180
default_huge_page_size(void)181 unsigned long default_huge_page_size(void)
182 {
183 unsigned long hps = 0;
184 char *line = NULL;
185 size_t linelen = 0;
186 FILE *f = fopen("/proc/meminfo", "r");
187
188 if (!f)
189 return 0;
190 while (getline(&line, &linelen, f) > 0) {
191 if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) {
192 hps <<= 10;
193 break;
194 }
195 }
196
197 free(line);
198 fclose(f);
199 return hps;
200 }
201
detect_hugetlb_page_sizes(size_t sizes[],int max)202 int detect_hugetlb_page_sizes(size_t sizes[], int max)
203 {
204 DIR *dir = opendir("/sys/kernel/mm/hugepages/");
205 int count = 0;
206
207 if (!dir)
208 return 0;
209
210 while (count < max) {
211 struct dirent *entry = readdir(dir);
212 size_t kb;
213
214 if (!entry)
215 break;
216 if (entry->d_type != DT_DIR)
217 continue;
218 if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1)
219 continue;
220 sizes[count++] = kb * 1024;
221 ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n",
222 kb);
223 }
224 closedir(dir);
225 return count;
226 }
227
228 /* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */
uffd_register_with_ioctls(int uffd,void * addr,uint64_t len,bool miss,bool wp,bool minor,uint64_t * ioctls)229 int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len,
230 bool miss, bool wp, bool minor, uint64_t *ioctls)
231 {
232 struct uffdio_register uffdio_register = { 0 };
233 uint64_t mode = 0;
234 int ret = 0;
235
236 if (miss)
237 mode |= UFFDIO_REGISTER_MODE_MISSING;
238 if (wp)
239 mode |= UFFDIO_REGISTER_MODE_WP;
240 if (minor)
241 mode |= UFFDIO_REGISTER_MODE_MINOR;
242
243 uffdio_register.range.start = (unsigned long)addr;
244 uffdio_register.range.len = len;
245 uffdio_register.mode = mode;
246
247 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1)
248 ret = -errno;
249 else if (ioctls)
250 *ioctls = uffdio_register.ioctls;
251
252 return ret;
253 }
254
uffd_register(int uffd,void * addr,uint64_t len,bool miss,bool wp,bool minor)255 int uffd_register(int uffd, void *addr, uint64_t len,
256 bool miss, bool wp, bool minor)
257 {
258 return uffd_register_with_ioctls(uffd, addr, len,
259 miss, wp, minor, NULL);
260 }
261
uffd_unregister(int uffd,void * addr,uint64_t len)262 int uffd_unregister(int uffd, void *addr, uint64_t len)
263 {
264 struct uffdio_range range = { .start = (uintptr_t)addr, .len = len };
265 int ret = 0;
266
267 if (ioctl(uffd, UFFDIO_UNREGISTER, &range) == -1)
268 ret = -errno;
269
270 return ret;
271 }
272