56 static void os_pages_unmap(void *addr, size_t size);
61 os_pages_map(void *addr, size_t size, size_t alignment, bool *commit) {  in os_pages_map()  argument
63 	assert(ALIGNMENT_CEILING(size, os_page) == size);  in os_pages_map()
64 	assert(size != 0);  in os_pages_map()
76 	ret = VirtualAlloc(addr, size, MEM_RESERVE | (*commit ? MEM_COMMIT : 0),  in os_pages_map()
86 		ret = mmap(addr, size, prot, mmap_flags, -1, 0);  in os_pages_map()
96 		os_pages_unmap(ret, size);  in os_pages_map()
103 		prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ret, size,  in os_pages_map()
113 os_pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size,  in os_pages_trim()  argument
117 	assert(alloc_size >= leadsize + size);  in os_pages_trim()
120 	void *new_addr = os_pages_map(ret, size, PAGE, commit);  in os_pages_trim()
125 		os_pages_unmap(new_addr, size);  in os_pages_trim()
129 	size_t trailsize = alloc_size - leadsize - size;  in os_pages_trim()
135 		os_pages_unmap((void *)((uintptr_t)ret + size), trailsize);  in os_pages_trim()
142 os_pages_unmap(void *addr, size_t size) {  in os_pages_unmap()  argument
144 	assert(ALIGNMENT_CEILING(size, os_page) == size);  in os_pages_unmap()
149 	if (munmap(addr, size) == -1)  in os_pages_unmap()
169 pages_map_slow(size_t size, size_t alignment, bool *commit) {  in pages_map_slow()  argument
170 	size_t alloc_size = size + alignment - os_page;  in pages_map_slow()
172 	if (alloc_size < size) {  in pages_map_slow()
184 		ret = os_pages_trim(pages, alloc_size, leadsize, size, commit);  in pages_map_slow()
193 pages_map(void *addr, size_t size, size_t alignment, bool *commit) {  in pages_map()  argument
211 	void *ret = os_pages_map(addr, size, os_page, commit);  in pages_map()
217 		os_pages_unmap(ret, size);  in pages_map()
218 		return pages_map_slow(size, alignment, commit);  in pages_map()
226 pages_unmap(void *addr, size_t size) {  in pages_unmap()  argument
228 	assert(PAGE_CEILING(size) == size);  in pages_unmap()
230 	os_pages_unmap(addr, size);  in pages_unmap()
234 pages_commit_impl(void *addr, size_t size, bool commit) {  in pages_commit_impl()  argument
236 	assert(PAGE_CEILING(size) == size);  in pages_commit_impl()
243 	return (commit ? (addr != VirtualAlloc(addr, size, MEM_COMMIT,  in pages_commit_impl()
244 	    PAGE_READWRITE)) : (!VirtualFree(addr, size, MEM_DECOMMIT)));  in pages_commit_impl()
248 		void *result = mmap(addr, size, prot, mmap_flags | MAP_FIXED,  in pages_commit_impl()
258 			os_pages_unmap(result, size);  in pages_commit_impl()
267 pages_commit(void *addr, size_t size) {  in pages_commit()  argument
268 	return pages_commit_impl(addr, size, true);  in pages_commit()
272 pages_decommit(void *addr, size_t size) {  in pages_decommit()  argument
273 	return pages_commit_impl(addr, size, false);  in pages_decommit()
277 pages_purge_lazy(void *addr, size_t size) {  in pages_purge_lazy()  argument
279 	assert(PAGE_CEILING(size) == size);  in pages_purge_lazy()
293 	VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);  in pages_purge_lazy()
296 	return (madvise(addr, size,  in pages_purge_lazy()
305 	return (madvise(addr, size, MADV_DONTNEED) != 0);  in pages_purge_lazy()
312 pages_purge_forced(void *addr, size_t size) {  in pages_purge_forced()  argument
314 	assert(PAGE_CEILING(size) == size);  in pages_purge_forced()
322 	return (madvise(addr, size, MADV_DONTNEED) != 0);  in pages_purge_forced()
325 	return pages_commit(addr, size);  in pages_purge_forced()
332 pages_huge_impl(void *addr, size_t size, bool aligned) {  in pages_huge_impl()  argument
335 		assert(HUGEPAGE_CEILING(size) == size);  in pages_huge_impl()
338 	return (madvise(addr, size, MADV_HUGEPAGE) != 0);  in pages_huge_impl()
345 pages_huge(void *addr, size_t size) {  in pages_huge()  argument
346 	return pages_huge_impl(addr, size, true);  in pages_huge()
350 pages_huge_unaligned(void *addr, size_t size) {  in pages_huge_unaligned()  argument
351 	return pages_huge_impl(addr, size, false);  in pages_huge_unaligned()
355 pages_nohuge_impl(void *addr, size_t size, bool aligned) {  in pages_nohuge_impl()  argument
358 		assert(HUGEPAGE_CEILING(size) == size);  in pages_nohuge_impl()
362 	return (madvise(addr, size, MADV_NOHUGEPAGE) != 0);  in pages_nohuge_impl()
369 pages_nohuge(void *addr, size_t size) {  in pages_nohuge()  argument
370 	return pages_nohuge_impl(addr, size, true);  in pages_nohuge()
374 pages_nohuge_unaligned(void *addr, size_t size) {  in pages_nohuge_unaligned()  argument
375 	return pages_nohuge_impl(addr, size, false);  in pages_nohuge_unaligned()
379 pages_dontdump(void *addr, size_t size) {  in pages_dontdump()  argument
381 	assert(PAGE_CEILING(size) == size);  in pages_dontdump()
383 	return madvise(addr, size, MADV_DONTDUMP) != 0;  in pages_dontdump()
390 pages_dodump(void *addr, size_t size) {  in pages_dodump()  argument
392 	assert(PAGE_CEILING(size) == size);  in pages_dodump()
394 	return madvise(addr, size, MADV_DODUMP) != 0;  in pages_dodump()
511 pages_set_thp_state (void *ptr, size_t size) {  in pages_set_thp_state()  argument
521 		pages_huge_unaligned(ptr, size);  in pages_set_thp_state()
525 		pages_nohuge_unaligned(ptr, size);  in pages_set_thp_state()