1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Page Size Emulation
4 *
5 * Copyright (c) 2024, Google LLC.
6 * Author: Kalesh Singh <kaleshsingh@goole.com>
7 */
8
9 #include <linux/errno.h>
10 #include <linux/init.h>
11 #include <linux/kstrtox.h>
12 #include <linux/mm.h>
13 #include <linux/page_size_compat.h>
14
15 #define MIN_PAGE_SHIFT_COMPAT (PAGE_SHIFT + 1)
16 #define MAX_PAGE_SHIFT_COMPAT 16 /* Max of 64KB */
17 #define __MMAP_RND_BITS(x) (x - (__PAGE_SHIFT - PAGE_SHIFT))
18
19 DEFINE_STATIC_KEY_FALSE(page_shift_compat_enabled);
20 EXPORT_SYMBOL_GPL(page_shift_compat_enabled);
21
22 int page_shift_compat = MIN_PAGE_SHIFT_COMPAT;
23 EXPORT_SYMBOL_GPL(page_shift_compat);
24
early_page_shift_compat(char * buf)25 static int __init early_page_shift_compat(char *buf)
26 {
27 int ret;
28
29 ret = kstrtoint(buf, 10, &page_shift_compat);
30 if (ret)
31 return ret;
32
33 /* Only supported on 4KB kernel */
34 if (PAGE_SHIFT != 12)
35 return -ENOTSUPP;
36
37 if (page_shift_compat < MIN_PAGE_SHIFT_COMPAT ||
38 page_shift_compat > MAX_PAGE_SHIFT_COMPAT)
39 return -EINVAL;
40
41 static_branch_enable(&page_shift_compat_enabled);
42
43 return 0;
44 }
45 early_param("page_shift", early_page_shift_compat);
46
init_mmap_rnd_bits(void)47 static int __init init_mmap_rnd_bits(void)
48 {
49 if (!static_branch_unlikely(&page_shift_compat_enabled))
50 return 0;
51
52 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
53 mmap_rnd_bits_min = __MMAP_RND_BITS(CONFIG_ARCH_MMAP_RND_BITS_MIN);
54 mmap_rnd_bits_max = __MMAP_RND_BITS(CONFIG_ARCH_MMAP_RND_BITS_MAX);
55 mmap_rnd_bits = __MMAP_RND_BITS(CONFIG_ARCH_MMAP_RND_BITS);
56 #endif
57
58 return 0;
59 }
60 core_initcall(init_mmap_rnd_bits);
61
62 /*
63 * Updates len to avoid mapping off the end of the file.
64 *
65 * The length of the original mapping must be updated before
66 * it's VMA is created to avoid an unaligned munmap in the
67 * MAP_FIXED fixup mapping.
68 */
___filemap_len(struct inode * inode,unsigned long pgoff,unsigned long len,unsigned long flags)69 unsigned long ___filemap_len(struct inode *inode, unsigned long pgoff, unsigned long len,
70 unsigned long flags)
71 {
72 unsigned long file_size;
73 unsigned long new_len;
74 pgoff_t max_pgcount;
75 pgoff_t last_pgoff;
76
77 if (flags & __MAP_NO_COMPAT)
78 return len;
79
80 file_size = (unsigned long) i_size_read(inode);
81
82 /*
83 * Round up, so that this is a count (not an index). This simplifies
84 * the following calculations.
85 */
86 max_pgcount = DIV_ROUND_UP(file_size, PAGE_SIZE);
87 last_pgoff = pgoff + (len >> PAGE_SHIFT);
88
89 if (unlikely(last_pgoff >= max_pgcount)) {
90 new_len = (max_pgcount - pgoff) << PAGE_SHIFT;
91 /* Careful of underflows in special files */
92 if (new_len > 0 && new_len < len)
93 return new_len;
94 }
95
96 return len;
97 }
98
is_shmem_fault(const struct vm_operations_struct * vm_ops)99 static inline bool is_shmem_fault(const struct vm_operations_struct *vm_ops)
100 {
101 #ifdef CONFIG_SHMEM
102 return vm_ops->fault == shmem_fault;
103 #else
104 return false;
105 #endif
106 }
107
is_f2fs_filemap_fault(const struct vm_operations_struct * vm_ops)108 static inline bool is_f2fs_filemap_fault(const struct vm_operations_struct *vm_ops)
109 {
110 #ifdef CONFIG_F2FS_FS
111 return vm_ops->fault == f2fs_filemap_fault;
112 #else
113 return false;
114 #endif
115 }
116
is_filemap_fault(const struct vm_operations_struct * vm_ops)117 static inline bool is_filemap_fault(const struct vm_operations_struct *vm_ops)
118 {
119 return vm_ops->fault == filemap_fault;
120 }
121
122 /*
123 * This is called to fill any holes created by ___filemap_len()
124 * with an anonymous mapping.
125 */
___filemap_fixup(unsigned long addr,unsigned long prot,unsigned long old_len,unsigned long new_len)126 void ___filemap_fixup(unsigned long addr, unsigned long prot, unsigned long old_len,
127 unsigned long new_len)
128 {
129 unsigned long anon_len = old_len - new_len;
130 unsigned long anon_addr = addr + new_len;
131 struct mm_struct *mm = current->mm;
132 unsigned long populate = 0;
133 struct vm_area_struct *vma;
134 const struct vm_operations_struct *vm_ops;
135
136 if (!anon_len)
137 return;
138
139 BUG_ON(new_len > old_len);
140
141 /* The original do_mmap() failed */
142 if (IS_ERR_VALUE(addr))
143 return;
144
145 vma = find_vma(mm, addr);
146
147 /*
148 * This should never happen, VMA was inserted and we still
149 * haven't released the mmap write lock.
150 */
151 BUG_ON(!vma);
152
153 vm_ops = vma->vm_ops;
154 if (!vm_ops)
155 return;
156
157 /*
158 * Insert fixup vmas for file backed and shmem backed VMAs.
159 *
160 * Faulting off the end of a file will result in SIGBUS since there is no
161 * file page for the given file offset.
162 *
163 * shmem pages live in page cache or swap cache. Looking up a page cache
164 * page with an index (pgoff) beyond the file is invalid and will result
165 * in shmem_get_folio_gfp() returning -EINVAL.
166 */
167 if (!is_filemap_fault(vm_ops) && !is_f2fs_filemap_fault(vm_ops) &&
168 !is_shmem_fault(vm_ops))
169 return;
170
171 /*
172 * Override the end of the file mapping that is off the file
173 * with an anonymous mapping.
174 */
175 anon_addr = do_mmap(NULL, anon_addr, anon_len, prot,
176 MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED|__MAP_NO_COMPAT,
177 0, 0, &populate, NULL);
178 }
179
180 /*
181 * Folds any anon fixup entries created by ___filemap_fixup()
182 * into the previous mapping so that /proc/<pid>/[s]maps don't
183 * show unaliged entries.
184 */
__fold_filemap_fixup_entry(struct vma_iterator * iter,unsigned long * end)185 void __fold_filemap_fixup_entry(struct vma_iterator *iter, unsigned long *end)
186 {
187 struct vm_area_struct *next_vma;
188
189 /* Not emulating page size? */
190 if (!static_branch_unlikely(&page_shift_compat_enabled))
191 return;
192
193 /* Advance iterator */
194 next_vma = vma_next(iter);
195
196 /* If fixup VMA, adjust the end to cover its extent */
197 if (next_vma && (next_vma->vm_flags & __VM_NO_COMPAT)) {
198 *end = next_vma->vm_end;
199 return;
200 }
201
202 /* Rewind iterator */
203 vma_prev(iter);
204 }
205