1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __LINUX_PAGE_SIZE_COMPAT_H
3 #define __LINUX_PAGE_SIZE_COMPAT_H
4
5 /*
6 * include/linux/page_size_compat.h
7 *
8 * Page Size Emulation
9 *
10 * Copyright (c) 2024, Google LLC.
11 * Author: Kalesh Singh <kaleshsingh@goole.com>
12
13 * Helper macros for page size emulation.
14 *
15 * The macros for use with the emulated page size are all
16 * namespaced by the prefix '__'.
17 *
18 * The valid range of androidboot.page_shift is [13, 16].
19 * In other words page sizes of 8KB, 16KB, 32KB and 64KB can
20 * be emulated.
21 */
22
23 #include <asm/page.h>
24
25 #define __MAX_PAGE_SHIFT 14
26 #define __MAX_PAGE_SIZE (_AC(1,UL) << __MAX_PAGE_SHIFT)
27 #define __MAX_PAGE_MASK (~(__MAX_PAGE_SIZE-1))
28
29 #ifndef __ASSEMBLY__
30
31 #include <linux/align.h>
32 #include <linux/jump_label.h>
33 #include <linux/mman.h>
34 #include <linux/printk.h>
35 #include <linux/sched.h>
36
37 #define pgcompat_err(fmt, ...) \
38 pr_err("pgcompat [%i (%s)]: " fmt, task_pid_nr(current), current->comm, ## __VA_ARGS__)
39
40 DECLARE_STATIC_KEY_FALSE(page_shift_compat_enabled);
41 extern int page_shift_compat;
42
43 #ifdef CONFIG_SHMEM
44 extern vm_fault_t shmem_fault(struct vm_fault *vmf);
45 #endif /* CONFIG_SHMEM */
46
47 #ifdef CONFIG_F2FS_FS
48 extern vm_fault_t f2fs_filemap_fault(struct vm_fault *vmf);
49 #endif /* CONFIG_F2FS_FS */
50
51 #ifdef CONFIG_X86_64
__page_shift(void)52 static __always_inline unsigned __page_shift(void)
53 {
54 if (static_branch_unlikely(&page_shift_compat_enabled))
55 return page_shift_compat;
56 else
57 return PAGE_SHIFT;
58 }
59 #else /* !CONFIG_X86_64 */
60 #define __page_shift() PAGE_SHIFT
61 #endif /* CONFIG_X86_64 */
62
63 #define __PAGE_SHIFT __page_shift()
64 #define __PAGE_SIZE (_AC(1,UL) << __PAGE_SHIFT)
65 #define __PAGE_MASK (~(__PAGE_SIZE-1))
66 #define __PAGE_ALIGN(addr) ALIGN(addr, __PAGE_SIZE)
67 #define __PAGE_ALIGN_DOWN(addr) ALIGN_DOWN(addr, __PAGE_SIZE)
68
69 #define __offset_in_page(p) ((unsigned long)(p) & ~__PAGE_MASK)
70
71 #define __offset_in_page_log(addr) \
72 ({ \
73 if (static_branch_unlikely(&page_shift_compat_enabled) && \
74 __offset_in_page(addr)) \
75 pgcompat_err("%s: addr (0x%08lx) not page aligned", __func__, addr); \
76 (__offset_in_page(addr)); \
77 })
78
79 #define __PAGE_ALIGNED(addr) (!__offset_in_page_log(addr))
80
81 /*
82 * Increases @size by an adequate amount to allow __PAGE_SIZE alignment
83 * by rounding up; given that @size is already a multiple of the
84 * base page size (PAGE_SIZE).
85 *
86 * Example:
87 * If __PAGE_SHIFT == PAGE_SHIFT == 12
88 * @size is increased by 0
89 * ((1 << (0)) - 1) << PAGE_SHIFT
90 * (1 ) - 1) << PAGE_SHIFT
91 * (0 ) << PAGE_SHIFT
92 *
93 * If __PAGE_SHIFT == 13 and PAGE_SHIFT == 12
94 * @size is increased by PAGE_SIZE (4KB):
95 * ((1 << (1)) - 1) << PAGE_SHIFT
96 * (2 ) - 1) << PAGE_SHIFT
97 * (1 ) << PAGE_SHIFT
98 * If __PAGE_SHIFT == 14 and PAGE_SHIFT == 12
99 * @size is increased by 3xPAGE_SIZE (12KB):
100 * ((1 << (2)) - 1) << PAGE_SHIFT
101 * (4 ) - 1) << PAGE_SHIFT
102 * (3 ) << PAGE_SHIFT
103 * ...
104 */
105 #define __PAGE_SIZE_ROUND_UP_ADJ(size) \
106 ((size) + (((1 << (__PAGE_SHIFT - PAGE_SHIFT)) - 1) << PAGE_SHIFT))
107
108 /*
109 * VMA is exempt from emulated page align requirements
110 *
111 * NOTE: __MAP_NO_COMPAT is not new UABI it is only ever set by the kernel
112 * in ___filemap_fixup()
113 */
114 #define __VM_NO_COMPAT _BITULL(58)
115 #define __MAP_NO_COMPAT _BITUL(31)
116
117 /*
118 * Conditional page-alignment based on mmap flags
119 *
120 * If the VMA is allowed to not respect the emulated page size, align using the
121 * base PAGE_SIZE, else align using the emulated __PAGE_SIZE.
122 */
123 #define __COMPAT_PAGE_ALIGN(size, flags) \
124 (flags & __MAP_NO_COMPAT) ? PAGE_ALIGN(size) : __PAGE_ALIGN(size)
125
126 /*
127 * Combines the mmap "flags" argument into "vm_flags"
128 *
129 * If page size emulation is enabled, adds translation of the no-compat flag.
130 */
calc_vm_flag_bits(unsigned long flags)131 static __always_inline unsigned long calc_vm_flag_bits(unsigned long flags)
132 {
133 unsigned long flag_bits = __calc_vm_flag_bits(flags);
134
135 if (static_branch_unlikely(&page_shift_compat_enabled))
136 flag_bits |= _calc_vm_trans(flags, __MAP_NO_COMPAT, __VM_NO_COMPAT );
137
138 return flag_bits;
139 }
140
141 extern unsigned long ___filemap_len(struct inode *inode, unsigned long pgoff,
142 unsigned long len, unsigned long flags);
143
144 extern void ___filemap_fixup(unsigned long addr, unsigned long prot, unsigned long old_len,
145 unsigned long new_len);
146
__filemap_len(struct inode * inode,unsigned long pgoff,unsigned long len,unsigned long flags)147 static __always_inline unsigned long __filemap_len(struct inode *inode, unsigned long pgoff,
148 unsigned long len, unsigned long flags)
149 {
150 if (static_branch_unlikely(&page_shift_compat_enabled))
151 return ___filemap_len(inode, pgoff, len, flags);
152 else
153 return len;
154 }
155
__filemap_fixup(unsigned long addr,unsigned long prot,unsigned long old_len,unsigned long new_len)156 static __always_inline void __filemap_fixup(unsigned long addr, unsigned long prot,
157 unsigned long old_len, unsigned long new_len)
158 {
159
160 if (static_branch_unlikely(&page_shift_compat_enabled))
161 ___filemap_fixup(addr, prot, old_len, new_len);
162 }
163
164 extern void __fold_filemap_fixup_entry(struct vma_iterator *iter, unsigned long *end);
165
166 #endif /* !__ASSEMBLY__ */
167
168 #endif /* __LINUX_PAGE_SIZE_COMPAT_H */
169