1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Clang Control Flow Integrity (CFI) error and slowpath handling.
4 *
5 * Copyright (C) 2019 Google LLC
6 */
7
8 #include <linux/gfp.h>
9 #include <linux/module.h>
10 #include <linux/mutex.h>
11 #include <linux/printk.h>
12 #include <linux/ratelimit.h>
13 #include <linux/rcupdate.h>
14 #include <asm/cacheflush.h>
15 #include <asm/set_memory.h>
16
17 /* Compiler-defined handler names */
18 #ifdef CONFIG_CFI_PERMISSIVE
19 #define cfi_failure_handler __ubsan_handle_cfi_check_fail
20 #define cfi_slowpath_handler __cfi_slowpath_diag
21 #else /* enforcing */
22 #define cfi_failure_handler __ubsan_handle_cfi_check_fail_abort
23 #define cfi_slowpath_handler __cfi_slowpath
24 #endif /* CONFIG_CFI_PERMISSIVE */
25
handle_cfi_failure(void * ptr)26 static inline void handle_cfi_failure(void *ptr)
27 {
28 if (IS_ENABLED(CONFIG_CFI_PERMISSIVE))
29 WARN_RATELIMIT(1, "CFI failure (target: %pS):\n", ptr);
30 else
31 panic("CFI failure (target: %pS)\n", ptr);
32 }
33
34 #ifdef CONFIG_MODULES
35 #ifdef CONFIG_CFI_CLANG_SHADOW
36 struct shadow_range {
37 /* Module address range */
38 unsigned long mod_min_addr;
39 unsigned long mod_max_addr;
40 /* Module page range */
41 unsigned long min_page;
42 unsigned long max_page;
43 };
44
45 #define SHADOW_ORDER 2
46 #define SHADOW_PAGES (1 << SHADOW_ORDER)
47 #define SHADOW_SIZE \
48 ((SHADOW_PAGES * PAGE_SIZE - sizeof(struct shadow_range)) / sizeof(u16))
49 #define SHADOW_INVALID 0xFFFF
50
51 struct cfi_shadow {
52 /* Page range covered by the shadow */
53 struct shadow_range r;
54 /* Page offsets to __cfi_check functions in modules */
55 u16 shadow[SHADOW_SIZE];
56 };
57
58 static DEFINE_MUTEX(shadow_update_lock);
59 static struct cfi_shadow __rcu *cfi_shadow __read_mostly;
60
ptr_to_shadow(const struct cfi_shadow * s,unsigned long ptr)61 static inline int ptr_to_shadow(const struct cfi_shadow *s, unsigned long ptr)
62 {
63 unsigned long index;
64 unsigned long page = ptr >> PAGE_SHIFT;
65
66 if (unlikely(page < s->r.min_page))
67 return -1; /* Outside of module area */
68
69 index = page - s->r.min_page;
70
71 if (index >= SHADOW_SIZE)
72 return -1; /* Cannot be addressed with shadow */
73
74 return (int)index;
75 }
76
shadow_to_ptr(const struct cfi_shadow * s,int index)77 static inline unsigned long shadow_to_ptr(const struct cfi_shadow *s,
78 int index)
79 {
80 if (unlikely(index < 0 || index >= SHADOW_SIZE))
81 return 0;
82
83 if (unlikely(s->shadow[index] == SHADOW_INVALID))
84 return 0;
85
86 return (s->r.min_page + s->shadow[index]) << PAGE_SHIFT;
87 }
88
shadow_to_page(const struct cfi_shadow * s,int index)89 static inline unsigned long shadow_to_page(const struct cfi_shadow *s,
90 int index)
91 {
92 if (unlikely(index < 0 || index >= SHADOW_SIZE))
93 return 0;
94
95 return (s->r.min_page + index) << PAGE_SHIFT;
96 }
97
prepare_next_shadow(const struct cfi_shadow __rcu * prev,struct cfi_shadow * next)98 static void prepare_next_shadow(const struct cfi_shadow __rcu *prev,
99 struct cfi_shadow *next)
100 {
101 int i, index, check;
102
103 /* Mark everything invalid */
104 memset(next->shadow, 0xFF, sizeof(next->shadow));
105
106 if (!prev)
107 return; /* No previous shadow */
108
109 /* If the base address didn't change, update is not needed */
110 if (prev->r.min_page == next->r.min_page) {
111 memcpy(next->shadow, prev->shadow, sizeof(next->shadow));
112 return;
113 }
114
115 /* Convert the previous shadow to the new address range */
116 for (i = 0; i < SHADOW_SIZE; ++i) {
117 if (prev->shadow[i] == SHADOW_INVALID)
118 continue;
119
120 index = ptr_to_shadow(next, shadow_to_page(prev, i));
121 if (index < 0)
122 continue;
123
124 check = ptr_to_shadow(next,
125 shadow_to_ptr(prev, prev->shadow[i]));
126 if (check < 0)
127 continue;
128
129 next->shadow[index] = (u16)check;
130 }
131 }
132
add_module_to_shadow(struct cfi_shadow * s,struct module * mod)133 static void add_module_to_shadow(struct cfi_shadow *s, struct module *mod)
134 {
135 unsigned long ptr;
136 unsigned long min_page_addr;
137 unsigned long max_page_addr;
138 unsigned long check = (unsigned long)mod->cfi_check;
139 int check_index = ptr_to_shadow(s, check);
140
141 if (unlikely((check & PAGE_MASK) != check))
142 return; /* Must be page aligned */
143
144 if (check_index < 0)
145 return; /* Module not addressable with shadow */
146
147 min_page_addr = (unsigned long)mod->core_layout.base & PAGE_MASK;
148 max_page_addr = (unsigned long)mod->core_layout.base +
149 mod->core_layout.text_size;
150 max_page_addr &= PAGE_MASK;
151
152 /* For each page, store the check function index in the shadow */
153 for (ptr = min_page_addr; ptr <= max_page_addr; ptr += PAGE_SIZE) {
154 int index = ptr_to_shadow(s, ptr);
155
156 if (index >= 0) {
157 /* Each page must only contain one module */
158 WARN_ON(s->shadow[index] != SHADOW_INVALID);
159 s->shadow[index] = (u16)check_index;
160 }
161 }
162 }
163
remove_module_from_shadow(struct cfi_shadow * s,struct module * mod)164 static void remove_module_from_shadow(struct cfi_shadow *s, struct module *mod)
165 {
166 unsigned long ptr;
167 unsigned long min_page_addr;
168 unsigned long max_page_addr;
169
170 min_page_addr = (unsigned long)mod->core_layout.base & PAGE_MASK;
171 max_page_addr = (unsigned long)mod->core_layout.base +
172 mod->core_layout.text_size;
173 max_page_addr &= PAGE_MASK;
174
175 for (ptr = min_page_addr; ptr <= max_page_addr; ptr += PAGE_SIZE) {
176 int index = ptr_to_shadow(s, ptr);
177
178 if (index >= 0)
179 s->shadow[index] = SHADOW_INVALID;
180 }
181 }
182
183 typedef void (*update_shadow_fn)(struct cfi_shadow *, struct module *);
184
update_shadow(struct module * mod,unsigned long min_addr,unsigned long max_addr,update_shadow_fn fn)185 static void update_shadow(struct module *mod, unsigned long min_addr,
186 unsigned long max_addr, update_shadow_fn fn)
187 {
188 struct cfi_shadow *prev;
189 struct cfi_shadow *next = (struct cfi_shadow *)
190 __get_free_pages(GFP_KERNEL, SHADOW_ORDER);
191
192 next->r.mod_min_addr = min_addr;
193 next->r.mod_max_addr = max_addr;
194 next->r.min_page = min_addr >> PAGE_SHIFT;
195 next->r.max_page = max_addr >> PAGE_SHIFT;
196
197 mutex_lock(&shadow_update_lock);
198 prev = rcu_dereference_protected(cfi_shadow, 1);
199 prepare_next_shadow(prev, next);
200
201 fn(next, mod);
202 set_memory_ro((unsigned long)next, SHADOW_PAGES);
203 rcu_assign_pointer(cfi_shadow, next);
204
205 mutex_unlock(&shadow_update_lock);
206 synchronize_rcu();
207
208 if (prev) {
209 set_memory_rw((unsigned long)prev, SHADOW_PAGES);
210 free_pages((unsigned long)prev, SHADOW_ORDER);
211 }
212 }
213
cfi_module_add(struct module * mod,unsigned long min_addr,unsigned long max_addr)214 void cfi_module_add(struct module *mod, unsigned long min_addr,
215 unsigned long max_addr)
216 {
217 update_shadow(mod, min_addr, max_addr, add_module_to_shadow);
218 }
219 EXPORT_SYMBOL(cfi_module_add);
220
cfi_module_remove(struct module * mod,unsigned long min_addr,unsigned long max_addr)221 void cfi_module_remove(struct module *mod, unsigned long min_addr,
222 unsigned long max_addr)
223 {
224 update_shadow(mod, min_addr, max_addr, remove_module_from_shadow);
225 }
226 EXPORT_SYMBOL(cfi_module_remove);
227
ptr_to_check_fn(const struct cfi_shadow __rcu * s,unsigned long ptr)228 static inline cfi_check_fn ptr_to_check_fn(const struct cfi_shadow __rcu *s,
229 unsigned long ptr)
230 {
231 int index;
232
233 if (unlikely(!s))
234 return NULL; /* No shadow available */
235
236 if (ptr < s->r.mod_min_addr || ptr > s->r.mod_max_addr)
237 return NULL; /* Not in a mapped module */
238
239 index = ptr_to_shadow(s, ptr);
240 if (index < 0)
241 return NULL; /* Cannot be addressed with shadow */
242
243 return (cfi_check_fn)shadow_to_ptr(s, index);
244 }
245 #endif /* CONFIG_CFI_CLANG_SHADOW */
246
find_module_cfi_check(void * ptr)247 static inline cfi_check_fn find_module_cfi_check(void *ptr)
248 {
249 struct module *mod;
250
251 preempt_disable();
252 mod = __module_address((unsigned long)ptr);
253 preempt_enable();
254
255 if (mod)
256 return mod->cfi_check;
257
258 return CFI_CHECK_FN;
259 }
260
find_cfi_check(void * ptr)261 static inline cfi_check_fn find_cfi_check(void *ptr)
262 {
263 #ifdef CONFIG_CFI_CLANG_SHADOW
264 cfi_check_fn f;
265
266 if (!rcu_access_pointer(cfi_shadow))
267 return CFI_CHECK_FN; /* No loaded modules */
268
269 /* Look up the __cfi_check function to use */
270 rcu_read_lock();
271 f = ptr_to_check_fn(rcu_dereference(cfi_shadow), (unsigned long)ptr);
272 rcu_read_unlock();
273
274 if (f)
275 return f;
276
277 /*
278 * Fall back to find_module_cfi_check, which works also for a larger
279 * module address space, but is slower.
280 */
281 #endif /* CONFIG_CFI_CLANG_SHADOW */
282
283 return find_module_cfi_check(ptr);
284 }
285
cfi_slowpath_handler(uint64_t id,void * ptr,void * diag)286 void cfi_slowpath_handler(uint64_t id, void *ptr, void *diag)
287 {
288 cfi_check_fn check = find_cfi_check(ptr);
289
290 if (likely(check))
291 check(id, ptr, diag);
292 else /* Don't allow unchecked modules */
293 handle_cfi_failure(ptr);
294 }
295 EXPORT_SYMBOL(cfi_slowpath_handler);
296 #endif /* CONFIG_MODULES */
297
cfi_failure_handler(void * data,void * ptr,void * vtable)298 void cfi_failure_handler(void *data, void *ptr, void *vtable)
299 {
300 handle_cfi_failure(ptr);
301 }
302 EXPORT_SYMBOL(cfi_failure_handler);
303
__cfi_check_fail(void * data,void * ptr)304 void __cfi_check_fail(void *data, void *ptr)
305 {
306 handle_cfi_failure(ptr);
307 }
308