1 /*
2 * CFI (Control Flow Integrity) error and slowpath handling
3 *
4 * Copyright (C) 2017 Google, Inc.
5 */
6
7 #include <linux/gfp.h>
8 #include <linux/module.h>
9 #include <linux/printk.h>
10 #include <linux/ratelimit.h>
11 #include <linux/rcupdate.h>
12 #include <linux/spinlock.h>
13 #include <asm/bug.h>
14 #include <asm/cacheflush.h>
15 #include <asm/memory.h>
16 #include <asm/set_memory.h>
17
18 /* Compiler-defined handler names */
19 #ifdef CONFIG_CFI_PERMISSIVE
20 #define cfi_failure_handler __ubsan_handle_cfi_check_fail
21 #define cfi_slowpath_handler __cfi_slowpath_diag
22 #else /* enforcing */
23 #define cfi_failure_handler __ubsan_handle_cfi_check_fail_abort
24 #define cfi_slowpath_handler __cfi_slowpath
25 #endif /* CONFIG_CFI_PERMISSIVE */
26
handle_cfi_failure(void * ptr)27 static inline void handle_cfi_failure(void *ptr)
28 {
29 #ifdef CONFIG_CFI_PERMISSIVE
30 WARN_RATELIMIT(1, "CFI failure (target: [<%px>] %pF):\n", ptr, ptr);
31 #else
32 pr_err("CFI failure (target: [<%px>] %pF):\n", ptr, ptr);
33 BUG();
34 #endif
35 }
36
37 #ifdef CONFIG_MODULES
38 #ifdef CONFIG_CFI_CLANG_SHADOW
39 struct shadow_range {
40 /* Module address range */
41 unsigned long mod_min_addr;
42 unsigned long mod_max_addr;
43 /* Module page range */
44 unsigned long min_page;
45 unsigned long max_page;
46 };
47
48 #define SHADOW_ORDER 1
49 #define SHADOW_PAGES (1 << SHADOW_ORDER)
50 #define SHADOW_SIZE \
51 ((SHADOW_PAGES * PAGE_SIZE - sizeof(struct shadow_range)) / sizeof(u16))
52 #define SHADOW_INVALID 0xFFFF
53
54 struct cfi_shadow {
55 /* Page range covered by the shadow */
56 struct shadow_range r;
57 /* Page offsets to __cfi_check functions in modules */
58 u16 shadow[SHADOW_SIZE];
59 };
60
61 static DEFINE_SPINLOCK(shadow_update_lock);
62 static struct cfi_shadow __rcu *cfi_shadow __read_mostly = NULL;
63
ptr_to_shadow(const struct cfi_shadow * s,unsigned long ptr)64 static inline int ptr_to_shadow(const struct cfi_shadow *s, unsigned long ptr)
65 {
66 unsigned long index;
67 unsigned long page = ptr >> PAGE_SHIFT;
68
69 if (unlikely(page < s->r.min_page))
70 return -1; /* Outside of module area */
71
72 index = page - s->r.min_page;
73
74 if (index >= SHADOW_SIZE)
75 return -1; /* Cannot be addressed with shadow */
76
77 return (int)index;
78 }
79
shadow_to_ptr(const struct cfi_shadow * s,int index)80 static inline unsigned long shadow_to_ptr(const struct cfi_shadow *s,
81 int index)
82 {
83 BUG_ON(index < 0 || index >= SHADOW_SIZE);
84
85 if (unlikely(s->shadow[index] == SHADOW_INVALID))
86 return 0;
87
88 return (s->r.min_page + s->shadow[index]) << PAGE_SHIFT;
89 }
90
shadow_to_page(const struct cfi_shadow * s,int index)91 static inline unsigned long shadow_to_page(const struct cfi_shadow *s,
92 int index)
93 {
94 BUG_ON(index < 0 || index >= SHADOW_SIZE);
95
96 return (s->r.min_page + index) << PAGE_SHIFT;
97 }
98
prepare_next_shadow(const struct cfi_shadow __rcu * prev,struct cfi_shadow * next)99 static void prepare_next_shadow(const struct cfi_shadow __rcu *prev,
100 struct cfi_shadow *next)
101 {
102 int i, index, check;
103
104 /* Mark everything invalid */
105 memset(next->shadow, 0xFF, sizeof(next->shadow));
106
107 if (!prev)
108 return; /* No previous shadow */
109
110 /* If the base address didn't change, update is not needed */
111 if (prev->r.min_page == next->r.min_page) {
112 memcpy(next->shadow, prev->shadow, sizeof(next->shadow));
113 return;
114 }
115
116 /* Convert the previous shadow to the new address range */
117 for (i = 0; i < SHADOW_SIZE; ++i) {
118 if (prev->shadow[i] == SHADOW_INVALID)
119 continue;
120
121 index = ptr_to_shadow(next, shadow_to_page(prev, i));
122 if (index < 0)
123 continue;
124
125 check = ptr_to_shadow(next,
126 shadow_to_ptr(prev, prev->shadow[i]));
127 if (check < 0)
128 continue;
129
130 next->shadow[index] = (u16)check;
131 }
132 }
133
add_module_to_shadow(struct cfi_shadow * s,struct module * mod)134 static void add_module_to_shadow(struct cfi_shadow *s, struct module *mod)
135 {
136 unsigned long ptr;
137 unsigned long min_page_addr;
138 unsigned long max_page_addr;
139 unsigned long check = (unsigned long)mod->cfi_check;
140 int check_index = ptr_to_shadow(s, check);
141
142 BUG_ON((check & PAGE_MASK) != check); /* Must be page aligned */
143
144 if (check_index < 0)
145 return; /* Module not addressable with shadow */
146
147 min_page_addr = (unsigned long)mod->core_layout.base & PAGE_MASK;
148 max_page_addr = (unsigned long)mod->core_layout.base +
149 mod->core_layout.text_size;
150 max_page_addr &= PAGE_MASK;
151
152 /* For each page, store the check function index in the shadow */
153 for (ptr = min_page_addr; ptr <= max_page_addr; ptr += PAGE_SIZE) {
154 int index = ptr_to_shadow(s, ptr);
155 if (index >= 0) {
156 /* Assume a page only contains code for one module */
157 BUG_ON(s->shadow[index] != SHADOW_INVALID);
158 s->shadow[index] = (u16)check_index;
159 }
160 }
161 }
162
remove_module_from_shadow(struct cfi_shadow * s,struct module * mod)163 static void remove_module_from_shadow(struct cfi_shadow *s, struct module *mod)
164 {
165 unsigned long ptr;
166 unsigned long min_page_addr;
167 unsigned long max_page_addr;
168
169 min_page_addr = (unsigned long)mod->core_layout.base & PAGE_MASK;
170 max_page_addr = (unsigned long)mod->core_layout.base +
171 mod->core_layout.text_size;
172 max_page_addr &= PAGE_MASK;
173
174 for (ptr = min_page_addr; ptr <= max_page_addr; ptr += PAGE_SIZE) {
175 int index = ptr_to_shadow(s, ptr);
176 if (index >= 0)
177 s->shadow[index] = SHADOW_INVALID;
178 }
179 }
180
181 typedef void (*update_shadow_fn)(struct cfi_shadow *, struct module *);
182
update_shadow(struct module * mod,unsigned long min_addr,unsigned long max_addr,update_shadow_fn fn)183 static void update_shadow(struct module *mod, unsigned long min_addr,
184 unsigned long max_addr, update_shadow_fn fn)
185 {
186 struct cfi_shadow *prev;
187 struct cfi_shadow *next = (struct cfi_shadow *)
188 __get_free_pages(GFP_KERNEL, SHADOW_ORDER);
189
190 BUG_ON(!next);
191
192 next->r.mod_min_addr = min_addr;
193 next->r.mod_max_addr = max_addr;
194 next->r.min_page = min_addr >> PAGE_SHIFT;
195 next->r.max_page = max_addr >> PAGE_SHIFT;
196
197 spin_lock(&shadow_update_lock);
198 prev = rcu_dereference_protected(cfi_shadow, 1);
199 prepare_next_shadow(prev, next);
200
201 fn(next, mod);
202 set_memory_ro((unsigned long)next, SHADOW_PAGES);
203 rcu_assign_pointer(cfi_shadow, next);
204
205 spin_unlock(&shadow_update_lock);
206 synchronize_rcu();
207
208 if (prev) {
209 set_memory_rw((unsigned long)prev, SHADOW_PAGES);
210 free_pages((unsigned long)prev, SHADOW_ORDER);
211 }
212 }
213
cfi_module_add(struct module * mod,unsigned long min_addr,unsigned long max_addr)214 void cfi_module_add(struct module *mod, unsigned long min_addr,
215 unsigned long max_addr)
216 {
217 update_shadow(mod, min_addr, max_addr, add_module_to_shadow);
218 }
219 EXPORT_SYMBOL(cfi_module_add);
220
cfi_module_remove(struct module * mod,unsigned long min_addr,unsigned long max_addr)221 void cfi_module_remove(struct module *mod, unsigned long min_addr,
222 unsigned long max_addr)
223 {
224 update_shadow(mod, min_addr, max_addr, remove_module_from_shadow);
225 }
226 EXPORT_SYMBOL(cfi_module_remove);
227
ptr_to_check_fn(const struct cfi_shadow __rcu * s,unsigned long ptr)228 static inline cfi_check_fn ptr_to_check_fn(const struct cfi_shadow __rcu *s,
229 unsigned long ptr)
230 {
231 int index;
232 unsigned long check;
233
234 if (unlikely(!s))
235 return NULL; /* No shadow available */
236
237 if (ptr < s->r.mod_min_addr || ptr > s->r.mod_max_addr)
238 return NULL; /* Not in a mapped module */
239
240 index = ptr_to_shadow(s, ptr);
241 if (index < 0)
242 return NULL; /* Cannot be addressed with shadow */
243
244 return (cfi_check_fn)shadow_to_ptr(s, index);
245 }
246 #endif /* CONFIG_CFI_CLANG_SHADOW */
247
find_module_cfi_check(void * ptr)248 static inline cfi_check_fn find_module_cfi_check(void *ptr)
249 {
250 struct module *mod;
251
252 preempt_disable();
253 mod = __module_address((unsigned long)ptr);
254 preempt_enable();
255
256 if (mod)
257 return mod->cfi_check;
258
259 return CFI_CHECK_FN;
260 }
261
find_cfi_check(void * ptr)262 static inline cfi_check_fn find_cfi_check(void *ptr)
263 {
264 #ifdef CONFIG_CFI_CLANG_SHADOW
265 cfi_check_fn f;
266
267 if (!rcu_access_pointer(cfi_shadow))
268 return CFI_CHECK_FN; /* No loaded modules */
269
270 /* Look up the __cfi_check function to use */
271 rcu_read_lock();
272 f = ptr_to_check_fn(rcu_dereference(cfi_shadow), (unsigned long)ptr);
273 rcu_read_unlock();
274
275 if (f)
276 return f;
277
278 /*
279 * Fall back to find_module_cfi_check, which works also for a larger
280 * module address space, but is slower.
281 */
282 #endif /* CONFIG_CFI_CLANG_SHADOW */
283
284 return find_module_cfi_check(ptr);
285 }
286
cfi_slowpath_handler(uint64_t id,void * ptr,void * diag)287 void cfi_slowpath_handler(uint64_t id, void *ptr, void *diag)
288 {
289 cfi_check_fn check = find_cfi_check(ptr);
290
291 if (likely(check))
292 check(id, ptr, diag);
293 else /* Don't allow unchecked modules */
294 handle_cfi_failure(ptr);
295 }
296 EXPORT_SYMBOL(cfi_slowpath_handler);
297 #endif /* CONFIG_MODULES */
298
cfi_failure_handler(void * data,void * ptr,void * vtable)299 void cfi_failure_handler(void *data, void *ptr, void *vtable)
300 {
301 handle_cfi_failure(ptr);
302 }
303 EXPORT_SYMBOL(cfi_failure_handler);
304
__cfi_check_fail(void * data,void * ptr)305 void __cfi_check_fail(void *data, void *ptr)
306 {
307 handle_cfi_failure(ptr);
308 }
309