1 /*
2 * CFI (Control Flow Integrity) error and slowpath handling
3 *
4 * Copyright (C) 2017 Google, Inc.
5 */
6
7 #include <linux/gfp.h>
8 #include <linux/module.h>
9 #include <linux/printk.h>
10 #include <linux/ratelimit.h>
11 #include <linux/rcupdate.h>
12 #include <linux/spinlock.h>
13 #include <asm/bug.h>
14 #include <asm/cacheflush.h>
15 #include <asm/memory.h>
16
17 /* Compiler-defined handler names */
18 #ifdef CONFIG_CFI_PERMISSIVE
19 #define cfi_failure_handler __ubsan_handle_cfi_check_fail
20 #define cfi_slowpath_handler __cfi_slowpath_diag
21 #else /* enforcing */
22 #define cfi_failure_handler __ubsan_handle_cfi_check_fail_abort
23 #define cfi_slowpath_handler __cfi_slowpath
24 #endif /* CONFIG_CFI_PERMISSIVE */
25
handle_cfi_failure()26 static inline void handle_cfi_failure()
27 {
28 #ifdef CONFIG_CFI_PERMISSIVE
29 WARN_RATELIMIT(1, "CFI failure:\n");
30 #else
31 pr_err("CFI failure:\n");
32 BUG();
33 #endif
34 }
35
36 #ifdef CONFIG_MODULES
37 #ifdef CONFIG_CFI_CLANG_SHADOW
38 struct shadow_range {
39 /* Module address range */
40 unsigned long mod_min_addr;
41 unsigned long mod_max_addr;
42 /* Module page range */
43 unsigned long min_page;
44 unsigned long max_page;
45 };
46
47 #define SHADOW_ORDER 1
48 #define SHADOW_PAGES (1 << SHADOW_ORDER)
49 #define SHADOW_SIZE \
50 ((SHADOW_PAGES * PAGE_SIZE - sizeof(struct shadow_range)) / sizeof(u16))
51 #define SHADOW_INVALID 0xFFFF
52
53 struct cfi_shadow {
54 /* Page range covered by the shadow */
55 struct shadow_range r;
56 /* Page offsets to __cfi_check functions in modules */
57 u16 shadow[SHADOW_SIZE];
58 };
59
60 static DEFINE_SPINLOCK(shadow_update_lock);
61 static struct cfi_shadow __rcu *cfi_shadow __read_mostly = NULL;
62
ptr_to_shadow(const struct cfi_shadow * s,unsigned long ptr)63 static inline int ptr_to_shadow(const struct cfi_shadow *s, unsigned long ptr)
64 {
65 unsigned long index;
66 unsigned long page = ptr >> PAGE_SHIFT;
67
68 if (unlikely(page < s->r.min_page))
69 return -1; /* Outside of module area */
70
71 index = page - s->r.min_page;
72
73 if (index >= SHADOW_SIZE)
74 return -1; /* Cannot be addressed with shadow */
75
76 return (int)index;
77 }
78
shadow_to_ptr(const struct cfi_shadow * s,int index)79 static inline unsigned long shadow_to_ptr(const struct cfi_shadow *s,
80 int index)
81 {
82 BUG_ON(index < 0 || index >= SHADOW_SIZE);
83
84 if (unlikely(s->shadow[index] == SHADOW_INVALID))
85 return 0;
86
87 return (s->r.min_page + s->shadow[index]) << PAGE_SHIFT;
88 }
89
prepare_next_shadow(const struct cfi_shadow __rcu * prev,struct cfi_shadow * next)90 static void prepare_next_shadow(const struct cfi_shadow __rcu *prev,
91 struct cfi_shadow *next)
92 {
93 int i, index, check;
94
95 /* Mark everything invalid */
96 memset(next->shadow, 0xFF, sizeof(next->shadow));
97
98 if (!prev)
99 return; /* No previous shadow */
100
101 /* If the base address didn't change, update is not needed */
102 if (prev->r.min_page == next->r.min_page) {
103 memcpy(next->shadow, prev->shadow, sizeof(next->shadow));
104 return;
105 }
106
107 /* Convert the previous shadow to the new address range */
108 for (i = 0; i < SHADOW_SIZE; ++i) {
109 if (prev->shadow[i] == SHADOW_INVALID)
110 continue;
111
112 index = ptr_to_shadow(next, shadow_to_ptr(prev, i));
113 if (index < 0)
114 continue;
115
116 check = ptr_to_shadow(next,
117 shadow_to_ptr(prev, prev->shadow[i]));
118 if (check < 0)
119 continue;
120
121 next->shadow[index] = (u16)check;
122 }
123 }
124
add_module_to_shadow(struct cfi_shadow * s,struct module * mod)125 static void add_module_to_shadow(struct cfi_shadow *s, struct module *mod)
126 {
127 unsigned long ptr;
128 unsigned long min_page_addr;
129 unsigned long max_page_addr;
130 unsigned long check = (unsigned long)mod->cfi_check;
131 int check_index = ptr_to_shadow(s, check);
132
133 BUG_ON((check & PAGE_MASK) != check); /* Must be page aligned */
134
135 if (check_index < 0)
136 return; /* Module not addressable with shadow */
137
138 min_page_addr = (unsigned long)mod->core_layout.base & PAGE_MASK;
139 max_page_addr = (unsigned long)mod->core_layout.base +
140 mod->core_layout.text_size;
141 max_page_addr &= PAGE_MASK;
142
143 /* For each page, store the check function index in the shadow */
144 for (ptr = min_page_addr; ptr <= max_page_addr; ptr += PAGE_SIZE) {
145 int index = ptr_to_shadow(s, ptr);
146 if (index >= 0) {
147 /* Assume a page only contains code for one module */
148 BUG_ON(s->shadow[index] != SHADOW_INVALID);
149 s->shadow[index] = (u16)check_index;
150 }
151 }
152 }
153
remove_module_from_shadow(struct cfi_shadow * s,struct module * mod)154 static void remove_module_from_shadow(struct cfi_shadow *s, struct module *mod)
155 {
156 unsigned long ptr;
157 unsigned long min_page_addr;
158 unsigned long max_page_addr;
159
160 min_page_addr = (unsigned long)mod->core_layout.base & PAGE_MASK;
161 max_page_addr = (unsigned long)mod->core_layout.base +
162 mod->core_layout.text_size;
163 max_page_addr &= PAGE_MASK;
164
165 for (ptr = min_page_addr; ptr <= max_page_addr; ptr += PAGE_SIZE) {
166 int index = ptr_to_shadow(s, ptr);
167 if (index >= 0)
168 s->shadow[index] = SHADOW_INVALID;
169 }
170 }
171
172 typedef void (*update_shadow_fn)(struct cfi_shadow *, struct module *);
173
update_shadow(struct module * mod,unsigned long min_addr,unsigned long max_addr,update_shadow_fn fn)174 static void update_shadow(struct module *mod, unsigned long min_addr,
175 unsigned long max_addr, update_shadow_fn fn)
176 {
177 struct cfi_shadow *prev;
178 struct cfi_shadow *next = (struct cfi_shadow *)
179 __get_free_pages(GFP_KERNEL, SHADOW_ORDER);
180
181 BUG_ON(!next);
182
183 next->r.mod_min_addr = min_addr;
184 next->r.mod_max_addr = max_addr;
185 next->r.min_page = min_addr >> PAGE_SHIFT;
186 next->r.max_page = max_addr >> PAGE_SHIFT;
187
188 spin_lock(&shadow_update_lock);
189 prev = rcu_dereference_protected(cfi_shadow, 1);
190 prepare_next_shadow(prev, next);
191
192 fn(next, mod);
193 set_memory_ro((unsigned long)next, SHADOW_PAGES);
194 rcu_assign_pointer(cfi_shadow, next);
195
196 spin_unlock(&shadow_update_lock);
197 synchronize_rcu();
198
199 if (prev) {
200 set_memory_rw((unsigned long)prev, SHADOW_PAGES);
201 free_pages((unsigned long)prev, SHADOW_ORDER);
202 }
203 }
204
cfi_module_add(struct module * mod,unsigned long min_addr,unsigned long max_addr)205 void cfi_module_add(struct module *mod, unsigned long min_addr,
206 unsigned long max_addr)
207 {
208 update_shadow(mod, min_addr, max_addr, add_module_to_shadow);
209 }
210 EXPORT_SYMBOL(cfi_module_add);
211
cfi_module_remove(struct module * mod,unsigned long min_addr,unsigned long max_addr)212 void cfi_module_remove(struct module *mod, unsigned long min_addr,
213 unsigned long max_addr)
214 {
215 update_shadow(mod, min_addr, max_addr, remove_module_from_shadow);
216 }
217 EXPORT_SYMBOL(cfi_module_remove);
218
ptr_to_check_fn(const struct cfi_shadow __rcu * s,unsigned long ptr)219 static inline cfi_check_fn ptr_to_check_fn(const struct cfi_shadow __rcu *s,
220 unsigned long ptr)
221 {
222 int index;
223 unsigned long check;
224
225 if (unlikely(!s))
226 return NULL; /* No shadow available */
227
228 if (ptr < s->r.mod_min_addr || ptr > s->r.mod_max_addr)
229 return NULL; /* Not in a mapped module */
230
231 index = ptr_to_shadow(s, ptr);
232 if (index < 0)
233 return NULL; /* Cannot be addressed with shadow */
234
235 return (cfi_check_fn)shadow_to_ptr(s, index);
236 }
237 #endif /* CONFIG_CFI_CLANG_SHADOW */
238
find_module_cfi_check(void * ptr)239 static inline cfi_check_fn find_module_cfi_check(void *ptr)
240 {
241 struct module *mod;
242
243 preempt_disable();
244 mod = __module_address((unsigned long)ptr);
245 preempt_enable();
246
247 if (mod)
248 return mod->cfi_check;
249
250 return CFI_CHECK_FN;
251 }
252
find_cfi_check(void * ptr)253 static inline cfi_check_fn find_cfi_check(void *ptr)
254 {
255 #ifdef CONFIG_CFI_CLANG_SHADOW
256 cfi_check_fn f;
257
258 if (!rcu_access_pointer(cfi_shadow))
259 return CFI_CHECK_FN; /* No loaded modules */
260
261 /* Look up the __cfi_check function to use */
262 rcu_read_lock();
263 f = ptr_to_check_fn(rcu_dereference(cfi_shadow), (unsigned long)ptr);
264 rcu_read_unlock();
265
266 if (f)
267 return f;
268
269 /*
270 * Fall back to find_module_cfi_check, which works also for a larger
271 * module address space, but is slower.
272 */
273 #endif /* CONFIG_CFI_CLANG_SHADOW */
274
275 return find_module_cfi_check(ptr);
276 }
277
cfi_slowpath_handler(uint64_t id,void * ptr,void * diag)278 void cfi_slowpath_handler(uint64_t id, void *ptr, void *diag)
279 {
280 cfi_check_fn check = find_cfi_check(ptr);
281
282 if (likely(check))
283 check(id, ptr, diag);
284 else /* Don't allow unchecked modules */
285 handle_cfi_failure();
286 }
287 EXPORT_SYMBOL(cfi_slowpath_handler);
288 #endif /* CONFIG_MODULES */
289
cfi_failure_handler(void * data,void * value,void * vtable)290 void cfi_failure_handler(void *data, void *value, void *vtable)
291 {
292 handle_cfi_failure();
293 }
294 EXPORT_SYMBOL(cfi_failure_handler);
295
__cfi_check_fail(void * data,void * value)296 void __cfi_check_fail(void *data, void *value)
297 {
298 handle_cfi_failure();
299 }
300