1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Based on arch/arm/include/asm/uaccess.h
4 *
5 * Copyright (C) 2012 ARM Ltd.
6 */
7 #ifndef __ASM_UACCESS_H
8 #define __ASM_UACCESS_H
9
10 #include <asm/alternative.h>
11 #include <asm/kernel-pgtable.h>
12 #include <asm/sysreg.h>
13
14 /*
15 * User space memory access functions
16 */
17 #include <linux/bitops.h>
18 #include <linux/kasan-checks.h>
19 #include <linux/string.h>
20
21 #include <asm/cpufeature.h>
22 #include <asm/mmu.h>
23 #include <asm/mte.h>
24 #include <asm/ptrace.h>
25 #include <asm/memory.h>
26 #include <asm/extable.h>
27
28 #define HAVE_GET_KERNEL_NOFAULT
29
30 #define get_fs() (current_thread_info()->addr_limit)
31
set_fs(mm_segment_t fs)32 static inline void set_fs(mm_segment_t fs)
33 {
34 current_thread_info()->addr_limit = fs;
35
36 /*
37 * Prevent a mispredicted conditional call to set_fs from forwarding
38 * the wrong address limit to access_ok under speculation.
39 */
40 spec_bar();
41
42 /* On user-mode return, check fs is correct */
43 set_thread_flag(TIF_FSCHECK);
44
45 /*
46 * Enable/disable UAO so that copy_to_user() etc can access
47 * kernel memory with the unprivileged instructions.
48 */
49 if (IS_ENABLED(CONFIG_ARM64_UAO) && fs == KERNEL_DS)
50 asm(ALTERNATIVE("nop", SET_PSTATE_UAO(1), ARM64_HAS_UAO));
51 else
52 asm(ALTERNATIVE("nop", SET_PSTATE_UAO(0), ARM64_HAS_UAO,
53 CONFIG_ARM64_UAO));
54 }
55
56 #define uaccess_kernel() (get_fs() == KERNEL_DS)
57
58 /*
59 * Test whether a block of memory is a valid user space address.
60 * Returns 1 if the range is valid, 0 otherwise.
61 *
62 * This is equivalent to the following test:
63 * (u65)addr + (u65)size <= (u65)current->addr_limit + 1
64 */
__range_ok(const void __user * addr,unsigned long size)65 static inline unsigned long __range_ok(const void __user *addr, unsigned long size)
66 {
67 unsigned long ret, limit = current_thread_info()->addr_limit;
68
69 /*
70 * Asynchronous I/O running in a kernel thread does not have the
71 * TIF_TAGGED_ADDR flag of the process owning the mm, so always untag
72 * the user address before checking.
73 */
74 if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) &&
75 (current->flags & PF_KTHREAD || test_thread_flag(TIF_TAGGED_ADDR)))
76 addr = untagged_addr(addr);
77
78 __chk_user_ptr(addr);
79 asm volatile(
80 // A + B <= C + 1 for all A,B,C, in four easy steps:
81 // 1: X = A + B; X' = X % 2^64
82 " adds %0, %3, %2\n"
83 // 2: Set C = 0 if X > 2^64, to guarantee X' > C in step 4
84 " csel %1, xzr, %1, hi\n"
85 // 3: Set X' = ~0 if X >= 2^64. For X == 2^64, this decrements X'
86 // to compensate for the carry flag being set in step 4. For
87 // X > 2^64, X' merely has to remain nonzero, which it does.
88 " csinv %0, %0, xzr, cc\n"
89 // 4: For X < 2^64, this gives us X' - C - 1 <= 0, where the -1
90 // comes from the carry in being clear. Otherwise, we are
91 // testing X' - C == 0, subject to the previous adjustments.
92 " sbcs xzr, %0, %1\n"
93 " cset %0, ls\n"
94 : "=&r" (ret), "+r" (limit) : "Ir" (size), "0" (addr) : "cc");
95
96 return ret;
97 }
98
99 #define access_ok(addr, size) __range_ok(addr, size)
100 #define user_addr_max get_fs
101
102 #define _ASM_EXTABLE(from, to) \
103 " .pushsection __ex_table, \"a\"\n" \
104 " .align 3\n" \
105 " .long (" #from " - .), (" #to " - .)\n" \
106 " .popsection\n"
107
108 /*
109 * User access enabling/disabling.
110 */
111 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
__uaccess_ttbr0_disable(void)112 static inline void __uaccess_ttbr0_disable(void)
113 {
114 unsigned long flags, ttbr;
115
116 local_irq_save(flags);
117 ttbr = read_sysreg(ttbr1_el1);
118 ttbr &= ~TTBR_ASID_MASK;
119 /* reserved_pg_dir placed before swapper_pg_dir */
120 write_sysreg(ttbr - PAGE_SIZE, ttbr0_el1);
121 isb();
122 /* Set reserved ASID */
123 write_sysreg(ttbr, ttbr1_el1);
124 isb();
125 local_irq_restore(flags);
126 }
127
__uaccess_ttbr0_enable(void)128 static inline void __uaccess_ttbr0_enable(void)
129 {
130 unsigned long flags, ttbr0, ttbr1;
131
132 /*
133 * Disable interrupts to avoid preemption between reading the 'ttbr0'
134 * variable and the MSR. A context switch could trigger an ASID
135 * roll-over and an update of 'ttbr0'.
136 */
137 local_irq_save(flags);
138 ttbr0 = READ_ONCE(current_thread_info()->ttbr0);
139
140 /* Restore active ASID */
141 ttbr1 = read_sysreg(ttbr1_el1);
142 ttbr1 &= ~TTBR_ASID_MASK; /* safety measure */
143 ttbr1 |= ttbr0 & TTBR_ASID_MASK;
144 write_sysreg(ttbr1, ttbr1_el1);
145 isb();
146
147 /* Restore user page table */
148 write_sysreg(ttbr0, ttbr0_el1);
149 isb();
150 local_irq_restore(flags);
151 }
152
uaccess_ttbr0_disable(void)153 static inline bool uaccess_ttbr0_disable(void)
154 {
155 if (!system_uses_ttbr0_pan())
156 return false;
157 __uaccess_ttbr0_disable();
158 return true;
159 }
160
uaccess_ttbr0_enable(void)161 static inline bool uaccess_ttbr0_enable(void)
162 {
163 if (!system_uses_ttbr0_pan())
164 return false;
165 __uaccess_ttbr0_enable();
166 return true;
167 }
168 #else
uaccess_ttbr0_disable(void)169 static inline bool uaccess_ttbr0_disable(void)
170 {
171 return false;
172 }
173
uaccess_ttbr0_enable(void)174 static inline bool uaccess_ttbr0_enable(void)
175 {
176 return false;
177 }
178 #endif
179
__uaccess_disable_hw_pan(void)180 static inline void __uaccess_disable_hw_pan(void)
181 {
182 asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN,
183 CONFIG_ARM64_PAN));
184 }
185
__uaccess_enable_hw_pan(void)186 static inline void __uaccess_enable_hw_pan(void)
187 {
188 asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN,
189 CONFIG_ARM64_PAN));
190 }
191
192 #define __uaccess_disable(alt) \
193 do { \
194 if (!uaccess_ttbr0_disable()) \
195 asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), alt, \
196 CONFIG_ARM64_PAN)); \
197 } while (0)
198
199 #define __uaccess_enable(alt) \
200 do { \
201 if (!uaccess_ttbr0_enable()) \
202 asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), alt, \
203 CONFIG_ARM64_PAN)); \
204 } while (0)
205
206 /*
207 * The Tag Check Flag (TCF) mode for MTE is per EL, hence TCF0
208 * affects EL0 and TCF affects EL1 irrespective of which TTBR is
209 * used.
210 * The kernel accesses TTBR0 usually with LDTR/STTR instructions
211 * when UAO is available, so these would act as EL0 accesses using
212 * TCF0.
213 * However futex.h code uses exclusives which would be executed as
214 * EL1, this can potentially cause a tag check fault even if the
215 * user disables TCF0.
216 *
217 * To address the problem we set the PSTATE.TCO bit in uaccess_enable()
218 * and reset it in uaccess_disable().
219 *
220 * The Tag check override (TCO) bit disables temporarily the tag checking
221 * preventing the issue.
222 */
__uaccess_disable_tco(void)223 static inline void __uaccess_disable_tco(void)
224 {
225 asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(0),
226 ARM64_MTE, CONFIG_KASAN_HW_TAGS));
227 }
228
__uaccess_enable_tco(void)229 static inline void __uaccess_enable_tco(void)
230 {
231 asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(1),
232 ARM64_MTE, CONFIG_KASAN_HW_TAGS));
233 }
234
235 /*
236 * These functions disable tag checking only if in MTE async mode
237 * since the sync mode generates exceptions synchronously and the
238 * nofault or load_unaligned_zeropad can handle them.
239 */
__uaccess_disable_tco_async(void)240 static inline void __uaccess_disable_tco_async(void)
241 {
242 if (system_uses_mte_async_or_asymm_mode())
243 __uaccess_disable_tco();
244 }
245
__uaccess_enable_tco_async(void)246 static inline void __uaccess_enable_tco_async(void)
247 {
248 if (system_uses_mte_async_or_asymm_mode())
249 __uaccess_enable_tco();
250 }
251
uaccess_disable_privileged(void)252 static inline void uaccess_disable_privileged(void)
253 {
254 __uaccess_disable_tco();
255
256 __uaccess_disable(ARM64_HAS_PAN);
257 }
258
uaccess_enable_privileged(void)259 static inline void uaccess_enable_privileged(void)
260 {
261 __uaccess_enable_tco();
262
263 __uaccess_enable(ARM64_HAS_PAN);
264 }
265
266 /*
267 * These functions are no-ops when UAO is present.
268 */
uaccess_disable_not_uao(void)269 static inline void uaccess_disable_not_uao(void)
270 {
271 __uaccess_disable(ARM64_ALT_PAN_NOT_UAO);
272 }
273
uaccess_enable_not_uao(void)274 static inline void uaccess_enable_not_uao(void)
275 {
276 __uaccess_enable(ARM64_ALT_PAN_NOT_UAO);
277 }
278
279 /*
280 * Sanitise a uaccess pointer such that it becomes NULL if above the
281 * current addr_limit. In case the pointer is tagged (has the top byte set),
282 * untag the pointer before checking.
283 */
284 #define uaccess_mask_ptr(ptr) (__typeof__(ptr))__uaccess_mask_ptr(ptr)
__uaccess_mask_ptr(const void __user * ptr)285 static inline void __user *__uaccess_mask_ptr(const void __user *ptr)
286 {
287 void __user *safe_ptr;
288
289 asm volatile(
290 " bics xzr, %3, %2\n"
291 " csel %0, %1, xzr, eq\n"
292 : "=&r" (safe_ptr)
293 : "r" (ptr), "r" (current_thread_info()->addr_limit),
294 "r" (untagged_addr(ptr))
295 : "cc");
296
297 csdb();
298 return safe_ptr;
299 }
300
301 /*
302 * The "__xxx" versions of the user access functions do not verify the address
303 * space - it must have been done previously with a separate "access_ok()"
304 * call.
305 *
306 * The "__xxx_error" versions set the third argument to -EFAULT if an error
307 * occurs, and leave it unchanged on success.
308 */
309 #define __get_mem_asm(load, reg, x, addr, err) \
310 asm volatile( \
311 "1: " load " " reg "1, [%2]\n" \
312 "2:\n" \
313 " .section .fixup, \"ax\"\n" \
314 " .align 2\n" \
315 "3: mov %w0, %3\n" \
316 " mov %1, #0\n" \
317 " b 2b\n" \
318 " .previous\n" \
319 _ASM_EXTABLE(1b, 3b) \
320 : "+r" (err), "=&r" (x) \
321 : "r" (addr), "i" (-EFAULT))
322
323 #define __raw_get_mem(ldr, x, ptr, err) \
324 do { \
325 unsigned long __gu_val; \
326 switch (sizeof(*(ptr))) { \
327 case 1: \
328 __get_mem_asm(ldr "b", "%w", __gu_val, (ptr), (err)); \
329 break; \
330 case 2: \
331 __get_mem_asm(ldr "h", "%w", __gu_val, (ptr), (err)); \
332 break; \
333 case 4: \
334 __get_mem_asm(ldr, "%w", __gu_val, (ptr), (err)); \
335 break; \
336 case 8: \
337 __get_mem_asm(ldr, "%x", __gu_val, (ptr), (err)); \
338 break; \
339 default: \
340 BUILD_BUG(); \
341 } \
342 (x) = (__force __typeof__(*(ptr)))__gu_val; \
343 } while (0)
344
345 /*
346 * We must not call into the scheduler between uaccess_enable_not_uao() and
347 * uaccess_disable_not_uao(). As `x` and `ptr` could contain blocking functions,
348 * we must evaluate these outside of the critical section.
349 */
350 #define __raw_get_user(x, ptr, err) \
351 do { \
352 __typeof__(*(ptr)) __user *__rgu_ptr = (ptr); \
353 __typeof__(x) __rgu_val; \
354 __chk_user_ptr(ptr); \
355 \
356 uaccess_enable_not_uao(); \
357 __raw_get_mem("ldtr", __rgu_val, __rgu_ptr, err); \
358 uaccess_disable_not_uao(); \
359 \
360 (x) = __rgu_val; \
361 } while (0)
362
363 #define __get_user_error(x, ptr, err) \
364 do { \
365 __typeof__(*(ptr)) __user *__p = (ptr); \
366 might_fault(); \
367 if (access_ok(__p, sizeof(*__p))) { \
368 __p = uaccess_mask_ptr(__p); \
369 __raw_get_user((x), __p, (err)); \
370 } else { \
371 (x) = (__force __typeof__(x))0; (err) = -EFAULT; \
372 } \
373 } while (0)
374
375 #define __get_user(x, ptr) \
376 ({ \
377 int __gu_err = 0; \
378 __get_user_error((x), (ptr), __gu_err); \
379 __gu_err; \
380 })
381
382 #define get_user __get_user
383
384 /*
385 * We must not call into the scheduler between __uaccess_enable_tco_async() and
386 * __uaccess_disable_tco_async(). As `dst` and `src` may contain blocking
387 * functions, we must evaluate these outside of the critical section.
388 */
389 #define __get_kernel_nofault(dst, src, type, err_label) \
390 do { \
391 __typeof__(dst) __gkn_dst = (dst); \
392 __typeof__(src) __gkn_src = (src); \
393 int __gkn_err = 0; \
394 \
395 __uaccess_enable_tco_async(); \
396 __raw_get_mem("ldr", *((type *)(__gkn_dst)), \
397 (__force type *)(__gkn_src), __gkn_err); \
398 __uaccess_disable_tco_async(); \
399 \
400 if (unlikely(__gkn_err)) \
401 goto err_label; \
402 } while (0)
403
404 #define __put_mem_asm(store, reg, x, addr, err) \
405 asm volatile( \
406 "1: " store " " reg "1, [%2]\n" \
407 "2:\n" \
408 " .section .fixup,\"ax\"\n" \
409 " .align 2\n" \
410 "3: mov %w0, %3\n" \
411 " b 2b\n" \
412 " .previous\n" \
413 _ASM_EXTABLE(1b, 3b) \
414 : "+r" (err) \
415 : "r" (x), "r" (addr), "i" (-EFAULT))
416
417 #define __raw_put_mem(str, x, ptr, err) \
418 do { \
419 __typeof__(*(ptr)) __pu_val = (x); \
420 switch (sizeof(*(ptr))) { \
421 case 1: \
422 __put_mem_asm(str "b", "%w", __pu_val, (ptr), (err)); \
423 break; \
424 case 2: \
425 __put_mem_asm(str "h", "%w", __pu_val, (ptr), (err)); \
426 break; \
427 case 4: \
428 __put_mem_asm(str, "%w", __pu_val, (ptr), (err)); \
429 break; \
430 case 8: \
431 __put_mem_asm(str, "%x", __pu_val, (ptr), (err)); \
432 break; \
433 default: \
434 BUILD_BUG(); \
435 } \
436 } while (0)
437
438 /*
439 * We must not call into the scheduler between uaccess_enable_not_uao() and
440 * uaccess_disable_not_uao(). As `x` and `ptr` could contain blocking functions,
441 * we must evaluate these outside of the critical section.
442 */
443 #define __raw_put_user(x, ptr, err) \
444 do { \
445 __typeof__(*(ptr)) __user *__rpu_ptr = (ptr); \
446 __typeof__(*(ptr)) __rpu_val = (x); \
447 __chk_user_ptr(__rpu_ptr); \
448 \
449 uaccess_enable_not_uao(); \
450 __raw_put_mem("sttr", __rpu_val, __rpu_ptr, err); \
451 uaccess_disable_not_uao(); \
452 } while (0)
453
454 #define __put_user_error(x, ptr, err) \
455 do { \
456 __typeof__(*(ptr)) __user *__p = (ptr); \
457 might_fault(); \
458 if (access_ok(__p, sizeof(*__p))) { \
459 __p = uaccess_mask_ptr(__p); \
460 __raw_put_user((x), __p, (err)); \
461 } else { \
462 (err) = -EFAULT; \
463 } \
464 } while (0)
465
466 #define __put_user(x, ptr) \
467 ({ \
468 int __pu_err = 0; \
469 __put_user_error((x), (ptr), __pu_err); \
470 __pu_err; \
471 })
472
473 #define put_user __put_user
474
475 /*
476 * We must not call into the scheduler between __uaccess_enable_tco_async() and
477 * __uaccess_disable_tco_async(). As `dst` and `src` may contain blocking
478 * functions, we must evaluate these outside of the critical section.
479 */
480 #define __put_kernel_nofault(dst, src, type, err_label) \
481 do { \
482 __typeof__(dst) __pkn_dst = (dst); \
483 __typeof__(src) __pkn_src = (src); \
484 int __pkn_err = 0; \
485 \
486 __uaccess_enable_tco_async(); \
487 __raw_put_mem("str", *((type *)(__pkn_src)), \
488 (__force type *)(__pkn_dst), __pkn_err); \
489 __uaccess_disable_tco_async(); \
490 \
491 if (unlikely(__pkn_err)) \
492 goto err_label; \
493 } while(0)
494
495 extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n);
496 #define raw_copy_from_user(to, from, n) \
497 ({ \
498 unsigned long __acfu_ret; \
499 uaccess_enable_not_uao(); \
500 __acfu_ret = __arch_copy_from_user((to), \
501 __uaccess_mask_ptr(from), (n)); \
502 uaccess_disable_not_uao(); \
503 __acfu_ret; \
504 })
505
506 extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n);
507 #define raw_copy_to_user(to, from, n) \
508 ({ \
509 unsigned long __actu_ret; \
510 uaccess_enable_not_uao(); \
511 __actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to), \
512 (from), (n)); \
513 uaccess_disable_not_uao(); \
514 __actu_ret; \
515 })
516
517 extern unsigned long __must_check __arch_copy_in_user(void __user *to, const void __user *from, unsigned long n);
518 #define raw_copy_in_user(to, from, n) \
519 ({ \
520 unsigned long __aciu_ret; \
521 uaccess_enable_not_uao(); \
522 __aciu_ret = __arch_copy_in_user(__uaccess_mask_ptr(to), \
523 __uaccess_mask_ptr(from), (n)); \
524 uaccess_disable_not_uao(); \
525 __aciu_ret; \
526 })
527
528 #define INLINE_COPY_TO_USER
529 #define INLINE_COPY_FROM_USER
530
531 extern unsigned long __must_check __arch_clear_user(void __user *to, unsigned long n);
__clear_user(void __user * to,unsigned long n)532 static inline unsigned long __must_check __clear_user(void __user *to, unsigned long n)
533 {
534 if (access_ok(to, n)) {
535 uaccess_enable_not_uao();
536 n = __arch_clear_user(__uaccess_mask_ptr(to), n);
537 uaccess_disable_not_uao();
538 }
539 return n;
540 }
541 #define clear_user __clear_user
542
543 extern long strncpy_from_user(char *dest, const char __user *src, long count);
544
545 extern __must_check long strnlen_user(const char __user *str, long n);
546
547 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
548 struct page;
549 void memcpy_page_flushcache(char *to, struct page *page, size_t offset, size_t len);
550 extern unsigned long __must_check __copy_user_flushcache(void *to, const void __user *from, unsigned long n);
551
__copy_from_user_flushcache(void * dst,const void __user * src,unsigned size)552 static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)
553 {
554 kasan_check_write(dst, size);
555 return __copy_user_flushcache(dst, __uaccess_mask_ptr(src), size);
556 }
557 #endif
558
559 #endif /* __ASM_UACCESS_H */
560