• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1  /*
2   * Based on arch/arm/include/asm/uaccess.h
3   *
4   * Copyright (C) 2012 ARM Ltd.
5   *
6   * This program is free software; you can redistribute it and/or modify
7   * it under the terms of the GNU General Public License version 2 as
8   * published by the Free Software Foundation.
9   *
10   * This program is distributed in the hope that it will be useful,
11   * but WITHOUT ANY WARRANTY; without even the implied warranty of
12   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13   * GNU General Public License for more details.
14   *
15   * You should have received a copy of the GNU General Public License
16   * along with this program.  If not, see <http://www.gnu.org/licenses/>.
17   */
18  #ifndef __ASM_UACCESS_H
19  #define __ASM_UACCESS_H
20  
21  #include <asm/alternative.h>
22  #include <asm/kernel-pgtable.h>
23  #include <asm/mmu.h>
24  #include <asm/sysreg.h>
25  
26  #ifndef __ASSEMBLY__
27  
28  /*
29   * User space memory access functions
30   */
31  #include <linux/bitops.h>
32  #include <linux/string.h>
33  #include <linux/thread_info.h>
34  
35  #include <asm/cpufeature.h>
36  #include <asm/ptrace.h>
37  #include <asm/errno.h>
38  #include <asm/memory.h>
39  #include <asm/compiler.h>
40  
41  #define VERIFY_READ 0
42  #define VERIFY_WRITE 1
43  
44  /*
45   * The exception table consists of pairs of addresses: the first is the
46   * address of an instruction that is allowed to fault, and the second is
47   * the address at which the program should continue.  No registers are
48   * modified, so it is entirely up to the continuation code to figure out
49   * what to do.
50   *
51   * All the routines below use bits of fixup code that are out of line
52   * with the main instruction path.  This means when everything is well,
53   * we don't even have to jump over them.  Further, they do not intrude
54   * on our cache or tlb entries.
55   */
56  
57  struct exception_table_entry
58  {
59  	unsigned long insn, fixup;
60  };
61  
62  extern int fixup_exception(struct pt_regs *regs);
63  
64  #define KERNEL_DS	(-1UL)
65  #define get_ds()	(KERNEL_DS)
66  
67  #define USER_DS		TASK_SIZE_64
68  #define get_fs()	(current_thread_info()->addr_limit)
69  
set_fs(mm_segment_t fs)70  static inline void set_fs(mm_segment_t fs)
71  {
72  	current_thread_info()->addr_limit = fs;
73  
74  	/*
75  	 * Enable/disable UAO so that copy_to_user() etc can access
76  	 * kernel memory with the unprivileged instructions.
77  	 */
78  	if (IS_ENABLED(CONFIG_ARM64_UAO) && fs == KERNEL_DS)
79  		asm(ALTERNATIVE("nop", SET_PSTATE_UAO(1), ARM64_HAS_UAO));
80  	else
81  		asm(ALTERNATIVE("nop", SET_PSTATE_UAO(0), ARM64_HAS_UAO,
82  				CONFIG_ARM64_UAO));
83  }
84  
85  #define segment_eq(a,b)	((a) == (b))
86  
87  /*
88   * Return 1 if addr < current->addr_limit, 0 otherwise.
89   */
90  #define __addr_ok(addr)							\
91  ({									\
92  	unsigned long flag;						\
93  	asm("cmp %1, %0; cset %0, lo"					\
94  		: "=&r" (flag)						\
95  		: "r" (addr), "0" (current_thread_info()->addr_limit)	\
96  		: "cc");						\
97  	flag;								\
98  })
99  
100  /*
101   * Test whether a block of memory is a valid user space address.
102   * Returns 1 if the range is valid, 0 otherwise.
103   *
104   * This is equivalent to the following test:
105   * (u65)addr + (u65)size <= current->addr_limit
106   *
107   * This needs 65-bit arithmetic.
108   */
109  #define __range_ok(addr, size)						\
110  ({									\
111  	unsigned long __addr = (unsigned long __force)(addr);		\
112  	unsigned long flag, roksum;					\
113  	__chk_user_ptr(addr);						\
114  	asm("adds %1, %1, %3; ccmp %1, %4, #2, cc; cset %0, ls"		\
115  		: "=&r" (flag), "=&r" (roksum)				\
116  		: "1" (__addr), "Ir" (size),				\
117  		  "r" (current_thread_info()->addr_limit)		\
118  		: "cc");						\
119  	flag;								\
120  })
121  
122  /*
123   * When dealing with data aborts, watchpoints, or instruction traps we may end
124   * up with a tagged userland pointer. Clear the tag to get a sane pointer to
125   * pass on to access_ok(), for instance.
126   */
127  #define untagged_addr(addr)		sign_extend64(addr, 55)
128  
129  #define access_ok(type, addr, size)	__range_ok(addr, size)
130  #define user_addr_max			get_fs
131  
132  /*
133   * User access enabling/disabling.
134   */
135  #ifdef CONFIG_ARM64_SW_TTBR0_PAN
__uaccess_ttbr0_disable(void)136  static inline void __uaccess_ttbr0_disable(void)
137  {
138  	unsigned long flags, ttbr;
139  
140  	local_irq_save(flags);
141  	ttbr = read_sysreg(ttbr1_el1);
142  	ttbr &= ~TTBR_ASID_MASK;
143  	/* reserved_ttbr0 placed at the end of swapper_pg_dir */
144  	write_sysreg(ttbr + SWAPPER_DIR_SIZE, ttbr0_el1);
145  	isb();
146  	/* Set reserved ASID */
147  	write_sysreg(ttbr, ttbr1_el1);
148  	isb();
149  	local_irq_restore(flags);
150  }
151  
__uaccess_ttbr0_enable(void)152  static inline void __uaccess_ttbr0_enable(void)
153  {
154  	unsigned long flags, ttbr0, ttbr1;
155  
156  	/*
157  	 * Disable interrupts to avoid preemption between reading the 'ttbr0'
158  	 * variable and the MSR. A context switch could trigger an ASID
159  	 * roll-over and an update of 'ttbr0'.
160  	 */
161  	local_irq_save(flags);
162  	ttbr0 = READ_ONCE(current_thread_info()->ttbr0);
163  
164  	/* Restore active ASID */
165  	ttbr1 = read_sysreg(ttbr1_el1);
166  	ttbr1 &= ~TTBR_ASID_MASK;		/* safety measure */
167  	ttbr1 |= ttbr0 & TTBR_ASID_MASK;
168  	write_sysreg(ttbr1, ttbr1_el1);
169  	isb();
170  
171  	/* Restore user page table */
172  	write_sysreg(ttbr0, ttbr0_el1);
173  	isb();
174  	local_irq_restore(flags);
175  }
176  
uaccess_ttbr0_disable(void)177  static inline bool uaccess_ttbr0_disable(void)
178  {
179  	if (!system_uses_ttbr0_pan())
180  		return false;
181  	__uaccess_ttbr0_disable();
182  	return true;
183  }
184  
uaccess_ttbr0_enable(void)185  static inline bool uaccess_ttbr0_enable(void)
186  {
187  	if (!system_uses_ttbr0_pan())
188  		return false;
189  	__uaccess_ttbr0_enable();
190  	return true;
191  }
192  #else
uaccess_ttbr0_disable(void)193  static inline bool uaccess_ttbr0_disable(void)
194  {
195  	return false;
196  }
197  
uaccess_ttbr0_enable(void)198  static inline bool uaccess_ttbr0_enable(void)
199  {
200  	return false;
201  }
202  #endif
203  
204  #define __uaccess_disable(alt)						\
205  do {									\
206  	if (!uaccess_ttbr0_disable())					\
207  		asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), alt,		\
208  				CONFIG_ARM64_PAN));			\
209  } while (0)
210  
211  #define __uaccess_enable(alt)						\
212  do {									\
213  	if (!uaccess_ttbr0_enable())					\
214  		asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), alt,		\
215  				CONFIG_ARM64_PAN));			\
216  } while (0)
217  
uaccess_disable(void)218  static inline void uaccess_disable(void)
219  {
220  	__uaccess_disable(ARM64_HAS_PAN);
221  }
222  
uaccess_enable(void)223  static inline void uaccess_enable(void)
224  {
225  	__uaccess_enable(ARM64_HAS_PAN);
226  }
227  
228  /*
229   * These functions are no-ops when UAO is present.
230   */
uaccess_disable_not_uao(void)231  static inline void uaccess_disable_not_uao(void)
232  {
233  	__uaccess_disable(ARM64_ALT_PAN_NOT_UAO);
234  }
235  
uaccess_enable_not_uao(void)236  static inline void uaccess_enable_not_uao(void)
237  {
238  	__uaccess_enable(ARM64_ALT_PAN_NOT_UAO);
239  }
240  
241  /*
242   * The "__xxx" versions of the user access functions do not verify the address
243   * space - it must have been done previously with a separate "access_ok()"
244   * call.
245   *
246   * The "__xxx_error" versions set the third argument to -EFAULT if an error
247   * occurs, and leave it unchanged on success.
248   */
249  #define __get_user_asm(instr, alt_instr, reg, x, addr, err, feature)	\
250  	asm volatile(							\
251  	"1:"ALTERNATIVE(instr "     " reg "1, [%2]\n",			\
252  			alt_instr " " reg "1, [%2]\n", feature)		\
253  	"2:\n"								\
254  	"	.section .fixup, \"ax\"\n"				\
255  	"	.align	2\n"						\
256  	"3:	mov	%w0, %3\n"					\
257  	"	mov	%1, #0\n"					\
258  	"	b	2b\n"						\
259  	"	.previous\n"						\
260  	"	.section __ex_table,\"a\"\n"				\
261  	"	.align	3\n"						\
262  	"	.quad	1b, 3b\n"					\
263  	"	.previous"						\
264  	: "+r" (err), "=&r" (x)						\
265  	: "r" (addr), "i" (-EFAULT))
266  
267  #define __get_user_err(x, ptr, err)					\
268  do {									\
269  	unsigned long __gu_val;						\
270  	__chk_user_ptr(ptr);						\
271  	uaccess_enable_not_uao();					\
272  	switch (sizeof(*(ptr))) {					\
273  	case 1:								\
274  		__get_user_asm("ldrb", "ldtrb", "%w", __gu_val, (ptr),  \
275  			       (err), ARM64_HAS_UAO);			\
276  		break;							\
277  	case 2:								\
278  		__get_user_asm("ldrh", "ldtrh", "%w", __gu_val, (ptr),  \
279  			       (err), ARM64_HAS_UAO);			\
280  		break;							\
281  	case 4:								\
282  		__get_user_asm("ldr", "ldtr", "%w", __gu_val, (ptr),	\
283  			       (err), ARM64_HAS_UAO);			\
284  		break;							\
285  	case 8:								\
286  		__get_user_asm("ldr", "ldtr", "%",  __gu_val, (ptr),	\
287  			       (err), ARM64_HAS_UAO);			\
288  		break;							\
289  	default:							\
290  		BUILD_BUG();						\
291  	}								\
292  	uaccess_disable_not_uao();					\
293  	(x) = (__force __typeof__(*(ptr)))__gu_val;			\
294  } while (0)
295  
296  #define __get_user(x, ptr)						\
297  ({									\
298  	int __gu_err = 0;						\
299  	__get_user_err((x), (ptr), __gu_err);				\
300  	__gu_err;							\
301  })
302  
303  #define __get_user_error(x, ptr, err)					\
304  ({									\
305  	__get_user_err((x), (ptr), (err));				\
306  	(void)0;							\
307  })
308  
309  #define __get_user_unaligned __get_user
310  
311  #define get_user(x, ptr)						\
312  ({									\
313  	__typeof__(*(ptr)) __user *__p = (ptr);				\
314  	might_fault();							\
315  	access_ok(VERIFY_READ, __p, sizeof(*__p)) ?			\
316  		__get_user((x), __p) :					\
317  		((x) = 0, -EFAULT);					\
318  })
319  
320  #define __put_user_asm(instr, alt_instr, reg, x, addr, err, feature)	\
321  	asm volatile(							\
322  	"1:"ALTERNATIVE(instr "     " reg "1, [%2]\n",			\
323  			alt_instr " " reg "1, [%2]\n", feature)		\
324  	"2:\n"								\
325  	"	.section .fixup,\"ax\"\n"				\
326  	"	.align	2\n"						\
327  	"3:	mov	%w0, %3\n"					\
328  	"	b	2b\n"						\
329  	"	.previous\n"						\
330  	"	.section __ex_table,\"a\"\n"				\
331  	"	.align	3\n"						\
332  	"	.quad	1b, 3b\n"					\
333  	"	.previous"						\
334  	: "+r" (err)							\
335  	: "r" (x), "r" (addr), "i" (-EFAULT))
336  
337  #define __put_user_err(x, ptr, err)					\
338  do {									\
339  	__typeof__(*(ptr)) __pu_val = (x);				\
340  	__chk_user_ptr(ptr);						\
341  	uaccess_enable_not_uao();					\
342  	switch (sizeof(*(ptr))) {					\
343  	case 1:								\
344  		__put_user_asm("strb", "sttrb", "%w", __pu_val, (ptr),	\
345  			       (err), ARM64_HAS_UAO);			\
346  		break;							\
347  	case 2:								\
348  		__put_user_asm("strh", "sttrh", "%w", __pu_val, (ptr),	\
349  			       (err), ARM64_HAS_UAO);			\
350  		break;							\
351  	case 4:								\
352  		__put_user_asm("str", "sttr", "%w", __pu_val, (ptr),	\
353  			       (err), ARM64_HAS_UAO);			\
354  		break;							\
355  	case 8:								\
356  		__put_user_asm("str", "sttr", "%", __pu_val, (ptr),	\
357  			       (err), ARM64_HAS_UAO);			\
358  		break;							\
359  	default:							\
360  		BUILD_BUG();						\
361  	}								\
362  	uaccess_disable_not_uao();					\
363  } while (0)
364  
365  #define __put_user(x, ptr)						\
366  ({									\
367  	int __pu_err = 0;						\
368  	__put_user_err((x), (ptr), __pu_err);				\
369  	__pu_err;							\
370  })
371  
372  #define __put_user_error(x, ptr, err)					\
373  ({									\
374  	__put_user_err((x), (ptr), (err));				\
375  	(void)0;							\
376  })
377  
378  #define __put_user_unaligned __put_user
379  
380  #define put_user(x, ptr)						\
381  ({									\
382  	__typeof__(*(ptr)) __user *__p = (ptr);				\
383  	might_fault();							\
384  	access_ok(VERIFY_WRITE, __p, sizeof(*__p)) ?			\
385  		__put_user((x), __p) :					\
386  		-EFAULT;						\
387  })
388  
389  extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n);
390  extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n);
391  extern unsigned long __must_check __copy_in_user(void __user *to, const void __user *from, unsigned long n);
392  extern unsigned long __must_check __clear_user(void __user *addr, unsigned long n);
393  
__copy_from_user(void * to,const void __user * from,unsigned long n)394  static inline unsigned long __must_check __copy_from_user(void *to, const void __user *from, unsigned long n)
395  {
396  	check_object_size(to, n, false);
397  	return __arch_copy_from_user(to, from, n);
398  }
399  
__copy_to_user(void __user * to,const void * from,unsigned long n)400  static inline unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n)
401  {
402  	check_object_size(from, n, true);
403  	return __arch_copy_to_user(to, from, n);
404  }
405  
copy_from_user(void * to,const void __user * from,unsigned long n)406  static inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long n)
407  {
408  	if (access_ok(VERIFY_READ, from, n)) {
409  		check_object_size(to, n, false);
410  		n = __arch_copy_from_user(to, from, n);
411  	} else /* security hole - plug it */
412  		memset(to, 0, n);
413  	return n;
414  }
415  
copy_to_user(void __user * to,const void * from,unsigned long n)416  static inline unsigned long __must_check copy_to_user(void __user *to, const void *from, unsigned long n)
417  {
418  	if (access_ok(VERIFY_WRITE, to, n)) {
419  		check_object_size(from, n, true);
420  		n = __arch_copy_to_user(to, from, n);
421  	}
422  	return n;
423  }
424  
copy_in_user(void __user * to,const void __user * from,unsigned long n)425  static inline unsigned long __must_check copy_in_user(void __user *to, const void __user *from, unsigned long n)
426  {
427  	if (access_ok(VERIFY_READ, from, n) && access_ok(VERIFY_WRITE, to, n))
428  		n = __copy_in_user(to, from, n);
429  	return n;
430  }
431  
432  #define __copy_to_user_inatomic __copy_to_user
433  #define __copy_from_user_inatomic __copy_from_user
434  
clear_user(void __user * to,unsigned long n)435  static inline unsigned long __must_check clear_user(void __user *to, unsigned long n)
436  {
437  	if (access_ok(VERIFY_WRITE, to, n))
438  		n = __clear_user(to, n);
439  	return n;
440  }
441  
442  extern long strncpy_from_user(char *dest, const char __user *src, long count);
443  
444  extern __must_check long strlen_user(const char __user *str);
445  extern __must_check long strnlen_user(const char __user *str, long n);
446  
447  #else	/* __ASSEMBLY__ */
448  
449  #include <asm/assembler.h>
450  
451  /*
452   * User access enabling/disabling macros.
453   */
454  #ifdef CONFIG_ARM64_SW_TTBR0_PAN
455  	.macro	__uaccess_ttbr0_disable, tmp1
456  	mrs	\tmp1, ttbr1_el1		// swapper_pg_dir
457  	bic	\tmp1, \tmp1, #TTBR_ASID_MASK
458  	add	\tmp1, \tmp1, #SWAPPER_DIR_SIZE	// reserved_ttbr0 at the end of swapper_pg_dir
459  	msr	ttbr0_el1, \tmp1		// set reserved TTBR0_EL1
460  	isb
461  	sub	\tmp1, \tmp1, #SWAPPER_DIR_SIZE
462  	msr	ttbr1_el1, \tmp1		// set reserved ASID
463  	isb
464  	.endm
465  
466  	.macro	__uaccess_ttbr0_enable, tmp1, tmp2
467  	get_thread_info \tmp1
468  	ldr	\tmp1, [\tmp1, #TSK_TI_TTBR0]	// load saved TTBR0_EL1
469  	mrs	\tmp2, ttbr1_el1
470  	extr    \tmp2, \tmp2, \tmp1, #48
471  	ror     \tmp2, \tmp2, #16
472  	msr	ttbr1_el1, \tmp2		// set the active ASID
473  	isb
474  	msr	ttbr0_el1, \tmp1		// set the non-PAN TTBR0_EL1
475  	isb
476  	.endm
477  
478  	.macro	uaccess_ttbr0_disable, tmp1, tmp2
479  alternative_if_not ARM64_HAS_PAN
480  	save_and_disable_irq \tmp2		// avoid preemption
481  	__uaccess_ttbr0_disable \tmp1
482  	restore_irq \tmp2
483  alternative_else_nop_endif
484  	.endm
485  
486  	.macro	uaccess_ttbr0_enable, tmp1, tmp2, tmp3
487  alternative_if_not ARM64_HAS_PAN
488  	save_and_disable_irq \tmp3		// avoid preemption
489  	__uaccess_ttbr0_enable \tmp1, \tmp2
490  	restore_irq \tmp3
491  alternative_else_nop_endif
492  	.endm
493  #else
494  	.macro	uaccess_ttbr0_disable, tmp1, tmp2
495  	.endm
496  
497  	.macro	uaccess_ttbr0_enable, tmp1, tmp2, tmp3
498  	.endm
499  #endif
500  
501  /*
502   * These macros are no-ops when UAO is present.
503   */
504  	.macro	uaccess_disable_not_uao, tmp1, tmp2
505  	uaccess_ttbr0_disable \tmp1, \tmp2
506  alternative_if ARM64_ALT_PAN_NOT_UAO
507  	SET_PSTATE_PAN(1)
508  alternative_else_nop_endif
509  	.endm
510  
511  	.macro	uaccess_enable_not_uao, tmp1, tmp2, tmp3
512  	uaccess_ttbr0_enable \tmp1, \tmp2, \tmp3
513  alternative_if ARM64_ALT_PAN_NOT_UAO
514  	SET_PSTATE_PAN(0)
515  alternative_else_nop_endif
516  	.endm
517  
518  #endif	/* __ASSEMBLY__ */
519  
520  #endif /* __ASM_UACCESS_H */
521