1 #ifndef _ASM_X86_UACCESS_H
2 #define _ASM_X86_UACCESS_H
3 /*
4 * User space memory access functions
5 */
6 #include <linux/errno.h>
7 #include <linux/compiler.h>
8 #include <linux/kasan-checks.h>
9 #include <linux/thread_info.h>
10 #include <linux/string.h>
11 #include <asm/asm.h>
12 #include <asm/page.h>
13 #include <asm/smap.h>
14 #include <asm/extable.h>
15
16 #define VERIFY_READ 0
17 #define VERIFY_WRITE 1
18
19 /*
20 * The fs value determines whether argument validity checking should be
21 * performed or not. If get_fs() == USER_DS, checking is performed, with
22 * get_fs() == KERNEL_DS, checking is bypassed.
23 *
24 * For historical reasons, these macros are grossly misnamed.
25 */
26
27 #define MAKE_MM_SEG(s) ((mm_segment_t) { (s) })
28
29 #define KERNEL_DS MAKE_MM_SEG(-1UL)
30 #define USER_DS MAKE_MM_SEG(TASK_SIZE_MAX)
31
32 #define get_ds() (KERNEL_DS)
33 #define get_fs() (current->thread.addr_limit)
34 #define set_fs(x) (current->thread.addr_limit = (x))
35
36 #define segment_eq(a, b) ((a).seg == (b).seg)
37
38 #define user_addr_max() (current->thread.addr_limit.seg)
39 #define __addr_ok(addr) \
40 ((unsigned long __force)(addr) < user_addr_max())
41
42 /*
43 * Test whether a block of memory is a valid user space address.
44 * Returns 0 if the range is valid, nonzero otherwise.
45 */
__chk_range_not_ok(unsigned long addr,unsigned long size,unsigned long limit)46 static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, unsigned long limit)
47 {
48 /*
49 * If we have used "sizeof()" for the size,
50 * we know it won't overflow the limit (but
51 * it might overflow the 'addr', so it's
52 * important to subtract the size from the
53 * limit, not add it to the address).
54 */
55 if (__builtin_constant_p(size))
56 return unlikely(addr > limit - size);
57
58 /* Arbitrary sizes? Be careful about overflow */
59 addr += size;
60 if (unlikely(addr < size))
61 return true;
62 return unlikely(addr > limit);
63 }
64
65 #define __range_not_ok(addr, size, limit) \
66 ({ \
67 __chk_user_ptr(addr); \
68 __chk_range_not_ok((unsigned long __force)(addr), size, limit); \
69 })
70
71 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
72 # define WARN_ON_IN_IRQ() WARN_ON_ONCE(!in_task())
73 #else
74 # define WARN_ON_IN_IRQ()
75 #endif
76
77 /**
78 * access_ok: - Checks if a user space pointer is valid
79 * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that
80 * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe
81 * to write to a block, it is always safe to read from it.
82 * @addr: User space pointer to start of block to check
83 * @size: Size of block to check
84 *
85 * Context: User context only. This function may sleep if pagefaults are
86 * enabled.
87 *
88 * Checks if a pointer to a block of memory in user space is valid.
89 *
90 * Returns true (nonzero) if the memory block may be valid, false (zero)
91 * if it is definitely invalid.
92 *
93 * Note that, depending on architecture, this function probably just
94 * checks that the pointer is in the user space range - after calling
95 * this function, memory access functions may still return -EFAULT.
96 */
97 #define access_ok(type, addr, size) \
98 ({ \
99 WARN_ON_IN_IRQ(); \
100 likely(!__range_not_ok(addr, size, user_addr_max())); \
101 })
102
103 /*
104 * These are the main single-value transfer routines. They automatically
105 * use the right size if we just have the right pointer type.
106 *
107 * This gets kind of ugly. We want to return _two_ values in "get_user()"
108 * and yet we don't want to do any pointers, because that is too much
109 * of a performance impact. Thus we have a few rather ugly macros here,
110 * and hide all the ugliness from the user.
111 *
112 * The "__xxx" versions of the user access functions are versions that
113 * do not verify the address space, that must have been done previously
114 * with a separate "access_ok()" call (this is used when we do multiple
115 * accesses to the same area of user memory).
116 */
117
118 extern int __get_user_1(void);
119 extern int __get_user_2(void);
120 extern int __get_user_4(void);
121 extern int __get_user_8(void);
122 extern int __get_user_bad(void);
123
124 #define __uaccess_begin() stac()
125 #define __uaccess_end() clac()
126 #define __uaccess_begin_nospec() \
127 ({ \
128 stac(); \
129 barrier_nospec(); \
130 })
131
132 /*
133 * This is a type: either unsigned long, if the argument fits into
134 * that type, or otherwise unsigned long long.
135 */
136 #define __inttype(x) \
137 __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
138
139 /**
140 * get_user: - Get a simple variable from user space.
141 * @x: Variable to store result.
142 * @ptr: Source address, in user space.
143 *
144 * Context: User context only. This function may sleep if pagefaults are
145 * enabled.
146 *
147 * This macro copies a single simple variable from user space to kernel
148 * space. It supports simple types like char and int, but not larger
149 * data types like structures or arrays.
150 *
151 * @ptr must have pointer-to-simple-variable type, and the result of
152 * dereferencing @ptr must be assignable to @x without a cast.
153 *
154 * Returns zero on success, or -EFAULT on error.
155 * On error, the variable @x is set to zero.
156 */
157 /*
158 * Careful: we have to cast the result to the type of the pointer
159 * for sign reasons.
160 *
161 * The use of _ASM_DX as the register specifier is a bit of a
162 * simplification, as gcc only cares about it as the starting point
163 * and not size: for a 64-bit value it will use %ecx:%edx on 32 bits
164 * (%ecx being the next register in gcc's x86 register sequence), and
165 * %rdx on 64 bits.
166 *
167 * Clang/LLVM cares about the size of the register, but still wants
168 * the base register for something that ends up being a pair.
169 */
170 #define get_user(x, ptr) \
171 ({ \
172 int __ret_gu; \
173 register __inttype(*(ptr)) __val_gu asm("%"_ASM_DX); \
174 __chk_user_ptr(ptr); \
175 might_fault(); \
176 asm volatile("call __get_user_%P4" \
177 : "=a" (__ret_gu), "=r" (__val_gu), \
178 ASM_CALL_CONSTRAINT \
179 : "0" (ptr), "i" (sizeof(*(ptr)))); \
180 (x) = (__force __typeof__(*(ptr))) __val_gu; \
181 __builtin_expect(__ret_gu, 0); \
182 })
183
184 #define __put_user_x(size, x, ptr, __ret_pu) \
185 asm volatile("call __put_user_" #size : "=a" (__ret_pu) \
186 : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
187
188
189
190 #ifdef CONFIG_X86_32
191 #define __put_user_asm_u64(x, addr, err, errret) \
192 asm volatile("\n" \
193 "1: movl %%eax,0(%2)\n" \
194 "2: movl %%edx,4(%2)\n" \
195 "3:" \
196 ".section .fixup,\"ax\"\n" \
197 "4: movl %3,%0\n" \
198 " jmp 3b\n" \
199 ".previous\n" \
200 _ASM_EXTABLE(1b, 4b) \
201 _ASM_EXTABLE(2b, 4b) \
202 : "=r" (err) \
203 : "A" (x), "r" (addr), "i" (errret), "0" (err))
204
205 #define __put_user_asm_ex_u64(x, addr) \
206 asm volatile("\n" \
207 "1: movl %%eax,0(%1)\n" \
208 "2: movl %%edx,4(%1)\n" \
209 "3:" \
210 _ASM_EXTABLE_EX(1b, 2b) \
211 _ASM_EXTABLE_EX(2b, 3b) \
212 : : "A" (x), "r" (addr))
213
214 #define __put_user_x8(x, ptr, __ret_pu) \
215 asm volatile("call __put_user_8" : "=a" (__ret_pu) \
216 : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
217 #else
218 #define __put_user_asm_u64(x, ptr, retval, errret) \
219 __put_user_asm(x, ptr, retval, "q", "", "er", errret)
220 #define __put_user_asm_ex_u64(x, addr) \
221 __put_user_asm_ex(x, addr, "q", "", "er")
222 #define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)
223 #endif
224
225 extern void __put_user_bad(void);
226
227 /*
228 * Strange magic calling convention: pointer in %ecx,
229 * value in %eax(:%edx), return value in %eax. clobbers %rbx
230 */
231 extern void __put_user_1(void);
232 extern void __put_user_2(void);
233 extern void __put_user_4(void);
234 extern void __put_user_8(void);
235
236 /**
237 * put_user: - Write a simple value into user space.
238 * @x: Value to copy to user space.
239 * @ptr: Destination address, in user space.
240 *
241 * Context: User context only. This function may sleep if pagefaults are
242 * enabled.
243 *
244 * This macro copies a single simple value from kernel space to user
245 * space. It supports simple types like char and int, but not larger
246 * data types like structures or arrays.
247 *
248 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
249 * to the result of dereferencing @ptr.
250 *
251 * Returns zero on success, or -EFAULT on error.
252 */
253 #define put_user(x, ptr) \
254 ({ \
255 int __ret_pu; \
256 __typeof__(*(ptr)) __pu_val; \
257 __chk_user_ptr(ptr); \
258 might_fault(); \
259 __pu_val = x; \
260 switch (sizeof(*(ptr))) { \
261 case 1: \
262 __put_user_x(1, __pu_val, ptr, __ret_pu); \
263 break; \
264 case 2: \
265 __put_user_x(2, __pu_val, ptr, __ret_pu); \
266 break; \
267 case 4: \
268 __put_user_x(4, __pu_val, ptr, __ret_pu); \
269 break; \
270 case 8: \
271 __put_user_x8(__pu_val, ptr, __ret_pu); \
272 break; \
273 default: \
274 __put_user_x(X, __pu_val, ptr, __ret_pu); \
275 break; \
276 } \
277 __builtin_expect(__ret_pu, 0); \
278 })
279
280 #define __put_user_size(x, ptr, size, retval, errret) \
281 do { \
282 retval = 0; \
283 __chk_user_ptr(ptr); \
284 switch (size) { \
285 case 1: \
286 __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \
287 break; \
288 case 2: \
289 __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \
290 break; \
291 case 4: \
292 __put_user_asm(x, ptr, retval, "l", "k", "ir", errret); \
293 break; \
294 case 8: \
295 __put_user_asm_u64((__typeof__(*ptr))(x), ptr, retval, \
296 errret); \
297 break; \
298 default: \
299 __put_user_bad(); \
300 } \
301 } while (0)
302
303 /*
304 * This doesn't do __uaccess_begin/end - the exception handling
305 * around it must do that.
306 */
307 #define __put_user_size_ex(x, ptr, size) \
308 do { \
309 __chk_user_ptr(ptr); \
310 switch (size) { \
311 case 1: \
312 __put_user_asm_ex(x, ptr, "b", "b", "iq"); \
313 break; \
314 case 2: \
315 __put_user_asm_ex(x, ptr, "w", "w", "ir"); \
316 break; \
317 case 4: \
318 __put_user_asm_ex(x, ptr, "l", "k", "ir"); \
319 break; \
320 case 8: \
321 __put_user_asm_ex_u64((__typeof__(*ptr))(x), ptr); \
322 break; \
323 default: \
324 __put_user_bad(); \
325 } \
326 } while (0)
327
328 #ifdef CONFIG_X86_32
329 #define __get_user_asm_u64(x, ptr, retval, errret) \
330 ({ \
331 __typeof__(ptr) __ptr = (ptr); \
332 asm volatile("\n" \
333 "1: movl %2,%%eax\n" \
334 "2: movl %3,%%edx\n" \
335 "3:\n" \
336 ".section .fixup,\"ax\"\n" \
337 "4: mov %4,%0\n" \
338 " xorl %%eax,%%eax\n" \
339 " xorl %%edx,%%edx\n" \
340 " jmp 3b\n" \
341 ".previous\n" \
342 _ASM_EXTABLE(1b, 4b) \
343 _ASM_EXTABLE(2b, 4b) \
344 : "=r" (retval), "=&A"(x) \
345 : "m" (__m(__ptr)), "m" __m(((u32 *)(__ptr)) + 1), \
346 "i" (errret), "0" (retval)); \
347 })
348
349 #define __get_user_asm_ex_u64(x, ptr) (x) = __get_user_bad()
350 #else
351 #define __get_user_asm_u64(x, ptr, retval, errret) \
352 __get_user_asm(x, ptr, retval, "q", "", "=r", errret)
353 #define __get_user_asm_ex_u64(x, ptr) \
354 __get_user_asm_ex(x, ptr, "q", "", "=r")
355 #endif
356
357 #define __get_user_size(x, ptr, size, retval, errret) \
358 do { \
359 retval = 0; \
360 __chk_user_ptr(ptr); \
361 switch (size) { \
362 case 1: \
363 __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \
364 break; \
365 case 2: \
366 __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \
367 break; \
368 case 4: \
369 __get_user_asm(x, ptr, retval, "l", "k", "=r", errret); \
370 break; \
371 case 8: \
372 __get_user_asm_u64(x, ptr, retval, errret); \
373 break; \
374 default: \
375 (x) = __get_user_bad(); \
376 } \
377 } while (0)
378
379 #define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \
380 asm volatile("\n" \
381 "1: mov"itype" %2,%"rtype"1\n" \
382 "2:\n" \
383 ".section .fixup,\"ax\"\n" \
384 "3: mov %3,%0\n" \
385 " xor"itype" %"rtype"1,%"rtype"1\n" \
386 " jmp 2b\n" \
387 ".previous\n" \
388 _ASM_EXTABLE(1b, 3b) \
389 : "=r" (err), ltype(x) \
390 : "m" (__m(addr)), "i" (errret), "0" (err))
391
392 /*
393 * This doesn't do __uaccess_begin/end - the exception handling
394 * around it must do that.
395 */
396 #define __get_user_size_ex(x, ptr, size) \
397 do { \
398 __chk_user_ptr(ptr); \
399 switch (size) { \
400 case 1: \
401 __get_user_asm_ex(x, ptr, "b", "b", "=q"); \
402 break; \
403 case 2: \
404 __get_user_asm_ex(x, ptr, "w", "w", "=r"); \
405 break; \
406 case 4: \
407 __get_user_asm_ex(x, ptr, "l", "k", "=r"); \
408 break; \
409 case 8: \
410 __get_user_asm_ex_u64(x, ptr); \
411 break; \
412 default: \
413 (x) = __get_user_bad(); \
414 } \
415 } while (0)
416
417 #define __get_user_asm_ex(x, addr, itype, rtype, ltype) \
418 asm volatile("1: mov"itype" %1,%"rtype"0\n" \
419 "2:\n" \
420 ".section .fixup,\"ax\"\n" \
421 "3:xor"itype" %"rtype"0,%"rtype"0\n" \
422 " jmp 2b\n" \
423 ".previous\n" \
424 _ASM_EXTABLE_EX(1b, 3b) \
425 : ltype(x) : "m" (__m(addr)))
426
427 #define __put_user_nocheck(x, ptr, size) \
428 ({ \
429 int __pu_err; \
430 __uaccess_begin(); \
431 __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \
432 __uaccess_end(); \
433 __builtin_expect(__pu_err, 0); \
434 })
435
436 #define __get_user_nocheck(x, ptr, size) \
437 ({ \
438 int __gu_err; \
439 __inttype(*(ptr)) __gu_val; \
440 __uaccess_begin_nospec(); \
441 __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \
442 __uaccess_end(); \
443 (x) = (__force __typeof__(*(ptr)))__gu_val; \
444 __builtin_expect(__gu_err, 0); \
445 })
446
447 /* FIXME: this hack is definitely wrong -AK */
448 struct __large_struct { unsigned long buf[100]; };
449 #define __m(x) (*(struct __large_struct __user *)(x))
450
451 /*
452 * Tell gcc we read from memory instead of writing: this is because
453 * we do not write to any memory gcc knows about, so there are no
454 * aliasing issues.
455 */
456 #define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \
457 asm volatile("\n" \
458 "1: mov"itype" %"rtype"1,%2\n" \
459 "2:\n" \
460 ".section .fixup,\"ax\"\n" \
461 "3: mov %3,%0\n" \
462 " jmp 2b\n" \
463 ".previous\n" \
464 _ASM_EXTABLE(1b, 3b) \
465 : "=r"(err) \
466 : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err))
467
468 #define __put_user_asm_ex(x, addr, itype, rtype, ltype) \
469 asm volatile("1: mov"itype" %"rtype"0,%1\n" \
470 "2:\n" \
471 _ASM_EXTABLE_EX(1b, 2b) \
472 : : ltype(x), "m" (__m(addr)))
473
474 /*
475 * uaccess_try and catch
476 */
477 #define uaccess_try do { \
478 current->thread.uaccess_err = 0; \
479 __uaccess_begin(); \
480 barrier();
481
482 #define uaccess_try_nospec do { \
483 current->thread.uaccess_err = 0; \
484 __uaccess_begin_nospec(); \
485
486 #define uaccess_catch(err) \
487 __uaccess_end(); \
488 (err) |= (current->thread.uaccess_err ? -EFAULT : 0); \
489 } while (0)
490
491 /**
492 * __get_user: - Get a simple variable from user space, with less checking.
493 * @x: Variable to store result.
494 * @ptr: Source address, in user space.
495 *
496 * Context: User context only. This function may sleep if pagefaults are
497 * enabled.
498 *
499 * This macro copies a single simple variable from user space to kernel
500 * space. It supports simple types like char and int, but not larger
501 * data types like structures or arrays.
502 *
503 * @ptr must have pointer-to-simple-variable type, and the result of
504 * dereferencing @ptr must be assignable to @x without a cast.
505 *
506 * Caller must check the pointer with access_ok() before calling this
507 * function.
508 *
509 * Returns zero on success, or -EFAULT on error.
510 * On error, the variable @x is set to zero.
511 */
512
513 #define __get_user(x, ptr) \
514 __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
515
516 /**
517 * __put_user: - Write a simple value into user space, with less checking.
518 * @x: Value to copy to user space.
519 * @ptr: Destination address, in user space.
520 *
521 * Context: User context only. This function may sleep if pagefaults are
522 * enabled.
523 *
524 * This macro copies a single simple value from kernel space to user
525 * space. It supports simple types like char and int, but not larger
526 * data types like structures or arrays.
527 *
528 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
529 * to the result of dereferencing @ptr.
530 *
531 * Caller must check the pointer with access_ok() before calling this
532 * function.
533 *
534 * Returns zero on success, or -EFAULT on error.
535 */
536
537 #define __put_user(x, ptr) \
538 __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
539
540 #define __get_user_unaligned __get_user
541 #define __put_user_unaligned __put_user
542
543 /*
544 * {get|put}_user_try and catch
545 *
546 * get_user_try {
547 * get_user_ex(...);
548 * } get_user_catch(err)
549 */
550 #define get_user_try uaccess_try_nospec
551 #define get_user_catch(err) uaccess_catch(err)
552
553 #define get_user_ex(x, ptr) do { \
554 unsigned long __gue_val; \
555 __get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr)))); \
556 (x) = (__force __typeof__(*(ptr)))__gue_val; \
557 } while (0)
558
559 #define put_user_try uaccess_try
560 #define put_user_catch(err) uaccess_catch(err)
561
562 #define put_user_ex(x, ptr) \
563 __put_user_size_ex((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
564
565 extern unsigned long
566 copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
567 extern __must_check long
568 strncpy_from_user(char *dst, const char __user *src, long count);
569
570 extern __must_check long strlen_user(const char __user *str);
571 extern __must_check long strnlen_user(const char __user *str, long n);
572
573 unsigned long __must_check clear_user(void __user *mem, unsigned long len);
574 unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
575
576 extern void __cmpxchg_wrong_size(void)
577 __compiletime_error("Bad argument size for cmpxchg");
578
579 #define __user_atomic_cmpxchg_inatomic(uval, ptr, old, new, size) \
580 ({ \
581 int __ret = 0; \
582 __typeof__(ptr) __uval = (uval); \
583 __typeof__(*(ptr)) __old = (old); \
584 __typeof__(*(ptr)) __new = (new); \
585 __uaccess_begin_nospec(); \
586 switch (size) { \
587 case 1: \
588 { \
589 asm volatile("\n" \
590 "1:\t" LOCK_PREFIX "cmpxchgb %4, %2\n" \
591 "2:\n" \
592 "\t.section .fixup, \"ax\"\n" \
593 "3:\tmov %3, %0\n" \
594 "\tjmp 2b\n" \
595 "\t.previous\n" \
596 _ASM_EXTABLE(1b, 3b) \
597 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
598 : "i" (-EFAULT), "q" (__new), "1" (__old) \
599 : "memory" \
600 ); \
601 break; \
602 } \
603 case 2: \
604 { \
605 asm volatile("\n" \
606 "1:\t" LOCK_PREFIX "cmpxchgw %4, %2\n" \
607 "2:\n" \
608 "\t.section .fixup, \"ax\"\n" \
609 "3:\tmov %3, %0\n" \
610 "\tjmp 2b\n" \
611 "\t.previous\n" \
612 _ASM_EXTABLE(1b, 3b) \
613 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
614 : "i" (-EFAULT), "r" (__new), "1" (__old) \
615 : "memory" \
616 ); \
617 break; \
618 } \
619 case 4: \
620 { \
621 asm volatile("\n" \
622 "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" \
623 "2:\n" \
624 "\t.section .fixup, \"ax\"\n" \
625 "3:\tmov %3, %0\n" \
626 "\tjmp 2b\n" \
627 "\t.previous\n" \
628 _ASM_EXTABLE(1b, 3b) \
629 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
630 : "i" (-EFAULT), "r" (__new), "1" (__old) \
631 : "memory" \
632 ); \
633 break; \
634 } \
635 case 8: \
636 { \
637 if (!IS_ENABLED(CONFIG_X86_64)) \
638 __cmpxchg_wrong_size(); \
639 \
640 asm volatile("\n" \
641 "1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n" \
642 "2:\n" \
643 "\t.section .fixup, \"ax\"\n" \
644 "3:\tmov %3, %0\n" \
645 "\tjmp 2b\n" \
646 "\t.previous\n" \
647 _ASM_EXTABLE(1b, 3b) \
648 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
649 : "i" (-EFAULT), "r" (__new), "1" (__old) \
650 : "memory" \
651 ); \
652 break; \
653 } \
654 default: \
655 __cmpxchg_wrong_size(); \
656 } \
657 __uaccess_end(); \
658 *__uval = __old; \
659 __ret; \
660 })
661
662 #define user_atomic_cmpxchg_inatomic(uval, ptr, old, new) \
663 ({ \
664 access_ok(VERIFY_WRITE, (ptr), sizeof(*(ptr))) ? \
665 __user_atomic_cmpxchg_inatomic((uval), (ptr), \
666 (old), (new), sizeof(*(ptr))) : \
667 -EFAULT; \
668 })
669
670 /*
671 * movsl can be slow when source and dest are not both 8-byte aligned
672 */
673 #ifdef CONFIG_X86_INTEL_USERCOPY
674 extern struct movsl_mask {
675 int mask;
676 } ____cacheline_aligned_in_smp movsl_mask;
677 #endif
678
679 #define ARCH_HAS_NOCACHE_UACCESS 1
680
681 #ifdef CONFIG_X86_32
682 # include <asm/uaccess_32.h>
683 #else
684 # include <asm/uaccess_64.h>
685 #endif
686
687 unsigned long __must_check _copy_from_user(void *to, const void __user *from,
688 unsigned n);
689 unsigned long __must_check _copy_to_user(void __user *to, const void *from,
690 unsigned n);
691
692 extern void __compiletime_error("usercopy buffer size is too small")
693 __bad_copy_user(void);
694
copy_user_overflow(int size,unsigned long count)695 static inline void copy_user_overflow(int size, unsigned long count)
696 {
697 WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count);
698 }
699
700 static __always_inline unsigned long __must_check
copy_from_user(void * to,const void __user * from,unsigned long n)701 copy_from_user(void *to, const void __user *from, unsigned long n)
702 {
703 int sz = __compiletime_object_size(to);
704
705 might_fault();
706
707 kasan_check_write(to, n);
708
709 if (likely(sz < 0 || sz >= n)) {
710 check_object_size(to, n, false);
711 n = _copy_from_user(to, from, n);
712 } else if (!__builtin_constant_p(n))
713 copy_user_overflow(sz, n);
714 else
715 __bad_copy_user();
716
717 return n;
718 }
719
720 static __always_inline unsigned long __must_check
copy_to_user(void __user * to,const void * from,unsigned long n)721 copy_to_user(void __user *to, const void *from, unsigned long n)
722 {
723 int sz = __compiletime_object_size(from);
724
725 kasan_check_read(from, n);
726
727 might_fault();
728
729 if (likely(sz < 0 || sz >= n)) {
730 check_object_size(from, n, true);
731 n = _copy_to_user(to, from, n);
732 } else if (!__builtin_constant_p(n))
733 copy_user_overflow(sz, n);
734 else
735 __bad_copy_user();
736
737 return n;
738 }
739
740 /*
741 * We rely on the nested NMI work to allow atomic faults from the NMI path; the
742 * nested NMI paths are careful to preserve CR2.
743 *
744 * Caller must use pagefault_enable/disable, or run in interrupt context,
745 * and also do a uaccess_ok() check
746 */
747 #define __copy_from_user_nmi __copy_from_user_inatomic
748
749 /*
750 * The "unsafe" user accesses aren't really "unsafe", but the naming
751 * is a big fat warning: you have to not only do the access_ok()
752 * checking before using them, but you have to surround them with the
753 * user_access_begin/end() pair.
754 */
755 #define user_access_begin() __uaccess_begin()
756 #define user_access_end() __uaccess_end()
757
758 #define unsafe_put_user(x, ptr, err_label) \
759 do { \
760 int __pu_err; \
761 __put_user_size((x), (ptr), sizeof(*(ptr)), __pu_err, -EFAULT); \
762 if (unlikely(__pu_err)) goto err_label; \
763 } while (0)
764
765 #define unsafe_get_user(x, ptr, err_label) \
766 do { \
767 int __gu_err; \
768 unsigned long __gu_val; \
769 __get_user_size(__gu_val, (ptr), sizeof(*(ptr)), __gu_err, -EFAULT); \
770 (x) = (__force __typeof__(*(ptr)))__gu_val; \
771 if (unlikely(__gu_err)) goto err_label; \
772 } while (0)
773
774 #endif /* _ASM_X86_UACCESS_H */
775
776