1 /*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1996, 1997, 1998, 1999, 2000, 03, 04 by Ralf Baechle
7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
8 * Copyright (C) 2007 Maciej W. Rozycki
9 * Copyright (C) 2014, Imagination Technologies Ltd.
10 */
11 #ifndef _ASM_UACCESS_H
12 #define _ASM_UACCESS_H
13
14 #include <linux/kernel.h>
15 #include <linux/errno.h>
16 #include <linux/thread_info.h>
17 #include <linux/string.h>
18 #include <asm/asm-eva.h>
19
20 /*
21 * The fs value determines whether argument validity checking should be
22 * performed or not. If get_fs() == USER_DS, checking is performed, with
23 * get_fs() == KERNEL_DS, checking is bypassed.
24 *
25 * For historical reasons, these macros are grossly misnamed.
26 */
27 #ifdef CONFIG_32BIT
28
29 #ifdef CONFIG_KVM_GUEST
30 #define __UA_LIMIT 0x40000000UL
31 #else
32 #define __UA_LIMIT 0x80000000UL
33 #endif
34
35 #define __UA_ADDR ".word"
36 #define __UA_LA "la"
37 #define __UA_ADDU "addu"
38 #define __UA_t0 "$8"
39 #define __UA_t1 "$9"
40
41 #endif /* CONFIG_32BIT */
42
43 #ifdef CONFIG_64BIT
44
45 extern u64 __ua_limit;
46
47 #define __UA_LIMIT __ua_limit
48
49 #define __UA_ADDR ".dword"
50 #define __UA_LA "dla"
51 #define __UA_ADDU "daddu"
52 #define __UA_t0 "$12"
53 #define __UA_t1 "$13"
54
55 #endif /* CONFIG_64BIT */
56
57 /*
58 * USER_DS is a bitmask that has the bits set that may not be set in a valid
59 * userspace address. Note that we limit 32-bit userspace to 0x7fff8000 but
60 * the arithmetic we're doing only works if the limit is a power of two, so
61 * we use 0x80000000 here on 32-bit kernels. If a process passes an invalid
62 * address in this range it's the process's problem, not ours :-)
63 */
64
65 #ifdef CONFIG_KVM_GUEST
66 #define KERNEL_DS ((mm_segment_t) { 0x80000000UL })
67 #define USER_DS ((mm_segment_t) { 0xC0000000UL })
68 #else
69 #define KERNEL_DS ((mm_segment_t) { 0UL })
70 #define USER_DS ((mm_segment_t) { __UA_LIMIT })
71 #endif
72
73 #define VERIFY_READ 0
74 #define VERIFY_WRITE 1
75
76 #define get_ds() (KERNEL_DS)
77 #define get_fs() (current_thread_info()->addr_limit)
78 #define set_fs(x) (current_thread_info()->addr_limit = (x))
79
80 #define segment_eq(a, b) ((a).seg == (b).seg)
81
82 /*
83 * eva_kernel_access() - determine whether kernel memory access on an EVA system
84 *
85 * Determines whether memory accesses should be performed to kernel memory
86 * on a system using Extended Virtual Addressing (EVA).
87 *
88 * Return: true if a kernel memory access on an EVA system, else false.
89 */
eva_kernel_access(void)90 static inline bool eva_kernel_access(void)
91 {
92 if (!config_enabled(CONFIG_EVA))
93 return false;
94
95 return segment_eq(get_fs(), get_ds());
96 }
97
98 /*
99 * Is a address valid? This does a straightforward calculation rather
100 * than tests.
101 *
102 * Address valid if:
103 * - "addr" doesn't have any high-bits set
104 * - AND "size" doesn't have any high-bits set
105 * - AND "addr+size" doesn't have any high-bits set
106 * - OR we are in kernel mode.
107 *
108 * __ua_size() is a trick to avoid runtime checking of positive constant
109 * sizes; for those we already know at compile time that the size is ok.
110 */
111 #define __ua_size(size) \
112 ((__builtin_constant_p(size) && (signed long) (size) > 0) ? 0 : (size))
113
114 /*
115 * access_ok: - Checks if a user space pointer is valid
116 * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that
117 * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe
118 * to write to a block, it is always safe to read from it.
119 * @addr: User space pointer to start of block to check
120 * @size: Size of block to check
121 *
122 * Context: User context only. This function may sleep if pagefaults are
123 * enabled.
124 *
125 * Checks if a pointer to a block of memory in user space is valid.
126 *
127 * Returns true (nonzero) if the memory block may be valid, false (zero)
128 * if it is definitely invalid.
129 *
130 * Note that, depending on architecture, this function probably just
131 * checks that the pointer is in the user space range - after calling
132 * this function, memory access functions may still return -EFAULT.
133 */
134
135 #define __access_mask get_fs().seg
136
137 #define __access_ok(addr, size, mask) \
138 ({ \
139 unsigned long __addr = (unsigned long) (addr); \
140 unsigned long __size = size; \
141 unsigned long __mask = mask; \
142 unsigned long __ok; \
143 \
144 __chk_user_ptr(addr); \
145 __ok = (signed long)(__mask & (__addr | (__addr + __size) | \
146 __ua_size(__size))); \
147 __ok == 0; \
148 })
149
150 #define access_ok(type, addr, size) \
151 likely(__access_ok((addr), (size), __access_mask))
152
153 /*
154 * put_user: - Write a simple value into user space.
155 * @x: Value to copy to user space.
156 * @ptr: Destination address, in user space.
157 *
158 * Context: User context only. This function may sleep if pagefaults are
159 * enabled.
160 *
161 * This macro copies a single simple value from kernel space to user
162 * space. It supports simple types like char and int, but not larger
163 * data types like structures or arrays.
164 *
165 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
166 * to the result of dereferencing @ptr.
167 *
168 * Returns zero on success, or -EFAULT on error.
169 */
170 #define put_user(x,ptr) \
171 __put_user_check((x), (ptr), sizeof(*(ptr)))
172
173 /*
174 * get_user: - Get a simple variable from user space.
175 * @x: Variable to store result.
176 * @ptr: Source address, in user space.
177 *
178 * Context: User context only. This function may sleep if pagefaults are
179 * enabled.
180 *
181 * This macro copies a single simple variable from user space to kernel
182 * space. It supports simple types like char and int, but not larger
183 * data types like structures or arrays.
184 *
185 * @ptr must have pointer-to-simple-variable type, and the result of
186 * dereferencing @ptr must be assignable to @x without a cast.
187 *
188 * Returns zero on success, or -EFAULT on error.
189 * On error, the variable @x is set to zero.
190 */
191 #define get_user(x,ptr) \
192 __get_user_check((x), (ptr), sizeof(*(ptr)))
193
194 /*
195 * __put_user: - Write a simple value into user space, with less checking.
196 * @x: Value to copy to user space.
197 * @ptr: Destination address, in user space.
198 *
199 * Context: User context only. This function may sleep if pagefaults are
200 * enabled.
201 *
202 * This macro copies a single simple value from kernel space to user
203 * space. It supports simple types like char and int, but not larger
204 * data types like structures or arrays.
205 *
206 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
207 * to the result of dereferencing @ptr.
208 *
209 * Caller must check the pointer with access_ok() before calling this
210 * function.
211 *
212 * Returns zero on success, or -EFAULT on error.
213 */
214 #define __put_user(x,ptr) \
215 __put_user_nocheck((x), (ptr), sizeof(*(ptr)))
216
217 /*
218 * __get_user: - Get a simple variable from user space, with less checking.
219 * @x: Variable to store result.
220 * @ptr: Source address, in user space.
221 *
222 * Context: User context only. This function may sleep if pagefaults are
223 * enabled.
224 *
225 * This macro copies a single simple variable from user space to kernel
226 * space. It supports simple types like char and int, but not larger
227 * data types like structures or arrays.
228 *
229 * @ptr must have pointer-to-simple-variable type, and the result of
230 * dereferencing @ptr must be assignable to @x without a cast.
231 *
232 * Caller must check the pointer with access_ok() before calling this
233 * function.
234 *
235 * Returns zero on success, or -EFAULT on error.
236 * On error, the variable @x is set to zero.
237 */
238 #define __get_user(x,ptr) \
239 __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
240
241 struct __large_struct { unsigned long buf[100]; };
242 #define __m(x) (*(struct __large_struct __user *)(x))
243
244 /*
245 * Yuck. We need two variants, one for 64bit operation and one
246 * for 32 bit mode and old iron.
247 */
248 #ifndef CONFIG_EVA
249 #define __get_kernel_common(val, size, ptr) __get_user_common(val, size, ptr)
250 #else
251 /*
252 * Kernel specific functions for EVA. We need to use normal load instructions
253 * to read data from kernel when operating in EVA mode. We use these macros to
254 * avoid redefining __get_user_asm for EVA.
255 */
256 #undef _loadd
257 #undef _loadw
258 #undef _loadh
259 #undef _loadb
260 #ifdef CONFIG_32BIT
261 #define _loadd _loadw
262 #else
263 #define _loadd(reg, addr) "ld " reg ", " addr
264 #endif
265 #define _loadw(reg, addr) "lw " reg ", " addr
266 #define _loadh(reg, addr) "lh " reg ", " addr
267 #define _loadb(reg, addr) "lb " reg ", " addr
268
269 #define __get_kernel_common(val, size, ptr) \
270 do { \
271 switch (size) { \
272 case 1: __get_data_asm(val, _loadb, ptr); break; \
273 case 2: __get_data_asm(val, _loadh, ptr); break; \
274 case 4: __get_data_asm(val, _loadw, ptr); break; \
275 case 8: __GET_DW(val, _loadd, ptr); break; \
276 default: __get_user_unknown(); break; \
277 } \
278 } while (0)
279 #endif
280
281 #ifdef CONFIG_32BIT
282 #define __GET_DW(val, insn, ptr) __get_data_asm_ll32(val, insn, ptr)
283 #endif
284 #ifdef CONFIG_64BIT
285 #define __GET_DW(val, insn, ptr) __get_data_asm(val, insn, ptr)
286 #endif
287
288 extern void __get_user_unknown(void);
289
290 #define __get_user_common(val, size, ptr) \
291 do { \
292 switch (size) { \
293 case 1: __get_data_asm(val, user_lb, ptr); break; \
294 case 2: __get_data_asm(val, user_lh, ptr); break; \
295 case 4: __get_data_asm(val, user_lw, ptr); break; \
296 case 8: __GET_DW(val, user_ld, ptr); break; \
297 default: __get_user_unknown(); break; \
298 } \
299 } while (0)
300
301 #define __get_user_nocheck(x, ptr, size) \
302 ({ \
303 int __gu_err; \
304 \
305 if (eva_kernel_access()) { \
306 __get_kernel_common((x), size, ptr); \
307 } else { \
308 __chk_user_ptr(ptr); \
309 __get_user_common((x), size, ptr); \
310 } \
311 __gu_err; \
312 })
313
314 #define __get_user_check(x, ptr, size) \
315 ({ \
316 int __gu_err = -EFAULT; \
317 const __typeof__(*(ptr)) __user * __gu_ptr = (ptr); \
318 \
319 might_fault(); \
320 if (likely(access_ok(VERIFY_READ, __gu_ptr, size))) { \
321 if (eva_kernel_access()) \
322 __get_kernel_common((x), size, __gu_ptr); \
323 else \
324 __get_user_common((x), size, __gu_ptr); \
325 } else \
326 (x) = 0; \
327 \
328 __gu_err; \
329 })
330
331 #define __get_data_asm(val, insn, addr) \
332 { \
333 long __gu_tmp; \
334 \
335 __asm__ __volatile__( \
336 "1: "insn("%1", "%3")" \n" \
337 "2: \n" \
338 " .insn \n" \
339 " .section .fixup,\"ax\" \n" \
340 "3: li %0, %4 \n" \
341 " move %1, $0 \n" \
342 " j 2b \n" \
343 " .previous \n" \
344 " .section __ex_table,\"a\" \n" \
345 " "__UA_ADDR "\t1b, 3b \n" \
346 " .previous \n" \
347 : "=r" (__gu_err), "=r" (__gu_tmp) \
348 : "0" (0), "o" (__m(addr)), "i" (-EFAULT)); \
349 \
350 (val) = (__typeof__(*(addr))) __gu_tmp; \
351 }
352
353 /*
354 * Get a long long 64 using 32 bit registers.
355 */
356 #define __get_data_asm_ll32(val, insn, addr) \
357 { \
358 union { \
359 unsigned long long l; \
360 __typeof__(*(addr)) t; \
361 } __gu_tmp; \
362 \
363 __asm__ __volatile__( \
364 "1: " insn("%1", "(%3)")" \n" \
365 "2: " insn("%D1", "4(%3)")" \n" \
366 "3: \n" \
367 " .insn \n" \
368 " .section .fixup,\"ax\" \n" \
369 "4: li %0, %4 \n" \
370 " move %1, $0 \n" \
371 " move %D1, $0 \n" \
372 " j 3b \n" \
373 " .previous \n" \
374 " .section __ex_table,\"a\" \n" \
375 " " __UA_ADDR " 1b, 4b \n" \
376 " " __UA_ADDR " 2b, 4b \n" \
377 " .previous \n" \
378 : "=r" (__gu_err), "=&r" (__gu_tmp.l) \
379 : "0" (0), "r" (addr), "i" (-EFAULT)); \
380 \
381 (val) = __gu_tmp.t; \
382 }
383
384 #ifndef CONFIG_EVA
385 #define __put_kernel_common(ptr, size) __put_user_common(ptr, size)
386 #else
387 /*
388 * Kernel specific functions for EVA. We need to use normal load instructions
389 * to read data from kernel when operating in EVA mode. We use these macros to
390 * avoid redefining __get_data_asm for EVA.
391 */
392 #undef _stored
393 #undef _storew
394 #undef _storeh
395 #undef _storeb
396 #ifdef CONFIG_32BIT
397 #define _stored _storew
398 #else
399 #define _stored(reg, addr) "ld " reg ", " addr
400 #endif
401
402 #define _storew(reg, addr) "sw " reg ", " addr
403 #define _storeh(reg, addr) "sh " reg ", " addr
404 #define _storeb(reg, addr) "sb " reg ", " addr
405
406 #define __put_kernel_common(ptr, size) \
407 do { \
408 switch (size) { \
409 case 1: __put_data_asm(_storeb, ptr); break; \
410 case 2: __put_data_asm(_storeh, ptr); break; \
411 case 4: __put_data_asm(_storew, ptr); break; \
412 case 8: __PUT_DW(_stored, ptr); break; \
413 default: __put_user_unknown(); break; \
414 } \
415 } while(0)
416 #endif
417
418 /*
419 * Yuck. We need two variants, one for 64bit operation and one
420 * for 32 bit mode and old iron.
421 */
422 #ifdef CONFIG_32BIT
423 #define __PUT_DW(insn, ptr) __put_data_asm_ll32(insn, ptr)
424 #endif
425 #ifdef CONFIG_64BIT
426 #define __PUT_DW(insn, ptr) __put_data_asm(insn, ptr)
427 #endif
428
429 #define __put_user_common(ptr, size) \
430 do { \
431 switch (size) { \
432 case 1: __put_data_asm(user_sb, ptr); break; \
433 case 2: __put_data_asm(user_sh, ptr); break; \
434 case 4: __put_data_asm(user_sw, ptr); break; \
435 case 8: __PUT_DW(user_sd, ptr); break; \
436 default: __put_user_unknown(); break; \
437 } \
438 } while (0)
439
440 #define __put_user_nocheck(x, ptr, size) \
441 ({ \
442 __typeof__(*(ptr)) __pu_val; \
443 int __pu_err = 0; \
444 \
445 __pu_val = (x); \
446 if (eva_kernel_access()) { \
447 __put_kernel_common(ptr, size); \
448 } else { \
449 __chk_user_ptr(ptr); \
450 __put_user_common(ptr, size); \
451 } \
452 __pu_err; \
453 })
454
455 #define __put_user_check(x, ptr, size) \
456 ({ \
457 __typeof__(*(ptr)) __user *__pu_addr = (ptr); \
458 __typeof__(*(ptr)) __pu_val = (x); \
459 int __pu_err = -EFAULT; \
460 \
461 might_fault(); \
462 if (likely(access_ok(VERIFY_WRITE, __pu_addr, size))) { \
463 if (eva_kernel_access()) \
464 __put_kernel_common(__pu_addr, size); \
465 else \
466 __put_user_common(__pu_addr, size); \
467 } \
468 \
469 __pu_err; \
470 })
471
472 #define __put_data_asm(insn, ptr) \
473 { \
474 __asm__ __volatile__( \
475 "1: "insn("%z2", "%3")" # __put_data_asm \n" \
476 "2: \n" \
477 " .insn \n" \
478 " .section .fixup,\"ax\" \n" \
479 "3: li %0, %4 \n" \
480 " j 2b \n" \
481 " .previous \n" \
482 " .section __ex_table,\"a\" \n" \
483 " " __UA_ADDR " 1b, 3b \n" \
484 " .previous \n" \
485 : "=r" (__pu_err) \
486 : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)), \
487 "i" (-EFAULT)); \
488 }
489
490 #define __put_data_asm_ll32(insn, ptr) \
491 { \
492 __asm__ __volatile__( \
493 "1: "insn("%2", "(%3)")" # __put_data_asm_ll32 \n" \
494 "2: "insn("%D2", "4(%3)")" \n" \
495 "3: \n" \
496 " .insn \n" \
497 " .section .fixup,\"ax\" \n" \
498 "4: li %0, %4 \n" \
499 " j 3b \n" \
500 " .previous \n" \
501 " .section __ex_table,\"a\" \n" \
502 " " __UA_ADDR " 1b, 4b \n" \
503 " " __UA_ADDR " 2b, 4b \n" \
504 " .previous" \
505 : "=r" (__pu_err) \
506 : "0" (0), "r" (__pu_val), "r" (ptr), \
507 "i" (-EFAULT)); \
508 }
509
510 extern void __put_user_unknown(void);
511
512 /*
513 * ul{b,h,w} are macros and there are no equivalent macros for EVA.
514 * EVA unaligned access is handled in the ADE exception handler.
515 */
516 #ifndef CONFIG_EVA
517 /*
518 * put_user_unaligned: - Write a simple value into user space.
519 * @x: Value to copy to user space.
520 * @ptr: Destination address, in user space.
521 *
522 * Context: User context only. This function may sleep if pagefaults are
523 * enabled.
524 *
525 * This macro copies a single simple value from kernel space to user
526 * space. It supports simple types like char and int, but not larger
527 * data types like structures or arrays.
528 *
529 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
530 * to the result of dereferencing @ptr.
531 *
532 * Returns zero on success, or -EFAULT on error.
533 */
534 #define put_user_unaligned(x,ptr) \
535 __put_user_unaligned_check((x),(ptr),sizeof(*(ptr)))
536
537 /*
538 * get_user_unaligned: - Get a simple variable from user space.
539 * @x: Variable to store result.
540 * @ptr: Source address, in user space.
541 *
542 * Context: User context only. This function may sleep if pagefaults are
543 * enabled.
544 *
545 * This macro copies a single simple variable from user space to kernel
546 * space. It supports simple types like char and int, but not larger
547 * data types like structures or arrays.
548 *
549 * @ptr must have pointer-to-simple-variable type, and the result of
550 * dereferencing @ptr must be assignable to @x without a cast.
551 *
552 * Returns zero on success, or -EFAULT on error.
553 * On error, the variable @x is set to zero.
554 */
555 #define get_user_unaligned(x,ptr) \
556 __get_user_unaligned_check((x),(ptr),sizeof(*(ptr)))
557
558 /*
559 * __put_user_unaligned: - Write a simple value into user space, with less checking.
560 * @x: Value to copy to user space.
561 * @ptr: Destination address, in user space.
562 *
563 * Context: User context only. This function may sleep if pagefaults are
564 * enabled.
565 *
566 * This macro copies a single simple value from kernel space to user
567 * space. It supports simple types like char and int, but not larger
568 * data types like structures or arrays.
569 *
570 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
571 * to the result of dereferencing @ptr.
572 *
573 * Caller must check the pointer with access_ok() before calling this
574 * function.
575 *
576 * Returns zero on success, or -EFAULT on error.
577 */
578 #define __put_user_unaligned(x,ptr) \
579 __put_user_unaligned_nocheck((x),(ptr),sizeof(*(ptr)))
580
581 /*
582 * __get_user_unaligned: - Get a simple variable from user space, with less checking.
583 * @x: Variable to store result.
584 * @ptr: Source address, in user space.
585 *
586 * Context: User context only. This function may sleep if pagefaults are
587 * enabled.
588 *
589 * This macro copies a single simple variable from user space to kernel
590 * space. It supports simple types like char and int, but not larger
591 * data types like structures or arrays.
592 *
593 * @ptr must have pointer-to-simple-variable type, and the result of
594 * dereferencing @ptr must be assignable to @x without a cast.
595 *
596 * Caller must check the pointer with access_ok() before calling this
597 * function.
598 *
599 * Returns zero on success, or -EFAULT on error.
600 * On error, the variable @x is set to zero.
601 */
602 #define __get_user_unaligned(x,ptr) \
603 __get_user_unaligned_nocheck((x),(ptr),sizeof(*(ptr)))
604
605 /*
606 * Yuck. We need two variants, one for 64bit operation and one
607 * for 32 bit mode and old iron.
608 */
609 #ifdef CONFIG_32BIT
610 #define __GET_USER_UNALIGNED_DW(val, ptr) \
611 __get_user_unaligned_asm_ll32(val, ptr)
612 #endif
613 #ifdef CONFIG_64BIT
614 #define __GET_USER_UNALIGNED_DW(val, ptr) \
615 __get_user_unaligned_asm(val, "uld", ptr)
616 #endif
617
618 extern void __get_user_unaligned_unknown(void);
619
620 #define __get_user_unaligned_common(val, size, ptr) \
621 do { \
622 switch (size) { \
623 case 1: __get_data_asm(val, "lb", ptr); break; \
624 case 2: __get_data_unaligned_asm(val, "ulh", ptr); break; \
625 case 4: __get_data_unaligned_asm(val, "ulw", ptr); break; \
626 case 8: __GET_USER_UNALIGNED_DW(val, ptr); break; \
627 default: __get_user_unaligned_unknown(); break; \
628 } \
629 } while (0)
630
631 #define __get_user_unaligned_nocheck(x,ptr,size) \
632 ({ \
633 int __gu_err; \
634 \
635 __get_user_unaligned_common((x), size, ptr); \
636 __gu_err; \
637 })
638
639 #define __get_user_unaligned_check(x,ptr,size) \
640 ({ \
641 int __gu_err = -EFAULT; \
642 const __typeof__(*(ptr)) __user * __gu_ptr = (ptr); \
643 \
644 if (likely(access_ok(VERIFY_READ, __gu_ptr, size))) \
645 __get_user_unaligned_common((x), size, __gu_ptr); \
646 \
647 __gu_err; \
648 })
649
650 #define __get_data_unaligned_asm(val, insn, addr) \
651 { \
652 long __gu_tmp; \
653 \
654 __asm__ __volatile__( \
655 "1: " insn " %1, %3 \n" \
656 "2: \n" \
657 " .insn \n" \
658 " .section .fixup,\"ax\" \n" \
659 "3: li %0, %4 \n" \
660 " move %1, $0 \n" \
661 " j 2b \n" \
662 " .previous \n" \
663 " .section __ex_table,\"a\" \n" \
664 " "__UA_ADDR "\t1b, 3b \n" \
665 " "__UA_ADDR "\t1b + 4, 3b \n" \
666 " .previous \n" \
667 : "=r" (__gu_err), "=r" (__gu_tmp) \
668 : "0" (0), "o" (__m(addr)), "i" (-EFAULT)); \
669 \
670 (val) = (__typeof__(*(addr))) __gu_tmp; \
671 }
672
673 /*
674 * Get a long long 64 using 32 bit registers.
675 */
676 #define __get_user_unaligned_asm_ll32(val, addr) \
677 { \
678 unsigned long long __gu_tmp; \
679 \
680 __asm__ __volatile__( \
681 "1: ulw %1, (%3) \n" \
682 "2: ulw %D1, 4(%3) \n" \
683 " move %0, $0 \n" \
684 "3: \n" \
685 " .insn \n" \
686 " .section .fixup,\"ax\" \n" \
687 "4: li %0, %4 \n" \
688 " move %1, $0 \n" \
689 " move %D1, $0 \n" \
690 " j 3b \n" \
691 " .previous \n" \
692 " .section __ex_table,\"a\" \n" \
693 " " __UA_ADDR " 1b, 4b \n" \
694 " " __UA_ADDR " 1b + 4, 4b \n" \
695 " " __UA_ADDR " 2b, 4b \n" \
696 " " __UA_ADDR " 2b + 4, 4b \n" \
697 " .previous \n" \
698 : "=r" (__gu_err), "=&r" (__gu_tmp) \
699 : "0" (0), "r" (addr), "i" (-EFAULT)); \
700 (val) = (__typeof__(*(addr))) __gu_tmp; \
701 }
702
703 /*
704 * Yuck. We need two variants, one for 64bit operation and one
705 * for 32 bit mode and old iron.
706 */
707 #ifdef CONFIG_32BIT
708 #define __PUT_USER_UNALIGNED_DW(ptr) __put_user_unaligned_asm_ll32(ptr)
709 #endif
710 #ifdef CONFIG_64BIT
711 #define __PUT_USER_UNALIGNED_DW(ptr) __put_user_unaligned_asm("usd", ptr)
712 #endif
713
714 #define __put_user_unaligned_common(ptr, size) \
715 do { \
716 switch (size) { \
717 case 1: __put_data_asm("sb", ptr); break; \
718 case 2: __put_user_unaligned_asm("ush", ptr); break; \
719 case 4: __put_user_unaligned_asm("usw", ptr); break; \
720 case 8: __PUT_USER_UNALIGNED_DW(ptr); break; \
721 default: __put_user_unaligned_unknown(); break; \
722 } while (0)
723
724 #define __put_user_unaligned_nocheck(x,ptr,size) \
725 ({ \
726 __typeof__(*(ptr)) __pu_val; \
727 int __pu_err = 0; \
728 \
729 __pu_val = (x); \
730 __put_user_unaligned_common(ptr, size); \
731 __pu_err; \
732 })
733
734 #define __put_user_unaligned_check(x,ptr,size) \
735 ({ \
736 __typeof__(*(ptr)) __user *__pu_addr = (ptr); \
737 __typeof__(*(ptr)) __pu_val = (x); \
738 int __pu_err = -EFAULT; \
739 \
740 if (likely(access_ok(VERIFY_WRITE, __pu_addr, size))) \
741 __put_user_unaligned_common(__pu_addr, size); \
742 \
743 __pu_err; \
744 })
745
746 #define __put_user_unaligned_asm(insn, ptr) \
747 { \
748 __asm__ __volatile__( \
749 "1: " insn " %z2, %3 # __put_user_unaligned_asm\n" \
750 "2: \n" \
751 " .insn \n" \
752 " .section .fixup,\"ax\" \n" \
753 "3: li %0, %4 \n" \
754 " j 2b \n" \
755 " .previous \n" \
756 " .section __ex_table,\"a\" \n" \
757 " " __UA_ADDR " 1b, 3b \n" \
758 " .previous \n" \
759 : "=r" (__pu_err) \
760 : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)), \
761 "i" (-EFAULT)); \
762 }
763
764 #define __put_user_unaligned_asm_ll32(ptr) \
765 { \
766 __asm__ __volatile__( \
767 "1: sw %2, (%3) # __put_user_unaligned_asm_ll32 \n" \
768 "2: sw %D2, 4(%3) \n" \
769 "3: \n" \
770 " .insn \n" \
771 " .section .fixup,\"ax\" \n" \
772 "4: li %0, %4 \n" \
773 " j 3b \n" \
774 " .previous \n" \
775 " .section __ex_table,\"a\" \n" \
776 " " __UA_ADDR " 1b, 4b \n" \
777 " " __UA_ADDR " 1b + 4, 4b \n" \
778 " " __UA_ADDR " 2b, 4b \n" \
779 " " __UA_ADDR " 2b + 4, 4b \n" \
780 " .previous" \
781 : "=r" (__pu_err) \
782 : "0" (0), "r" (__pu_val), "r" (ptr), \
783 "i" (-EFAULT)); \
784 }
785
786 extern void __put_user_unaligned_unknown(void);
787 #endif
788
789 /*
790 * We're generating jump to subroutines which will be outside the range of
791 * jump instructions
792 */
793 #ifdef MODULE
794 #define __MODULE_JAL(destination) \
795 ".set\tnoat\n\t" \
796 __UA_LA "\t$1, " #destination "\n\t" \
797 "jalr\t$1\n\t" \
798 ".set\tat\n\t"
799 #else
800 #define __MODULE_JAL(destination) \
801 "jal\t" #destination "\n\t"
802 #endif
803
804 #if defined(CONFIG_CPU_DADDI_WORKAROUNDS) || (defined(CONFIG_EVA) && \
805 defined(CONFIG_CPU_HAS_PREFETCH))
806 #define DADDI_SCRATCH "$3"
807 #else
808 #define DADDI_SCRATCH "$0"
809 #endif
810
811 extern size_t __copy_user(void *__to, const void *__from, size_t __n);
812
813 #ifndef CONFIG_EVA
814 #define __invoke_copy_to_user(to, from, n) \
815 ({ \
816 register void __user *__cu_to_r __asm__("$4"); \
817 register const void *__cu_from_r __asm__("$5"); \
818 register long __cu_len_r __asm__("$6"); \
819 \
820 __cu_to_r = (to); \
821 __cu_from_r = (from); \
822 __cu_len_r = (n); \
823 __asm__ __volatile__( \
824 __MODULE_JAL(__copy_user) \
825 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \
826 : \
827 : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \
828 DADDI_SCRATCH, "memory"); \
829 __cu_len_r; \
830 })
831
832 #define __invoke_copy_to_kernel(to, from, n) \
833 __invoke_copy_to_user(to, from, n)
834
835 #endif
836
837 /*
838 * __copy_to_user: - Copy a block of data into user space, with less checking.
839 * @to: Destination address, in user space.
840 * @from: Source address, in kernel space.
841 * @n: Number of bytes to copy.
842 *
843 * Context: User context only. This function may sleep if pagefaults are
844 * enabled.
845 *
846 * Copy data from kernel space to user space. Caller must check
847 * the specified block with access_ok() before calling this function.
848 *
849 * Returns number of bytes that could not be copied.
850 * On success, this will be zero.
851 */
852 #define __copy_to_user(to, from, n) \
853 ({ \
854 void __user *__cu_to; \
855 const void *__cu_from; \
856 long __cu_len; \
857 \
858 __cu_to = (to); \
859 __cu_from = (from); \
860 __cu_len = (n); \
861 might_fault(); \
862 if (eva_kernel_access()) \
863 __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from, \
864 __cu_len); \
865 else \
866 __cu_len = __invoke_copy_to_user(__cu_to, __cu_from, \
867 __cu_len); \
868 __cu_len; \
869 })
870
871 extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n);
872
873 #define __copy_to_user_inatomic(to, from, n) \
874 ({ \
875 void __user *__cu_to; \
876 const void *__cu_from; \
877 long __cu_len; \
878 \
879 __cu_to = (to); \
880 __cu_from = (from); \
881 __cu_len = (n); \
882 if (eva_kernel_access()) \
883 __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from, \
884 __cu_len); \
885 else \
886 __cu_len = __invoke_copy_to_user(__cu_to, __cu_from, \
887 __cu_len); \
888 __cu_len; \
889 })
890
891 #define __copy_from_user_inatomic(to, from, n) \
892 ({ \
893 void *__cu_to; \
894 const void __user *__cu_from; \
895 long __cu_len; \
896 \
897 __cu_to = (to); \
898 __cu_from = (from); \
899 __cu_len = (n); \
900 if (eva_kernel_access()) \
901 __cu_len = __invoke_copy_from_kernel_inatomic(__cu_to, \
902 __cu_from,\
903 __cu_len);\
904 else \
905 __cu_len = __invoke_copy_from_user_inatomic(__cu_to, \
906 __cu_from, \
907 __cu_len); \
908 __cu_len; \
909 })
910
911 /*
912 * copy_to_user: - Copy a block of data into user space.
913 * @to: Destination address, in user space.
914 * @from: Source address, in kernel space.
915 * @n: Number of bytes to copy.
916 *
917 * Context: User context only. This function may sleep if pagefaults are
918 * enabled.
919 *
920 * Copy data from kernel space to user space.
921 *
922 * Returns number of bytes that could not be copied.
923 * On success, this will be zero.
924 */
925 #define copy_to_user(to, from, n) \
926 ({ \
927 void __user *__cu_to; \
928 const void *__cu_from; \
929 long __cu_len; \
930 \
931 __cu_to = (to); \
932 __cu_from = (from); \
933 __cu_len = (n); \
934 if (eva_kernel_access()) { \
935 __cu_len = __invoke_copy_to_kernel(__cu_to, \
936 __cu_from, \
937 __cu_len); \
938 } else { \
939 if (access_ok(VERIFY_WRITE, __cu_to, __cu_len)) { \
940 might_fault(); \
941 __cu_len = __invoke_copy_to_user(__cu_to, \
942 __cu_from, \
943 __cu_len); \
944 } \
945 } \
946 __cu_len; \
947 })
948
949 #ifndef CONFIG_EVA
950
951 #define __invoke_copy_from_user(to, from, n) \
952 ({ \
953 register void *__cu_to_r __asm__("$4"); \
954 register const void __user *__cu_from_r __asm__("$5"); \
955 register long __cu_len_r __asm__("$6"); \
956 \
957 __cu_to_r = (to); \
958 __cu_from_r = (from); \
959 __cu_len_r = (n); \
960 __asm__ __volatile__( \
961 ".set\tnoreorder\n\t" \
962 __MODULE_JAL(__copy_user) \
963 ".set\tnoat\n\t" \
964 __UA_ADDU "\t$1, %1, %2\n\t" \
965 ".set\tat\n\t" \
966 ".set\treorder" \
967 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \
968 : \
969 : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \
970 DADDI_SCRATCH, "memory"); \
971 __cu_len_r; \
972 })
973
974 #define __invoke_copy_from_kernel(to, from, n) \
975 __invoke_copy_from_user(to, from, n)
976
977 /* For userland <-> userland operations */
978 #define ___invoke_copy_in_user(to, from, n) \
979 __invoke_copy_from_user(to, from, n)
980
981 /* For kernel <-> kernel operations */
982 #define ___invoke_copy_in_kernel(to, from, n) \
983 __invoke_copy_from_user(to, from, n)
984
985 #define __invoke_copy_from_user_inatomic(to, from, n) \
986 ({ \
987 register void *__cu_to_r __asm__("$4"); \
988 register const void __user *__cu_from_r __asm__("$5"); \
989 register long __cu_len_r __asm__("$6"); \
990 \
991 __cu_to_r = (to); \
992 __cu_from_r = (from); \
993 __cu_len_r = (n); \
994 __asm__ __volatile__( \
995 ".set\tnoreorder\n\t" \
996 __MODULE_JAL(__copy_user_inatomic) \
997 ".set\tnoat\n\t" \
998 __UA_ADDU "\t$1, %1, %2\n\t" \
999 ".set\tat\n\t" \
1000 ".set\treorder" \
1001 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \
1002 : \
1003 : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \
1004 DADDI_SCRATCH, "memory"); \
1005 __cu_len_r; \
1006 })
1007
1008 #define __invoke_copy_from_kernel_inatomic(to, from, n) \
1009 __invoke_copy_from_user_inatomic(to, from, n) \
1010
1011 #else
1012
1013 /* EVA specific functions */
1014
1015 extern size_t __copy_user_inatomic_eva(void *__to, const void *__from,
1016 size_t __n);
1017 extern size_t __copy_from_user_eva(void *__to, const void *__from,
1018 size_t __n);
1019 extern size_t __copy_to_user_eva(void *__to, const void *__from,
1020 size_t __n);
1021 extern size_t __copy_in_user_eva(void *__to, const void *__from, size_t __n);
1022
1023 #define __invoke_copy_from_user_eva_generic(to, from, n, func_ptr) \
1024 ({ \
1025 register void *__cu_to_r __asm__("$4"); \
1026 register const void __user *__cu_from_r __asm__("$5"); \
1027 register long __cu_len_r __asm__("$6"); \
1028 \
1029 __cu_to_r = (to); \
1030 __cu_from_r = (from); \
1031 __cu_len_r = (n); \
1032 __asm__ __volatile__( \
1033 ".set\tnoreorder\n\t" \
1034 __MODULE_JAL(func_ptr) \
1035 ".set\tnoat\n\t" \
1036 __UA_ADDU "\t$1, %1, %2\n\t" \
1037 ".set\tat\n\t" \
1038 ".set\treorder" \
1039 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \
1040 : \
1041 : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \
1042 DADDI_SCRATCH, "memory"); \
1043 __cu_len_r; \
1044 })
1045
1046 #define __invoke_copy_to_user_eva_generic(to, from, n, func_ptr) \
1047 ({ \
1048 register void *__cu_to_r __asm__("$4"); \
1049 register const void __user *__cu_from_r __asm__("$5"); \
1050 register long __cu_len_r __asm__("$6"); \
1051 \
1052 __cu_to_r = (to); \
1053 __cu_from_r = (from); \
1054 __cu_len_r = (n); \
1055 __asm__ __volatile__( \
1056 __MODULE_JAL(func_ptr) \
1057 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \
1058 : \
1059 : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \
1060 DADDI_SCRATCH, "memory"); \
1061 __cu_len_r; \
1062 })
1063
1064 /*
1065 * Source or destination address is in userland. We need to go through
1066 * the TLB
1067 */
1068 #define __invoke_copy_from_user(to, from, n) \
1069 __invoke_copy_from_user_eva_generic(to, from, n, __copy_from_user_eva)
1070
1071 #define __invoke_copy_from_user_inatomic(to, from, n) \
1072 __invoke_copy_from_user_eva_generic(to, from, n, \
1073 __copy_user_inatomic_eva)
1074
1075 #define __invoke_copy_to_user(to, from, n) \
1076 __invoke_copy_to_user_eva_generic(to, from, n, __copy_to_user_eva)
1077
1078 #define ___invoke_copy_in_user(to, from, n) \
1079 __invoke_copy_from_user_eva_generic(to, from, n, __copy_in_user_eva)
1080
1081 /*
1082 * Source or destination address in the kernel. We are not going through
1083 * the TLB
1084 */
1085 #define __invoke_copy_from_kernel(to, from, n) \
1086 __invoke_copy_from_user_eva_generic(to, from, n, __copy_user)
1087
1088 #define __invoke_copy_from_kernel_inatomic(to, from, n) \
1089 __invoke_copy_from_user_eva_generic(to, from, n, __copy_user_inatomic)
1090
1091 #define __invoke_copy_to_kernel(to, from, n) \
1092 __invoke_copy_to_user_eva_generic(to, from, n, __copy_user)
1093
1094 #define ___invoke_copy_in_kernel(to, from, n) \
1095 __invoke_copy_from_user_eva_generic(to, from, n, __copy_user)
1096
1097 #endif /* CONFIG_EVA */
1098
1099 /*
1100 * __copy_from_user: - Copy a block of data from user space, with less checking.
1101 * @to: Destination address, in kernel space.
1102 * @from: Source address, in user space.
1103 * @n: Number of bytes to copy.
1104 *
1105 * Context: User context only. This function may sleep if pagefaults are
1106 * enabled.
1107 *
1108 * Copy data from user space to kernel space. Caller must check
1109 * the specified block with access_ok() before calling this function.
1110 *
1111 * Returns number of bytes that could not be copied.
1112 * On success, this will be zero.
1113 *
1114 * If some data could not be copied, this function will pad the copied
1115 * data to the requested size using zero bytes.
1116 */
1117 #define __copy_from_user(to, from, n) \
1118 ({ \
1119 void *__cu_to; \
1120 const void __user *__cu_from; \
1121 long __cu_len; \
1122 \
1123 __cu_to = (to); \
1124 __cu_from = (from); \
1125 __cu_len = (n); \
1126 if (eva_kernel_access()) { \
1127 __cu_len = __invoke_copy_from_kernel(__cu_to, \
1128 __cu_from, \
1129 __cu_len); \
1130 } else { \
1131 might_fault(); \
1132 __cu_len = __invoke_copy_from_user(__cu_to, __cu_from, \
1133 __cu_len); \
1134 } \
1135 __cu_len; \
1136 })
1137
1138 /*
1139 * copy_from_user: - Copy a block of data from user space.
1140 * @to: Destination address, in kernel space.
1141 * @from: Source address, in user space.
1142 * @n: Number of bytes to copy.
1143 *
1144 * Context: User context only. This function may sleep if pagefaults are
1145 * enabled.
1146 *
1147 * Copy data from user space to kernel space.
1148 *
1149 * Returns number of bytes that could not be copied.
1150 * On success, this will be zero.
1151 *
1152 * If some data could not be copied, this function will pad the copied
1153 * data to the requested size using zero bytes.
1154 */
1155 #define copy_from_user(to, from, n) \
1156 ({ \
1157 void *__cu_to; \
1158 const void __user *__cu_from; \
1159 long __cu_len; \
1160 \
1161 __cu_to = (to); \
1162 __cu_from = (from); \
1163 __cu_len = (n); \
1164 if (eva_kernel_access()) { \
1165 __cu_len = __invoke_copy_from_kernel(__cu_to, \
1166 __cu_from, \
1167 __cu_len); \
1168 } else { \
1169 if (access_ok(VERIFY_READ, __cu_from, __cu_len)) { \
1170 might_fault(); \
1171 __cu_len = __invoke_copy_from_user(__cu_to, \
1172 __cu_from, \
1173 __cu_len); \
1174 } else { \
1175 memset(__cu_to, 0, __cu_len); \
1176 } \
1177 } \
1178 __cu_len; \
1179 })
1180
1181 #define __copy_in_user(to, from, n) \
1182 ({ \
1183 void __user *__cu_to; \
1184 const void __user *__cu_from; \
1185 long __cu_len; \
1186 \
1187 __cu_to = (to); \
1188 __cu_from = (from); \
1189 __cu_len = (n); \
1190 if (eva_kernel_access()) { \
1191 __cu_len = ___invoke_copy_in_kernel(__cu_to, __cu_from, \
1192 __cu_len); \
1193 } else { \
1194 might_fault(); \
1195 __cu_len = ___invoke_copy_in_user(__cu_to, __cu_from, \
1196 __cu_len); \
1197 } \
1198 __cu_len; \
1199 })
1200
1201 #define copy_in_user(to, from, n) \
1202 ({ \
1203 void __user *__cu_to; \
1204 const void __user *__cu_from; \
1205 long __cu_len; \
1206 \
1207 __cu_to = (to); \
1208 __cu_from = (from); \
1209 __cu_len = (n); \
1210 if (eva_kernel_access()) { \
1211 __cu_len = ___invoke_copy_in_kernel(__cu_to,__cu_from, \
1212 __cu_len); \
1213 } else { \
1214 if (likely(access_ok(VERIFY_READ, __cu_from, __cu_len) &&\
1215 access_ok(VERIFY_WRITE, __cu_to, __cu_len))) {\
1216 might_fault(); \
1217 __cu_len = ___invoke_copy_in_user(__cu_to, \
1218 __cu_from, \
1219 __cu_len); \
1220 } \
1221 } \
1222 __cu_len; \
1223 })
1224
1225 /*
1226 * __clear_user: - Zero a block of memory in user space, with less checking.
1227 * @to: Destination address, in user space.
1228 * @n: Number of bytes to zero.
1229 *
1230 * Zero a block of memory in user space. Caller must check
1231 * the specified block with access_ok() before calling this function.
1232 *
1233 * Returns number of bytes that could not be cleared.
1234 * On success, this will be zero.
1235 */
1236 static inline __kernel_size_t
__clear_user(void __user * addr,__kernel_size_t size)1237 __clear_user(void __user *addr, __kernel_size_t size)
1238 {
1239 __kernel_size_t res;
1240
1241 #ifdef CONFIG_CPU_MICROMIPS
1242 /* micromips memset / bzero also clobbers t7 & t8 */
1243 #define bzero_clobbers "$4", "$5", "$6", __UA_t0, __UA_t1, "$15", "$24", "$31"
1244 #else
1245 #define bzero_clobbers "$4", "$5", "$6", __UA_t0, __UA_t1, "$31"
1246 #endif /* CONFIG_CPU_MICROMIPS */
1247
1248 if (eva_kernel_access()) {
1249 __asm__ __volatile__(
1250 "move\t$4, %1\n\t"
1251 "move\t$5, $0\n\t"
1252 "move\t$6, %2\n\t"
1253 __MODULE_JAL(__bzero_kernel)
1254 "move\t%0, $6"
1255 : "=r" (res)
1256 : "r" (addr), "r" (size)
1257 : bzero_clobbers);
1258 } else {
1259 might_fault();
1260 __asm__ __volatile__(
1261 "move\t$4, %1\n\t"
1262 "move\t$5, $0\n\t"
1263 "move\t$6, %2\n\t"
1264 __MODULE_JAL(__bzero)
1265 "move\t%0, $6"
1266 : "=r" (res)
1267 : "r" (addr), "r" (size)
1268 : bzero_clobbers);
1269 }
1270
1271 return res;
1272 }
1273
1274 #define clear_user(addr,n) \
1275 ({ \
1276 void __user * __cl_addr = (addr); \
1277 unsigned long __cl_size = (n); \
1278 if (__cl_size && access_ok(VERIFY_WRITE, \
1279 __cl_addr, __cl_size)) \
1280 __cl_size = __clear_user(__cl_addr, __cl_size); \
1281 __cl_size; \
1282 })
1283
1284 /*
1285 * __strncpy_from_user: - Copy a NUL terminated string from userspace, with less checking.
1286 * @dst: Destination address, in kernel space. This buffer must be at
1287 * least @count bytes long.
1288 * @src: Source address, in user space.
1289 * @count: Maximum number of bytes to copy, including the trailing NUL.
1290 *
1291 * Copies a NUL-terminated string from userspace to kernel space.
1292 * Caller must check the specified block with access_ok() before calling
1293 * this function.
1294 *
1295 * On success, returns the length of the string (not including the trailing
1296 * NUL).
1297 *
1298 * If access to userspace fails, returns -EFAULT (some data may have been
1299 * copied).
1300 *
1301 * If @count is smaller than the length of the string, copies @count bytes
1302 * and returns @count.
1303 */
1304 static inline long
__strncpy_from_user(char * __to,const char __user * __from,long __len)1305 __strncpy_from_user(char *__to, const char __user *__from, long __len)
1306 {
1307 long res;
1308
1309 if (eva_kernel_access()) {
1310 __asm__ __volatile__(
1311 "move\t$4, %1\n\t"
1312 "move\t$5, %2\n\t"
1313 "move\t$6, %3\n\t"
1314 __MODULE_JAL(__strncpy_from_kernel_nocheck_asm)
1315 "move\t%0, $2"
1316 : "=r" (res)
1317 : "r" (__to), "r" (__from), "r" (__len)
1318 : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1319 } else {
1320 might_fault();
1321 __asm__ __volatile__(
1322 "move\t$4, %1\n\t"
1323 "move\t$5, %2\n\t"
1324 "move\t$6, %3\n\t"
1325 __MODULE_JAL(__strncpy_from_user_nocheck_asm)
1326 "move\t%0, $2"
1327 : "=r" (res)
1328 : "r" (__to), "r" (__from), "r" (__len)
1329 : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1330 }
1331
1332 return res;
1333 }
1334
1335 /*
1336 * strncpy_from_user: - Copy a NUL terminated string from userspace.
1337 * @dst: Destination address, in kernel space. This buffer must be at
1338 * least @count bytes long.
1339 * @src: Source address, in user space.
1340 * @count: Maximum number of bytes to copy, including the trailing NUL.
1341 *
1342 * Copies a NUL-terminated string from userspace to kernel space.
1343 *
1344 * On success, returns the length of the string (not including the trailing
1345 * NUL).
1346 *
1347 * If access to userspace fails, returns -EFAULT (some data may have been
1348 * copied).
1349 *
1350 * If @count is smaller than the length of the string, copies @count bytes
1351 * and returns @count.
1352 */
1353 static inline long
strncpy_from_user(char * __to,const char __user * __from,long __len)1354 strncpy_from_user(char *__to, const char __user *__from, long __len)
1355 {
1356 long res;
1357
1358 if (eva_kernel_access()) {
1359 __asm__ __volatile__(
1360 "move\t$4, %1\n\t"
1361 "move\t$5, %2\n\t"
1362 "move\t$6, %3\n\t"
1363 __MODULE_JAL(__strncpy_from_kernel_asm)
1364 "move\t%0, $2"
1365 : "=r" (res)
1366 : "r" (__to), "r" (__from), "r" (__len)
1367 : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1368 } else {
1369 might_fault();
1370 __asm__ __volatile__(
1371 "move\t$4, %1\n\t"
1372 "move\t$5, %2\n\t"
1373 "move\t$6, %3\n\t"
1374 __MODULE_JAL(__strncpy_from_user_asm)
1375 "move\t%0, $2"
1376 : "=r" (res)
1377 : "r" (__to), "r" (__from), "r" (__len)
1378 : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1379 }
1380
1381 return res;
1382 }
1383
1384 /*
1385 * strlen_user: - Get the size of a string in user space.
1386 * @str: The string to measure.
1387 *
1388 * Context: User context only. This function may sleep if pagefaults are
1389 * enabled.
1390 *
1391 * Get the size of a NUL-terminated string in user space.
1392 *
1393 * Returns the size of the string INCLUDING the terminating NUL.
1394 * On exception, returns 0.
1395 *
1396 * If there is a limit on the length of a valid string, you may wish to
1397 * consider using strnlen_user() instead.
1398 */
strlen_user(const char __user * s)1399 static inline long strlen_user(const char __user *s)
1400 {
1401 long res;
1402
1403 if (eva_kernel_access()) {
1404 __asm__ __volatile__(
1405 "move\t$4, %1\n\t"
1406 __MODULE_JAL(__strlen_kernel_asm)
1407 "move\t%0, $2"
1408 : "=r" (res)
1409 : "r" (s)
1410 : "$2", "$4", __UA_t0, "$31");
1411 } else {
1412 might_fault();
1413 __asm__ __volatile__(
1414 "move\t$4, %1\n\t"
1415 __MODULE_JAL(__strlen_user_asm)
1416 "move\t%0, $2"
1417 : "=r" (res)
1418 : "r" (s)
1419 : "$2", "$4", __UA_t0, "$31");
1420 }
1421
1422 return res;
1423 }
1424
1425 /* Returns: 0 if bad, string length+1 (memory size) of string if ok */
__strnlen_user(const char __user * s,long n)1426 static inline long __strnlen_user(const char __user *s, long n)
1427 {
1428 long res;
1429
1430 if (eva_kernel_access()) {
1431 __asm__ __volatile__(
1432 "move\t$4, %1\n\t"
1433 "move\t$5, %2\n\t"
1434 __MODULE_JAL(__strnlen_kernel_nocheck_asm)
1435 "move\t%0, $2"
1436 : "=r" (res)
1437 : "r" (s), "r" (n)
1438 : "$2", "$4", "$5", __UA_t0, "$31");
1439 } else {
1440 might_fault();
1441 __asm__ __volatile__(
1442 "move\t$4, %1\n\t"
1443 "move\t$5, %2\n\t"
1444 __MODULE_JAL(__strnlen_user_nocheck_asm)
1445 "move\t%0, $2"
1446 : "=r" (res)
1447 : "r" (s), "r" (n)
1448 : "$2", "$4", "$5", __UA_t0, "$31");
1449 }
1450
1451 return res;
1452 }
1453
1454 /*
1455 * strnlen_user: - Get the size of a string in user space.
1456 * @str: The string to measure.
1457 *
1458 * Context: User context only. This function may sleep if pagefaults are
1459 * enabled.
1460 *
1461 * Get the size of a NUL-terminated string in user space.
1462 *
1463 * Returns the size of the string INCLUDING the terminating NUL.
1464 * On exception, returns 0.
1465 * If the string is too long, returns a value greater than @n.
1466 */
strnlen_user(const char __user * s,long n)1467 static inline long strnlen_user(const char __user *s, long n)
1468 {
1469 long res;
1470
1471 might_fault();
1472 if (eva_kernel_access()) {
1473 __asm__ __volatile__(
1474 "move\t$4, %1\n\t"
1475 "move\t$5, %2\n\t"
1476 __MODULE_JAL(__strnlen_kernel_asm)
1477 "move\t%0, $2"
1478 : "=r" (res)
1479 : "r" (s), "r" (n)
1480 : "$2", "$4", "$5", __UA_t0, "$31");
1481 } else {
1482 __asm__ __volatile__(
1483 "move\t$4, %1\n\t"
1484 "move\t$5, %2\n\t"
1485 __MODULE_JAL(__strnlen_user_asm)
1486 "move\t%0, $2"
1487 : "=r" (res)
1488 : "r" (s), "r" (n)
1489 : "$2", "$4", "$5", __UA_t0, "$31");
1490 }
1491
1492 return res;
1493 }
1494
1495 struct exception_table_entry
1496 {
1497 unsigned long insn;
1498 unsigned long nextinsn;
1499 };
1500
1501 extern int fixup_exception(struct pt_regs *regs);
1502
1503 #endif /* _ASM_UACCESS_H */
1504