1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_STRING_64_H
3 #define _ASM_X86_STRING_64_H
4
5 #ifdef __KERNEL__
6 #include <linux/jump_label.h>
7
8 /* Written 2002 by Andi Kleen */
9
10 /* Only used for special circumstances. Stolen from i386/string.h */
__inline_memcpy(void * to,const void * from,size_t n)11 static __always_inline void *__inline_memcpy(void *to, const void *from, size_t n)
12 {
13 unsigned long d0, d1, d2;
14 asm volatile("rep ; movsl\n\t"
15 "testb $2,%b4\n\t"
16 "je 1f\n\t"
17 "movsw\n"
18 "1:\ttestb $1,%b4\n\t"
19 "je 2f\n\t"
20 "movsb\n"
21 "2:"
22 : "=&c" (d0), "=&D" (d1), "=&S" (d2)
23 : "0" (n / 4), "q" (n), "1" ((long)to), "2" ((long)from)
24 : "memory");
25 return to;
26 }
27
28 /* Even with __builtin_ the compiler may decide to use the out of line
29 function. */
30
31 #define __HAVE_ARCH_MEMCPY 1
32 extern void *memcpy(void *to, const void *from, size_t len);
33 extern void *__memcpy(void *to, const void *from, size_t len);
34
35 #ifndef CONFIG_FORTIFY_SOURCE
36 #if (__GNUC__ == 4 && __GNUC_MINOR__ < 3) || __GNUC__ < 4
37 #define memcpy(dst, src, len) \
38 ({ \
39 size_t __len = (len); \
40 void *__ret; \
41 if (__builtin_constant_p(len) && __len >= 64) \
42 __ret = __memcpy((dst), (src), __len); \
43 else \
44 __ret = __builtin_memcpy((dst), (src), __len); \
45 __ret; \
46 })
47 #endif
48 #endif /* !CONFIG_FORTIFY_SOURCE */
49
50 #define __HAVE_ARCH_MEMSET
51 void *memset(void *s, int c, size_t n);
52 void *__memset(void *s, int c, size_t n);
53
54 #define __HAVE_ARCH_MEMSET16
memset16(uint16_t * s,uint16_t v,size_t n)55 static inline void *memset16(uint16_t *s, uint16_t v, size_t n)
56 {
57 long d0, d1;
58 asm volatile("rep\n\t"
59 "stosw"
60 : "=&c" (d0), "=&D" (d1)
61 : "a" (v), "1" (s), "0" (n)
62 : "memory");
63 return s;
64 }
65
66 #define __HAVE_ARCH_MEMSET32
memset32(uint32_t * s,uint32_t v,size_t n)67 static inline void *memset32(uint32_t *s, uint32_t v, size_t n)
68 {
69 long d0, d1;
70 asm volatile("rep\n\t"
71 "stosl"
72 : "=&c" (d0), "=&D" (d1)
73 : "a" (v), "1" (s), "0" (n)
74 : "memory");
75 return s;
76 }
77
78 #define __HAVE_ARCH_MEMSET64
memset64(uint64_t * s,uint64_t v,size_t n)79 static inline void *memset64(uint64_t *s, uint64_t v, size_t n)
80 {
81 long d0, d1;
82 asm volatile("rep\n\t"
83 "stosq"
84 : "=&c" (d0), "=&D" (d1)
85 : "a" (v), "1" (s), "0" (n)
86 : "memory");
87 return s;
88 }
89
90 #define __HAVE_ARCH_MEMMOVE
91 void *memmove(void *dest, const void *src, size_t count);
92 void *__memmove(void *dest, const void *src, size_t count);
93
94 int memcmp(const void *cs, const void *ct, size_t count);
95 size_t strlen(const char *s);
96 char *strcpy(char *dest, const char *src);
97 char *strcat(char *dest, const char *src);
98 int strcmp(const char *cs, const char *ct);
99
100 #if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__)
101
102 /*
103 * For files that not instrumented (e.g. mm/slub.c) we
104 * should use not instrumented version of mem* functions.
105 */
106
107 #undef memcpy
108 #define memcpy(dst, src, len) __memcpy(dst, src, len)
109 #define memmove(dst, src, len) __memmove(dst, src, len)
110 #define memset(s, c, n) __memset(s, c, n)
111
112 #ifndef __NO_FORTIFY
113 #define __NO_FORTIFY /* FORTIFY_SOURCE uses __builtin_memcpy, etc. */
114 #endif
115
116 #endif
117
118 #define __HAVE_ARCH_MEMCPY_MCSAFE 1
119 __must_check int memcpy_mcsafe_unrolled(void *dst, const void *src, size_t cnt);
120 DECLARE_STATIC_KEY_FALSE(mcsafe_key);
121
122 /**
123 * memcpy_mcsafe - copy memory with indication if a machine check happened
124 *
125 * @dst: destination address
126 * @src: source address
127 * @cnt: number of bytes to copy
128 *
129 * Low level memory copy function that catches machine checks
130 * We only call into the "safe" function on systems that can
131 * actually do machine check recovery. Everyone else can just
132 * use memcpy().
133 *
134 * Return 0 for success, -EFAULT for fail
135 */
136 static __always_inline __must_check int
memcpy_mcsafe(void * dst,const void * src,size_t cnt)137 memcpy_mcsafe(void *dst, const void *src, size_t cnt)
138 {
139 #ifdef CONFIG_X86_MCE
140 if (static_branch_unlikely(&mcsafe_key))
141 return memcpy_mcsafe_unrolled(dst, src, cnt);
142 else
143 #endif
144 memcpy(dst, src, cnt);
145 return 0;
146 }
147
148 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
149 #define __HAVE_ARCH_MEMCPY_FLUSHCACHE 1
150 void memcpy_flushcache(void *dst, const void *src, size_t cnt);
151 #endif
152
153 #endif /* __KERNEL__ */
154
155 #endif /* _ASM_X86_STRING_64_H */
156