1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_SPECIAL_INSNS_H
3 #define _ASM_X86_SPECIAL_INSNS_H
4
5
6 #ifdef __KERNEL__
7
8 #include <asm/nops.h>
9 #include <asm/processor-flags.h>
10 #include <linux/jump_label.h>
11
12 /*
13 * The compiler should not reorder volatile asm statements with respect to each
14 * other: they should execute in program order. However GCC 4.9.x and 5.x have
15 * a bug (which was fixed in 8.1, 7.3 and 6.5) where they might reorder
16 * volatile asm. The write functions are not affected since they have memory
17 * clobbers preventing reordering. To prevent reads from being reordered with
18 * respect to writes, use a dummy memory operand.
19 */
20
21 #define __FORCE_ORDER "m"(*(unsigned int *)0x1000UL)
22
23 void native_write_cr0(unsigned long val);
24
native_read_cr0(void)25 static inline unsigned long native_read_cr0(void)
26 {
27 unsigned long val;
28 asm volatile("mov %%cr0,%0\n\t" : "=r" (val) : __FORCE_ORDER);
29 return val;
30 }
31
native_read_cr2(void)32 static inline unsigned long native_read_cr2(void)
33 {
34 unsigned long val;
35 asm volatile("mov %%cr2,%0\n\t" : "=r" (val) : __FORCE_ORDER);
36 return val;
37 }
38
native_write_cr2(unsigned long val)39 static inline void native_write_cr2(unsigned long val)
40 {
41 asm volatile("mov %0,%%cr2": : "r" (val) : "memory");
42 }
43
__native_read_cr3(void)44 static inline unsigned long __native_read_cr3(void)
45 {
46 unsigned long val;
47 asm volatile("mov %%cr3,%0\n\t" : "=r" (val) : __FORCE_ORDER);
48 return val;
49 }
50
native_write_cr3(unsigned long val)51 static inline void native_write_cr3(unsigned long val)
52 {
53 asm volatile("mov %0,%%cr3": : "r" (val) : "memory");
54 }
55
native_read_cr4(void)56 static inline unsigned long native_read_cr4(void)
57 {
58 unsigned long val;
59 #ifdef CONFIG_X86_32
60 /*
61 * This could fault if CR4 does not exist. Non-existent CR4
62 * is functionally equivalent to CR4 == 0. Keep it simple and pretend
63 * that CR4 == 0 on CPUs that don't have CR4.
64 */
65 asm volatile("1: mov %%cr4, %0\n"
66 "2:\n"
67 _ASM_EXTABLE(1b, 2b)
68 : "=r" (val) : "0" (0), __FORCE_ORDER);
69 #else
70 /* CR4 always exists on x86_64. */
71 asm volatile("mov %%cr4,%0\n\t" : "=r" (val) : __FORCE_ORDER);
72 #endif
73 return val;
74 }
75
76 void native_write_cr4(unsigned long val);
77
78 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
rdpkru(void)79 static inline u32 rdpkru(void)
80 {
81 u32 ecx = 0;
82 u32 edx, pkru;
83
84 /*
85 * "rdpkru" instruction. Places PKRU contents in to EAX,
86 * clears EDX and requires that ecx=0.
87 */
88 asm volatile(".byte 0x0f,0x01,0xee\n\t"
89 : "=a" (pkru), "=d" (edx)
90 : "c" (ecx));
91 return pkru;
92 }
93
wrpkru(u32 pkru)94 static inline void wrpkru(u32 pkru)
95 {
96 u32 ecx = 0, edx = 0;
97
98 /*
99 * "wrpkru" instruction. Loads contents in EAX to PKRU,
100 * requires that ecx = edx = 0.
101 */
102 asm volatile(".byte 0x0f,0x01,0xef\n\t"
103 : : "a" (pkru), "c"(ecx), "d"(edx));
104 }
105
__write_pkru(u32 pkru)106 static inline void __write_pkru(u32 pkru)
107 {
108 /*
109 * WRPKRU is relatively expensive compared to RDPKRU.
110 * Avoid WRPKRU when it would not change the value.
111 */
112 if (pkru == rdpkru())
113 return;
114
115 wrpkru(pkru);
116 }
117
118 #else
rdpkru(void)119 static inline u32 rdpkru(void)
120 {
121 return 0;
122 }
123
__write_pkru(u32 pkru)124 static inline void __write_pkru(u32 pkru)
125 {
126 }
127 #endif
128
native_wbinvd(void)129 static inline void native_wbinvd(void)
130 {
131 asm volatile("wbinvd": : :"memory");
132 }
133
134 extern asmlinkage void native_load_gs_index(unsigned);
135
__read_cr4(void)136 static inline unsigned long __read_cr4(void)
137 {
138 return native_read_cr4();
139 }
140
141 #ifdef CONFIG_PARAVIRT_XXL
142 #include <asm/paravirt.h>
143 #else
144
read_cr0(void)145 static inline unsigned long read_cr0(void)
146 {
147 return native_read_cr0();
148 }
149
write_cr0(unsigned long x)150 static inline void write_cr0(unsigned long x)
151 {
152 native_write_cr0(x);
153 }
154
read_cr2(void)155 static inline unsigned long read_cr2(void)
156 {
157 return native_read_cr2();
158 }
159
write_cr2(unsigned long x)160 static inline void write_cr2(unsigned long x)
161 {
162 native_write_cr2(x);
163 }
164
165 /*
166 * Careful! CR3 contains more than just an address. You probably want
167 * read_cr3_pa() instead.
168 */
__read_cr3(void)169 static inline unsigned long __read_cr3(void)
170 {
171 return __native_read_cr3();
172 }
173
write_cr3(unsigned long x)174 static inline void write_cr3(unsigned long x)
175 {
176 native_write_cr3(x);
177 }
178
__write_cr4(unsigned long x)179 static inline void __write_cr4(unsigned long x)
180 {
181 native_write_cr4(x);
182 }
183
wbinvd(void)184 static inline void wbinvd(void)
185 {
186 native_wbinvd();
187 }
188
189 #ifdef CONFIG_X86_64
190
load_gs_index(unsigned selector)191 static inline void load_gs_index(unsigned selector)
192 {
193 native_load_gs_index(selector);
194 }
195
196 #endif
197
198 #endif /* CONFIG_PARAVIRT_XXL */
199
clflush(volatile void * __p)200 static inline void clflush(volatile void *__p)
201 {
202 asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p));
203 }
204
clflushopt(volatile void * __p)205 static inline void clflushopt(volatile void *__p)
206 {
207 alternative_io(".byte " __stringify(NOP_DS_PREFIX) "; clflush %P0",
208 ".byte 0x66; clflush %P0",
209 X86_FEATURE_CLFLUSHOPT,
210 "+m" (*(volatile char __force *)__p));
211 }
212
clwb(volatile void * __p)213 static inline void clwb(volatile void *__p)
214 {
215 volatile struct { char x[64]; } *p = __p;
216
217 asm volatile(ALTERNATIVE_2(
218 ".byte " __stringify(NOP_DS_PREFIX) "; clflush (%[pax])",
219 ".byte 0x66; clflush (%[pax])", /* clflushopt (%%rax) */
220 X86_FEATURE_CLFLUSHOPT,
221 ".byte 0x66, 0x0f, 0xae, 0x30", /* clwb (%%rax) */
222 X86_FEATURE_CLWB)
223 : [p] "+m" (*p)
224 : [pax] "a" (p));
225 }
226
227 #define nop() asm volatile ("nop")
228
229
230 #endif /* __KERNEL__ */
231
232 #endif /* _ASM_X86_SPECIAL_INSNS_H */
233