1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (C) 1994 Linus Torvalds
4 *
5 * Pentium III FXSR, SSE support
6 * General FPU state handling cleanups
7 * Gareth Hughes <gareth@valinux.com>, May 2000
8 * x86-64 work by Andi Kleen 2002
9 */
10
11 #ifndef _ASM_X86_FPU_INTERNAL_H
12 #define _ASM_X86_FPU_INTERNAL_H
13
14 #include <linux/compat.h>
15 #include <linux/sched.h>
16 #include <linux/slab.h>
17 #include <linux/mm.h>
18
19 #include <asm/user.h>
20 #include <asm/fpu/api.h>
21 #include <asm/fpu/xstate.h>
22 #include <asm/fpu/xcr.h>
23 #include <asm/cpufeature.h>
24 #include <asm/trace/fpu.h>
25
26 /*
27 * High level FPU state handling functions:
28 */
29 extern int fpu__restore_sig(void __user *buf, int ia32_frame);
30 extern void fpu__drop(struct fpu *fpu);
31 extern void fpu__clear_user_states(struct fpu *fpu);
32 extern int fpu__exception_code(struct fpu *fpu, int trap_nr);
33
34 extern void fpu_sync_fpstate(struct fpu *fpu);
35
36 /* Clone and exit operations */
37 extern int fpu_clone(struct task_struct *dst);
38 extern void fpu_flush_thread(void);
39
40 /*
41 * Boot time FPU initialization functions:
42 */
43 extern void fpu__init_cpu(void);
44 extern void fpu__init_system_xstate(void);
45 extern void fpu__init_cpu_xstate(void);
46 extern void fpu__init_system(void);
47 extern void fpu__init_check_bugs(void);
48 extern void fpu__resume_cpu(void);
49
50 /*
51 * Debugging facility:
52 */
53 #ifdef CONFIG_X86_DEBUG_FPU
54 # define WARN_ON_FPU(x) WARN_ON_ONCE(x)
55 #else
56 # define WARN_ON_FPU(x) ({ (void)(x); 0; })
57 #endif
58
59 /*
60 * FPU related CPU feature flag helper routines:
61 */
use_xsaveopt(void)62 static __always_inline __pure bool use_xsaveopt(void)
63 {
64 return static_cpu_has(X86_FEATURE_XSAVEOPT);
65 }
66
use_xsave(void)67 static __always_inline __pure bool use_xsave(void)
68 {
69 return static_cpu_has(X86_FEATURE_XSAVE);
70 }
71
use_fxsr(void)72 static __always_inline __pure bool use_fxsr(void)
73 {
74 return static_cpu_has(X86_FEATURE_FXSR);
75 }
76
77 /*
78 * fpstate handling functions:
79 */
80
81 extern union fpregs_state init_fpstate;
82
83 extern void fpstate_init(union fpregs_state *state);
84 #ifdef CONFIG_MATH_EMULATION
85 extern void fpstate_init_soft(struct swregs_state *soft);
86 #else
fpstate_init_soft(struct swregs_state * soft)87 static inline void fpstate_init_soft(struct swregs_state *soft) {}
88 #endif
89 extern void save_fpregs_to_fpstate(struct fpu *fpu);
90
91 /* Returns 0 or the negated trap number, which results in -EFAULT for #PF */
92 #define user_insn(insn, output, input...) \
93 ({ \
94 int err; \
95 \
96 might_fault(); \
97 \
98 asm volatile(ASM_STAC "\n" \
99 "1: " #insn "\n" \
100 "2: " ASM_CLAC "\n" \
101 ".section .fixup,\"ax\"\n" \
102 "3: negl %%eax\n" \
103 " jmp 2b\n" \
104 ".previous\n" \
105 _ASM_EXTABLE_FAULT(1b, 3b) \
106 : [err] "=a" (err), output \
107 : "0"(0), input); \
108 err; \
109 })
110
111 #define kernel_insn_err(insn, output, input...) \
112 ({ \
113 int err; \
114 asm volatile("1:" #insn "\n\t" \
115 "2:\n" \
116 ".section .fixup,\"ax\"\n" \
117 "3: movl $-1,%[err]\n" \
118 " jmp 2b\n" \
119 ".previous\n" \
120 _ASM_EXTABLE(1b, 3b) \
121 : [err] "=r" (err), output \
122 : "0"(0), input); \
123 err; \
124 })
125
126 #define kernel_insn(insn, output, input...) \
127 asm volatile("1:" #insn "\n\t" \
128 "2:\n" \
129 _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_FPU_RESTORE) \
130 : output : input)
131
fnsave_to_user_sigframe(struct fregs_state __user * fx)132 static inline int fnsave_to_user_sigframe(struct fregs_state __user *fx)
133 {
134 return user_insn(fnsave %[fx]; fwait, [fx] "=m" (*fx), "m" (*fx));
135 }
136
fxsave_to_user_sigframe(struct fxregs_state __user * fx)137 static inline int fxsave_to_user_sigframe(struct fxregs_state __user *fx)
138 {
139 if (IS_ENABLED(CONFIG_X86_32))
140 return user_insn(fxsave %[fx], [fx] "=m" (*fx), "m" (*fx));
141 else
142 return user_insn(fxsaveq %[fx], [fx] "=m" (*fx), "m" (*fx));
143
144 }
145
fxrstor(struct fxregs_state * fx)146 static inline void fxrstor(struct fxregs_state *fx)
147 {
148 if (IS_ENABLED(CONFIG_X86_32))
149 kernel_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx));
150 else
151 kernel_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx));
152 }
153
fxrstor_safe(struct fxregs_state * fx)154 static inline int fxrstor_safe(struct fxregs_state *fx)
155 {
156 if (IS_ENABLED(CONFIG_X86_32))
157 return kernel_insn_err(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx));
158 else
159 return kernel_insn_err(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx));
160 }
161
fxrstor_from_user_sigframe(struct fxregs_state __user * fx)162 static inline int fxrstor_from_user_sigframe(struct fxregs_state __user *fx)
163 {
164 if (IS_ENABLED(CONFIG_X86_32))
165 return user_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx));
166 else
167 return user_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx));
168 }
169
frstor(struct fregs_state * fx)170 static inline void frstor(struct fregs_state *fx)
171 {
172 kernel_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx));
173 }
174
frstor_safe(struct fregs_state * fx)175 static inline int frstor_safe(struct fregs_state *fx)
176 {
177 return kernel_insn_err(frstor %[fx], "=m" (*fx), [fx] "m" (*fx));
178 }
179
frstor_from_user_sigframe(struct fregs_state __user * fx)180 static inline int frstor_from_user_sigframe(struct fregs_state __user *fx)
181 {
182 return user_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx));
183 }
184
fxsave(struct fxregs_state * fx)185 static inline void fxsave(struct fxregs_state *fx)
186 {
187 if (IS_ENABLED(CONFIG_X86_32))
188 asm volatile( "fxsave %[fx]" : [fx] "=m" (*fx));
189 else
190 asm volatile("fxsaveq %[fx]" : [fx] "=m" (*fx));
191 }
192
193 /* These macros all use (%edi)/(%rdi) as the single memory argument. */
194 #define XSAVE ".byte " REX_PREFIX "0x0f,0xae,0x27"
195 #define XSAVEOPT ".byte " REX_PREFIX "0x0f,0xae,0x37"
196 #define XSAVES ".byte " REX_PREFIX "0x0f,0xc7,0x2f"
197 #define XRSTOR ".byte " REX_PREFIX "0x0f,0xae,0x2f"
198 #define XRSTORS ".byte " REX_PREFIX "0x0f,0xc7,0x1f"
199
200 /*
201 * After this @err contains 0 on success or the negated trap number when
202 * the operation raises an exception. For faults this results in -EFAULT.
203 */
204 #define XSTATE_OP(op, st, lmask, hmask, err) \
205 asm volatile("1:" op "\n\t" \
206 "xor %[err], %[err]\n" \
207 "2:\n\t" \
208 ".pushsection .fixup,\"ax\"\n\t" \
209 "3: negl %%eax\n\t" \
210 "jmp 2b\n\t" \
211 ".popsection\n\t" \
212 _ASM_EXTABLE_FAULT(1b, 3b) \
213 : [err] "=a" (err) \
214 : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \
215 : "memory")
216
217 /*
218 * If XSAVES is enabled, it replaces XSAVEOPT because it supports a compact
219 * format and supervisor states in addition to modified optimization in
220 * XSAVEOPT.
221 *
222 * Otherwise, if XSAVEOPT is enabled, XSAVEOPT replaces XSAVE because XSAVEOPT
223 * supports modified optimization which is not supported by XSAVE.
224 *
225 * We use XSAVE as a fallback.
226 *
227 * The 661 label is defined in the ALTERNATIVE* macros as the address of the
228 * original instruction which gets replaced. We need to use it here as the
229 * address of the instruction where we might get an exception at.
230 */
231 #define XSTATE_XSAVE(st, lmask, hmask, err) \
232 asm volatile(ALTERNATIVE_2(XSAVE, \
233 XSAVEOPT, X86_FEATURE_XSAVEOPT, \
234 XSAVES, X86_FEATURE_XSAVES) \
235 "\n" \
236 "xor %[err], %[err]\n" \
237 "3:\n" \
238 ".pushsection .fixup,\"ax\"\n" \
239 "4: movl $-2, %[err]\n" \
240 "jmp 3b\n" \
241 ".popsection\n" \
242 _ASM_EXTABLE(661b, 4b) \
243 : [err] "=r" (err) \
244 : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \
245 : "memory")
246
247 /*
248 * Use XRSTORS to restore context if it is enabled. XRSTORS supports compact
249 * XSAVE area format.
250 */
251 #define XSTATE_XRESTORE(st, lmask, hmask) \
252 asm volatile(ALTERNATIVE(XRSTOR, \
253 XRSTORS, X86_FEATURE_XSAVES) \
254 "\n" \
255 "3:\n" \
256 _ASM_EXTABLE_TYPE(661b, 3b, EX_TYPE_FPU_RESTORE) \
257 : \
258 : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \
259 : "memory")
260
261 /*
262 * This function is called only during boot time when x86 caps are not set
263 * up and alternative can not be used yet.
264 */
os_xrstor_booting(struct xregs_state * xstate)265 static inline void os_xrstor_booting(struct xregs_state *xstate)
266 {
267 u64 mask = xfeatures_mask_fpstate();
268 u32 lmask = mask;
269 u32 hmask = mask >> 32;
270 int err;
271
272 WARN_ON(system_state != SYSTEM_BOOTING);
273
274 if (boot_cpu_has(X86_FEATURE_XSAVES))
275 XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);
276 else
277 XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);
278
279 /*
280 * We should never fault when copying from a kernel buffer, and the FPU
281 * state we set at boot time should be valid.
282 */
283 WARN_ON_FPU(err);
284 }
285
286 /*
287 * Save processor xstate to xsave area.
288 *
289 * Uses either XSAVE or XSAVEOPT or XSAVES depending on the CPU features
290 * and command line options. The choice is permanent until the next reboot.
291 */
os_xsave(struct xregs_state * xstate)292 static inline void os_xsave(struct xregs_state *xstate)
293 {
294 u64 mask = xfeatures_mask_all;
295 u32 lmask = mask;
296 u32 hmask = mask >> 32;
297 int err;
298
299 WARN_ON_FPU(!alternatives_patched);
300
301 XSTATE_XSAVE(xstate, lmask, hmask, err);
302
303 /* We should never fault when copying to a kernel buffer: */
304 WARN_ON_FPU(err);
305 }
306
307 /*
308 * Restore processor xstate from xsave area.
309 *
310 * Uses XRSTORS when XSAVES is used, XRSTOR otherwise.
311 */
os_xrstor(struct xregs_state * xstate,u64 mask)312 static inline void os_xrstor(struct xregs_state *xstate, u64 mask)
313 {
314 u32 lmask = mask;
315 u32 hmask = mask >> 32;
316
317 XSTATE_XRESTORE(xstate, lmask, hmask);
318 }
319
320 /*
321 * Save xstate to user space xsave area.
322 *
323 * We don't use modified optimization because xrstor/xrstors might track
324 * a different application.
325 *
326 * We don't use compacted format xsave area for
327 * backward compatibility for old applications which don't understand
328 * compacted format of xsave area.
329 */
xsave_to_user_sigframe(struct xregs_state __user * buf)330 static inline int xsave_to_user_sigframe(struct xregs_state __user *buf)
331 {
332 /*
333 * Include the features which are not xsaved/rstored by the kernel
334 * internally, e.g. PKRU. That's user space ABI and also required
335 * to allow the signal handler to modify PKRU.
336 */
337 u64 mask = xfeatures_mask_uabi();
338 u32 lmask = mask;
339 u32 hmask = mask >> 32;
340 int err;
341
342 /*
343 * Clear the xsave header first, so that reserved fields are
344 * initialized to zero.
345 */
346 err = __clear_user(&buf->header, sizeof(buf->header));
347 if (unlikely(err))
348 return -EFAULT;
349
350 stac();
351 XSTATE_OP(XSAVE, buf, lmask, hmask, err);
352 clac();
353
354 return err;
355 }
356
357 /*
358 * Restore xstate from user space xsave area.
359 */
xrstor_from_user_sigframe(struct xregs_state __user * buf,u64 mask)360 static inline int xrstor_from_user_sigframe(struct xregs_state __user *buf, u64 mask)
361 {
362 struct xregs_state *xstate = ((__force struct xregs_state *)buf);
363 u32 lmask = mask;
364 u32 hmask = mask >> 32;
365 int err;
366
367 stac();
368 XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);
369 clac();
370
371 return err;
372 }
373
374 /*
375 * Restore xstate from kernel space xsave area, return an error code instead of
376 * an exception.
377 */
os_xrstor_safe(struct xregs_state * xstate,u64 mask)378 static inline int os_xrstor_safe(struct xregs_state *xstate, u64 mask)
379 {
380 u32 lmask = mask;
381 u32 hmask = mask >> 32;
382 int err;
383
384 if (cpu_feature_enabled(X86_FEATURE_XSAVES))
385 XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);
386 else
387 XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);
388
389 return err;
390 }
391
392 extern void __restore_fpregs_from_fpstate(union fpregs_state *fpstate, u64 mask);
393
restore_fpregs_from_fpstate(union fpregs_state * fpstate)394 static inline void restore_fpregs_from_fpstate(union fpregs_state *fpstate)
395 {
396 __restore_fpregs_from_fpstate(fpstate, xfeatures_mask_fpstate());
397 }
398
399 extern int copy_fpstate_to_sigframe(void __user *buf, void __user *fp, int size);
400
401 /*
402 * FPU context switch related helper methods:
403 */
404
405 DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
406
407 /*
408 * The in-register FPU state for an FPU context on a CPU is assumed to be
409 * valid if the fpu->last_cpu matches the CPU, and the fpu_fpregs_owner_ctx
410 * matches the FPU.
411 *
412 * If the FPU register state is valid, the kernel can skip restoring the
413 * FPU state from memory.
414 *
415 * Any code that clobbers the FPU registers or updates the in-memory
416 * FPU state for a task MUST let the rest of the kernel know that the
417 * FPU registers are no longer valid for this task.
418 *
419 * Invalidate a resource you control: CPU if using the CPU for something else
420 * (with preemption disabled), FPU for the current task, or a task that
421 * is prevented from running by the current task.
422 */
__cpu_invalidate_fpregs_state(void)423 static inline void __cpu_invalidate_fpregs_state(void)
424 {
425 __this_cpu_write(fpu_fpregs_owner_ctx, NULL);
426 }
427
__fpu_invalidate_fpregs_state(struct fpu * fpu)428 static inline void __fpu_invalidate_fpregs_state(struct fpu *fpu)
429 {
430 fpu->last_cpu = -1;
431 }
432
fpregs_state_valid(struct fpu * fpu,unsigned int cpu)433 static inline int fpregs_state_valid(struct fpu *fpu, unsigned int cpu)
434 {
435 return fpu == this_cpu_read(fpu_fpregs_owner_ctx) && cpu == fpu->last_cpu;
436 }
437
438 /*
439 * These generally need preemption protection to work,
440 * do try to avoid using these on their own:
441 */
fpregs_deactivate(struct fpu * fpu)442 static inline void fpregs_deactivate(struct fpu *fpu)
443 {
444 this_cpu_write(fpu_fpregs_owner_ctx, NULL);
445 trace_x86_fpu_regs_deactivated(fpu);
446 }
447
fpregs_activate(struct fpu * fpu)448 static inline void fpregs_activate(struct fpu *fpu)
449 {
450 this_cpu_write(fpu_fpregs_owner_ctx, fpu);
451 trace_x86_fpu_regs_activated(fpu);
452 }
453
454 /* Internal helper for switch_fpu_return() and signal frame setup */
fpregs_restore_userregs(void)455 static inline void fpregs_restore_userregs(void)
456 {
457 struct fpu *fpu = ¤t->thread.fpu;
458 int cpu = smp_processor_id();
459
460 if (WARN_ON_ONCE(current->flags & PF_KTHREAD))
461 return;
462
463 if (!fpregs_state_valid(fpu, cpu)) {
464 u64 mask;
465
466 /*
467 * This restores _all_ xstate which has not been
468 * established yet.
469 *
470 * If PKRU is enabled, then the PKRU value is already
471 * correct because it was either set in switch_to() or in
472 * flush_thread(). So it is excluded because it might be
473 * not up to date in current->thread.fpu.xsave state.
474 */
475 mask = xfeatures_mask_restore_user() |
476 xfeatures_mask_supervisor();
477 __restore_fpregs_from_fpstate(&fpu->state, mask);
478
479 fpregs_activate(fpu);
480 fpu->last_cpu = cpu;
481 }
482 clear_thread_flag(TIF_NEED_FPU_LOAD);
483 }
484
485 /*
486 * FPU state switching for scheduling.
487 *
488 * This is a two-stage process:
489 *
490 * - switch_fpu_prepare() saves the old state.
491 * This is done within the context of the old process.
492 *
493 * - switch_fpu_finish() sets TIF_NEED_FPU_LOAD; the floating point state
494 * will get loaded on return to userspace, or when the kernel needs it.
495 *
496 * If TIF_NEED_FPU_LOAD is cleared then the CPU's FPU registers
497 * are saved in the current thread's FPU register state.
498 *
499 * If TIF_NEED_FPU_LOAD is set then CPU's FPU registers may not
500 * hold current()'s FPU registers. It is required to load the
501 * registers before returning to userland or using the content
502 * otherwise.
503 *
504 * The FPU context is only stored/restored for a user task and
505 * PF_KTHREAD is used to distinguish between kernel and user threads.
506 */
switch_fpu_prepare(struct fpu * old_fpu,int cpu)507 static inline void switch_fpu_prepare(struct fpu *old_fpu, int cpu)
508 {
509 if (static_cpu_has(X86_FEATURE_FPU) && !(current->flags & PF_KTHREAD)) {
510 save_fpregs_to_fpstate(old_fpu);
511 /*
512 * The save operation preserved register state, so the
513 * fpu_fpregs_owner_ctx is still @old_fpu. Store the
514 * current CPU number in @old_fpu, so the next return
515 * to user space can avoid the FPU register restore
516 * when is returns on the same CPU and still owns the
517 * context.
518 */
519 old_fpu->last_cpu = cpu;
520
521 trace_x86_fpu_regs_deactivated(old_fpu);
522 }
523 }
524
525 /*
526 * Misc helper functions:
527 */
528
529 /*
530 * Delay loading of the complete FPU state until the return to userland.
531 * PKRU is handled separately.
532 */
switch_fpu_finish(struct fpu * new_fpu)533 static inline void switch_fpu_finish(struct fpu *new_fpu)
534 {
535 if (cpu_feature_enabled(X86_FEATURE_FPU))
536 set_thread_flag(TIF_NEED_FPU_LOAD);
537 }
538
539 #endif /* _ASM_X86_FPU_INTERNAL_H */
540