• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * FPU signal frame handling routines.
4  */
5 
6 #include <linux/compat.h>
7 #include <linux/cpu.h>
8 #include <linux/pagemap.h>
9 
10 #include <asm/fpu/internal.h>
11 #include <asm/fpu/signal.h>
12 #include <asm/fpu/regset.h>
13 #include <asm/fpu/xstate.h>
14 
15 #include <asm/sigframe.h>
16 #include <asm/trace/fpu.h>
17 
18 static struct _fpx_sw_bytes fx_sw_reserved __ro_after_init;
19 static struct _fpx_sw_bytes fx_sw_reserved_ia32 __ro_after_init;
20 
21 /*
22  * Check for the presence of extended state information in the
23  * user fpstate pointer in the sigcontext.
24  */
check_xstate_in_sigframe(struct fxregs_state __user * fxbuf,struct _fpx_sw_bytes * fx_sw)25 static inline int check_xstate_in_sigframe(struct fxregs_state __user *fxbuf,
26 					   struct _fpx_sw_bytes *fx_sw)
27 {
28 	int min_xstate_size = sizeof(struct fxregs_state) +
29 			      sizeof(struct xstate_header);
30 	void __user *fpstate = fxbuf;
31 	unsigned int magic2;
32 
33 	if (__copy_from_user(fx_sw, &fxbuf->sw_reserved[0], sizeof(*fx_sw)))
34 		return -EFAULT;
35 
36 	/* Check for the first magic field and other error scenarios. */
37 	if (fx_sw->magic1 != FP_XSTATE_MAGIC1 ||
38 	    fx_sw->xstate_size < min_xstate_size ||
39 	    fx_sw->xstate_size > fpu_user_xstate_size ||
40 	    fx_sw->xstate_size > fx_sw->extended_size)
41 		goto setfx;
42 
43 	/*
44 	 * Check for the presence of second magic word at the end of memory
45 	 * layout. This detects the case where the user just copied the legacy
46 	 * fpstate layout with out copying the extended state information
47 	 * in the memory layout.
48 	 */
49 	if (__get_user(magic2, (__u32 __user *)(fpstate + fx_sw->xstate_size)))
50 		return -EFAULT;
51 
52 	if (likely(magic2 == FP_XSTATE_MAGIC2))
53 		return 0;
54 setfx:
55 	trace_x86_fpu_xstate_check_failed(&current->thread.fpu);
56 
57 	/* Set the parameters for fx only state */
58 	fx_sw->magic1 = 0;
59 	fx_sw->xstate_size = sizeof(struct fxregs_state);
60 	fx_sw->xfeatures = XFEATURE_MASK_FPSSE;
61 	return 0;
62 }
63 
64 /*
65  * Signal frame handlers.
66  */
save_fsave_header(struct task_struct * tsk,void __user * buf)67 static inline int save_fsave_header(struct task_struct *tsk, void __user *buf)
68 {
69 	if (use_fxsr()) {
70 		struct xregs_state *xsave = &tsk->thread.fpu.state.xsave;
71 		struct user_i387_ia32_struct env;
72 		struct _fpstate_32 __user *fp = buf;
73 
74 		fpregs_lock();
75 		if (!test_thread_flag(TIF_NEED_FPU_LOAD))
76 			fxsave(&tsk->thread.fpu.state.fxsave);
77 		fpregs_unlock();
78 
79 		convert_from_fxsr(&env, tsk);
80 
81 		if (__copy_to_user(buf, &env, sizeof(env)) ||
82 		    __put_user(xsave->i387.swd, &fp->status) ||
83 		    __put_user(X86_FXSR_MAGIC, &fp->magic))
84 			return -1;
85 	} else {
86 		struct fregs_state __user *fp = buf;
87 		u32 swd;
88 		if (__get_user(swd, &fp->swd) || __put_user(swd, &fp->status))
89 			return -1;
90 	}
91 
92 	return 0;
93 }
94 
save_xstate_epilog(void __user * buf,int ia32_frame)95 static inline int save_xstate_epilog(void __user *buf, int ia32_frame)
96 {
97 	struct xregs_state __user *x = buf;
98 	struct _fpx_sw_bytes *sw_bytes;
99 	u32 xfeatures;
100 	int err;
101 
102 	/* Setup the bytes not touched by the [f]xsave and reserved for SW. */
103 	sw_bytes = ia32_frame ? &fx_sw_reserved_ia32 : &fx_sw_reserved;
104 	err = __copy_to_user(&x->i387.sw_reserved, sw_bytes, sizeof(*sw_bytes));
105 
106 	if (!use_xsave())
107 		return err;
108 
109 	err |= __put_user(FP_XSTATE_MAGIC2,
110 			  (__u32 __user *)(buf + fpu_user_xstate_size));
111 
112 	/*
113 	 * Read the xfeatures which we copied (directly from the cpu or
114 	 * from the state in task struct) to the user buffers.
115 	 */
116 	err |= __get_user(xfeatures, (__u32 __user *)&x->header.xfeatures);
117 
118 	/*
119 	 * For legacy compatible, we always set FP/SSE bits in the bit
120 	 * vector while saving the state to the user context. This will
121 	 * enable us capturing any changes(during sigreturn) to
122 	 * the FP/SSE bits by the legacy applications which don't touch
123 	 * xfeatures in the xsave header.
124 	 *
125 	 * xsave aware apps can change the xfeatures in the xsave
126 	 * header as well as change any contents in the memory layout.
127 	 * xrestore as part of sigreturn will capture all the changes.
128 	 */
129 	xfeatures |= XFEATURE_MASK_FPSSE;
130 
131 	err |= __put_user(xfeatures, (__u32 __user *)&x->header.xfeatures);
132 
133 	return err;
134 }
135 
copy_fpregs_to_sigframe(struct xregs_state __user * buf)136 static inline int copy_fpregs_to_sigframe(struct xregs_state __user *buf)
137 {
138 	int err;
139 
140 	if (use_xsave())
141 		err = xsave_to_user_sigframe(buf);
142 	else if (use_fxsr())
143 		err = fxsave_to_user_sigframe((struct fxregs_state __user *) buf);
144 	else
145 		err = fnsave_to_user_sigframe((struct fregs_state __user *) buf);
146 
147 	if (unlikely(err) && __clear_user(buf, fpu_user_xstate_size))
148 		err = -EFAULT;
149 	return err;
150 }
151 
152 /*
153  * Save the fpu, extended register state to the user signal frame.
154  *
155  * 'buf_fx' is the 64-byte aligned pointer at which the [f|fx|x]save
156  *  state is copied.
157  *  'buf' points to the 'buf_fx' or to the fsave header followed by 'buf_fx'.
158  *
159  *	buf == buf_fx for 64-bit frames and 32-bit fsave frame.
160  *	buf != buf_fx for 32-bit frames with fxstate.
161  *
162  * Try to save it directly to the user frame with disabled page fault handler.
163  * If this fails then do the slow path where the FPU state is first saved to
164  * task's fpu->state and then copy it to the user frame pointed to by the
165  * aligned pointer 'buf_fx'.
166  *
167  * If this is a 32-bit frame with fxstate, put a fsave header before
168  * the aligned state at 'buf_fx'.
169  *
170  * For [f]xsave state, update the SW reserved fields in the [f]xsave frame
171  * indicating the absence/presence of the extended state to the user.
172  */
copy_fpstate_to_sigframe(void __user * buf,void __user * buf_fx,int size)173 int copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size)
174 {
175 	struct task_struct *tsk = current;
176 	int ia32_fxstate = (buf != buf_fx);
177 	int ret;
178 
179 	ia32_fxstate &= (IS_ENABLED(CONFIG_X86_32) ||
180 			 IS_ENABLED(CONFIG_IA32_EMULATION));
181 
182 	if (!static_cpu_has(X86_FEATURE_FPU)) {
183 		struct user_i387_ia32_struct fp;
184 		fpregs_soft_get(current, NULL, (struct membuf){.p = &fp,
185 						.left = sizeof(fp)});
186 		return copy_to_user(buf, &fp, sizeof(fp)) ? -EFAULT : 0;
187 	}
188 
189 	if (!access_ok(buf, size))
190 		return -EACCES;
191 retry:
192 	/*
193 	 * Load the FPU registers if they are not valid for the current task.
194 	 * With a valid FPU state we can attempt to save the state directly to
195 	 * userland's stack frame which will likely succeed. If it does not,
196 	 * resolve the fault in the user memory and try again.
197 	 */
198 	fpregs_lock();
199 	if (test_thread_flag(TIF_NEED_FPU_LOAD))
200 		fpregs_restore_userregs();
201 
202 	pagefault_disable();
203 	ret = copy_fpregs_to_sigframe(buf_fx);
204 	pagefault_enable();
205 	fpregs_unlock();
206 
207 	if (ret) {
208 		if (!fault_in_writeable(buf_fx, fpu_user_xstate_size))
209 			goto retry;
210 		return -EFAULT;
211 	}
212 
213 	/* Save the fsave header for the 32-bit frames. */
214 	if ((ia32_fxstate || !use_fxsr()) && save_fsave_header(tsk, buf))
215 		return -1;
216 
217 	if (use_fxsr() && save_xstate_epilog(buf_fx, ia32_fxstate))
218 		return -1;
219 
220 	return 0;
221 }
222 
__restore_fpregs_from_user(void __user * buf,u64 xrestore,bool fx_only)223 static int __restore_fpregs_from_user(void __user *buf, u64 xrestore,
224 				      bool fx_only)
225 {
226 	if (use_xsave()) {
227 		u64 init_bv = xfeatures_mask_uabi() & ~xrestore;
228 		int ret;
229 
230 		if (likely(!fx_only))
231 			ret = xrstor_from_user_sigframe(buf, xrestore);
232 		else
233 			ret = fxrstor_from_user_sigframe(buf);
234 
235 		if (!ret && unlikely(init_bv))
236 			os_xrstor(&init_fpstate.xsave, init_bv);
237 		return ret;
238 	} else if (use_fxsr()) {
239 		return fxrstor_from_user_sigframe(buf);
240 	} else {
241 		return frstor_from_user_sigframe(buf);
242 	}
243 }
244 
245 /*
246  * Attempt to restore the FPU registers directly from user memory.
247  * Pagefaults are handled and any errors returned are fatal.
248  */
restore_fpregs_from_user(void __user * buf,u64 xrestore,bool fx_only,unsigned int size)249 static int restore_fpregs_from_user(void __user *buf, u64 xrestore,
250 				    bool fx_only, unsigned int size)
251 {
252 	struct fpu *fpu = &current->thread.fpu;
253 	int ret;
254 
255 retry:
256 	fpregs_lock();
257 	pagefault_disable();
258 	ret = __restore_fpregs_from_user(buf, xrestore, fx_only);
259 	pagefault_enable();
260 
261 	if (unlikely(ret)) {
262 		/*
263 		 * The above did an FPU restore operation, restricted to
264 		 * the user portion of the registers, and failed, but the
265 		 * microcode might have modified the FPU registers
266 		 * nevertheless.
267 		 *
268 		 * If the FPU registers do not belong to current, then
269 		 * invalidate the FPU register state otherwise the task
270 		 * might preempt current and return to user space with
271 		 * corrupted FPU registers.
272 		 */
273 		if (test_thread_flag(TIF_NEED_FPU_LOAD))
274 			__cpu_invalidate_fpregs_state();
275 		fpregs_unlock();
276 
277 		/* Try to handle #PF, but anything else is fatal. */
278 		if (ret != -EFAULT)
279 			return -EINVAL;
280 
281 		if (!fault_in_readable(buf, size))
282 			goto retry;
283 		return -EFAULT;
284 	}
285 
286 	/*
287 	 * Restore supervisor states: previous context switch etc has done
288 	 * XSAVES and saved the supervisor states in the kernel buffer from
289 	 * which they can be restored now.
290 	 *
291 	 * It would be optimal to handle this with a single XRSTORS, but
292 	 * this does not work because the rest of the FPU registers have
293 	 * been restored from a user buffer directly.
294 	 */
295 	if (test_thread_flag(TIF_NEED_FPU_LOAD) && xfeatures_mask_supervisor())
296 		os_xrstor(&fpu->state.xsave, xfeatures_mask_supervisor());
297 
298 	fpregs_mark_activate();
299 	fpregs_unlock();
300 	return 0;
301 }
302 
__fpu_restore_sig(void __user * buf,void __user * buf_fx,bool ia32_fxstate)303 static int __fpu_restore_sig(void __user *buf, void __user *buf_fx,
304 			     bool ia32_fxstate)
305 {
306 	int state_size = fpu_kernel_xstate_size;
307 	struct task_struct *tsk = current;
308 	struct fpu *fpu = &tsk->thread.fpu;
309 	struct user_i387_ia32_struct env;
310 	u64 user_xfeatures = 0;
311 	bool fx_only = false;
312 	int ret;
313 
314 	if (use_xsave()) {
315 		struct _fpx_sw_bytes fx_sw_user;
316 
317 		ret = check_xstate_in_sigframe(buf_fx, &fx_sw_user);
318 		if (unlikely(ret))
319 			return ret;
320 
321 		fx_only = !fx_sw_user.magic1;
322 		state_size = fx_sw_user.xstate_size;
323 		user_xfeatures = fx_sw_user.xfeatures;
324 	} else {
325 		user_xfeatures = XFEATURE_MASK_FPSSE;
326 	}
327 
328 	if (likely(!ia32_fxstate)) {
329 		/*
330 		 * Attempt to restore the FPU registers directly from user
331 		 * memory. For that to succeed, the user access cannot cause page
332 		 * faults. If it does, fall back to the slow path below, going
333 		 * through the kernel buffer with the enabled pagefault handler.
334 		 */
335 		return restore_fpregs_from_user(buf_fx, user_xfeatures, fx_only,
336 						state_size);
337 	}
338 
339 	/*
340 	 * Copy the legacy state because the FP portion of the FX frame has
341 	 * to be ignored for histerical raisins. The legacy state is folded
342 	 * in once the larger state has been copied.
343 	 */
344 	ret = __copy_from_user(&env, buf, sizeof(env));
345 	if (ret)
346 		return ret;
347 
348 	/*
349 	 * By setting TIF_NEED_FPU_LOAD it is ensured that our xstate is
350 	 * not modified on context switch and that the xstate is considered
351 	 * to be loaded again on return to userland (overriding last_cpu avoids
352 	 * the optimisation).
353 	 */
354 	fpregs_lock();
355 	if (!test_thread_flag(TIF_NEED_FPU_LOAD)) {
356 		/*
357 		 * If supervisor states are available then save the
358 		 * hardware state in current's fpstate so that the
359 		 * supervisor state is preserved. Save the full state for
360 		 * simplicity. There is no point in optimizing this by only
361 		 * saving the supervisor states and then shuffle them to
362 		 * the right place in memory. It's ia32 mode. Shrug.
363 		 */
364 		if (xfeatures_mask_supervisor())
365 			os_xsave(&fpu->state.xsave);
366 		set_thread_flag(TIF_NEED_FPU_LOAD);
367 	}
368 	__fpu_invalidate_fpregs_state(fpu);
369 	__cpu_invalidate_fpregs_state();
370 	fpregs_unlock();
371 
372 	if (use_xsave() && !fx_only) {
373 		ret = copy_sigframe_from_user_to_xstate(tsk, buf_fx);
374 		if (ret)
375 			return ret;
376 	} else {
377 		if (__copy_from_user(&fpu->state.fxsave, buf_fx,
378 				     sizeof(fpu->state.fxsave)))
379 			return -EFAULT;
380 
381 		if (IS_ENABLED(CONFIG_X86_64)) {
382 			/* Reject invalid MXCSR values. */
383 			if (fpu->state.fxsave.mxcsr & ~mxcsr_feature_mask)
384 				return -EINVAL;
385 		} else {
386 			/* Mask invalid bits out for historical reasons (broken hardware). */
387 			fpu->state.fxsave.mxcsr &= mxcsr_feature_mask;
388 		}
389 
390 		/* Enforce XFEATURE_MASK_FPSSE when XSAVE is enabled */
391 		if (use_xsave())
392 			fpu->state.xsave.header.xfeatures |= XFEATURE_MASK_FPSSE;
393 	}
394 
395 	/* Fold the legacy FP storage */
396 	convert_to_fxsr(&fpu->state.fxsave, &env);
397 
398 	fpregs_lock();
399 	if (use_xsave()) {
400 		/*
401 		 * Remove all UABI feature bits not set in user_xfeatures
402 		 * from the memory xstate header which makes the full
403 		 * restore below bring them into init state. This works for
404 		 * fx_only mode as well because that has only FP and SSE
405 		 * set in user_xfeatures.
406 		 *
407 		 * Preserve supervisor states!
408 		 */
409 		u64 mask = user_xfeatures | xfeatures_mask_supervisor();
410 
411 		fpu->state.xsave.header.xfeatures &= mask;
412 		ret = os_xrstor_safe(&fpu->state.xsave, xfeatures_mask_all);
413 	} else {
414 		ret = fxrstor_safe(&fpu->state.fxsave);
415 	}
416 
417 	if (likely(!ret))
418 		fpregs_mark_activate();
419 
420 	fpregs_unlock();
421 	return ret;
422 }
xstate_sigframe_size(void)423 static inline int xstate_sigframe_size(void)
424 {
425 	return use_xsave() ? fpu_user_xstate_size + FP_XSTATE_MAGIC2_SIZE :
426 			fpu_user_xstate_size;
427 }
428 
429 /*
430  * Restore FPU state from a sigframe:
431  */
fpu__restore_sig(void __user * buf,int ia32_frame)432 int fpu__restore_sig(void __user *buf, int ia32_frame)
433 {
434 	unsigned int size = xstate_sigframe_size();
435 	struct fpu *fpu = &current->thread.fpu;
436 	void __user *buf_fx = buf;
437 	bool ia32_fxstate = false;
438 	int ret;
439 
440 	if (unlikely(!buf)) {
441 		fpu__clear_user_states(fpu);
442 		return 0;
443 	}
444 
445 	ia32_frame &= (IS_ENABLED(CONFIG_X86_32) ||
446 		       IS_ENABLED(CONFIG_IA32_EMULATION));
447 
448 	/*
449 	 * Only FXSR enabled systems need the FX state quirk.
450 	 * FRSTOR does not need it and can use the fast path.
451 	 */
452 	if (ia32_frame && use_fxsr()) {
453 		buf_fx = buf + sizeof(struct fregs_state);
454 		size += sizeof(struct fregs_state);
455 		ia32_fxstate = true;
456 	}
457 
458 	if (!access_ok(buf, size)) {
459 		ret = -EACCES;
460 		goto out;
461 	}
462 
463 	if (!IS_ENABLED(CONFIG_X86_64) && !cpu_feature_enabled(X86_FEATURE_FPU)) {
464 		ret = fpregs_soft_set(current, NULL, 0,
465 				      sizeof(struct user_i387_ia32_struct),
466 				      NULL, buf);
467 	} else {
468 		ret = __fpu_restore_sig(buf, buf_fx, ia32_fxstate);
469 	}
470 
471 out:
472 	if (unlikely(ret))
473 		fpu__clear_user_states(fpu);
474 	return ret;
475 }
476 
477 unsigned long
fpu__alloc_mathframe(unsigned long sp,int ia32_frame,unsigned long * buf_fx,unsigned long * size)478 fpu__alloc_mathframe(unsigned long sp, int ia32_frame,
479 		     unsigned long *buf_fx, unsigned long *size)
480 {
481 	unsigned long frame_size = xstate_sigframe_size();
482 
483 	*buf_fx = sp = round_down(sp - frame_size, 64);
484 	if (ia32_frame && use_fxsr()) {
485 		frame_size += sizeof(struct fregs_state);
486 		sp -= sizeof(struct fregs_state);
487 	}
488 
489 	*size = frame_size;
490 
491 	return sp;
492 }
493 
fpu__get_fpstate_size(void)494 unsigned long fpu__get_fpstate_size(void)
495 {
496 	unsigned long ret = xstate_sigframe_size();
497 
498 	/*
499 	 * This space is needed on (most) 32-bit kernels, or when a 32-bit
500 	 * app is running on a 64-bit kernel. To keep things simple, just
501 	 * assume the worst case and always include space for 'freg_state',
502 	 * even for 64-bit apps on 64-bit kernels. This wastes a bit of
503 	 * space, but keeps the code simple.
504 	 */
505 	if ((IS_ENABLED(CONFIG_IA32_EMULATION) ||
506 	     IS_ENABLED(CONFIG_X86_32)) && use_fxsr())
507 		ret += sizeof(struct fregs_state);
508 
509 	return ret;
510 }
511 
512 /*
513  * Prepare the SW reserved portion of the fxsave memory layout, indicating
514  * the presence of the extended state information in the memory layout
515  * pointed by the fpstate pointer in the sigcontext.
516  * This will be saved when ever the FP and extended state context is
517  * saved on the user stack during the signal handler delivery to the user.
518  */
fpu__init_prepare_fx_sw_frame(void)519 void fpu__init_prepare_fx_sw_frame(void)
520 {
521 	int size = fpu_user_xstate_size + FP_XSTATE_MAGIC2_SIZE;
522 
523 	fx_sw_reserved.magic1 = FP_XSTATE_MAGIC1;
524 	fx_sw_reserved.extended_size = size;
525 	fx_sw_reserved.xfeatures = xfeatures_mask_uabi();
526 	fx_sw_reserved.xstate_size = fpu_user_xstate_size;
527 
528 	if (IS_ENABLED(CONFIG_IA32_EMULATION) ||
529 	    IS_ENABLED(CONFIG_X86_32)) {
530 		int fsave_header_size = sizeof(struct fregs_state);
531 
532 		fx_sw_reserved_ia32 = fx_sw_reserved;
533 		fx_sw_reserved_ia32.extended_size = size + fsave_header_size;
534 	}
535 }
536 
537