• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * xsave/xrstor support.
4  *
5  * Author: Suresh Siddha <suresh.b.siddha@intel.com>
6  */
7 #include <linux/compat.h>
8 #include <linux/cpu.h>
9 #include <linux/mman.h>
10 #include <linux/pkeys.h>
11 #include <linux/seq_file.h>
12 #include <linux/proc_fs.h>
13 
14 #include <asm/fpu/api.h>
15 #include <asm/fpu/internal.h>
16 #include <asm/fpu/signal.h>
17 #include <asm/fpu/regset.h>
18 #include <asm/fpu/xstate.h>
19 
20 #include <asm/tlbflush.h>
21 #include <asm/cpufeature.h>
22 
23 /*
24  * Although we spell it out in here, the Processor Trace
25  * xfeature is completely unused.  We use other mechanisms
26  * to save/restore PT state in Linux.
27  */
28 static const char *xfeature_names[] =
29 {
30 	"x87 floating point registers"	,
31 	"SSE registers"			,
32 	"AVX registers"			,
33 	"MPX bounds registers"		,
34 	"MPX CSR"			,
35 	"AVX-512 opmask"		,
36 	"AVX-512 Hi256"			,
37 	"AVX-512 ZMM_Hi256"		,
38 	"Processor Trace (unused)"	,
39 	"Protection Keys User registers",
40 	"PASID state",
41 	"unknown xstate feature"	,
42 };
43 
44 static short xsave_cpuid_features[] __initdata = {
45 	X86_FEATURE_FPU,
46 	X86_FEATURE_XMM,
47 	X86_FEATURE_AVX,
48 	X86_FEATURE_MPX,
49 	X86_FEATURE_MPX,
50 	X86_FEATURE_AVX512F,
51 	X86_FEATURE_AVX512F,
52 	X86_FEATURE_AVX512F,
53 	X86_FEATURE_INTEL_PT,
54 	X86_FEATURE_PKU,
55 	X86_FEATURE_ENQCMD,
56 };
57 
58 /*
59  * This represents the full set of bits that should ever be set in a kernel
60  * XSAVE buffer, both supervisor and user xstates.
61  */
62 u64 xfeatures_mask_all __read_mostly;
63 
64 static unsigned int xstate_offsets[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1};
65 static unsigned int xstate_sizes[XFEATURE_MAX]   = { [ 0 ... XFEATURE_MAX - 1] = -1};
66 static unsigned int xstate_comp_offsets[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1};
67 static unsigned int xstate_supervisor_only_offsets[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1};
68 
69 /*
70  * The XSAVE area of kernel can be in standard or compacted format;
71  * it is always in standard format for user mode. This is the user
72  * mode standard format size used for signal and ptrace frames.
73  */
74 unsigned int fpu_user_xstate_size;
75 
76 /*
77  * Return whether the system supports a given xfeature.
78  *
79  * Also return the name of the (most advanced) feature that the caller requested:
80  */
cpu_has_xfeatures(u64 xfeatures_needed,const char ** feature_name)81 int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name)
82 {
83 	u64 xfeatures_missing = xfeatures_needed & ~xfeatures_mask_all;
84 
85 	if (unlikely(feature_name)) {
86 		long xfeature_idx, max_idx;
87 		u64 xfeatures_print;
88 		/*
89 		 * So we use FLS here to be able to print the most advanced
90 		 * feature that was requested but is missing. So if a driver
91 		 * asks about "XFEATURE_MASK_SSE | XFEATURE_MASK_YMM" we'll print the
92 		 * missing AVX feature - this is the most informative message
93 		 * to users:
94 		 */
95 		if (xfeatures_missing)
96 			xfeatures_print = xfeatures_missing;
97 		else
98 			xfeatures_print = xfeatures_needed;
99 
100 		xfeature_idx = fls64(xfeatures_print)-1;
101 		max_idx = ARRAY_SIZE(xfeature_names)-1;
102 		xfeature_idx = min(xfeature_idx, max_idx);
103 
104 		*feature_name = xfeature_names[xfeature_idx];
105 	}
106 
107 	if (xfeatures_missing)
108 		return 0;
109 
110 	return 1;
111 }
112 EXPORT_SYMBOL_GPL(cpu_has_xfeatures);
113 
xfeature_is_supervisor(int xfeature_nr)114 static bool xfeature_is_supervisor(int xfeature_nr)
115 {
116 	/*
117 	 * Extended State Enumeration Sub-leaves (EAX = 0DH, ECX = n, n > 1)
118 	 * returns ECX[0] set to (1) for a supervisor state, and cleared (0)
119 	 * for a user state.
120 	 */
121 	u32 eax, ebx, ecx, edx;
122 
123 	cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
124 	return ecx & 1;
125 }
126 
127 /*
128  * When executing XSAVEOPT (or other optimized XSAVE instructions), if
129  * a processor implementation detects that an FPU state component is still
130  * (or is again) in its initialized state, it may clear the corresponding
131  * bit in the header.xfeatures field, and can skip the writeout of registers
132  * to the corresponding memory layout.
133  *
134  * This means that when the bit is zero, the state component might still contain
135  * some previous - non-initialized register state.
136  *
137  * Before writing xstate information to user-space we sanitize those components,
138  * to always ensure that the memory layout of a feature will be in the init state
139  * if the corresponding header bit is zero. This is to ensure that user-space doesn't
140  * see some stale state in the memory layout during signal handling, debugging etc.
141  */
fpstate_sanitize_xstate(struct fpu * fpu)142 void fpstate_sanitize_xstate(struct fpu *fpu)
143 {
144 	struct fxregs_state *fx = &fpu->state.fxsave;
145 	int feature_bit;
146 	u64 xfeatures;
147 
148 	if (!use_xsaveopt())
149 		return;
150 
151 	xfeatures = fpu->state.xsave.header.xfeatures;
152 
153 	/*
154 	 * None of the feature bits are in init state. So nothing else
155 	 * to do for us, as the memory layout is up to date.
156 	 */
157 	if ((xfeatures & xfeatures_mask_all) == xfeatures_mask_all)
158 		return;
159 
160 	/*
161 	 * FP is in init state
162 	 */
163 	if (!(xfeatures & XFEATURE_MASK_FP)) {
164 		fx->cwd = 0x37f;
165 		fx->swd = 0;
166 		fx->twd = 0;
167 		fx->fop = 0;
168 		fx->rip = 0;
169 		fx->rdp = 0;
170 		memset(&fx->st_space[0], 0, 128);
171 	}
172 
173 	/*
174 	 * SSE is in init state
175 	 */
176 	if (!(xfeatures & XFEATURE_MASK_SSE))
177 		memset(&fx->xmm_space[0], 0, 256);
178 
179 	/*
180 	 * First two features are FPU and SSE, which above we handled
181 	 * in a special way already:
182 	 */
183 	feature_bit = 0x2;
184 	xfeatures = (xfeatures_mask_user() & ~xfeatures) >> 2;
185 
186 	/*
187 	 * Update all the remaining memory layouts according to their
188 	 * standard xstate layout, if their header bit is in the init
189 	 * state:
190 	 */
191 	while (xfeatures) {
192 		if (xfeatures & 0x1) {
193 			int offset = xstate_comp_offsets[feature_bit];
194 			int size = xstate_sizes[feature_bit];
195 
196 			memcpy((void *)fx + offset,
197 			       (void *)&init_fpstate.xsave + offset,
198 			       size);
199 		}
200 
201 		xfeatures >>= 1;
202 		feature_bit++;
203 	}
204 }
205 
206 /*
207  * Enable the extended processor state save/restore feature.
208  * Called once per CPU onlining.
209  */
fpu__init_cpu_xstate(void)210 void fpu__init_cpu_xstate(void)
211 {
212 	u64 unsup_bits;
213 
214 	if (!boot_cpu_has(X86_FEATURE_XSAVE) || !xfeatures_mask_all)
215 		return;
216 	/*
217 	 * Unsupported supervisor xstates should not be found in
218 	 * the xfeatures mask.
219 	 */
220 	unsup_bits = xfeatures_mask_all & XFEATURE_MASK_SUPERVISOR_UNSUPPORTED;
221 	WARN_ONCE(unsup_bits, "x86/fpu: Found unsupported supervisor xstates: 0x%llx\n",
222 		  unsup_bits);
223 
224 	xfeatures_mask_all &= ~XFEATURE_MASK_SUPERVISOR_UNSUPPORTED;
225 
226 	cr4_set_bits(X86_CR4_OSXSAVE);
227 
228 	/*
229 	 * XCR_XFEATURE_ENABLED_MASK (aka. XCR0) sets user features
230 	 * managed by XSAVE{C, OPT, S} and XRSTOR{S}.  Only XSAVE user
231 	 * states can be set here.
232 	 */
233 	xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_user());
234 
235 	/*
236 	 * MSR_IA32_XSS sets supervisor states managed by XSAVES.
237 	 */
238 	if (boot_cpu_has(X86_FEATURE_XSAVES)) {
239 		wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() |
240 				     xfeatures_mask_dynamic());
241 	}
242 }
243 
xfeature_enabled(enum xfeature xfeature)244 static bool xfeature_enabled(enum xfeature xfeature)
245 {
246 	return xfeatures_mask_all & BIT_ULL(xfeature);
247 }
248 
249 /*
250  * Record the offsets and sizes of various xstates contained
251  * in the XSAVE state memory layout.
252  */
setup_xstate_features(void)253 static void __init setup_xstate_features(void)
254 {
255 	u32 eax, ebx, ecx, edx, i;
256 	/* start at the beginnning of the "extended state" */
257 	unsigned int last_good_offset = offsetof(struct xregs_state,
258 						 extended_state_area);
259 	/*
260 	 * The FP xstates and SSE xstates are legacy states. They are always
261 	 * in the fixed offsets in the xsave area in either compacted form
262 	 * or standard form.
263 	 */
264 	xstate_offsets[XFEATURE_FP]	= 0;
265 	xstate_sizes[XFEATURE_FP]	= offsetof(struct fxregs_state,
266 						   xmm_space);
267 
268 	xstate_offsets[XFEATURE_SSE]	= xstate_sizes[XFEATURE_FP];
269 	xstate_sizes[XFEATURE_SSE]	= sizeof_field(struct fxregs_state,
270 						       xmm_space);
271 
272 	for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
273 		if (!xfeature_enabled(i))
274 			continue;
275 
276 		cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
277 
278 		xstate_sizes[i] = eax;
279 
280 		/*
281 		 * If an xfeature is supervisor state, the offset in EBX is
282 		 * invalid, leave it to -1.
283 		 */
284 		if (xfeature_is_supervisor(i))
285 			continue;
286 
287 		xstate_offsets[i] = ebx;
288 
289 		/*
290 		 * In our xstate size checks, we assume that the highest-numbered
291 		 * xstate feature has the highest offset in the buffer.  Ensure
292 		 * it does.
293 		 */
294 		WARN_ONCE(last_good_offset > xstate_offsets[i],
295 			  "x86/fpu: misordered xstate at %d\n", last_good_offset);
296 
297 		last_good_offset = xstate_offsets[i];
298 	}
299 }
300 
print_xstate_feature(u64 xstate_mask)301 static void __init print_xstate_feature(u64 xstate_mask)
302 {
303 	const char *feature_name;
304 
305 	if (cpu_has_xfeatures(xstate_mask, &feature_name))
306 		pr_info("x86/fpu: Supporting XSAVE feature 0x%03Lx: '%s'\n", xstate_mask, feature_name);
307 }
308 
309 /*
310  * Print out all the supported xstate features:
311  */
print_xstate_features(void)312 static void __init print_xstate_features(void)
313 {
314 	print_xstate_feature(XFEATURE_MASK_FP);
315 	print_xstate_feature(XFEATURE_MASK_SSE);
316 	print_xstate_feature(XFEATURE_MASK_YMM);
317 	print_xstate_feature(XFEATURE_MASK_BNDREGS);
318 	print_xstate_feature(XFEATURE_MASK_BNDCSR);
319 	print_xstate_feature(XFEATURE_MASK_OPMASK);
320 	print_xstate_feature(XFEATURE_MASK_ZMM_Hi256);
321 	print_xstate_feature(XFEATURE_MASK_Hi16_ZMM);
322 	print_xstate_feature(XFEATURE_MASK_PKRU);
323 	print_xstate_feature(XFEATURE_MASK_PASID);
324 }
325 
326 /*
327  * This check is important because it is easy to get XSTATE_*
328  * confused with XSTATE_BIT_*.
329  */
330 #define CHECK_XFEATURE(nr) do {		\
331 	WARN_ON(nr < FIRST_EXTENDED_XFEATURE);	\
332 	WARN_ON(nr >= XFEATURE_MAX);	\
333 } while (0)
334 
335 /*
336  * We could cache this like xstate_size[], but we only use
337  * it here, so it would be a waste of space.
338  */
xfeature_is_aligned(int xfeature_nr)339 static int xfeature_is_aligned(int xfeature_nr)
340 {
341 	u32 eax, ebx, ecx, edx;
342 
343 	CHECK_XFEATURE(xfeature_nr);
344 
345 	if (!xfeature_enabled(xfeature_nr)) {
346 		WARN_ONCE(1, "Checking alignment of disabled xfeature %d\n",
347 			  xfeature_nr);
348 		return 0;
349 	}
350 
351 	cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
352 	/*
353 	 * The value returned by ECX[1] indicates the alignment
354 	 * of state component 'i' when the compacted format
355 	 * of the extended region of an XSAVE area is used:
356 	 */
357 	return !!(ecx & 2);
358 }
359 
360 /*
361  * This function sets up offsets and sizes of all extended states in
362  * xsave area. This supports both standard format and compacted format
363  * of the xsave area.
364  */
setup_xstate_comp_offsets(void)365 static void __init setup_xstate_comp_offsets(void)
366 {
367 	unsigned int next_offset;
368 	int i;
369 
370 	/*
371 	 * The FP xstates and SSE xstates are legacy states. They are always
372 	 * in the fixed offsets in the xsave area in either compacted form
373 	 * or standard form.
374 	 */
375 	xstate_comp_offsets[XFEATURE_FP] = 0;
376 	xstate_comp_offsets[XFEATURE_SSE] = offsetof(struct fxregs_state,
377 						     xmm_space);
378 
379 	if (!boot_cpu_has(X86_FEATURE_XSAVES)) {
380 		for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
381 			if (xfeature_enabled(i))
382 				xstate_comp_offsets[i] = xstate_offsets[i];
383 		}
384 		return;
385 	}
386 
387 	next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE;
388 
389 	for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
390 		if (!xfeature_enabled(i))
391 			continue;
392 
393 		if (xfeature_is_aligned(i))
394 			next_offset = ALIGN(next_offset, 64);
395 
396 		xstate_comp_offsets[i] = next_offset;
397 		next_offset += xstate_sizes[i];
398 	}
399 }
400 
401 /*
402  * Setup offsets of a supervisor-state-only XSAVES buffer:
403  *
404  * The offsets stored in xstate_comp_offsets[] only work for one specific
405  * value of the Requested Feature BitMap (RFBM).  In cases where a different
406  * RFBM value is used, a different set of offsets is required.  This set of
407  * offsets is for when RFBM=xfeatures_mask_supervisor().
408  */
setup_supervisor_only_offsets(void)409 static void __init setup_supervisor_only_offsets(void)
410 {
411 	unsigned int next_offset;
412 	int i;
413 
414 	next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE;
415 
416 	for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
417 		if (!xfeature_enabled(i) || !xfeature_is_supervisor(i))
418 			continue;
419 
420 		if (xfeature_is_aligned(i))
421 			next_offset = ALIGN(next_offset, 64);
422 
423 		xstate_supervisor_only_offsets[i] = next_offset;
424 		next_offset += xstate_sizes[i];
425 	}
426 }
427 
428 /*
429  * Print out xstate component offsets and sizes
430  */
print_xstate_offset_size(void)431 static void __init print_xstate_offset_size(void)
432 {
433 	int i;
434 
435 	for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
436 		if (!xfeature_enabled(i))
437 			continue;
438 		pr_info("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n",
439 			 i, xstate_comp_offsets[i], i, xstate_sizes[i]);
440 	}
441 }
442 
443 /*
444  * All supported features have either init state all zeros or are
445  * handled in setup_init_fpu() individually. This is an explicit
446  * feature list and does not use XFEATURE_MASK*SUPPORTED to catch
447  * newly added supported features at build time and make people
448  * actually look at the init state for the new feature.
449  */
450 #define XFEATURES_INIT_FPSTATE_HANDLED		\
451 	(XFEATURE_MASK_FP |			\
452 	 XFEATURE_MASK_SSE |			\
453 	 XFEATURE_MASK_YMM |			\
454 	 XFEATURE_MASK_OPMASK |			\
455 	 XFEATURE_MASK_ZMM_Hi256 |		\
456 	 XFEATURE_MASK_Hi16_ZMM	 |		\
457 	 XFEATURE_MASK_PKRU |			\
458 	 XFEATURE_MASK_BNDREGS |		\
459 	 XFEATURE_MASK_BNDCSR |			\
460 	 XFEATURE_MASK_PASID)
461 
462 /*
463  * setup the xstate image representing the init state
464  */
setup_init_fpu_buf(void)465 static void __init setup_init_fpu_buf(void)
466 {
467 	static int on_boot_cpu __initdata = 1;
468 
469 	BUILD_BUG_ON((XFEATURE_MASK_USER_SUPPORTED |
470 		      XFEATURE_MASK_SUPERVISOR_SUPPORTED) !=
471 		     XFEATURES_INIT_FPSTATE_HANDLED);
472 
473 	WARN_ON_FPU(!on_boot_cpu);
474 	on_boot_cpu = 0;
475 
476 	if (!boot_cpu_has(X86_FEATURE_XSAVE))
477 		return;
478 
479 	setup_xstate_features();
480 	print_xstate_features();
481 
482 	if (boot_cpu_has(X86_FEATURE_XSAVES))
483 		init_fpstate.xsave.header.xcomp_bv = XCOMP_BV_COMPACTED_FORMAT |
484 						     xfeatures_mask_all;
485 
486 	/*
487 	 * Init all the features state with header.xfeatures being 0x0
488 	 */
489 	copy_kernel_to_xregs_booting(&init_fpstate.xsave);
490 
491 	/*
492 	 * All components are now in init state. Read the state back so
493 	 * that init_fpstate contains all non-zero init state. This only
494 	 * works with XSAVE, but not with XSAVEOPT and XSAVES because
495 	 * those use the init optimization which skips writing data for
496 	 * components in init state.
497 	 *
498 	 * XSAVE could be used, but that would require to reshuffle the
499 	 * data when XSAVES is available because XSAVES uses xstate
500 	 * compaction. But doing so is a pointless exercise because most
501 	 * components have an all zeros init state except for the legacy
502 	 * ones (FP and SSE). Those can be saved with FXSAVE into the
503 	 * legacy area. Adding new features requires to ensure that init
504 	 * state is all zeroes or if not to add the necessary handling
505 	 * here.
506 	 */
507 	fxsave(&init_fpstate.fxsave);
508 }
509 
xfeature_uncompacted_offset(int xfeature_nr)510 static int xfeature_uncompacted_offset(int xfeature_nr)
511 {
512 	u32 eax, ebx, ecx, edx;
513 
514 	/*
515 	 * Only XSAVES supports supervisor states and it uses compacted
516 	 * format. Checking a supervisor state's uncompacted offset is
517 	 * an error.
518 	 */
519 	if (XFEATURE_MASK_SUPERVISOR_ALL & BIT_ULL(xfeature_nr)) {
520 		WARN_ONCE(1, "No fixed offset for xstate %d\n", xfeature_nr);
521 		return -1;
522 	}
523 
524 	CHECK_XFEATURE(xfeature_nr);
525 	cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
526 	return ebx;
527 }
528 
xfeature_size(int xfeature_nr)529 int xfeature_size(int xfeature_nr)
530 {
531 	u32 eax, ebx, ecx, edx;
532 
533 	CHECK_XFEATURE(xfeature_nr);
534 	cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
535 	return eax;
536 }
537 
538 /*
539  * 'XSAVES' implies two different things:
540  * 1. saving of supervisor/system state
541  * 2. using the compacted format
542  *
543  * Use this function when dealing with the compacted format so
544  * that it is obvious which aspect of 'XSAVES' is being handled
545  * by the calling code.
546  */
using_compacted_format(void)547 int using_compacted_format(void)
548 {
549 	return boot_cpu_has(X86_FEATURE_XSAVES);
550 }
551 
552 /* Validate an xstate header supplied by userspace (ptrace or sigreturn) */
validate_user_xstate_header(const struct xstate_header * hdr)553 int validate_user_xstate_header(const struct xstate_header *hdr)
554 {
555 	/* No unknown or supervisor features may be set */
556 	if (hdr->xfeatures & ~xfeatures_mask_user())
557 		return -EINVAL;
558 
559 	/* Userspace must use the uncompacted format */
560 	if (hdr->xcomp_bv)
561 		return -EINVAL;
562 
563 	/*
564 	 * If 'reserved' is shrunken to add a new field, make sure to validate
565 	 * that new field here!
566 	 */
567 	BUILD_BUG_ON(sizeof(hdr->reserved) != 48);
568 
569 	/* No reserved bits may be set */
570 	if (memchr_inv(hdr->reserved, 0, sizeof(hdr->reserved)))
571 		return -EINVAL;
572 
573 	return 0;
574 }
575 
__xstate_dump_leaves(void)576 static void __xstate_dump_leaves(void)
577 {
578 	int i;
579 	u32 eax, ebx, ecx, edx;
580 	static int should_dump = 1;
581 
582 	if (!should_dump)
583 		return;
584 	should_dump = 0;
585 	/*
586 	 * Dump out a few leaves past the ones that we support
587 	 * just in case there are some goodies up there
588 	 */
589 	for (i = 0; i < XFEATURE_MAX + 10; i++) {
590 		cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
591 		pr_warn("CPUID[%02x, %02x]: eax=%08x ebx=%08x ecx=%08x edx=%08x\n",
592 			XSTATE_CPUID, i, eax, ebx, ecx, edx);
593 	}
594 }
595 
596 #define XSTATE_WARN_ON(x) do {							\
597 	if (WARN_ONCE(x, "XSAVE consistency problem, dumping leaves")) {	\
598 		__xstate_dump_leaves();						\
599 	}									\
600 } while (0)
601 
602 #define XCHECK_SZ(sz, nr, nr_macro, __struct) do {			\
603 	if ((nr == nr_macro) &&						\
604 	    WARN_ONCE(sz != sizeof(__struct),				\
605 		"%s: struct is %zu bytes, cpu state %d bytes\n",	\
606 		__stringify(nr_macro), sizeof(__struct), sz)) {		\
607 		__xstate_dump_leaves();					\
608 	}								\
609 } while (0)
610 
611 /*
612  * We have a C struct for each 'xstate'.  We need to ensure
613  * that our software representation matches what the CPU
614  * tells us about the state's size.
615  */
check_xstate_against_struct(int nr)616 static void check_xstate_against_struct(int nr)
617 {
618 	/*
619 	 * Ask the CPU for the size of the state.
620 	 */
621 	int sz = xfeature_size(nr);
622 	/*
623 	 * Match each CPU state with the corresponding software
624 	 * structure.
625 	 */
626 	XCHECK_SZ(sz, nr, XFEATURE_YMM,       struct ymmh_struct);
627 	XCHECK_SZ(sz, nr, XFEATURE_BNDREGS,   struct mpx_bndreg_state);
628 	XCHECK_SZ(sz, nr, XFEATURE_BNDCSR,    struct mpx_bndcsr_state);
629 	XCHECK_SZ(sz, nr, XFEATURE_OPMASK,    struct avx_512_opmask_state);
630 	XCHECK_SZ(sz, nr, XFEATURE_ZMM_Hi256, struct avx_512_zmm_uppers_state);
631 	XCHECK_SZ(sz, nr, XFEATURE_Hi16_ZMM,  struct avx_512_hi16_state);
632 	XCHECK_SZ(sz, nr, XFEATURE_PKRU,      struct pkru_state);
633 	XCHECK_SZ(sz, nr, XFEATURE_PASID,     struct ia32_pasid_state);
634 
635 	/*
636 	 * Make *SURE* to add any feature numbers in below if
637 	 * there are "holes" in the xsave state component
638 	 * numbers.
639 	 */
640 	if ((nr < XFEATURE_YMM) ||
641 	    (nr >= XFEATURE_MAX) ||
642 	    (nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR) ||
643 	    ((nr >= XFEATURE_RSRVD_COMP_11) && (nr <= XFEATURE_LBR))) {
644 		WARN_ONCE(1, "no structure for xstate: %d\n", nr);
645 		XSTATE_WARN_ON(1);
646 	}
647 }
648 
649 /*
650  * This essentially double-checks what the cpu told us about
651  * how large the XSAVE buffer needs to be.  We are recalculating
652  * it to be safe.
653  *
654  * Dynamic XSAVE features allocate their own buffers and are not
655  * covered by these checks. Only the size of the buffer for task->fpu
656  * is checked here.
657  */
do_extra_xstate_size_checks(void)658 static void do_extra_xstate_size_checks(void)
659 {
660 	int paranoid_xstate_size = FXSAVE_SIZE + XSAVE_HDR_SIZE;
661 	int i;
662 
663 	for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
664 		if (!xfeature_enabled(i))
665 			continue;
666 
667 		check_xstate_against_struct(i);
668 		/*
669 		 * Supervisor state components can be managed only by
670 		 * XSAVES, which is compacted-format only.
671 		 */
672 		if (!using_compacted_format())
673 			XSTATE_WARN_ON(xfeature_is_supervisor(i));
674 
675 		/* Align from the end of the previous feature */
676 		if (xfeature_is_aligned(i))
677 			paranoid_xstate_size = ALIGN(paranoid_xstate_size, 64);
678 		/*
679 		 * The offset of a given state in the non-compacted
680 		 * format is given to us in a CPUID leaf.  We check
681 		 * them for being ordered (increasing offsets) in
682 		 * setup_xstate_features().
683 		 */
684 		if (!using_compacted_format())
685 			paranoid_xstate_size = xfeature_uncompacted_offset(i);
686 		/*
687 		 * The compacted-format offset always depends on where
688 		 * the previous state ended.
689 		 */
690 		paranoid_xstate_size += xfeature_size(i);
691 	}
692 	XSTATE_WARN_ON(paranoid_xstate_size != fpu_kernel_xstate_size);
693 }
694 
695 
696 /*
697  * Get total size of enabled xstates in XCR0 | IA32_XSS.
698  *
699  * Note the SDM's wording here.  "sub-function 0" only enumerates
700  * the size of the *user* states.  If we use it to size a buffer
701  * that we use 'XSAVES' on, we could potentially overflow the
702  * buffer because 'XSAVES' saves system states too.
703  */
get_xsaves_size(void)704 static unsigned int __init get_xsaves_size(void)
705 {
706 	unsigned int eax, ebx, ecx, edx;
707 	/*
708 	 * - CPUID function 0DH, sub-function 1:
709 	 *    EBX enumerates the size (in bytes) required by
710 	 *    the XSAVES instruction for an XSAVE area
711 	 *    containing all the state components
712 	 *    corresponding to bits currently set in
713 	 *    XCR0 | IA32_XSS.
714 	 */
715 	cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx);
716 	return ebx;
717 }
718 
719 /*
720  * Get the total size of the enabled xstates without the dynamic supervisor
721  * features.
722  */
get_xsaves_size_no_dynamic(void)723 static unsigned int __init get_xsaves_size_no_dynamic(void)
724 {
725 	u64 mask = xfeatures_mask_dynamic();
726 	unsigned int size;
727 
728 	if (!mask)
729 		return get_xsaves_size();
730 
731 	/* Disable dynamic features. */
732 	wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor());
733 
734 	/*
735 	 * Ask the hardware what size is required of the buffer.
736 	 * This is the size required for the task->fpu buffer.
737 	 */
738 	size = get_xsaves_size();
739 
740 	/* Re-enable dynamic features so XSAVES will work on them again. */
741 	wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | mask);
742 
743 	return size;
744 }
745 
get_xsave_size(void)746 static unsigned int __init get_xsave_size(void)
747 {
748 	unsigned int eax, ebx, ecx, edx;
749 	/*
750 	 * - CPUID function 0DH, sub-function 0:
751 	 *    EBX enumerates the size (in bytes) required by
752 	 *    the XSAVE instruction for an XSAVE area
753 	 *    containing all the *user* state components
754 	 *    corresponding to bits currently set in XCR0.
755 	 */
756 	cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
757 	return ebx;
758 }
759 
760 /*
761  * Will the runtime-enumerated 'xstate_size' fit in the init
762  * task's statically-allocated buffer?
763  */
is_supported_xstate_size(unsigned int test_xstate_size)764 static bool is_supported_xstate_size(unsigned int test_xstate_size)
765 {
766 	if (test_xstate_size <= sizeof(union fpregs_state))
767 		return true;
768 
769 	pr_warn("x86/fpu: xstate buffer too small (%zu < %d), disabling xsave\n",
770 			sizeof(union fpregs_state), test_xstate_size);
771 	return false;
772 }
773 
init_xstate_size(void)774 static int __init init_xstate_size(void)
775 {
776 	/* Recompute the context size for enabled features: */
777 	unsigned int possible_xstate_size;
778 	unsigned int xsave_size;
779 
780 	xsave_size = get_xsave_size();
781 
782 	if (boot_cpu_has(X86_FEATURE_XSAVES))
783 		possible_xstate_size = get_xsaves_size_no_dynamic();
784 	else
785 		possible_xstate_size = xsave_size;
786 
787 	/* Ensure we have the space to store all enabled: */
788 	if (!is_supported_xstate_size(possible_xstate_size))
789 		return -EINVAL;
790 
791 	/*
792 	 * The size is OK, we are definitely going to use xsave,
793 	 * make it known to the world that we need more space.
794 	 */
795 	fpu_kernel_xstate_size = possible_xstate_size;
796 	do_extra_xstate_size_checks();
797 
798 	/*
799 	 * User space is always in standard format.
800 	 */
801 	fpu_user_xstate_size = xsave_size;
802 	return 0;
803 }
804 
805 /*
806  * We enabled the XSAVE hardware, but something went wrong and
807  * we can not use it.  Disable it.
808  */
fpu__init_disable_system_xstate(void)809 static void fpu__init_disable_system_xstate(void)
810 {
811 	xfeatures_mask_all = 0;
812 	cr4_clear_bits(X86_CR4_OSXSAVE);
813 	setup_clear_cpu_cap(X86_FEATURE_XSAVE);
814 }
815 
816 /*
817  * Enable and initialize the xsave feature.
818  * Called once per system bootup.
819  */
fpu__init_system_xstate(void)820 void __init fpu__init_system_xstate(void)
821 {
822 	unsigned int eax, ebx, ecx, edx;
823 	static int on_boot_cpu __initdata = 1;
824 	int err;
825 	int i;
826 
827 	WARN_ON_FPU(!on_boot_cpu);
828 	on_boot_cpu = 0;
829 
830 	if (!boot_cpu_has(X86_FEATURE_FPU)) {
831 		pr_info("x86/fpu: No FPU detected\n");
832 		return;
833 	}
834 
835 	if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
836 		pr_info("x86/fpu: x87 FPU will use %s\n",
837 			boot_cpu_has(X86_FEATURE_FXSR) ? "FXSAVE" : "FSAVE");
838 		return;
839 	}
840 
841 	if (boot_cpu_data.cpuid_level < XSTATE_CPUID) {
842 		WARN_ON_FPU(1);
843 		return;
844 	}
845 
846 	/*
847 	 * Find user xstates supported by the processor.
848 	 */
849 	cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
850 	xfeatures_mask_all = eax + ((u64)edx << 32);
851 
852 	/*
853 	 * Find supervisor xstates supported by the processor.
854 	 */
855 	cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx);
856 	xfeatures_mask_all |= ecx + ((u64)edx << 32);
857 
858 	if ((xfeatures_mask_user() & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) {
859 		/*
860 		 * This indicates that something really unexpected happened
861 		 * with the enumeration.  Disable XSAVE and try to continue
862 		 * booting without it.  This is too early to BUG().
863 		 */
864 		pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n",
865 		       xfeatures_mask_all);
866 		goto out_disable;
867 	}
868 
869 	/*
870 	 * Clear XSAVE features that are disabled in the normal CPUID.
871 	 */
872 	for (i = 0; i < ARRAY_SIZE(xsave_cpuid_features); i++) {
873 		if (!boot_cpu_has(xsave_cpuid_features[i]))
874 			xfeatures_mask_all &= ~BIT_ULL(i);
875 	}
876 
877 	xfeatures_mask_all &= fpu__get_supported_xfeatures_mask();
878 
879 	/* Enable xstate instructions to be able to continue with initialization: */
880 	fpu__init_cpu_xstate();
881 	err = init_xstate_size();
882 	if (err)
883 		goto out_disable;
884 
885 	/*
886 	 * Update info used for ptrace frames; use standard-format size and no
887 	 * supervisor xstates:
888 	 */
889 	update_regset_xstate_info(fpu_user_xstate_size, xfeatures_mask_user());
890 
891 	fpu__init_prepare_fx_sw_frame();
892 	setup_init_fpu_buf();
893 	setup_xstate_comp_offsets();
894 	setup_supervisor_only_offsets();
895 
896 	/*
897 	 * CPU capabilities initialization runs before FPU init. So
898 	 * X86_FEATURE_OSXSAVE is not set. Now that XSAVE is completely
899 	 * functional, set the feature bit so depending code works.
900 	 */
901 	setup_force_cpu_cap(X86_FEATURE_OSXSAVE);
902 
903 	print_xstate_offset_size();
904 
905 	pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n",
906 		xfeatures_mask_all,
907 		fpu_kernel_xstate_size,
908 		boot_cpu_has(X86_FEATURE_XSAVES) ? "compacted" : "standard");
909 	return;
910 
911 out_disable:
912 	/* something went wrong, try to boot without any XSAVE support */
913 	fpu__init_disable_system_xstate();
914 }
915 
916 /*
917  * Restore minimal FPU state after suspend:
918  */
fpu__resume_cpu(void)919 void fpu__resume_cpu(void)
920 {
921 	/*
922 	 * Restore XCR0 on xsave capable CPUs:
923 	 */
924 	if (boot_cpu_has(X86_FEATURE_XSAVE))
925 		xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_user());
926 
927 	/*
928 	 * Restore IA32_XSS. The same CPUID bit enumerates support
929 	 * of XSAVES and MSR_IA32_XSS.
930 	 */
931 	if (boot_cpu_has(X86_FEATURE_XSAVES)) {
932 		wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor()  |
933 				     xfeatures_mask_dynamic());
934 	}
935 }
936 
937 /*
938  * Given an xstate feature nr, calculate where in the xsave
939  * buffer the state is.  Callers should ensure that the buffer
940  * is valid.
941  */
__raw_xsave_addr(struct xregs_state * xsave,int xfeature_nr)942 static void *__raw_xsave_addr(struct xregs_state *xsave, int xfeature_nr)
943 {
944 	if (!xfeature_enabled(xfeature_nr)) {
945 		WARN_ON_FPU(1);
946 		return NULL;
947 	}
948 
949 	return (void *)xsave + xstate_comp_offsets[xfeature_nr];
950 }
951 /*
952  * Given the xsave area and a state inside, this function returns the
953  * address of the state.
954  *
955  * This is the API that is called to get xstate address in either
956  * standard format or compacted format of xsave area.
957  *
958  * Note that if there is no data for the field in the xsave buffer
959  * this will return NULL.
960  *
961  * Inputs:
962  *	xstate: the thread's storage area for all FPU data
963  *	xfeature_nr: state which is defined in xsave.h (e.g. XFEATURE_FP,
964  *	XFEATURE_SSE, etc...)
965  * Output:
966  *	address of the state in the xsave area, or NULL if the
967  *	field is not present in the xsave buffer.
968  */
get_xsave_addr(struct xregs_state * xsave,int xfeature_nr)969 void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr)
970 {
971 	/*
972 	 * Do we even *have* xsave state?
973 	 */
974 	if (!boot_cpu_has(X86_FEATURE_XSAVE))
975 		return NULL;
976 
977 	/*
978 	 * We should not ever be requesting features that we
979 	 * have not enabled.
980 	 */
981 	WARN_ONCE(!(xfeatures_mask_all & BIT_ULL(xfeature_nr)),
982 		  "get of unsupported state");
983 	/*
984 	 * This assumes the last 'xsave*' instruction to
985 	 * have requested that 'xfeature_nr' be saved.
986 	 * If it did not, we might be seeing and old value
987 	 * of the field in the buffer.
988 	 *
989 	 * This can happen because the last 'xsave' did not
990 	 * request that this feature be saved (unlikely)
991 	 * or because the "init optimization" caused it
992 	 * to not be saved.
993 	 */
994 	if (!(xsave->header.xfeatures & BIT_ULL(xfeature_nr)))
995 		return NULL;
996 
997 	return __raw_xsave_addr(xsave, xfeature_nr);
998 }
999 EXPORT_SYMBOL_GPL(get_xsave_addr);
1000 
1001 /*
1002  * This wraps up the common operations that need to occur when retrieving
1003  * data from xsave state.  It first ensures that the current task was
1004  * using the FPU and retrieves the data in to a buffer.  It then calculates
1005  * the offset of the requested field in the buffer.
1006  *
1007  * This function is safe to call whether the FPU is in use or not.
1008  *
1009  * Note that this only works on the current task.
1010  *
1011  * Inputs:
1012  *	@xfeature_nr: state which is defined in xsave.h (e.g. XFEATURE_FP,
1013  *	XFEATURE_SSE, etc...)
1014  * Output:
1015  *	address of the state in the xsave area or NULL if the state
1016  *	is not present or is in its 'init state'.
1017  */
get_xsave_field_ptr(int xfeature_nr)1018 const void *get_xsave_field_ptr(int xfeature_nr)
1019 {
1020 	struct fpu *fpu = &current->thread.fpu;
1021 
1022 	/*
1023 	 * fpu__save() takes the CPU's xstate registers
1024 	 * and saves them off to the 'fpu memory buffer.
1025 	 */
1026 	fpu__save(fpu);
1027 
1028 	return get_xsave_addr(&fpu->state.xsave, xfeature_nr);
1029 }
1030 
1031 #ifdef CONFIG_ARCH_HAS_PKEYS
1032 
1033 /*
1034  * This will go out and modify PKRU register to set the access
1035  * rights for @pkey to @init_val.
1036  */
arch_set_user_pkey_access(struct task_struct * tsk,int pkey,unsigned long init_val)1037 int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
1038 		unsigned long init_val)
1039 {
1040 	u32 old_pkru;
1041 	int pkey_shift = (pkey * PKRU_BITS_PER_PKEY);
1042 	u32 new_pkru_bits = 0;
1043 
1044 	/*
1045 	 * This check implies XSAVE support.  OSPKE only gets
1046 	 * set if we enable XSAVE and we enable PKU in XCR0.
1047 	 */
1048 	if (!boot_cpu_has(X86_FEATURE_OSPKE))
1049 		return -EINVAL;
1050 
1051 	/*
1052 	 * This code should only be called with valid 'pkey'
1053 	 * values originating from in-kernel users.  Complain
1054 	 * if a bad value is observed.
1055 	 */
1056 	WARN_ON_ONCE(pkey >= arch_max_pkey());
1057 
1058 	/* Set the bits we need in PKRU:  */
1059 	if (init_val & PKEY_DISABLE_ACCESS)
1060 		new_pkru_bits |= PKRU_AD_BIT;
1061 	if (init_val & PKEY_DISABLE_WRITE)
1062 		new_pkru_bits |= PKRU_WD_BIT;
1063 
1064 	/* Shift the bits in to the correct place in PKRU for pkey: */
1065 	new_pkru_bits <<= pkey_shift;
1066 
1067 	/* Get old PKRU and mask off any old bits in place: */
1068 	old_pkru = read_pkru();
1069 	old_pkru &= ~((PKRU_AD_BIT|PKRU_WD_BIT) << pkey_shift);
1070 
1071 	/* Write old part along with new part: */
1072 	write_pkru(old_pkru | new_pkru_bits);
1073 
1074 	return 0;
1075 }
1076 #endif /* ! CONFIG_ARCH_HAS_PKEYS */
1077 
1078 /*
1079  * Weird legacy quirk: SSE and YMM states store information in the
1080  * MXCSR and MXCSR_FLAGS fields of the FP area. That means if the FP
1081  * area is marked as unused in the xfeatures header, we need to copy
1082  * MXCSR and MXCSR_FLAGS if either SSE or YMM are in use.
1083  */
xfeatures_mxcsr_quirk(u64 xfeatures)1084 static inline bool xfeatures_mxcsr_quirk(u64 xfeatures)
1085 {
1086 	if (!(xfeatures & (XFEATURE_MASK_SSE|XFEATURE_MASK_YMM)))
1087 		return false;
1088 
1089 	if (xfeatures & XFEATURE_MASK_FP)
1090 		return false;
1091 
1092 	return true;
1093 }
1094 
copy_feature(bool from_xstate,struct membuf * to,void * xstate,void * init_xstate,unsigned int size)1095 static void copy_feature(bool from_xstate, struct membuf *to, void *xstate,
1096 			 void *init_xstate, unsigned int size)
1097 {
1098 	membuf_write(to, from_xstate ? xstate : init_xstate, size);
1099 }
1100 
1101 /*
1102  * Convert from kernel XSAVES compacted format to standard format and copy
1103  * to a kernel-space ptrace buffer.
1104  *
1105  * It supports partial copy but pos always starts from zero. This is called
1106  * from xstateregs_get() and there we check the CPU has XSAVES.
1107  */
copy_xstate_to_kernel(struct membuf to,struct xregs_state * xsave)1108 void copy_xstate_to_kernel(struct membuf to, struct xregs_state *xsave)
1109 {
1110 	const unsigned int off_mxcsr = offsetof(struct fxregs_state, mxcsr);
1111 	struct xregs_state *xinit = &init_fpstate.xsave;
1112 	struct xstate_header header;
1113 	unsigned int zerofrom;
1114 	int i;
1115 
1116 	/*
1117 	 * The destination is a ptrace buffer; we put in only user xstates:
1118 	 */
1119 	memset(&header, 0, sizeof(header));
1120 	header.xfeatures = xsave->header.xfeatures;
1121 	header.xfeatures &= xfeatures_mask_user();
1122 
1123 	/* Copy FP state up to MXCSR */
1124 	copy_feature(header.xfeatures & XFEATURE_MASK_FP, &to, &xsave->i387,
1125 		     &xinit->i387, off_mxcsr);
1126 
1127 	/* Copy MXCSR when SSE or YMM are set in the feature mask */
1128 	copy_feature(header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM),
1129 		     &to, &xsave->i387.mxcsr, &xinit->i387.mxcsr,
1130 		     MXCSR_AND_FLAGS_SIZE);
1131 
1132 	/* Copy the remaining FP state */
1133 	copy_feature(header.xfeatures & XFEATURE_MASK_FP,
1134 		     &to, &xsave->i387.st_space, &xinit->i387.st_space,
1135 		     sizeof(xsave->i387.st_space));
1136 
1137 	/* Copy the SSE state - shared with YMM, but independently managed */
1138 	copy_feature(header.xfeatures & XFEATURE_MASK_SSE,
1139 		     &to, &xsave->i387.xmm_space, &xinit->i387.xmm_space,
1140 		     sizeof(xsave->i387.xmm_space));
1141 
1142 	/* Zero the padding area */
1143 	membuf_zero(&to, sizeof(xsave->i387.padding));
1144 
1145 	/* Copy xsave->i387.sw_reserved */
1146 	membuf_write(&to, xstate_fx_sw_bytes, sizeof(xsave->i387.sw_reserved));
1147 
1148 	/* Copy the user space relevant state of @xsave->header */
1149 	membuf_write(&to, &header, sizeof(header));
1150 
1151 	zerofrom = offsetof(struct xregs_state, extended_state_area);
1152 
1153 	for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
1154 		/*
1155 		 * The ptrace buffer is in non-compacted XSAVE format.
1156 		 * In non-compacted format disabled features still occupy
1157 		 * state space, but there is no state to copy from in the
1158 		 * compacted init_fpstate. The gap tracking will zero this
1159 		 * later.
1160 		 */
1161 		if (!(xfeatures_mask_user() & BIT_ULL(i)))
1162 			continue;
1163 
1164 		/*
1165 		 * If there was a feature or alignment gap, zero the space
1166 		 * in the destination buffer.
1167 		 */
1168 		if (zerofrom < xstate_offsets[i])
1169 			membuf_zero(&to, xstate_offsets[i] - zerofrom);
1170 
1171 		copy_feature(header.xfeatures & BIT_ULL(i), &to,
1172 			     __raw_xsave_addr(xsave, i),
1173 			     __raw_xsave_addr(xinit, i),
1174 			     xstate_sizes[i]);
1175 
1176 		/*
1177 		 * Keep track of the last copied state in the non-compacted
1178 		 * target buffer for gap zeroing.
1179 		 */
1180 		zerofrom = xstate_offsets[i] + xstate_sizes[i];
1181 	}
1182 
1183 	if (to.left)
1184 		membuf_zero(&to, to.left);
1185 }
1186 
1187 /*
1188  * Convert from a ptrace standard-format kernel buffer to kernel XSAVES format
1189  * and copy to the target thread. This is called from xstateregs_set().
1190  */
copy_kernel_to_xstate(struct xregs_state * xsave,const void * kbuf)1191 int copy_kernel_to_xstate(struct xregs_state *xsave, const void *kbuf)
1192 {
1193 	unsigned int offset, size;
1194 	int i;
1195 	struct xstate_header hdr;
1196 
1197 	offset = offsetof(struct xregs_state, header);
1198 	size = sizeof(hdr);
1199 
1200 	memcpy(&hdr, kbuf + offset, size);
1201 
1202 	if (validate_user_xstate_header(&hdr))
1203 		return -EINVAL;
1204 
1205 	for (i = 0; i < XFEATURE_MAX; i++) {
1206 		u64 mask = ((u64)1 << i);
1207 
1208 		if (hdr.xfeatures & mask) {
1209 			void *dst = __raw_xsave_addr(xsave, i);
1210 
1211 			offset = xstate_offsets[i];
1212 			size = xstate_sizes[i];
1213 
1214 			memcpy(dst, kbuf + offset, size);
1215 		}
1216 	}
1217 
1218 	if (xfeatures_mxcsr_quirk(hdr.xfeatures)) {
1219 		offset = offsetof(struct fxregs_state, mxcsr);
1220 		size = MXCSR_AND_FLAGS_SIZE;
1221 		memcpy(&xsave->i387.mxcsr, kbuf + offset, size);
1222 	}
1223 
1224 	/*
1225 	 * The state that came in from userspace was user-state only.
1226 	 * Mask all the user states out of 'xfeatures':
1227 	 */
1228 	xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR_ALL;
1229 
1230 	/*
1231 	 * Add back in the features that came in from userspace:
1232 	 */
1233 	xsave->header.xfeatures |= hdr.xfeatures;
1234 
1235 	return 0;
1236 }
1237 
1238 /*
1239  * Convert from a ptrace or sigreturn standard-format user-space buffer to
1240  * kernel XSAVES format and copy to the target thread. This is called from
1241  * xstateregs_set(), as well as potentially from the sigreturn() and
1242  * rt_sigreturn() system calls.
1243  */
copy_user_to_xstate(struct xregs_state * xsave,const void __user * ubuf)1244 int copy_user_to_xstate(struct xregs_state *xsave, const void __user *ubuf)
1245 {
1246 	unsigned int offset, size;
1247 	int i;
1248 	struct xstate_header hdr;
1249 
1250 	offset = offsetof(struct xregs_state, header);
1251 	size = sizeof(hdr);
1252 
1253 	if (__copy_from_user(&hdr, ubuf + offset, size))
1254 		return -EFAULT;
1255 
1256 	if (validate_user_xstate_header(&hdr))
1257 		return -EINVAL;
1258 
1259 	for (i = 0; i < XFEATURE_MAX; i++) {
1260 		u64 mask = ((u64)1 << i);
1261 
1262 		if (hdr.xfeatures & mask) {
1263 			void *dst = __raw_xsave_addr(xsave, i);
1264 
1265 			offset = xstate_offsets[i];
1266 			size = xstate_sizes[i];
1267 
1268 			if (__copy_from_user(dst, ubuf + offset, size))
1269 				return -EFAULT;
1270 		}
1271 	}
1272 
1273 	if (xfeatures_mxcsr_quirk(hdr.xfeatures)) {
1274 		offset = offsetof(struct fxregs_state, mxcsr);
1275 		size = MXCSR_AND_FLAGS_SIZE;
1276 		if (__copy_from_user(&xsave->i387.mxcsr, ubuf + offset, size))
1277 			return -EFAULT;
1278 	}
1279 
1280 	/*
1281 	 * The state that came in from userspace was user-state only.
1282 	 * Mask all the user states out of 'xfeatures':
1283 	 */
1284 	xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR_ALL;
1285 
1286 	/*
1287 	 * Add back in the features that came in from userspace:
1288 	 */
1289 	xsave->header.xfeatures |= hdr.xfeatures;
1290 
1291 	return 0;
1292 }
1293 
1294 /*
1295  * Save only supervisor states to the kernel buffer.  This blows away all
1296  * old states, and is intended to be used only in __fpu__restore_sig(), where
1297  * user states are restored from the user buffer.
1298  */
copy_supervisor_to_kernel(struct xregs_state * xstate)1299 void copy_supervisor_to_kernel(struct xregs_state *xstate)
1300 {
1301 	struct xstate_header *header;
1302 	u64 max_bit, min_bit;
1303 	u32 lmask, hmask;
1304 	int err, i;
1305 
1306 	if (WARN_ON(!boot_cpu_has(X86_FEATURE_XSAVES)))
1307 		return;
1308 
1309 	if (!xfeatures_mask_supervisor())
1310 		return;
1311 
1312 	max_bit = __fls(xfeatures_mask_supervisor());
1313 	min_bit = __ffs(xfeatures_mask_supervisor());
1314 
1315 	lmask = xfeatures_mask_supervisor();
1316 	hmask = xfeatures_mask_supervisor() >> 32;
1317 	XSTATE_OP(XSAVES, xstate, lmask, hmask, err);
1318 
1319 	/* We should never fault when copying to a kernel buffer: */
1320 	if (WARN_ON_FPU(err))
1321 		return;
1322 
1323 	/*
1324 	 * At this point, the buffer has only supervisor states and must be
1325 	 * converted back to normal kernel format.
1326 	 */
1327 	header = &xstate->header;
1328 	header->xcomp_bv |= xfeatures_mask_all;
1329 
1330 	/*
1331 	 * This only moves states up in the buffer.  Start with
1332 	 * the last state and move backwards so that states are
1333 	 * not overwritten until after they are moved.  Note:
1334 	 * memmove() allows overlapping src/dst buffers.
1335 	 */
1336 	for (i = max_bit; i >= min_bit; i--) {
1337 		u8 *xbuf = (u8 *)xstate;
1338 
1339 		if (!((header->xfeatures >> i) & 1))
1340 			continue;
1341 
1342 		/* Move xfeature 'i' into its normal location */
1343 		memmove(xbuf + xstate_comp_offsets[i],
1344 			xbuf + xstate_supervisor_only_offsets[i],
1345 			xstate_sizes[i]);
1346 	}
1347 }
1348 
1349 /**
1350  * copy_dynamic_supervisor_to_kernel() - Save dynamic supervisor states to
1351  *                                       an xsave area
1352  * @xstate: A pointer to an xsave area
1353  * @mask: Represent the dynamic supervisor features saved into the xsave area
1354  *
1355  * Only the dynamic supervisor states sets in the mask are saved into the xsave
1356  * area (See the comment in XFEATURE_MASK_DYNAMIC for the details of dynamic
1357  * supervisor feature). Besides the dynamic supervisor states, the legacy
1358  * region and XSAVE header are also saved into the xsave area. The supervisor
1359  * features in the XFEATURE_MASK_SUPERVISOR_SUPPORTED and
1360  * XFEATURE_MASK_SUPERVISOR_UNSUPPORTED are not saved.
1361  *
1362  * The xsave area must be 64-bytes aligned.
1363  */
copy_dynamic_supervisor_to_kernel(struct xregs_state * xstate,u64 mask)1364 void copy_dynamic_supervisor_to_kernel(struct xregs_state *xstate, u64 mask)
1365 {
1366 	u64 dynamic_mask = xfeatures_mask_dynamic() & mask;
1367 	u32 lmask, hmask;
1368 	int err;
1369 
1370 	if (WARN_ON_FPU(!boot_cpu_has(X86_FEATURE_XSAVES)))
1371 		return;
1372 
1373 	if (WARN_ON_FPU(!dynamic_mask))
1374 		return;
1375 
1376 	lmask = dynamic_mask;
1377 	hmask = dynamic_mask >> 32;
1378 
1379 	XSTATE_OP(XSAVES, xstate, lmask, hmask, err);
1380 
1381 	/* Should never fault when copying to a kernel buffer */
1382 	WARN_ON_FPU(err);
1383 }
1384 
1385 /**
1386  * copy_kernel_to_dynamic_supervisor() - Restore dynamic supervisor states from
1387  *                                       an xsave area
1388  * @xstate: A pointer to an xsave area
1389  * @mask: Represent the dynamic supervisor features restored from the xsave area
1390  *
1391  * Only the dynamic supervisor states sets in the mask are restored from the
1392  * xsave area (See the comment in XFEATURE_MASK_DYNAMIC for the details of
1393  * dynamic supervisor feature). Besides the dynamic supervisor states, the
1394  * legacy region and XSAVE header are also restored from the xsave area. The
1395  * supervisor features in the XFEATURE_MASK_SUPERVISOR_SUPPORTED and
1396  * XFEATURE_MASK_SUPERVISOR_UNSUPPORTED are not restored.
1397  *
1398  * The xsave area must be 64-bytes aligned.
1399  */
copy_kernel_to_dynamic_supervisor(struct xregs_state * xstate,u64 mask)1400 void copy_kernel_to_dynamic_supervisor(struct xregs_state *xstate, u64 mask)
1401 {
1402 	u64 dynamic_mask = xfeatures_mask_dynamic() & mask;
1403 	u32 lmask, hmask;
1404 	int err;
1405 
1406 	if (WARN_ON_FPU(!boot_cpu_has(X86_FEATURE_XSAVES)))
1407 		return;
1408 
1409 	if (WARN_ON_FPU(!dynamic_mask))
1410 		return;
1411 
1412 	lmask = dynamic_mask;
1413 	hmask = dynamic_mask >> 32;
1414 
1415 	XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);
1416 
1417 	/* Should never fault when copying from a kernel buffer */
1418 	WARN_ON_FPU(err);
1419 }
1420 
1421 #ifdef CONFIG_PROC_PID_ARCH_STATUS
1422 /*
1423  * Report the amount of time elapsed in millisecond since last AVX512
1424  * use in the task.
1425  */
avx512_status(struct seq_file * m,struct task_struct * task)1426 static void avx512_status(struct seq_file *m, struct task_struct *task)
1427 {
1428 	unsigned long timestamp = READ_ONCE(task->thread.fpu.avx512_timestamp);
1429 	long delta;
1430 
1431 	if (!timestamp) {
1432 		/*
1433 		 * Report -1 if no AVX512 usage
1434 		 */
1435 		delta = -1;
1436 	} else {
1437 		delta = (long)(jiffies - timestamp);
1438 		/*
1439 		 * Cap to LONG_MAX if time difference > LONG_MAX
1440 		 */
1441 		if (delta < 0)
1442 			delta = LONG_MAX;
1443 		delta = jiffies_to_msecs(delta);
1444 	}
1445 
1446 	seq_put_decimal_ll(m, "AVX512_elapsed_ms:\t", delta);
1447 	seq_putc(m, '\n');
1448 }
1449 
1450 /*
1451  * Report architecture specific information
1452  */
proc_pid_arch_status(struct seq_file * m,struct pid_namespace * ns,struct pid * pid,struct task_struct * task)1453 int proc_pid_arch_status(struct seq_file *m, struct pid_namespace *ns,
1454 			struct pid *pid, struct task_struct *task)
1455 {
1456 	/*
1457 	 * Report AVX512 state if the processor and build option supported.
1458 	 */
1459 	if (cpu_feature_enabled(X86_FEATURE_AVX512F))
1460 		avx512_status(m, task);
1461 
1462 	return 0;
1463 }
1464 #endif /* CONFIG_PROC_PID_ARCH_STATUS */
1465