1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_PROCESSOR_H
3 #define _ASM_X86_PROCESSOR_H
4
5 #include <asm/processor-flags.h>
6
7 /* Forward declaration, a strange C thing */
8 struct task_struct;
9 struct mm_struct;
10 struct vm86;
11
12 #include <asm/math_emu.h>
13 #include <asm/segment.h>
14 #include <asm/types.h>
15 #include <uapi/asm/sigcontext.h>
16 #include <asm/current.h>
17 #include <asm/cpufeatures.h>
18 #include <asm/page.h>
19 #include <asm/pgtable_types.h>
20 #include <asm/percpu.h>
21 #include <asm/msr.h>
22 #include <asm/desc_defs.h>
23 #include <asm/nops.h>
24 #include <asm/special_insns.h>
25 #include <asm/fpu/types.h>
26 #include <asm/unwind_hints.h>
27 #include <asm/vdso/processor.h>
28
29 #include <linux/personality.h>
30 #include <linux/cache.h>
31 #include <linux/threads.h>
32 #include <linux/math64.h>
33 #include <linux/err.h>
34 #include <linux/irqflags.h>
35 #include <linux/mem_encrypt.h>
36
37 /*
38 * We handle most unaligned accesses in hardware. On the other hand
39 * unaligned DMA can be quite expensive on some Nehalem processors.
40 *
41 * Based on this we disable the IP header alignment in network drivers.
42 */
43 #define NET_IP_ALIGN 0
44
45 #define HBP_NUM 4
46
47 /*
48 * These alignment constraints are for performance in the vSMP case,
49 * but in the task_struct case we must also meet hardware imposed
50 * alignment requirements of the FPU state:
51 */
52 #ifdef CONFIG_X86_VSMP
53 # define ARCH_MIN_TASKALIGN (1 << INTERNODE_CACHE_SHIFT)
54 # define ARCH_MIN_MMSTRUCT_ALIGN (1 << INTERNODE_CACHE_SHIFT)
55 #else
56 # define ARCH_MIN_TASKALIGN __alignof__(union fpregs_state)
57 # define ARCH_MIN_MMSTRUCT_ALIGN 0
58 #endif
59
60 enum tlb_infos {
61 ENTRIES,
62 NR_INFO
63 };
64
65 extern u16 __read_mostly tlb_lli_4k[NR_INFO];
66 extern u16 __read_mostly tlb_lli_2m[NR_INFO];
67 extern u16 __read_mostly tlb_lli_4m[NR_INFO];
68 extern u16 __read_mostly tlb_lld_4k[NR_INFO];
69 extern u16 __read_mostly tlb_lld_2m[NR_INFO];
70 extern u16 __read_mostly tlb_lld_4m[NR_INFO];
71 extern u16 __read_mostly tlb_lld_1g[NR_INFO];
72
73 /*
74 * CPU type and hardware bug flags. Kept separately for each CPU.
75 * Members of this structure are referenced in head_32.S, so think twice
76 * before touching them. [mj]
77 */
78
79 struct cpuinfo_x86 {
80 __u8 x86; /* CPU family */
81 __u8 x86_vendor; /* CPU vendor */
82 __u8 x86_model;
83 __u8 x86_stepping;
84 #ifdef CONFIG_X86_64
85 /* Number of 4K pages in DTLB/ITLB combined(in pages): */
86 int x86_tlbsize;
87 #endif
88 __u8 x86_virt_bits;
89 __u8 x86_phys_bits;
90 /* CPUID returned core id bits: */
91 __u8 x86_coreid_bits;
92 __u8 cu_id;
93 /* Max extended CPUID function supported: */
94 __u32 extended_cpuid_level;
95 /* Maximum supported CPUID level, -1=no CPUID: */
96 int cpuid_level;
97 __u32 x86_capability[NCAPINTS + NBUGINTS];
98 char x86_vendor_id[16];
99 char x86_model_id[64];
100 /* in KB - valid for CPUS which support this call: */
101 unsigned int x86_cache_size;
102 int x86_cache_alignment; /* In bytes */
103 /* Cache QoS architectural values: */
104 int x86_cache_max_rmid; /* max index */
105 int x86_cache_occ_scale; /* scale to bytes */
106 int x86_power;
107 unsigned long loops_per_jiffy;
108 /* cpuid returned max cores value: */
109 u16 x86_max_cores;
110 u16 apicid;
111 u16 initial_apicid;
112 u16 x86_clflush_size;
113 /* number of cores as seen by the OS: */
114 u16 booted_cores;
115 /* Physical processor id: */
116 u16 phys_proc_id;
117 /* Logical processor id: */
118 u16 logical_proc_id;
119 /* Core id: */
120 u16 cpu_core_id;
121 u16 cpu_die_id;
122 u16 logical_die_id;
123 /* Index into per_cpu list: */
124 u16 cpu_index;
125 u32 microcode;
126 /* Address space bits used by the cache internally */
127 u8 x86_cache_bits;
128 unsigned initialized : 1;
129 } __randomize_layout;
130
131 struct cpuid_regs {
132 u32 eax, ebx, ecx, edx;
133 };
134
135 enum cpuid_regs_idx {
136 CPUID_EAX = 0,
137 CPUID_EBX,
138 CPUID_ECX,
139 CPUID_EDX,
140 };
141
142 #define X86_VENDOR_INTEL 0
143 #define X86_VENDOR_CYRIX 1
144 #define X86_VENDOR_AMD 2
145 #define X86_VENDOR_UMC 3
146 #define X86_VENDOR_CENTAUR 5
147 #define X86_VENDOR_TRANSMETA 7
148 #define X86_VENDOR_NSC 8
149 #define X86_VENDOR_HYGON 9
150 #define X86_VENDOR_ZHAOXIN 10
151 #define X86_VENDOR_NUM 11
152
153 #define X86_VENDOR_UNKNOWN 0xff
154
155 /*
156 * capabilities of CPUs
157 */
158 extern struct cpuinfo_x86 boot_cpu_data;
159 extern struct cpuinfo_x86 new_cpu_data;
160
161 extern struct x86_hw_tss doublefault_tss;
162 extern __u32 cpu_caps_cleared[NCAPINTS + NBUGINTS];
163 extern __u32 cpu_caps_set[NCAPINTS + NBUGINTS];
164
165 #ifdef CONFIG_SMP
166 DECLARE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
167 #define cpu_data(cpu) per_cpu(cpu_info, cpu)
168 #else
169 #define cpu_info boot_cpu_data
170 #define cpu_data(cpu) boot_cpu_data
171 #endif
172
173 extern const struct seq_operations cpuinfo_op;
174
175 #define cache_line_size() (boot_cpu_data.x86_cache_alignment)
176
177 extern void cpu_detect(struct cpuinfo_x86 *c);
178
l1tf_pfn_limit(void)179 static inline unsigned long long l1tf_pfn_limit(void)
180 {
181 return BIT_ULL(boot_cpu_data.x86_cache_bits - 1 - PAGE_SHIFT);
182 }
183
184 extern void early_cpu_init(void);
185 extern void identify_boot_cpu(void);
186 extern void identify_secondary_cpu(struct cpuinfo_x86 *);
187 extern void print_cpu_info(struct cpuinfo_x86 *);
188 void print_cpu_msr(struct cpuinfo_x86 *);
189
190 #ifdef CONFIG_X86_32
191 extern int have_cpuid_p(void);
192 #else
have_cpuid_p(void)193 static inline int have_cpuid_p(void)
194 {
195 return 1;
196 }
197 #endif
native_cpuid(unsigned int * eax,unsigned int * ebx,unsigned int * ecx,unsigned int * edx)198 static inline void native_cpuid(unsigned int *eax, unsigned int *ebx,
199 unsigned int *ecx, unsigned int *edx)
200 {
201 /* ecx is often an input as well as an output. */
202 asm volatile("cpuid"
203 : "=a" (*eax),
204 "=b" (*ebx),
205 "=c" (*ecx),
206 "=d" (*edx)
207 : "0" (*eax), "2" (*ecx)
208 : "memory");
209 }
210
211 #define native_cpuid_reg(reg) \
212 static inline unsigned int native_cpuid_##reg(unsigned int op) \
213 { \
214 unsigned int eax = op, ebx, ecx = 0, edx; \
215 \
216 native_cpuid(&eax, &ebx, &ecx, &edx); \
217 \
218 return reg; \
219 }
220
221 /*
222 * Native CPUID functions returning a single datum.
223 */
224 native_cpuid_reg(eax)
native_cpuid_reg(ebx)225 native_cpuid_reg(ebx)
226 native_cpuid_reg(ecx)
227 native_cpuid_reg(edx)
228
229 /*
230 * Friendlier CR3 helpers.
231 */
232 static inline unsigned long read_cr3_pa(void)
233 {
234 return __read_cr3() & CR3_ADDR_MASK;
235 }
236
native_read_cr3_pa(void)237 static inline unsigned long native_read_cr3_pa(void)
238 {
239 return __native_read_cr3() & CR3_ADDR_MASK;
240 }
241
load_cr3(pgd_t * pgdir)242 static inline void load_cr3(pgd_t *pgdir)
243 {
244 write_cr3(__sme_pa(pgdir));
245 }
246
247 /*
248 * Note that while the legacy 'TSS' name comes from 'Task State Segment',
249 * on modern x86 CPUs the TSS also holds information important to 64-bit mode,
250 * unrelated to the task-switch mechanism:
251 */
252 #ifdef CONFIG_X86_32
253 /* This is the TSS defined by the hardware. */
254 struct x86_hw_tss {
255 unsigned short back_link, __blh;
256 unsigned long sp0;
257 unsigned short ss0, __ss0h;
258 unsigned long sp1;
259
260 /*
261 * We don't use ring 1, so ss1 is a convenient scratch space in
262 * the same cacheline as sp0. We use ss1 to cache the value in
263 * MSR_IA32_SYSENTER_CS. When we context switch
264 * MSR_IA32_SYSENTER_CS, we first check if the new value being
265 * written matches ss1, and, if it's not, then we wrmsr the new
266 * value and update ss1.
267 *
268 * The only reason we context switch MSR_IA32_SYSENTER_CS is
269 * that we set it to zero in vm86 tasks to avoid corrupting the
270 * stack if we were to go through the sysenter path from vm86
271 * mode.
272 */
273 unsigned short ss1; /* MSR_IA32_SYSENTER_CS */
274
275 unsigned short __ss1h;
276 unsigned long sp2;
277 unsigned short ss2, __ss2h;
278 unsigned long __cr3;
279 unsigned long ip;
280 unsigned long flags;
281 unsigned long ax;
282 unsigned long cx;
283 unsigned long dx;
284 unsigned long bx;
285 unsigned long sp;
286 unsigned long bp;
287 unsigned long si;
288 unsigned long di;
289 unsigned short es, __esh;
290 unsigned short cs, __csh;
291 unsigned short ss, __ssh;
292 unsigned short ds, __dsh;
293 unsigned short fs, __fsh;
294 unsigned short gs, __gsh;
295 unsigned short ldt, __ldth;
296 unsigned short trace;
297 unsigned short io_bitmap_base;
298
299 } __attribute__((packed));
300 #else
301 struct x86_hw_tss {
302 u32 reserved1;
303 u64 sp0;
304
305 /*
306 * We store cpu_current_top_of_stack in sp1 so it's always accessible.
307 * Linux does not use ring 1, so sp1 is not otherwise needed.
308 */
309 u64 sp1;
310
311 /*
312 * Since Linux does not use ring 2, the 'sp2' slot is unused by
313 * hardware. entry_SYSCALL_64 uses it as scratch space to stash
314 * the user RSP value.
315 */
316 u64 sp2;
317
318 u64 reserved2;
319 u64 ist[7];
320 u32 reserved3;
321 u32 reserved4;
322 u16 reserved5;
323 u16 io_bitmap_base;
324
325 } __attribute__((packed));
326 #endif
327
328 /*
329 * IO-bitmap sizes:
330 */
331 #define IO_BITMAP_BITS 65536
332 #define IO_BITMAP_BYTES (IO_BITMAP_BITS/8)
333 #define IO_BITMAP_LONGS (IO_BITMAP_BYTES/sizeof(long))
334 #define IO_BITMAP_OFFSET (offsetof(struct tss_struct, io_bitmap) - offsetof(struct tss_struct, x86_tss))
335 #define INVALID_IO_BITMAP_OFFSET 0x8000
336
337 struct entry_stack {
338 char stack[PAGE_SIZE];
339 };
340
341 struct entry_stack_page {
342 struct entry_stack stack;
343 } __aligned(PAGE_SIZE);
344
345 struct tss_struct {
346 /*
347 * The fixed hardware portion. This must not cross a page boundary
348 * at risk of violating the SDM's advice and potentially triggering
349 * errata.
350 */
351 struct x86_hw_tss x86_tss;
352
353 /*
354 * The extra 1 is there because the CPU will access an
355 * additional byte beyond the end of the IO permission
356 * bitmap. The extra byte must be all 1 bits, and must
357 * be within the limit.
358 */
359 unsigned long io_bitmap[IO_BITMAP_LONGS + 1];
360 } __aligned(PAGE_SIZE);
361
362 DECLARE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss_rw);
363
364 /*
365 * sizeof(unsigned long) coming from an extra "long" at the end
366 * of the iobitmap.
367 *
368 * -1? seg base+limit should be pointing to the address of the
369 * last valid byte
370 */
371 #define __KERNEL_TSS_LIMIT \
372 (IO_BITMAP_OFFSET + IO_BITMAP_BYTES + sizeof(unsigned long) - 1)
373
374 /* Per CPU interrupt stacks */
375 struct irq_stack {
376 char stack[IRQ_STACK_SIZE];
377 } __aligned(IRQ_STACK_SIZE);
378
379 DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);
380
381 #ifdef CONFIG_X86_32
382 DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack);
383 #else
384 /* The RO copy can't be accessed with this_cpu_xyz(), so use the RW copy. */
385 #define cpu_current_top_of_stack cpu_tss_rw.x86_tss.sp1
386 #endif
387
388 #ifdef CONFIG_X86_64
389 struct fixed_percpu_data {
390 /*
391 * GCC hardcodes the stack canary as %gs:40. Since the
392 * irq_stack is the object at %gs:0, we reserve the bottom
393 * 48 bytes of the irq stack for the canary.
394 */
395 char gs_base[40];
396 unsigned long stack_canary;
397 };
398
399 DECLARE_PER_CPU_FIRST(struct fixed_percpu_data, fixed_percpu_data) __visible;
400 DECLARE_INIT_PER_CPU(fixed_percpu_data);
401
cpu_kernelmode_gs_base(int cpu)402 static inline unsigned long cpu_kernelmode_gs_base(int cpu)
403 {
404 return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu);
405 }
406
407 DECLARE_PER_CPU(unsigned int, irq_count);
408 extern asmlinkage void ignore_sysret(void);
409
410 #if IS_ENABLED(CONFIG_KVM)
411 /* Save actual FS/GS selectors and bases to current->thread */
412 void save_fsgs_for_kvm(void);
413 #endif
414 #else /* X86_64 */
415 #ifdef CONFIG_STACKPROTECTOR
416 /*
417 * Make sure stack canary segment base is cached-aligned:
418 * "For Intel Atom processors, avoid non zero segment base address
419 * that is not aligned to cache line boundary at all cost."
420 * (Optim Ref Manual Assembly/Compiler Coding Rule 15.)
421 */
422 struct stack_canary {
423 char __pad[20]; /* canary at %gs:20 */
424 unsigned long canary;
425 };
426 DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
427 #endif
428 /* Per CPU softirq stack pointer */
429 DECLARE_PER_CPU(struct irq_stack *, softirq_stack_ptr);
430 #endif /* X86_64 */
431
432 extern unsigned int fpu_kernel_xstate_size;
433 extern unsigned int fpu_user_xstate_size;
434
435 struct perf_event;
436
437 typedef struct {
438 unsigned long seg;
439 } mm_segment_t;
440
441 struct thread_struct {
442 /* Cached TLS descriptors: */
443 struct desc_struct tls_array[GDT_ENTRY_TLS_ENTRIES];
444 #ifdef CONFIG_X86_32
445 unsigned long sp0;
446 #endif
447 unsigned long sp;
448 #ifdef CONFIG_X86_32
449 unsigned long sysenter_cs;
450 #else
451 unsigned short es;
452 unsigned short ds;
453 unsigned short fsindex;
454 unsigned short gsindex;
455 #endif
456
457 #ifdef CONFIG_X86_64
458 unsigned long fsbase;
459 unsigned long gsbase;
460 #else
461 /*
462 * XXX: this could presumably be unsigned short. Alternatively,
463 * 32-bit kernels could be taught to use fsindex instead.
464 */
465 unsigned long fs;
466 unsigned long gs;
467 #endif
468
469 /* Save middle states of ptrace breakpoints */
470 struct perf_event *ptrace_bps[HBP_NUM];
471 /* Debug status used for traps, single steps, etc... */
472 unsigned long debugreg6;
473 /* Keep track of the exact dr7 value set by the user */
474 unsigned long ptrace_dr7;
475 /* Fault info: */
476 unsigned long cr2;
477 unsigned long trap_nr;
478 unsigned long error_code;
479 #ifdef CONFIG_VM86
480 /* Virtual 86 mode info */
481 struct vm86 *vm86;
482 #endif
483 /* IO permissions: */
484 unsigned long *io_bitmap_ptr;
485 unsigned long iopl;
486 /* Max allowed port in the bitmap, in bytes: */
487 unsigned io_bitmap_max;
488
489 mm_segment_t addr_limit;
490
491 unsigned int sig_on_uaccess_err:1;
492 unsigned int uaccess_err:1; /* uaccess failed */
493
494 /* Floating point and extended processor state */
495 struct fpu fpu;
496 /*
497 * WARNING: 'fpu' is dynamically-sized. It *MUST* be at
498 * the end.
499 */
500 };
501
502 /* Whitelist the FPU state from the task_struct for hardened usercopy. */
arch_thread_struct_whitelist(unsigned long * offset,unsigned long * size)503 static inline void arch_thread_struct_whitelist(unsigned long *offset,
504 unsigned long *size)
505 {
506 *offset = offsetof(struct thread_struct, fpu.state);
507 *size = fpu_kernel_xstate_size;
508 }
509
510 /*
511 * Set IOPL bits in EFLAGS from given mask
512 */
native_set_iopl_mask(unsigned mask)513 static inline void native_set_iopl_mask(unsigned mask)
514 {
515 #ifdef CONFIG_X86_32
516 unsigned int reg;
517
518 asm volatile ("pushfl;"
519 "popl %0;"
520 "andl %1, %0;"
521 "orl %2, %0;"
522 "pushl %0;"
523 "popfl"
524 : "=&r" (reg)
525 : "i" (~X86_EFLAGS_IOPL), "r" (mask));
526 #endif
527 }
528
529 static inline void
native_load_sp0(unsigned long sp0)530 native_load_sp0(unsigned long sp0)
531 {
532 this_cpu_write(cpu_tss_rw.x86_tss.sp0, sp0);
533 }
534
native_swapgs(void)535 static inline void native_swapgs(void)
536 {
537 #ifdef CONFIG_X86_64
538 asm volatile("swapgs" ::: "memory");
539 #endif
540 }
541
current_top_of_stack(void)542 static inline unsigned long current_top_of_stack(void)
543 {
544 /*
545 * We can't read directly from tss.sp0: sp0 on x86_32 is special in
546 * and around vm86 mode and sp0 on x86_64 is special because of the
547 * entry trampoline.
548 */
549 return this_cpu_read_stable(cpu_current_top_of_stack);
550 }
551
on_thread_stack(void)552 static inline bool on_thread_stack(void)
553 {
554 return (unsigned long)(current_top_of_stack() -
555 current_stack_pointer) < THREAD_SIZE;
556 }
557
558 #ifdef CONFIG_PARAVIRT_XXL
559 #include <asm/paravirt.h>
560 #else
561 #define __cpuid native_cpuid
562
load_sp0(unsigned long sp0)563 static inline void load_sp0(unsigned long sp0)
564 {
565 native_load_sp0(sp0);
566 }
567
568 #define set_iopl_mask native_set_iopl_mask
569 #endif /* CONFIG_PARAVIRT_XXL */
570
571 /* Free all resources held by a thread. */
572 extern void release_thread(struct task_struct *);
573
574 unsigned long get_wchan(struct task_struct *p);
575
576 /*
577 * Generic CPUID function
578 * clear %ecx since some cpus (Cyrix MII) do not set or clear %ecx
579 * resulting in stale register contents being returned.
580 */
cpuid(unsigned int op,unsigned int * eax,unsigned int * ebx,unsigned int * ecx,unsigned int * edx)581 static inline void cpuid(unsigned int op,
582 unsigned int *eax, unsigned int *ebx,
583 unsigned int *ecx, unsigned int *edx)
584 {
585 *eax = op;
586 *ecx = 0;
587 __cpuid(eax, ebx, ecx, edx);
588 }
589
590 /* Some CPUID calls want 'count' to be placed in ecx */
cpuid_count(unsigned int op,int count,unsigned int * eax,unsigned int * ebx,unsigned int * ecx,unsigned int * edx)591 static inline void cpuid_count(unsigned int op, int count,
592 unsigned int *eax, unsigned int *ebx,
593 unsigned int *ecx, unsigned int *edx)
594 {
595 *eax = op;
596 *ecx = count;
597 __cpuid(eax, ebx, ecx, edx);
598 }
599
600 /*
601 * CPUID functions returning a single datum
602 */
cpuid_eax(unsigned int op)603 static inline unsigned int cpuid_eax(unsigned int op)
604 {
605 unsigned int eax, ebx, ecx, edx;
606
607 cpuid(op, &eax, &ebx, &ecx, &edx);
608
609 return eax;
610 }
611
cpuid_ebx(unsigned int op)612 static inline unsigned int cpuid_ebx(unsigned int op)
613 {
614 unsigned int eax, ebx, ecx, edx;
615
616 cpuid(op, &eax, &ebx, &ecx, &edx);
617
618 return ebx;
619 }
620
cpuid_ecx(unsigned int op)621 static inline unsigned int cpuid_ecx(unsigned int op)
622 {
623 unsigned int eax, ebx, ecx, edx;
624
625 cpuid(op, &eax, &ebx, &ecx, &edx);
626
627 return ecx;
628 }
629
cpuid_edx(unsigned int op)630 static inline unsigned int cpuid_edx(unsigned int op)
631 {
632 unsigned int eax, ebx, ecx, edx;
633
634 cpuid(op, &eax, &ebx, &ecx, &edx);
635
636 return edx;
637 }
638
639 /*
640 * This function forces the icache and prefetched instruction stream to
641 * catch up with reality in two very specific cases:
642 *
643 * a) Text was modified using one virtual address and is about to be executed
644 * from the same physical page at a different virtual address.
645 *
646 * b) Text was modified on a different CPU, may subsequently be
647 * executed on this CPU, and you want to make sure the new version
648 * gets executed. This generally means you're calling this in a IPI.
649 *
650 * If you're calling this for a different reason, you're probably doing
651 * it wrong.
652 */
sync_core(void)653 static inline void sync_core(void)
654 {
655 /*
656 * There are quite a few ways to do this. IRET-to-self is nice
657 * because it works on every CPU, at any CPL (so it's compatible
658 * with paravirtualization), and it never exits to a hypervisor.
659 * The only down sides are that it's a bit slow (it seems to be
660 * a bit more than 2x slower than the fastest options) and that
661 * it unmasks NMIs. The "push %cs" is needed because, in
662 * paravirtual environments, __KERNEL_CS may not be a valid CS
663 * value when we do IRET directly.
664 *
665 * In case NMI unmasking or performance ever becomes a problem,
666 * the next best option appears to be MOV-to-CR2 and an
667 * unconditional jump. That sequence also works on all CPUs,
668 * but it will fault at CPL3 (i.e. Xen PV).
669 *
670 * CPUID is the conventional way, but it's nasty: it doesn't
671 * exist on some 486-like CPUs, and it usually exits to a
672 * hypervisor.
673 *
674 * Like all of Linux's memory ordering operations, this is a
675 * compiler barrier as well.
676 */
677 #ifdef CONFIG_X86_32
678 asm volatile (
679 "pushfl\n\t"
680 "pushl %%cs\n\t"
681 "pushl $1f\n\t"
682 "iret\n\t"
683 "1:"
684 : ASM_CALL_CONSTRAINT : : "memory");
685 #else
686 unsigned int tmp;
687
688 asm volatile (
689 UNWIND_HINT_SAVE
690 "mov %%ss, %0\n\t"
691 "pushq %q0\n\t"
692 "pushq %%rsp\n\t"
693 "addq $8, (%%rsp)\n\t"
694 "pushfq\n\t"
695 "mov %%cs, %0\n\t"
696 "pushq %q0\n\t"
697 "pushq $1f\n\t"
698 "iretq\n\t"
699 UNWIND_HINT_RESTORE
700 "1:"
701 : "=&r" (tmp), ASM_CALL_CONSTRAINT : : "cc", "memory");
702 #endif
703 }
704
705 extern void select_idle_routine(const struct cpuinfo_x86 *c);
706 extern void amd_e400_c1e_apic_setup(void);
707
708 extern unsigned long boot_option_idle_override;
709
710 enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT,
711 IDLE_POLL};
712
713 extern void enable_sep_cpu(void);
714 extern int sysenter_setup(void);
715
716
717 /* Defined in head.S */
718 extern struct desc_ptr early_gdt_descr;
719
720 extern void switch_to_new_gdt(int);
721 extern void load_direct_gdt(int);
722 extern void load_fixmap_gdt(int);
723 extern void load_percpu_segment(int);
724 extern void cpu_init(void);
725 extern void cr4_init(void);
726
get_debugctlmsr(void)727 static inline unsigned long get_debugctlmsr(void)
728 {
729 unsigned long debugctlmsr = 0;
730
731 #ifndef CONFIG_X86_DEBUGCTLMSR
732 if (boot_cpu_data.x86 < 6)
733 return 0;
734 #endif
735 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
736
737 return debugctlmsr;
738 }
739
update_debugctlmsr(unsigned long debugctlmsr)740 static inline void update_debugctlmsr(unsigned long debugctlmsr)
741 {
742 #ifndef CONFIG_X86_DEBUGCTLMSR
743 if (boot_cpu_data.x86 < 6)
744 return;
745 #endif
746 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
747 }
748
749 extern void set_task_blockstep(struct task_struct *task, bool on);
750
751 /* Boot loader type from the setup header: */
752 extern int bootloader_type;
753 extern int bootloader_version;
754
755 extern char ignore_fpu_irq;
756
757 #define HAVE_ARCH_PICK_MMAP_LAYOUT 1
758 #define ARCH_HAS_PREFETCHW
759 #define ARCH_HAS_SPINLOCK_PREFETCH
760
761 #ifdef CONFIG_X86_32
762 # define BASE_PREFETCH ""
763 # define ARCH_HAS_PREFETCH
764 #else
765 # define BASE_PREFETCH "prefetcht0 %P1"
766 #endif
767
768 /*
769 * Prefetch instructions for Pentium III (+) and AMD Athlon (+)
770 *
771 * It's not worth to care about 3dnow prefetches for the K6
772 * because they are microcoded there and very slow.
773 */
prefetch(const void * x)774 static inline void prefetch(const void *x)
775 {
776 alternative_input(BASE_PREFETCH, "prefetchnta %P1",
777 X86_FEATURE_XMM,
778 "m" (*(const char *)x));
779 }
780
781 /*
782 * 3dnow prefetch to get an exclusive cache line.
783 * Useful for spinlocks to avoid one state transition in the
784 * cache coherency protocol:
785 */
prefetchw(const void * x)786 static inline void prefetchw(const void *x)
787 {
788 alternative_input(BASE_PREFETCH, "prefetchw %P1",
789 X86_FEATURE_3DNOWPREFETCH,
790 "m" (*(const char *)x));
791 }
792
spin_lock_prefetch(const void * x)793 static inline void spin_lock_prefetch(const void *x)
794 {
795 prefetchw(x);
796 }
797
798 #define TOP_OF_INIT_STACK ((unsigned long)&init_stack + sizeof(init_stack) - \
799 TOP_OF_KERNEL_STACK_PADDING)
800
801 #define task_top_of_stack(task) ((unsigned long)(task_pt_regs(task) + 1))
802
803 #define task_pt_regs(task) \
804 ({ \
805 unsigned long __ptr = (unsigned long)task_stack_page(task); \
806 __ptr += THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING; \
807 ((struct pt_regs *)__ptr) - 1; \
808 })
809
810 #ifdef CONFIG_X86_32
811 /*
812 * User space process size: 3GB (default).
813 */
814 #define IA32_PAGE_OFFSET PAGE_OFFSET
815 #define TASK_SIZE PAGE_OFFSET
816 #define TASK_SIZE_LOW TASK_SIZE
817 #define TASK_SIZE_MAX TASK_SIZE
818 #define DEFAULT_MAP_WINDOW TASK_SIZE
819 #define STACK_TOP TASK_SIZE
820 #define STACK_TOP_MAX STACK_TOP
821
822 #define INIT_THREAD { \
823 .sp0 = TOP_OF_INIT_STACK, \
824 .sysenter_cs = __KERNEL_CS, \
825 .io_bitmap_ptr = NULL, \
826 .addr_limit = KERNEL_DS, \
827 }
828
829 #define KSTK_ESP(task) (task_pt_regs(task)->sp)
830
831 #else
832 /*
833 * User space process size. This is the first address outside the user range.
834 * There are a few constraints that determine this:
835 *
836 * On Intel CPUs, if a SYSCALL instruction is at the highest canonical
837 * address, then that syscall will enter the kernel with a
838 * non-canonical return address, and SYSRET will explode dangerously.
839 * We avoid this particular problem by preventing anything executable
840 * from being mapped at the maximum canonical address.
841 *
842 * On AMD CPUs in the Ryzen family, there's a nasty bug in which the
843 * CPUs malfunction if they execute code from the highest canonical page.
844 * They'll speculate right off the end of the canonical space, and
845 * bad things happen. This is worked around in the same way as the
846 * Intel problem.
847 *
848 * With page table isolation enabled, we map the LDT in ... [stay tuned]
849 */
850 #define TASK_SIZE_MAX ((1UL << __VIRTUAL_MASK_SHIFT) - PAGE_SIZE)
851
852 #define DEFAULT_MAP_WINDOW ((1UL << 47) - PAGE_SIZE)
853
854 /* This decides where the kernel will search for a free chunk of vm
855 * space during mmap's.
856 */
857 #define IA32_PAGE_OFFSET ((current->personality & ADDR_LIMIT_3GB) ? \
858 0xc0000000 : 0xFFFFe000)
859
860 #define TASK_SIZE_LOW (test_thread_flag(TIF_ADDR32) ? \
861 IA32_PAGE_OFFSET : DEFAULT_MAP_WINDOW)
862 #define TASK_SIZE (test_thread_flag(TIF_ADDR32) ? \
863 IA32_PAGE_OFFSET : TASK_SIZE_MAX)
864 #define TASK_SIZE_OF(child) ((test_tsk_thread_flag(child, TIF_ADDR32)) ? \
865 IA32_PAGE_OFFSET : TASK_SIZE_MAX)
866
867 #define STACK_TOP TASK_SIZE_LOW
868 #define STACK_TOP_MAX TASK_SIZE_MAX
869
870 #define INIT_THREAD { \
871 .addr_limit = KERNEL_DS, \
872 }
873
874 extern unsigned long KSTK_ESP(struct task_struct *task);
875
876 #endif /* CONFIG_X86_64 */
877
878 extern void start_thread(struct pt_regs *regs, unsigned long new_ip,
879 unsigned long new_sp);
880
881 /*
882 * This decides where the kernel will search for a free chunk of vm
883 * space during mmap's.
884 */
885 #define __TASK_UNMAPPED_BASE(task_size) (PAGE_ALIGN(task_size / 3))
886 #define TASK_UNMAPPED_BASE __TASK_UNMAPPED_BASE(TASK_SIZE_LOW)
887
888 #define KSTK_EIP(task) (task_pt_regs(task)->ip)
889
890 /* Get/set a process' ability to use the timestamp counter instruction */
891 #define GET_TSC_CTL(adr) get_tsc_mode((adr))
892 #define SET_TSC_CTL(val) set_tsc_mode((val))
893
894 extern int get_tsc_mode(unsigned long adr);
895 extern int set_tsc_mode(unsigned int val);
896
897 DECLARE_PER_CPU(u64, msr_misc_features_shadow);
898
899 /* Register/unregister a process' MPX related resource */
900 #define MPX_ENABLE_MANAGEMENT() mpx_enable_management()
901 #define MPX_DISABLE_MANAGEMENT() mpx_disable_management()
902
903 #ifdef CONFIG_X86_INTEL_MPX
904 extern int mpx_enable_management(void);
905 extern int mpx_disable_management(void);
906 #else
mpx_enable_management(void)907 static inline int mpx_enable_management(void)
908 {
909 return -EINVAL;
910 }
mpx_disable_management(void)911 static inline int mpx_disable_management(void)
912 {
913 return -EINVAL;
914 }
915 #endif /* CONFIG_X86_INTEL_MPX */
916
917 #ifdef CONFIG_CPU_SUP_AMD
918 extern u16 amd_get_nb_id(int cpu);
919 extern u32 amd_get_nodes_per_socket(void);
920 #else
amd_get_nb_id(int cpu)921 static inline u16 amd_get_nb_id(int cpu) { return 0; }
amd_get_nodes_per_socket(void)922 static inline u32 amd_get_nodes_per_socket(void) { return 0; }
923 #endif
924
hypervisor_cpuid_base(const char * sig,uint32_t leaves)925 static inline uint32_t hypervisor_cpuid_base(const char *sig, uint32_t leaves)
926 {
927 uint32_t base, eax, signature[3];
928
929 for (base = 0x40000000; base < 0x40010000; base += 0x100) {
930 cpuid(base, &eax, &signature[0], &signature[1], &signature[2]);
931
932 if (!memcmp(sig, signature, 12) &&
933 (leaves == 0 || ((eax - base) >= leaves)))
934 return base;
935 }
936
937 return 0;
938 }
939
940 extern unsigned long arch_align_stack(unsigned long sp);
941 void free_init_pages(const char *what, unsigned long begin, unsigned long end);
942 extern void free_kernel_image_pages(void *begin, void *end);
943
944 void default_idle(void);
945 #ifdef CONFIG_XEN
946 bool xen_set_default_idle(void);
947 #else
948 #define xen_set_default_idle 0
949 #endif
950
951 void stop_this_cpu(void *dummy);
952 void df_debug(struct pt_regs *regs, long error_code);
953 void microcode_check(void);
954
955 enum l1tf_mitigations {
956 L1TF_MITIGATION_OFF,
957 L1TF_MITIGATION_FLUSH_NOWARN,
958 L1TF_MITIGATION_FLUSH,
959 L1TF_MITIGATION_FLUSH_NOSMT,
960 L1TF_MITIGATION_FULL,
961 L1TF_MITIGATION_FULL_FORCE
962 };
963
964 extern enum l1tf_mitigations l1tf_mitigation;
965
966 enum mds_mitigations {
967 MDS_MITIGATION_OFF,
968 MDS_MITIGATION_FULL,
969 MDS_MITIGATION_VMWERV,
970 };
971
972 enum taa_mitigations {
973 TAA_MITIGATION_OFF,
974 TAA_MITIGATION_UCODE_NEEDED,
975 TAA_MITIGATION_VERW,
976 TAA_MITIGATION_TSX_DISABLED,
977 };
978
979 extern bool gds_ucode_mitigated(void);
980
981 #endif /* _ASM_X86_PROCESSOR_H */
982