• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Asm versions of Xen pv-ops, suitable for direct use.
4 *
5 * We only bother with direct forms (ie, vcpu in percpu data) of the
6 * operations here; the indirect forms are better handled in C.
7 */
8
9#include <asm/errno.h>
10#include <asm/asm-offsets.h>
11#include <asm/percpu.h>
12#include <asm/processor-flags.h>
13#include <asm/segment.h>
14#include <asm/thread_info.h>
15#include <asm/asm.h>
16#include <asm/frame.h>
17#include <asm/unwind_hints.h>
18
19#include <xen/interface/xen.h>
20
21#include <linux/init.h>
22#include <linux/linkage.h>
23#include <../entry/calling.h>
24
25/*
26 * Enable events.  This clears the event mask and tests the pending
27 * event status with one and operation.  If there are pending events,
28 * then enter the hypervisor to get them handled.
29 */
30SYM_FUNC_START(xen_irq_enable_direct)
31	FRAME_BEGIN
32	/* Unmask events */
33	movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
34
35	/*
36	 * Preempt here doesn't matter because that will deal with any
37	 * pending interrupts.  The pending check may end up being run
38	 * on the wrong CPU, but that doesn't hurt.
39	 */
40
41	/* Test for pending */
42	testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
43	jz 1f
44
45	call check_events
461:
47	FRAME_END
48	RET
49SYM_FUNC_END(xen_irq_enable_direct)
50
51
52/*
53 * Disabling events is simply a matter of making the event mask
54 * non-zero.
55 */
56SYM_FUNC_START(xen_irq_disable_direct)
57	movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
58	RET
59SYM_FUNC_END(xen_irq_disable_direct)
60
61/*
62 * (xen_)save_fl is used to get the current interrupt enable status.
63 * Callers expect the status to be in X86_EFLAGS_IF, and other bits
64 * may be set in the return value.  We take advantage of this by
65 * making sure that X86_EFLAGS_IF has the right value (and other bits
66 * in that byte are 0), but other bits in the return value are
67 * undefined.  We need to toggle the state of the bit, because Xen and
68 * x86 use opposite senses (mask vs enable).
69 */
70SYM_FUNC_START(xen_save_fl_direct)
71	testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
72	setz %ah
73	addb %ah, %ah
74	RET
75SYM_FUNC_END(xen_save_fl_direct)
76
77/*
78 * Force an event check by making a hypercall, but preserve regs
79 * before making the call.
80 */
81SYM_FUNC_START(check_events)
82	FRAME_BEGIN
83	push %rax
84	push %rcx
85	push %rdx
86	push %rsi
87	push %rdi
88	push %r8
89	push %r9
90	push %r10
91	push %r11
92	call xen_force_evtchn_callback
93	pop %r11
94	pop %r10
95	pop %r9
96	pop %r8
97	pop %rdi
98	pop %rsi
99	pop %rdx
100	pop %rcx
101	pop %rax
102	FRAME_END
103	RET
104SYM_FUNC_END(check_events)
105
106SYM_FUNC_START(xen_read_cr2)
107	FRAME_BEGIN
108	_ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX
109	_ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX
110	FRAME_END
111	RET
112SYM_FUNC_END(xen_read_cr2);
113
114SYM_FUNC_START(xen_read_cr2_direct)
115	FRAME_BEGIN
116	_ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX
117	FRAME_END
118	RET
119SYM_FUNC_END(xen_read_cr2_direct);
120
121.macro xen_pv_trap name
122SYM_CODE_START(xen_\name)
123	UNWIND_HINT_ENTRY
124	pop %rcx
125	pop %r11
126	jmp  \name
127SYM_CODE_END(xen_\name)
128_ASM_NOKPROBE(xen_\name)
129.endm
130
131xen_pv_trap asm_exc_divide_error
132xen_pv_trap asm_xenpv_exc_debug
133xen_pv_trap asm_exc_int3
134xen_pv_trap asm_xenpv_exc_nmi
135xen_pv_trap asm_exc_overflow
136xen_pv_trap asm_exc_bounds
137xen_pv_trap asm_exc_invalid_op
138xen_pv_trap asm_exc_device_not_available
139xen_pv_trap asm_xenpv_exc_double_fault
140xen_pv_trap asm_exc_coproc_segment_overrun
141xen_pv_trap asm_exc_invalid_tss
142xen_pv_trap asm_exc_segment_not_present
143xen_pv_trap asm_exc_stack_segment
144xen_pv_trap asm_exc_general_protection
145xen_pv_trap asm_exc_page_fault
146xen_pv_trap asm_exc_spurious_interrupt_bug
147xen_pv_trap asm_exc_coprocessor_error
148xen_pv_trap asm_exc_alignment_check
149#ifdef CONFIG_X86_MCE
150xen_pv_trap asm_xenpv_exc_machine_check
151#endif /* CONFIG_X86_MCE */
152xen_pv_trap asm_exc_simd_coprocessor_error
153#ifdef CONFIG_IA32_EMULATION
154xen_pv_trap entry_INT80_compat
155#endif
156xen_pv_trap asm_exc_xen_unknown_trap
157xen_pv_trap asm_exc_xen_hypervisor_callback
158
159	__INIT
160SYM_CODE_START(xen_early_idt_handler_array)
161	i = 0
162	.rept NUM_EXCEPTION_VECTORS
163	UNWIND_HINT_EMPTY
164	pop %rcx
165	pop %r11
166	jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
167	i = i + 1
168	.fill xen_early_idt_handler_array + i*XEN_EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
169	.endr
170SYM_CODE_END(xen_early_idt_handler_array)
171	__FINIT
172
173hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
174/*
175 * Xen64 iret frame:
176 *
177 *	ss
178 *	rsp
179 *	rflags
180 *	cs
181 *	rip		<-- standard iret frame
182 *
183 *	flags
184 *
185 *	rcx		}
186 *	r11		}<-- pushed by hypercall page
187 * rsp->rax		}
188 */
189SYM_CODE_START(xen_iret)
190	UNWIND_HINT_EMPTY
191	pushq $0
192	jmp hypercall_iret
193SYM_CODE_END(xen_iret)
194
195/*
196 * XEN pv doesn't use trampoline stack, PER_CPU_VAR(cpu_tss_rw + TSS_sp0) is
197 * also the kernel stack.  Reusing swapgs_restore_regs_and_return_to_usermode()
198 * in XEN pv would cause %rsp to move up to the top of the kernel stack and
199 * leave the IRET frame below %rsp, which is dangerous to be corrupted if #NMI
200 * interrupts. And swapgs_restore_regs_and_return_to_usermode() pushing the IRET
201 * frame at the same address is useless.
202 */
203SYM_CODE_START(xenpv_restore_regs_and_return_to_usermode)
204	UNWIND_HINT_REGS
205	POP_REGS
206
207	/* stackleak_erase() can work safely on the kernel stack. */
208	STACKLEAK_ERASE_NOCLOBBER
209
210	addq	$8, %rsp	/* skip regs->orig_ax */
211	jmp xen_iret
212SYM_CODE_END(xenpv_restore_regs_and_return_to_usermode)
213
214/*
215 * Xen handles syscall callbacks much like ordinary exceptions, which
216 * means we have:
217 * - kernel gs
218 * - kernel rsp
219 * - an iret-like stack frame on the stack (including rcx and r11):
220 *	ss
221 *	rsp
222 *	rflags
223 *	cs
224 *	rip
225 *	r11
226 * rsp->rcx
227 */
228
229/* Normal 64-bit system call target */
230SYM_CODE_START(xen_entry_SYSCALL_64)
231	UNWIND_HINT_ENTRY
232	popq %rcx
233	popq %r11
234
235	/*
236	 * Neither Xen nor the kernel really knows what the old SS and
237	 * CS were.  The kernel expects __USER_DS and __USER_CS, so
238	 * report those values even though Xen will guess its own values.
239	 */
240	movq $__USER_DS, 4*8(%rsp)
241	movq $__USER_CS, 1*8(%rsp)
242
243	jmp entry_SYSCALL_64_after_hwframe
244SYM_CODE_END(xen_entry_SYSCALL_64)
245
246#ifdef CONFIG_IA32_EMULATION
247
248/* 32-bit compat syscall target */
249SYM_CODE_START(xen_entry_SYSCALL_compat)
250	UNWIND_HINT_ENTRY
251	popq %rcx
252	popq %r11
253
254	/*
255	 * Neither Xen nor the kernel really knows what the old SS and
256	 * CS were.  The kernel expects __USER32_DS and __USER32_CS, so
257	 * report those values even though Xen will guess its own values.
258	 */
259	movq $__USER32_DS, 4*8(%rsp)
260	movq $__USER32_CS, 1*8(%rsp)
261
262	jmp entry_SYSCALL_compat_after_hwframe
263SYM_CODE_END(xen_entry_SYSCALL_compat)
264
265/* 32-bit compat sysenter target */
266SYM_CODE_START(xen_entry_SYSENTER_compat)
267	UNWIND_HINT_ENTRY
268	/*
269	 * NB: Xen is polite and clears TF from EFLAGS for us.  This means
270	 * that we don't need to guard against single step exceptions here.
271	 */
272	popq %rcx
273	popq %r11
274
275	/*
276	 * Neither Xen nor the kernel really knows what the old SS and
277	 * CS were.  The kernel expects __USER32_DS and __USER32_CS, so
278	 * report those values even though Xen will guess its own values.
279	 */
280	movq $__USER32_DS, 4*8(%rsp)
281	movq $__USER32_CS, 1*8(%rsp)
282
283	jmp entry_SYSENTER_compat_after_hwframe
284SYM_CODE_END(xen_entry_SYSENTER_compat)
285
286#else /* !CONFIG_IA32_EMULATION */
287
288SYM_CODE_START(xen_entry_SYSCALL_compat)
289SYM_CODE_START(xen_entry_SYSENTER_compat)
290	UNWIND_HINT_ENTRY
291	lea 16(%rsp), %rsp	/* strip %rcx, %r11 */
292	mov $-ENOSYS, %rax
293	pushq $0
294	jmp hypercall_iret
295SYM_CODE_END(xen_entry_SYSENTER_compat)
296SYM_CODE_END(xen_entry_SYSCALL_compat)
297
298#endif	/* CONFIG_IA32_EMULATION */
299