1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Asm versions of Xen pv-ops, suitable for direct use. 4 * 5 * We only bother with direct forms (ie, vcpu in pda) of the 6 * operations here; the indirect forms are better handled in C. 7 */ 8 9#include <asm/errno.h> 10#include <asm/percpu.h> 11#include <asm/processor-flags.h> 12#include <asm/segment.h> 13#include <asm/asm-offsets.h> 14#include <asm/thread_info.h> 15#include <asm/asm.h> 16 17#include <xen/interface/xen.h> 18 19#include <linux/init.h> 20#include <linux/linkage.h> 21 22.macro xen_pv_trap name 23ENTRY(xen_\name) 24 pop %rcx 25 pop %r11 26 jmp \name 27END(xen_\name) 28_ASM_NOKPROBE(xen_\name) 29.endm 30 31xen_pv_trap divide_error 32xen_pv_trap debug 33xen_pv_trap xendebug 34xen_pv_trap int3 35xen_pv_trap xennmi 36xen_pv_trap overflow 37xen_pv_trap bounds 38xen_pv_trap invalid_op 39xen_pv_trap device_not_available 40xen_pv_trap double_fault 41xen_pv_trap coprocessor_segment_overrun 42xen_pv_trap invalid_TSS 43xen_pv_trap segment_not_present 44xen_pv_trap stack_segment 45xen_pv_trap general_protection 46xen_pv_trap page_fault 47xen_pv_trap spurious_interrupt_bug 48xen_pv_trap coprocessor_error 49xen_pv_trap alignment_check 50#ifdef CONFIG_X86_MCE 51xen_pv_trap machine_check 52#endif /* CONFIG_X86_MCE */ 53xen_pv_trap simd_coprocessor_error 54#ifdef CONFIG_IA32_EMULATION 55xen_pv_trap entry_INT80_compat 56#endif 57xen_pv_trap hypervisor_callback 58 59 __INIT 60ENTRY(xen_early_idt_handler_array) 61 i = 0 62 .rept NUM_EXCEPTION_VECTORS 63 pop %rcx 64 pop %r11 65 jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE 66 i = i + 1 67 .fill xen_early_idt_handler_array + i*XEN_EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc 68 .endr 69END(xen_early_idt_handler_array) 70 __FINIT 71 72hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32 73/* 74 * Xen64 iret frame: 75 * 76 * ss 77 * rsp 78 * rflags 79 * cs 80 * rip <-- standard iret frame 81 * 82 * flags 83 * 84 * rcx } 85 * r11 }<-- pushed by hypercall page 86 * rsp->rax } 87 */ 88ENTRY(xen_iret) 89 pushq $0 90 jmp hypercall_iret 91 92ENTRY(xen_sysret64) 93 /* 94 * We're already on the usermode stack at this point, but 95 * still with the kernel gs, so we can easily switch back 96 */ 97 movq %rsp, PER_CPU_VAR(rsp_scratch) 98 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp 99 100 pushq $__USER_DS 101 pushq PER_CPU_VAR(rsp_scratch) 102 pushq %r11 103 pushq $__USER_CS 104 pushq %rcx 105 106 pushq $VGCF_in_syscall 107 jmp hypercall_iret 108 109/* 110 * Xen handles syscall callbacks much like ordinary exceptions, which 111 * means we have: 112 * - kernel gs 113 * - kernel rsp 114 * - an iret-like stack frame on the stack (including rcx and r11): 115 * ss 116 * rsp 117 * rflags 118 * cs 119 * rip 120 * r11 121 * rsp->rcx 122 */ 123 124/* Normal 64-bit system call target */ 125ENTRY(xen_syscall_target) 126 popq %rcx 127 popq %r11 128 129 /* 130 * Neither Xen nor the kernel really knows what the old SS and 131 * CS were. The kernel expects __USER_DS and __USER_CS, so 132 * report those values even though Xen will guess its own values. 133 */ 134 movq $__USER_DS, 4*8(%rsp) 135 movq $__USER_CS, 1*8(%rsp) 136 137 jmp entry_SYSCALL_64_after_hwframe 138ENDPROC(xen_syscall_target) 139 140#ifdef CONFIG_IA32_EMULATION 141 142/* 32-bit compat syscall target */ 143ENTRY(xen_syscall32_target) 144 popq %rcx 145 popq %r11 146 147 /* 148 * Neither Xen nor the kernel really knows what the old SS and 149 * CS were. The kernel expects __USER32_DS and __USER32_CS, so 150 * report those values even though Xen will guess its own values. 151 */ 152 movq $__USER32_DS, 4*8(%rsp) 153 movq $__USER32_CS, 1*8(%rsp) 154 155 jmp entry_SYSCALL_compat_after_hwframe 156ENDPROC(xen_syscall32_target) 157 158/* 32-bit compat sysenter target */ 159ENTRY(xen_sysenter_target) 160 mov 0*8(%rsp), %rcx 161 mov 1*8(%rsp), %r11 162 mov 5*8(%rsp), %rsp 163 jmp entry_SYSENTER_compat 164ENDPROC(xen_sysenter_target) 165 166#else /* !CONFIG_IA32_EMULATION */ 167 168ENTRY(xen_syscall32_target) 169ENTRY(xen_sysenter_target) 170 lea 16(%rsp), %rsp /* strip %rcx, %r11 */ 171 mov $-ENOSYS, %rax 172 pushq $0 173 jmp hypercall_iret 174ENDPROC(xen_syscall32_target) 175ENDPROC(xen_sysenter_target) 176 177#endif /* CONFIG_IA32_EMULATION */ 178