1/* SPDX-License-Identifier: GPL-2.0 */ 2 3#include <linux/stringify.h> 4#include <linux/linkage.h> 5#include <asm/dwarf2.h> 6#include <asm/cpufeatures.h> 7#include <asm/alternative.h> 8#include <asm/export.h> 9#include <asm/nospec-branch.h> 10#include <asm/unwind_hints.h> 11#include <asm/frame.h> 12 13 .section .text.__x86.indirect_thunk 14 15.macro RETPOLINE reg 16 ANNOTATE_INTRA_FUNCTION_CALL 17 call .Ldo_rop_\@ 18.Lspec_trap_\@: 19 UNWIND_HINT_EMPTY 20 pause 21 lfence 22 jmp .Lspec_trap_\@ 23.Ldo_rop_\@: 24 mov %\reg, (%_ASM_SP) 25 UNWIND_HINT_FUNC 26 RET 27.endm 28 29.macro THUNK reg 30 31 .align RETPOLINE_THUNK_SIZE 32SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL) 33 UNWIND_HINT_EMPTY 34 35 ALTERNATIVE_2 __stringify(RETPOLINE \reg), \ 36 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg; int3), X86_FEATURE_RETPOLINE_LFENCE, \ 37 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), ALT_NOT(X86_FEATURE_RETPOLINE) 38 39.endm 40 41/* 42 * Despite being an assembler file we can't just use .irp here 43 * because __KSYM_DEPS__ only uses the C preprocessor and would 44 * only see one instance of "__x86_indirect_thunk_\reg" rather 45 * than one per register with the correct names. So we do it 46 * the simple and nasty way... 47 * 48 * Worse, you can only have a single EXPORT_SYMBOL per line, 49 * and CPP can't insert newlines, so we have to repeat everything 50 * at least twice. 51 */ 52 53#define __EXPORT_THUNK(sym) _ASM_NOKPROBE(sym); EXPORT_SYMBOL(sym) 54#define EXPORT_THUNK(reg) __EXPORT_THUNK(__x86_indirect_thunk_ ## reg) 55 56 .align RETPOLINE_THUNK_SIZE 57SYM_CODE_START(__x86_indirect_thunk_array) 58 59#define GEN(reg) THUNK reg 60#include <asm/GEN-for-each-reg.h> 61#undef GEN 62 63 .align RETPOLINE_THUNK_SIZE 64SYM_CODE_END(__x86_indirect_thunk_array) 65 66#define GEN(reg) EXPORT_THUNK(reg) 67#include <asm/GEN-for-each-reg.h> 68#undef GEN 69 70/* 71 * This function name is magical and is used by -mfunction-return=thunk-extern 72 * for the compiler to generate JMPs to it. 73 */ 74#ifdef CONFIG_RETHUNK 75 76 .section .text.__x86.return_thunk 77 78/* 79 * Safety details here pertain to the AMD Zen{1,2} microarchitecture: 80 * 1) The RET at __x86_return_thunk must be on a 64 byte boundary, for 81 * alignment within the BTB. 82 * 2) The instruction at zen_untrain_ret must contain, and not 83 * end with, the 0xc3 byte of the RET. 84 * 3) STIBP must be enabled, or SMT disabled, to prevent the sibling thread 85 * from re-poisioning the BTB prediction. 86 */ 87 .align 64 88 .skip 63, 0xcc 89SYM_FUNC_START_NOALIGN(zen_untrain_ret); 90 91 /* 92 * As executed from zen_untrain_ret, this is: 93 * 94 * TEST $0xcc, %bl 95 * LFENCE 96 * JMP __x86_return_thunk 97 * 98 * Executing the TEST instruction has a side effect of evicting any BTB 99 * prediction (potentially attacker controlled) attached to the RET, as 100 * __x86_return_thunk + 1 isn't an instruction boundary at the moment. 101 */ 102 .byte 0xf6 103 104 /* 105 * As executed from __x86_return_thunk, this is a plain RET. 106 * 107 * As part of the TEST above, RET is the ModRM byte, and INT3 the imm8. 108 * 109 * We subsequently jump backwards and architecturally execute the RET. 110 * This creates a correct BTB prediction (type=ret), but in the 111 * meantime we suffer Straight Line Speculation (because the type was 112 * no branch) which is halted by the INT3. 113 * 114 * With SMT enabled and STIBP active, a sibling thread cannot poison 115 * RET's prediction to a type of its choice, but can evict the 116 * prediction due to competitive sharing. If the prediction is 117 * evicted, __x86_return_thunk will suffer Straight Line Speculation 118 * which will be contained safely by the INT3. 119 */ 120SYM_INNER_LABEL(__x86_return_thunk, SYM_L_GLOBAL) 121 ret 122 int3 123SYM_CODE_END(__x86_return_thunk) 124 125 /* 126 * Ensure the TEST decoding / BTB invalidation is complete. 127 */ 128 lfence 129 130 /* 131 * Jump back and execute the RET in the middle of the TEST instruction. 132 * INT3 is for SLS protection. 133 */ 134 jmp __x86_return_thunk 135 int3 136SYM_FUNC_END(zen_untrain_ret) 137__EXPORT_THUNK(zen_untrain_ret) 138 139EXPORT_SYMBOL(__x86_return_thunk) 140 141#endif /* CONFIG_RETHUNK */ 142