1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Intel Transactional Synchronization Extensions (TSX) control.
4 *
5 * Copyright (C) 2019-2021 Intel Corporation
6 *
7 * Author:
8 * Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
9 */
10
11 #include <linux/cpufeature.h>
12
13 #include <asm/cmdline.h>
14
15 #include "cpu.h"
16
17 #undef pr_fmt
18 #define pr_fmt(fmt) "tsx: " fmt
19
20 enum tsx_ctrl_states tsx_ctrl_state __ro_after_init = TSX_CTRL_NOT_SUPPORTED;
21
tsx_disable(void)22 static void tsx_disable(void)
23 {
24 u64 tsx;
25
26 rdmsrl(MSR_IA32_TSX_CTRL, tsx);
27
28 /* Force all transactions to immediately abort */
29 tsx |= TSX_CTRL_RTM_DISABLE;
30
31 /*
32 * Ensure TSX support is not enumerated in CPUID.
33 * This is visible to userspace and will ensure they
34 * do not waste resources trying TSX transactions that
35 * will always abort.
36 */
37 tsx |= TSX_CTRL_CPUID_CLEAR;
38
39 wrmsrl(MSR_IA32_TSX_CTRL, tsx);
40 }
41
tsx_enable(void)42 static void tsx_enable(void)
43 {
44 u64 tsx;
45
46 rdmsrl(MSR_IA32_TSX_CTRL, tsx);
47
48 /* Enable the RTM feature in the cpu */
49 tsx &= ~TSX_CTRL_RTM_DISABLE;
50
51 /*
52 * Ensure TSX support is enumerated in CPUID.
53 * This is visible to userspace and will ensure they
54 * can enumerate and use the TSX feature.
55 */
56 tsx &= ~TSX_CTRL_CPUID_CLEAR;
57
58 wrmsrl(MSR_IA32_TSX_CTRL, tsx);
59 }
60
x86_get_tsx_auto_mode(void)61 static enum tsx_ctrl_states x86_get_tsx_auto_mode(void)
62 {
63 if (boot_cpu_has_bug(X86_BUG_TAA))
64 return TSX_CTRL_DISABLE;
65
66 return TSX_CTRL_ENABLE;
67 }
68
69 /*
70 * Disabling TSX is not a trivial business.
71 *
72 * First of all, there's a CPUID bit: X86_FEATURE_RTM_ALWAYS_ABORT
73 * which says that TSX is practically disabled (all transactions are
74 * aborted by default). When that bit is set, the kernel unconditionally
75 * disables TSX.
76 *
77 * In order to do that, however, it needs to dance a bit:
78 *
79 * 1. The first method to disable it is through MSR_TSX_FORCE_ABORT and
80 * the MSR is present only when *two* CPUID bits are set:
81 *
82 * - X86_FEATURE_RTM_ALWAYS_ABORT
83 * - X86_FEATURE_TSX_FORCE_ABORT
84 *
85 * 2. The second method is for CPUs which do not have the above-mentioned
86 * MSR: those use a different MSR - MSR_IA32_TSX_CTRL and disable TSX
87 * through that one. Those CPUs can also have the initially mentioned
88 * CPUID bit X86_FEATURE_RTM_ALWAYS_ABORT set and for those the same strategy
89 * applies: TSX gets disabled unconditionally.
90 *
91 * When either of the two methods are present, the kernel disables TSX and
92 * clears the respective RTM and HLE feature flags.
93 *
94 * An additional twist in the whole thing presents late microcode loading
95 * which, when done, may cause for the X86_FEATURE_RTM_ALWAYS_ABORT CPUID
96 * bit to be set after the update.
97 *
98 * A subsequent hotplug operation on any logical CPU except the BSP will
99 * cause for the supported CPUID feature bits to get re-detected and, if
100 * RTM and HLE get cleared all of a sudden, but, userspace did consult
101 * them before the update, then funny explosions will happen. Long story
102 * short: the kernel doesn't modify CPUID feature bits after booting.
103 *
104 * That's why, this function's call in init_intel() doesn't clear the
105 * feature flags.
106 */
tsx_clear_cpuid(void)107 static void tsx_clear_cpuid(void)
108 {
109 u64 msr;
110
111 /*
112 * MSR_TFA_TSX_CPUID_CLEAR bit is only present when both CPUID
113 * bits RTM_ALWAYS_ABORT and TSX_FORCE_ABORT are present.
114 */
115 if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT) &&
116 boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)) {
117 rdmsrl(MSR_TSX_FORCE_ABORT, msr);
118 msr |= MSR_TFA_TSX_CPUID_CLEAR;
119 wrmsrl(MSR_TSX_FORCE_ABORT, msr);
120 } else if (cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL)) {
121 rdmsrl(MSR_IA32_TSX_CTRL, msr);
122 msr |= TSX_CTRL_CPUID_CLEAR;
123 wrmsrl(MSR_IA32_TSX_CTRL, msr);
124 }
125 }
126
127 /*
128 * Disable TSX development mode
129 *
130 * When the microcode released in Feb 2022 is applied, TSX will be disabled by
131 * default on some processors. MSR 0x122 (TSX_CTRL) and MSR 0x123
132 * (IA32_MCU_OPT_CTRL) can be used to re-enable TSX for development, doing so is
133 * not recommended for production deployments. In particular, applying MD_CLEAR
134 * flows for mitigation of the Intel TSX Asynchronous Abort (TAA) transient
135 * execution attack may not be effective on these processors when Intel TSX is
136 * enabled with updated microcode.
137 */
tsx_dev_mode_disable(void)138 static void tsx_dev_mode_disable(void)
139 {
140 u64 mcu_opt_ctrl;
141
142 /* Check if RTM_ALLOW exists */
143 if (!boot_cpu_has_bug(X86_BUG_TAA) ||
144 !cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL) ||
145 !cpu_feature_enabled(X86_FEATURE_SRBDS_CTRL))
146 return;
147
148 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl);
149
150 if (mcu_opt_ctrl & RTM_ALLOW) {
151 mcu_opt_ctrl &= ~RTM_ALLOW;
152 wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl);
153 setup_force_cpu_cap(X86_FEATURE_RTM_ALWAYS_ABORT);
154 }
155 }
156
tsx_init(void)157 void __init tsx_init(void)
158 {
159 char arg[5] = {};
160 int ret;
161
162 tsx_dev_mode_disable();
163
164 /*
165 * Hardware will always abort a TSX transaction when the CPUID bit
166 * RTM_ALWAYS_ABORT is set. In this case, it is better not to enumerate
167 * CPUID.RTM and CPUID.HLE bits. Clear them here.
168 */
169 if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT)) {
170 tsx_ctrl_state = TSX_CTRL_RTM_ALWAYS_ABORT;
171 tsx_clear_cpuid();
172 setup_clear_cpu_cap(X86_FEATURE_RTM);
173 setup_clear_cpu_cap(X86_FEATURE_HLE);
174 return;
175 }
176
177 /*
178 * TSX is controlled via MSR_IA32_TSX_CTRL. However, support for this
179 * MSR is enumerated by ARCH_CAP_TSX_MSR bit in MSR_IA32_ARCH_CAPABILITIES.
180 *
181 * TSX control (aka MSR_IA32_TSX_CTRL) is only available after a
182 * microcode update on CPUs that have their MSR_IA32_ARCH_CAPABILITIES
183 * bit MDS_NO=1. CPUs with MDS_NO=0 are not planned to get
184 * MSR_IA32_TSX_CTRL support even after a microcode update. Thus,
185 * tsx= cmdline requests will do nothing on CPUs without
186 * MSR_IA32_TSX_CTRL support.
187 */
188 if (x86_read_arch_cap_msr() & ARCH_CAP_TSX_CTRL_MSR) {
189 setup_force_cpu_cap(X86_FEATURE_MSR_TSX_CTRL);
190 } else {
191 tsx_ctrl_state = TSX_CTRL_NOT_SUPPORTED;
192 return;
193 }
194
195 ret = cmdline_find_option(boot_command_line, "tsx", arg, sizeof(arg));
196 if (ret >= 0) {
197 if (!strcmp(arg, "on")) {
198 tsx_ctrl_state = TSX_CTRL_ENABLE;
199 } else if (!strcmp(arg, "off")) {
200 tsx_ctrl_state = TSX_CTRL_DISABLE;
201 } else if (!strcmp(arg, "auto")) {
202 tsx_ctrl_state = x86_get_tsx_auto_mode();
203 } else {
204 tsx_ctrl_state = TSX_CTRL_DISABLE;
205 pr_err("invalid option, defaulting to off\n");
206 }
207 } else {
208 /* tsx= not provided */
209 if (IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_AUTO))
210 tsx_ctrl_state = x86_get_tsx_auto_mode();
211 else if (IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_OFF))
212 tsx_ctrl_state = TSX_CTRL_DISABLE;
213 else
214 tsx_ctrl_state = TSX_CTRL_ENABLE;
215 }
216
217 if (tsx_ctrl_state == TSX_CTRL_DISABLE) {
218 tsx_disable();
219
220 /*
221 * tsx_disable() will change the state of the RTM and HLE CPUID
222 * bits. Clear them here since they are now expected to be not
223 * set.
224 */
225 setup_clear_cpu_cap(X86_FEATURE_RTM);
226 setup_clear_cpu_cap(X86_FEATURE_HLE);
227 } else if (tsx_ctrl_state == TSX_CTRL_ENABLE) {
228
229 /*
230 * HW defaults TSX to be enabled at bootup.
231 * We may still need the TSX enable support
232 * during init for special cases like
233 * kexec after TSX is disabled.
234 */
235 tsx_enable();
236
237 /*
238 * tsx_enable() will change the state of the RTM and HLE CPUID
239 * bits. Force them here since they are now expected to be set.
240 */
241 setup_force_cpu_cap(X86_FEATURE_RTM);
242 setup_force_cpu_cap(X86_FEATURE_HLE);
243 }
244 }
245
tsx_ap_init(void)246 void tsx_ap_init(void)
247 {
248 tsx_dev_mode_disable();
249
250 if (tsx_ctrl_state == TSX_CTRL_ENABLE)
251 tsx_enable();
252 else if (tsx_ctrl_state == TSX_CTRL_DISABLE)
253 tsx_disable();
254 else if (tsx_ctrl_state == TSX_CTRL_RTM_ALWAYS_ABORT)
255 /* See comment over that function for more details. */
256 tsx_clear_cpuid();
257 }
258