1 /*
2 * VMware Detection code.
3 *
4 * Copyright (C) 2008, VMware, Inc.
5 * Author : Alok N Kataria <akataria@vmware.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
15 * NON INFRINGEMENT. See the GNU General Public License for more
16 * details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
21 *
22 */
23
24 #include <linux/dmi.h>
25 #include <linux/init.h>
26 #include <linux/export.h>
27 #include <linux/clocksource.h>
28 #include <linux/cpu.h>
29 #include <linux/reboot.h>
30 #include <asm/div64.h>
31 #include <asm/x86_init.h>
32 #include <asm/hypervisor.h>
33 #include <asm/timer.h>
34 #include <asm/apic.h>
35 #include <asm/vmware.h>
36 #include <asm/svm.h>
37
38 #undef pr_fmt
39 #define pr_fmt(fmt) "vmware: " fmt
40
41 #define CPUID_VMWARE_INFO_LEAF 0x40000000
42 #define CPUID_VMWARE_FEATURES_LEAF 0x40000010
43 #define CPUID_VMWARE_FEATURES_ECX_VMMCALL BIT(0)
44 #define CPUID_VMWARE_FEATURES_ECX_VMCALL BIT(1)
45
46 #define VMWARE_HYPERVISOR_MAGIC 0x564D5868
47
48 #define VMWARE_CMD_GETVERSION 10
49 #define VMWARE_CMD_GETHZ 45
50 #define VMWARE_CMD_GETVCPU_INFO 68
51 #define VMWARE_CMD_LEGACY_X2APIC 3
52 #define VMWARE_CMD_VCPU_RESERVED 31
53 #define VMWARE_CMD_STEALCLOCK 91
54
55 #define STEALCLOCK_NOT_AVAILABLE (-1)
56 #define STEALCLOCK_DISABLED 0
57 #define STEALCLOCK_ENABLED 1
58
59 #define VMWARE_PORT(cmd, eax, ebx, ecx, edx) \
60 __asm__("inl (%%dx), %%eax" : \
61 "=a"(eax), "=c"(ecx), "=d"(edx), "=b"(ebx) : \
62 "a"(VMWARE_HYPERVISOR_MAGIC), \
63 "c"(VMWARE_CMD_##cmd), \
64 "d"(VMWARE_HYPERVISOR_PORT), "b"(UINT_MAX) : \
65 "memory")
66
67 #define VMWARE_VMCALL(cmd, eax, ebx, ecx, edx) \
68 __asm__("vmcall" : \
69 "=a"(eax), "=c"(ecx), "=d"(edx), "=b"(ebx) : \
70 "a"(VMWARE_HYPERVISOR_MAGIC), \
71 "c"(VMWARE_CMD_##cmd), \
72 "d"(0), "b"(UINT_MAX) : \
73 "memory")
74
75 #define VMWARE_VMMCALL(cmd, eax, ebx, ecx, edx) \
76 __asm__("vmmcall" : \
77 "=a"(eax), "=c"(ecx), "=d"(edx), "=b"(ebx) : \
78 "a"(VMWARE_HYPERVISOR_MAGIC), \
79 "c"(VMWARE_CMD_##cmd), \
80 "d"(0), "b"(UINT_MAX) : \
81 "memory")
82
83 #define VMWARE_CMD(cmd, eax, ebx, ecx, edx) do { \
84 switch (vmware_hypercall_mode) { \
85 case CPUID_VMWARE_FEATURES_ECX_VMCALL: \
86 VMWARE_VMCALL(cmd, eax, ebx, ecx, edx); \
87 break; \
88 case CPUID_VMWARE_FEATURES_ECX_VMMCALL: \
89 VMWARE_VMMCALL(cmd, eax, ebx, ecx, edx); \
90 break; \
91 default: \
92 VMWARE_PORT(cmd, eax, ebx, ecx, edx); \
93 break; \
94 } \
95 } while (0)
96
97 struct vmware_steal_time {
98 union {
99 uint64_t clock; /* stolen time counter in units of vtsc */
100 struct {
101 /* only for little-endian */
102 uint32_t clock_low;
103 uint32_t clock_high;
104 };
105 };
106 uint64_t reserved[7];
107 };
108
109 static unsigned long vmware_tsc_khz __ro_after_init;
110 static u8 vmware_hypercall_mode __ro_after_init;
111
__vmware_platform(void)112 static inline int __vmware_platform(void)
113 {
114 uint32_t eax, ebx, ecx, edx;
115 VMWARE_CMD(GETVERSION, eax, ebx, ecx, edx);
116 return eax != (uint32_t)-1 && ebx == VMWARE_HYPERVISOR_MAGIC;
117 }
118
vmware_get_tsc_khz(void)119 static unsigned long vmware_get_tsc_khz(void)
120 {
121 return vmware_tsc_khz;
122 }
123
124 #ifdef CONFIG_PARAVIRT
125 static struct cyc2ns_data vmware_cyc2ns __ro_after_init;
126 static bool vmw_sched_clock __initdata = true;
127 static DEFINE_PER_CPU_DECRYPTED(struct vmware_steal_time, vmw_steal_time) __aligned(64);
128 static bool has_steal_clock;
129 static bool steal_acc __initdata = true; /* steal time accounting */
130
setup_vmw_sched_clock(char * s)131 static __init int setup_vmw_sched_clock(char *s)
132 {
133 vmw_sched_clock = false;
134 return 0;
135 }
136 early_param("no-vmw-sched-clock", setup_vmw_sched_clock);
137
parse_no_stealacc(char * arg)138 static __init int parse_no_stealacc(char *arg)
139 {
140 steal_acc = false;
141 return 0;
142 }
143 early_param("no-steal-acc", parse_no_stealacc);
144
vmware_sched_clock(void)145 static unsigned long long notrace vmware_sched_clock(void)
146 {
147 unsigned long long ns;
148
149 ns = mul_u64_u32_shr(rdtsc(), vmware_cyc2ns.cyc2ns_mul,
150 vmware_cyc2ns.cyc2ns_shift);
151 ns -= vmware_cyc2ns.cyc2ns_offset;
152 return ns;
153 }
154
vmware_cyc2ns_setup(void)155 static void __init vmware_cyc2ns_setup(void)
156 {
157 struct cyc2ns_data *d = &vmware_cyc2ns;
158 unsigned long long tsc_now = rdtsc();
159
160 clocks_calc_mult_shift(&d->cyc2ns_mul, &d->cyc2ns_shift,
161 vmware_tsc_khz, NSEC_PER_MSEC, 0);
162 d->cyc2ns_offset = mul_u64_u32_shr(tsc_now, d->cyc2ns_mul,
163 d->cyc2ns_shift);
164
165 pr_info("using clock offset of %llu ns\n", d->cyc2ns_offset);
166 }
167
vmware_cmd_stealclock(uint32_t arg1,uint32_t arg2)168 static int vmware_cmd_stealclock(uint32_t arg1, uint32_t arg2)
169 {
170 uint32_t result, info;
171
172 asm volatile (VMWARE_HYPERCALL :
173 "=a"(result),
174 "=c"(info) :
175 "a"(VMWARE_HYPERVISOR_MAGIC),
176 "b"(0),
177 "c"(VMWARE_CMD_STEALCLOCK),
178 "d"(0),
179 "S"(arg1),
180 "D"(arg2) :
181 "memory");
182 return result;
183 }
184
stealclock_enable(phys_addr_t pa)185 static bool stealclock_enable(phys_addr_t pa)
186 {
187 return vmware_cmd_stealclock(upper_32_bits(pa),
188 lower_32_bits(pa)) == STEALCLOCK_ENABLED;
189 }
190
__stealclock_disable(void)191 static int __stealclock_disable(void)
192 {
193 return vmware_cmd_stealclock(0, 1);
194 }
195
stealclock_disable(void)196 static void stealclock_disable(void)
197 {
198 __stealclock_disable();
199 }
200
vmware_is_stealclock_available(void)201 static bool vmware_is_stealclock_available(void)
202 {
203 return __stealclock_disable() != STEALCLOCK_NOT_AVAILABLE;
204 }
205
206 /**
207 * vmware_steal_clock() - read the per-cpu steal clock
208 * @cpu: the cpu number whose steal clock we want to read
209 *
210 * The function reads the steal clock if we are on a 64-bit system, otherwise
211 * reads it in parts, checking that the high part didn't change in the
212 * meantime.
213 *
214 * Return:
215 * The steal clock reading in ns.
216 */
vmware_steal_clock(int cpu)217 static uint64_t vmware_steal_clock(int cpu)
218 {
219 struct vmware_steal_time *steal = &per_cpu(vmw_steal_time, cpu);
220 uint64_t clock;
221
222 if (IS_ENABLED(CONFIG_64BIT))
223 clock = READ_ONCE(steal->clock);
224 else {
225 uint32_t initial_high, low, high;
226
227 do {
228 initial_high = READ_ONCE(steal->clock_high);
229 /* Do not reorder initial_high and high readings */
230 virt_rmb();
231 low = READ_ONCE(steal->clock_low);
232 /* Keep low reading in between */
233 virt_rmb();
234 high = READ_ONCE(steal->clock_high);
235 } while (initial_high != high);
236
237 clock = ((uint64_t)high << 32) | low;
238 }
239
240 return mul_u64_u32_shr(clock, vmware_cyc2ns.cyc2ns_mul,
241 vmware_cyc2ns.cyc2ns_shift);
242 }
243
vmware_register_steal_time(void)244 static void vmware_register_steal_time(void)
245 {
246 int cpu = smp_processor_id();
247 struct vmware_steal_time *st = &per_cpu(vmw_steal_time, cpu);
248
249 if (!has_steal_clock)
250 return;
251
252 if (!stealclock_enable(slow_virt_to_phys(st))) {
253 has_steal_clock = false;
254 return;
255 }
256
257 pr_info("vmware-stealtime: cpu %d, pa %llx\n",
258 cpu, (unsigned long long) slow_virt_to_phys(st));
259 }
260
vmware_disable_steal_time(void)261 static void vmware_disable_steal_time(void)
262 {
263 if (!has_steal_clock)
264 return;
265
266 stealclock_disable();
267 }
268
vmware_guest_cpu_init(void)269 static void vmware_guest_cpu_init(void)
270 {
271 if (has_steal_clock)
272 vmware_register_steal_time();
273 }
274
vmware_pv_guest_cpu_reboot(void * unused)275 static void vmware_pv_guest_cpu_reboot(void *unused)
276 {
277 vmware_disable_steal_time();
278 }
279
vmware_pv_reboot_notify(struct notifier_block * nb,unsigned long code,void * unused)280 static int vmware_pv_reboot_notify(struct notifier_block *nb,
281 unsigned long code, void *unused)
282 {
283 if (code == SYS_RESTART)
284 on_each_cpu(vmware_pv_guest_cpu_reboot, NULL, 1);
285 return NOTIFY_DONE;
286 }
287
288 static struct notifier_block vmware_pv_reboot_nb = {
289 .notifier_call = vmware_pv_reboot_notify,
290 };
291
292 #ifdef CONFIG_SMP
vmware_smp_prepare_boot_cpu(void)293 static void __init vmware_smp_prepare_boot_cpu(void)
294 {
295 vmware_guest_cpu_init();
296 native_smp_prepare_boot_cpu();
297 }
298
vmware_cpu_online(unsigned int cpu)299 static int vmware_cpu_online(unsigned int cpu)
300 {
301 local_irq_disable();
302 vmware_guest_cpu_init();
303 local_irq_enable();
304 return 0;
305 }
306
vmware_cpu_down_prepare(unsigned int cpu)307 static int vmware_cpu_down_prepare(unsigned int cpu)
308 {
309 local_irq_disable();
310 vmware_disable_steal_time();
311 local_irq_enable();
312 return 0;
313 }
314 #endif
315
activate_jump_labels(void)316 static __init int activate_jump_labels(void)
317 {
318 if (has_steal_clock) {
319 static_key_slow_inc(¶virt_steal_enabled);
320 if (steal_acc)
321 static_key_slow_inc(¶virt_steal_rq_enabled);
322 }
323
324 return 0;
325 }
326 arch_initcall(activate_jump_labels);
327
vmware_paravirt_ops_setup(void)328 static void __init vmware_paravirt_ops_setup(void)
329 {
330 pv_info.name = "VMware hypervisor";
331 pv_ops.cpu.io_delay = paravirt_nop;
332
333 if (vmware_tsc_khz == 0)
334 return;
335
336 vmware_cyc2ns_setup();
337
338 if (vmw_sched_clock)
339 pv_ops.time.sched_clock = vmware_sched_clock;
340
341 if (vmware_is_stealclock_available()) {
342 has_steal_clock = true;
343 pv_ops.time.steal_clock = vmware_steal_clock;
344
345 /* We use reboot notifier only to disable steal clock */
346 register_reboot_notifier(&vmware_pv_reboot_nb);
347
348 #ifdef CONFIG_SMP
349 smp_ops.smp_prepare_boot_cpu =
350 vmware_smp_prepare_boot_cpu;
351 if (cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
352 "x86/vmware:online",
353 vmware_cpu_online,
354 vmware_cpu_down_prepare) < 0)
355 pr_err("vmware_guest: Failed to install cpu hotplug callbacks\n");
356 #else
357 vmware_guest_cpu_init();
358 #endif
359 }
360 }
361 #else
362 #define vmware_paravirt_ops_setup() do {} while (0)
363 #endif
364
365 /*
366 * VMware hypervisor takes care of exporting a reliable TSC to the guest.
367 * Still, due to timing difference when running on virtual cpus, the TSC can
368 * be marked as unstable in some cases. For example, the TSC sync check at
369 * bootup can fail due to a marginal offset between vcpus' TSCs (though the
370 * TSCs do not drift from each other). Also, the ACPI PM timer clocksource
371 * is not suitable as a watchdog when running on a hypervisor because the
372 * kernel may miss a wrap of the counter if the vcpu is descheduled for a
373 * long time. To skip these checks at runtime we set these capability bits,
374 * so that the kernel could just trust the hypervisor with providing a
375 * reliable virtual TSC that is suitable for timekeeping.
376 */
vmware_set_capabilities(void)377 static void __init vmware_set_capabilities(void)
378 {
379 setup_force_cpu_cap(X86_FEATURE_CONSTANT_TSC);
380 setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE);
381 if (vmware_hypercall_mode == CPUID_VMWARE_FEATURES_ECX_VMCALL)
382 setup_force_cpu_cap(X86_FEATURE_VMCALL);
383 else if (vmware_hypercall_mode == CPUID_VMWARE_FEATURES_ECX_VMMCALL)
384 setup_force_cpu_cap(X86_FEATURE_VMW_VMMCALL);
385 }
386
vmware_platform_setup(void)387 static void __init vmware_platform_setup(void)
388 {
389 uint32_t eax, ebx, ecx, edx;
390 uint64_t lpj, tsc_khz;
391
392 VMWARE_CMD(GETHZ, eax, ebx, ecx, edx);
393
394 if (ebx != UINT_MAX) {
395 lpj = tsc_khz = eax | (((uint64_t)ebx) << 32);
396 do_div(tsc_khz, 1000);
397 WARN_ON(tsc_khz >> 32);
398 pr_info("TSC freq read from hypervisor : %lu.%03lu MHz\n",
399 (unsigned long) tsc_khz / 1000,
400 (unsigned long) tsc_khz % 1000);
401
402 if (!preset_lpj) {
403 do_div(lpj, HZ);
404 preset_lpj = lpj;
405 }
406
407 vmware_tsc_khz = tsc_khz;
408 x86_platform.calibrate_tsc = vmware_get_tsc_khz;
409 x86_platform.calibrate_cpu = vmware_get_tsc_khz;
410
411 #ifdef CONFIG_X86_LOCAL_APIC
412 /* Skip lapic calibration since we know the bus frequency. */
413 lapic_timer_period = ecx / HZ;
414 pr_info("Host bus clock speed read from hypervisor : %u Hz\n",
415 ecx);
416 #endif
417 } else {
418 pr_warn("Failed to get TSC freq from the hypervisor\n");
419 }
420
421 vmware_paravirt_ops_setup();
422
423 #ifdef CONFIG_X86_IO_APIC
424 no_timer_check = 1;
425 #endif
426
427 vmware_set_capabilities();
428 }
429
vmware_select_hypercall(void)430 static u8 __init vmware_select_hypercall(void)
431 {
432 int eax, ebx, ecx, edx;
433
434 cpuid(CPUID_VMWARE_FEATURES_LEAF, &eax, &ebx, &ecx, &edx);
435 return (ecx & (CPUID_VMWARE_FEATURES_ECX_VMMCALL |
436 CPUID_VMWARE_FEATURES_ECX_VMCALL));
437 }
438
439 /*
440 * While checking the dmi string information, just checking the product
441 * serial key should be enough, as this will always have a VMware
442 * specific string when running under VMware hypervisor.
443 * If !boot_cpu_has(X86_FEATURE_HYPERVISOR), vmware_hypercall_mode
444 * intentionally defaults to 0.
445 */
vmware_platform(void)446 static uint32_t __init vmware_platform(void)
447 {
448 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
449 unsigned int eax;
450 unsigned int hyper_vendor_id[3];
451
452 cpuid(CPUID_VMWARE_INFO_LEAF, &eax, &hyper_vendor_id[0],
453 &hyper_vendor_id[1], &hyper_vendor_id[2]);
454 if (!memcmp(hyper_vendor_id, "VMwareVMware", 12)) {
455 if (eax >= CPUID_VMWARE_FEATURES_LEAF)
456 vmware_hypercall_mode =
457 vmware_select_hypercall();
458
459 pr_info("hypercall mode: 0x%02x\n",
460 (unsigned int) vmware_hypercall_mode);
461
462 return CPUID_VMWARE_INFO_LEAF;
463 }
464 } else if (dmi_available && dmi_name_in_serial("VMware") &&
465 __vmware_platform())
466 return 1;
467
468 return 0;
469 }
470
471 /* Checks if hypervisor supports x2apic without VT-D interrupt remapping. */
vmware_legacy_x2apic_available(void)472 static bool __init vmware_legacy_x2apic_available(void)
473 {
474 uint32_t eax, ebx, ecx, edx;
475 VMWARE_CMD(GETVCPU_INFO, eax, ebx, ecx, edx);
476 return (eax & (1 << VMWARE_CMD_VCPU_RESERVED)) == 0 &&
477 (eax & (1 << VMWARE_CMD_LEGACY_X2APIC)) != 0;
478 }
479
480 #ifdef CONFIG_AMD_MEM_ENCRYPT
vmware_sev_es_hcall_prepare(struct ghcb * ghcb,struct pt_regs * regs)481 static void vmware_sev_es_hcall_prepare(struct ghcb *ghcb,
482 struct pt_regs *regs)
483 {
484 /* Copy VMWARE specific Hypercall parameters to the GHCB */
485 ghcb_set_rip(ghcb, regs->ip);
486 ghcb_set_rbx(ghcb, regs->bx);
487 ghcb_set_rcx(ghcb, regs->cx);
488 ghcb_set_rdx(ghcb, regs->dx);
489 ghcb_set_rsi(ghcb, regs->si);
490 ghcb_set_rdi(ghcb, regs->di);
491 ghcb_set_rbp(ghcb, regs->bp);
492 }
493
vmware_sev_es_hcall_finish(struct ghcb * ghcb,struct pt_regs * regs)494 static bool vmware_sev_es_hcall_finish(struct ghcb *ghcb, struct pt_regs *regs)
495 {
496 if (!(ghcb_rbx_is_valid(ghcb) &&
497 ghcb_rcx_is_valid(ghcb) &&
498 ghcb_rdx_is_valid(ghcb) &&
499 ghcb_rsi_is_valid(ghcb) &&
500 ghcb_rdi_is_valid(ghcb) &&
501 ghcb_rbp_is_valid(ghcb)))
502 return false;
503
504 regs->bx = ghcb->save.rbx;
505 regs->cx = ghcb->save.rcx;
506 regs->dx = ghcb->save.rdx;
507 regs->si = ghcb->save.rsi;
508 regs->di = ghcb->save.rdi;
509 regs->bp = ghcb->save.rbp;
510
511 return true;
512 }
513 #endif
514
515 const __initconst struct hypervisor_x86 x86_hyper_vmware = {
516 .name = "VMware",
517 .detect = vmware_platform,
518 .type = X86_HYPER_VMWARE,
519 .init.init_platform = vmware_platform_setup,
520 .init.x2apic_available = vmware_legacy_x2apic_available,
521 #ifdef CONFIG_AMD_MEM_ENCRYPT
522 .runtime.sev_es_hcall_prepare = vmware_sev_es_hcall_prepare,
523 .runtime.sev_es_hcall_finish = vmware_sev_es_hcall_finish,
524 #endif
525 };
526