• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * KVM paravirt_ops implementation
4  *
5  * Copyright (C) 2007, Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
6  * Copyright IBM Corporation, 2007
7  *   Authors: Anthony Liguori <aliguori@us.ibm.com>
8  */
9 
10 #include <linux/context_tracking.h>
11 #include <linux/init.h>
12 #include <linux/kernel.h>
13 #include <linux/kvm_para.h>
14 #include <linux/cpu.h>
15 #include <linux/mm.h>
16 #include <linux/highmem.h>
17 #include <linux/hardirq.h>
18 #include <linux/notifier.h>
19 #include <linux/reboot.h>
20 #include <linux/hash.h>
21 #include <linux/sched.h>
22 #include <linux/slab.h>
23 #include <linux/kprobes.h>
24 #include <linux/debugfs.h>
25 #include <linux/nmi.h>
26 #include <linux/swait.h>
27 #include <linux/syscore_ops.h>
28 #include <asm/timer.h>
29 #include <asm/cpu.h>
30 #include <asm/traps.h>
31 #include <asm/desc.h>
32 #include <asm/tlbflush.h>
33 #include <asm/apic.h>
34 #include <asm/apicdef.h>
35 #include <asm/hypervisor.h>
36 #include <asm/tlb.h>
37 #include <asm/reboot.h>
38 
39 static int kvmapf = 1;
40 
parse_no_kvmapf(char * arg)41 static int __init parse_no_kvmapf(char *arg)
42 {
43         kvmapf = 0;
44         return 0;
45 }
46 
47 early_param("no-kvmapf", parse_no_kvmapf);
48 
49 static int steal_acc = 1;
parse_no_stealacc(char * arg)50 static int __init parse_no_stealacc(char *arg)
51 {
52         steal_acc = 0;
53         return 0;
54 }
55 
56 early_param("no-steal-acc", parse_no_stealacc);
57 
58 static DEFINE_PER_CPU_DECRYPTED(struct kvm_vcpu_pv_apf_data, apf_reason) __aligned(64);
59 DEFINE_PER_CPU_DECRYPTED(struct kvm_steal_time, steal_time) __aligned(64) __visible;
60 static int has_steal_clock = 0;
61 
62 static int has_guest_poll = 0;
63 /*
64  * No need for any "IO delay" on KVM
65  */
kvm_io_delay(void)66 static void kvm_io_delay(void)
67 {
68 }
69 
70 #define KVM_TASK_SLEEP_HASHBITS 8
71 #define KVM_TASK_SLEEP_HASHSIZE (1<<KVM_TASK_SLEEP_HASHBITS)
72 
73 struct kvm_task_sleep_node {
74 	struct hlist_node link;
75 	struct swait_queue_head wq;
76 	u32 token;
77 	int cpu;
78 	bool halted;
79 };
80 
81 static struct kvm_task_sleep_head {
82 	raw_spinlock_t lock;
83 	struct hlist_head list;
84 } async_pf_sleepers[KVM_TASK_SLEEP_HASHSIZE];
85 
_find_apf_task(struct kvm_task_sleep_head * b,u32 token)86 static struct kvm_task_sleep_node *_find_apf_task(struct kvm_task_sleep_head *b,
87 						  u32 token)
88 {
89 	struct hlist_node *p;
90 
91 	hlist_for_each(p, &b->list) {
92 		struct kvm_task_sleep_node *n =
93 			hlist_entry(p, typeof(*n), link);
94 		if (n->token == token)
95 			return n;
96 	}
97 
98 	return NULL;
99 }
100 
101 /*
102  * @interrupt_kernel: Is this called from a routine which interrupts the kernel
103  * 		      (other than user space)?
104  */
kvm_async_pf_task_wait(u32 token,int interrupt_kernel)105 void kvm_async_pf_task_wait(u32 token, int interrupt_kernel)
106 {
107 	u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS);
108 	struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
109 	struct kvm_task_sleep_node n, *e;
110 	DECLARE_SWAITQUEUE(wait);
111 
112 	rcu_irq_enter();
113 
114 	raw_spin_lock(&b->lock);
115 	e = _find_apf_task(b, token);
116 	if (e) {
117 		/* dummy entry exist -> wake up was delivered ahead of PF */
118 		hlist_del(&e->link);
119 		kfree(e);
120 		raw_spin_unlock(&b->lock);
121 
122 		rcu_irq_exit();
123 		return;
124 	}
125 
126 	n.token = token;
127 	n.cpu = smp_processor_id();
128 	n.halted = is_idle_task(current) ||
129 		   (IS_ENABLED(CONFIG_PREEMPT_COUNT)
130 		    ? preempt_count() > 1 || rcu_preempt_depth()
131 		    : interrupt_kernel);
132 	init_swait_queue_head(&n.wq);
133 	hlist_add_head(&n.link, &b->list);
134 	raw_spin_unlock(&b->lock);
135 
136 	for (;;) {
137 		if (!n.halted)
138 			prepare_to_swait_exclusive(&n.wq, &wait, TASK_UNINTERRUPTIBLE);
139 		if (hlist_unhashed(&n.link))
140 			break;
141 
142 		rcu_irq_exit();
143 
144 		if (!n.halted) {
145 			local_irq_enable();
146 			schedule();
147 			local_irq_disable();
148 		} else {
149 			/*
150 			 * We cannot reschedule. So halt.
151 			 */
152 			native_safe_halt();
153 			local_irq_disable();
154 		}
155 
156 		rcu_irq_enter();
157 	}
158 	if (!n.halted)
159 		finish_swait(&n.wq, &wait);
160 
161 	rcu_irq_exit();
162 	return;
163 }
164 EXPORT_SYMBOL_GPL(kvm_async_pf_task_wait);
165 
apf_task_wake_one(struct kvm_task_sleep_node * n)166 static void apf_task_wake_one(struct kvm_task_sleep_node *n)
167 {
168 	hlist_del_init(&n->link);
169 	if (n->halted)
170 		smp_send_reschedule(n->cpu);
171 	else if (swq_has_sleeper(&n->wq))
172 		swake_up_one(&n->wq);
173 }
174 
apf_task_wake_all(void)175 static void apf_task_wake_all(void)
176 {
177 	int i;
178 
179 	for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) {
180 		struct hlist_node *p, *next;
181 		struct kvm_task_sleep_head *b = &async_pf_sleepers[i];
182 		raw_spin_lock(&b->lock);
183 		hlist_for_each_safe(p, next, &b->list) {
184 			struct kvm_task_sleep_node *n =
185 				hlist_entry(p, typeof(*n), link);
186 			if (n->cpu == smp_processor_id())
187 				apf_task_wake_one(n);
188 		}
189 		raw_spin_unlock(&b->lock);
190 	}
191 }
192 
kvm_async_pf_task_wake(u32 token)193 void kvm_async_pf_task_wake(u32 token)
194 {
195 	u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS);
196 	struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
197 	struct kvm_task_sleep_node *n;
198 
199 	if (token == ~0) {
200 		apf_task_wake_all();
201 		return;
202 	}
203 
204 again:
205 	raw_spin_lock(&b->lock);
206 	n = _find_apf_task(b, token);
207 	if (!n) {
208 		/*
209 		 * async PF was not yet handled.
210 		 * Add dummy entry for the token.
211 		 */
212 		n = kzalloc(sizeof(*n), GFP_ATOMIC);
213 		if (!n) {
214 			/*
215 			 * Allocation failed! Busy wait while other cpu
216 			 * handles async PF.
217 			 */
218 			raw_spin_unlock(&b->lock);
219 			cpu_relax();
220 			goto again;
221 		}
222 		n->token = token;
223 		n->cpu = smp_processor_id();
224 		init_swait_queue_head(&n->wq);
225 		hlist_add_head(&n->link, &b->list);
226 	} else
227 		apf_task_wake_one(n);
228 	raw_spin_unlock(&b->lock);
229 	return;
230 }
231 EXPORT_SYMBOL_GPL(kvm_async_pf_task_wake);
232 
kvm_read_and_reset_pf_reason(void)233 u32 kvm_read_and_reset_pf_reason(void)
234 {
235 	u32 reason = 0;
236 
237 	if (__this_cpu_read(apf_reason.enabled)) {
238 		reason = __this_cpu_read(apf_reason.reason);
239 		__this_cpu_write(apf_reason.reason, 0);
240 	}
241 
242 	return reason;
243 }
244 EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason);
245 NOKPROBE_SYMBOL(kvm_read_and_reset_pf_reason);
246 
247 dotraplinkage void
do_async_page_fault(struct pt_regs * regs,unsigned long error_code,unsigned long address)248 do_async_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address)
249 {
250 	enum ctx_state prev_state;
251 
252 	switch (kvm_read_and_reset_pf_reason()) {
253 	default:
254 		do_page_fault(regs, error_code, address);
255 		break;
256 	case KVM_PV_REASON_PAGE_NOT_PRESENT:
257 		/* page is swapped out by the host. */
258 		prev_state = exception_enter();
259 		kvm_async_pf_task_wait((u32)address, !user_mode(regs));
260 		exception_exit(prev_state);
261 		break;
262 	case KVM_PV_REASON_PAGE_READY:
263 		rcu_irq_enter();
264 		kvm_async_pf_task_wake((u32)address);
265 		rcu_irq_exit();
266 		break;
267 	}
268 }
269 NOKPROBE_SYMBOL(do_async_page_fault);
270 
paravirt_ops_setup(void)271 static void __init paravirt_ops_setup(void)
272 {
273 	pv_info.name = "KVM";
274 
275 	if (kvm_para_has_feature(KVM_FEATURE_NOP_IO_DELAY))
276 		pv_ops.cpu.io_delay = kvm_io_delay;
277 
278 #ifdef CONFIG_X86_IO_APIC
279 	no_timer_check = 1;
280 #endif
281 }
282 
kvm_register_steal_time(void)283 static void kvm_register_steal_time(void)
284 {
285 	int cpu = smp_processor_id();
286 	struct kvm_steal_time *st = &per_cpu(steal_time, cpu);
287 
288 	if (!has_steal_clock)
289 		return;
290 
291 	wrmsrl(MSR_KVM_STEAL_TIME, (slow_virt_to_phys(st) | KVM_MSR_ENABLED));
292 	pr_info("kvm-stealtime: cpu %d, msr %llx\n",
293 		cpu, (unsigned long long) slow_virt_to_phys(st));
294 }
295 
296 static DEFINE_PER_CPU_DECRYPTED(unsigned long, kvm_apic_eoi) = KVM_PV_EOI_DISABLED;
297 
kvm_guest_apic_eoi_write(u32 reg,u32 val)298 static notrace void kvm_guest_apic_eoi_write(u32 reg, u32 val)
299 {
300 	/**
301 	 * This relies on __test_and_clear_bit to modify the memory
302 	 * in a way that is atomic with respect to the local CPU.
303 	 * The hypervisor only accesses this memory from the local CPU so
304 	 * there's no need for lock or memory barriers.
305 	 * An optimization barrier is implied in apic write.
306 	 */
307 	if (__test_and_clear_bit(KVM_PV_EOI_BIT, this_cpu_ptr(&kvm_apic_eoi)))
308 		return;
309 	apic->native_eoi_write(APIC_EOI, APIC_EOI_ACK);
310 }
311 
kvm_guest_cpu_init(void)312 static void kvm_guest_cpu_init(void)
313 {
314 	if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF) && kvmapf) {
315 		u64 pa = slow_virt_to_phys(this_cpu_ptr(&apf_reason));
316 
317 #ifdef CONFIG_PREEMPTION
318 		pa |= KVM_ASYNC_PF_SEND_ALWAYS;
319 #endif
320 		pa |= KVM_ASYNC_PF_ENABLED;
321 
322 		if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_VMEXIT))
323 			pa |= KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
324 
325 		wrmsrl(MSR_KVM_ASYNC_PF_EN, pa);
326 		__this_cpu_write(apf_reason.enabled, 1);
327 		printk(KERN_INFO"KVM setup async PF for cpu %d\n",
328 		       smp_processor_id());
329 	}
330 
331 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) {
332 		unsigned long pa;
333 		/* Size alignment is implied but just to make it explicit. */
334 		BUILD_BUG_ON(__alignof__(kvm_apic_eoi) < 4);
335 		__this_cpu_write(kvm_apic_eoi, 0);
336 		pa = slow_virt_to_phys(this_cpu_ptr(&kvm_apic_eoi))
337 			| KVM_MSR_ENABLED;
338 		wrmsrl(MSR_KVM_PV_EOI_EN, pa);
339 	}
340 
341 	if (has_steal_clock)
342 		kvm_register_steal_time();
343 }
344 
kvm_pv_disable_apf(void)345 static void kvm_pv_disable_apf(void)
346 {
347 	if (!__this_cpu_read(apf_reason.enabled))
348 		return;
349 
350 	wrmsrl(MSR_KVM_ASYNC_PF_EN, 0);
351 	__this_cpu_write(apf_reason.enabled, 0);
352 
353 	printk(KERN_INFO"Unregister pv shared memory for cpu %d\n",
354 	       smp_processor_id());
355 }
356 
kvm_disable_steal_time(void)357 static void kvm_disable_steal_time(void)
358 {
359 	if (!has_steal_clock)
360 		return;
361 
362 	wrmsr(MSR_KVM_STEAL_TIME, 0, 0);
363 }
364 
kvm_pv_guest_cpu_reboot(void * unused)365 static void kvm_pv_guest_cpu_reboot(void *unused)
366 {
367 	/*
368 	 * We disable PV EOI before we load a new kernel by kexec,
369 	 * since MSR_KVM_PV_EOI_EN stores a pointer into old kernel's memory.
370 	 * New kernel can re-enable when it boots.
371 	 */
372 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
373 		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
374 	kvm_pv_disable_apf();
375 	kvm_disable_steal_time();
376 }
377 
kvm_pv_reboot_notify(struct notifier_block * nb,unsigned long code,void * unused)378 static int kvm_pv_reboot_notify(struct notifier_block *nb,
379 				unsigned long code, void *unused)
380 {
381 	if (code == SYS_RESTART)
382 		on_each_cpu(kvm_pv_guest_cpu_reboot, NULL, 1);
383 	return NOTIFY_DONE;
384 }
385 
386 static struct notifier_block kvm_pv_reboot_nb = {
387 	.notifier_call = kvm_pv_reboot_notify,
388 };
389 
kvm_steal_clock(int cpu)390 static u64 kvm_steal_clock(int cpu)
391 {
392 	u64 steal;
393 	struct kvm_steal_time *src;
394 	int version;
395 
396 	src = &per_cpu(steal_time, cpu);
397 	do {
398 		version = src->version;
399 		virt_rmb();
400 		steal = src->steal;
401 		virt_rmb();
402 	} while ((version & 1) || (version != src->version));
403 
404 	return steal;
405 }
406 
__set_percpu_decrypted(void * ptr,unsigned long size)407 static inline void __set_percpu_decrypted(void *ptr, unsigned long size)
408 {
409 	early_set_memory_decrypted((unsigned long) ptr, size);
410 }
411 
412 /*
413  * Iterate through all possible CPUs and map the memory region pointed
414  * by apf_reason, steal_time and kvm_apic_eoi as decrypted at once.
415  *
416  * Note: we iterate through all possible CPUs to ensure that CPUs
417  * hotplugged will have their per-cpu variable already mapped as
418  * decrypted.
419  */
sev_map_percpu_data(void)420 static void __init sev_map_percpu_data(void)
421 {
422 	int cpu;
423 
424 	if (!sev_active())
425 		return;
426 
427 	for_each_possible_cpu(cpu) {
428 		__set_percpu_decrypted(&per_cpu(apf_reason, cpu), sizeof(apf_reason));
429 		__set_percpu_decrypted(&per_cpu(steal_time, cpu), sizeof(steal_time));
430 		__set_percpu_decrypted(&per_cpu(kvm_apic_eoi, cpu), sizeof(kvm_apic_eoi));
431 	}
432 }
433 
kvm_guest_cpu_offline(bool shutdown)434 static void kvm_guest_cpu_offline(bool shutdown)
435 {
436 	kvm_disable_steal_time();
437 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
438 		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
439 	kvm_pv_disable_apf();
440 	if (!shutdown)
441 		apf_task_wake_all();
442 	kvmclock_disable();
443 }
444 
kvm_cpu_online(unsigned int cpu)445 static int kvm_cpu_online(unsigned int cpu)
446 {
447 	unsigned long flags;
448 
449 	local_irq_save(flags);
450 	kvm_guest_cpu_init();
451 	local_irq_restore(flags);
452 	return 0;
453 }
454 
455 #ifdef CONFIG_SMP
456 #define KVM_IPI_CLUSTER_SIZE	(2 * BITS_PER_LONG)
457 
__send_ipi_mask(const struct cpumask * mask,int vector)458 static void __send_ipi_mask(const struct cpumask *mask, int vector)
459 {
460 	unsigned long flags;
461 	int cpu, apic_id, icr;
462 	int min = 0, max = 0;
463 #ifdef CONFIG_X86_64
464 	__uint128_t ipi_bitmap = 0;
465 #else
466 	u64 ipi_bitmap = 0;
467 #endif
468 	long ret;
469 
470 	if (cpumask_empty(mask))
471 		return;
472 
473 	local_irq_save(flags);
474 
475 	switch (vector) {
476 	default:
477 		icr = APIC_DM_FIXED | vector;
478 		break;
479 	case NMI_VECTOR:
480 		icr = APIC_DM_NMI;
481 		break;
482 	}
483 
484 	for_each_cpu(cpu, mask) {
485 		apic_id = per_cpu(x86_cpu_to_apicid, cpu);
486 		if (!ipi_bitmap) {
487 			min = max = apic_id;
488 		} else if (apic_id < min && max - apic_id < KVM_IPI_CLUSTER_SIZE) {
489 			ipi_bitmap <<= min - apic_id;
490 			min = apic_id;
491 		} else if (apic_id > min && apic_id < min + KVM_IPI_CLUSTER_SIZE) {
492 			max = apic_id < max ? max : apic_id;
493 		} else {
494 			ret = kvm_hypercall4(KVM_HC_SEND_IPI, (unsigned long)ipi_bitmap,
495 				(unsigned long)(ipi_bitmap >> BITS_PER_LONG), min, icr);
496 			WARN_ONCE(ret < 0, "KVM: failed to send PV IPI: %ld", ret);
497 			min = max = apic_id;
498 			ipi_bitmap = 0;
499 		}
500 		__set_bit(apic_id - min, (unsigned long *)&ipi_bitmap);
501 	}
502 
503 	if (ipi_bitmap) {
504 		ret = kvm_hypercall4(KVM_HC_SEND_IPI, (unsigned long)ipi_bitmap,
505 			(unsigned long)(ipi_bitmap >> BITS_PER_LONG), min, icr);
506 		WARN_ONCE(ret < 0, "KVM: failed to send PV IPI: %ld", ret);
507 	}
508 
509 	local_irq_restore(flags);
510 }
511 
kvm_send_ipi_mask(const struct cpumask * mask,int vector)512 static void kvm_send_ipi_mask(const struct cpumask *mask, int vector)
513 {
514 	__send_ipi_mask(mask, vector);
515 }
516 
kvm_send_ipi_mask_allbutself(const struct cpumask * mask,int vector)517 static void kvm_send_ipi_mask_allbutself(const struct cpumask *mask, int vector)
518 {
519 	unsigned int this_cpu = smp_processor_id();
520 	struct cpumask new_mask;
521 	const struct cpumask *local_mask;
522 
523 	cpumask_copy(&new_mask, mask);
524 	cpumask_clear_cpu(this_cpu, &new_mask);
525 	local_mask = &new_mask;
526 	__send_ipi_mask(local_mask, vector);
527 }
528 
529 /*
530  * Set the IPI entry points
531  */
kvm_setup_pv_ipi(void)532 static void kvm_setup_pv_ipi(void)
533 {
534 	apic->send_IPI_mask = kvm_send_ipi_mask;
535 	apic->send_IPI_mask_allbutself = kvm_send_ipi_mask_allbutself;
536 	pr_info("KVM setup pv IPIs\n");
537 }
538 
kvm_smp_send_call_func_ipi(const struct cpumask * mask)539 static void kvm_smp_send_call_func_ipi(const struct cpumask *mask)
540 {
541 	int cpu;
542 
543 	native_send_call_func_ipi(mask);
544 
545 	/* Make sure other vCPUs get a chance to run if they need to. */
546 	for_each_cpu(cpu, mask) {
547 		if (vcpu_is_preempted(cpu)) {
548 			kvm_hypercall1(KVM_HC_SCHED_YIELD, per_cpu(x86_cpu_to_apicid, cpu));
549 			break;
550 		}
551 	}
552 }
553 
kvm_smp_prepare_cpus(unsigned int max_cpus)554 static void __init kvm_smp_prepare_cpus(unsigned int max_cpus)
555 {
556 	native_smp_prepare_cpus(max_cpus);
557 	if (kvm_para_has_hint(KVM_HINTS_REALTIME))
558 		static_branch_disable(&virt_spin_lock_key);
559 }
560 
kvm_smp_prepare_boot_cpu(void)561 static void __init kvm_smp_prepare_boot_cpu(void)
562 {
563 	/*
564 	 * Map the per-cpu variables as decrypted before kvm_guest_cpu_init()
565 	 * shares the guest physical address with the hypervisor.
566 	 */
567 	sev_map_percpu_data();
568 
569 	kvm_guest_cpu_init();
570 	native_smp_prepare_boot_cpu();
571 	kvm_spinlock_init();
572 }
573 
kvm_cpu_down_prepare(unsigned int cpu)574 static int kvm_cpu_down_prepare(unsigned int cpu)
575 {
576 	unsigned long flags;
577 
578 	local_irq_save(flags);
579 	kvm_guest_cpu_offline(false);
580 	local_irq_restore(flags);
581 	return 0;
582 }
583 
584 #endif
585 
kvm_suspend(void)586 static int kvm_suspend(void)
587 {
588 	u64 val = 0;
589 
590 	kvm_guest_cpu_offline(false);
591 
592 #ifdef CONFIG_ARCH_CPUIDLE_HALTPOLL
593 	if (kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL))
594 		rdmsrl(MSR_KVM_POLL_CONTROL, val);
595 	has_guest_poll = !(val & 1);
596 #endif
597 	return 0;
598 }
599 
kvm_resume(void)600 static void kvm_resume(void)
601 {
602 	kvm_cpu_online(raw_smp_processor_id());
603 
604 #ifdef CONFIG_ARCH_CPUIDLE_HALTPOLL
605 	if (kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL) && has_guest_poll)
606 		wrmsrl(MSR_KVM_POLL_CONTROL, 0);
607 #endif
608 }
609 
610 static struct syscore_ops kvm_syscore_ops = {
611 	.suspend	= kvm_suspend,
612 	.resume		= kvm_resume,
613 };
614 
615 /*
616  * After a PV feature is registered, the host will keep writing to the
617  * registered memory location. If the guest happens to shutdown, this memory
618  * won't be valid. In cases like kexec, in which you install a new kernel, this
619  * means a random memory location will be kept being written.
620  */
621 #ifdef CONFIG_KEXEC_CORE
kvm_crash_shutdown(struct pt_regs * regs)622 static void kvm_crash_shutdown(struct pt_regs *regs)
623 {
624 	kvm_guest_cpu_offline(true);
625 	native_machine_crash_shutdown(regs);
626 }
627 #endif
628 
kvm_apf_trap_init(void)629 static void __init kvm_apf_trap_init(void)
630 {
631 	update_intr_gate(X86_TRAP_PF, async_page_fault);
632 }
633 
634 static DEFINE_PER_CPU(cpumask_var_t, __pv_tlb_mask);
635 
kvm_flush_tlb_others(const struct cpumask * cpumask,const struct flush_tlb_info * info)636 static void kvm_flush_tlb_others(const struct cpumask *cpumask,
637 			const struct flush_tlb_info *info)
638 {
639 	u8 state;
640 	int cpu;
641 	struct kvm_steal_time *src;
642 	struct cpumask *flushmask = this_cpu_cpumask_var_ptr(__pv_tlb_mask);
643 
644 	cpumask_copy(flushmask, cpumask);
645 	/*
646 	 * We have to call flush only on online vCPUs. And
647 	 * queue flush_on_enter for pre-empted vCPUs
648 	 */
649 	for_each_cpu(cpu, flushmask) {
650 		src = &per_cpu(steal_time, cpu);
651 		state = READ_ONCE(src->preempted);
652 		if ((state & KVM_VCPU_PREEMPTED)) {
653 			if (try_cmpxchg(&src->preempted, &state,
654 					state | KVM_VCPU_FLUSH_TLB))
655 				__cpumask_clear_cpu(cpu, flushmask);
656 		}
657 	}
658 
659 	native_flush_tlb_others(flushmask, info);
660 }
661 
kvm_guest_init(void)662 static void __init kvm_guest_init(void)
663 {
664 	int i;
665 
666 	paravirt_ops_setup();
667 	register_reboot_notifier(&kvm_pv_reboot_nb);
668 	for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++)
669 		raw_spin_lock_init(&async_pf_sleepers[i].lock);
670 	if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF))
671 		x86_init.irqs.trap_init = kvm_apf_trap_init;
672 
673 	if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
674 		has_steal_clock = 1;
675 		pv_ops.time.steal_clock = kvm_steal_clock;
676 	}
677 
678 	if (kvm_para_has_feature(KVM_FEATURE_PV_TLB_FLUSH) &&
679 	    !kvm_para_has_hint(KVM_HINTS_REALTIME) &&
680 	    kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
681 		pv_ops.mmu.flush_tlb_others = kvm_flush_tlb_others;
682 		pv_ops.mmu.tlb_remove_table = tlb_remove_table;
683 	}
684 
685 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
686 		apic_set_eoi_write(kvm_guest_apic_eoi_write);
687 
688 #ifdef CONFIG_SMP
689 	smp_ops.smp_prepare_cpus = kvm_smp_prepare_cpus;
690 	smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu;
691 	if (kvm_para_has_feature(KVM_FEATURE_PV_SCHED_YIELD) &&
692 	    !kvm_para_has_hint(KVM_HINTS_REALTIME) &&
693 	    kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
694 		smp_ops.send_call_func_ipi = kvm_smp_send_call_func_ipi;
695 		pr_info("KVM setup pv sched yield\n");
696 	}
697 	if (cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "x86/kvm:online",
698 				      kvm_cpu_online, kvm_cpu_down_prepare) < 0)
699 		pr_err("kvm_guest: Failed to install cpu hotplug callbacks\n");
700 #else
701 	sev_map_percpu_data();
702 	kvm_guest_cpu_init();
703 #endif
704 
705 #ifdef CONFIG_KEXEC_CORE
706 	machine_ops.crash_shutdown = kvm_crash_shutdown;
707 #endif
708 
709 	register_syscore_ops(&kvm_syscore_ops);
710 
711 	/*
712 	 * Hard lockup detection is enabled by default. Disable it, as guests
713 	 * can get false positives too easily, for example if the host is
714 	 * overcommitted.
715 	 */
716 	hardlockup_detector_disable();
717 }
718 
__kvm_cpuid_base(void)719 static noinline uint32_t __kvm_cpuid_base(void)
720 {
721 	if (boot_cpu_data.cpuid_level < 0)
722 		return 0;	/* So we don't blow up on old processors */
723 
724 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
725 		return hypervisor_cpuid_base("KVMKVMKVM\0\0\0", 0);
726 
727 	return 0;
728 }
729 
kvm_cpuid_base(void)730 static inline uint32_t kvm_cpuid_base(void)
731 {
732 	static int kvm_cpuid_base = -1;
733 
734 	if (kvm_cpuid_base == -1)
735 		kvm_cpuid_base = __kvm_cpuid_base();
736 
737 	return kvm_cpuid_base;
738 }
739 
kvm_para_available(void)740 bool kvm_para_available(void)
741 {
742 	return kvm_cpuid_base() != 0;
743 }
744 EXPORT_SYMBOL_GPL(kvm_para_available);
745 
kvm_arch_para_features(void)746 unsigned int kvm_arch_para_features(void)
747 {
748 	return cpuid_eax(kvm_cpuid_base() | KVM_CPUID_FEATURES);
749 }
750 
kvm_arch_para_hints(void)751 unsigned int kvm_arch_para_hints(void)
752 {
753 	return cpuid_edx(kvm_cpuid_base() | KVM_CPUID_FEATURES);
754 }
755 EXPORT_SYMBOL_GPL(kvm_arch_para_hints);
756 
kvm_detect(void)757 static uint32_t __init kvm_detect(void)
758 {
759 	return kvm_cpuid_base();
760 }
761 
kvm_apic_init(void)762 static void __init kvm_apic_init(void)
763 {
764 #if defined(CONFIG_SMP)
765 	if (kvm_para_has_feature(KVM_FEATURE_PV_SEND_IPI))
766 		kvm_setup_pv_ipi();
767 #endif
768 }
769 
kvm_init_platform(void)770 static void __init kvm_init_platform(void)
771 {
772 	kvmclock_init();
773 	x86_platform.apic_post_init = kvm_apic_init;
774 }
775 
776 const __initconst struct hypervisor_x86 x86_hyper_kvm = {
777 	.name			= "KVM",
778 	.detect			= kvm_detect,
779 	.type			= X86_HYPER_KVM,
780 	.init.guest_late_init	= kvm_guest_init,
781 	.init.x2apic_available	= kvm_para_available,
782 	.init.init_platform	= kvm_init_platform,
783 };
784 
activate_jump_labels(void)785 static __init int activate_jump_labels(void)
786 {
787 	if (has_steal_clock) {
788 		static_key_slow_inc(&paravirt_steal_enabled);
789 		if (steal_acc)
790 			static_key_slow_inc(&paravirt_steal_rq_enabled);
791 	}
792 
793 	return 0;
794 }
795 arch_initcall(activate_jump_labels);
796 
kvm_setup_pv_tlb_flush(void)797 static __init int kvm_setup_pv_tlb_flush(void)
798 {
799 	int cpu;
800 
801 	if (kvm_para_has_feature(KVM_FEATURE_PV_TLB_FLUSH) &&
802 	    !kvm_para_has_hint(KVM_HINTS_REALTIME) &&
803 	    kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
804 		for_each_possible_cpu(cpu) {
805 			zalloc_cpumask_var_node(per_cpu_ptr(&__pv_tlb_mask, cpu),
806 				GFP_KERNEL, cpu_to_node(cpu));
807 		}
808 		pr_info("KVM setup pv remote TLB flush\n");
809 	}
810 
811 	return 0;
812 }
813 arch_initcall(kvm_setup_pv_tlb_flush);
814 
815 #ifdef CONFIG_PARAVIRT_SPINLOCKS
816 
817 /* Kick a cpu by its apicid. Used to wake up a halted vcpu */
kvm_kick_cpu(int cpu)818 static void kvm_kick_cpu(int cpu)
819 {
820 	int apicid;
821 	unsigned long flags = 0;
822 
823 	apicid = per_cpu(x86_cpu_to_apicid, cpu);
824 	kvm_hypercall2(KVM_HC_KICK_CPU, flags, apicid);
825 }
826 
827 #include <asm/qspinlock.h>
828 
kvm_wait(u8 * ptr,u8 val)829 static void kvm_wait(u8 *ptr, u8 val)
830 {
831 	unsigned long flags;
832 
833 	if (in_nmi())
834 		return;
835 
836 	local_irq_save(flags);
837 
838 	if (READ_ONCE(*ptr) != val)
839 		goto out;
840 
841 	/*
842 	 * halt until it's our turn and kicked. Note that we do safe halt
843 	 * for irq enabled case to avoid hang when lock info is overwritten
844 	 * in irq spinlock slowpath and no spurious interrupt occur to save us.
845 	 */
846 	if (arch_irqs_disabled_flags(flags))
847 		halt();
848 	else
849 		safe_halt();
850 
851 out:
852 	local_irq_restore(flags);
853 }
854 
855 #ifdef CONFIG_X86_32
__kvm_vcpu_is_preempted(long cpu)856 __visible bool __kvm_vcpu_is_preempted(long cpu)
857 {
858 	struct kvm_steal_time *src = &per_cpu(steal_time, cpu);
859 
860 	return !!(src->preempted & KVM_VCPU_PREEMPTED);
861 }
862 PV_CALLEE_SAVE_REGS_THUNK(__kvm_vcpu_is_preempted);
863 
864 #else
865 
866 #include <asm/asm-offsets.h>
867 
868 extern bool __raw_callee_save___kvm_vcpu_is_preempted(long);
869 
870 /*
871  * Hand-optimize version for x86-64 to avoid 8 64-bit register saving and
872  * restoring to/from the stack.
873  */
874 asm(
875 ".pushsection .text;"
876 ".global __raw_callee_save___kvm_vcpu_is_preempted;"
877 ".type __raw_callee_save___kvm_vcpu_is_preempted, @function;"
878 "__raw_callee_save___kvm_vcpu_is_preempted:"
879 "movq	__per_cpu_offset(,%rdi,8), %rax;"
880 "cmpb	$0, " __stringify(KVM_STEAL_TIME_preempted) "+steal_time(%rax);"
881 "setne	%al;"
882 "ret;"
883 ".size __raw_callee_save___kvm_vcpu_is_preempted, .-__raw_callee_save___kvm_vcpu_is_preempted;"
884 ".popsection");
885 
886 #endif
887 
888 /*
889  * Setup pv_lock_ops to exploit KVM_FEATURE_PV_UNHALT if present.
890  */
kvm_spinlock_init(void)891 void __init kvm_spinlock_init(void)
892 {
893 	/* Does host kernel support KVM_FEATURE_PV_UNHALT? */
894 	if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT))
895 		return;
896 
897 	if (kvm_para_has_hint(KVM_HINTS_REALTIME))
898 		return;
899 
900 	/* Don't use the pvqspinlock code if there is only 1 vCPU. */
901 	if (num_possible_cpus() == 1)
902 		return;
903 
904 	__pv_init_lock_hash();
905 	pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath;
906 	pv_ops.lock.queued_spin_unlock =
907 		PV_CALLEE_SAVE(__pv_queued_spin_unlock);
908 	pv_ops.lock.wait = kvm_wait;
909 	pv_ops.lock.kick = kvm_kick_cpu;
910 
911 	if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
912 		pv_ops.lock.vcpu_is_preempted =
913 			PV_CALLEE_SAVE(__kvm_vcpu_is_preempted);
914 	}
915 }
916 
917 #endif	/* CONFIG_PARAVIRT_SPINLOCKS */
918 
919 #ifdef CONFIG_ARCH_CPUIDLE_HALTPOLL
920 
kvm_disable_host_haltpoll(void * i)921 static void kvm_disable_host_haltpoll(void *i)
922 {
923 	wrmsrl(MSR_KVM_POLL_CONTROL, 0);
924 }
925 
kvm_enable_host_haltpoll(void * i)926 static void kvm_enable_host_haltpoll(void *i)
927 {
928 	wrmsrl(MSR_KVM_POLL_CONTROL, 1);
929 }
930 
arch_haltpoll_enable(unsigned int cpu)931 void arch_haltpoll_enable(unsigned int cpu)
932 {
933 	if (!kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL)) {
934 		pr_err_once("kvm: host does not support poll control\n");
935 		pr_err_once("kvm: host upgrade recommended\n");
936 		return;
937 	}
938 
939 	/* Enable guest halt poll disables host halt poll */
940 	smp_call_function_single(cpu, kvm_disable_host_haltpoll, NULL, 1);
941 }
942 EXPORT_SYMBOL_GPL(arch_haltpoll_enable);
943 
arch_haltpoll_disable(unsigned int cpu)944 void arch_haltpoll_disable(unsigned int cpu)
945 {
946 	if (!kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL))
947 		return;
948 
949 	/* Enable guest halt poll disables host halt poll */
950 	smp_call_function_single(cpu, kvm_enable_host_haltpoll, NULL, 1);
951 }
952 EXPORT_SYMBOL_GPL(arch_haltpoll_disable);
953 #endif
954