1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * KVM Microsoft Hyper-V emulation
4 *
5 * derived from arch/x86/kvm/x86.c
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 * Copyright (C) 2008 Qumranet, Inc.
9 * Copyright IBM Corporation, 2008
10 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
11 * Copyright (C) 2015 Andrey Smetanin <asmetanin@virtuozzo.com>
12 *
13 * Authors:
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
16 * Amit Shah <amit.shah@qumranet.com>
17 * Ben-Ami Yassour <benami@il.ibm.com>
18 * Andrey Smetanin <asmetanin@virtuozzo.com>
19 */
20
21 #include "x86.h"
22 #include "lapic.h"
23 #include "ioapic.h"
24 #include "cpuid.h"
25 #include "hyperv.h"
26 #include "xen.h"
27
28 #include <linux/cpu.h>
29 #include <linux/kvm_host.h>
30 #include <linux/highmem.h>
31 #include <linux/sched/cputime.h>
32 #include <linux/eventfd.h>
33
34 #include <asm/apicdef.h>
35 #include <trace/events/kvm.h>
36
37 #include "trace.h"
38 #include "irq.h"
39 #include "fpu.h"
40
41 /* "Hv#1" signature */
42 #define HYPERV_CPUID_SIGNATURE_EAX 0x31237648
43
44 #define KVM_HV_MAX_SPARSE_VCPU_SET_BITS DIV_ROUND_UP(KVM_MAX_VCPUS, 64)
45
46 static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer,
47 bool vcpu_kick);
48
synic_read_sint(struct kvm_vcpu_hv_synic * synic,int sint)49 static inline u64 synic_read_sint(struct kvm_vcpu_hv_synic *synic, int sint)
50 {
51 return atomic64_read(&synic->sint[sint]);
52 }
53
synic_get_sint_vector(u64 sint_value)54 static inline int synic_get_sint_vector(u64 sint_value)
55 {
56 if (sint_value & HV_SYNIC_SINT_MASKED)
57 return -1;
58 return sint_value & HV_SYNIC_SINT_VECTOR_MASK;
59 }
60
synic_has_vector_connected(struct kvm_vcpu_hv_synic * synic,int vector)61 static bool synic_has_vector_connected(struct kvm_vcpu_hv_synic *synic,
62 int vector)
63 {
64 int i;
65
66 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) {
67 if (synic_get_sint_vector(synic_read_sint(synic, i)) == vector)
68 return true;
69 }
70 return false;
71 }
72
synic_has_vector_auto_eoi(struct kvm_vcpu_hv_synic * synic,int vector)73 static bool synic_has_vector_auto_eoi(struct kvm_vcpu_hv_synic *synic,
74 int vector)
75 {
76 int i;
77 u64 sint_value;
78
79 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) {
80 sint_value = synic_read_sint(synic, i);
81 if (synic_get_sint_vector(sint_value) == vector &&
82 sint_value & HV_SYNIC_SINT_AUTO_EOI)
83 return true;
84 }
85 return false;
86 }
87
synic_update_vector(struct kvm_vcpu_hv_synic * synic,int vector)88 static void synic_update_vector(struct kvm_vcpu_hv_synic *synic,
89 int vector)
90 {
91 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic);
92 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
93 int auto_eoi_old, auto_eoi_new;
94
95 if (vector < HV_SYNIC_FIRST_VALID_VECTOR)
96 return;
97
98 if (synic_has_vector_connected(synic, vector))
99 __set_bit(vector, synic->vec_bitmap);
100 else
101 __clear_bit(vector, synic->vec_bitmap);
102
103 auto_eoi_old = bitmap_weight(synic->auto_eoi_bitmap, 256);
104
105 if (synic_has_vector_auto_eoi(synic, vector))
106 __set_bit(vector, synic->auto_eoi_bitmap);
107 else
108 __clear_bit(vector, synic->auto_eoi_bitmap);
109
110 auto_eoi_new = bitmap_weight(synic->auto_eoi_bitmap, 256);
111
112 if (!!auto_eoi_old == !!auto_eoi_new)
113 return;
114
115 mutex_lock(&vcpu->kvm->arch.apicv_update_lock);
116
117 if (auto_eoi_new)
118 hv->synic_auto_eoi_used++;
119 else
120 hv->synic_auto_eoi_used--;
121
122 __kvm_request_apicv_update(vcpu->kvm,
123 !hv->synic_auto_eoi_used,
124 APICV_INHIBIT_REASON_HYPERV);
125
126 mutex_unlock(&vcpu->kvm->arch.apicv_update_lock);
127 }
128
synic_set_sint(struct kvm_vcpu_hv_synic * synic,int sint,u64 data,bool host)129 static int synic_set_sint(struct kvm_vcpu_hv_synic *synic, int sint,
130 u64 data, bool host)
131 {
132 int vector, old_vector;
133 bool masked;
134
135 vector = data & HV_SYNIC_SINT_VECTOR_MASK;
136 masked = data & HV_SYNIC_SINT_MASKED;
137
138 /*
139 * Valid vectors are 16-255, however, nested Hyper-V attempts to write
140 * default '0x10000' value on boot and this should not #GP. We need to
141 * allow zero-initing the register from host as well.
142 */
143 if (vector < HV_SYNIC_FIRST_VALID_VECTOR && !host && !masked)
144 return 1;
145 /*
146 * Guest may configure multiple SINTs to use the same vector, so
147 * we maintain a bitmap of vectors handled by synic, and a
148 * bitmap of vectors with auto-eoi behavior. The bitmaps are
149 * updated here, and atomically queried on fast paths.
150 */
151 old_vector = synic_read_sint(synic, sint) & HV_SYNIC_SINT_VECTOR_MASK;
152
153 atomic64_set(&synic->sint[sint], data);
154
155 synic_update_vector(synic, old_vector);
156
157 synic_update_vector(synic, vector);
158
159 /* Load SynIC vectors into EOI exit bitmap */
160 kvm_make_request(KVM_REQ_SCAN_IOAPIC, hv_synic_to_vcpu(synic));
161 return 0;
162 }
163
get_vcpu_by_vpidx(struct kvm * kvm,u32 vpidx)164 static struct kvm_vcpu *get_vcpu_by_vpidx(struct kvm *kvm, u32 vpidx)
165 {
166 struct kvm_vcpu *vcpu = NULL;
167 int i;
168
169 if (vpidx >= KVM_MAX_VCPUS)
170 return NULL;
171
172 vcpu = kvm_get_vcpu(kvm, vpidx);
173 if (vcpu && kvm_hv_get_vpindex(vcpu) == vpidx)
174 return vcpu;
175 kvm_for_each_vcpu(i, vcpu, kvm)
176 if (kvm_hv_get_vpindex(vcpu) == vpidx)
177 return vcpu;
178 return NULL;
179 }
180
synic_get(struct kvm * kvm,u32 vpidx)181 static struct kvm_vcpu_hv_synic *synic_get(struct kvm *kvm, u32 vpidx)
182 {
183 struct kvm_vcpu *vcpu;
184 struct kvm_vcpu_hv_synic *synic;
185
186 vcpu = get_vcpu_by_vpidx(kvm, vpidx);
187 if (!vcpu || !to_hv_vcpu(vcpu))
188 return NULL;
189 synic = to_hv_synic(vcpu);
190 return (synic->active) ? synic : NULL;
191 }
192
kvm_hv_notify_acked_sint(struct kvm_vcpu * vcpu,u32 sint)193 static void kvm_hv_notify_acked_sint(struct kvm_vcpu *vcpu, u32 sint)
194 {
195 struct kvm *kvm = vcpu->kvm;
196 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu);
197 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
198 struct kvm_vcpu_hv_stimer *stimer;
199 int gsi, idx;
200
201 trace_kvm_hv_notify_acked_sint(vcpu->vcpu_id, sint);
202
203 /* Try to deliver pending Hyper-V SynIC timers messages */
204 for (idx = 0; idx < ARRAY_SIZE(hv_vcpu->stimer); idx++) {
205 stimer = &hv_vcpu->stimer[idx];
206 if (stimer->msg_pending && stimer->config.enable &&
207 !stimer->config.direct_mode &&
208 stimer->config.sintx == sint)
209 stimer_mark_pending(stimer, false);
210 }
211
212 idx = srcu_read_lock(&kvm->irq_srcu);
213 gsi = atomic_read(&synic->sint_to_gsi[sint]);
214 if (gsi != -1)
215 kvm_notify_acked_gsi(kvm, gsi);
216 srcu_read_unlock(&kvm->irq_srcu, idx);
217 }
218
synic_exit(struct kvm_vcpu_hv_synic * synic,u32 msr)219 static void synic_exit(struct kvm_vcpu_hv_synic *synic, u32 msr)
220 {
221 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic);
222 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
223
224 hv_vcpu->exit.type = KVM_EXIT_HYPERV_SYNIC;
225 hv_vcpu->exit.u.synic.msr = msr;
226 hv_vcpu->exit.u.synic.control = synic->control;
227 hv_vcpu->exit.u.synic.evt_page = synic->evt_page;
228 hv_vcpu->exit.u.synic.msg_page = synic->msg_page;
229
230 kvm_make_request(KVM_REQ_HV_EXIT, vcpu);
231 }
232
synic_set_msr(struct kvm_vcpu_hv_synic * synic,u32 msr,u64 data,bool host)233 static int synic_set_msr(struct kvm_vcpu_hv_synic *synic,
234 u32 msr, u64 data, bool host)
235 {
236 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic);
237 int ret;
238
239 if (!synic->active && (!host || data))
240 return 1;
241
242 trace_kvm_hv_synic_set_msr(vcpu->vcpu_id, msr, data, host);
243
244 ret = 0;
245 switch (msr) {
246 case HV_X64_MSR_SCONTROL:
247 synic->control = data;
248 if (!host)
249 synic_exit(synic, msr);
250 break;
251 case HV_X64_MSR_SVERSION:
252 if (!host) {
253 ret = 1;
254 break;
255 }
256 synic->version = data;
257 break;
258 case HV_X64_MSR_SIEFP:
259 if ((data & HV_SYNIC_SIEFP_ENABLE) && !host &&
260 !synic->dont_zero_synic_pages)
261 if (kvm_clear_guest(vcpu->kvm,
262 data & PAGE_MASK, PAGE_SIZE)) {
263 ret = 1;
264 break;
265 }
266 synic->evt_page = data;
267 if (!host)
268 synic_exit(synic, msr);
269 break;
270 case HV_X64_MSR_SIMP:
271 if ((data & HV_SYNIC_SIMP_ENABLE) && !host &&
272 !synic->dont_zero_synic_pages)
273 if (kvm_clear_guest(vcpu->kvm,
274 data & PAGE_MASK, PAGE_SIZE)) {
275 ret = 1;
276 break;
277 }
278 synic->msg_page = data;
279 if (!host)
280 synic_exit(synic, msr);
281 break;
282 case HV_X64_MSR_EOM: {
283 int i;
284
285 if (!synic->active)
286 break;
287
288 for (i = 0; i < ARRAY_SIZE(synic->sint); i++)
289 kvm_hv_notify_acked_sint(vcpu, i);
290 break;
291 }
292 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15:
293 ret = synic_set_sint(synic, msr - HV_X64_MSR_SINT0, data, host);
294 break;
295 default:
296 ret = 1;
297 break;
298 }
299 return ret;
300 }
301
kvm_hv_is_syndbg_enabled(struct kvm_vcpu * vcpu)302 static bool kvm_hv_is_syndbg_enabled(struct kvm_vcpu *vcpu)
303 {
304 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
305
306 return hv_vcpu->cpuid_cache.syndbg_cap_eax &
307 HV_X64_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING;
308 }
309
kvm_hv_syndbg_complete_userspace(struct kvm_vcpu * vcpu)310 static int kvm_hv_syndbg_complete_userspace(struct kvm_vcpu *vcpu)
311 {
312 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
313
314 if (vcpu->run->hyperv.u.syndbg.msr == HV_X64_MSR_SYNDBG_CONTROL)
315 hv->hv_syndbg.control.status =
316 vcpu->run->hyperv.u.syndbg.status;
317 return 1;
318 }
319
syndbg_exit(struct kvm_vcpu * vcpu,u32 msr)320 static void syndbg_exit(struct kvm_vcpu *vcpu, u32 msr)
321 {
322 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu);
323 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
324
325 hv_vcpu->exit.type = KVM_EXIT_HYPERV_SYNDBG;
326 hv_vcpu->exit.u.syndbg.msr = msr;
327 hv_vcpu->exit.u.syndbg.control = syndbg->control.control;
328 hv_vcpu->exit.u.syndbg.send_page = syndbg->control.send_page;
329 hv_vcpu->exit.u.syndbg.recv_page = syndbg->control.recv_page;
330 hv_vcpu->exit.u.syndbg.pending_page = syndbg->control.pending_page;
331 vcpu->arch.complete_userspace_io =
332 kvm_hv_syndbg_complete_userspace;
333
334 kvm_make_request(KVM_REQ_HV_EXIT, vcpu);
335 }
336
syndbg_set_msr(struct kvm_vcpu * vcpu,u32 msr,u64 data,bool host)337 static int syndbg_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host)
338 {
339 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu);
340
341 if (!kvm_hv_is_syndbg_enabled(vcpu) && !host)
342 return 1;
343
344 trace_kvm_hv_syndbg_set_msr(vcpu->vcpu_id,
345 to_hv_vcpu(vcpu)->vp_index, msr, data);
346 switch (msr) {
347 case HV_X64_MSR_SYNDBG_CONTROL:
348 syndbg->control.control = data;
349 if (!host)
350 syndbg_exit(vcpu, msr);
351 break;
352 case HV_X64_MSR_SYNDBG_STATUS:
353 syndbg->control.status = data;
354 break;
355 case HV_X64_MSR_SYNDBG_SEND_BUFFER:
356 syndbg->control.send_page = data;
357 break;
358 case HV_X64_MSR_SYNDBG_RECV_BUFFER:
359 syndbg->control.recv_page = data;
360 break;
361 case HV_X64_MSR_SYNDBG_PENDING_BUFFER:
362 syndbg->control.pending_page = data;
363 if (!host)
364 syndbg_exit(vcpu, msr);
365 break;
366 case HV_X64_MSR_SYNDBG_OPTIONS:
367 syndbg->options = data;
368 break;
369 default:
370 break;
371 }
372
373 return 0;
374 }
375
syndbg_get_msr(struct kvm_vcpu * vcpu,u32 msr,u64 * pdata,bool host)376 static int syndbg_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)
377 {
378 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu);
379
380 if (!kvm_hv_is_syndbg_enabled(vcpu) && !host)
381 return 1;
382
383 switch (msr) {
384 case HV_X64_MSR_SYNDBG_CONTROL:
385 *pdata = syndbg->control.control;
386 break;
387 case HV_X64_MSR_SYNDBG_STATUS:
388 *pdata = syndbg->control.status;
389 break;
390 case HV_X64_MSR_SYNDBG_SEND_BUFFER:
391 *pdata = syndbg->control.send_page;
392 break;
393 case HV_X64_MSR_SYNDBG_RECV_BUFFER:
394 *pdata = syndbg->control.recv_page;
395 break;
396 case HV_X64_MSR_SYNDBG_PENDING_BUFFER:
397 *pdata = syndbg->control.pending_page;
398 break;
399 case HV_X64_MSR_SYNDBG_OPTIONS:
400 *pdata = syndbg->options;
401 break;
402 default:
403 break;
404 }
405
406 trace_kvm_hv_syndbg_get_msr(vcpu->vcpu_id, kvm_hv_get_vpindex(vcpu), msr, *pdata);
407
408 return 0;
409 }
410
synic_get_msr(struct kvm_vcpu_hv_synic * synic,u32 msr,u64 * pdata,bool host)411 static int synic_get_msr(struct kvm_vcpu_hv_synic *synic, u32 msr, u64 *pdata,
412 bool host)
413 {
414 int ret;
415
416 if (!synic->active && !host)
417 return 1;
418
419 ret = 0;
420 switch (msr) {
421 case HV_X64_MSR_SCONTROL:
422 *pdata = synic->control;
423 break;
424 case HV_X64_MSR_SVERSION:
425 *pdata = synic->version;
426 break;
427 case HV_X64_MSR_SIEFP:
428 *pdata = synic->evt_page;
429 break;
430 case HV_X64_MSR_SIMP:
431 *pdata = synic->msg_page;
432 break;
433 case HV_X64_MSR_EOM:
434 *pdata = 0;
435 break;
436 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15:
437 *pdata = atomic64_read(&synic->sint[msr - HV_X64_MSR_SINT0]);
438 break;
439 default:
440 ret = 1;
441 break;
442 }
443 return ret;
444 }
445
synic_set_irq(struct kvm_vcpu_hv_synic * synic,u32 sint)446 static int synic_set_irq(struct kvm_vcpu_hv_synic *synic, u32 sint)
447 {
448 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic);
449 struct kvm_lapic_irq irq;
450 int ret, vector;
451
452 if (KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm))
453 return -EINVAL;
454
455 if (sint >= ARRAY_SIZE(synic->sint))
456 return -EINVAL;
457
458 vector = synic_get_sint_vector(synic_read_sint(synic, sint));
459 if (vector < 0)
460 return -ENOENT;
461
462 memset(&irq, 0, sizeof(irq));
463 irq.shorthand = APIC_DEST_SELF;
464 irq.dest_mode = APIC_DEST_PHYSICAL;
465 irq.delivery_mode = APIC_DM_FIXED;
466 irq.vector = vector;
467 irq.level = 1;
468
469 ret = kvm_irq_delivery_to_apic(vcpu->kvm, vcpu->arch.apic, &irq, NULL);
470 trace_kvm_hv_synic_set_irq(vcpu->vcpu_id, sint, irq.vector, ret);
471 return ret;
472 }
473
kvm_hv_synic_set_irq(struct kvm * kvm,u32 vpidx,u32 sint)474 int kvm_hv_synic_set_irq(struct kvm *kvm, u32 vpidx, u32 sint)
475 {
476 struct kvm_vcpu_hv_synic *synic;
477
478 synic = synic_get(kvm, vpidx);
479 if (!synic)
480 return -EINVAL;
481
482 return synic_set_irq(synic, sint);
483 }
484
kvm_hv_synic_send_eoi(struct kvm_vcpu * vcpu,int vector)485 void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector)
486 {
487 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu);
488 int i;
489
490 trace_kvm_hv_synic_send_eoi(vcpu->vcpu_id, vector);
491
492 for (i = 0; i < ARRAY_SIZE(synic->sint); i++)
493 if (synic_get_sint_vector(synic_read_sint(synic, i)) == vector)
494 kvm_hv_notify_acked_sint(vcpu, i);
495 }
496
kvm_hv_set_sint_gsi(struct kvm * kvm,u32 vpidx,u32 sint,int gsi)497 static int kvm_hv_set_sint_gsi(struct kvm *kvm, u32 vpidx, u32 sint, int gsi)
498 {
499 struct kvm_vcpu_hv_synic *synic;
500
501 synic = synic_get(kvm, vpidx);
502 if (!synic)
503 return -EINVAL;
504
505 if (sint >= ARRAY_SIZE(synic->sint_to_gsi))
506 return -EINVAL;
507
508 atomic_set(&synic->sint_to_gsi[sint], gsi);
509 return 0;
510 }
511
kvm_hv_irq_routing_update(struct kvm * kvm)512 void kvm_hv_irq_routing_update(struct kvm *kvm)
513 {
514 struct kvm_irq_routing_table *irq_rt;
515 struct kvm_kernel_irq_routing_entry *e;
516 u32 gsi;
517
518 irq_rt = srcu_dereference_check(kvm->irq_routing, &kvm->irq_srcu,
519 lockdep_is_held(&kvm->irq_lock));
520
521 for (gsi = 0; gsi < irq_rt->nr_rt_entries; gsi++) {
522 hlist_for_each_entry(e, &irq_rt->map[gsi], link) {
523 if (e->type == KVM_IRQ_ROUTING_HV_SINT)
524 kvm_hv_set_sint_gsi(kvm, e->hv_sint.vcpu,
525 e->hv_sint.sint, gsi);
526 }
527 }
528 }
529
synic_init(struct kvm_vcpu_hv_synic * synic)530 static void synic_init(struct kvm_vcpu_hv_synic *synic)
531 {
532 int i;
533
534 memset(synic, 0, sizeof(*synic));
535 synic->version = HV_SYNIC_VERSION_1;
536 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) {
537 atomic64_set(&synic->sint[i], HV_SYNIC_SINT_MASKED);
538 atomic_set(&synic->sint_to_gsi[i], -1);
539 }
540 }
541
get_time_ref_counter(struct kvm * kvm)542 static u64 get_time_ref_counter(struct kvm *kvm)
543 {
544 struct kvm_hv *hv = to_kvm_hv(kvm);
545 struct kvm_vcpu *vcpu;
546 u64 tsc;
547
548 /*
549 * Fall back to get_kvmclock_ns() when TSC page hasn't been set up,
550 * is broken, disabled or being updated.
551 */
552 if (hv->hv_tsc_page_status != HV_TSC_PAGE_SET)
553 return div_u64(get_kvmclock_ns(kvm), 100);
554
555 vcpu = kvm_get_vcpu(kvm, 0);
556 tsc = kvm_read_l1_tsc(vcpu, rdtsc());
557 return mul_u64_u64_shr(tsc, hv->tsc_ref.tsc_scale, 64)
558 + hv->tsc_ref.tsc_offset;
559 }
560
stimer_mark_pending(struct kvm_vcpu_hv_stimer * stimer,bool vcpu_kick)561 static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer,
562 bool vcpu_kick)
563 {
564 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
565
566 set_bit(stimer->index,
567 to_hv_vcpu(vcpu)->stimer_pending_bitmap);
568 kvm_make_request(KVM_REQ_HV_STIMER, vcpu);
569 if (vcpu_kick)
570 kvm_vcpu_kick(vcpu);
571 }
572
stimer_cleanup(struct kvm_vcpu_hv_stimer * stimer)573 static void stimer_cleanup(struct kvm_vcpu_hv_stimer *stimer)
574 {
575 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
576
577 trace_kvm_hv_stimer_cleanup(hv_stimer_to_vcpu(stimer)->vcpu_id,
578 stimer->index);
579
580 hrtimer_cancel(&stimer->timer);
581 clear_bit(stimer->index,
582 to_hv_vcpu(vcpu)->stimer_pending_bitmap);
583 stimer->msg_pending = false;
584 stimer->exp_time = 0;
585 }
586
stimer_timer_callback(struct hrtimer * timer)587 static enum hrtimer_restart stimer_timer_callback(struct hrtimer *timer)
588 {
589 struct kvm_vcpu_hv_stimer *stimer;
590
591 stimer = container_of(timer, struct kvm_vcpu_hv_stimer, timer);
592 trace_kvm_hv_stimer_callback(hv_stimer_to_vcpu(stimer)->vcpu_id,
593 stimer->index);
594 stimer_mark_pending(stimer, true);
595
596 return HRTIMER_NORESTART;
597 }
598
599 /*
600 * stimer_start() assumptions:
601 * a) stimer->count is not equal to 0
602 * b) stimer->config has HV_STIMER_ENABLE flag
603 */
stimer_start(struct kvm_vcpu_hv_stimer * stimer)604 static int stimer_start(struct kvm_vcpu_hv_stimer *stimer)
605 {
606 u64 time_now;
607 ktime_t ktime_now;
608
609 time_now = get_time_ref_counter(hv_stimer_to_vcpu(stimer)->kvm);
610 ktime_now = ktime_get();
611
612 if (stimer->config.periodic) {
613 if (stimer->exp_time) {
614 if (time_now >= stimer->exp_time) {
615 u64 remainder;
616
617 div64_u64_rem(time_now - stimer->exp_time,
618 stimer->count, &remainder);
619 stimer->exp_time =
620 time_now + (stimer->count - remainder);
621 }
622 } else
623 stimer->exp_time = time_now + stimer->count;
624
625 trace_kvm_hv_stimer_start_periodic(
626 hv_stimer_to_vcpu(stimer)->vcpu_id,
627 stimer->index,
628 time_now, stimer->exp_time);
629
630 hrtimer_start(&stimer->timer,
631 ktime_add_ns(ktime_now,
632 100 * (stimer->exp_time - time_now)),
633 HRTIMER_MODE_ABS);
634 return 0;
635 }
636 stimer->exp_time = stimer->count;
637 if (time_now >= stimer->count) {
638 /*
639 * Expire timer according to Hypervisor Top-Level Functional
640 * specification v4(15.3.1):
641 * "If a one shot is enabled and the specified count is in
642 * the past, it will expire immediately."
643 */
644 stimer_mark_pending(stimer, false);
645 return 0;
646 }
647
648 trace_kvm_hv_stimer_start_one_shot(hv_stimer_to_vcpu(stimer)->vcpu_id,
649 stimer->index,
650 time_now, stimer->count);
651
652 hrtimer_start(&stimer->timer,
653 ktime_add_ns(ktime_now, 100 * (stimer->count - time_now)),
654 HRTIMER_MODE_ABS);
655 return 0;
656 }
657
stimer_set_config(struct kvm_vcpu_hv_stimer * stimer,u64 config,bool host)658 static int stimer_set_config(struct kvm_vcpu_hv_stimer *stimer, u64 config,
659 bool host)
660 {
661 union hv_stimer_config new_config = {.as_uint64 = config},
662 old_config = {.as_uint64 = stimer->config.as_uint64};
663 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
664 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
665 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu);
666
667 if (!synic->active && (!host || config))
668 return 1;
669
670 if (unlikely(!host && hv_vcpu->enforce_cpuid && new_config.direct_mode &&
671 !(hv_vcpu->cpuid_cache.features_edx &
672 HV_STIMER_DIRECT_MODE_AVAILABLE)))
673 return 1;
674
675 trace_kvm_hv_stimer_set_config(hv_stimer_to_vcpu(stimer)->vcpu_id,
676 stimer->index, config, host);
677
678 stimer_cleanup(stimer);
679 if (old_config.enable &&
680 !new_config.direct_mode && new_config.sintx == 0)
681 new_config.enable = 0;
682 stimer->config.as_uint64 = new_config.as_uint64;
683
684 if (stimer->config.enable)
685 stimer_mark_pending(stimer, false);
686
687 return 0;
688 }
689
stimer_set_count(struct kvm_vcpu_hv_stimer * stimer,u64 count,bool host)690 static int stimer_set_count(struct kvm_vcpu_hv_stimer *stimer, u64 count,
691 bool host)
692 {
693 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
694 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu);
695
696 if (!synic->active && (!host || count))
697 return 1;
698
699 trace_kvm_hv_stimer_set_count(hv_stimer_to_vcpu(stimer)->vcpu_id,
700 stimer->index, count, host);
701
702 stimer_cleanup(stimer);
703 stimer->count = count;
704 if (!host) {
705 if (stimer->count == 0)
706 stimer->config.enable = 0;
707 else if (stimer->config.auto_enable)
708 stimer->config.enable = 1;
709 }
710
711 if (stimer->config.enable)
712 stimer_mark_pending(stimer, false);
713
714 return 0;
715 }
716
stimer_get_config(struct kvm_vcpu_hv_stimer * stimer,u64 * pconfig)717 static int stimer_get_config(struct kvm_vcpu_hv_stimer *stimer, u64 *pconfig)
718 {
719 *pconfig = stimer->config.as_uint64;
720 return 0;
721 }
722
stimer_get_count(struct kvm_vcpu_hv_stimer * stimer,u64 * pcount)723 static int stimer_get_count(struct kvm_vcpu_hv_stimer *stimer, u64 *pcount)
724 {
725 *pcount = stimer->count;
726 return 0;
727 }
728
synic_deliver_msg(struct kvm_vcpu_hv_synic * synic,u32 sint,struct hv_message * src_msg,bool no_retry)729 static int synic_deliver_msg(struct kvm_vcpu_hv_synic *synic, u32 sint,
730 struct hv_message *src_msg, bool no_retry)
731 {
732 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic);
733 int msg_off = offsetof(struct hv_message_page, sint_message[sint]);
734 gfn_t msg_page_gfn;
735 struct hv_message_header hv_hdr;
736 int r;
737
738 if (!(synic->msg_page & HV_SYNIC_SIMP_ENABLE))
739 return -ENOENT;
740
741 msg_page_gfn = synic->msg_page >> PAGE_SHIFT;
742
743 /*
744 * Strictly following the spec-mandated ordering would assume setting
745 * .msg_pending before checking .message_type. However, this function
746 * is only called in vcpu context so the entire update is atomic from
747 * guest POV and thus the exact order here doesn't matter.
748 */
749 r = kvm_vcpu_read_guest_page(vcpu, msg_page_gfn, &hv_hdr.message_type,
750 msg_off + offsetof(struct hv_message,
751 header.message_type),
752 sizeof(hv_hdr.message_type));
753 if (r < 0)
754 return r;
755
756 if (hv_hdr.message_type != HVMSG_NONE) {
757 if (no_retry)
758 return 0;
759
760 hv_hdr.message_flags.msg_pending = 1;
761 r = kvm_vcpu_write_guest_page(vcpu, msg_page_gfn,
762 &hv_hdr.message_flags,
763 msg_off +
764 offsetof(struct hv_message,
765 header.message_flags),
766 sizeof(hv_hdr.message_flags));
767 if (r < 0)
768 return r;
769 return -EAGAIN;
770 }
771
772 r = kvm_vcpu_write_guest_page(vcpu, msg_page_gfn, src_msg, msg_off,
773 sizeof(src_msg->header) +
774 src_msg->header.payload_size);
775 if (r < 0)
776 return r;
777
778 r = synic_set_irq(synic, sint);
779 if (r < 0)
780 return r;
781 if (r == 0)
782 return -EFAULT;
783 return 0;
784 }
785
stimer_send_msg(struct kvm_vcpu_hv_stimer * stimer)786 static int stimer_send_msg(struct kvm_vcpu_hv_stimer *stimer)
787 {
788 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
789 struct hv_message *msg = &stimer->msg;
790 struct hv_timer_message_payload *payload =
791 (struct hv_timer_message_payload *)&msg->u.payload;
792
793 /*
794 * To avoid piling up periodic ticks, don't retry message
795 * delivery for them (within "lazy" lost ticks policy).
796 */
797 bool no_retry = stimer->config.periodic;
798
799 payload->expiration_time = stimer->exp_time;
800 payload->delivery_time = get_time_ref_counter(vcpu->kvm);
801 return synic_deliver_msg(to_hv_synic(vcpu),
802 stimer->config.sintx, msg,
803 no_retry);
804 }
805
stimer_notify_direct(struct kvm_vcpu_hv_stimer * stimer)806 static int stimer_notify_direct(struct kvm_vcpu_hv_stimer *stimer)
807 {
808 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
809 struct kvm_lapic_irq irq = {
810 .delivery_mode = APIC_DM_FIXED,
811 .vector = stimer->config.apic_vector
812 };
813
814 if (lapic_in_kernel(vcpu))
815 return !kvm_apic_set_irq(vcpu, &irq, NULL);
816 return 0;
817 }
818
stimer_expiration(struct kvm_vcpu_hv_stimer * stimer)819 static void stimer_expiration(struct kvm_vcpu_hv_stimer *stimer)
820 {
821 int r, direct = stimer->config.direct_mode;
822
823 stimer->msg_pending = true;
824 if (!direct)
825 r = stimer_send_msg(stimer);
826 else
827 r = stimer_notify_direct(stimer);
828 trace_kvm_hv_stimer_expiration(hv_stimer_to_vcpu(stimer)->vcpu_id,
829 stimer->index, direct, r);
830 if (!r) {
831 stimer->msg_pending = false;
832 if (!(stimer->config.periodic))
833 stimer->config.enable = 0;
834 }
835 }
836
kvm_hv_process_stimers(struct kvm_vcpu * vcpu)837 void kvm_hv_process_stimers(struct kvm_vcpu *vcpu)
838 {
839 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
840 struct kvm_vcpu_hv_stimer *stimer;
841 u64 time_now, exp_time;
842 int i;
843
844 if (!hv_vcpu)
845 return;
846
847 for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++)
848 if (test_and_clear_bit(i, hv_vcpu->stimer_pending_bitmap)) {
849 stimer = &hv_vcpu->stimer[i];
850 if (stimer->config.enable) {
851 exp_time = stimer->exp_time;
852
853 if (exp_time) {
854 time_now =
855 get_time_ref_counter(vcpu->kvm);
856 if (time_now >= exp_time)
857 stimer_expiration(stimer);
858 }
859
860 if ((stimer->config.enable) &&
861 stimer->count) {
862 if (!stimer->msg_pending)
863 stimer_start(stimer);
864 } else
865 stimer_cleanup(stimer);
866 }
867 }
868 }
869
kvm_hv_vcpu_uninit(struct kvm_vcpu * vcpu)870 void kvm_hv_vcpu_uninit(struct kvm_vcpu *vcpu)
871 {
872 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
873 int i;
874
875 if (!hv_vcpu)
876 return;
877
878 for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++)
879 stimer_cleanup(&hv_vcpu->stimer[i]);
880
881 kfree(hv_vcpu);
882 vcpu->arch.hyperv = NULL;
883 }
884
kvm_hv_assist_page_enabled(struct kvm_vcpu * vcpu)885 bool kvm_hv_assist_page_enabled(struct kvm_vcpu *vcpu)
886 {
887 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
888
889 if (!hv_vcpu)
890 return false;
891
892 if (!(hv_vcpu->hv_vapic & HV_X64_MSR_VP_ASSIST_PAGE_ENABLE))
893 return false;
894 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
895 }
896 EXPORT_SYMBOL_GPL(kvm_hv_assist_page_enabled);
897
kvm_hv_get_assist_page(struct kvm_vcpu * vcpu,struct hv_vp_assist_page * assist_page)898 bool kvm_hv_get_assist_page(struct kvm_vcpu *vcpu,
899 struct hv_vp_assist_page *assist_page)
900 {
901 if (!kvm_hv_assist_page_enabled(vcpu))
902 return false;
903 return !kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data,
904 assist_page, sizeof(*assist_page));
905 }
906 EXPORT_SYMBOL_GPL(kvm_hv_get_assist_page);
907
stimer_prepare_msg(struct kvm_vcpu_hv_stimer * stimer)908 static void stimer_prepare_msg(struct kvm_vcpu_hv_stimer *stimer)
909 {
910 struct hv_message *msg = &stimer->msg;
911 struct hv_timer_message_payload *payload =
912 (struct hv_timer_message_payload *)&msg->u.payload;
913
914 memset(&msg->header, 0, sizeof(msg->header));
915 msg->header.message_type = HVMSG_TIMER_EXPIRED;
916 msg->header.payload_size = sizeof(*payload);
917
918 payload->timer_index = stimer->index;
919 payload->expiration_time = 0;
920 payload->delivery_time = 0;
921 }
922
stimer_init(struct kvm_vcpu_hv_stimer * stimer,int timer_index)923 static void stimer_init(struct kvm_vcpu_hv_stimer *stimer, int timer_index)
924 {
925 memset(stimer, 0, sizeof(*stimer));
926 stimer->index = timer_index;
927 hrtimer_init(&stimer->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
928 stimer->timer.function = stimer_timer_callback;
929 stimer_prepare_msg(stimer);
930 }
931
kvm_hv_vcpu_init(struct kvm_vcpu * vcpu)932 static int kvm_hv_vcpu_init(struct kvm_vcpu *vcpu)
933 {
934 struct kvm_vcpu_hv *hv_vcpu;
935 int i;
936
937 hv_vcpu = kzalloc(sizeof(struct kvm_vcpu_hv), GFP_KERNEL_ACCOUNT);
938 if (!hv_vcpu)
939 return -ENOMEM;
940
941 vcpu->arch.hyperv = hv_vcpu;
942 hv_vcpu->vcpu = vcpu;
943
944 synic_init(&hv_vcpu->synic);
945
946 bitmap_zero(hv_vcpu->stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT);
947 for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++)
948 stimer_init(&hv_vcpu->stimer[i], i);
949
950 hv_vcpu->vp_index = vcpu->vcpu_idx;
951
952 return 0;
953 }
954
kvm_hv_activate_synic(struct kvm_vcpu * vcpu,bool dont_zero_synic_pages)955 int kvm_hv_activate_synic(struct kvm_vcpu *vcpu, bool dont_zero_synic_pages)
956 {
957 struct kvm_vcpu_hv_synic *synic;
958 int r;
959
960 if (!to_hv_vcpu(vcpu)) {
961 r = kvm_hv_vcpu_init(vcpu);
962 if (r)
963 return r;
964 }
965
966 synic = to_hv_synic(vcpu);
967
968 synic->active = true;
969 synic->dont_zero_synic_pages = dont_zero_synic_pages;
970 synic->control = HV_SYNIC_CONTROL_ENABLE;
971 return 0;
972 }
973
kvm_hv_msr_partition_wide(u32 msr)974 static bool kvm_hv_msr_partition_wide(u32 msr)
975 {
976 bool r = false;
977
978 switch (msr) {
979 case HV_X64_MSR_GUEST_OS_ID:
980 case HV_X64_MSR_HYPERCALL:
981 case HV_X64_MSR_REFERENCE_TSC:
982 case HV_X64_MSR_TIME_REF_COUNT:
983 case HV_X64_MSR_CRASH_CTL:
984 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
985 case HV_X64_MSR_RESET:
986 case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
987 case HV_X64_MSR_TSC_EMULATION_CONTROL:
988 case HV_X64_MSR_TSC_EMULATION_STATUS:
989 case HV_X64_MSR_SYNDBG_OPTIONS:
990 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER:
991 r = true;
992 break;
993 }
994
995 return r;
996 }
997
kvm_hv_msr_get_crash_data(struct kvm * kvm,u32 index,u64 * pdata)998 static int kvm_hv_msr_get_crash_data(struct kvm *kvm, u32 index, u64 *pdata)
999 {
1000 struct kvm_hv *hv = to_kvm_hv(kvm);
1001 size_t size = ARRAY_SIZE(hv->hv_crash_param);
1002
1003 if (WARN_ON_ONCE(index >= size))
1004 return -EINVAL;
1005
1006 *pdata = hv->hv_crash_param[array_index_nospec(index, size)];
1007 return 0;
1008 }
1009
kvm_hv_msr_get_crash_ctl(struct kvm * kvm,u64 * pdata)1010 static int kvm_hv_msr_get_crash_ctl(struct kvm *kvm, u64 *pdata)
1011 {
1012 struct kvm_hv *hv = to_kvm_hv(kvm);
1013
1014 *pdata = hv->hv_crash_ctl;
1015 return 0;
1016 }
1017
kvm_hv_msr_set_crash_ctl(struct kvm * kvm,u64 data)1018 static int kvm_hv_msr_set_crash_ctl(struct kvm *kvm, u64 data)
1019 {
1020 struct kvm_hv *hv = to_kvm_hv(kvm);
1021
1022 hv->hv_crash_ctl = data & HV_CRASH_CTL_CRASH_NOTIFY;
1023
1024 return 0;
1025 }
1026
kvm_hv_msr_set_crash_data(struct kvm * kvm,u32 index,u64 data)1027 static int kvm_hv_msr_set_crash_data(struct kvm *kvm, u32 index, u64 data)
1028 {
1029 struct kvm_hv *hv = to_kvm_hv(kvm);
1030 size_t size = ARRAY_SIZE(hv->hv_crash_param);
1031
1032 if (WARN_ON_ONCE(index >= size))
1033 return -EINVAL;
1034
1035 hv->hv_crash_param[array_index_nospec(index, size)] = data;
1036 return 0;
1037 }
1038
1039 /*
1040 * The kvmclock and Hyper-V TSC page use similar formulas, and converting
1041 * between them is possible:
1042 *
1043 * kvmclock formula:
1044 * nsec = (ticks - tsc_timestamp) * tsc_to_system_mul * 2^(tsc_shift-32)
1045 * + system_time
1046 *
1047 * Hyper-V formula:
1048 * nsec/100 = ticks * scale / 2^64 + offset
1049 *
1050 * When tsc_timestamp = system_time = 0, offset is zero in the Hyper-V formula.
1051 * By dividing the kvmclock formula by 100 and equating what's left we get:
1052 * ticks * scale / 2^64 = ticks * tsc_to_system_mul * 2^(tsc_shift-32) / 100
1053 * scale / 2^64 = tsc_to_system_mul * 2^(tsc_shift-32) / 100
1054 * scale = tsc_to_system_mul * 2^(32+tsc_shift) / 100
1055 *
1056 * Now expand the kvmclock formula and divide by 100:
1057 * nsec = ticks * tsc_to_system_mul * 2^(tsc_shift-32)
1058 * - tsc_timestamp * tsc_to_system_mul * 2^(tsc_shift-32)
1059 * + system_time
1060 * nsec/100 = ticks * tsc_to_system_mul * 2^(tsc_shift-32) / 100
1061 * - tsc_timestamp * tsc_to_system_mul * 2^(tsc_shift-32) / 100
1062 * + system_time / 100
1063 *
1064 * Replace tsc_to_system_mul * 2^(tsc_shift-32) / 100 by scale / 2^64:
1065 * nsec/100 = ticks * scale / 2^64
1066 * - tsc_timestamp * scale / 2^64
1067 * + system_time / 100
1068 *
1069 * Equate with the Hyper-V formula so that ticks * scale / 2^64 cancels out:
1070 * offset = system_time / 100 - tsc_timestamp * scale / 2^64
1071 *
1072 * These two equivalencies are implemented in this function.
1073 */
compute_tsc_page_parameters(struct pvclock_vcpu_time_info * hv_clock,struct ms_hyperv_tsc_page * tsc_ref)1074 static bool compute_tsc_page_parameters(struct pvclock_vcpu_time_info *hv_clock,
1075 struct ms_hyperv_tsc_page *tsc_ref)
1076 {
1077 u64 max_mul;
1078
1079 if (!(hv_clock->flags & PVCLOCK_TSC_STABLE_BIT))
1080 return false;
1081
1082 /*
1083 * check if scale would overflow, if so we use the time ref counter
1084 * tsc_to_system_mul * 2^(tsc_shift+32) / 100 >= 2^64
1085 * tsc_to_system_mul / 100 >= 2^(32-tsc_shift)
1086 * tsc_to_system_mul >= 100 * 2^(32-tsc_shift)
1087 */
1088 max_mul = 100ull << (32 - hv_clock->tsc_shift);
1089 if (hv_clock->tsc_to_system_mul >= max_mul)
1090 return false;
1091
1092 /*
1093 * Otherwise compute the scale and offset according to the formulas
1094 * derived above.
1095 */
1096 tsc_ref->tsc_scale =
1097 mul_u64_u32_div(1ULL << (32 + hv_clock->tsc_shift),
1098 hv_clock->tsc_to_system_mul,
1099 100);
1100
1101 tsc_ref->tsc_offset = hv_clock->system_time;
1102 do_div(tsc_ref->tsc_offset, 100);
1103 tsc_ref->tsc_offset -=
1104 mul_u64_u64_shr(hv_clock->tsc_timestamp, tsc_ref->tsc_scale, 64);
1105 return true;
1106 }
1107
1108 /*
1109 * Don't touch TSC page values if the guest has opted for TSC emulation after
1110 * migration. KVM doesn't fully support reenlightenment notifications and TSC
1111 * access emulation and Hyper-V is known to expect the values in TSC page to
1112 * stay constant before TSC access emulation is disabled from guest side
1113 * (HV_X64_MSR_TSC_EMULATION_STATUS). KVM userspace is expected to preserve TSC
1114 * frequency and guest visible TSC value across migration (and prevent it when
1115 * TSC scaling is unsupported).
1116 */
tsc_page_update_unsafe(struct kvm_hv * hv)1117 static inline bool tsc_page_update_unsafe(struct kvm_hv *hv)
1118 {
1119 return (hv->hv_tsc_page_status != HV_TSC_PAGE_GUEST_CHANGED) &&
1120 hv->hv_tsc_emulation_control;
1121 }
1122
kvm_hv_setup_tsc_page(struct kvm * kvm,struct pvclock_vcpu_time_info * hv_clock)1123 void kvm_hv_setup_tsc_page(struct kvm *kvm,
1124 struct pvclock_vcpu_time_info *hv_clock)
1125 {
1126 struct kvm_hv *hv = to_kvm_hv(kvm);
1127 u32 tsc_seq;
1128 u64 gfn;
1129
1130 BUILD_BUG_ON(sizeof(tsc_seq) != sizeof(hv->tsc_ref.tsc_sequence));
1131 BUILD_BUG_ON(offsetof(struct ms_hyperv_tsc_page, tsc_sequence) != 0);
1132
1133 if (hv->hv_tsc_page_status == HV_TSC_PAGE_BROKEN ||
1134 hv->hv_tsc_page_status == HV_TSC_PAGE_UNSET)
1135 return;
1136
1137 mutex_lock(&hv->hv_lock);
1138 if (!(hv->hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE))
1139 goto out_unlock;
1140
1141 gfn = hv->hv_tsc_page >> HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT;
1142 /*
1143 * Because the TSC parameters only vary when there is a
1144 * change in the master clock, do not bother with caching.
1145 */
1146 if (unlikely(kvm_read_guest(kvm, gfn_to_gpa(gfn),
1147 &tsc_seq, sizeof(tsc_seq))))
1148 goto out_err;
1149
1150 if (tsc_seq && tsc_page_update_unsafe(hv)) {
1151 if (kvm_read_guest(kvm, gfn_to_gpa(gfn), &hv->tsc_ref, sizeof(hv->tsc_ref)))
1152 goto out_err;
1153
1154 hv->hv_tsc_page_status = HV_TSC_PAGE_SET;
1155 goto out_unlock;
1156 }
1157
1158 /*
1159 * While we're computing and writing the parameters, force the
1160 * guest to use the time reference count MSR.
1161 */
1162 hv->tsc_ref.tsc_sequence = 0;
1163 if (kvm_write_guest(kvm, gfn_to_gpa(gfn),
1164 &hv->tsc_ref, sizeof(hv->tsc_ref.tsc_sequence)))
1165 goto out_err;
1166
1167 if (!compute_tsc_page_parameters(hv_clock, &hv->tsc_ref))
1168 goto out_err;
1169
1170 /* Ensure sequence is zero before writing the rest of the struct. */
1171 smp_wmb();
1172 if (kvm_write_guest(kvm, gfn_to_gpa(gfn), &hv->tsc_ref, sizeof(hv->tsc_ref)))
1173 goto out_err;
1174
1175 /*
1176 * Now switch to the TSC page mechanism by writing the sequence.
1177 */
1178 tsc_seq++;
1179 if (tsc_seq == 0xFFFFFFFF || tsc_seq == 0)
1180 tsc_seq = 1;
1181
1182 /* Write the struct entirely before the non-zero sequence. */
1183 smp_wmb();
1184
1185 hv->tsc_ref.tsc_sequence = tsc_seq;
1186 if (kvm_write_guest(kvm, gfn_to_gpa(gfn),
1187 &hv->tsc_ref, sizeof(hv->tsc_ref.tsc_sequence)))
1188 goto out_err;
1189
1190 hv->hv_tsc_page_status = HV_TSC_PAGE_SET;
1191 goto out_unlock;
1192
1193 out_err:
1194 hv->hv_tsc_page_status = HV_TSC_PAGE_BROKEN;
1195 out_unlock:
1196 mutex_unlock(&hv->hv_lock);
1197 }
1198
kvm_hv_invalidate_tsc_page(struct kvm * kvm)1199 void kvm_hv_invalidate_tsc_page(struct kvm *kvm)
1200 {
1201 struct kvm_hv *hv = to_kvm_hv(kvm);
1202 u64 gfn;
1203 int idx;
1204
1205 if (hv->hv_tsc_page_status == HV_TSC_PAGE_BROKEN ||
1206 hv->hv_tsc_page_status == HV_TSC_PAGE_UNSET ||
1207 tsc_page_update_unsafe(hv))
1208 return;
1209
1210 mutex_lock(&hv->hv_lock);
1211
1212 if (!(hv->hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE))
1213 goto out_unlock;
1214
1215 /* Preserve HV_TSC_PAGE_GUEST_CHANGED/HV_TSC_PAGE_HOST_CHANGED states */
1216 if (hv->hv_tsc_page_status == HV_TSC_PAGE_SET)
1217 hv->hv_tsc_page_status = HV_TSC_PAGE_UPDATING;
1218
1219 gfn = hv->hv_tsc_page >> HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT;
1220
1221 hv->tsc_ref.tsc_sequence = 0;
1222
1223 /*
1224 * Take the srcu lock as memslots will be accessed to check the gfn
1225 * cache generation against the memslots generation.
1226 */
1227 idx = srcu_read_lock(&kvm->srcu);
1228 if (kvm_write_guest(kvm, gfn_to_gpa(gfn),
1229 &hv->tsc_ref, sizeof(hv->tsc_ref.tsc_sequence)))
1230 hv->hv_tsc_page_status = HV_TSC_PAGE_BROKEN;
1231 srcu_read_unlock(&kvm->srcu, idx);
1232
1233 out_unlock:
1234 mutex_unlock(&hv->hv_lock);
1235 }
1236
1237
hv_check_msr_access(struct kvm_vcpu_hv * hv_vcpu,u32 msr)1238 static bool hv_check_msr_access(struct kvm_vcpu_hv *hv_vcpu, u32 msr)
1239 {
1240 if (!hv_vcpu->enforce_cpuid)
1241 return true;
1242
1243 switch (msr) {
1244 case HV_X64_MSR_GUEST_OS_ID:
1245 case HV_X64_MSR_HYPERCALL:
1246 return hv_vcpu->cpuid_cache.features_eax &
1247 HV_MSR_HYPERCALL_AVAILABLE;
1248 case HV_X64_MSR_VP_RUNTIME:
1249 return hv_vcpu->cpuid_cache.features_eax &
1250 HV_MSR_VP_RUNTIME_AVAILABLE;
1251 case HV_X64_MSR_TIME_REF_COUNT:
1252 return hv_vcpu->cpuid_cache.features_eax &
1253 HV_MSR_TIME_REF_COUNT_AVAILABLE;
1254 case HV_X64_MSR_VP_INDEX:
1255 return hv_vcpu->cpuid_cache.features_eax &
1256 HV_MSR_VP_INDEX_AVAILABLE;
1257 case HV_X64_MSR_RESET:
1258 return hv_vcpu->cpuid_cache.features_eax &
1259 HV_MSR_RESET_AVAILABLE;
1260 case HV_X64_MSR_REFERENCE_TSC:
1261 return hv_vcpu->cpuid_cache.features_eax &
1262 HV_MSR_REFERENCE_TSC_AVAILABLE;
1263 case HV_X64_MSR_SCONTROL:
1264 case HV_X64_MSR_SVERSION:
1265 case HV_X64_MSR_SIEFP:
1266 case HV_X64_MSR_SIMP:
1267 case HV_X64_MSR_EOM:
1268 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15:
1269 return hv_vcpu->cpuid_cache.features_eax &
1270 HV_MSR_SYNIC_AVAILABLE;
1271 case HV_X64_MSR_STIMER0_CONFIG:
1272 case HV_X64_MSR_STIMER1_CONFIG:
1273 case HV_X64_MSR_STIMER2_CONFIG:
1274 case HV_X64_MSR_STIMER3_CONFIG:
1275 case HV_X64_MSR_STIMER0_COUNT:
1276 case HV_X64_MSR_STIMER1_COUNT:
1277 case HV_X64_MSR_STIMER2_COUNT:
1278 case HV_X64_MSR_STIMER3_COUNT:
1279 return hv_vcpu->cpuid_cache.features_eax &
1280 HV_MSR_SYNTIMER_AVAILABLE;
1281 case HV_X64_MSR_EOI:
1282 case HV_X64_MSR_ICR:
1283 case HV_X64_MSR_TPR:
1284 case HV_X64_MSR_VP_ASSIST_PAGE:
1285 return hv_vcpu->cpuid_cache.features_eax &
1286 HV_MSR_APIC_ACCESS_AVAILABLE;
1287 break;
1288 case HV_X64_MSR_TSC_FREQUENCY:
1289 case HV_X64_MSR_APIC_FREQUENCY:
1290 return hv_vcpu->cpuid_cache.features_eax &
1291 HV_ACCESS_FREQUENCY_MSRS;
1292 case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
1293 case HV_X64_MSR_TSC_EMULATION_CONTROL:
1294 case HV_X64_MSR_TSC_EMULATION_STATUS:
1295 return hv_vcpu->cpuid_cache.features_eax &
1296 HV_ACCESS_REENLIGHTENMENT;
1297 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
1298 case HV_X64_MSR_CRASH_CTL:
1299 return hv_vcpu->cpuid_cache.features_edx &
1300 HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE;
1301 case HV_X64_MSR_SYNDBG_OPTIONS:
1302 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER:
1303 return hv_vcpu->cpuid_cache.features_edx &
1304 HV_FEATURE_DEBUG_MSRS_AVAILABLE;
1305 default:
1306 break;
1307 }
1308
1309 return false;
1310 }
1311
kvm_hv_set_msr_pw(struct kvm_vcpu * vcpu,u32 msr,u64 data,bool host)1312 static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data,
1313 bool host)
1314 {
1315 struct kvm *kvm = vcpu->kvm;
1316 struct kvm_hv *hv = to_kvm_hv(kvm);
1317
1318 if (unlikely(!host && !hv_check_msr_access(to_hv_vcpu(vcpu), msr)))
1319 return 1;
1320
1321 switch (msr) {
1322 case HV_X64_MSR_GUEST_OS_ID:
1323 hv->hv_guest_os_id = data;
1324 /* setting guest os id to zero disables hypercall page */
1325 if (!hv->hv_guest_os_id)
1326 hv->hv_hypercall &= ~HV_X64_MSR_HYPERCALL_ENABLE;
1327 break;
1328 case HV_X64_MSR_HYPERCALL: {
1329 u8 instructions[9];
1330 int i = 0;
1331 u64 addr;
1332
1333 /* if guest os id is not set hypercall should remain disabled */
1334 if (!hv->hv_guest_os_id)
1335 break;
1336 if (!(data & HV_X64_MSR_HYPERCALL_ENABLE)) {
1337 hv->hv_hypercall = data;
1338 break;
1339 }
1340
1341 /*
1342 * If Xen and Hyper-V hypercalls are both enabled, disambiguate
1343 * the same way Xen itself does, by setting the bit 31 of EAX
1344 * which is RsvdZ in the 32-bit Hyper-V hypercall ABI and just
1345 * going to be clobbered on 64-bit.
1346 */
1347 if (kvm_xen_hypercall_enabled(kvm)) {
1348 /* orl $0x80000000, %eax */
1349 instructions[i++] = 0x0d;
1350 instructions[i++] = 0x00;
1351 instructions[i++] = 0x00;
1352 instructions[i++] = 0x00;
1353 instructions[i++] = 0x80;
1354 }
1355
1356 /* vmcall/vmmcall */
1357 static_call(kvm_x86_patch_hypercall)(vcpu, instructions + i);
1358 i += 3;
1359
1360 /* ret */
1361 ((unsigned char *)instructions)[i++] = 0xc3;
1362
1363 addr = data & HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_MASK;
1364 if (kvm_vcpu_write_guest(vcpu, addr, instructions, i))
1365 return 1;
1366 hv->hv_hypercall = data;
1367 break;
1368 }
1369 case HV_X64_MSR_REFERENCE_TSC:
1370 hv->hv_tsc_page = data;
1371 if (hv->hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE) {
1372 if (!host)
1373 hv->hv_tsc_page_status = HV_TSC_PAGE_GUEST_CHANGED;
1374 else
1375 hv->hv_tsc_page_status = HV_TSC_PAGE_HOST_CHANGED;
1376 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
1377 } else {
1378 hv->hv_tsc_page_status = HV_TSC_PAGE_UNSET;
1379 }
1380 break;
1381 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
1382 return kvm_hv_msr_set_crash_data(kvm,
1383 msr - HV_X64_MSR_CRASH_P0,
1384 data);
1385 case HV_X64_MSR_CRASH_CTL:
1386 if (host)
1387 return kvm_hv_msr_set_crash_ctl(kvm, data);
1388
1389 if (data & HV_CRASH_CTL_CRASH_NOTIFY) {
1390 vcpu_debug(vcpu, "hv crash (0x%llx 0x%llx 0x%llx 0x%llx 0x%llx)\n",
1391 hv->hv_crash_param[0],
1392 hv->hv_crash_param[1],
1393 hv->hv_crash_param[2],
1394 hv->hv_crash_param[3],
1395 hv->hv_crash_param[4]);
1396
1397 /* Send notification about crash to user space */
1398 kvm_make_request(KVM_REQ_HV_CRASH, vcpu);
1399 }
1400 break;
1401 case HV_X64_MSR_RESET:
1402 if (data == 1) {
1403 vcpu_debug(vcpu, "hyper-v reset requested\n");
1404 kvm_make_request(KVM_REQ_HV_RESET, vcpu);
1405 }
1406 break;
1407 case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
1408 hv->hv_reenlightenment_control = data;
1409 break;
1410 case HV_X64_MSR_TSC_EMULATION_CONTROL:
1411 hv->hv_tsc_emulation_control = data;
1412 break;
1413 case HV_X64_MSR_TSC_EMULATION_STATUS:
1414 if (data && !host)
1415 return 1;
1416
1417 hv->hv_tsc_emulation_status = data;
1418 break;
1419 case HV_X64_MSR_TIME_REF_COUNT:
1420 /* read-only, but still ignore it if host-initiated */
1421 if (!host)
1422 return 1;
1423 break;
1424 case HV_X64_MSR_SYNDBG_OPTIONS:
1425 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER:
1426 return syndbg_set_msr(vcpu, msr, data, host);
1427 default:
1428 vcpu_unimpl(vcpu, "Hyper-V unhandled wrmsr: 0x%x data 0x%llx\n",
1429 msr, data);
1430 return 1;
1431 }
1432 return 0;
1433 }
1434
1435 /* Calculate cpu time spent by current task in 100ns units */
current_task_runtime_100ns(void)1436 static u64 current_task_runtime_100ns(void)
1437 {
1438 u64 utime, stime;
1439
1440 task_cputime_adjusted(current, &utime, &stime);
1441
1442 return div_u64(utime + stime, 100);
1443 }
1444
kvm_hv_set_msr(struct kvm_vcpu * vcpu,u32 msr,u64 data,bool host)1445 static int kvm_hv_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host)
1446 {
1447 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
1448
1449 if (unlikely(!host && !hv_check_msr_access(hv_vcpu, msr)))
1450 return 1;
1451
1452 switch (msr) {
1453 case HV_X64_MSR_VP_INDEX: {
1454 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
1455 u32 new_vp_index = (u32)data;
1456
1457 if (!host || new_vp_index >= KVM_MAX_VCPUS)
1458 return 1;
1459
1460 if (new_vp_index == hv_vcpu->vp_index)
1461 return 0;
1462
1463 /*
1464 * The VP index is initialized to vcpu_index by
1465 * kvm_hv_vcpu_postcreate so they initially match. Now the
1466 * VP index is changing, adjust num_mismatched_vp_indexes if
1467 * it now matches or no longer matches vcpu_idx.
1468 */
1469 if (hv_vcpu->vp_index == vcpu->vcpu_idx)
1470 atomic_inc(&hv->num_mismatched_vp_indexes);
1471 else if (new_vp_index == vcpu->vcpu_idx)
1472 atomic_dec(&hv->num_mismatched_vp_indexes);
1473
1474 hv_vcpu->vp_index = new_vp_index;
1475 break;
1476 }
1477 case HV_X64_MSR_VP_ASSIST_PAGE: {
1478 u64 gfn;
1479 unsigned long addr;
1480
1481 if (!(data & HV_X64_MSR_VP_ASSIST_PAGE_ENABLE)) {
1482 hv_vcpu->hv_vapic = data;
1483 if (kvm_lapic_enable_pv_eoi(vcpu, 0, 0))
1484 return 1;
1485 break;
1486 }
1487 gfn = data >> HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT;
1488 addr = kvm_vcpu_gfn_to_hva(vcpu, gfn);
1489 if (kvm_is_error_hva(addr))
1490 return 1;
1491
1492 /*
1493 * Clear apic_assist portion of struct hv_vp_assist_page
1494 * only, there can be valuable data in the rest which needs
1495 * to be preserved e.g. on migration.
1496 */
1497 if (__put_user(0, (u32 __user *)addr))
1498 return 1;
1499 hv_vcpu->hv_vapic = data;
1500 kvm_vcpu_mark_page_dirty(vcpu, gfn);
1501 if (kvm_lapic_enable_pv_eoi(vcpu,
1502 gfn_to_gpa(gfn) | KVM_MSR_ENABLED,
1503 sizeof(struct hv_vp_assist_page)))
1504 return 1;
1505 break;
1506 }
1507 case HV_X64_MSR_EOI:
1508 return kvm_hv_vapic_msr_write(vcpu, APIC_EOI, data);
1509 case HV_X64_MSR_ICR:
1510 return kvm_hv_vapic_msr_write(vcpu, APIC_ICR, data);
1511 case HV_X64_MSR_TPR:
1512 return kvm_hv_vapic_msr_write(vcpu, APIC_TASKPRI, data);
1513 case HV_X64_MSR_VP_RUNTIME:
1514 if (!host)
1515 return 1;
1516 hv_vcpu->runtime_offset = data - current_task_runtime_100ns();
1517 break;
1518 case HV_X64_MSR_SCONTROL:
1519 case HV_X64_MSR_SVERSION:
1520 case HV_X64_MSR_SIEFP:
1521 case HV_X64_MSR_SIMP:
1522 case HV_X64_MSR_EOM:
1523 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15:
1524 return synic_set_msr(to_hv_synic(vcpu), msr, data, host);
1525 case HV_X64_MSR_STIMER0_CONFIG:
1526 case HV_X64_MSR_STIMER1_CONFIG:
1527 case HV_X64_MSR_STIMER2_CONFIG:
1528 case HV_X64_MSR_STIMER3_CONFIG: {
1529 int timer_index = (msr - HV_X64_MSR_STIMER0_CONFIG)/2;
1530
1531 return stimer_set_config(to_hv_stimer(vcpu, timer_index),
1532 data, host);
1533 }
1534 case HV_X64_MSR_STIMER0_COUNT:
1535 case HV_X64_MSR_STIMER1_COUNT:
1536 case HV_X64_MSR_STIMER2_COUNT:
1537 case HV_X64_MSR_STIMER3_COUNT: {
1538 int timer_index = (msr - HV_X64_MSR_STIMER0_COUNT)/2;
1539
1540 return stimer_set_count(to_hv_stimer(vcpu, timer_index),
1541 data, host);
1542 }
1543 case HV_X64_MSR_TSC_FREQUENCY:
1544 case HV_X64_MSR_APIC_FREQUENCY:
1545 /* read-only, but still ignore it if host-initiated */
1546 if (!host)
1547 return 1;
1548 break;
1549 default:
1550 vcpu_unimpl(vcpu, "Hyper-V unhandled wrmsr: 0x%x data 0x%llx\n",
1551 msr, data);
1552 return 1;
1553 }
1554
1555 return 0;
1556 }
1557
kvm_hv_get_msr_pw(struct kvm_vcpu * vcpu,u32 msr,u64 * pdata,bool host)1558 static int kvm_hv_get_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata,
1559 bool host)
1560 {
1561 u64 data = 0;
1562 struct kvm *kvm = vcpu->kvm;
1563 struct kvm_hv *hv = to_kvm_hv(kvm);
1564
1565 if (unlikely(!host && !hv_check_msr_access(to_hv_vcpu(vcpu), msr)))
1566 return 1;
1567
1568 switch (msr) {
1569 case HV_X64_MSR_GUEST_OS_ID:
1570 data = hv->hv_guest_os_id;
1571 break;
1572 case HV_X64_MSR_HYPERCALL:
1573 data = hv->hv_hypercall;
1574 break;
1575 case HV_X64_MSR_TIME_REF_COUNT:
1576 data = get_time_ref_counter(kvm);
1577 break;
1578 case HV_X64_MSR_REFERENCE_TSC:
1579 data = hv->hv_tsc_page;
1580 break;
1581 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
1582 return kvm_hv_msr_get_crash_data(kvm,
1583 msr - HV_X64_MSR_CRASH_P0,
1584 pdata);
1585 case HV_X64_MSR_CRASH_CTL:
1586 return kvm_hv_msr_get_crash_ctl(kvm, pdata);
1587 case HV_X64_MSR_RESET:
1588 data = 0;
1589 break;
1590 case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
1591 data = hv->hv_reenlightenment_control;
1592 break;
1593 case HV_X64_MSR_TSC_EMULATION_CONTROL:
1594 data = hv->hv_tsc_emulation_control;
1595 break;
1596 case HV_X64_MSR_TSC_EMULATION_STATUS:
1597 data = hv->hv_tsc_emulation_status;
1598 break;
1599 case HV_X64_MSR_SYNDBG_OPTIONS:
1600 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER:
1601 return syndbg_get_msr(vcpu, msr, pdata, host);
1602 default:
1603 vcpu_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
1604 return 1;
1605 }
1606
1607 *pdata = data;
1608 return 0;
1609 }
1610
kvm_hv_get_msr(struct kvm_vcpu * vcpu,u32 msr,u64 * pdata,bool host)1611 static int kvm_hv_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata,
1612 bool host)
1613 {
1614 u64 data = 0;
1615 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
1616
1617 if (unlikely(!host && !hv_check_msr_access(hv_vcpu, msr)))
1618 return 1;
1619
1620 switch (msr) {
1621 case HV_X64_MSR_VP_INDEX:
1622 data = hv_vcpu->vp_index;
1623 break;
1624 case HV_X64_MSR_EOI:
1625 return kvm_hv_vapic_msr_read(vcpu, APIC_EOI, pdata);
1626 case HV_X64_MSR_ICR:
1627 return kvm_hv_vapic_msr_read(vcpu, APIC_ICR, pdata);
1628 case HV_X64_MSR_TPR:
1629 return kvm_hv_vapic_msr_read(vcpu, APIC_TASKPRI, pdata);
1630 case HV_X64_MSR_VP_ASSIST_PAGE:
1631 data = hv_vcpu->hv_vapic;
1632 break;
1633 case HV_X64_MSR_VP_RUNTIME:
1634 data = current_task_runtime_100ns() + hv_vcpu->runtime_offset;
1635 break;
1636 case HV_X64_MSR_SCONTROL:
1637 case HV_X64_MSR_SVERSION:
1638 case HV_X64_MSR_SIEFP:
1639 case HV_X64_MSR_SIMP:
1640 case HV_X64_MSR_EOM:
1641 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15:
1642 return synic_get_msr(to_hv_synic(vcpu), msr, pdata, host);
1643 case HV_X64_MSR_STIMER0_CONFIG:
1644 case HV_X64_MSR_STIMER1_CONFIG:
1645 case HV_X64_MSR_STIMER2_CONFIG:
1646 case HV_X64_MSR_STIMER3_CONFIG: {
1647 int timer_index = (msr - HV_X64_MSR_STIMER0_CONFIG)/2;
1648
1649 return stimer_get_config(to_hv_stimer(vcpu, timer_index),
1650 pdata);
1651 }
1652 case HV_X64_MSR_STIMER0_COUNT:
1653 case HV_X64_MSR_STIMER1_COUNT:
1654 case HV_X64_MSR_STIMER2_COUNT:
1655 case HV_X64_MSR_STIMER3_COUNT: {
1656 int timer_index = (msr - HV_X64_MSR_STIMER0_COUNT)/2;
1657
1658 return stimer_get_count(to_hv_stimer(vcpu, timer_index),
1659 pdata);
1660 }
1661 case HV_X64_MSR_TSC_FREQUENCY:
1662 data = (u64)vcpu->arch.virtual_tsc_khz * 1000;
1663 break;
1664 case HV_X64_MSR_APIC_FREQUENCY:
1665 data = APIC_BUS_FREQUENCY;
1666 break;
1667 default:
1668 vcpu_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
1669 return 1;
1670 }
1671 *pdata = data;
1672 return 0;
1673 }
1674
kvm_hv_set_msr_common(struct kvm_vcpu * vcpu,u32 msr,u64 data,bool host)1675 int kvm_hv_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host)
1676 {
1677 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
1678
1679 if (!host && !vcpu->arch.hyperv_enabled)
1680 return 1;
1681
1682 if (!to_hv_vcpu(vcpu)) {
1683 if (kvm_hv_vcpu_init(vcpu))
1684 return 1;
1685 }
1686
1687 if (kvm_hv_msr_partition_wide(msr)) {
1688 int r;
1689
1690 mutex_lock(&hv->hv_lock);
1691 r = kvm_hv_set_msr_pw(vcpu, msr, data, host);
1692 mutex_unlock(&hv->hv_lock);
1693 return r;
1694 } else
1695 return kvm_hv_set_msr(vcpu, msr, data, host);
1696 }
1697
kvm_hv_get_msr_common(struct kvm_vcpu * vcpu,u32 msr,u64 * pdata,bool host)1698 int kvm_hv_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)
1699 {
1700 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
1701
1702 if (!host && !vcpu->arch.hyperv_enabled)
1703 return 1;
1704
1705 if (!to_hv_vcpu(vcpu)) {
1706 if (kvm_hv_vcpu_init(vcpu))
1707 return 1;
1708 }
1709
1710 if (kvm_hv_msr_partition_wide(msr)) {
1711 int r;
1712
1713 mutex_lock(&hv->hv_lock);
1714 r = kvm_hv_get_msr_pw(vcpu, msr, pdata, host);
1715 mutex_unlock(&hv->hv_lock);
1716 return r;
1717 } else
1718 return kvm_hv_get_msr(vcpu, msr, pdata, host);
1719 }
1720
sparse_set_to_vcpu_mask(struct kvm * kvm,u64 * sparse_banks,u64 valid_bank_mask,u64 * vp_bitmap,unsigned long * vcpu_bitmap)1721 static __always_inline unsigned long *sparse_set_to_vcpu_mask(
1722 struct kvm *kvm, u64 *sparse_banks, u64 valid_bank_mask,
1723 u64 *vp_bitmap, unsigned long *vcpu_bitmap)
1724 {
1725 struct kvm_hv *hv = to_kvm_hv(kvm);
1726 struct kvm_vcpu *vcpu;
1727 int i, bank, sbank = 0;
1728
1729 memset(vp_bitmap, 0,
1730 KVM_HV_MAX_SPARSE_VCPU_SET_BITS * sizeof(*vp_bitmap));
1731 for_each_set_bit(bank, (unsigned long *)&valid_bank_mask,
1732 KVM_HV_MAX_SPARSE_VCPU_SET_BITS)
1733 vp_bitmap[bank] = sparse_banks[sbank++];
1734
1735 if (likely(!atomic_read(&hv->num_mismatched_vp_indexes))) {
1736 /* for all vcpus vp_index == vcpu_idx */
1737 return (unsigned long *)vp_bitmap;
1738 }
1739
1740 bitmap_zero(vcpu_bitmap, KVM_MAX_VCPUS);
1741 kvm_for_each_vcpu(i, vcpu, kvm) {
1742 if (test_bit(kvm_hv_get_vpindex(vcpu), (unsigned long *)vp_bitmap))
1743 __set_bit(i, vcpu_bitmap);
1744 }
1745 return vcpu_bitmap;
1746 }
1747
1748 struct kvm_hv_hcall {
1749 u64 param;
1750 u64 ingpa;
1751 u64 outgpa;
1752 u16 code;
1753 u16 rep_cnt;
1754 u16 rep_idx;
1755 bool fast;
1756 bool rep;
1757 sse128_t xmm[HV_HYPERCALL_MAX_XMM_REGISTERS];
1758 };
1759
kvm_hv_flush_tlb(struct kvm_vcpu * vcpu,struct kvm_hv_hcall * hc)1760 static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
1761 {
1762 int i;
1763 gpa_t gpa;
1764 struct kvm *kvm = vcpu->kvm;
1765 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
1766 struct hv_tlb_flush_ex flush_ex;
1767 struct hv_tlb_flush flush;
1768 u64 vp_bitmap[KVM_HV_MAX_SPARSE_VCPU_SET_BITS];
1769 DECLARE_BITMAP(vcpu_bitmap, KVM_MAX_VCPUS);
1770 unsigned long *vcpu_mask;
1771 u64 valid_bank_mask;
1772 u64 sparse_banks[64];
1773 int sparse_banks_len;
1774 bool all_cpus;
1775
1776 if (hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST ||
1777 hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE) {
1778 if (hc->fast) {
1779 flush.address_space = hc->ingpa;
1780 flush.flags = hc->outgpa;
1781 flush.processor_mask = sse128_lo(hc->xmm[0]);
1782 } else {
1783 if (unlikely(kvm_read_guest(kvm, hc->ingpa,
1784 &flush, sizeof(flush))))
1785 return HV_STATUS_INVALID_HYPERCALL_INPUT;
1786 }
1787
1788 trace_kvm_hv_flush_tlb(flush.processor_mask,
1789 flush.address_space, flush.flags);
1790
1791 valid_bank_mask = BIT_ULL(0);
1792 sparse_banks[0] = flush.processor_mask;
1793
1794 /*
1795 * Work around possible WS2012 bug: it sends hypercalls
1796 * with processor_mask = 0x0 and HV_FLUSH_ALL_PROCESSORS clear,
1797 * while also expecting us to flush something and crashing if
1798 * we don't. Let's treat processor_mask == 0 same as
1799 * HV_FLUSH_ALL_PROCESSORS.
1800 */
1801 all_cpus = (flush.flags & HV_FLUSH_ALL_PROCESSORS) ||
1802 flush.processor_mask == 0;
1803 } else {
1804 if (hc->fast) {
1805 flush_ex.address_space = hc->ingpa;
1806 flush_ex.flags = hc->outgpa;
1807 memcpy(&flush_ex.hv_vp_set,
1808 &hc->xmm[0], sizeof(hc->xmm[0]));
1809 } else {
1810 if (unlikely(kvm_read_guest(kvm, hc->ingpa, &flush_ex,
1811 sizeof(flush_ex))))
1812 return HV_STATUS_INVALID_HYPERCALL_INPUT;
1813 }
1814
1815 trace_kvm_hv_flush_tlb_ex(flush_ex.hv_vp_set.valid_bank_mask,
1816 flush_ex.hv_vp_set.format,
1817 flush_ex.address_space,
1818 flush_ex.flags);
1819
1820 valid_bank_mask = flush_ex.hv_vp_set.valid_bank_mask;
1821 all_cpus = flush_ex.hv_vp_set.format !=
1822 HV_GENERIC_SET_SPARSE_4K;
1823
1824 sparse_banks_len = bitmap_weight((unsigned long *)&valid_bank_mask, 64);
1825
1826 if (!sparse_banks_len && !all_cpus)
1827 goto ret_success;
1828
1829 if (!all_cpus) {
1830 if (hc->fast) {
1831 /* XMM0 is already consumed, each XMM holds two sparse banks. */
1832 if (sparse_banks_len > 2 * (HV_HYPERCALL_MAX_XMM_REGISTERS - 1))
1833 return HV_STATUS_INVALID_HYPERCALL_INPUT;
1834 for (i = 0; i < sparse_banks_len; i += 2) {
1835 sparse_banks[i] = sse128_lo(hc->xmm[i / 2 + 1]);
1836 sparse_banks[i + 1] = sse128_hi(hc->xmm[i / 2 + 1]);
1837 }
1838 } else {
1839 gpa = hc->ingpa + offsetof(struct hv_tlb_flush_ex,
1840 hv_vp_set.bank_contents);
1841 if (unlikely(kvm_read_guest(kvm, gpa, sparse_banks,
1842 sparse_banks_len *
1843 sizeof(sparse_banks[0]))))
1844 return HV_STATUS_INVALID_HYPERCALL_INPUT;
1845 }
1846 }
1847 }
1848
1849 cpumask_clear(&hv_vcpu->tlb_flush);
1850
1851 /*
1852 * vcpu->arch.cr3 may not be up-to-date for running vCPUs so we can't
1853 * analyze it here, flush TLB regardless of the specified address space.
1854 */
1855 if (all_cpus) {
1856 kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH_GUEST);
1857 } else {
1858 vcpu_mask = sparse_set_to_vcpu_mask(kvm, sparse_banks, valid_bank_mask,
1859 vp_bitmap, vcpu_bitmap);
1860
1861 kvm_make_vcpus_request_mask(kvm, KVM_REQ_TLB_FLUSH_GUEST,
1862 NULL, vcpu_mask, &hv_vcpu->tlb_flush);
1863 }
1864
1865 ret_success:
1866 /* We always do full TLB flush, set 'Reps completed' = 'Rep Count' */
1867 return (u64)HV_STATUS_SUCCESS |
1868 ((u64)hc->rep_cnt << HV_HYPERCALL_REP_COMP_OFFSET);
1869 }
1870
kvm_send_ipi_to_many(struct kvm * kvm,u32 vector,unsigned long * vcpu_bitmap)1871 static void kvm_send_ipi_to_many(struct kvm *kvm, u32 vector,
1872 unsigned long *vcpu_bitmap)
1873 {
1874 struct kvm_lapic_irq irq = {
1875 .delivery_mode = APIC_DM_FIXED,
1876 .vector = vector
1877 };
1878 struct kvm_vcpu *vcpu;
1879 int i;
1880
1881 kvm_for_each_vcpu(i, vcpu, kvm) {
1882 if (vcpu_bitmap && !test_bit(i, vcpu_bitmap))
1883 continue;
1884
1885 /* We fail only when APIC is disabled */
1886 kvm_apic_set_irq(vcpu, &irq, NULL);
1887 }
1888 }
1889
kvm_hv_send_ipi(struct kvm_vcpu * vcpu,struct kvm_hv_hcall * hc)1890 static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
1891 {
1892 struct kvm *kvm = vcpu->kvm;
1893 struct hv_send_ipi_ex send_ipi_ex;
1894 struct hv_send_ipi send_ipi;
1895 u64 vp_bitmap[KVM_HV_MAX_SPARSE_VCPU_SET_BITS];
1896 DECLARE_BITMAP(vcpu_bitmap, KVM_MAX_VCPUS);
1897 unsigned long *vcpu_mask;
1898 unsigned long valid_bank_mask;
1899 u64 sparse_banks[64];
1900 int sparse_banks_len;
1901 u32 vector;
1902 bool all_cpus;
1903 int i;
1904
1905 if (hc->code == HVCALL_SEND_IPI) {
1906 if (!hc->fast) {
1907 if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi,
1908 sizeof(send_ipi))))
1909 return HV_STATUS_INVALID_HYPERCALL_INPUT;
1910 sparse_banks[0] = send_ipi.cpu_mask;
1911 vector = send_ipi.vector;
1912 } else {
1913 /* 'reserved' part of hv_send_ipi should be 0 */
1914 if (unlikely(hc->ingpa >> 32 != 0))
1915 return HV_STATUS_INVALID_HYPERCALL_INPUT;
1916 sparse_banks[0] = hc->outgpa;
1917 vector = (u32)hc->ingpa;
1918 }
1919 all_cpus = false;
1920 valid_bank_mask = BIT_ULL(0);
1921
1922 trace_kvm_hv_send_ipi(vector, sparse_banks[0]);
1923 } else {
1924 if (!hc->fast) {
1925 if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi_ex,
1926 sizeof(send_ipi_ex))))
1927 return HV_STATUS_INVALID_HYPERCALL_INPUT;
1928 } else {
1929 send_ipi_ex.vector = (u32)hc->ingpa;
1930 send_ipi_ex.vp_set.format = hc->outgpa;
1931 send_ipi_ex.vp_set.valid_bank_mask = sse128_lo(hc->xmm[0]);
1932 }
1933
1934 trace_kvm_hv_send_ipi_ex(send_ipi_ex.vector,
1935 send_ipi_ex.vp_set.format,
1936 send_ipi_ex.vp_set.valid_bank_mask);
1937
1938 vector = send_ipi_ex.vector;
1939 valid_bank_mask = send_ipi_ex.vp_set.valid_bank_mask;
1940 sparse_banks_len = bitmap_weight(&valid_bank_mask, 64);
1941
1942 all_cpus = send_ipi_ex.vp_set.format == HV_GENERIC_SET_ALL;
1943
1944 if (all_cpus)
1945 goto check_and_send_ipi;
1946
1947 if (!sparse_banks_len)
1948 goto ret_success;
1949
1950 if (!hc->fast) {
1951 if (kvm_read_guest(kvm,
1952 hc->ingpa + offsetof(struct hv_send_ipi_ex,
1953 vp_set.bank_contents),
1954 sparse_banks,
1955 sparse_banks_len * sizeof(sparse_banks[0])))
1956 return HV_STATUS_INVALID_HYPERCALL_INPUT;
1957 } else {
1958 /*
1959 * The lower half of XMM0 is already consumed, each XMM holds
1960 * two sparse banks.
1961 */
1962 if (sparse_banks_len > (2 * HV_HYPERCALL_MAX_XMM_REGISTERS - 1))
1963 return HV_STATUS_INVALID_HYPERCALL_INPUT;
1964 for (i = 0; i < sparse_banks_len; i++) {
1965 if (i % 2)
1966 sparse_banks[i] = sse128_lo(hc->xmm[(i + 1) / 2]);
1967 else
1968 sparse_banks[i] = sse128_hi(hc->xmm[i / 2]);
1969 }
1970 }
1971 }
1972
1973 check_and_send_ipi:
1974 if ((vector < HV_IPI_LOW_VECTOR) || (vector > HV_IPI_HIGH_VECTOR))
1975 return HV_STATUS_INVALID_HYPERCALL_INPUT;
1976
1977 vcpu_mask = all_cpus ? NULL :
1978 sparse_set_to_vcpu_mask(kvm, sparse_banks, valid_bank_mask,
1979 vp_bitmap, vcpu_bitmap);
1980
1981 kvm_send_ipi_to_many(kvm, vector, vcpu_mask);
1982
1983 ret_success:
1984 return HV_STATUS_SUCCESS;
1985 }
1986
kvm_hv_set_cpuid(struct kvm_vcpu * vcpu)1987 void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu)
1988 {
1989 struct kvm_cpuid_entry2 *entry;
1990 struct kvm_vcpu_hv *hv_vcpu;
1991
1992 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_INTERFACE, 0);
1993 if (entry && entry->eax == HYPERV_CPUID_SIGNATURE_EAX) {
1994 vcpu->arch.hyperv_enabled = true;
1995 } else {
1996 vcpu->arch.hyperv_enabled = false;
1997 return;
1998 }
1999
2000 if (!to_hv_vcpu(vcpu) && kvm_hv_vcpu_init(vcpu))
2001 return;
2002
2003 hv_vcpu = to_hv_vcpu(vcpu);
2004
2005 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_FEATURES, 0);
2006 if (entry) {
2007 hv_vcpu->cpuid_cache.features_eax = entry->eax;
2008 hv_vcpu->cpuid_cache.features_ebx = entry->ebx;
2009 hv_vcpu->cpuid_cache.features_edx = entry->edx;
2010 } else {
2011 hv_vcpu->cpuid_cache.features_eax = 0;
2012 hv_vcpu->cpuid_cache.features_ebx = 0;
2013 hv_vcpu->cpuid_cache.features_edx = 0;
2014 }
2015
2016 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_ENLIGHTMENT_INFO, 0);
2017 if (entry) {
2018 hv_vcpu->cpuid_cache.enlightenments_eax = entry->eax;
2019 hv_vcpu->cpuid_cache.enlightenments_ebx = entry->ebx;
2020 } else {
2021 hv_vcpu->cpuid_cache.enlightenments_eax = 0;
2022 hv_vcpu->cpuid_cache.enlightenments_ebx = 0;
2023 }
2024
2025 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES, 0);
2026 if (entry)
2027 hv_vcpu->cpuid_cache.syndbg_cap_eax = entry->eax;
2028 else
2029 hv_vcpu->cpuid_cache.syndbg_cap_eax = 0;
2030 }
2031
kvm_hv_set_enforce_cpuid(struct kvm_vcpu * vcpu,bool enforce)2032 int kvm_hv_set_enforce_cpuid(struct kvm_vcpu *vcpu, bool enforce)
2033 {
2034 struct kvm_vcpu_hv *hv_vcpu;
2035 int ret = 0;
2036
2037 if (!to_hv_vcpu(vcpu)) {
2038 if (enforce) {
2039 ret = kvm_hv_vcpu_init(vcpu);
2040 if (ret)
2041 return ret;
2042 } else {
2043 return 0;
2044 }
2045 }
2046
2047 hv_vcpu = to_hv_vcpu(vcpu);
2048 hv_vcpu->enforce_cpuid = enforce;
2049
2050 return ret;
2051 }
2052
kvm_hv_hypercall_enabled(struct kvm_vcpu * vcpu)2053 bool kvm_hv_hypercall_enabled(struct kvm_vcpu *vcpu)
2054 {
2055 return vcpu->arch.hyperv_enabled && to_kvm_hv(vcpu->kvm)->hv_guest_os_id;
2056 }
2057
kvm_hv_hypercall_set_result(struct kvm_vcpu * vcpu,u64 result)2058 static void kvm_hv_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result)
2059 {
2060 bool longmode;
2061
2062 longmode = is_64_bit_hypercall(vcpu);
2063 if (longmode)
2064 kvm_rax_write(vcpu, result);
2065 else {
2066 kvm_rdx_write(vcpu, result >> 32);
2067 kvm_rax_write(vcpu, result & 0xffffffff);
2068 }
2069 }
2070
kvm_hv_hypercall_complete(struct kvm_vcpu * vcpu,u64 result)2071 static int kvm_hv_hypercall_complete(struct kvm_vcpu *vcpu, u64 result)
2072 {
2073 trace_kvm_hv_hypercall_done(result);
2074 kvm_hv_hypercall_set_result(vcpu, result);
2075 ++vcpu->stat.hypercalls;
2076 return kvm_skip_emulated_instruction(vcpu);
2077 }
2078
kvm_hv_hypercall_complete_userspace(struct kvm_vcpu * vcpu)2079 static int kvm_hv_hypercall_complete_userspace(struct kvm_vcpu *vcpu)
2080 {
2081 return kvm_hv_hypercall_complete(vcpu, vcpu->run->hyperv.u.hcall.result);
2082 }
2083
kvm_hvcall_signal_event(struct kvm_vcpu * vcpu,struct kvm_hv_hcall * hc)2084 static u16 kvm_hvcall_signal_event(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
2085 {
2086 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
2087 struct eventfd_ctx *eventfd;
2088
2089 if (unlikely(!hc->fast)) {
2090 int ret;
2091 gpa_t gpa = hc->ingpa;
2092
2093 if ((gpa & (__alignof__(hc->ingpa) - 1)) ||
2094 offset_in_page(gpa) + sizeof(hc->ingpa) > PAGE_SIZE)
2095 return HV_STATUS_INVALID_ALIGNMENT;
2096
2097 ret = kvm_vcpu_read_guest(vcpu, gpa,
2098 &hc->ingpa, sizeof(hc->ingpa));
2099 if (ret < 0)
2100 return HV_STATUS_INVALID_ALIGNMENT;
2101 }
2102
2103 /*
2104 * Per spec, bits 32-47 contain the extra "flag number". However, we
2105 * have no use for it, and in all known usecases it is zero, so just
2106 * report lookup failure if it isn't.
2107 */
2108 if (hc->ingpa & 0xffff00000000ULL)
2109 return HV_STATUS_INVALID_PORT_ID;
2110 /* remaining bits are reserved-zero */
2111 if (hc->ingpa & ~KVM_HYPERV_CONN_ID_MASK)
2112 return HV_STATUS_INVALID_HYPERCALL_INPUT;
2113
2114 /* the eventfd is protected by vcpu->kvm->srcu, but conn_to_evt isn't */
2115 rcu_read_lock();
2116 eventfd = idr_find(&hv->conn_to_evt, hc->ingpa);
2117 rcu_read_unlock();
2118 if (!eventfd)
2119 return HV_STATUS_INVALID_PORT_ID;
2120
2121 eventfd_signal(eventfd, 1);
2122 return HV_STATUS_SUCCESS;
2123 }
2124
is_xmm_fast_hypercall(struct kvm_hv_hcall * hc)2125 static bool is_xmm_fast_hypercall(struct kvm_hv_hcall *hc)
2126 {
2127 switch (hc->code) {
2128 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST:
2129 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE:
2130 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX:
2131 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX:
2132 case HVCALL_SEND_IPI_EX:
2133 return true;
2134 }
2135
2136 return false;
2137 }
2138
kvm_hv_hypercall_read_xmm(struct kvm_hv_hcall * hc)2139 static void kvm_hv_hypercall_read_xmm(struct kvm_hv_hcall *hc)
2140 {
2141 int reg;
2142
2143 kvm_fpu_get();
2144 for (reg = 0; reg < HV_HYPERCALL_MAX_XMM_REGISTERS; reg++)
2145 _kvm_read_sse_reg(reg, &hc->xmm[reg]);
2146 kvm_fpu_put();
2147 }
2148
hv_check_hypercall_access(struct kvm_vcpu_hv * hv_vcpu,u16 code)2149 static bool hv_check_hypercall_access(struct kvm_vcpu_hv *hv_vcpu, u16 code)
2150 {
2151 if (!hv_vcpu->enforce_cpuid)
2152 return true;
2153
2154 switch (code) {
2155 case HVCALL_NOTIFY_LONG_SPIN_WAIT:
2156 return hv_vcpu->cpuid_cache.enlightenments_ebx &&
2157 hv_vcpu->cpuid_cache.enlightenments_ebx != U32_MAX;
2158 case HVCALL_POST_MESSAGE:
2159 return hv_vcpu->cpuid_cache.features_ebx & HV_POST_MESSAGES;
2160 case HVCALL_SIGNAL_EVENT:
2161 return hv_vcpu->cpuid_cache.features_ebx & HV_SIGNAL_EVENTS;
2162 case HVCALL_POST_DEBUG_DATA:
2163 case HVCALL_RETRIEVE_DEBUG_DATA:
2164 case HVCALL_RESET_DEBUG_SESSION:
2165 /*
2166 * Return 'true' when SynDBG is disabled so the resulting code
2167 * will be HV_STATUS_INVALID_HYPERCALL_CODE.
2168 */
2169 return !kvm_hv_is_syndbg_enabled(hv_vcpu->vcpu) ||
2170 hv_vcpu->cpuid_cache.features_ebx & HV_DEBUGGING;
2171 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX:
2172 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX:
2173 if (!(hv_vcpu->cpuid_cache.enlightenments_eax &
2174 HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED))
2175 return false;
2176 fallthrough;
2177 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST:
2178 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE:
2179 return hv_vcpu->cpuid_cache.enlightenments_eax &
2180 HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED;
2181 case HVCALL_SEND_IPI_EX:
2182 if (!(hv_vcpu->cpuid_cache.enlightenments_eax &
2183 HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED))
2184 return false;
2185 fallthrough;
2186 case HVCALL_SEND_IPI:
2187 return hv_vcpu->cpuid_cache.enlightenments_eax &
2188 HV_X64_CLUSTER_IPI_RECOMMENDED;
2189 default:
2190 break;
2191 }
2192
2193 return true;
2194 }
2195
kvm_hv_hypercall(struct kvm_vcpu * vcpu)2196 int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
2197 {
2198 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
2199 struct kvm_hv_hcall hc;
2200 u64 ret = HV_STATUS_SUCCESS;
2201
2202 /*
2203 * hypercall generates UD from non zero cpl and real mode
2204 * per HYPER-V spec
2205 */
2206 if (static_call(kvm_x86_get_cpl)(vcpu) != 0 || !is_protmode(vcpu)) {
2207 kvm_queue_exception(vcpu, UD_VECTOR);
2208 return 1;
2209 }
2210
2211 #ifdef CONFIG_X86_64
2212 if (is_64_bit_hypercall(vcpu)) {
2213 hc.param = kvm_rcx_read(vcpu);
2214 hc.ingpa = kvm_rdx_read(vcpu);
2215 hc.outgpa = kvm_r8_read(vcpu);
2216 } else
2217 #endif
2218 {
2219 hc.param = ((u64)kvm_rdx_read(vcpu) << 32) |
2220 (kvm_rax_read(vcpu) & 0xffffffff);
2221 hc.ingpa = ((u64)kvm_rbx_read(vcpu) << 32) |
2222 (kvm_rcx_read(vcpu) & 0xffffffff);
2223 hc.outgpa = ((u64)kvm_rdi_read(vcpu) << 32) |
2224 (kvm_rsi_read(vcpu) & 0xffffffff);
2225 }
2226
2227 hc.code = hc.param & 0xffff;
2228 hc.fast = !!(hc.param & HV_HYPERCALL_FAST_BIT);
2229 hc.rep_cnt = (hc.param >> HV_HYPERCALL_REP_COMP_OFFSET) & 0xfff;
2230 hc.rep_idx = (hc.param >> HV_HYPERCALL_REP_START_OFFSET) & 0xfff;
2231 hc.rep = !!(hc.rep_cnt || hc.rep_idx);
2232
2233 trace_kvm_hv_hypercall(hc.code, hc.fast, hc.rep_cnt, hc.rep_idx,
2234 hc.ingpa, hc.outgpa);
2235
2236 if (unlikely(!hv_check_hypercall_access(hv_vcpu, hc.code))) {
2237 ret = HV_STATUS_ACCESS_DENIED;
2238 goto hypercall_complete;
2239 }
2240
2241 if (hc.fast && is_xmm_fast_hypercall(&hc)) {
2242 if (unlikely(hv_vcpu->enforce_cpuid &&
2243 !(hv_vcpu->cpuid_cache.features_edx &
2244 HV_X64_HYPERCALL_XMM_INPUT_AVAILABLE))) {
2245 kvm_queue_exception(vcpu, UD_VECTOR);
2246 return 1;
2247 }
2248
2249 kvm_hv_hypercall_read_xmm(&hc);
2250 }
2251
2252 switch (hc.code) {
2253 case HVCALL_NOTIFY_LONG_SPIN_WAIT:
2254 if (unlikely(hc.rep)) {
2255 ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
2256 break;
2257 }
2258 kvm_vcpu_on_spin(vcpu, true);
2259 break;
2260 case HVCALL_SIGNAL_EVENT:
2261 if (unlikely(hc.rep)) {
2262 ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
2263 break;
2264 }
2265 ret = kvm_hvcall_signal_event(vcpu, &hc);
2266 if (ret != HV_STATUS_INVALID_PORT_ID)
2267 break;
2268 fallthrough; /* maybe userspace knows this conn_id */
2269 case HVCALL_POST_MESSAGE:
2270 /* don't bother userspace if it has no way to handle it */
2271 if (unlikely(hc.rep || !to_hv_synic(vcpu)->active)) {
2272 ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
2273 break;
2274 }
2275 vcpu->run->exit_reason = KVM_EXIT_HYPERV;
2276 vcpu->run->hyperv.type = KVM_EXIT_HYPERV_HCALL;
2277 vcpu->run->hyperv.u.hcall.input = hc.param;
2278 vcpu->run->hyperv.u.hcall.params[0] = hc.ingpa;
2279 vcpu->run->hyperv.u.hcall.params[1] = hc.outgpa;
2280 vcpu->arch.complete_userspace_io =
2281 kvm_hv_hypercall_complete_userspace;
2282 return 0;
2283 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST:
2284 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX:
2285 if (unlikely(!hc.rep_cnt || hc.rep_idx)) {
2286 ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
2287 break;
2288 }
2289 ret = kvm_hv_flush_tlb(vcpu, &hc);
2290 break;
2291 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE:
2292 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX:
2293 if (unlikely(hc.rep)) {
2294 ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
2295 break;
2296 }
2297 ret = kvm_hv_flush_tlb(vcpu, &hc);
2298 break;
2299 case HVCALL_SEND_IPI:
2300 case HVCALL_SEND_IPI_EX:
2301 if (unlikely(hc.rep)) {
2302 ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
2303 break;
2304 }
2305 ret = kvm_hv_send_ipi(vcpu, &hc);
2306 break;
2307 case HVCALL_POST_DEBUG_DATA:
2308 case HVCALL_RETRIEVE_DEBUG_DATA:
2309 if (unlikely(hc.fast)) {
2310 ret = HV_STATUS_INVALID_PARAMETER;
2311 break;
2312 }
2313 fallthrough;
2314 case HVCALL_RESET_DEBUG_SESSION: {
2315 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu);
2316
2317 if (!kvm_hv_is_syndbg_enabled(vcpu)) {
2318 ret = HV_STATUS_INVALID_HYPERCALL_CODE;
2319 break;
2320 }
2321
2322 if (!(syndbg->options & HV_X64_SYNDBG_OPTION_USE_HCALLS)) {
2323 ret = HV_STATUS_OPERATION_DENIED;
2324 break;
2325 }
2326 vcpu->run->exit_reason = KVM_EXIT_HYPERV;
2327 vcpu->run->hyperv.type = KVM_EXIT_HYPERV_HCALL;
2328 vcpu->run->hyperv.u.hcall.input = hc.param;
2329 vcpu->run->hyperv.u.hcall.params[0] = hc.ingpa;
2330 vcpu->run->hyperv.u.hcall.params[1] = hc.outgpa;
2331 vcpu->arch.complete_userspace_io =
2332 kvm_hv_hypercall_complete_userspace;
2333 return 0;
2334 }
2335 default:
2336 ret = HV_STATUS_INVALID_HYPERCALL_CODE;
2337 break;
2338 }
2339
2340 hypercall_complete:
2341 return kvm_hv_hypercall_complete(vcpu, ret);
2342 }
2343
kvm_hv_init_vm(struct kvm * kvm)2344 void kvm_hv_init_vm(struct kvm *kvm)
2345 {
2346 struct kvm_hv *hv = to_kvm_hv(kvm);
2347
2348 mutex_init(&hv->hv_lock);
2349 idr_init(&hv->conn_to_evt);
2350 }
2351
kvm_hv_destroy_vm(struct kvm * kvm)2352 void kvm_hv_destroy_vm(struct kvm *kvm)
2353 {
2354 struct kvm_hv *hv = to_kvm_hv(kvm);
2355 struct eventfd_ctx *eventfd;
2356 int i;
2357
2358 idr_for_each_entry(&hv->conn_to_evt, eventfd, i)
2359 eventfd_ctx_put(eventfd);
2360 idr_destroy(&hv->conn_to_evt);
2361 }
2362
kvm_hv_eventfd_assign(struct kvm * kvm,u32 conn_id,int fd)2363 static int kvm_hv_eventfd_assign(struct kvm *kvm, u32 conn_id, int fd)
2364 {
2365 struct kvm_hv *hv = to_kvm_hv(kvm);
2366 struct eventfd_ctx *eventfd;
2367 int ret;
2368
2369 eventfd = eventfd_ctx_fdget(fd);
2370 if (IS_ERR(eventfd))
2371 return PTR_ERR(eventfd);
2372
2373 mutex_lock(&hv->hv_lock);
2374 ret = idr_alloc(&hv->conn_to_evt, eventfd, conn_id, conn_id + 1,
2375 GFP_KERNEL_ACCOUNT);
2376 mutex_unlock(&hv->hv_lock);
2377
2378 if (ret >= 0)
2379 return 0;
2380
2381 if (ret == -ENOSPC)
2382 ret = -EEXIST;
2383 eventfd_ctx_put(eventfd);
2384 return ret;
2385 }
2386
kvm_hv_eventfd_deassign(struct kvm * kvm,u32 conn_id)2387 static int kvm_hv_eventfd_deassign(struct kvm *kvm, u32 conn_id)
2388 {
2389 struct kvm_hv *hv = to_kvm_hv(kvm);
2390 struct eventfd_ctx *eventfd;
2391
2392 mutex_lock(&hv->hv_lock);
2393 eventfd = idr_remove(&hv->conn_to_evt, conn_id);
2394 mutex_unlock(&hv->hv_lock);
2395
2396 if (!eventfd)
2397 return -ENOENT;
2398
2399 synchronize_srcu(&kvm->srcu);
2400 eventfd_ctx_put(eventfd);
2401 return 0;
2402 }
2403
kvm_vm_ioctl_hv_eventfd(struct kvm * kvm,struct kvm_hyperv_eventfd * args)2404 int kvm_vm_ioctl_hv_eventfd(struct kvm *kvm, struct kvm_hyperv_eventfd *args)
2405 {
2406 if ((args->flags & ~KVM_HYPERV_EVENTFD_DEASSIGN) ||
2407 (args->conn_id & ~KVM_HYPERV_CONN_ID_MASK))
2408 return -EINVAL;
2409
2410 if (args->flags == KVM_HYPERV_EVENTFD_DEASSIGN)
2411 return kvm_hv_eventfd_deassign(kvm, args->conn_id);
2412 return kvm_hv_eventfd_assign(kvm, args->conn_id, args->fd);
2413 }
2414
kvm_get_hv_cpuid(struct kvm_vcpu * vcpu,struct kvm_cpuid2 * cpuid,struct kvm_cpuid_entry2 __user * entries)2415 int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid,
2416 struct kvm_cpuid_entry2 __user *entries)
2417 {
2418 uint16_t evmcs_ver = 0;
2419 struct kvm_cpuid_entry2 cpuid_entries[] = {
2420 { .function = HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS },
2421 { .function = HYPERV_CPUID_INTERFACE },
2422 { .function = HYPERV_CPUID_VERSION },
2423 { .function = HYPERV_CPUID_FEATURES },
2424 { .function = HYPERV_CPUID_ENLIGHTMENT_INFO },
2425 { .function = HYPERV_CPUID_IMPLEMENT_LIMITS },
2426 { .function = HYPERV_CPUID_SYNDBG_VENDOR_AND_MAX_FUNCTIONS },
2427 { .function = HYPERV_CPUID_SYNDBG_INTERFACE },
2428 { .function = HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES },
2429 { .function = HYPERV_CPUID_NESTED_FEATURES },
2430 };
2431 int i, nent = ARRAY_SIZE(cpuid_entries);
2432
2433 if (kvm_x86_ops.nested_ops->get_evmcs_version)
2434 evmcs_ver = kvm_x86_ops.nested_ops->get_evmcs_version(vcpu);
2435
2436 /* Skip NESTED_FEATURES if eVMCS is not supported */
2437 if (!evmcs_ver)
2438 --nent;
2439
2440 if (cpuid->nent < nent)
2441 return -E2BIG;
2442
2443 if (cpuid->nent > nent)
2444 cpuid->nent = nent;
2445
2446 for (i = 0; i < nent; i++) {
2447 struct kvm_cpuid_entry2 *ent = &cpuid_entries[i];
2448 u32 signature[3];
2449
2450 switch (ent->function) {
2451 case HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS:
2452 memcpy(signature, "Linux KVM Hv", 12);
2453
2454 ent->eax = HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES;
2455 ent->ebx = signature[0];
2456 ent->ecx = signature[1];
2457 ent->edx = signature[2];
2458 break;
2459
2460 case HYPERV_CPUID_INTERFACE:
2461 ent->eax = HYPERV_CPUID_SIGNATURE_EAX;
2462 break;
2463
2464 case HYPERV_CPUID_VERSION:
2465 /*
2466 * We implement some Hyper-V 2016 functions so let's use
2467 * this version.
2468 */
2469 ent->eax = 0x00003839;
2470 ent->ebx = 0x000A0000;
2471 break;
2472
2473 case HYPERV_CPUID_FEATURES:
2474 ent->eax |= HV_MSR_VP_RUNTIME_AVAILABLE;
2475 ent->eax |= HV_MSR_TIME_REF_COUNT_AVAILABLE;
2476 ent->eax |= HV_MSR_SYNIC_AVAILABLE;
2477 ent->eax |= HV_MSR_SYNTIMER_AVAILABLE;
2478 ent->eax |= HV_MSR_APIC_ACCESS_AVAILABLE;
2479 ent->eax |= HV_MSR_HYPERCALL_AVAILABLE;
2480 ent->eax |= HV_MSR_VP_INDEX_AVAILABLE;
2481 ent->eax |= HV_MSR_RESET_AVAILABLE;
2482 ent->eax |= HV_MSR_REFERENCE_TSC_AVAILABLE;
2483 ent->eax |= HV_ACCESS_FREQUENCY_MSRS;
2484 ent->eax |= HV_ACCESS_REENLIGHTENMENT;
2485
2486 ent->ebx |= HV_POST_MESSAGES;
2487 ent->ebx |= HV_SIGNAL_EVENTS;
2488
2489 ent->edx |= HV_X64_HYPERCALL_XMM_INPUT_AVAILABLE;
2490 ent->edx |= HV_FEATURE_FREQUENCY_MSRS_AVAILABLE;
2491 ent->edx |= HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE;
2492
2493 ent->ebx |= HV_DEBUGGING;
2494 ent->edx |= HV_X64_GUEST_DEBUGGING_AVAILABLE;
2495 ent->edx |= HV_FEATURE_DEBUG_MSRS_AVAILABLE;
2496
2497 /*
2498 * Direct Synthetic timers only make sense with in-kernel
2499 * LAPIC
2500 */
2501 if (!vcpu || lapic_in_kernel(vcpu))
2502 ent->edx |= HV_STIMER_DIRECT_MODE_AVAILABLE;
2503
2504 break;
2505
2506 case HYPERV_CPUID_ENLIGHTMENT_INFO:
2507 ent->eax |= HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED;
2508 ent->eax |= HV_X64_APIC_ACCESS_RECOMMENDED;
2509 ent->eax |= HV_X64_RELAXED_TIMING_RECOMMENDED;
2510 ent->eax |= HV_X64_CLUSTER_IPI_RECOMMENDED;
2511 ent->eax |= HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED;
2512 if (evmcs_ver)
2513 ent->eax |= HV_X64_ENLIGHTENED_VMCS_RECOMMENDED;
2514 if (!cpu_smt_possible())
2515 ent->eax |= HV_X64_NO_NONARCH_CORESHARING;
2516
2517 ent->eax |= HV_DEPRECATING_AEOI_RECOMMENDED;
2518 /*
2519 * Default number of spinlock retry attempts, matches
2520 * HyperV 2016.
2521 */
2522 ent->ebx = 0x00000FFF;
2523
2524 break;
2525
2526 case HYPERV_CPUID_IMPLEMENT_LIMITS:
2527 /* Maximum number of virtual processors */
2528 ent->eax = KVM_MAX_VCPUS;
2529 /*
2530 * Maximum number of logical processors, matches
2531 * HyperV 2016.
2532 */
2533 ent->ebx = 64;
2534
2535 break;
2536
2537 case HYPERV_CPUID_NESTED_FEATURES:
2538 ent->eax = evmcs_ver;
2539
2540 break;
2541
2542 case HYPERV_CPUID_SYNDBG_VENDOR_AND_MAX_FUNCTIONS:
2543 memcpy(signature, "Linux KVM Hv", 12);
2544
2545 ent->eax = 0;
2546 ent->ebx = signature[0];
2547 ent->ecx = signature[1];
2548 ent->edx = signature[2];
2549 break;
2550
2551 case HYPERV_CPUID_SYNDBG_INTERFACE:
2552 memcpy(signature, "VS#1\0\0\0\0\0\0\0\0", 12);
2553 ent->eax = signature[0];
2554 break;
2555
2556 case HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES:
2557 ent->eax |= HV_X64_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING;
2558 break;
2559
2560 default:
2561 break;
2562 }
2563 }
2564
2565 if (copy_to_user(entries, cpuid_entries,
2566 nent * sizeof(struct kvm_cpuid_entry2)))
2567 return -EFAULT;
2568
2569 return 0;
2570 }
2571