1 // SPDX-License-Identifier: GPL-2.0-only
3  * Kernel-based Virtual Machine driver for Linux
16  *   Ben-Ami Yassour <benami@il.ibm.com>
48 #include <linux/user-return-notifier.h>
62 #include <linux/entry-kvm.h>
103 	((struct kvm_vcpu *)(ctxt)->vcpu)
106  * - enable syscall per default because its emulated by KVM
107  * - enable LME and LMA per default on 64 bit KVM
140 				*(((struct kvm_x86_ops *)0)->func));
143 #include <asm/kvm-x86-ops.h>
160 /* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
165  * lapic timer advance (tscdeadline mode only) in nanoseconds.  '-1' enables
167  * advancement entirely.  Any other value is used as-is and disables adaptive
170 static int __read_mostly lapic_timer_advance_ns = -1;
181  * Flags to manipulate forced emulation behavior (any non-zero value will
188 int __read_mostly pi_inject_timer = -1;
350 					  size - useroffset, NULL);  in kvm_alloc_emulator_cache()
359 		vcpu->arch.apf.gfns[i] = ~0;  in kvm_async_pf_hash_reset()
375 	if (msrs->registered) {  in kvm_on_user_return()
376 		msrs->registered = false;  in kvm_on_user_return()
381 		values = &msrs->values[slot];  in kvm_on_user_return()
382 		if (values->host != values->curr) {  in kvm_on_user_return()
383 			wrmsrl(kvm_uret_msrs_list[slot], values->host);  in kvm_on_user_return()
384 			values->curr = values->host;  in kvm_on_user_return()
409 		return -1;  in kvm_add_user_return_msr()
424 	return -1;  in kvm_find_user_return_msr()
437 		msrs->values[i].host = value;  in kvm_user_return_msr_cpu_online()
438 		msrs->values[i].curr = value;  in kvm_user_return_msr_cpu_online()
448 	value = (value & mask) | (msrs->values[slot].host & ~mask);  in kvm_set_user_return_msr()
449 	if (value == msrs->values[slot].curr)  in kvm_set_user_return_msr()
455 	msrs->values[slot].curr = value;  in kvm_set_user_return_msr()
456 	if (!msrs->registered) {  in kvm_set_user_return_msr()
457 		msrs->urn.on_user_return = kvm_on_user_return;  in kvm_set_user_return_msr()
458 		user_return_notifier_register(&msrs->urn);  in kvm_set_user_return_msr()
459 		msrs->registered = true;  in kvm_set_user_return_msr()
470 	if (msrs->registered)  in drop_user_return_notifiers()
471 		kvm_on_user_return(&msrs->urn);  in drop_user_return_notifiers()
476 	return vcpu->arch.apic_base;  in kvm_get_apic_base()
488 	enum lapic_mode new_mode = kvm_apic_mode(msr_info->data);  in kvm_set_apic_base()
492 	if ((msr_info->data & reserved_bits) != 0 || new_mode == LAPIC_MODE_INVALID)  in kvm_set_apic_base()
494 	if (!msr_info->host_initiated) {  in kvm_set_apic_base()
501 	kvm_lapic_set_base(vcpu, msr_info->data);  in kvm_set_apic_base()
502 	kvm_recalculate_apic_map(vcpu->kvm);  in kvm_set_apic_base()
557 	 * #DBs can be trap-like or fault-like, the caller must check other CPU  in exception_type()
576 	if (!ex->has_payload)  in kvm_deliver_exception_payload()
579 	switch (ex->vector) {  in kvm_deliver_exception_payload()
582 		 * "Certain debug exceptions may clear bit 0-3.  The  in kvm_deliver_exception_payload()
586 		vcpu->arch.dr6 &= ~DR_TRAP_BITS;  in kvm_deliver_exception_payload()
595 		 * Active low bits should be cleared if 1-setting in payload.  in kvm_deliver_exception_payload()
596 		 * Active high bits should be set if 1-setting in payload.  in kvm_deliver_exception_payload()
603 		vcpu->arch.dr6 |= DR6_ACTIVE_LOW;  in kvm_deliver_exception_payload()
604 		vcpu->arch.dr6 |= ex->payload;  in kvm_deliver_exception_payload()
605 		vcpu->arch.dr6 ^= ex->payload & DR6_ACTIVE_LOW;  in kvm_deliver_exception_payload()
613 		vcpu->arch.dr6 &= ~BIT(12);  in kvm_deliver_exception_payload()
616 		vcpu->arch.cr2 = ex->payload;  in kvm_deliver_exception_payload()
620 	ex->has_payload = false;  in kvm_deliver_exception_payload()
621 	ex->payload = 0;  in kvm_deliver_exception_payload()
629 	struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit;  in kvm_queue_exception_vmexit()
631 	ex->vector = vector;  in kvm_queue_exception_vmexit()
632 	ex->injected = false;  in kvm_queue_exception_vmexit()
633 	ex->pending = true;  in kvm_queue_exception_vmexit()
634 	ex->has_error_code = has_error_code;  in kvm_queue_exception_vmexit()
635 	ex->error_code = error_code;  in kvm_queue_exception_vmexit()
636 	ex->has_payload = has_payload;  in kvm_queue_exception_vmexit()
637 	ex->payload = payload;  in kvm_queue_exception_vmexit()
643 	kvm_x86_ops.nested_ops->leave_nested(vcpu);  in kvm_leave_nested()
657 	 * morph it to a VM-Exit if L1 wants to intercept the exception.  A  in kvm_multiple_exception()
659 	 * when it was original queued, and re-checking is incorrect if _L1_  in kvm_multiple_exception()
663 	    kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, nr, error_code)) {  in kvm_multiple_exception()
669 	if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {  in kvm_multiple_exception()
673 			 * On VM-Entry, an exception can be pending if and only  in kvm_multiple_exception()
680 			vcpu->arch.exception.injected = true;  in kvm_multiple_exception()
690 			vcpu->arch.exception.pending = true;  in kvm_multiple_exception()
691 			vcpu->arch.exception.injected = false;  in kvm_multiple_exception()
693 		vcpu->arch.exception.has_error_code = has_error;  in kvm_multiple_exception()
694 		vcpu->arch.exception.vector = nr;  in kvm_multiple_exception()
695 		vcpu->arch.exception.error_code = error_code;  in kvm_multiple_exception()
696 		vcpu->arch.exception.has_payload = has_payload;  in kvm_multiple_exception()
697 		vcpu->arch.exception.payload = payload;  in kvm_multiple_exception()
700 						      &vcpu->arch.exception);  in kvm_multiple_exception()
705 	prev_nr = vcpu->arch.exception.vector;  in kvm_multiple_exception()
707 		/* triple fault -> shutdown */  in kvm_multiple_exception()
719 		vcpu->arch.exception.injected = false;  in kvm_multiple_exception()
720 		vcpu->arch.exception.pending = false;  in kvm_multiple_exception()
725 		   that instruction re-execution will regenerate lost  in kvm_multiple_exception()
781 	++vcpu->stat.pf_guest;  in kvm_inject_page_fault()
784 	 * Async #PF in L2 is always forwarded to L1 as a VM-Exit regardless of  in kvm_inject_page_fault()
787 	if (is_guest_mode(vcpu) && fault->async_page_fault)  in kvm_inject_page_fault()
789 					   true, fault->error_code,  in kvm_inject_page_fault()
790 					   true, fault->address);  in kvm_inject_page_fault()
792 		kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,  in kvm_inject_page_fault()
793 					fault->address);  in kvm_inject_page_fault()
800 	WARN_ON_ONCE(fault->vector != PF_VECTOR);  in kvm_inject_emulated_page_fault()
802 	fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :  in kvm_inject_emulated_page_fault()
803 					       vcpu->arch.walk_mmu;  in kvm_inject_emulated_page_fault()
809 	if ((fault->error_code & PFERR_PRESENT_MASK) &&  in kvm_inject_emulated_page_fault()
810 	    !(fault->error_code & PFERR_RSVD_MASK))  in kvm_inject_emulated_page_fault()
811 		kvm_mmu_invalidate_addr(vcpu, fault_mmu, fault->address,  in kvm_inject_emulated_page_fault()
814 	fault_mmu->inject_page_fault(vcpu, fault);  in kvm_inject_emulated_page_fault()
820 	atomic_inc(&vcpu->arch.nmi_queued);  in kvm_inject_nmi()
860 	return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2);  in pdptr_rsvd_bits()
868 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in load_pdptrs()
873 	u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];  in load_pdptrs()
901 	if (!tdp_enabled && memcmp(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs)))  in load_pdptrs()
902 		kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT);  in load_pdptrs()
904 	memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));  in load_pdptrs()
907 	vcpu->arch.pdptrs_from_userspace = false;  in load_pdptrs()
932 	 * CR0.WP is incorporated into the MMU role, but only for non-nested,  in kvm_post_set_cr0()
965 	    kvm_arch_has_noncoherent_dma(vcpu->kvm) &&  in kvm_post_set_cr0()
966 	    !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))  in kvm_post_set_cr0()
967 		kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);  in kvm_post_set_cr0()
984 	if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) &&  in kvm_set_cr0()
995 	if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) &&  in kvm_set_cr0()
1020 	if (vcpu->arch.guest_state_protected)  in kvm_load_guest_xsave_state()
1025 		if (vcpu->arch.xcr0 != host_xcr0)  in kvm_load_guest_xsave_state()
1026 			xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);  in kvm_load_guest_xsave_state()
1029 		    vcpu->arch.ia32_xss != host_xss)  in kvm_load_guest_xsave_state()
1030 			wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);  in kvm_load_guest_xsave_state()
1034 	    vcpu->arch.pkru != vcpu->arch.host_pkru &&  in kvm_load_guest_xsave_state()
1035 	    ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||  in kvm_load_guest_xsave_state()
1037 		write_pkru(vcpu->arch.pkru);  in kvm_load_guest_xsave_state()
1043 	if (vcpu->arch.guest_state_protected)  in kvm_load_host_xsave_state()
1047 	    ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||  in kvm_load_host_xsave_state()
1049 		vcpu->arch.pkru = rdpkru();  in kvm_load_host_xsave_state()
1050 		if (vcpu->arch.pkru != vcpu->arch.host_pkru)  in kvm_load_host_xsave_state()
1051 			write_pkru(vcpu->arch.host_pkru);  in kvm_load_host_xsave_state()
1056 		if (vcpu->arch.xcr0 != host_xcr0)  in kvm_load_host_xsave_state()
1060 		    vcpu->arch.ia32_xss != host_xss)  in kvm_load_host_xsave_state()
1070 	return vcpu->arch.guest_supported_xcr0 & XFEATURE_MASK_USER_DYNAMIC;  in kvm_guest_supported_xfd()
1077 	u64 old_xcr0 = vcpu->arch.xcr0;  in __kvm_set_xcr()
1093 	valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;  in __kvm_set_xcr()
1112 	vcpu->arch.xcr0 = xcr0;  in __kvm_set_xcr()
1137 	if (cr4 & vcpu->arch.cr4_guest_rsvd_bits)  in __kvm_is_valid_cr4()
1156 	 * If CR4.PCIDE is changed 0 -> 1, there is no need to flush the TLB  in kvm_post_set_cr4()
1170 	 * - CR4.PCIDE is changed from 1 to 0  in kvm_post_set_cr4()
1171 	 * - CR4.PGE is toggled  in kvm_post_set_cr4()
1182 	 * - CR4.SMEP is changed from 0 to 1  in kvm_post_set_cr4()
1183 	 * - CR4.PAE is toggled  in kvm_post_set_cr4()
1225 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in kvm_invalidate_pcid()
1260 		if (kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd) == pcid)  in kvm_invalidate_pcid()
1263 	kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free);  in kvm_invalidate_pcid()
1296 	vcpu->arch.cr3 = cr3;  in kvm_set_cr3()
1305 	 * and it's impossible to use a non-zero PCID when PCID is disabled,  in kvm_set_cr3()
1322 		vcpu->arch.cr8 = cr8;  in kvm_set_cr8()
1332 		return vcpu->arch.cr8;  in kvm_get_cr8()
1340 	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {  in kvm_update_dr0123()
1342 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];  in kvm_update_dr0123()
1350 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)  in kvm_update_dr7()
1351 		dr7 = vcpu->arch.guest_debug_dr7;  in kvm_update_dr7()
1353 		dr7 = vcpu->arch.dr7;  in kvm_update_dr7()
1355 	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;  in kvm_update_dr7()
1357 		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;  in kvm_update_dr7()
1375 	size_t size = ARRAY_SIZE(vcpu->arch.db);  in kvm_set_dr()
1379 		vcpu->arch.db[array_index_nospec(dr, size)] = val;  in kvm_set_dr()
1380 		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))  in kvm_set_dr()
1381 			vcpu->arch.eff_db[dr] = val;  in kvm_set_dr()
1387 		vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);  in kvm_set_dr()
1393 		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;  in kvm_set_dr()
1404 	size_t size = ARRAY_SIZE(vcpu->arch.db);  in kvm_get_dr()
1408 		*val = vcpu->arch.db[array_index_nospec(dr, size)];  in kvm_get_dr()
1412 		*val = vcpu->arch.dr6;  in kvm_get_dr()
1416 		*val = vcpu->arch.dr7;  in kvm_get_dr()
1573  * List of MSRs that control the existence of MSR-based features, i.e. MSRs
1585 			      (KVM_LAST_EMULATED_VMX_MSR - KVM_FIRST_EMULATED_VMX_MSR + 1)];
1610  *   10 - MISC_PACKAGE_CTRLS
1611  *   11 - ENERGY_FILTERING_CTL
1612  *   12 - DOITM
1613  *   18 - FB_CLEAR_CTRL
1614  *   21 - XAPIC_DISABLE_STATUS
1615  *   23 - OVERCLOCKING_STATUS
1686 	switch (msr->index) {  in kvm_get_msr_feature()
1688 		msr->data = kvm_get_arch_capabilities();  in kvm_get_msr_feature()
1691 		msr->data = kvm_caps.supported_perf_cap;  in kvm_get_msr_feature()
1694 		rdmsrl_safe(msr->index, &msr->data);  in kvm_get_msr_feature()
1757 	u64 old_efer = vcpu->arch.efer;  in set_efer()
1758 	u64 efer = msr_info->data;  in set_efer()
1764 	if (!msr_info->host_initiated) {  in set_efer()
1769 		    (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))  in set_efer()
1774 	efer |= vcpu->arch.efer & EFER_LMA;  in set_efer()
1798 	struct kvm *kvm = vcpu->kvm;  in kvm_msr_allowed()
1807 	idx = srcu_read_lock(&kvm->srcu);  in kvm_msr_allowed()
1809 	msr_filter = srcu_dereference(kvm->arch.msr_filter, &kvm->srcu);  in kvm_msr_allowed()
1815 	allowed = msr_filter->default_allow;  in kvm_msr_allowed()
1816 	ranges = msr_filter->ranges;  in kvm_msr_allowed()
1818 	for (i = 0; i < msr_filter->count; i++) {  in kvm_msr_allowed()
1819 		u32 start = ranges[i].base;  in kvm_msr_allowed()
1825 			allowed = test_bit(index - start, bitmap);  in kvm_msr_allowed()
1831 	srcu_read_unlock(&kvm->srcu, idx);  in kvm_msr_allowed()
1840  * Returns 0 on success, non-0 otherwise.
1861 		 * non-canonical address is written on Intel but not on  in __kvm_set_msr()
1862 		 * AMD (which ignores the top 32-bits, because it does  in __kvm_set_msr()
1863 		 * not implement 64-bit SYSENTER).  in __kvm_set_msr()
1865 		 * 64-bit code should hence be able to write a non-canonical  in __kvm_set_msr()
1867 		 * vmentry does not fail on Intel after writing a non-canonical  in __kvm_set_msr()
1869 		 * invokes 64-bit SYSENTER.  in __kvm_set_msr()
1888 		 * the bits in all other cases.  This ensures cross-vendor  in __kvm_set_msr()
1920  * Returns 0 on success, non-0 otherwise.
1993 	if (!vcpu->run->msr.error) {  in complete_userspace_rdmsr()
1994 		kvm_rax_write(vcpu, (u32)vcpu->run->msr.data);  in complete_userspace_rdmsr()
1995 		kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);  in complete_userspace_rdmsr()
2001 	return complete_emulated_insn_gp(vcpu, vcpu->run->msr.error);  in complete_emulated_msr_access()
2012 	return static_call(kvm_x86_complete_emulated_msr)(vcpu, vcpu->run->msr.error);  in complete_fast_msr_access()
2041 	if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason))  in kvm_msr_user_space()
2044 	vcpu->run->exit_reason = exit_reason;  in kvm_msr_user_space()
2045 	vcpu->run->msr.error = 0;  in kvm_msr_user_space()
2046 	memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad));  in kvm_msr_user_space()
2047 	vcpu->run->msr.reason = msr_reason;  in kvm_msr_user_space()
2048 	vcpu->run->msr.index = index;  in kvm_msr_user_space()
2049 	vcpu->run->msr.data = data;  in kvm_msr_user_space()
2050 	vcpu->arch.complete_userspace_io = completion;  in kvm_msr_user_space()
2066 		kvm_rax_write(vcpu, data & -1u);  in kvm_emulate_rdmsr()
2067 		kvm_rdx_write(vcpu, (data >> 32) & -1u);  in kvm_emulate_rdmsr()
2127 	if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS) &&  in kvm_emulate_monitor_mwait()
2149 	return vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) ||  in kvm_vcpu_exit_request()
2155  * i.e. the sending of IPI, sending IPI early in the VM-Exit flow reduces
2162 	if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic))  in handle_fastpath_set_x2apic_icr_irqoff()
2169 		return kvm_x2apic_icr_write(vcpu->arch.apic, data);  in handle_fastpath_set_x2apic_icr_irqoff()
2240 			return -EINVAL;  in do_set_msr()
2275 	write_seqcount_begin(&vdata->seq);  in update_pvclock_gtod()
2278 	vdata->clock.vclock_mode	= tk->tkr_mono.clock->vdso_clock_mode;  in update_pvclock_gtod()
2279 	vdata->clock.cycle_last		= tk->tkr_mono.cycle_last;  in update_pvclock_gtod()
2280 	vdata->clock.mask		= tk->tkr_mono.mask;  in update_pvclock_gtod()
2281 	vdata->clock.mult		= tk->tkr_mono.mult;  in update_pvclock_gtod()
2282 	vdata->clock.shift		= tk->tkr_mono.shift;  in update_pvclock_gtod()
2283 	vdata->clock.base_cycles	= tk->tkr_mono.xtime_nsec;  in update_pvclock_gtod()
2284 	vdata->clock.offset		= tk->tkr_mono.base;  in update_pvclock_gtod()
2286 	vdata->raw_clock.vclock_mode	= tk->tkr_raw.clock->vdso_clock_mode;  in update_pvclock_gtod()
2287 	vdata->raw_clock.cycle_last	= tk->tkr_raw.cycle_last;  in update_pvclock_gtod()
2288 	vdata->raw_clock.mask		= tk->tkr_raw.mask;  in update_pvclock_gtod()
2289 	vdata->raw_clock.mult		= tk->tkr_raw.mult;  in update_pvclock_gtod()
2290 	vdata->raw_clock.shift		= tk->tkr_raw.shift;  in update_pvclock_gtod()
2291 	vdata->raw_clock.base_cycles	= tk->tkr_raw.xtime_nsec;  in update_pvclock_gtod()
2292 	vdata->raw_clock.offset		= tk->tkr_raw.base;  in update_pvclock_gtod()
2294 	vdata->wall_time_sec            = tk->xtime_sec;  in update_pvclock_gtod()
2296 	vdata->offs_boot		= tk->offs_boot;  in update_pvclock_gtod()
2298 	write_seqcount_end(&vdata->seq);  in update_pvclock_gtod()
2342 	wall_nsec = ktime_get_real_ns() - get_kvmclock_ns(kvm);  in kvm_write_wall_clock()
2363 	struct kvm_arch *ka = &vcpu->kvm->arch;  in kvm_write_system_time()
2365 	if (vcpu->vcpu_id == 0 && !host_initiated) {  in kvm_write_system_time()
2366 		if (ka->boot_vcpu_runs_old_kvmclock != old_msr)  in kvm_write_system_time()
2369 		ka->boot_vcpu_runs_old_kvmclock = old_msr;  in kvm_write_system_time()
2372 	vcpu->arch.time = system_time;  in kvm_write_system_time()
2377 		kvm_gpc_activate(&vcpu->arch.pv_time, system_time & ~1ULL,  in kvm_write_system_time()
2380 		kvm_gpc_deactivate(&vcpu->arch.pv_time);  in kvm_write_system_time()
2403 		shift--;  in kvm_get_time_scale()
2448 			vcpu->arch.tsc_catchup = 1;  in set_tsc_khz()
2449 			vcpu->arch.tsc_always_catchup = 1;  in set_tsc_khz()
2453 			return -1;  in set_tsc_khz()
2457 	/* TSC scaling required  - calculate ratio */  in set_tsc_khz()
2462 		pr_warn_ratelimited("Invalid TSC scaling ratio - virtual-tsc-khz=%u\n",  in set_tsc_khz()
2464 		return -1;  in set_tsc_khz()
2480 		return -1;  in kvm_set_tsc_khz()
2485 			   &vcpu->arch.virtual_tsc_shift,  in kvm_set_tsc_khz()
2486 			   &vcpu->arch.virtual_tsc_mult);  in kvm_set_tsc_khz()
2487 	vcpu->arch.virtual_tsc_khz = user_tsc_khz;  in kvm_set_tsc_khz()
2495 	thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm);  in kvm_set_tsc_khz()
2507 	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,  in compute_guest_tsc()
2508 				      vcpu->arch.virtual_tsc_mult,  in compute_guest_tsc()
2509 				      vcpu->arch.virtual_tsc_shift);  in compute_guest_tsc()
2510 	tsc += vcpu->arch.this_tsc_write;  in compute_guest_tsc()
2525 	struct kvm_arch *ka = &vcpu->kvm->arch;  in kvm_track_tsc_matching()
2528 	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==  in kvm_track_tsc_matching()
2529 			 atomic_read(&vcpu->kvm->online_vcpus));  in kvm_track_tsc_matching()
2539 	if (ka->use_master_clock ||  in kvm_track_tsc_matching()
2540 	    (gtod_is_based_on_tsc(gtod->clock.vclock_mode) && vcpus_matched))  in kvm_track_tsc_matching()
2543 	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,  in kvm_track_tsc_matching()
2544 			    atomic_read(&vcpu->kvm->online_vcpus),  in kvm_track_tsc_matching()
2545 		            ka->use_master_clock, gtod->clock.vclock_mode);  in kvm_track_tsc_matching()
2552  * The most significant 64-N bits (mult) of ratio represent the
2555  * point number (mult + frac * 2^(-N)).
2578 	tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio);  in kvm_compute_l1_tsc_offset()
2580 	return target_tsc - tsc;  in kvm_compute_l1_tsc_offset()
2585 	return vcpu->arch.l1_tsc_offset +  in kvm_read_l1_tsc()
2586 		kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio);  in kvm_read_l1_tsc()
2617 	trace_kvm_write_tsc_offset(vcpu->vcpu_id,  in kvm_vcpu_write_tsc_offset()
2618 				   vcpu->arch.l1_tsc_offset,  in kvm_vcpu_write_tsc_offset()
2621 	vcpu->arch.l1_tsc_offset = l1_offset;  in kvm_vcpu_write_tsc_offset()
2629 		vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(  in kvm_vcpu_write_tsc_offset()
2634 		vcpu->arch.tsc_offset = l1_offset;  in kvm_vcpu_write_tsc_offset()
2641 	vcpu->arch.l1_tsc_scaling_ratio = l1_multiplier;  in kvm_vcpu_write_tsc_multiplier()
2645 		vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier(  in kvm_vcpu_write_tsc_multiplier()
2649 		vcpu->arch.tsc_scaling_ratio = l1_multiplier;  in kvm_vcpu_write_tsc_multiplier()
2659 	 * TSC is marked unstable when we're running on Hyper-V,  in kvm_check_tsc_unstable()
2676 	struct kvm *kvm = vcpu->kvm;  in __kvm_synchronize_tsc()
2678 	lockdep_assert_held(&kvm->arch.tsc_write_lock);  in __kvm_synchronize_tsc()
2684 	kvm->arch.last_tsc_nsec = ns;  in __kvm_synchronize_tsc()
2685 	kvm->arch.last_tsc_write = tsc;  in __kvm_synchronize_tsc()
2686 	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;  in __kvm_synchronize_tsc()
2687 	kvm->arch.last_tsc_offset = offset;  in __kvm_synchronize_tsc()
2689 	vcpu->arch.last_guest_tsc = tsc;  in __kvm_synchronize_tsc()
2701 		 * These values are tracked in kvm->arch.cur_xxx variables.  in __kvm_synchronize_tsc()
2703 		kvm->arch.cur_tsc_generation++;  in __kvm_synchronize_tsc()
2704 		kvm->arch.cur_tsc_nsec = ns;  in __kvm_synchronize_tsc()
2705 		kvm->arch.cur_tsc_write = tsc;  in __kvm_synchronize_tsc()
2706 		kvm->arch.cur_tsc_offset = offset;  in __kvm_synchronize_tsc()
2707 		kvm->arch.nr_vcpus_matched_tsc = 0;  in __kvm_synchronize_tsc()
2708 	} else if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) {  in __kvm_synchronize_tsc()
2709 		kvm->arch.nr_vcpus_matched_tsc++;  in __kvm_synchronize_tsc()
2713 	vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;  in __kvm_synchronize_tsc()
2714 	vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;  in __kvm_synchronize_tsc()
2715 	vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;  in __kvm_synchronize_tsc()
2722 	struct kvm *kvm = vcpu->kvm;  in kvm_synchronize_tsc()
2728 	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);  in kvm_synchronize_tsc()
2731 	elapsed = ns - kvm->arch.last_tsc_nsec;  in kvm_synchronize_tsc()
2733 	if (vcpu->arch.virtual_tsc_khz) {  in kvm_synchronize_tsc()
2736 			 * detection of vcpu initialization -- need to sync  in kvm_synchronize_tsc()
2742 			u64 tsc_exp = kvm->arch.last_tsc_write +  in kvm_synchronize_tsc()
2744 			u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;  in kvm_synchronize_tsc()
2762 	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {  in kvm_synchronize_tsc()
2764 			offset = kvm->arch.cur_tsc_offset;  in kvm_synchronize_tsc()
2774 	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);  in kvm_synchronize_tsc()
2780 	u64 tsc_offset = vcpu->arch.l1_tsc_offset;  in adjust_tsc_offset_guest()
2786 	if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio)  in adjust_tsc_offset_host()
2789 				   vcpu->arch.l1_tsc_scaling_ratio);  in adjust_tsc_offset_host()
2821 	switch (clock->vclock_mode) {  in vgettsc()
2827 			v = (tsc_pg_val - clock->cycle_last) &  in vgettsc()
2828 				clock->mask;  in vgettsc()
2837 		v = (*tsc_timestamp - clock->cycle_last) &  in vgettsc()
2838 			clock->mask;  in vgettsc()
2847 	return v * clock->mult;  in vgettsc()
2858 		seq = read_seqcount_begin(&gtod->seq);  in do_monotonic_raw()
2859 		ns = gtod->raw_clock.base_cycles;  in do_monotonic_raw()
2860 		ns += vgettsc(&gtod->raw_clock, tsc_timestamp, &mode);  in do_monotonic_raw()
2861 		ns >>= gtod->raw_clock.shift;  in do_monotonic_raw()
2862 		ns += ktime_to_ns(ktime_add(gtod->raw_clock.offset, gtod->offs_boot));  in do_monotonic_raw()
2863 	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));  in do_monotonic_raw()
2877 		seq = read_seqcount_begin(&gtod->seq);  in do_realtime()
2878 		ts->tv_sec = gtod->wall_time_sec;  in do_realtime()
2879 		ns = gtod->clock.base_cycles;  in do_realtime()
2880 		ns += vgettsc(&gtod->clock, tsc_timestamp, &mode);  in do_realtime()
2881 		ns >>= gtod->clock.shift;  in do_realtime()
2882 	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));  in do_realtime()
2884 	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);  in do_realtime()
2885 	ts->tv_nsec = ns;  in do_realtime()
2929  * 4. ret0 = timespec0 + (rdtsc - tsc0) |
2930  * 5.				        | ret1 = timespec1 + (rdtsc - tsc1)
2931  * 				        | ret1 = timespec0 + N + (rdtsc - (tsc0 + M))
2935  * 	- ret0 < ret1
2936  *	- timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M))
2938  *	- 0 < N - M => M < N
2957 	struct kvm_arch *ka = &kvm->arch;  in pvclock_update_vm_gtod_copy()
2961 	lockdep_assert_held(&kvm->arch.tsc_write_lock);  in pvclock_update_vm_gtod_copy()
2962 	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==  in pvclock_update_vm_gtod_copy()
2963 			atomic_read(&kvm->online_vcpus));  in pvclock_update_vm_gtod_copy()
2970 					&ka->master_kernel_ns,  in pvclock_update_vm_gtod_copy()
2971 					&ka->master_cycle_now);  in pvclock_update_vm_gtod_copy()
2973 	ka->use_master_clock = host_tsc_clocksource && vcpus_matched  in pvclock_update_vm_gtod_copy()
2974 				&& !ka->backwards_tsc_observed  in pvclock_update_vm_gtod_copy()
2975 				&& !ka->boot_vcpu_runs_old_kvmclock;  in pvclock_update_vm_gtod_copy()
2977 	if (ka->use_master_clock)  in pvclock_update_vm_gtod_copy()
2981 	trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,  in pvclock_update_vm_gtod_copy()
2993 	raw_spin_lock_irq(&kvm->arch.tsc_write_lock);  in __kvm_start_pvclock_update()
2994 	write_seqcount_begin(&kvm->arch.pvclock_sc);  in __kvm_start_pvclock_update()
3007 	struct kvm_arch *ka = &kvm->arch;  in kvm_end_pvclock_update()
3011 	write_seqcount_end(&ka->pvclock_sc);  in kvm_end_pvclock_update()
3012 	raw_spin_unlock_irq(&ka->tsc_write_lock);  in kvm_end_pvclock_update()
3031  * per-CPU value (which may be zero if a CPU is going offline).  Note, tsc_khz
3045 /* Called within read_seqcount_begin/retry for kvm->pvclock_sc.  */
3048 	struct kvm_arch *ka = &kvm->arch;  in __get_kvmclock()
3054 	data->flags = 0;  in __get_kvmclock()
3055 	if (ka->use_master_clock &&  in __get_kvmclock()
3060 		if (kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) {  in __get_kvmclock()
3061 			data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec;  in __get_kvmclock()
3062 			data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC;  in __get_kvmclock()
3065 		data->host_tsc = rdtsc();  in __get_kvmclock()
3067 		data->flags |= KVM_CLOCK_TSC_STABLE;  in __get_kvmclock()
3068 		hv_clock.tsc_timestamp = ka->master_cycle_now;  in __get_kvmclock()
3069 		hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;  in __get_kvmclock()
3073 		data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc);  in __get_kvmclock()
3075 		data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset;  in __get_kvmclock()
3083 	struct kvm_arch *ka = &kvm->arch;  in get_kvmclock()
3087 		seq = read_seqcount_begin(&ka->pvclock_sc);  in get_kvmclock()
3089 	} while (read_seqcount_retry(&ka->pvclock_sc, seq));  in get_kvmclock()
3104 	struct kvm_vcpu_arch *vcpu = &v->arch;  in kvm_setup_guest_pvclock()
3108 	read_lock_irqsave(&gpc->lock, flags);  in kvm_setup_guest_pvclock()
3110 		read_unlock_irqrestore(&gpc->lock, flags);  in kvm_setup_guest_pvclock()
3115 		read_lock_irqsave(&gpc->lock, flags);  in kvm_setup_guest_pvclock()
3118 	guest_hv_clock = (void *)(gpc->khva + offset);  in kvm_setup_guest_pvclock()
3127 	guest_hv_clock->version = vcpu->hv_clock.version = (guest_hv_clock->version + 1) | 1;  in kvm_setup_guest_pvclock()
3131 	vcpu->hv_clock.flags |= (guest_hv_clock->flags & PVCLOCK_GUEST_STOPPED);  in kvm_setup_guest_pvclock()
3133 	if (vcpu->pvclock_set_guest_stopped_request) {  in kvm_setup_guest_pvclock()
3134 		vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED;  in kvm_setup_guest_pvclock()
3135 		vcpu->pvclock_set_guest_stopped_request = false;  in kvm_setup_guest_pvclock()
3138 	memcpy(guest_hv_clock, &vcpu->hv_clock, sizeof(*guest_hv_clock));  in kvm_setup_guest_pvclock()
3141 	guest_hv_clock->version = ++vcpu->hv_clock.version;  in kvm_setup_guest_pvclock()
3143 	mark_page_dirty_in_slot(v->kvm, gpc->memslot, gpc->gpa >> PAGE_SHIFT);  in kvm_setup_guest_pvclock()
3144 	read_unlock_irqrestore(&gpc->lock, flags);  in kvm_setup_guest_pvclock()
3146 	trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);  in kvm_setup_guest_pvclock()
3153 	struct kvm_vcpu_arch *vcpu = &v->arch;  in kvm_guest_time_update()
3154 	struct kvm_arch *ka = &v->kvm->arch;  in kvm_guest_time_update()
3168 		seq = read_seqcount_begin(&ka->pvclock_sc);  in kvm_guest_time_update()
3169 		use_master_clock = ka->use_master_clock;  in kvm_guest_time_update()
3171 			host_tsc = ka->master_cycle_now;  in kvm_guest_time_update()
3172 			kernel_ns = ka->master_kernel_ns;  in kvm_guest_time_update()
3174 	} while (read_seqcount_retry(&ka->pvclock_sc, seq));  in kvm_guest_time_update()
3195 	 *   2) Broken TSC compensation resets the base at each VCPU  in kvm_guest_time_update()
3201 	if (vcpu->tsc_catchup) {  in kvm_guest_time_update()
3204 			adjust_tsc_offset_guest(v, tsc - tsc_timestamp);  in kvm_guest_time_update()
3215 					    v->arch.l1_tsc_scaling_ratio);  in kvm_guest_time_update()
3217 	if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {  in kvm_guest_time_update()
3219 				   &vcpu->hv_clock.tsc_shift,  in kvm_guest_time_update()
3220 				   &vcpu->hv_clock.tsc_to_system_mul);  in kvm_guest_time_update()
3221 		vcpu->hw_tsc_khz = tgt_tsc_khz;  in kvm_guest_time_update()
3225 	vcpu->hv_clock.tsc_timestamp = tsc_timestamp;  in kvm_guest_time_update()
3226 	vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;  in kvm_guest_time_update()
3227 	vcpu->last_guest_tsc = tsc_timestamp;  in kvm_guest_time_update()
3234 	vcpu->hv_clock.flags = pvclock_flags;  in kvm_guest_time_update()
3236 	if (vcpu->pv_time.active)  in kvm_guest_time_update()
3237 		kvm_setup_guest_pvclock(v, &vcpu->pv_time, 0);  in kvm_guest_time_update()
3238 	if (vcpu->xen.vcpu_info_cache.active)  in kvm_guest_time_update()
3239 		kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_info_cache,  in kvm_guest_time_update()
3241 	if (vcpu->xen.vcpu_time_info_cache.active)  in kvm_guest_time_update()
3242 		kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_time_info_cache, 0);  in kvm_guest_time_update()
3243 	kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);  in kvm_guest_time_update()
3249  * vcpu->cpu migration, should not allow system_timestamp from
3255  * We need to rate-limit these requests though, as they can
3258  * by the delay we use to rate-limit the updates.
3280 	struct kvm *kvm = v->kvm;  in kvm_gen_kvmclock_update()
3283 	schedule_delayed_work(&kvm->arch.kvmclock_update_work,  in kvm_gen_kvmclock_update()
3299 	schedule_delayed_work(&kvm->arch.kvmclock_update_work, 0);  in kvmclock_sync_fn()
3300 	schedule_delayed_work(&kvm->arch.kvmclock_sync_work,  in kvmclock_sync_fn()
3321 		return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));  in can_set_mci_status()
3328 	u64 mcg_cap = vcpu->arch.mcg_cap;  in set_msr_mce()
3330 	u32 msr = msr_info->index;  in set_msr_mce()
3331 	u64 data = msr_info->data;  in set_msr_mce()
3336 		vcpu->arch.mcg_status = data;  in set_msr_mce()
3340 		    (data || !msr_info->host_initiated))  in set_msr_mce()
3344 		vcpu->arch.mcg_ctl = data;  in set_msr_mce()
3346 	case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:  in set_msr_mce()
3347 		last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1;  in set_msr_mce()
3351 		if (!(mcg_cap & MCG_CMCI_P) && (data || !msr_info->host_initiated))  in set_msr_mce()
3356 		offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2,  in set_msr_mce()
3357 					    last_msr + 1 - MSR_IA32_MC0_CTL2);  in set_msr_mce()
3358 		vcpu->arch.mci_ctl2_banks[offset] = data;  in set_msr_mce()
3360 	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:  in set_msr_mce()
3361 		last_msr = MSR_IA32_MCx_CTL(bank_num) - 1;  in set_msr_mce()
3373 		 * single-bit ECC data errors.  in set_msr_mce()
3381 		 * AMD-based CPUs allow non-zero values, but if and only if  in set_msr_mce()
3384 		if (!msr_info->host_initiated && is_mci_status_msr(msr) &&  in set_msr_mce()
3388 		offset = array_index_nospec(msr - MSR_IA32_MC0_CTL,  in set_msr_mce()
3389 					    last_msr + 1 - MSR_IA32_MC0_CTL);  in set_msr_mce()
3390 		vcpu->arch.mce_banks[offset] = data;  in set_msr_mce()
3402 	return (vcpu->arch.apf.msr_en_val & mask) == mask;  in kvm_pv_async_pf_enabled()
3424 	vcpu->arch.apf.msr_en_val = data;  in kvm_pv_enable_async_pf()
3432 	if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,  in kvm_pv_enable_async_pf()
3436 	vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);  in kvm_pv_enable_async_pf()
3437 	vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;  in kvm_pv_enable_async_pf()
3446 	/* Bits 8-63 are reserved */  in kvm_pv_enable_async_pf_int()
3453 	vcpu->arch.apf.msr_int_val = data;  in kvm_pv_enable_async_pf_int()
3455 	vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK;  in kvm_pv_enable_async_pf_int()
3462 	kvm_gpc_deactivate(&vcpu->arch.pv_time);  in kvmclock_reset()
3463 	vcpu->arch.time = 0;  in kvmclock_reset()
3468 	++vcpu->stat.tlb_flush;  in kvm_vcpu_flush_tlb_all()
3477 	++vcpu->stat.tlb_flush;  in kvm_vcpu_flush_tlb_guest()
3493 	 * Flushing all "guest" TLB is always a superset of Hyper-V's fine  in kvm_vcpu_flush_tlb_guest()
3502 	++vcpu->stat.tlb_flush;  in kvm_vcpu_flush_tlb_current()
3510  * prior before nested VM-Enter/VM-Exit.
3524 	struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;  in record_steal_time()
3527 	gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;  in record_steal_time()
3531 	if (kvm_xen_msr_enabled(vcpu->kvm)) {  in record_steal_time()
3536 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))  in record_steal_time()
3539 	if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm))  in record_steal_time()
3542 	slots = kvm_memslots(vcpu->kvm);  in record_steal_time()
3544 	if (unlikely(slots->generation != ghc->generation ||  in record_steal_time()
3545 		     gpa != ghc->gpa ||  in record_steal_time()
3546 		     kvm_is_error_hva(ghc->hva) || !ghc->memslot)) {  in record_steal_time()
3548 		BUILD_BUG_ON((sizeof(*st) - 1) & KVM_STEAL_VALID_BITS);  in record_steal_time()
3550 		if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st)) ||  in record_steal_time()
3551 		    kvm_is_error_hva(ghc->hva) || !ghc->memslot)  in record_steal_time()
3555 	st = (struct kvm_steal_time __user *)ghc->hva;  in record_steal_time()
3562 		int err = -EFAULT;  in record_steal_time()
3573 			       "+m" (st->preempted));  in record_steal_time()
3579 		vcpu->arch.st.preempted = 0;  in record_steal_time()
3581 		trace_kvm_pv_tlb_flush(vcpu->vcpu_id,  in record_steal_time()
3592 		unsafe_put_user(0, &st->preempted, out);  in record_steal_time()
3593 		vcpu->arch.st.preempted = 0;  in record_steal_time()
3596 	unsafe_get_user(version, &st->version, out);  in record_steal_time()
3601 	unsafe_put_user(version, &st->version, out);  in record_steal_time()
3605 	unsafe_get_user(steal, &st->steal, out);  in record_steal_time()
3606 	steal += current->sched_info.run_delay -  in record_steal_time()
3607 		vcpu->arch.st.last_steal;  in record_steal_time()
3608 	vcpu->arch.st.last_steal = current->sched_info.run_delay;  in record_steal_time()
3609 	unsafe_put_user(steal, &st->steal, out);  in record_steal_time()
3612 	unsafe_put_user(version, &st->version, out);  in record_steal_time()
3617 	mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));  in record_steal_time()
3634 	u32 msr = msr_info->index;  in kvm_set_msr_common()
3635 	u64 data = msr_info->data;  in kvm_set_msr_common()
3637 	if (msr && msr == vcpu->kvm->arch.xen_hvm_config.msr)  in kvm_set_msr_common()
3652 		if (msr_info->host_initiated)  in kvm_set_msr_common()
3653 			vcpu->arch.microcode_version = data;  in kvm_set_msr_common()
3656 		if (!msr_info->host_initiated)  in kvm_set_msr_common()
3658 		vcpu->arch.arch_capabilities = data;  in kvm_set_msr_common()
3661 		if (!msr_info->host_initiated)  in kvm_set_msr_common()
3671 		if (vcpu->arch.perf_capabilities == data)  in kvm_set_msr_common()
3674 		vcpu->arch.perf_capabilities = data;  in kvm_set_msr_common()
3678 		if (!msr_info->host_initiated && !guest_has_pred_cmd_msr(vcpu))  in kvm_set_msr_common()
3689 		if (!msr_info->host_initiated &&  in kvm_set_msr_common()
3709 			vcpu->arch.msr_hwcr = data;  in kvm_set_msr_common()
3725 		vcpu->arch.pat = data;  in kvm_set_msr_common()
3739 			if (!msr_info->host_initiated) {  in kvm_set_msr_common()
3740 				s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;  in kvm_set_msr_common()
3747 			vcpu->arch.ia32_tsc_adjust_msr = data;  in kvm_set_msr_common()
3751 		u64 old_val = vcpu->arch.ia32_misc_enable_msr;  in kvm_set_msr_common()
3753 		if (!msr_info->host_initiated) {  in kvm_set_msr_common()
3763 		if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&  in kvm_set_msr_common()
3767 			vcpu->arch.ia32_misc_enable_msr = data;  in kvm_set_msr_common()
3770 			vcpu->arch.ia32_misc_enable_msr = data;  in kvm_set_msr_common()
3775 		if (!IS_ENABLED(CONFIG_KVM_SMM) || !msr_info->host_initiated)  in kvm_set_msr_common()
3777 		vcpu->arch.smbase = data;  in kvm_set_msr_common()
3780 		vcpu->arch.msr_ia32_power_ctl = data;  in kvm_set_msr_common()
3783 		if (msr_info->host_initiated) {  in kvm_set_msr_common()
3786 			u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset;  in kvm_set_msr_common()
3788 			vcpu->arch.ia32_tsc_adjust_msr += adj;  in kvm_set_msr_common()
3792 		if (!msr_info->host_initiated &&  in kvm_set_msr_common()
3802 		vcpu->arch.ia32_xss = data;  in kvm_set_msr_common()
3806 		if (!msr_info->host_initiated)  in kvm_set_msr_common()
3808 		vcpu->arch.smi_count = data;  in kvm_set_msr_common()
3814 		vcpu->kvm->arch.wall_clock = data;  in kvm_set_msr_common()
3815 		kvm_write_wall_clock(vcpu->kvm, data, 0);  in kvm_set_msr_common()
3821 		vcpu->kvm->arch.wall_clock = data;  in kvm_set_msr_common()
3822 		kvm_write_wall_clock(vcpu->kvm, data, 0);  in kvm_set_msr_common()
3828 		kvm_write_system_time(vcpu, data, false, msr_info->host_initiated);  in kvm_set_msr_common()
3834 		kvm_write_system_time(vcpu, data, true,  msr_info->host_initiated);  in kvm_set_msr_common()
3854 			vcpu->arch.apf.pageready_pending = false;  in kvm_set_msr_common()
3868 		vcpu->arch.st.msr_val = data;  in kvm_set_msr_common()
3889 		if (data & (-1ULL << 1))  in kvm_set_msr_common()
3892 		vcpu->arch.msr_kvm_poll_control = data;  in kvm_set_msr_common()
3897 	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:  in kvm_set_msr_common()
3898 	case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:  in kvm_set_msr_common()
3915 		 * all pre-dating SVM, but a recommended workaround from  in kvm_set_msr_common()
3932 					     msr_info->host_initiated);  in kvm_set_msr_common()
3934 		/* Drop writes to this legacy MSR -- see rdmsr  in kvm_set_msr_common()
3942 		vcpu->arch.osvw.length = data;  in kvm_set_msr_common()
3947 		vcpu->arch.osvw.status = data;  in kvm_set_msr_common()
3950 		if (!msr_info->host_initiated ||  in kvm_set_msr_common()
3954 		vcpu->arch.msr_platform_info = data;  in kvm_set_msr_common()
3961 		vcpu->arch.msr_misc_features_enables = data;  in kvm_set_msr_common()
3965 		if (!msr_info->host_initiated &&  in kvm_set_msr_common()
3972 		fpu_update_guest_xfd(&vcpu->arch.guest_fpu, data);  in kvm_set_msr_common()
3975 		if (!msr_info->host_initiated &&  in kvm_set_msr_common()
3982 		vcpu->arch.guest_fpu.xfd_err = data;  in kvm_set_msr_common()
3991 		 * as to-be-saved, even if an MSRs isn't fully supported.  in kvm_set_msr_common()
3993 		if (msr_info->host_initiated && !data &&  in kvm_set_msr_common()
4006 	u64 mcg_cap = vcpu->arch.mcg_cap;  in get_msr_mce()
4016 		data = vcpu->arch.mcg_cap;  in get_msr_mce()
4021 		data = vcpu->arch.mcg_ctl;  in get_msr_mce()
4024 		data = vcpu->arch.mcg_status;  in get_msr_mce()
4026 	case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:  in get_msr_mce()
4027 		last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1;  in get_msr_mce()
4033 		offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2,  in get_msr_mce()
4034 					    last_msr + 1 - MSR_IA32_MC0_CTL2);  in get_msr_mce()
4035 		data = vcpu->arch.mci_ctl2_banks[offset];  in get_msr_mce()
4037 	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:  in get_msr_mce()
4038 		last_msr = MSR_IA32_MCx_CTL(bank_num) - 1;  in get_msr_mce()
4042 		offset = array_index_nospec(msr - MSR_IA32_MC0_CTL,  in get_msr_mce()
4043 					    last_msr + 1 - MSR_IA32_MC0_CTL);  in get_msr_mce()
4044 		data = vcpu->arch.mce_banks[offset];  in get_msr_mce()
4055 	switch (msr_info->index) {  in kvm_get_msr_common()
4078 	 * so for existing CPU-specific MSRs.  in kvm_get_msr_common()
4085 		msr_info->data = 0;  in kvm_get_msr_common()
4091 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))  in kvm_get_msr_common()
4093 		msr_info->data = 0;  in kvm_get_msr_common()
4096 		msr_info->data = vcpu->arch.microcode_version;  in kvm_get_msr_common()
4099 		if (!msr_info->host_initiated &&  in kvm_get_msr_common()
4102 		msr_info->data = vcpu->arch.arch_capabilities;  in kvm_get_msr_common()
4105 		if (!msr_info->host_initiated &&  in kvm_get_msr_common()
4108 		msr_info->data = vcpu->arch.perf_capabilities;  in kvm_get_msr_common()
4111 		msr_info->data = vcpu->arch.msr_ia32_power_ctl;  in kvm_get_msr_common()
4120 		 * return L1's TSC value to ensure backwards-compatible  in kvm_get_msr_common()
4125 		if (msr_info->host_initiated) {  in kvm_get_msr_common()
4126 			offset = vcpu->arch.l1_tsc_offset;  in kvm_get_msr_common()
4127 			ratio = vcpu->arch.l1_tsc_scaling_ratio;  in kvm_get_msr_common()
4129 			offset = vcpu->arch.tsc_offset;  in kvm_get_msr_common()
4130 			ratio = vcpu->arch.tsc_scaling_ratio;  in kvm_get_msr_common()
4133 		msr_info->data = kvm_scale_tsc(rdtsc(), ratio) + offset;  in kvm_get_msr_common()
4137 		msr_info->data = vcpu->arch.pat;  in kvm_get_msr_common()
4142 		return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);  in kvm_get_msr_common()
4144 		msr_info->data = 3;  in kvm_get_msr_common()
4158 		msr_info->data = 1 << 24;  in kvm_get_msr_common()
4161 		msr_info->data = kvm_get_apic_base(vcpu);  in kvm_get_msr_common()
4164 		return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);  in kvm_get_msr_common()
4166 		msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);  in kvm_get_msr_common()
4169 		msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;  in kvm_get_msr_common()
4172 		msr_info->data = vcpu->arch.ia32_misc_enable_msr;  in kvm_get_msr_common()
4175 		if (!IS_ENABLED(CONFIG_KVM_SMM) || !msr_info->host_initiated)  in kvm_get_msr_common()
4177 		msr_info->data = vcpu->arch.smbase;  in kvm_get_msr_common()
4180 		msr_info->data = vcpu->arch.smi_count;  in kvm_get_msr_common()
4184 		msr_info->data = 1000ULL;  in kvm_get_msr_common()
4186 		msr_info->data |= (((uint64_t)4ULL) << 40);  in kvm_get_msr_common()
4189 		msr_info->data = vcpu->arch.efer;  in kvm_get_msr_common()
4195 		msr_info->data = vcpu->kvm->arch.wall_clock;  in kvm_get_msr_common()
4201 		msr_info->data = vcpu->kvm->arch.wall_clock;  in kvm_get_msr_common()
4207 		msr_info->data = vcpu->arch.time;  in kvm_get_msr_common()
4213 		msr_info->data = vcpu->arch.time;  in kvm_get_msr_common()
4219 		msr_info->data = vcpu->arch.apf.msr_en_val;  in kvm_get_msr_common()
4225 		msr_info->data = vcpu->arch.apf.msr_int_val;  in kvm_get_msr_common()
4231 		msr_info->data = 0;  in kvm_get_msr_common()
4237 		msr_info->data = vcpu->arch.st.msr_val;  in kvm_get_msr_common()
4243 		msr_info->data = vcpu->arch.pv_eoi.msr_val;  in kvm_get_msr_common()
4249 		msr_info->data = vcpu->arch.msr_kvm_poll_control;  in kvm_get_msr_common()
4256 	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:  in kvm_get_msr_common()
4257 	case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:  in kvm_get_msr_common()
4258 		return get_msr_mce(vcpu, msr_info->index, &msr_info->data,  in kvm_get_msr_common()
4259 				   msr_info->host_initiated);  in kvm_get_msr_common()
4261 		if (!msr_info->host_initiated &&  in kvm_get_msr_common()
4264 		msr_info->data = vcpu->arch.ia32_xss;  in kvm_get_msr_common()
4268 		 * Provide expected ramp-up count for K7. All other  in kvm_get_msr_common()
4276 		msr_info->data = 0x20000000;  in kvm_get_msr_common()
4289 					     msr_info->index, &msr_info->data,  in kvm_get_msr_common()
4290 					     msr_info->host_initiated);  in kvm_get_msr_common()
4302 		msr_info->data = 0xbe702111;  in kvm_get_msr_common()
4307 		msr_info->data = vcpu->arch.osvw.length;  in kvm_get_msr_common()
4312 		msr_info->data = vcpu->arch.osvw.status;  in kvm_get_msr_common()
4315 		if (!msr_info->host_initiated &&  in kvm_get_msr_common()
4316 		    !vcpu->kvm->arch.guest_can_read_msr_platform_info)  in kvm_get_msr_common()
4318 		msr_info->data = vcpu->arch.msr_platform_info;  in kvm_get_msr_common()
4321 		msr_info->data = vcpu->arch.msr_misc_features_enables;  in kvm_get_msr_common()
4324 		msr_info->data = vcpu->arch.msr_hwcr;  in kvm_get_msr_common()
4328 		if (!msr_info->host_initiated &&  in kvm_get_msr_common()
4332 		msr_info->data = vcpu->arch.guest_fpu.fpstate->xfd;  in kvm_get_msr_common()
4335 		if (!msr_info->host_initiated &&  in kvm_get_msr_common()
4339 		msr_info->data = vcpu->arch.guest_fpu.xfd_err;  in kvm_get_msr_common()
4343 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))  in kvm_get_msr_common()
4348 		 * to-be-saved, even if an MSR isn't fully supported.  in kvm_get_msr_common()
4350 		if (msr_info->host_initiated &&  in kvm_get_msr_common()
4351 		    kvm_is_msr_to_save(msr_info->index)) {  in kvm_get_msr_common()
4352 			msr_info->data = 0;  in kvm_get_msr_common()
4374 	for (i = 0; i < msrs->nmsrs; ++i)  in __msr_io()
4396 	r = -EFAULT;  in msr_io()
4400 	r = -E2BIG;  in msr_io()
4405 	entries = memdup_user(user_msrs->entries, size);  in msr_io()
4413 	if (writeback && copy_to_user(user_msrs->entries, entries, size))  in msr_io()
4414 		r = -EFAULT;  in msr_io()
4434 	r = -EFAULT;  in kvm_ioctl_get_supported_hv_cpuid()
4438 	r = kvm_get_hv_cpuid(vcpu, &cpuid, cpuid_arg->entries);  in kvm_ioctl_get_supported_hv_cpuid()
4442 	r = -EFAULT;  in kvm_ioctl_get_supported_hv_cpuid()
4602 		r = kvm_x86_ops.nested_ops->get_state ?  in kvm_vm_ioctl_check_extension()
4603 			kvm_x86_ops.nested_ops->get_state(NULL, NULL, 0) : 0;  in kvm_vm_ioctl_check_extension()
4609 		r = kvm_x86_ops.nested_ops->enable_evmcs != NULL;  in kvm_vm_ioctl_check_extension()
4647 	void __user *uaddr = (void __user*)(unsigned long)attr->addr;  in kvm_get_attr_addr()
4649 	if ((u64)(unsigned long)uaddr != attr->addr)  in kvm_get_attr_addr()
4650 		return ERR_PTR_USR(-EFAULT);  in kvm_get_attr_addr()
4658 	if (attr->group)  in kvm_x86_dev_get_attr()
4659 		return -ENXIO;  in kvm_x86_dev_get_attr()
4664 	switch (attr->attr) {  in kvm_x86_dev_get_attr()
4667 			return -EFAULT;  in kvm_x86_dev_get_attr()
4670 		return -ENXIO;  in kvm_x86_dev_get_attr()
4676 	if (attr->group)  in kvm_x86_dev_has_attr()
4677 		return -ENXIO;  in kvm_x86_dev_has_attr()
4679 	switch (attr->attr) {  in kvm_x86_dev_has_attr()
4683 		return -ENXIO;  in kvm_x86_dev_has_attr()
4699 		r = -EFAULT;  in kvm_arch_dev_ioctl()
4706 		r = -E2BIG;  in kvm_arch_dev_ioctl()
4709 		r = -EFAULT;  in kvm_arch_dev_ioctl()
4710 		if (copy_to_user(user_msr_list->indices, &msrs_to_save,  in kvm_arch_dev_ioctl()
4713 		if (copy_to_user(user_msr_list->indices + num_msrs_to_save,  in kvm_arch_dev_ioctl()
4725 		r = -EFAULT;  in kvm_arch_dev_ioctl()
4729 		r = kvm_dev_ioctl_get_cpuid(&cpuid, cpuid_arg->entries,  in kvm_arch_dev_ioctl()
4734 		r = -EFAULT;  in kvm_arch_dev_ioctl()
4741 		r = -EFAULT;  in kvm_arch_dev_ioctl()
4752 		r = -EFAULT;  in kvm_arch_dev_ioctl()
4759 		r = -E2BIG;  in kvm_arch_dev_ioctl()
4762 		r = -EFAULT;  in kvm_arch_dev_ioctl()
4763 		if (copy_to_user(user_msr_list->indices, &msr_based_features,  in kvm_arch_dev_ioctl()
4777 		r = -EFAULT;  in kvm_arch_dev_ioctl()
4785 		r = -EFAULT;  in kvm_arch_dev_ioctl()
4792 		r = -EINVAL;  in kvm_arch_dev_ioctl()
4806 	return kvm_arch_has_noncoherent_dma(vcpu->kvm);  in need_emulate_wbinvd()
4814 			cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);  in kvm_arch_vcpu_load()
4815 		else if (vcpu->cpu != -1 && vcpu->cpu != cpu)  in kvm_arch_vcpu_load()
4816 			smp_call_function_single(vcpu->cpu,  in kvm_arch_vcpu_load()
4823 	vcpu->arch.host_pkru = read_pkru();  in kvm_arch_vcpu_load()
4826 	if (unlikely(vcpu->arch.tsc_offset_adjustment)) {  in kvm_arch_vcpu_load()
4827 		adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);  in kvm_arch_vcpu_load()
4828 		vcpu->arch.tsc_offset_adjustment = 0;  in kvm_arch_vcpu_load()
4832 	if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {  in kvm_arch_vcpu_load()
4833 		s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :  in kvm_arch_vcpu_load()
4834 				rdtsc() - vcpu->arch.last_host_tsc;  in kvm_arch_vcpu_load()
4840 						vcpu->arch.last_guest_tsc);  in kvm_arch_vcpu_load()
4842 			vcpu->arch.tsc_catchup = 1;  in kvm_arch_vcpu_load()
4850 		 * kvmclock on vcpu->cpu migration  in kvm_arch_vcpu_load()
4852 		if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)  in kvm_arch_vcpu_load()
4854 		if (vcpu->cpu != cpu)  in kvm_arch_vcpu_load()
4856 		vcpu->cpu = cpu;  in kvm_arch_vcpu_load()
4864 	struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;  in kvm_steal_time_set_preempted()
4868 	gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;  in kvm_steal_time_set_preempted()
4871 	 * The vCPU can be marked preempted if and only if the VM-Exit was on  in kvm_steal_time_set_preempted()
4875 	 * preempted if and only if the VM-Exit was due to a host interrupt.  in kvm_steal_time_set_preempted()
4877 	if (!vcpu->arch.at_instruction_boundary) {  in kvm_steal_time_set_preempted()
4878 		vcpu->stat.preemption_other++;  in kvm_steal_time_set_preempted()
4882 	vcpu->stat.preemption_reported++;  in kvm_steal_time_set_preempted()
4883 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))  in kvm_steal_time_set_preempted()
4886 	if (vcpu->arch.st.preempted)  in kvm_steal_time_set_preempted()
4890 	if (unlikely(current->mm != vcpu->kvm->mm))  in kvm_steal_time_set_preempted()
4893 	slots = kvm_memslots(vcpu->kvm);  in kvm_steal_time_set_preempted()
4895 	if (unlikely(slots->generation != ghc->generation ||  in kvm_steal_time_set_preempted()
4896 		     gpa != ghc->gpa ||  in kvm_steal_time_set_preempted()
4897 		     kvm_is_error_hva(ghc->hva) || !ghc->memslot))  in kvm_steal_time_set_preempted()
4900 	st = (struct kvm_steal_time __user *)ghc->hva;  in kvm_steal_time_set_preempted()
4901 	BUILD_BUG_ON(sizeof(st->preempted) != sizeof(preempted));  in kvm_steal_time_set_preempted()
4903 	if (!copy_to_user_nofault(&st->preempted, &preempted, sizeof(preempted)))  in kvm_steal_time_set_preempted()
4904 		vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;  in kvm_steal_time_set_preempted()
4906 	mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));  in kvm_steal_time_set_preempted()
4913 	if (vcpu->preempted) {  in kvm_arch_vcpu_put()
4914 		if (!vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_put()
4915 			vcpu->arch.preempted_in_kernel = !static_call(kvm_x86_get_cpl)(vcpu);  in kvm_arch_vcpu_put()
4921 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_put()
4922 		if (kvm_xen_msr_enabled(vcpu->kvm))  in kvm_arch_vcpu_put()
4926 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_put()
4930 	vcpu->arch.last_host_tsc = rdtsc();  in kvm_arch_vcpu_put()
4977 	 * instruction boundary and with no events half-injected.  in kvm_vcpu_ready_for_interrupt_injection()
4988 	if (irq->irq >= KVM_NR_INTERRUPTS)  in kvm_vcpu_ioctl_interrupt()
4989 		return -EINVAL;  in kvm_vcpu_ioctl_interrupt()
4991 	if (!irqchip_in_kernel(vcpu->kvm)) {  in kvm_vcpu_ioctl_interrupt()
4992 		kvm_queue_interrupt(vcpu, irq->irq, false);  in kvm_vcpu_ioctl_interrupt()
4998 	 * With in-kernel LAPIC, we only use this to inject EXTINT, so  in kvm_vcpu_ioctl_interrupt()
4999 	 * fail for in-kernel 8259.  in kvm_vcpu_ioctl_interrupt()
5001 	if (pic_in_kernel(vcpu->kvm))  in kvm_vcpu_ioctl_interrupt()
5002 		return -ENXIO;  in kvm_vcpu_ioctl_interrupt()
5004 	if (vcpu->arch.pending_external_vector != -1)  in kvm_vcpu_ioctl_interrupt()
5005 		return -EEXIST;  in kvm_vcpu_ioctl_interrupt()
5007 	vcpu->arch.pending_external_vector = irq->irq;  in kvm_vcpu_ioctl_interrupt()
5022 	if (tac->flags)  in vcpu_ioctl_tpr_access_reporting()
5023 		return -EINVAL;  in vcpu_ioctl_tpr_access_reporting()
5024 	vcpu->arch.tpr_access_reporting = !!tac->enabled;  in vcpu_ioctl_tpr_access_reporting()
5034 	r = -EINVAL;  in kvm_vcpu_ioctl_x86_setup_mce()
5040 	vcpu->arch.mcg_cap = mcg_cap;  in kvm_vcpu_ioctl_x86_setup_mce()
5043 		vcpu->arch.mcg_ctl = ~(u64)0;  in kvm_vcpu_ioctl_x86_setup_mce()
5046 		vcpu->arch.mce_banks[bank*4] = ~(u64)0;  in kvm_vcpu_ioctl_x86_setup_mce()
5048 			vcpu->arch.mci_ctl2_banks[bank] = 0;  in kvm_vcpu_ioctl_x86_setup_mce()
5061  * - none of the bits for Machine Check Exceptions are set
5062  * - both the VAL (valid) and UC (uncorrectable) bits are set
5063  * MCI_STATUS_PCC - Processor Context Corrupted
5064  * MCI_STATUS_S - Signaled as a Machine Check Exception
5065  * MCI_STATUS_AR - Software recoverable Action Required
5069 	return	!mce->mcg_status &&  in is_ucna()
5070 		!(mce->status & (MCI_STATUS_PCC | MCI_STATUS_S | MCI_STATUS_AR)) &&  in is_ucna()
5071 		(mce->status & MCI_STATUS_VAL) &&  in is_ucna()
5072 		(mce->status & MCI_STATUS_UC);  in is_ucna()
5077 	u64 mcg_cap = vcpu->arch.mcg_cap;  in kvm_vcpu_x86_set_ucna()
5079 	banks[1] = mce->status;  in kvm_vcpu_x86_set_ucna()
5080 	banks[2] = mce->addr;  in kvm_vcpu_x86_set_ucna()
5081 	banks[3] = mce->misc;  in kvm_vcpu_x86_set_ucna()
5082 	vcpu->arch.mcg_status = mce->mcg_status;  in kvm_vcpu_x86_set_ucna()
5085 	    !(vcpu->arch.mci_ctl2_banks[mce->bank] & MCI_CTL2_CMCI_EN))  in kvm_vcpu_x86_set_ucna()
5089 		kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTCMCI);  in kvm_vcpu_x86_set_ucna()
5097 	u64 mcg_cap = vcpu->arch.mcg_cap;  in kvm_vcpu_ioctl_x86_set_mce()
5099 	u64 *banks = vcpu->arch.mce_banks;  in kvm_vcpu_ioctl_x86_set_mce()
5101 	if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))  in kvm_vcpu_ioctl_x86_set_mce()
5102 		return -EINVAL;  in kvm_vcpu_ioctl_x86_set_mce()
5104 	banks += array_index_nospec(4 * mce->bank, 4 * bank_num);  in kvm_vcpu_ioctl_x86_set_mce()
5113 	if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&  in kvm_vcpu_ioctl_x86_set_mce()
5114 	    vcpu->arch.mcg_ctl != ~(u64)0)  in kvm_vcpu_ioctl_x86_set_mce()
5120 	if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)  in kvm_vcpu_ioctl_x86_set_mce()
5122 	if (mce->status & MCI_STATUS_UC) {  in kvm_vcpu_ioctl_x86_set_mce()
5123 		if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||  in kvm_vcpu_ioctl_x86_set_mce()
5129 			mce->status |= MCI_STATUS_OVER;  in kvm_vcpu_ioctl_x86_set_mce()
5130 		banks[2] = mce->addr;  in kvm_vcpu_ioctl_x86_set_mce()
5131 		banks[3] = mce->misc;  in kvm_vcpu_ioctl_x86_set_mce()
5132 		vcpu->arch.mcg_status = mce->mcg_status;  in kvm_vcpu_ioctl_x86_set_mce()
5133 		banks[1] = mce->status;  in kvm_vcpu_ioctl_x86_set_mce()
5138 			mce->status |= MCI_STATUS_OVER;  in kvm_vcpu_ioctl_x86_set_mce()
5139 		banks[2] = mce->addr;  in kvm_vcpu_ioctl_x86_set_mce()
5140 		banks[3] = mce->misc;  in kvm_vcpu_ioctl_x86_set_mce()
5141 		banks[1] = mce->status;  in kvm_vcpu_ioctl_x86_set_mce()
5162 	 * non-exiting _injected_ exception, and a pending exiting exception.  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5163 	 * In that case, ignore the VM-Exiting exception as it's an extension  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5166 	if (vcpu->arch.exception_vmexit.pending &&  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5167 	    !vcpu->arch.exception.pending &&  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5168 	    !vcpu->arch.exception.injected)  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5169 		ex = &vcpu->arch.exception_vmexit;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5171 		ex = &vcpu->arch.exception;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5176 	 * intercepts #PF, ditto for DR6 and #DBs.  If the per-VM capability,  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5181 	if (!vcpu->kvm->arch.exception_payload_enabled &&  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5182 	    ex->pending && ex->has_payload)  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5193 	if (!kvm_exception_is_soft(ex->vector)) {  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5194 		events->exception.injected = ex->injected;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5195 		events->exception.pending = ex->pending;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5201 		if (!vcpu->kvm->arch.exception_payload_enabled)  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5202 			events->exception.injected |= ex->pending;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5204 	events->exception.nr = ex->vector;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5205 	events->exception.has_error_code = ex->has_error_code;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5206 	events->exception.error_code = ex->error_code;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5207 	events->exception_has_payload = ex->has_payload;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5208 	events->exception_payload = ex->payload;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5210 	events->interrupt.injected =  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5211 		vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5212 	events->interrupt.nr = vcpu->arch.interrupt.nr;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5213 	events->interrupt.shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5215 	events->nmi.injected = vcpu->arch.nmi_injected;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5216 	events->nmi.pending = kvm_get_nr_pending_nmis(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5217 	events->nmi.masked = static_call(kvm_x86_get_nmi_mask)(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5219 	/* events->sipi_vector is never valid when reporting to user space */  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5222 	events->smi.smm = is_smm(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5223 	events->smi.pending = vcpu->arch.smi_pending;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5224 	events->smi.smm_inside_nmi =  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5225 		!!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5227 	events->smi.latched_init = kvm_lapic_latched_init(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5229 	events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5232 	if (vcpu->kvm->arch.exception_payload_enabled)  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5233 		events->flags |= KVM_VCPUEVENT_VALID_PAYLOAD;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5234 	if (vcpu->kvm->arch.triple_fault_event) {  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5235 		events->triple_fault.pending = kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5236 		events->flags |= KVM_VCPUEVENT_VALID_TRIPLE_FAULT;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5243 	if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5249 		return -EINVAL;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5251 	if (events->flags & KVM_VCPUEVENT_VALID_PAYLOAD) {  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5252 		if (!vcpu->kvm->arch.exception_payload_enabled)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5253 			return -EINVAL;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5254 		if (events->exception.pending)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5255 			events->exception.injected = 0;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5257 			events->exception_has_payload = 0;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5259 		events->exception.pending = 0;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5260 		events->exception_has_payload = 0;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5263 	if ((events->exception.injected || events->exception.pending) &&  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5264 	    (events->exception.nr > 31 || events->exception.nr == NMI_VECTOR))  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5265 		return -EINVAL;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5268 	if (events->flags & KVM_VCPUEVENT_VALID_SMM &&  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5269 	    (events->smi.smm || events->smi.pending) &&  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5270 	    vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5271 		return -EINVAL;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5277 	 * morph the exception to a VM-Exit if appropriate.  Do this only for  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5278 	 * pending exceptions, already-injected exceptions are not subject to  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5281 	 * pending exception, which in turn may cause a spurious VM-Exit.  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5283 	vcpu->arch.exception_from_userspace = events->exception.pending;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5285 	vcpu->arch.exception_vmexit.pending = false;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5287 	vcpu->arch.exception.injected = events->exception.injected;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5288 	vcpu->arch.exception.pending = events->exception.pending;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5289 	vcpu->arch.exception.vector = events->exception.nr;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5290 	vcpu->arch.exception.has_error_code = events->exception.has_error_code;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5291 	vcpu->arch.exception.error_code = events->exception.error_code;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5292 	vcpu->arch.exception.has_payload = events->exception_has_payload;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5293 	vcpu->arch.exception.payload = events->exception_payload;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5295 	vcpu->arch.interrupt.injected = events->interrupt.injected;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5296 	vcpu->arch.interrupt.nr = events->interrupt.nr;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5297 	vcpu->arch.interrupt.soft = events->interrupt.soft;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5298 	if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5300 						events->interrupt.shadow);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5302 	vcpu->arch.nmi_injected = events->nmi.injected;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5303 	if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING) {  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5304 		vcpu->arch.nmi_pending = 0;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5305 		atomic_set(&vcpu->arch.nmi_queued, events->nmi.pending);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5306 		if (events->nmi.pending)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5309 	static_call(kvm_x86_set_nmi_mask)(vcpu, events->nmi.masked);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5311 	if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR &&  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5313 		vcpu->arch.apic->sipi_vector = events->sipi_vector;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5315 	if (events->flags & KVM_VCPUEVENT_VALID_SMM) {  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5317 		if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5319 			kvm_smm_changed(vcpu, events->smi.smm);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5322 		vcpu->arch.smi_pending = events->smi.pending;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5324 		if (events->smi.smm) {  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5325 			if (events->smi.smm_inside_nmi)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5326 				vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5328 				vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5332 		if (events->smi.smm || events->smi.pending ||  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5333 		    events->smi.smm_inside_nmi)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5334 			return -EINVAL;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5338 			if (events->smi.latched_init)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5339 				set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5341 				clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5345 	if (events->flags & KVM_VCPUEVENT_VALID_TRIPLE_FAULT) {  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5346 		if (!vcpu->kvm->arch.triple_fault_event)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5347 			return -EINVAL;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5348 		if (events->triple_fault.pending)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5365 	memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));  in kvm_vcpu_ioctl_x86_get_debugregs()
5367 	dbgregs->dr6 = val;  in kvm_vcpu_ioctl_x86_get_debugregs()
5368 	dbgregs->dr7 = vcpu->arch.dr7;  in kvm_vcpu_ioctl_x86_get_debugregs()
5374 	if (dbgregs->flags)  in kvm_vcpu_ioctl_x86_set_debugregs()
5375 		return -EINVAL;  in kvm_vcpu_ioctl_x86_set_debugregs()
5377 	if (!kvm_dr6_valid(dbgregs->dr6))  in kvm_vcpu_ioctl_x86_set_debugregs()
5378 		return -EINVAL;  in kvm_vcpu_ioctl_x86_set_debugregs()
5379 	if (!kvm_dr7_valid(dbgregs->dr7))  in kvm_vcpu_ioctl_x86_set_debugregs()
5380 		return -EINVAL;  in kvm_vcpu_ioctl_x86_set_debugregs()
5382 	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));  in kvm_vcpu_ioctl_x86_set_debugregs()
5384 	vcpu->arch.dr6 = dbgregs->dr6;  in kvm_vcpu_ioctl_x86_set_debugregs()
5385 	vcpu->arch.dr7 = dbgregs->dr7;  in kvm_vcpu_ioctl_x86_set_debugregs()
5407 	u64 supported_xcr0 = vcpu->arch.guest_supported_xcr0 |  in kvm_vcpu_ioctl_x86_get_xsave2()
5410 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_vcpu_ioctl_x86_get_xsave2()
5413 	fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu, state, size,  in kvm_vcpu_ioctl_x86_get_xsave2()
5414 				       supported_xcr0, vcpu->arch.pkru);  in kvm_vcpu_ioctl_x86_get_xsave2()
5420 	return kvm_vcpu_ioctl_x86_get_xsave2(vcpu, (void *)guest_xsave->region,  in kvm_vcpu_ioctl_x86_get_xsave()
5421 					     sizeof(guest_xsave->region));  in kvm_vcpu_ioctl_x86_get_xsave()
5427 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_vcpu_ioctl_x86_set_xsave()
5430 	return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu,  in kvm_vcpu_ioctl_x86_set_xsave()
5431 					      guest_xsave->region,  in kvm_vcpu_ioctl_x86_set_xsave()
5433 					      &vcpu->arch.pkru);  in kvm_vcpu_ioctl_x86_set_xsave()
5440 		guest_xcrs->nr_xcrs = 0;  in kvm_vcpu_ioctl_x86_get_xcrs()
5444 	guest_xcrs->nr_xcrs = 1;  in kvm_vcpu_ioctl_x86_get_xcrs()
5445 	guest_xcrs->flags = 0;  in kvm_vcpu_ioctl_x86_get_xcrs()
5446 	guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK;  in kvm_vcpu_ioctl_x86_get_xcrs()
5447 	guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;  in kvm_vcpu_ioctl_x86_get_xcrs()
5456 		return -EINVAL;  in kvm_vcpu_ioctl_x86_set_xcrs()
5458 	if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags)  in kvm_vcpu_ioctl_x86_set_xcrs()
5459 		return -EINVAL;  in kvm_vcpu_ioctl_x86_set_xcrs()
5461 	for (i = 0; i < guest_xcrs->nr_xcrs; i++)  in kvm_vcpu_ioctl_x86_set_xcrs()
5463 		if (guest_xcrs->xcrs[i].xcr == XCR_XFEATURE_ENABLED_MASK) {  in kvm_vcpu_ioctl_x86_set_xcrs()
5465 				guest_xcrs->xcrs[i].value);  in kvm_vcpu_ioctl_x86_set_xcrs()
5469 		r = -EINVAL;  in kvm_vcpu_ioctl_x86_set_xcrs()
5481 	if (!vcpu->arch.pv_time.active)  in kvm_set_guest_paused()
5482 		return -EINVAL;  in kvm_set_guest_paused()
5483 	vcpu->arch.pvclock_set_guest_stopped_request = true;  in kvm_set_guest_paused()
5493 	switch (attr->attr) {  in kvm_arch_tsc_has_attr()
5498 		r = -ENXIO;  in kvm_arch_tsc_has_attr()
5513 	switch (attr->attr) {  in kvm_arch_tsc_get_attr()
5515 		r = -EFAULT;  in kvm_arch_tsc_get_attr()
5516 		if (put_user(vcpu->arch.l1_tsc_offset, uaddr))  in kvm_arch_tsc_get_attr()
5521 		r = -ENXIO;  in kvm_arch_tsc_get_attr()
5531 	struct kvm *kvm = vcpu->kvm;  in kvm_arch_tsc_set_attr()
5537 	switch (attr->attr) {  in kvm_arch_tsc_set_attr()
5543 		r = -EFAULT;  in kvm_arch_tsc_set_attr()
5547 		raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);  in kvm_arch_tsc_set_attr()
5549 		matched = (vcpu->arch.virtual_tsc_khz &&  in kvm_arch_tsc_set_attr()
5550 			   kvm->arch.last_tsc_khz == vcpu->arch.virtual_tsc_khz &&  in kvm_arch_tsc_set_attr()
5551 			   kvm->arch.last_tsc_offset == offset);  in kvm_arch_tsc_set_attr()
5553 		tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio) + offset;  in kvm_arch_tsc_set_attr()
5557 		raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);  in kvm_arch_tsc_set_attr()
5563 		r = -ENXIO;  in kvm_arch_tsc_set_attr()
5577 		return -EFAULT;  in kvm_vcpu_ioctl_device_attr()
5580 		return -ENXIO;  in kvm_vcpu_ioctl_device_attr()
5604 	if (cap->flags)  in kvm_vcpu_ioctl_enable_cap()
5605 		return -EINVAL;  in kvm_vcpu_ioctl_enable_cap()
5607 	switch (cap->cap) {  in kvm_vcpu_ioctl_enable_cap()
5609 		if (cap->args[0])  in kvm_vcpu_ioctl_enable_cap()
5610 			return -EINVAL;  in kvm_vcpu_ioctl_enable_cap()
5614 		if (!irqchip_in_kernel(vcpu->kvm))  in kvm_vcpu_ioctl_enable_cap()
5615 			return -EINVAL;  in kvm_vcpu_ioctl_enable_cap()
5616 		return kvm_hv_activate_synic(vcpu, cap->cap ==  in kvm_vcpu_ioctl_enable_cap()
5619 		if (!kvm_x86_ops.nested_ops->enable_evmcs)  in kvm_vcpu_ioctl_enable_cap()
5620 			return -ENOTTY;  in kvm_vcpu_ioctl_enable_cap()
5621 		r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version);  in kvm_vcpu_ioctl_enable_cap()
5623 			user_ptr = (void __user *)(uintptr_t)cap->args[0];  in kvm_vcpu_ioctl_enable_cap()
5626 				r = -EFAULT;  in kvm_vcpu_ioctl_enable_cap()
5631 			return -ENOTTY;  in kvm_vcpu_ioctl_enable_cap()
5636 		return kvm_hv_set_enforce_cpuid(vcpu, cap->args[0]);  in kvm_vcpu_ioctl_enable_cap()
5639 		vcpu->arch.pv_cpuid.enforce = cap->args[0];  in kvm_vcpu_ioctl_enable_cap()
5640 		if (vcpu->arch.pv_cpuid.enforce)  in kvm_vcpu_ioctl_enable_cap()
5645 		return -EINVAL;  in kvm_vcpu_ioctl_enable_cap()
5652 	struct kvm_vcpu *vcpu = filp->private_data;  in kvm_arch_vcpu_ioctl()
5668 		r = -EINVAL;  in kvm_arch_vcpu_ioctl()
5674 		r = -ENOMEM;  in kvm_arch_vcpu_ioctl()
5680 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5687 		r = -EINVAL;  in kvm_arch_vcpu_ioctl()
5702 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5720 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5723 		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);  in kvm_arch_vcpu_ioctl()
5730 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5734 					      cpuid_arg->entries);  in kvm_arch_vcpu_ioctl()
5741 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5745 					      cpuid_arg->entries);  in kvm_arch_vcpu_ioctl()
5748 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5755 		int idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
5757 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
5761 		int idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
5763 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
5769 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5775 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5785 		r = -EINVAL;  in kvm_arch_vcpu_ioctl()
5788 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5791 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
5793 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
5799 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5808 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5819 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5828 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5840 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5850 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5859 		r = -EINVAL;  in kvm_arch_vcpu_ioctl()
5860 		if (vcpu->arch.guest_fpu.uabi_size > sizeof(struct kvm_xsave))  in kvm_arch_vcpu_ioctl()
5864 		r = -ENOMEM;  in kvm_arch_vcpu_ioctl()
5870 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5877 		int size = vcpu->arch.guest_fpu.uabi_size;  in kvm_arch_vcpu_ioctl()
5890 		int size = vcpu->arch.guest_fpu.uabi_size;  in kvm_arch_vcpu_ioctl()
5893 		r = -ENOMEM;  in kvm_arch_vcpu_ioctl()
5899 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5909 		r = -ENOMEM;  in kvm_arch_vcpu_ioctl()
5915 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5935 		r = -EINVAL;  in kvm_arch_vcpu_ioctl()
5951 		r = vcpu->arch.virtual_tsc_khz;  in kvm_arch_vcpu_ioctl()
5961 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5971 		r = -EINVAL;  in kvm_arch_vcpu_ioctl()
5972 		if (!kvm_x86_ops.nested_ops->get_state)  in kvm_arch_vcpu_ioctl()
5975 		BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size));  in kvm_arch_vcpu_ioctl()
5976 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5977 		if (get_user(user_data_size, &user_kvm_nested_state->size))  in kvm_arch_vcpu_ioctl()
5980 		r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state,  in kvm_arch_vcpu_ioctl()
5986 			if (put_user(r, &user_kvm_nested_state->size))  in kvm_arch_vcpu_ioctl()
5987 				r = -EFAULT;  in kvm_arch_vcpu_ioctl()
5989 				r = -E2BIG;  in kvm_arch_vcpu_ioctl()
6001 		r = -EINVAL;  in kvm_arch_vcpu_ioctl()
6002 		if (!kvm_x86_ops.nested_ops->set_state)  in kvm_arch_vcpu_ioctl()
6005 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
6009 		r = -EINVAL;  in kvm_arch_vcpu_ioctl()
6024 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
6025 		r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state);  in kvm_arch_vcpu_ioctl()
6026 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
6036 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
6041 			r = -EFAULT;  in kvm_arch_vcpu_ioctl()
6047 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
6056 		r = -ENOMEM;  in kvm_arch_vcpu_ioctl()
6060 		r = -EFAULT;  in kvm_arch_vcpu_ioctl()
6082 		r = -EINVAL;  in kvm_arch_vcpu_ioctl()
6100 	if (addr > (unsigned int)(-3 * PAGE_SIZE))  in kvm_vm_ioctl_set_tss_addr()
6101 		return -EINVAL;  in kvm_vm_ioctl_set_tss_addr()
6116 		return -EINVAL;  in kvm_vm_ioctl_set_nr_mmu_pages()
6118 	mutex_lock(&kvm->slots_lock);  in kvm_vm_ioctl_set_nr_mmu_pages()
6121 	kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;  in kvm_vm_ioctl_set_nr_mmu_pages()
6123 	mutex_unlock(&kvm->slots_lock);  in kvm_vm_ioctl_set_nr_mmu_pages()
6129 	struct kvm_pic *pic = kvm->arch.vpic;  in kvm_vm_ioctl_get_irqchip()  local
6133 	switch (chip->chip_id) {  in kvm_vm_ioctl_get_irqchip()
6135 		memcpy(&chip->chip.pic, &pic->pics[0],  in kvm_vm_ioctl_get_irqchip()
6139 		memcpy(&chip->chip.pic, &pic->pics[1],  in kvm_vm_ioctl_get_irqchip()
6143 		kvm_get_ioapic(kvm, &chip->chip.ioapic);  in kvm_vm_ioctl_get_irqchip()
6146 		r = -EINVAL;  in kvm_vm_ioctl_get_irqchip()
6154 	struct kvm_pic *pic = kvm->arch.vpic;  in kvm_vm_ioctl_set_irqchip()  local
6158 	switch (chip->chip_id) {  in kvm_vm_ioctl_set_irqchip()
6160 		spin_lock(&pic->lock);  in kvm_vm_ioctl_set_irqchip()
6161 		memcpy(&pic->pics[0], &chip->chip.pic,  in kvm_vm_ioctl_set_irqchip()
6163 		spin_unlock(&pic->lock);  in kvm_vm_ioctl_set_irqchip()
6166 		spin_lock(&pic->lock);  in kvm_vm_ioctl_set_irqchip()
6167 		memcpy(&pic->pics[1], &chip->chip.pic,  in kvm_vm_ioctl_set_irqchip()
6169 		spin_unlock(&pic->lock);  in kvm_vm_ioctl_set_irqchip()
6172 		kvm_set_ioapic(kvm, &chip->chip.ioapic);  in kvm_vm_ioctl_set_irqchip()
6175 		r = -EINVAL;  in kvm_vm_ioctl_set_irqchip()
6178 	kvm_pic_update_irq(pic);  in kvm_vm_ioctl_set_irqchip()
6184 	struct kvm_kpit_state *kps = &kvm->arch.vpit->pit_state;  in kvm_vm_ioctl_get_pit()
6186 	BUILD_BUG_ON(sizeof(*ps) != sizeof(kps->channels));  in kvm_vm_ioctl_get_pit()
6188 	mutex_lock(&kps->lock);  in kvm_vm_ioctl_get_pit()
6189 	memcpy(ps, &kps->channels, sizeof(*ps));  in kvm_vm_ioctl_get_pit()
6190 	mutex_unlock(&kps->lock);  in kvm_vm_ioctl_get_pit()
6197 	struct kvm_pit *pit = kvm->arch.vpit;  in kvm_vm_ioctl_set_pit()
6199 	mutex_lock(&pit->pit_state.lock);  in kvm_vm_ioctl_set_pit()
6200 	memcpy(&pit->pit_state.channels, ps, sizeof(*ps));  in kvm_vm_ioctl_set_pit()
6202 		kvm_pit_load_count(pit, i, ps->channels[i].count, 0);  in kvm_vm_ioctl_set_pit()
6203 	mutex_unlock(&pit->pit_state.lock);  in kvm_vm_ioctl_set_pit()
6209 	mutex_lock(&kvm->arch.vpit->pit_state.lock);  in kvm_vm_ioctl_get_pit2()
6210 	memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,  in kvm_vm_ioctl_get_pit2()
6211 		sizeof(ps->channels));  in kvm_vm_ioctl_get_pit2()
6212 	ps->flags = kvm->arch.vpit->pit_state.flags;  in kvm_vm_ioctl_get_pit2()
6213 	mutex_unlock(&kvm->arch.vpit->pit_state.lock);  in kvm_vm_ioctl_get_pit2()
6214 	memset(&ps->reserved, 0, sizeof(ps->reserved));  in kvm_vm_ioctl_get_pit2()
6223 	struct kvm_pit *pit = kvm->arch.vpit;  in kvm_vm_ioctl_set_pit2()
6225 	mutex_lock(&pit->pit_state.lock);  in kvm_vm_ioctl_set_pit2()
6226 	prev_legacy = pit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;  in kvm_vm_ioctl_set_pit2()
6227 	cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;  in kvm_vm_ioctl_set_pit2()
6230 	memcpy(&pit->pit_state.channels, &ps->channels,  in kvm_vm_ioctl_set_pit2()
6231 	       sizeof(pit->pit_state.channels));  in kvm_vm_ioctl_set_pit2()
6232 	pit->pit_state.flags = ps->flags;  in kvm_vm_ioctl_set_pit2()
6234 		kvm_pit_load_count(pit, i, pit->pit_state.channels[i].count,  in kvm_vm_ioctl_set_pit2()
6236 	mutex_unlock(&pit->pit_state.lock);  in kvm_vm_ioctl_set_pit2()
6243 	struct kvm_pit *pit = kvm->arch.vpit;  in kvm_vm_ioctl_reinject()
6245 	/* pit->pit_state.lock was overloaded to prevent userspace from getting  in kvm_vm_ioctl_reinject()
6249 	mutex_lock(&pit->pit_state.lock);  in kvm_vm_ioctl_reinject()
6250 	kvm_pit_set_reinject(pit, control->pit_reinject);  in kvm_vm_ioctl_reinject()
6251 	mutex_unlock(&pit->pit_state.lock);  in kvm_vm_ioctl_reinject()
6262 	 * on all VM-Exits, thus we only need to kick running vCPUs to force a  in kvm_arch_sync_dirty_log()
6263 	 * VM-Exit.  in kvm_arch_sync_dirty_log()
6276 		return -ENXIO;  in kvm_vm_ioctl_irq_line()
6278 	irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,  in kvm_vm_ioctl_irq_line()
6279 					irq_event->irq, irq_event->level,  in kvm_vm_ioctl_irq_line()
6289 	if (cap->flags)  in kvm_vm_ioctl_enable_cap()
6290 		return -EINVAL;  in kvm_vm_ioctl_enable_cap()
6292 	switch (cap->cap) {  in kvm_vm_ioctl_enable_cap()
6294 		r = -EINVAL;  in kvm_vm_ioctl_enable_cap()
6295 		if (cap->args[0] & ~KVM_X86_VALID_QUIRKS)  in kvm_vm_ioctl_enable_cap()
6299 		kvm->arch.disabled_quirks = cap->args[0];  in kvm_vm_ioctl_enable_cap()
6303 		mutex_lock(&kvm->lock);  in kvm_vm_ioctl_enable_cap()
6304 		r = -EINVAL;  in kvm_vm_ioctl_enable_cap()
6305 		if (cap->args[0] > MAX_NR_RESERVED_IOAPIC_PINS)  in kvm_vm_ioctl_enable_cap()
6307 		r = -EEXIST;  in kvm_vm_ioctl_enable_cap()
6310 		if (kvm->created_vcpus)  in kvm_vm_ioctl_enable_cap()
6317 		kvm->arch.irqchip_mode = KVM_IRQCHIP_SPLIT;  in kvm_vm_ioctl_enable_cap()
6318 		kvm->arch.nr_reserved_ioapic_pins = cap->args[0];  in kvm_vm_ioctl_enable_cap()
6322 		mutex_unlock(&kvm->lock);  in kvm_vm_ioctl_enable_cap()
6326 		r = -EINVAL;  in kvm_vm_ioctl_enable_cap()
6327 		if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS)  in kvm_vm_ioctl_enable_cap()
6330 		if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS)  in kvm_vm_ioctl_enable_cap()
6331 			kvm->arch.x2apic_format = true;  in kvm_vm_ioctl_enable_cap()
6332 		if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)  in kvm_vm_ioctl_enable_cap()
6333 			kvm->arch.x2apic_broadcast_quirk_disabled = true;  in kvm_vm_ioctl_enable_cap()
6338 		r = -EINVAL;  in kvm_vm_ioctl_enable_cap()
6339 		if (cap->args[0] & ~KVM_X86_DISABLE_VALID_EXITS)  in kvm_vm_ioctl_enable_cap()
6342 		if (cap->args[0] & KVM_X86_DISABLE_EXITS_PAUSE)  in kvm_vm_ioctl_enable_cap()
6343 			kvm->arch.pause_in_guest = true;  in kvm_vm_ioctl_enable_cap()
6345 #define SMT_RSB_MSG "This processor is affected by the Cross-Thread Return Predictions vulnerabilit…  in kvm_vm_ioctl_enable_cap()
6350 			    (cap->args[0] & ~KVM_X86_DISABLE_EXITS_PAUSE))  in kvm_vm_ioctl_enable_cap()
6353 			if ((cap->args[0] & KVM_X86_DISABLE_EXITS_MWAIT) &&  in kvm_vm_ioctl_enable_cap()
6355 				kvm->arch.mwait_in_guest = true;  in kvm_vm_ioctl_enable_cap()
6356 			if (cap->args[0] & KVM_X86_DISABLE_EXITS_HLT)  in kvm_vm_ioctl_enable_cap()
6357 				kvm->arch.hlt_in_guest = true;  in kvm_vm_ioctl_enable_cap()
6358 			if (cap->args[0] & KVM_X86_DISABLE_EXITS_CSTATE)  in kvm_vm_ioctl_enable_cap()
6359 				kvm->arch.cstate_in_guest = true;  in kvm_vm_ioctl_enable_cap()
6365 		kvm->arch.guest_can_read_msr_platform_info = cap->args[0];  in kvm_vm_ioctl_enable_cap()
6369 		kvm->arch.exception_payload_enabled = cap->args[0];  in kvm_vm_ioctl_enable_cap()
6373 		kvm->arch.triple_fault_event = cap->args[0];  in kvm_vm_ioctl_enable_cap()
6377 		r = -EINVAL;  in kvm_vm_ioctl_enable_cap()
6378 		if (cap->args[0] & ~KVM_MSR_EXIT_REASON_VALID_MASK)  in kvm_vm_ioctl_enable_cap()
6380 		kvm->arch.user_space_msr_mask = cap->args[0];  in kvm_vm_ioctl_enable_cap()
6384 		r = -EINVAL;  in kvm_vm_ioctl_enable_cap()
6385 		if (cap->args[0] & ~KVM_BUS_LOCK_DETECTION_VALID_MODE)  in kvm_vm_ioctl_enable_cap()
6388 		if ((cap->args[0] & KVM_BUS_LOCK_DETECTION_OFF) &&  in kvm_vm_ioctl_enable_cap()
6389 		    (cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT))  in kvm_vm_ioctl_enable_cap()
6393 		    cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT)  in kvm_vm_ioctl_enable_cap()
6394 			kvm->arch.bus_lock_detection_enabled = true;  in kvm_vm_ioctl_enable_cap()
6401 		r = sgx_set_attribute(&allowed_attributes, cap->args[0]);  in kvm_vm_ioctl_enable_cap()
6408 			kvm->arch.sgx_provisioning_allowed = true;  in kvm_vm_ioctl_enable_cap()
6410 			r = -EINVAL;  in kvm_vm_ioctl_enable_cap()
6415 		r = -EINVAL;  in kvm_vm_ioctl_enable_cap()
6419 		r = static_call(kvm_x86_vm_copy_enc_context_from)(kvm, cap->args[0]);  in kvm_vm_ioctl_enable_cap()
6422 		r = -EINVAL;  in kvm_vm_ioctl_enable_cap()
6426 		r = static_call(kvm_x86_vm_move_enc_context_from)(kvm, cap->args[0]);  in kvm_vm_ioctl_enable_cap()
6429 		if (cap->args[0] & ~KVM_EXIT_HYPERCALL_VALID_MASK) {  in kvm_vm_ioctl_enable_cap()
6430 			r = -EINVAL;  in kvm_vm_ioctl_enable_cap()
6433 		kvm->arch.hypercall_exit_enabled = cap->args[0];  in kvm_vm_ioctl_enable_cap()
6437 		r = -EINVAL;  in kvm_vm_ioctl_enable_cap()
6438 		if (cap->args[0] & ~1)  in kvm_vm_ioctl_enable_cap()
6440 		kvm->arch.exit_on_emulation_error = cap->args[0];  in kvm_vm_ioctl_enable_cap()
6444 		r = -EINVAL;  in kvm_vm_ioctl_enable_cap()
6445 		if (!enable_pmu || (cap->args[0] & ~KVM_CAP_PMU_VALID_MASK))  in kvm_vm_ioctl_enable_cap()
6448 		mutex_lock(&kvm->lock);  in kvm_vm_ioctl_enable_cap()
6449 		if (!kvm->created_vcpus) {  in kvm_vm_ioctl_enable_cap()
6450 			kvm->arch.enable_pmu = !(cap->args[0] & KVM_PMU_CAP_DISABLE);  in kvm_vm_ioctl_enable_cap()
6453 		mutex_unlock(&kvm->lock);  in kvm_vm_ioctl_enable_cap()
6456 		r = -EINVAL;  in kvm_vm_ioctl_enable_cap()
6457 		if (cap->args[0] > KVM_MAX_VCPU_IDS)  in kvm_vm_ioctl_enable_cap()
6460 		mutex_lock(&kvm->lock);  in kvm_vm_ioctl_enable_cap()
6461 		if (kvm->arch.max_vcpu_ids == cap->args[0]) {  in kvm_vm_ioctl_enable_cap()
6463 		} else if (!kvm->arch.max_vcpu_ids) {  in kvm_vm_ioctl_enable_cap()
6464 			kvm->arch.max_vcpu_ids = cap->args[0];  in kvm_vm_ioctl_enable_cap()
6467 		mutex_unlock(&kvm->lock);  in kvm_vm_ioctl_enable_cap()
6470 		r = -EINVAL;  in kvm_vm_ioctl_enable_cap()
6471 		if ((u32)cap->args[0] & ~KVM_X86_NOTIFY_VMEXIT_VALID_BITS)  in kvm_vm_ioctl_enable_cap()
6475 		if (!((u32)cap->args[0] & KVM_X86_NOTIFY_VMEXIT_ENABLED))  in kvm_vm_ioctl_enable_cap()
6477 		mutex_lock(&kvm->lock);  in kvm_vm_ioctl_enable_cap()
6478 		if (!kvm->created_vcpus) {  in kvm_vm_ioctl_enable_cap()
6479 			kvm->arch.notify_window = cap->args[0] >> 32;  in kvm_vm_ioctl_enable_cap()
6480 			kvm->arch.notify_vmexit_flags = (u32)cap->args[0];  in kvm_vm_ioctl_enable_cap()
6483 		mutex_unlock(&kvm->lock);  in kvm_vm_ioctl_enable_cap()
6486 		r = -EINVAL;  in kvm_vm_ioctl_enable_cap()
6500 			r = -EPERM;  in kvm_vm_ioctl_enable_cap()
6504 		if (cap->args[0])  in kvm_vm_ioctl_enable_cap()
6507 		mutex_lock(&kvm->lock);  in kvm_vm_ioctl_enable_cap()
6508 		if (!kvm->created_vcpus) {  in kvm_vm_ioctl_enable_cap()
6509 			kvm->arch.disable_nx_huge_pages = true;  in kvm_vm_ioctl_enable_cap()
6512 		mutex_unlock(&kvm->lock);  in kvm_vm_ioctl_enable_cap()
6515 		r = -EINVAL;  in kvm_vm_ioctl_enable_cap()
6529 	msr_filter->default_allow = default_allow;  in kvm_alloc_msr_filter()
6540 	for (i = 0; i < msr_filter->count; i++)  in kvm_free_msr_filter()
6541 		kfree(msr_filter->ranges[i].bitmap);  in kvm_free_msr_filter()
6552 	if (!user_range->nmsrs)  in kvm_add_msr_filter()
6555 	if (user_range->flags & ~KVM_MSR_FILTER_RANGE_VALID_MASK)  in kvm_add_msr_filter()
6556 		return -EINVAL;  in kvm_add_msr_filter()
6558 	if (!user_range->flags)  in kvm_add_msr_filter()
6559 		return -EINVAL;  in kvm_add_msr_filter()
6561 	bitmap_size = BITS_TO_LONGS(user_range->nmsrs) * sizeof(long);  in kvm_add_msr_filter()
6563 		return -EINVAL;  in kvm_add_msr_filter()
6565 	bitmap = memdup_user((__user u8*)user_range->bitmap, bitmap_size);  in kvm_add_msr_filter()
6569 	msr_filter->ranges[msr_filter->count] = (struct msr_bitmap_range) {  in kvm_add_msr_filter()
6570 		.flags = user_range->flags,  in kvm_add_msr_filter()
6571 		.base = user_range->base,  in kvm_add_msr_filter()
6572 		.nmsrs = user_range->nmsrs,  in kvm_add_msr_filter()
6576 	msr_filter->count++;  in kvm_add_msr_filter()
6589 	if (filter->flags & ~KVM_MSR_FILTER_VALID_MASK)  in kvm_vm_ioctl_set_msr_filter()
6590 		return -EINVAL;  in kvm_vm_ioctl_set_msr_filter()
6592 	for (i = 0; i < ARRAY_SIZE(filter->ranges); i++)  in kvm_vm_ioctl_set_msr_filter()
6593 		empty &= !filter->ranges[i].nmsrs;  in kvm_vm_ioctl_set_msr_filter()
6595 	default_allow = !(filter->flags & KVM_MSR_FILTER_DEFAULT_DENY);  in kvm_vm_ioctl_set_msr_filter()
6597 		return -EINVAL;  in kvm_vm_ioctl_set_msr_filter()
6601 		return -ENOMEM;  in kvm_vm_ioctl_set_msr_filter()
6603 	for (i = 0; i < ARRAY_SIZE(filter->ranges); i++) {  in kvm_vm_ioctl_set_msr_filter()
6604 		r = kvm_add_msr_filter(new_filter, &filter->ranges[i]);  in kvm_vm_ioctl_set_msr_filter()
6611 	mutex_lock(&kvm->lock);  in kvm_vm_ioctl_set_msr_filter()
6612 	old_filter = rcu_replace_pointer(kvm->arch.msr_filter, new_filter,  in kvm_vm_ioctl_set_msr_filter()
6613 					 mutex_is_locked(&kvm->lock));  in kvm_vm_ioctl_set_msr_filter()
6614 	mutex_unlock(&kvm->lock);  in kvm_vm_ioctl_set_msr_filter()
6615 	synchronize_srcu(&kvm->srcu);  in kvm_vm_ioctl_set_msr_filter()
6629 	__u32 base;  member
6644 	struct kvm *kvm = filp->private_data;  in kvm_arch_vm_compat_ioctl()
6645 	long r = -ENOTTY;  in kvm_arch_vm_compat_ioctl()
6656 			return -EFAULT;  in kvm_arch_vm_compat_ioctl()
6664 				.flags = cr->flags,  in kvm_arch_vm_compat_ioctl()
6665 				.nmsrs = cr->nmsrs,  in kvm_arch_vm_compat_ioctl()
6666 				.base = cr->base,  in kvm_arch_vm_compat_ioctl()
6667 				.bitmap = (__u8 *)(ulong)cr->bitmap,  in kvm_arch_vm_compat_ioctl()
6687 	mutex_lock(&kvm->lock);  in kvm_arch_suspend_notifier()
6689 		if (!vcpu->arch.pv_time.active)  in kvm_arch_suspend_notifier()
6695 				vcpu->vcpu_id, ret);  in kvm_arch_suspend_notifier()
6699 	mutex_unlock(&kvm->lock);  in kvm_arch_suspend_notifier()
6722 		return -EFAULT;  in kvm_vm_ioctl_get_clock()
6729 	struct kvm_arch *ka = &kvm->arch;  in kvm_vm_ioctl_set_clock()
6734 		return -EFAULT;  in kvm_vm_ioctl_set_clock()
6741 		return -EINVAL;  in kvm_vm_ioctl_set_clock()
6761 			data.clock += now_real_ns - data.realtime;  in kvm_vm_ioctl_set_clock()
6764 	if (ka->use_master_clock)  in kvm_vm_ioctl_set_clock()
6765 		now_raw_ns = ka->master_kernel_ns;  in kvm_vm_ioctl_set_clock()
6768 	ka->kvmclock_offset = data.clock - now_raw_ns;  in kvm_vm_ioctl_set_clock()
6775 	struct kvm *kvm = filp->private_data;  in kvm_arch_vm_ioctl()
6777 	int r = -ENOTTY;  in kvm_arch_vm_ioctl()
6779 	 * This union makes it completely explicit to gcc-3.x  in kvm_arch_vm_ioctl()
6796 		mutex_lock(&kvm->lock);  in kvm_arch_vm_ioctl()
6797 		r = -EINVAL;  in kvm_arch_vm_ioctl()
6798 		if (kvm->created_vcpus)  in kvm_arch_vm_ioctl()
6800 		r = -EFAULT;  in kvm_arch_vm_ioctl()
6805 		mutex_unlock(&kvm->lock);  in kvm_arch_vm_ioctl()
6812 		mutex_lock(&kvm->lock);  in kvm_arch_vm_ioctl()
6814 		r = -EEXIST;  in kvm_arch_vm_ioctl()
6818 		r = -EINVAL;  in kvm_arch_vm_ioctl()
6819 		if (kvm->created_vcpus)  in kvm_arch_vm_ioctl()
6838 		/* Write kvm->irq_routing before enabling irqchip_in_kernel. */  in kvm_arch_vm_ioctl()
6840 		kvm->arch.irqchip_mode = KVM_IRQCHIP_KERNEL;  in kvm_arch_vm_ioctl()
6843 		mutex_unlock(&kvm->lock);  in kvm_arch_vm_ioctl()
6850 		r = -EFAULT;  in kvm_arch_vm_ioctl()
6855 		mutex_lock(&kvm->lock);  in kvm_arch_vm_ioctl()
6856 		r = -EEXIST;  in kvm_arch_vm_ioctl()
6857 		if (kvm->arch.vpit)  in kvm_arch_vm_ioctl()
6859 		r = -ENOMEM;  in kvm_arch_vm_ioctl()
6860 		kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);  in kvm_arch_vm_ioctl()
6861 		if (kvm->arch.vpit)  in kvm_arch_vm_ioctl()
6864 		mutex_unlock(&kvm->lock);  in kvm_arch_vm_ioctl()
6867 		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */  in kvm_arch_vm_ioctl()
6876 		r = -ENXIO;  in kvm_arch_vm_ioctl()
6882 		r = -EFAULT;  in kvm_arch_vm_ioctl()
6891 		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */  in kvm_arch_vm_ioctl()
6900 		r = -ENXIO;  in kvm_arch_vm_ioctl()
6909 		r = -EFAULT;  in kvm_arch_vm_ioctl()
6912 		r = -ENXIO;  in kvm_arch_vm_ioctl()
6913 		if (!kvm->arch.vpit)  in kvm_arch_vm_ioctl()
6918 		r = -EFAULT;  in kvm_arch_vm_ioctl()
6925 		r = -EFAULT;  in kvm_arch_vm_ioctl()
6928 		mutex_lock(&kvm->lock);  in kvm_arch_vm_ioctl()
6929 		r = -ENXIO;  in kvm_arch_vm_ioctl()
6930 		if (!kvm->arch.vpit)  in kvm_arch_vm_ioctl()
6934 		mutex_unlock(&kvm->lock);  in kvm_arch_vm_ioctl()
6938 		r = -ENXIO;  in kvm_arch_vm_ioctl()
6939 		if (!kvm->arch.vpit)  in kvm_arch_vm_ioctl()
6944 		r = -EFAULT;  in kvm_arch_vm_ioctl()
6951 		r = -EFAULT;  in kvm_arch_vm_ioctl()
6954 		mutex_lock(&kvm->lock);  in kvm_arch_vm_ioctl()
6955 		r = -ENXIO;  in kvm_arch_vm_ioctl()
6956 		if (!kvm->arch.vpit)  in kvm_arch_vm_ioctl()
6960 		mutex_unlock(&kvm->lock);  in kvm_arch_vm_ioctl()
6965 		r =  -EFAULT;  in kvm_arch_vm_ioctl()
6968 		r = -ENXIO;  in kvm_arch_vm_ioctl()
6969 		if (!kvm->arch.vpit)  in kvm_arch_vm_ioctl()
6976 		mutex_lock(&kvm->lock);  in kvm_arch_vm_ioctl()
6977 		if (kvm->created_vcpus)  in kvm_arch_vm_ioctl()
6978 			r = -EBUSY;  in kvm_arch_vm_ioctl()
6980 			kvm->arch.bsp_vcpu_id = arg;  in kvm_arch_vm_ioctl()
6981 		mutex_unlock(&kvm->lock);  in kvm_arch_vm_ioctl()
6986 		r = -EFAULT;  in kvm_arch_vm_ioctl()
6995 		r = -EFAULT;  in kvm_arch_vm_ioctl()
7000 			r = -EFAULT;  in kvm_arch_vm_ioctl()
7006 		r = -EFAULT;  in kvm_arch_vm_ioctl()
7015 		r = -EFAULT;  in kvm_arch_vm_ioctl()
7031 		r = -EINVAL;  in kvm_arch_vm_ioctl()
7041 		WRITE_ONCE(kvm->arch.default_tsc_khz, user_tsc_khz);  in kvm_arch_vm_ioctl()
7047 		r = READ_ONCE(kvm->arch.default_tsc_khz);  in kvm_arch_vm_ioctl()
7051 		r = -ENOTTY;  in kvm_arch_vm_ioctl()
7061 		r = -EFAULT;  in kvm_arch_vm_ioctl()
7065 		r = -ENOTTY;  in kvm_arch_vm_ioctl()
7075 		r = -EFAULT;  in kvm_arch_vm_ioctl()
7079 		r = -ENOTTY;  in kvm_arch_vm_ioctl()
7089 		r = -EFAULT;  in kvm_arch_vm_ioctl()
7103 			return -EFAULT;  in kvm_arch_vm_ioctl()
7109 		r = -ENOTTY;  in kvm_arch_vm_ioctl()
7171 		    (msr_index - MSR_IA32_RTIT_ADDR0_A >=  in kvm_probe_msr_to_save()
7176 		if (msr_index - MSR_ARCH_PERFMON_PERFCTR0 >=  in kvm_probe_msr_to_save()
7181 		if (msr_index - MSR_ARCH_PERFMON_EVENTSEL0 >=  in kvm_probe_msr_to_save()
7186 		if (msr_index - MSR_ARCH_PERFMON_FIXED_CTR0 >=  in kvm_probe_msr_to_save()
7254 		      !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))  in vcpu_mmio_write()
7259 		len -= n;  in vcpu_mmio_write()
7274 		      !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,  in vcpu_mmio_read()
7281 		len -= n;  in vcpu_mmio_read()
7303 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in translate_nested_gpa()
7308 	/* NPT walks are always user-walks */  in translate_nested_gpa()
7310 	t_gpa  = mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception);  in translate_nested_gpa()
7318 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_mmu_gva_to_gpa_read()
7321 	return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);  in kvm_mmu_gva_to_gpa_read()
7328 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_mmu_gva_to_gpa_write()
7332 	return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);  in kvm_mmu_gva_to_gpa_write()
7340 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_mmu_gva_to_gpa_system()
7342 	return mmu->gva_to_gpa(vcpu, mmu, gva, 0, exception);  in kvm_mmu_gva_to_gpa_system()
7349 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_read_guest_virt_helper()
7354 		gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);  in kvm_read_guest_virt_helper()
7355 		unsigned offset = addr & (PAGE_SIZE-1);  in kvm_read_guest_virt_helper()
7356 		unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);  in kvm_read_guest_virt_helper()
7368 		bytes -= toread;  in kvm_read_guest_virt_helper()
7382 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_fetch_guest_virt()
7388 	gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access|PFERR_FETCH_MASK,  in kvm_fetch_guest_virt()
7393 	offset = addr & (PAGE_SIZE-1);  in kvm_fetch_guest_virt()
7395 		bytes = (unsigned)PAGE_SIZE - offset;  in kvm_fetch_guest_virt()
7441 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_write_guest_virt_helper()
7446 		gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);  in kvm_write_guest_virt_helper()
7447 		unsigned offset = addr & (PAGE_SIZE-1);  in kvm_write_guest_virt_helper()
7448 		unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);  in kvm_write_guest_virt_helper()
7459 		bytes -= towrite;  in kvm_write_guest_virt_helper()
7487 	vcpu->arch.l1tf_flush_l1d = true;  in kvm_write_guest_virt_system()
7545 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in vcpu_mmio_gva_to_gpa()
7555 	    !permission_fault(vcpu, vcpu->arch.walk_mmu,  in vcpu_mmio_gva_to_gpa()
7556 			      vcpu->arch.mmio_access, 0, access))) {  in vcpu_mmio_gva_to_gpa()
7557 		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |  in vcpu_mmio_gva_to_gpa()
7558 					(gva & (PAGE_SIZE - 1));  in vcpu_mmio_gva_to_gpa()
7563 	*gpa = mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);  in vcpu_mmio_gva_to_gpa()
7566 		return -1;  in vcpu_mmio_gva_to_gpa()
7597 	if (vcpu->mmio_read_completed) {  in read_prepare()
7599 			       vcpu->mmio_fragments[0].gpa, val);  in read_prepare()
7600 		vcpu->mmio_read_completed = 0;  in read_prepare()
7635 	struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];  in write_exit_mmio()
7637 	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));  in write_exit_mmio()
7663 	bool write = ops->write;  in emulator_read_write_onepage()
7665 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in emulator_read_write_onepage()
7674 	if (ctxt->gpa_available && emulator_can_use_gpa(ctxt) &&  in emulator_read_write_onepage()
7675 	    (addr & ~PAGE_MASK) == (ctxt->gpa_val & ~PAGE_MASK)) {  in emulator_read_write_onepage()
7676 		gpa = ctxt->gpa_val;  in emulator_read_write_onepage()
7684 	if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))  in emulator_read_write_onepage()
7690 	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);  in emulator_read_write_onepage()
7695 	bytes -= handled;  in emulator_read_write_onepage()
7698 	WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);  in emulator_read_write_onepage()
7699 	frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];  in emulator_read_write_onepage()
7700 	frag->gpa = gpa;  in emulator_read_write_onepage()
7701 	frag->data = val;  in emulator_read_write_onepage()
7702 	frag->len = bytes;  in emulator_read_write_onepage()
7716 	if (ops->read_write_prepare &&  in emulator_read_write()
7717 		  ops->read_write_prepare(vcpu, val, bytes))  in emulator_read_write()
7720 	vcpu->mmio_nr_fragments = 0;  in emulator_read_write()
7723 	if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {  in emulator_read_write()
7726 		now = -addr & ~PAGE_MASK;  in emulator_read_write()
7733 		if (ctxt->mode != X86EMUL_MODE_PROT64)  in emulator_read_write()
7736 		bytes -= now;  in emulator_read_write()
7744 	if (!vcpu->mmio_nr_fragments)  in emulator_read_write()
7747 	gpa = vcpu->mmio_fragments[0].gpa;  in emulator_read_write()
7749 	vcpu->mmio_needed = 1;  in emulator_read_write()
7750 	vcpu->mmio_cur_fragment = 0;  in emulator_read_write()
7752 	vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);  in emulator_read_write()
7753 	vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;  in emulator_read_write()
7754 	vcpu->run->exit_reason = KVM_EXIT_MMIO;  in emulator_read_write()
7755 	vcpu->run->mmio.phys_addr = gpa;  in emulator_read_write()
7757 	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);  in emulator_read_write()
7797 	if (bytes > 8 || (bytes & (bytes - 1)))  in emulator_cmpxchg_emulated()
7811 		page_line_mask = ~(cache_line_size() - 1);  in emulator_cmpxchg_emulated()
7815 	if (((gpa + bytes - 1) & page_line_mask) != (gpa & page_line_mask))  in emulator_cmpxchg_emulated()
7863 	WARN_ON_ONCE(vcpu->arch.pio.count);  in emulator_pio_in_out()
7879 				memset(data, 0, size * (count - i));  in emulator_pio_in_out()
7888 	vcpu->arch.pio.port = port;  in emulator_pio_in_out()
7889 	vcpu->arch.pio.in = in;  in emulator_pio_in_out()
7890 	vcpu->arch.pio.count = count;  in emulator_pio_in_out()
7891 	vcpu->arch.pio.size = size;  in emulator_pio_in_out()
7894 		memset(vcpu->arch.pio_data, 0, size * count);  in emulator_pio_in_out()
7896 		memcpy(vcpu->arch.pio_data, data, size * count);  in emulator_pio_in_out()
7898 	vcpu->run->exit_reason = KVM_EXIT_IO;  in emulator_pio_in_out()
7899 	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;  in emulator_pio_in_out()
7900 	vcpu->run->io.size = size;  in emulator_pio_in_out()
7901 	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;  in emulator_pio_in_out()
7902 	vcpu->run->io.count = count;  in emulator_pio_in_out()
7903 	vcpu->run->io.port = port;  in emulator_pio_in_out()
7919 	int size = vcpu->arch.pio.size;  in complete_emulator_pio_in()
7920 	unsigned int count = vcpu->arch.pio.count;  in complete_emulator_pio_in()
7921 	memcpy(val, vcpu->arch.pio_data, size * count);  in complete_emulator_pio_in()
7922 	trace_kvm_pio(KVM_PIO_IN, vcpu->arch.pio.port, size, count, vcpu->arch.pio_data);  in complete_emulator_pio_in()
7923 	vcpu->arch.pio.count = 0;  in complete_emulator_pio_in()
7931 	if (vcpu->arch.pio.count) {  in emulator_pio_in_emulated()
7979 		cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);  in kvm_emulate_wbinvd_noskip()
7980 		on_each_cpu_mask(vcpu->arch.wbinvd_dirty_mask,  in kvm_emulate_wbinvd_noskip()
7983 		cpumask_clear(vcpu->arch.wbinvd_dirty_mask);  in kvm_emulate_wbinvd_noskip()
8018 	return (curr_cr & ~((1ULL << 32) - 1)) | new_val;  in mk_cr_64()
8031 		value = vcpu->arch.cr2;  in emulator_get_cr()
8060 		vcpu->arch.cr2 = val;  in emulator_set_cr()
8073 		res = -1;  in emulator_set_cr()
8129 	set_desc_base(desc, (unsigned long)var.base);  in emulator_get_segment()
8132 		*base3 = var.base >> 32;  in emulator_get_segment()
8134 	desc->type = var.type;  in emulator_get_segment()
8135 	desc->s = var.s;  in emulator_get_segment()
8136 	desc->dpl = var.dpl;  in emulator_get_segment()
8137 	desc->p = var.present;  in emulator_get_segment()
8138 	desc->avl = var.avl;  in emulator_get_segment()
8139 	desc->l = var.l;  in emulator_get_segment()
8140 	desc->d = var.db;  in emulator_get_segment()
8141 	desc->g = var.g;  in emulator_get_segment()
8154 	var.base = get_desc_base(desc);  in emulator_set_segment()
8156 	var.base |= ((u64)base3) << 32;  in emulator_set_segment()
8159 	if (desc->g)  in emulator_set_segment()
8161 	var.type = desc->type;  in emulator_set_segment()
8162 	var.dpl = desc->dpl;  in emulator_set_segment()
8163 	var.db = desc->d;  in emulator_set_segment()
8164 	var.s = desc->s;  in emulator_set_segment()
8165 	var.l = desc->l;  in emulator_set_segment()
8166 	var.g = desc->g;  in emulator_set_segment()
8167 	var.avl = desc->avl;  in emulator_set_segment()
8168 	var.present = desc->p;  in emulator_set_segment()
8233 	return -EINVAL;  in emulator_check_pmc()
8244 	emul_to_vcpu(ctxt)->arch.halt_request = 1;  in emulator_halt()
8252 					    &ctxt->exception);  in emulator_intercept()
8322 	struct kvm *kvm = emul_to_vcpu(ctxt)->kvm;  in emulator_vm_bugged()
8324 	if (!kvm->vm_bugged)  in emulator_vm_bugged()
8395 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in inject_emulated_exception()
8397 	if (ctxt->exception.vector == PF_VECTOR)  in inject_emulated_exception()
8398 		kvm_inject_emulated_page_fault(vcpu, &ctxt->exception);  in inject_emulated_exception()
8399 	else if (ctxt->exception.error_code_valid)  in inject_emulated_exception()
8400 		kvm_queue_exception_e(vcpu, ctxt->exception.vector,  in inject_emulated_exception()
8401 				      ctxt->exception.error_code);  in inject_emulated_exception()
8403 		kvm_queue_exception(vcpu, ctxt->exception.vector);  in inject_emulated_exception()
8416 	ctxt->vcpu = vcpu;  in alloc_emulate_ctxt()
8417 	ctxt->ops = &emulate_ops;  in alloc_emulate_ctxt()
8418 	vcpu->arch.emulate_ctxt = ctxt;  in alloc_emulate_ctxt()
8425 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in init_emulate_ctxt()
8430 	ctxt->gpa_available = false;  in init_emulate_ctxt()
8431 	ctxt->eflags = kvm_get_rflags(vcpu);  in init_emulate_ctxt()
8432 	ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;  in init_emulate_ctxt()
8434 	ctxt->eip = kvm_rip_read(vcpu);  in init_emulate_ctxt()
8435 	ctxt->mode = (!is_protmode(vcpu))		? X86EMUL_MODE_REAL :  in init_emulate_ctxt()
8436 		     (ctxt->eflags & X86_EFLAGS_VM)	? X86EMUL_MODE_VM86 :  in init_emulate_ctxt()
8440 	ctxt->interruptibility = 0;  in init_emulate_ctxt()
8441 	ctxt->have_exception = false;  in init_emulate_ctxt()
8442 	ctxt->exception.vector = -1;  in init_emulate_ctxt()
8443 	ctxt->perm_ok = false;  in init_emulate_ctxt()
8446 	vcpu->arch.emulate_regs_need_sync_from_vcpu = false;  in init_emulate_ctxt()
8451 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in kvm_inject_realmode_interrupt()
8456 	ctxt->op_bytes = 2;  in kvm_inject_realmode_interrupt()
8457 	ctxt->ad_bytes = 2;  in kvm_inject_realmode_interrupt()
8458 	ctxt->_eip = ctxt->eip + inc_eip;  in kvm_inject_realmode_interrupt()
8464 		ctxt->eip = ctxt->_eip;  in kvm_inject_realmode_interrupt()
8465 		kvm_rip_write(vcpu, ctxt->eip);  in kvm_inject_realmode_interrupt()
8466 		kvm_set_rflags(vcpu, ctxt->eflags);  in kvm_inject_realmode_interrupt()
8474 	struct kvm_run *run = vcpu->run;  in prepare_emulation_failure_exit()
8488 	run->exit_reason = KVM_EXIT_INTERNAL_ERROR;  in prepare_emulation_failure_exit()
8489 	run->emulation_failure.suberror = KVM_INTERNAL_ERROR_EMULATION;  in prepare_emulation_failure_exit()
8501 	run->emulation_failure.flags = 0;  in prepare_emulation_failure_exit()
8504 		BUILD_BUG_ON((sizeof(run->emulation_failure.insn_size) +  in prepare_emulation_failure_exit()
8505 			      sizeof(run->emulation_failure.insn_bytes) != 16));  in prepare_emulation_failure_exit()
8507 		run->emulation_failure.flags |=  in prepare_emulation_failure_exit()
8509 		run->emulation_failure.insn_size = insn_size;  in prepare_emulation_failure_exit()
8510 		memset(run->emulation_failure.insn_bytes, 0x90,  in prepare_emulation_failure_exit()
8511 		       sizeof(run->emulation_failure.insn_bytes));  in prepare_emulation_failure_exit()
8512 		memcpy(run->emulation_failure.insn_bytes, insn_bytes, insn_size);  in prepare_emulation_failure_exit()
8515 	memcpy(&run->internal.data[info_start], info, sizeof(info));  in prepare_emulation_failure_exit()
8516 	memcpy(&run->internal.data[info_start + ARRAY_SIZE(info)], data,  in prepare_emulation_failure_exit()
8519 	run->emulation_failure.ndata = info_start + ARRAY_SIZE(info) + ndata;  in prepare_emulation_failure_exit()
8524 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in prepare_emulation_ctxt_failure_exit()
8526 	prepare_emulation_failure_exit(vcpu, NULL, 0, ctxt->fetch.data,  in prepare_emulation_ctxt_failure_exit()
8527 				       ctxt->fetch.end - ctxt->fetch.data);  in prepare_emulation_ctxt_failure_exit()
8545 	struct kvm *kvm = vcpu->kvm;  in handle_emulation_failure()
8547 	++vcpu->stat.insn_emulation_fail;  in handle_emulation_failure()
8555 	if (kvm->arch.exit_on_emulation_error ||  in handle_emulation_failure()
8584 	if (!vcpu->arch.mmu->root_role.direct) {  in reexecute_instruction()
8602 	 * retry instruction -> write #PF -> emulation fail -> retry  in reexecute_instruction()
8603 	 * instruction -> ...  in reexecute_instruction()
8605 	pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));  in reexecute_instruction()
8616 	/* The instructions are well-emulated on direct mmu. */  in reexecute_instruction()
8617 	if (vcpu->arch.mmu->root_role.direct) {  in reexecute_instruction()
8620 		write_lock(&vcpu->kvm->mmu_lock);  in reexecute_instruction()
8621 		indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;  in reexecute_instruction()
8622 		write_unlock(&vcpu->kvm->mmu_lock);  in reexecute_instruction()
8625 			kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));  in reexecute_instruction()
8632 	 * and it failed try to unshadow page and re-enter the  in reexecute_instruction()
8635 	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));  in reexecute_instruction()
8651 	last_retry_eip = vcpu->arch.last_retry_eip;  in retry_instruction()
8652 	last_retry_addr = vcpu->arch.last_retry_addr;  in retry_instruction()
8655 	 * If the emulation is caused by #PF and it is non-page_table  in retry_instruction()
8656 	 * writing instruction, it means the VM-EXIT is caused by shadow  in retry_instruction()
8660 	 * Note: if the guest uses a non-page-table modifying instruction  in retry_instruction()
8667 	vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;  in retry_instruction()
8679 	if (ctxt->eip == last_retry_eip && last_retry_addr == cr2_or_gpa)  in retry_instruction()
8682 	vcpu->arch.last_retry_eip = ctxt->eip;  in retry_instruction()
8683 	vcpu->arch.last_retry_addr = cr2_or_gpa;  in retry_instruction()
8685 	if (!vcpu->arch.mmu->root_role.direct)  in retry_instruction()
8688 	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));  in retry_instruction()
8713 	struct kvm_run *kvm_run = vcpu->run;  in kvm_vcpu_do_singlestep()
8715 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {  in kvm_vcpu_do_singlestep()
8716 		kvm_run->debug.arch.dr6 = DR6_BS | DR6_ACTIVE_LOW;  in kvm_vcpu_do_singlestep()
8717 		kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);  in kvm_vcpu_do_singlestep()
8718 		kvm_run->debug.arch.exception = DB_VECTOR;  in kvm_vcpu_do_singlestep()
8719 		kvm_run->exit_reason = KVM_EXIT_DEBUG;  in kvm_vcpu_do_singlestep()
8791 	if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&  in kvm_vcpu_check_code_breakpoint()
8792 	    (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {  in kvm_vcpu_check_code_breakpoint()
8793 		struct kvm_run *kvm_run = vcpu->run;  in kvm_vcpu_check_code_breakpoint()
8796 					   vcpu->arch.guest_debug_dr7,  in kvm_vcpu_check_code_breakpoint()
8797 					   vcpu->arch.eff_db);  in kvm_vcpu_check_code_breakpoint()
8800 			kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW;  in kvm_vcpu_check_code_breakpoint()
8801 			kvm_run->debug.arch.pc = eip;  in kvm_vcpu_check_code_breakpoint()
8802 			kvm_run->debug.arch.exception = DB_VECTOR;  in kvm_vcpu_check_code_breakpoint()
8803 			kvm_run->exit_reason = KVM_EXIT_DEBUG;  in kvm_vcpu_check_code_breakpoint()
8809 	if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&  in kvm_vcpu_check_code_breakpoint()
8813 					   vcpu->arch.dr7,  in kvm_vcpu_check_code_breakpoint()
8814 					   vcpu->arch.db);  in kvm_vcpu_check_code_breakpoint()
8828 	switch (ctxt->opcode_len) {  in is_vmware_backdoor_opcode()
8830 		switch (ctxt->b) {  in is_vmware_backdoor_opcode()
8847 		switch (ctxt->b) {  in is_vmware_backdoor_opcode()
8860  * (and wrong) when emulating on an intercepted fault-like exception[*], as
8870 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in x86_decode_emulated_instruction()
8878 	++vcpu->stat.insn_emulation;  in x86_decode_emulated_instruction()
8888 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in x86_emulate_instruction()
8894 	vcpu->arch.l1tf_flush_l1d = true;  in x86_emulate_instruction()
8901 		 * are fault-like and are higher priority than any faults on  in x86_emulate_instruction()
8919 			if (ctxt->have_exception &&  in x86_emulate_instruction()
8922 				 * #UD should result in just EMULATION_FAILED, and trap-like  in x86_emulate_instruction()
8925 				WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR ||  in x86_emulate_instruction()
8926 					     exception_type(ctxt->exception.vector) == EXCPT_TRAP);  in x86_emulate_instruction()
8944 	 * injecting single-step #DBs.  in x86_emulate_instruction()
8947 		if (ctxt->mode != X86EMUL_MODE_PROT64)  in x86_emulate_instruction()
8948 			ctxt->eip = (u32)ctxt->_eip;  in x86_emulate_instruction()
8950 			ctxt->eip = ctxt->_eip;  in x86_emulate_instruction()
8957 		kvm_rip_write(vcpu, ctxt->eip);  in x86_emulate_instruction()
8958 		if (ctxt->eflags & X86_EFLAGS_RF)  in x86_emulate_instruction()
8959 			kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);  in x86_emulate_instruction()
8968 	if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {  in x86_emulate_instruction()
8969 		vcpu->arch.emulate_regs_need_sync_from_vcpu = false;  in x86_emulate_instruction()
8976 		ctxt->exception.address = cr2_or_gpa;  in x86_emulate_instruction()
8979 		if (vcpu->arch.mmu->root_role.direct) {  in x86_emulate_instruction()
8980 			ctxt->gpa_available = true;  in x86_emulate_instruction()
8981 			ctxt->gpa_val = cr2_or_gpa;  in x86_emulate_instruction()
8985 		ctxt->exception.address = 0;  in x86_emulate_instruction()
9000 	if (ctxt->have_exception) {  in x86_emulate_instruction()
9001 		WARN_ON_ONCE(vcpu->mmio_needed && !vcpu->mmio_is_write);  in x86_emulate_instruction()
9002 		vcpu->mmio_needed = false;  in x86_emulate_instruction()
9005 	} else if (vcpu->arch.pio.count) {  in x86_emulate_instruction()
9006 		if (!vcpu->arch.pio.in) {  in x86_emulate_instruction()
9007 			/* FIXME: return into emulator if single-stepping.  */  in x86_emulate_instruction()
9008 			vcpu->arch.pio.count = 0;  in x86_emulate_instruction()
9011 			vcpu->arch.complete_userspace_io = complete_emulated_pio;  in x86_emulate_instruction()
9014 	} else if (vcpu->mmio_needed) {  in x86_emulate_instruction()
9015 		++vcpu->stat.mmio_exits;  in x86_emulate_instruction()
9017 		if (!vcpu->mmio_is_write)  in x86_emulate_instruction()
9020 		vcpu->arch.complete_userspace_io = complete_emulated_mmio;  in x86_emulate_instruction()
9021 	} else if (vcpu->arch.complete_userspace_io) {  in x86_emulate_instruction()
9032 		toggle_interruptibility(vcpu, ctxt->interruptibility);  in x86_emulate_instruction()
9033 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;  in x86_emulate_instruction()
9036 		 * Note, EXCPT_DB is assumed to be fault-like as the emulator  in x86_emulate_instruction()
9038 		 * of which are fault-like.  in x86_emulate_instruction()
9040 		if (!ctxt->have_exception ||  in x86_emulate_instruction()
9041 		    exception_type(ctxt->exception.vector) == EXCPT_TRAP) {  in x86_emulate_instruction()
9043 			if (ctxt->is_branch)  in x86_emulate_instruction()
9045 			kvm_rip_write(vcpu, ctxt->eip);  in x86_emulate_instruction()
9046 			if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)))  in x86_emulate_instruction()
9049 			__kvm_set_rflags(vcpu, ctxt->eflags);  in x86_emulate_instruction()
9058 		if (unlikely((ctxt->eflags & ~rflags) & X86_EFLAGS_IF))  in x86_emulate_instruction()
9061 		vcpu->arch.emulate_regs_need_sync_to_vcpu = true;  in x86_emulate_instruction()
9081 	vcpu->arch.pio.count = 0;  in complete_fast_pio_out_port_0x7e()
9087 	vcpu->arch.pio.count = 0;  in complete_fast_pio_out()
9089 	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))  in complete_fast_pio_out()
9109 	    kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {  in kvm_fast_pio_out()
9110 		vcpu->arch.complete_userspace_io =  in kvm_fast_pio_out()
9114 		vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);  in kvm_fast_pio_out()
9115 		vcpu->arch.complete_userspace_io = complete_fast_pio_out;  in kvm_fast_pio_out()
9125 	BUG_ON(vcpu->arch.pio.count != 1);  in complete_fast_pio_in()
9127 	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {  in complete_fast_pio_in()
9128 		vcpu->arch.pio.count = 0;  in complete_fast_pio_in()
9133 	val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;  in complete_fast_pio_in()
9156 	vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);  in kvm_fast_pio_in()
9157 	vcpu->arch.complete_userspace_io = complete_fast_pio_in;  in kvm_fast_pio_in()
9188 		khz = freq->new;  in tsc_khz_changed()
9209 	/* TSC frequency always matches when on Hyper-V */  in kvm_hyperv_tsc_notifier()
9277 			if (vcpu->cpu != cpu)  in __kvmclock_cpufreq_notifier()
9280 			if (vcpu->cpu != raw_smp_processor_id())  in __kvmclock_cpufreq_notifier()
9286 	if (freq->old < freq->new && send_ipi) {  in __kvmclock_cpufreq_notifier()
9309 	if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)  in kvmclock_cpufreq_notifier()
9311 	if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)  in kvmclock_cpufreq_notifier()
9314 	for_each_cpu(cpu, freq->policy->cpus)  in kvmclock_cpufreq_notifier()
9342 				if (policy->cpuinfo.max_freq)  in kvm_timer_init()
9343 					max_tsc_khz = policy->cpuinfo.max_freq;  in kvm_timer_init()
9401 	if (!gtod_is_based_on_tsc(gtod->clock.vclock_mode) &&  in pvclock_gtod_notify()
9414 	memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));  in kvm_ops_update()
9424 #include <asm/kvm-x86-ops.h>  in kvm_ops_update()
9427 	kvm_pmu_ops_update(ops->pmu_ops);  in kvm_ops_update()
9445 		return -EIO;  in kvm_x86_check_processor_compatibility()
9462 		return -EEXIST;  in __kvm_x86_vendor_init()
9472 		return -EOPNOTSUPP;  in __kvm_x86_vendor_init()
9477 		return -EOPNOTSUPP;  in __kvm_x86_vendor_init()
9490 		return -EIO;  in __kvm_x86_vendor_init()
9496 		return -ENOMEM;  in __kvm_x86_vendor_init()
9502 		r = -ENOMEM;  in __kvm_x86_vendor_init()
9521 	kvm_init_pmu_capability(ops->pmu_ops);  in __kvm_x86_vendor_init()
9526 	r = ops->hardware_setup();  in __kvm_x86_vendor_init()
9545 	if (pi_inject_timer == -1)  in __kvm_x86_vendor_init()
9554 	kvm_register_perf_callbacks(ops->handle_intel_pt_intr);  in __kvm_x86_vendor_init()
9640 	 * local APIC is in-kernel, the run loop will detect the non-runnable  in __kvm_emulate_halt()
9645 	++vcpu->stat.halt_exits;  in __kvm_emulate_halt()
9647 		vcpu->arch.mp_state = state;  in __kvm_emulate_halt()
9650 		vcpu->run->exit_reason = reason;  in __kvm_emulate_halt()
9665 	 * TODO: we might be squashing a GUESTDBG_SINGLESTEP-triggered  in kvm_emulate_halt()
9691 		return -KVM_EOPNOTSUPP;  in kvm_pv_clock_pairing()
9697 	if (vcpu->arch.tsc_always_catchup)  in kvm_pv_clock_pairing()
9698 		return -KVM_EOPNOTSUPP;  in kvm_pv_clock_pairing()
9701 		return -KVM_EOPNOTSUPP;  in kvm_pv_clock_pairing()
9710 	if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,  in kvm_pv_clock_pairing()
9712 		ret = -KVM_EFAULT;  in kvm_pv_clock_pairing()
9721  * @apicid - apicid of vcpu to be kicked.
9741 	return (READ_ONCE(kvm->arch.apicv_inhibit_reasons) == 0);  in kvm_apicv_activated()
9747 	ulong vm_reasons = READ_ONCE(vcpu->kvm->arch.apicv_inhibit_reasons);  in kvm_vcpu_apicv_activated()
9767 	unsigned long *inhibits = &kvm->arch.apicv_inhibit_reasons;  in kvm_apicv_init()
9769 	init_rwsem(&kvm->arch.apicv_update_lock);  in kvm_apicv_init()
9783 	vcpu->stat.directed_yield_attempted++;  in kvm_sched_yield()
9789 	map = rcu_dereference(vcpu->kvm->arch.apic_map);  in kvm_sched_yield()
9791 	if (likely(map) && dest_id <= map->max_apic_id && map->phys_map[dest_id])  in kvm_sched_yield()
9792 		target = map->phys_map[dest_id]->vcpu;  in kvm_sched_yield()
9796 	if (!target || !READ_ONCE(target->ready))  in kvm_sched_yield()
9806 	vcpu->stat.directed_yield_successful++;  in kvm_sched_yield()
9814 	u64 ret = vcpu->run->hypercall.ret;  in complete_hypercall_exit()
9819 	++vcpu->stat.hypercalls;  in complete_hypercall_exit()
9828 	if (kvm_xen_hypercall_enabled(vcpu->kvm))  in kvm_emulate_hypercall()
9852 		ret = -KVM_EPERM;  in kvm_emulate_hypercall()
9856 	ret = -KVM_ENOSYS;  in kvm_emulate_hypercall()
9866 		kvm_pv_kick_cpu_op(vcpu->kvm, a1);  in kvm_emulate_hypercall()
9879 		ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);  in kvm_emulate_hypercall()
9891 		ret = -KVM_ENOSYS;  in kvm_emulate_hypercall()
9892 		if (!(vcpu->kvm->arch.hypercall_exit_enabled & (1 << KVM_HC_MAP_GPA_RANGE)))  in kvm_emulate_hypercall()
9897 			ret = -KVM_EINVAL;  in kvm_emulate_hypercall()
9901 		vcpu->run->exit_reason        = KVM_EXIT_HYPERCALL;  in kvm_emulate_hypercall()
9902 		vcpu->run->hypercall.nr       = KVM_HC_MAP_GPA_RANGE;  in kvm_emulate_hypercall()
9903 		vcpu->run->hypercall.args[0]  = gpa;  in kvm_emulate_hypercall()
9904 		vcpu->run->hypercall.args[1]  = npages;  in kvm_emulate_hypercall()
9905 		vcpu->run->hypercall.args[2]  = attrs;  in kvm_emulate_hypercall()
9906 		vcpu->run->hypercall.flags    = 0;  in kvm_emulate_hypercall()
9908 			vcpu->run->hypercall.flags |= KVM_EXIT_HYPERCALL_LONG_MODE;  in kvm_emulate_hypercall()
9910 		WARN_ON_ONCE(vcpu->run->hypercall.flags & KVM_EXIT_HYPERCALL_MBZ);  in kvm_emulate_hypercall()
9911 		vcpu->arch.complete_userspace_io = complete_hypercall_exit;  in kvm_emulate_hypercall()
9915 		ret = -KVM_ENOSYS;  in kvm_emulate_hypercall()
9923 	++vcpu->stat.hypercalls;  in kvm_emulate_hypercall()
9938 	if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_FIX_HYPERCALL_INSN)) {  in emulator_fix_hypercall()
9939 		ctxt->exception.error_code_valid = false;  in emulator_fix_hypercall()
9940 		ctxt->exception.vector = UD_VECTOR;  in emulator_fix_hypercall()
9941 		ctxt->have_exception = true;  in emulator_fix_hypercall()
9948 		&ctxt->exception);  in emulator_fix_hypercall()
9953 	return vcpu->run->request_interrupt_window &&  in dm_request_for_irq_injection()
9954 		likely(!pic_in_kernel(vcpu->kvm));  in dm_request_for_irq_injection()
9957 /* Called within kvm->srcu read side.  */
9960 	struct kvm_run *kvm_run = vcpu->run;  in post_kvm_run_save()
9962 	kvm_run->if_flag = static_call(kvm_x86_get_if_flag)(vcpu);  in post_kvm_run_save()
9963 	kvm_run->cr8 = kvm_get_cr8(vcpu);  in post_kvm_run_save()
9964 	kvm_run->apic_base = kvm_get_apic_base(vcpu);  in post_kvm_run_save()
9966 	kvm_run->ready_for_interrupt_injection =  in post_kvm_run_save()
9967 		pic_in_kernel(vcpu->kvm) ||  in post_kvm_run_save()
9971 		kvm_run->flags |= KVM_RUN_X86_SMM;  in post_kvm_run_save()
9984 	if (vcpu->arch.apic->apicv_active)  in update_cr8_intercept()
9987 	if (!vcpu->arch.apic->vapic_addr)  in update_cr8_intercept()
9990 		max_irr = -1;  in update_cr8_intercept()
9992 	if (max_irr != -1)  in update_cr8_intercept()
10004 		kvm_x86_ops.nested_ops->triple_fault(vcpu);  in kvm_check_nested_events()
10008 	return kvm_x86_ops.nested_ops->check_events(vcpu);  in kvm_check_nested_events()
10017 	 * is injected as intercepted #PF VM-Exits for AMD's Paged Real Mode do  in kvm_inject_exception()
10020 	vcpu->arch.exception.has_error_code &= is_protmode(vcpu);  in kvm_inject_exception()
10022 	trace_kvm_inj_exception(vcpu->arch.exception.vector,  in kvm_inject_exception()
10023 				vcpu->arch.exception.has_error_code,  in kvm_inject_exception()
10024 				vcpu->arch.exception.error_code,  in kvm_inject_exception()
10025 				vcpu->arch.exception.injected);  in kvm_inject_exception()
10035  * injected as part of a previous VM-Enter, but weren't successfully delivered
10036  * and need to be re-injected.
10041  * also be able to re-inject NMIs and IRQs in the middle of an instruction.
10042  * I.e. for exceptions and re-injected events, NOT invoking this on instruction
10047  * instruction boundaries for asynchronous events.  However, because VM-Exits
10053  * But, if a VM-Exit occurs during instruction execution, and KVM does NOT skip
10076 	 * Process nested events first, as nested VM-Exit supercedes event  in kvm_check_and_inject_events()
10077 	 * re-injection.  If there's an event queued for re-injection, it will  in kvm_check_and_inject_events()
10078 	 * be saved into the appropriate vmc{b,s}12 fields on nested VM-Exit.  in kvm_check_and_inject_events()
10086 	 * Re-inject exceptions and events *especially* if immediate entry+exit  in kvm_check_and_inject_events()
10090 	 * Don't re-inject an NMI or interrupt if there is a pending exception.  in kvm_check_and_inject_events()
10099 	 * as the exception "occurred" before the exit to userspace.  Trap-like  in kvm_check_and_inject_events()
10101 	 * And while fault-like exceptions, e.g. #GP and #PF, are the lowest  in kvm_check_and_inject_events()
10104 	 * Thus a pending fault-like exception means the fault occurred on the  in kvm_check_and_inject_events()
10108 	if (vcpu->arch.exception.injected)  in kvm_check_and_inject_events()
10112 	else if (vcpu->arch.nmi_injected)  in kvm_check_and_inject_events()
10114 	else if (vcpu->arch.interrupt.injected)  in kvm_check_and_inject_events()
10118 	 * Exceptions that morph to VM-Exits are handled above, and pending  in kvm_check_and_inject_events()
10119 	 * exceptions on top of injected exceptions that do not VM-Exit should  in kvm_check_and_inject_events()
10122 	WARN_ON_ONCE(vcpu->arch.exception.injected &&  in kvm_check_and_inject_events()
10123 		     vcpu->arch.exception.pending);  in kvm_check_and_inject_events()
10127 	 * nested VM-Enter or event re-injection so that a different pending  in kvm_check_and_inject_events()
10130 	 * Otherwise, continue processing events even if VM-Exit occurred.  The  in kvm_check_and_inject_events()
10131 	 * VM-Exit will have cleared exceptions that were meant for L2, but  in kvm_check_and_inject_events()
10138 	 * A pending exception VM-Exit should either result in nested VM-Exit  in kvm_check_and_inject_events()
10139 	 * or force an immediate re-entry and exit to/from L2, and exception  in kvm_check_and_inject_events()
10140 	 * VM-Exits cannot be injected (flag should _never_ be set).  in kvm_check_and_inject_events()
10142 	WARN_ON_ONCE(vcpu->arch.exception_vmexit.injected ||  in kvm_check_and_inject_events()
10143 		     vcpu->arch.exception_vmexit.pending);  in kvm_check_and_inject_events()
10147 	 * to re-inject a previous event.  See above comments on re-injecting  in kvm_check_and_inject_events()
10152 	if (vcpu->arch.exception.pending) {  in kvm_check_and_inject_events()
10154 		 * Fault-class exceptions, except #DBs, set RF=1 in the RFLAGS  in kvm_check_and_inject_events()
10155 		 * value pushed on the stack.  Trap-like exception and all #DBs  in kvm_check_and_inject_events()
10156 		 * leave RF as-is (KVM follows Intel's behavior in this regard;  in kvm_check_and_inject_events()
10161 		 * fault-like.  They do _not_ set RF, a la code breakpoints.  in kvm_check_and_inject_events()
10163 		if (exception_type(vcpu->arch.exception.vector) == EXCPT_FAULT)  in kvm_check_and_inject_events()
10167 		if (vcpu->arch.exception.vector == DB_VECTOR) {  in kvm_check_and_inject_events()
10168 			kvm_deliver_exception_payload(vcpu, &vcpu->arch.exception);  in kvm_check_and_inject_events()
10169 			if (vcpu->arch.dr7 & DR7_GD) {  in kvm_check_and_inject_events()
10170 				vcpu->arch.dr7 &= ~DR7_GD;  in kvm_check_and_inject_events()
10177 		vcpu->arch.exception.pending = false;  in kvm_check_and_inject_events()
10178 		vcpu->arch.exception.injected = true;  in kvm_check_and_inject_events()
10184 	if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ)  in kvm_check_and_inject_events()
10189 	 * due to architectural conditions (e.g. IF=0) a window-open exit  in kvm_check_and_inject_events()
10190 	 * will re-request KVM_REQ_EVENT.  Sometimes however an event is pending  in kvm_check_and_inject_events()
10196 	 * The kvm_x86_ops hooks communicate this by returning -EBUSY.  in kvm_check_and_inject_events()
10199 	if (vcpu->arch.smi_pending) {  in kvm_check_and_inject_events()
10200 		r = can_inject ? static_call(kvm_x86_smi_allowed)(vcpu, true) : -EBUSY;  in kvm_check_and_inject_events()
10204 			vcpu->arch.smi_pending = false;  in kvm_check_and_inject_events()
10205 			++vcpu->arch.smi_count;  in kvm_check_and_inject_events()
10213 	if (vcpu->arch.nmi_pending) {  in kvm_check_and_inject_events()
10214 		r = can_inject ? static_call(kvm_x86_nmi_allowed)(vcpu, true) : -EBUSY;  in kvm_check_and_inject_events()
10218 			--vcpu->arch.nmi_pending;  in kvm_check_and_inject_events()
10219 			vcpu->arch.nmi_injected = true;  in kvm_check_and_inject_events()
10224 		if (vcpu->arch.nmi_pending)  in kvm_check_and_inject_events()
10229 		r = can_inject ? static_call(kvm_x86_interrupt_allowed)(vcpu, true) : -EBUSY;  in kvm_check_and_inject_events()
10235 			if (!WARN_ON_ONCE(irq == -1)) {  in kvm_check_and_inject_events()
10246 	    kvm_x86_ops.nested_ops->has_events &&  in kvm_check_and_inject_events()
10247 	    kvm_x86_ops.nested_ops->has_events(vcpu))  in kvm_check_and_inject_events()
10252 	 * is done emulating and should only propagate the to-be-injected event  in kvm_check_and_inject_events()
10254 	 * infinite loop as KVM will bail from VM-Enter to inject the pending  in kvm_check_and_inject_events()
10262 	WARN_ON_ONCE(vcpu->arch.exception.pending ||  in kvm_check_and_inject_events()
10263 		     vcpu->arch.exception_vmexit.pending);  in kvm_check_and_inject_events()
10267 	if (r == -EBUSY) {  in kvm_check_and_inject_events()
10288 	if (static_call(kvm_x86_get_nmi_mask)(vcpu) || vcpu->arch.nmi_injected)  in process_nmi()
10295 	 * tracked in vcpu->arch.nmi_pending.  in process_nmi()
10298 		limit--;  in process_nmi()
10300 	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);  in process_nmi()
10301 	vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);  in process_nmi()
10303 	if (vcpu->arch.nmi_pending &&  in process_nmi()
10305 		vcpu->arch.nmi_pending--;  in process_nmi()
10307 	if (vcpu->arch.nmi_pending)  in process_nmi()
10314 	return vcpu->arch.nmi_pending +  in kvm_get_nr_pending_nmis()
10331 	struct kvm_lapic *apic = vcpu->arch.apic;  in __kvm_vcpu_update_apicv()
10337 	down_read(&vcpu->kvm->arch.apicv_update_lock);  in __kvm_vcpu_update_apicv()
10344 	if (apic->apicv_active == activate)  in __kvm_vcpu_update_apicv()
10347 	apic->apicv_active = activate;  in __kvm_vcpu_update_apicv()
10357 	if (!apic->apicv_active)  in __kvm_vcpu_update_apicv()
10362 	up_read(&vcpu->kvm->arch.apicv_update_lock);  in __kvm_vcpu_update_apicv()
10377 	 * running vCPUs, but KVM must not create SPTEs for the APIC base as  in kvm_vcpu_update_apicv()
10382 	if (apic_x2apic_mode(vcpu->arch.apic) &&  in kvm_vcpu_update_apicv()
10394 	lockdep_assert_held_write(&kvm->arch.apicv_update_lock);  in __kvm_set_or_clear_apicv_inhibit()
10399 	old = new = kvm->arch.apicv_inhibit_reasons;  in __kvm_set_or_clear_apicv_inhibit()
10417 		kvm->arch.apicv_inhibit_reasons = new;  in __kvm_set_or_clear_apicv_inhibit()
10420 			int idx = srcu_read_lock(&kvm->srcu);  in __kvm_set_or_clear_apicv_inhibit()
10423 			srcu_read_unlock(&kvm->srcu, idx);  in __kvm_set_or_clear_apicv_inhibit()
10426 		kvm->arch.apicv_inhibit_reasons = new;  in __kvm_set_or_clear_apicv_inhibit()
10436 	down_write(&kvm->arch.apicv_update_lock);  in kvm_set_or_clear_apicv_inhibit()
10438 	up_write(&kvm->arch.apicv_update_lock);  in kvm_set_or_clear_apicv_inhibit()
10447 	bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);  in vcpu_scan_ioapic()
10449 	if (irqchip_split(vcpu->kvm))  in vcpu_scan_ioapic()
10450 		kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);  in vcpu_scan_ioapic()
10453 		if (ioapic_in_kernel(vcpu->kvm))  in vcpu_scan_ioapic()
10454 			kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);  in vcpu_scan_ioapic()
10458 		vcpu->arch.load_eoi_exitmap_pending = true;  in vcpu_scan_ioapic()
10467 	if (!kvm_apic_hw_enabled(vcpu->arch.apic))  in vcpu_load_eoi_exitmap()
10472 			  vcpu->arch.ioapic_handled_vectors,  in vcpu_load_eoi_exitmap()
10473 			  to_hv_synic(vcpu)->vec_bitmap, 256);  in vcpu_load_eoi_exitmap()
10479 		vcpu, (u64 *)vcpu->arch.ioapic_handled_vectors);  in vcpu_load_eoi_exitmap()
10497 	smp_send_reschedule(vcpu->cpu);  in __kvm_request_immediate_exit()
10502  * Called within kvm->srcu read side.
10519 			r = -EIO;  in vcpu_enter_guest()
10529 			if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) {  in vcpu_enter_guest()
10539 			kvm_update_masterclock(vcpu->kvm);  in vcpu_enter_guest()
10563 		 * Fall back to a "full" guest flush if Hyper-V's precise  in vcpu_enter_guest()
10564 		 * flushing fails.  Note, Hyper-V's flushing is per-vCPU, but  in vcpu_enter_guest()
10573 			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;  in vcpu_enter_guest()
10579 				kvm_x86_ops.nested_ops->triple_fault(vcpu);  in vcpu_enter_guest()
10582 				vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;  in vcpu_enter_guest()
10583 				vcpu->mmio_needed = 0;  in vcpu_enter_guest()
10590 			vcpu->arch.apf.halted = true;  in vcpu_enter_guest()
10607 			BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);  in vcpu_enter_guest()
10608 			if (test_bit(vcpu->arch.pending_ioapic_eoi,  in vcpu_enter_guest()
10609 				     vcpu->arch.ioapic_handled_vectors)) {  in vcpu_enter_guest()
10610 				vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;  in vcpu_enter_guest()
10611 				vcpu->run->eoi.vector =  in vcpu_enter_guest()
10612 						vcpu->arch.pending_ioapic_eoi;  in vcpu_enter_guest()
10624 			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;  in vcpu_enter_guest()
10625 			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH;  in vcpu_enter_guest()
10626 			vcpu->run->system_event.ndata = 0;  in vcpu_enter_guest()
10631 			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;  in vcpu_enter_guest()
10632 			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET;  in vcpu_enter_guest()
10633 			vcpu->run->system_event.ndata = 0;  in vcpu_enter_guest()
10640 			vcpu->run->exit_reason = KVM_EXIT_HYPERV;  in vcpu_enter_guest()
10641 			vcpu->run->hyperv = hv_vcpu->exit;  in vcpu_enter_guest()
10648 		 * KVM_REQ_CLOCK_UPDATE, because Hyper-V SynIC timers  in vcpu_enter_guest()
10649 		 * depend on the guest clock being up-to-date  in vcpu_enter_guest()
10666 		++vcpu->stat.req_event;  in vcpu_enter_guest()
10672 		if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {  in vcpu_enter_guest()
10707 	/* Store vcpu->apicv_active before vcpu->mode.  */  in vcpu_enter_guest()
10708 	smp_store_release(&vcpu->mode, IN_GUEST_MODE);  in vcpu_enter_guest()
10713 	 * 1) We should set ->mode before checking ->requests.  Please see  in vcpu_enter_guest()
10716 	 * 2) For APICv, we should set ->mode before checking PID.ON. This  in vcpu_enter_guest()
10737 		vcpu->mode = OUTSIDE_GUEST_MODE;  in vcpu_enter_guest()
10755 	if (vcpu->arch.guest_fpu.xfd_err)  in vcpu_enter_guest()
10756 		wrmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);  in vcpu_enter_guest()
10758 	if (unlikely(vcpu->arch.switch_db_regs)) {  in vcpu_enter_guest()
10760 		set_debugreg(vcpu->arch.eff_db[0], 0);  in vcpu_enter_guest()
10761 		set_debugreg(vcpu->arch.eff_db[1], 1);  in vcpu_enter_guest()
10762 		set_debugreg(vcpu->arch.eff_db[2], 2);  in vcpu_enter_guest()
10763 		set_debugreg(vcpu->arch.eff_db[3], 3);  in vcpu_enter_guest()
10774 		 * per-VM state, and responsing vCPUs must wait for the update  in vcpu_enter_guest()
10792 		/* Note, VM-Exits that go down the "slow" path are accounted below. */  in vcpu_enter_guest()
10793 		++vcpu->stat.exits;  in vcpu_enter_guest()
10802 	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {  in vcpu_enter_guest()
10803 		WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);  in vcpu_enter_guest()
10819 	vcpu->arch.last_vmentry_cpu = vcpu->cpu;  in vcpu_enter_guest()
10820 	vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());  in vcpu_enter_guest()
10822 	vcpu->mode = OUTSIDE_GUEST_MODE;  in vcpu_enter_guest()
10827 	 * rely on the fact that guest_fpu::xfd is up-to-date (e.g.  in vcpu_enter_guest()
10830 	if (vcpu->arch.xfd_no_write_intercept)  in vcpu_enter_guest()
10835 	if (vcpu->arch.guest_fpu.xfd_err)  in vcpu_enter_guest()
10840 	 * VM-Exit on SVM and any ticks that occur between VM-Exit and now.  in vcpu_enter_guest()
10847 	++vcpu->stat.exits;  in vcpu_enter_guest()
10873 	if (unlikely(vcpu->arch.tsc_always_catchup))  in vcpu_enter_guest()
10876 	if (vcpu->arch.apic_attention)  in vcpu_enter_guest()
10886 	if (unlikely(vcpu->arch.apic_attention))  in vcpu_enter_guest()
10892 /* Called within kvm->srcu read side.  */
10899 		 * Switch to the software timer before halt-polling/blocking as  in vcpu_block()
10902 		 * Switch before halt-polling so that KVM recognizes an expired  in vcpu_block()
10910 		if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)  in vcpu_block()
10931 	 * state field (AMD does not have a similar field and a VM-Exit always  in vcpu_block()
10941 	switch(vcpu->arch.mp_state) {  in vcpu_block()
10944 		vcpu->arch.pv.pv_unhalted = false;  in vcpu_block()
10945 		vcpu->arch.mp_state =  in vcpu_block()
10949 		vcpu->arch.apf.halted = false;  in vcpu_block()
10962 	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&  in kvm_vcpu_running()
10963 		!vcpu->arch.apf.halted);  in kvm_vcpu_running()
10966 /* Called within kvm->srcu read side.  */
10971 	vcpu->arch.l1tf_flush_l1d = true;  in vcpu_run()
10980 		vcpu->arch.at_instruction_boundary = false;  in vcpu_run()
11000 			vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;  in vcpu_run()
11001 			++vcpu->stat.request_irq_exits;  in vcpu_run()
11024 	BUG_ON(!vcpu->arch.pio.count);  in complete_emulated_pio()
11049 	struct kvm_run *run = vcpu->run;  in complete_emulated_mmio()
11053 	BUG_ON(!vcpu->mmio_needed);  in complete_emulated_mmio()
11056 	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];  in complete_emulated_mmio()
11057 	len = min(8u, frag->len);  in complete_emulated_mmio()
11058 	if (!vcpu->mmio_is_write)  in complete_emulated_mmio()
11059 		memcpy(frag->data, run->mmio.data, len);  in complete_emulated_mmio()
11061 	if (frag->len <= 8) {  in complete_emulated_mmio()
11064 		vcpu->mmio_cur_fragment++;  in complete_emulated_mmio()
11067 		frag->data += len;  in complete_emulated_mmio()
11068 		frag->gpa += len;  in complete_emulated_mmio()
11069 		frag->len -= len;  in complete_emulated_mmio()
11072 	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {  in complete_emulated_mmio()
11073 		vcpu->mmio_needed = 0;  in complete_emulated_mmio()
11075 		/* FIXME: return into emulator if single-stepping.  */  in complete_emulated_mmio()
11076 		if (vcpu->mmio_is_write)  in complete_emulated_mmio()
11078 		vcpu->mmio_read_completed = 1;  in complete_emulated_mmio()
11082 	run->exit_reason = KVM_EXIT_MMIO;  in complete_emulated_mmio()
11083 	run->mmio.phys_addr = frag->gpa;  in complete_emulated_mmio()
11084 	if (vcpu->mmio_is_write)  in complete_emulated_mmio()
11085 		memcpy(run->mmio.data, frag->data, min(8u, frag->len));  in complete_emulated_mmio()
11086 	run->mmio.len = min(8u, frag->len);  in complete_emulated_mmio()
11087 	run->mmio.is_write = vcpu->mmio_is_write;  in complete_emulated_mmio()
11088 	vcpu->arch.complete_userspace_io = complete_emulated_mmio;  in complete_emulated_mmio()
11095 	/* Exclude PKRU, it's restored separately immediately after VM-Exit. */  in kvm_load_guest_fpu()
11096 	fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, true);  in kvm_load_guest_fpu()
11103 	fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, false);  in kvm_put_guest_fpu()
11104 	++vcpu->stat.fpu_reload;  in kvm_put_guest_fpu()
11110 	struct kvm_queued_exception *ex = &vcpu->arch.exception;  in kvm_arch_vcpu_ioctl_run()
11111 	struct kvm_run *kvm_run = vcpu->run;  in kvm_arch_vcpu_ioctl_run()
11116 	kvm_run->flags = 0;  in kvm_arch_vcpu_ioctl_run()
11120 	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {  in kvm_arch_vcpu_ioctl_run()
11121 		if (kvm_run->immediate_exit) {  in kvm_arch_vcpu_ioctl_run()
11122 			r = -EINTR;  in kvm_arch_vcpu_ioctl_run()
11144 		r = -EAGAIN;  in kvm_arch_vcpu_ioctl_run()
11146 			r = -EINTR;  in kvm_arch_vcpu_ioctl_run()
11147 			kvm_run->exit_reason = KVM_EXIT_INTR;  in kvm_arch_vcpu_ioctl_run()
11148 			++vcpu->stat.signal_exits;  in kvm_arch_vcpu_ioctl_run()
11153 	if ((kvm_run->kvm_valid_regs & ~KVM_SYNC_X86_VALID_FIELDS) ||  in kvm_arch_vcpu_ioctl_run()
11154 	    (kvm_run->kvm_dirty_regs & ~KVM_SYNC_X86_VALID_FIELDS)) {  in kvm_arch_vcpu_ioctl_run()
11155 		r = -EINVAL;  in kvm_arch_vcpu_ioctl_run()
11159 	if (kvm_run->kvm_dirty_regs) {  in kvm_arch_vcpu_ioctl_run()
11165 	/* re-sync apic's tpr */  in kvm_arch_vcpu_ioctl_run()
11167 		if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {  in kvm_arch_vcpu_ioctl_run()
11168 			r = -EINVAL;  in kvm_arch_vcpu_ioctl_run()
11175 	 * a pending VM-Exit if L1 wants to intercept the exception.  in kvm_arch_vcpu_ioctl_run()
11177 	if (vcpu->arch.exception_from_userspace && is_guest_mode(vcpu) &&  in kvm_arch_vcpu_ioctl_run()
11178 	    kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, ex->vector,  in kvm_arch_vcpu_ioctl_run()
11179 							ex->error_code)) {  in kvm_arch_vcpu_ioctl_run()
11180 		kvm_queue_exception_vmexit(vcpu, ex->vector,  in kvm_arch_vcpu_ioctl_run()
11181 					   ex->has_error_code, ex->error_code,  in kvm_arch_vcpu_ioctl_run()
11182 					   ex->has_payload, ex->payload);  in kvm_arch_vcpu_ioctl_run()
11183 		ex->injected = false;  in kvm_arch_vcpu_ioctl_run()
11184 		ex->pending = false;  in kvm_arch_vcpu_ioctl_run()
11186 	vcpu->arch.exception_from_userspace = false;  in kvm_arch_vcpu_ioctl_run()
11188 	if (unlikely(vcpu->arch.complete_userspace_io)) {  in kvm_arch_vcpu_ioctl_run()
11189 		int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;  in kvm_arch_vcpu_ioctl_run()
11190 		vcpu->arch.complete_userspace_io = NULL;  in kvm_arch_vcpu_ioctl_run()
11195 		WARN_ON_ONCE(vcpu->arch.pio.count);  in kvm_arch_vcpu_ioctl_run()
11196 		WARN_ON_ONCE(vcpu->mmio_needed);  in kvm_arch_vcpu_ioctl_run()
11199 	if (kvm_run->immediate_exit) {  in kvm_arch_vcpu_ioctl_run()
11200 		r = -EINTR;  in kvm_arch_vcpu_ioctl_run()
11212 	if (kvm_run->kvm_valid_regs)  in kvm_arch_vcpu_ioctl_run()
11224 	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {  in __get_regs()
11232 		emulator_writeback_register_cache(vcpu->arch.emulate_ctxt);  in __get_regs()
11233 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;  in __get_regs()
11235 	regs->rax = kvm_rax_read(vcpu);  in __get_regs()
11236 	regs->rbx = kvm_rbx_read(vcpu);  in __get_regs()
11237 	regs->rcx = kvm_rcx_read(vcpu);  in __get_regs()
11238 	regs->rdx = kvm_rdx_read(vcpu);  in __get_regs()
11239 	regs->rsi = kvm_rsi_read(vcpu);  in __get_regs()
11240 	regs->rdi = kvm_rdi_read(vcpu);  in __get_regs()
11241 	regs->rsp = kvm_rsp_read(vcpu);  in __get_regs()
11242 	regs->rbp = kvm_rbp_read(vcpu);  in __get_regs()
11244 	regs->r8 = kvm_r8_read(vcpu);  in __get_regs()
11245 	regs->r9 = kvm_r9_read(vcpu);  in __get_regs()
11246 	regs->r10 = kvm_r10_read(vcpu);  in __get_regs()
11247 	regs->r11 = kvm_r11_read(vcpu);  in __get_regs()
11248 	regs->r12 = kvm_r12_read(vcpu);  in __get_regs()
11249 	regs->r13 = kvm_r13_read(vcpu);  in __get_regs()
11250 	regs->r14 = kvm_r14_read(vcpu);  in __get_regs()
11251 	regs->r15 = kvm_r15_read(vcpu);  in __get_regs()
11254 	regs->rip = kvm_rip_read(vcpu);  in __get_regs()
11255 	regs->rflags = kvm_get_rflags(vcpu);  in __get_regs()
11268 	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;  in __set_regs()
11269 	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;  in __set_regs()
11271 	kvm_rax_write(vcpu, regs->rax);  in __set_regs()
11272 	kvm_rbx_write(vcpu, regs->rbx);  in __set_regs()
11273 	kvm_rcx_write(vcpu, regs->rcx);  in __set_regs()
11274 	kvm_rdx_write(vcpu, regs->rdx);  in __set_regs()
11275 	kvm_rsi_write(vcpu, regs->rsi);  in __set_regs()
11276 	kvm_rdi_write(vcpu, regs->rdi);  in __set_regs()
11277 	kvm_rsp_write(vcpu, regs->rsp);  in __set_regs()
11278 	kvm_rbp_write(vcpu, regs->rbp);  in __set_regs()
11280 	kvm_r8_write(vcpu, regs->r8);  in __set_regs()
11281 	kvm_r9_write(vcpu, regs->r9);  in __set_regs()
11282 	kvm_r10_write(vcpu, regs->r10);  in __set_regs()
11283 	kvm_r11_write(vcpu, regs->r11);  in __set_regs()
11284 	kvm_r12_write(vcpu, regs->r12);  in __set_regs()
11285 	kvm_r13_write(vcpu, regs->r13);  in __set_regs()
11286 	kvm_r14_write(vcpu, regs->r14);  in __set_regs()
11287 	kvm_r15_write(vcpu, regs->r15);  in __set_regs()
11290 	kvm_rip_write(vcpu, regs->rip);  in __set_regs()
11291 	kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);  in __set_regs()
11293 	vcpu->arch.exception.pending = false;  in __set_regs()
11294 	vcpu->arch.exception_vmexit.pending = false;  in __set_regs()
11311 	if (vcpu->arch.guest_state_protected)  in __get_sregs_common()
11314 	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);  in __get_sregs_common()
11315 	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);  in __get_sregs_common()
11316 	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);  in __get_sregs_common()
11317 	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);  in __get_sregs_common()
11318 	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);  in __get_sregs_common()
11319 	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);  in __get_sregs_common()
11321 	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);  in __get_sregs_common()
11322 	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);  in __get_sregs_common()
11325 	sregs->idt.limit = dt.size;  in __get_sregs_common()
11326 	sregs->idt.base = dt.address;  in __get_sregs_common()
11328 	sregs->gdt.limit = dt.size;  in __get_sregs_common()
11329 	sregs->gdt.base = dt.address;  in __get_sregs_common()
11331 	sregs->cr2 = vcpu->arch.cr2;  in __get_sregs_common()
11332 	sregs->cr3 = kvm_read_cr3(vcpu);  in __get_sregs_common()
11335 	sregs->cr0 = kvm_read_cr0(vcpu);  in __get_sregs_common()
11336 	sregs->cr4 = kvm_read_cr4(vcpu);  in __get_sregs_common()
11337 	sregs->cr8 = kvm_get_cr8(vcpu);  in __get_sregs_common()
11338 	sregs->efer = vcpu->arch.efer;  in __get_sregs_common()
11339 	sregs->apic_base = kvm_get_apic_base(vcpu);  in __get_sregs_common()
11346 	if (vcpu->arch.guest_state_protected)  in __get_sregs()
11349 	if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)  in __get_sregs()
11350 		set_bit(vcpu->arch.interrupt.nr,  in __get_sregs()
11351 			(unsigned long *)sregs->interrupt_bitmap);  in __get_sregs()
11360 	if (vcpu->arch.guest_state_protected)  in __get_sregs2()
11365 			sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i);  in __get_sregs2()
11366 		sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID;  in __get_sregs2()
11393 	if ((vcpu->arch.mp_state == KVM_MP_STATE_HALTED ||  in kvm_arch_vcpu_ioctl_get_mpstate()
11394 	     vcpu->arch.mp_state == KVM_MP_STATE_AP_RESET_HOLD) &&  in kvm_arch_vcpu_ioctl_get_mpstate()
11395 	    vcpu->arch.pv.pv_unhalted)  in kvm_arch_vcpu_ioctl_get_mpstate()
11396 		mp_state->mp_state = KVM_MP_STATE_RUNNABLE;  in kvm_arch_vcpu_ioctl_get_mpstate()
11398 		mp_state->mp_state = vcpu->arch.mp_state;  in kvm_arch_vcpu_ioctl_get_mpstate()
11410 	int ret = -EINVAL;  in kvm_arch_vcpu_ioctl_set_mpstate()
11414 	switch (mp_state->mp_state) {  in kvm_arch_vcpu_ioctl_set_mpstate()
11437 	if ((!kvm_apic_init_sipi_allowed(vcpu) || vcpu->arch.smi_pending) &&  in kvm_arch_vcpu_ioctl_set_mpstate()
11438 	    (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED ||  in kvm_arch_vcpu_ioctl_set_mpstate()
11439 	     mp_state->mp_state == KVM_MP_STATE_INIT_RECEIVED))  in kvm_arch_vcpu_ioctl_set_mpstate()
11442 	if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) {  in kvm_arch_vcpu_ioctl_set_mpstate()
11443 		vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;  in kvm_arch_vcpu_ioctl_set_mpstate()
11444 		set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);  in kvm_arch_vcpu_ioctl_set_mpstate()
11446 		vcpu->arch.mp_state = mp_state->mp_state;  in kvm_arch_vcpu_ioctl_set_mpstate()
11458 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in kvm_task_switch()
11466 		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;  in kvm_task_switch()
11467 		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;  in kvm_task_switch()
11468 		vcpu->run->internal.ndata = 0;  in kvm_task_switch()
11472 	kvm_rip_write(vcpu, ctxt->eip);  in kvm_task_switch()
11473 	kvm_set_rflags(vcpu, ctxt->eflags);  in kvm_task_switch()
11480 	if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) {  in kvm_is_valid_sregs()
11483 		 * 64-bit mode (though maybe in a 32-bit code segment).  in kvm_is_valid_sregs()
11486 		if (!(sregs->cr4 & X86_CR4_PAE) || !(sregs->efer & EFER_LMA))  in kvm_is_valid_sregs()
11488 		if (kvm_vcpu_is_illegal_gpa(vcpu, sregs->cr3))  in kvm_is_valid_sregs()
11492 		 * Not in 64-bit mode: EFER.LMA is clear and the code  in kvm_is_valid_sregs()
11493 		 * segment cannot be 64-bit.  in kvm_is_valid_sregs()
11495 		if (sregs->efer & EFER_LMA || sregs->cs.l)  in kvm_is_valid_sregs()
11499 	return kvm_is_valid_cr4(vcpu, sregs->cr4) &&  in kvm_is_valid_sregs()
11500 	       kvm_is_valid_cr0(vcpu, sregs->cr0);  in kvm_is_valid_sregs()
11511 		return -EINVAL;  in __set_sregs_common()
11513 	apic_base_msr.data = sregs->apic_base;  in __set_sregs_common()
11516 		return -EINVAL;  in __set_sregs_common()
11518 	if (vcpu->arch.guest_state_protected)  in __set_sregs_common()
11521 	dt.size = sregs->idt.limit;  in __set_sregs_common()
11522 	dt.address = sregs->idt.base;  in __set_sregs_common()
11524 	dt.size = sregs->gdt.limit;  in __set_sregs_common()
11525 	dt.address = sregs->gdt.base;  in __set_sregs_common()
11528 	vcpu->arch.cr2 = sregs->cr2;  in __set_sregs_common()
11529 	*mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;  in __set_sregs_common()
11530 	vcpu->arch.cr3 = sregs->cr3;  in __set_sregs_common()
11532 	static_call_cond(kvm_x86_post_set_cr3)(vcpu, sregs->cr3);  in __set_sregs_common()
11534 	kvm_set_cr8(vcpu, sregs->cr8);  in __set_sregs_common()
11536 	*mmu_reset_needed |= vcpu->arch.efer != sregs->efer;  in __set_sregs_common()
11537 	static_call(kvm_x86_set_efer)(vcpu, sregs->efer);  in __set_sregs_common()
11539 	*mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;  in __set_sregs_common()
11540 	static_call(kvm_x86_set_cr0)(vcpu, sregs->cr0);  in __set_sregs_common()
11541 	vcpu->arch.cr0 = sregs->cr0;  in __set_sregs_common()
11543 	*mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;  in __set_sregs_common()
11544 	static_call(kvm_x86_set_cr4)(vcpu, sregs->cr4);  in __set_sregs_common()
11547 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in __set_sregs_common()
11552 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in __set_sregs_common()
11555 	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);  in __set_sregs_common()
11556 	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);  in __set_sregs_common()
11557 	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);  in __set_sregs_common()
11558 	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);  in __set_sregs_common()
11559 	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);  in __set_sregs_common()
11560 	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);  in __set_sregs_common()
11562 	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);  in __set_sregs_common()
11563 	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);  in __set_sregs_common()
11569 	    sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&  in __set_sregs_common()
11571 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;  in __set_sregs_common()
11590 		(const unsigned long *)sregs->interrupt_bitmap, max_bits);  in __set_sregs()
11603 	bool valid_pdptrs = sregs2->flags & KVM_SREGS2_FLAGS_PDPTRS_VALID;  in __set_sregs2()
11604 	bool pae = (sregs2->cr0 & X86_CR0_PG) && (sregs2->cr4 & X86_CR4_PAE) &&  in __set_sregs2()
11605 		!(sregs2->efer & EFER_LMA);  in __set_sregs2()
11608 	if (sregs2->flags & ~KVM_SREGS2_FLAGS_PDPTRS_VALID)  in __set_sregs2()
11609 		return -EINVAL;  in __set_sregs2()
11611 	if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected))  in __set_sregs2()
11612 		return -EINVAL;  in __set_sregs2()
11621 			kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]);  in __set_sregs2()
11625 		vcpu->arch.pdptrs_from_userspace = true;  in __set_sregs2()
11652 	down_write(&kvm->arch.apicv_update_lock);  in kvm_arch_vcpu_guestdbg_update_apicv_inhibit()
11655 		if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) {  in kvm_arch_vcpu_guestdbg_update_apicv_inhibit()
11661 	up_write(&kvm->arch.apicv_update_lock);  in kvm_arch_vcpu_guestdbg_update_apicv_inhibit()
11670 	if (vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_ioctl_set_guest_debug()
11671 		return -EINVAL;  in kvm_arch_vcpu_ioctl_set_guest_debug()
11675 	if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {  in kvm_arch_vcpu_ioctl_set_guest_debug()
11676 		r = -EBUSY;  in kvm_arch_vcpu_ioctl_set_guest_debug()
11679 		if (dbg->control & KVM_GUESTDBG_INJECT_DB)  in kvm_arch_vcpu_ioctl_set_guest_debug()
11691 	vcpu->guest_debug = dbg->control;  in kvm_arch_vcpu_ioctl_set_guest_debug()
11692 	if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))  in kvm_arch_vcpu_ioctl_set_guest_debug()
11693 		vcpu->guest_debug = 0;  in kvm_arch_vcpu_ioctl_set_guest_debug()
11695 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {  in kvm_arch_vcpu_ioctl_set_guest_debug()
11697 			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];  in kvm_arch_vcpu_ioctl_set_guest_debug()
11698 		vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];  in kvm_arch_vcpu_ioctl_set_guest_debug()
11701 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];  in kvm_arch_vcpu_ioctl_set_guest_debug()
11705 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)  in kvm_arch_vcpu_ioctl_set_guest_debug()
11706 		vcpu->arch.singlestep_rip = kvm_get_linear_rip(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11716 	kvm_arch_vcpu_guestdbg_update_apicv_inhibit(vcpu->kvm);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11731 	unsigned long vaddr = tr->linear_address;  in kvm_arch_vcpu_ioctl_translate()
11737 	idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl_translate()
11739 	srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl_translate()
11740 	tr->physical_address = gpa;  in kvm_arch_vcpu_ioctl_translate()
11741 	tr->valid = gpa != INVALID_GPA;  in kvm_arch_vcpu_ioctl_translate()
11742 	tr->writeable = 1;  in kvm_arch_vcpu_ioctl_translate()
11743 	tr->usermode = 0;  in kvm_arch_vcpu_ioctl_translate()
11753 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_arch_vcpu_ioctl_get_fpu()
11758 	fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;  in kvm_arch_vcpu_ioctl_get_fpu()
11759 	memcpy(fpu->fpr, fxsave->st_space, 128);  in kvm_arch_vcpu_ioctl_get_fpu()
11760 	fpu->fcw = fxsave->cwd;  in kvm_arch_vcpu_ioctl_get_fpu()
11761 	fpu->fsw = fxsave->swd;  in kvm_arch_vcpu_ioctl_get_fpu()
11762 	fpu->ftwx = fxsave->twd;  in kvm_arch_vcpu_ioctl_get_fpu()
11763 	fpu->last_opcode = fxsave->fop;  in kvm_arch_vcpu_ioctl_get_fpu()
11764 	fpu->last_ip = fxsave->rip;  in kvm_arch_vcpu_ioctl_get_fpu()
11765 	fpu->last_dp = fxsave->rdp;  in kvm_arch_vcpu_ioctl_get_fpu()
11766 	memcpy(fpu->xmm, fxsave->xmm_space, sizeof(fxsave->xmm_space));  in kvm_arch_vcpu_ioctl_get_fpu()
11776 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_arch_vcpu_ioctl_set_fpu()
11781 	fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;  in kvm_arch_vcpu_ioctl_set_fpu()
11783 	memcpy(fxsave->st_space, fpu->fpr, 128);  in kvm_arch_vcpu_ioctl_set_fpu()
11784 	fxsave->cwd = fpu->fcw;  in kvm_arch_vcpu_ioctl_set_fpu()
11785 	fxsave->swd = fpu->fsw;  in kvm_arch_vcpu_ioctl_set_fpu()
11786 	fxsave->twd = fpu->ftwx;  in kvm_arch_vcpu_ioctl_set_fpu()
11787 	fxsave->fop = fpu->last_opcode;  in kvm_arch_vcpu_ioctl_set_fpu()
11788 	fxsave->rip = fpu->last_ip;  in kvm_arch_vcpu_ioctl_set_fpu()
11789 	fxsave->rdp = fpu->last_dp;  in kvm_arch_vcpu_ioctl_set_fpu()
11790 	memcpy(fxsave->xmm_space, fpu->xmm, sizeof(fxsave->xmm_space));  in kvm_arch_vcpu_ioctl_set_fpu()
11800 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)  in store_regs()
11801 		__get_regs(vcpu, &vcpu->run->s.regs.regs);  in store_regs()
11803 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)  in store_regs()
11804 		__get_sregs(vcpu, &vcpu->run->s.regs.sregs);  in store_regs()
11806 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)  in store_regs()
11808 				vcpu, &vcpu->run->s.regs.events);  in store_regs()
11813 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {  in sync_regs()
11814 		__set_regs(vcpu, &vcpu->run->s.regs.regs);  in sync_regs()
11815 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;  in sync_regs()
11818 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {  in sync_regs()
11819 		struct kvm_sregs sregs = vcpu->run->s.regs.sregs;  in sync_regs()
11822 			return -EINVAL;  in sync_regs()
11824 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;  in sync_regs()
11827 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {  in sync_regs()
11828 		struct kvm_vcpu_events events = vcpu->run->s.regs.events;  in sync_regs()
11831 			return -EINVAL;  in sync_regs()
11833 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;  in sync_regs()
11841 	if (kvm_check_tsc_unstable() && kvm->created_vcpus)  in kvm_arch_vcpu_precreate()
11845 	if (!kvm->arch.max_vcpu_ids)  in kvm_arch_vcpu_precreate()
11846 		kvm->arch.max_vcpu_ids = KVM_MAX_VCPU_IDS;  in kvm_arch_vcpu_precreate()
11848 	if (id >= kvm->arch.max_vcpu_ids)  in kvm_arch_vcpu_precreate()
11849 		return -EINVAL;  in kvm_arch_vcpu_precreate()
11859 	vcpu->arch.last_vmentry_cpu = -1;  in kvm_arch_vcpu_create()
11860 	vcpu->arch.regs_avail = ~0;  in kvm_arch_vcpu_create()
11861 	vcpu->arch.regs_dirty = ~0;  in kvm_arch_vcpu_create()
11863 	kvm_gpc_init(&vcpu->arch.pv_time, vcpu->kvm, vcpu, KVM_HOST_USES_PFN);  in kvm_arch_vcpu_create()
11865 	if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))  in kvm_arch_vcpu_create()
11866 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;  in kvm_arch_vcpu_create()
11868 		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;  in kvm_arch_vcpu_create()
11874 	if (irqchip_in_kernel(vcpu->kvm)) {  in kvm_arch_vcpu_create()
11885 		 * Ignore the current per-VM APICv state so that vCPU creation  in kvm_arch_vcpu_create()
11887 		 * will ensure the vCPU gets the correct state before VM-Entry.  in kvm_arch_vcpu_create()
11890 			vcpu->arch.apic->apicv_active = true;  in kvm_arch_vcpu_create()
11896 	r = -ENOMEM;  in kvm_arch_vcpu_create()
11901 	vcpu->arch.pio_data = page_address(page);  in kvm_arch_vcpu_create()
11903 	vcpu->arch.mce_banks = kcalloc(KVM_MAX_MCE_BANKS * 4, sizeof(u64),  in kvm_arch_vcpu_create()
11905 	vcpu->arch.mci_ctl2_banks = kcalloc(KVM_MAX_MCE_BANKS, sizeof(u64),  in kvm_arch_vcpu_create()
11907 	if (!vcpu->arch.mce_banks || !vcpu->arch.mci_ctl2_banks)  in kvm_arch_vcpu_create()
11909 	vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;  in kvm_arch_vcpu_create()
11911 	if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,  in kvm_arch_vcpu_create()
11918 	if (!fpu_alloc_guest_fpstate(&vcpu->arch.guest_fpu)) {  in kvm_arch_vcpu_create()
11923 	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);  in kvm_arch_vcpu_create()
11924 	vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu);  in kvm_arch_vcpu_create()
11926 	vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;  in kvm_arch_vcpu_create()
11930 	vcpu->arch.perf_capabilities = kvm_caps.supported_perf_cap;  in kvm_arch_vcpu_create()
11933 	vcpu->arch.pending_external_vector = -1;  in kvm_arch_vcpu_create()
11934 	vcpu->arch.preempted_in_kernel = false;  in kvm_arch_vcpu_create()
11937 	vcpu->arch.hv_root_tdp = INVALID_PAGE;  in kvm_arch_vcpu_create()
11944 	vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();  in kvm_arch_vcpu_create()
11945 	vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;  in kvm_arch_vcpu_create()
11949 	kvm_set_tsc_khz(vcpu, vcpu->kvm->arch.default_tsc_khz);  in kvm_arch_vcpu_create()
11956 	fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);  in kvm_arch_vcpu_create()
11958 	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);  in kvm_arch_vcpu_create()
11960 	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);  in kvm_arch_vcpu_create()
11962 	kfree(vcpu->arch.mce_banks);  in kvm_arch_vcpu_create()
11963 	kfree(vcpu->arch.mci_ctl2_banks);  in kvm_arch_vcpu_create()
11964 	free_page((unsigned long)vcpu->arch.pio_data);  in kvm_arch_vcpu_create()
11974 	struct kvm *kvm = vcpu->kvm;  in kvm_arch_vcpu_postcreate()
11976 	if (mutex_lock_killable(&vcpu->mutex))  in kvm_arch_vcpu_postcreate()
11983 	vcpu->arch.msr_kvm_poll_control = 1;  in kvm_arch_vcpu_postcreate()
11985 	mutex_unlock(&vcpu->mutex);  in kvm_arch_vcpu_postcreate()
11987 	if (kvmclock_periodic_sync && vcpu->vcpu_idx == 0)  in kvm_arch_vcpu_postcreate()
11988 		schedule_delayed_work(&kvm->arch.kvmclock_sync_work,  in kvm_arch_vcpu_postcreate()
12000 	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);  in kvm_arch_vcpu_destroy()
12001 	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);  in kvm_arch_vcpu_destroy()
12002 	fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);  in kvm_arch_vcpu_destroy()
12007 	kfree(vcpu->arch.mce_banks);  in kvm_arch_vcpu_destroy()
12008 	kfree(vcpu->arch.mci_ctl2_banks);  in kvm_arch_vcpu_destroy()
12010 	idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_destroy()
12012 	srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_destroy()
12013 	free_page((unsigned long)vcpu->arch.pio_data);  in kvm_arch_vcpu_destroy()
12014 	kvfree(vcpu->arch.cpuid_entries);  in kvm_arch_vcpu_destroy()
12026 	 * Several of the "set" flows, e.g. ->set_cr0(), read other registers  in kvm_vcpu_reset()
12036 	 * SVM doesn't unconditionally VM-Exit on INIT and SHUTDOWN, thus it's  in kvm_vcpu_reset()
12047 	vcpu->arch.hflags = 0;  in kvm_vcpu_reset()
12049 	vcpu->arch.smi_pending = 0;  in kvm_vcpu_reset()
12050 	vcpu->arch.smi_count = 0;  in kvm_vcpu_reset()
12051 	atomic_set(&vcpu->arch.nmi_queued, 0);  in kvm_vcpu_reset()
12052 	vcpu->arch.nmi_pending = 0;  in kvm_vcpu_reset()
12053 	vcpu->arch.nmi_injected = false;  in kvm_vcpu_reset()
12057 	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));  in kvm_vcpu_reset()
12059 	vcpu->arch.dr6 = DR6_ACTIVE_LOW;  in kvm_vcpu_reset()
12060 	vcpu->arch.dr7 = DR7_FIXED_1;  in kvm_vcpu_reset()
12063 	vcpu->arch.cr2 = 0;  in kvm_vcpu_reset()
12066 	vcpu->arch.apf.msr_en_val = 0;  in kvm_vcpu_reset()
12067 	vcpu->arch.apf.msr_int_val = 0;  in kvm_vcpu_reset()
12068 	vcpu->arch.st.msr_val = 0;  in kvm_vcpu_reset()
12074 	vcpu->arch.apf.halted = false;  in kvm_vcpu_reset()
12076 	if (vcpu->arch.guest_fpu.fpstate && kvm_mpx_supported()) {  in kvm_vcpu_reset()
12077 		struct fpstate *fpstate = vcpu->arch.guest_fpu.fpstate;  in kvm_vcpu_reset()
12095 		vcpu->arch.smbase = 0x30000;  in kvm_vcpu_reset()
12097 		vcpu->arch.msr_misc_features_enables = 0;  in kvm_vcpu_reset()
12098 		vcpu->arch.ia32_misc_enable_msr = MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL |  in kvm_vcpu_reset()
12106 	memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));  in kvm_vcpu_reset()
12117 	kvm_rdx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600);  in kvm_vcpu_reset()
12124 	vcpu->arch.cr3 = 0;  in kvm_vcpu_reset()
12146 	 * which PCIDs have to be flushed.  However, CR0.WP and the paging-related  in kvm_vcpu_reset()
12176 	cs.base = vector << 12;  in kvm_vcpu_deliver_sipi_vector()
12206 			if (!stable && vcpu->cpu == smp_processor_id())  in kvm_arch_hardware_enable()
12208 			if (stable && vcpu->arch.last_host_tsc > local_tsc) {  in kvm_arch_hardware_enable()
12210 				if (vcpu->arch.last_host_tsc > max_tsc)  in kvm_arch_hardware_enable()
12211 					max_tsc = vcpu->arch.last_host_tsc;  in kvm_arch_hardware_enable()
12241 	 * N.B. - this code below runs only on platforms with reliable TSC,  in kvm_arch_hardware_enable()
12255 		u64 delta_cyc = max_tsc - local_tsc;  in kvm_arch_hardware_enable()
12257 			kvm->arch.backwards_tsc_observed = true;  in kvm_arch_hardware_enable()
12259 				vcpu->arch.tsc_offset_adjustment += delta_cyc;  in kvm_arch_hardware_enable()
12260 				vcpu->arch.last_host_tsc = local_tsc;  in kvm_arch_hardware_enable()
12270 			kvm->arch.last_tsc_nsec = 0;  in kvm_arch_hardware_enable()
12271 			kvm->arch.last_tsc_write = 0;  in kvm_arch_hardware_enable()
12286 	return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;  in kvm_vcpu_is_reset_bsp()
12291 	return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;  in kvm_vcpu_is_bsp()
12301 	vcpu->arch.l1tf_flush_l1d = true;  in kvm_arch_sched_in()
12302 	if (pmu->version && unlikely(pmu->event_count)) {  in kvm_arch_sched_in()
12303 		pmu->need_cleanup = true;  in kvm_arch_sched_in()
12311 	kfree(to_kvm_hv(kvm)->hv_pa_pg);  in kvm_arch_free_vm()
12322 		return -EINVAL;  in kvm_arch_init_vm()
12334 	INIT_HLIST_HEAD(&kvm->arch.mask_notifier_list);  in kvm_arch_init_vm()
12335 	INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);  in kvm_arch_init_vm()
12336 	atomic_set(&kvm->arch.noncoherent_dma_count, 0);  in kvm_arch_init_vm()
12339 	set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);  in kvm_arch_init_vm()
12340 	/* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */  in kvm_arch_init_vm()
12342 		&kvm->arch.irq_sources_bitmap);  in kvm_arch_init_vm()
12344 	raw_spin_lock_init(&kvm->arch.tsc_write_lock);  in kvm_arch_init_vm()
12345 	mutex_init(&kvm->arch.apic_map_lock);  in kvm_arch_init_vm()
12346 	seqcount_raw_spinlock_init(&kvm->arch.pvclock_sc, &kvm->arch.tsc_write_lock);  in kvm_arch_init_vm()
12347 	kvm->arch.kvmclock_offset = -get_kvmclock_base_ns();  in kvm_arch_init_vm()
12349 	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);  in kvm_arch_init_vm()
12351 	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);  in kvm_arch_init_vm()
12353 	kvm->arch.default_tsc_khz = max_tsc_khz ? : tsc_khz;  in kvm_arch_init_vm()
12354 	kvm->arch.guest_can_read_msr_platform_info = true;  in kvm_arch_init_vm()
12355 	kvm->arch.enable_pmu = enable_pmu;  in kvm_arch_init_vm()
12358 	spin_lock_init(&kvm->arch.hv_root_tdp_lock);  in kvm_arch_init_vm()
12359 	kvm->arch.hv_root_tdp = INVALID_PAGE;  in kvm_arch_init_vm()
12362 	INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn);  in kvm_arch_init_vm()
12363 	INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn);  in kvm_arch_init_vm()
12403 	cancel_delayed_work_sync(&kvm->arch.kvmclock_sync_work);  in kvm_arch_sync_events()
12404 	cancel_delayed_work_sync(&kvm->arch.kvmclock_update_work);  in kvm_arch_sync_events()
12421  *   -errno:        on error
12426  * GPA->HVA translation will not change.  However, the HVA is a user
12438 	/* Called with kvm->slots_lock held.  */  in __x86_set_memory_region()
12440 		return ERR_PTR_USR(-EINVAL);  in __x86_set_memory_region()
12444 		if (slot && slot->npages)  in __x86_set_memory_region()
12445 			return ERR_PTR_USR(-EEXIST);  in __x86_set_memory_region()
12456 		if (!slot || !slot->npages)  in __x86_set_memory_region()
12459 		old_npages = slot->npages;  in __x86_set_memory_region()
12460 		hva = slot->userspace_addr;  in __x86_set_memory_region()
12490 	if (current->mm == kvm->mm) {  in kvm_arch_destroy_vm()
12496 		mutex_lock(&kvm->slots_lock);  in kvm_arch_destroy_vm()
12502 		mutex_unlock(&kvm->slots_lock);  in kvm_arch_destroy_vm()
12506 	kvm_free_msr_filter(srcu_dereference_check(kvm->arch.msr_filter, &kvm->srcu, 1));  in kvm_arch_destroy_vm()
12510 	kvfree(rcu_dereference_check(kvm->arch.apic_map, 1));  in kvm_arch_destroy_vm()
12511 	kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1));  in kvm_arch_destroy_vm()
12523 		kvfree(slot->arch.rmap[i]);  in memslot_rmap_free()
12524 		slot->arch.rmap[i] = NULL;  in memslot_rmap_free()
12535 		kvfree(slot->arch.lpage_info[i - 1]);  in kvm_arch_free_memslot()
12536 		slot->arch.lpage_info[i - 1] = NULL;  in kvm_arch_free_memslot()
12544 	const int sz = sizeof(*slot->arch.rmap[0]);  in memslot_rmap_alloc()
12551 		if (slot->arch.rmap[i])  in memslot_rmap_alloc()
12554 		slot->arch.rmap[i] = __vcalloc(lpages, sz, GFP_KERNEL_ACCOUNT);  in memslot_rmap_alloc()
12555 		if (!slot->arch.rmap[i]) {  in memslot_rmap_alloc()
12557 			return -ENOMEM;  in memslot_rmap_alloc()
12567 	unsigned long npages = slot->npages;  in kvm_alloc_memslot_metadata()
12575 	memset(&slot->arch, 0, sizeof(slot->arch));  in kvm_alloc_memslot_metadata()
12595 		slot->arch.lpage_info[i - 1] = linfo;  in kvm_alloc_memslot_metadata()
12597 		if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))  in kvm_alloc_memslot_metadata()
12599 		if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1))  in kvm_alloc_memslot_metadata()
12600 			linfo[lpages - 1].disallow_lpage = 1;  in kvm_alloc_memslot_metadata()
12601 		ugfn = slot->userspace_addr >> PAGE_SHIFT;  in kvm_alloc_memslot_metadata()
12606 		if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1)) {  in kvm_alloc_memslot_metadata()
12623 		kvfree(slot->arch.lpage_info[i - 1]);  in kvm_alloc_memslot_metadata()
12624 		slot->arch.lpage_info[i - 1] = NULL;  in kvm_alloc_memslot_metadata()
12626 	return -ENOMEM;  in kvm_alloc_memslot_metadata()
12635 	 * memslots->generation has been incremented.  in kvm_arch_memslots_updated()
12640 	/* Force re-initialization of steal_time cache */  in kvm_arch_memslots_updated()
12655 		return -EINVAL;  in kvm_arch_prepare_memory_region()
12658 		if ((new->base_gfn + new->npages - 1) > kvm_mmu_max_gfn())  in kvm_arch_prepare_memory_region()
12659 			return -EINVAL;  in kvm_arch_prepare_memory_region()
12665 		memcpy(&new->arch, &old->arch, sizeof(old->arch));  in kvm_arch_prepare_memory_region()
12667 		return -EIO;  in kvm_arch_prepare_memory_region()
12680 	nr_slots = atomic_read(&kvm->nr_memslots_dirty_logging);  in kvm_mmu_update_cpu_dirty_logging()
12690 	u32 old_flags = old ? old->flags : 0;  in kvm_mmu_slot_apply_flags()
12691 	u32 new_flags = new ? new->flags : 0;  in kvm_mmu_slot_apply_flags()
12711 	 * CREATE:      No shadow pages exist, thus nothing to write-protect  in kvm_mmu_slot_apply_flags()
12720 	 * READONLY and non-flags changes were filtered out above, and the only  in kvm_mmu_slot_apply_flags()
12738 		 * which can be collapsed into a single large-page spte.  Later  in kvm_mmu_slot_apply_flags()
12739 		 * page faults will create the large-page sptes.  in kvm_mmu_slot_apply_flags()
12744 		 * Initially-all-set does not require write protecting any page,  in kvm_mmu_slot_apply_flags()
12769 		 * write-protected before returning to userspace, i.e. before  in kvm_mmu_slot_apply_flags()
12776 		 * Specifically, KVM also write-protects guest page tables to  in kvm_mmu_slot_apply_flags()
12785 		 * To handle these scenarios, KVM uses a separate software-only  in kvm_mmu_slot_apply_flags()
12786 		 * bit (MMU-writable) to track if a SPTE is !writable due to  in kvm_mmu_slot_apply_flags()
12787 		 * a guest page table being write-protected (KVM clears the  in kvm_mmu_slot_apply_flags()
12788 		 * MMU-writable flag when write-protecting for shadow paging).  in kvm_mmu_slot_apply_flags()
12790 		 * The use of MMU-writable is also the primary motivation for  in kvm_mmu_slot_apply_flags()
12793 		 * !MMU-writable SPTE, KVM must flush if it encounters any  in kvm_mmu_slot_apply_flags()
12794 		 * MMU-writable SPTE regardless of whether the actual hardware  in kvm_mmu_slot_apply_flags()
12797 		 * write access" helpers to ignore MMU-writable entirely.  in kvm_mmu_slot_apply_flags()
12800 		 * access-tracked SPTEs is particularly relevant).  in kvm_mmu_slot_apply_flags()
12814 	if (!kvm->arch.n_requested_mmu_pages &&  in kvm_arch_commit_memory_region()
12818 		nr_mmu_pages = kvm->nr_memslot_pages / KVM_MEMSLOT_PAGES_TO_MMU_PAGES_RATIO;  in kvm_arch_commit_memory_region()
12838 	if (!list_empty_careful(&vcpu->async_pf.done))  in kvm_vcpu_has_events()
12845 	if (vcpu->arch.pv.pv_unhalted)  in kvm_vcpu_has_events()
12852 	    (vcpu->arch.nmi_pending &&  in kvm_vcpu_has_events()
12858 	    (vcpu->arch.smi_pending &&  in kvm_vcpu_has_events()
12875 	    kvm_x86_ops.nested_ops->has_events &&  in kvm_vcpu_has_events()
12876 	    kvm_x86_ops.nested_ops->has_events(vcpu))  in kvm_vcpu_has_events()
12901 	if (READ_ONCE(vcpu->arch.pv.pv_unhalted))  in kvm_arch_dy_runnable()
12916 	if (vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_in_kernel()
12919 	return vcpu->arch.preempted_in_kernel;  in kvm_arch_vcpu_in_kernel()
12940 	if (vcpu->arch.guest_state_protected)  in kvm_get_linear_rip()
12961 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)  in kvm_get_rflags()
12969 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&  in __kvm_set_rflags()
12970 	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))  in __kvm_set_rflags()
12991 	return (key + 1) & (ASYNC_PF_PER_VCPU - 1);  in kvm_async_pf_next_probe()
12998 	while (vcpu->arch.apf.gfns[key] != ~0)  in kvm_add_async_pf_gfn()
13001 	vcpu->arch.apf.gfns[key] = gfn;  in kvm_add_async_pf_gfn()
13010 		     (vcpu->arch.apf.gfns[key] != gfn &&  in kvm_async_pf_gfn_slot()
13011 		      vcpu->arch.apf.gfns[key] != ~0); i++)  in kvm_async_pf_gfn_slot()
13019 	return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;  in kvm_find_async_pf_gfn()
13028 	if (WARN_ON_ONCE(vcpu->arch.apf.gfns[i] != gfn))  in kvm_del_async_pf_gfn()
13032 		vcpu->arch.apf.gfns[i] = ~0;  in kvm_del_async_pf_gfn()
13035 			if (vcpu->arch.apf.gfns[j] == ~0)  in kvm_del_async_pf_gfn()
13037 			k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);  in kvm_del_async_pf_gfn()
13044 		vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];  in kvm_del_async_pf_gfn()
13053 	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason,  in apf_put_user_notpresent()
13061 	return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,  in apf_put_user_ready()
13070 	if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,  in apf_pageready_slot_free()
13083 	if (vcpu->arch.apf.send_user_only &&  in kvm_can_deliver_async_pf()
13092 		return vcpu->arch.apf.delivery_as_pf_vmexit;  in kvm_can_deliver_async_pf()
13110 	if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))  in kvm_can_do_async_pf()
13125 	trace_kvm_async_pf_not_present(work->arch.token, work->cr2_or_gpa);  in kvm_arch_async_page_not_present()
13126 	kvm_add_async_pf_gfn(vcpu, work->arch.gfn);  in kvm_arch_async_page_not_present()
13134 		fault.address = work->arch.token;  in kvm_arch_async_page_not_present()
13157 		.vector = vcpu->arch.apf.vec  in kvm_arch_async_page_present()
13160 	if (work->wakeup_all)  in kvm_arch_async_page_present()
13161 		work->arch.token = ~0; /* broadcast wakeup */  in kvm_arch_async_page_present()
13163 		kvm_del_async_pf_gfn(vcpu, work->arch.gfn);  in kvm_arch_async_page_present()
13164 	trace_kvm_async_pf_ready(work->arch.token, work->cr2_or_gpa);  in kvm_arch_async_page_present()
13166 	if ((work->wakeup_all || work->notpresent_injected) &&  in kvm_arch_async_page_present()
13168 	    !apf_put_user_ready(vcpu, work->arch.token)) {  in kvm_arch_async_page_present()
13169 		vcpu->arch.apf.pageready_pending = true;  in kvm_arch_async_page_present()
13173 	vcpu->arch.apf.halted = false;  in kvm_arch_async_page_present()
13174 	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;  in kvm_arch_async_page_present()
13180 	if (!vcpu->arch.apf.pageready_pending)  in kvm_arch_async_page_present_queued()
13194 	if (atomic_inc_return(&kvm->arch.assigned_device_count) == 1)  in kvm_arch_start_assignment()
13201 	atomic_dec(&kvm->arch.assigned_device_count);  in kvm_arch_end_assignment()
13207 	return raw_atomic_read(&kvm->arch.assigned_device_count);  in kvm_arch_has_assigned_device()
13213 	atomic_inc(&kvm->arch.noncoherent_dma_count);  in kvm_arch_register_noncoherent_dma()
13219 	atomic_dec(&kvm->arch.noncoherent_dma_count);  in kvm_arch_unregister_noncoherent_dma()
13225 	return atomic_read(&kvm->arch.noncoherent_dma_count);  in kvm_arch_has_noncoherent_dma()
13241 	irqfd->producer = prod;  in kvm_arch_irq_bypass_add_producer()
13242 	kvm_arch_start_assignment(irqfd->kvm);  in kvm_arch_irq_bypass_add_producer()
13243 	ret = static_call(kvm_x86_pi_update_irte)(irqfd->kvm,  in kvm_arch_irq_bypass_add_producer()
13244 					 prod->irq, irqfd->gsi, 1);  in kvm_arch_irq_bypass_add_producer()
13247 		kvm_arch_end_assignment(irqfd->kvm);  in kvm_arch_irq_bypass_add_producer()
13259 	WARN_ON(irqfd->producer != prod);  in kvm_arch_irq_bypass_del_producer()
13260 	irqfd->producer = NULL;  in kvm_arch_irq_bypass_del_producer()
13264 	 * remapped mode, so we can re-use the current implementation  in kvm_arch_irq_bypass_del_producer()
13268 	ret = static_call(kvm_x86_pi_update_irte)(irqfd->kvm, prod->irq, irqfd->gsi, 0);  in kvm_arch_irq_bypass_del_producer()
13271 		       " fails: %d\n", irqfd->consumer.token, ret);  in kvm_arch_irq_bypass_del_producer()
13273 	kvm_arch_end_assignment(irqfd->kvm);  in kvm_arch_irq_bypass_del_producer()
13285 	if (new->type != KVM_IRQ_ROUTING_MSI)  in kvm_arch_irqfd_route_changed()
13288 	return !!memcmp(&old->msi, &new->msi, sizeof(new->msi));  in kvm_arch_irqfd_route_changed()
13298 	return (vcpu->arch.msr_kvm_poll_control & 1) == 0;  in kvm_arch_no_poll()
13331 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_fixup_and_inject_pf_error()
13337 	    mmu->gva_to_gpa(vcpu, mmu, gva, access, &fault) != INVALID_GPA) {  in kvm_fixup_and_inject_pf_error()
13339 		 * If vcpu->arch.walk_mmu->gva_to_gpa succeeded, the page  in kvm_fixup_and_inject_pf_error()
13350 	vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault);  in kvm_fixup_and_inject_pf_error()
13363 		if (KVM_BUG_ON(!e, vcpu->kvm))  in kvm_handle_memory_failure()
13364 			return -EIO;  in kvm_handle_memory_failure()
13374 	 * doesn't seem to be a real use-case behind such requests, just return  in kvm_handle_memory_failure()
13426 		 * page tables, so a non-global flush just degenerates to a  in kvm_handle_invpcid()
13445 	struct kvm_run *run = vcpu->run;  in complete_sev_es_emulated_mmio()
13449 	BUG_ON(!vcpu->mmio_needed);  in complete_sev_es_emulated_mmio()
13452 	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];  in complete_sev_es_emulated_mmio()
13453 	len = min(8u, frag->len);  in complete_sev_es_emulated_mmio()
13454 	if (!vcpu->mmio_is_write)  in complete_sev_es_emulated_mmio()
13455 		memcpy(frag->data, run->mmio.data, len);  in complete_sev_es_emulated_mmio()
13457 	if (frag->len <= 8) {  in complete_sev_es_emulated_mmio()
13460 		vcpu->mmio_cur_fragment++;  in complete_sev_es_emulated_mmio()
13463 		frag->data += len;  in complete_sev_es_emulated_mmio()
13464 		frag->gpa += len;  in complete_sev_es_emulated_mmio()
13465 		frag->len -= len;  in complete_sev_es_emulated_mmio()
13468 	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {  in complete_sev_es_emulated_mmio()
13469 		vcpu->mmio_needed = 0;  in complete_sev_es_emulated_mmio()
13477 	run->mmio.phys_addr = frag->gpa;  in complete_sev_es_emulated_mmio()
13478 	run->mmio.len = min(8u, frag->len);  in complete_sev_es_emulated_mmio()
13479 	run->mmio.is_write = vcpu->mmio_is_write;  in complete_sev_es_emulated_mmio()
13480 	if (run->mmio.is_write)  in complete_sev_es_emulated_mmio()
13481 		memcpy(run->mmio.data, frag->data, min(8u, frag->len));  in complete_sev_es_emulated_mmio()
13482 	run->exit_reason = KVM_EXIT_MMIO;  in complete_sev_es_emulated_mmio()
13484 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;  in complete_sev_es_emulated_mmio()
13496 		return -EINVAL;  in kvm_sev_es_mmio_write()
13502 	bytes -= handled;  in kvm_sev_es_mmio_write()
13507 	frag = vcpu->mmio_fragments;  in kvm_sev_es_mmio_write()
13508 	vcpu->mmio_nr_fragments = 1;  in kvm_sev_es_mmio_write()
13509 	frag->len = bytes;  in kvm_sev_es_mmio_write()
13510 	frag->gpa = gpa;  in kvm_sev_es_mmio_write()
13511 	frag->data = data;  in kvm_sev_es_mmio_write()
13513 	vcpu->mmio_needed = 1;  in kvm_sev_es_mmio_write()
13514 	vcpu->mmio_cur_fragment = 0;  in kvm_sev_es_mmio_write()
13516 	vcpu->run->mmio.phys_addr = gpa;  in kvm_sev_es_mmio_write()
13517 	vcpu->run->mmio.len = min(8u, frag->len);  in kvm_sev_es_mmio_write()
13518 	vcpu->run->mmio.is_write = 1;  in kvm_sev_es_mmio_write()
13519 	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));  in kvm_sev_es_mmio_write()
13520 	vcpu->run->exit_reason = KVM_EXIT_MMIO;  in kvm_sev_es_mmio_write()
13522 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;  in kvm_sev_es_mmio_write()
13535 		return -EINVAL;  in kvm_sev_es_mmio_read()
13541 	bytes -= handled;  in kvm_sev_es_mmio_read()
13546 	frag = vcpu->mmio_fragments;  in kvm_sev_es_mmio_read()
13547 	vcpu->mmio_nr_fragments = 1;  in kvm_sev_es_mmio_read()
13548 	frag->len = bytes;  in kvm_sev_es_mmio_read()
13549 	frag->gpa = gpa;  in kvm_sev_es_mmio_read()
13550 	frag->data = data;  in kvm_sev_es_mmio_read()
13552 	vcpu->mmio_needed = 1;  in kvm_sev_es_mmio_read()
13553 	vcpu->mmio_cur_fragment = 0;  in kvm_sev_es_mmio_read()
13555 	vcpu->run->mmio.phys_addr = gpa;  in kvm_sev_es_mmio_read()
13556 	vcpu->run->mmio.len = min(8u, frag->len);  in kvm_sev_es_mmio_read()
13557 	vcpu->run->mmio.is_write = 0;  in kvm_sev_es_mmio_read()
13558 	vcpu->run->exit_reason = KVM_EXIT_MMIO;  in kvm_sev_es_mmio_read()
13560 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;  in kvm_sev_es_mmio_read()
13568 	vcpu->arch.sev_pio_count -= count;  in advance_sev_es_emulated_pio()
13569 	vcpu->arch.sev_pio_data += count * size;  in advance_sev_es_emulated_pio()
13577 	int size = vcpu->arch.pio.size;  in complete_sev_es_emulated_outs()
13578 	int port = vcpu->arch.pio.port;  in complete_sev_es_emulated_outs()
13580 	vcpu->arch.pio.count = 0;  in complete_sev_es_emulated_outs()
13581 	if (vcpu->arch.sev_pio_count)  in complete_sev_es_emulated_outs()
13591 			min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);  in kvm_sev_es_outs()
13592 		int ret = emulator_pio_out(vcpu, size, port, vcpu->arch.sev_pio_data, count);  in kvm_sev_es_outs()
13600 		if (!vcpu->arch.sev_pio_count)  in kvm_sev_es_outs()
13604 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_outs;  in kvm_sev_es_outs()
13613 	unsigned count = vcpu->arch.pio.count;  in complete_sev_es_emulated_ins()
13614 	int size = vcpu->arch.pio.size;  in complete_sev_es_emulated_ins()
13615 	int port = vcpu->arch.pio.port;  in complete_sev_es_emulated_ins()
13617 	complete_emulator_pio_in(vcpu, vcpu->arch.sev_pio_data);  in complete_sev_es_emulated_ins()
13619 	if (vcpu->arch.sev_pio_count)  in complete_sev_es_emulated_ins()
13629 			min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);  in kvm_sev_es_ins()
13630 		if (!emulator_pio_in(vcpu, size, port, vcpu->arch.sev_pio_data, count))  in kvm_sev_es_ins()
13635 		if (!vcpu->arch.sev_pio_count)  in kvm_sev_es_ins()
13639 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_ins;  in kvm_sev_es_ins()
13647 	vcpu->arch.sev_pio_data = data;  in kvm_sev_es_string_io()
13648 	vcpu->arch.sev_pio_count = count;  in kvm_sev_es_string_io()