103 	((struct kvm_vcpu *)(ctxt)->vcpu)
125 static void update_cr8_intercept(struct kvm_vcpu *vcpu);
126 static void process_nmi(struct kvm_vcpu *vcpu);
127 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
128 static void store_regs(struct kvm_vcpu *vcpu);
129 static int sync_regs(struct kvm_vcpu *vcpu);
130 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu);
132 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
133 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
272 	STATS_DESC_COUNTER(VCPU, pf_taken),
273 	STATS_DESC_COUNTER(VCPU, pf_fixed),
274 	STATS_DESC_COUNTER(VCPU, pf_emulate),
275 	STATS_DESC_COUNTER(VCPU, pf_spurious),
276 	STATS_DESC_COUNTER(VCPU, pf_fast),
277 	STATS_DESC_COUNTER(VCPU, pf_mmio_spte_created),
278 	STATS_DESC_COUNTER(VCPU, pf_guest),
279 	STATS_DESC_COUNTER(VCPU, tlb_flush),
280 	STATS_DESC_COUNTER(VCPU, invlpg),
281 	STATS_DESC_COUNTER(VCPU, exits),
282 	STATS_DESC_COUNTER(VCPU, io_exits),
283 	STATS_DESC_COUNTER(VCPU, mmio_exits),
284 	STATS_DESC_COUNTER(VCPU, signal_exits),
285 	STATS_DESC_COUNTER(VCPU, irq_window_exits),
286 	STATS_DESC_COUNTER(VCPU, nmi_window_exits),
287 	STATS_DESC_COUNTER(VCPU, l1d_flush),
288 	STATS_DESC_COUNTER(VCPU, halt_exits),
289 	STATS_DESC_COUNTER(VCPU, request_irq_exits),
290 	STATS_DESC_COUNTER(VCPU, irq_exits),
291 	STATS_DESC_COUNTER(VCPU, host_state_reload),
292 	STATS_DESC_COUNTER(VCPU, fpu_reload),
293 	STATS_DESC_COUNTER(VCPU, insn_emulation),
294 	STATS_DESC_COUNTER(VCPU, insn_emulation_fail),
295 	STATS_DESC_COUNTER(VCPU, hypercalls),
296 	STATS_DESC_COUNTER(VCPU, irq_injections),
297 	STATS_DESC_COUNTER(VCPU, nmi_injections),
298 	STATS_DESC_COUNTER(VCPU, req_event),
299 	STATS_DESC_COUNTER(VCPU, nested_run),
300 	STATS_DESC_COUNTER(VCPU, directed_yield_attempted),
301 	STATS_DESC_COUNTER(VCPU, directed_yield_successful),
302 	STATS_DESC_COUNTER(VCPU, preemption_reported),
303 	STATS_DESC_COUNTER(VCPU, preemption_other),
304 	STATS_DESC_IBOOLEAN(VCPU, guest_mode),
305 	STATS_DESC_COUNTER(VCPU, notify_window_exits),
355 static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)  in kvm_async_pf_hash_reset()  argument
359 		vcpu->arch.apf.gfns[i] = ~0;  in kvm_async_pf_hash_reset()
474 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)  in kvm_get_apic_base()  argument
476 	return vcpu->arch.apic_base;  in kvm_get_apic_base()
479 enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)  in kvm_get_apic_mode()  argument
481 	return kvm_apic_mode(kvm_get_apic_base(vcpu));  in kvm_get_apic_mode()
485 int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in kvm_set_apic_base()  argument
487 	enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);  in kvm_set_apic_base()
489 	u64 reserved_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu) | 0x2ff |  in kvm_set_apic_base()
490 		(guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);  in kvm_set_apic_base()
501 	kvm_lapic_set_base(vcpu, msr_info->data);  in kvm_set_apic_base()
502 	kvm_recalculate_apic_map(vcpu->kvm);  in kvm_set_apic_base()
573 void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu,  in kvm_deliver_exception_payload()  argument
586 		vcpu->arch.dr6 &= ~DR_TRAP_BITS;  in kvm_deliver_exception_payload()
603 		vcpu->arch.dr6 |= DR6_ACTIVE_LOW;  in kvm_deliver_exception_payload()
604 		vcpu->arch.dr6 |= ex->payload;  in kvm_deliver_exception_payload()
605 		vcpu->arch.dr6 ^= ex->payload & DR6_ACTIVE_LOW;  in kvm_deliver_exception_payload()
613 		vcpu->arch.dr6 &= ~BIT(12);  in kvm_deliver_exception_payload()
616 		vcpu->arch.cr2 = ex->payload;  in kvm_deliver_exception_payload()
625 static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vector,  in kvm_queue_exception_vmexit()  argument
629 	struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit;  in kvm_queue_exception_vmexit()
640 /* Forcibly leave the nested mode in cases like a vCPU reset */
641 static void kvm_leave_nested(struct kvm_vcpu *vcpu)  in kvm_leave_nested()  argument
643 	kvm_x86_ops.nested_ops->leave_nested(vcpu);  in kvm_leave_nested()
646 static void kvm_multiple_exception(struct kvm_vcpu *vcpu,  in kvm_multiple_exception()  argument
653 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_multiple_exception()
662 	if (!reinject && is_guest_mode(vcpu) &&  in kvm_multiple_exception()
663 	    kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, nr, error_code)) {  in kvm_multiple_exception()
664 		kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code,  in kvm_multiple_exception()
669 	if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {  in kvm_multiple_exception()
679 			WARN_ON_ONCE(kvm_is_exception_pending(vcpu));  in kvm_multiple_exception()
680 			vcpu->arch.exception.injected = true;  in kvm_multiple_exception()
690 			vcpu->arch.exception.pending = true;  in kvm_multiple_exception()
691 			vcpu->arch.exception.injected = false;  in kvm_multiple_exception()
693 		vcpu->arch.exception.has_error_code = has_error;  in kvm_multiple_exception()
694 		vcpu->arch.exception.vector = nr;  in kvm_multiple_exception()
695 		vcpu->arch.exception.error_code = error_code;  in kvm_multiple_exception()
696 		vcpu->arch.exception.has_payload = has_payload;  in kvm_multiple_exception()
697 		vcpu->arch.exception.payload = payload;  in kvm_multiple_exception()
698 		if (!is_guest_mode(vcpu))  in kvm_multiple_exception()
699 			kvm_deliver_exception_payload(vcpu,  in kvm_multiple_exception()
700 						      &vcpu->arch.exception);  in kvm_multiple_exception()
705 	prev_nr = vcpu->arch.exception.vector;  in kvm_multiple_exception()
708 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_multiple_exception()
719 		vcpu->arch.exception.injected = false;  in kvm_multiple_exception()
720 		vcpu->arch.exception.pending = false;  in kvm_multiple_exception()
722 		kvm_queue_exception_e(vcpu, DF_VECTOR, 0);  in kvm_multiple_exception()
731 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)  in kvm_queue_exception()  argument
733 	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);  in kvm_queue_exception()
737 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)  in kvm_requeue_exception()  argument
739 	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);  in kvm_requeue_exception()
743 void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,  in kvm_queue_exception_p()  argument
746 	kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);  in kvm_queue_exception_p()
750 static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,  in kvm_queue_exception_e_p()  argument
753 	kvm_multiple_exception(vcpu, nr, true, error_code,  in kvm_queue_exception_e_p()
757 int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)  in kvm_complete_insn_gp()  argument
760 		kvm_inject_gp(vcpu, 0);  in kvm_complete_insn_gp()
762 		return kvm_skip_emulated_instruction(vcpu);  in kvm_complete_insn_gp()
768 static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err)  in complete_emulated_insn_gp()  argument
771 		kvm_inject_gp(vcpu, 0);  in complete_emulated_insn_gp()
775 	return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE | EMULTYPE_SKIP |  in complete_emulated_insn_gp()
779 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)  in kvm_inject_page_fault()  argument
781 	++vcpu->stat.pf_guest;  in kvm_inject_page_fault()
787 	if (is_guest_mode(vcpu) && fault->async_page_fault)  in kvm_inject_page_fault()
788 		kvm_queue_exception_vmexit(vcpu, PF_VECTOR,  in kvm_inject_page_fault()
792 		kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,  in kvm_inject_page_fault()
796 void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,  in kvm_inject_emulated_page_fault()  argument
802 	fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :  in kvm_inject_emulated_page_fault()
803 					       vcpu->arch.walk_mmu;  in kvm_inject_emulated_page_fault()
811 		kvm_mmu_invalidate_addr(vcpu, fault_mmu, fault->address,  in kvm_inject_emulated_page_fault()
814 	fault_mmu->inject_page_fault(vcpu, fault);  in kvm_inject_emulated_page_fault()
818 void kvm_inject_nmi(struct kvm_vcpu *vcpu)  in kvm_inject_nmi()  argument
820 	atomic_inc(&vcpu->arch.nmi_queued);  in kvm_inject_nmi()
821 	kvm_make_request(KVM_REQ_NMI, vcpu);  in kvm_inject_nmi()
824 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)  in kvm_queue_exception_e()  argument
826 	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);  in kvm_queue_exception_e()
830 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)  in kvm_requeue_exception_e()  argument
832 	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);  in kvm_requeue_exception_e()
840 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)  in kvm_require_cpl()  argument
842 	if (static_call(kvm_x86_get_cpl)(vcpu) <= required_cpl)  in kvm_require_cpl()
844 	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);  in kvm_require_cpl()
848 bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)  in kvm_require_dr()  argument
850 	if ((dr != 4 && dr != 5) || !kvm_is_cr4_bit_set(vcpu, X86_CR4_DE))  in kvm_require_dr()
853 	kvm_queue_exception(vcpu, UD_VECTOR);  in kvm_require_dr()
858 static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)  in pdptr_rsvd_bits()  argument
860 	return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2);  in pdptr_rsvd_bits()
866 int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)  in load_pdptrs()  argument
868 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in load_pdptrs()
879 	real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn),  in load_pdptrs()
885 	ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte,  in load_pdptrs()
892 		    (pdpte[i] & pdptr_rsvd_bits(vcpu))) {  in load_pdptrs()
902 		kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT);  in load_pdptrs()
905 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);  in load_pdptrs()
906 	kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu);  in load_pdptrs()
907 	vcpu->arch.pdptrs_from_userspace = false;  in load_pdptrs()
913 static bool kvm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)  in kvm_is_valid_cr0()  argument
926 	return static_call(kvm_x86_is_valid_cr0)(vcpu, cr0);  in kvm_is_valid_cr0()
929 void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0)  in kvm_post_set_cr0()  argument
944 			kvm_init_mmu(vcpu);  in kvm_post_set_cr0()
950 		kvm_clear_async_pf_completion_queue(vcpu);  in kvm_post_set_cr0()
951 		kvm_async_pf_hash_reset(vcpu);  in kvm_post_set_cr0()
958 			kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_post_set_cr0()
962 		kvm_mmu_reset_context(vcpu);  in kvm_post_set_cr0()
965 	    kvm_arch_has_noncoherent_dma(vcpu->kvm) &&  in kvm_post_set_cr0()
966 	    !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))  in kvm_post_set_cr0()
967 		kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);  in kvm_post_set_cr0()
971 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)  in kvm_set_cr0()  argument
973 	unsigned long old_cr0 = kvm_read_cr0(vcpu);  in kvm_set_cr0()
975 	if (!kvm_is_valid_cr0(vcpu, cr0))  in kvm_set_cr0()
984 	if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) &&  in kvm_set_cr0()
988 		if (!is_pae(vcpu))  in kvm_set_cr0()
990 		static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);  in kvm_set_cr0()
995 	if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) &&  in kvm_set_cr0()
996 	    is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) &&  in kvm_set_cr0()
997 	    !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))  in kvm_set_cr0()
1001 	    (is_64_bit_mode(vcpu) || kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)))  in kvm_set_cr0()
1004 	static_call(kvm_x86_set_cr0)(vcpu, cr0);  in kvm_set_cr0()
1006 	kvm_post_set_cr0(vcpu, old_cr0, cr0);  in kvm_set_cr0()
1012 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)  in kvm_lmsw()  argument
1014 	(void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));  in kvm_lmsw()
1018 void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu)  in kvm_load_guest_xsave_state()  argument
1020 	if (vcpu->arch.guest_state_protected)  in kvm_load_guest_xsave_state()
1023 	if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) {  in kvm_load_guest_xsave_state()
1025 		if (vcpu->arch.xcr0 != host_xcr0)  in kvm_load_guest_xsave_state()
1026 			xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);  in kvm_load_guest_xsave_state()
1028 		if (guest_can_use(vcpu, X86_FEATURE_XSAVES) &&  in kvm_load_guest_xsave_state()
1029 		    vcpu->arch.ia32_xss != host_xss)  in kvm_load_guest_xsave_state()
1030 			wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);  in kvm_load_guest_xsave_state()
1034 	    vcpu->arch.pkru != vcpu->arch.host_pkru &&  in kvm_load_guest_xsave_state()
1035 	    ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||  in kvm_load_guest_xsave_state()
1036 	     kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE)))  in kvm_load_guest_xsave_state()
1037 		write_pkru(vcpu->arch.pkru);  in kvm_load_guest_xsave_state()
1041 void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu)  in kvm_load_host_xsave_state()  argument
1043 	if (vcpu->arch.guest_state_protected)  in kvm_load_host_xsave_state()
1047 	    ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||  in kvm_load_host_xsave_state()
1048 	     kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE))) {  in kvm_load_host_xsave_state()
1049 		vcpu->arch.pkru = rdpkru();  in kvm_load_host_xsave_state()
1050 		if (vcpu->arch.pkru != vcpu->arch.host_pkru)  in kvm_load_host_xsave_state()
1051 			write_pkru(vcpu->arch.host_pkru);  in kvm_load_host_xsave_state()
1054 	if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) {  in kvm_load_host_xsave_state()
1056 		if (vcpu->arch.xcr0 != host_xcr0)  in kvm_load_host_xsave_state()
1059 		if (guest_can_use(vcpu, X86_FEATURE_XSAVES) &&  in kvm_load_host_xsave_state()
1060 		    vcpu->arch.ia32_xss != host_xss)  in kvm_load_host_xsave_state()
1068 static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu)  in kvm_guest_supported_xfd()  argument
1070 	return vcpu->arch.guest_supported_xcr0 & XFEATURE_MASK_USER_DYNAMIC;  in kvm_guest_supported_xfd()
1074 static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)  in __kvm_set_xcr()  argument
1077 	u64 old_xcr0 = vcpu->arch.xcr0;  in __kvm_set_xcr()
1093 	valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;  in __kvm_set_xcr()
1112 	vcpu->arch.xcr0 = xcr0;  in __kvm_set_xcr()
1115 		kvm_update_cpuid_runtime(vcpu);  in __kvm_set_xcr()
1119 int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu)  in kvm_emulate_xsetbv()  argument
1122 	if (static_call(kvm_x86_get_cpl)(vcpu) != 0 ||  in kvm_emulate_xsetbv()
1123 	    __kvm_set_xcr(vcpu, kvm_rcx_read(vcpu), kvm_read_edx_eax(vcpu))) {  in kvm_emulate_xsetbv()
1124 		kvm_inject_gp(vcpu, 0);  in kvm_emulate_xsetbv()
1128 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_xsetbv()
1132 bool __kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)  in __kvm_is_valid_cr4()  argument
1137 	if (cr4 & vcpu->arch.cr4_guest_rsvd_bits)  in __kvm_is_valid_cr4()
1144 static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)  in kvm_is_valid_cr4()  argument
1146 	return __kvm_is_valid_cr4(vcpu, cr4) &&  in kvm_is_valid_cr4()
1147 	       static_call(kvm_x86_is_valid_cr4)(vcpu, cr4);  in kvm_is_valid_cr4()
1150 void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4)  in kvm_post_set_cr4()  argument
1153 		kvm_mmu_reset_context(vcpu);  in kvm_post_set_cr4()
1165 		kvm_mmu_unload(vcpu);  in kvm_post_set_cr4()
1177 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_post_set_cr4()
1187 		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);  in kvm_post_set_cr4()
1192 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)  in kvm_set_cr4()  argument
1194 	unsigned long old_cr4 = kvm_read_cr4(vcpu);  in kvm_set_cr4()
1196 	if (!kvm_is_valid_cr4(vcpu, cr4))  in kvm_set_cr4()
1199 	if (is_long_mode(vcpu)) {  in kvm_set_cr4()
1204 	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)  in kvm_set_cr4()
1206 		   && !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))  in kvm_set_cr4()
1211 		if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))  in kvm_set_cr4()
1215 	static_call(kvm_x86_set_cr4)(vcpu, cr4);  in kvm_set_cr4()
1217 	kvm_post_set_cr4(vcpu, old_cr4, cr4);  in kvm_set_cr4()
1223 static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid)  in kvm_invalidate_pcid()  argument
1225 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in kvm_invalidate_pcid()
1237 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_invalidate_pcid()
1246 	if (kvm_get_active_pcid(vcpu) == pcid) {  in kvm_invalidate_pcid()
1247 		kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);  in kvm_invalidate_pcid()
1248 		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);  in kvm_invalidate_pcid()
1256 	if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE))  in kvm_invalidate_pcid()
1260 		if (kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd) == pcid)  in kvm_invalidate_pcid()
1263 	kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free);  in kvm_invalidate_pcid()
1266 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)  in kvm_set_cr3()  argument
1271 	if (kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)) {  in kvm_set_cr3()
1279 	if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu))  in kvm_set_cr3()
1285 	 * the current vCPU mode is accurate.  in kvm_set_cr3()
1287 	if (kvm_vcpu_is_illegal_gpa(vcpu, cr3))  in kvm_set_cr3()
1290 	if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3))  in kvm_set_cr3()
1293 	if (cr3 != kvm_read_cr3(vcpu))  in kvm_set_cr3()
1294 		kvm_mmu_new_pgd(vcpu, cr3);  in kvm_set_cr3()
1296 	vcpu->arch.cr3 = cr3;  in kvm_set_cr3()
1297 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);  in kvm_set_cr3()
1309 		kvm_invalidate_pcid(vcpu, pcid);  in kvm_set_cr3()
1315 int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)  in kvm_set_cr8()  argument
1319 	if (lapic_in_kernel(vcpu))  in kvm_set_cr8()
1320 		kvm_lapic_set_tpr(vcpu, cr8);  in kvm_set_cr8()
1322 		vcpu->arch.cr8 = cr8;  in kvm_set_cr8()
1327 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)  in kvm_get_cr8()  argument
1329 	if (lapic_in_kernel(vcpu))  in kvm_get_cr8()
1330 		return kvm_lapic_get_cr8(vcpu);  in kvm_get_cr8()
1332 		return vcpu->arch.cr8;  in kvm_get_cr8()
1336 static void kvm_update_dr0123(struct kvm_vcpu *vcpu)  in kvm_update_dr0123()  argument
1340 	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {  in kvm_update_dr0123()
1342 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];  in kvm_update_dr0123()
1346 void kvm_update_dr7(struct kvm_vcpu *vcpu)  in kvm_update_dr7()  argument
1350 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)  in kvm_update_dr7()
1351 		dr7 = vcpu->arch.guest_debug_dr7;  in kvm_update_dr7()
1353 		dr7 = vcpu->arch.dr7;  in kvm_update_dr7()
1354 	static_call(kvm_x86_set_dr7)(vcpu, dr7);  in kvm_update_dr7()
1355 	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;  in kvm_update_dr7()
1357 		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;  in kvm_update_dr7()
1361 static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)  in kvm_dr6_fixed()  argument
1365 	if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM))  in kvm_dr6_fixed()
1368 	if (!guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))  in kvm_dr6_fixed()
1373 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)  in kvm_set_dr()  argument
1375 	size_t size = ARRAY_SIZE(vcpu->arch.db);  in kvm_set_dr()
1379 		vcpu->arch.db[array_index_nospec(dr, size)] = val;  in kvm_set_dr()
1380 		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))  in kvm_set_dr()
1381 			vcpu->arch.eff_db[dr] = val;  in kvm_set_dr()
1387 		vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);  in kvm_set_dr()
1393 		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;  in kvm_set_dr()
1394 		kvm_update_dr7(vcpu);  in kvm_set_dr()
1402 void kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)  in kvm_get_dr()  argument
1404 	size_t size = ARRAY_SIZE(vcpu->arch.db);  in kvm_get_dr()
1408 		*val = vcpu->arch.db[array_index_nospec(dr, size)];  in kvm_get_dr()
1412 		*val = vcpu->arch.dr6;  in kvm_get_dr()
1416 		*val = vcpu->arch.dr7;  in kvm_get_dr()
1422 int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu)  in kvm_emulate_rdpmc()  argument
1424 	u32 ecx = kvm_rcx_read(vcpu);  in kvm_emulate_rdpmc()
1427 	if (kvm_pmu_rdpmc(vcpu, ecx, &data)) {  in kvm_emulate_rdpmc()
1428 		kvm_inject_gp(vcpu, 0);  in kvm_emulate_rdpmc()
1432 	kvm_rax_write(vcpu, (u32)data);  in kvm_emulate_rdpmc()
1433 	kvm_rdx_write(vcpu, data >> 32);  in kvm_emulate_rdpmc()
1434 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_rdpmc()
1590  * patch, are immutable once the vCPU model is defined.
1702 static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data)  in do_get_msr_feature()  argument
1725 static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)  in __kvm_valid_efer()  argument
1727 	if (efer & EFER_AUTOIBRS && !guest_cpuid_has(vcpu, X86_FEATURE_AUTOIBRS))  in __kvm_valid_efer()
1730 	if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT))  in __kvm_valid_efer()
1733 	if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM))  in __kvm_valid_efer()
1737 	    !guest_cpuid_has(vcpu, X86_FEATURE_LM))  in __kvm_valid_efer()
1740 	if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX))  in __kvm_valid_efer()
1746 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)  in kvm_valid_efer()  argument
1751 	return __kvm_valid_efer(vcpu, efer);  in kvm_valid_efer()
1755 static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in set_efer()  argument
1757 	u64 old_efer = vcpu->arch.efer;  in set_efer()
1765 		if (!__kvm_valid_efer(vcpu, efer))  in set_efer()
1768 		if (is_paging(vcpu) &&  in set_efer()
1769 		    (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))  in set_efer()
1774 	efer |= vcpu->arch.efer & EFER_LMA;  in set_efer()
1776 	r = static_call(kvm_x86_set_efer)(vcpu, efer);  in set_efer()
1783 		kvm_mmu_reset_context(vcpu);  in set_efer()
1794 bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type)  in kvm_msr_allowed()  argument
1798 	struct kvm *kvm = vcpu->kvm;  in kvm_msr_allowed()
1843 static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,  in __kvm_set_msr()  argument
1854 		if (is_noncanonical_address(data, vcpu))  in __kvm_set_msr()
1871 		data = __canonical_address(data, vcpu_virt_addr_bits(vcpu));  in __kvm_set_msr()
1878 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) &&  in __kvm_set_msr()
1879 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDPID))  in __kvm_set_msr()
1891 		if (guest_cpuid_is_intel(vcpu) && (data >> 32) != 0)  in __kvm_set_msr()
1902 	return static_call(kvm_x86_set_msr)(vcpu, &msr);  in __kvm_set_msr()
1905 static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu,  in kvm_set_msr_ignored_check()  argument
1908 	int ret = __kvm_set_msr(vcpu, index, data, host_initiated);  in kvm_set_msr_ignored_check()
1923 int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,  in __kvm_get_msr()  argument
1935 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) &&  in __kvm_get_msr()
1936 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDPID))  in __kvm_get_msr()
1944 	ret = static_call(kvm_x86_get_msr)(vcpu, &msr);  in __kvm_get_msr()
1950 static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu,  in kvm_get_msr_ignored_check()  argument
1953 	int ret = __kvm_get_msr(vcpu, index, data, host_initiated);  in kvm_get_msr_ignored_check()
1965 static int kvm_get_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 *data)  in kvm_get_msr_with_filter()  argument
1967 	if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ))  in kvm_get_msr_with_filter()
1969 	return kvm_get_msr_ignored_check(vcpu, index, data, false);  in kvm_get_msr_with_filter()
1972 static int kvm_set_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 data)  in kvm_set_msr_with_filter()  argument
1974 	if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE))  in kvm_set_msr_with_filter()
1976 	return kvm_set_msr_ignored_check(vcpu, index, data, false);  in kvm_set_msr_with_filter()
1979 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data)  in kvm_get_msr()  argument
1981 	return kvm_get_msr_ignored_check(vcpu, index, data, false);  in kvm_get_msr()
1985 int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)  in kvm_set_msr()  argument
1987 	return kvm_set_msr_ignored_check(vcpu, index, data, false);  in kvm_set_msr()
1991 static void complete_userspace_rdmsr(struct kvm_vcpu *vcpu)  in complete_userspace_rdmsr()  argument
1993 	if (!vcpu->run->msr.error) {  in complete_userspace_rdmsr()
1994 		kvm_rax_write(vcpu, (u32)vcpu->run->msr.data);  in complete_userspace_rdmsr()
1995 		kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);  in complete_userspace_rdmsr()
1999 static int complete_emulated_msr_access(struct kvm_vcpu *vcpu)  in complete_emulated_msr_access()  argument
2001 	return complete_emulated_insn_gp(vcpu, vcpu->run->msr.error);  in complete_emulated_msr_access()
2004 static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu)  in complete_emulated_rdmsr()  argument
2006 	complete_userspace_rdmsr(vcpu);  in complete_emulated_rdmsr()
2007 	return complete_emulated_msr_access(vcpu);  in complete_emulated_rdmsr()
2010 static int complete_fast_msr_access(struct kvm_vcpu *vcpu)  in complete_fast_msr_access()  argument
2012 	return static_call(kvm_x86_complete_emulated_msr)(vcpu, vcpu->run->msr.error);  in complete_fast_msr_access()
2015 static int complete_fast_rdmsr(struct kvm_vcpu *vcpu)  in complete_fast_rdmsr()  argument
2017 	complete_userspace_rdmsr(vcpu);  in complete_fast_rdmsr()
2018 	return complete_fast_msr_access(vcpu);  in complete_fast_rdmsr()
2033 static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index,  in kvm_msr_user_space()  argument
2035 			      int (*completion)(struct kvm_vcpu *vcpu),  in kvm_msr_user_space()  argument
2041 	if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason))  in kvm_msr_user_space()
2044 	vcpu->run->exit_reason = exit_reason;  in kvm_msr_user_space()
2045 	vcpu->run->msr.error = 0;  in kvm_msr_user_space()
2046 	memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad));  in kvm_msr_user_space()
2047 	vcpu->run->msr.reason = msr_reason;  in kvm_msr_user_space()
2048 	vcpu->run->msr.index = index;  in kvm_msr_user_space()
2049 	vcpu->run->msr.data = data;  in kvm_msr_user_space()
2050 	vcpu->arch.complete_userspace_io = completion;  in kvm_msr_user_space()
2055 int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)  in kvm_emulate_rdmsr()  argument
2057 	u32 ecx = kvm_rcx_read(vcpu);  in kvm_emulate_rdmsr()
2061 	r = kvm_get_msr_with_filter(vcpu, ecx, &data);  in kvm_emulate_rdmsr()
2066 		kvm_rax_write(vcpu, data & -1u);  in kvm_emulate_rdmsr()
2067 		kvm_rdx_write(vcpu, (data >> 32) & -1u);  in kvm_emulate_rdmsr()
2070 		if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_RDMSR, 0,  in kvm_emulate_rdmsr()
2076 	return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);  in kvm_emulate_rdmsr()
2080 int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)  in kvm_emulate_wrmsr()  argument
2082 	u32 ecx = kvm_rcx_read(vcpu);  in kvm_emulate_wrmsr()
2083 	u64 data = kvm_read_edx_eax(vcpu);  in kvm_emulate_wrmsr()
2086 	r = kvm_set_msr_with_filter(vcpu, ecx, data);  in kvm_emulate_wrmsr()
2092 		if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_WRMSR, data,  in kvm_emulate_wrmsr()
2101 	return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);  in kvm_emulate_wrmsr()
2105 int kvm_emulate_as_nop(struct kvm_vcpu *vcpu)  in kvm_emulate_as_nop()  argument
2107 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_as_nop()
2110 int kvm_emulate_invd(struct kvm_vcpu *vcpu)  in kvm_emulate_invd()  argument
2113 	return kvm_emulate_as_nop(vcpu);  in kvm_emulate_invd()
2117 int kvm_handle_invalid_op(struct kvm_vcpu *vcpu)  in kvm_handle_invalid_op()  argument
2119 	kvm_queue_exception(vcpu, UD_VECTOR);  in kvm_handle_invalid_op()
2125 static int kvm_emulate_monitor_mwait(struct kvm_vcpu *vcpu, const char *insn)  in kvm_emulate_monitor_mwait()  argument
2127 	if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS) &&  in kvm_emulate_monitor_mwait()
2128 	    !guest_cpuid_has(vcpu, X86_FEATURE_MWAIT))  in kvm_emulate_monitor_mwait()
2129 		return kvm_handle_invalid_op(vcpu);  in kvm_emulate_monitor_mwait()
2132 	return kvm_emulate_as_nop(vcpu);  in kvm_emulate_monitor_mwait()
2134 int kvm_emulate_mwait(struct kvm_vcpu *vcpu)  in kvm_emulate_mwait()  argument
2136 	return kvm_emulate_monitor_mwait(vcpu, "MWAIT");  in kvm_emulate_mwait()
2140 int kvm_emulate_monitor(struct kvm_vcpu *vcpu)  in kvm_emulate_monitor()  argument
2142 	return kvm_emulate_monitor_mwait(vcpu, "MONITOR");  in kvm_emulate_monitor()
2146 static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)  in kvm_vcpu_exit_request()  argument
2149 	return vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) ||  in kvm_vcpu_exit_request()
2160 static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data)  in handle_fastpath_set_x2apic_icr_irqoff()  argument
2162 	if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic))  in handle_fastpath_set_x2apic_icr_irqoff()
2169 		return kvm_x2apic_icr_write(vcpu->arch.apic, data);  in handle_fastpath_set_x2apic_icr_irqoff()
2174 static int handle_fastpath_set_tscdeadline(struct kvm_vcpu *vcpu, u64 data)  in handle_fastpath_set_tscdeadline()  argument
2176 	if (!kvm_can_use_hv_timer(vcpu))  in handle_fastpath_set_tscdeadline()
2179 	kvm_set_lapic_tscdeadline_msr(vcpu, data);  in handle_fastpath_set_tscdeadline()
2183 fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu)  in handle_fastpath_set_msr_irqoff()  argument
2185 	u32 msr = kvm_rcx_read(vcpu);  in handle_fastpath_set_msr_irqoff()
2189 	kvm_vcpu_srcu_read_lock(vcpu);  in handle_fastpath_set_msr_irqoff()
2193 		data = kvm_read_edx_eax(vcpu);  in handle_fastpath_set_msr_irqoff()
2194 		if (!handle_fastpath_set_x2apic_icr_irqoff(vcpu, data)) {  in handle_fastpath_set_msr_irqoff()
2195 			kvm_skip_emulated_instruction(vcpu);  in handle_fastpath_set_msr_irqoff()
2200 		data = kvm_read_edx_eax(vcpu);  in handle_fastpath_set_msr_irqoff()
2201 		if (!handle_fastpath_set_tscdeadline(vcpu, data)) {  in handle_fastpath_set_msr_irqoff()
2202 			kvm_skip_emulated_instruction(vcpu);  in handle_fastpath_set_msr_irqoff()
2213 	kvm_vcpu_srcu_read_unlock(vcpu);  in handle_fastpath_set_msr_irqoff()
2222 static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)  in do_get_msr()  argument
2224 	return kvm_get_msr_ignored_check(vcpu, index, data, true);  in do_get_msr()
2227 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)  in do_set_msr()  argument
2233 	 * not support modifying the guest vCPU model on the fly, e.g. changing  in do_set_msr()
2238 	if (kvm_vcpu_has_run(vcpu) && kvm_is_immutable_feature_msr(index)) {  in do_set_msr()
2239 		if (do_get_msr(vcpu, index, &val) || *data != val)  in do_set_msr()
2245 	return kvm_set_msr_ignored_check(vcpu, index, *data, true);  in do_set_msr()
2360 static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time,  in kvm_write_system_time()  argument
2363 	struct kvm_arch *ka = &vcpu->kvm->arch;  in kvm_write_system_time()
2365 	if (vcpu->vcpu_id == 0 && !host_initiated) {  in kvm_write_system_time()
2367 			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in kvm_write_system_time()
2372 	vcpu->arch.time = system_time;  in kvm_write_system_time()
2373 	kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);  in kvm_write_system_time()
2377 		kvm_gpc_activate(&vcpu->arch.pv_time, system_time & ~1ULL,  in kvm_write_system_time()
2380 		kvm_gpc_deactivate(&vcpu->arch.pv_time);  in kvm_write_system_time()
2433 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier);
2435 static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)  in set_tsc_khz()  argument
2441 		kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);  in set_tsc_khz()
2448 			vcpu->arch.tsc_catchup = 1;  in set_tsc_khz()
2449 			vcpu->arch.tsc_always_catchup = 1;  in set_tsc_khz()
2467 	kvm_vcpu_write_tsc_multiplier(vcpu, ratio);  in set_tsc_khz()
2471 static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)  in kvm_set_tsc_khz()  argument
2479 		kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);  in kvm_set_tsc_khz()
2485 			   &vcpu->arch.virtual_tsc_shift,  in kvm_set_tsc_khz()
2486 			   &vcpu->arch.virtual_tsc_mult);  in kvm_set_tsc_khz()
2487 	vcpu->arch.virtual_tsc_khz = user_tsc_khz;  in kvm_set_tsc_khz()
2502 	return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);  in kvm_set_tsc_khz()
2505 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)  in compute_guest_tsc()  argument
2507 	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,  in compute_guest_tsc()
2508 				      vcpu->arch.virtual_tsc_mult,  in compute_guest_tsc()
2509 				      vcpu->arch.virtual_tsc_shift);  in compute_guest_tsc()
2510 	tsc += vcpu->arch.this_tsc_write;  in compute_guest_tsc()
2521 static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)  in kvm_track_tsc_matching()  argument
2525 	struct kvm_arch *ka = &vcpu->kvm->arch;  in kvm_track_tsc_matching()
2529 			 atomic_read(&vcpu->kvm->online_vcpus));  in kvm_track_tsc_matching()
2541 		kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in kvm_track_tsc_matching()
2543 	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,  in kvm_track_tsc_matching()
2544 			    atomic_read(&vcpu->kvm->online_vcpus),  in kvm_track_tsc_matching()
2574 static u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)  in kvm_compute_l1_tsc_offset()  argument
2578 	tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio);  in kvm_compute_l1_tsc_offset()
2583 u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)  in kvm_read_l1_tsc()  argument
2585 	return vcpu->arch.l1_tsc_offset +  in kvm_read_l1_tsc()
2586 		kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio);  in kvm_read_l1_tsc()
2615 static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset)  in kvm_vcpu_write_tsc_offset()  argument
2617 	trace_kvm_write_tsc_offset(vcpu->vcpu_id,  in kvm_vcpu_write_tsc_offset()
2618 				   vcpu->arch.l1_tsc_offset,  in kvm_vcpu_write_tsc_offset()
2621 	vcpu->arch.l1_tsc_offset = l1_offset;  in kvm_vcpu_write_tsc_offset()
2628 	if (is_guest_mode(vcpu))  in kvm_vcpu_write_tsc_offset()
2629 		vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(  in kvm_vcpu_write_tsc_offset()
2631 			static_call(kvm_x86_get_l2_tsc_offset)(vcpu),  in kvm_vcpu_write_tsc_offset()
2632 			static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu));  in kvm_vcpu_write_tsc_offset()
2634 		vcpu->arch.tsc_offset = l1_offset;  in kvm_vcpu_write_tsc_offset()
2636 	static_call(kvm_x86_write_tsc_offset)(vcpu);  in kvm_vcpu_write_tsc_offset()
2639 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier)  in kvm_vcpu_write_tsc_multiplier()  argument
2641 	vcpu->arch.l1_tsc_scaling_ratio = l1_multiplier;  in kvm_vcpu_write_tsc_multiplier()
2644 	if (is_guest_mode(vcpu))  in kvm_vcpu_write_tsc_multiplier()
2645 		vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier(  in kvm_vcpu_write_tsc_multiplier()
2647 			static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu));  in kvm_vcpu_write_tsc_multiplier()
2649 		vcpu->arch.tsc_scaling_ratio = l1_multiplier;  in kvm_vcpu_write_tsc_multiplier()
2652 		static_call(kvm_x86_write_tsc_multiplier)(vcpu);  in kvm_vcpu_write_tsc_multiplier()
2670  * offset for the vcpu and tracks the TSC matching generation that the vcpu
2673 static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,  in __kvm_synchronize_tsc()  argument
2676 	struct kvm *kvm = vcpu->kvm;  in __kvm_synchronize_tsc()
2686 	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;  in __kvm_synchronize_tsc()
2689 	vcpu->arch.last_guest_tsc = tsc;  in __kvm_synchronize_tsc()
2691 	kvm_vcpu_write_tsc_offset(vcpu, offset);  in __kvm_synchronize_tsc()
2708 	} else if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) {  in __kvm_synchronize_tsc()
2712 	/* Keep track of which generation this VCPU has synchronized to */  in __kvm_synchronize_tsc()
2713 	vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;  in __kvm_synchronize_tsc()
2714 	vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;  in __kvm_synchronize_tsc()
2715 	vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;  in __kvm_synchronize_tsc()
2717 	kvm_track_tsc_matching(vcpu);  in __kvm_synchronize_tsc()
2720 static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 data)  in kvm_synchronize_tsc()  argument
2722 	struct kvm *kvm = vcpu->kvm;  in kvm_synchronize_tsc()
2729 	offset = kvm_compute_l1_tsc_offset(vcpu, data);  in kvm_synchronize_tsc()
2733 	if (vcpu->arch.virtual_tsc_khz) {  in kvm_synchronize_tsc()
2736 			 * detection of vcpu initialization -- need to sync  in kvm_synchronize_tsc()
2743 						nsec_to_cycles(vcpu, elapsed);  in kvm_synchronize_tsc()
2744 			u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;  in kvm_synchronize_tsc()
2762 	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {  in kvm_synchronize_tsc()
2766 			u64 delta = nsec_to_cycles(vcpu, elapsed);  in kvm_synchronize_tsc()
2768 			offset = kvm_compute_l1_tsc_offset(vcpu, data);  in kvm_synchronize_tsc()
2773 	__kvm_synchronize_tsc(vcpu, offset, data, ns, matched);  in kvm_synchronize_tsc()
2777 static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,  in adjust_tsc_offset_guest()  argument
2780 	u64 tsc_offset = vcpu->arch.l1_tsc_offset;  in adjust_tsc_offset_guest()
2781 	kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);  in adjust_tsc_offset_guest()
2784 static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)  in adjust_tsc_offset_host()  argument
2786 	if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio)  in adjust_tsc_offset_host()
2789 				   vcpu->arch.l1_tsc_scaling_ratio);  in adjust_tsc_offset_host()
2790 	adjust_tsc_offset_guest(vcpu, adjustment);  in adjust_tsc_offset_host()
3008 	struct kvm_vcpu *vcpu;  in kvm_end_pvclock_update()  local
3013 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_end_pvclock_update()
3014 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_end_pvclock_update()
3017 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_end_pvclock_update()
3018 		kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);  in kvm_end_pvclock_update()
3104 	struct kvm_vcpu_arch *vcpu = &v->arch;  in kvm_setup_guest_pvclock()  local
3121 	 * This VCPU is paused, but it's legal for a guest to read another  in kvm_setup_guest_pvclock()
3122 	 * VCPU's kvmclock, so we really have to follow the specification where  in kvm_setup_guest_pvclock()
3127 	guest_hv_clock->version = vcpu->hv_clock.version = (guest_hv_clock->version + 1) | 1;  in kvm_setup_guest_pvclock()
3131 	vcpu->hv_clock.flags |= (guest_hv_clock->flags & PVCLOCK_GUEST_STOPPED);  in kvm_setup_guest_pvclock()
3133 	if (vcpu->pvclock_set_guest_stopped_request) {  in kvm_setup_guest_pvclock()
3134 		vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED;  in kvm_setup_guest_pvclock()
3135 		vcpu->pvclock_set_guest_stopped_request = false;  in kvm_setup_guest_pvclock()
3138 	memcpy(guest_hv_clock, &vcpu->hv_clock, sizeof(*guest_hv_clock));  in kvm_setup_guest_pvclock()
3141 	guest_hv_clock->version = ++vcpu->hv_clock.version;  in kvm_setup_guest_pvclock()
3146 	trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);  in kvm_setup_guest_pvclock()
3153 	struct kvm_vcpu_arch *vcpu = &v->arch;  in kvm_guest_time_update()  local
3195 	 *   2) Broken TSC compensation resets the base at each VCPU  in kvm_guest_time_update()
3201 	if (vcpu->tsc_catchup) {  in kvm_guest_time_update()
3217 	if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {  in kvm_guest_time_update()
3219 				   &vcpu->hv_clock.tsc_shift,  in kvm_guest_time_update()
3220 				   &vcpu->hv_clock.tsc_to_system_mul);  in kvm_guest_time_update()
3221 		vcpu->hw_tsc_khz = tgt_tsc_khz;  in kvm_guest_time_update()
3225 	vcpu->hv_clock.tsc_timestamp = tsc_timestamp;  in kvm_guest_time_update()
3226 	vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;  in kvm_guest_time_update()
3227 	vcpu->last_guest_tsc = tsc_timestamp;  in kvm_guest_time_update()
3234 	vcpu->hv_clock.flags = pvclock_flags;  in kvm_guest_time_update()
3236 	if (vcpu->pv_time.active)  in kvm_guest_time_update()
3237 		kvm_setup_guest_pvclock(v, &vcpu->pv_time, 0);  in kvm_guest_time_update()
3238 	if (vcpu->xen.vcpu_info_cache.active)  in kvm_guest_time_update()
3239 		kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_info_cache,  in kvm_guest_time_update()
3241 	if (vcpu->xen.vcpu_time_info_cache.active)  in kvm_guest_time_update()
3242 		kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_time_info_cache, 0);  in kvm_guest_time_update()
3243 	kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);  in kvm_guest_time_update()
3248  * kvmclock updates which are isolated to a given vcpu, such as
3249  * vcpu->cpu migration, should not allow system_timestamp from
3251  * correction applies to one vcpu's system_timestamp but not
3257  * The time for a remote vcpu to update its kvmclock is bound
3270 	struct kvm_vcpu *vcpu;  in kvmclock_update_fn()  local
3272 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvmclock_update_fn()
3273 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvmclock_update_fn()
3274 		kvm_vcpu_kick(vcpu);  in kvmclock_update_fn()
3317 static bool can_set_mci_status(struct kvm_vcpu *vcpu)  in can_set_mci_status()  argument
3320 	if (guest_cpuid_is_amd_or_hygon(vcpu))  in can_set_mci_status()
3321 		return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));  in can_set_mci_status()
3326 static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in set_msr_mce()  argument
3328 	u64 mcg_cap = vcpu->arch.mcg_cap;  in set_msr_mce()
3336 		vcpu->arch.mcg_status = data;  in set_msr_mce()
3344 		vcpu->arch.mcg_ctl = data;  in set_msr_mce()
3358 		vcpu->arch.mci_ctl2_banks[offset] = data;  in set_msr_mce()
3385 		    data != 0 && !can_set_mci_status(vcpu))  in set_msr_mce()
3390 		vcpu->arch.mce_banks[offset] = data;  in set_msr_mce()
3398 static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)  in kvm_pv_async_pf_enabled()  argument
3402 	return (vcpu->arch.apf.msr_en_val & mask) == mask;  in kvm_pv_async_pf_enabled()
3405 static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)  in kvm_pv_enable_async_pf()  argument
3413 	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_VMEXIT) &&  in kvm_pv_enable_async_pf()
3417 	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT) &&  in kvm_pv_enable_async_pf()
3421 	if (!lapic_in_kernel(vcpu))  in kvm_pv_enable_async_pf()
3424 	vcpu->arch.apf.msr_en_val = data;  in kvm_pv_enable_async_pf()
3426 	if (!kvm_pv_async_pf_enabled(vcpu)) {  in kvm_pv_enable_async_pf()
3427 		kvm_clear_async_pf_completion_queue(vcpu);  in kvm_pv_enable_async_pf()
3428 		kvm_async_pf_hash_reset(vcpu);  in kvm_pv_enable_async_pf()
3432 	if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,  in kvm_pv_enable_async_pf()
3436 	vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);  in kvm_pv_enable_async_pf()
3437 	vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;  in kvm_pv_enable_async_pf()
3439 	kvm_async_pf_wakeup_all(vcpu);  in kvm_pv_enable_async_pf()
3444 static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data)  in kvm_pv_enable_async_pf_int()  argument
3450 	if (!lapic_in_kernel(vcpu))  in kvm_pv_enable_async_pf_int()
3453 	vcpu->arch.apf.msr_int_val = data;  in kvm_pv_enable_async_pf_int()
3455 	vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK;  in kvm_pv_enable_async_pf_int()
3460 static void kvmclock_reset(struct kvm_vcpu *vcpu)  in kvmclock_reset()  argument
3462 	kvm_gpc_deactivate(&vcpu->arch.pv_time);  in kvmclock_reset()
3463 	vcpu->arch.time = 0;  in kvmclock_reset()
3466 static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu)  in kvm_vcpu_flush_tlb_all()  argument
3468 	++vcpu->stat.tlb_flush;  in kvm_vcpu_flush_tlb_all()
3469 	static_call(kvm_x86_flush_tlb_all)(vcpu);  in kvm_vcpu_flush_tlb_all()
3472 	kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);  in kvm_vcpu_flush_tlb_all()
3475 static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)  in kvm_vcpu_flush_tlb_guest()  argument
3477 	++vcpu->stat.tlb_flush;  in kvm_vcpu_flush_tlb_guest()
3486 		kvm_mmu_sync_roots(vcpu);  in kvm_vcpu_flush_tlb_guest()
3487 		kvm_mmu_sync_prev_roots(vcpu);  in kvm_vcpu_flush_tlb_guest()
3490 	static_call(kvm_x86_flush_tlb_guest)(vcpu);  in kvm_vcpu_flush_tlb_guest()
3496 	kvm_hv_vcpu_purge_flush_tlb(vcpu);  in kvm_vcpu_flush_tlb_guest()
3500 static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu)  in kvm_vcpu_flush_tlb_current()  argument
3502 	++vcpu->stat.tlb_flush;  in kvm_vcpu_flush_tlb_current()
3503 	static_call(kvm_x86_flush_tlb_current)(vcpu);  in kvm_vcpu_flush_tlb_current()
3512 void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu)  in kvm_service_local_tlb_flush_requests()  argument
3514 	if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))  in kvm_service_local_tlb_flush_requests()
3515 		kvm_vcpu_flush_tlb_current(vcpu);  in kvm_service_local_tlb_flush_requests()
3517 	if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu))  in kvm_service_local_tlb_flush_requests()
3518 		kvm_vcpu_flush_tlb_guest(vcpu);  in kvm_service_local_tlb_flush_requests()
3522 static void record_steal_time(struct kvm_vcpu *vcpu)  in record_steal_time()  argument
3524 	struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;  in record_steal_time()
3527 	gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;  in record_steal_time()
3531 	if (kvm_xen_msr_enabled(vcpu->kvm)) {  in record_steal_time()
3532 		kvm_xen_runstate_set_running(vcpu);  in record_steal_time()
3536 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))  in record_steal_time()
3539 	if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm))  in record_steal_time()
3542 	slots = kvm_memslots(vcpu->kvm);  in record_steal_time()
3550 		if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st)) ||  in record_steal_time()
3560 	if (guest_pv_has(vcpu, KVM_FEATURE_PV_TLB_FLUSH)) {  in record_steal_time()
3579 		vcpu->arch.st.preempted = 0;  in record_steal_time()
3581 		trace_kvm_pv_tlb_flush(vcpu->vcpu_id,  in record_steal_time()
3584 			kvm_vcpu_flush_tlb_guest(vcpu);  in record_steal_time()
3593 		vcpu->arch.st.preempted = 0;  in record_steal_time()
3607 		vcpu->arch.st.last_steal;  in record_steal_time()
3608 	vcpu->arch.st.last_steal = current->sched_info.run_delay;  in record_steal_time()
3617 	mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));  in record_steal_time()
3632 int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in kvm_set_msr_common()  argument
3637 	if (msr && msr == vcpu->kvm->arch.xen_hvm_config.msr)  in kvm_set_msr_common()
3638 		return kvm_xen_write_hypercall_page(vcpu, data);  in kvm_set_msr_common()
3653 			vcpu->arch.microcode_version = data;  in kvm_set_msr_common()
3658 		vcpu->arch.arch_capabilities = data;  in kvm_set_msr_common()
3668 		 * disallows changing feature MSRs after the vCPU has run; PMU  in kvm_set_msr_common()
3669 		 * refresh will bug the VM if called after the vCPU has run.  in kvm_set_msr_common()
3671 		if (vcpu->arch.perf_capabilities == data)  in kvm_set_msr_common()
3674 		vcpu->arch.perf_capabilities = data;  in kvm_set_msr_common()
3675 		kvm_pmu_refresh(vcpu);  in kvm_set_msr_common()
3678 		if (!msr_info->host_initiated && !guest_has_pred_cmd_msr(vcpu))  in kvm_set_msr_common()
3690 		    !guest_cpuid_has(vcpu, X86_FEATURE_FLUSH_L1D))  in kvm_set_msr_common()
3701 		return set_efer(vcpu, msr_info);  in kvm_set_msr_common()
3709 			vcpu->arch.msr_hwcr = data;  in kvm_set_msr_common()
3711 			kvm_pr_unimpl_wrmsr(vcpu, msr, data);  in kvm_set_msr_common()
3717 			kvm_pr_unimpl_wrmsr(vcpu, msr, data);  in kvm_set_msr_common()
3725 		vcpu->arch.pat = data;  in kvm_set_msr_common()
3729 		return kvm_mtrr_set_msr(vcpu, msr, data);  in kvm_set_msr_common()
3731 		return kvm_set_apic_base(vcpu, msr_info);  in kvm_set_msr_common()
3733 		return kvm_x2apic_msr_write(vcpu, msr, data);  in kvm_set_msr_common()
3735 		kvm_set_lapic_tscdeadline_msr(vcpu, data);  in kvm_set_msr_common()
3738 		if (guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST)) {  in kvm_set_msr_common()
3740 				s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;  in kvm_set_msr_common()
3741 				adjust_tsc_offset_guest(vcpu, adj);  in kvm_set_msr_common()
3745 				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_set_msr_common()
3747 			vcpu->arch.ia32_tsc_adjust_msr = data;  in kvm_set_msr_common()
3751 		u64 old_val = vcpu->arch.ia32_misc_enable_msr;  in kvm_set_msr_common()
3763 		if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&  in kvm_set_msr_common()
3765 			if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3))  in kvm_set_msr_common()
3767 			vcpu->arch.ia32_misc_enable_msr = data;  in kvm_set_msr_common()
3768 			kvm_update_cpuid_runtime(vcpu);  in kvm_set_msr_common()
3770 			vcpu->arch.ia32_misc_enable_msr = data;  in kvm_set_msr_common()
3777 		vcpu->arch.smbase = data;  in kvm_set_msr_common()
3780 		vcpu->arch.msr_ia32_power_ctl = data;  in kvm_set_msr_common()
3784 			kvm_synchronize_tsc(vcpu, data);  in kvm_set_msr_common()
3786 			u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset;  in kvm_set_msr_common()
3787 			adjust_tsc_offset_guest(vcpu, adj);  in kvm_set_msr_common()
3788 			vcpu->arch.ia32_tsc_adjust_msr += adj;  in kvm_set_msr_common()
3793 		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))  in kvm_set_msr_common()
3802 		vcpu->arch.ia32_xss = data;  in kvm_set_msr_common()
3803 		kvm_update_cpuid_runtime(vcpu);  in kvm_set_msr_common()
3808 		vcpu->arch.smi_count = data;  in kvm_set_msr_common()
3811 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_set_msr_common()
3814 		vcpu->kvm->arch.wall_clock = data;  in kvm_set_msr_common()
3815 		kvm_write_wall_clock(vcpu->kvm, data, 0);  in kvm_set_msr_common()
3818 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_set_msr_common()
3821 		vcpu->kvm->arch.wall_clock = data;  in kvm_set_msr_common()
3822 		kvm_write_wall_clock(vcpu->kvm, data, 0);  in kvm_set_msr_common()
3825 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_set_msr_common()
3828 		kvm_write_system_time(vcpu, data, false, msr_info->host_initiated);  in kvm_set_msr_common()
3831 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_set_msr_common()
3834 		kvm_write_system_time(vcpu, data, true,  msr_info->host_initiated);  in kvm_set_msr_common()
3837 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))  in kvm_set_msr_common()
3840 		if (kvm_pv_enable_async_pf(vcpu, data))  in kvm_set_msr_common()
3844 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))  in kvm_set_msr_common()
3847 		if (kvm_pv_enable_async_pf_int(vcpu, data))  in kvm_set_msr_common()
3851 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))  in kvm_set_msr_common()
3854 			vcpu->arch.apf.pageready_pending = false;  in kvm_set_msr_common()
3855 			kvm_check_async_pf_completion(vcpu);  in kvm_set_msr_common()
3859 		if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))  in kvm_set_msr_common()
3868 		vcpu->arch.st.msr_val = data;  in kvm_set_msr_common()
3873 		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);  in kvm_set_msr_common()
3877 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))  in kvm_set_msr_common()
3880 		if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8)))  in kvm_set_msr_common()
3885 		if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))  in kvm_set_msr_common()
3892 		vcpu->arch.msr_kvm_poll_control = data;  in kvm_set_msr_common()
3899 		return set_msr_mce(vcpu, msr_info);  in kvm_set_msr_common()
3905 		if (kvm_pmu_is_valid_msr(vcpu, msr))  in kvm_set_msr_common()
3906 			return kvm_pmu_set_msr(vcpu, msr_info);  in kvm_set_msr_common()
3909 			kvm_pr_unimpl_wrmsr(vcpu, msr, data);  in kvm_set_msr_common()
3931 		return kvm_hv_set_msr_common(vcpu, msr, data,  in kvm_set_msr_common()
3937 		kvm_pr_unimpl_wrmsr(vcpu, msr, data);  in kvm_set_msr_common()
3940 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))  in kvm_set_msr_common()
3942 		vcpu->arch.osvw.length = data;  in kvm_set_msr_common()
3945 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))  in kvm_set_msr_common()
3947 		vcpu->arch.osvw.status = data;  in kvm_set_msr_common()
3952 		     cpuid_fault_enabled(vcpu)))  in kvm_set_msr_common()
3954 		vcpu->arch.msr_platform_info = data;  in kvm_set_msr_common()
3959 		     !supports_cpuid_fault(vcpu)))  in kvm_set_msr_common()
3961 		vcpu->arch.msr_misc_features_enables = data;  in kvm_set_msr_common()
3966 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))  in kvm_set_msr_common()
3969 		if (data & ~kvm_guest_supported_xfd(vcpu))  in kvm_set_msr_common()
3972 		fpu_update_guest_xfd(&vcpu->arch.guest_fpu, data);  in kvm_set_msr_common()
3976 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))  in kvm_set_msr_common()
3979 		if (data & ~kvm_guest_supported_xfd(vcpu))  in kvm_set_msr_common()
3982 		vcpu->arch.guest_fpu.xfd_err = data;  in kvm_set_msr_common()
3986 		if (kvm_pmu_is_valid_msr(vcpu, msr))  in kvm_set_msr_common()
3987 			return kvm_pmu_set_msr(vcpu, msr_info);  in kvm_set_msr_common()
4003 static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)  in get_msr_mce()  argument
4006 	u64 mcg_cap = vcpu->arch.mcg_cap;  in get_msr_mce()
4016 		data = vcpu->arch.mcg_cap;  in get_msr_mce()
4021 		data = vcpu->arch.mcg_ctl;  in get_msr_mce()
4024 		data = vcpu->arch.mcg_status;  in get_msr_mce()
4035 		data = vcpu->arch.mci_ctl2_banks[offset];  in get_msr_mce()
4044 		data = vcpu->arch.mce_banks[offset];  in get_msr_mce()
4053 int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in kvm_get_msr_common()  argument
4091 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))  in kvm_get_msr_common()
4092 			return kvm_pmu_get_msr(vcpu, msr_info);  in kvm_get_msr_common()
4096 		msr_info->data = vcpu->arch.microcode_version;  in kvm_get_msr_common()
4100 		    !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))  in kvm_get_msr_common()
4102 		msr_info->data = vcpu->arch.arch_capabilities;  in kvm_get_msr_common()
4106 		    !guest_cpuid_has(vcpu, X86_FEATURE_PDCM))  in kvm_get_msr_common()
4108 		msr_info->data = vcpu->arch.perf_capabilities;  in kvm_get_msr_common()
4111 		msr_info->data = vcpu->arch.msr_ia32_power_ctl;  in kvm_get_msr_common()
4126 			offset = vcpu->arch.l1_tsc_offset;  in kvm_get_msr_common()
4127 			ratio = vcpu->arch.l1_tsc_scaling_ratio;  in kvm_get_msr_common()
4129 			offset = vcpu->arch.tsc_offset;  in kvm_get_msr_common()
4130 			ratio = vcpu->arch.tsc_scaling_ratio;  in kvm_get_msr_common()
4137 		msr_info->data = vcpu->arch.pat;  in kvm_get_msr_common()
4142 		return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);  in kvm_get_msr_common()
4161 		msr_info->data = kvm_get_apic_base(vcpu);  in kvm_get_msr_common()
4164 		return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);  in kvm_get_msr_common()
4166 		msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);  in kvm_get_msr_common()
4169 		msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;  in kvm_get_msr_common()
4172 		msr_info->data = vcpu->arch.ia32_misc_enable_msr;  in kvm_get_msr_common()
4177 		msr_info->data = vcpu->arch.smbase;  in kvm_get_msr_common()
4180 		msr_info->data = vcpu->arch.smi_count;  in kvm_get_msr_common()
4189 		msr_info->data = vcpu->arch.efer;  in kvm_get_msr_common()
4192 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_get_msr_common()
4195 		msr_info->data = vcpu->kvm->arch.wall_clock;  in kvm_get_msr_common()
4198 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_get_msr_common()
4201 		msr_info->data = vcpu->kvm->arch.wall_clock;  in kvm_get_msr_common()
4204 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_get_msr_common()
4207 		msr_info->data = vcpu->arch.time;  in kvm_get_msr_common()
4210 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_get_msr_common()
4213 		msr_info->data = vcpu->arch.time;  in kvm_get_msr_common()
4216 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))  in kvm_get_msr_common()
4219 		msr_info->data = vcpu->arch.apf.msr_en_val;  in kvm_get_msr_common()
4222 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))  in kvm_get_msr_common()
4225 		msr_info->data = vcpu->arch.apf.msr_int_val;  in kvm_get_msr_common()
4228 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))  in kvm_get_msr_common()
4234 		if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))  in kvm_get_msr_common()
4237 		msr_info->data = vcpu->arch.st.msr_val;  in kvm_get_msr_common()
4240 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))  in kvm_get_msr_common()
4243 		msr_info->data = vcpu->arch.pv_eoi.msr_val;  in kvm_get_msr_common()
4246 		if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))  in kvm_get_msr_common()
4249 		msr_info->data = vcpu->arch.msr_kvm_poll_control;  in kvm_get_msr_common()
4258 		return get_msr_mce(vcpu, msr_info->index, &msr_info->data,  in kvm_get_msr_common()
4262 		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))  in kvm_get_msr_common()
4264 		msr_info->data = vcpu->arch.ia32_xss;  in kvm_get_msr_common()
4288 		return kvm_hv_get_msr_common(vcpu,  in kvm_get_msr_common()
4305 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))  in kvm_get_msr_common()
4307 		msr_info->data = vcpu->arch.osvw.length;  in kvm_get_msr_common()
4310 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))  in kvm_get_msr_common()
4312 		msr_info->data = vcpu->arch.osvw.status;  in kvm_get_msr_common()
4316 		    !vcpu->kvm->arch.guest_can_read_msr_platform_info)  in kvm_get_msr_common()
4318 		msr_info->data = vcpu->arch.msr_platform_info;  in kvm_get_msr_common()
4321 		msr_info->data = vcpu->arch.msr_misc_features_enables;  in kvm_get_msr_common()
4324 		msr_info->data = vcpu->arch.msr_hwcr;  in kvm_get_msr_common()
4329 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))  in kvm_get_msr_common()
4332 		msr_info->data = vcpu->arch.guest_fpu.fpstate->xfd;  in kvm_get_msr_common()
4336 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))  in kvm_get_msr_common()
4339 		msr_info->data = vcpu->arch.guest_fpu.xfd_err;  in kvm_get_msr_common()
4343 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))  in kvm_get_msr_common()
4344 			return kvm_pmu_get_msr(vcpu, msr_info);  in kvm_get_msr_common()
4367 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,  in __msr_io()  argument
4369 		    int (*do_msr)(struct kvm_vcpu *vcpu,  in __msr_io()  argument
4375 		if (do_msr(vcpu, entries[i].index, &entries[i].data))  in __msr_io()
4386 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,  in msr_io()  argument
4387 		  int (*do_msr)(struct kvm_vcpu *vcpu,  in msr_io()  argument
4411 	r = __msr_io(vcpu, &msrs, entries, do_msr);  in msr_io()
4428 static int kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu *vcpu,  in kvm_ioctl_get_supported_hv_cpuid()  argument
4438 	r = kvm_get_hv_cpuid(vcpu, &cpuid, cpuid_arg->entries);  in kvm_ioctl_get_supported_hv_cpuid()
4804 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)  in need_emulate_wbinvd()  argument
4806 	return kvm_arch_has_noncoherent_dma(vcpu->kvm);  in need_emulate_wbinvd()
4809 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)  in kvm_arch_vcpu_load()  argument
4812 	if (need_emulate_wbinvd(vcpu)) {  in kvm_arch_vcpu_load()
4814 			cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);  in kvm_arch_vcpu_load()
4815 		else if (vcpu->cpu != -1 && vcpu->cpu != cpu)  in kvm_arch_vcpu_load()
4816 			smp_call_function_single(vcpu->cpu,  in kvm_arch_vcpu_load()
4820 	static_call(kvm_x86_vcpu_load)(vcpu, cpu);  in kvm_arch_vcpu_load()
4823 	vcpu->arch.host_pkru = read_pkru();  in kvm_arch_vcpu_load()
4826 	if (unlikely(vcpu->arch.tsc_offset_adjustment)) {  in kvm_arch_vcpu_load()
4827 		adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);  in kvm_arch_vcpu_load()
4828 		vcpu->arch.tsc_offset_adjustment = 0;  in kvm_arch_vcpu_load()
4829 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_arch_vcpu_load()
4832 	if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {  in kvm_arch_vcpu_load()
4833 		s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :  in kvm_arch_vcpu_load()
4834 				rdtsc() - vcpu->arch.last_host_tsc;  in kvm_arch_vcpu_load()
4839 			u64 offset = kvm_compute_l1_tsc_offset(vcpu,  in kvm_arch_vcpu_load()
4840 						vcpu->arch.last_guest_tsc);  in kvm_arch_vcpu_load()
4841 			kvm_vcpu_write_tsc_offset(vcpu, offset);  in kvm_arch_vcpu_load()
4842 			vcpu->arch.tsc_catchup = 1;  in kvm_arch_vcpu_load()
4845 		if (kvm_lapic_hv_timer_in_use(vcpu))  in kvm_arch_vcpu_load()
4846 			kvm_lapic_restart_hv_timer(vcpu);  in kvm_arch_vcpu_load()
4850 		 * kvmclock on vcpu->cpu migration  in kvm_arch_vcpu_load()
4852 		if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)  in kvm_arch_vcpu_load()
4853 			kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);  in kvm_arch_vcpu_load()
4854 		if (vcpu->cpu != cpu)  in kvm_arch_vcpu_load()
4855 			kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);  in kvm_arch_vcpu_load()
4856 		vcpu->cpu = cpu;  in kvm_arch_vcpu_load()
4859 	kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);  in kvm_arch_vcpu_load()
4862 static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)  in kvm_steal_time_set_preempted()  argument
4864 	struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;  in kvm_steal_time_set_preempted()
4868 	gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;  in kvm_steal_time_set_preempted()
4871 	 * The vCPU can be marked preempted if and only if the VM-Exit was on  in kvm_steal_time_set_preempted()
4874 	 * when this is true, for example allowing the vCPU to be marked  in kvm_steal_time_set_preempted()
4877 	if (!vcpu->arch.at_instruction_boundary) {  in kvm_steal_time_set_preempted()
4878 		vcpu->stat.preemption_other++;  in kvm_steal_time_set_preempted()
4882 	vcpu->stat.preemption_reported++;  in kvm_steal_time_set_preempted()
4883 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))  in kvm_steal_time_set_preempted()
4886 	if (vcpu->arch.st.preempted)  in kvm_steal_time_set_preempted()
4890 	if (unlikely(current->mm != vcpu->kvm->mm))  in kvm_steal_time_set_preempted()
4893 	slots = kvm_memslots(vcpu->kvm);  in kvm_steal_time_set_preempted()
4904 		vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;  in kvm_steal_time_set_preempted()
4906 	mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));  in kvm_steal_time_set_preempted()
4909 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_put()  argument
4913 	if (vcpu->preempted) {  in kvm_arch_vcpu_put()
4914 		if (!vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_put()
4915 			vcpu->arch.preempted_in_kernel = !static_call(kvm_x86_get_cpl)(vcpu);  in kvm_arch_vcpu_put()
4921 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_put()
4922 		if (kvm_xen_msr_enabled(vcpu->kvm))  in kvm_arch_vcpu_put()
4923 			kvm_xen_runstate_set_preempted(vcpu);  in kvm_arch_vcpu_put()
4925 			kvm_steal_time_set_preempted(vcpu);  in kvm_arch_vcpu_put()
4926 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_put()
4929 	static_call(kvm_x86_vcpu_put)(vcpu);  in kvm_arch_vcpu_put()
4930 	vcpu->arch.last_host_tsc = rdtsc();  in kvm_arch_vcpu_put()
4933 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_get_lapic()  argument
4936 	static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);  in kvm_vcpu_ioctl_get_lapic()
4938 	return kvm_apic_get_state(vcpu, s);  in kvm_vcpu_ioctl_get_lapic()
4941 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_set_lapic()  argument
4946 	r = kvm_apic_set_state(vcpu, s);  in kvm_vcpu_ioctl_set_lapic()
4949 	update_cr8_intercept(vcpu);  in kvm_vcpu_ioctl_set_lapic()
4954 static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu)  in kvm_cpu_accept_dm_intr()  argument
4962 	if (kvm_cpu_has_extint(vcpu))  in kvm_cpu_accept_dm_intr()
4966 	return (!lapic_in_kernel(vcpu) ||  in kvm_cpu_accept_dm_intr()
4967 		kvm_apic_accept_pic_intr(vcpu));  in kvm_cpu_accept_dm_intr()
4970 static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)  in kvm_vcpu_ready_for_interrupt_injection()  argument
4979 	return (kvm_arch_interrupt_allowed(vcpu) &&  in kvm_vcpu_ready_for_interrupt_injection()
4980 		kvm_cpu_accept_dm_intr(vcpu) &&  in kvm_vcpu_ready_for_interrupt_injection()
4981 		!kvm_event_needs_reinjection(vcpu) &&  in kvm_vcpu_ready_for_interrupt_injection()
4982 		!kvm_is_exception_pending(vcpu));  in kvm_vcpu_ready_for_interrupt_injection()
4985 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_interrupt()  argument
4991 	if (!irqchip_in_kernel(vcpu->kvm)) {  in kvm_vcpu_ioctl_interrupt()
4992 		kvm_queue_interrupt(vcpu, irq->irq, false);  in kvm_vcpu_ioctl_interrupt()
4993 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_ioctl_interrupt()
5001 	if (pic_in_kernel(vcpu->kvm))  in kvm_vcpu_ioctl_interrupt()
5004 	if (vcpu->arch.pending_external_vector != -1)  in kvm_vcpu_ioctl_interrupt()
5007 	vcpu->arch.pending_external_vector = irq->irq;  in kvm_vcpu_ioctl_interrupt()
5008 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_ioctl_interrupt()
5012 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)  in kvm_vcpu_ioctl_nmi()  argument
5014 	kvm_inject_nmi(vcpu);  in kvm_vcpu_ioctl_nmi()
5019 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,  in vcpu_ioctl_tpr_access_reporting()  argument
5024 	vcpu->arch.tpr_access_reporting = !!tac->enabled;  in vcpu_ioctl_tpr_access_reporting()
5028 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_setup_mce()  argument
5040 	vcpu->arch.mcg_cap = mcg_cap;  in kvm_vcpu_ioctl_x86_setup_mce()
5043 		vcpu->arch.mcg_ctl = ~(u64)0;  in kvm_vcpu_ioctl_x86_setup_mce()
5046 		vcpu->arch.mce_banks[bank*4] = ~(u64)0;  in kvm_vcpu_ioctl_x86_setup_mce()
5048 			vcpu->arch.mci_ctl2_banks[bank] = 0;  in kvm_vcpu_ioctl_x86_setup_mce()
5051 	kvm_apic_after_set_mcg_cap(vcpu);  in kvm_vcpu_ioctl_x86_setup_mce()
5053 	static_call(kvm_x86_setup_mce)(vcpu);  in kvm_vcpu_ioctl_x86_setup_mce()
5075 static int kvm_vcpu_x86_set_ucna(struct kvm_vcpu *vcpu, struct kvm_x86_mce *mce, u64* banks)  in kvm_vcpu_x86_set_ucna()  argument
5077 	u64 mcg_cap = vcpu->arch.mcg_cap;  in kvm_vcpu_x86_set_ucna()
5082 	vcpu->arch.mcg_status = mce->mcg_status;  in kvm_vcpu_x86_set_ucna()
5085 	    !(vcpu->arch.mci_ctl2_banks[mce->bank] & MCI_CTL2_CMCI_EN))  in kvm_vcpu_x86_set_ucna()
5088 	if (lapic_in_kernel(vcpu))  in kvm_vcpu_x86_set_ucna()
5089 		kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTCMCI);  in kvm_vcpu_x86_set_ucna()
5094 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_mce()  argument
5097 	u64 mcg_cap = vcpu->arch.mcg_cap;  in kvm_vcpu_ioctl_x86_set_mce()
5099 	u64 *banks = vcpu->arch.mce_banks;  in kvm_vcpu_ioctl_x86_set_mce()
5107 		return kvm_vcpu_x86_set_ucna(vcpu, mce, banks);  in kvm_vcpu_ioctl_x86_set_mce()
5114 	    vcpu->arch.mcg_ctl != ~(u64)0)  in kvm_vcpu_ioctl_x86_set_mce()
5123 		if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||  in kvm_vcpu_ioctl_x86_set_mce()
5124 		    !kvm_is_cr4_bit_set(vcpu, X86_CR4_MCE)) {  in kvm_vcpu_ioctl_x86_set_mce()
5125 			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_vcpu_ioctl_x86_set_mce()
5132 		vcpu->arch.mcg_status = mce->mcg_status;  in kvm_vcpu_ioctl_x86_set_mce()
5134 		kvm_queue_exception(vcpu, MC_VECTOR);  in kvm_vcpu_ioctl_x86_set_mce()
5147 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_vcpu_events()  argument
5152 	process_nmi(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5155 	if (kvm_check_request(KVM_REQ_SMI, vcpu))  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5156 		process_smi(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5166 	if (vcpu->arch.exception_vmexit.pending &&  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5167 	    !vcpu->arch.exception.pending &&  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5168 	    !vcpu->arch.exception.injected)  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5169 		ex = &vcpu->arch.exception_vmexit;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5171 		ex = &vcpu->arch.exception;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5181 	if (!vcpu->kvm->arch.exception_payload_enabled &&  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5183 		kvm_deliver_exception_payload(vcpu, ex);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5201 		if (!vcpu->kvm->arch.exception_payload_enabled)  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5211 		vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5212 	events->interrupt.nr = vcpu->arch.interrupt.nr;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5213 	events->interrupt.shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5215 	events->nmi.injected = vcpu->arch.nmi_injected;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5216 	events->nmi.pending = kvm_get_nr_pending_nmis(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5217 	events->nmi.masked = static_call(kvm_x86_get_nmi_mask)(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5222 	events->smi.smm = is_smm(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5223 	events->smi.pending = vcpu->arch.smi_pending;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5225 		!!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5227 	events->smi.latched_init = kvm_lapic_latched_init(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5232 	if (vcpu->kvm->arch.exception_payload_enabled)  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5234 	if (vcpu->kvm->arch.triple_fault_event) {  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5235 		events->triple_fault.pending = kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5240 static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_vcpu_events()  argument
5252 		if (!vcpu->kvm->arch.exception_payload_enabled)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5270 	    vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5273 	process_nmi(vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5283 	vcpu->arch.exception_from_userspace = events->exception.pending;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5285 	vcpu->arch.exception_vmexit.pending = false;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5287 	vcpu->arch.exception.injected = events->exception.injected;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5288 	vcpu->arch.exception.pending = events->exception.pending;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5289 	vcpu->arch.exception.vector = events->exception.nr;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5290 	vcpu->arch.exception.has_error_code = events->exception.has_error_code;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5291 	vcpu->arch.exception.error_code = events->exception.error_code;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5292 	vcpu->arch.exception.has_payload = events->exception_has_payload;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5293 	vcpu->arch.exception.payload = events->exception_payload;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5295 	vcpu->arch.interrupt.injected = events->interrupt.injected;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5296 	vcpu->arch.interrupt.nr = events->interrupt.nr;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5297 	vcpu->arch.interrupt.soft = events->interrupt.soft;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5299 		static_call(kvm_x86_set_interrupt_shadow)(vcpu,  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5302 	vcpu->arch.nmi_injected = events->nmi.injected;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5304 		vcpu->arch.nmi_pending = 0;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5305 		atomic_set(&vcpu->arch.nmi_queued, events->nmi.pending);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5307 			kvm_make_request(KVM_REQ_NMI, vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5309 	static_call(kvm_x86_set_nmi_mask)(vcpu, events->nmi.masked);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5312 	    lapic_in_kernel(vcpu))  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5313 		vcpu->arch.apic->sipi_vector = events->sipi_vector;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5317 		if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5318 			kvm_leave_nested(vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5319 			kvm_smm_changed(vcpu, events->smi.smm);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5322 		vcpu->arch.smi_pending = events->smi.pending;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5326 				vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5328 				vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5337 		if (lapic_in_kernel(vcpu)) {  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5339 				set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5341 				clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5346 		if (!vcpu->kvm->arch.triple_fault_event)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5349 			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5351 			kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5354 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5359 static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_debugregs()  argument
5365 	memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));  in kvm_vcpu_ioctl_x86_get_debugregs()
5366 	kvm_get_dr(vcpu, 6, &val);  in kvm_vcpu_ioctl_x86_get_debugregs()
5368 	dbgregs->dr7 = vcpu->arch.dr7;  in kvm_vcpu_ioctl_x86_get_debugregs()
5371 static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_debugregs()  argument
5382 	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));  in kvm_vcpu_ioctl_x86_set_debugregs()
5383 	kvm_update_dr0123(vcpu);  in kvm_vcpu_ioctl_x86_set_debugregs()
5384 	vcpu->arch.dr6 = dbgregs->dr6;  in kvm_vcpu_ioctl_x86_set_debugregs()
5385 	vcpu->arch.dr7 = dbgregs->dr7;  in kvm_vcpu_ioctl_x86_set_debugregs()
5386 	kvm_update_dr7(vcpu);  in kvm_vcpu_ioctl_x86_set_debugregs()
5392 static void kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_xsave2()  argument
5407 	u64 supported_xcr0 = vcpu->arch.guest_supported_xcr0 |  in kvm_vcpu_ioctl_x86_get_xsave2()
5410 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_vcpu_ioctl_x86_get_xsave2()
5413 	fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu, state, size,  in kvm_vcpu_ioctl_x86_get_xsave2()
5414 				       supported_xcr0, vcpu->arch.pkru);  in kvm_vcpu_ioctl_x86_get_xsave2()
5417 static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_xsave()  argument
5420 	return kvm_vcpu_ioctl_x86_get_xsave2(vcpu, (void *)guest_xsave->region,  in kvm_vcpu_ioctl_x86_get_xsave()
5424 static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_xsave()  argument
5427 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_vcpu_ioctl_x86_set_xsave()
5430 	return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu,  in kvm_vcpu_ioctl_x86_set_xsave()
5433 					      &vcpu->arch.pkru);  in kvm_vcpu_ioctl_x86_set_xsave()
5436 static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_xcrs()  argument
5447 	guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;  in kvm_vcpu_ioctl_x86_get_xcrs()
5450 static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_xcrs()  argument
5464 			r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,  in kvm_vcpu_ioctl_x86_set_xcrs()
5479 static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)  in kvm_set_guest_paused()  argument
5481 	if (!vcpu->arch.pv_time.active)  in kvm_set_guest_paused()
5483 	vcpu->arch.pvclock_set_guest_stopped_request = true;  in kvm_set_guest_paused()
5484 	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_set_guest_paused()
5488 static int kvm_arch_tsc_has_attr(struct kvm_vcpu *vcpu,  in kvm_arch_tsc_has_attr()  argument
5504 static int kvm_arch_tsc_get_attr(struct kvm_vcpu *vcpu,  in kvm_arch_tsc_get_attr()  argument
5516 		if (put_user(vcpu->arch.l1_tsc_offset, uaddr))  in kvm_arch_tsc_get_attr()
5527 static int kvm_arch_tsc_set_attr(struct kvm_vcpu *vcpu,  in kvm_arch_tsc_set_attr()  argument
5531 	struct kvm *kvm = vcpu->kvm;  in kvm_arch_tsc_set_attr()
5549 		matched = (vcpu->arch.virtual_tsc_khz &&  in kvm_arch_tsc_set_attr()
5550 			   kvm->arch.last_tsc_khz == vcpu->arch.virtual_tsc_khz &&  in kvm_arch_tsc_set_attr()
5553 		tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio) + offset;  in kvm_arch_tsc_set_attr()
5556 		__kvm_synchronize_tsc(vcpu, offset, tsc, ns, matched);  in kvm_arch_tsc_set_attr()
5569 static int kvm_vcpu_ioctl_device_attr(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_device_attr()  argument
5584 		r = kvm_arch_tsc_has_attr(vcpu, &attr);  in kvm_vcpu_ioctl_device_attr()
5587 		r = kvm_arch_tsc_get_attr(vcpu, &attr);  in kvm_vcpu_ioctl_device_attr()
5590 		r = kvm_arch_tsc_set_attr(vcpu, &attr);  in kvm_vcpu_ioctl_device_attr()
5597 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_enable_cap()  argument
5614 		if (!irqchip_in_kernel(vcpu->kvm))  in kvm_vcpu_ioctl_enable_cap()
5616 		return kvm_hv_activate_synic(vcpu, cap->cap ==  in kvm_vcpu_ioctl_enable_cap()
5621 		r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version);  in kvm_vcpu_ioctl_enable_cap()
5633 		return static_call(kvm_x86_enable_l2_tlb_flush)(vcpu);  in kvm_vcpu_ioctl_enable_cap()
5636 		return kvm_hv_set_enforce_cpuid(vcpu, cap->args[0]);  in kvm_vcpu_ioctl_enable_cap()
5639 		vcpu->arch.pv_cpuid.enforce = cap->args[0];  in kvm_vcpu_ioctl_enable_cap()
5640 		if (vcpu->arch.pv_cpuid.enforce)  in kvm_vcpu_ioctl_enable_cap()
5641 			kvm_update_pv_runtime(vcpu);  in kvm_vcpu_ioctl_enable_cap()
5652 	struct kvm_vcpu *vcpu = filp->private_data;  in kvm_arch_vcpu_ioctl()  local
5663 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl()
5669 		if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_ioctl()
5677 		r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);  in kvm_arch_vcpu_ioctl()
5688 		if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_ioctl()
5696 		r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);  in kvm_arch_vcpu_ioctl()
5705 		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);  in kvm_arch_vcpu_ioctl()
5709 		r = kvm_vcpu_ioctl_nmi(vcpu);  in kvm_arch_vcpu_ioctl()
5713 		r = kvm_inject_smi(vcpu);  in kvm_arch_vcpu_ioctl()
5723 		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);  in kvm_arch_vcpu_ioctl()
5733 		r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,  in kvm_arch_vcpu_ioctl()
5744 		r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,  in kvm_arch_vcpu_ioctl()
5755 		int idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
5756 		r = msr_io(vcpu, argp, do_get_msr, 1);  in kvm_arch_vcpu_ioctl()
5757 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
5761 		int idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
5762 		r = msr_io(vcpu, argp, do_set_msr, 0);  in kvm_arch_vcpu_ioctl()
5763 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
5772 		r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);  in kvm_arch_vcpu_ioctl()
5786 		if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_ioctl()
5791 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
5792 		r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);  in kvm_arch_vcpu_ioctl()
5793 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
5802 		r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);  in kvm_arch_vcpu_ioctl()
5811 		r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);  in kvm_arch_vcpu_ioctl()
5817 		kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);  in kvm_arch_vcpu_ioctl()
5832 		r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);  in kvm_arch_vcpu_ioctl()
5838 		kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);  in kvm_arch_vcpu_ioctl()
5855 		r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);  in kvm_arch_vcpu_ioctl()
5860 		if (vcpu->arch.guest_fpu.uabi_size > sizeof(struct kvm_xsave))  in kvm_arch_vcpu_ioctl()
5868 		kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);  in kvm_arch_vcpu_ioctl()
5877 		int size = vcpu->arch.guest_fpu.uabi_size;  in kvm_arch_vcpu_ioctl()
5885 		r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);  in kvm_arch_vcpu_ioctl()
5890 		int size = vcpu->arch.guest_fpu.uabi_size;  in kvm_arch_vcpu_ioctl()
5897 		kvm_vcpu_ioctl_x86_get_xsave2(vcpu, u.buffer, size);  in kvm_arch_vcpu_ioctl()
5913 		kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);  in kvm_arch_vcpu_ioctl()
5929 		r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);  in kvm_arch_vcpu_ioctl()
5945 		if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))  in kvm_arch_vcpu_ioctl()
5951 		r = vcpu->arch.virtual_tsc_khz;  in kvm_arch_vcpu_ioctl()
5955 		r = kvm_set_guest_paused(vcpu);  in kvm_arch_vcpu_ioctl()
5964 		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);  in kvm_arch_vcpu_ioctl()
5980 		r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state,  in kvm_arch_vcpu_ioctl()
6024 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
6025 		r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state);  in kvm_arch_vcpu_ioctl()
6026 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
6030 		r = kvm_ioctl_get_supported_hv_cpuid(vcpu, argp);  in kvm_arch_vcpu_ioctl()
6039 		r = kvm_xen_vcpu_get_attr(vcpu, &xva);  in kvm_arch_vcpu_ioctl()
6050 		r = kvm_xen_vcpu_set_attr(vcpu, &xva);  in kvm_arch_vcpu_ioctl()
6059 		__get_sregs2(vcpu, u.sregs2);  in kvm_arch_vcpu_ioctl()
6073 		r = __set_sregs2(vcpu, u.sregs2);  in kvm_arch_vcpu_ioctl()
6079 		r = kvm_vcpu_ioctl_device_attr(vcpu, ioctl, argp);  in kvm_arch_vcpu_ioctl()
6087 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl()
6091 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)  in kvm_arch_vcpu_fault()  argument
6265 	struct kvm_vcpu *vcpu;  in kvm_arch_sync_dirty_log()  local
6268 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_arch_sync_dirty_log()
6269 		kvm_vcpu_kick(vcpu);  in kvm_arch_sync_dirty_log()
6683 	struct kvm_vcpu *vcpu;  in kvm_arch_suspend_notifier()  local
6688 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_arch_suspend_notifier()
6689 		if (!vcpu->arch.pv_time.active)  in kvm_arch_suspend_notifier()
6692 		ret = kvm_set_guest_paused(vcpu);  in kvm_arch_suspend_notifier()
6694 			kvm_err("Failed to pause guest VCPU%d: %d\n",  in kvm_arch_suspend_notifier()
6695 				vcpu->vcpu_id, ret);  in kvm_arch_suspend_notifier()
7245 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,  in vcpu_mmio_write()  argument
7253 		if (!(lapic_in_kernel(vcpu) &&  in vcpu_mmio_write()
7254 		      !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))  in vcpu_mmio_write()
7255 		    && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))  in vcpu_mmio_write()
7266 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)  in vcpu_mmio_read()  argument
7273 		if (!(lapic_in_kernel(vcpu) &&  in vcpu_mmio_read()
7274 		      !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,  in vcpu_mmio_read()
7276 		    && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))  in vcpu_mmio_read()
7288 void kvm_set_segment(struct kvm_vcpu *vcpu,  in kvm_set_segment()  argument
7291 	static_call(kvm_x86_set_segment)(vcpu, var, seg);  in kvm_set_segment()
7294 void kvm_get_segment(struct kvm_vcpu *vcpu,  in kvm_get_segment()  argument
7297 	static_call(kvm_x86_get_segment)(vcpu, var, seg);  in kvm_get_segment()
7300 gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u64 access,  in translate_nested_gpa()  argument
7303 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in translate_nested_gpa()
7306 	BUG_ON(!mmu_is_nested(vcpu));  in translate_nested_gpa()
7310 	t_gpa  = mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception);  in translate_nested_gpa()
7315 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,  in kvm_mmu_gva_to_gpa_read()  argument
7318 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_mmu_gva_to_gpa_read()
7320 	u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_mmu_gva_to_gpa_read()
7321 	return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);  in kvm_mmu_gva_to_gpa_read()
7325 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,  in kvm_mmu_gva_to_gpa_write()  argument
7328 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_mmu_gva_to_gpa_write()
7330 	u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_mmu_gva_to_gpa_write()
7332 	return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);  in kvm_mmu_gva_to_gpa_write()
7337 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,  in kvm_mmu_gva_to_gpa_system()  argument
7340 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_mmu_gva_to_gpa_system()
7342 	return mmu->gva_to_gpa(vcpu, mmu, gva, 0, exception);  in kvm_mmu_gva_to_gpa_system()
7346 				      struct kvm_vcpu *vcpu, u64 access,  in kvm_read_guest_virt_helper()  argument
7349 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_read_guest_virt_helper()
7354 		gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);  in kvm_read_guest_virt_helper()
7361 		ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data,  in kvm_read_guest_virt_helper()
7381 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in kvm_fetch_guest_virt()  local
7382 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_fetch_guest_virt()
7383 	u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_fetch_guest_virt()
7388 	gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access|PFERR_FETCH_MASK,  in kvm_fetch_guest_virt()
7396 	ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val,  in kvm_fetch_guest_virt()
7404 int kvm_read_guest_virt(struct kvm_vcpu *vcpu,  in kvm_read_guest_virt()  argument
7408 	u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_read_guest_virt()
7417 	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,  in kvm_read_guest_virt()
7426 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_read_std()  local
7431 	else if (static_call(kvm_x86_get_cpl)(vcpu) == 3)  in emulator_read_std()
7434 	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);  in emulator_read_std()
7438 				      struct kvm_vcpu *vcpu, u64 access,  in kvm_write_guest_virt_helper()  argument
7441 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_write_guest_virt_helper()
7446 		gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);  in kvm_write_guest_virt_helper()
7453 		ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite);  in kvm_write_guest_virt_helper()
7471 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_write_std()  local
7476 	else if (static_call(kvm_x86_get_cpl)(vcpu) == 3)  in emulator_write_std()
7479 	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,  in emulator_write_std()
7483 int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,  in kvm_write_guest_virt_system()  argument
7487 	vcpu->arch.l1tf_flush_l1d = true;  in kvm_write_guest_virt_system()
7489 	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,  in kvm_write_guest_virt_system()
7494 static int kvm_can_emulate_insn(struct kvm_vcpu *vcpu, int emul_type,  in kvm_can_emulate_insn()  argument
7497 	return static_call(kvm_x86_can_emulate_instruction)(vcpu, emul_type,  in kvm_can_emulate_insn()
7501 int handle_ud(struct kvm_vcpu *vcpu)  in handle_ud()  argument
7509 	if (unlikely(!kvm_can_emulate_insn(vcpu, emul_type, NULL, 0)))  in handle_ud()
7513 	    kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),  in handle_ud()
7517 			kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) & ~X86_EFLAGS_RF);  in handle_ud()
7518 		kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));  in handle_ud()
7522 	return kvm_emulate_instruction(vcpu, emul_type);  in handle_ud()
7526 static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva,  in vcpu_is_mmio_gpa()  argument
7533 	if (vcpu_match_mmio_gpa(vcpu, gpa)) {  in vcpu_is_mmio_gpa()
7541 static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,  in vcpu_mmio_gva_to_gpa()  argument
7545 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in vcpu_mmio_gva_to_gpa()
7546 	u64 access = ((static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0)  in vcpu_mmio_gva_to_gpa()
7554 	if (vcpu_match_mmio_gva(vcpu, gva) && (!is_paging(vcpu) ||  in vcpu_mmio_gva_to_gpa()
7555 	    !permission_fault(vcpu, vcpu->arch.walk_mmu,  in vcpu_mmio_gva_to_gpa()
7556 			      vcpu->arch.mmio_access, 0, access))) {  in vcpu_mmio_gva_to_gpa()
7557 		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |  in vcpu_mmio_gva_to_gpa()
7563 	*gpa = mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);  in vcpu_mmio_gva_to_gpa()
7568 	return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);  in vcpu_mmio_gva_to_gpa()
7571 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,  in emulator_write_phys()  argument
7576 	ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);  in emulator_write_phys()
7579 	kvm_page_track_write(vcpu, gpa, val, bytes);  in emulator_write_phys()
7584 	int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
7586 	int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
7588 	int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
7590 	int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
7595 static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)  in read_prepare()  argument
7597 	if (vcpu->mmio_read_completed) {  in read_prepare()
7599 			       vcpu->mmio_fragments[0].gpa, val);  in read_prepare()
7600 		vcpu->mmio_read_completed = 0;  in read_prepare()
7607 static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,  in read_emulate()  argument
7610 	return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);  in read_emulate()
7613 static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,  in write_emulate()  argument
7616 	return emulator_write_phys(vcpu, gpa, val, bytes);  in write_emulate()
7619 static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)  in write_mmio()  argument
7622 	return vcpu_mmio_write(vcpu, gpa, bytes, val);  in write_mmio()
7625 static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,  in read_exit_mmio()  argument
7632 static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,  in write_exit_mmio()  argument
7635 	struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];  in write_exit_mmio()
7637 	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));  in write_exit_mmio()
7658 				       struct kvm_vcpu *vcpu,  in emulator_read_write_onepage()  argument
7665 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in emulator_read_write_onepage()
7677 		ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write);  in emulator_read_write_onepage()
7679 		ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);  in emulator_read_write_onepage()
7684 	if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))  in emulator_read_write_onepage()
7690 	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);  in emulator_read_write_onepage()
7698 	WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);  in emulator_read_write_onepage()
7699 	frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];  in emulator_read_write_onepage()
7712 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_read_write()  local
7717 		  ops->read_write_prepare(vcpu, val, bytes))  in emulator_read_write()
7720 	vcpu->mmio_nr_fragments = 0;  in emulator_read_write()
7728 						 vcpu, ops);  in emulator_read_write()
7740 					 vcpu, ops);  in emulator_read_write()
7744 	if (!vcpu->mmio_nr_fragments)  in emulator_read_write()
7747 	gpa = vcpu->mmio_fragments[0].gpa;  in emulator_read_write()
7749 	vcpu->mmio_needed = 1;  in emulator_read_write()
7750 	vcpu->mmio_cur_fragment = 0;  in emulator_read_write()
7752 	vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);  in emulator_read_write()
7753 	vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;  in emulator_read_write()
7754 	vcpu->run->exit_reason = KVM_EXIT_MMIO;  in emulator_read_write()
7755 	vcpu->run->mmio.phys_addr = gpa;  in emulator_read_write()
7757 	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);  in emulator_read_write()
7790 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_cmpxchg_emulated()  local
7800 	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);  in emulator_cmpxchg_emulated()
7818 	hva = kvm_vcpu_gfn_to_hva(vcpu, gpa_to_gfn(gpa));  in emulator_cmpxchg_emulated()
7846 	kvm_page_track_write(vcpu, gpa, new, bytes);  in emulator_cmpxchg_emulated()
7856 static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,  in emulator_pio_in_out()  argument
7863 	WARN_ON_ONCE(vcpu->arch.pio.count);  in emulator_pio_in_out()
7866 			r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, port, size, data);  in emulator_pio_in_out()
7868 			r = kvm_io_bus_write(vcpu, KVM_PIO_BUS, port, size, data);  in emulator_pio_in_out()
7888 	vcpu->arch.pio.port = port;  in emulator_pio_in_out()
7889 	vcpu->arch.pio.in = in;  in emulator_pio_in_out()
7890 	vcpu->arch.pio.count = count;  in emulator_pio_in_out()
7891 	vcpu->arch.pio.size = size;  in emulator_pio_in_out()
7894 		memset(vcpu->arch.pio_data, 0, size * count);  in emulator_pio_in_out()
7896 		memcpy(vcpu->arch.pio_data, data, size * count);  in emulator_pio_in_out()
7898 	vcpu->run->exit_reason = KVM_EXIT_IO;  in emulator_pio_in_out()
7899 	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;  in emulator_pio_in_out()
7900 	vcpu->run->io.size = size;  in emulator_pio_in_out()
7901 	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;  in emulator_pio_in_out()
7902 	vcpu->run->io.count = count;  in emulator_pio_in_out()
7903 	vcpu->run->io.port = port;  in emulator_pio_in_out()
7907 static int emulator_pio_in(struct kvm_vcpu *vcpu, int size,  in emulator_pio_in()  argument
7910 	int r = emulator_pio_in_out(vcpu, size, port, val, count, true);  in emulator_pio_in()
7917 static void complete_emulator_pio_in(struct kvm_vcpu *vcpu, void *val)  in complete_emulator_pio_in()  argument
7919 	int size = vcpu->arch.pio.size;  in complete_emulator_pio_in()
7920 	unsigned int count = vcpu->arch.pio.count;  in complete_emulator_pio_in()
7921 	memcpy(val, vcpu->arch.pio_data, size * count);  in complete_emulator_pio_in()
7922 	trace_kvm_pio(KVM_PIO_IN, vcpu->arch.pio.port, size, count, vcpu->arch.pio_data);  in complete_emulator_pio_in()
7923 	vcpu->arch.pio.count = 0;  in complete_emulator_pio_in()
7930 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_pio_in_emulated()  local
7931 	if (vcpu->arch.pio.count) {  in emulator_pio_in_emulated()
7935 		 * can modify ECX before rerunning the vCPU.  Ignore any such  in emulator_pio_in_emulated()
7939 		complete_emulator_pio_in(vcpu, val);  in emulator_pio_in_emulated()
7943 	return emulator_pio_in(vcpu, size, port, val, count);  in emulator_pio_in_emulated()
7946 static int emulator_pio_out(struct kvm_vcpu *vcpu, int size,  in emulator_pio_out()  argument
7951 	return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);  in emulator_pio_out()
7961 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)  in get_segment_base()  argument
7963 	return static_call(kvm_x86_get_segment_base)(vcpu, seg);  in get_segment_base()
7971 static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)  in kvm_emulate_wbinvd_noskip()  argument
7973 	if (!need_emulate_wbinvd(vcpu))  in kvm_emulate_wbinvd_noskip()
7979 		cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);  in kvm_emulate_wbinvd_noskip()
7980 		on_each_cpu_mask(vcpu->arch.wbinvd_dirty_mask,  in kvm_emulate_wbinvd_noskip()
7983 		cpumask_clear(vcpu->arch.wbinvd_dirty_mask);  in kvm_emulate_wbinvd_noskip()
7989 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)  in kvm_emulate_wbinvd()  argument
7991 	kvm_emulate_wbinvd_noskip(vcpu);  in kvm_emulate_wbinvd()
7992 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_wbinvd()
8023 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_get_cr()  local
8028 		value = kvm_read_cr0(vcpu);  in emulator_get_cr()
8031 		value = vcpu->arch.cr2;  in emulator_get_cr()
8034 		value = kvm_read_cr3(vcpu);  in emulator_get_cr()
8037 		value = kvm_read_cr4(vcpu);  in emulator_get_cr()
8040 		value = kvm_get_cr8(vcpu);  in emulator_get_cr()
8052 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_cr()  local
8057 		res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));  in emulator_set_cr()
8060 		vcpu->arch.cr2 = val;  in emulator_set_cr()
8063 		res = kvm_set_cr3(vcpu, val);  in emulator_set_cr()
8066 		res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));  in emulator_set_cr()
8069 		res = kvm_set_cr8(vcpu, val);  in emulator_set_cr()
8150 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_segment()  local
8172 	kvm_set_segment(vcpu, &var, seg);  in emulator_set_segment()
8179 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_get_msr_with_filter()  local
8182 	r = kvm_get_msr_with_filter(vcpu, msr_index, pdata);  in emulator_get_msr_with_filter()
8187 		if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0,  in emulator_get_msr_with_filter()
8202 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_msr_with_filter()  local
8205 	r = kvm_set_msr_with_filter(vcpu, msr_index, data);  in emulator_set_msr_with_filter()
8210 		if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data,  in emulator_set_msr_with_filter()
8374 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)  in toggle_interruptibility()  argument
8376 	u32 int_shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);  in toggle_interruptibility()
8387 		static_call(kvm_x86_set_interrupt_shadow)(vcpu, mask);  in toggle_interruptibility()
8389 			kvm_make_request(KVM_REQ_EVENT, vcpu);  in toggle_interruptibility()
8393 static void inject_emulated_exception(struct kvm_vcpu *vcpu)  in inject_emulated_exception()  argument
8395 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in inject_emulated_exception()
8398 		kvm_inject_emulated_page_fault(vcpu, &ctxt->exception);  in inject_emulated_exception()
8400 		kvm_queue_exception_e(vcpu, ctxt->exception.vector,  in inject_emulated_exception()
8403 		kvm_queue_exception(vcpu, ctxt->exception.vector);  in inject_emulated_exception()
8406 static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu)  in alloc_emulate_ctxt()  argument
8412 		pr_err("failed to allocate vcpu's emulator\n");  in alloc_emulate_ctxt()
8416 	ctxt->vcpu = vcpu;  in alloc_emulate_ctxt()
8418 	vcpu->arch.emulate_ctxt = ctxt;  in alloc_emulate_ctxt()
8423 static void init_emulate_ctxt(struct kvm_vcpu *vcpu)  in init_emulate_ctxt()  argument
8425 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in init_emulate_ctxt()
8428 	static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);  in init_emulate_ctxt()
8431 	ctxt->eflags = kvm_get_rflags(vcpu);  in init_emulate_ctxt()
8434 	ctxt->eip = kvm_rip_read(vcpu);  in init_emulate_ctxt()
8435 	ctxt->mode = (!is_protmode(vcpu))		? X86EMUL_MODE_REAL :  in init_emulate_ctxt()
8437 		     (cs_l && is_long_mode(vcpu))	? X86EMUL_MODE_PROT64 :  in init_emulate_ctxt()
8446 	vcpu->arch.emulate_regs_need_sync_from_vcpu = false;  in init_emulate_ctxt()
8449 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)  in kvm_inject_realmode_interrupt()  argument
8451 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in kvm_inject_realmode_interrupt()
8454 	init_emulate_ctxt(vcpu);  in kvm_inject_realmode_interrupt()
8462 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_inject_realmode_interrupt()
8465 		kvm_rip_write(vcpu, ctxt->eip);  in kvm_inject_realmode_interrupt()
8466 		kvm_set_rflags(vcpu, ctxt->eflags);  in kvm_inject_realmode_interrupt()
8471 static void prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data,  in prepare_emulation_failure_exit()  argument
8474 	struct kvm_run *run = vcpu->run;  in prepare_emulation_failure_exit()
8484 	static_call(kvm_x86_get_exit_info)(vcpu, (u32 *)&info[0], &info[1],  in prepare_emulation_failure_exit()
8522 static void prepare_emulation_ctxt_failure_exit(struct kvm_vcpu *vcpu)  in prepare_emulation_ctxt_failure_exit()  argument
8524 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in prepare_emulation_ctxt_failure_exit()
8526 	prepare_emulation_failure_exit(vcpu, NULL, 0, ctxt->fetch.data,  in prepare_emulation_ctxt_failure_exit()
8530 void __kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data,  in __kvm_prepare_emulation_failure_exit()  argument
8533 	prepare_emulation_failure_exit(vcpu, data, ndata, NULL, 0);  in __kvm_prepare_emulation_failure_exit()
8537 void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu)  in kvm_prepare_emulation_failure_exit()  argument
8539 	__kvm_prepare_emulation_failure_exit(vcpu, NULL, 0);  in kvm_prepare_emulation_failure_exit()
8543 static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)  in handle_emulation_failure()  argument
8545 	struct kvm *kvm = vcpu->kvm;  in handle_emulation_failure()
8547 	++vcpu->stat.insn_emulation_fail;  in handle_emulation_failure()
8548 	trace_kvm_emulate_insn_failed(vcpu);  in handle_emulation_failure()
8551 		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);  in handle_emulation_failure()
8557 		prepare_emulation_ctxt_failure_exit(vcpu);  in handle_emulation_failure()
8561 	kvm_queue_exception(vcpu, UD_VECTOR);  in handle_emulation_failure()
8563 	if (!is_guest_mode(vcpu) && static_call(kvm_x86_get_cpl)(vcpu) == 0) {  in handle_emulation_failure()
8564 		prepare_emulation_ctxt_failure_exit(vcpu);  in handle_emulation_failure()
8571 static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,  in reexecute_instruction()  argument
8580 	if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||  in reexecute_instruction()
8584 	if (!vcpu->arch.mmu->root_role.direct) {  in reexecute_instruction()
8589 		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);  in reexecute_instruction()
8605 	pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));  in reexecute_instruction()
8617 	if (vcpu->arch.mmu->root_role.direct) {  in reexecute_instruction()
8620 		write_lock(&vcpu->kvm->mmu_lock);  in reexecute_instruction()
8621 		indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;  in reexecute_instruction()
8622 		write_unlock(&vcpu->kvm->mmu_lock);  in reexecute_instruction()
8625 			kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));  in reexecute_instruction()
8635 	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));  in reexecute_instruction()
8648 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in retry_instruction()  local
8651 	last_retry_eip = vcpu->arch.last_retry_eip;  in retry_instruction()
8652 	last_retry_addr = vcpu->arch.last_retry_addr;  in retry_instruction()
8667 	vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;  in retry_instruction()
8672 	if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||  in retry_instruction()
8682 	vcpu->arch.last_retry_eip = ctxt->eip;  in retry_instruction()
8683 	vcpu->arch.last_retry_addr = cr2_or_gpa;  in retry_instruction()
8685 	if (!vcpu->arch.mmu->root_role.direct)  in retry_instruction()
8686 		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);  in retry_instruction()
8688 	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));  in retry_instruction()
8693 static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
8694 static int complete_emulated_pio(struct kvm_vcpu *vcpu);
8711 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu)  in kvm_vcpu_do_singlestep()  argument
8713 	struct kvm_run *kvm_run = vcpu->run;  in kvm_vcpu_do_singlestep()
8715 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {  in kvm_vcpu_do_singlestep()
8717 		kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);  in kvm_vcpu_do_singlestep()
8722 	kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS);  in kvm_vcpu_do_singlestep()
8726 int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)  in kvm_skip_emulated_instruction()  argument
8728 	unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);  in kvm_skip_emulated_instruction()
8731 	r = static_call(kvm_x86_skip_emulated_instruction)(vcpu);  in kvm_skip_emulated_instruction()
8735 	kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS);  in kvm_skip_emulated_instruction()
8746 		r = kvm_vcpu_do_singlestep(vcpu);  in kvm_skip_emulated_instruction()
8751 static bool kvm_is_code_breakpoint_inhibited(struct kvm_vcpu *vcpu)  in kvm_is_code_breakpoint_inhibited()  argument
8755 	if (kvm_get_rflags(vcpu) & X86_EFLAGS_RF)  in kvm_is_code_breakpoint_inhibited()
8763 	shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);  in kvm_is_code_breakpoint_inhibited()
8765 	       guest_cpuid_is_intel(vcpu);  in kvm_is_code_breakpoint_inhibited()
8768 static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu,  in kvm_vcpu_check_code_breakpoint()  argument
8791 	if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&  in kvm_vcpu_check_code_breakpoint()
8792 	    (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {  in kvm_vcpu_check_code_breakpoint()
8793 		struct kvm_run *kvm_run = vcpu->run;  in kvm_vcpu_check_code_breakpoint()
8794 		unsigned long eip = kvm_get_linear_rip(vcpu);  in kvm_vcpu_check_code_breakpoint()
8796 					   vcpu->arch.guest_debug_dr7,  in kvm_vcpu_check_code_breakpoint()
8797 					   vcpu->arch.eff_db);  in kvm_vcpu_check_code_breakpoint()
8809 	if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&  in kvm_vcpu_check_code_breakpoint()
8810 	    !kvm_is_code_breakpoint_inhibited(vcpu)) {  in kvm_vcpu_check_code_breakpoint()
8811 		unsigned long eip = kvm_get_linear_rip(vcpu);  in kvm_vcpu_check_code_breakpoint()
8813 					   vcpu->arch.dr7,  in kvm_vcpu_check_code_breakpoint()
8814 					   vcpu->arch.db);  in kvm_vcpu_check_code_breakpoint()
8817 			kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);  in kvm_vcpu_check_code_breakpoint()
8867 int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type,  in x86_decode_emulated_instruction()  argument
8870 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in x86_decode_emulated_instruction()
8873 	init_emulate_ctxt(vcpu);  in x86_decode_emulated_instruction()
8877 	trace_kvm_emulate_insn_start(vcpu);  in x86_decode_emulated_instruction()
8878 	++vcpu->stat.insn_emulation;  in x86_decode_emulated_instruction()
8884 int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,  in x86_emulate_instruction()  argument
8888 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in x86_emulate_instruction()
8891 	if (unlikely(!kvm_can_emulate_insn(vcpu, emulation_type, insn, insn_len)))  in x86_emulate_instruction()
8894 	vcpu->arch.l1tf_flush_l1d = true;  in x86_emulate_instruction()
8897 		kvm_clear_exception_queue(vcpu);  in x86_emulate_instruction()
8904 		if (kvm_vcpu_check_code_breakpoint(vcpu, emulation_type, &r))  in x86_emulate_instruction()
8907 		r = x86_decode_emulated_instruction(vcpu, emulation_type,  in x86_emulate_instruction()
8912 				kvm_queue_exception(vcpu, UD_VECTOR);  in x86_emulate_instruction()
8915 			if (reexecute_instruction(vcpu, cr2_or_gpa,  in x86_emulate_instruction()
8927 				inject_emulated_exception(vcpu);  in x86_emulate_instruction()
8930 			return handle_emulation_failure(vcpu, emulation_type);  in x86_emulate_instruction()
8936 		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);  in x86_emulate_instruction()
8957 		kvm_rip_write(vcpu, ctxt->eip);  in x86_emulate_instruction()
8959 			kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);  in x86_emulate_instruction()
8968 	if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {  in x86_emulate_instruction()
8969 		vcpu->arch.emulate_regs_need_sync_from_vcpu = false;  in x86_emulate_instruction()
8979 		if (vcpu->arch.mmu->root_role.direct) {  in x86_emulate_instruction()
8994 		if (reexecute_instruction(vcpu, cr2_or_gpa, emulation_type))  in x86_emulate_instruction()
8997 		return handle_emulation_failure(vcpu, emulation_type);  in x86_emulate_instruction()
9001 		WARN_ON_ONCE(vcpu->mmio_needed && !vcpu->mmio_is_write);  in x86_emulate_instruction()
9002 		vcpu->mmio_needed = false;  in x86_emulate_instruction()
9004 		inject_emulated_exception(vcpu);  in x86_emulate_instruction()
9005 	} else if (vcpu->arch.pio.count) {  in x86_emulate_instruction()
9006 		if (!vcpu->arch.pio.in) {  in x86_emulate_instruction()
9008 			vcpu->arch.pio.count = 0;  in x86_emulate_instruction()
9011 			vcpu->arch.complete_userspace_io = complete_emulated_pio;  in x86_emulate_instruction()
9014 	} else if (vcpu->mmio_needed) {  in x86_emulate_instruction()
9015 		++vcpu->stat.mmio_exits;  in x86_emulate_instruction()
9017 		if (!vcpu->mmio_is_write)  in x86_emulate_instruction()
9020 		vcpu->arch.complete_userspace_io = complete_emulated_mmio;  in x86_emulate_instruction()
9021 	} else if (vcpu->arch.complete_userspace_io) {  in x86_emulate_instruction()
9031 		unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);  in x86_emulate_instruction()
9032 		toggle_interruptibility(vcpu, ctxt->interruptibility);  in x86_emulate_instruction()
9033 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;  in x86_emulate_instruction()
9042 			kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS);  in x86_emulate_instruction()
9044 				kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_BRANCH_INSTRUCTIONS);  in x86_emulate_instruction()
9045 			kvm_rip_write(vcpu, ctxt->eip);  in x86_emulate_instruction()
9046 			if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)))  in x86_emulate_instruction()
9047 				r = kvm_vcpu_do_singlestep(vcpu);  in x86_emulate_instruction()
9048 			static_call_cond(kvm_x86_update_emulated_instruction)(vcpu);  in x86_emulate_instruction()
9049 			__kvm_set_rflags(vcpu, ctxt->eflags);  in x86_emulate_instruction()
9059 			kvm_make_request(KVM_REQ_EVENT, vcpu);  in x86_emulate_instruction()
9061 		vcpu->arch.emulate_regs_need_sync_to_vcpu = true;  in x86_emulate_instruction()
9066 int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type)  in kvm_emulate_instruction()  argument
9068 	return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);  in kvm_emulate_instruction()
9072 int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,  in kvm_emulate_instruction_from_buffer()  argument
9075 	return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len);  in kvm_emulate_instruction_from_buffer()
9079 static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu)  in complete_fast_pio_out_port_0x7e()  argument
9081 	vcpu->arch.pio.count = 0;  in complete_fast_pio_out_port_0x7e()
9085 static int complete_fast_pio_out(struct kvm_vcpu *vcpu)  in complete_fast_pio_out()  argument
9087 	vcpu->arch.pio.count = 0;  in complete_fast_pio_out()
9089 	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))  in complete_fast_pio_out()
9092 	return kvm_skip_emulated_instruction(vcpu);  in complete_fast_pio_out()
9095 static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,  in kvm_fast_pio_out()  argument
9098 	unsigned long val = kvm_rax_read(vcpu);  in kvm_fast_pio_out()
9099 	int ret = emulator_pio_out(vcpu, size, port, &val, 1);  in kvm_fast_pio_out()
9109 	    kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {  in kvm_fast_pio_out()
9110 		vcpu->arch.complete_userspace_io =  in kvm_fast_pio_out()
9112 		kvm_skip_emulated_instruction(vcpu);  in kvm_fast_pio_out()
9114 		vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);  in kvm_fast_pio_out()
9115 		vcpu->arch.complete_userspace_io = complete_fast_pio_out;  in kvm_fast_pio_out()
9120 static int complete_fast_pio_in(struct kvm_vcpu *vcpu)  in complete_fast_pio_in()  argument
9125 	BUG_ON(vcpu->arch.pio.count != 1);  in complete_fast_pio_in()
9127 	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {  in complete_fast_pio_in()
9128 		vcpu->arch.pio.count = 0;  in complete_fast_pio_in()
9133 	val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;  in complete_fast_pio_in()
9135 	complete_emulator_pio_in(vcpu, &val);  in complete_fast_pio_in()
9136 	kvm_rax_write(vcpu, val);  in complete_fast_pio_in()
9138 	return kvm_skip_emulated_instruction(vcpu);  in complete_fast_pio_in()
9141 static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size,  in kvm_fast_pio_in()  argument
9148 	val = (size < 4) ? kvm_rax_read(vcpu) : 0;  in kvm_fast_pio_in()
9150 	ret = emulator_pio_in(vcpu, size, port, &val, 1);  in kvm_fast_pio_in()
9152 		kvm_rax_write(vcpu, val);  in kvm_fast_pio_in()
9156 	vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);  in kvm_fast_pio_in()
9157 	vcpu->arch.complete_userspace_io = complete_fast_pio_in;  in kvm_fast_pio_in()
9162 int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in)  in kvm_fast_pio()  argument
9167 		ret = kvm_fast_pio_in(vcpu, size, port);  in kvm_fast_pio()
9169 		ret = kvm_fast_pio_out(vcpu, size, port);  in kvm_fast_pio()
9170 	return ret && kvm_skip_emulated_instruction(vcpu);  in kvm_fast_pio()
9229 	struct kvm_vcpu *vcpu;  in __kvmclock_cpufreq_notifier()  local
9244 	 * the TSC for each VCPU.  We must flag these local variables  in __kvmclock_cpufreq_notifier()
9263 	 * anytime after the setting of the VCPU's request bit, the  in __kvmclock_cpufreq_notifier()
9276 		kvm_for_each_vcpu(i, vcpu, kvm) {  in __kvmclock_cpufreq_notifier()
9277 			if (vcpu->cpu != cpu)  in __kvmclock_cpufreq_notifier()
9279 			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in __kvmclock_cpufreq_notifier()
9280 			if (vcpu->cpu != raw_smp_processor_id())  in __kvmclock_cpufreq_notifier()
9360 	struct kvm_vcpu *vcpu;  in pvclock_gtod_update_fn()  local
9365 		kvm_for_each_vcpu(i, vcpu, kvm)  in pvclock_gtod_update_fn()
9366 			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in pvclock_gtod_update_fn()
9468 	 * vCPU's FPU state as a fxregs_state struct.  in __kvm_x86_vendor_init()
9636 static int __kvm_emulate_halt(struct kvm_vcpu *vcpu, int state, int reason)  in __kvm_emulate_halt()  argument
9639 	 * The vCPU has halted, e.g. executed HLT.  Update the run state if the  in __kvm_emulate_halt()
9641 	 * state and halt the vCPU.  Exit to userspace if the local APIC is  in __kvm_emulate_halt()
9645 	++vcpu->stat.halt_exits;  in __kvm_emulate_halt()
9646 	if (lapic_in_kernel(vcpu)) {  in __kvm_emulate_halt()
9647 		vcpu->arch.mp_state = state;  in __kvm_emulate_halt()
9650 		vcpu->run->exit_reason = reason;  in __kvm_emulate_halt()
9655 int kvm_emulate_halt_noskip(struct kvm_vcpu *vcpu)  in kvm_emulate_halt_noskip()  argument
9657 	return __kvm_emulate_halt(vcpu, KVM_MP_STATE_HALTED, KVM_EXIT_HLT);  in kvm_emulate_halt_noskip()
9661 int kvm_emulate_halt(struct kvm_vcpu *vcpu)  in kvm_emulate_halt()  argument
9663 	int ret = kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_halt()
9668 	return kvm_emulate_halt_noskip(vcpu) && ret;  in kvm_emulate_halt()
9672 int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu)  in kvm_emulate_ap_reset_hold()  argument
9674 	int ret = kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_ap_reset_hold()
9676 	return __kvm_emulate_halt(vcpu, KVM_MP_STATE_AP_RESET_HOLD,  in kvm_emulate_ap_reset_hold()
9682 static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,  in kvm_pv_clock_pairing()  argument
9697 	if (vcpu->arch.tsc_always_catchup)  in kvm_pv_clock_pairing()
9705 	clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);  in kvm_pv_clock_pairing()
9710 	if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,  in kvm_pv_clock_pairing()
9719  * kvm_pv_kick_cpu_op:  Kick a vcpu.
9721  * @apicid - apicid of vcpu to be kicked.
9745 bool kvm_vcpu_apicv_activated(struct kvm_vcpu *vcpu)  in kvm_vcpu_apicv_activated()  argument
9747 	ulong vm_reasons = READ_ONCE(vcpu->kvm->arch.apicv_inhibit_reasons);  in kvm_vcpu_apicv_activated()
9748 	ulong vcpu_reasons = static_call(kvm_x86_vcpu_get_apicv_inhibit_reasons)(vcpu);  in kvm_vcpu_apicv_activated()
9778 static void kvm_sched_yield(struct kvm_vcpu *vcpu, unsigned long dest_id)  in kvm_sched_yield()  argument
9783 	vcpu->stat.directed_yield_attempted++;  in kvm_sched_yield()
9789 	map = rcu_dereference(vcpu->kvm->arch.apic_map);  in kvm_sched_yield()
9792 		target = map->phys_map[dest_id]->vcpu;  in kvm_sched_yield()
9800 	if (vcpu == target)  in kvm_sched_yield()
9806 	vcpu->stat.directed_yield_successful++;  in kvm_sched_yield()
9812 static int complete_hypercall_exit(struct kvm_vcpu *vcpu)  in complete_hypercall_exit()  argument
9814 	u64 ret = vcpu->run->hypercall.ret;  in complete_hypercall_exit()
9816 	if (!is_64_bit_mode(vcpu))  in complete_hypercall_exit()
9818 	kvm_rax_write(vcpu, ret);  in complete_hypercall_exit()
9819 	++vcpu->stat.hypercalls;  in complete_hypercall_exit()
9820 	return kvm_skip_emulated_instruction(vcpu);  in complete_hypercall_exit()
9823 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)  in kvm_emulate_hypercall()  argument
9828 	if (kvm_xen_hypercall_enabled(vcpu->kvm))  in kvm_emulate_hypercall()
9829 		return kvm_xen_hypercall(vcpu);  in kvm_emulate_hypercall()
9831 	if (kvm_hv_hypercall_enabled(vcpu))  in kvm_emulate_hypercall()
9832 		return kvm_hv_hypercall(vcpu);  in kvm_emulate_hypercall()
9834 	nr = kvm_rax_read(vcpu);  in kvm_emulate_hypercall()
9835 	a0 = kvm_rbx_read(vcpu);  in kvm_emulate_hypercall()
9836 	a1 = kvm_rcx_read(vcpu);  in kvm_emulate_hypercall()
9837 	a2 = kvm_rdx_read(vcpu);  in kvm_emulate_hypercall()
9838 	a3 = kvm_rsi_read(vcpu);  in kvm_emulate_hypercall()
9842 	op_64_bit = is_64_bit_hypercall(vcpu);  in kvm_emulate_hypercall()
9851 	if (static_call(kvm_x86_get_cpl)(vcpu) != 0) {  in kvm_emulate_hypercall()
9863 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_UNHALT))  in kvm_emulate_hypercall()
9866 		kvm_pv_kick_cpu_op(vcpu->kvm, a1);  in kvm_emulate_hypercall()
9867 		kvm_sched_yield(vcpu, a1);  in kvm_emulate_hypercall()
9872 		ret = kvm_pv_clock_pairing(vcpu, a0, a1);  in kvm_emulate_hypercall()
9876 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SEND_IPI))  in kvm_emulate_hypercall()
9879 		ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);  in kvm_emulate_hypercall()
9882 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SCHED_YIELD))  in kvm_emulate_hypercall()
9885 		kvm_sched_yield(vcpu, a0);  in kvm_emulate_hypercall()
9892 		if (!(vcpu->kvm->arch.hypercall_exit_enabled & (1 << KVM_HC_MAP_GPA_RANGE)))  in kvm_emulate_hypercall()
9901 		vcpu->run->exit_reason        = KVM_EXIT_HYPERCALL;  in kvm_emulate_hypercall()
9902 		vcpu->run->hypercall.nr       = KVM_HC_MAP_GPA_RANGE;  in kvm_emulate_hypercall()
9903 		vcpu->run->hypercall.args[0]  = gpa;  in kvm_emulate_hypercall()
9904 		vcpu->run->hypercall.args[1]  = npages;  in kvm_emulate_hypercall()
9905 		vcpu->run->hypercall.args[2]  = attrs;  in kvm_emulate_hypercall()
9906 		vcpu->run->hypercall.flags    = 0;  in kvm_emulate_hypercall()
9908 			vcpu->run->hypercall.flags |= KVM_EXIT_HYPERCALL_LONG_MODE;  in kvm_emulate_hypercall()
9910 		WARN_ON_ONCE(vcpu->run->hypercall.flags & KVM_EXIT_HYPERCALL_MBZ);  in kvm_emulate_hypercall()
9911 		vcpu->arch.complete_userspace_io = complete_hypercall_exit;  in kvm_emulate_hypercall()
9921 	kvm_rax_write(vcpu, ret);  in kvm_emulate_hypercall()
9923 	++vcpu->stat.hypercalls;  in kvm_emulate_hypercall()
9924 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_hypercall()
9930 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_fix_hypercall()  local
9932 	unsigned long rip = kvm_rip_read(vcpu);  in emulator_fix_hypercall()
9938 	if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_FIX_HYPERCALL_INSN)) {  in emulator_fix_hypercall()
9945 	static_call(kvm_x86_patch_hypercall)(vcpu, instruction);  in emulator_fix_hypercall()
9951 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)  in dm_request_for_irq_injection()  argument
9953 	return vcpu->run->request_interrupt_window &&  in dm_request_for_irq_injection()
9954 		likely(!pic_in_kernel(vcpu->kvm));  in dm_request_for_irq_injection()
9958 static void post_kvm_run_save(struct kvm_vcpu *vcpu)  in post_kvm_run_save()  argument
9960 	struct kvm_run *kvm_run = vcpu->run;  in post_kvm_run_save()
9962 	kvm_run->if_flag = static_call(kvm_x86_get_if_flag)(vcpu);  in post_kvm_run_save()
9963 	kvm_run->cr8 = kvm_get_cr8(vcpu);  in post_kvm_run_save()
9964 	kvm_run->apic_base = kvm_get_apic_base(vcpu);  in post_kvm_run_save()
9967 		pic_in_kernel(vcpu->kvm) ||  in post_kvm_run_save()
9968 		kvm_vcpu_ready_for_interrupt_injection(vcpu);  in post_kvm_run_save()
9970 	if (is_smm(vcpu))  in post_kvm_run_save()
9974 static void update_cr8_intercept(struct kvm_vcpu *vcpu)  in update_cr8_intercept()  argument
9981 	if (!lapic_in_kernel(vcpu))  in update_cr8_intercept()
9984 	if (vcpu->arch.apic->apicv_active)  in update_cr8_intercept()
9987 	if (!vcpu->arch.apic->vapic_addr)  in update_cr8_intercept()
9988 		max_irr = kvm_lapic_find_highest_irr(vcpu);  in update_cr8_intercept()
9995 	tpr = kvm_lapic_get_cr8(vcpu);  in update_cr8_intercept()
9997 	static_call(kvm_x86_update_cr8_intercept)(vcpu, tpr, max_irr);  in update_cr8_intercept()
10001 int kvm_check_nested_events(struct kvm_vcpu *vcpu)  in kvm_check_nested_events()  argument
10003 	if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {  in kvm_check_nested_events()
10004 		kvm_x86_ops.nested_ops->triple_fault(vcpu);  in kvm_check_nested_events()
10008 	return kvm_x86_ops.nested_ops->check_events(vcpu);  in kvm_check_nested_events()
10011 static void kvm_inject_exception(struct kvm_vcpu *vcpu)  in kvm_inject_exception()  argument
10014 	 * Suppress the error code if the vCPU is in Real Mode, as Real Mode  in kvm_inject_exception()
10020 	vcpu->arch.exception.has_error_code &= is_protmode(vcpu);  in kvm_inject_exception()
10022 	trace_kvm_inj_exception(vcpu->arch.exception.vector,  in kvm_inject_exception()
10023 				vcpu->arch.exception.has_error_code,  in kvm_inject_exception()
10024 				vcpu->arch.exception.error_code,  in kvm_inject_exception()
10025 				vcpu->arch.exception.injected);  in kvm_inject_exception()
10027 	static_call(kvm_x86_inject_exception)(vcpu);  in kvm_inject_exception()
10069 static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu,  in kvm_check_and_inject_events()  argument
10080 	if (is_guest_mode(vcpu))  in kvm_check_and_inject_events()
10081 		r = kvm_check_nested_events(vcpu);  in kvm_check_and_inject_events()
10108 	if (vcpu->arch.exception.injected)  in kvm_check_and_inject_events()
10109 		kvm_inject_exception(vcpu);  in kvm_check_and_inject_events()
10110 	else if (kvm_is_exception_pending(vcpu))  in kvm_check_and_inject_events()
10112 	else if (vcpu->arch.nmi_injected)  in kvm_check_and_inject_events()
10113 		static_call(kvm_x86_inject_nmi)(vcpu);  in kvm_check_and_inject_events()
10114 	else if (vcpu->arch.interrupt.injected)  in kvm_check_and_inject_events()
10115 		static_call(kvm_x86_inject_irq)(vcpu, true);  in kvm_check_and_inject_events()
10122 	WARN_ON_ONCE(vcpu->arch.exception.injected &&  in kvm_check_and_inject_events()
10123 		     vcpu->arch.exception.pending);  in kvm_check_and_inject_events()
10142 	WARN_ON_ONCE(vcpu->arch.exception_vmexit.injected ||  in kvm_check_and_inject_events()
10143 		     vcpu->arch.exception_vmexit.pending);  in kvm_check_and_inject_events()
10150 	can_inject = !kvm_event_needs_reinjection(vcpu);  in kvm_check_and_inject_events()
10152 	if (vcpu->arch.exception.pending) {  in kvm_check_and_inject_events()
10163 		if (exception_type(vcpu->arch.exception.vector) == EXCPT_FAULT)  in kvm_check_and_inject_events()
10164 			__kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |  in kvm_check_and_inject_events()
10167 		if (vcpu->arch.exception.vector == DB_VECTOR) {  in kvm_check_and_inject_events()
10168 			kvm_deliver_exception_payload(vcpu, &vcpu->arch.exception);  in kvm_check_and_inject_events()
10169 			if (vcpu->arch.dr7 & DR7_GD) {  in kvm_check_and_inject_events()
10170 				vcpu->arch.dr7 &= ~DR7_GD;  in kvm_check_and_inject_events()
10171 				kvm_update_dr7(vcpu);  in kvm_check_and_inject_events()
10175 		kvm_inject_exception(vcpu);  in kvm_check_and_inject_events()
10177 		vcpu->arch.exception.pending = false;  in kvm_check_and_inject_events()
10178 		vcpu->arch.exception.injected = true;  in kvm_check_and_inject_events()
10184 	if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ)  in kvm_check_and_inject_events()
10199 	if (vcpu->arch.smi_pending) {  in kvm_check_and_inject_events()
10200 		r = can_inject ? static_call(kvm_x86_smi_allowed)(vcpu, true) : -EBUSY;  in kvm_check_and_inject_events()
10204 			vcpu->arch.smi_pending = false;  in kvm_check_and_inject_events()
10205 			++vcpu->arch.smi_count;  in kvm_check_and_inject_events()
10206 			enter_smm(vcpu);  in kvm_check_and_inject_events()
10209 			static_call(kvm_x86_enable_smi_window)(vcpu);  in kvm_check_and_inject_events()
10213 	if (vcpu->arch.nmi_pending) {  in kvm_check_and_inject_events()
10214 		r = can_inject ? static_call(kvm_x86_nmi_allowed)(vcpu, true) : -EBUSY;  in kvm_check_and_inject_events()
10218 			--vcpu->arch.nmi_pending;  in kvm_check_and_inject_events()
10219 			vcpu->arch.nmi_injected = true;  in kvm_check_and_inject_events()
10220 			static_call(kvm_x86_inject_nmi)(vcpu);  in kvm_check_and_inject_events()
10222 			WARN_ON(static_call(kvm_x86_nmi_allowed)(vcpu, true) < 0);  in kvm_check_and_inject_events()
10224 		if (vcpu->arch.nmi_pending)  in kvm_check_and_inject_events()
10225 			static_call(kvm_x86_enable_nmi_window)(vcpu);  in kvm_check_and_inject_events()
10228 	if (kvm_cpu_has_injectable_intr(vcpu)) {  in kvm_check_and_inject_events()
10229 		r = can_inject ? static_call(kvm_x86_interrupt_allowed)(vcpu, true) : -EBUSY;  in kvm_check_and_inject_events()
10233 			int irq = kvm_cpu_get_interrupt(vcpu);  in kvm_check_and_inject_events()
10236 				kvm_queue_interrupt(vcpu, irq, false);  in kvm_check_and_inject_events()
10237 				static_call(kvm_x86_inject_irq)(vcpu, false);  in kvm_check_and_inject_events()
10238 				WARN_ON(static_call(kvm_x86_interrupt_allowed)(vcpu, true) < 0);  in kvm_check_and_inject_events()
10241 		if (kvm_cpu_has_injectable_intr(vcpu))  in kvm_check_and_inject_events()
10242 			static_call(kvm_x86_enable_irq_window)(vcpu);  in kvm_check_and_inject_events()
10245 	if (is_guest_mode(vcpu) &&  in kvm_check_and_inject_events()
10247 	    kvm_x86_ops.nested_ops->has_events(vcpu))  in kvm_check_and_inject_events()
10253 	 * to the VMCS/VMCB.  Queueing a new exception can put the vCPU into an  in kvm_check_and_inject_events()
10258 	 * vCPU into an infinite loop.  Triple fault can be queued when running  in kvm_check_and_inject_events()
10262 	WARN_ON_ONCE(vcpu->arch.exception.pending ||  in kvm_check_and_inject_events()
10263 		     vcpu->arch.exception_vmexit.pending);  in kvm_check_and_inject_events()
10274 static void process_nmi(struct kvm_vcpu *vcpu)  in process_nmi()  argument
10280 	 * incoming NMIs as quickly as bare metal, e.g. if the vCPU is  in process_nmi()
10288 	if (static_call(kvm_x86_get_nmi_mask)(vcpu) || vcpu->arch.nmi_injected)  in process_nmi()
10295 	 * tracked in vcpu->arch.nmi_pending.  in process_nmi()
10297 	if (static_call(kvm_x86_is_vnmi_pending)(vcpu))  in process_nmi()
10300 	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);  in process_nmi()
10301 	vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);  in process_nmi()
10303 	if (vcpu->arch.nmi_pending &&  in process_nmi()
10304 	    (static_call(kvm_x86_set_vnmi_pending)(vcpu)))  in process_nmi()
10305 		vcpu->arch.nmi_pending--;  in process_nmi()
10307 	if (vcpu->arch.nmi_pending)  in process_nmi()
10308 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in process_nmi()
10312 int kvm_get_nr_pending_nmis(struct kvm_vcpu *vcpu)  in kvm_get_nr_pending_nmis()  argument
10314 	return vcpu->arch.nmi_pending +  in kvm_get_nr_pending_nmis()
10315 	       static_call(kvm_x86_is_vnmi_pending)(vcpu);  in kvm_get_nr_pending_nmis()
10329 void __kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)  in __kvm_vcpu_update_apicv()  argument
10331 	struct kvm_lapic *apic = vcpu->arch.apic;  in __kvm_vcpu_update_apicv()
10334 	if (!lapic_in_kernel(vcpu))  in __kvm_vcpu_update_apicv()
10337 	down_read(&vcpu->kvm->arch.apicv_update_lock);  in __kvm_vcpu_update_apicv()
10341 	activate = kvm_vcpu_apicv_activated(vcpu) &&  in __kvm_vcpu_update_apicv()
10342 		   (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED);  in __kvm_vcpu_update_apicv()
10348 	kvm_apic_update_apicv(vcpu);  in __kvm_vcpu_update_apicv()
10349 	static_call(kvm_x86_refresh_apicv_exec_ctrl)(vcpu);  in __kvm_vcpu_update_apicv()
10358 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in __kvm_vcpu_update_apicv()
10362 	up_read(&vcpu->kvm->arch.apicv_update_lock);  in __kvm_vcpu_update_apicv()
10366 static void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)  in kvm_vcpu_update_apicv()  argument
10368 	if (!lapic_in_kernel(vcpu))  in kvm_vcpu_update_apicv()
10373 	 * deleted if any vCPU has xAPIC virtualization and x2APIC enabled, but  in kvm_vcpu_update_apicv()
10378 	 * the vCPU would incorrectly be able to access the vAPIC page via MMIO  in kvm_vcpu_update_apicv()
10382 	if (apic_x2apic_mode(vcpu->arch.apic) &&  in kvm_vcpu_update_apicv()
10384 		kvm_inhibit_apic_access_page(vcpu);  in kvm_vcpu_update_apicv()
10386 	__kvm_vcpu_update_apicv(vcpu);  in kvm_vcpu_update_apicv()
10442 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)  in vcpu_scan_ioapic()  argument
10444 	if (!kvm_apic_present(vcpu))  in vcpu_scan_ioapic()
10447 	bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);  in vcpu_scan_ioapic()
10449 	if (irqchip_split(vcpu->kvm))  in vcpu_scan_ioapic()
10450 		kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);  in vcpu_scan_ioapic()
10452 		static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);  in vcpu_scan_ioapic()
10453 		if (ioapic_in_kernel(vcpu->kvm))  in vcpu_scan_ioapic()
10454 			kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);  in vcpu_scan_ioapic()
10457 	if (is_guest_mode(vcpu))  in vcpu_scan_ioapic()
10458 		vcpu->arch.load_eoi_exitmap_pending = true;  in vcpu_scan_ioapic()
10460 		kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);  in vcpu_scan_ioapic()
10463 static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu)  in vcpu_load_eoi_exitmap()  argument
10467 	if (!kvm_apic_hw_enabled(vcpu->arch.apic))  in vcpu_load_eoi_exitmap()
10470 	if (to_hv_vcpu(vcpu)) {  in vcpu_load_eoi_exitmap()
10472 			  vcpu->arch.ioapic_handled_vectors,  in vcpu_load_eoi_exitmap()
10473 			  to_hv_synic(vcpu)->vec_bitmap, 256);  in vcpu_load_eoi_exitmap()
10474 		static_call_cond(kvm_x86_load_eoi_exitmap)(vcpu, eoi_exit_bitmap);  in vcpu_load_eoi_exitmap()
10479 		vcpu, (u64 *)vcpu->arch.ioapic_handled_vectors);  in vcpu_load_eoi_exitmap()
10487 static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)  in kvm_vcpu_reload_apic_access_page()  argument
10489 	if (!lapic_in_kernel(vcpu))  in kvm_vcpu_reload_apic_access_page()
10492 	static_call_cond(kvm_x86_set_apic_access_page_addr)(vcpu);  in kvm_vcpu_reload_apic_access_page()
10495 void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)  in __kvm_request_immediate_exit()  argument
10497 	smp_send_reschedule(vcpu->cpu);  in __kvm_request_immediate_exit()
10507 static int vcpu_enter_guest(struct kvm_vcpu *vcpu)  in vcpu_enter_guest()  argument
10511 		dm_request_for_irq_injection(vcpu) &&  in vcpu_enter_guest()
10512 		kvm_cpu_accept_dm_intr(vcpu);  in vcpu_enter_guest()
10517 	if (kvm_request_pending(vcpu)) {  in vcpu_enter_guest()
10518 		if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) {  in vcpu_enter_guest()
10523 		if (kvm_dirty_ring_check_request(vcpu)) {  in vcpu_enter_guest()
10528 		if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {  in vcpu_enter_guest()
10529 			if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) {  in vcpu_enter_guest()
10534 		if (kvm_check_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu))  in vcpu_enter_guest()
10535 			kvm_mmu_free_obsolete_roots(vcpu);  in vcpu_enter_guest()
10536 		if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))  in vcpu_enter_guest()
10537 			__kvm_migrate_timers(vcpu);  in vcpu_enter_guest()
10538 		if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))  in vcpu_enter_guest()
10539 			kvm_update_masterclock(vcpu->kvm);  in vcpu_enter_guest()
10540 		if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))  in vcpu_enter_guest()
10541 			kvm_gen_kvmclock_update(vcpu);  in vcpu_enter_guest()
10542 		if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {  in vcpu_enter_guest()
10543 			r = kvm_guest_time_update(vcpu);  in vcpu_enter_guest()
10547 		if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))  in vcpu_enter_guest()
10548 			kvm_mmu_sync_roots(vcpu);  in vcpu_enter_guest()
10549 		if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu))  in vcpu_enter_guest()
10550 			kvm_mmu_load_pgd(vcpu);  in vcpu_enter_guest()
10557 		if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))  in vcpu_enter_guest()
10558 			kvm_vcpu_flush_tlb_all(vcpu);  in vcpu_enter_guest()
10560 		kvm_service_local_tlb_flush_requests(vcpu);  in vcpu_enter_guest()
10564 		 * flushing fails.  Note, Hyper-V's flushing is per-vCPU, but  in vcpu_enter_guest()
10568 		if (kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu) &&  in vcpu_enter_guest()
10569 		    kvm_hv_vcpu_flush_tlb(vcpu))  in vcpu_enter_guest()
10570 			kvm_vcpu_flush_tlb_guest(vcpu);  in vcpu_enter_guest()
10572 		if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {  in vcpu_enter_guest()
10573 			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;  in vcpu_enter_guest()
10577 		if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {  in vcpu_enter_guest()
10578 			if (is_guest_mode(vcpu))  in vcpu_enter_guest()
10579 				kvm_x86_ops.nested_ops->triple_fault(vcpu);  in vcpu_enter_guest()
10581 			if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {  in vcpu_enter_guest()
10582 				vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;  in vcpu_enter_guest()
10583 				vcpu->mmio_needed = 0;  in vcpu_enter_guest()
10588 		if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {  in vcpu_enter_guest()
10590 			vcpu->arch.apf.halted = true;  in vcpu_enter_guest()
10594 		if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))  in vcpu_enter_guest()
10595 			record_steal_time(vcpu);  in vcpu_enter_guest()
10597 		if (kvm_check_request(KVM_REQ_SMI, vcpu))  in vcpu_enter_guest()
10598 			process_smi(vcpu);  in vcpu_enter_guest()
10600 		if (kvm_check_request(KVM_REQ_NMI, vcpu))  in vcpu_enter_guest()
10601 			process_nmi(vcpu);  in vcpu_enter_guest()
10602 		if (kvm_check_request(KVM_REQ_PMU, vcpu))  in vcpu_enter_guest()
10603 			kvm_pmu_handle_event(vcpu);  in vcpu_enter_guest()
10604 		if (kvm_check_request(KVM_REQ_PMI, vcpu))  in vcpu_enter_guest()
10605 			kvm_pmu_deliver_pmi(vcpu);  in vcpu_enter_guest()
10606 		if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {  in vcpu_enter_guest()
10607 			BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);  in vcpu_enter_guest()
10608 			if (test_bit(vcpu->arch.pending_ioapic_eoi,  in vcpu_enter_guest()
10609 				     vcpu->arch.ioapic_handled_vectors)) {  in vcpu_enter_guest()
10610 				vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;  in vcpu_enter_guest()
10611 				vcpu->run->eoi.vector =  in vcpu_enter_guest()
10612 						vcpu->arch.pending_ioapic_eoi;  in vcpu_enter_guest()
10617 		if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))  in vcpu_enter_guest()
10618 			vcpu_scan_ioapic(vcpu);  in vcpu_enter_guest()
10619 		if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu))  in vcpu_enter_guest()
10620 			vcpu_load_eoi_exitmap(vcpu);  in vcpu_enter_guest()
10621 		if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))  in vcpu_enter_guest()
10622 			kvm_vcpu_reload_apic_access_page(vcpu);  in vcpu_enter_guest()
10623 		if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) {  in vcpu_enter_guest()
10624 			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;  in vcpu_enter_guest()
10625 			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH;  in vcpu_enter_guest()
10626 			vcpu->run->system_event.ndata = 0;  in vcpu_enter_guest()
10630 		if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) {  in vcpu_enter_guest()
10631 			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;  in vcpu_enter_guest()
10632 			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET;  in vcpu_enter_guest()
10633 			vcpu->run->system_event.ndata = 0;  in vcpu_enter_guest()
10637 		if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {  in vcpu_enter_guest()
10638 			struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);  in vcpu_enter_guest()
10640 			vcpu->run->exit_reason = KVM_EXIT_HYPERV;  in vcpu_enter_guest()
10641 			vcpu->run->hyperv = hv_vcpu->exit;  in vcpu_enter_guest()
10651 		if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))  in vcpu_enter_guest()
10652 			kvm_hv_process_stimers(vcpu);  in vcpu_enter_guest()
10653 		if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu))  in vcpu_enter_guest()
10654 			kvm_vcpu_update_apicv(vcpu);  in vcpu_enter_guest()
10655 		if (kvm_check_request(KVM_REQ_APF_READY, vcpu))  in vcpu_enter_guest()
10656 			kvm_check_async_pf_completion(vcpu);  in vcpu_enter_guest()
10657 		if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu))  in vcpu_enter_guest()
10658 			static_call(kvm_x86_msr_filter_changed)(vcpu);  in vcpu_enter_guest()
10660 		if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu))  in vcpu_enter_guest()
10661 			static_call(kvm_x86_update_cpu_dirty_logging)(vcpu);  in vcpu_enter_guest()
10664 	if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win ||  in vcpu_enter_guest()
10665 	    kvm_xen_has_interrupt(vcpu)) {  in vcpu_enter_guest()
10666 		++vcpu->stat.req_event;  in vcpu_enter_guest()
10667 		r = kvm_apic_accept_events(vcpu);  in vcpu_enter_guest()
10672 		if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {  in vcpu_enter_guest()
10677 		r = kvm_check_and_inject_events(vcpu, &req_immediate_exit);  in vcpu_enter_guest()
10683 			static_call(kvm_x86_enable_irq_window)(vcpu);  in vcpu_enter_guest()
10685 		if (kvm_lapic_enabled(vcpu)) {  in vcpu_enter_guest()
10686 			update_cr8_intercept(vcpu);  in vcpu_enter_guest()
10687 			kvm_lapic_sync_to_vapic(vcpu);  in vcpu_enter_guest()
10691 	r = kvm_mmu_reload(vcpu);  in vcpu_enter_guest()
10698 	static_call(kvm_x86_prepare_switch_to_guest)(vcpu);  in vcpu_enter_guest()
10707 	/* Store vcpu->apicv_active before vcpu->mode.  */  in vcpu_enter_guest()
10708 	smp_store_release(&vcpu->mode, IN_GUEST_MODE);  in vcpu_enter_guest()
10710 	kvm_vcpu_srcu_read_unlock(vcpu);  in vcpu_enter_guest()
10721 	 * tables done while the VCPU is running.  Please see the comment  in vcpu_enter_guest()
10729 	 * target vCPU wasn't running).  Do this regardless of the vCPU's APICv  in vcpu_enter_guest()
10733 	if (kvm_lapic_enabled(vcpu))  in vcpu_enter_guest()
10734 		static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);  in vcpu_enter_guest()
10736 	if (kvm_vcpu_exit_request(vcpu)) {  in vcpu_enter_guest()
10737 		vcpu->mode = OUTSIDE_GUEST_MODE;  in vcpu_enter_guest()
10741 		kvm_vcpu_srcu_read_lock(vcpu);  in vcpu_enter_guest()
10747 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in vcpu_enter_guest()
10748 		static_call(kvm_x86_request_immediate_exit)(vcpu);  in vcpu_enter_guest()
10755 	if (vcpu->arch.guest_fpu.xfd_err)  in vcpu_enter_guest()
10756 		wrmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);  in vcpu_enter_guest()
10758 	if (unlikely(vcpu->arch.switch_db_regs)) {  in vcpu_enter_guest()
10760 		set_debugreg(vcpu->arch.eff_db[0], 0);  in vcpu_enter_guest()
10761 		set_debugreg(vcpu->arch.eff_db[1], 1);  in vcpu_enter_guest()
10762 		set_debugreg(vcpu->arch.eff_db[2], 2);  in vcpu_enter_guest()
10763 		set_debugreg(vcpu->arch.eff_db[3], 3);  in vcpu_enter_guest()
10772 		 * Assert that vCPU vs. VM APICv state is consistent.  An APICv  in vcpu_enter_guest()
10777 		WARN_ON_ONCE((kvm_vcpu_apicv_activated(vcpu) != kvm_vcpu_apicv_active(vcpu)) &&  in vcpu_enter_guest()
10778 			     (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED));  in vcpu_enter_guest()
10780 		exit_fastpath = static_call(kvm_x86_vcpu_run)(vcpu);  in vcpu_enter_guest()
10784 		if (kvm_lapic_enabled(vcpu))  in vcpu_enter_guest()
10785 			static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);  in vcpu_enter_guest()
10787 		if (unlikely(kvm_vcpu_exit_request(vcpu))) {  in vcpu_enter_guest()
10793 		++vcpu->stat.exits;  in vcpu_enter_guest()
10802 	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {  in vcpu_enter_guest()
10803 		WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);  in vcpu_enter_guest()
10804 		static_call(kvm_x86_sync_dirty_debug_regs)(vcpu);  in vcpu_enter_guest()
10805 		kvm_update_dr0123(vcpu);  in vcpu_enter_guest()
10806 		kvm_update_dr7(vcpu);  in vcpu_enter_guest()
10819 	vcpu->arch.last_vmentry_cpu = vcpu->cpu;  in vcpu_enter_guest()
10820 	vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());  in vcpu_enter_guest()
10822 	vcpu->mode = OUTSIDE_GUEST_MODE;  in vcpu_enter_guest()
10830 	if (vcpu->arch.xfd_no_write_intercept)  in vcpu_enter_guest()
10833 	static_call(kvm_x86_handle_exit_irqoff)(vcpu);  in vcpu_enter_guest()
10835 	if (vcpu->arch.guest_fpu.xfd_err)  in vcpu_enter_guest()
10845 	kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);  in vcpu_enter_guest()
10847 	++vcpu->stat.exits;  in vcpu_enter_guest()
10849 	kvm_after_interrupt(vcpu);  in vcpu_enter_guest()
10863 	kvm_vcpu_srcu_read_lock(vcpu);  in vcpu_enter_guest()
10869 		unsigned long rip = kvm_rip_read(vcpu);  in vcpu_enter_guest()
10873 	if (unlikely(vcpu->arch.tsc_always_catchup))  in vcpu_enter_guest()
10874 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in vcpu_enter_guest()
10876 	if (vcpu->arch.apic_attention)  in vcpu_enter_guest()
10877 		kvm_lapic_sync_from_vapic(vcpu);  in vcpu_enter_guest()
10879 	r = static_call(kvm_x86_handle_exit)(vcpu, exit_fastpath);  in vcpu_enter_guest()
10884 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in vcpu_enter_guest()
10885 	static_call(kvm_x86_cancel_injection)(vcpu);  in vcpu_enter_guest()
10886 	if (unlikely(vcpu->arch.apic_attention))  in vcpu_enter_guest()
10887 		kvm_lapic_sync_from_vapic(vcpu);  in vcpu_enter_guest()
10893 static inline int vcpu_block(struct kvm_vcpu *vcpu)  in vcpu_block()  argument
10897 	if (!kvm_arch_vcpu_runnable(vcpu)) {  in vcpu_block()
10900 		 * the guest's timer may be a break event for the vCPU, and the  in vcpu_block()
10905 		hv_timer = kvm_lapic_hv_timer_in_use(vcpu);  in vcpu_block()
10907 			kvm_lapic_switch_to_sw_timer(vcpu);  in vcpu_block()
10909 		kvm_vcpu_srcu_read_unlock(vcpu);  in vcpu_block()
10910 		if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)  in vcpu_block()
10911 			kvm_vcpu_halt(vcpu);  in vcpu_block()
10913 			kvm_vcpu_block(vcpu);  in vcpu_block()
10914 		kvm_vcpu_srcu_read_lock(vcpu);  in vcpu_block()
10917 			kvm_lapic_switch_to_hv_timer(vcpu);  in vcpu_block()
10920 		 * If the vCPU is not runnable, a signal or another host event  in vcpu_block()
10922 		 * vCPU's activity state.  in vcpu_block()
10924 		if (!kvm_arch_vcpu_runnable(vcpu))  in vcpu_block()
10934 	if (is_guest_mode(vcpu)) {  in vcpu_block()
10935 		if (kvm_check_nested_events(vcpu) < 0)  in vcpu_block()
10939 	if (kvm_apic_accept_events(vcpu) < 0)  in vcpu_block()
10941 	switch(vcpu->arch.mp_state) {  in vcpu_block()
10944 		vcpu->arch.pv.pv_unhalted = false;  in vcpu_block()
10945 		vcpu->arch.mp_state =  in vcpu_block()
10949 		vcpu->arch.apf.halted = false;  in vcpu_block()
10960 static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu)  in kvm_vcpu_running()  argument
10962 	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&  in kvm_vcpu_running()
10963 		!vcpu->arch.apf.halted);  in kvm_vcpu_running()
10967 static int vcpu_run(struct kvm_vcpu *vcpu)  in vcpu_run()  argument
10971 	vcpu->arch.l1tf_flush_l1d = true;  in vcpu_run()
10975 		 * If another guest vCPU requests a PV TLB flush in the middle  in vcpu_run()
10980 		vcpu->arch.at_instruction_boundary = false;  in vcpu_run()
10981 		if (kvm_vcpu_running(vcpu)) {  in vcpu_run()
10982 			r = vcpu_enter_guest(vcpu);  in vcpu_run()
10984 			r = vcpu_block(vcpu);  in vcpu_run()
10990 		kvm_clear_request(KVM_REQ_UNBLOCK, vcpu);  in vcpu_run()
10991 		if (kvm_xen_has_pending_events(vcpu))  in vcpu_run()
10992 			kvm_xen_inject_pending_events(vcpu);  in vcpu_run()
10994 		if (kvm_cpu_has_pending_timer(vcpu))  in vcpu_run()
10995 			kvm_inject_pending_timer_irqs(vcpu);  in vcpu_run()
10997 		if (dm_request_for_irq_injection(vcpu) &&  in vcpu_run()
10998 			kvm_vcpu_ready_for_interrupt_injection(vcpu)) {  in vcpu_run()
11000 			vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;  in vcpu_run()
11001 			++vcpu->stat.request_irq_exits;  in vcpu_run()
11006 			kvm_vcpu_srcu_read_unlock(vcpu);  in vcpu_run()
11007 			r = xfer_to_guest_mode_handle_work(vcpu);  in vcpu_run()
11008 			kvm_vcpu_srcu_read_lock(vcpu);  in vcpu_run()
11017 static inline int complete_emulated_io(struct kvm_vcpu *vcpu)  in complete_emulated_io()  argument
11019 	return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE);  in complete_emulated_io()
11022 static int complete_emulated_pio(struct kvm_vcpu *vcpu)  in complete_emulated_pio()  argument
11024 	BUG_ON(!vcpu->arch.pio.count);  in complete_emulated_pio()
11026 	return complete_emulated_io(vcpu);  in complete_emulated_pio()
11047 static int complete_emulated_mmio(struct kvm_vcpu *vcpu)  in complete_emulated_mmio()  argument
11049 	struct kvm_run *run = vcpu->run;  in complete_emulated_mmio()
11053 	BUG_ON(!vcpu->mmio_needed);  in complete_emulated_mmio()
11056 	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];  in complete_emulated_mmio()
11058 	if (!vcpu->mmio_is_write)  in complete_emulated_mmio()
11064 		vcpu->mmio_cur_fragment++;  in complete_emulated_mmio()
11072 	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {  in complete_emulated_mmio()
11073 		vcpu->mmio_needed = 0;  in complete_emulated_mmio()
11076 		if (vcpu->mmio_is_write)  in complete_emulated_mmio()
11078 		vcpu->mmio_read_completed = 1;  in complete_emulated_mmio()
11079 		return complete_emulated_io(vcpu);  in complete_emulated_mmio()
11084 	if (vcpu->mmio_is_write)  in complete_emulated_mmio()
11087 	run->mmio.is_write = vcpu->mmio_is_write;  in complete_emulated_mmio()
11088 	vcpu->arch.complete_userspace_io = complete_emulated_mmio;  in complete_emulated_mmio()
11093 static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)  in kvm_load_guest_fpu()  argument
11096 	fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, true);  in kvm_load_guest_fpu()
11101 static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)  in kvm_put_guest_fpu()  argument
11103 	fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, false);  in kvm_put_guest_fpu()
11104 	++vcpu->stat.fpu_reload;  in kvm_put_guest_fpu()
11108 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_ioctl_run()  argument
11110 	struct kvm_queued_exception *ex = &vcpu->arch.exception;  in kvm_arch_vcpu_ioctl_run()
11111 	struct kvm_run *kvm_run = vcpu->run;  in kvm_arch_vcpu_ioctl_run()
11114 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_run()
11115 	kvm_sigset_activate(vcpu);  in kvm_arch_vcpu_ioctl_run()
11117 	kvm_load_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_run()
11119 	kvm_vcpu_srcu_read_lock(vcpu);  in kvm_arch_vcpu_ioctl_run()
11120 	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {  in kvm_arch_vcpu_ioctl_run()
11129 		 * APIC timer to be active is if userspace stuffed vCPU state,  in kvm_arch_vcpu_ioctl_run()
11130 		 * i.e. put the vCPU into a nonsensical state.  Only an INIT  in kvm_arch_vcpu_ioctl_run()
11131 		 * will transition the vCPU out of UNINITIALIZED (without more  in kvm_arch_vcpu_ioctl_run()
11136 		kvm_vcpu_srcu_read_unlock(vcpu);  in kvm_arch_vcpu_ioctl_run()
11137 		kvm_vcpu_block(vcpu);  in kvm_arch_vcpu_ioctl_run()
11138 		kvm_vcpu_srcu_read_lock(vcpu);  in kvm_arch_vcpu_ioctl_run()
11140 		if (kvm_apic_accept_events(vcpu) < 0) {  in kvm_arch_vcpu_ioctl_run()
11148 			++vcpu->stat.signal_exits;  in kvm_arch_vcpu_ioctl_run()
11160 		r = sync_regs(vcpu);  in kvm_arch_vcpu_ioctl_run()
11166 	if (!lapic_in_kernel(vcpu)) {  in kvm_arch_vcpu_ioctl_run()
11167 		if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {  in kvm_arch_vcpu_ioctl_run()
11177 	if (vcpu->arch.exception_from_userspace && is_guest_mode(vcpu) &&  in kvm_arch_vcpu_ioctl_run()
11178 	    kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, ex->vector,  in kvm_arch_vcpu_ioctl_run()
11180 		kvm_queue_exception_vmexit(vcpu, ex->vector,  in kvm_arch_vcpu_ioctl_run()
11186 	vcpu->arch.exception_from_userspace = false;  in kvm_arch_vcpu_ioctl_run()
11188 	if (unlikely(vcpu->arch.complete_userspace_io)) {  in kvm_arch_vcpu_ioctl_run()
11189 		int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;  in kvm_arch_vcpu_ioctl_run()
11190 		vcpu->arch.complete_userspace_io = NULL;  in kvm_arch_vcpu_ioctl_run()
11191 		r = cui(vcpu);  in kvm_arch_vcpu_ioctl_run()
11195 		WARN_ON_ONCE(vcpu->arch.pio.count);  in kvm_arch_vcpu_ioctl_run()
11196 		WARN_ON_ONCE(vcpu->mmio_needed);  in kvm_arch_vcpu_ioctl_run()
11204 	r = static_call(kvm_x86_vcpu_pre_run)(vcpu);  in kvm_arch_vcpu_ioctl_run()
11208 	r = vcpu_run(vcpu);  in kvm_arch_vcpu_ioctl_run()
11211 	kvm_put_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_run()
11213 		store_regs(vcpu);  in kvm_arch_vcpu_ioctl_run()
11214 	post_kvm_run_save(vcpu);  in kvm_arch_vcpu_ioctl_run()
11215 	kvm_vcpu_srcu_read_unlock(vcpu);  in kvm_arch_vcpu_ioctl_run()
11217 	kvm_sigset_deactivate(vcpu);  in kvm_arch_vcpu_ioctl_run()
11218 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_run()
11222 static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in __get_regs()  argument
11224 	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {  in __get_regs()
11228 		 * back from emulation context to vcpu. Userspace shouldn't do  in __get_regs()
11232 		emulator_writeback_register_cache(vcpu->arch.emulate_ctxt);  in __get_regs()
11233 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;  in __get_regs()
11235 	regs->rax = kvm_rax_read(vcpu);  in __get_regs()
11236 	regs->rbx = kvm_rbx_read(vcpu);  in __get_regs()
11237 	regs->rcx = kvm_rcx_read(vcpu);  in __get_regs()
11238 	regs->rdx = kvm_rdx_read(vcpu);  in __get_regs()
11239 	regs->rsi = kvm_rsi_read(vcpu);  in __get_regs()
11240 	regs->rdi = kvm_rdi_read(vcpu);  in __get_regs()
11241 	regs->rsp = kvm_rsp_read(vcpu);  in __get_regs()
11242 	regs->rbp = kvm_rbp_read(vcpu);  in __get_regs()
11244 	regs->r8 = kvm_r8_read(vcpu);  in __get_regs()
11245 	regs->r9 = kvm_r9_read(vcpu);  in __get_regs()
11246 	regs->r10 = kvm_r10_read(vcpu);  in __get_regs()
11247 	regs->r11 = kvm_r11_read(vcpu);  in __get_regs()
11248 	regs->r12 = kvm_r12_read(vcpu);  in __get_regs()
11249 	regs->r13 = kvm_r13_read(vcpu);  in __get_regs()
11250 	regs->r14 = kvm_r14_read(vcpu);  in __get_regs()
11251 	regs->r15 = kvm_r15_read(vcpu);  in __get_regs()
11254 	regs->rip = kvm_rip_read(vcpu);  in __get_regs()
11255 	regs->rflags = kvm_get_rflags(vcpu);  in __get_regs()
11258 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in kvm_arch_vcpu_ioctl_get_regs()  argument
11260 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_regs()
11261 	__get_regs(vcpu, regs);  in kvm_arch_vcpu_ioctl_get_regs()
11262 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_regs()
11266 static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in __set_regs()  argument
11268 	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;  in __set_regs()
11269 	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;  in __set_regs()
11271 	kvm_rax_write(vcpu, regs->rax);  in __set_regs()
11272 	kvm_rbx_write(vcpu, regs->rbx);  in __set_regs()
11273 	kvm_rcx_write(vcpu, regs->rcx);  in __set_regs()
11274 	kvm_rdx_write(vcpu, regs->rdx);  in __set_regs()
11275 	kvm_rsi_write(vcpu, regs->rsi);  in __set_regs()
11276 	kvm_rdi_write(vcpu, regs->rdi);  in __set_regs()
11277 	kvm_rsp_write(vcpu, regs->rsp);  in __set_regs()
11278 	kvm_rbp_write(vcpu, regs->rbp);  in __set_regs()
11280 	kvm_r8_write(vcpu, regs->r8);  in __set_regs()
11281 	kvm_r9_write(vcpu, regs->r9);  in __set_regs()
11282 	kvm_r10_write(vcpu, regs->r10);  in __set_regs()
11283 	kvm_r11_write(vcpu, regs->r11);  in __set_regs()
11284 	kvm_r12_write(vcpu, regs->r12);  in __set_regs()
11285 	kvm_r13_write(vcpu, regs->r13);  in __set_regs()
11286 	kvm_r14_write(vcpu, regs->r14);  in __set_regs()
11287 	kvm_r15_write(vcpu, regs->r15);  in __set_regs()
11290 	kvm_rip_write(vcpu, regs->rip);  in __set_regs()
11291 	kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);  in __set_regs()
11293 	vcpu->arch.exception.pending = false;  in __set_regs()
11294 	vcpu->arch.exception_vmexit.pending = false;  in __set_regs()
11296 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in __set_regs()
11299 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in kvm_arch_vcpu_ioctl_set_regs()  argument
11301 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_regs()
11302 	__set_regs(vcpu, regs);  in kvm_arch_vcpu_ioctl_set_regs()
11303 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_regs()
11307 static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in __get_sregs_common()  argument
11311 	if (vcpu->arch.guest_state_protected)  in __get_sregs_common()
11314 	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);  in __get_sregs_common()
11315 	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);  in __get_sregs_common()
11316 	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);  in __get_sregs_common()
11317 	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);  in __get_sregs_common()
11318 	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);  in __get_sregs_common()
11319 	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);  in __get_sregs_common()
11321 	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);  in __get_sregs_common()
11322 	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);  in __get_sregs_common()
11324 	static_call(kvm_x86_get_idt)(vcpu, &dt);  in __get_sregs_common()
11327 	static_call(kvm_x86_get_gdt)(vcpu, &dt);  in __get_sregs_common()
11331 	sregs->cr2 = vcpu->arch.cr2;  in __get_sregs_common()
11332 	sregs->cr3 = kvm_read_cr3(vcpu);  in __get_sregs_common()
11335 	sregs->cr0 = kvm_read_cr0(vcpu);  in __get_sregs_common()
11336 	sregs->cr4 = kvm_read_cr4(vcpu);  in __get_sregs_common()
11337 	sregs->cr8 = kvm_get_cr8(vcpu);  in __get_sregs_common()
11338 	sregs->efer = vcpu->arch.efer;  in __get_sregs_common()
11339 	sregs->apic_base = kvm_get_apic_base(vcpu);  in __get_sregs_common()
11342 static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in __get_sregs()  argument
11344 	__get_sregs_common(vcpu, sregs);  in __get_sregs()
11346 	if (vcpu->arch.guest_state_protected)  in __get_sregs()
11349 	if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)  in __get_sregs()
11350 		set_bit(vcpu->arch.interrupt.nr,  in __get_sregs()
11354 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)  in __get_sregs2()  argument
11358 	__get_sregs_common(vcpu, (struct kvm_sregs *)sregs2);  in __get_sregs2()
11360 	if (vcpu->arch.guest_state_protected)  in __get_sregs2()
11363 	if (is_pae_paging(vcpu)) {  in __get_sregs2()
11365 			sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i);  in __get_sregs2()
11370 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_get_sregs()  argument
11373 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_sregs()
11374 	__get_sregs(vcpu, sregs);  in kvm_arch_vcpu_ioctl_get_sregs()
11375 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_sregs()
11379 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_get_mpstate()  argument
11384 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11386 		kvm_load_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11388 	r = kvm_apic_accept_events(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11393 	if ((vcpu->arch.mp_state == KVM_MP_STATE_HALTED ||  in kvm_arch_vcpu_ioctl_get_mpstate()
11394 	     vcpu->arch.mp_state == KVM_MP_STATE_AP_RESET_HOLD) &&  in kvm_arch_vcpu_ioctl_get_mpstate()
11395 	    vcpu->arch.pv.pv_unhalted)  in kvm_arch_vcpu_ioctl_get_mpstate()
11398 		mp_state->mp_state = vcpu->arch.mp_state;  in kvm_arch_vcpu_ioctl_get_mpstate()
11402 		kvm_put_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11403 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11407 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_set_mpstate()  argument
11412 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_mpstate()
11420 		if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_ioctl_set_mpstate()
11437 	if ((!kvm_apic_init_sipi_allowed(vcpu) || vcpu->arch.smi_pending) &&  in kvm_arch_vcpu_ioctl_set_mpstate()
11443 		vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;  in kvm_arch_vcpu_ioctl_set_mpstate()
11444 		set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);  in kvm_arch_vcpu_ioctl_set_mpstate()
11446 		vcpu->arch.mp_state = mp_state->mp_state;  in kvm_arch_vcpu_ioctl_set_mpstate()
11447 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_arch_vcpu_ioctl_set_mpstate()
11451 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_mpstate()
11455 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,  in kvm_task_switch()  argument
11458 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in kvm_task_switch()
11461 	init_emulate_ctxt(vcpu);  in kvm_task_switch()
11466 		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;  in kvm_task_switch()
11467 		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;  in kvm_task_switch()
11468 		vcpu->run->internal.ndata = 0;  in kvm_task_switch()
11472 	kvm_rip_write(vcpu, ctxt->eip);  in kvm_task_switch()
11473 	kvm_set_rflags(vcpu, ctxt->eflags);  in kvm_task_switch()
11478 static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in kvm_is_valid_sregs()  argument
11488 		if (kvm_vcpu_is_illegal_gpa(vcpu, sregs->cr3))  in kvm_is_valid_sregs()
11499 	return kvm_is_valid_cr4(vcpu, sregs->cr4) &&  in kvm_is_valid_sregs()
11500 	       kvm_is_valid_cr0(vcpu, sregs->cr0);  in kvm_is_valid_sregs()
11503 static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs,  in __set_sregs_common()  argument
11510 	if (!kvm_is_valid_sregs(vcpu, sregs))  in __set_sregs_common()
11515 	if (kvm_set_apic_base(vcpu, &apic_base_msr))  in __set_sregs_common()
11518 	if (vcpu->arch.guest_state_protected)  in __set_sregs_common()
11523 	static_call(kvm_x86_set_idt)(vcpu, &dt);  in __set_sregs_common()
11526 	static_call(kvm_x86_set_gdt)(vcpu, &dt);  in __set_sregs_common()
11528 	vcpu->arch.cr2 = sregs->cr2;  in __set_sregs_common()
11529 	*mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;  in __set_sregs_common()
11530 	vcpu->arch.cr3 = sregs->cr3;  in __set_sregs_common()
11531 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);  in __set_sregs_common()
11532 	static_call_cond(kvm_x86_post_set_cr3)(vcpu, sregs->cr3);  in __set_sregs_common()
11534 	kvm_set_cr8(vcpu, sregs->cr8);  in __set_sregs_common()
11536 	*mmu_reset_needed |= vcpu->arch.efer != sregs->efer;  in __set_sregs_common()
11537 	static_call(kvm_x86_set_efer)(vcpu, sregs->efer);  in __set_sregs_common()
11539 	*mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;  in __set_sregs_common()
11540 	static_call(kvm_x86_set_cr0)(vcpu, sregs->cr0);  in __set_sregs_common()
11541 	vcpu->arch.cr0 = sregs->cr0;  in __set_sregs_common()
11543 	*mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;  in __set_sregs_common()
11544 	static_call(kvm_x86_set_cr4)(vcpu, sregs->cr4);  in __set_sregs_common()
11547 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in __set_sregs_common()
11548 		if (is_pae_paging(vcpu)) {  in __set_sregs_common()
11549 			load_pdptrs(vcpu, kvm_read_cr3(vcpu));  in __set_sregs_common()
11552 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in __set_sregs_common()
11555 	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);  in __set_sregs_common()
11556 	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);  in __set_sregs_common()
11557 	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);  in __set_sregs_common()
11558 	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);  in __set_sregs_common()
11559 	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);  in __set_sregs_common()
11560 	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);  in __set_sregs_common()
11562 	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);  in __set_sregs_common()
11563 	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);  in __set_sregs_common()
11565 	update_cr8_intercept(vcpu);  in __set_sregs_common()
11567 	/* Older userspace won't unhalt the vcpu on reset. */  in __set_sregs_common()
11568 	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&  in __set_sregs_common()
11570 	    !is_protmode(vcpu))  in __set_sregs_common()
11571 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;  in __set_sregs_common()
11576 static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in __set_sregs()  argument
11580 	int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true);  in __set_sregs()
11586 		kvm_mmu_reset_context(vcpu);  in __set_sregs()
11593 		kvm_queue_interrupt(vcpu, pending_vec, false);  in __set_sregs()
11595 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in __set_sregs()
11600 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)  in __set_sregs2()  argument
11611 	if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected))  in __set_sregs2()
11614 	ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2,  in __set_sregs2()
11621 			kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]);  in __set_sregs2()
11623 		kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);  in __set_sregs2()
11625 		vcpu->arch.pdptrs_from_userspace = true;  in __set_sregs2()
11628 		kvm_mmu_reset_context(vcpu);  in __set_sregs2()
11632 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_set_sregs()  argument
11637 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_sregs()
11638 	ret = __set_sregs(vcpu, sregs);  in kvm_arch_vcpu_ioctl_set_sregs()
11639 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_sregs()
11646 	struct kvm_vcpu *vcpu;  in kvm_arch_vcpu_guestdbg_update_apicv_inhibit()  local
11654 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_arch_vcpu_guestdbg_update_apicv_inhibit()
11655 		if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) {  in kvm_arch_vcpu_guestdbg_update_apicv_inhibit()
11664 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_set_guest_debug()  argument
11670 	if (vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_ioctl_set_guest_debug()
11673 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11677 		if (kvm_is_exception_pending(vcpu))  in kvm_arch_vcpu_ioctl_set_guest_debug()
11680 			kvm_queue_exception(vcpu, DB_VECTOR);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11682 			kvm_queue_exception(vcpu, BP_VECTOR);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11689 	rflags = kvm_get_rflags(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11691 	vcpu->guest_debug = dbg->control;  in kvm_arch_vcpu_ioctl_set_guest_debug()
11692 	if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))  in kvm_arch_vcpu_ioctl_set_guest_debug()
11693 		vcpu->guest_debug = 0;  in kvm_arch_vcpu_ioctl_set_guest_debug()
11695 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {  in kvm_arch_vcpu_ioctl_set_guest_debug()
11697 			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];  in kvm_arch_vcpu_ioctl_set_guest_debug()
11698 		vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];  in kvm_arch_vcpu_ioctl_set_guest_debug()
11701 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];  in kvm_arch_vcpu_ioctl_set_guest_debug()
11703 	kvm_update_dr7(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11705 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)  in kvm_arch_vcpu_ioctl_set_guest_debug()
11706 		vcpu->arch.singlestep_rip = kvm_get_linear_rip(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11712 	kvm_set_rflags(vcpu, rflags);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11714 	static_call(kvm_x86_update_exception_bitmap)(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11716 	kvm_arch_vcpu_guestdbg_update_apicv_inhibit(vcpu->kvm);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11721 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11728 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_translate()  argument
11735 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_translate()
11737 	idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl_translate()
11738 	gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);  in kvm_arch_vcpu_ioctl_translate()
11739 	srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl_translate()
11745 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_translate()
11749 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)  in kvm_arch_vcpu_ioctl_get_fpu()  argument
11753 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_arch_vcpu_ioctl_get_fpu()
11756 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_fpu()
11758 	fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;  in kvm_arch_vcpu_ioctl_get_fpu()
11768 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_fpu()
11772 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)  in kvm_arch_vcpu_ioctl_set_fpu()  argument
11776 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_arch_vcpu_ioctl_set_fpu()
11779 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_fpu()
11781 	fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;  in kvm_arch_vcpu_ioctl_set_fpu()
11792 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_fpu()
11796 static void store_regs(struct kvm_vcpu *vcpu)  in store_regs()  argument
11800 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)  in store_regs()
11801 		__get_regs(vcpu, &vcpu->run->s.regs.regs);  in store_regs()
11803 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)  in store_regs()
11804 		__get_sregs(vcpu, &vcpu->run->s.regs.sregs);  in store_regs()
11806 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)  in store_regs()
11808 				vcpu, &vcpu->run->s.regs.events);  in store_regs()
11811 static int sync_regs(struct kvm_vcpu *vcpu)  in sync_regs()  argument
11813 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {  in sync_regs()
11814 		__set_regs(vcpu, &vcpu->run->s.regs.regs);  in sync_regs()
11815 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;  in sync_regs()
11818 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {  in sync_regs()
11819 		struct kvm_sregs sregs = vcpu->run->s.regs.sregs;  in sync_regs()
11821 		if (__set_sregs(vcpu, &sregs))  in sync_regs()
11824 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;  in sync_regs()
11827 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {  in sync_regs()
11828 		struct kvm_vcpu_events events = vcpu->run->s.regs.events;  in sync_regs()
11830 		if (kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events))  in sync_regs()
11833 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;  in sync_regs()
11854 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_create()  argument
11859 	vcpu->arch.last_vmentry_cpu = -1;  in kvm_arch_vcpu_create()
11860 	vcpu->arch.regs_avail = ~0;  in kvm_arch_vcpu_create()
11861 	vcpu->arch.regs_dirty = ~0;  in kvm_arch_vcpu_create()
11863 	kvm_gpc_init(&vcpu->arch.pv_time, vcpu->kvm, vcpu, KVM_HOST_USES_PFN);  in kvm_arch_vcpu_create()
11865 	if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))  in kvm_arch_vcpu_create()
11866 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;  in kvm_arch_vcpu_create()
11868 		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;  in kvm_arch_vcpu_create()
11870 	r = kvm_mmu_create(vcpu);  in kvm_arch_vcpu_create()
11874 	if (irqchip_in_kernel(vcpu->kvm)) {  in kvm_arch_vcpu_create()
11875 		r = kvm_create_lapic(vcpu, lapic_timer_advance_ns);  in kvm_arch_vcpu_create()
11880 		 * Defer evaluating inhibits until the vCPU is first run, as  in kvm_arch_vcpu_create()
11881 		 * this vCPU will not get notified of any changes until this  in kvm_arch_vcpu_create()
11882 		 * vCPU is visible to other vCPUs (marked online and added to  in kvm_arch_vcpu_create()
11885 		 * Ignore the current per-VM APICv state so that vCPU creation  in kvm_arch_vcpu_create()
11887 		 * will ensure the vCPU gets the correct state before VM-Entry.  in kvm_arch_vcpu_create()
11890 			vcpu->arch.apic->apicv_active = true;  in kvm_arch_vcpu_create()
11891 			kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);  in kvm_arch_vcpu_create()
11901 	vcpu->arch.pio_data = page_address(page);  in kvm_arch_vcpu_create()
11903 	vcpu->arch.mce_banks = kcalloc(KVM_MAX_MCE_BANKS * 4, sizeof(u64),  in kvm_arch_vcpu_create()
11905 	vcpu->arch.mci_ctl2_banks = kcalloc(KVM_MAX_MCE_BANKS, sizeof(u64),  in kvm_arch_vcpu_create()
11907 	if (!vcpu->arch.mce_banks || !vcpu->arch.mci_ctl2_banks)  in kvm_arch_vcpu_create()
11909 	vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;  in kvm_arch_vcpu_create()
11911 	if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,  in kvm_arch_vcpu_create()
11915 	if (!alloc_emulate_ctxt(vcpu))  in kvm_arch_vcpu_create()
11918 	if (!fpu_alloc_guest_fpstate(&vcpu->arch.guest_fpu)) {  in kvm_arch_vcpu_create()
11919 		pr_err("failed to allocate vcpu's fpu\n");  in kvm_arch_vcpu_create()
11923 	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);  in kvm_arch_vcpu_create()
11924 	vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu);  in kvm_arch_vcpu_create()
11926 	vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;  in kvm_arch_vcpu_create()
11928 	kvm_async_pf_hash_reset(vcpu);  in kvm_arch_vcpu_create()
11930 	vcpu->arch.perf_capabilities = kvm_caps.supported_perf_cap;  in kvm_arch_vcpu_create()
11931 	kvm_pmu_init(vcpu);  in kvm_arch_vcpu_create()
11933 	vcpu->arch.pending_external_vector = -1;  in kvm_arch_vcpu_create()
11934 	vcpu->arch.preempted_in_kernel = false;  in kvm_arch_vcpu_create()
11937 	vcpu->arch.hv_root_tdp = INVALID_PAGE;  in kvm_arch_vcpu_create()
11940 	r = static_call(kvm_x86_vcpu_create)(vcpu);  in kvm_arch_vcpu_create()
11944 	vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();  in kvm_arch_vcpu_create()
11945 	vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;  in kvm_arch_vcpu_create()
11946 	kvm_xen_init_vcpu(vcpu);  in kvm_arch_vcpu_create()
11947 	kvm_vcpu_mtrr_init(vcpu);  in kvm_arch_vcpu_create()
11948 	vcpu_load(vcpu);  in kvm_arch_vcpu_create()
11949 	kvm_set_tsc_khz(vcpu, vcpu->kvm->arch.default_tsc_khz);  in kvm_arch_vcpu_create()
11950 	kvm_vcpu_reset(vcpu, false);  in kvm_arch_vcpu_create()
11951 	kvm_init_mmu(vcpu);  in kvm_arch_vcpu_create()
11952 	vcpu_put(vcpu);  in kvm_arch_vcpu_create()
11956 	fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);  in kvm_arch_vcpu_create()
11958 	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);  in kvm_arch_vcpu_create()
11960 	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);  in kvm_arch_vcpu_create()
11962 	kfree(vcpu->arch.mce_banks);  in kvm_arch_vcpu_create()
11963 	kfree(vcpu->arch.mci_ctl2_banks);  in kvm_arch_vcpu_create()
11964 	free_page((unsigned long)vcpu->arch.pio_data);  in kvm_arch_vcpu_create()
11966 	kvm_free_lapic(vcpu);  in kvm_arch_vcpu_create()
11968 	kvm_mmu_destroy(vcpu);  in kvm_arch_vcpu_create()
11972 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_postcreate()  argument
11974 	struct kvm *kvm = vcpu->kvm;  in kvm_arch_vcpu_postcreate()
11976 	if (mutex_lock_killable(&vcpu->mutex))  in kvm_arch_vcpu_postcreate()
11978 	vcpu_load(vcpu);  in kvm_arch_vcpu_postcreate()
11979 	kvm_synchronize_tsc(vcpu, 0);  in kvm_arch_vcpu_postcreate()
11980 	vcpu_put(vcpu);  in kvm_arch_vcpu_postcreate()
11983 	vcpu->arch.msr_kvm_poll_control = 1;  in kvm_arch_vcpu_postcreate()
11985 	mutex_unlock(&vcpu->mutex);  in kvm_arch_vcpu_postcreate()
11987 	if (kvmclock_periodic_sync && vcpu->vcpu_idx == 0)  in kvm_arch_vcpu_postcreate()
11992 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_destroy()  argument
11996 	kvmclock_reset(vcpu);  in kvm_arch_vcpu_destroy()
11998 	static_call(kvm_x86_vcpu_free)(vcpu);  in kvm_arch_vcpu_destroy()
12000 	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);  in kvm_arch_vcpu_destroy()
12001 	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);  in kvm_arch_vcpu_destroy()
12002 	fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);  in kvm_arch_vcpu_destroy()
12004 	kvm_xen_destroy_vcpu(vcpu);  in kvm_arch_vcpu_destroy()
12005 	kvm_hv_vcpu_uninit(vcpu);  in kvm_arch_vcpu_destroy()
12006 	kvm_pmu_destroy(vcpu);  in kvm_arch_vcpu_destroy()
12007 	kfree(vcpu->arch.mce_banks);  in kvm_arch_vcpu_destroy()
12008 	kfree(vcpu->arch.mci_ctl2_banks);  in kvm_arch_vcpu_destroy()
12009 	kvm_free_lapic(vcpu);  in kvm_arch_vcpu_destroy()
12010 	idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_destroy()
12011 	kvm_mmu_destroy(vcpu);  in kvm_arch_vcpu_destroy()
12012 	srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_destroy()
12013 	free_page((unsigned long)vcpu->arch.pio_data);  in kvm_arch_vcpu_destroy()
12014 	kvfree(vcpu->arch.cpuid_entries);  in kvm_arch_vcpu_destroy()
12015 	if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_destroy()
12019 void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)  in kvm_vcpu_reset()  argument
12022 	unsigned long old_cr0 = kvm_read_cr0(vcpu);  in kvm_vcpu_reset()
12029 	 * into hardware, to be zeroed at vCPU creation.  Use CRs as a sentinel  in kvm_vcpu_reset()
12033 		     (old_cr0 || kvm_read_cr3(vcpu) || kvm_read_cr4(vcpu)));  in kvm_vcpu_reset()
12037 	 * possible to INIT the vCPU while L2 is active.  Force the vCPU back  in kvm_vcpu_reset()
12041 	if (is_guest_mode(vcpu))  in kvm_vcpu_reset()
12042 		kvm_leave_nested(vcpu);  in kvm_vcpu_reset()
12044 	kvm_lapic_reset(vcpu, init_event);  in kvm_vcpu_reset()
12046 	WARN_ON_ONCE(is_guest_mode(vcpu) || is_smm(vcpu));  in kvm_vcpu_reset()
12047 	vcpu->arch.hflags = 0;  in kvm_vcpu_reset()
12049 	vcpu->arch.smi_pending = 0;  in kvm_vcpu_reset()
12050 	vcpu->arch.smi_count = 0;  in kvm_vcpu_reset()
12051 	atomic_set(&vcpu->arch.nmi_queued, 0);  in kvm_vcpu_reset()
12052 	vcpu->arch.nmi_pending = 0;  in kvm_vcpu_reset()
12053 	vcpu->arch.nmi_injected = false;  in kvm_vcpu_reset()
12054 	kvm_clear_interrupt_queue(vcpu);  in kvm_vcpu_reset()
12055 	kvm_clear_exception_queue(vcpu);  in kvm_vcpu_reset()
12057 	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));  in kvm_vcpu_reset()
12058 	kvm_update_dr0123(vcpu);  in kvm_vcpu_reset()
12059 	vcpu->arch.dr6 = DR6_ACTIVE_LOW;  in kvm_vcpu_reset()
12060 	vcpu->arch.dr7 = DR7_FIXED_1;  in kvm_vcpu_reset()
12061 	kvm_update_dr7(vcpu);  in kvm_vcpu_reset()
12063 	vcpu->arch.cr2 = 0;  in kvm_vcpu_reset()
12065 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_reset()
12066 	vcpu->arch.apf.msr_en_val = 0;  in kvm_vcpu_reset()
12067 	vcpu->arch.apf.msr_int_val = 0;  in kvm_vcpu_reset()
12068 	vcpu->arch.st.msr_val = 0;  in kvm_vcpu_reset()
12070 	kvmclock_reset(vcpu);  in kvm_vcpu_reset()
12072 	kvm_clear_async_pf_completion_queue(vcpu);  in kvm_vcpu_reset()
12073 	kvm_async_pf_hash_reset(vcpu);  in kvm_vcpu_reset()
12074 	vcpu->arch.apf.halted = false;  in kvm_vcpu_reset()
12076 	if (vcpu->arch.guest_fpu.fpstate && kvm_mpx_supported()) {  in kvm_vcpu_reset()
12077 		struct fpstate *fpstate = vcpu->arch.guest_fpu.fpstate;  in kvm_vcpu_reset()
12084 			kvm_put_guest_fpu(vcpu);  in kvm_vcpu_reset()
12090 			kvm_load_guest_fpu(vcpu);  in kvm_vcpu_reset()
12094 		kvm_pmu_reset(vcpu);  in kvm_vcpu_reset()
12095 		vcpu->arch.smbase = 0x30000;  in kvm_vcpu_reset()
12097 		vcpu->arch.msr_misc_features_enables = 0;  in kvm_vcpu_reset()
12098 		vcpu->arch.ia32_misc_enable_msr = MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL |  in kvm_vcpu_reset()
12101 		__kvm_set_xcr(vcpu, 0, XFEATURE_MASK_FP);  in kvm_vcpu_reset()
12102 		__kvm_set_msr(vcpu, MSR_IA32_XSS, 0, true);  in kvm_vcpu_reset()
12106 	memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));  in kvm_vcpu_reset()
12107 	kvm_register_mark_dirty(vcpu, VCPU_REGS_RSP);  in kvm_vcpu_reset()
12112 	 * RESET since KVM emulates RESET before exposing the vCPU to userspace,  in kvm_vcpu_reset()
12116 	cpuid_0x1 = kvm_find_cpuid_entry(vcpu, 1);  in kvm_vcpu_reset()
12117 	kvm_rdx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600);  in kvm_vcpu_reset()
12119 	static_call(kvm_x86_vcpu_reset)(vcpu, init_event);  in kvm_vcpu_reset()
12121 	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);  in kvm_vcpu_reset()
12122 	kvm_rip_write(vcpu, 0xfff0);  in kvm_vcpu_reset()
12124 	vcpu->arch.cr3 = 0;  in kvm_vcpu_reset()
12125 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);  in kvm_vcpu_reset()
12138 	static_call(kvm_x86_set_cr0)(vcpu, new_cr0);  in kvm_vcpu_reset()
12139 	static_call(kvm_x86_set_cr4)(vcpu, 0);  in kvm_vcpu_reset()
12140 	static_call(kvm_x86_set_efer)(vcpu, 0);  in kvm_vcpu_reset()
12141 	static_call(kvm_x86_update_exception_bitmap)(vcpu);  in kvm_vcpu_reset()
12152 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_vcpu_reset()
12153 		kvm_mmu_reset_context(vcpu);  in kvm_vcpu_reset()
12166 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_vcpu_reset()
12170 void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)  in kvm_vcpu_deliver_sipi_vector()  argument
12174 	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);  in kvm_vcpu_deliver_sipi_vector()
12177 	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);  in kvm_vcpu_deliver_sipi_vector()
12178 	kvm_rip_write(vcpu, 0);  in kvm_vcpu_deliver_sipi_vector()
12185 	struct kvm_vcpu *vcpu;  in kvm_arch_hardware_enable()  local
12205 		kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_arch_hardware_enable()
12206 			if (!stable && vcpu->cpu == smp_processor_id())  in kvm_arch_hardware_enable()
12207 				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_arch_hardware_enable()
12208 			if (stable && vcpu->arch.last_host_tsc > local_tsc) {  in kvm_arch_hardware_enable()
12210 				if (vcpu->arch.last_host_tsc > max_tsc)  in kvm_arch_hardware_enable()
12211 					max_tsc = vcpu->arch.last_host_tsc;  in kvm_arch_hardware_enable()
12228 	 * adjustment to TSC in each VCPU.  When the VCPU later gets loaded,  in kvm_arch_hardware_enable()
12230 	 * adjustments, in case multiple suspend cycles happen before some VCPU  in kvm_arch_hardware_enable()
12258 			kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_arch_hardware_enable()
12259 				vcpu->arch.tsc_offset_adjustment += delta_cyc;  in kvm_arch_hardware_enable()
12260 				vcpu->arch.last_host_tsc = local_tsc;  in kvm_arch_hardware_enable()
12261 				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in kvm_arch_hardware_enable()
12284 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)  in kvm_vcpu_is_reset_bsp()  argument
12286 	return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;  in kvm_vcpu_is_reset_bsp()
12289 bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)  in kvm_vcpu_is_bsp()  argument
12291 	return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;  in kvm_vcpu_is_bsp()
12297 void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)  in kvm_arch_sched_in()  argument
12299 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);  in kvm_arch_sched_in()
12301 	vcpu->arch.l1tf_flush_l1d = true;  in kvm_arch_sched_in()
12304 		kvm_make_request(KVM_REQ_PMU, vcpu);  in kvm_arch_sched_in()
12306 	static_call(kvm_x86_sched_in)(vcpu, cpu);  in kvm_arch_sched_in()
12383 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)  in kvm_unload_vcpu_mmu()  argument
12385 	vcpu_load(vcpu);  in kvm_unload_vcpu_mmu()
12386 	kvm_mmu_unload(vcpu);  in kvm_unload_vcpu_mmu()
12387 	vcpu_put(vcpu);  in kvm_unload_vcpu_mmu()
12393 	struct kvm_vcpu *vcpu;  in kvm_unload_vcpu_mmus()  local
12395 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_unload_vcpu_mmus()
12396 		kvm_clear_async_pf_completion_queue(vcpu);  in kvm_unload_vcpu_mmus()
12397 		kvm_unload_vcpu_mmu(vcpu);  in kvm_unload_vcpu_mmus()
12631 	struct kvm_vcpu *vcpu;  in kvm_arch_memslots_updated()  local
12641 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_arch_memslots_updated()
12642 		kvm_vcpu_kick(vcpu);  in kvm_arch_memslots_updated()
12830 static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)  in kvm_guest_apic_has_interrupt()  argument
12832 	return (is_guest_mode(vcpu) &&  in kvm_guest_apic_has_interrupt()
12833 		static_call(kvm_x86_guest_apic_has_interrupt)(vcpu));  in kvm_guest_apic_has_interrupt()
12836 static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)  in kvm_vcpu_has_events()  argument
12838 	if (!list_empty_careful(&vcpu->async_pf.done))  in kvm_vcpu_has_events()
12841 	if (kvm_apic_has_pending_init_or_sipi(vcpu) &&  in kvm_vcpu_has_events()
12842 	    kvm_apic_init_sipi_allowed(vcpu))  in kvm_vcpu_has_events()
12845 	if (vcpu->arch.pv.pv_unhalted)  in kvm_vcpu_has_events()
12848 	if (kvm_is_exception_pending(vcpu))  in kvm_vcpu_has_events()
12851 	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||  in kvm_vcpu_has_events()
12852 	    (vcpu->arch.nmi_pending &&  in kvm_vcpu_has_events()
12853 	     static_call(kvm_x86_nmi_allowed)(vcpu, false)))  in kvm_vcpu_has_events()
12857 	if (kvm_test_request(KVM_REQ_SMI, vcpu) ||  in kvm_vcpu_has_events()
12858 	    (vcpu->arch.smi_pending &&  in kvm_vcpu_has_events()
12859 	     static_call(kvm_x86_smi_allowed)(vcpu, false)))  in kvm_vcpu_has_events()
12863 	if (kvm_test_request(KVM_REQ_PMI, vcpu))  in kvm_vcpu_has_events()
12866 	if (kvm_arch_interrupt_allowed(vcpu) &&  in kvm_vcpu_has_events()
12867 	    (kvm_cpu_has_interrupt(vcpu) ||  in kvm_vcpu_has_events()
12868 	    kvm_guest_apic_has_interrupt(vcpu)))  in kvm_vcpu_has_events()
12871 	if (kvm_hv_has_stimer_pending(vcpu))  in kvm_vcpu_has_events()
12874 	if (is_guest_mode(vcpu) &&  in kvm_vcpu_has_events()
12876 	    kvm_x86_ops.nested_ops->has_events(vcpu))  in kvm_vcpu_has_events()
12879 	if (kvm_xen_has_pending_events(vcpu))  in kvm_vcpu_has_events()
12885 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_runnable()  argument
12887 	return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu);  in kvm_arch_vcpu_runnable()
12890 bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu)  in kvm_arch_dy_has_pending_interrupt()  argument
12892 	if (kvm_vcpu_apicv_active(vcpu) &&  in kvm_arch_dy_has_pending_interrupt()
12893 	    static_call(kvm_x86_dy_apicv_has_pending_interrupt)(vcpu))  in kvm_arch_dy_has_pending_interrupt()
12899 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)  in kvm_arch_dy_runnable()  argument
12901 	if (READ_ONCE(vcpu->arch.pv.pv_unhalted))  in kvm_arch_dy_runnable()
12904 	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||  in kvm_arch_dy_runnable()
12906 		kvm_test_request(KVM_REQ_SMI, vcpu) ||  in kvm_arch_dy_runnable()
12908 		 kvm_test_request(KVM_REQ_EVENT, vcpu))  in kvm_arch_dy_runnable()
12911 	return kvm_arch_dy_has_pending_interrupt(vcpu);  in kvm_arch_dy_runnable()
12914 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_in_kernel()  argument
12916 	if (vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_in_kernel()
12919 	return vcpu->arch.preempted_in_kernel;  in kvm_arch_vcpu_in_kernel()
12922 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_get_ip()  argument
12924 	return kvm_rip_read(vcpu);  in kvm_arch_vcpu_get_ip()
12927 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_should_kick()  argument
12929 	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;  in kvm_arch_vcpu_should_kick()
12932 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)  in kvm_arch_interrupt_allowed()  argument
12934 	return static_call(kvm_x86_interrupt_allowed)(vcpu, false);  in kvm_arch_interrupt_allowed()
12937 unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)  in kvm_get_linear_rip()  argument
12940 	if (vcpu->arch.guest_state_protected)  in kvm_get_linear_rip()
12943 	if (is_64_bit_mode(vcpu))  in kvm_get_linear_rip()
12944 		return kvm_rip_read(vcpu);  in kvm_get_linear_rip()
12945 	return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +  in kvm_get_linear_rip()
12946 		     kvm_rip_read(vcpu));  in kvm_get_linear_rip()
12950 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)  in kvm_is_linear_rip()  argument
12952 	return kvm_get_linear_rip(vcpu) == linear_rip;  in kvm_is_linear_rip()
12956 unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)  in kvm_get_rflags()  argument
12960 	rflags = static_call(kvm_x86_get_rflags)(vcpu);  in kvm_get_rflags()
12961 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)  in kvm_get_rflags()
12967 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)  in __kvm_set_rflags()  argument
12969 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&  in __kvm_set_rflags()
12970 	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))  in __kvm_set_rflags()
12972 	static_call(kvm_x86_set_rflags)(vcpu, rflags);  in __kvm_set_rflags()
12975 void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)  in kvm_set_rflags()  argument
12977 	__kvm_set_rflags(vcpu, rflags);  in kvm_set_rflags()
12978 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_set_rflags()
12994 static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_add_async_pf_gfn()  argument
12998 	while (vcpu->arch.apf.gfns[key] != ~0)  in kvm_add_async_pf_gfn()
13001 	vcpu->arch.apf.gfns[key] = gfn;  in kvm_add_async_pf_gfn()
13004 static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_async_pf_gfn_slot()  argument
13010 		     (vcpu->arch.apf.gfns[key] != gfn &&  in kvm_async_pf_gfn_slot()
13011 		      vcpu->arch.apf.gfns[key] != ~0); i++)  in kvm_async_pf_gfn_slot()
13017 bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_find_async_pf_gfn()  argument
13019 	return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;  in kvm_find_async_pf_gfn()
13022 static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_del_async_pf_gfn()  argument
13026 	i = j = kvm_async_pf_gfn_slot(vcpu, gfn);  in kvm_del_async_pf_gfn()
13028 	if (WARN_ON_ONCE(vcpu->arch.apf.gfns[i] != gfn))  in kvm_del_async_pf_gfn()
13032 		vcpu->arch.apf.gfns[i] = ~0;  in kvm_del_async_pf_gfn()
13035 			if (vcpu->arch.apf.gfns[j] == ~0)  in kvm_del_async_pf_gfn()
13037 			k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);  in kvm_del_async_pf_gfn()
13044 		vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];  in kvm_del_async_pf_gfn()
13049 static inline int apf_put_user_notpresent(struct kvm_vcpu *vcpu)  in apf_put_user_notpresent()  argument
13053 	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason,  in apf_put_user_notpresent()
13057 static inline int apf_put_user_ready(struct kvm_vcpu *vcpu, u32 token)  in apf_put_user_ready()  argument
13061 	return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,  in apf_put_user_ready()
13065 static inline bool apf_pageready_slot_free(struct kvm_vcpu *vcpu)  in apf_pageready_slot_free()  argument
13070 	if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,  in apf_pageready_slot_free()
13077 static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)  in kvm_can_deliver_async_pf()  argument
13080 	if (!kvm_pv_async_pf_enabled(vcpu))  in kvm_can_deliver_async_pf()
13083 	if (vcpu->arch.apf.send_user_only &&  in kvm_can_deliver_async_pf()
13084 	    static_call(kvm_x86_get_cpl)(vcpu) == 0)  in kvm_can_deliver_async_pf()
13087 	if (is_guest_mode(vcpu)) {  in kvm_can_deliver_async_pf()
13092 		return vcpu->arch.apf.delivery_as_pf_vmexit;  in kvm_can_deliver_async_pf()
13099 		return is_paging(vcpu);  in kvm_can_deliver_async_pf()
13103 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)  in kvm_can_do_async_pf()  argument
13105 	if (unlikely(!lapic_in_kernel(vcpu) ||  in kvm_can_do_async_pf()
13106 		     kvm_event_needs_reinjection(vcpu) ||  in kvm_can_do_async_pf()
13107 		     kvm_is_exception_pending(vcpu)))  in kvm_can_do_async_pf()
13110 	if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))  in kvm_can_do_async_pf()
13117 	return kvm_arch_interrupt_allowed(vcpu);  in kvm_can_do_async_pf()
13120 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,  in kvm_arch_async_page_not_present()  argument
13126 	kvm_add_async_pf_gfn(vcpu, work->arch.gfn);  in kvm_arch_async_page_not_present()
13128 	if (kvm_can_deliver_async_pf(vcpu) &&  in kvm_arch_async_page_not_present()
13129 	    !apf_put_user_notpresent(vcpu)) {  in kvm_arch_async_page_not_present()
13136 		kvm_inject_page_fault(vcpu, &fault);  in kvm_arch_async_page_not_present()
13147 		kvm_make_request(KVM_REQ_APF_HALT, vcpu);  in kvm_arch_async_page_not_present()
13152 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,  in kvm_arch_async_page_present()  argument
13157 		.vector = vcpu->arch.apf.vec  in kvm_arch_async_page_present()
13163 		kvm_del_async_pf_gfn(vcpu, work->arch.gfn);  in kvm_arch_async_page_present()
13167 	    kvm_pv_async_pf_enabled(vcpu) &&  in kvm_arch_async_page_present()
13168 	    !apf_put_user_ready(vcpu, work->arch.token)) {  in kvm_arch_async_page_present()
13169 		vcpu->arch.apf.pageready_pending = true;  in kvm_arch_async_page_present()
13170 		kvm_apic_set_irq(vcpu, &irq, NULL);  in kvm_arch_async_page_present()
13173 	vcpu->arch.apf.halted = false;  in kvm_arch_async_page_present()
13174 	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;  in kvm_arch_async_page_present()
13177 void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu)  in kvm_arch_async_page_present_queued()  argument
13179 	kvm_make_request(KVM_REQ_APF_READY, vcpu);  in kvm_arch_async_page_present_queued()
13180 	if (!vcpu->arch.apf.pageready_pending)  in kvm_arch_async_page_present_queued()
13181 		kvm_vcpu_kick(vcpu);  in kvm_arch_async_page_present_queued()
13184 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)  in kvm_arch_can_dequeue_async_page_present()  argument
13186 	if (!kvm_pv_async_pf_enabled(vcpu))  in kvm_arch_can_dequeue_async_page_present()
13189 		return kvm_lapic_enabled(vcpu) && apf_pageready_slot_free(vcpu);  in kvm_arch_can_dequeue_async_page_present()
13296 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)  in kvm_arch_no_poll()  argument
13298 	return (vcpu->arch.msr_kvm_poll_control & 1) == 0;  in kvm_arch_no_poll()
13329 void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code)  in kvm_fixup_and_inject_pf_error()  argument
13331 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_fixup_and_inject_pf_error()
13337 	    mmu->gva_to_gpa(vcpu, mmu, gva, access, &fault) != INVALID_GPA) {  in kvm_fixup_and_inject_pf_error()
13339 		 * If vcpu->arch.walk_mmu->gva_to_gpa succeeded, the page  in kvm_fixup_and_inject_pf_error()
13350 	vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault);  in kvm_fixup_and_inject_pf_error()
13359 int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r,  in kvm_handle_memory_failure()  argument
13363 		if (KVM_BUG_ON(!e, vcpu->kvm))  in kvm_handle_memory_failure()
13366 		kvm_inject_emulated_page_fault(vcpu, e);  in kvm_handle_memory_failure()
13377 	kvm_prepare_emulation_failure_exit(vcpu);  in kvm_handle_memory_failure()
13383 int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva)  in kvm_handle_invpcid()  argument
13393 	r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);  in kvm_handle_invpcid()
13395 		return kvm_handle_memory_failure(vcpu, r, &e);  in kvm_handle_invpcid()
13398 		kvm_inject_gp(vcpu, 0);  in kvm_handle_invpcid()
13402 	pcid_enabled = kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE);  in kvm_handle_invpcid()
13407 		    is_noncanonical_address(operand.gla, vcpu)) {  in kvm_handle_invpcid()
13408 			kvm_inject_gp(vcpu, 0);  in kvm_handle_invpcid()
13411 		kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid);  in kvm_handle_invpcid()
13412 		return kvm_skip_emulated_instruction(vcpu);  in kvm_handle_invpcid()
13416 			kvm_inject_gp(vcpu, 0);  in kvm_handle_invpcid()
13420 		kvm_invalidate_pcid(vcpu, operand.pcid);  in kvm_handle_invpcid()
13421 		return kvm_skip_emulated_instruction(vcpu);  in kvm_handle_invpcid()
13433 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_handle_invpcid()
13434 		return kvm_skip_emulated_instruction(vcpu);  in kvm_handle_invpcid()
13437 		kvm_inject_gp(vcpu, 0);  in kvm_handle_invpcid()
13443 static int complete_sev_es_emulated_mmio(struct kvm_vcpu *vcpu)  in complete_sev_es_emulated_mmio()  argument
13445 	struct kvm_run *run = vcpu->run;  in complete_sev_es_emulated_mmio()
13449 	BUG_ON(!vcpu->mmio_needed);  in complete_sev_es_emulated_mmio()
13452 	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];  in complete_sev_es_emulated_mmio()
13454 	if (!vcpu->mmio_is_write)  in complete_sev_es_emulated_mmio()
13460 		vcpu->mmio_cur_fragment++;  in complete_sev_es_emulated_mmio()
13468 	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {  in complete_sev_es_emulated_mmio()
13469 		vcpu->mmio_needed = 0;  in complete_sev_es_emulated_mmio()
13479 	run->mmio.is_write = vcpu->mmio_is_write;  in complete_sev_es_emulated_mmio()
13484 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;  in complete_sev_es_emulated_mmio()
13489 int kvm_sev_es_mmio_write(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,  in kvm_sev_es_mmio_write()  argument
13498 	handled = write_emultor.read_write_mmio(vcpu, gpa, bytes, data);  in kvm_sev_es_mmio_write()
13507 	frag = vcpu->mmio_fragments;  in kvm_sev_es_mmio_write()
13508 	vcpu->mmio_nr_fragments = 1;  in kvm_sev_es_mmio_write()
13513 	vcpu->mmio_needed = 1;  in kvm_sev_es_mmio_write()
13514 	vcpu->mmio_cur_fragment = 0;  in kvm_sev_es_mmio_write()
13516 	vcpu->run->mmio.phys_addr = gpa;  in kvm_sev_es_mmio_write()
13517 	vcpu->run->mmio.len = min(8u, frag->len);  in kvm_sev_es_mmio_write()
13518 	vcpu->run->mmio.is_write = 1;  in kvm_sev_es_mmio_write()
13519 	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));  in kvm_sev_es_mmio_write()
13520 	vcpu->run->exit_reason = KVM_EXIT_MMIO;  in kvm_sev_es_mmio_write()
13522 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;  in kvm_sev_es_mmio_write()
13528 int kvm_sev_es_mmio_read(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,  in kvm_sev_es_mmio_read()  argument
13537 	handled = read_emultor.read_write_mmio(vcpu, gpa, bytes, data);  in kvm_sev_es_mmio_read()
13546 	frag = vcpu->mmio_fragments;  in kvm_sev_es_mmio_read()
13547 	vcpu->mmio_nr_fragments = 1;  in kvm_sev_es_mmio_read()
13552 	vcpu->mmio_needed = 1;  in kvm_sev_es_mmio_read()
13553 	vcpu->mmio_cur_fragment = 0;  in kvm_sev_es_mmio_read()
13555 	vcpu->run->mmio.phys_addr = gpa;  in kvm_sev_es_mmio_read()
13556 	vcpu->run->mmio.len = min(8u, frag->len);  in kvm_sev_es_mmio_read()
13557 	vcpu->run->mmio.is_write = 0;  in kvm_sev_es_mmio_read()
13558 	vcpu->run->exit_reason = KVM_EXIT_MMIO;  in kvm_sev_es_mmio_read()
13560 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;  in kvm_sev_es_mmio_read()
13566 static void advance_sev_es_emulated_pio(struct kvm_vcpu *vcpu, unsigned count, int size)  in advance_sev_es_emulated_pio()  argument
13568 	vcpu->arch.sev_pio_count -= count;  in advance_sev_es_emulated_pio()
13569 	vcpu->arch.sev_pio_data += count * size;  in advance_sev_es_emulated_pio()
13572 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,
13575 static int complete_sev_es_emulated_outs(struct kvm_vcpu *vcpu)  in complete_sev_es_emulated_outs()  argument
13577 	int size = vcpu->arch.pio.size;  in complete_sev_es_emulated_outs()
13578 	int port = vcpu->arch.pio.port;  in complete_sev_es_emulated_outs()
13580 	vcpu->arch.pio.count = 0;  in complete_sev_es_emulated_outs()
13581 	if (vcpu->arch.sev_pio_count)  in complete_sev_es_emulated_outs()
13582 		return kvm_sev_es_outs(vcpu, size, port);  in complete_sev_es_emulated_outs()
13586 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,  in kvm_sev_es_outs()  argument
13591 			min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);  in kvm_sev_es_outs()
13592 		int ret = emulator_pio_out(vcpu, size, port, vcpu->arch.sev_pio_data, count);  in kvm_sev_es_outs()
13595 		advance_sev_es_emulated_pio(vcpu, count, size);  in kvm_sev_es_outs()
13600 		if (!vcpu->arch.sev_pio_count)  in kvm_sev_es_outs()
13604 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_outs;  in kvm_sev_es_outs()
13608 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,
13611 static int complete_sev_es_emulated_ins(struct kvm_vcpu *vcpu)  in complete_sev_es_emulated_ins()  argument
13613 	unsigned count = vcpu->arch.pio.count;  in complete_sev_es_emulated_ins()
13614 	int size = vcpu->arch.pio.size;  in complete_sev_es_emulated_ins()
13615 	int port = vcpu->arch.pio.port;  in complete_sev_es_emulated_ins()
13617 	complete_emulator_pio_in(vcpu, vcpu->arch.sev_pio_data);  in complete_sev_es_emulated_ins()
13618 	advance_sev_es_emulated_pio(vcpu, count, size);  in complete_sev_es_emulated_ins()
13619 	if (vcpu->arch.sev_pio_count)  in complete_sev_es_emulated_ins()
13620 		return kvm_sev_es_ins(vcpu, size, port);  in complete_sev_es_emulated_ins()
13624 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,  in kvm_sev_es_ins()  argument
13629 			min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);  in kvm_sev_es_ins()
13630 		if (!emulator_pio_in(vcpu, size, port, vcpu->arch.sev_pio_data, count))  in kvm_sev_es_ins()
13634 		advance_sev_es_emulated_pio(vcpu, count, size);  in kvm_sev_es_ins()
13635 		if (!vcpu->arch.sev_pio_count)  in kvm_sev_es_ins()
13639 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_ins;  in kvm_sev_es_ins()
13643 int kvm_sev_es_string_io(struct kvm_vcpu *vcpu, unsigned int size,  in kvm_sev_es_string_io()  argument
13647 	vcpu->arch.sev_pio_data = data;  in kvm_sev_es_string_io()
13648 	vcpu->arch.sev_pio_count = count;  in kvm_sev_es_string_io()
13649 	return in ? kvm_sev_es_ins(vcpu, size, port)  in kvm_sev_es_string_io()
13650 		  : kvm_sev_es_outs(vcpu, size, port);  in kvm_sev_es_string_io()