169 	 * The kernel can not handle #VE's when accessing normal kernel  in tdx_parse_tdinfo()
170 	 * memory.  Ensure that no #VE will be delivered for accesses to  in tdx_parse_tdinfo()
171 	 * TD-private memory.  Only VMM-shared memory (MMIO) will #VE.  in tdx_parse_tdinfo()
186  * The TDX module spec states that #VE may be injected for a limited set of
189  *  - Emulation of the architectural #VE injection on EPT violation;
203  * information if #VE occurred due to instruction execution, but not for EPT
206 static int ve_instr_len(struct ve_info *ve)  in ve_instr_len()  argument
208 	switch (ve->exit_reason) {  in ve_instr_len()
214 		/* It is safe to use ve->instr_len for #VE due instructions */  in ve_instr_len()
215 		return ve->instr_len;  in ve_instr_len()
218 		 * For EPT violations, ve->insn_len is not defined. For those,  in ve_instr_len()
222 		WARN_ONCE(1, "ve->instr_len is not defined for EPT violations");  in ve_instr_len()
225 		WARN_ONCE(1, "Unexpected #VE-type: %lld\n", ve->exit_reason);  in ve_instr_len()
226 		return ve->instr_len;  in ve_instr_len()
253 static int handle_halt(struct ve_info *ve)  in handle_halt()  argument
260 	return ve_instr_len(ve);  in handle_halt()
274 static int read_msr(struct pt_regs *regs, struct ve_info *ve)  in read_msr()  argument
292 	return ve_instr_len(ve);  in read_msr()
295 static int write_msr(struct pt_regs *regs, struct ve_info *ve)  in write_msr()  argument
312 	return ve_instr_len(ve);  in write_msr()
315 static int handle_cpuid(struct pt_regs *regs, struct ve_info *ve)  in handle_cpuid()  argument
333 		return ve_instr_len(ve);  in handle_cpuid()
354 	return ve_instr_len(ve);  in handle_cpuid()
380 static int handle_mmio(struct pt_regs *regs, struct ve_info *ve)  in handle_mmio()  argument
426 		if (!mmio_write(size, ve->gpa, val))  in handle_mmio()
431 		if (!mmio_write(size, ve->gpa, val))  in handle_mmio()
453 	if (!mmio_read(size, ve->gpa, &val))  in handle_mmio()
533 static int handle_io(struct pt_regs *regs, struct ve_info *ve)  in handle_io()  argument
535 	u32 exit_qual = ve->exit_qual;  in handle_io()
554 	return ve_instr_len(ve);  in handle_io()
558  * Early #VE exception handler. Only handles a subset of port I/O.
563 	struct ve_info ve;  in tdx_early_handle_ve()  local
566 	tdx_get_ve_info(&ve);  in tdx_early_handle_ve()
568 	if (ve.exit_reason != EXIT_REASON_IO_INSTRUCTION)  in tdx_early_handle_ve()
571 	insn_len = handle_io(regs, &ve);  in tdx_early_handle_ve()
579 void tdx_get_ve_info(struct ve_info *ve)  in tdx_get_ve_info()  argument
584 	 * Called during #VE handling to retrieve the #VE info from the  in tdx_get_ve_info()
587 	 * This has to be called early in #VE handling.  A "nested" #VE which  in tdx_get_ve_info()
590 	 * The call retrieves the #VE info from the TDX module, which also  in tdx_get_ve_info()
591 	 * clears the "#VE valid" flag. This must be done before anything else  in tdx_get_ve_info()
592 	 * because any #VE that occurs while the valid flag is set will lead to  in tdx_get_ve_info()
595 	 * Note, the TDX module treats virtual NMIs as inhibited if the #VE  in tdx_get_ve_info()
596 	 * valid flag is set. It means that NMI=>#VE will not result in a #DF.  in tdx_get_ve_info()
601 	ve->exit_reason = out.rcx;  in tdx_get_ve_info()
602 	ve->exit_qual   = out.rdx;  in tdx_get_ve_info()
603 	ve->gla         = out.r8;  in tdx_get_ve_info()
604 	ve->gpa         = out.r9;  in tdx_get_ve_info()
605 	ve->instr_len   = lower_32_bits(out.r10);  in tdx_get_ve_info()
606 	ve->instr_info  = upper_32_bits(out.r10);  in tdx_get_ve_info()
610  * Handle the user initiated #VE.
615 static int virt_exception_user(struct pt_regs *regs, struct ve_info *ve)  in virt_exception_user()  argument
617 	switch (ve->exit_reason) {  in virt_exception_user()
619 		return handle_cpuid(regs, ve);  in virt_exception_user()
621 		pr_warn("Unexpected #VE: %lld\n", ve->exit_reason);  in virt_exception_user()
632  * Handle the kernel #VE.
637 static int virt_exception_kernel(struct pt_regs *regs, struct ve_info *ve)  in virt_exception_kernel()  argument
639 	switch (ve->exit_reason) {  in virt_exception_kernel()
641 		return handle_halt(ve);  in virt_exception_kernel()
643 		return read_msr(regs, ve);  in virt_exception_kernel()
645 		return write_msr(regs, ve);  in virt_exception_kernel()
647 		return handle_cpuid(regs, ve);  in virt_exception_kernel()
649 		if (is_private_gpa(ve->gpa))  in virt_exception_kernel()
651 		return handle_mmio(regs, ve);  in virt_exception_kernel()
653 		return handle_io(regs, ve);  in virt_exception_kernel()
655 		pr_warn("Unexpected #VE: %lld\n", ve->exit_reason);  in virt_exception_kernel()
660 bool tdx_handle_virt_exception(struct pt_regs *regs, struct ve_info *ve)  in tdx_handle_virt_exception()  argument
665 		insn_len = virt_exception_user(regs, ve);  in tdx_handle_virt_exception()
667 		insn_len = virt_exception_kernel(regs, ve);  in tdx_handle_virt_exception()
671 	/* After successful #VE handling, move the IP */  in tdx_handle_virt_exception()
795          *   - Shared mapping  => Private Page == Recoverable #VE  in tdx_early_init()
804 	 * which can result in a #VE.  But, there is never a private mapping to  in tdx_early_init()
815 	 * bringup low level code. That raises #VE which cannot be handled  in tdx_early_init()