37 static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);
43 static struct ghcb __initdata *boot_ghcb;
47 	struct ghcb ghcb_page;
56 	struct ghcb backup_ghcb;
83 	struct ghcb *ghcb;  member
157 static noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)  in __sev_get_ghcb()
160 	struct ghcb *ghcb;  in __sev_get_ghcb()  local
165 	ghcb = &data->ghcb_page;  in __sev_get_ghcb()
188 		state->ghcb = &data->backup_ghcb;  in __sev_get_ghcb()
191 		*state->ghcb = *ghcb;  in __sev_get_ghcb()
193 		state->ghcb = NULL;  in __sev_get_ghcb()
197 	return ghcb;  in __sev_get_ghcb()
416 static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt,  in vc_slow_virt_to_phys()  argument
457 	struct ghcb *ghcb;  in __sev_put_ghcb()  local
462 	ghcb = &data->ghcb_page;  in __sev_put_ghcb()
464 	if (state->ghcb) {  in __sev_put_ghcb()
466 		*ghcb = *state->ghcb;  in __sev_put_ghcb()
468 		state->ghcb = NULL;  in __sev_put_ghcb()
474 		vc_ghcb_invalidate(ghcb);  in __sev_put_ghcb()
482 	struct ghcb *ghcb;  in __sev_es_nmi_complete()  local
484 	ghcb = __sev_get_ghcb(&state);  in __sev_es_nmi_complete()
486 	vc_ghcb_invalidate(ghcb);  in __sev_es_nmi_complete()
487 	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE);  in __sev_es_nmi_complete()
488 	ghcb_set_sw_exit_info_1(ghcb, 0);  in __sev_es_nmi_complete()
489 	ghcb_set_sw_exit_info_2(ghcb, 0);  in __sev_es_nmi_complete()
491 	sev_es_wr_ghcb_msr(__pa_nodebug(ghcb));  in __sev_es_nmi_complete()
501 	struct ghcb *ghcb;  in get_jump_table_addr()  local
506 	ghcb = __sev_get_ghcb(&state);  in get_jump_table_addr()
508 	vc_ghcb_invalidate(ghcb);  in get_jump_table_addr()
509 	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_JUMP_TABLE);  in get_jump_table_addr()
510 	ghcb_set_sw_exit_info_1(ghcb, SVM_VMGEXIT_GET_AP_JUMP_TABLE);  in get_jump_table_addr()
511 	ghcb_set_sw_exit_info_2(ghcb, 0);  in get_jump_table_addr()
513 	sev_es_wr_ghcb_msr(__pa(ghcb));  in get_jump_table_addr()
516 	if (ghcb_sw_exit_info_1_is_valid(ghcb) &&  in get_jump_table_addr()
517 	    ghcb_sw_exit_info_2_is_valid(ghcb))  in get_jump_table_addr()
518 		ret = ghcb->save.sw_exit_info_2;  in get_jump_table_addr()
592 static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt)  in vc_handle_msr()  argument
601 	ghcb_set_rcx(ghcb, regs->cx);  in vc_handle_msr()
603 		ghcb_set_rax(ghcb, regs->ax);  in vc_handle_msr()
604 		ghcb_set_rdx(ghcb, regs->dx);  in vc_handle_msr()
607 	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, exit_info_1, 0);  in vc_handle_msr()
610 		regs->ax = ghcb->save.rax;  in vc_handle_msr()
611 		regs->dx = ghcb->save.rdx;  in vc_handle_msr()
643 	struct ghcb *ghcb;  in sev_es_ap_hlt_loop()  local
645 	ghcb = __sev_get_ghcb(&state);  in sev_es_ap_hlt_loop()
648 		vc_ghcb_invalidate(ghcb);  in sev_es_ap_hlt_loop()
649 		ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_HLT_LOOP);  in sev_es_ap_hlt_loop()
650 		ghcb_set_sw_exit_info_1(ghcb, 0);  in sev_es_ap_hlt_loop()
651 		ghcb_set_sw_exit_info_2(ghcb, 0);  in sev_es_ap_hlt_loop()
653 		sev_es_wr_ghcb_msr(__pa(ghcb));  in sev_es_ap_hlt_loop()
657 		if (ghcb_sw_exit_info_2_is_valid(ghcb) &&  in sev_es_ap_hlt_loop()
658 		    ghcb->save.sw_exit_info_2)  in sev_es_ap_hlt_loop()
796 static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt,  in vc_do_mmio()  argument
800 	unsigned long ghcb_pa = __pa(ghcb);  in vc_do_mmio()
811 	res = vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr);  in vc_do_mmio()
823 	ghcb_set_sw_scratch(ghcb, ghcb_pa + offsetof(struct ghcb, shared_buffer));  in vc_do_mmio()
825 	return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2);  in vc_do_mmio()
828 static enum es_result vc_handle_mmio_twobyte_ops(struct ghcb *ghcb,  in vc_handle_mmio_twobyte_ops()  argument
846 		ret = vc_do_mmio(ghcb, ctxt, bytes, true);  in vc_handle_mmio_twobyte_ops()
857 		memcpy(reg_data, ghcb->shared_buffer, bytes);  in vc_handle_mmio_twobyte_ops()
868 		ret = vc_do_mmio(ghcb, ctxt, bytes, true);  in vc_handle_mmio_twobyte_ops()
878 			u8 *val = (u8 *)ghcb->shared_buffer;  in vc_handle_mmio_twobyte_ops()
882 			u16 *val = (u16 *)ghcb->shared_buffer;  in vc_handle_mmio_twobyte_ops()
888 		memcpy(reg_data, ghcb->shared_buffer, bytes);  in vc_handle_mmio_twobyte_ops()
965 static enum es_result vc_handle_mmio(struct ghcb *ghcb,  in vc_handle_mmio()  argument
986 		memcpy(ghcb->shared_buffer, reg_data, bytes);  in vc_handle_mmio()
988 		ret = vc_do_mmio(ghcb, ctxt, bytes, false);  in vc_handle_mmio()
998 		memcpy(ghcb->shared_buffer, insn->immediate1.bytes, bytes);  in vc_handle_mmio()
1000 		ret = vc_do_mmio(ghcb, ctxt, bytes, false);  in vc_handle_mmio()
1011 		ret = vc_do_mmio(ghcb, ctxt, bytes, true);  in vc_handle_mmio()
1023 		memcpy(reg_data, ghcb->shared_buffer, bytes);  in vc_handle_mmio()
1038 		ret = vc_handle_mmio_twobyte_ops(ghcb, ctxt);  in vc_handle_mmio()
1047 static enum es_result vc_handle_dr7_write(struct ghcb *ghcb,  in vc_handle_dr7_write()  argument
1074 	ghcb_set_rax(ghcb, val);  in vc_handle_dr7_write()
1075 	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0);  in vc_handle_dr7_write()
1085 static enum es_result vc_handle_dr7_read(struct ghcb *ghcb,  in vc_handle_dr7_read()  argument
1102 static enum es_result vc_handle_wbinvd(struct ghcb *ghcb,  in vc_handle_wbinvd()  argument
1105 	return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0);  in vc_handle_wbinvd()
1108 static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt)  in vc_handle_rdpmc()  argument
1112 	ghcb_set_rcx(ghcb, ctxt->regs->cx);  in vc_handle_rdpmc()
1114 	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0);  in vc_handle_rdpmc()
1118 	if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb)))  in vc_handle_rdpmc()
1121 	ctxt->regs->ax = ghcb->save.rax;  in vc_handle_rdpmc()
1122 	ctxt->regs->dx = ghcb->save.rdx;  in vc_handle_rdpmc()
1127 static enum es_result vc_handle_monitor(struct ghcb *ghcb,  in vc_handle_monitor()  argument
1137 static enum es_result vc_handle_mwait(struct ghcb *ghcb,  in vc_handle_mwait()  argument
1144 static enum es_result vc_handle_vmmcall(struct ghcb *ghcb,  in vc_handle_vmmcall()  argument
1149 	ghcb_set_rax(ghcb, ctxt->regs->ax);  in vc_handle_vmmcall()
1150 	ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0);  in vc_handle_vmmcall()
1153 		x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs);  in vc_handle_vmmcall()
1155 	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0);  in vc_handle_vmmcall()
1159 	if (!ghcb_rax_is_valid(ghcb))  in vc_handle_vmmcall()
1162 	ctxt->regs->ax = ghcb->save.rax;  in vc_handle_vmmcall()
1170 	    !x86_platform.hyper.sev_es_hcall_finish(ghcb, ctxt->regs))  in vc_handle_vmmcall()
1176 static enum es_result vc_handle_trap_ac(struct ghcb *ghcb,  in vc_handle_trap_ac()  argument
1190 					 struct ghcb *ghcb,  in vc_handle_exitcode()  argument
1197 		result = vc_handle_dr7_read(ghcb, ctxt);  in vc_handle_exitcode()
1200 		result = vc_handle_dr7_write(ghcb, ctxt);  in vc_handle_exitcode()
1203 		result = vc_handle_trap_ac(ghcb, ctxt);  in vc_handle_exitcode()
1207 		result = vc_handle_rdtsc(ghcb, ctxt, exit_code);  in vc_handle_exitcode()
1210 		result = vc_handle_rdpmc(ghcb, ctxt);  in vc_handle_exitcode()
1217 		result = vc_handle_cpuid(ghcb, ctxt);  in vc_handle_exitcode()
1220 		result = vc_handle_ioio(ghcb, ctxt);  in vc_handle_exitcode()
1223 		result = vc_handle_msr(ghcb, ctxt);  in vc_handle_exitcode()
1226 		result = vc_handle_vmmcall(ghcb, ctxt);  in vc_handle_exitcode()
1229 		result = vc_handle_wbinvd(ghcb, ctxt);  in vc_handle_exitcode()
1232 		result = vc_handle_monitor(ghcb, ctxt);  in vc_handle_exitcode()
1235 		result = vc_handle_mwait(ghcb, ctxt);  in vc_handle_exitcode()
1238 		result = vc_handle_mmio(ghcb, ctxt);  in vc_handle_exitcode()
1289 	struct ghcb *ghcb;  in vc_raw_handle_exception()  local
1292 	ghcb = __sev_get_ghcb(&state);  in vc_raw_handle_exception()
1294 	vc_ghcb_invalidate(ghcb);  in vc_raw_handle_exception()
1298 		result = vc_handle_exitcode(&ctxt, ghcb, error_code);  in vc_raw_handle_exception()