1 /*
2 * Copyright (C) 2012-2015 - ARM Ltd
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18 #include <linux/compiler.h>
19 #include <linux/kvm_host.h>
20
21 #include <asm/kvm_asm.h>
22 #include <asm/kvm_hyp.h>
23
24 /* Yes, this does nothing, on purpose */
__sysreg_do_nothing(struct kvm_cpu_context * ctxt)25 static void __hyp_text __sysreg_do_nothing(struct kvm_cpu_context *ctxt) { }
26
27 /*
28 * Non-VHE: Both host and guest must save everything.
29 *
30 * VHE: Host must save tpidr*_el0, actlr_el1, mdscr_el1, sp_el0,
31 * and guest must save everything.
32 */
33
__sysreg_save_common_state(struct kvm_cpu_context * ctxt)34 static void __hyp_text __sysreg_save_common_state(struct kvm_cpu_context *ctxt)
35 {
36 ctxt->sys_regs[ACTLR_EL1] = read_sysreg(actlr_el1);
37 ctxt->sys_regs[TPIDR_EL0] = read_sysreg(tpidr_el0);
38 ctxt->sys_regs[TPIDRRO_EL0] = read_sysreg(tpidrro_el0);
39 ctxt->sys_regs[MDSCR_EL1] = read_sysreg(mdscr_el1);
40 ctxt->gp_regs.regs.sp = read_sysreg(sp_el0);
41 }
42
__sysreg_save_state(struct kvm_cpu_context * ctxt)43 static void __hyp_text __sysreg_save_state(struct kvm_cpu_context *ctxt)
44 {
45 ctxt->sys_regs[MPIDR_EL1] = read_sysreg(vmpidr_el2);
46 ctxt->sys_regs[CSSELR_EL1] = read_sysreg(csselr_el1);
47 ctxt->sys_regs[SCTLR_EL1] = read_sysreg_el1(sctlr);
48 ctxt->sys_regs[CPACR_EL1] = read_sysreg_el1(cpacr);
49 ctxt->sys_regs[TTBR0_EL1] = read_sysreg_el1(ttbr0);
50 ctxt->sys_regs[TTBR1_EL1] = read_sysreg_el1(ttbr1);
51 ctxt->sys_regs[TCR_EL1] = read_sysreg_el1(tcr);
52 ctxt->sys_regs[ESR_EL1] = read_sysreg_el1(esr);
53 ctxt->sys_regs[AFSR0_EL1] = read_sysreg_el1(afsr0);
54 ctxt->sys_regs[AFSR1_EL1] = read_sysreg_el1(afsr1);
55 ctxt->sys_regs[FAR_EL1] = read_sysreg_el1(far);
56 ctxt->sys_regs[MAIR_EL1] = read_sysreg_el1(mair);
57 ctxt->sys_regs[VBAR_EL1] = read_sysreg_el1(vbar);
58 ctxt->sys_regs[CONTEXTIDR_EL1] = read_sysreg_el1(contextidr);
59 ctxt->sys_regs[AMAIR_EL1] = read_sysreg_el1(amair);
60 ctxt->sys_regs[CNTKCTL_EL1] = read_sysreg_el1(cntkctl);
61 ctxt->sys_regs[PAR_EL1] = read_sysreg(par_el1);
62 ctxt->sys_regs[TPIDR_EL1] = read_sysreg(tpidr_el1);
63
64 ctxt->gp_regs.sp_el1 = read_sysreg(sp_el1);
65 ctxt->gp_regs.elr_el1 = read_sysreg_el1(elr);
66 ctxt->gp_regs.spsr[KVM_SPSR_EL1]= read_sysreg_el1(spsr);
67 ctxt->gp_regs.regs.pc = read_sysreg_el2(elr);
68 ctxt->gp_regs.regs.pstate = read_sysreg_el2(spsr);
69 }
70
71 static hyp_alternate_select(__sysreg_call_save_host_state,
72 __sysreg_save_state, __sysreg_do_nothing,
73 ARM64_HAS_VIRT_HOST_EXTN);
74
__sysreg_save_host_state(struct kvm_cpu_context * ctxt)75 void __hyp_text __sysreg_save_host_state(struct kvm_cpu_context *ctxt)
76 {
77 __sysreg_call_save_host_state()(ctxt);
78 __sysreg_save_common_state(ctxt);
79 }
80
__sysreg_save_guest_state(struct kvm_cpu_context * ctxt)81 void __hyp_text __sysreg_save_guest_state(struct kvm_cpu_context *ctxt)
82 {
83 __sysreg_save_state(ctxt);
84 __sysreg_save_common_state(ctxt);
85 }
86
__sysreg_restore_common_state(struct kvm_cpu_context * ctxt)87 static void __hyp_text __sysreg_restore_common_state(struct kvm_cpu_context *ctxt)
88 {
89 write_sysreg(ctxt->sys_regs[ACTLR_EL1], actlr_el1);
90 write_sysreg(ctxt->sys_regs[TPIDR_EL0], tpidr_el0);
91 write_sysreg(ctxt->sys_regs[TPIDRRO_EL0], tpidrro_el0);
92 write_sysreg(ctxt->sys_regs[MDSCR_EL1], mdscr_el1);
93 write_sysreg(ctxt->gp_regs.regs.sp, sp_el0);
94 }
95
__sysreg_restore_state(struct kvm_cpu_context * ctxt)96 static void __hyp_text __sysreg_restore_state(struct kvm_cpu_context *ctxt)
97 {
98 write_sysreg(ctxt->sys_regs[MPIDR_EL1], vmpidr_el2);
99 write_sysreg(ctxt->sys_regs[CSSELR_EL1], csselr_el1);
100 write_sysreg_el1(ctxt->sys_regs[SCTLR_EL1], sctlr);
101 write_sysreg_el1(ctxt->sys_regs[CPACR_EL1], cpacr);
102 write_sysreg_el1(ctxt->sys_regs[TTBR0_EL1], ttbr0);
103 write_sysreg_el1(ctxt->sys_regs[TTBR1_EL1], ttbr1);
104 write_sysreg_el1(ctxt->sys_regs[TCR_EL1], tcr);
105 write_sysreg_el1(ctxt->sys_regs[ESR_EL1], esr);
106 write_sysreg_el1(ctxt->sys_regs[AFSR0_EL1], afsr0);
107 write_sysreg_el1(ctxt->sys_regs[AFSR1_EL1], afsr1);
108 write_sysreg_el1(ctxt->sys_regs[FAR_EL1], far);
109 write_sysreg_el1(ctxt->sys_regs[MAIR_EL1], mair);
110 write_sysreg_el1(ctxt->sys_regs[VBAR_EL1], vbar);
111 write_sysreg_el1(ctxt->sys_regs[CONTEXTIDR_EL1],contextidr);
112 write_sysreg_el1(ctxt->sys_regs[AMAIR_EL1], amair);
113 write_sysreg_el1(ctxt->sys_regs[CNTKCTL_EL1], cntkctl);
114 write_sysreg(ctxt->sys_regs[PAR_EL1], par_el1);
115 write_sysreg(ctxt->sys_regs[TPIDR_EL1], tpidr_el1);
116
117 write_sysreg(ctxt->gp_regs.sp_el1, sp_el1);
118 write_sysreg_el1(ctxt->gp_regs.elr_el1, elr);
119 write_sysreg_el1(ctxt->gp_regs.spsr[KVM_SPSR_EL1],spsr);
120 write_sysreg_el2(ctxt->gp_regs.regs.pc, elr);
121 write_sysreg_el2(ctxt->gp_regs.regs.pstate, spsr);
122 }
123
124 static hyp_alternate_select(__sysreg_call_restore_host_state,
125 __sysreg_restore_state, __sysreg_do_nothing,
126 ARM64_HAS_VIRT_HOST_EXTN);
127
__sysreg_restore_host_state(struct kvm_cpu_context * ctxt)128 void __hyp_text __sysreg_restore_host_state(struct kvm_cpu_context *ctxt)
129 {
130 __sysreg_call_restore_host_state()(ctxt);
131 __sysreg_restore_common_state(ctxt);
132 }
133
__sysreg_restore_guest_state(struct kvm_cpu_context * ctxt)134 void __hyp_text __sysreg_restore_guest_state(struct kvm_cpu_context *ctxt)
135 {
136 __sysreg_restore_state(ctxt);
137 __sysreg_restore_common_state(ctxt);
138 }
139
__sysreg32_save_state(struct kvm_vcpu * vcpu)140 void __hyp_text __sysreg32_save_state(struct kvm_vcpu *vcpu)
141 {
142 u64 *spsr, *sysreg;
143
144 if (read_sysreg(hcr_el2) & HCR_RW)
145 return;
146
147 spsr = vcpu->arch.ctxt.gp_regs.spsr;
148 sysreg = vcpu->arch.ctxt.sys_regs;
149
150 spsr[KVM_SPSR_ABT] = read_sysreg(spsr_abt);
151 spsr[KVM_SPSR_UND] = read_sysreg(spsr_und);
152 spsr[KVM_SPSR_IRQ] = read_sysreg(spsr_irq);
153 spsr[KVM_SPSR_FIQ] = read_sysreg(spsr_fiq);
154
155 sysreg[DACR32_EL2] = read_sysreg(dacr32_el2);
156 sysreg[IFSR32_EL2] = read_sysreg(ifsr32_el2);
157
158 if (__fpsimd_enabled())
159 sysreg[FPEXC32_EL2] = read_sysreg(fpexc32_el2);
160
161 if (vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
162 sysreg[DBGVCR32_EL2] = read_sysreg(dbgvcr32_el2);
163 }
164
__sysreg32_restore_state(struct kvm_vcpu * vcpu)165 void __hyp_text __sysreg32_restore_state(struct kvm_vcpu *vcpu)
166 {
167 u64 *spsr, *sysreg;
168
169 if (read_sysreg(hcr_el2) & HCR_RW)
170 return;
171
172 spsr = vcpu->arch.ctxt.gp_regs.spsr;
173 sysreg = vcpu->arch.ctxt.sys_regs;
174
175 write_sysreg(spsr[KVM_SPSR_ABT], spsr_abt);
176 write_sysreg(spsr[KVM_SPSR_UND], spsr_und);
177 write_sysreg(spsr[KVM_SPSR_IRQ], spsr_irq);
178 write_sysreg(spsr[KVM_SPSR_FIQ], spsr_fiq);
179
180 write_sysreg(sysreg[DACR32_EL2], dacr32_el2);
181 write_sysreg(sysreg[IFSR32_EL2], ifsr32_el2);
182
183 if (vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
184 write_sysreg(sysreg[DBGVCR32_EL2], dbgvcr32_el2);
185 }
186
__kvm_set_tpidr_el2(u64 tpidr_el2)187 void __hyp_text __kvm_set_tpidr_el2(u64 tpidr_el2)
188 {
189 asm("msr tpidr_el2, %0": : "r" (tpidr_el2));
190 }
191
__kvm_enable_ssbs(void)192 void __hyp_text __kvm_enable_ssbs(void)
193 {
194 u64 tmp;
195
196 asm volatile(
197 "mrs %0, sctlr_el2\n"
198 "orr %0, %0, %1\n"
199 "msr sctlr_el2, %0"
200 : "=&r" (tmp) : "L" (SCTLR_ELx_DSSBS));
201 }
202