1/* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License, version 2, as 4 * published by the Free Software Foundation. 5 * 6 * This program is distributed in the hope that it will be useful, 7 * but WITHOUT ANY WARRANTY; without even the implied warranty of 8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 * GNU General Public License for more details. 10 * 11 * You should have received a copy of the GNU General Public License 12 * along with this program; if not, write to the Free Software 13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 14 * 15 * Copyright IBM Corp. 2007 16 * 17 * Authors: Hollis Blanchard <hollisb@us.ibm.com> 18 */ 19 20#include <asm/ppc_asm.h> 21#include <asm/kvm_asm.h> 22#include <asm/reg.h> 23#include <asm/mmu-44x.h> 24#include <asm/page.h> 25#include <asm/asm-offsets.h> 26 27#define KVMPPC_MSR_MASK (MSR_CE|MSR_EE|MSR_PR|MSR_DE|MSR_ME|MSR_IS|MSR_DS) 28 29#define VCPU_GPR(n) (VCPU_GPRS + (n * 4)) 30 31/* The host stack layout: */ 32#define HOST_R1 0 /* Implied by stwu. */ 33#define HOST_CALLEE_LR 4 34#define HOST_RUN 8 35/* r2 is special: it holds 'current', and it made nonvolatile in the 36 * kernel with the -ffixed-r2 gcc option. */ 37#define HOST_R2 12 38#define HOST_NV_GPRS 16 39#define HOST_NV_GPR(n) (HOST_NV_GPRS + ((n - 14) * 4)) 40#define HOST_MIN_STACK_SIZE (HOST_NV_GPR(31) + 4) 41#define HOST_STACK_SIZE (((HOST_MIN_STACK_SIZE + 15) / 16) * 16) /* Align. */ 42#define HOST_STACK_LR (HOST_STACK_SIZE + 4) /* In caller stack frame. */ 43 44#define NEED_INST_MASK ((1<<BOOKE_INTERRUPT_PROGRAM) | \ 45 (1<<BOOKE_INTERRUPT_DTLB_MISS) | \ 46 (1<<BOOKE_INTERRUPT_DEBUG)) 47 48#define NEED_DEAR_MASK ((1<<BOOKE_INTERRUPT_DATA_STORAGE) | \ 49 (1<<BOOKE_INTERRUPT_DTLB_MISS)) 50 51#define NEED_ESR_MASK ((1<<BOOKE_INTERRUPT_DATA_STORAGE) | \ 52 (1<<BOOKE_INTERRUPT_INST_STORAGE) | \ 53 (1<<BOOKE_INTERRUPT_PROGRAM) | \ 54 (1<<BOOKE_INTERRUPT_DTLB_MISS)) 55 56.macro KVM_HANDLER ivor_nr 57_GLOBAL(kvmppc_handler_\ivor_nr) 58 /* Get pointer to vcpu and record exit number. */ 59 mtspr SPRN_SPRG0, r4 60 mfspr r4, SPRN_SPRG1 61 stw r5, VCPU_GPR(r5)(r4) 62 stw r6, VCPU_GPR(r6)(r4) 63 mfctr r5 64 lis r6, kvmppc_resume_host@h 65 stw r5, VCPU_CTR(r4) 66 li r5, \ivor_nr 67 ori r6, r6, kvmppc_resume_host@l 68 mtctr r6 69 bctr 70.endm 71 72_GLOBAL(kvmppc_handlers_start) 73KVM_HANDLER BOOKE_INTERRUPT_CRITICAL 74KVM_HANDLER BOOKE_INTERRUPT_MACHINE_CHECK 75KVM_HANDLER BOOKE_INTERRUPT_DATA_STORAGE 76KVM_HANDLER BOOKE_INTERRUPT_INST_STORAGE 77KVM_HANDLER BOOKE_INTERRUPT_EXTERNAL 78KVM_HANDLER BOOKE_INTERRUPT_ALIGNMENT 79KVM_HANDLER BOOKE_INTERRUPT_PROGRAM 80KVM_HANDLER BOOKE_INTERRUPT_FP_UNAVAIL 81KVM_HANDLER BOOKE_INTERRUPT_SYSCALL 82KVM_HANDLER BOOKE_INTERRUPT_AP_UNAVAIL 83KVM_HANDLER BOOKE_INTERRUPT_DECREMENTER 84KVM_HANDLER BOOKE_INTERRUPT_FIT 85KVM_HANDLER BOOKE_INTERRUPT_WATCHDOG 86KVM_HANDLER BOOKE_INTERRUPT_DTLB_MISS 87KVM_HANDLER BOOKE_INTERRUPT_ITLB_MISS 88KVM_HANDLER BOOKE_INTERRUPT_DEBUG 89 90_GLOBAL(kvmppc_handler_len) 91 .long kvmppc_handler_1 - kvmppc_handler_0 92 93 94/* Registers: 95 * SPRG0: guest r4 96 * r4: vcpu pointer 97 * r5: KVM exit number 98 */ 99_GLOBAL(kvmppc_resume_host) 100 stw r3, VCPU_GPR(r3)(r4) 101 mfcr r3 102 stw r3, VCPU_CR(r4) 103 stw r7, VCPU_GPR(r7)(r4) 104 stw r8, VCPU_GPR(r8)(r4) 105 stw r9, VCPU_GPR(r9)(r4) 106 107 li r6, 1 108 slw r6, r6, r5 109 110#ifdef CONFIG_KVM_EXIT_TIMING 111 /* save exit time */ 1121: 113 mfspr r7, SPRN_TBRU 114 mfspr r8, SPRN_TBRL 115 mfspr r9, SPRN_TBRU 116 cmpw r9, r7 117 bne 1b 118 stw r8, VCPU_TIMING_EXIT_TBL(r4) 119 stw r9, VCPU_TIMING_EXIT_TBU(r4) 120#endif 121 122 /* Save the faulting instruction and all GPRs for emulation. */ 123 andi. r7, r6, NEED_INST_MASK 124 beq ..skip_inst_copy 125 mfspr r9, SPRN_SRR0 126 mfmsr r8 127 ori r7, r8, MSR_DS 128 mtmsr r7 129 isync 130 lwz r9, 0(r9) 131 mtmsr r8 132 isync 133 stw r9, VCPU_LAST_INST(r4) 134 135 stw r15, VCPU_GPR(r15)(r4) 136 stw r16, VCPU_GPR(r16)(r4) 137 stw r17, VCPU_GPR(r17)(r4) 138 stw r18, VCPU_GPR(r18)(r4) 139 stw r19, VCPU_GPR(r19)(r4) 140 stw r20, VCPU_GPR(r20)(r4) 141 stw r21, VCPU_GPR(r21)(r4) 142 stw r22, VCPU_GPR(r22)(r4) 143 stw r23, VCPU_GPR(r23)(r4) 144 stw r24, VCPU_GPR(r24)(r4) 145 stw r25, VCPU_GPR(r25)(r4) 146 stw r26, VCPU_GPR(r26)(r4) 147 stw r27, VCPU_GPR(r27)(r4) 148 stw r28, VCPU_GPR(r28)(r4) 149 stw r29, VCPU_GPR(r29)(r4) 150 stw r30, VCPU_GPR(r30)(r4) 151 stw r31, VCPU_GPR(r31)(r4) 152..skip_inst_copy: 153 154 /* Also grab DEAR and ESR before the host can clobber them. */ 155 156 andi. r7, r6, NEED_DEAR_MASK 157 beq ..skip_dear 158 mfspr r9, SPRN_DEAR 159 stw r9, VCPU_FAULT_DEAR(r4) 160..skip_dear: 161 162 andi. r7, r6, NEED_ESR_MASK 163 beq ..skip_esr 164 mfspr r9, SPRN_ESR 165 stw r9, VCPU_FAULT_ESR(r4) 166..skip_esr: 167 168 /* Save remaining volatile guest register state to vcpu. */ 169 stw r0, VCPU_GPR(r0)(r4) 170 stw r1, VCPU_GPR(r1)(r4) 171 stw r2, VCPU_GPR(r2)(r4) 172 stw r10, VCPU_GPR(r10)(r4) 173 stw r11, VCPU_GPR(r11)(r4) 174 stw r12, VCPU_GPR(r12)(r4) 175 stw r13, VCPU_GPR(r13)(r4) 176 stw r14, VCPU_GPR(r14)(r4) /* We need a NV GPR below. */ 177 mflr r3 178 stw r3, VCPU_LR(r4) 179 mfxer r3 180 stw r3, VCPU_XER(r4) 181 mfspr r3, SPRN_SPRG0 182 stw r3, VCPU_GPR(r4)(r4) 183 mfspr r3, SPRN_SRR0 184 stw r3, VCPU_PC(r4) 185 186 /* Restore host stack pointer and PID before IVPR, since the host 187 * exception handlers use them. */ 188 lwz r1, VCPU_HOST_STACK(r4) 189 lwz r3, VCPU_HOST_PID(r4) 190 mtspr SPRN_PID, r3 191 192 /* Restore host IVPR before re-enabling interrupts. We cheat and know 193 * that Linux IVPR is always 0xc0000000. */ 194 lis r3, 0xc000 195 mtspr SPRN_IVPR, r3 196 197 /* Switch to kernel stack and jump to handler. */ 198 LOAD_REG_ADDR(r3, kvmppc_handle_exit) 199 mtctr r3 200 lwz r3, HOST_RUN(r1) 201 lwz r2, HOST_R2(r1) 202 mr r14, r4 /* Save vcpu pointer. */ 203 204 bctrl /* kvmppc_handle_exit() */ 205 206 /* Restore vcpu pointer and the nonvolatiles we used. */ 207 mr r4, r14 208 lwz r14, VCPU_GPR(r14)(r4) 209 210 /* Sometimes instruction emulation must restore complete GPR state. */ 211 andi. r5, r3, RESUME_FLAG_NV 212 beq ..skip_nv_load 213 lwz r15, VCPU_GPR(r15)(r4) 214 lwz r16, VCPU_GPR(r16)(r4) 215 lwz r17, VCPU_GPR(r17)(r4) 216 lwz r18, VCPU_GPR(r18)(r4) 217 lwz r19, VCPU_GPR(r19)(r4) 218 lwz r20, VCPU_GPR(r20)(r4) 219 lwz r21, VCPU_GPR(r21)(r4) 220 lwz r22, VCPU_GPR(r22)(r4) 221 lwz r23, VCPU_GPR(r23)(r4) 222 lwz r24, VCPU_GPR(r24)(r4) 223 lwz r25, VCPU_GPR(r25)(r4) 224 lwz r26, VCPU_GPR(r26)(r4) 225 lwz r27, VCPU_GPR(r27)(r4) 226 lwz r28, VCPU_GPR(r28)(r4) 227 lwz r29, VCPU_GPR(r29)(r4) 228 lwz r30, VCPU_GPR(r30)(r4) 229 lwz r31, VCPU_GPR(r31)(r4) 230..skip_nv_load: 231 232 /* Should we return to the guest? */ 233 andi. r5, r3, RESUME_FLAG_HOST 234 beq lightweight_exit 235 236 srawi r3, r3, 2 /* Shift -ERR back down. */ 237 238heavyweight_exit: 239 /* Not returning to guest. */ 240 241 /* We already saved guest volatile register state; now save the 242 * non-volatiles. */ 243 stw r15, VCPU_GPR(r15)(r4) 244 stw r16, VCPU_GPR(r16)(r4) 245 stw r17, VCPU_GPR(r17)(r4) 246 stw r18, VCPU_GPR(r18)(r4) 247 stw r19, VCPU_GPR(r19)(r4) 248 stw r20, VCPU_GPR(r20)(r4) 249 stw r21, VCPU_GPR(r21)(r4) 250 stw r22, VCPU_GPR(r22)(r4) 251 stw r23, VCPU_GPR(r23)(r4) 252 stw r24, VCPU_GPR(r24)(r4) 253 stw r25, VCPU_GPR(r25)(r4) 254 stw r26, VCPU_GPR(r26)(r4) 255 stw r27, VCPU_GPR(r27)(r4) 256 stw r28, VCPU_GPR(r28)(r4) 257 stw r29, VCPU_GPR(r29)(r4) 258 stw r30, VCPU_GPR(r30)(r4) 259 stw r31, VCPU_GPR(r31)(r4) 260 261 /* Load host non-volatile register state from host stack. */ 262 lwz r14, HOST_NV_GPR(r14)(r1) 263 lwz r15, HOST_NV_GPR(r15)(r1) 264 lwz r16, HOST_NV_GPR(r16)(r1) 265 lwz r17, HOST_NV_GPR(r17)(r1) 266 lwz r18, HOST_NV_GPR(r18)(r1) 267 lwz r19, HOST_NV_GPR(r19)(r1) 268 lwz r20, HOST_NV_GPR(r20)(r1) 269 lwz r21, HOST_NV_GPR(r21)(r1) 270 lwz r22, HOST_NV_GPR(r22)(r1) 271 lwz r23, HOST_NV_GPR(r23)(r1) 272 lwz r24, HOST_NV_GPR(r24)(r1) 273 lwz r25, HOST_NV_GPR(r25)(r1) 274 lwz r26, HOST_NV_GPR(r26)(r1) 275 lwz r27, HOST_NV_GPR(r27)(r1) 276 lwz r28, HOST_NV_GPR(r28)(r1) 277 lwz r29, HOST_NV_GPR(r29)(r1) 278 lwz r30, HOST_NV_GPR(r30)(r1) 279 lwz r31, HOST_NV_GPR(r31)(r1) 280 281 /* Return to kvm_vcpu_run(). */ 282 lwz r4, HOST_STACK_LR(r1) 283 addi r1, r1, HOST_STACK_SIZE 284 mtlr r4 285 /* r3 still contains the return code from kvmppc_handle_exit(). */ 286 blr 287 288 289/* Registers: 290 * r3: kvm_run pointer 291 * r4: vcpu pointer 292 */ 293_GLOBAL(__kvmppc_vcpu_run) 294 stwu r1, -HOST_STACK_SIZE(r1) 295 stw r1, VCPU_HOST_STACK(r4) /* Save stack pointer to vcpu. */ 296 297 /* Save host state to stack. */ 298 stw r3, HOST_RUN(r1) 299 mflr r3 300 stw r3, HOST_STACK_LR(r1) 301 302 /* Save host non-volatile register state to stack. */ 303 stw r14, HOST_NV_GPR(r14)(r1) 304 stw r15, HOST_NV_GPR(r15)(r1) 305 stw r16, HOST_NV_GPR(r16)(r1) 306 stw r17, HOST_NV_GPR(r17)(r1) 307 stw r18, HOST_NV_GPR(r18)(r1) 308 stw r19, HOST_NV_GPR(r19)(r1) 309 stw r20, HOST_NV_GPR(r20)(r1) 310 stw r21, HOST_NV_GPR(r21)(r1) 311 stw r22, HOST_NV_GPR(r22)(r1) 312 stw r23, HOST_NV_GPR(r23)(r1) 313 stw r24, HOST_NV_GPR(r24)(r1) 314 stw r25, HOST_NV_GPR(r25)(r1) 315 stw r26, HOST_NV_GPR(r26)(r1) 316 stw r27, HOST_NV_GPR(r27)(r1) 317 stw r28, HOST_NV_GPR(r28)(r1) 318 stw r29, HOST_NV_GPR(r29)(r1) 319 stw r30, HOST_NV_GPR(r30)(r1) 320 stw r31, HOST_NV_GPR(r31)(r1) 321 322 /* Load guest non-volatiles. */ 323 lwz r14, VCPU_GPR(r14)(r4) 324 lwz r15, VCPU_GPR(r15)(r4) 325 lwz r16, VCPU_GPR(r16)(r4) 326 lwz r17, VCPU_GPR(r17)(r4) 327 lwz r18, VCPU_GPR(r18)(r4) 328 lwz r19, VCPU_GPR(r19)(r4) 329 lwz r20, VCPU_GPR(r20)(r4) 330 lwz r21, VCPU_GPR(r21)(r4) 331 lwz r22, VCPU_GPR(r22)(r4) 332 lwz r23, VCPU_GPR(r23)(r4) 333 lwz r24, VCPU_GPR(r24)(r4) 334 lwz r25, VCPU_GPR(r25)(r4) 335 lwz r26, VCPU_GPR(r26)(r4) 336 lwz r27, VCPU_GPR(r27)(r4) 337 lwz r28, VCPU_GPR(r28)(r4) 338 lwz r29, VCPU_GPR(r29)(r4) 339 lwz r30, VCPU_GPR(r30)(r4) 340 lwz r31, VCPU_GPR(r31)(r4) 341 342lightweight_exit: 343 stw r2, HOST_R2(r1) 344 345 mfspr r3, SPRN_PID 346 stw r3, VCPU_HOST_PID(r4) 347 lwz r3, VCPU_SHADOW_PID(r4) 348 mtspr SPRN_PID, r3 349 350 iccci 0, 0 /* XXX hack */ 351 352 /* Load some guest volatiles. */ 353 lwz r0, VCPU_GPR(r0)(r4) 354 lwz r2, VCPU_GPR(r2)(r4) 355 lwz r9, VCPU_GPR(r9)(r4) 356 lwz r10, VCPU_GPR(r10)(r4) 357 lwz r11, VCPU_GPR(r11)(r4) 358 lwz r12, VCPU_GPR(r12)(r4) 359 lwz r13, VCPU_GPR(r13)(r4) 360 lwz r3, VCPU_LR(r4) 361 mtlr r3 362 lwz r3, VCPU_XER(r4) 363 mtxer r3 364 365 /* Switch the IVPR. XXX If we take a TLB miss after this we're screwed, 366 * so how do we make sure vcpu won't fault? */ 367 lis r8, kvmppc_booke_handlers@ha 368 lwz r8, kvmppc_booke_handlers@l(r8) 369 mtspr SPRN_IVPR, r8 370 371 /* Save vcpu pointer for the exception handlers. */ 372 mtspr SPRN_SPRG1, r4 373 374 /* Can't switch the stack pointer until after IVPR is switched, 375 * because host interrupt handlers would get confused. */ 376 lwz r1, VCPU_GPR(r1)(r4) 377 378 /* XXX handle USPRG0 */ 379 /* Host interrupt handlers may have clobbered these guest-readable 380 * SPRGs, so we need to reload them here with the guest's values. */ 381 lwz r3, VCPU_SPRG4(r4) 382 mtspr SPRN_SPRG4, r3 383 lwz r3, VCPU_SPRG5(r4) 384 mtspr SPRN_SPRG5, r3 385 lwz r3, VCPU_SPRG6(r4) 386 mtspr SPRN_SPRG6, r3 387 lwz r3, VCPU_SPRG7(r4) 388 mtspr SPRN_SPRG7, r3 389 390#ifdef CONFIG_KVM_EXIT_TIMING 391 /* save enter time */ 3921: 393 mfspr r6, SPRN_TBRU 394 mfspr r7, SPRN_TBRL 395 mfspr r8, SPRN_TBRU 396 cmpw r8, r6 397 bne 1b 398 stw r7, VCPU_TIMING_LAST_ENTER_TBL(r4) 399 stw r8, VCPU_TIMING_LAST_ENTER_TBU(r4) 400#endif 401 402 /* Finish loading guest volatiles and jump to guest. */ 403 lwz r3, VCPU_CTR(r4) 404 mtctr r3 405 lwz r3, VCPU_CR(r4) 406 mtcr r3 407 lwz r5, VCPU_GPR(r5)(r4) 408 lwz r6, VCPU_GPR(r6)(r4) 409 lwz r7, VCPU_GPR(r7)(r4) 410 lwz r8, VCPU_GPR(r8)(r4) 411 lwz r3, VCPU_PC(r4) 412 mtsrr0 r3 413 lwz r3, VCPU_MSR(r4) 414 oris r3, r3, KVMPPC_MSR_MASK@h 415 ori r3, r3, KVMPPC_MSR_MASK@l 416 mtsrr1 r3 417 418 /* Clear any debug events which occurred since we disabled MSR[DE]. 419 * XXX This gives us a 3-instruction window in which a breakpoint 420 * intended for guest context could fire in the host instead. */ 421 lis r3, 0xffff 422 ori r3, r3, 0xffff 423 mtspr SPRN_DBSR, r3 424 425 lwz r3, VCPU_GPR(r3)(r4) 426 lwz r4, VCPU_GPR(r4)(r4) 427 rfi 428