1/* 2 * ultra.S: Don't expand these all over the place... 3 * 4 * Copyright (C) 1997, 2000, 2008 David S. Miller (davem@davemloft.net) 5 */ 6 7#include <asm/asi.h> 8#include <asm/pgtable.h> 9#include <asm/page.h> 10#include <asm/spitfire.h> 11#include <asm/mmu_context.h> 12#include <asm/mmu.h> 13#include <asm/pil.h> 14#include <asm/head.h> 15#include <asm/thread_info.h> 16#include <asm/cacheflush.h> 17#include <asm/hypervisor.h> 18#include <asm/cpudata.h> 19 20 /* Basically, most of the Spitfire vs. Cheetah madness 21 * has to do with the fact that Cheetah does not support 22 * IMMU flushes out of the secondary context. Someone needs 23 * to throw a south lake birthday party for the folks 24 * in Microelectronics who refused to fix this shit. 25 */ 26 27 /* This file is meant to be read efficiently by the CPU, not humans. 28 * Staraj sie tego nikomu nie pierdolnac... 29 */ 30 .text 31 .align 32 32 .globl __flush_tlb_mm 33__flush_tlb_mm: /* 18 insns */ 34 /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ 35 ldxa [%o1] ASI_DMMU, %g2 36 cmp %g2, %o0 37 bne,pn %icc, __spitfire_flush_tlb_mm_slow 38 mov 0x50, %g3 39 stxa %g0, [%g3] ASI_DMMU_DEMAP 40 stxa %g0, [%g3] ASI_IMMU_DEMAP 41 sethi %hi(KERNBASE), %g3 42 flush %g3 43 retl 44 nop 45 nop 46 nop 47 nop 48 nop 49 nop 50 nop 51 nop 52 nop 53 nop 54 55 .align 32 56 .globl __flush_tlb_pending 57__flush_tlb_pending: /* 26 insns */ 58 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ 59 rdpr %pstate, %g7 60 sllx %o1, 3, %o1 61 andn %g7, PSTATE_IE, %g2 62 wrpr %g2, %pstate 63 mov SECONDARY_CONTEXT, %o4 64 ldxa [%o4] ASI_DMMU, %g2 65 stxa %o0, [%o4] ASI_DMMU 661: sub %o1, (1 << 3), %o1 67 ldx [%o2 + %o1], %o3 68 andcc %o3, 1, %g0 69 andn %o3, 1, %o3 70 be,pn %icc, 2f 71 or %o3, 0x10, %o3 72 stxa %g0, [%o3] ASI_IMMU_DEMAP 732: stxa %g0, [%o3] ASI_DMMU_DEMAP 74 membar #Sync 75 brnz,pt %o1, 1b 76 nop 77 stxa %g2, [%o4] ASI_DMMU 78 sethi %hi(KERNBASE), %o4 79 flush %o4 80 retl 81 wrpr %g7, 0x0, %pstate 82 nop 83 nop 84 nop 85 nop 86 87 .align 32 88 .globl __flush_tlb_kernel_range 89__flush_tlb_kernel_range: /* 16 insns */ 90 /* %o0=start, %o1=end */ 91 cmp %o0, %o1 92 be,pn %xcc, 2f 93 sethi %hi(PAGE_SIZE), %o4 94 sub %o1, %o0, %o3 95 sub %o3, %o4, %o3 96 or %o0, 0x20, %o0 ! Nucleus 971: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP 98 stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP 99 membar #Sync 100 brnz,pt %o3, 1b 101 sub %o3, %o4, %o3 1022: sethi %hi(KERNBASE), %o3 103 flush %o3 104 retl 105 nop 106 nop 107 108__spitfire_flush_tlb_mm_slow: 109 rdpr %pstate, %g1 110 wrpr %g1, PSTATE_IE, %pstate 111 stxa %o0, [%o1] ASI_DMMU 112 stxa %g0, [%g3] ASI_DMMU_DEMAP 113 stxa %g0, [%g3] ASI_IMMU_DEMAP 114 flush %g6 115 stxa %g2, [%o1] ASI_DMMU 116 sethi %hi(KERNBASE), %o1 117 flush %o1 118 retl 119 wrpr %g1, 0, %pstate 120 121/* 122 * The following code flushes one page_size worth. 123 */ 124 .section .kprobes.text, "ax" 125 .align 32 126 .globl __flush_icache_page 127__flush_icache_page: /* %o0 = phys_page */ 128 srlx %o0, PAGE_SHIFT, %o0 129 sethi %uhi(PAGE_OFFSET), %g1 130 sllx %o0, PAGE_SHIFT, %o0 131 sethi %hi(PAGE_SIZE), %g2 132 sllx %g1, 32, %g1 133 add %o0, %g1, %o0 1341: subcc %g2, 32, %g2 135 bne,pt %icc, 1b 136 flush %o0 + %g2 137 retl 138 nop 139 140#ifdef DCACHE_ALIASING_POSSIBLE 141 142#if (PAGE_SHIFT != 13) 143#error only page shift of 13 is supported by dcache flush 144#endif 145 146#define DTAG_MASK 0x3 147 148 /* This routine is Spitfire specific so the hardcoded 149 * D-cache size and line-size are OK. 150 */ 151 .align 64 152 .globl __flush_dcache_page 153__flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */ 154 sethi %uhi(PAGE_OFFSET), %g1 155 sllx %g1, 32, %g1 156 sub %o0, %g1, %o0 ! physical address 157 srlx %o0, 11, %o0 ! make D-cache TAG 158 sethi %hi(1 << 14), %o2 ! D-cache size 159 sub %o2, (1 << 5), %o2 ! D-cache line size 1601: ldxa [%o2] ASI_DCACHE_TAG, %o3 ! load D-cache TAG 161 andcc %o3, DTAG_MASK, %g0 ! Valid? 162 be,pn %xcc, 2f ! Nope, branch 163 andn %o3, DTAG_MASK, %o3 ! Clear valid bits 164 cmp %o3, %o0 ! TAG match? 165 bne,pt %xcc, 2f ! Nope, branch 166 nop 167 stxa %g0, [%o2] ASI_DCACHE_TAG ! Invalidate TAG 168 membar #Sync 1692: brnz,pt %o2, 1b 170 sub %o2, (1 << 5), %o2 ! D-cache line size 171 172 /* The I-cache does not snoop local stores so we 173 * better flush that too when necessary. 174 */ 175 brnz,pt %o1, __flush_icache_page 176 sllx %o0, 11, %o0 177 retl 178 nop 179 180#endif /* DCACHE_ALIASING_POSSIBLE */ 181 182 .previous 183 184 /* Cheetah specific versions, patched at boot time. */ 185__cheetah_flush_tlb_mm: /* 19 insns */ 186 rdpr %pstate, %g7 187 andn %g7, PSTATE_IE, %g2 188 wrpr %g2, 0x0, %pstate 189 wrpr %g0, 1, %tl 190 mov PRIMARY_CONTEXT, %o2 191 mov 0x40, %g3 192 ldxa [%o2] ASI_DMMU, %g2 193 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o1 194 sllx %o1, CTX_PGSZ1_NUC_SHIFT, %o1 195 or %o0, %o1, %o0 /* Preserve nucleus page size fields */ 196 stxa %o0, [%o2] ASI_DMMU 197 stxa %g0, [%g3] ASI_DMMU_DEMAP 198 stxa %g0, [%g3] ASI_IMMU_DEMAP 199 stxa %g2, [%o2] ASI_DMMU 200 sethi %hi(KERNBASE), %o2 201 flush %o2 202 wrpr %g0, 0, %tl 203 retl 204 wrpr %g7, 0x0, %pstate 205 206__cheetah_flush_tlb_pending: /* 27 insns */ 207 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ 208 rdpr %pstate, %g7 209 sllx %o1, 3, %o1 210 andn %g7, PSTATE_IE, %g2 211 wrpr %g2, 0x0, %pstate 212 wrpr %g0, 1, %tl 213 mov PRIMARY_CONTEXT, %o4 214 ldxa [%o4] ASI_DMMU, %g2 215 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 216 sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 217 or %o0, %o3, %o0 /* Preserve nucleus page size fields */ 218 stxa %o0, [%o4] ASI_DMMU 2191: sub %o1, (1 << 3), %o1 220 ldx [%o2 + %o1], %o3 221 andcc %o3, 1, %g0 222 be,pn %icc, 2f 223 andn %o3, 1, %o3 224 stxa %g0, [%o3] ASI_IMMU_DEMAP 2252: stxa %g0, [%o3] ASI_DMMU_DEMAP 226 membar #Sync 227 brnz,pt %o1, 1b 228 nop 229 stxa %g2, [%o4] ASI_DMMU 230 sethi %hi(KERNBASE), %o4 231 flush %o4 232 wrpr %g0, 0, %tl 233 retl 234 wrpr %g7, 0x0, %pstate 235 236#ifdef DCACHE_ALIASING_POSSIBLE 237__cheetah_flush_dcache_page: /* 11 insns */ 238 sethi %uhi(PAGE_OFFSET), %g1 239 sllx %g1, 32, %g1 240 sub %o0, %g1, %o0 241 sethi %hi(PAGE_SIZE), %o4 2421: subcc %o4, (1 << 5), %o4 243 stxa %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE 244 membar #Sync 245 bne,pt %icc, 1b 246 nop 247 retl /* I-cache flush never needed on Cheetah, see callers. */ 248 nop 249#endif /* DCACHE_ALIASING_POSSIBLE */ 250 251 /* Hypervisor specific versions, patched at boot time. */ 252__hypervisor_tlb_tl0_error: 253 save %sp, -192, %sp 254 mov %i0, %o0 255 call hypervisor_tlbop_error 256 mov %i1, %o1 257 ret 258 restore 259 260__hypervisor_flush_tlb_mm: /* 10 insns */ 261 mov %o0, %o2 /* ARG2: mmu context */ 262 mov 0, %o0 /* ARG0: CPU lists unimplemented */ 263 mov 0, %o1 /* ARG1: CPU lists unimplemented */ 264 mov HV_MMU_ALL, %o3 /* ARG3: flags */ 265 mov HV_FAST_MMU_DEMAP_CTX, %o5 266 ta HV_FAST_TRAP 267 brnz,pn %o0, __hypervisor_tlb_tl0_error 268 mov HV_FAST_MMU_DEMAP_CTX, %o1 269 retl 270 nop 271 272__hypervisor_flush_tlb_pending: /* 16 insns */ 273 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ 274 sllx %o1, 3, %g1 275 mov %o2, %g2 276 mov %o0, %g3 2771: sub %g1, (1 << 3), %g1 278 ldx [%g2 + %g1], %o0 /* ARG0: vaddr + IMMU-bit */ 279 mov %g3, %o1 /* ARG1: mmu context */ 280 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 281 srlx %o0, PAGE_SHIFT, %o0 282 sllx %o0, PAGE_SHIFT, %o0 283 ta HV_MMU_UNMAP_ADDR_TRAP 284 brnz,pn %o0, __hypervisor_tlb_tl0_error 285 mov HV_MMU_UNMAP_ADDR_TRAP, %o1 286 brnz,pt %g1, 1b 287 nop 288 retl 289 nop 290 291__hypervisor_flush_tlb_kernel_range: /* 16 insns */ 292 /* %o0=start, %o1=end */ 293 cmp %o0, %o1 294 be,pn %xcc, 2f 295 sethi %hi(PAGE_SIZE), %g3 296 mov %o0, %g1 297 sub %o1, %g1, %g2 298 sub %g2, %g3, %g2 2991: add %g1, %g2, %o0 /* ARG0: virtual address */ 300 mov 0, %o1 /* ARG1: mmu context */ 301 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 302 ta HV_MMU_UNMAP_ADDR_TRAP 303 brnz,pn %o0, __hypervisor_tlb_tl0_error 304 mov HV_MMU_UNMAP_ADDR_TRAP, %o1 305 brnz,pt %g2, 1b 306 sub %g2, %g3, %g2 3072: retl 308 nop 309 310#ifdef DCACHE_ALIASING_POSSIBLE 311 /* XXX Niagara and friends have an 8K cache, so no aliasing is 312 * XXX possible, but nothing explicit in the Hypervisor API 313 * XXX guarantees this. 314 */ 315__hypervisor_flush_dcache_page: /* 2 insns */ 316 retl 317 nop 318#endif 319 320tlb_patch_one: 3211: lduw [%o1], %g1 322 stw %g1, [%o0] 323 flush %o0 324 subcc %o2, 1, %o2 325 add %o1, 4, %o1 326 bne,pt %icc, 1b 327 add %o0, 4, %o0 328 retl 329 nop 330 331 .globl cheetah_patch_cachetlbops 332cheetah_patch_cachetlbops: 333 save %sp, -128, %sp 334 335 sethi %hi(__flush_tlb_mm), %o0 336 or %o0, %lo(__flush_tlb_mm), %o0 337 sethi %hi(__cheetah_flush_tlb_mm), %o1 338 or %o1, %lo(__cheetah_flush_tlb_mm), %o1 339 call tlb_patch_one 340 mov 19, %o2 341 342 sethi %hi(__flush_tlb_pending), %o0 343 or %o0, %lo(__flush_tlb_pending), %o0 344 sethi %hi(__cheetah_flush_tlb_pending), %o1 345 or %o1, %lo(__cheetah_flush_tlb_pending), %o1 346 call tlb_patch_one 347 mov 27, %o2 348 349#ifdef DCACHE_ALIASING_POSSIBLE 350 sethi %hi(__flush_dcache_page), %o0 351 or %o0, %lo(__flush_dcache_page), %o0 352 sethi %hi(__cheetah_flush_dcache_page), %o1 353 or %o1, %lo(__cheetah_flush_dcache_page), %o1 354 call tlb_patch_one 355 mov 11, %o2 356#endif /* DCACHE_ALIASING_POSSIBLE */ 357 358 ret 359 restore 360 361#ifdef CONFIG_SMP 362 /* These are all called by the slaves of a cross call, at 363 * trap level 1, with interrupts fully disabled. 364 * 365 * Register usage: 366 * %g5 mm->context (all tlb flushes) 367 * %g1 address arg 1 (tlb page and range flushes) 368 * %g7 address arg 2 (tlb range flush only) 369 * 370 * %g6 scratch 1 371 * %g2 scratch 2 372 * %g3 scratch 3 373 * %g4 scratch 4 374 */ 375 .align 32 376 .globl xcall_flush_tlb_mm 377xcall_flush_tlb_mm: /* 21 insns */ 378 mov PRIMARY_CONTEXT, %g2 379 ldxa [%g2] ASI_DMMU, %g3 380 srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4 381 sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 382 or %g5, %g4, %g5 /* Preserve nucleus page size fields */ 383 stxa %g5, [%g2] ASI_DMMU 384 mov 0x40, %g4 385 stxa %g0, [%g4] ASI_DMMU_DEMAP 386 stxa %g0, [%g4] ASI_IMMU_DEMAP 387 stxa %g3, [%g2] ASI_DMMU 388 retry 389 nop 390 nop 391 nop 392 nop 393 nop 394 nop 395 nop 396 nop 397 nop 398 nop 399 400 .globl xcall_flush_tlb_pending 401xcall_flush_tlb_pending: /* 21 insns */ 402 /* %g5=context, %g1=nr, %g7=vaddrs[] */ 403 sllx %g1, 3, %g1 404 mov PRIMARY_CONTEXT, %g4 405 ldxa [%g4] ASI_DMMU, %g2 406 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4 407 sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 408 or %g5, %g4, %g5 409 mov PRIMARY_CONTEXT, %g4 410 stxa %g5, [%g4] ASI_DMMU 4111: sub %g1, (1 << 3), %g1 412 ldx [%g7 + %g1], %g5 413 andcc %g5, 0x1, %g0 414 be,pn %icc, 2f 415 416 andn %g5, 0x1, %g5 417 stxa %g0, [%g5] ASI_IMMU_DEMAP 4182: stxa %g0, [%g5] ASI_DMMU_DEMAP 419 membar #Sync 420 brnz,pt %g1, 1b 421 nop 422 stxa %g2, [%g4] ASI_DMMU 423 retry 424 nop 425 426 .globl xcall_flush_tlb_kernel_range 427xcall_flush_tlb_kernel_range: /* 25 insns */ 428 sethi %hi(PAGE_SIZE - 1), %g2 429 or %g2, %lo(PAGE_SIZE - 1), %g2 430 andn %g1, %g2, %g1 431 andn %g7, %g2, %g7 432 sub %g7, %g1, %g3 433 add %g2, 1, %g2 434 sub %g3, %g2, %g3 435 or %g1, 0x20, %g1 ! Nucleus 4361: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP 437 stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP 438 membar #Sync 439 brnz,pt %g3, 1b 440 sub %g3, %g2, %g3 441 retry 442 nop 443 nop 444 nop 445 nop 446 nop 447 nop 448 nop 449 nop 450 nop 451 nop 452 nop 453 454 /* This runs in a very controlled environment, so we do 455 * not need to worry about BH races etc. 456 */ 457 .globl xcall_sync_tick 458xcall_sync_tick: 459 460661: rdpr %pstate, %g2 461 wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate 462 .section .sun4v_2insn_patch, "ax" 463 .word 661b 464 nop 465 nop 466 .previous 467 468 rdpr %pil, %g2 469 wrpr %g0, PIL_NORMAL_MAX, %pil 470 sethi %hi(109f), %g7 471 b,pt %xcc, etrap_irq 472109: or %g7, %lo(109b), %g7 473#ifdef CONFIG_TRACE_IRQFLAGS 474 call trace_hardirqs_off 475 nop 476#endif 477 call smp_synchronize_tick_client 478 nop 479 b rtrap_xcall 480 ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 481 482 .globl xcall_fetch_glob_regs 483xcall_fetch_glob_regs: 484 sethi %hi(global_reg_snapshot), %g1 485 or %g1, %lo(global_reg_snapshot), %g1 486 __GET_CPUID(%g2) 487 sllx %g2, 6, %g3 488 add %g1, %g3, %g1 489 rdpr %tstate, %g7 490 stx %g7, [%g1 + GR_SNAP_TSTATE] 491 rdpr %tpc, %g7 492 stx %g7, [%g1 + GR_SNAP_TPC] 493 rdpr %tnpc, %g7 494 stx %g7, [%g1 + GR_SNAP_TNPC] 495 stx %o7, [%g1 + GR_SNAP_O7] 496 stx %i7, [%g1 + GR_SNAP_I7] 497 /* Don't try this at home kids... */ 498 rdpr %cwp, %g2 499 sub %g2, 1, %g7 500 wrpr %g7, %cwp 501 mov %i7, %g7 502 wrpr %g2, %cwp 503 stx %g7, [%g1 + GR_SNAP_RPC] 504 sethi %hi(trap_block), %g7 505 or %g7, %lo(trap_block), %g7 506 sllx %g2, TRAP_BLOCK_SZ_SHIFT, %g2 507 add %g7, %g2, %g7 508 ldx [%g7 + TRAP_PER_CPU_THREAD], %g3 509 stx %g3, [%g1 + GR_SNAP_THREAD] 510 retry 511 512#ifdef DCACHE_ALIASING_POSSIBLE 513 .align 32 514 .globl xcall_flush_dcache_page_cheetah 515xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */ 516 sethi %hi(PAGE_SIZE), %g3 5171: subcc %g3, (1 << 5), %g3 518 stxa %g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE 519 membar #Sync 520 bne,pt %icc, 1b 521 nop 522 retry 523 nop 524#endif /* DCACHE_ALIASING_POSSIBLE */ 525 526 .globl xcall_flush_dcache_page_spitfire 527xcall_flush_dcache_page_spitfire: /* %g1 == physical page address 528 %g7 == kernel page virtual address 529 %g5 == (page->mapping != NULL) */ 530#ifdef DCACHE_ALIASING_POSSIBLE 531 srlx %g1, (13 - 2), %g1 ! Form tag comparitor 532 sethi %hi(L1DCACHE_SIZE), %g3 ! D$ size == 16K 533 sub %g3, (1 << 5), %g3 ! D$ linesize == 32 5341: ldxa [%g3] ASI_DCACHE_TAG, %g2 535 andcc %g2, 0x3, %g0 536 be,pn %xcc, 2f 537 andn %g2, 0x3, %g2 538 cmp %g2, %g1 539 540 bne,pt %xcc, 2f 541 nop 542 stxa %g0, [%g3] ASI_DCACHE_TAG 543 membar #Sync 5442: cmp %g3, 0 545 bne,pt %xcc, 1b 546 sub %g3, (1 << 5), %g3 547 548 brz,pn %g5, 2f 549#endif /* DCACHE_ALIASING_POSSIBLE */ 550 sethi %hi(PAGE_SIZE), %g3 551 5521: flush %g7 553 subcc %g3, (1 << 5), %g3 554 bne,pt %icc, 1b 555 add %g7, (1 << 5), %g7 556 5572: retry 558 nop 559 nop 560 561 /* %g5: error 562 * %g6: tlb op 563 */ 564__hypervisor_tlb_xcall_error: 565 mov %g5, %g4 566 mov %g6, %g5 567 ba,pt %xcc, etrap 568 rd %pc, %g7 569 mov %l4, %o0 570 call hypervisor_tlbop_error_xcall 571 mov %l5, %o1 572 ba,a,pt %xcc, rtrap 573 574 .globl __hypervisor_xcall_flush_tlb_mm 575__hypervisor_xcall_flush_tlb_mm: /* 21 insns */ 576 /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */ 577 mov %o0, %g2 578 mov %o1, %g3 579 mov %o2, %g4 580 mov %o3, %g1 581 mov %o5, %g7 582 clr %o0 /* ARG0: CPU lists unimplemented */ 583 clr %o1 /* ARG1: CPU lists unimplemented */ 584 mov %g5, %o2 /* ARG2: mmu context */ 585 mov HV_MMU_ALL, %o3 /* ARG3: flags */ 586 mov HV_FAST_MMU_DEMAP_CTX, %o5 587 ta HV_FAST_TRAP 588 mov HV_FAST_MMU_DEMAP_CTX, %g6 589 brnz,pn %o0, __hypervisor_tlb_xcall_error 590 mov %o0, %g5 591 mov %g2, %o0 592 mov %g3, %o1 593 mov %g4, %o2 594 mov %g1, %o3 595 mov %g7, %o5 596 membar #Sync 597 retry 598 599 .globl __hypervisor_xcall_flush_tlb_pending 600__hypervisor_xcall_flush_tlb_pending: /* 21 insns */ 601 /* %g5=ctx, %g1=nr, %g7=vaddrs[], %g2,%g3,%g4,g6=scratch */ 602 sllx %g1, 3, %g1 603 mov %o0, %g2 604 mov %o1, %g3 605 mov %o2, %g4 6061: sub %g1, (1 << 3), %g1 607 ldx [%g7 + %g1], %o0 /* ARG0: virtual address */ 608 mov %g5, %o1 /* ARG1: mmu context */ 609 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 610 srlx %o0, PAGE_SHIFT, %o0 611 sllx %o0, PAGE_SHIFT, %o0 612 ta HV_MMU_UNMAP_ADDR_TRAP 613 mov HV_MMU_UNMAP_ADDR_TRAP, %g6 614 brnz,a,pn %o0, __hypervisor_tlb_xcall_error 615 mov %o0, %g5 616 brnz,pt %g1, 1b 617 nop 618 mov %g2, %o0 619 mov %g3, %o1 620 mov %g4, %o2 621 membar #Sync 622 retry 623 624 .globl __hypervisor_xcall_flush_tlb_kernel_range 625__hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */ 626 /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */ 627 sethi %hi(PAGE_SIZE - 1), %g2 628 or %g2, %lo(PAGE_SIZE - 1), %g2 629 andn %g1, %g2, %g1 630 andn %g7, %g2, %g7 631 sub %g7, %g1, %g3 632 add %g2, 1, %g2 633 sub %g3, %g2, %g3 634 mov %o0, %g2 635 mov %o1, %g4 636 mov %o2, %g7 6371: add %g1, %g3, %o0 /* ARG0: virtual address */ 638 mov 0, %o1 /* ARG1: mmu context */ 639 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 640 ta HV_MMU_UNMAP_ADDR_TRAP 641 mov HV_MMU_UNMAP_ADDR_TRAP, %g6 642 brnz,pn %o0, __hypervisor_tlb_xcall_error 643 mov %o0, %g5 644 sethi %hi(PAGE_SIZE), %o2 645 brnz,pt %g3, 1b 646 sub %g3, %o2, %g3 647 mov %g2, %o0 648 mov %g4, %o1 649 mov %g7, %o2 650 membar #Sync 651 retry 652 653 /* These just get rescheduled to PIL vectors. */ 654 .globl xcall_call_function 655xcall_call_function: 656 wr %g0, (1 << PIL_SMP_CALL_FUNC), %set_softint 657 retry 658 659 .globl xcall_call_function_single 660xcall_call_function_single: 661 wr %g0, (1 << PIL_SMP_CALL_FUNC_SNGL), %set_softint 662 retry 663 664 .globl xcall_receive_signal 665xcall_receive_signal: 666 wr %g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint 667 retry 668 669 .globl xcall_capture 670xcall_capture: 671 wr %g0, (1 << PIL_SMP_CAPTURE), %set_softint 672 retry 673 674 .globl xcall_new_mmu_context_version 675xcall_new_mmu_context_version: 676 wr %g0, (1 << PIL_SMP_CTX_NEW_VERSION), %set_softint 677 retry 678 679#ifdef CONFIG_KGDB 680 .globl xcall_kgdb_capture 681xcall_kgdb_capture: 682 wr %g0, (1 << PIL_KGDB_CAPTURE), %set_softint 683 retry 684#endif 685 686#endif /* CONFIG_SMP */ 687 688 689 .globl hypervisor_patch_cachetlbops 690hypervisor_patch_cachetlbops: 691 save %sp, -128, %sp 692 693 sethi %hi(__flush_tlb_mm), %o0 694 or %o0, %lo(__flush_tlb_mm), %o0 695 sethi %hi(__hypervisor_flush_tlb_mm), %o1 696 or %o1, %lo(__hypervisor_flush_tlb_mm), %o1 697 call tlb_patch_one 698 mov 10, %o2 699 700 sethi %hi(__flush_tlb_pending), %o0 701 or %o0, %lo(__flush_tlb_pending), %o0 702 sethi %hi(__hypervisor_flush_tlb_pending), %o1 703 or %o1, %lo(__hypervisor_flush_tlb_pending), %o1 704 call tlb_patch_one 705 mov 16, %o2 706 707 sethi %hi(__flush_tlb_kernel_range), %o0 708 or %o0, %lo(__flush_tlb_kernel_range), %o0 709 sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1 710 or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1 711 call tlb_patch_one 712 mov 16, %o2 713 714#ifdef DCACHE_ALIASING_POSSIBLE 715 sethi %hi(__flush_dcache_page), %o0 716 or %o0, %lo(__flush_dcache_page), %o0 717 sethi %hi(__hypervisor_flush_dcache_page), %o1 718 or %o1, %lo(__hypervisor_flush_dcache_page), %o1 719 call tlb_patch_one 720 mov 2, %o2 721#endif /* DCACHE_ALIASING_POSSIBLE */ 722 723#ifdef CONFIG_SMP 724 sethi %hi(xcall_flush_tlb_mm), %o0 725 or %o0, %lo(xcall_flush_tlb_mm), %o0 726 sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1 727 or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1 728 call tlb_patch_one 729 mov 21, %o2 730 731 sethi %hi(xcall_flush_tlb_pending), %o0 732 or %o0, %lo(xcall_flush_tlb_pending), %o0 733 sethi %hi(__hypervisor_xcall_flush_tlb_pending), %o1 734 or %o1, %lo(__hypervisor_xcall_flush_tlb_pending), %o1 735 call tlb_patch_one 736 mov 21, %o2 737 738 sethi %hi(xcall_flush_tlb_kernel_range), %o0 739 or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 740 sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1 741 or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1 742 call tlb_patch_one 743 mov 25, %o2 744#endif /* CONFIG_SMP */ 745 746 ret 747 restore 748