1/* 2 * ppc64 MMU hashtable management routines 3 * 4 * (c) Copyright IBM Corp. 2003, 2005 5 * 6 * Maintained by: Benjamin Herrenschmidt 7 * <benh@kernel.crashing.org> 8 * 9 * This file is covered by the GNU Public Licence v2 as 10 * described in the kernel's COPYING file. 11 */ 12 13#include <asm/reg.h> 14#include <asm/pgtable.h> 15#include <asm/mmu.h> 16#include <asm/page.h> 17#include <asm/types.h> 18#include <asm/ppc_asm.h> 19#include <asm/asm-offsets.h> 20#include <asm/cputable.h> 21 22 .text 23 24/* 25 * Stackframe: 26 * 27 * +-> Back chain (SP + 256) 28 * | General register save area (SP + 112) 29 * | Parameter save area (SP + 48) 30 * | TOC save area (SP + 40) 31 * | link editor doubleword (SP + 32) 32 * | compiler doubleword (SP + 24) 33 * | LR save area (SP + 16) 34 * | CR save area (SP + 8) 35 * SP ---> +-- Back chain (SP + 0) 36 */ 37#define STACKFRAMESIZE 256 38 39/* Save parameters offsets */ 40#define STK_PARM(i) (STACKFRAMESIZE + 48 + ((i)-3)*8) 41 42/* Save non-volatile offsets */ 43#define STK_REG(i) (112 + ((i)-14)*8) 44 45 46#ifndef CONFIG_PPC_64K_PAGES 47 48/***************************************************************************** 49 * * 50 * 4K SW & 4K HW pages implementation * 51 * * 52 *****************************************************************************/ 53 54 55/* 56 * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, 57 * pte_t *ptep, unsigned long trap, int local, int ssize) 58 * 59 * Adds a 4K page to the hash table in a segment of 4K pages only 60 */ 61 62_GLOBAL(__hash_page_4K) 63 mflr r0 64 std r0,16(r1) 65 stdu r1,-STACKFRAMESIZE(r1) 66 /* Save all params that we need after a function call */ 67 std r6,STK_PARM(r6)(r1) 68 std r8,STK_PARM(r8)(r1) 69 std r9,STK_PARM(r9)(r1) 70 71 /* Add _PAGE_PRESENT to access */ 72 ori r4,r4,_PAGE_PRESENT 73 74 /* Save non-volatile registers. 75 * r31 will hold "old PTE" 76 * r30 is "new PTE" 77 * r29 is "va" 78 * r28 is a hash value 79 * r27 is hashtab mask (maybe dynamic patched instead ?) 80 */ 81 std r27,STK_REG(r27)(r1) 82 std r28,STK_REG(r28)(r1) 83 std r29,STK_REG(r29)(r1) 84 std r30,STK_REG(r30)(r1) 85 std r31,STK_REG(r31)(r1) 86 87 /* Step 1: 88 * 89 * Check permissions, atomically mark the linux PTE busy 90 * and hashed. 91 */ 921: 93 ldarx r31,0,r6 94 /* Check access rights (access & ~(pte_val(*ptep))) */ 95 andc. r0,r4,r31 96 bne- htab_wrong_access 97 /* Check if PTE is busy */ 98 andi. r0,r31,_PAGE_BUSY 99 /* If so, just bail out and refault if needed. Someone else 100 * is changing this PTE anyway and might hash it. 101 */ 102 bne- htab_bail_ok 103 104 /* Prepare new PTE value (turn access RW into DIRTY, then 105 * add BUSY,HASHPTE and ACCESSED) 106 */ 107 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ 108 or r30,r30,r31 109 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE 110 /* Write the linux PTE atomically (setting busy) */ 111 stdcx. r30,0,r6 112 bne- 1b 113 isync 114 115 /* Step 2: 116 * 117 * Insert/Update the HPTE in the hash table. At this point, 118 * r4 (access) is re-useable, we use it for the new HPTE flags 119 */ 120 121BEGIN_FTR_SECTION 122 cmpdi r9,0 /* check segment size */ 123 bne 3f 124END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT) 125 /* Calc va and put it in r29 */ 126 rldicr r29,r5,28,63-28 127 rldicl r3,r3,0,36 128 or r29,r3,r29 129 130 /* Calculate hash value for primary slot and store it in r28 */ 131 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ 132 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ 133 xor r28,r5,r0 134 b 4f 135 1363: /* Calc VA and hash in r29 and r28 for 1T segment */ 137 sldi r29,r5,40 /* vsid << 40 */ 138 clrldi r3,r3,24 /* ea & 0xffffffffff */ 139 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */ 140 clrldi r5,r5,40 /* vsid & 0xffffff */ 141 rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */ 142 xor r28,r28,r5 143 or r29,r3,r29 /* VA */ 144 xor r28,r28,r0 /* hash */ 145 146 /* Convert linux PTE bits into HW equivalents */ 1474: andi. r3,r30,0x1fe /* Get basic set of flags */ 148 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ 149 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ 150 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ 151 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ 152 andc r0,r30,r0 /* r0 = pte & ~r0 */ 153 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ 154 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ 155 156 /* We eventually do the icache sync here (maybe inline that 157 * code rather than call a C function...) 158 */ 159BEGIN_FTR_SECTION 160 mr r4,r30 161 mr r5,r7 162 bl .hash_page_do_lazy_icache 163END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) 164 165 /* At this point, r3 contains new PP bits, save them in 166 * place of "access" in the param area (sic) 167 */ 168 std r3,STK_PARM(r4)(r1) 169 170 /* Get htab_hash_mask */ 171 ld r4,htab_hash_mask@got(2) 172 ld r27,0(r4) /* htab_hash_mask -> r27 */ 173 174 /* Check if we may already be in the hashtable, in this case, we 175 * go to out-of-line code to try to modify the HPTE 176 */ 177 andi. r0,r31,_PAGE_HASHPTE 178 bne htab_modify_pte 179 180htab_insert_pte: 181 /* Clear hpte bits in new pte (we also clear BUSY btw) and 182 * add _PAGE_HASHPTE 183 */ 184 lis r0,_PAGE_HPTEFLAGS@h 185 ori r0,r0,_PAGE_HPTEFLAGS@l 186 andc r30,r30,r0 187 ori r30,r30,_PAGE_HASHPTE 188 189 /* physical address r5 */ 190 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT 191 sldi r5,r5,PAGE_SHIFT 192 193 /* Calculate primary group hash */ 194 and r0,r28,r27 195 rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */ 196 197 /* Call ppc_md.hpte_insert */ 198 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ 199 mr r4,r29 /* Retreive va */ 200 li r7,0 /* !bolted, !secondary */ 201 li r8,MMU_PAGE_4K /* page size */ 202 ld r9,STK_PARM(r9)(r1) /* segment size */ 203_GLOBAL(htab_call_hpte_insert1) 204 bl . /* Patched by htab_finish_init() */ 205 cmpdi 0,r3,0 206 bge htab_pte_insert_ok /* Insertion successful */ 207 cmpdi 0,r3,-2 /* Critical failure */ 208 beq- htab_pte_insert_failure 209 210 /* Now try secondary slot */ 211 212 /* physical address r5 */ 213 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT 214 sldi r5,r5,PAGE_SHIFT 215 216 /* Calculate secondary group hash */ 217 andc r0,r27,r28 218 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ 219 220 /* Call ppc_md.hpte_insert */ 221 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ 222 mr r4,r29 /* Retreive va */ 223 li r7,HPTE_V_SECONDARY /* !bolted, secondary */ 224 li r8,MMU_PAGE_4K /* page size */ 225 ld r9,STK_PARM(r9)(r1) /* segment size */ 226_GLOBAL(htab_call_hpte_insert2) 227 bl . /* Patched by htab_finish_init() */ 228 cmpdi 0,r3,0 229 bge+ htab_pte_insert_ok /* Insertion successful */ 230 cmpdi 0,r3,-2 /* Critical failure */ 231 beq- htab_pte_insert_failure 232 233 /* Both are full, we need to evict something */ 234 mftb r0 235 /* Pick a random group based on TB */ 236 andi. r0,r0,1 237 mr r5,r28 238 bne 2f 239 not r5,r5 2402: and r0,r5,r27 241 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 242 /* Call ppc_md.hpte_remove */ 243_GLOBAL(htab_call_hpte_remove) 244 bl . /* Patched by htab_finish_init() */ 245 246 /* Try all again */ 247 b htab_insert_pte 248 249htab_bail_ok: 250 li r3,0 251 b htab_bail 252 253htab_pte_insert_ok: 254 /* Insert slot number & secondary bit in PTE */ 255 rldimi r30,r3,12,63-15 256 257 /* Write out the PTE with a normal write 258 * (maybe add eieio may be good still ?) 259 */ 260htab_write_out_pte: 261 ld r6,STK_PARM(r6)(r1) 262 std r30,0(r6) 263 li r3, 0 264htab_bail: 265 ld r27,STK_REG(r27)(r1) 266 ld r28,STK_REG(r28)(r1) 267 ld r29,STK_REG(r29)(r1) 268 ld r30,STK_REG(r30)(r1) 269 ld r31,STK_REG(r31)(r1) 270 addi r1,r1,STACKFRAMESIZE 271 ld r0,16(r1) 272 mtlr r0 273 blr 274 275htab_modify_pte: 276 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ 277 mr r4,r3 278 rlwinm r3,r31,32-12,29,31 279 280 /* Secondary group ? if yes, get a inverted hash value */ 281 mr r5,r28 282 andi. r0,r31,_PAGE_SECONDARY 283 beq 1f 284 not r5,r5 2851: 286 /* Calculate proper slot value for ppc_md.hpte_updatepp */ 287 and r0,r5,r27 288 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 289 add r3,r0,r3 /* add slot idx */ 290 291 /* Call ppc_md.hpte_updatepp */ 292 mr r5,r29 /* va */ 293 li r6,MMU_PAGE_4K /* page size */ 294 ld r7,STK_PARM(r9)(r1) /* segment size */ 295 ld r8,STK_PARM(r8)(r1) /* get "local" param */ 296_GLOBAL(htab_call_hpte_updatepp) 297 bl . /* Patched by htab_finish_init() */ 298 299 /* if we failed because typically the HPTE wasn't really here 300 * we try an insertion. 301 */ 302 cmpdi 0,r3,-1 303 beq- htab_insert_pte 304 305 /* Clear the BUSY bit and Write out the PTE */ 306 li r0,_PAGE_BUSY 307 andc r30,r30,r0 308 b htab_write_out_pte 309 310htab_wrong_access: 311 /* Bail out clearing reservation */ 312 stdcx. r31,0,r6 313 li r3,1 314 b htab_bail 315 316htab_pte_insert_failure: 317 /* Bail out restoring old PTE */ 318 ld r6,STK_PARM(r6)(r1) 319 std r31,0(r6) 320 li r3,-1 321 b htab_bail 322 323 324#else /* CONFIG_PPC_64K_PAGES */ 325 326 327/***************************************************************************** 328 * * 329 * 64K SW & 4K or 64K HW in a 4K segment pages implementation * 330 * * 331 *****************************************************************************/ 332 333/* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, 334 * pte_t *ptep, unsigned long trap, int local, int ssize, 335 * int subpg_prot) 336 */ 337 338/* 339 * For now, we do NOT implement Admixed pages 340 */ 341_GLOBAL(__hash_page_4K) 342 mflr r0 343 std r0,16(r1) 344 stdu r1,-STACKFRAMESIZE(r1) 345 /* Save all params that we need after a function call */ 346 std r6,STK_PARM(r6)(r1) 347 std r8,STK_PARM(r8)(r1) 348 std r9,STK_PARM(r9)(r1) 349 350 /* Add _PAGE_PRESENT to access */ 351 ori r4,r4,_PAGE_PRESENT 352 353 /* Save non-volatile registers. 354 * r31 will hold "old PTE" 355 * r30 is "new PTE" 356 * r29 is "va" 357 * r28 is a hash value 358 * r27 is hashtab mask (maybe dynamic patched instead ?) 359 * r26 is the hidx mask 360 * r25 is the index in combo page 361 */ 362 std r25,STK_REG(r25)(r1) 363 std r26,STK_REG(r26)(r1) 364 std r27,STK_REG(r27)(r1) 365 std r28,STK_REG(r28)(r1) 366 std r29,STK_REG(r29)(r1) 367 std r30,STK_REG(r30)(r1) 368 std r31,STK_REG(r31)(r1) 369 370 /* Step 1: 371 * 372 * Check permissions, atomically mark the linux PTE busy 373 * and hashed. 374 */ 3751: 376 ldarx r31,0,r6 377 /* Check access rights (access & ~(pte_val(*ptep))) */ 378 andc. r0,r4,r31 379 bne- htab_wrong_access 380 /* Check if PTE is busy */ 381 andi. r0,r31,_PAGE_BUSY 382 /* If so, just bail out and refault if needed. Someone else 383 * is changing this PTE anyway and might hash it. 384 */ 385 bne- htab_bail_ok 386 /* Prepare new PTE value (turn access RW into DIRTY, then 387 * add BUSY and ACCESSED) 388 */ 389 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ 390 or r30,r30,r31 391 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED 392 oris r30,r30,_PAGE_COMBO@h 393 /* Write the linux PTE atomically (setting busy) */ 394 stdcx. r30,0,r6 395 bne- 1b 396 isync 397 398 /* Step 2: 399 * 400 * Insert/Update the HPTE in the hash table. At this point, 401 * r4 (access) is re-useable, we use it for the new HPTE flags 402 */ 403 404 /* Load the hidx index */ 405 rldicl r25,r3,64-12,60 406 407BEGIN_FTR_SECTION 408 cmpdi r9,0 /* check segment size */ 409 bne 3f 410END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT) 411 /* Calc va and put it in r29 */ 412 rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */ 413 rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */ 414 or r29,r3,r29 /* r29 = va */ 415 416 /* Calculate hash value for primary slot and store it in r28 */ 417 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ 418 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ 419 xor r28,r5,r0 420 b 4f 421 4223: /* Calc VA and hash in r29 and r28 for 1T segment */ 423 sldi r29,r5,40 /* vsid << 40 */ 424 clrldi r3,r3,24 /* ea & 0xffffffffff */ 425 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */ 426 clrldi r5,r5,40 /* vsid & 0xffffff */ 427 rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */ 428 xor r28,r28,r5 429 or r29,r3,r29 /* VA */ 430 xor r28,r28,r0 /* hash */ 431 432 /* Convert linux PTE bits into HW equivalents */ 4334: 434#ifdef CONFIG_PPC_SUBPAGE_PROT 435 andc r10,r30,r10 436 andi. r3,r10,0x1fe /* Get basic set of flags */ 437 rlwinm r0,r10,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ 438#else 439 andi. r3,r30,0x1fe /* Get basic set of flags */ 440 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ 441#endif 442 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ 443 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ 444 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ 445 andc r0,r3,r0 /* r0 = pte & ~r0 */ 446 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ 447 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ 448 449 /* We eventually do the icache sync here (maybe inline that 450 * code rather than call a C function...) 451 */ 452BEGIN_FTR_SECTION 453 mr r4,r30 454 mr r5,r7 455 bl .hash_page_do_lazy_icache 456END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) 457 458 /* At this point, r3 contains new PP bits, save them in 459 * place of "access" in the param area (sic) 460 */ 461 std r3,STK_PARM(r4)(r1) 462 463 /* Get htab_hash_mask */ 464 ld r4,htab_hash_mask@got(2) 465 ld r27,0(r4) /* htab_hash_mask -> r27 */ 466 467 /* Check if we may already be in the hashtable, in this case, we 468 * go to out-of-line code to try to modify the HPTE. We look for 469 * the bit at (1 >> (index + 32)) 470 */ 471 rldicl. r0,r31,64-12,48 472 li r26,0 /* Default hidx */ 473 beq htab_insert_pte 474 475 /* 476 * Check if the pte was already inserted into the hash table 477 * as a 64k HW page, and invalidate the 64k HPTE if so. 478 */ 479 andis. r0,r31,_PAGE_COMBO@h 480 beq htab_inval_old_hpte 481 482 ld r6,STK_PARM(r6)(r1) 483 ori r26,r6,0x8000 /* Load the hidx mask */ 484 ld r26,0(r26) 485 addi r5,r25,36 /* Check actual HPTE_SUB bit, this */ 486 rldcr. r0,r31,r5,0 /* must match pgtable.h definition */ 487 bne htab_modify_pte 488 489htab_insert_pte: 490 /* real page number in r5, PTE RPN value + index */ 491 andis. r0,r31,_PAGE_4K_PFN@h 492 srdi r5,r31,PTE_RPN_SHIFT 493 bne- htab_special_pfn 494 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT 495 add r5,r5,r25 496htab_special_pfn: 497 sldi r5,r5,HW_PAGE_SHIFT 498 499 /* Calculate primary group hash */ 500 and r0,r28,r27 501 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 502 503 /* Call ppc_md.hpte_insert */ 504 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ 505 mr r4,r29 /* Retreive va */ 506 li r7,0 /* !bolted, !secondary */ 507 li r8,MMU_PAGE_4K /* page size */ 508 ld r9,STK_PARM(r9)(r1) /* segment size */ 509_GLOBAL(htab_call_hpte_insert1) 510 bl . /* patched by htab_finish_init() */ 511 cmpdi 0,r3,0 512 bge htab_pte_insert_ok /* Insertion successful */ 513 cmpdi 0,r3,-2 /* Critical failure */ 514 beq- htab_pte_insert_failure 515 516 /* Now try secondary slot */ 517 518 /* real page number in r5, PTE RPN value + index */ 519 andis. r0,r31,_PAGE_4K_PFN@h 520 srdi r5,r31,PTE_RPN_SHIFT 521 bne- 3f 522 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT 523 add r5,r5,r25 5243: sldi r5,r5,HW_PAGE_SHIFT 525 526 /* Calculate secondary group hash */ 527 andc r0,r27,r28 528 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ 529 530 /* Call ppc_md.hpte_insert */ 531 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ 532 mr r4,r29 /* Retreive va */ 533 li r7,HPTE_V_SECONDARY /* !bolted, secondary */ 534 li r8,MMU_PAGE_4K /* page size */ 535 ld r9,STK_PARM(r9)(r1) /* segment size */ 536_GLOBAL(htab_call_hpte_insert2) 537 bl . /* patched by htab_finish_init() */ 538 cmpdi 0,r3,0 539 bge+ htab_pte_insert_ok /* Insertion successful */ 540 cmpdi 0,r3,-2 /* Critical failure */ 541 beq- htab_pte_insert_failure 542 543 /* Both are full, we need to evict something */ 544 mftb r0 545 /* Pick a random group based on TB */ 546 andi. r0,r0,1 547 mr r5,r28 548 bne 2f 549 not r5,r5 5502: and r0,r5,r27 551 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 552 /* Call ppc_md.hpte_remove */ 553_GLOBAL(htab_call_hpte_remove) 554 bl . /* patched by htab_finish_init() */ 555 556 /* Try all again */ 557 b htab_insert_pte 558 559 /* 560 * Call out to C code to invalidate an 64k HW HPTE that is 561 * useless now that the segment has been switched to 4k pages. 562 */ 563htab_inval_old_hpte: 564 mr r3,r29 /* virtual addr */ 565 mr r4,r31 /* PTE.pte */ 566 li r5,0 /* PTE.hidx */ 567 li r6,MMU_PAGE_64K /* psize */ 568 ld r7,STK_PARM(r9)(r1) /* ssize */ 569 ld r8,STK_PARM(r8)(r1) /* local */ 570 bl .flush_hash_page 571 /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */ 572 lis r0,_PAGE_HPTE_SUB@h 573 ori r0,r0,_PAGE_HPTE_SUB@l 574 andc r30,r30,r0 575 b htab_insert_pte 576 577htab_bail_ok: 578 li r3,0 579 b htab_bail 580 581htab_pte_insert_ok: 582 /* Insert slot number & secondary bit in PTE second half, 583 * clear _PAGE_BUSY and set approriate HPTE slot bit 584 */ 585 ld r6,STK_PARM(r6)(r1) 586 li r0,_PAGE_BUSY 587 andc r30,r30,r0 588 /* HPTE SUB bit */ 589 li r0,1 590 subfic r5,r25,27 /* Must match bit position in */ 591 sld r0,r0,r5 /* pgtable.h */ 592 or r30,r30,r0 593 /* hindx */ 594 sldi r5,r25,2 595 sld r3,r3,r5 596 li r4,0xf 597 sld r4,r4,r5 598 andc r26,r26,r4 599 or r26,r26,r3 600 ori r5,r6,0x8000 601 std r26,0(r5) 602 lwsync 603 std r30,0(r6) 604 li r3, 0 605htab_bail: 606 ld r25,STK_REG(r25)(r1) 607 ld r26,STK_REG(r26)(r1) 608 ld r27,STK_REG(r27)(r1) 609 ld r28,STK_REG(r28)(r1) 610 ld r29,STK_REG(r29)(r1) 611 ld r30,STK_REG(r30)(r1) 612 ld r31,STK_REG(r31)(r1) 613 addi r1,r1,STACKFRAMESIZE 614 ld r0,16(r1) 615 mtlr r0 616 blr 617 618htab_modify_pte: 619 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ 620 mr r4,r3 621 sldi r5,r25,2 622 srd r3,r26,r5 623 624 /* Secondary group ? if yes, get a inverted hash value */ 625 mr r5,r28 626 andi. r0,r3,0x8 /* page secondary ? */ 627 beq 1f 628 not r5,r5 6291: andi. r3,r3,0x7 /* extract idx alone */ 630 631 /* Calculate proper slot value for ppc_md.hpte_updatepp */ 632 and r0,r5,r27 633 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 634 add r3,r0,r3 /* add slot idx */ 635 636 /* Call ppc_md.hpte_updatepp */ 637 mr r5,r29 /* va */ 638 li r6,MMU_PAGE_4K /* page size */ 639 ld r7,STK_PARM(r9)(r1) /* segment size */ 640 ld r8,STK_PARM(r8)(r1) /* get "local" param */ 641_GLOBAL(htab_call_hpte_updatepp) 642 bl . /* patched by htab_finish_init() */ 643 644 /* if we failed because typically the HPTE wasn't really here 645 * we try an insertion. 646 */ 647 cmpdi 0,r3,-1 648 beq- htab_insert_pte 649 650 /* Clear the BUSY bit and Write out the PTE */ 651 li r0,_PAGE_BUSY 652 andc r30,r30,r0 653 ld r6,STK_PARM(r6)(r1) 654 std r30,0(r6) 655 li r3,0 656 b htab_bail 657 658htab_wrong_access: 659 /* Bail out clearing reservation */ 660 stdcx. r31,0,r6 661 li r3,1 662 b htab_bail 663 664htab_pte_insert_failure: 665 /* Bail out restoring old PTE */ 666 ld r6,STK_PARM(r6)(r1) 667 std r31,0(r6) 668 li r3,-1 669 b htab_bail 670 671#endif /* CONFIG_PPC_64K_PAGES */ 672 673#ifdef CONFIG_PPC_HAS_HASH_64K 674 675/***************************************************************************** 676 * * 677 * 64K SW & 64K HW in a 64K segment pages implementation * 678 * * 679 *****************************************************************************/ 680 681_GLOBAL(__hash_page_64K) 682 mflr r0 683 std r0,16(r1) 684 stdu r1,-STACKFRAMESIZE(r1) 685 /* Save all params that we need after a function call */ 686 std r6,STK_PARM(r6)(r1) 687 std r8,STK_PARM(r8)(r1) 688 std r9,STK_PARM(r9)(r1) 689 690 /* Add _PAGE_PRESENT to access */ 691 ori r4,r4,_PAGE_PRESENT 692 693 /* Save non-volatile registers. 694 * r31 will hold "old PTE" 695 * r30 is "new PTE" 696 * r29 is "va" 697 * r28 is a hash value 698 * r27 is hashtab mask (maybe dynamic patched instead ?) 699 */ 700 std r27,STK_REG(r27)(r1) 701 std r28,STK_REG(r28)(r1) 702 std r29,STK_REG(r29)(r1) 703 std r30,STK_REG(r30)(r1) 704 std r31,STK_REG(r31)(r1) 705 706 /* Step 1: 707 * 708 * Check permissions, atomically mark the linux PTE busy 709 * and hashed. 710 */ 7111: 712 ldarx r31,0,r6 713 /* Check access rights (access & ~(pte_val(*ptep))) */ 714 andc. r0,r4,r31 715 bne- ht64_wrong_access 716 /* Check if PTE is busy */ 717 andi. r0,r31,_PAGE_BUSY 718 /* If so, just bail out and refault if needed. Someone else 719 * is changing this PTE anyway and might hash it. 720 */ 721 bne- ht64_bail_ok 722BEGIN_FTR_SECTION 723 /* Check if PTE has the cache-inhibit bit set */ 724 andi. r0,r31,_PAGE_NO_CACHE 725 /* If so, bail out and refault as a 4k page */ 726 bne- ht64_bail_ok 727END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE) 728 /* Prepare new PTE value (turn access RW into DIRTY, then 729 * add BUSY and ACCESSED) 730 */ 731 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ 732 or r30,r30,r31 733 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED 734 /* Write the linux PTE atomically (setting busy) */ 735 stdcx. r30,0,r6 736 bne- 1b 737 isync 738 739 /* Step 2: 740 * 741 * Insert/Update the HPTE in the hash table. At this point, 742 * r4 (access) is re-useable, we use it for the new HPTE flags 743 */ 744 745BEGIN_FTR_SECTION 746 cmpdi r9,0 /* check segment size */ 747 bne 3f 748END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT) 749 /* Calc va and put it in r29 */ 750 rldicr r29,r5,28,63-28 751 rldicl r3,r3,0,36 752 or r29,r3,r29 753 754 /* Calculate hash value for primary slot and store it in r28 */ 755 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ 756 rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */ 757 xor r28,r5,r0 758 b 4f 759 7603: /* Calc VA and hash in r29 and r28 for 1T segment */ 761 sldi r29,r5,40 /* vsid << 40 */ 762 clrldi r3,r3,24 /* ea & 0xffffffffff */ 763 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */ 764 clrldi r5,r5,40 /* vsid & 0xffffff */ 765 rldicl r0,r3,64-16,40 /* (ea >> 16) & 0xffffff */ 766 xor r28,r28,r5 767 or r29,r3,r29 /* VA */ 768 xor r28,r28,r0 /* hash */ 769 770 /* Convert linux PTE bits into HW equivalents */ 7714: andi. r3,r30,0x1fe /* Get basic set of flags */ 772 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ 773 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ 774 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ 775 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ 776 andc r0,r30,r0 /* r0 = pte & ~r0 */ 777 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ 778 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ 779 780 /* We eventually do the icache sync here (maybe inline that 781 * code rather than call a C function...) 782 */ 783BEGIN_FTR_SECTION 784 mr r4,r30 785 mr r5,r7 786 bl .hash_page_do_lazy_icache 787END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) 788 789 /* At this point, r3 contains new PP bits, save them in 790 * place of "access" in the param area (sic) 791 */ 792 std r3,STK_PARM(r4)(r1) 793 794 /* Get htab_hash_mask */ 795 ld r4,htab_hash_mask@got(2) 796 ld r27,0(r4) /* htab_hash_mask -> r27 */ 797 798 /* Check if we may already be in the hashtable, in this case, we 799 * go to out-of-line code to try to modify the HPTE 800 */ 801 rldicl. r0,r31,64-12,48 802 bne ht64_modify_pte 803 804ht64_insert_pte: 805 /* Clear hpte bits in new pte (we also clear BUSY btw) and 806 * add _PAGE_HPTE_SUB0 807 */ 808 lis r0,_PAGE_HPTEFLAGS@h 809 ori r0,r0,_PAGE_HPTEFLAGS@l 810 andc r30,r30,r0 811#ifdef CONFIG_PPC_64K_PAGES 812 oris r30,r30,_PAGE_HPTE_SUB0@h 813#else 814 ori r30,r30,_PAGE_HASHPTE 815#endif 816 /* Phyical address in r5 */ 817 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT 818 sldi r5,r5,PAGE_SHIFT 819 820 /* Calculate primary group hash */ 821 and r0,r28,r27 822 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 823 824 /* Call ppc_md.hpte_insert */ 825 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ 826 mr r4,r29 /* Retreive va */ 827 li r7,0 /* !bolted, !secondary */ 828 li r8,MMU_PAGE_64K 829 ld r9,STK_PARM(r9)(r1) /* segment size */ 830_GLOBAL(ht64_call_hpte_insert1) 831 bl . /* patched by htab_finish_init() */ 832 cmpdi 0,r3,0 833 bge ht64_pte_insert_ok /* Insertion successful */ 834 cmpdi 0,r3,-2 /* Critical failure */ 835 beq- ht64_pte_insert_failure 836 837 /* Now try secondary slot */ 838 839 /* Phyical address in r5 */ 840 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT 841 sldi r5,r5,PAGE_SHIFT 842 843 /* Calculate secondary group hash */ 844 andc r0,r27,r28 845 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ 846 847 /* Call ppc_md.hpte_insert */ 848 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ 849 mr r4,r29 /* Retreive va */ 850 li r7,HPTE_V_SECONDARY /* !bolted, secondary */ 851 li r8,MMU_PAGE_64K 852 ld r9,STK_PARM(r9)(r1) /* segment size */ 853_GLOBAL(ht64_call_hpte_insert2) 854 bl . /* patched by htab_finish_init() */ 855 cmpdi 0,r3,0 856 bge+ ht64_pte_insert_ok /* Insertion successful */ 857 cmpdi 0,r3,-2 /* Critical failure */ 858 beq- ht64_pte_insert_failure 859 860 /* Both are full, we need to evict something */ 861 mftb r0 862 /* Pick a random group based on TB */ 863 andi. r0,r0,1 864 mr r5,r28 865 bne 2f 866 not r5,r5 8672: and r0,r5,r27 868 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 869 /* Call ppc_md.hpte_remove */ 870_GLOBAL(ht64_call_hpte_remove) 871 bl . /* patched by htab_finish_init() */ 872 873 /* Try all again */ 874 b ht64_insert_pte 875 876ht64_bail_ok: 877 li r3,0 878 b ht64_bail 879 880ht64_pte_insert_ok: 881 /* Insert slot number & secondary bit in PTE */ 882 rldimi r30,r3,12,63-15 883 884 /* Write out the PTE with a normal write 885 * (maybe add eieio may be good still ?) 886 */ 887ht64_write_out_pte: 888 ld r6,STK_PARM(r6)(r1) 889 std r30,0(r6) 890 li r3, 0 891ht64_bail: 892 ld r27,STK_REG(r27)(r1) 893 ld r28,STK_REG(r28)(r1) 894 ld r29,STK_REG(r29)(r1) 895 ld r30,STK_REG(r30)(r1) 896 ld r31,STK_REG(r31)(r1) 897 addi r1,r1,STACKFRAMESIZE 898 ld r0,16(r1) 899 mtlr r0 900 blr 901 902ht64_modify_pte: 903 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ 904 mr r4,r3 905 rlwinm r3,r31,32-12,29,31 906 907 /* Secondary group ? if yes, get a inverted hash value */ 908 mr r5,r28 909 andi. r0,r31,_PAGE_F_SECOND 910 beq 1f 911 not r5,r5 9121: 913 /* Calculate proper slot value for ppc_md.hpte_updatepp */ 914 and r0,r5,r27 915 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 916 add r3,r0,r3 /* add slot idx */ 917 918 /* Call ppc_md.hpte_updatepp */ 919 mr r5,r29 /* va */ 920 li r6,MMU_PAGE_64K 921 ld r7,STK_PARM(r9)(r1) /* segment size */ 922 ld r8,STK_PARM(r8)(r1) /* get "local" param */ 923_GLOBAL(ht64_call_hpte_updatepp) 924 bl . /* patched by htab_finish_init() */ 925 926 /* if we failed because typically the HPTE wasn't really here 927 * we try an insertion. 928 */ 929 cmpdi 0,r3,-1 930 beq- ht64_insert_pte 931 932 /* Clear the BUSY bit and Write out the PTE */ 933 li r0,_PAGE_BUSY 934 andc r30,r30,r0 935 b ht64_write_out_pte 936 937ht64_wrong_access: 938 /* Bail out clearing reservation */ 939 stdcx. r31,0,r6 940 li r3,1 941 b ht64_bail 942 943ht64_pte_insert_failure: 944 /* Bail out restoring old PTE */ 945 ld r6,STK_PARM(r6)(r1) 946 std r31,0(r6) 947 li r3,-1 948 b ht64_bail 949 950 951#endif /* CONFIG_PPC_HAS_HASH_64K */ 952 953 954/***************************************************************************** 955 * * 956 * Huge pages implementation is in hugetlbpage.c * 957 * * 958 *****************************************************************************/ 959