Lines Matching refs:p
205 static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance) in uasm_bgezl_hazard() argument
209 uasm_il_bgezl(p, r, 0, label_tlbw_hazard_0 + instance); in uasm_bgezl_hazard()
216 static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance) in uasm_bgezl_label() argument
220 uasm_build_label(l, *p, label_tlbw_hazard_0 + instance); in uasm_bgezl_label()
359 static struct work_registers build_get_work_registers(u32 **p) in build_get_work_registers() argument
365 UASM_i_MTC0(p, 1, c0_kscratch(), scratch_reg); in build_get_work_registers()
374 UASM_i_CPUID_MFC0(p, K0, SMP_CPUID_REG); in build_get_work_registers()
375 UASM_i_SRL_SAFE(p, K0, K0, SMP_CPUID_REGSHIFT); in build_get_work_registers()
378 UASM_i_SLL(p, K0, K0, ilog2(sizeof(struct tlb_reg_save))); in build_get_work_registers()
380 UASM_i_LA(p, K1, (long)&handler_reg_save); in build_get_work_registers()
381 UASM_i_ADDU(p, K0, K0, K1); in build_get_work_registers()
383 UASM_i_LA(p, K0, (long)&handler_reg_save); in build_get_work_registers()
386 UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), K0); in build_get_work_registers()
387 UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), K0); in build_get_work_registers()
395 static void build_restore_work_registers(u32 **p) in build_restore_work_registers() argument
398 uasm_i_ehb(p); in build_restore_work_registers()
399 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); in build_restore_work_registers()
403 UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), K0); in build_restore_work_registers()
404 UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), K0); in build_restore_work_registers()
418 u32 *p; in build_r3000_tlb_refill_handler() local
421 p = tlb_handler; in build_r3000_tlb_refill_handler()
423 uasm_i_mfc0(&p, K0, C0_BADVADDR); in build_r3000_tlb_refill_handler()
424 uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */ in build_r3000_tlb_refill_handler()
425 uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1); in build_r3000_tlb_refill_handler()
426 uasm_i_srl(&p, K0, K0, 22); /* load delay */ in build_r3000_tlb_refill_handler()
427 uasm_i_sll(&p, K0, K0, 2); in build_r3000_tlb_refill_handler()
428 uasm_i_addu(&p, K1, K1, K0); in build_r3000_tlb_refill_handler()
429 uasm_i_mfc0(&p, K0, C0_CONTEXT); in build_r3000_tlb_refill_handler()
430 uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */ in build_r3000_tlb_refill_handler()
431 uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */ in build_r3000_tlb_refill_handler()
432 uasm_i_addu(&p, K1, K1, K0); in build_r3000_tlb_refill_handler()
433 uasm_i_lw(&p, K0, 0, K1); in build_r3000_tlb_refill_handler()
434 uasm_i_nop(&p); /* load delay */ in build_r3000_tlb_refill_handler()
435 uasm_i_mtc0(&p, K0, C0_ENTRYLO0); in build_r3000_tlb_refill_handler()
436 uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */ in build_r3000_tlb_refill_handler()
437 uasm_i_tlbwr(&p); /* cp0 delay */ in build_r3000_tlb_refill_handler()
438 uasm_i_jr(&p, K1); in build_r3000_tlb_refill_handler()
439 uasm_i_rfe(&p); /* branch delay */ in build_r3000_tlb_refill_handler()
441 if (p > tlb_handler + 32) in build_r3000_tlb_refill_handler()
445 (unsigned int)(p - tlb_handler)); in build_r3000_tlb_refill_handler()
484 static void __maybe_unused build_tlb_probe_entry(u32 **p) in build_tlb_probe_entry() argument
492 uasm_i_nop(p); in build_tlb_probe_entry()
493 uasm_i_tlbp(p); in build_tlb_probe_entry()
497 uasm_i_tlbp(p); in build_tlb_probe_entry()
502 void build_tlb_write_entry(u32 **p, struct uasm_label **l, in build_tlb_write_entry() argument
515 uasm_i_ehb(p); in build_tlb_write_entry()
516 tlbw(p); in build_tlb_write_entry()
531 uasm_bgezl_hazard(p, r, hazard_instance); in build_tlb_write_entry()
532 tlbw(p); in build_tlb_write_entry()
533 uasm_bgezl_label(l, p, hazard_instance); in build_tlb_write_entry()
535 uasm_i_nop(p); in build_tlb_write_entry()
540 uasm_i_nop(p); in build_tlb_write_entry()
541 tlbw(p); in build_tlb_write_entry()
542 uasm_i_nop(p); in build_tlb_write_entry()
547 uasm_i_nop(p); /* QED specifies 2 nops hazard */ in build_tlb_write_entry()
548 uasm_i_nop(p); /* QED specifies 2 nops hazard */ in build_tlb_write_entry()
549 tlbw(p); in build_tlb_write_entry()
556 uasm_i_nop(p); in build_tlb_write_entry()
557 tlbw(p); in build_tlb_write_entry()
582 uasm_i_nop(p); in build_tlb_write_entry()
585 tlbw(p); in build_tlb_write_entry()
589 uasm_i_nop(p); in build_tlb_write_entry()
590 uasm_i_nop(p); in build_tlb_write_entry()
591 uasm_i_nop(p); in build_tlb_write_entry()
592 uasm_i_nop(p); in build_tlb_write_entry()
593 tlbw(p); in build_tlb_write_entry()
601 uasm_i_nop(p); in build_tlb_write_entry()
602 uasm_i_nop(p); in build_tlb_write_entry()
603 tlbw(p); in build_tlb_write_entry()
604 uasm_i_nop(p); in build_tlb_write_entry()
605 uasm_i_nop(p); in build_tlb_write_entry()
610 uasm_i_nop(p); in build_tlb_write_entry()
611 uasm_i_nop(p); in build_tlb_write_entry()
612 tlbw(p); in build_tlb_write_entry()
616 tlbw(p); in build_tlb_write_entry()
617 uasm_i_nop(p); in build_tlb_write_entry()
628 static __maybe_unused void build_convert_pte_to_entrylo(u32 **p, in build_convert_pte_to_entrylo() argument
638 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL)); in build_convert_pte_to_entrylo()
640 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_NO_EXEC)); in build_convert_pte_to_entrylo()
641 UASM_i_ROTR(p, reg, reg, in build_convert_pte_to_entrylo()
646 uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL)); in build_convert_pte_to_entrylo()
648 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL)); in build_convert_pte_to_entrylo()
655 static void build_restore_pagemask(u32 **p, struct uasm_reloc **r, in build_restore_pagemask() argument
665 uasm_i_ehb(p); in build_restore_pagemask()
669 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); in build_restore_pagemask()
670 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); in build_restore_pagemask()
671 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_restore_pagemask()
672 uasm_il_b(p, r, lid); in build_restore_pagemask()
674 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); in build_restore_pagemask()
675 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_restore_pagemask()
676 uasm_il_b(p, r, lid); in build_restore_pagemask()
678 uasm_i_mtc0(p, 0, C0_PAGEMASK); in build_restore_pagemask()
679 uasm_il_b(p, r, lid); in build_restore_pagemask()
682 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); in build_restore_pagemask()
684 UASM_i_LW(p, 1, scratchpad_offset(0), 0); in build_restore_pagemask()
688 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); in build_restore_pagemask()
689 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); in build_restore_pagemask()
690 uasm_il_b(p, r, lid); in build_restore_pagemask()
691 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_restore_pagemask()
693 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); in build_restore_pagemask()
694 uasm_il_b(p, r, lid); in build_restore_pagemask()
695 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_restore_pagemask()
697 uasm_il_b(p, r, lid); in build_restore_pagemask()
698 uasm_i_mtc0(p, 0, C0_PAGEMASK); in build_restore_pagemask()
703 static void build_huge_tlb_write_entry(u32 **p, struct uasm_label **l, in build_huge_tlb_write_entry() argument
710 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16); in build_huge_tlb_write_entry()
711 uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff); in build_huge_tlb_write_entry()
712 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_huge_tlb_write_entry()
714 build_tlb_write_entry(p, l, r, wmode); in build_huge_tlb_write_entry()
716 build_restore_pagemask(p, r, tmp, label_leave, restore_scratch); in build_huge_tlb_write_entry()
723 build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, in build_is_huge_pte() argument
726 UASM_i_LW(p, tmp, 0, pmd); in build_is_huge_pte()
728 uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid); in build_is_huge_pte()
730 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); in build_is_huge_pte()
731 uasm_il_bnez(p, r, tmp, lid); in build_is_huge_pte()
735 static void build_huge_update_entries(u32 **p, unsigned int pte, in build_huge_update_entries() argument
753 uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16)); in build_huge_update_entries()
755 build_convert_pte_to_entrylo(p, pte); in build_huge_update_entries()
756 UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */ in build_huge_update_entries()
759 UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7); in build_huge_update_entries()
761 UASM_i_ADDU(p, pte, pte, tmp); in build_huge_update_entries()
763 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */ in build_huge_update_entries()
766 static void build_huge_handler_tail(u32 **p, struct uasm_reloc **r, in build_huge_handler_tail() argument
773 UASM_i_SC(p, pte, 0, ptr); in build_huge_handler_tail()
774 uasm_il_beqz(p, r, pte, label_tlb_huge_update); in build_huge_handler_tail()
775 UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */ in build_huge_handler_tail()
777 UASM_i_SW(p, pte, 0, ptr); in build_huge_handler_tail()
782 UASM_i_MFC0(p, ptr, C0_ENTRYHI); in build_huge_handler_tail()
783 uasm_i_ori(p, ptr, ptr, MIPS_ENTRYHI_EHINV); in build_huge_handler_tail()
784 UASM_i_MTC0(p, ptr, C0_ENTRYHI); in build_huge_handler_tail()
785 build_tlb_write_entry(p, l, r, tlb_indexed); in build_huge_handler_tail()
787 uasm_i_xori(p, ptr, ptr, MIPS_ENTRYHI_EHINV); in build_huge_handler_tail()
788 UASM_i_MTC0(p, ptr, C0_ENTRYHI); in build_huge_handler_tail()
789 build_huge_update_entries(p, pte, ptr); in build_huge_handler_tail()
790 build_huge_tlb_write_entry(p, l, r, pte, tlb_random, 0); in build_huge_handler_tail()
795 build_huge_update_entries(p, pte, ptr); in build_huge_handler_tail()
796 build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0); in build_huge_handler_tail()
805 void build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, in build_get_pmde64() argument
814 uasm_i_dmfc0(p, tmp, C0_BADVADDR); in build_get_pmde64()
828 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); in build_get_pmde64()
829 uasm_il_bnez(p, r, ptr, label_vmalloc); in build_get_pmde64()
831 uasm_il_bltz(p, r, tmp, label_vmalloc); in build_get_pmde64()
838 UASM_i_MFC0(p, ptr, C0_PWBASE); in build_get_pmde64()
840 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); in build_get_pmde64()
846 UASM_i_MFC0(p, ptr, C0_CONTEXT); in build_get_pmde64()
849 uasm_i_dins(p, ptr, 0, 0, 23); in build_get_pmde64()
852 uasm_i_ori(p, ptr, ptr, 0x540); in build_get_pmde64()
853 uasm_i_drotr(p, ptr, ptr, 11); in build_get_pmde64()
855 UASM_i_CPUID_MFC0(p, ptr, SMP_CPUID_REG); in build_get_pmde64()
856 uasm_i_dsrl_safe(p, ptr, ptr, SMP_CPUID_PTRSHIFT); in build_get_pmde64()
857 UASM_i_LA_mostly(p, tmp, pgdc); in build_get_pmde64()
858 uasm_i_daddu(p, ptr, ptr, tmp); in build_get_pmde64()
859 uasm_i_dmfc0(p, tmp, C0_BADVADDR); in build_get_pmde64()
860 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); in build_get_pmde64()
862 UASM_i_LA_mostly(p, ptr, pgdc); in build_get_pmde64()
863 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); in build_get_pmde64()
867 uasm_l_vmalloc_done(l, *p); in build_get_pmde64()
870 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3); in build_get_pmde64()
872 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); in build_get_pmde64()
873 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */ in build_get_pmde64()
875 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ in build_get_pmde64()
876 uasm_i_ld(p, ptr, 0, ptr); /* get pud pointer */ in build_get_pmde64()
877 uasm_i_dsrl_safe(p, tmp, tmp, PUD_SHIFT - 3); /* get pud offset in bytes */ in build_get_pmde64()
878 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PUD - 1) << 3); in build_get_pmde64()
879 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pud offset */ in build_get_pmde64()
882 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ in build_get_pmde64()
883 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */ in build_get_pmde64()
884 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ in build_get_pmde64()
885 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); in build_get_pmde64()
886 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */ in build_get_pmde64()
896 build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, in build_get_pgd_vmalloc64() argument
906 uasm_l_vmalloc(l, *p); in build_get_pgd_vmalloc64()
910 uasm_il_bltz(p, r, bvaddr, label_vmalloc_done); in build_get_pgd_vmalloc64()
911 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); in build_get_pgd_vmalloc64()
915 uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault); in build_get_pgd_vmalloc64()
920 uasm_il_b(p, r, label_vmalloc_done); in build_get_pgd_vmalloc64()
921 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); in build_get_pgd_vmalloc64()
923 UASM_i_LA_mostly(p, ptr, swpd); in build_get_pgd_vmalloc64()
924 uasm_il_b(p, r, label_vmalloc_done); in build_get_pgd_vmalloc64()
926 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd)); in build_get_pgd_vmalloc64()
928 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); in build_get_pgd_vmalloc64()
932 uasm_l_large_segbits_fault(l, *p); in build_get_pgd_vmalloc64()
935 uasm_i_ehb(p); in build_get_pgd_vmalloc64()
950 uasm_i_sync(p, 0); in build_get_pgd_vmalloc64()
951 UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0); in build_get_pgd_vmalloc64()
952 uasm_i_jr(p, ptr); in build_get_pgd_vmalloc64()
956 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); in build_get_pgd_vmalloc64()
958 UASM_i_LW(p, 1, scratchpad_offset(0), 0); in build_get_pgd_vmalloc64()
960 uasm_i_nop(p); in build_get_pgd_vmalloc64()
971 void build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) in build_get_pgde32() argument
975 uasm_i_mfc0(p, ptr, c0_kscratch(), pgd_reg); in build_get_pgde32()
976 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ in build_get_pgde32()
982 uasm_i_mfc0(p, ptr, SMP_CPUID_REG); in build_get_pgde32()
983 UASM_i_LA_mostly(p, tmp, pgdc); in build_get_pgde32()
984 uasm_i_srl(p, ptr, ptr, SMP_CPUID_PTRSHIFT); in build_get_pgde32()
985 uasm_i_addu(p, ptr, tmp, ptr); in build_get_pgde32()
987 UASM_i_LA_mostly(p, ptr, pgdc); in build_get_pgde32()
989 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ in build_get_pgde32()
990 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); in build_get_pgde32()
992 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */ in build_get_pgde32()
993 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2); in build_get_pgde32()
994 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ in build_get_pgde32()
1000 static void build_adjust_context(u32 **p, unsigned int ctx) in build_adjust_context() argument
1022 UASM_i_SRL(p, ctx, ctx, shift); in build_adjust_context()
1023 uasm_i_andi(p, ctx, ctx, mask); in build_adjust_context()
1026 void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) in build_get_ptep() argument
1037 UASM_i_LW(p, ptr, 0, ptr); in build_get_ptep()
1038 GET_CONTEXT(p, tmp); /* get context reg */ in build_get_ptep()
1042 GET_CONTEXT(p, tmp); /* get context reg */ in build_get_ptep()
1043 UASM_i_LW(p, ptr, 0, ptr); in build_get_ptep()
1047 build_adjust_context(p, tmp); in build_get_ptep()
1048 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ in build_get_ptep()
1052 void build_update_entries(u32 **p, unsigned int tmp, unsigned int ptep) in build_update_entries() argument
1064 uasm_i_lw(p, tmp, pte_off_even, ptep); /* even pte */ in build_update_entries()
1065 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); in build_update_entries()
1066 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); in build_update_entries()
1069 uasm_i_lw(p, tmp, 0, ptep); in build_update_entries()
1070 uasm_i_ext(p, tmp, tmp, 0, 24); in build_update_entries()
1071 uasm_i_mthc0(p, tmp, C0_ENTRYLO0); in build_update_entries()
1074 uasm_i_lw(p, tmp, pte_off_odd, ptep); /* odd pte */ in build_update_entries()
1075 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); in build_update_entries()
1076 UASM_i_MTC0(p, tmp, C0_ENTRYLO1); in build_update_entries()
1079 uasm_i_lw(p, tmp, sizeof(pte_t), ptep); in build_update_entries()
1080 uasm_i_ext(p, tmp, tmp, 0, 24); in build_update_entries()
1081 uasm_i_mthc0(p, tmp, C0_ENTRYLO1); in build_update_entries()
1086 UASM_i_LW(p, tmp, pte_off_even, ptep); /* get even pte */ in build_update_entries()
1087 UASM_i_LW(p, ptep, pte_off_odd, ptep); /* get odd pte */ in build_update_entries()
1089 build_tlb_probe_entry(p); in build_update_entries()
1090 build_convert_pte_to_entrylo(p, tmp); in build_update_entries()
1092 UASM_i_MTC0(p, 0, C0_ENTRYLO0); in build_update_entries()
1093 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ in build_update_entries()
1094 build_convert_pte_to_entrylo(p, ptep); in build_update_entries()
1096 uasm_i_mfc0(p, tmp, C0_INDEX); in build_update_entries()
1098 UASM_i_MTC0(p, 0, C0_ENTRYLO1); in build_update_entries()
1099 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ in build_update_entries()
1110 build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, in build_fast_tlb_refill_handler() argument
1124 UASM_i_MFC0(p, tmp, C0_BADVADDR); in build_fast_tlb_refill_handler()
1127 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); in build_fast_tlb_refill_handler()
1129 UASM_i_MFC0(p, ptr, C0_CONTEXT); in build_fast_tlb_refill_handler()
1132 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg); in build_fast_tlb_refill_handler()
1134 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); in build_fast_tlb_refill_handler()
1136 uasm_i_dsrl_safe(p, scratch, tmp, in build_fast_tlb_refill_handler()
1138 uasm_il_bnez(p, r, scratch, label_vmalloc); in build_fast_tlb_refill_handler()
1143 uasm_i_dins(p, ptr, 0, 0, 23); in build_fast_tlb_refill_handler()
1147 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); in build_fast_tlb_refill_handler()
1149 UASM_i_MFC0(p, ptr, C0_CONTEXT); in build_fast_tlb_refill_handler()
1151 UASM_i_MFC0(p, tmp, C0_BADVADDR); in build_fast_tlb_refill_handler()
1154 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg); in build_fast_tlb_refill_handler()
1156 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); in build_fast_tlb_refill_handler()
1160 uasm_i_dins(p, ptr, 0, 0, 23); in build_fast_tlb_refill_handler()
1162 uasm_il_bltz(p, r, tmp, label_vmalloc); in build_fast_tlb_refill_handler()
1168 uasm_i_ori(p, ptr, ptr, 0x540); in build_fast_tlb_refill_handler()
1169 uasm_i_drotr(p, ptr, ptr, 11); in build_fast_tlb_refill_handler()
1180 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); in build_fast_tlb_refill_handler()
1182 uasm_l_vmalloc_done(l, *p); in build_fast_tlb_refill_handler()
1192 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); in build_fast_tlb_refill_handler()
1195 GET_CONTEXT(p, tmp); /* get context reg */ in build_fast_tlb_refill_handler()
1197 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3); in build_fast_tlb_refill_handler()
1200 UASM_i_LWX(p, LOC_PTEP, scratch, ptr); in build_fast_tlb_refill_handler()
1202 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */ in build_fast_tlb_refill_handler()
1203 uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */ in build_fast_tlb_refill_handler()
1208 uasm_i_dsrl_safe(p, scratch, tmp, PUD_SHIFT - 3); in build_fast_tlb_refill_handler()
1209 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PUD - 1) << 3); in build_fast_tlb_refill_handler()
1212 UASM_i_LWX(p, ptr, scratch, ptr); in build_fast_tlb_refill_handler()
1214 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */ in build_fast_tlb_refill_handler()
1215 UASM_i_LW(p, ptr, 0, ptr); in build_fast_tlb_refill_handler()
1223 uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3); in build_fast_tlb_refill_handler()
1224 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3); in build_fast_tlb_refill_handler()
1225 GET_CONTEXT(p, tmp); /* get context reg */ in build_fast_tlb_refill_handler()
1228 UASM_i_LWX(p, scratch, scratch, ptr); in build_fast_tlb_refill_handler()
1230 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */ in build_fast_tlb_refill_handler()
1231 UASM_i_LW(p, scratch, 0, ptr); in build_fast_tlb_refill_handler()
1235 build_adjust_context(p, tmp); in build_fast_tlb_refill_handler()
1238 uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update); in build_fast_tlb_refill_handler()
1245 uasm_i_nop(p); in build_fast_tlb_refill_handler()
1253 UASM_i_LWX(p, even, scratch, tmp); in build_fast_tlb_refill_handler()
1254 UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t)); in build_fast_tlb_refill_handler()
1255 UASM_i_LWX(p, odd, scratch, tmp); in build_fast_tlb_refill_handler()
1257 UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */ in build_fast_tlb_refill_handler()
1260 UASM_i_LW(p, even, 0, ptr); /* get even pte */ in build_fast_tlb_refill_handler()
1261 UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */ in build_fast_tlb_refill_handler()
1264 uasm_i_drotr(p, even, even, ilog2(_PAGE_GLOBAL)); in build_fast_tlb_refill_handler()
1265 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ in build_fast_tlb_refill_handler()
1266 uasm_i_drotr(p, odd, odd, ilog2(_PAGE_GLOBAL)); in build_fast_tlb_refill_handler()
1268 uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL)); in build_fast_tlb_refill_handler()
1269 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ in build_fast_tlb_refill_handler()
1270 uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL)); in build_fast_tlb_refill_handler()
1272 UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */ in build_fast_tlb_refill_handler()
1275 uasm_i_ehb(p); in build_fast_tlb_refill_handler()
1276 UASM_i_MFC0(p, scratch, c0_kscratch(), c0_scratch_reg); in build_fast_tlb_refill_handler()
1277 build_tlb_write_entry(p, l, r, tlb_random); in build_fast_tlb_refill_handler()
1278 uasm_l_leave(l, *p); in build_fast_tlb_refill_handler()
1281 build_tlb_write_entry(p, l, r, tlb_random); in build_fast_tlb_refill_handler()
1282 uasm_l_leave(l, *p); in build_fast_tlb_refill_handler()
1283 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); in build_fast_tlb_refill_handler()
1285 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); in build_fast_tlb_refill_handler()
1286 build_tlb_write_entry(p, l, r, tlb_random); in build_fast_tlb_refill_handler()
1287 uasm_l_leave(l, *p); in build_fast_tlb_refill_handler()
1291 uasm_i_eret(p); /* return from trap */ in build_fast_tlb_refill_handler()
1306 u32 *p = tlb_handler; in build_r4000_tlb_refill_handler() local
1320 htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1, in build_r4000_tlb_refill_handler()
1334 uasm_i_dmfc0(&p, K0, C0_BADVADDR); in build_r4000_tlb_refill_handler()
1335 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); in build_r4000_tlb_refill_handler()
1336 uasm_i_xor(&p, K0, K0, K1); in build_r4000_tlb_refill_handler()
1337 uasm_i_dsrl_safe(&p, K1, K0, 62); in build_r4000_tlb_refill_handler()
1338 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); in build_r4000_tlb_refill_handler()
1339 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); in build_r4000_tlb_refill_handler()
1340 uasm_i_or(&p, K0, K0, K1); in build_r4000_tlb_refill_handler()
1341 uasm_il_bnez(&p, &r, K0, label_leave); in build_r4000_tlb_refill_handler()
1346 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ in build_r4000_tlb_refill_handler()
1348 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ in build_r4000_tlb_refill_handler()
1352 build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update); in build_r4000_tlb_refill_handler()
1355 build_get_ptep(&p, K0, K1); in build_r4000_tlb_refill_handler()
1356 build_update_entries(&p, K0, K1); in build_r4000_tlb_refill_handler()
1357 build_tlb_write_entry(&p, &l, &r, tlb_random); in build_r4000_tlb_refill_handler()
1358 uasm_l_leave(&l, p); in build_r4000_tlb_refill_handler()
1359 uasm_i_eret(&p); /* return from trap */ in build_r4000_tlb_refill_handler()
1362 uasm_l_tlb_huge_update(&l, p); in build_r4000_tlb_refill_handler()
1364 UASM_i_LW(&p, htlb_info.huge_pte, 0, K1); in build_r4000_tlb_refill_handler()
1365 build_huge_update_entries(&p, htlb_info.huge_pte, K1); in build_r4000_tlb_refill_handler()
1366 build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random, in build_r4000_tlb_refill_handler()
1371 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode); in build_r4000_tlb_refill_handler()
1386 if ((p - tlb_handler) > 64) in build_r4000_tlb_refill_handler()
1393 uasm_copy_handler(relocs, labels, tlb_handler, p, f); in build_r4000_tlb_refill_handler()
1394 final_len = p - tlb_handler; in build_r4000_tlb_refill_handler()
1397 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1) in build_r4000_tlb_refill_handler()
1398 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3) in build_r4000_tlb_refill_handler()
1406 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) { in build_r4000_tlb_refill_handler()
1408 uasm_copy_handler(relocs, labels, tlb_handler, p, f); in build_r4000_tlb_refill_handler()
1409 final_len = p - tlb_handler; in build_r4000_tlb_refill_handler()
1429 split < p - MIPS64_REFILL_INSNS) in build_r4000_tlb_refill_handler()
1468 uasm_copy_handler(relocs, labels, split, p, final_handler); in build_r4000_tlb_refill_handler()
1470 (p - split); in build_r4000_tlb_refill_handler()
1534 u32 *p = tlb_handler; in build_loongson3_tlb_refill_handler() local
1543 uasm_i_dmfc0(&p, K0, C0_BADVADDR); in build_loongson3_tlb_refill_handler()
1544 uasm_i_dsrl_safe(&p, K1, K0, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); in build_loongson3_tlb_refill_handler()
1545 uasm_il_beqz(&p, &r, K1, label_vmalloc); in build_loongson3_tlb_refill_handler()
1546 uasm_i_nop(&p); in build_loongson3_tlb_refill_handler()
1548 uasm_il_bgez(&p, &r, K0, label_large_segbits_fault); in build_loongson3_tlb_refill_handler()
1549 uasm_i_nop(&p); in build_loongson3_tlb_refill_handler()
1550 uasm_l_vmalloc(&l, p); in build_loongson3_tlb_refill_handler()
1553 uasm_i_dmfc0(&p, K1, C0_PGD); in build_loongson3_tlb_refill_handler()
1555 uasm_i_lddir(&p, K0, K1, 3); /* global page dir */ in build_loongson3_tlb_refill_handler()
1557 uasm_i_lddir(&p, K1, K0, 1); /* middle page dir */ in build_loongson3_tlb_refill_handler()
1559 uasm_i_ldpte(&p, K1, 0); /* even */ in build_loongson3_tlb_refill_handler()
1560 uasm_i_ldpte(&p, K1, 1); /* odd */ in build_loongson3_tlb_refill_handler()
1561 uasm_i_tlbwr(&p); in build_loongson3_tlb_refill_handler()
1565 uasm_i_lui(&p, K0, PM_DEFAULT_MASK >> 16); in build_loongson3_tlb_refill_handler()
1566 uasm_i_ori(&p, K0, K0, PM_DEFAULT_MASK & 0xffff); in build_loongson3_tlb_refill_handler()
1567 uasm_i_mtc0(&p, K0, C0_PAGEMASK); in build_loongson3_tlb_refill_handler()
1569 uasm_i_ori(&p, K0, 0, PM_DEFAULT_MASK); in build_loongson3_tlb_refill_handler()
1570 uasm_i_mtc0(&p, K0, C0_PAGEMASK); in build_loongson3_tlb_refill_handler()
1572 uasm_i_mtc0(&p, 0, C0_PAGEMASK); in build_loongson3_tlb_refill_handler()
1575 uasm_i_eret(&p); in build_loongson3_tlb_refill_handler()
1578 uasm_l_large_segbits_fault(&l, p); in build_loongson3_tlb_refill_handler()
1579 UASM_i_LA(&p, K1, (unsigned long)tlb_do_page_fault_0); in build_loongson3_tlb_refill_handler()
1580 uasm_i_jr(&p, K1); in build_loongson3_tlb_refill_handler()
1581 uasm_i_nop(&p); in build_loongson3_tlb_refill_handler()
1596 u32 *p = (u32 *)msk_isa16_mode((ulong)tlbmiss_handler_setup_pgd); in build_setup_pgd() local
1601 memset(p, 0, tlbmiss_handler_setup_pgd_end - (char *)p); in build_setup_pgd()
1617 UASM_i_SRA(&p, a1, a0, 29); in build_setup_pgd()
1618 UASM_i_ADDIU(&p, a1, a1, 4); in build_setup_pgd()
1619 uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1); in build_setup_pgd()
1620 uasm_i_nop(&p); in build_setup_pgd()
1621 uasm_i_dinsm(&p, a0, 0, 29, 64 - 29); in build_setup_pgd()
1622 uasm_l_tlbl_goaround1(&l, p); in build_setup_pgd()
1623 UASM_i_SLL(&p, a0, a0, 11); in build_setup_pgd()
1624 UASM_i_MTC0(&p, a0, C0_CONTEXT); in build_setup_pgd()
1625 uasm_i_jr(&p, 31); in build_setup_pgd()
1626 uasm_i_ehb(&p); in build_setup_pgd()
1630 UASM_i_MTC0(&p, a0, C0_PWBASE); in build_setup_pgd()
1632 UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg); in build_setup_pgd()
1633 uasm_i_jr(&p, 31); in build_setup_pgd()
1634 uasm_i_ehb(&p); in build_setup_pgd()
1639 UASM_i_CPUID_MFC0(&p, a1, SMP_CPUID_REG); in build_setup_pgd()
1640 UASM_i_SRL_SAFE(&p, a1, a1, SMP_CPUID_PTRSHIFT); in build_setup_pgd()
1641 UASM_i_LA_mostly(&p, a2, pgdc); in build_setup_pgd()
1642 UASM_i_ADDU(&p, a2, a2, a1); in build_setup_pgd()
1643 UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2); in build_setup_pgd()
1645 UASM_i_LA_mostly(&p, a2, pgdc); in build_setup_pgd()
1646 UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2); in build_setup_pgd()
1651 UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg); in build_setup_pgd()
1652 uasm_i_jr(&p, 31); in build_setup_pgd()
1653 uasm_i_ehb(&p); in build_setup_pgd()
1655 uasm_i_jr(&p, 31); in build_setup_pgd()
1656 uasm_i_nop(&p); in build_setup_pgd()
1659 if (p >= (u32 *)tlbmiss_handler_setup_pgd_end) in build_setup_pgd()
1664 (unsigned int)(p - (u32 *)tlbmiss_handler_setup_pgd)); in build_setup_pgd()
1671 iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) in iPTE_LW() argument
1675 uasm_i_sync(p, 0); in iPTE_LW()
1678 uasm_i_lld(p, pte, 0, ptr); in iPTE_LW()
1681 UASM_i_LL(p, pte, 0, ptr); in iPTE_LW()
1685 uasm_i_ld(p, pte, 0, ptr); in iPTE_LW()
1688 UASM_i_LW(p, pte, 0, ptr); in iPTE_LW()
1693 iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, in iPTE_SW() argument
1700 uasm_i_lui(p, scratch, swmode >> 16); in iPTE_SW()
1701 uasm_i_or(p, pte, pte, scratch); in iPTE_SW()
1704 uasm_i_ori(p, pte, pte, mode); in iPTE_SW()
1710 uasm_i_scd(p, pte, 0, ptr); in iPTE_SW()
1713 UASM_i_SC(p, pte, 0, ptr); in iPTE_SW()
1716 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change); in iPTE_SW()
1718 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); in iPTE_SW()
1723 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr); in iPTE_SW()
1724 uasm_i_ori(p, pte, pte, hwmode); in iPTE_SW()
1726 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr); in iPTE_SW()
1727 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); in iPTE_SW()
1729 uasm_i_lw(p, pte, 0, ptr); in iPTE_SW()
1731 uasm_i_nop(p); in iPTE_SW()
1733 uasm_i_nop(p); in iPTE_SW()
1738 uasm_i_sd(p, pte, 0, ptr); in iPTE_SW()
1741 UASM_i_SW(p, pte, 0, ptr); in iPTE_SW()
1745 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr); in iPTE_SW()
1746 uasm_i_ori(p, pte, pte, hwmode); in iPTE_SW()
1748 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr); in iPTE_SW()
1749 uasm_i_lw(p, pte, 0, ptr); in iPTE_SW()
1761 build_pte_present(u32 **p, struct uasm_reloc **r, in build_pte_present() argument
1769 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid); in build_pte_present()
1770 uasm_i_nop(p); in build_pte_present()
1773 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT); in build_pte_present()
1776 uasm_i_andi(p, t, cur, 1); in build_pte_present()
1777 uasm_il_beqz(p, r, t, lid); in build_pte_present()
1780 iPTE_LW(p, pte, ptr); in build_pte_present()
1784 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT); in build_pte_present()
1787 uasm_i_andi(p, t, cur, in build_pte_present()
1789 uasm_i_xori(p, t, t, _PAGE_PRESENT >> _PAGE_PRESENT_SHIFT); in build_pte_present()
1790 uasm_il_bnez(p, r, t, lid); in build_pte_present()
1793 iPTE_LW(p, pte, ptr); in build_pte_present()
1799 build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, in build_make_valid() argument
1804 iPTE_SW(p, r, pte, ptr, mode, scratch); in build_make_valid()
1812 build_pte_writable(u32 **p, struct uasm_reloc **r, in build_pte_writable() argument
1820 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT); in build_pte_writable()
1823 uasm_i_andi(p, t, cur, in build_pte_writable()
1825 uasm_i_xori(p, t, t, in build_pte_writable()
1827 uasm_il_bnez(p, r, t, lid); in build_pte_writable()
1830 iPTE_LW(p, pte, ptr); in build_pte_writable()
1832 uasm_i_nop(p); in build_pte_writable()
1839 build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, in build_make_write() argument
1845 iPTE_SW(p, r, pte, ptr, mode, scratch); in build_make_write()
1853 build_pte_modifiable(u32 **p, struct uasm_reloc **r, in build_pte_modifiable() argument
1858 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid); in build_pte_modifiable()
1859 uasm_i_nop(p); in build_pte_modifiable()
1862 uasm_i_srl(p, t, pte, _PAGE_WRITE_SHIFT); in build_pte_modifiable()
1863 uasm_i_andi(p, t, t, 1); in build_pte_modifiable()
1864 uasm_il_beqz(p, r, t, lid); in build_pte_modifiable()
1867 iPTE_LW(p, pte, ptr); in build_pte_modifiable()
1883 build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) in build_r3000_pte_reload_tlbwi() argument
1885 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ in build_r3000_pte_reload_tlbwi()
1886 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */ in build_r3000_pte_reload_tlbwi()
1887 uasm_i_tlbwi(p); in build_r3000_pte_reload_tlbwi()
1888 uasm_i_jr(p, tmp); in build_r3000_pte_reload_tlbwi()
1889 uasm_i_rfe(p); /* branch delay */ in build_r3000_pte_reload_tlbwi()
1899 build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, in build_r3000_tlb_reload_write() argument
1903 uasm_i_mfc0(p, tmp, C0_INDEX); in build_r3000_tlb_reload_write()
1904 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ in build_r3000_tlb_reload_write()
1905 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */ in build_r3000_tlb_reload_write()
1906 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */ in build_r3000_tlb_reload_write()
1907 uasm_i_tlbwi(p); /* cp0 delay */ in build_r3000_tlb_reload_write()
1908 uasm_i_jr(p, tmp); in build_r3000_tlb_reload_write()
1909 uasm_i_rfe(p); /* branch delay */ in build_r3000_tlb_reload_write()
1910 uasm_l_r3000_write_probe_fail(l, *p); in build_r3000_tlb_reload_write()
1911 uasm_i_tlbwr(p); /* cp0 delay */ in build_r3000_tlb_reload_write()
1912 uasm_i_jr(p, tmp); in build_r3000_tlb_reload_write()
1913 uasm_i_rfe(p); /* branch delay */ in build_r3000_tlb_reload_write()
1917 build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, in build_r3000_tlbchange_handler_head() argument
1922 uasm_i_mfc0(p, pte, C0_BADVADDR); in build_r3000_tlbchange_handler_head()
1923 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */ in build_r3000_tlbchange_handler_head()
1924 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); in build_r3000_tlbchange_handler_head()
1925 uasm_i_srl(p, pte, pte, 22); /* load delay */ in build_r3000_tlbchange_handler_head()
1926 uasm_i_sll(p, pte, pte, 2); in build_r3000_tlbchange_handler_head()
1927 uasm_i_addu(p, ptr, ptr, pte); in build_r3000_tlbchange_handler_head()
1928 uasm_i_mfc0(p, pte, C0_CONTEXT); in build_r3000_tlbchange_handler_head()
1929 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */ in build_r3000_tlbchange_handler_head()
1930 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */ in build_r3000_tlbchange_handler_head()
1931 uasm_i_addu(p, ptr, ptr, pte); in build_r3000_tlbchange_handler_head()
1932 uasm_i_lw(p, pte, 0, ptr); in build_r3000_tlbchange_handler_head()
1933 uasm_i_tlbp(p); /* load delay */ in build_r3000_tlbchange_handler_head()
1938 u32 *p = (u32 *)handle_tlbl; in build_r3000_tlb_load_handler() local
1942 memset(p, 0, handle_tlbl_end - (char *)p); in build_r3000_tlb_load_handler()
1946 build_r3000_tlbchange_handler_head(&p, K0, K1); in build_r3000_tlb_load_handler()
1947 build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl); in build_r3000_tlb_load_handler()
1948 uasm_i_nop(&p); /* load delay */ in build_r3000_tlb_load_handler()
1949 build_make_valid(&p, &r, K0, K1, -1); in build_r3000_tlb_load_handler()
1950 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); in build_r3000_tlb_load_handler()
1952 uasm_l_nopage_tlbl(&l, p); in build_r3000_tlb_load_handler()
1953 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); in build_r3000_tlb_load_handler()
1954 uasm_i_nop(&p); in build_r3000_tlb_load_handler()
1956 if (p >= (u32 *)handle_tlbl_end) in build_r3000_tlb_load_handler()
1961 (unsigned int)(p - (u32 *)handle_tlbl)); in build_r3000_tlb_load_handler()
1968 u32 *p = (u32 *)handle_tlbs; in build_r3000_tlb_store_handler() local
1972 memset(p, 0, handle_tlbs_end - (char *)p); in build_r3000_tlb_store_handler()
1976 build_r3000_tlbchange_handler_head(&p, K0, K1); in build_r3000_tlb_store_handler()
1977 build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs); in build_r3000_tlb_store_handler()
1978 uasm_i_nop(&p); /* load delay */ in build_r3000_tlb_store_handler()
1979 build_make_write(&p, &r, K0, K1, -1); in build_r3000_tlb_store_handler()
1980 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); in build_r3000_tlb_store_handler()
1982 uasm_l_nopage_tlbs(&l, p); in build_r3000_tlb_store_handler()
1983 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); in build_r3000_tlb_store_handler()
1984 uasm_i_nop(&p); in build_r3000_tlb_store_handler()
1986 if (p >= (u32 *)handle_tlbs_end) in build_r3000_tlb_store_handler()
1991 (unsigned int)(p - (u32 *)handle_tlbs)); in build_r3000_tlb_store_handler()
1998 u32 *p = (u32 *)handle_tlbm; in build_r3000_tlb_modify_handler() local
2002 memset(p, 0, handle_tlbm_end - (char *)p); in build_r3000_tlb_modify_handler()
2006 build_r3000_tlbchange_handler_head(&p, K0, K1); in build_r3000_tlb_modify_handler()
2007 build_pte_modifiable(&p, &r, K0, K1, -1, label_nopage_tlbm); in build_r3000_tlb_modify_handler()
2008 uasm_i_nop(&p); /* load delay */ in build_r3000_tlb_modify_handler()
2009 build_make_write(&p, &r, K0, K1, -1); in build_r3000_tlb_modify_handler()
2010 build_r3000_pte_reload_tlbwi(&p, K0, K1); in build_r3000_tlb_modify_handler()
2012 uasm_l_nopage_tlbm(&l, p); in build_r3000_tlb_modify_handler()
2013 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); in build_r3000_tlb_modify_handler()
2014 uasm_i_nop(&p); in build_r3000_tlb_modify_handler()
2016 if (p >= (u32 *)handle_tlbm_end) in build_r3000_tlb_modify_handler()
2021 (unsigned int)(p - (u32 *)handle_tlbm)); in build_r3000_tlb_modify_handler()
2051 build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, in build_r4000_tlbchange_handler_head() argument
2054 struct work_registers wr = build_get_work_registers(p); in build_r4000_tlbchange_handler_head()
2057 build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */ in build_r4000_tlbchange_handler_head()
2059 build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */ in build_r4000_tlbchange_handler_head()
2068 build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update); in build_r4000_tlbchange_handler_head()
2071 UASM_i_MFC0(p, wr.r1, C0_BADVADDR); in build_r4000_tlbchange_handler_head()
2072 UASM_i_LW(p, wr.r2, 0, wr.r2); in build_r4000_tlbchange_handler_head()
2073 UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); in build_r4000_tlbchange_handler_head()
2074 uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2); in build_r4000_tlbchange_handler_head()
2075 UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1); in build_r4000_tlbchange_handler_head()
2078 uasm_l_smp_pgtable_change(l, *p); in build_r4000_tlbchange_handler_head()
2080 iPTE_LW(p, wr.r1, wr.r2); /* get even pte */ in build_r4000_tlbchange_handler_head()
2082 build_tlb_probe_entry(p); in build_r4000_tlbchange_handler_head()
2085 uasm_i_ehb(p); in build_r4000_tlbchange_handler_head()
2086 uasm_i_mfc0(p, wr.r3, C0_INDEX); in build_r4000_tlbchange_handler_head()
2087 uasm_il_bltz(p, r, wr.r3, label_leave); in build_r4000_tlbchange_handler_head()
2088 uasm_i_nop(p); in build_r4000_tlbchange_handler_head()
2095 build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, in build_r4000_tlbchange_handler_tail() argument
2099 uasm_i_ori(p, ptr, ptr, sizeof(pte_t)); in build_r4000_tlbchange_handler_tail()
2100 uasm_i_xori(p, ptr, ptr, sizeof(pte_t)); in build_r4000_tlbchange_handler_tail()
2101 build_update_entries(p, tmp, ptr); in build_r4000_tlbchange_handler_tail()
2102 build_tlb_write_entry(p, l, r, tlb_indexed); in build_r4000_tlbchange_handler_tail()
2103 uasm_l_leave(l, *p); in build_r4000_tlbchange_handler_tail()
2104 build_restore_work_registers(p); in build_r4000_tlbchange_handler_tail()
2105 uasm_i_eret(p); /* return from trap */ in build_r4000_tlbchange_handler_tail()
2108 build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill); in build_r4000_tlbchange_handler_tail()
2114 u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbl); in build_r4000_tlb_load_handler() local
2119 memset(p, 0, handle_tlbl_end - (char *)p); in build_r4000_tlb_load_handler()
2126 uasm_i_dmfc0(&p, K0, C0_BADVADDR); in build_r4000_tlb_load_handler()
2127 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); in build_r4000_tlb_load_handler()
2128 uasm_i_xor(&p, K0, K0, K1); in build_r4000_tlb_load_handler()
2129 uasm_i_dsrl_safe(&p, K1, K0, 62); in build_r4000_tlb_load_handler()
2130 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); in build_r4000_tlb_load_handler()
2131 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); in build_r4000_tlb_load_handler()
2132 uasm_i_or(&p, K0, K0, K1); in build_r4000_tlb_load_handler()
2133 uasm_il_bnez(&p, &r, K0, label_leave); in build_r4000_tlb_load_handler()
2137 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); in build_r4000_tlb_load_handler()
2138 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); in build_r4000_tlb_load_handler()
2140 build_tlb_probe_entry(&p); in build_r4000_tlb_load_handler()
2148 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), in build_r4000_tlb_load_handler()
2151 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); in build_r4000_tlb_load_handler()
2152 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1); in build_r4000_tlb_load_handler()
2154 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2164 uasm_i_tlbr(&p); in build_r4000_tlb_load_handler()
2169 uasm_i_ehb(&p); in build_r4000_tlb_load_handler()
2180 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); in build_r4000_tlb_load_handler()
2182 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); in build_r4000_tlb_load_handler()
2183 uasm_i_beqz(&p, wr.r3, 8); in build_r4000_tlb_load_handler()
2186 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); in build_r4000_tlb_load_handler()
2188 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); in build_r4000_tlb_load_handler()
2194 uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl); in build_r4000_tlb_load_handler()
2195 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2196 uasm_l_tlbl_goaround1(&l, p); in build_r4000_tlb_load_handler()
2198 uasm_i_andi(&p, wr.r3, wr.r3, 2); in build_r4000_tlb_load_handler()
2199 uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl); in build_r4000_tlb_load_handler()
2200 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2202 uasm_l_tlbl_goaround1(&l, p); in build_r4000_tlb_load_handler()
2204 build_make_valid(&p, &r, wr.r1, wr.r2, wr.r3); in build_r4000_tlb_load_handler()
2205 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); in build_r4000_tlb_load_handler()
2212 uasm_l_tlb_huge_update(&l, p); in build_r4000_tlb_load_handler()
2213 iPTE_LW(&p, wr.r1, wr.r2); in build_r4000_tlb_load_handler()
2214 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); in build_r4000_tlb_load_handler()
2215 build_tlb_probe_entry(&p); in build_r4000_tlb_load_handler()
2223 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), in build_r4000_tlb_load_handler()
2226 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); in build_r4000_tlb_load_handler()
2227 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); in build_r4000_tlb_load_handler()
2229 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2239 uasm_i_tlbr(&p); in build_r4000_tlb_load_handler()
2244 uasm_i_ehb(&p); in build_r4000_tlb_load_handler()
2255 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); in build_r4000_tlb_load_handler()
2257 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); in build_r4000_tlb_load_handler()
2258 uasm_i_beqz(&p, wr.r3, 8); in build_r4000_tlb_load_handler()
2261 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); in build_r4000_tlb_load_handler()
2263 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); in build_r4000_tlb_load_handler()
2269 uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2); in build_r4000_tlb_load_handler()
2271 uasm_i_andi(&p, wr.r3, wr.r3, 2); in build_r4000_tlb_load_handler()
2272 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); in build_r4000_tlb_load_handler()
2275 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2280 build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0); in build_r4000_tlb_load_handler()
2282 uasm_l_tlbl_goaround2(&l, p); in build_r4000_tlb_load_handler()
2284 uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID)); in build_r4000_tlb_load_handler()
2285 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 1); in build_r4000_tlb_load_handler()
2288 uasm_l_nopage_tlbl(&l, p); in build_r4000_tlb_load_handler()
2290 uasm_i_sync(&p, 0); in build_r4000_tlb_load_handler()
2291 build_restore_work_registers(&p); in build_r4000_tlb_load_handler()
2294 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_0)); in build_r4000_tlb_load_handler()
2295 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_0)); in build_r4000_tlb_load_handler()
2296 uasm_i_jr(&p, K0); in build_r4000_tlb_load_handler()
2299 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); in build_r4000_tlb_load_handler()
2300 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2302 if (p >= (u32 *)handle_tlbl_end) in build_r4000_tlb_load_handler()
2307 (unsigned int)(p - (u32 *)handle_tlbl)); in build_r4000_tlb_load_handler()
2314 u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbs); in build_r4000_tlb_store_handler() local
2319 memset(p, 0, handle_tlbs_end - (char *)p); in build_r4000_tlb_store_handler()
2323 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); in build_r4000_tlb_store_handler()
2324 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); in build_r4000_tlb_store_handler()
2326 build_tlb_probe_entry(&p); in build_r4000_tlb_store_handler()
2327 build_make_write(&p, &r, wr.r1, wr.r2, wr.r3); in build_r4000_tlb_store_handler()
2328 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); in build_r4000_tlb_store_handler()
2335 uasm_l_tlb_huge_update(&l, p); in build_r4000_tlb_store_handler()
2336 iPTE_LW(&p, wr.r1, wr.r2); in build_r4000_tlb_store_handler()
2337 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); in build_r4000_tlb_store_handler()
2338 build_tlb_probe_entry(&p); in build_r4000_tlb_store_handler()
2339 uasm_i_ori(&p, wr.r1, wr.r1, in build_r4000_tlb_store_handler()
2341 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 1); in build_r4000_tlb_store_handler()
2344 uasm_l_nopage_tlbs(&l, p); in build_r4000_tlb_store_handler()
2346 uasm_i_sync(&p, 0); in build_r4000_tlb_store_handler()
2347 build_restore_work_registers(&p); in build_r4000_tlb_store_handler()
2350 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1)); in build_r4000_tlb_store_handler()
2351 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1)); in build_r4000_tlb_store_handler()
2352 uasm_i_jr(&p, K0); in build_r4000_tlb_store_handler()
2355 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); in build_r4000_tlb_store_handler()
2356 uasm_i_nop(&p); in build_r4000_tlb_store_handler()
2358 if (p >= (u32 *)handle_tlbs_end) in build_r4000_tlb_store_handler()
2363 (unsigned int)(p - (u32 *)handle_tlbs)); in build_r4000_tlb_store_handler()
2370 u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbm); in build_r4000_tlb_modify_handler() local
2375 memset(p, 0, handle_tlbm_end - (char *)p); in build_r4000_tlb_modify_handler()
2379 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); in build_r4000_tlb_modify_handler()
2380 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); in build_r4000_tlb_modify_handler()
2382 build_tlb_probe_entry(&p); in build_r4000_tlb_modify_handler()
2384 build_make_write(&p, &r, wr.r1, wr.r2, wr.r3); in build_r4000_tlb_modify_handler()
2385 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); in build_r4000_tlb_modify_handler()
2392 uasm_l_tlb_huge_update(&l, p); in build_r4000_tlb_modify_handler()
2393 iPTE_LW(&p, wr.r1, wr.r2); in build_r4000_tlb_modify_handler()
2394 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); in build_r4000_tlb_modify_handler()
2395 build_tlb_probe_entry(&p); in build_r4000_tlb_modify_handler()
2396 uasm_i_ori(&p, wr.r1, wr.r1, in build_r4000_tlb_modify_handler()
2398 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 0); in build_r4000_tlb_modify_handler()
2401 uasm_l_nopage_tlbm(&l, p); in build_r4000_tlb_modify_handler()
2403 uasm_i_sync(&p, 0); in build_r4000_tlb_modify_handler()
2404 build_restore_work_registers(&p); in build_r4000_tlb_modify_handler()
2407 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1)); in build_r4000_tlb_modify_handler()
2408 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1)); in build_r4000_tlb_modify_handler()
2409 uasm_i_jr(&p, K0); in build_r4000_tlb_modify_handler()
2412 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); in build_r4000_tlb_modify_handler()
2413 uasm_i_nop(&p); in build_r4000_tlb_modify_handler()
2415 if (p >= (u32 *)handle_tlbm_end) in build_r4000_tlb_modify_handler()
2420 (unsigned int)(p - (u32 *)handle_tlbm)); in build_r4000_tlb_modify_handler()