Lines Matching +full:llp +full:- +full:based
1 // SPDX-License-Identifier: GPL-2.0-or-later
6 * Based on earlier code written by:
12 #include <asm/asm-prototypes.h>
17 #include <asm/ppc-opcode.h>
27 #include <asm/code-patching.h>
84 * ignores all other bits from 0-27, so just clear them all. in assert_slb_presence()
86 ea &= ~((1UL << SID_SHIFT) - 1); in assert_slb_presence()
104 WRITE_ONCE(p->save_area[index].esid, 0); in slb_shadow_update()
105 WRITE_ONCE(p->save_area[index].vsid, cpu_to_be64(mk_vsid_data(ea, ssize, flags))); in slb_shadow_update()
106 WRITE_ONCE(p->save_area[index].esid, cpu_to_be64(mk_esid_data(ea, ssize, index))); in slb_shadow_update()
111 WRITE_ONCE(get_slb_shadow()->save_area[index].esid, cpu_to_be64(index)); in slb_shadow_clear()
144 : "r" (be64_to_cpu(p->save_area[index].vsid)), in __slb_restore_bolted_realmode()
145 "r" (be64_to_cpu(p->save_area[index].esid))); in __slb_restore_bolted_realmode()
148 assert_slb_presence(true, local_paca->kstack); in __slb_restore_bolted_realmode()
157 get_paca()->slb_cache_ptr = 0; in slb_restore_bolted_realmode()
159 get_paca()->slb_kern_bitmap = (1U << SLB_NUM_BOLTED) - 1; in slb_restore_bolted_realmode()
160 get_paca()->slb_used_bitmap = get_paca()->slb_kern_bitmap; in slb_restore_bolted_realmode()
182 * Older processors will ignore this optimisation. Over-invalidation in __slb_flush_and_restore_bolted()
190 ksp_esid_data = be64_to_cpu(p->save_area[KSTACK_INDEX].esid); in __slb_flush_and_restore_bolted()
191 ksp_vsid_data = be64_to_cpu(p->save_area[KSTACK_INDEX].vsid); in __slb_flush_and_restore_bolted()
202 * This flushes non-bolted entries, it can be run in virtual mode. Must
221 assert_slb_presence(true, get_paca()->kstack); in slb_flush_and_restore_bolted()
223 get_paca()->slb_cache_ptr = 0; in slb_flush_and_restore_bolted()
225 get_paca()->slb_kern_bitmap = (1U << SLB_NUM_BOLTED) - 1; in slb_flush_and_restore_bolted()
226 get_paca()->slb_used_bitmap = get_paca()->slb_kern_bitmap; in slb_flush_and_restore_bolted()
235 get_paca()->slb_save_cache_ptr = get_paca()->slb_cache_ptr; in slb_save_contents()
243 slb_ptr->esid = e; in slb_save_contents()
244 slb_ptr->vsid = v; in slb_save_contents()
253 unsigned long llp; in slb_dump_contents() local
259 pr_err("Last SLB entry inserted at slot %d\n", get_paca()->stab_rr); in slb_dump_contents()
262 e = slb_ptr->esid; in slb_dump_contents()
263 v = slb_ptr->vsid; in slb_dump_contents()
275 llp = v & SLB_VSID_LLP; in slb_dump_contents()
277 pr_err(" 1T ESID=%9lx VSID=%13lx LLP:%3lx\n", in slb_dump_contents()
279 (v & ~SLB_VSID_B) >> SLB_VSID_SHIFT_1T, llp); in slb_dump_contents()
281 pr_err(" 256M ESID=%9lx VSID=%13lx LLP:%3lx\n", in slb_dump_contents()
283 (v & ~SLB_VSID_B) >> SLB_VSID_SHIFT, llp); in slb_dump_contents()
286 pr_err("----------------------------------\n"); in slb_dump_contents()
289 pr_err("SLB cache ptr value = %d\n", get_paca()->slb_save_cache_ptr); in slb_dump_contents()
291 n = min_t(int, get_paca()->slb_save_cache_ptr, SLB_CACHE_ENTRIES); in slb_dump_contents()
293 pr_err("%02d EA[0-35]=%9x\n", i, get_paca()->slb_cache[i]); in slb_dump_contents()
296 pr_err("%02d EA[0-35]=%9x\n", i, get_paca()->slb_cache[i]); in slb_dump_contents()
302 * vmalloc is not bolted, so just have to flush non-bolted. in slb_vmalloc_update()
311 for (i = 0; i < ti->slb_preload_nr; i++) { in preload_hit()
314 idx = (ti->slb_preload_tail + i) % SLB_PRELOAD_NR; in preload_hit()
315 if (esid == ti->slb_preload_esid[idx]) in preload_hit()
337 idx = (ti->slb_preload_tail + ti->slb_preload_nr) % SLB_PRELOAD_NR; in preload_add()
338 ti->slb_preload_esid[idx] = esid; in preload_add()
339 if (ti->slb_preload_nr == SLB_PRELOAD_NR) in preload_add()
340 ti->slb_preload_tail = (ti->slb_preload_tail + 1) % SLB_PRELOAD_NR; in preload_add()
342 ti->slb_preload_nr++; in preload_add()
349 if (!ti->slb_preload_nr) in preload_age()
351 ti->slb_preload_nr--; in preload_age()
352 ti->slb_preload_tail = (ti->slb_preload_tail + 1) % SLB_PRELOAD_NR; in preload_age()
358 struct mm_struct *mm = current->mm; in slb_setup_new_exec()
367 if (ti->slb_preload_nr + 2 > SLB_PRELOAD_NR) in slb_setup_new_exec()
394 if (!is_kernel_addr(mm->mmap_base)) { in slb_setup_new_exec()
395 if (preload_add(ti, mm->mmap_base)) in slb_setup_new_exec()
396 slb_allocate_user(mm, mm->mmap_base); in slb_setup_new_exec()
408 struct mm_struct *mm = current->mm; in preload_new_slb_context()
409 unsigned long heap = mm->start_brk; in preload_new_slb_context()
414 if (ti->slb_preload_nr + 3 > SLB_PRELOAD_NR) in preload_new_slb_context()
445 unsigned long slbie_data = get_paca()->slb_cache[index]; in slb_cache_slbie_kernel()
446 unsigned long ksp = get_paca()->kstack; in slb_cache_slbie_kernel()
459 unsigned long slbie_data = get_paca()->slb_cache[index]; in slb_cache_slbie_user()
475 * We need interrupts hard-disabled here, not just soft-disabled, in switch_slb()
485 get_paca()->slb_cache_ptr = 0; in switch_slb()
486 get_paca()->slb_kern_bitmap = (1U << SLB_NUM_BOLTED) - 1; in switch_slb()
498 unsigned long offset = get_paca()->slb_cache_ptr; in switch_slb()
520 get_paca()->slb_kern_bitmap = (1U << SLB_NUM_BOLTED) - 1; in switch_slb()
523 get_paca()->slb_cache_ptr = 0; in switch_slb()
525 get_paca()->slb_used_bitmap = get_paca()->slb_kern_bitmap; in switch_slb()
535 tsk->thread.load_slb++; in switch_slb()
536 if (!tsk->thread.load_slb) { in switch_slb()
543 for (i = 0; i < ti->slb_preload_nr; i++) { in switch_slb()
547 idx = (ti->slb_preload_tail + i) % SLB_PRELOAD_NR; in switch_slb()
548 ea = (unsigned long)ti->slb_preload_esid[idx] << SID_SHIFT; in switch_slb()
575 /* Prepare our SLB miss handler based on our page size */ in slb_initialize()
579 get_paca()->vmalloc_sllp = SLB_VSID_KERNEL | vmalloc_llp; in slb_initialize()
585 pr_devel("SLB: linear LLP = %04lx\n", linear_llp); in slb_initialize()
586 pr_devel("SLB: io LLP = %04lx\n", io_llp); in slb_initialize()
588 pr_devel("SLB: vmemmap LLP = %04lx\n", vmemmap_llp); in slb_initialize()
592 get_paca()->stab_rr = SLB_NUM_BOLTED - 1; in slb_initialize()
593 get_paca()->slb_kern_bitmap = (1U << SLB_NUM_BOLTED) - 1; in slb_initialize()
594 get_paca()->slb_used_bitmap = get_paca()->slb_kern_bitmap; in slb_initialize()
607 * get_paca()->kstack hasn't been initialized yet. in slb_initialize()
612 (get_paca()->kstack & slb_esid_mask(mmu_kernel_ssize)) > PAGE_OFFSET) in slb_initialize()
613 create_shadowed_slbe(get_paca()->kstack, in slb_initialize()
632 slb_cache_index = local_paca->slb_cache_ptr; in slb_cache_update()
638 local_paca->slb_cache[slb_cache_index++] = esid_data >> SID_SHIFT; in slb_cache_update()
639 local_paca->slb_cache_ptr++; in slb_cache_update()
646 local_paca->slb_cache_ptr = SLB_CACHE_ENTRIES + 1; in slb_cache_update()
670 if (local_paca->slb_used_bitmap != U32_MAX) { in alloc_slb_index()
671 index = ffz(local_paca->slb_used_bitmap); in alloc_slb_index()
672 local_paca->slb_used_bitmap |= 1U << index; in alloc_slb_index()
674 local_paca->slb_kern_bitmap |= 1U << index; in alloc_slb_index()
676 /* round-robin replacement of slb starting at SLB_NUM_BOLTED. */ in alloc_slb_index()
677 index = local_paca->stab_rr; in alloc_slb_index()
678 if (index < (mmu_slb_size - 1)) in alloc_slb_index()
682 local_paca->stab_rr = index; in alloc_slb_index()
685 local_paca->slb_kern_bitmap |= 1U << index; in alloc_slb_index()
687 local_paca->slb_kern_bitmap &= ~(1U << index); in alloc_slb_index()
704 return -EFAULT; in slb_insert_entry()
730 int slb_cache_index = local_paca->slb_cache_ptr; in slb_insert_entry()
734 * cache of inserted (non-bolted) kernel SLB entries. All in slb_insert_entry()
735 * non-bolted kernel entries are flushed on any user fault, in slb_insert_entry()
736 * or if there are already 3 non-boled kernel entries. in slb_insert_entry()
748 local_paca->slb_cache[slb_cache_index++] = esid_data >> SID_SHIFT; in slb_insert_entry()
749 local_paca->slb_cache_ptr = slb_cache_index; in slb_insert_entry()
771 return -EFAULT; in slb_allocate_kernel()
779 return -EFAULT; in slb_allocate_kernel()
786 return -EFAULT; in slb_allocate_kernel()
788 flags = local_paca->vmalloc_sllp; in slb_allocate_kernel()
793 return -EFAULT; in slb_allocate_kernel()
798 return -EFAULT; in slb_allocate_kernel()
821 if (ea >= mm_ctx_slb_addr_limit(&mm->context)) in slb_allocate_user()
822 return -EFAULT; in slb_allocate_user()
824 context = get_user_context(&mm->context, ea); in slb_allocate_user()
826 return -EFAULT; in slb_allocate_user()
830 return -EFAULT; in slb_allocate_user()
848 if (unlikely(!(regs->msr & MSR_RI))) in do_slb_fault()
849 return -EINVAL; in do_slb_fault()
854 * mm->context stuff is not. in do_slb_fault()
860 * looking at possible non-bolted memory. We could test user vs in do_slb_fault()
870 BUG_ON(local_paca->in_kernel_slb_handler); in do_slb_fault()
871 local_paca->in_kernel_slb_handler = 1; in do_slb_fault()
875 local_paca->in_kernel_slb_handler = 0; in do_slb_fault()
879 struct mm_struct *mm = current->mm; in do_slb_fault()
883 return -EFAULT; in do_slb_fault()
895 if (err == -EFAULT) { in do_bad_slb_fault()
900 } else if (err == -EINVAL) { in do_bad_slb_fault()