Lines Matching full:table
3 * Page table allocation functions
54 void crst_table_free(struct mm_struct *mm, unsigned long *table) in crst_table_free() argument
56 pagetable_free(virt_to_ptdesc(table)); in crst_table_free()
144 u64 *table; in page_table_alloc_pgste() local
148 table = (u64 *)ptdesc_to_virt(ptdesc); in page_table_alloc_pgste()
149 arch_set_page_dat(virt_to_page(table), 0); in page_table_alloc_pgste()
150 memset64(table, _PAGE_INVALID, PTRS_PER_PTE); in page_table_alloc_pgste()
151 memset64(table + PTRS_PER_PTE, 0, PTRS_PER_PTE); in page_table_alloc_pgste()
245 unsigned long *table; in page_table_alloc() local
249 /* Try to get a fragment of a 4K page as a 2K page table */ in page_table_alloc()
251 table = NULL; in page_table_alloc()
268 table = (unsigned long *) ptdesc_to_virt(ptdesc); in page_table_alloc()
271 table += PTRS_PER_PTE; in page_table_alloc()
278 if (table) in page_table_alloc()
279 return table; in page_table_alloc()
290 /* Initialize page table */ in page_table_alloc()
291 table = (unsigned long *) ptdesc_to_virt(ptdesc); in page_table_alloc()
293 /* Return 4K page table with PGSTEs */ in page_table_alloc()
296 memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE); in page_table_alloc()
297 memset64((u64 *)table + PTRS_PER_PTE, 0, PTRS_PER_PTE); in page_table_alloc()
301 memset64((u64 *)table, _PAGE_INVALID, 2 * PTRS_PER_PTE); in page_table_alloc()
306 return table; in page_table_alloc()
309 static void page_table_release_check(struct page *page, void *table, in page_table_release_check() argument
320 table, half, mask); in page_table_release_check()
333 void page_table_free(struct mm_struct *mm, unsigned long *table) in page_table_free() argument
336 struct ptdesc *ptdesc = virt_to_ptdesc(table); in page_table_free()
339 /* Free 2K page table fragment of a 4K page */ in page_table_free()
340 bit = ((unsigned long) table & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t)); in page_table_free()
372 page_table_release_check(ptdesc_page(ptdesc), table, half, mask); in page_table_free()
379 void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table, in page_table_free_rcu() argument
384 struct ptdesc *ptdesc = virt_to_ptdesc(table); in page_table_free_rcu()
388 gmap_unlink(mm, table, vmaddr); in page_table_free_rcu()
389 table = (unsigned long *) ((unsigned long)table | 0x03U); in page_table_free_rcu()
390 tlb_remove_ptdesc(tlb, table); in page_table_free_rcu()
393 bit = ((unsigned long) table & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t)); in page_table_free_rcu()
415 table = (unsigned long *) ((unsigned long) table | (0x01U << bit)); in page_table_free_rcu()
416 tlb_remove_table(tlb, table); in page_table_free_rcu()
422 void *table = (void *)((unsigned long) _table ^ mask); in __tlb_remove_table() local
423 struct ptdesc *ptdesc = virt_to_ptdesc(table); in __tlb_remove_table()
429 case 0x01U: /* lower 2K of a 4K page table */ in __tlb_remove_table()
430 case 0x02U: /* higher 2K of a 4K page table */ in __tlb_remove_table()
436 case 0x03U: /* 4K page table with pgstes */ in __tlb_remove_table()
442 page_table_release_check(ptdesc_page(ptdesc), table, half, mask); in __tlb_remove_table()
474 unsigned long *table; in base_pgt_alloc() local
476 table = kmem_cache_alloc(base_pgt_cache, GFP_KERNEL); in base_pgt_alloc()
477 if (table) in base_pgt_alloc()
478 memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE); in base_pgt_alloc()
479 return table; in base_pgt_alloc()
482 static void base_pgt_free(unsigned long *table) in base_pgt_free() argument
484 kmem_cache_free(base_pgt_cache, table); in base_pgt_free()
489 unsigned long *table; in base_crst_alloc() local
495 table = ptdesc_address(ptdesc); in base_crst_alloc()
497 crst_table_init(table, val); in base_crst_alloc()
498 return table; in base_crst_alloc()
501 static void base_crst_free(unsigned long *table) in base_crst_free() argument
503 pagetable_free(virt_to_ptdesc(table)); in base_crst_free()
550 unsigned long *ste, next, *table; in base_segment_walk() local
560 table = base_pgt_alloc(); in base_segment_walk()
561 if (!table) in base_segment_walk()
563 *ste = __pa(table) | _SEGMENT_ENTRY; in base_segment_walk()
565 table = __va(*ste & _SEGMENT_ENTRY_ORIGIN); in base_segment_walk()
566 rc = base_page_walk(table, addr, next, alloc); in base_segment_walk()
570 base_pgt_free(table); in base_segment_walk()
579 unsigned long *rtte, next, *table; in base_region3_walk() local
589 table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY); in base_region3_walk()
590 if (!table) in base_region3_walk()
592 *rtte = __pa(table) | _REGION3_ENTRY; in base_region3_walk()
594 table = __va(*rtte & _REGION_ENTRY_ORIGIN); in base_region3_walk()
595 rc = base_segment_walk(table, addr, next, alloc); in base_region3_walk()
599 base_crst_free(table); in base_region3_walk()
607 unsigned long *rste, next, *table; in base_region2_walk() local
617 table = base_crst_alloc(_REGION3_ENTRY_EMPTY); in base_region2_walk()
618 if (!table) in base_region2_walk()
620 *rste = __pa(table) | _REGION2_ENTRY; in base_region2_walk()
622 table = __va(*rste & _REGION_ENTRY_ORIGIN); in base_region2_walk()
623 rc = base_region3_walk(table, addr, next, alloc); in base_region2_walk()
627 base_crst_free(table); in base_region2_walk()
635 unsigned long *rfte, next, *table; in base_region1_walk() local
645 table = base_crst_alloc(_REGION2_ENTRY_EMPTY); in base_region1_walk()
646 if (!table) in base_region1_walk()
648 *rfte = __pa(table) | _REGION1_ENTRY; in base_region1_walk()
650 table = __va(*rfte & _REGION_ENTRY_ORIGIN); in base_region1_walk()
651 rc = base_region2_walk(table, addr, next, alloc); in base_region1_walk()
655 base_crst_free(table); in base_region1_walk()
669 unsigned long *table = __va(asce & _ASCE_ORIGIN); in base_asce_free() local
675 base_segment_walk(table, 0, _REGION3_SIZE, 0); in base_asce_free()
678 base_region3_walk(table, 0, _REGION2_SIZE, 0); in base_asce_free()
681 base_region2_walk(table, 0, _REGION1_SIZE, 0); in base_asce_free()
684 base_region1_walk(table, 0, TASK_SIZE_MAX, 0); in base_asce_free()
687 base_crst_free(table); in base_asce_free()
721 unsigned long asce, *table, end; in base_asce_alloc() local
728 table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY); in base_asce_alloc()
729 if (!table) in base_asce_alloc()
731 rc = base_segment_walk(table, addr, end, 1); in base_asce_alloc()
732 asce = __pa(table) | _ASCE_TYPE_SEGMENT | _ASCE_TABLE_LENGTH; in base_asce_alloc()
734 table = base_crst_alloc(_REGION3_ENTRY_EMPTY); in base_asce_alloc()
735 if (!table) in base_asce_alloc()
737 rc = base_region3_walk(table, addr, end, 1); in base_asce_alloc()
738 asce = __pa(table) | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; in base_asce_alloc()
740 table = base_crst_alloc(_REGION2_ENTRY_EMPTY); in base_asce_alloc()
741 if (!table) in base_asce_alloc()
743 rc = base_region2_walk(table, addr, end, 1); in base_asce_alloc()
744 asce = __pa(table) | _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH; in base_asce_alloc()
746 table = base_crst_alloc(_REGION1_ENTRY_EMPTY); in base_asce_alloc()
747 if (!table) in base_asce_alloc()
749 rc = base_region1_walk(table, addr, end, 1); in base_asce_alloc()
750 asce = __pa(table) | _ASCE_TYPE_REGION1 | _ASCE_TABLE_LENGTH; in base_asce_alloc()