Lines Matching full:table
3 * Page table allocation functions
66 void crst_table_free(struct mm_struct *mm, unsigned long *table) in crst_table_free() argument
68 free_pages((unsigned long) table, 2); in crst_table_free()
94 unsigned long *table, *pgd; in crst_table_upgrade() local
102 table = crst_table_alloc(mm); in crst_table_upgrade()
103 if (!table) { in crst_table_upgrade()
110 crst_table_init(table, _REGION2_ENTRY_EMPTY); in crst_table_upgrade()
111 p4d_populate(mm, (p4d_t *) table, (pud_t *) pgd); in crst_table_upgrade()
112 mm->pgd = (pgd_t *) table; in crst_table_upgrade()
118 crst_table_init(table, _REGION1_ENTRY_EMPTY); in crst_table_upgrade()
119 pgd_populate(mm, (pgd_t *) table, (p4d_t *) pgd); in crst_table_upgrade()
120 mm->pgd = (pgd_t *) table; in crst_table_upgrade()
172 u64 *table; in page_table_alloc_pgste() local
176 table = (u64 *)page_to_phys(page); in page_table_alloc_pgste()
177 memset64(table, _PAGE_INVALID, PTRS_PER_PTE); in page_table_alloc_pgste()
178 memset64(table + PTRS_PER_PTE, 0, PTRS_PER_PTE); in page_table_alloc_pgste()
191 * page table entry allocation/free routines.
195 unsigned long *table; in page_table_alloc() local
199 /* Try to get a fragment of a 4K page as a 2K page table */ in page_table_alloc()
201 table = NULL; in page_table_alloc()
209 table = (unsigned long *) page_to_phys(page); in page_table_alloc()
212 table += PTRS_PER_PTE; in page_table_alloc()
219 if (table) in page_table_alloc()
220 return table; in page_table_alloc()
231 /* Initialize page table */ in page_table_alloc()
232 table = (unsigned long *) page_to_phys(page); in page_table_alloc()
234 /* Return 4K page table with PGSTEs */ in page_table_alloc()
236 memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE); in page_table_alloc()
237 memset64((u64 *)table + PTRS_PER_PTE, 0, PTRS_PER_PTE); in page_table_alloc()
241 memset64((u64 *)table, _PAGE_INVALID, 2 * PTRS_PER_PTE); in page_table_alloc()
246 return table; in page_table_alloc()
249 void page_table_free(struct mm_struct *mm, unsigned long *table) in page_table_free() argument
254 page = pfn_to_page(__pa(table) >> PAGE_SHIFT); in page_table_free()
256 /* Free 2K page table fragment of a 4K page */ in page_table_free()
257 bit = (__pa(table) & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t)); in page_table_free()
276 void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table, in page_table_free_rcu() argument
284 page = pfn_to_page(__pa(table) >> PAGE_SHIFT); in page_table_free_rcu()
286 gmap_unlink(mm, table, vmaddr); in page_table_free_rcu()
287 table = (unsigned long *) (__pa(table) | 3); in page_table_free_rcu()
288 tlb_remove_table(tlb, table); in page_table_free_rcu()
291 bit = (__pa(table) & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t)); in page_table_free_rcu()
300 table = (unsigned long *) (__pa(table) | (1U << bit)); in page_table_free_rcu()
301 tlb_remove_table(tlb, table); in page_table_free_rcu()
307 void *table = (void *)((unsigned long) _table ^ mask); in __tlb_remove_table() local
308 struct page *page = pfn_to_page(__pa(table) >> PAGE_SHIFT); in __tlb_remove_table()
312 free_pages((unsigned long) table, 2); in __tlb_remove_table()
314 case 1: /* lower 2K of a 4K page table */ in __tlb_remove_table()
315 case 2: /* higher 2K of a 4K page table */ in __tlb_remove_table()
321 case 3: /* 4K page table with pgstes */ in __tlb_remove_table()
335 static void tlb_remove_table_one(void *table) in tlb_remove_table_one() argument
341 * It is however sufficient for software page-table walkers that rely in tlb_remove_table_one()
345 __tlb_remove_table(table); in tlb_remove_table_one()
371 void tlb_remove_table(struct mmu_gather *tlb, void *table) in tlb_remove_table() argument
381 tlb_remove_table_one(table); in tlb_remove_table()
386 (*batch)->tables[(*batch)->nr++] = table; in tlb_remove_table()
400 u64 *table; in base_pgt_alloc() local
402 table = kmem_cache_alloc(base_pgt_cache, GFP_KERNEL); in base_pgt_alloc()
403 if (table) in base_pgt_alloc()
404 memset64(table, _PAGE_INVALID, PTRS_PER_PTE); in base_pgt_alloc()
405 return (unsigned long) table; in base_pgt_alloc()
408 static void base_pgt_free(unsigned long table) in base_pgt_free() argument
410 kmem_cache_free(base_pgt_cache, (void *) table); in base_pgt_free()
415 unsigned long table; in base_crst_alloc() local
417 table = __get_free_pages(GFP_KERNEL, CRST_ALLOC_ORDER); in base_crst_alloc()
418 if (table) in base_crst_alloc()
419 crst_table_init((unsigned long *)table, val); in base_crst_alloc()
420 return table; in base_crst_alloc()
423 static void base_crst_free(unsigned long table) in base_crst_free() argument
425 free_pages(table, CRST_ALLOC_ORDER); in base_crst_free()
472 unsigned long *ste, next, table; in base_segment_walk() local
482 table = base_pgt_alloc(); in base_segment_walk()
483 if (!table) in base_segment_walk()
485 *ste = table | _SEGMENT_ENTRY; in base_segment_walk()
487 table = *ste & _SEGMENT_ENTRY_ORIGIN; in base_segment_walk()
488 rc = base_page_walk(table, addr, next, alloc); in base_segment_walk()
492 base_pgt_free(table); in base_segment_walk()
501 unsigned long *rtte, next, table; in base_region3_walk() local
511 table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY); in base_region3_walk()
512 if (!table) in base_region3_walk()
514 *rtte = table | _REGION3_ENTRY; in base_region3_walk()
516 table = *rtte & _REGION_ENTRY_ORIGIN; in base_region3_walk()
517 rc = base_segment_walk(table, addr, next, alloc); in base_region3_walk()
521 base_crst_free(table); in base_region3_walk()
529 unsigned long *rste, next, table; in base_region2_walk() local
539 table = base_crst_alloc(_REGION3_ENTRY_EMPTY); in base_region2_walk()
540 if (!table) in base_region2_walk()
542 *rste = table | _REGION2_ENTRY; in base_region2_walk()
544 table = *rste & _REGION_ENTRY_ORIGIN; in base_region2_walk()
545 rc = base_region3_walk(table, addr, next, alloc); in base_region2_walk()
549 base_crst_free(table); in base_region2_walk()
557 unsigned long *rfte, next, table; in base_region1_walk() local
567 table = base_crst_alloc(_REGION2_ENTRY_EMPTY); in base_region1_walk()
568 if (!table) in base_region1_walk()
570 *rfte = table | _REGION1_ENTRY; in base_region1_walk()
572 table = *rfte & _REGION_ENTRY_ORIGIN; in base_region1_walk()
573 rc = base_region2_walk(table, addr, next, alloc); in base_region1_walk()
577 base_crst_free(table); in base_region1_walk()
591 unsigned long table = asce & _ASCE_ORIGIN; in base_asce_free() local
597 base_segment_walk(table, 0, _REGION3_SIZE, 0); in base_asce_free()
600 base_region3_walk(table, 0, _REGION2_SIZE, 0); in base_asce_free()
603 base_region2_walk(table, 0, _REGION1_SIZE, 0); in base_asce_free()
606 base_region1_walk(table, 0, -_PAGE_SIZE, 0); in base_asce_free()
609 base_crst_free(table); in base_asce_free()
643 unsigned long asce, table, end; in base_asce_alloc() local
650 table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY); in base_asce_alloc()
651 if (!table) in base_asce_alloc()
653 rc = base_segment_walk(table, addr, end, 1); in base_asce_alloc()
654 asce = table | _ASCE_TYPE_SEGMENT | _ASCE_TABLE_LENGTH; in base_asce_alloc()
656 table = base_crst_alloc(_REGION3_ENTRY_EMPTY); in base_asce_alloc()
657 if (!table) in base_asce_alloc()
659 rc = base_region3_walk(table, addr, end, 1); in base_asce_alloc()
660 asce = table | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; in base_asce_alloc()
662 table = base_crst_alloc(_REGION2_ENTRY_EMPTY); in base_asce_alloc()
663 if (!table) in base_asce_alloc()
665 rc = base_region2_walk(table, addr, end, 1); in base_asce_alloc()
666 asce = table | _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH; in base_asce_alloc()
668 table = base_crst_alloc(_REGION1_ENTRY_EMPTY); in base_asce_alloc()
669 if (!table) in base_asce_alloc()
671 rc = base_region1_walk(table, addr, end, 1); in base_asce_alloc()
672 asce = table | _ASCE_TYPE_REGION1 | _ASCE_TABLE_LENGTH; in base_asce_alloc()