Lines Matching refs:table
45 void crst_table_free(struct mm_struct *mm, unsigned long *table) in crst_table_free() argument
47 free_pages((unsigned long) table, ALLOC_ORDER); in crst_table_free()
53 unsigned long *table, *pgd; in crst_table_upgrade() local
58 table = crst_table_alloc(mm); in crst_table_upgrade()
59 if (!table) in crst_table_upgrade()
77 crst_table_init(table, entry); in crst_table_upgrade()
78 pgd_populate(mm, (pgd_t *) table, (pud_t *) pgd); in crst_table_upgrade()
79 mm->pgd = (pgd_t *) table; in crst_table_upgrade()
81 table = NULL; in crst_table_upgrade()
84 if (table) in crst_table_upgrade()
85 crst_table_free(mm, table); in crst_table_upgrade()
132 unsigned long *table; in gmap_alloc() local
143 table = (unsigned long *) page_to_phys(page); in gmap_alloc()
144 crst_table_init(table, _REGION1_ENTRY_EMPTY); in gmap_alloc()
145 gmap->table = table; in gmap_alloc()
147 _ASCE_USER_BITS | __pa(table); in gmap_alloc()
158 static int gmap_unlink_segment(struct gmap *gmap, unsigned long *table) in gmap_unlink_segment() argument
164 if (*table & _SEGMENT_ENTRY_INV) in gmap_unlink_segment()
166 page = pfn_to_page(*table >> PAGE_SHIFT); in gmap_unlink_segment()
169 if (rmap->entry != table) in gmap_unlink_segment()
175 *table = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO | mp->vmaddr; in gmap_unlink_segment()
182 __tlb_flush_idte((unsigned long) gmap->table | in gmap_flush_tlb()
195 unsigned long *table; in gmap_free() local
201 __tlb_flush_idte((unsigned long) gmap->table | in gmap_free()
210 table = (unsigned long *) page_to_phys(page); in gmap_free()
211 if ((*table & _REGION_ENTRY_TYPE_MASK) == 0) in gmap_free()
213 for (i = 0; i < PTRS_PER_PMD; i++, table++) in gmap_free()
214 gmap_unlink_segment(gmap, table); in gmap_free()
248 unsigned long *table, unsigned long init) in gmap_alloc_table() argument
261 if (*table & _REGION_ENTRY_INV) { in gmap_alloc_table()
263 *table = (unsigned long) new | _REGION_ENTRY_LENGTH | in gmap_alloc_table()
264 (*table & _REGION_ENTRY_TYPE_MASK); in gmap_alloc_table()
280 unsigned long *table; in gmap_unmap_segment() local
294 table = gmap->table + (((to + off) >> 53) & 0x7ff); in gmap_unmap_segment()
295 if (*table & _REGION_ENTRY_INV) in gmap_unmap_segment()
297 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); in gmap_unmap_segment()
298 table = table + (((to + off) >> 42) & 0x7ff); in gmap_unmap_segment()
299 if (*table & _REGION_ENTRY_INV) in gmap_unmap_segment()
301 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); in gmap_unmap_segment()
302 table = table + (((to + off) >> 31) & 0x7ff); in gmap_unmap_segment()
303 if (*table & _REGION_ENTRY_INV) in gmap_unmap_segment()
305 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); in gmap_unmap_segment()
306 table = table + (((to + off) >> 20) & 0x7ff); in gmap_unmap_segment()
309 flush |= gmap_unlink_segment(gmap, table); in gmap_unmap_segment()
310 *table = _SEGMENT_ENTRY_INV; in gmap_unmap_segment()
332 unsigned long *table; in gmap_map_segment() local
347 table = gmap->table + (((to + off) >> 53) & 0x7ff); in gmap_map_segment()
348 if ((*table & _REGION_ENTRY_INV) && in gmap_map_segment()
349 gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY)) in gmap_map_segment()
351 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); in gmap_map_segment()
352 table = table + (((to + off) >> 42) & 0x7ff); in gmap_map_segment()
353 if ((*table & _REGION_ENTRY_INV) && in gmap_map_segment()
354 gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY)) in gmap_map_segment()
356 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); in gmap_map_segment()
357 table = table + (((to + off) >> 31) & 0x7ff); in gmap_map_segment()
358 if ((*table & _REGION_ENTRY_INV) && in gmap_map_segment()
359 gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY)) in gmap_map_segment()
361 table = (unsigned long *) (*table & _REGION_ENTRY_ORIGIN); in gmap_map_segment()
362 table = table + (((to + off) >> 20) & 0x7ff); in gmap_map_segment()
365 flush |= gmap_unlink_segment(gmap, table); in gmap_map_segment()
366 *table = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO | (from + off); in gmap_map_segment()
387 unsigned long *table, vmaddr, segment; in __gmap_fault() local
400 table = gmap->table + ((address >> 53) & 0x7ff); in __gmap_fault()
401 if (unlikely(*table & _REGION_ENTRY_INV)) in __gmap_fault()
403 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); in __gmap_fault()
404 table = table + ((address >> 42) & 0x7ff); in __gmap_fault()
405 if (unlikely(*table & _REGION_ENTRY_INV)) in __gmap_fault()
407 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); in __gmap_fault()
408 table = table + ((address >> 31) & 0x7ff); in __gmap_fault()
409 if (unlikely(*table & _REGION_ENTRY_INV)) in __gmap_fault()
411 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); in __gmap_fault()
412 table = table + ((address >> 20) & 0x7ff); in __gmap_fault()
415 segment = *table; in __gmap_fault()
444 rmap->entry = table; in __gmap_fault()
449 *table = pmd_val(*pmd) & PAGE_MASK; in __gmap_fault()
470 unsigned long *table, address, size; in gmap_discard() local
479 table = gmap->table + ((address >> 53) & 0x7ff); in gmap_discard()
480 if (unlikely(*table & _REGION_ENTRY_INV)) { in gmap_discard()
484 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); in gmap_discard()
485 table = table + ((address >> 42) & 0x7ff); in gmap_discard()
486 if (unlikely(*table & _REGION_ENTRY_INV)) { in gmap_discard()
490 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); in gmap_discard()
491 table = table + ((address >> 31) & 0x7ff); in gmap_discard()
492 if (unlikely(*table & _REGION_ENTRY_INV)) { in gmap_discard()
496 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); in gmap_discard()
497 table = table + ((address >> 20) & 0x7ff); in gmap_discard()
498 if (unlikely(*table & _SEGMENT_ENTRY_INV)) { in gmap_discard()
502 page = pfn_to_page(*table >> PAGE_SHIFT); in gmap_discard()
514 void gmap_unmap_notifier(struct mm_struct *mm, unsigned long *table) in gmap_unmap_notifier() argument
523 page = pfn_to_page(__pa(table) >> PAGE_SHIFT); in gmap_unmap_notifier()
541 unsigned long *table; in page_table_alloc_pgste() local
557 table = (unsigned long *) page_to_phys(page); in page_table_alloc_pgste()
558 clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE/2); in page_table_alloc_pgste()
559 clear_table(table + PTRS_PER_PTE, 0, PAGE_SIZE/2); in page_table_alloc_pgste()
560 return table; in page_table_alloc_pgste()
563 static inline void page_table_free_pgste(unsigned long *table) in page_table_free_pgste() argument
568 page = pfn_to_page(__pa(table) >> PAGE_SHIFT); in page_table_free_pgste()
585 static inline void page_table_free_pgste(unsigned long *table) in page_table_free_pgste() argument
590 unsigned long *table) in gmap_unmap_notifier() argument
613 unsigned long *table; in page_table_alloc() local
624 table = (unsigned long *) page_to_phys(page); in page_table_alloc()
635 table = (unsigned long *) page_to_phys(page); in page_table_alloc()
636 clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE); in page_table_alloc()
641 table += PTRS_PER_PTE; in page_table_alloc()
647 return table; in page_table_alloc()
650 void page_table_free(struct mm_struct *mm, unsigned long *table) in page_table_free() argument
656 gmap_unmap_notifier(mm, table); in page_table_free()
657 return page_table_free_pgste(table); in page_table_free()
660 page = pfn_to_page(__pa(table) >> PAGE_SHIFT); in page_table_free()
661 bit = 1 << ((__pa(table) & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t))); in page_table_free()
676 static void __page_table_free_rcu(void *table, unsigned bit) in __page_table_free_rcu() argument
681 return page_table_free_pgste(table); in __page_table_free_rcu()
683 page = pfn_to_page(__pa(table) >> PAGE_SHIFT); in __page_table_free_rcu()
691 void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table) in page_table_free_rcu() argument
699 gmap_unmap_notifier(mm, table); in page_table_free_rcu()
700 table = (unsigned long *) (__pa(table) | FRAG_MASK); in page_table_free_rcu()
701 tlb_remove_table(tlb, table); in page_table_free_rcu()
704 bit = 1 << ((__pa(table) & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t))); in page_table_free_rcu()
705 page = pfn_to_page(__pa(table) >> PAGE_SHIFT); in page_table_free_rcu()
713 table = (unsigned long *) (__pa(table) | (bit << 4)); in page_table_free_rcu()
714 tlb_remove_table(tlb, table); in page_table_free_rcu()
720 void *table = (void *)((unsigned long) _table & ~mask); in __tlb_remove_table() local
724 __page_table_free_rcu(table, type); in __tlb_remove_table()
726 free_pages((unsigned long) table, ALLOC_ORDER); in __tlb_remove_table()
734 static void tlb_remove_table_one(void *table) in tlb_remove_table_one() argument
744 __tlb_remove_table(table); in tlb_remove_table_one()
771 void tlb_remove_table(struct mmu_gather *tlb, void *table) in tlb_remove_table() argument
780 tlb_remove_table_one(table); in tlb_remove_table()
785 (*batch)->tables[(*batch)->nr++] = table; in tlb_remove_table()