| /mm/ |
| D | mmu_gather.c | 16 static bool tlb_next_batch(struct mmu_gather *tlb) in tlb_next_batch() argument
20 batch = tlb->active; in tlb_next_batch()
22 tlb->active = batch->next; in tlb_next_batch()
26 if (tlb->batch_count == MAX_GATHER_BATCH_COUNT) in tlb_next_batch()
33 tlb->batch_count++; in tlb_next_batch()
38 tlb->active->next = batch; in tlb_next_batch()
39 tlb->active = batch; in tlb_next_batch()
44 static void tlb_batch_pages_flush(struct mmu_gather *tlb) in tlb_batch_pages_flush() argument
48 for (batch = &tlb->local; batch && batch->nr; batch = batch->next) { in tlb_batch_pages_flush()
52 tlb->active = &tlb->local; in tlb_batch_pages_flush()
[all …]
|
| D | madvise.c | 42 struct mmu_gather *tlb; member
322 struct mmu_gather *tlb = private->tlb; in madvise_cold_or_pageout_pte_range() local
325 struct mm_struct *mm = tlb->mm; in madvise_cold_or_pageout_pte_range()
340 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); in madvise_cold_or_pageout_pte_range()
383 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); in madvise_cold_or_pageout_pte_range()
408 tlb_change_page_size(tlb, PAGE_SIZE); in madvise_cold_or_pageout_pte_range()
468 tlb->fullmm); in madvise_cold_or_pageout_pte_range()
471 tlb_remove_tlb_entry(tlb, pte, addr); in madvise_cold_or_pageout_pte_range()
506 static void madvise_cold_page_range(struct mmu_gather *tlb, in madvise_cold_page_range() argument
512 .tlb = tlb, in madvise_cold_page_range()
[all …]
|
| D | memory.c | 269 static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, in free_pte_range() argument
280 spinlock_t *ptl = pmd_lock(tlb->mm, pmd); in free_pte_range()
292 pte_free_tlb(tlb, token, addr); in free_pte_range()
293 mm_dec_nr_ptes(tlb->mm); in free_pte_range()
296 static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud, in free_pmd_range() argument
310 free_pte_range(tlb, pmd, addr); in free_pmd_range()
326 pmd_free_tlb(tlb, pmd, start); in free_pmd_range()
327 mm_dec_nr_pmds(tlb->mm); in free_pmd_range()
330 static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d, in free_pud_range() argument
344 free_pmd_range(tlb, pud, addr, next, floor, ceiling); in free_pud_range()
[all …]
|
| D | huge_memory.c | 1501 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, in madvise_free_huge_pmd() argument
1507 struct mm_struct *mm = tlb->mm; in madvise_free_huge_pmd()
1510 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); in madvise_free_huge_pmd()
1560 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); in madvise_free_huge_pmd()
1580 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, in zap_huge_pmd() argument
1586 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); in zap_huge_pmd()
1598 tlb->fullmm); in zap_huge_pmd()
1599 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); in zap_huge_pmd()
1602 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1605 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
[all …]
|
| D | oom_kill.c | 548 struct mmu_gather tlb; in __oom_reap_task_mm() local
553 tlb_gather_mmu(&tlb, mm); in __oom_reap_task_mm()
555 tlb_finish_mmu(&tlb); in __oom_reap_task_mm()
559 unmap_page_range(&tlb, vma, range.start, range.end, NULL); in __oom_reap_task_mm()
561 tlb_finish_mmu(&tlb); in __oom_reap_task_mm()
|
| D | mmap.c | 2666 struct mmu_gather tlb; in unmap_region() local
2670 tlb_gather_mmu(&tlb, mm); in unmap_region()
2672 unmap_vmas(&tlb, vma, start, end); in unmap_region()
2684 tlb_flush_mmu(&tlb); in unmap_region()
2689 free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, in unmap_region()
2691 tlb_finish_mmu(&tlb); in unmap_region()
3167 struct mmu_gather tlb; in exit_mmap() local
3211 tlb_gather_mmu_fullmm(&tlb, mm); in exit_mmap()
3214 unmap_vmas(&tlb, vma, 0, -1); in exit_mmap()
3215 free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING); in exit_mmap()
[all …]
|
| D | internal.h | 40 void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
48 void unmap_page_range(struct mmu_gather *tlb,
|
| D | hugetlb.c | 4451 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, in __unmap_hugepage_range() argument
4474 tlb_change_page_size(tlb, sz); in __unmap_hugepage_range()
4475 tlb_start_vma(tlb, vma); in __unmap_hugepage_range()
4493 tlb_flush_pmd_range(tlb, address & PUD_MASK, PUD_SIZE); in __unmap_hugepage_range()
4534 tlb_remove_huge_tlb_entry(h, tlb, ptep, address); in __unmap_hugepage_range()
4542 tlb_remove_page_size(tlb, page, huge_page_size(h)); in __unmap_hugepage_range()
4550 tlb_end_vma(tlb, vma); in __unmap_hugepage_range()
4566 tlb_flush_mmu_tlbonly(tlb); in __unmap_hugepage_range()
4569 void __unmap_hugepage_range_final(struct mmu_gather *tlb, in __unmap_hugepage_range_final() argument
4573 __unmap_hugepage_range(tlb, vma, start, end, ref_page); in __unmap_hugepage_range_final()
[all …]
|
| D | Kconfig | 386 huge tlb transparently to the applications whenever possible.
389 allocation, by reducing the number of tlb misses and by speeding
|