1 // SPDX-License-Identifier: GPL-2.0
2
3 #include <asm/pgalloc.h>
4 #include <linux/gfp.h>
5 #include <linux/kernel.h>
6 #include <linux/pgtable.h>
7
8 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
p4d_set_huge(p4d_t * p4d,phys_addr_t addr,pgprot_t prot)9 int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot)
10 {
11 return 0;
12 }
13
p4d_clear_huge(p4d_t * p4d)14 void p4d_clear_huge(p4d_t *p4d)
15 {
16 }
17
pud_set_huge(pud_t * pud,phys_addr_t phys,pgprot_t prot)18 int pud_set_huge(pud_t *pud, phys_addr_t phys, pgprot_t prot)
19 {
20 pud_t new_pud = pfn_pud(__phys_to_pfn(phys), prot);
21
22 set_pud(pud, new_pud);
23 return 1;
24 }
25
pud_clear_huge(pud_t * pud)26 int pud_clear_huge(pud_t *pud)
27 {
28 if (!pud_leaf(READ_ONCE(*pud)))
29 return 0;
30 pud_clear(pud);
31 return 1;
32 }
33
pud_free_pmd_page(pud_t * pud,unsigned long addr)34 int pud_free_pmd_page(pud_t *pud, unsigned long addr)
35 {
36 pmd_t *pmd = pud_pgtable(*pud);
37 int i;
38
39 pud_clear(pud);
40
41 flush_tlb_kernel_range(addr, addr + PUD_SIZE);
42
43 for (i = 0; i < PTRS_PER_PMD; i++) {
44 if (!pmd_none(pmd[i])) {
45 pte_t *pte = (pte_t *)pmd_page_vaddr(pmd[i]);
46
47 pte_free_kernel(NULL, pte);
48 }
49 }
50
51 pmd_free(NULL, pmd);
52
53 return 1;
54 }
55
pmd_set_huge(pmd_t * pmd,phys_addr_t phys,pgprot_t prot)56 int pmd_set_huge(pmd_t *pmd, phys_addr_t phys, pgprot_t prot)
57 {
58 pmd_t new_pmd = pfn_pmd(__phys_to_pfn(phys), prot);
59
60 set_pmd(pmd, new_pmd);
61 return 1;
62 }
63
pmd_clear_huge(pmd_t * pmd)64 int pmd_clear_huge(pmd_t *pmd)
65 {
66 if (!pmd_leaf(READ_ONCE(*pmd)))
67 return 0;
68 pmd_clear(pmd);
69 return 1;
70 }
71
pmd_free_pte_page(pmd_t * pmd,unsigned long addr)72 int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
73 {
74 pte_t *pte = (pte_t *)pmd_page_vaddr(*pmd);
75
76 pmd_clear(pmd);
77
78 flush_tlb_kernel_range(addr, addr + PMD_SIZE);
79 pte_free_kernel(NULL, pte);
80 return 1;
81 }
82
83 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
84 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
pmdp_collapse_flush(struct vm_area_struct * vma,unsigned long address,pmd_t * pmdp)85 pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
86 unsigned long address, pmd_t *pmdp)
87 {
88 pmd_t pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
89
90 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
91 VM_BUG_ON(pmd_trans_huge(*pmdp));
92 /*
93 * When leaf PTE entries (regular pages) are collapsed into a leaf
94 * PMD entry (huge page), a valid non-leaf PTE is converted into a
95 * valid leaf PTE at the level 1 page table. Since the sfence.vma
96 * forms that specify an address only apply to leaf PTEs, we need a
97 * global flush here. collapse_huge_page() assumes these flushes are
98 * eager, so just do the fence here.
99 */
100 flush_tlb_mm(vma->vm_mm);
101 return pmd;
102 }
103 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
104