1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_POWERPC_BOOK3S_64_HASH_64K_H
3 #define _ASM_POWERPC_BOOK3S_64_HASH_64K_H
4
5 #define H_PTE_INDEX_SIZE 8
6 #define H_PMD_INDEX_SIZE 10
7 #define H_PUD_INDEX_SIZE 7
8 #define H_PGD_INDEX_SIZE 8
9
10 /*
11 * 64k aligned address free up few of the lower bits of RPN for us
12 * We steal that here. For more deatils look at pte_pfn/pfn_pte()
13 */
14 #define H_PAGE_COMBO _RPAGE_RPN0 /* this is a combo 4k page */
15 #define H_PAGE_4K_PFN _RPAGE_RPN1 /* PFN is for a single 4k page */
16 /*
17 * We need to differentiate between explicit huge page and THP huge
18 * page, since THP huge page also need to track real subpage details
19 */
20 #define H_PAGE_THP_HUGE H_PAGE_4K_PFN
21
22 /*
23 * Used to track subpage group valid if H_PAGE_COMBO is set
24 * This overloads H_PAGE_F_GIX and H_PAGE_F_SECOND
25 */
26 #define H_PAGE_COMBO_VALID (H_PAGE_F_GIX | H_PAGE_F_SECOND)
27
28 /* PTE flags to conserve for HPTE identification */
29 #define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_F_SECOND | \
30 H_PAGE_F_GIX | H_PAGE_HASHPTE | H_PAGE_COMBO)
31 /*
32 * we support 16 fragments per PTE page of 64K size.
33 */
34 #define H_PTE_FRAG_NR 16
35 /*
36 * We use a 2K PTE page fragment and another 2K for storing
37 * real_pte_t hash index
38 */
39 #define H_PTE_FRAG_SIZE_SHIFT 12
40 #define PTE_FRAG_SIZE (1UL << PTE_FRAG_SIZE_SHIFT)
41
42 #ifndef __ASSEMBLY__
43 #include <asm/errno.h>
44
45 /*
46 * With 64K pages on hash table, we have a special PTE format that
47 * uses a second "half" of the page table to encode sub-page information
48 * in order to deal with 64K made of 4K HW pages. Thus we override the
49 * generic accessors and iterators here
50 */
51 #define __real_pte __real_pte
__real_pte(pte_t pte,pte_t * ptep)52 static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep)
53 {
54 real_pte_t rpte;
55 unsigned long *hidxp;
56
57 rpte.pte = pte;
58 rpte.hidx = 0;
59 if (pte_val(pte) & H_PAGE_COMBO) {
60 /*
61 * Make sure we order the hidx load against the H_PAGE_COMBO
62 * check. The store side ordering is done in __hash_page_4K
63 */
64 smp_rmb();
65 hidxp = (unsigned long *)(ptep + PTRS_PER_PTE);
66 rpte.hidx = *hidxp;
67 }
68 return rpte;
69 }
70
__rpte_to_hidx(real_pte_t rpte,unsigned long index)71 static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index)
72 {
73 if ((pte_val(rpte.pte) & H_PAGE_COMBO))
74 return (rpte.hidx >> (index<<2)) & 0xf;
75 return (pte_val(rpte.pte) >> H_PAGE_F_GIX_SHIFT) & 0xf;
76 }
77
78 #define __rpte_to_pte(r) ((r).pte)
79 extern bool __rpte_sub_valid(real_pte_t rpte, unsigned long index);
80 /*
81 * Trick: we set __end to va + 64k, which happens works for
82 * a 16M page as well as we want only one iteration
83 */
84 #define pte_iterate_hashed_subpages(rpte, psize, vpn, index, shift) \
85 do { \
86 unsigned long __end = vpn + (1UL << (PAGE_SHIFT - VPN_SHIFT)); \
87 unsigned __split = (psize == MMU_PAGE_4K || \
88 psize == MMU_PAGE_64K_AP); \
89 shift = mmu_psize_defs[psize].shift; \
90 for (index = 0; vpn < __end; index++, \
91 vpn += (1L << (shift - VPN_SHIFT))) { \
92 if (!__split || __rpte_sub_valid(rpte, index)) \
93 do {
94
95 #define pte_iterate_hashed_end() } while(0); } } while(0)
96
97 #define pte_pagesize_index(mm, addr, pte) \
98 (((pte) & H_PAGE_COMBO)? MMU_PAGE_4K: MMU_PAGE_64K)
99
100 extern int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
101 unsigned long pfn, unsigned long size, pgprot_t);
hash__remap_4k_pfn(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn,pgprot_t prot)102 static inline int hash__remap_4k_pfn(struct vm_area_struct *vma, unsigned long addr,
103 unsigned long pfn, pgprot_t prot)
104 {
105 if (pfn > (PTE_RPN_MASK >> PAGE_SHIFT)) {
106 WARN(1, "remap_4k_pfn called with wrong pfn value\n");
107 return -EINVAL;
108 }
109 return remap_pfn_range(vma, addr, pfn, PAGE_SIZE,
110 __pgprot(pgprot_val(prot) | H_PAGE_4K_PFN));
111 }
112
113 #define H_PTE_TABLE_SIZE PTE_FRAG_SIZE
114 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
115 #define H_PMD_TABLE_SIZE ((sizeof(pmd_t) << PMD_INDEX_SIZE) + \
116 (sizeof(unsigned long) << PMD_INDEX_SIZE))
117 #else
118 #define H_PMD_TABLE_SIZE (sizeof(pmd_t) << PMD_INDEX_SIZE)
119 #endif
120 #define H_PUD_TABLE_SIZE (sizeof(pud_t) << PUD_INDEX_SIZE)
121 #define H_PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE)
122
123 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
get_hpte_slot_array(pmd_t * pmdp)124 static inline char *get_hpte_slot_array(pmd_t *pmdp)
125 {
126 /*
127 * The hpte hindex is stored in the pgtable whose address is in the
128 * second half of the PMD
129 *
130 * Order this load with the test for pmd_trans_huge in the caller
131 */
132 smp_rmb();
133 return *(char **)(pmdp + PTRS_PER_PMD);
134
135
136 }
137 /*
138 * The linux hugepage PMD now include the pmd entries followed by the address
139 * to the stashed pgtable_t. The stashed pgtable_t contains the hpte bits.
140 * [ 000 | 1 bit secondary | 3 bit hidx | 1 bit valid]. We use one byte per
141 * each HPTE entry. With 16MB hugepage and 64K HPTE we need 256 entries and
142 * with 4K HPTE we need 4096 entries. Both will fit in a 4K pgtable_t.
143 *
144 * The top three bits are intentionally left as zero. This memory location
145 * are also used as normal page PTE pointers. So if we have any pointers
146 * left around while we collapse a hugepage, we need to make sure
147 * _PAGE_PRESENT bit of that is zero when we look at them
148 */
hpte_valid(unsigned char * hpte_slot_array,int index)149 static inline unsigned int hpte_valid(unsigned char *hpte_slot_array, int index)
150 {
151 return hpte_slot_array[index] & 0x1;
152 }
153
hpte_hash_index(unsigned char * hpte_slot_array,int index)154 static inline unsigned int hpte_hash_index(unsigned char *hpte_slot_array,
155 int index)
156 {
157 return hpte_slot_array[index] >> 1;
158 }
159
mark_hpte_slot_valid(unsigned char * hpte_slot_array,unsigned int index,unsigned int hidx)160 static inline void mark_hpte_slot_valid(unsigned char *hpte_slot_array,
161 unsigned int index, unsigned int hidx)
162 {
163 hpte_slot_array[index] = (hidx << 1) | 0x1;
164 }
165
166 /*
167 *
168 * For core kernel code by design pmd_trans_huge is never run on any hugetlbfs
169 * page. The hugetlbfs page table walking and mangling paths are totally
170 * separated form the core VM paths and they're differentiated by
171 * VM_HUGETLB being set on vm_flags well before any pmd_trans_huge could run.
172 *
173 * pmd_trans_huge() is defined as false at build time if
174 * CONFIG_TRANSPARENT_HUGEPAGE=n to optimize away code blocks at build
175 * time in such case.
176 *
177 * For ppc64 we need to differntiate from explicit hugepages from THP, because
178 * for THP we also track the subpage details at the pmd level. We don't do
179 * that for explicit huge pages.
180 *
181 */
hash__pmd_trans_huge(pmd_t pmd)182 static inline int hash__pmd_trans_huge(pmd_t pmd)
183 {
184 return !!((pmd_val(pmd) & (_PAGE_PTE | H_PAGE_THP_HUGE)) ==
185 (_PAGE_PTE | H_PAGE_THP_HUGE));
186 }
187
hash__pmd_same(pmd_t pmd_a,pmd_t pmd_b)188 static inline int hash__pmd_same(pmd_t pmd_a, pmd_t pmd_b)
189 {
190 return (((pmd_raw(pmd_a) ^ pmd_raw(pmd_b)) & ~cpu_to_be64(_PAGE_HPTEFLAGS)) == 0);
191 }
192
hash__pmd_mkhuge(pmd_t pmd)193 static inline pmd_t hash__pmd_mkhuge(pmd_t pmd)
194 {
195 return __pmd(pmd_val(pmd) | (_PAGE_PTE | H_PAGE_THP_HUGE));
196 }
197
198 extern unsigned long hash__pmd_hugepage_update(struct mm_struct *mm,
199 unsigned long addr, pmd_t *pmdp,
200 unsigned long clr, unsigned long set);
201 extern pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma,
202 unsigned long address, pmd_t *pmdp);
203 extern void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
204 pgtable_t pgtable);
205 extern pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
206 extern void hash__pmdp_huge_split_prepare(struct vm_area_struct *vma,
207 unsigned long address, pmd_t *pmdp);
208 extern pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm,
209 unsigned long addr, pmd_t *pmdp);
210 extern int hash__has_transparent_hugepage(void);
211 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
212 #endif /* __ASSEMBLY__ */
213
214 #endif /* _ASM_POWERPC_BOOK3S_64_HASH_64K_H */
215