1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * PPC64 Huge TLB Page Support for hash based MMUs (POWER4 and later)
4 *
5 * Copyright (C) 2003 David Gibson, IBM Corporation.
6 *
7 * Based on the IA-32 version:
8 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
9 */
10
11 #include <linux/mm.h>
12 #include <linux/hugetlb.h>
13 #include <asm/cacheflush.h>
14 #include <asm/machdep.h>
15
16 unsigned int hpage_shift;
17 EXPORT_SYMBOL(hpage_shift);
18
19 extern long hpte_insert_repeating(unsigned long hash, unsigned long vpn,
20 unsigned long pa, unsigned long rlags,
21 unsigned long vflags, int psize, int ssize);
22
__hash_page_huge(unsigned long ea,unsigned long access,unsigned long vsid,pte_t * ptep,unsigned long trap,unsigned long flags,int ssize,unsigned int shift,unsigned int mmu_psize)23 int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
24 pte_t *ptep, unsigned long trap, unsigned long flags,
25 int ssize, unsigned int shift, unsigned int mmu_psize)
26 {
27 real_pte_t rpte;
28 unsigned long vpn;
29 unsigned long old_pte, new_pte;
30 unsigned long rflags, pa;
31 long slot, offset;
32
33 BUG_ON(shift != mmu_psize_defs[mmu_psize].shift);
34
35 /* Search the Linux page table for a match with va */
36 vpn = hpt_vpn(ea, vsid, ssize);
37
38 /*
39 * At this point, we have a pte (old_pte) which can be used to build
40 * or update an HPTE. There are 2 cases:
41 *
42 * 1. There is a valid (present) pte with no associated HPTE (this is
43 * the most common case)
44 * 2. There is a valid (present) pte with an associated HPTE. The
45 * current values of the pp bits in the HPTE prevent access
46 * because we are doing software DIRTY bit management and the
47 * page is currently not DIRTY.
48 */
49
50
51 do {
52 old_pte = pte_val(*ptep);
53 /* If PTE busy, retry the access */
54 if (unlikely(old_pte & H_PAGE_BUSY))
55 return 0;
56 /* If PTE permissions don't match, take page fault */
57 if (unlikely(!check_pte_access(access, old_pte)))
58 return 1;
59
60 /*
61 * Try to lock the PTE, add ACCESSED and DIRTY if it was
62 * a write access
63 */
64 new_pte = old_pte | H_PAGE_BUSY | _PAGE_ACCESSED;
65 if (access & _PAGE_WRITE)
66 new_pte |= _PAGE_DIRTY;
67 } while(!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));
68
69 /* Make sure this is a hugetlb entry */
70 if (old_pte & (H_PAGE_THP_HUGE | _PAGE_DEVMAP))
71 return 0;
72
73 rflags = htab_convert_pte_flags(new_pte);
74 if (unlikely(mmu_psize == MMU_PAGE_16G))
75 offset = PTRS_PER_PUD;
76 else
77 offset = PTRS_PER_PMD;
78 rpte = __real_pte(__pte(old_pte), ptep, offset);
79
80 if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
81 /*
82 * No CPU has hugepages but lacks no execute, so we
83 * don't need to worry about that case
84 */
85 rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);
86
87 /* Check if pte already has an hpte (case 2) */
88 if (unlikely(old_pte & H_PAGE_HASHPTE)) {
89 /* There MIGHT be an HPTE for this pte */
90 unsigned long gslot;
91
92 gslot = pte_get_hash_gslot(vpn, shift, ssize, rpte, 0);
93 if (mmu_hash_ops.hpte_updatepp(gslot, rflags, vpn, mmu_psize,
94 mmu_psize, ssize, flags) == -1)
95 old_pte &= ~_PAGE_HPTEFLAGS;
96 }
97
98 if (likely(!(old_pte & H_PAGE_HASHPTE))) {
99 unsigned long hash = hpt_hash(vpn, shift, ssize);
100
101 pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
102
103 /* clear HPTE slot informations in new PTE */
104 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | H_PAGE_HASHPTE;
105
106 slot = hpte_insert_repeating(hash, vpn, pa, rflags, 0,
107 mmu_psize, ssize);
108
109 /*
110 * Hypervisor failure. Restore old pte and return -1
111 * similar to __hash_page_*
112 */
113 if (unlikely(slot == -2)) {
114 *ptep = __pte(old_pte);
115 hash_failure_debug(ea, access, vsid, trap, ssize,
116 mmu_psize, mmu_psize, old_pte);
117 return -1;
118 }
119
120 new_pte |= pte_set_hidx(ptep, rpte, 0, slot, offset);
121 }
122
123 /*
124 * No need to use ldarx/stdcx here
125 */
126 *ptep = __pte(new_pte & ~H_PAGE_BUSY);
127 return 0;
128 }
129
huge_ptep_modify_prot_start(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep)130 pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma,
131 unsigned long addr, pte_t *ptep)
132 {
133 unsigned long pte_val;
134 /*
135 * Clear the _PAGE_PRESENT so that no hardware parallel update is
136 * possible. Also keep the pte_present true so that we don't take
137 * wrong fault.
138 */
139 pte_val = pte_update(vma->vm_mm, addr, ptep,
140 _PAGE_PRESENT, _PAGE_INVALID, 1);
141
142 return __pte(pte_val);
143 }
144
huge_ptep_modify_prot_commit(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep,pte_t old_pte,pte_t pte)145 void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr,
146 pte_t *ptep, pte_t old_pte, pte_t pte)
147 {
148
149 if (radix_enabled())
150 return radix__huge_ptep_modify_prot_commit(vma, addr, ptep,
151 old_pte, pte);
152 set_huge_pte_at(vma->vm_mm, addr, ptep, pte);
153 }
154
hugetlbpage_init_default(void)155 void hugetlbpage_init_default(void)
156 {
157 /* Set default large page size. Currently, we pick 16M or 1M
158 * depending on what is available
159 */
160 if (mmu_psize_defs[MMU_PAGE_16M].shift)
161 hpage_shift = mmu_psize_defs[MMU_PAGE_16M].shift;
162 else if (mmu_psize_defs[MMU_PAGE_1M].shift)
163 hpage_shift = mmu_psize_defs[MMU_PAGE_1M].shift;
164 else if (mmu_psize_defs[MMU_PAGE_2M].shift)
165 hpage_shift = mmu_psize_defs[MMU_PAGE_2M].shift;
166 }
167