1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * PPC64 Huge TLB Page Support for hash based MMUs (POWER4 and later)
4 *
5 * Copyright (C) 2003 David Gibson, IBM Corporation.
6 *
7 * Based on the IA-32 version:
8 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
9 */
10
11 #include <linux/mm.h>
12 #include <linux/hugetlb.h>
13 #include <asm/cacheflush.h>
14 #include <asm/machdep.h>
15
16 unsigned int hpage_shift;
17 EXPORT_SYMBOL(hpage_shift);
18
__hash_page_huge(unsigned long ea,unsigned long access,unsigned long vsid,pte_t * ptep,unsigned long trap,unsigned long flags,int ssize,unsigned int shift,unsigned int mmu_psize)19 int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
20 pte_t *ptep, unsigned long trap, unsigned long flags,
21 int ssize, unsigned int shift, unsigned int mmu_psize)
22 {
23 real_pte_t rpte;
24 unsigned long vpn;
25 unsigned long old_pte, new_pte;
26 unsigned long rflags, pa;
27 long slot, offset;
28
29 BUG_ON(shift != mmu_psize_defs[mmu_psize].shift);
30
31 /* Search the Linux page table for a match with va */
32 vpn = hpt_vpn(ea, vsid, ssize);
33
34 /*
35 * At this point, we have a pte (old_pte) which can be used to build
36 * or update an HPTE. There are 2 cases:
37 *
38 * 1. There is a valid (present) pte with no associated HPTE (this is
39 * the most common case)
40 * 2. There is a valid (present) pte with an associated HPTE. The
41 * current values of the pp bits in the HPTE prevent access
42 * because we are doing software DIRTY bit management and the
43 * page is currently not DIRTY.
44 */
45
46
47 do {
48 old_pte = pte_val(*ptep);
49 /* If PTE busy, retry the access */
50 if (unlikely(old_pte & H_PAGE_BUSY))
51 return 0;
52 /* If PTE permissions don't match, take page fault */
53 if (unlikely(!check_pte_access(access, old_pte)))
54 return 1;
55
56 /*
57 * Try to lock the PTE, add ACCESSED and DIRTY if it was
58 * a write access
59 */
60 new_pte = old_pte | H_PAGE_BUSY | _PAGE_ACCESSED;
61 if (access & _PAGE_WRITE)
62 new_pte |= _PAGE_DIRTY;
63 } while(!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));
64
65 /* Make sure this is a hugetlb entry */
66 if (old_pte & (H_PAGE_THP_HUGE | _PAGE_DEVMAP))
67 return 0;
68
69 rflags = htab_convert_pte_flags(new_pte, flags);
70 if (unlikely(mmu_psize == MMU_PAGE_16G))
71 offset = PTRS_PER_PUD;
72 else
73 offset = PTRS_PER_PMD;
74 rpte = __real_pte(__pte(old_pte), ptep, offset);
75
76 if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
77 /*
78 * No CPU has hugepages but lacks no execute, so we
79 * don't need to worry about that case
80 */
81 rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);
82
83 /* Check if pte already has an hpte (case 2) */
84 if (unlikely(old_pte & H_PAGE_HASHPTE)) {
85 /* There MIGHT be an HPTE for this pte */
86 unsigned long gslot;
87
88 gslot = pte_get_hash_gslot(vpn, shift, ssize, rpte, 0);
89 if (mmu_hash_ops.hpte_updatepp(gslot, rflags, vpn, mmu_psize,
90 mmu_psize, ssize, flags) == -1)
91 old_pte &= ~_PAGE_HPTEFLAGS;
92 }
93
94 if (likely(!(old_pte & H_PAGE_HASHPTE))) {
95 unsigned long hash = hpt_hash(vpn, shift, ssize);
96
97 pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
98
99 /* clear HPTE slot informations in new PTE */
100 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | H_PAGE_HASHPTE;
101
102 slot = hpte_insert_repeating(hash, vpn, pa, rflags, 0,
103 mmu_psize, ssize);
104
105 /*
106 * Hypervisor failure. Restore old pte and return -1
107 * similar to __hash_page_*
108 */
109 if (unlikely(slot == -2)) {
110 *ptep = __pte(old_pte);
111 hash_failure_debug(ea, access, vsid, trap, ssize,
112 mmu_psize, mmu_psize, old_pte);
113 return -1;
114 }
115
116 new_pte |= pte_set_hidx(ptep, rpte, 0, slot, offset);
117 }
118
119 /*
120 * No need to use ldarx/stdcx here
121 */
122 *ptep = __pte(new_pte & ~H_PAGE_BUSY);
123 return 0;
124 }
125
huge_ptep_modify_prot_start(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep)126 pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma,
127 unsigned long addr, pte_t *ptep)
128 {
129 unsigned long pte_val;
130 /*
131 * Clear the _PAGE_PRESENT so that no hardware parallel update is
132 * possible. Also keep the pte_present true so that we don't take
133 * wrong fault.
134 */
135 pte_val = pte_update(vma->vm_mm, addr, ptep,
136 _PAGE_PRESENT, _PAGE_INVALID, 1);
137
138 return __pte(pte_val);
139 }
140
huge_ptep_modify_prot_commit(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep,pte_t old_pte,pte_t pte)141 void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr,
142 pte_t *ptep, pte_t old_pte, pte_t pte)
143 {
144
145 if (radix_enabled())
146 return radix__huge_ptep_modify_prot_commit(vma, addr, ptep,
147 old_pte, pte);
148 set_huge_pte_at(vma->vm_mm, addr, ptep, pte);
149 }
150
hugetlbpage_init_default(void)151 void hugetlbpage_init_default(void)
152 {
153 /* Set default large page size. Currently, we pick 16M or 1M
154 * depending on what is available
155 */
156 if (mmu_psize_defs[MMU_PAGE_16M].shift)
157 hpage_shift = mmu_psize_defs[MMU_PAGE_16M].shift;
158 else if (mmu_psize_defs[MMU_PAGE_1M].shift)
159 hpage_shift = mmu_psize_defs[MMU_PAGE_1M].shift;
160 else if (mmu_psize_defs[MMU_PAGE_2M].shift)
161 hpage_shift = mmu_psize_defs[MMU_PAGE_2M].shift;
162 }
163