1 #ifndef _ASM_X86_TLBFLUSH_H
2 #define _ASM_X86_TLBFLUSH_H
3
4 #include <linux/mm.h>
5 #include <linux/sched.h>
6
7 #include <asm/processor.h>
8 #include <asm/special_insns.h>
9
10 #ifdef CONFIG_PARAVIRT
11 #include <asm/paravirt.h>
12 #else
13 #define __flush_tlb() __native_flush_tlb()
14 #define __flush_tlb_global() __native_flush_tlb_global()
15 #define __flush_tlb_single(addr) __native_flush_tlb_single(addr)
16 #endif
17
18 struct tlb_state {
19 #ifdef CONFIG_SMP
20 struct mm_struct *active_mm;
21 int state;
22 #endif
23
24 /*
25 * Access to this CR4 shadow and to H/W CR4 is protected by
26 * disabling interrupts when modifying either one.
27 */
28 unsigned long cr4;
29 };
30 DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);
31
32 /* Initialize cr4 shadow for this CPU. */
cr4_init_shadow(void)33 static inline void cr4_init_shadow(void)
34 {
35 this_cpu_write(cpu_tlbstate.cr4, __read_cr4_safe());
36 }
37
38 /* Set in this cpu's CR4. */
cr4_set_bits(unsigned long mask)39 static inline void cr4_set_bits(unsigned long mask)
40 {
41 unsigned long cr4;
42
43 cr4 = this_cpu_read(cpu_tlbstate.cr4);
44 if ((cr4 | mask) != cr4) {
45 cr4 |= mask;
46 this_cpu_write(cpu_tlbstate.cr4, cr4);
47 __write_cr4(cr4);
48 }
49 }
50
51 /* Clear in this cpu's CR4. */
cr4_clear_bits(unsigned long mask)52 static inline void cr4_clear_bits(unsigned long mask)
53 {
54 unsigned long cr4;
55
56 cr4 = this_cpu_read(cpu_tlbstate.cr4);
57 if ((cr4 & ~mask) != cr4) {
58 cr4 &= ~mask;
59 this_cpu_write(cpu_tlbstate.cr4, cr4);
60 __write_cr4(cr4);
61 }
62 }
63
64 /* Read the CR4 shadow. */
cr4_read_shadow(void)65 static inline unsigned long cr4_read_shadow(void)
66 {
67 return this_cpu_read(cpu_tlbstate.cr4);
68 }
69
70 /*
71 * Save some of cr4 feature set we're using (e.g. Pentium 4MB
72 * enable and PPro Global page enable), so that any CPU's that boot
73 * up after us can get the correct flags. This should only be used
74 * during boot on the boot cpu.
75 */
76 extern unsigned long mmu_cr4_features;
77 extern u32 *trampoline_cr4_features;
78
cr4_set_bits_and_update_boot(unsigned long mask)79 static inline void cr4_set_bits_and_update_boot(unsigned long mask)
80 {
81 mmu_cr4_features |= mask;
82 if (trampoline_cr4_features)
83 *trampoline_cr4_features = mmu_cr4_features;
84 cr4_set_bits(mask);
85 }
86
__native_flush_tlb(void)87 static inline void __native_flush_tlb(void)
88 {
89 /*
90 * If current->mm == NULL then we borrow a mm which may change during a
91 * task switch and therefore we must not be preempted while we write CR3
92 * back:
93 */
94 preempt_disable();
95 native_write_cr3(native_read_cr3());
96 preempt_enable();
97 }
98
__native_flush_tlb_global_irq_disabled(void)99 static inline void __native_flush_tlb_global_irq_disabled(void)
100 {
101 unsigned long cr4;
102
103 cr4 = this_cpu_read(cpu_tlbstate.cr4);
104 /* clear PGE */
105 native_write_cr4(cr4 & ~X86_CR4_PGE);
106 /* write old PGE again and flush TLBs */
107 native_write_cr4(cr4);
108 }
109
__native_flush_tlb_global(void)110 static inline void __native_flush_tlb_global(void)
111 {
112 unsigned long flags;
113
114 /*
115 * Read-modify-write to CR4 - protect it from preemption and
116 * from interrupts. (Use the raw variant because this code can
117 * be called from deep inside debugging code.)
118 */
119 raw_local_irq_save(flags);
120
121 __native_flush_tlb_global_irq_disabled();
122
123 raw_local_irq_restore(flags);
124 }
125
__native_flush_tlb_single(unsigned long addr)126 static inline void __native_flush_tlb_single(unsigned long addr)
127 {
128 asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
129 }
130
__flush_tlb_all(void)131 static inline void __flush_tlb_all(void)
132 {
133 if (cpu_has_pge)
134 __flush_tlb_global();
135 else
136 __flush_tlb();
137 }
138
__flush_tlb_one(unsigned long addr)139 static inline void __flush_tlb_one(unsigned long addr)
140 {
141 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE);
142 __flush_tlb_single(addr);
143 }
144
145 #define TLB_FLUSH_ALL -1UL
146
147 /*
148 * TLB flushing:
149 *
150 * - flush_tlb() flushes the current mm struct TLBs
151 * - flush_tlb_all() flushes all processes TLBs
152 * - flush_tlb_mm(mm) flushes the specified mm context TLB's
153 * - flush_tlb_page(vma, vmaddr) flushes one page
154 * - flush_tlb_range(vma, start, end) flushes a range of pages
155 * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages
156 * - flush_tlb_others(cpumask, mm, start, end) flushes TLBs on other cpus
157 *
158 * ..but the i386 has somewhat limited tlb flushing capabilities,
159 * and page-granular flushes are available only on i486 and up.
160 */
161
162 #ifndef CONFIG_SMP
163
164 /* "_up" is for UniProcessor.
165 *
166 * This is a helper for other header functions. *Not* intended to be called
167 * directly. All global TLB flushes need to either call this, or to bump the
168 * vm statistics themselves.
169 */
__flush_tlb_up(void)170 static inline void __flush_tlb_up(void)
171 {
172 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
173 __flush_tlb();
174 }
175
flush_tlb_all(void)176 static inline void flush_tlb_all(void)
177 {
178 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
179 __flush_tlb_all();
180 }
181
flush_tlb(void)182 static inline void flush_tlb(void)
183 {
184 __flush_tlb_up();
185 }
186
local_flush_tlb(void)187 static inline void local_flush_tlb(void)
188 {
189 __flush_tlb_up();
190 }
191
flush_tlb_mm(struct mm_struct * mm)192 static inline void flush_tlb_mm(struct mm_struct *mm)
193 {
194 if (mm == current->active_mm)
195 __flush_tlb_up();
196 }
197
flush_tlb_page(struct vm_area_struct * vma,unsigned long addr)198 static inline void flush_tlb_page(struct vm_area_struct *vma,
199 unsigned long addr)
200 {
201 if (vma->vm_mm == current->active_mm)
202 __flush_tlb_one(addr);
203 }
204
flush_tlb_range(struct vm_area_struct * vma,unsigned long start,unsigned long end)205 static inline void flush_tlb_range(struct vm_area_struct *vma,
206 unsigned long start, unsigned long end)
207 {
208 if (vma->vm_mm == current->active_mm)
209 __flush_tlb_up();
210 }
211
flush_tlb_mm_range(struct mm_struct * mm,unsigned long start,unsigned long end,unsigned long vmflag)212 static inline void flush_tlb_mm_range(struct mm_struct *mm,
213 unsigned long start, unsigned long end, unsigned long vmflag)
214 {
215 if (mm == current->active_mm)
216 __flush_tlb_up();
217 }
218
native_flush_tlb_others(const struct cpumask * cpumask,struct mm_struct * mm,unsigned long start,unsigned long end)219 static inline void native_flush_tlb_others(const struct cpumask *cpumask,
220 struct mm_struct *mm,
221 unsigned long start,
222 unsigned long end)
223 {
224 }
225
reset_lazy_tlbstate(void)226 static inline void reset_lazy_tlbstate(void)
227 {
228 }
229
flush_tlb_kernel_range(unsigned long start,unsigned long end)230 static inline void flush_tlb_kernel_range(unsigned long start,
231 unsigned long end)
232 {
233 flush_tlb_all();
234 }
235
236 #else /* SMP */
237
238 #include <asm/smp.h>
239
240 #define local_flush_tlb() __flush_tlb()
241
242 #define flush_tlb_mm(mm) flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL)
243
244 #define flush_tlb_range(vma, start, end) \
245 flush_tlb_mm_range(vma->vm_mm, start, end, vma->vm_flags)
246
247 extern void flush_tlb_all(void);
248 extern void flush_tlb_current_task(void);
249 extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
250 extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
251 unsigned long end, unsigned long vmflag);
252 extern void flush_tlb_kernel_range(unsigned long start, unsigned long end);
253
254 #define flush_tlb() flush_tlb_current_task()
255
256 void native_flush_tlb_others(const struct cpumask *cpumask,
257 struct mm_struct *mm,
258 unsigned long start, unsigned long end);
259
260 #define TLBSTATE_OK 1
261 #define TLBSTATE_LAZY 2
262
reset_lazy_tlbstate(void)263 static inline void reset_lazy_tlbstate(void)
264 {
265 this_cpu_write(cpu_tlbstate.state, 0);
266 this_cpu_write(cpu_tlbstate.active_mm, &init_mm);
267 }
268
269 #endif /* SMP */
270
271 #ifndef CONFIG_PARAVIRT
272 #define flush_tlb_others(mask, mm, start, end) \
273 native_flush_tlb_others(mask, mm, start, end)
274 #endif
275
276 #endif /* _ASM_X86_TLBFLUSH_H */
277