• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 #ifndef _ASM_X86_TLBFLUSH_H
2 #define _ASM_X86_TLBFLUSH_H
3 
4 #include <linux/mm.h>
5 #include <linux/sched.h>
6 
7 #include <asm/processor.h>
8 #include <asm/special_insns.h>
9 
10 #ifdef CONFIG_PARAVIRT
11 #include <asm/paravirt.h>
12 #else
13 #define __flush_tlb() __native_flush_tlb()
14 #define __flush_tlb_global() __native_flush_tlb_global()
15 #define __flush_tlb_single(addr) __native_flush_tlb_single(addr)
16 #endif
17 
18 struct tlb_state {
19 #ifdef CONFIG_SMP
20 	struct mm_struct *active_mm;
21 	int state;
22 #endif
23 
24 	/*
25 	 * Access to this CR4 shadow and to H/W CR4 is protected by
26 	 * disabling interrupts when modifying either one.
27 	 */
28 	unsigned long cr4;
29 };
30 DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);
31 
32 /* Initialize cr4 shadow for this CPU. */
cr4_init_shadow(void)33 static inline void cr4_init_shadow(void)
34 {
35 	this_cpu_write(cpu_tlbstate.cr4, __read_cr4_safe());
36 }
37 
38 /* Set in this cpu's CR4. */
cr4_set_bits(unsigned long mask)39 static inline void cr4_set_bits(unsigned long mask)
40 {
41 	unsigned long cr4;
42 
43 	cr4 = this_cpu_read(cpu_tlbstate.cr4);
44 	if ((cr4 | mask) != cr4) {
45 		cr4 |= mask;
46 		this_cpu_write(cpu_tlbstate.cr4, cr4);
47 		__write_cr4(cr4);
48 	}
49 }
50 
51 /* Clear in this cpu's CR4. */
cr4_clear_bits(unsigned long mask)52 static inline void cr4_clear_bits(unsigned long mask)
53 {
54 	unsigned long cr4;
55 
56 	cr4 = this_cpu_read(cpu_tlbstate.cr4);
57 	if ((cr4 & ~mask) != cr4) {
58 		cr4 &= ~mask;
59 		this_cpu_write(cpu_tlbstate.cr4, cr4);
60 		__write_cr4(cr4);
61 	}
62 }
63 
64 /* Read the CR4 shadow. */
cr4_read_shadow(void)65 static inline unsigned long cr4_read_shadow(void)
66 {
67 	return this_cpu_read(cpu_tlbstate.cr4);
68 }
69 
70 /*
71  * Save some of cr4 feature set we're using (e.g.  Pentium 4MB
72  * enable and PPro Global page enable), so that any CPU's that boot
73  * up after us can get the correct flags.  This should only be used
74  * during boot on the boot cpu.
75  */
76 extern unsigned long mmu_cr4_features;
77 extern u32 *trampoline_cr4_features;
78 
cr4_set_bits_and_update_boot(unsigned long mask)79 static inline void cr4_set_bits_and_update_boot(unsigned long mask)
80 {
81 	mmu_cr4_features |= mask;
82 	if (trampoline_cr4_features)
83 		*trampoline_cr4_features = mmu_cr4_features;
84 	cr4_set_bits(mask);
85 }
86 
__native_flush_tlb(void)87 static inline void __native_flush_tlb(void)
88 {
89 	/*
90 	 * If current->mm == NULL then we borrow a mm which may change during a
91 	 * task switch and therefore we must not be preempted while we write CR3
92 	 * back:
93 	 */
94 	preempt_disable();
95 	native_write_cr3(native_read_cr3());
96 	preempt_enable();
97 }
98 
__native_flush_tlb_global_irq_disabled(void)99 static inline void __native_flush_tlb_global_irq_disabled(void)
100 {
101 	unsigned long cr4;
102 
103 	cr4 = this_cpu_read(cpu_tlbstate.cr4);
104 	/* clear PGE */
105 	native_write_cr4(cr4 & ~X86_CR4_PGE);
106 	/* write old PGE again and flush TLBs */
107 	native_write_cr4(cr4);
108 }
109 
__native_flush_tlb_global(void)110 static inline void __native_flush_tlb_global(void)
111 {
112 	unsigned long flags;
113 
114 	/*
115 	 * Read-modify-write to CR4 - protect it from preemption and
116 	 * from interrupts. (Use the raw variant because this code can
117 	 * be called from deep inside debugging code.)
118 	 */
119 	raw_local_irq_save(flags);
120 
121 	__native_flush_tlb_global_irq_disabled();
122 
123 	raw_local_irq_restore(flags);
124 }
125 
__native_flush_tlb_single(unsigned long addr)126 static inline void __native_flush_tlb_single(unsigned long addr)
127 {
128 	asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
129 }
130 
__flush_tlb_all(void)131 static inline void __flush_tlb_all(void)
132 {
133 	if (cpu_has_pge)
134 		__flush_tlb_global();
135 	else
136 		__flush_tlb();
137 }
138 
__flush_tlb_one(unsigned long addr)139 static inline void __flush_tlb_one(unsigned long addr)
140 {
141 	count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE);
142 	__flush_tlb_single(addr);
143 }
144 
145 #define TLB_FLUSH_ALL	-1UL
146 
147 /*
148  * TLB flushing:
149  *
150  *  - flush_tlb() flushes the current mm struct TLBs
151  *  - flush_tlb_all() flushes all processes TLBs
152  *  - flush_tlb_mm(mm) flushes the specified mm context TLB's
153  *  - flush_tlb_page(vma, vmaddr) flushes one page
154  *  - flush_tlb_range(vma, start, end) flushes a range of pages
155  *  - flush_tlb_kernel_range(start, end) flushes a range of kernel pages
156  *  - flush_tlb_others(cpumask, mm, start, end) flushes TLBs on other cpus
157  *
158  * ..but the i386 has somewhat limited tlb flushing capabilities,
159  * and page-granular flushes are available only on i486 and up.
160  */
161 
162 #ifndef CONFIG_SMP
163 
164 /* "_up" is for UniProcessor.
165  *
166  * This is a helper for other header functions.  *Not* intended to be called
167  * directly.  All global TLB flushes need to either call this, or to bump the
168  * vm statistics themselves.
169  */
__flush_tlb_up(void)170 static inline void __flush_tlb_up(void)
171 {
172 	count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
173 	__flush_tlb();
174 }
175 
flush_tlb_all(void)176 static inline void flush_tlb_all(void)
177 {
178 	count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
179 	__flush_tlb_all();
180 }
181 
flush_tlb(void)182 static inline void flush_tlb(void)
183 {
184 	__flush_tlb_up();
185 }
186 
local_flush_tlb(void)187 static inline void local_flush_tlb(void)
188 {
189 	__flush_tlb_up();
190 }
191 
flush_tlb_mm(struct mm_struct * mm)192 static inline void flush_tlb_mm(struct mm_struct *mm)
193 {
194 	if (mm == current->active_mm)
195 		__flush_tlb_up();
196 }
197 
flush_tlb_page(struct vm_area_struct * vma,unsigned long addr)198 static inline void flush_tlb_page(struct vm_area_struct *vma,
199 				  unsigned long addr)
200 {
201 	if (vma->vm_mm == current->active_mm)
202 		__flush_tlb_one(addr);
203 }
204 
flush_tlb_range(struct vm_area_struct * vma,unsigned long start,unsigned long end)205 static inline void flush_tlb_range(struct vm_area_struct *vma,
206 				   unsigned long start, unsigned long end)
207 {
208 	if (vma->vm_mm == current->active_mm)
209 		__flush_tlb_up();
210 }
211 
flush_tlb_mm_range(struct mm_struct * mm,unsigned long start,unsigned long end,unsigned long vmflag)212 static inline void flush_tlb_mm_range(struct mm_struct *mm,
213 	   unsigned long start, unsigned long end, unsigned long vmflag)
214 {
215 	if (mm == current->active_mm)
216 		__flush_tlb_up();
217 }
218 
native_flush_tlb_others(const struct cpumask * cpumask,struct mm_struct * mm,unsigned long start,unsigned long end)219 static inline void native_flush_tlb_others(const struct cpumask *cpumask,
220 					   struct mm_struct *mm,
221 					   unsigned long start,
222 					   unsigned long end)
223 {
224 }
225 
reset_lazy_tlbstate(void)226 static inline void reset_lazy_tlbstate(void)
227 {
228 }
229 
flush_tlb_kernel_range(unsigned long start,unsigned long end)230 static inline void flush_tlb_kernel_range(unsigned long start,
231 					  unsigned long end)
232 {
233 	flush_tlb_all();
234 }
235 
236 #else  /* SMP */
237 
238 #include <asm/smp.h>
239 
240 #define local_flush_tlb() __flush_tlb()
241 
242 #define flush_tlb_mm(mm)	flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL)
243 
244 #define flush_tlb_range(vma, start, end)	\
245 		flush_tlb_mm_range(vma->vm_mm, start, end, vma->vm_flags)
246 
247 extern void flush_tlb_all(void);
248 extern void flush_tlb_current_task(void);
249 extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
250 extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
251 				unsigned long end, unsigned long vmflag);
252 extern void flush_tlb_kernel_range(unsigned long start, unsigned long end);
253 
254 #define flush_tlb()	flush_tlb_current_task()
255 
256 void native_flush_tlb_others(const struct cpumask *cpumask,
257 				struct mm_struct *mm,
258 				unsigned long start, unsigned long end);
259 
260 #define TLBSTATE_OK	1
261 #define TLBSTATE_LAZY	2
262 
reset_lazy_tlbstate(void)263 static inline void reset_lazy_tlbstate(void)
264 {
265 	this_cpu_write(cpu_tlbstate.state, 0);
266 	this_cpu_write(cpu_tlbstate.active_mm, &init_mm);
267 }
268 
269 #endif	/* SMP */
270 
271 #ifndef CONFIG_PARAVIRT
272 #define flush_tlb_others(mask, mm, start, end)	\
273 	native_flush_tlb_others(mask, mm, start, end)
274 #endif
275 
276 #endif /* _ASM_X86_TLBFLUSH_H */
277