• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  *  arch/arm/include/asm/tlbflush.h
3  *
4  *  Copyright (C) 1999-2003 Russell King
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 as
8  * published by the Free Software Foundation.
9  */
10 #ifndef _ASMARM_TLBFLUSH_H
11 #define _ASMARM_TLBFLUSH_H
12 
13 #ifdef CONFIG_MMU
14 
15 #include <asm/glue.h>
16 
17 #define TLB_V4_U_PAGE	(1 << 1)
18 #define TLB_V4_D_PAGE	(1 << 2)
19 #define TLB_V4_I_PAGE	(1 << 3)
20 #define TLB_V6_U_PAGE	(1 << 4)
21 #define TLB_V6_D_PAGE	(1 << 5)
22 #define TLB_V6_I_PAGE	(1 << 6)
23 
24 #define TLB_V4_U_FULL	(1 << 9)
25 #define TLB_V4_D_FULL	(1 << 10)
26 #define TLB_V4_I_FULL	(1 << 11)
27 #define TLB_V6_U_FULL	(1 << 12)
28 #define TLB_V6_D_FULL	(1 << 13)
29 #define TLB_V6_I_FULL	(1 << 14)
30 
31 #define TLB_V6_U_ASID	(1 << 16)
32 #define TLB_V6_D_ASID	(1 << 17)
33 #define TLB_V6_I_ASID	(1 << 18)
34 
35 #define TLB_V6_BP	(1 << 19)
36 
37 /* Unified Inner Shareable TLB operations (ARMv7 MP extensions) */
38 #define TLB_V7_UIS_PAGE	(1 << 20)
39 #define TLB_V7_UIS_FULL (1 << 21)
40 #define TLB_V7_UIS_ASID (1 << 22)
41 #define TLB_V7_UIS_BP	(1 << 23)
42 
43 #define TLB_BARRIER	(1 << 28)
44 #define TLB_L2CLEAN_FR	(1 << 29)		/* Feroceon */
45 #define TLB_DCLEAN	(1 << 30)
46 #define TLB_WB		(1 << 31)
47 
48 /*
49  *	MMU TLB Model
50  *	=============
51  *
52  *	We have the following to choose from:
53  *	  v4    - ARMv4 without write buffer
54  *	  v4wb  - ARMv4 with write buffer without I TLB flush entry instruction
55  *	  v4wbi - ARMv4 with write buffer with I TLB flush entry instruction
56  *	  fr    - Feroceon (v4wbi with non-outer-cacheable page table walks)
57  *	  fa    - Faraday (v4 with write buffer with UTLB)
58  *	  v6wbi - ARMv6 with write buffer with I TLB flush entry instruction
59  *	  v7wbi - identical to v6wbi
60  */
61 #undef _TLB
62 #undef MULTI_TLB
63 
64 #ifdef CONFIG_SMP_ON_UP
65 #define MULTI_TLB 1
66 #endif
67 
68 #define v4_tlb_flags	(TLB_V4_U_FULL | TLB_V4_U_PAGE)
69 
70 #ifdef CONFIG_CPU_TLB_V4WT
71 # define v4_possible_flags	v4_tlb_flags
72 # define v4_always_flags	v4_tlb_flags
73 # ifdef _TLB
74 #  define MULTI_TLB 1
75 # else
76 #  define _TLB v4
77 # endif
78 #else
79 # define v4_possible_flags	0
80 # define v4_always_flags	(-1UL)
81 #endif
82 
83 #define fa_tlb_flags	(TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
84 			 TLB_V4_U_FULL | TLB_V4_U_PAGE)
85 
86 #ifdef CONFIG_CPU_TLB_FA
87 # define fa_possible_flags	fa_tlb_flags
88 # define fa_always_flags	fa_tlb_flags
89 # ifdef _TLB
90 #  define MULTI_TLB 1
91 # else
92 #  define _TLB fa
93 # endif
94 #else
95 # define fa_possible_flags	0
96 # define fa_always_flags	(-1UL)
97 #endif
98 
99 #define v4wbi_tlb_flags	(TLB_WB | TLB_DCLEAN | \
100 			 TLB_V4_I_FULL | TLB_V4_D_FULL | \
101 			 TLB_V4_I_PAGE | TLB_V4_D_PAGE)
102 
103 #ifdef CONFIG_CPU_TLB_V4WBI
104 # define v4wbi_possible_flags	v4wbi_tlb_flags
105 # define v4wbi_always_flags	v4wbi_tlb_flags
106 # ifdef _TLB
107 #  define MULTI_TLB 1
108 # else
109 #  define _TLB v4wbi
110 # endif
111 #else
112 # define v4wbi_possible_flags	0
113 # define v4wbi_always_flags	(-1UL)
114 #endif
115 
116 #define fr_tlb_flags	(TLB_WB | TLB_DCLEAN | TLB_L2CLEAN_FR | \
117 			 TLB_V4_I_FULL | TLB_V4_D_FULL | \
118 			 TLB_V4_I_PAGE | TLB_V4_D_PAGE)
119 
120 #ifdef CONFIG_CPU_TLB_FEROCEON
121 # define fr_possible_flags	fr_tlb_flags
122 # define fr_always_flags	fr_tlb_flags
123 # ifdef _TLB
124 #  define MULTI_TLB 1
125 # else
126 #  define _TLB v4wbi
127 # endif
128 #else
129 # define fr_possible_flags	0
130 # define fr_always_flags	(-1UL)
131 #endif
132 
133 #define v4wb_tlb_flags	(TLB_WB | TLB_DCLEAN | \
134 			 TLB_V4_I_FULL | TLB_V4_D_FULL | \
135 			 TLB_V4_D_PAGE)
136 
137 #ifdef CONFIG_CPU_TLB_V4WB
138 # define v4wb_possible_flags	v4wb_tlb_flags
139 # define v4wb_always_flags	v4wb_tlb_flags
140 # ifdef _TLB
141 #  define MULTI_TLB 1
142 # else
143 #  define _TLB v4wb
144 # endif
145 #else
146 # define v4wb_possible_flags	0
147 # define v4wb_always_flags	(-1UL)
148 #endif
149 
150 #define v6wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
151 			 TLB_V6_I_FULL | TLB_V6_D_FULL | \
152 			 TLB_V6_I_PAGE | TLB_V6_D_PAGE | \
153 			 TLB_V6_I_ASID | TLB_V6_D_ASID | \
154 			 TLB_V6_BP)
155 
156 #ifdef CONFIG_CPU_TLB_V6
157 # define v6wbi_possible_flags	v6wbi_tlb_flags
158 # define v6wbi_always_flags	v6wbi_tlb_flags
159 # ifdef _TLB
160 #  define MULTI_TLB 1
161 # else
162 #  define _TLB v6wbi
163 # endif
164 #else
165 # define v6wbi_possible_flags	0
166 # define v6wbi_always_flags	(-1UL)
167 #endif
168 
169 #define v7wbi_tlb_flags_smp	(TLB_WB | TLB_BARRIER | \
170 				 TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | \
171 				 TLB_V7_UIS_ASID | TLB_V7_UIS_BP)
172 #define v7wbi_tlb_flags_up	(TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
173 				 TLB_V6_U_FULL | TLB_V6_U_PAGE | \
174 				 TLB_V6_U_ASID | TLB_V6_BP)
175 
176 #ifdef CONFIG_CPU_TLB_V7
177 
178 # ifdef CONFIG_SMP_ON_UP
179 #  define v7wbi_possible_flags	(v7wbi_tlb_flags_smp | v7wbi_tlb_flags_up)
180 #  define v7wbi_always_flags	(v7wbi_tlb_flags_smp & v7wbi_tlb_flags_up)
181 # elif defined(CONFIG_SMP)
182 #  define v7wbi_possible_flags	v7wbi_tlb_flags_smp
183 #  define v7wbi_always_flags	v7wbi_tlb_flags_smp
184 # else
185 #  define v7wbi_possible_flags	v7wbi_tlb_flags_up
186 #  define v7wbi_always_flags	v7wbi_tlb_flags_up
187 # endif
188 # ifdef _TLB
189 #  define MULTI_TLB 1
190 # else
191 #  define _TLB v7wbi
192 # endif
193 #else
194 # define v7wbi_possible_flags	0
195 # define v7wbi_always_flags	(-1UL)
196 #endif
197 
198 #ifndef _TLB
199 #error Unknown TLB model
200 #endif
201 
202 #ifndef __ASSEMBLY__
203 
204 #include <linux/sched.h>
205 
206 struct cpu_tlb_fns {
207 	void (*flush_user_range)(unsigned long, unsigned long, struct vm_area_struct *);
208 	void (*flush_kern_range)(unsigned long, unsigned long);
209 	unsigned long tlb_flags;
210 };
211 
212 /*
213  * Select the calling method
214  */
215 #ifdef MULTI_TLB
216 
217 #define __cpu_flush_user_tlb_range	cpu_tlb.flush_user_range
218 #define __cpu_flush_kern_tlb_range	cpu_tlb.flush_kern_range
219 
220 #else
221 
222 #define __cpu_flush_user_tlb_range	__glue(_TLB,_flush_user_tlb_range)
223 #define __cpu_flush_kern_tlb_range	__glue(_TLB,_flush_kern_tlb_range)
224 
225 extern void __cpu_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
226 extern void __cpu_flush_kern_tlb_range(unsigned long, unsigned long);
227 
228 #endif
229 
230 extern struct cpu_tlb_fns cpu_tlb;
231 
232 #define __cpu_tlb_flags			cpu_tlb.tlb_flags
233 
234 /*
235  *	TLB Management
236  *	==============
237  *
238  *	The arch/arm/mm/tlb-*.S files implement these methods.
239  *
240  *	The TLB specific code is expected to perform whatever tests it
241  *	needs to determine if it should invalidate the TLB for each
242  *	call.  Start addresses are inclusive and end addresses are
243  *	exclusive; it is safe to round these addresses down.
244  *
245  *	flush_tlb_all()
246  *
247  *		Invalidate the entire TLB.
248  *
249  *	flush_tlb_mm(mm)
250  *
251  *		Invalidate all TLB entries in a particular address
252  *		space.
253  *		- mm	- mm_struct describing address space
254  *
255  *	flush_tlb_range(mm,start,end)
256  *
257  *		Invalidate a range of TLB entries in the specified
258  *		address space.
259  *		- mm	- mm_struct describing address space
260  *		- start - start address (may not be aligned)
261  *		- end	- end address (exclusive, may not be aligned)
262  *
263  *	flush_tlb_page(vaddr,vma)
264  *
265  *		Invalidate the specified page in the specified address range.
266  *		- vaddr - virtual address (may not be aligned)
267  *		- vma	- vma_struct describing address range
268  *
269  *	flush_kern_tlb_page(kaddr)
270  *
271  *		Invalidate the TLB entry for the specified page.  The address
272  *		will be in the kernels virtual memory space.  Current uses
273  *		only require the D-TLB to be invalidated.
274  *		- kaddr - Kernel virtual memory address
275  */
276 
277 /*
278  * We optimise the code below by:
279  *  - building a set of TLB flags that might be set in __cpu_tlb_flags
280  *  - building a set of TLB flags that will always be set in __cpu_tlb_flags
281  *  - if we're going to need __cpu_tlb_flags, access it once and only once
282  *
283  * This allows us to build optimal assembly for the single-CPU type case,
284  * and as close to optimal given the compiler constrants for multi-CPU
285  * case.  We could do better for the multi-CPU case if the compiler
286  * implemented the "%?" method, but this has been discontinued due to too
287  * many people getting it wrong.
288  */
289 #define possible_tlb_flags	(v4_possible_flags | \
290 				 v4wbi_possible_flags | \
291 				 fr_possible_flags | \
292 				 v4wb_possible_flags | \
293 				 fa_possible_flags | \
294 				 v6wbi_possible_flags | \
295 				 v7wbi_possible_flags)
296 
297 #define always_tlb_flags	(v4_always_flags & \
298 				 v4wbi_always_flags & \
299 				 fr_always_flags & \
300 				 v4wb_always_flags & \
301 				 fa_always_flags & \
302 				 v6wbi_always_flags & \
303 				 v7wbi_always_flags)
304 
305 #define tlb_flag(f)	((always_tlb_flags & (f)) || (__tlb_flag & possible_tlb_flags & (f)))
306 
307 #define __tlb_op(f, insnarg, arg)					\
308 	do {								\
309 		if (always_tlb_flags & (f))				\
310 			asm("mcr " insnarg				\
311 			    : : "r" (arg) : "cc");			\
312 		else if (possible_tlb_flags & (f))			\
313 			asm("tst %1, %2\n\t"				\
314 			    "mcrne " insnarg				\
315 			    : : "r" (arg), "r" (__tlb_flag), "Ir" (f)	\
316 			    : "cc");					\
317 	} while (0)
318 
319 #define tlb_op(f, regs, arg)	__tlb_op(f, "p15, 0, %0, " regs, arg)
320 #define tlb_l2_op(f, regs, arg)	__tlb_op(f, "p15, 1, %0, " regs, arg)
321 
local_flush_tlb_all(void)322 static inline void local_flush_tlb_all(void)
323 {
324 	const int zero = 0;
325 	const unsigned int __tlb_flag = __cpu_tlb_flags;
326 
327 	if (tlb_flag(TLB_WB))
328 		dsb();
329 
330 	tlb_op(TLB_V4_U_FULL | TLB_V6_U_FULL, "c8, c7, 0", zero);
331 	tlb_op(TLB_V4_D_FULL | TLB_V6_D_FULL, "c8, c6, 0", zero);
332 	tlb_op(TLB_V4_I_FULL | TLB_V6_I_FULL, "c8, c5, 0", zero);
333 	tlb_op(TLB_V7_UIS_FULL, "c8, c3, 0", zero);
334 
335 	if (tlb_flag(TLB_BARRIER)) {
336 		dsb();
337 		isb();
338 	}
339 }
340 
local_flush_tlb_mm(struct mm_struct * mm)341 static inline void local_flush_tlb_mm(struct mm_struct *mm)
342 {
343 	const int zero = 0;
344 	const int asid = ASID(mm);
345 	const unsigned int __tlb_flag = __cpu_tlb_flags;
346 
347 	if (tlb_flag(TLB_WB))
348 		dsb();
349 
350 	if (possible_tlb_flags & (TLB_V4_U_FULL|TLB_V4_D_FULL|TLB_V4_I_FULL)) {
351 		if (cpumask_test_cpu(get_cpu(), mm_cpumask(mm))) {
352 			tlb_op(TLB_V4_U_FULL, "c8, c7, 0", zero);
353 			tlb_op(TLB_V4_D_FULL, "c8, c6, 0", zero);
354 			tlb_op(TLB_V4_I_FULL, "c8, c5, 0", zero);
355 		}
356 		put_cpu();
357 	}
358 
359 	tlb_op(TLB_V6_U_ASID, "c8, c7, 2", asid);
360 	tlb_op(TLB_V6_D_ASID, "c8, c6, 2", asid);
361 	tlb_op(TLB_V6_I_ASID, "c8, c5, 2", asid);
362 #ifdef CONFIG_ARM_ERRATA_720789
363 	tlb_op(TLB_V7_UIS_ASID, "c8, c3, 0", zero);
364 #else
365 	tlb_op(TLB_V7_UIS_ASID, "c8, c3, 2", asid);
366 #endif
367 
368 	if (tlb_flag(TLB_BARRIER))
369 		dsb();
370 }
371 
372 static inline void
local_flush_tlb_page(struct vm_area_struct * vma,unsigned long uaddr)373 local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
374 {
375 	const int zero = 0;
376 	const unsigned int __tlb_flag = __cpu_tlb_flags;
377 
378 	uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm);
379 
380 	if (tlb_flag(TLB_WB))
381 		dsb();
382 
383 	if (possible_tlb_flags & (TLB_V4_U_PAGE|TLB_V4_D_PAGE|TLB_V4_I_PAGE|TLB_V4_I_FULL) &&
384 	    cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) {
385 		tlb_op(TLB_V4_U_PAGE, "c8, c7, 1", uaddr);
386 		tlb_op(TLB_V4_D_PAGE, "c8, c6, 1", uaddr);
387 		tlb_op(TLB_V4_I_PAGE, "c8, c5, 1", uaddr);
388 		if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL))
389 			asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc");
390 	}
391 
392 	tlb_op(TLB_V6_U_PAGE, "c8, c7, 1", uaddr);
393 	tlb_op(TLB_V6_D_PAGE, "c8, c6, 1", uaddr);
394 	tlb_op(TLB_V6_I_PAGE, "c8, c5, 1", uaddr);
395 #ifdef CONFIG_ARM_ERRATA_720789
396 	tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 3", uaddr & PAGE_MASK);
397 #else
398 	tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 1", uaddr);
399 #endif
400 
401 	if (tlb_flag(TLB_BARRIER))
402 		dsb();
403 }
404 
local_flush_tlb_kernel_page(unsigned long kaddr)405 static inline void local_flush_tlb_kernel_page(unsigned long kaddr)
406 {
407 	const int zero = 0;
408 	const unsigned int __tlb_flag = __cpu_tlb_flags;
409 
410 	kaddr &= PAGE_MASK;
411 
412 	if (tlb_flag(TLB_WB))
413 		dsb();
414 
415 	tlb_op(TLB_V4_U_PAGE, "c8, c7, 1", kaddr);
416 	tlb_op(TLB_V4_D_PAGE, "c8, c6, 1", kaddr);
417 	tlb_op(TLB_V4_I_PAGE, "c8, c5, 1", kaddr);
418 	if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL))
419 		asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc");
420 
421 	tlb_op(TLB_V6_U_PAGE, "c8, c7, 1", kaddr);
422 	tlb_op(TLB_V6_D_PAGE, "c8, c6, 1", kaddr);
423 	tlb_op(TLB_V6_I_PAGE, "c8, c5, 1", kaddr);
424 	tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 1", kaddr);
425 
426 	if (tlb_flag(TLB_BARRIER)) {
427 		dsb();
428 		isb();
429 	}
430 }
431 
local_flush_bp_all(void)432 static inline void local_flush_bp_all(void)
433 {
434 	const int zero = 0;
435 	const unsigned int __tlb_flag = __cpu_tlb_flags;
436 
437 	if (tlb_flag(TLB_V7_UIS_BP))
438 		asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero));
439 	else if (tlb_flag(TLB_V6_BP))
440 		asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero));
441 
442 	if (tlb_flag(TLB_BARRIER))
443 		isb();
444 }
445 
446 #ifdef CONFIG_ARM_ERRATA_798181
dummy_flush_tlb_a15_erratum(void)447 static inline void dummy_flush_tlb_a15_erratum(void)
448 {
449 	/*
450 	 * Dummy TLBIMVAIS. Using the unmapped address 0 and ASID 0.
451 	 */
452 	asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0));
453 	dsb();
454 }
455 #else
dummy_flush_tlb_a15_erratum(void)456 static inline void dummy_flush_tlb_a15_erratum(void)
457 {
458 }
459 #endif
460 
461 /*
462  *	flush_pmd_entry
463  *
464  *	Flush a PMD entry (word aligned, or double-word aligned) to
465  *	RAM if the TLB for the CPU we are running on requires this.
466  *	This is typically used when we are creating PMD entries.
467  *
468  *	clean_pmd_entry
469  *
470  *	Clean (but don't drain the write buffer) if the CPU requires
471  *	these operations.  This is typically used when we are removing
472  *	PMD entries.
473  */
flush_pmd_entry(void * pmd)474 static inline void flush_pmd_entry(void *pmd)
475 {
476 	const unsigned int __tlb_flag = __cpu_tlb_flags;
477 
478 	tlb_op(TLB_DCLEAN, "c7, c10, 1	@ flush_pmd", pmd);
479 	tlb_l2_op(TLB_L2CLEAN_FR, "c15, c9, 1  @ L2 flush_pmd", pmd);
480 
481 	if (tlb_flag(TLB_WB))
482 		dsb();
483 }
484 
clean_pmd_entry(void * pmd)485 static inline void clean_pmd_entry(void *pmd)
486 {
487 	const unsigned int __tlb_flag = __cpu_tlb_flags;
488 
489 	tlb_op(TLB_DCLEAN, "c7, c10, 1	@ flush_pmd", pmd);
490 	tlb_l2_op(TLB_L2CLEAN_FR, "c15, c9, 1  @ L2 flush_pmd", pmd);
491 }
492 
493 #undef tlb_op
494 #undef tlb_flag
495 #undef always_tlb_flags
496 #undef possible_tlb_flags
497 
498 /*
499  * Convert calls to our calling convention.
500  */
501 #define local_flush_tlb_range(vma,start,end)	__cpu_flush_user_tlb_range(start,end,vma)
502 #define local_flush_tlb_kernel_range(s,e)	__cpu_flush_kern_tlb_range(s,e)
503 
504 #ifndef CONFIG_SMP
505 #define flush_tlb_all		local_flush_tlb_all
506 #define flush_tlb_mm		local_flush_tlb_mm
507 #define flush_tlb_page		local_flush_tlb_page
508 #define flush_tlb_kernel_page	local_flush_tlb_kernel_page
509 #define flush_tlb_range		local_flush_tlb_range
510 #define flush_tlb_kernel_range	local_flush_tlb_kernel_range
511 #define flush_bp_all		local_flush_bp_all
512 #else
513 extern void flush_tlb_all(void);
514 extern void flush_tlb_mm(struct mm_struct *mm);
515 extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr);
516 extern void flush_tlb_kernel_page(unsigned long kaddr);
517 extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
518 extern void flush_tlb_kernel_range(unsigned long start, unsigned long end);
519 extern void flush_bp_all(void);
520 #endif
521 
522 /*
523  * If PG_dcache_clean is not set for the page, we need to ensure that any
524  * cache entries for the kernels virtual memory range are written
525  * back to the page. On ARMv6 and later, the cache coherency is handled via
526  * the set_pte_at() function.
527  */
528 #if __LINUX_ARM_ARCH__ < 6
529 extern void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr,
530 	pte_t *ptep);
531 #else
update_mmu_cache(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep)532 static inline void update_mmu_cache(struct vm_area_struct *vma,
533 				    unsigned long addr, pte_t *ptep)
534 {
535 }
536 #endif
537 
538 #endif
539 
540 #endif /* CONFIG_MMU */
541 
542 #endif
543