1 /*
2 * linux/arch/arm/mm/mmu.c
3 *
4 * Copyright (C) 1995-2005 Russell King
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10 #include <linux/module.h>
11 #include <linux/kernel.h>
12 #include <linux/errno.h>
13 #include <linux/init.h>
14 #include <linux/mman.h>
15 #include <linux/nodemask.h>
16 #include <linux/memblock.h>
17 #include <linux/fs.h>
18 #include <linux/vmalloc.h>
19 #include <linux/sizes.h>
20
21 #include <asm/cp15.h>
22 #include <asm/cputype.h>
23 #include <asm/sections.h>
24 #include <asm/cachetype.h>
25 #include <asm/fixmap.h>
26 #include <asm/sections.h>
27 #include <asm/setup.h>
28 #include <asm/smp_plat.h>
29 #include <asm/tlb.h>
30 #include <asm/highmem.h>
31 #include <asm/system_info.h>
32 #include <asm/traps.h>
33 #include <asm/procinfo.h>
34 #include <asm/memory.h>
35
36 #include <asm/mach/arch.h>
37 #include <asm/mach/map.h>
38 #include <asm/mach/pci.h>
39 #include <asm/fixmap.h>
40
41 #include "mm.h"
42 #include "tcm.h"
43
44 /*
45 * empty_zero_page is a special page that is used for
46 * zero-initialized data and COW.
47 */
48 struct page *empty_zero_page;
49 EXPORT_SYMBOL(empty_zero_page);
50
51 /*
52 * The pmd table for the upper-most set of pages.
53 */
54 pmd_t *top_pmd;
55
56 pmdval_t user_pmd_table = _PAGE_USER_TABLE;
57
58 #define CPOLICY_UNCACHED 0
59 #define CPOLICY_BUFFERED 1
60 #define CPOLICY_WRITETHROUGH 2
61 #define CPOLICY_WRITEBACK 3
62 #define CPOLICY_WRITEALLOC 4
63
64 static unsigned int cachepolicy __initdata = CPOLICY_WRITEBACK;
65 static unsigned int ecc_mask __initdata = 0;
66 pgprot_t pgprot_user;
67 pgprot_t pgprot_kernel;
68 pgprot_t pgprot_hyp_device;
69 pgprot_t pgprot_s2;
70 pgprot_t pgprot_s2_device;
71
72 EXPORT_SYMBOL(pgprot_user);
73 EXPORT_SYMBOL(pgprot_kernel);
74
75 struct cachepolicy {
76 const char policy[16];
77 unsigned int cr_mask;
78 pmdval_t pmd;
79 pteval_t pte;
80 pteval_t pte_s2;
81 };
82
83 #ifdef CONFIG_ARM_LPAE
84 #define s2_policy(policy) policy
85 #else
86 #define s2_policy(policy) 0
87 #endif
88
89 static struct cachepolicy cache_policies[] __initdata = {
90 {
91 .policy = "uncached",
92 .cr_mask = CR_W|CR_C,
93 .pmd = PMD_SECT_UNCACHED,
94 .pte = L_PTE_MT_UNCACHED,
95 .pte_s2 = s2_policy(L_PTE_S2_MT_UNCACHED),
96 }, {
97 .policy = "buffered",
98 .cr_mask = CR_C,
99 .pmd = PMD_SECT_BUFFERED,
100 .pte = L_PTE_MT_BUFFERABLE,
101 .pte_s2 = s2_policy(L_PTE_S2_MT_UNCACHED),
102 }, {
103 .policy = "writethrough",
104 .cr_mask = 0,
105 .pmd = PMD_SECT_WT,
106 .pte = L_PTE_MT_WRITETHROUGH,
107 .pte_s2 = s2_policy(L_PTE_S2_MT_WRITETHROUGH),
108 }, {
109 .policy = "writeback",
110 .cr_mask = 0,
111 .pmd = PMD_SECT_WB,
112 .pte = L_PTE_MT_WRITEBACK,
113 .pte_s2 = s2_policy(L_PTE_S2_MT_WRITEBACK),
114 }, {
115 .policy = "writealloc",
116 .cr_mask = 0,
117 .pmd = PMD_SECT_WBWA,
118 .pte = L_PTE_MT_WRITEALLOC,
119 .pte_s2 = s2_policy(L_PTE_S2_MT_WRITEBACK),
120 }
121 };
122
123 #ifdef CONFIG_CPU_CP15
124 static unsigned long initial_pmd_value __initdata = 0;
125
126 /*
127 * Initialise the cache_policy variable with the initial state specified
128 * via the "pmd" value. This is used to ensure that on ARMv6 and later,
129 * the C code sets the page tables up with the same policy as the head
130 * assembly code, which avoids an illegal state where the TLBs can get
131 * confused. See comments in early_cachepolicy() for more information.
132 */
init_default_cache_policy(unsigned long pmd)133 void __init init_default_cache_policy(unsigned long pmd)
134 {
135 int i;
136
137 initial_pmd_value = pmd;
138
139 pmd &= PMD_SECT_TEX(1) | PMD_SECT_BUFFERABLE | PMD_SECT_CACHEABLE;
140
141 for (i = 0; i < ARRAY_SIZE(cache_policies); i++)
142 if (cache_policies[i].pmd == pmd) {
143 cachepolicy = i;
144 break;
145 }
146
147 if (i == ARRAY_SIZE(cache_policies))
148 pr_err("ERROR: could not find cache policy\n");
149 }
150
151 /*
152 * These are useful for identifying cache coherency problems by allowing
153 * the cache or the cache and writebuffer to be turned off. (Note: the
154 * write buffer should not be on and the cache off).
155 */
early_cachepolicy(char * p)156 static int __init early_cachepolicy(char *p)
157 {
158 int i, selected = -1;
159
160 for (i = 0; i < ARRAY_SIZE(cache_policies); i++) {
161 int len = strlen(cache_policies[i].policy);
162
163 if (memcmp(p, cache_policies[i].policy, len) == 0) {
164 selected = i;
165 break;
166 }
167 }
168
169 if (selected == -1)
170 pr_err("ERROR: unknown or unsupported cache policy\n");
171
172 /*
173 * This restriction is partly to do with the way we boot; it is
174 * unpredictable to have memory mapped using two different sets of
175 * memory attributes (shared, type, and cache attribs). We can not
176 * change these attributes once the initial assembly has setup the
177 * page tables.
178 */
179 if (cpu_architecture() >= CPU_ARCH_ARMv6 && selected != cachepolicy) {
180 pr_warn("Only cachepolicy=%s supported on ARMv6 and later\n",
181 cache_policies[cachepolicy].policy);
182 return 0;
183 }
184
185 if (selected != cachepolicy) {
186 unsigned long cr = __clear_cr(cache_policies[selected].cr_mask);
187 cachepolicy = selected;
188 flush_cache_all();
189 set_cr(cr);
190 }
191 return 0;
192 }
193 early_param("cachepolicy", early_cachepolicy);
194
early_nocache(char * __unused)195 static int __init early_nocache(char *__unused)
196 {
197 char *p = "buffered";
198 printk(KERN_WARNING "nocache is deprecated; use cachepolicy=%s\n", p);
199 early_cachepolicy(p);
200 return 0;
201 }
202 early_param("nocache", early_nocache);
203
early_nowrite(char * __unused)204 static int __init early_nowrite(char *__unused)
205 {
206 char *p = "uncached";
207 printk(KERN_WARNING "nowb is deprecated; use cachepolicy=%s\n", p);
208 early_cachepolicy(p);
209 return 0;
210 }
211 early_param("nowb", early_nowrite);
212
213 #ifndef CONFIG_ARM_LPAE
early_ecc(char * p)214 static int __init early_ecc(char *p)
215 {
216 if (memcmp(p, "on", 2) == 0)
217 ecc_mask = PMD_PROTECTION;
218 else if (memcmp(p, "off", 3) == 0)
219 ecc_mask = 0;
220 return 0;
221 }
222 early_param("ecc", early_ecc);
223 #endif
224
225 #else /* ifdef CONFIG_CPU_CP15 */
226
early_cachepolicy(char * p)227 static int __init early_cachepolicy(char *p)
228 {
229 pr_warn("cachepolicy kernel parameter not supported without cp15\n");
230 }
231 early_param("cachepolicy", early_cachepolicy);
232
noalign_setup(char * __unused)233 static int __init noalign_setup(char *__unused)
234 {
235 pr_warn("noalign kernel parameter not supported without cp15\n");
236 }
237 __setup("noalign", noalign_setup);
238
239 #endif /* ifdef CONFIG_CPU_CP15 / else */
240
241 #define PROT_PTE_DEVICE L_PTE_PRESENT|L_PTE_YOUNG|L_PTE_DIRTY|L_PTE_XN
242 #define PROT_PTE_S2_DEVICE PROT_PTE_DEVICE
243 #define PROT_SECT_DEVICE PMD_TYPE_SECT|PMD_SECT_AP_WRITE
244
245 static struct mem_type mem_types[] = {
246 [MT_DEVICE] = { /* Strongly ordered / ARMv6 shared device */
247 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_SHARED |
248 L_PTE_SHARED,
249 .prot_pte_s2 = s2_policy(PROT_PTE_S2_DEVICE) |
250 s2_policy(L_PTE_S2_MT_DEV_SHARED) |
251 L_PTE_SHARED,
252 .prot_l1 = PMD_TYPE_TABLE,
253 .prot_sect = PROT_SECT_DEVICE | PMD_SECT_S,
254 .domain = DOMAIN_IO,
255 },
256 [MT_DEVICE_NONSHARED] = { /* ARMv6 non-shared device */
257 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_NONSHARED,
258 .prot_l1 = PMD_TYPE_TABLE,
259 .prot_sect = PROT_SECT_DEVICE,
260 .domain = DOMAIN_IO,
261 },
262 [MT_DEVICE_CACHED] = { /* ioremap_cached */
263 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_CACHED,
264 .prot_l1 = PMD_TYPE_TABLE,
265 .prot_sect = PROT_SECT_DEVICE | PMD_SECT_WB,
266 .domain = DOMAIN_IO,
267 },
268 [MT_DEVICE_WC] = { /* ioremap_wc */
269 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_WC,
270 .prot_l1 = PMD_TYPE_TABLE,
271 .prot_sect = PROT_SECT_DEVICE,
272 .domain = DOMAIN_IO,
273 },
274 [MT_UNCACHED] = {
275 .prot_pte = PROT_PTE_DEVICE,
276 .prot_l1 = PMD_TYPE_TABLE,
277 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
278 .domain = DOMAIN_IO,
279 },
280 [MT_CACHECLEAN] = {
281 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
282 .domain = DOMAIN_KERNEL,
283 },
284 #ifndef CONFIG_ARM_LPAE
285 [MT_MINICLEAN] = {
286 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_MINICACHE,
287 .domain = DOMAIN_KERNEL,
288 },
289 #endif
290 [MT_LOW_VECTORS] = {
291 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
292 L_PTE_RDONLY,
293 .prot_l1 = PMD_TYPE_TABLE,
294 .domain = DOMAIN_VECTORS,
295 },
296 [MT_HIGH_VECTORS] = {
297 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
298 L_PTE_USER | L_PTE_RDONLY,
299 .prot_l1 = PMD_TYPE_TABLE,
300 .domain = DOMAIN_VECTORS,
301 },
302 [MT_MEMORY_RWX] = {
303 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
304 .prot_l1 = PMD_TYPE_TABLE,
305 .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
306 .domain = DOMAIN_KERNEL,
307 },
308 [MT_MEMORY_RW] = {
309 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
310 L_PTE_XN,
311 .prot_l1 = PMD_TYPE_TABLE,
312 .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
313 .domain = DOMAIN_KERNEL,
314 },
315 [MT_ROM] = {
316 .prot_sect = PMD_TYPE_SECT,
317 .domain = DOMAIN_KERNEL,
318 },
319 [MT_MEMORY_RWX_NONCACHED] = {
320 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
321 L_PTE_MT_BUFFERABLE,
322 .prot_l1 = PMD_TYPE_TABLE,
323 .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
324 .domain = DOMAIN_KERNEL,
325 },
326 [MT_MEMORY_RW_DTCM] = {
327 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
328 L_PTE_XN,
329 .prot_l1 = PMD_TYPE_TABLE,
330 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
331 .domain = DOMAIN_KERNEL,
332 },
333 [MT_MEMORY_RWX_ITCM] = {
334 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
335 .prot_l1 = PMD_TYPE_TABLE,
336 .domain = DOMAIN_KERNEL,
337 },
338 [MT_MEMORY_RW_SO] = {
339 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
340 L_PTE_MT_UNCACHED | L_PTE_XN,
341 .prot_l1 = PMD_TYPE_TABLE,
342 .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_S |
343 PMD_SECT_UNCACHED | PMD_SECT_XN,
344 .domain = DOMAIN_KERNEL,
345 },
346 [MT_MEMORY_DMA_READY] = {
347 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
348 L_PTE_XN,
349 .prot_l1 = PMD_TYPE_TABLE,
350 .domain = DOMAIN_KERNEL,
351 },
352 };
353
get_mem_type(unsigned int type)354 const struct mem_type *get_mem_type(unsigned int type)
355 {
356 return type < ARRAY_SIZE(mem_types) ? &mem_types[type] : NULL;
357 }
358 EXPORT_SYMBOL(get_mem_type);
359
360 #define PTE_SET_FN(_name, pteop) \
361 static int pte_set_##_name(pte_t *ptep, pgtable_t token, unsigned long addr, \
362 void *data) \
363 { \
364 pte_t pte = pteop(*ptep); \
365 \
366 set_pte_ext(ptep, pte, 0); \
367 return 0; \
368 } \
369
370 #define SET_MEMORY_FN(_name, callback) \
371 int set_memory_##_name(unsigned long addr, int numpages) \
372 { \
373 unsigned long start = addr; \
374 unsigned long size = PAGE_SIZE*numpages; \
375 unsigned end = start + size; \
376 \
377 if (start < MODULES_VADDR || start >= MODULES_END) \
378 return -EINVAL;\
379 \
380 if (end < MODULES_VADDR || end >= MODULES_END) \
381 return -EINVAL; \
382 \
383 apply_to_page_range(&init_mm, start, size, callback, NULL); \
384 flush_tlb_kernel_range(start, end); \
385 return 0;\
386 }
387
PTE_SET_FN(ro,pte_wrprotect)388 PTE_SET_FN(ro, pte_wrprotect)
389 PTE_SET_FN(rw, pte_mkwrite)
390 PTE_SET_FN(x, pte_mkexec)
391 PTE_SET_FN(nx, pte_mknexec)
392
393 SET_MEMORY_FN(ro, pte_set_ro)
394 SET_MEMORY_FN(rw, pte_set_rw)
395 SET_MEMORY_FN(x, pte_set_x)
396 SET_MEMORY_FN(nx, pte_set_nx)
397
398 /*
399 * To avoid TLB flush broadcasts, this uses local_flush_tlb_kernel_range().
400 * As a result, this can only be called with preemption disabled, as under
401 * stop_machine().
402 */
403 void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot)
404 {
405 unsigned long vaddr = __fix_to_virt(idx);
406 pte_t *pte = pte_offset_kernel(pmd_off_k(vaddr), vaddr);
407
408 /* Make sure fixmap region does not exceed available allocation. */
409 BUILD_BUG_ON(FIXADDR_START + (__end_of_fixed_addresses * PAGE_SIZE) >
410 FIXADDR_END);
411 BUG_ON(idx >= __end_of_fixed_addresses);
412
413 if (pgprot_val(prot))
414 set_pte_at(NULL, vaddr, pte,
415 pfn_pte(phys >> PAGE_SHIFT, prot));
416 else
417 pte_clear(NULL, vaddr, pte);
418 local_flush_tlb_kernel_range(vaddr, vaddr + PAGE_SIZE);
419 }
420
421 /*
422 * Adjust the PMD section entries according to the CPU in use.
423 */
build_mem_type_table(void)424 static void __init build_mem_type_table(void)
425 {
426 struct cachepolicy *cp;
427 unsigned int cr = get_cr();
428 pteval_t user_pgprot, kern_pgprot, vecs_pgprot;
429 pteval_t hyp_device_pgprot, s2_pgprot, s2_device_pgprot;
430 int cpu_arch = cpu_architecture();
431 int i;
432
433 if (cpu_arch < CPU_ARCH_ARMv6) {
434 #if defined(CONFIG_CPU_DCACHE_DISABLE)
435 if (cachepolicy > CPOLICY_BUFFERED)
436 cachepolicy = CPOLICY_BUFFERED;
437 #elif defined(CONFIG_CPU_DCACHE_WRITETHROUGH)
438 if (cachepolicy > CPOLICY_WRITETHROUGH)
439 cachepolicy = CPOLICY_WRITETHROUGH;
440 #endif
441 }
442 if (cpu_arch < CPU_ARCH_ARMv5) {
443 if (cachepolicy >= CPOLICY_WRITEALLOC)
444 cachepolicy = CPOLICY_WRITEBACK;
445 ecc_mask = 0;
446 }
447
448 if (is_smp()) {
449 if (cachepolicy != CPOLICY_WRITEALLOC) {
450 pr_warn("Forcing write-allocate cache policy for SMP\n");
451 cachepolicy = CPOLICY_WRITEALLOC;
452 }
453 if (!(initial_pmd_value & PMD_SECT_S)) {
454 pr_warn("Forcing shared mappings for SMP\n");
455 initial_pmd_value |= PMD_SECT_S;
456 }
457 }
458
459 /*
460 * Strip out features not present on earlier architectures.
461 * Pre-ARMv5 CPUs don't have TEX bits. Pre-ARMv6 CPUs or those
462 * without extended page tables don't have the 'Shared' bit.
463 */
464 if (cpu_arch < CPU_ARCH_ARMv5)
465 for (i = 0; i < ARRAY_SIZE(mem_types); i++)
466 mem_types[i].prot_sect &= ~PMD_SECT_TEX(7);
467 if ((cpu_arch < CPU_ARCH_ARMv6 || !(cr & CR_XP)) && !cpu_is_xsc3())
468 for (i = 0; i < ARRAY_SIZE(mem_types); i++)
469 mem_types[i].prot_sect &= ~PMD_SECT_S;
470
471 /*
472 * ARMv5 and lower, bit 4 must be set for page tables (was: cache
473 * "update-able on write" bit on ARM610). However, Xscale and
474 * Xscale3 require this bit to be cleared.
475 */
476 if (cpu_is_xscale() || cpu_is_xsc3()) {
477 for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
478 mem_types[i].prot_sect &= ~PMD_BIT4;
479 mem_types[i].prot_l1 &= ~PMD_BIT4;
480 }
481 } else if (cpu_arch < CPU_ARCH_ARMv6) {
482 for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
483 if (mem_types[i].prot_l1)
484 mem_types[i].prot_l1 |= PMD_BIT4;
485 if (mem_types[i].prot_sect)
486 mem_types[i].prot_sect |= PMD_BIT4;
487 }
488 }
489
490 /*
491 * Mark the device areas according to the CPU/architecture.
492 */
493 if (cpu_is_xsc3() || (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP))) {
494 if (!cpu_is_xsc3()) {
495 /*
496 * Mark device regions on ARMv6+ as execute-never
497 * to prevent speculative instruction fetches.
498 */
499 mem_types[MT_DEVICE].prot_sect |= PMD_SECT_XN;
500 mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_XN;
501 mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_XN;
502 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_XN;
503
504 /* Also setup NX memory mapping */
505 mem_types[MT_MEMORY_RW].prot_sect |= PMD_SECT_XN;
506 }
507 if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) {
508 /*
509 * For ARMv7 with TEX remapping,
510 * - shared device is SXCB=1100
511 * - nonshared device is SXCB=0100
512 * - write combine device mem is SXCB=0001
513 * (Uncached Normal memory)
514 */
515 mem_types[MT_DEVICE].prot_sect |= PMD_SECT_TEX(1);
516 mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(1);
517 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_BUFFERABLE;
518 } else if (cpu_is_xsc3()) {
519 /*
520 * For Xscale3,
521 * - shared device is TEXCB=00101
522 * - nonshared device is TEXCB=01000
523 * - write combine device mem is TEXCB=00100
524 * (Inner/Outer Uncacheable in xsc3 parlance)
525 */
526 mem_types[MT_DEVICE].prot_sect |= PMD_SECT_TEX(1) | PMD_SECT_BUFFERED;
527 mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(2);
528 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_TEX(1);
529 } else {
530 /*
531 * For ARMv6 and ARMv7 without TEX remapping,
532 * - shared device is TEXCB=00001
533 * - nonshared device is TEXCB=01000
534 * - write combine device mem is TEXCB=00100
535 * (Uncached Normal in ARMv6 parlance).
536 */
537 mem_types[MT_DEVICE].prot_sect |= PMD_SECT_BUFFERED;
538 mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(2);
539 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_TEX(1);
540 }
541 } else {
542 /*
543 * On others, write combining is "Uncached/Buffered"
544 */
545 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_BUFFERABLE;
546 }
547
548 /*
549 * Now deal with the memory-type mappings
550 */
551 cp = &cache_policies[cachepolicy];
552 vecs_pgprot = kern_pgprot = user_pgprot = cp->pte;
553 s2_pgprot = cp->pte_s2;
554 hyp_device_pgprot = mem_types[MT_DEVICE].prot_pte;
555 s2_device_pgprot = mem_types[MT_DEVICE].prot_pte_s2;
556
557 /*
558 * We don't use domains on ARMv6 (since this causes problems with
559 * v6/v7 kernels), so we must use a separate memory type for user
560 * r/o, kernel r/w to map the vectors page.
561 */
562 #ifndef CONFIG_ARM_LPAE
563 if (cpu_arch == CPU_ARCH_ARMv6)
564 vecs_pgprot |= L_PTE_MT_VECTORS;
565 #endif
566
567 #ifndef CONFIG_ARM_LPAE
568 /*
569 * We don't use domains on ARMv6 (since this causes problems with
570 * v6/v7 kernels), so we must use a separate memory type for user
571 * r/o, kernel r/w to map the vectors page.
572 */
573 if (cpu_arch == CPU_ARCH_ARMv6)
574 vecs_pgprot |= L_PTE_MT_VECTORS;
575
576 /*
577 * Check is it with support for the PXN bit
578 * in the Short-descriptor translation table format descriptors.
579 */
580 if (cpu_arch == CPU_ARCH_ARMv7 &&
581 (read_cpuid_ext(CPUID_EXT_MMFR0) & 0xF) >= 4) {
582 user_pmd_table |= PMD_PXNTABLE;
583 }
584 #endif
585
586 /*
587 * ARMv6 and above have extended page tables.
588 */
589 if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) {
590 #ifndef CONFIG_ARM_LPAE
591 /*
592 * Mark cache clean areas and XIP ROM read only
593 * from SVC mode and no access from userspace.
594 */
595 mem_types[MT_ROM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
596 mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
597 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
598 #endif
599
600 /*
601 * If the initial page tables were created with the S bit
602 * set, then we need to do the same here for the same
603 * reasons given in early_cachepolicy().
604 */
605 if (initial_pmd_value & PMD_SECT_S) {
606 user_pgprot |= L_PTE_SHARED;
607 kern_pgprot |= L_PTE_SHARED;
608 vecs_pgprot |= L_PTE_SHARED;
609 s2_pgprot |= L_PTE_SHARED;
610 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_S;
611 mem_types[MT_DEVICE_WC].prot_pte |= L_PTE_SHARED;
612 mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S;
613 mem_types[MT_DEVICE_CACHED].prot_pte |= L_PTE_SHARED;
614 mem_types[MT_MEMORY_RWX].prot_sect |= PMD_SECT_S;
615 mem_types[MT_MEMORY_RWX].prot_pte |= L_PTE_SHARED;
616 mem_types[MT_MEMORY_RW].prot_sect |= PMD_SECT_S;
617 mem_types[MT_MEMORY_RW].prot_pte |= L_PTE_SHARED;
618 mem_types[MT_MEMORY_DMA_READY].prot_pte |= L_PTE_SHARED;
619 mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |= PMD_SECT_S;
620 mem_types[MT_MEMORY_RWX_NONCACHED].prot_pte |= L_PTE_SHARED;
621 }
622 }
623
624 /*
625 * Non-cacheable Normal - intended for memory areas that must
626 * not cause dirty cache line writebacks when used
627 */
628 if (cpu_arch >= CPU_ARCH_ARMv6) {
629 if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) {
630 /* Non-cacheable Normal is XCB = 001 */
631 mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |=
632 PMD_SECT_BUFFERED;
633 } else {
634 /* For both ARMv6 and non-TEX-remapping ARMv7 */
635 mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |=
636 PMD_SECT_TEX(1);
637 }
638 } else {
639 mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |= PMD_SECT_BUFFERABLE;
640 }
641
642 #ifdef CONFIG_ARM_LPAE
643 /*
644 * Do not generate access flag faults for the kernel mappings.
645 */
646 for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
647 mem_types[i].prot_pte |= PTE_EXT_AF;
648 if (mem_types[i].prot_sect)
649 mem_types[i].prot_sect |= PMD_SECT_AF;
650 }
651 kern_pgprot |= PTE_EXT_AF;
652 vecs_pgprot |= PTE_EXT_AF;
653
654 /*
655 * Set PXN for user mappings
656 */
657 user_pgprot |= PTE_EXT_PXN;
658 #endif
659
660 for (i = 0; i < 16; i++) {
661 pteval_t v = pgprot_val(protection_map[i]);
662 protection_map[i] = __pgprot(v | user_pgprot);
663 }
664
665 mem_types[MT_LOW_VECTORS].prot_pte |= vecs_pgprot;
666 mem_types[MT_HIGH_VECTORS].prot_pte |= vecs_pgprot;
667
668 pgprot_user = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | user_pgprot);
669 pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG |
670 L_PTE_DIRTY | kern_pgprot);
671 pgprot_s2 = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | s2_pgprot);
672 pgprot_s2_device = __pgprot(s2_device_pgprot);
673 pgprot_hyp_device = __pgprot(hyp_device_pgprot);
674
675 mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask;
676 mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask;
677 mem_types[MT_MEMORY_RWX].prot_sect |= ecc_mask | cp->pmd;
678 mem_types[MT_MEMORY_RWX].prot_pte |= kern_pgprot;
679 mem_types[MT_MEMORY_RW].prot_sect |= ecc_mask | cp->pmd;
680 mem_types[MT_MEMORY_RW].prot_pte |= kern_pgprot;
681 mem_types[MT_MEMORY_DMA_READY].prot_pte |= kern_pgprot;
682 mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |= ecc_mask;
683 mem_types[MT_ROM].prot_sect |= cp->pmd;
684
685 switch (cp->pmd) {
686 case PMD_SECT_WT:
687 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WT;
688 break;
689 case PMD_SECT_WB:
690 case PMD_SECT_WBWA:
691 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WB;
692 break;
693 }
694 pr_info("Memory policy: %sData cache %s\n",
695 ecc_mask ? "ECC enabled, " : "", cp->policy);
696
697 for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
698 struct mem_type *t = &mem_types[i];
699 if (t->prot_l1)
700 t->prot_l1 |= PMD_DOMAIN(t->domain);
701 if (t->prot_sect)
702 t->prot_sect |= PMD_DOMAIN(t->domain);
703 }
704 }
705
706 #ifdef CONFIG_ARM_DMA_MEM_BUFFERABLE
phys_mem_access_prot(struct file * file,unsigned long pfn,unsigned long size,pgprot_t vma_prot)707 pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
708 unsigned long size, pgprot_t vma_prot)
709 {
710 if (!pfn_valid(pfn))
711 return pgprot_noncached(vma_prot);
712 else if (file->f_flags & O_SYNC)
713 return pgprot_writecombine(vma_prot);
714 return vma_prot;
715 }
716 EXPORT_SYMBOL(phys_mem_access_prot);
717 #endif
718
719 #define vectors_base() (vectors_high() ? 0xffff0000 : 0)
720
early_alloc_aligned(unsigned long sz,unsigned long align)721 static void __init *early_alloc_aligned(unsigned long sz, unsigned long align)
722 {
723 void *ptr = __va(memblock_alloc(sz, align));
724 memset(ptr, 0, sz);
725 return ptr;
726 }
727
early_alloc(unsigned long sz)728 static void __init *early_alloc(unsigned long sz)
729 {
730 return early_alloc_aligned(sz, sz);
731 }
732
early_pte_alloc(pmd_t * pmd,unsigned long addr,unsigned long prot)733 static pte_t * __init early_pte_alloc(pmd_t *pmd, unsigned long addr, unsigned long prot)
734 {
735 if (pmd_none(*pmd)) {
736 pte_t *pte = early_alloc(PTE_HWTABLE_OFF + PTE_HWTABLE_SIZE);
737 __pmd_populate(pmd, __pa(pte), prot);
738 }
739 BUG_ON(pmd_bad(*pmd));
740 return pte_offset_kernel(pmd, addr);
741 }
742
alloc_init_pte(pmd_t * pmd,unsigned long addr,unsigned long end,unsigned long pfn,const struct mem_type * type)743 static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
744 unsigned long end, unsigned long pfn,
745 const struct mem_type *type)
746 {
747 pte_t *pte = early_pte_alloc(pmd, addr, type->prot_l1);
748 do {
749 set_pte_ext(pte, pfn_pte(pfn, __pgprot(type->prot_pte)), 0);
750 pfn++;
751 } while (pte++, addr += PAGE_SIZE, addr != end);
752 }
753
__map_init_section(pmd_t * pmd,unsigned long addr,unsigned long end,phys_addr_t phys,const struct mem_type * type)754 static void __init __map_init_section(pmd_t *pmd, unsigned long addr,
755 unsigned long end, phys_addr_t phys,
756 const struct mem_type *type)
757 {
758 pmd_t *p = pmd;
759
760 #ifndef CONFIG_ARM_LPAE
761 /*
762 * In classic MMU format, puds and pmds are folded in to
763 * the pgds. pmd_offset gives the PGD entry. PGDs refer to a
764 * group of L1 entries making up one logical pointer to
765 * an L2 table (2MB), where as PMDs refer to the individual
766 * L1 entries (1MB). Hence increment to get the correct
767 * offset for odd 1MB sections.
768 * (See arch/arm/include/asm/pgtable-2level.h)
769 */
770 if (addr & SECTION_SIZE)
771 pmd++;
772 #endif
773 do {
774 *pmd = __pmd(phys | type->prot_sect);
775 phys += SECTION_SIZE;
776 } while (pmd++, addr += SECTION_SIZE, addr != end);
777
778 flush_pmd_entry(p);
779 }
780
alloc_init_pmd(pud_t * pud,unsigned long addr,unsigned long end,phys_addr_t phys,const struct mem_type * type)781 static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
782 unsigned long end, phys_addr_t phys,
783 const struct mem_type *type)
784 {
785 pmd_t *pmd = pmd_offset(pud, addr);
786 unsigned long next;
787
788 do {
789 /*
790 * With LPAE, we must loop over to map
791 * all the pmds for the given range.
792 */
793 next = pmd_addr_end(addr, end);
794
795 /*
796 * Try a section mapping - addr, next and phys must all be
797 * aligned to a section boundary.
798 */
799 if (type->prot_sect &&
800 ((addr | next | phys) & ~SECTION_MASK) == 0) {
801 __map_init_section(pmd, addr, next, phys, type);
802 } else {
803 alloc_init_pte(pmd, addr, next,
804 __phys_to_pfn(phys), type);
805 }
806
807 phys += next - addr;
808
809 } while (pmd++, addr = next, addr != end);
810 }
811
alloc_init_pud(pgd_t * pgd,unsigned long addr,unsigned long end,phys_addr_t phys,const struct mem_type * type)812 static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
813 unsigned long end, phys_addr_t phys,
814 const struct mem_type *type)
815 {
816 pud_t *pud = pud_offset(pgd, addr);
817 unsigned long next;
818
819 do {
820 next = pud_addr_end(addr, end);
821 alloc_init_pmd(pud, addr, next, phys, type);
822 phys += next - addr;
823 } while (pud++, addr = next, addr != end);
824 }
825
826 #ifndef CONFIG_ARM_LPAE
create_36bit_mapping(struct map_desc * md,const struct mem_type * type)827 static void __init create_36bit_mapping(struct map_desc *md,
828 const struct mem_type *type)
829 {
830 unsigned long addr, length, end;
831 phys_addr_t phys;
832 pgd_t *pgd;
833
834 addr = md->virtual;
835 phys = __pfn_to_phys(md->pfn);
836 length = PAGE_ALIGN(md->length);
837
838 if (!(cpu_architecture() >= CPU_ARCH_ARMv6 || cpu_is_xsc3())) {
839 printk(KERN_ERR "MM: CPU does not support supersection "
840 "mapping for 0x%08llx at 0x%08lx\n",
841 (long long)__pfn_to_phys((u64)md->pfn), addr);
842 return;
843 }
844
845 /* N.B. ARMv6 supersections are only defined to work with domain 0.
846 * Since domain assignments can in fact be arbitrary, the
847 * 'domain == 0' check below is required to insure that ARMv6
848 * supersections are only allocated for domain 0 regardless
849 * of the actual domain assignments in use.
850 */
851 if (type->domain) {
852 printk(KERN_ERR "MM: invalid domain in supersection "
853 "mapping for 0x%08llx at 0x%08lx\n",
854 (long long)__pfn_to_phys((u64)md->pfn), addr);
855 return;
856 }
857
858 if ((addr | length | __pfn_to_phys(md->pfn)) & ~SUPERSECTION_MASK) {
859 printk(KERN_ERR "MM: cannot create mapping for 0x%08llx"
860 " at 0x%08lx invalid alignment\n",
861 (long long)__pfn_to_phys((u64)md->pfn), addr);
862 return;
863 }
864
865 /*
866 * Shift bits [35:32] of address into bits [23:20] of PMD
867 * (See ARMv6 spec).
868 */
869 phys |= (((md->pfn >> (32 - PAGE_SHIFT)) & 0xF) << 20);
870
871 pgd = pgd_offset_k(addr);
872 end = addr + length;
873 do {
874 pud_t *pud = pud_offset(pgd, addr);
875 pmd_t *pmd = pmd_offset(pud, addr);
876 int i;
877
878 for (i = 0; i < 16; i++)
879 *pmd++ = __pmd(phys | type->prot_sect | PMD_SECT_SUPER);
880
881 addr += SUPERSECTION_SIZE;
882 phys += SUPERSECTION_SIZE;
883 pgd += SUPERSECTION_SIZE >> PGDIR_SHIFT;
884 } while (addr != end);
885 }
886 #endif /* !CONFIG_ARM_LPAE */
887
888 /*
889 * Create the page directory entries and any necessary
890 * page tables for the mapping specified by `md'. We
891 * are able to cope here with varying sizes and address
892 * offsets, and we take full advantage of sections and
893 * supersections.
894 */
create_mapping(struct map_desc * md)895 static void __init create_mapping(struct map_desc *md)
896 {
897 unsigned long addr, length, end;
898 phys_addr_t phys;
899 const struct mem_type *type;
900 pgd_t *pgd;
901
902 if (md->virtual != vectors_base() && md->virtual < TASK_SIZE) {
903 printk(KERN_WARNING "BUG: not creating mapping for 0x%08llx"
904 " at 0x%08lx in user region\n",
905 (long long)__pfn_to_phys((u64)md->pfn), md->virtual);
906 return;
907 }
908
909 if ((md->type == MT_DEVICE || md->type == MT_ROM) &&
910 md->virtual >= PAGE_OFFSET &&
911 (md->virtual < VMALLOC_START || md->virtual >= VMALLOC_END)) {
912 printk(KERN_WARNING "BUG: mapping for 0x%08llx"
913 " at 0x%08lx out of vmalloc space\n",
914 (long long)__pfn_to_phys((u64)md->pfn), md->virtual);
915 }
916
917 type = &mem_types[md->type];
918
919 #ifndef CONFIG_ARM_LPAE
920 /*
921 * Catch 36-bit addresses
922 */
923 if (md->pfn >= 0x100000) {
924 create_36bit_mapping(md, type);
925 return;
926 }
927 #endif
928
929 addr = md->virtual & PAGE_MASK;
930 phys = __pfn_to_phys(md->pfn);
931 length = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK));
932
933 if (type->prot_l1 == 0 && ((addr | phys | length) & ~SECTION_MASK)) {
934 printk(KERN_WARNING "BUG: map for 0x%08llx at 0x%08lx can not "
935 "be mapped using pages, ignoring.\n",
936 (long long)__pfn_to_phys(md->pfn), addr);
937 return;
938 }
939
940 pgd = pgd_offset_k(addr);
941 end = addr + length;
942 do {
943 unsigned long next = pgd_addr_end(addr, end);
944
945 alloc_init_pud(pgd, addr, next, phys, type);
946
947 phys += next - addr;
948 addr = next;
949 } while (pgd++, addr != end);
950 }
951
952 /*
953 * Create the architecture specific mappings
954 */
iotable_init(struct map_desc * io_desc,int nr)955 void __init iotable_init(struct map_desc *io_desc, int nr)
956 {
957 struct map_desc *md;
958 struct vm_struct *vm;
959 struct static_vm *svm;
960
961 if (!nr)
962 return;
963
964 svm = early_alloc_aligned(sizeof(*svm) * nr, __alignof__(*svm));
965
966 for (md = io_desc; nr; md++, nr--) {
967 create_mapping(md);
968
969 vm = &svm->vm;
970 vm->addr = (void *)(md->virtual & PAGE_MASK);
971 vm->size = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK));
972 vm->phys_addr = __pfn_to_phys(md->pfn);
973 vm->flags = VM_IOREMAP | VM_ARM_STATIC_MAPPING;
974 vm->flags |= VM_ARM_MTYPE(md->type);
975 vm->caller = iotable_init;
976 add_static_vm_early(svm++);
977 }
978 }
979
vm_reserve_area_early(unsigned long addr,unsigned long size,void * caller)980 void __init vm_reserve_area_early(unsigned long addr, unsigned long size,
981 void *caller)
982 {
983 struct vm_struct *vm;
984 struct static_vm *svm;
985
986 svm = early_alloc_aligned(sizeof(*svm), __alignof__(*svm));
987
988 vm = &svm->vm;
989 vm->addr = (void *)addr;
990 vm->size = size;
991 vm->flags = VM_IOREMAP | VM_ARM_EMPTY_MAPPING;
992 vm->caller = caller;
993 add_static_vm_early(svm);
994 }
995
996 #ifndef CONFIG_ARM_LPAE
997
998 /*
999 * The Linux PMD is made of two consecutive section entries covering 2MB
1000 * (see definition in include/asm/pgtable-2level.h). However a call to
1001 * create_mapping() may optimize static mappings by using individual
1002 * 1MB section mappings. This leaves the actual PMD potentially half
1003 * initialized if the top or bottom section entry isn't used, leaving it
1004 * open to problems if a subsequent ioremap() or vmalloc() tries to use
1005 * the virtual space left free by that unused section entry.
1006 *
1007 * Let's avoid the issue by inserting dummy vm entries covering the unused
1008 * PMD halves once the static mappings are in place.
1009 */
1010
pmd_empty_section_gap(unsigned long addr)1011 static void __init pmd_empty_section_gap(unsigned long addr)
1012 {
1013 vm_reserve_area_early(addr, SECTION_SIZE, pmd_empty_section_gap);
1014 }
1015
fill_pmd_gaps(void)1016 static void __init fill_pmd_gaps(void)
1017 {
1018 struct static_vm *svm;
1019 struct vm_struct *vm;
1020 unsigned long addr, next = 0;
1021 pmd_t *pmd;
1022
1023 list_for_each_entry(svm, &static_vmlist, list) {
1024 vm = &svm->vm;
1025 addr = (unsigned long)vm->addr;
1026 if (addr < next)
1027 continue;
1028
1029 /*
1030 * Check if this vm starts on an odd section boundary.
1031 * If so and the first section entry for this PMD is free
1032 * then we block the corresponding virtual address.
1033 */
1034 if ((addr & ~PMD_MASK) == SECTION_SIZE) {
1035 pmd = pmd_off_k(addr);
1036 if (pmd_none(*pmd))
1037 pmd_empty_section_gap(addr & PMD_MASK);
1038 }
1039
1040 /*
1041 * Then check if this vm ends on an odd section boundary.
1042 * If so and the second section entry for this PMD is empty
1043 * then we block the corresponding virtual address.
1044 */
1045 addr += vm->size;
1046 if ((addr & ~PMD_MASK) == SECTION_SIZE) {
1047 pmd = pmd_off_k(addr) + 1;
1048 if (pmd_none(*pmd))
1049 pmd_empty_section_gap(addr);
1050 }
1051
1052 /* no need to look at any vm entry until we hit the next PMD */
1053 next = (addr + PMD_SIZE - 1) & PMD_MASK;
1054 }
1055 }
1056
1057 #else
1058 #define fill_pmd_gaps() do { } while (0)
1059 #endif
1060
1061 #if defined(CONFIG_PCI) && !defined(CONFIG_NEED_MACH_IO_H)
pci_reserve_io(void)1062 static void __init pci_reserve_io(void)
1063 {
1064 struct static_vm *svm;
1065
1066 svm = find_static_vm_vaddr((void *)PCI_IO_VIRT_BASE);
1067 if (svm)
1068 return;
1069
1070 vm_reserve_area_early(PCI_IO_VIRT_BASE, SZ_2M, pci_reserve_io);
1071 }
1072 #else
1073 #define pci_reserve_io() do { } while (0)
1074 #endif
1075
1076 #ifdef CONFIG_DEBUG_LL
debug_ll_io_init(void)1077 void __init debug_ll_io_init(void)
1078 {
1079 struct map_desc map;
1080
1081 debug_ll_addr(&map.pfn, &map.virtual);
1082 if (!map.pfn || !map.virtual)
1083 return;
1084 map.pfn = __phys_to_pfn(map.pfn);
1085 map.virtual &= PAGE_MASK;
1086 map.length = PAGE_SIZE;
1087 map.type = MT_DEVICE;
1088 iotable_init(&map, 1);
1089 }
1090 #endif
1091
1092 static void * __initdata vmalloc_min =
1093 (void *)(VMALLOC_END - (240 << 20) - VMALLOC_OFFSET);
1094
1095 /*
1096 * vmalloc=size forces the vmalloc area to be exactly 'size'
1097 * bytes. This can be used to increase (or decrease) the vmalloc
1098 * area - the default is 240m.
1099 */
early_vmalloc(char * arg)1100 static int __init early_vmalloc(char *arg)
1101 {
1102 unsigned long vmalloc_reserve = memparse(arg, NULL);
1103
1104 if (vmalloc_reserve < SZ_16M) {
1105 vmalloc_reserve = SZ_16M;
1106 printk(KERN_WARNING
1107 "vmalloc area too small, limiting to %luMB\n",
1108 vmalloc_reserve >> 20);
1109 }
1110
1111 if (vmalloc_reserve > VMALLOC_END - (PAGE_OFFSET + SZ_32M)) {
1112 vmalloc_reserve = VMALLOC_END - (PAGE_OFFSET + SZ_32M);
1113 printk(KERN_WARNING
1114 "vmalloc area is too big, limiting to %luMB\n",
1115 vmalloc_reserve >> 20);
1116 }
1117
1118 vmalloc_min = (void *)(VMALLOC_END - vmalloc_reserve);
1119 return 0;
1120 }
1121 early_param("vmalloc", early_vmalloc);
1122
1123 phys_addr_t arm_lowmem_limit __initdata = 0;
1124
sanity_check_meminfo(void)1125 void __init sanity_check_meminfo(void)
1126 {
1127 phys_addr_t memblock_limit = 0;
1128 int highmem = 0;
1129 phys_addr_t vmalloc_limit = __pa(vmalloc_min - 1) + 1;
1130 struct memblock_region *reg;
1131
1132 for_each_memblock(memory, reg) {
1133 phys_addr_t block_start = reg->base;
1134 phys_addr_t block_end = reg->base + reg->size;
1135 phys_addr_t size_limit = reg->size;
1136
1137 if (reg->base >= vmalloc_limit)
1138 highmem = 1;
1139 else
1140 size_limit = vmalloc_limit - reg->base;
1141
1142
1143 if (!IS_ENABLED(CONFIG_HIGHMEM) || cache_is_vipt_aliasing()) {
1144
1145 if (highmem) {
1146 pr_notice("Ignoring RAM at %pa-%pa (!CONFIG_HIGHMEM)\n",
1147 &block_start, &block_end);
1148 memblock_remove(reg->base, reg->size);
1149 continue;
1150 }
1151
1152 if (reg->size > size_limit) {
1153 phys_addr_t overlap_size = reg->size - size_limit;
1154
1155 pr_notice("Truncating RAM at %pa-%pa to -%pa",
1156 &block_start, &block_end, &vmalloc_limit);
1157 memblock_remove(vmalloc_limit, overlap_size);
1158 block_end = vmalloc_limit;
1159 }
1160 }
1161
1162 if (!highmem) {
1163 if (block_end > arm_lowmem_limit) {
1164 if (reg->size > size_limit)
1165 arm_lowmem_limit = vmalloc_limit;
1166 else
1167 arm_lowmem_limit = block_end;
1168 }
1169
1170 /*
1171 * Find the first non-pmd-aligned page, and point
1172 * memblock_limit at it. This relies on rounding the
1173 * limit down to be pmd-aligned, which happens at the
1174 * end of this function.
1175 *
1176 * With this algorithm, the start or end of almost any
1177 * bank can be non-pmd-aligned. The only exception is
1178 * that the start of the bank 0 must be section-
1179 * aligned, since otherwise memory would need to be
1180 * allocated when mapping the start of bank 0, which
1181 * occurs before any free memory is mapped.
1182 */
1183 if (!memblock_limit) {
1184 if (!IS_ALIGNED(block_start, PMD_SIZE))
1185 memblock_limit = block_start;
1186 else if (!IS_ALIGNED(block_end, PMD_SIZE))
1187 memblock_limit = arm_lowmem_limit;
1188 }
1189
1190 }
1191 }
1192
1193 high_memory = __va(arm_lowmem_limit - 1) + 1;
1194
1195 if (!memblock_limit)
1196 memblock_limit = arm_lowmem_limit;
1197
1198 /*
1199 * Round the memblock limit down to a pmd size. This
1200 * helps to ensure that we will allocate memory from the
1201 * last full pmd, which should be mapped.
1202 */
1203 memblock_limit = round_down(memblock_limit, PMD_SIZE);
1204
1205 memblock_set_current_limit(memblock_limit);
1206 }
1207
prepare_page_table(void)1208 static inline void prepare_page_table(void)
1209 {
1210 unsigned long addr;
1211 phys_addr_t end;
1212
1213 /*
1214 * Clear out all the mappings below the kernel image.
1215 */
1216 for (addr = 0; addr < MODULES_VADDR; addr += PMD_SIZE)
1217 pmd_clear(pmd_off_k(addr));
1218
1219 #ifdef CONFIG_XIP_KERNEL
1220 /* The XIP kernel is mapped in the module area -- skip over it */
1221 addr = ((unsigned long)_etext + PMD_SIZE - 1) & PMD_MASK;
1222 #endif
1223 for ( ; addr < PAGE_OFFSET; addr += PMD_SIZE)
1224 pmd_clear(pmd_off_k(addr));
1225
1226 /*
1227 * Find the end of the first block of lowmem.
1228 */
1229 end = memblock.memory.regions[0].base + memblock.memory.regions[0].size;
1230 if (end >= arm_lowmem_limit)
1231 end = arm_lowmem_limit;
1232
1233 /*
1234 * Clear out all the kernel space mappings, except for the first
1235 * memory bank, up to the vmalloc region.
1236 */
1237 for (addr = __phys_to_virt(end);
1238 addr < VMALLOC_START; addr += PMD_SIZE)
1239 pmd_clear(pmd_off_k(addr));
1240 }
1241
1242 #ifdef CONFIG_ARM_LPAE
1243 /* the first page is reserved for pgd */
1244 #define SWAPPER_PG_DIR_SIZE (PAGE_SIZE + \
1245 PTRS_PER_PGD * PTRS_PER_PMD * sizeof(pmd_t))
1246 #else
1247 #define SWAPPER_PG_DIR_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
1248 #endif
1249
1250 /*
1251 * Reserve the special regions of memory
1252 */
arm_mm_memblock_reserve(void)1253 void __init arm_mm_memblock_reserve(void)
1254 {
1255 /*
1256 * Reserve the page tables. These are already in use,
1257 * and can only be in node 0.
1258 */
1259 memblock_reserve(__pa(swapper_pg_dir), SWAPPER_PG_DIR_SIZE);
1260
1261 #ifdef CONFIG_SA1111
1262 /*
1263 * Because of the SA1111 DMA bug, we want to preserve our
1264 * precious DMA-able memory...
1265 */
1266 memblock_reserve(PHYS_OFFSET, __pa(swapper_pg_dir) - PHYS_OFFSET);
1267 #endif
1268 }
1269
1270 /*
1271 * Set up the device mappings. Since we clear out the page tables for all
1272 * mappings above VMALLOC_START, we will remove any debug device mappings.
1273 * This means you have to be careful how you debug this function, or any
1274 * called function. This means you can't use any function or debugging
1275 * method which may touch any device, otherwise the kernel _will_ crash.
1276 */
devicemaps_init(const struct machine_desc * mdesc)1277 static void __init devicemaps_init(const struct machine_desc *mdesc)
1278 {
1279 struct map_desc map;
1280 unsigned long addr;
1281 void *vectors;
1282
1283 /*
1284 * Allocate the vector page early.
1285 */
1286 vectors = early_alloc(PAGE_SIZE * 2);
1287
1288 early_trap_init(vectors);
1289
1290 for (addr = VMALLOC_START; addr; addr += PMD_SIZE)
1291 pmd_clear(pmd_off_k(addr));
1292
1293 /*
1294 * Map the kernel if it is XIP.
1295 * It is always first in the modulearea.
1296 */
1297 #ifdef CONFIG_XIP_KERNEL
1298 map.pfn = __phys_to_pfn(CONFIG_XIP_PHYS_ADDR & SECTION_MASK);
1299 map.virtual = MODULES_VADDR;
1300 map.length = ((unsigned long)_etext - map.virtual + ~SECTION_MASK) & SECTION_MASK;
1301 map.type = MT_ROM;
1302 create_mapping(&map);
1303 #endif
1304
1305 /*
1306 * Map the cache flushing regions.
1307 */
1308 #ifdef FLUSH_BASE
1309 map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS);
1310 map.virtual = FLUSH_BASE;
1311 map.length = SZ_1M;
1312 map.type = MT_CACHECLEAN;
1313 create_mapping(&map);
1314 #endif
1315 #ifdef FLUSH_BASE_MINICACHE
1316 map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS + SZ_1M);
1317 map.virtual = FLUSH_BASE_MINICACHE;
1318 map.length = SZ_1M;
1319 map.type = MT_MINICLEAN;
1320 create_mapping(&map);
1321 #endif
1322
1323 /*
1324 * Create a mapping for the machine vectors at the high-vectors
1325 * location (0xffff0000). If we aren't using high-vectors, also
1326 * create a mapping at the low-vectors virtual address.
1327 */
1328 map.pfn = __phys_to_pfn(virt_to_phys(vectors));
1329 map.virtual = 0xffff0000;
1330 map.length = PAGE_SIZE;
1331 #ifdef CONFIG_KUSER_HELPERS
1332 map.type = MT_HIGH_VECTORS;
1333 #else
1334 map.type = MT_LOW_VECTORS;
1335 #endif
1336 create_mapping(&map);
1337
1338 if (!vectors_high()) {
1339 map.virtual = 0;
1340 map.length = PAGE_SIZE * 2;
1341 map.type = MT_LOW_VECTORS;
1342 create_mapping(&map);
1343 }
1344
1345 /* Now create a kernel read-only mapping */
1346 map.pfn += 1;
1347 map.virtual = 0xffff0000 + PAGE_SIZE;
1348 map.length = PAGE_SIZE;
1349 map.type = MT_LOW_VECTORS;
1350 create_mapping(&map);
1351
1352 /*
1353 * Ask the machine support to map in the statically mapped devices.
1354 */
1355 if (mdesc->map_io)
1356 mdesc->map_io();
1357 else
1358 debug_ll_io_init();
1359 fill_pmd_gaps();
1360
1361 /* Reserve fixed i/o space in VMALLOC region */
1362 pci_reserve_io();
1363
1364 /*
1365 * Finally flush the caches and tlb to ensure that we're in a
1366 * consistent state wrt the writebuffer. This also ensures that
1367 * any write-allocated cache lines in the vector page are written
1368 * back. After this point, we can start to touch devices again.
1369 */
1370 local_flush_tlb_all();
1371 flush_cache_all();
1372 }
1373
kmap_init(void)1374 static void __init kmap_init(void)
1375 {
1376 #ifdef CONFIG_HIGHMEM
1377 pkmap_page_table = early_pte_alloc(pmd_off_k(PKMAP_BASE),
1378 PKMAP_BASE, _PAGE_KERNEL_TABLE);
1379 #endif
1380
1381 early_pte_alloc(pmd_off_k(FIXADDR_START), FIXADDR_START,
1382 _PAGE_KERNEL_TABLE);
1383 }
1384
map_lowmem(void)1385 static void __init map_lowmem(void)
1386 {
1387 struct memblock_region *reg;
1388 unsigned long kernel_x_start = round_down(__pa(_stext), SECTION_SIZE);
1389 unsigned long kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE);
1390
1391 /* Map all the lowmem memory banks. */
1392 for_each_memblock(memory, reg) {
1393 phys_addr_t start = reg->base;
1394 phys_addr_t end = start + reg->size;
1395 struct map_desc map;
1396
1397 if (end > arm_lowmem_limit)
1398 end = arm_lowmem_limit;
1399 if (start >= end)
1400 break;
1401
1402 if (end < kernel_x_start) {
1403 map.pfn = __phys_to_pfn(start);
1404 map.virtual = __phys_to_virt(start);
1405 map.length = end - start;
1406 map.type = MT_MEMORY_RWX;
1407
1408 create_mapping(&map);
1409 } else if (start >= kernel_x_end) {
1410 map.pfn = __phys_to_pfn(start);
1411 map.virtual = __phys_to_virt(start);
1412 map.length = end - start;
1413 map.type = MT_MEMORY_RW;
1414
1415 create_mapping(&map);
1416 } else {
1417 /* This better cover the entire kernel */
1418 if (start < kernel_x_start) {
1419 map.pfn = __phys_to_pfn(start);
1420 map.virtual = __phys_to_virt(start);
1421 map.length = kernel_x_start - start;
1422 map.type = MT_MEMORY_RW;
1423
1424 create_mapping(&map);
1425 }
1426
1427 map.pfn = __phys_to_pfn(kernel_x_start);
1428 map.virtual = __phys_to_virt(kernel_x_start);
1429 map.length = kernel_x_end - kernel_x_start;
1430 map.type = MT_MEMORY_RWX;
1431
1432 create_mapping(&map);
1433
1434 if (kernel_x_end < end) {
1435 map.pfn = __phys_to_pfn(kernel_x_end);
1436 map.virtual = __phys_to_virt(kernel_x_end);
1437 map.length = end - kernel_x_end;
1438 map.type = MT_MEMORY_RW;
1439
1440 create_mapping(&map);
1441 }
1442 }
1443 }
1444 }
1445
1446 #ifdef CONFIG_ARM_PV_FIXUP
1447 extern unsigned long __atags_pointer;
1448 typedef void pgtables_remap(long long offset, unsigned long pgd, void *bdata);
1449 pgtables_remap lpae_pgtables_remap_asm;
1450
1451 /*
1452 * early_paging_init() recreates boot time page table setup, allowing machines
1453 * to switch over to a high (>4G) address space on LPAE systems
1454 */
early_paging_init(const struct machine_desc * mdesc,struct proc_info_list * procinfo)1455 void __init early_paging_init(const struct machine_desc *mdesc,
1456 struct proc_info_list *procinfo)
1457 {
1458 pgtables_remap *lpae_pgtables_remap;
1459 unsigned long pa_pgd;
1460 unsigned int cr, ttbcr;
1461 long long offset;
1462 void *boot_data;
1463
1464 if (!(mdesc->init_meminfo))
1465 return;
1466
1467 offset = mdesc->init_meminfo();
1468 if (offset == 0)
1469 return;
1470
1471 /* Re-set the phys pfn offset, and the pv offset */
1472 __pv_offset += offset;
1473 __pv_phys_pfn_offset += PFN_DOWN(offset);
1474
1475 /*
1476 * Get the address of the remap function in the 1:1 identity
1477 * mapping setup by the early page table assembly code. We
1478 * must get this prior to the pv update. The following barrier
1479 * ensures that this is complete before we fixup any P:V offsets.
1480 */
1481 lpae_pgtables_remap = (pgtables_remap *)(unsigned long)__pa(lpae_pgtables_remap_asm);
1482 pa_pgd = __pa(swapper_pg_dir);
1483 boot_data = __va(__atags_pointer);
1484 barrier();
1485
1486 /* Run the patch stub to update the constants */
1487 fixup_pv_table(&__pv_table_begin,
1488 (&__pv_table_end - &__pv_table_begin) << 2);
1489
1490 /*
1491 * We changing not only the virtual to physical mapping, but also
1492 * the physical addresses used to access memory. We need to flush
1493 * all levels of cache in the system with caching disabled to
1494 * ensure that all data is written back, and nothing is prefetched
1495 * into the caches. We also need to prevent the TLB walkers
1496 * allocating into the caches too. Note that this is ARMv7 LPAE
1497 * specific.
1498 */
1499 cr = get_cr();
1500 set_cr(cr & ~(CR_I | CR_C));
1501 asm("mrc p15, 0, %0, c2, c0, 2" : "=r" (ttbcr));
1502 asm volatile("mcr p15, 0, %0, c2, c0, 2"
1503 : : "r" (ttbcr & ~(3 << 8 | 3 << 10)));
1504 flush_cache_all();
1505
1506 /*
1507 * Fixup the page tables - this must be in the idmap region as
1508 * we need to disable the MMU to do this safely, and hence it
1509 * needs to be assembly. It's fairly simple, as we're using the
1510 * temporary tables setup by the initial assembly code.
1511 */
1512 lpae_pgtables_remap(offset, pa_pgd, boot_data);
1513
1514 /* Re-enable the caches and cacheable TLB walks */
1515 asm volatile("mcr p15, 0, %0, c2, c0, 2" : : "r" (ttbcr));
1516 set_cr(cr);
1517 }
1518
1519 #else
1520
early_paging_init(const struct machine_desc * mdesc,struct proc_info_list * procinfo)1521 void __init early_paging_init(const struct machine_desc *mdesc,
1522 struct proc_info_list *procinfo)
1523 {
1524 long long offset;
1525
1526 if (!mdesc->init_meminfo)
1527 return;
1528
1529 offset = mdesc->init_meminfo();
1530 if (offset == 0)
1531 return;
1532
1533 pr_crit("Physical address space modification is only to support Keystone2.\n");
1534 pr_crit("Please enable ARM_LPAE and ARM_PATCH_PHYS_VIRT support to use this\n");
1535 pr_crit("feature. Your kernel may crash now, have a good day.\n");
1536 add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
1537 }
1538
1539 #endif
1540
1541 /*
1542 * paging_init() sets up the page tables, initialises the zone memory
1543 * maps, and sets up the zero page, bad page and bad page tables.
1544 */
paging_init(const struct machine_desc * mdesc)1545 void __init paging_init(const struct machine_desc *mdesc)
1546 {
1547 void *zero_page;
1548
1549 build_mem_type_table();
1550 prepare_page_table();
1551 map_lowmem();
1552 dma_contiguous_remap();
1553 devicemaps_init(mdesc);
1554 kmap_init();
1555 tcm_init();
1556
1557 top_pmd = pmd_off_k(0xffff0000);
1558
1559 /* allocate the zero page. */
1560 zero_page = early_alloc(PAGE_SIZE);
1561
1562 bootmem_init();
1563
1564 empty_zero_page = virt_to_page(zero_page);
1565 __flush_dcache_page(NULL, empty_zero_page);
1566 }
1567