1 /*
2 * Copyright (c) 2016-2017, Linaro Limited. All rights reserved.
3 * Copyright (c) 2014-2019, Arm Limited. All rights reserved.
4 * Copyright (c) 2014, STMicroelectronics International N.V.
5 * All rights reserved.
6 *
7 * SPDX-License-Identifier: BSD-3-Clause
8 */
9
10 #include <assert.h>
11 #include <stdio.h>
12 #include <string.h>
13
14 #include <platform_def.h>
15
16 #include <arch.h>
17 #include <arch_helpers.h>
18 #include <common/debug.h>
19 #include <lib/cassert.h>
20 #include <lib/utils.h>
21 #include <lib/xlat_tables/xlat_tables.h>
22
23 #include "../xlat_tables_private.h"
24
25 #ifdef ARMV7_SUPPORTS_LARGE_PAGE_ADDRESSING
26 #error "ARMV7_SUPPORTS_LARGE_PAGE_ADDRESSING flag is set. \
27 This module is to be used when LPAE is not supported"
28 #endif
29
30 CASSERT(PLAT_VIRT_ADDR_SPACE_SIZE == (1ULL << 32), invalid_vaddr_space_size);
31 CASSERT(PLAT_PHY_ADDR_SPACE_SIZE == (1ULL << 32), invalid_paddr_space_size);
32
33 #define MMU32B_UNSET_DESC ~0ul
34 #define MMU32B_INVALID_DESC 0ul
35
36 #define MT_UNKNOWN ~0U
37
38 /*
39 * MMU related values
40 */
41
42 /* Sharable */
43 #define MMU32B_TTB_S (1 << 1)
44
45 /* Not Outer Sharable */
46 #define MMU32B_TTB_NOS (1 << 5)
47
48 /* Normal memory, Inner Non-cacheable */
49 #define MMU32B_TTB_IRGN_NC 0
50
51 /* Normal memory, Inner Write-Back Write-Allocate Cacheable */
52 #define MMU32B_TTB_IRGN_WBWA (1 << 6)
53
54 /* Normal memory, Inner Write-Through Cacheable */
55 #define MMU32B_TTB_IRGN_WT 1
56
57 /* Normal memory, Inner Write-Back no Write-Allocate Cacheable */
58 #define MMU32B_TTB_IRGN_WB (1 | (1 << 6))
59
60 /* Normal memory, Outer Write-Back Write-Allocate Cacheable */
61 #define MMU32B_TTB_RNG_WBWA (1 << 3)
62
63 #define MMU32B_DEFAULT_ATTRS \
64 (MMU32B_TTB_S | MMU32B_TTB_NOS | \
65 MMU32B_TTB_IRGN_WBWA | MMU32B_TTB_RNG_WBWA)
66
67 /* armv7 memory mapping attributes: section mapping */
68 #define SECTION_SECURE (0 << 19)
69 #define SECTION_NOTSECURE (1 << 19)
70 #define SECTION_SHARED (1 << 16)
71 #define SECTION_NOTGLOBAL (1 << 17)
72 #define SECTION_ACCESS_FLAG (1 << 10)
73 #define SECTION_UNPRIV (1 << 11)
74 #define SECTION_RO (1 << 15)
75 #define SECTION_TEX(tex) ((((tex) >> 2) << 12) | \
76 ((((tex) >> 1) & 0x1) << 3) | \
77 (((tex) & 0x1) << 2))
78 #define SECTION_DEVICE SECTION_TEX(MMU32B_ATTR_DEVICE_INDEX)
79 #define SECTION_NORMAL SECTION_TEX(MMU32B_ATTR_DEVICE_INDEX)
80 #define SECTION_NORMAL_CACHED \
81 SECTION_TEX(MMU32B_ATTR_IWBWA_OWBWA_INDEX)
82
83 #define SECTION_XN (1 << 4)
84 #define SECTION_PXN (1 << 0)
85 #define SECTION_SECTION (2 << 0)
86
87 #define SECTION_PT_NOTSECURE (1 << 3)
88 #define SECTION_PT_PT (1 << 0)
89
90 #define SMALL_PAGE_SMALL_PAGE (1 << 1)
91 #define SMALL_PAGE_SHARED (1 << 10)
92 #define SMALL_PAGE_NOTGLOBAL (1 << 11)
93 #define SMALL_PAGE_TEX(tex) ((((tex) >> 2) << 6) | \
94 ((((tex) >> 1) & 0x1) << 3) | \
95 (((tex) & 0x1) << 2))
96 #define SMALL_PAGE_DEVICE \
97 SMALL_PAGE_TEX(MMU32B_ATTR_DEVICE_INDEX)
98 #define SMALL_PAGE_NORMAL \
99 SMALL_PAGE_TEX(MMU32B_ATTR_DEVICE_INDEX)
100 #define SMALL_PAGE_NORMAL_CACHED \
101 SMALL_PAGE_TEX(MMU32B_ATTR_IWBWA_OWBWA_INDEX)
102 #define SMALL_PAGE_ACCESS_FLAG (1 << 4)
103 #define SMALL_PAGE_UNPRIV (1 << 5)
104 #define SMALL_PAGE_RO (1 << 9)
105 #define SMALL_PAGE_XN (1 << 0)
106
107 /* The TEX, C and B bits concatenated */
108 #define MMU32B_ATTR_DEVICE_INDEX 0x0
109 #define MMU32B_ATTR_IWBWA_OWBWA_INDEX 0x1
110
111 #define MMU32B_PRRR_IDX(idx, tr, nos) (((tr) << (2 * (idx))) | \
112 ((uint32_t)(nos) << ((idx) + 24)))
113 #define MMU32B_NMRR_IDX(idx, ir, or) (((ir) << (2 * (idx))) | \
114 ((uint32_t)(or) << (2 * (idx) + 16)))
115 #define MMU32B_PRRR_DS0 (1 << 16)
116 #define MMU32B_PRRR_DS1 (1 << 17)
117 #define MMU32B_PRRR_NS0 (1 << 18)
118 #define MMU32B_PRRR_NS1 (1 << 19)
119
120 #define DACR_DOMAIN(num, perm) ((perm) << ((num) * 2))
121 #define DACR_DOMAIN_PERM_NO_ACCESS 0x0
122 #define DACR_DOMAIN_PERM_CLIENT 0x1
123 #define DACR_DOMAIN_PERM_MANAGER 0x3
124
125 #define NUM_1MB_IN_4GB (1U << 12)
126 #define NUM_4K_IN_1MB (1U << 8)
127
128 #define ONE_MB_SHIFT 20
129
130 /* mmu 32b integration */
131 #define MMU32B_L1_TABLE_SIZE (NUM_1MB_IN_4GB * 4)
132 #define MMU32B_L2_TABLE_SIZE (NUM_4K_IN_1MB * 4)
133 #define MMU32B_L1_TABLE_ALIGN (1 << 14)
134 #define MMU32B_L2_TABLE_ALIGN (1 << 10)
135
136 static unsigned int next_xlat;
137 static unsigned long long xlat_max_pa;
138 static uintptr_t xlat_max_va;
139
140 static uint32_t mmu_l1_base[NUM_1MB_IN_4GB]
141 __aligned(MMU32B_L1_TABLE_ALIGN) __attribute__((section("xlat_table")));
142
143 static uint32_t mmu_l2_base[MAX_XLAT_TABLES][NUM_4K_IN_1MB]
144 __aligned(MMU32B_L2_TABLE_ALIGN) __attribute__((section("xlat_table")));
145
146 /*
147 * Array of all memory regions stored in order of ascending base address.
148 * The list is terminated by the first entry with size == 0.
149 */
150 static mmap_region_t mmap[MAX_MMAP_REGIONS + 1];
151
print_mmap(void)152 void print_mmap(void)
153 {
154 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE
155 mmap_region_t *mm = mmap;
156
157 printf("init xlat - l1:%p l2:%p (%d)\n",
158 (void *)mmu_l1_base, (void *)mmu_l2_base, MAX_XLAT_TABLES);
159 printf("mmap:\n");
160 while (mm->size) {
161 printf(" VA:%p PA:0x%llx size:0x%zx attr:0x%x\n",
162 (void *)mm->base_va, mm->base_pa,
163 mm->size, mm->attr);
164 ++mm;
165 };
166 printf("\n");
167 #endif
168 }
169
mmap_add(const mmap_region_t * mm)170 void mmap_add(const mmap_region_t *mm)
171 {
172 const mmap_region_t *mm_cursor = mm;
173
174 while ((mm_cursor->size != 0U) || (mm_cursor->attr != 0U)) {
175 mmap_add_region(mm_cursor->base_pa, mm_cursor->base_va,
176 mm_cursor->size, mm_cursor->attr);
177 mm_cursor++;
178 }
179 }
180
mmap_add_region(unsigned long long base_pa,uintptr_t base_va,size_t size,unsigned int attr)181 void mmap_add_region(unsigned long long base_pa, uintptr_t base_va,
182 size_t size, unsigned int attr)
183 {
184 mmap_region_t *mm = mmap;
185 const mmap_region_t *mm_last = mm + ARRAY_SIZE(mmap) - 1U;
186 unsigned long long end_pa = base_pa + size - 1U;
187 uintptr_t end_va = base_va + size - 1U;
188
189 assert(IS_PAGE_ALIGNED(base_pa));
190 assert(IS_PAGE_ALIGNED(base_va));
191 assert(IS_PAGE_ALIGNED(size));
192
193 if (size == 0U)
194 return;
195
196 assert(base_pa < end_pa); /* Check for overflows */
197 assert(base_va < end_va);
198
199 assert((base_va + (uintptr_t)size - (uintptr_t)1) <=
200 (PLAT_VIRT_ADDR_SPACE_SIZE - 1U));
201 assert((base_pa + (unsigned long long)size - 1ULL) <=
202 (PLAT_PHY_ADDR_SPACE_SIZE - 1U));
203
204 #if ENABLE_ASSERTIONS
205
206 /* Check for PAs and VAs overlaps with all other regions */
207 for (mm = mmap; mm->size; ++mm) {
208
209 uintptr_t mm_end_va = mm->base_va + mm->size - 1U;
210
211 /*
212 * Check if one of the regions is completely inside the other
213 * one.
214 */
215 bool fully_overlapped_va =
216 ((base_va >= mm->base_va) && (end_va <= mm_end_va)) ||
217 ((mm->base_va >= base_va) && (mm_end_va <= end_va));
218
219 /*
220 * Full VA overlaps are only allowed if both regions are
221 * identity mapped (zero offset) or have the same VA to PA
222 * offset. Also, make sure that it's not the exact same area.
223 */
224 if (fully_overlapped_va) {
225 assert((mm->base_va - mm->base_pa) ==
226 (base_va - base_pa));
227 assert((base_va != mm->base_va) || (size != mm->size));
228 } else {
229 /*
230 * If the regions do not have fully overlapping VAs,
231 * then they must have fully separated VAs and PAs.
232 * Partial overlaps are not allowed
233 */
234
235 unsigned long long mm_end_pa =
236 mm->base_pa + mm->size - 1;
237
238 bool separated_pa = (end_pa < mm->base_pa) ||
239 (base_pa > mm_end_pa);
240 bool separated_va = (end_va < mm->base_va) ||
241 (base_va > mm_end_va);
242
243 assert(separated_va && separated_pa);
244 }
245 }
246
247 mm = mmap; /* Restore pointer to the start of the array */
248
249 #endif /* ENABLE_ASSERTIONS */
250
251 /* Find correct place in mmap to insert new region */
252 while ((mm->base_va < base_va) && (mm->size != 0U))
253 ++mm;
254
255 /*
256 * If a section is contained inside another one with the same base
257 * address, it must be placed after the one it is contained in:
258 *
259 * 1st |-----------------------|
260 * 2nd |------------|
261 * 3rd |------|
262 *
263 * This is required for mmap_region_attr() to get the attributes of the
264 * small region correctly.
265 */
266 while ((mm->base_va == base_va) && (mm->size > size))
267 ++mm;
268
269 /* Make room for new region by moving other regions up by one place */
270 (void)memmove(mm + 1, mm, (uintptr_t)mm_last - (uintptr_t)mm);
271
272 /* Check we haven't lost the empty sentinal from the end of the array */
273 assert(mm_last->size == 0U);
274
275 mm->base_pa = base_pa;
276 mm->base_va = base_va;
277 mm->size = size;
278 mm->attr = attr;
279
280 if (end_pa > xlat_max_pa)
281 xlat_max_pa = end_pa;
282 if (end_va > xlat_max_va)
283 xlat_max_va = end_va;
284 }
285
286 /* map all memory as shared/global/domain0/no-usr access */
mmap_desc(unsigned attr,unsigned int addr_pa,unsigned int level)287 static uint32_t mmap_desc(unsigned attr, unsigned int addr_pa,
288 unsigned int level)
289 {
290 uint32_t desc;
291
292 switch (level) {
293 case 1:
294 assert(!(addr_pa & (MMU32B_L1_TABLE_ALIGN - 1)));
295
296 desc = SECTION_SECTION | SECTION_SHARED;
297
298 desc |= attr & MT_NS ? SECTION_NOTSECURE : 0;
299
300 desc |= SECTION_ACCESS_FLAG;
301 desc |= attr & MT_RW ? 0 : SECTION_RO;
302
303 desc |= attr & MT_MEMORY ?
304 SECTION_NORMAL_CACHED : SECTION_DEVICE;
305
306 if ((attr & MT_RW) || !(attr & MT_MEMORY))
307 desc |= SECTION_XN;
308 break;
309 case 2:
310 assert(!(addr_pa & (MMU32B_L2_TABLE_ALIGN - 1)));
311
312 desc = SMALL_PAGE_SMALL_PAGE | SMALL_PAGE_SHARED;
313
314 desc |= SMALL_PAGE_ACCESS_FLAG;
315 desc |= attr & MT_RW ? 0 : SMALL_PAGE_RO;
316
317 desc |= attr & MT_MEMORY ?
318 SMALL_PAGE_NORMAL_CACHED : SMALL_PAGE_DEVICE;
319
320 if ((attr & MT_RW) || !(attr & MT_MEMORY))
321 desc |= SMALL_PAGE_XN;
322 break;
323 default:
324 panic();
325 }
326 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE
327 /* dump only the non-lpae level 2 tables */
328 if (level == 2) {
329 printf(attr & MT_MEMORY ? "MEM" : "dev");
330 printf(attr & MT_RW ? "-rw" : "-RO");
331 printf(attr & MT_NS ? "-NS" : "-S");
332 }
333 #endif
334 return desc | addr_pa;
335 }
336
mmap_region_attr(const mmap_region_t * mm,uintptr_t base_va,size_t size,unsigned int * attr)337 static unsigned int mmap_region_attr(const mmap_region_t *mm, uintptr_t base_va,
338 size_t size, unsigned int *attr)
339 {
340 /* Don't assume that the area is contained in the first region */
341 unsigned int ret = MT_UNKNOWN;
342
343 /*
344 * Get attributes from last (innermost) region that contains the
345 * requested area. Don't stop as soon as one region doesn't contain it
346 * because there may be other internal regions that contain this area:
347 *
348 * |-----------------------------1-----------------------------|
349 * |----2----| |-------3-------| |----5----|
350 * |--4--|
351 *
352 * |---| <- Area we want the attributes of.
353 *
354 * In this example, the area is contained in regions 1, 3 and 4 but not
355 * in region 2. The loop shouldn't stop at region 2 as inner regions
356 * have priority over outer regions, it should stop at region 5.
357 */
358 for ( ; ; ++mm) {
359
360 if (mm->size == 0U)
361 return ret; /* Reached end of list */
362
363 if (mm->base_va > (base_va + size - 1U))
364 return ret; /* Next region is after area so end */
365
366 if ((mm->base_va + mm->size - 1U) < base_va)
367 continue; /* Next region has already been overtaken */
368
369 if ((ret == 0U) && (mm->attr == *attr))
370 continue; /* Region doesn't override attribs so skip */
371
372 if ((mm->base_va > base_va) ||
373 ((mm->base_va + mm->size - 1U) < (base_va + size - 1U)))
374 return MT_UNKNOWN; /* Region doesn't fully cover area */
375
376 *attr = mm->attr;
377 ret = 0U;
378 }
379 return ret;
380 }
381
init_xlation_table_inner(mmap_region_t * mm,unsigned int base_va,uint32_t * table,unsigned int level)382 static mmap_region_t *init_xlation_table_inner(mmap_region_t *mm,
383 unsigned int base_va,
384 uint32_t *table,
385 unsigned int level)
386 {
387 unsigned int level_size_shift = (level == 1) ?
388 ONE_MB_SHIFT : FOUR_KB_SHIFT;
389 unsigned int level_size = 1 << level_size_shift;
390 unsigned int level_index_mask = (level == 1) ?
391 (NUM_1MB_IN_4GB - 1) << ONE_MB_SHIFT :
392 (NUM_4K_IN_1MB - 1) << FOUR_KB_SHIFT;
393
394 assert(level == 1 || level == 2);
395
396 VERBOSE("init xlat table at %p (level%1d)\n", (void *)table, level);
397
398 do {
399 uint32_t desc = MMU32B_UNSET_DESC;
400
401 if (mm->base_va + mm->size <= base_va) {
402 /* Area now after the region so skip it */
403 ++mm;
404 continue;
405 }
406 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE
407 /* dump only non-lpae level 2 tables content */
408 if (level == 2)
409 printf(" 0x%lx %x " + 6 - 2 * level,
410 base_va, level_size);
411 #endif
412 if (mm->base_va >= base_va + level_size) {
413 /* Next region is after area so nothing to map yet */
414 desc = MMU32B_INVALID_DESC;
415 } else if (mm->base_va <= base_va && mm->base_va + mm->size >=
416 base_va + level_size) {
417 /* Next region covers all of area */
418 unsigned int attr = mm->attr;
419 unsigned int r = mmap_region_attr(mm, base_va,
420 level_size, &attr);
421
422 if (r == 0U) {
423 desc = mmap_desc(attr,
424 base_va - mm->base_va + mm->base_pa,
425 level);
426 }
427 }
428
429 if (desc == MMU32B_UNSET_DESC) {
430 uintptr_t xlat_table;
431
432 /*
433 * Area not covered by a region so need finer table
434 * Reuse next level table if any (assert attrib matching).
435 * Otherwise allocate a xlat table.
436 */
437 if (*table) {
438 assert((*table & 3) == SECTION_PT_PT);
439 assert(!(*table & SECTION_PT_NOTSECURE) ==
440 !(mm->attr & MT_NS));
441
442 xlat_table = (*table) &
443 ~(MMU32B_L1_TABLE_ALIGN - 1);
444 desc = *table;
445 } else {
446 xlat_table = (uintptr_t)mmu_l2_base +
447 next_xlat * MMU32B_L2_TABLE_SIZE;
448 next_xlat++;
449 assert(next_xlat <= MAX_XLAT_TABLES);
450 memset((char *)xlat_table, 0,
451 MMU32B_L2_TABLE_SIZE);
452
453 desc = xlat_table | SECTION_PT_PT;
454 desc |= mm->attr & MT_NS ?
455 SECTION_PT_NOTSECURE : 0;
456 }
457 /* Recurse to fill in new table */
458 mm = init_xlation_table_inner(mm, base_va,
459 (uint32_t *)xlat_table,
460 level + 1);
461 }
462 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE
463 /* dump only non-lpae level 2 tables content */
464 if (level == 2)
465 printf("\n");
466 #endif
467 *table++ = desc;
468 base_va += level_size;
469 } while (mm->size && (base_va & level_index_mask));
470
471 return mm;
472 }
473
init_xlat_tables(void)474 void init_xlat_tables(void)
475 {
476 print_mmap();
477
478 assert(!((unsigned int)mmu_l1_base & (MMU32B_L1_TABLE_ALIGN - 1)));
479 assert(!((unsigned int)mmu_l2_base & (MMU32B_L2_TABLE_ALIGN - 1)));
480
481 memset(mmu_l1_base, 0, MMU32B_L1_TABLE_SIZE);
482
483 init_xlation_table_inner(mmap, 0, (uint32_t *)mmu_l1_base, 1);
484
485 VERBOSE("init xlat - max_va=%p, max_pa=%llx\n",
486 (void *)xlat_max_va, xlat_max_pa);
487 assert(xlat_max_va <= PLAT_VIRT_ADDR_SPACE_SIZE - 1);
488 assert(xlat_max_pa <= PLAT_VIRT_ADDR_SPACE_SIZE - 1);
489 }
490
491 /*******************************************************************************
492 * Function for enabling the MMU in Secure PL1, assuming that the
493 * page-tables have already been created.
494 ******************************************************************************/
enable_mmu_svc_mon(unsigned int flags)495 void enable_mmu_svc_mon(unsigned int flags)
496 {
497 unsigned int prrr;
498 unsigned int nmrr;
499 unsigned int sctlr;
500
501 assert(IS_IN_SECURE());
502 assert((read_sctlr() & SCTLR_M_BIT) == 0);
503
504 /* Enable Access flag (simplified access permissions) and TEX remap */
505 write_sctlr(read_sctlr() | SCTLR_AFE_BIT | SCTLR_TRE_BIT);
506
507 prrr = MMU32B_PRRR_IDX(MMU32B_ATTR_DEVICE_INDEX, 1, 0) \
508 | MMU32B_PRRR_IDX(MMU32B_ATTR_IWBWA_OWBWA_INDEX, 2, 1);
509 nmrr = MMU32B_NMRR_IDX(MMU32B_ATTR_DEVICE_INDEX, 0, 0) \
510 | MMU32B_NMRR_IDX(MMU32B_ATTR_IWBWA_OWBWA_INDEX, 1, 1);
511
512 prrr |= MMU32B_PRRR_NS1 | MMU32B_PRRR_DS1;
513
514 write_prrr(prrr);
515 write_nmrr(nmrr);
516
517 /* Program Domain access control register: domain 0 only */
518 write_dacr(DACR_DOMAIN(0, DACR_DOMAIN_PERM_CLIENT));
519
520 /* Invalidate TLBs at the current exception level */
521 tlbiall();
522
523 /* set MMU base xlat table entry (use only TTBR0) */
524 write_ttbr0((uint32_t)mmu_l1_base | MMU32B_DEFAULT_ATTRS);
525 write_ttbr1(0);
526
527 /*
528 * Ensure all translation table writes have drained
529 * into memory, the TLB invalidation is complete,
530 * and translation register writes are committed
531 * before enabling the MMU
532 */
533 dsb();
534 isb();
535
536 sctlr = read_sctlr();
537 sctlr |= SCTLR_M_BIT;
538 #if ARMV7_SUPPORTS_VIRTUALIZATION
539 sctlr |= SCTLR_WXN_BIT;
540 #endif
541
542 if (flags & DISABLE_DCACHE)
543 sctlr &= ~SCTLR_C_BIT;
544 else
545 sctlr |= SCTLR_C_BIT;
546
547 write_sctlr(sctlr);
548
549 /* Ensure the MMU enable takes effect immediately */
550 isb();
551 }
552