1 /*
2 * Copyright (c) 2017-2021, Arm Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <assert.h>
8 #include <errno.h>
9 #include <stdbool.h>
10 #include <stdint.h>
11 #include <stdio.h>
12
13 #include <platform_def.h>
14
15 #include <arch_helpers.h>
16 #include <common/debug.h>
17 #include <lib/utils_def.h>
18 #include <lib/xlat_tables/xlat_tables_defs.h>
19 #include <lib/xlat_tables/xlat_tables_v2.h>
20
21 #include "xlat_tables_private.h"
22
23 #if LOG_LEVEL < LOG_LEVEL_VERBOSE
24
xlat_mmap_print(__unused const mmap_region_t * mmap)25 void xlat_mmap_print(__unused const mmap_region_t *mmap)
26 {
27 /* Empty */
28 }
29
xlat_tables_print(__unused xlat_ctx_t * ctx)30 void xlat_tables_print(__unused xlat_ctx_t *ctx)
31 {
32 /* Empty */
33 }
34
35 #else /* if LOG_LEVEL >= LOG_LEVEL_VERBOSE */
36
xlat_mmap_print(const mmap_region_t * mmap)37 void xlat_mmap_print(const mmap_region_t *mmap)
38 {
39 printf("mmap:\n");
40 const mmap_region_t *mm = mmap;
41
42 while (mm->size != 0U) {
43 printf(" VA:0x%lx PA:0x%llx size:0x%zx attr:0x%x granularity:0x%zx\n",
44 mm->base_va, mm->base_pa, mm->size, mm->attr,
45 mm->granularity);
46 ++mm;
47 };
48 printf("\n");
49 }
50
51 /* Print the attributes of the specified block descriptor. */
xlat_desc_print(const xlat_ctx_t * ctx,uint64_t desc)52 static void xlat_desc_print(const xlat_ctx_t *ctx, uint64_t desc)
53 {
54 uint64_t mem_type_index = ATTR_INDEX_GET(desc);
55 int xlat_regime = ctx->xlat_regime;
56
57 if (mem_type_index == ATTR_IWBWA_OWBWA_NTR_INDEX) {
58 printf("MEM");
59 } else if (mem_type_index == ATTR_NON_CACHEABLE_INDEX) {
60 printf("NC");
61 } else {
62 assert(mem_type_index == ATTR_DEVICE_INDEX);
63 printf("DEV");
64 }
65
66 if ((xlat_regime == EL3_REGIME) || (xlat_regime == EL2_REGIME)) {
67 /* For EL3 and EL2 only check the AP[2] and XN bits. */
68 printf(((desc & LOWER_ATTRS(AP_RO)) != 0ULL) ? "-RO" : "-RW");
69 printf(((desc & UPPER_ATTRS(XN)) != 0ULL) ? "-XN" : "-EXEC");
70 } else {
71 assert(xlat_regime == EL1_EL0_REGIME);
72 /*
73 * For EL0 and EL1:
74 * - In AArch64 PXN and UXN can be set independently but in
75 * AArch32 there is no UXN (XN affects both privilege levels).
76 * For consistency, we set them simultaneously in both cases.
77 * - RO and RW permissions must be the same in EL1 and EL0. If
78 * EL0 can access that memory region, so can EL1, with the
79 * same permissions.
80 */
81 #if ENABLE_ASSERTIONS
82 uint64_t xn_mask = xlat_arch_regime_get_xn_desc(EL1_EL0_REGIME);
83 uint64_t xn_perm = desc & xn_mask;
84
85 assert((xn_perm == xn_mask) || (xn_perm == 0ULL));
86 #endif
87 printf(((desc & LOWER_ATTRS(AP_RO)) != 0ULL) ? "-RO" : "-RW");
88 /* Only check one of PXN and UXN, the other one is the same. */
89 printf(((desc & UPPER_ATTRS(PXN)) != 0ULL) ? "-XN" : "-EXEC");
90 /*
91 * Privileged regions can only be accessed from EL1, user
92 * regions can be accessed from EL1 and EL0.
93 */
94 printf(((desc & LOWER_ATTRS(AP_ACCESS_UNPRIVILEGED)) != 0ULL)
95 ? "-USER" : "-PRIV");
96 }
97
98 #if ENABLE_RME
99 switch (desc & LOWER_ATTRS(EL3_S1_NSE | NS)) {
100 case 0ULL:
101 printf("-S");
102 break;
103 case LOWER_ATTRS(NS):
104 printf("-NS");
105 break;
106 case LOWER_ATTRS(EL3_S1_NSE):
107 printf("-RT");
108 break;
109 default: /* LOWER_ATTRS(EL3_S1_NSE | NS) */
110 printf("-RL");
111 }
112 #else
113 printf(((LOWER_ATTRS(NS) & desc) != 0ULL) ? "-NS" : "-S");
114 #endif
115
116 #ifdef __aarch64__
117 /* Check Guarded Page bit */
118 if ((desc & GP) != 0ULL) {
119 printf("-GP");
120 }
121 #endif
122 }
123
124 static const char * const level_spacers[] = {
125 "[LV0] ",
126 " [LV1] ",
127 " [LV2] ",
128 " [LV3] "
129 };
130
131 static const char *invalid_descriptors_ommited =
132 "%s(%d invalid descriptors omitted)\n";
133
134 /*
135 * Recursive function that reads the translation tables passed as an argument
136 * and prints their status.
137 */
xlat_tables_print_internal(xlat_ctx_t * ctx,uintptr_t table_base_va,const uint64_t * table_base,unsigned int table_entries,unsigned int level)138 static void xlat_tables_print_internal(xlat_ctx_t *ctx, uintptr_t table_base_va,
139 const uint64_t *table_base, unsigned int table_entries,
140 unsigned int level)
141 {
142 assert(level <= XLAT_TABLE_LEVEL_MAX);
143
144 uint64_t desc;
145 uintptr_t table_idx_va = table_base_va;
146 unsigned int table_idx = 0U;
147 size_t level_size = XLAT_BLOCK_SIZE(level);
148
149 /*
150 * Keep track of how many invalid descriptors are counted in a row.
151 * Whenever multiple invalid descriptors are found, only the first one
152 * is printed, and a line is added to inform about how many descriptors
153 * have been omitted.
154 */
155 int invalid_row_count = 0;
156
157 while (table_idx < table_entries) {
158
159 desc = table_base[table_idx];
160
161 if ((desc & DESC_MASK) == INVALID_DESC) {
162
163 if (invalid_row_count == 0) {
164 printf("%sVA:0x%lx size:0x%zx\n",
165 level_spacers[level],
166 table_idx_va, level_size);
167 }
168 invalid_row_count++;
169
170 } else {
171
172 if (invalid_row_count > 1) {
173 printf(invalid_descriptors_ommited,
174 level_spacers[level],
175 invalid_row_count - 1);
176 }
177 invalid_row_count = 0;
178
179 /*
180 * Check if this is a table or a block. Tables are only
181 * allowed in levels other than 3, but DESC_PAGE has the
182 * same value as DESC_TABLE, so we need to check.
183 */
184 if (((desc & DESC_MASK) == TABLE_DESC) &&
185 (level < XLAT_TABLE_LEVEL_MAX)) {
186 /*
187 * Do not print any PA for a table descriptor,
188 * as it doesn't directly map physical memory
189 * but instead points to the next translation
190 * table in the translation table walk.
191 */
192 printf("%sVA:0x%lx size:0x%zx\n",
193 level_spacers[level],
194 table_idx_va, level_size);
195
196 uintptr_t addr_inner = desc & TABLE_ADDR_MASK;
197
198 xlat_tables_print_internal(ctx, table_idx_va,
199 (uint64_t *)addr_inner,
200 XLAT_TABLE_ENTRIES, level + 1U);
201 } else {
202 printf("%sVA:0x%lx PA:0x%llx size:0x%zx ",
203 level_spacers[level], table_idx_va,
204 (uint64_t)(desc & TABLE_ADDR_MASK),
205 level_size);
206 xlat_desc_print(ctx, desc);
207 printf("\n");
208 }
209 }
210
211 table_idx++;
212 table_idx_va += level_size;
213 }
214
215 if (invalid_row_count > 1) {
216 printf(invalid_descriptors_ommited,
217 level_spacers[level], invalid_row_count - 1);
218 }
219 }
220
xlat_tables_print(xlat_ctx_t * ctx)221 void xlat_tables_print(xlat_ctx_t *ctx)
222 {
223 const char *xlat_regime_str;
224 int used_page_tables;
225
226 if (ctx->xlat_regime == EL1_EL0_REGIME) {
227 xlat_regime_str = "1&0";
228 } else if (ctx->xlat_regime == EL2_REGIME) {
229 xlat_regime_str = "2";
230 } else {
231 assert(ctx->xlat_regime == EL3_REGIME);
232 xlat_regime_str = "3";
233 }
234 VERBOSE("Translation tables state:\n");
235 VERBOSE(" Xlat regime: EL%s\n", xlat_regime_str);
236 VERBOSE(" Max allowed PA: 0x%llx\n", ctx->pa_max_address);
237 VERBOSE(" Max allowed VA: 0x%lx\n", ctx->va_max_address);
238 VERBOSE(" Max mapped PA: 0x%llx\n", ctx->max_pa);
239 VERBOSE(" Max mapped VA: 0x%lx\n", ctx->max_va);
240
241 VERBOSE(" Initial lookup level: %u\n", ctx->base_level);
242 VERBOSE(" Entries @initial lookup level: %u\n",
243 ctx->base_table_entries);
244
245 #if PLAT_XLAT_TABLES_DYNAMIC
246 used_page_tables = 0;
247 for (int i = 0; i < ctx->tables_num; ++i) {
248 if (ctx->tables_mapped_regions[i] != 0)
249 ++used_page_tables;
250 }
251 #else
252 used_page_tables = ctx->next_table;
253 #endif
254 VERBOSE(" Used %d sub-tables out of %d (spare: %d)\n",
255 used_page_tables, ctx->tables_num,
256 ctx->tables_num - used_page_tables);
257
258 xlat_tables_print_internal(ctx, 0U, ctx->base_table,
259 ctx->base_table_entries, ctx->base_level);
260 }
261
262 #endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */
263
264 /*
265 * Do a translation table walk to find the block or page descriptor that maps
266 * virtual_addr.
267 *
268 * On success, return the address of the descriptor within the translation
269 * table. Its lookup level is stored in '*out_level'.
270 * On error, return NULL.
271 *
272 * xlat_table_base
273 * Base address for the initial lookup level.
274 * xlat_table_base_entries
275 * Number of entries in the translation table for the initial lookup level.
276 * virt_addr_space_size
277 * Size in bytes of the virtual address space.
278 */
find_xlat_table_entry(uintptr_t virtual_addr,void * xlat_table_base,unsigned int xlat_table_base_entries,unsigned long long virt_addr_space_size,unsigned int * out_level)279 static uint64_t *find_xlat_table_entry(uintptr_t virtual_addr,
280 void *xlat_table_base,
281 unsigned int xlat_table_base_entries,
282 unsigned long long virt_addr_space_size,
283 unsigned int *out_level)
284 {
285 unsigned int start_level;
286 uint64_t *table;
287 unsigned int entries;
288
289 start_level = GET_XLAT_TABLE_LEVEL_BASE(virt_addr_space_size);
290
291 table = xlat_table_base;
292 entries = xlat_table_base_entries;
293
294 for (unsigned int level = start_level;
295 level <= XLAT_TABLE_LEVEL_MAX;
296 ++level) {
297 uint64_t idx, desc, desc_type;
298
299 idx = XLAT_TABLE_IDX(virtual_addr, level);
300 if (idx >= entries) {
301 WARN("Missing xlat table entry at address 0x%lx\n",
302 virtual_addr);
303 return NULL;
304 }
305
306 desc = table[idx];
307 desc_type = desc & DESC_MASK;
308
309 if (desc_type == INVALID_DESC) {
310 VERBOSE("Invalid entry (memory not mapped)\n");
311 return NULL;
312 }
313
314 if (level == XLAT_TABLE_LEVEL_MAX) {
315 /*
316 * Only page descriptors allowed at the final lookup
317 * level.
318 */
319 assert(desc_type == PAGE_DESC);
320 *out_level = level;
321 return &table[idx];
322 }
323
324 if (desc_type == BLOCK_DESC) {
325 *out_level = level;
326 return &table[idx];
327 }
328
329 assert(desc_type == TABLE_DESC);
330 table = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK);
331 entries = XLAT_TABLE_ENTRIES;
332 }
333
334 /*
335 * This shouldn't be reached, the translation table walk should end at
336 * most at level XLAT_TABLE_LEVEL_MAX and return from inside the loop.
337 */
338 assert(false);
339
340 return NULL;
341 }
342
343
xlat_get_mem_attributes_internal(const xlat_ctx_t * ctx,uintptr_t base_va,uint32_t * attributes,uint64_t ** table_entry,unsigned long long * addr_pa,unsigned int * table_level)344 static int xlat_get_mem_attributes_internal(const xlat_ctx_t *ctx,
345 uintptr_t base_va, uint32_t *attributes, uint64_t **table_entry,
346 unsigned long long *addr_pa, unsigned int *table_level)
347 {
348 uint64_t *entry;
349 uint64_t desc;
350 unsigned int level;
351 unsigned long long virt_addr_space_size;
352
353 /*
354 * Sanity-check arguments.
355 */
356 assert(ctx != NULL);
357 assert(ctx->initialized);
358 assert((ctx->xlat_regime == EL1_EL0_REGIME) ||
359 (ctx->xlat_regime == EL2_REGIME) ||
360 (ctx->xlat_regime == EL3_REGIME));
361
362 virt_addr_space_size = (unsigned long long)ctx->va_max_address + 1ULL;
363 assert(virt_addr_space_size > 0U);
364
365 entry = find_xlat_table_entry(base_va,
366 ctx->base_table,
367 ctx->base_table_entries,
368 virt_addr_space_size,
369 &level);
370 if (entry == NULL) {
371 WARN("Address 0x%lx is not mapped.\n", base_va);
372 return -EINVAL;
373 }
374
375 if (addr_pa != NULL) {
376 *addr_pa = *entry & TABLE_ADDR_MASK;
377 }
378
379 if (table_entry != NULL) {
380 *table_entry = entry;
381 }
382
383 if (table_level != NULL) {
384 *table_level = level;
385 }
386
387 desc = *entry;
388
389 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE
390 VERBOSE("Attributes: ");
391 xlat_desc_print(ctx, desc);
392 printf("\n");
393 #endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */
394
395 assert(attributes != NULL);
396 *attributes = 0U;
397
398 uint64_t attr_index = (desc >> ATTR_INDEX_SHIFT) & ATTR_INDEX_MASK;
399
400 if (attr_index == ATTR_IWBWA_OWBWA_NTR_INDEX) {
401 *attributes |= MT_MEMORY;
402 } else if (attr_index == ATTR_NON_CACHEABLE_INDEX) {
403 *attributes |= MT_NON_CACHEABLE;
404 } else {
405 assert(attr_index == ATTR_DEVICE_INDEX);
406 *attributes |= MT_DEVICE;
407 }
408
409 uint64_t ap2_bit = (desc >> AP2_SHIFT) & 1U;
410
411 if (ap2_bit == AP2_RW)
412 *attributes |= MT_RW;
413
414 if (ctx->xlat_regime == EL1_EL0_REGIME) {
415 uint64_t ap1_bit = (desc >> AP1_SHIFT) & 1U;
416
417 if (ap1_bit == AP1_ACCESS_UNPRIVILEGED)
418 *attributes |= MT_USER;
419 }
420
421 uint64_t ns_bit = (desc >> NS_SHIFT) & 1U;
422
423 if (ns_bit == 1U)
424 *attributes |= MT_NS;
425
426 uint64_t xn_mask = xlat_arch_regime_get_xn_desc(ctx->xlat_regime);
427
428 if ((desc & xn_mask) == xn_mask) {
429 *attributes |= MT_EXECUTE_NEVER;
430 } else {
431 assert((desc & xn_mask) == 0U);
432 }
433
434 return 0;
435 }
436
437
xlat_get_mem_attributes_ctx(const xlat_ctx_t * ctx,uintptr_t base_va,uint32_t * attr)438 int xlat_get_mem_attributes_ctx(const xlat_ctx_t *ctx, uintptr_t base_va,
439 uint32_t *attr)
440 {
441 return xlat_get_mem_attributes_internal(ctx, base_va, attr,
442 NULL, NULL, NULL);
443 }
444
445
xlat_change_mem_attributes_ctx(const xlat_ctx_t * ctx,uintptr_t base_va,size_t size,uint32_t attr)446 int xlat_change_mem_attributes_ctx(const xlat_ctx_t *ctx, uintptr_t base_va,
447 size_t size, uint32_t attr)
448 {
449 /* Note: This implementation isn't optimized. */
450
451 assert(ctx != NULL);
452 assert(ctx->initialized);
453
454 unsigned long long virt_addr_space_size =
455 (unsigned long long)ctx->va_max_address + 1U;
456 assert(virt_addr_space_size > 0U);
457
458 if (!IS_PAGE_ALIGNED(base_va)) {
459 WARN("%s: Address 0x%lx is not aligned on a page boundary.\n",
460 __func__, base_va);
461 return -EINVAL;
462 }
463
464 if (size == 0U) {
465 WARN("%s: Size is 0.\n", __func__);
466 return -EINVAL;
467 }
468
469 if ((size % PAGE_SIZE) != 0U) {
470 WARN("%s: Size 0x%zx is not a multiple of a page size.\n",
471 __func__, size);
472 return -EINVAL;
473 }
474
475 if (((attr & MT_EXECUTE_NEVER) == 0U) && ((attr & MT_RW) != 0U)) {
476 WARN("%s: Mapping memory as read-write and executable not allowed.\n",
477 __func__);
478 return -EINVAL;
479 }
480
481 size_t pages_count = size / PAGE_SIZE;
482
483 VERBOSE("Changing memory attributes of %zu pages starting from address 0x%lx...\n",
484 pages_count, base_va);
485
486 uintptr_t base_va_original = base_va;
487
488 /*
489 * Sanity checks.
490 */
491 for (unsigned int i = 0U; i < pages_count; ++i) {
492 const uint64_t *entry;
493 uint64_t desc, attr_index;
494 unsigned int level;
495
496 entry = find_xlat_table_entry(base_va,
497 ctx->base_table,
498 ctx->base_table_entries,
499 virt_addr_space_size,
500 &level);
501 if (entry == NULL) {
502 WARN("Address 0x%lx is not mapped.\n", base_va);
503 return -EINVAL;
504 }
505
506 desc = *entry;
507
508 /*
509 * Check that all the required pages are mapped at page
510 * granularity.
511 */
512 if (((desc & DESC_MASK) != PAGE_DESC) ||
513 (level != XLAT_TABLE_LEVEL_MAX)) {
514 WARN("Address 0x%lx is not mapped at the right granularity.\n",
515 base_va);
516 WARN("Granularity is 0x%lx, should be 0x%lx.\n",
517 XLAT_BLOCK_SIZE(level), PAGE_SIZE);
518 return -EINVAL;
519 }
520
521 /*
522 * If the region type is device, it shouldn't be executable.
523 */
524 attr_index = (desc >> ATTR_INDEX_SHIFT) & ATTR_INDEX_MASK;
525 if (attr_index == ATTR_DEVICE_INDEX) {
526 if ((attr & MT_EXECUTE_NEVER) == 0U) {
527 WARN("Setting device memory as executable at address 0x%lx.",
528 base_va);
529 return -EINVAL;
530 }
531 }
532
533 base_va += PAGE_SIZE;
534 }
535
536 /* Restore original value. */
537 base_va = base_va_original;
538
539 for (unsigned int i = 0U; i < pages_count; ++i) {
540
541 uint32_t old_attr = 0U, new_attr;
542 uint64_t *entry = NULL;
543 unsigned int level = 0U;
544 unsigned long long addr_pa = 0ULL;
545
546 (void) xlat_get_mem_attributes_internal(ctx, base_va, &old_attr,
547 &entry, &addr_pa, &level);
548
549 /*
550 * From attr, only MT_RO/MT_RW, MT_EXECUTE/MT_EXECUTE_NEVER and
551 * MT_USER/MT_PRIVILEGED are taken into account. Any other
552 * information is ignored.
553 */
554
555 /* Clean the old attributes so that they can be rebuilt. */
556 new_attr = old_attr & ~(MT_RW | MT_EXECUTE_NEVER | MT_USER);
557
558 /*
559 * Update attributes, but filter out the ones this function
560 * isn't allowed to change.
561 */
562 new_attr |= attr & (MT_RW | MT_EXECUTE_NEVER | MT_USER);
563
564 /*
565 * The break-before-make sequence requires writing an invalid
566 * descriptor and making sure that the system sees the change
567 * before writing the new descriptor.
568 */
569 *entry = INVALID_DESC;
570 #if !HW_ASSISTED_COHERENCY
571 dccvac((uintptr_t)entry);
572 #endif
573 /* Invalidate any cached copy of this mapping in the TLBs. */
574 xlat_arch_tlbi_va(base_va, ctx->xlat_regime);
575
576 /* Ensure completion of the invalidation. */
577 xlat_arch_tlbi_va_sync();
578
579 /* Write new descriptor */
580 *entry = xlat_desc(ctx, new_attr, addr_pa, level);
581 #if !HW_ASSISTED_COHERENCY
582 dccvac((uintptr_t)entry);
583 #endif
584 base_va += PAGE_SIZE;
585 }
586
587 /* Ensure that the last descriptor writen is seen by the system. */
588 dsbish();
589
590 return 0;
591 }
592