1/* 2 * Copyright (c) 2014 Google Inc. All rights reserved 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining 5 * a copy of this software and associated documentation files 6 * (the "Software"), to deal in the Software without restriction, 7 * including without limitation the rights to use, copy, modify, merge, 8 * publish, distribute, sublicense, and/or sell copies of the Software, 9 * and to permit persons to whom the Software is furnished to do so, 10 * subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be 13 * included in all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 18 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 19 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 20 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 21 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 */ 23 24#include <arch/arm64/mmu.h> 25#include <arch/ops.h> 26#include <assert.h> 27#include <bits.h> 28#include <debug.h> 29#include <err.h> 30#include <kernel/thread.h> 31#include <kernel/vm.h> 32#include <lib/heap.h> 33#include <stdbool.h> 34#include <stdlib.h> 35#include <string.h> 36#include <sys/types.h> 37#include <trace.h> 38#include <inttypes.h> 39 40#define LOCAL_TRACE 0 41#define TRACE_CONTEXT_SWITCH 0 42 43#define ARM64_ASID_BITS (8) /* TODO: Use 16 bit ASIDs when hardware supports it */ 44 45STATIC_ASSERT(((long)KERNEL_BASE >> MMU_KERNEL_SIZE_SHIFT) == -1); 46STATIC_ASSERT(((long)KERNEL_ASPACE_BASE >> MMU_KERNEL_SIZE_SHIFT) == -1); 47STATIC_ASSERT(MMU_KERNEL_SIZE_SHIFT <= 48); 48STATIC_ASSERT(MMU_KERNEL_SIZE_SHIFT >= 25); 49STATIC_ASSERT(USER_ASPACE_BASE + USER_ASPACE_SIZE <= 1UL << MMU_USER_SIZE_SHIFT); 50 51/* the main translation table */ 52extern pte_t arm64_kernel_translation_table[]; 53 54/* This is explicitly a check for overflows, so don't sanitize it */ 55__attribute__((no_sanitize("unsigned-integer-overflow"))) 56static inline bool wrap_check(vaddr_t vaddr, size_t size) { 57 return vaddr + size - 1 > vaddr; 58} 59 60static uint64_t arch_mmu_asid(arch_aspace_t *aspace) 61{ 62 return aspace->asid & BIT_MASK(ARM64_ASID_BITS); 63} 64 65static inline vaddr_t adjusted_vaddr(arch_aspace_t *aspace, vaddr_t vaddr) 66{ 67 return arch_adjusted_vaddr(vaddr, aspace->flags & ARCH_ASPACE_FLAG_KERNEL); 68} 69 70static inline bool is_valid_vaddr(arch_aspace_t *aspace, vaddr_t vaddr) 71{ 72 vaddr = adjusted_vaddr(aspace, vaddr); 73 return (aspace->size && vaddr >= aspace->base && vaddr <= aspace->base + (aspace->size - 1)); 74} 75 76/* convert user level mmu flags to flags that go in L1 descriptors */ 77static bool mmu_flags_to_pte_attr(const uint aspace_flags, const uint flags, pte_t* out_attr) 78{ 79 pte_t attr = MMU_PTE_ATTR_AF; 80 81 DEBUG_ASSERT((aspace_flags & ~ARCH_ASPACE_FLAG_ALL) == 0); 82 83 /* Enforce that executable mappings are not also writable */ 84 if ((flags & (ARCH_MMU_FLAG_PERM_RO | ARCH_MMU_FLAG_PERM_NO_EXECUTE)) == 0) { 85 return false; 86 } 87 88 if (flags & ARCH_MMU_FLAG_TAGGED) { 89 if ((flags & ARCH_MMU_FLAG_CACHE_MASK) & ~ARCH_MMU_FLAG_CACHED) { 90 /* only normal memory can be tagged */ 91 return false; 92 } 93 } 94 95 switch (flags & ARCH_MMU_FLAG_CACHE_MASK) { 96 case ARCH_MMU_FLAG_CACHED: 97 if (flags & ARCH_MMU_FLAG_TAGGED) { 98 attr |= MMU_PTE_ATTR_NORMAL_MEMORY_TAGGED; 99 } else { 100 attr |= MMU_PTE_ATTR_NORMAL_MEMORY; 101 } 102 attr |= MMU_PTE_ATTR_SH_INNER_SHAREABLE; 103 break; 104 case ARCH_MMU_FLAG_UNCACHED: 105 attr |= MMU_PTE_ATTR_STRONGLY_ORDERED; 106 break; 107 case ARCH_MMU_FLAG_UNCACHED_DEVICE: 108 attr |= MMU_PTE_ATTR_DEVICE; 109 break; 110 default: 111 /* invalid user-supplied flag */ 112 DEBUG_ASSERT(0); 113 return false; 114 } 115 116 switch (flags & (ARCH_MMU_FLAG_PERM_USER | ARCH_MMU_FLAG_PERM_RO)) { 117 case 0: 118 attr |= MMU_PTE_ATTR_AP_P_RW_U_NA; 119 break; 120 case ARCH_MMU_FLAG_PERM_RO: 121 attr |= MMU_PTE_ATTR_AP_P_RO_U_NA; 122 break; 123 case ARCH_MMU_FLAG_PERM_USER: 124 attr |= MMU_PTE_ATTR_AP_P_RW_U_RW; 125 break; 126 case ARCH_MMU_FLAG_PERM_USER | ARCH_MMU_FLAG_PERM_RO: 127 attr |= MMU_PTE_ATTR_AP_P_RO_U_RO; 128 break; 129 } 130 131 if (flags & ARCH_MMU_FLAG_PERM_NO_EXECUTE) { 132 attr |= MMU_PTE_ATTR_UXN | MMU_PTE_ATTR_PXN; 133 } else if (flags & ARCH_MMU_FLAG_PERM_USER) { 134 /* User executable page, marked privileged execute never. */ 135 attr |= MMU_PTE_ATTR_PXN; 136 137 if (aspace_flags & ARCH_ASPACE_FLAG_BTI) { 138 attr |= MMU_PTE_ATTR_GP; 139 } 140 } else { 141 /* Privileged executable page, marked user execute never. */ 142 attr |= MMU_PTE_ATTR_UXN; 143 144 if (aspace_flags & ARCH_ASPACE_FLAG_BTI) { 145 attr |= MMU_PTE_ATTR_GP; 146 } 147 } 148 149 if (flags & ARCH_MMU_FLAG_NS) { 150 attr |= MMU_PTE_ATTR_NON_SECURE; 151 } 152 153 /* GP bit is undefined/MBZ if BTI is not supported */ 154 if ((attr & MMU_PTE_ATTR_GP) && !arch_bti_supported()) { 155 return false; 156 } 157 158 *out_attr = attr; 159 return true; 160} 161 162#ifndef EARLY_MMU 163status_t arch_mmu_query(arch_aspace_t *aspace, vaddr_t vaddr, paddr_t *paddr, uint *flags) 164{ 165 uint index; 166 uint index_shift; 167 uint page_size_shift; 168 pte_t pte; 169 pte_t pte_addr; 170 uint descriptor_type; 171 pte_t *page_table; 172 vaddr_t vaddr_rem; 173 174 LTRACEF("aspace %p, vaddr 0x%lx\n", aspace, vaddr); 175 176 DEBUG_ASSERT(aspace); 177 DEBUG_ASSERT(aspace->tt_virt); 178 179 DEBUG_ASSERT(is_valid_vaddr(aspace, vaddr)); 180 if (!is_valid_vaddr(aspace, vaddr)) 181 return ERR_OUT_OF_RANGE; 182 183 vaddr = adjusted_vaddr(aspace, vaddr); 184 /* compute shift values based on if this address space is for kernel or user space */ 185 if (aspace->flags & ARCH_ASPACE_FLAG_KERNEL) { 186 index_shift = MMU_KERNEL_TOP_SHIFT; 187 page_size_shift = MMU_KERNEL_PAGE_SIZE_SHIFT; 188 189 vaddr_t kernel_base = ~0UL << MMU_KERNEL_SIZE_SHIFT; 190 vaddr_rem = vaddr - kernel_base; 191 192 index = vaddr_rem >> index_shift; 193 ASSERT(index < MMU_KERNEL_PAGE_TABLE_ENTRIES_TOP); 194 } else { 195 index_shift = MMU_USER_TOP_SHIFT; 196 page_size_shift = MMU_USER_PAGE_SIZE_SHIFT; 197 198 vaddr_rem = vaddr; 199 index = vaddr_rem >> index_shift; 200 ASSERT(index < MMU_USER_PAGE_TABLE_ENTRIES_TOP); 201 } 202 203 page_table = aspace->tt_virt; 204 205 while (true) { 206 index = vaddr_rem >> index_shift; 207 vaddr_rem -= (vaddr_t)index << index_shift; 208 pte = page_table[index]; 209 descriptor_type = pte & MMU_PTE_DESCRIPTOR_MASK; 210 pte_addr = pte & MMU_PTE_OUTPUT_ADDR_MASK; 211 212 LTRACEF("va 0x%lx, index %d, index_shift %d, rem 0x%lx, pte 0x%" PRIx64 "\n", 213 vaddr, index, index_shift, vaddr_rem, pte); 214 215 if (descriptor_type == MMU_PTE_DESCRIPTOR_INVALID) 216 return ERR_NOT_FOUND; 217 218 if (descriptor_type == ((index_shift > page_size_shift) ? 219 MMU_PTE_L012_DESCRIPTOR_BLOCK : 220 MMU_PTE_L3_DESCRIPTOR_PAGE)) { 221 break; 222 } 223 224 if (index_shift <= page_size_shift || 225 descriptor_type != MMU_PTE_L012_DESCRIPTOR_TABLE) { 226 PANIC_UNIMPLEMENTED; 227 } 228 229 page_table = paddr_to_kvaddr(pte_addr); 230 index_shift -= page_size_shift - 3; 231 } 232 233 if (paddr) 234 *paddr = pte_addr + vaddr_rem; 235 if (flags) { 236 uint mmu_flags = 0; 237 if (pte & MMU_PTE_ATTR_NON_SECURE) 238 mmu_flags |= ARCH_MMU_FLAG_NS; 239 switch (pte & MMU_PTE_ATTR_ATTR_INDEX_MASK) { 240 case MMU_PTE_ATTR_STRONGLY_ORDERED: 241 mmu_flags |= ARCH_MMU_FLAG_UNCACHED; 242 break; 243 case MMU_PTE_ATTR_DEVICE: 244 mmu_flags |= ARCH_MMU_FLAG_UNCACHED_DEVICE; 245 break; 246 case MMU_PTE_ATTR_NORMAL_MEMORY_TAGGED: 247 mmu_flags |= ARCH_MMU_FLAG_TAGGED; 248 break; 249 case MMU_PTE_ATTR_NORMAL_MEMORY: 250 mmu_flags |= ARCH_MMU_FLAG_CACHED; 251 break; 252 default: 253 PANIC_UNIMPLEMENTED; 254 } 255 switch (pte & MMU_PTE_ATTR_AP_MASK) { 256 case MMU_PTE_ATTR_AP_P_RW_U_NA: 257 break; 258 case MMU_PTE_ATTR_AP_P_RW_U_RW: 259 mmu_flags |= ARCH_MMU_FLAG_PERM_USER; 260 break; 261 case MMU_PTE_ATTR_AP_P_RO_U_NA: 262 mmu_flags |= ARCH_MMU_FLAG_PERM_RO; 263 break; 264 case MMU_PTE_ATTR_AP_P_RO_U_RO: 265 mmu_flags |= ARCH_MMU_FLAG_PERM_USER | ARCH_MMU_FLAG_PERM_RO; 266 break; 267 } 268 /* 269 * Based on whether or not this is a user page, check UXN or PXN 270 * bit to determine if it's an executable page. 271 */ 272 if (mmu_flags & ARCH_MMU_FLAG_PERM_USER) { 273 DEBUG_ASSERT(pte & MMU_PTE_ATTR_PXN); 274 if (pte & MMU_PTE_ATTR_UXN) { 275 mmu_flags |= ARCH_MMU_FLAG_PERM_NO_EXECUTE; 276 } 277 } else { 278 DEBUG_ASSERT(pte & MMU_PTE_ATTR_UXN); 279 if (pte & MMU_PTE_ATTR_PXN) { 280 /* Privileged page, check the PXN bit. */ 281 mmu_flags |= ARCH_MMU_FLAG_PERM_NO_EXECUTE; 282 } 283 } 284 *flags = mmu_flags; 285 } 286 LTRACEF("va 0x%lx, paddr 0x%lx, flags 0x%x\n", 287 vaddr, paddr ? *paddr : ~0UL, flags ? *flags : ~0U); 288 return 0; 289} 290 291static int alloc_page_table(paddr_t *paddrp, uint page_size_shift) 292{ 293 size_t size = 1U << page_size_shift; 294 295 LTRACEF("page_size_shift %u\n", page_size_shift); 296 297 if (size >= PAGE_SIZE) { 298 size_t count = size / PAGE_SIZE; 299 size_t ret = pmm_alloc_contiguous(count, page_size_shift, paddrp, NULL); 300 if (ret != count) 301 return ERR_NO_MEMORY; 302 } else { 303 void *vaddr = memalign(size, size); 304 if (!vaddr) 305 return ERR_NO_MEMORY; 306 *paddrp = vaddr_to_paddr(vaddr); 307 if (*paddrp == 0) { 308 free(vaddr); 309 return ERR_NO_MEMORY; 310 } 311 } 312 313 LTRACEF("allocated 0x%lx\n", *paddrp); 314 return 0; 315} 316 317static void free_page_table(void *vaddr, paddr_t paddr, uint page_size_shift) 318{ 319 LTRACEF("vaddr %p paddr 0x%lx page_size_shift %u\n", vaddr, paddr, page_size_shift); 320 321 size_t size = 1U << page_size_shift; 322 vm_page_t *page; 323 324 if (size >= PAGE_SIZE) { 325 page = paddr_to_vm_page(paddr); 326 if (!page) 327 panic("bad page table paddr 0x%lx\n", paddr); 328 pmm_free_page(page); 329 } else { 330 free(vaddr); 331 } 332} 333#endif /* EARLY_MMU */ 334 335static pte_t *arm64_mmu_get_page_table(vaddr_t index, uint page_size_shift, pte_t *page_table) 336{ 337 pte_t pte; 338 paddr_t paddr; 339 void *vaddr; 340 int ret; 341 342 pte = page_table[index]; 343 switch (pte & MMU_PTE_DESCRIPTOR_MASK) { 344 case MMU_PTE_DESCRIPTOR_INVALID: 345 ret = alloc_page_table(&paddr, page_size_shift); 346 if (ret) { 347 TRACEF("failed to allocate page table\n"); 348 return NULL; 349 } 350 vaddr = paddr_to_kvaddr(paddr); 351 352 LTRACEF("allocated page table, vaddr %p, paddr 0x%lx\n", vaddr, paddr); 353 memset(vaddr, MMU_PTE_DESCRIPTOR_INVALID, 1U << page_size_shift); 354 355 __asm__ volatile("dmb ishst" ::: "memory"); 356 357 pte = paddr | MMU_PTE_L012_DESCRIPTOR_TABLE; 358 page_table[index] = pte; 359 LTRACEF("pte %p[0x%lx] = 0x%" PRIx64 "\n", page_table, index, pte); 360 return vaddr; 361 362 case MMU_PTE_L012_DESCRIPTOR_TABLE: 363 paddr = pte & MMU_PTE_OUTPUT_ADDR_MASK; 364 LTRACEF("found page table 0x%lx\n", paddr); 365 return paddr_to_kvaddr(paddr); 366 367 case MMU_PTE_L012_DESCRIPTOR_BLOCK: 368 return NULL; 369 370 default: 371 PANIC_UNIMPLEMENTED; 372 } 373} 374 375static bool page_table_is_clear(pte_t *page_table, uint page_size_shift) 376{ 377 int i; 378 int count = 1U << (page_size_shift - 3); 379 pte_t pte; 380 381 for (i = 0; i < count; i++) { 382 pte = page_table[i]; 383 if (pte != MMU_PTE_DESCRIPTOR_INVALID) { 384 LTRACEF("page_table at %p still in use, index %d is 0x%" PRIx64 "\n", 385 page_table, i, pte); 386 return false; 387 } 388 } 389 390 LTRACEF("page table at %p is clear\n", page_table); 391 return true; 392} 393 394static void arm64_mmu_unmap_pt(vaddr_t vaddr, vaddr_t vaddr_rel, 395 size_t size, 396 uint index_shift, uint page_size_shift, 397 pte_t *page_table, uint asid) 398{ 399 pte_t *next_page_table; 400 vaddr_t index; 401 size_t chunk_size; 402 vaddr_t vaddr_rem; 403 vaddr_t block_size; 404 vaddr_t block_mask; 405 pte_t pte; 406 paddr_t page_table_paddr; 407 408 LTRACEF("vaddr 0x%lx, vaddr_rel 0x%lx, size 0x%lx, index shift %d, page_size_shift %d, page_table %p\n", 409 vaddr, vaddr_rel, size, index_shift, page_size_shift, page_table); 410 411 while (size) { 412 block_size = 1UL << index_shift; 413 block_mask = block_size - 1; 414 vaddr_rem = vaddr_rel & block_mask; 415 chunk_size = MIN(size, block_size - vaddr_rem); 416 index = vaddr_rel >> index_shift; 417 418 pte = page_table[index]; 419 420 if (index_shift > page_size_shift && 421 (pte & MMU_PTE_DESCRIPTOR_MASK) == MMU_PTE_L012_DESCRIPTOR_TABLE) { 422 page_table_paddr = pte & MMU_PTE_OUTPUT_ADDR_MASK; 423 next_page_table = paddr_to_kvaddr(page_table_paddr); 424 arm64_mmu_unmap_pt(vaddr, vaddr_rem, chunk_size, 425 index_shift - (page_size_shift - 3), 426 page_size_shift, 427 next_page_table, asid); 428 if (chunk_size == block_size || 429 page_table_is_clear(next_page_table, page_size_shift)) { 430 LTRACEF("pte %p[0x%lx] = 0 (was page table)\n", page_table, index); 431 page_table[index] = MMU_PTE_DESCRIPTOR_INVALID; 432 __asm__ volatile("dmb ishst" ::: "memory"); 433 free_page_table(next_page_table, page_table_paddr, page_size_shift); 434 } 435 } else if (pte) { 436 LTRACEF("pte %p[0x%lx] = 0\n", page_table, index); 437 page_table[index] = MMU_PTE_DESCRIPTOR_INVALID; 438 CF; 439 if (asid == MMU_ARM64_GLOBAL_ASID) 440 ARM64_TLBI(vaae1is, vaddr >> 12); 441 else 442 ARM64_TLBI(vae1is, vaddr >> 12 | (vaddr_t)asid << 48); 443 } else { 444 LTRACEF("pte %p[0x%lx] already clear\n", page_table, index); 445 } 446 size -= chunk_size; 447 if (!size) { 448 break; 449 } 450 /* Early out avoids a benign overflow. */ 451 vaddr += chunk_size; 452 vaddr_rel += chunk_size; 453 } 454} 455 456static int arm64_mmu_map_pt(vaddr_t vaddr_in, vaddr_t vaddr_rel_in, 457 paddr_t paddr_in, 458 size_t size_in, pte_t attrs, 459 uint index_shift, uint page_size_shift, 460 pte_t *page_table, uint asid, bool replace) 461{ 462 int ret; 463 pte_t *next_page_table; 464 vaddr_t index; 465 vaddr_t vaddr = vaddr_in; 466 vaddr_t vaddr_rel = vaddr_rel_in; 467 paddr_t paddr = paddr_in; 468 size_t size = size_in; 469 size_t chunk_size; 470 vaddr_t vaddr_rem; 471 vaddr_t block_size; 472 vaddr_t block_mask; 473 pte_t pte; 474 475 LTRACEF("vaddr 0x%lx, vaddr_rel 0x%lx, paddr 0x%lx, size 0x%lx, attrs 0x%" PRIx64 ", index shift %d, page_size_shift %d, page_table %p\n", 476 vaddr, vaddr_rel, paddr, size, attrs, 477 index_shift, page_size_shift, page_table); 478 479 if ((vaddr_rel | paddr | size) & ((1UL << page_size_shift) - 1)) { 480 TRACEF("not page aligned\n"); 481 return ERR_INVALID_ARGS; 482 } 483 484 while (size) { 485 block_size = 1UL << index_shift; 486 block_mask = block_size - 1; 487 vaddr_rem = vaddr_rel & block_mask; 488 chunk_size = MIN(size, block_size - vaddr_rem); 489 index = vaddr_rel >> index_shift; 490 491 if (((vaddr_rel | paddr) & block_mask) || 492 (chunk_size != block_size) || 493 (index_shift > MMU_PTE_DESCRIPTOR_BLOCK_MAX_SHIFT)) { 494 next_page_table = arm64_mmu_get_page_table(index, page_size_shift, 495 page_table); 496 if (!next_page_table) 497 goto err; 498 499 ret = arm64_mmu_map_pt(vaddr, vaddr_rem, paddr, chunk_size, attrs, 500 index_shift - (page_size_shift - 3), 501 page_size_shift, next_page_table, asid, 502 replace); 503 if (ret) 504 goto err; 505 } else { 506 pte = page_table[index]; 507 if (!pte && replace) { 508 TRACEF("page table entry does not exist yet, index 0x%lx, 0x%" PRIx64 "\n", 509 index, pte); 510 goto err; 511 } 512 if (pte && !replace) { 513 TRACEF("page table entry already in use, index 0x%lx, 0x%" PRIx64 "\n", 514 index, pte); 515 goto err; 516 } 517 518 pte = paddr | attrs; 519 if (index_shift > page_size_shift) 520 pte |= MMU_PTE_L012_DESCRIPTOR_BLOCK; 521 else 522 pte |= MMU_PTE_L3_DESCRIPTOR_PAGE; 523 524 LTRACEF("pte %p[0x%lx] = 0x%" PRIx64 "\n", page_table, index, pte); 525 page_table[index] = pte; 526 if (replace) { 527 CF; 528 if (asid == MMU_ARM64_GLOBAL_ASID) 529 ARM64_TLBI(vaae1is, vaddr >> 12); 530 else 531 ARM64_TLBI(vae1is, vaddr >> 12 | (vaddr_t)asid << 48); 532 } 533 } 534 size -= chunk_size; 535 if (!size) { 536 break; 537 } 538 /* Note: early out avoids a benign overflow. */ 539 vaddr += chunk_size; 540 vaddr_rel += chunk_size; 541 paddr += chunk_size; 542 } 543 544 return 0; 545 546err: 547 arm64_mmu_unmap_pt(vaddr_in, vaddr_rel_in, size_in - size, 548 index_shift, page_size_shift, page_table, asid); 549 DSB; 550 return ERR_GENERIC; 551} 552 553#ifndef EARLY_MMU 554int arm64_mmu_map(vaddr_t vaddr, paddr_t paddr, size_t size, pte_t attrs, 555 vaddr_t vaddr_base, uint top_size_shift, 556 uint top_index_shift, uint page_size_shift, 557 pte_t *top_page_table, uint asid, bool replace) 558{ 559 int ret; 560 vaddr_t vaddr_rel = vaddr - vaddr_base; 561 vaddr_t vaddr_rel_max = 1UL << top_size_shift; 562 563 LTRACEF("vaddr 0x%lx, paddr 0x%lx, size 0x%lx, attrs 0x%" PRIx64 ", asid 0x%x\n", 564 vaddr, paddr, size, attrs, asid); 565 566 if (vaddr_rel > vaddr_rel_max - size || size > vaddr_rel_max) { 567 TRACEF("vaddr 0x%lx, size 0x%lx out of range vaddr 0x%lx, size 0x%lx\n", 568 vaddr, size, vaddr_base, vaddr_rel_max); 569 return ERR_INVALID_ARGS; 570 } 571 572 if (!top_page_table) { 573 TRACEF("page table is NULL\n"); 574 return ERR_INVALID_ARGS; 575 } 576 577 ret = arm64_mmu_map_pt(vaddr, vaddr_rel, paddr, size, attrs, 578 top_index_shift, page_size_shift, top_page_table, 579 asid, replace); 580 DSB; 581 return ret; 582} 583 584int arm64_mmu_unmap(vaddr_t vaddr, size_t size, 585 vaddr_t vaddr_base, uint top_size_shift, 586 uint top_index_shift, uint page_size_shift, 587 pte_t *top_page_table, uint asid) 588{ 589 vaddr_t vaddr_rel = vaddr - vaddr_base; 590 vaddr_t vaddr_rel_max = 1UL << top_size_shift; 591 592 LTRACEF("vaddr 0x%lx, size 0x%lx, asid 0x%x\n", vaddr, size, asid); 593 594 if (vaddr_rel > vaddr_rel_max - size || size > vaddr_rel_max) { 595 TRACEF("vaddr 0x%lx, size 0x%lx out of range vaddr 0x%lx, size 0x%lx\n", 596 vaddr, size, vaddr_base, vaddr_rel_max); 597 return ERR_INVALID_ARGS; 598 } 599 600 if (!top_page_table) { 601 TRACEF("page table is NULL\n"); 602 return ERR_INVALID_ARGS; 603 } 604 605 arm64_mmu_unmap_pt(vaddr, vaddr_rel, size, 606 top_index_shift, page_size_shift, top_page_table, asid); 607 DSB; 608 return 0; 609} 610 611static void arm64_tlbflush_if_asid_changed(arch_aspace_t *aspace, asid_t asid) 612{ 613 THREAD_LOCK(state); 614 if (asid != arch_mmu_asid(aspace)) { 615 TRACEF("asid changed for aspace %p while mapping or unmapping memory, 0x%" PRIxASID " -> 0x%" PRIxASID ", flush all tlbs\n", 616 aspace, asid, aspace->asid); 617 ARM64_TLBI_NOADDR(vmalle1is); 618 DSB; 619 } 620 THREAD_UNLOCK(state); 621} 622 623static int arm64_mmu_map_aspace(arch_aspace_t *aspace, vaddr_t vaddr, paddr_t paddr, size_t count, uint flags, 624 bool replace) 625{ 626 LTRACEF("vaddr 0x%lx paddr 0x%lx count %zu flags 0x%x\n", vaddr, paddr, count, flags); 627 628 DEBUG_ASSERT(aspace); 629 DEBUG_ASSERT(aspace->tt_virt); 630 631 DEBUG_ASSERT(is_valid_vaddr(aspace, vaddr)); 632 if (!is_valid_vaddr(aspace, vaddr)) 633 return ERR_OUT_OF_RANGE; 634 635 vaddr = adjusted_vaddr(aspace, vaddr); 636 637 /* paddr and vaddr must be aligned */ 638 DEBUG_ASSERT(IS_PAGE_ALIGNED(vaddr)); 639 DEBUG_ASSERT(IS_PAGE_ALIGNED(paddr)); 640 if (!IS_PAGE_ALIGNED(vaddr) || !IS_PAGE_ALIGNED(paddr)) 641 return ERR_INVALID_ARGS; 642 643 if (paddr & ~MMU_PTE_OUTPUT_ADDR_MASK) { 644 return ERR_INVALID_ARGS; 645 } 646 647 if (count == 0) 648 return NO_ERROR; 649 650 pte_t pte_attr; 651 if (!mmu_flags_to_pte_attr(aspace->flags, flags, &pte_attr)) { 652 return ERR_INVALID_ARGS; 653 } 654 655 int ret; 656 if (aspace->flags & ARCH_ASPACE_FLAG_KERNEL) { 657 ret = arm64_mmu_map(vaddr, paddr, count * PAGE_SIZE, 658 pte_attr, 659 ~0UL << MMU_KERNEL_SIZE_SHIFT, MMU_KERNEL_SIZE_SHIFT, 660 MMU_KERNEL_TOP_SHIFT, MMU_KERNEL_PAGE_SIZE_SHIFT, 661 aspace->tt_virt, MMU_ARM64_GLOBAL_ASID, replace); 662 } else { 663 asid_t asid = arch_mmu_asid(aspace); 664 ret = arm64_mmu_map(vaddr, paddr, count * PAGE_SIZE, 665 pte_attr | MMU_PTE_ATTR_NON_GLOBAL, 666 0, MMU_USER_SIZE_SHIFT, 667 MMU_USER_TOP_SHIFT, MMU_USER_PAGE_SIZE_SHIFT, 668 aspace->tt_virt, asid, replace); 669 arm64_tlbflush_if_asid_changed(aspace, asid); 670 } 671 672 return ret; 673} 674 675int arch_mmu_map(arch_aspace_t *aspace, vaddr_t vaddr, paddr_t paddr, size_t count, uint flags) 676{ 677 return arm64_mmu_map_aspace(aspace, vaddr, paddr, count, flags, false); 678} 679 680int arch_mmu_map_replace(arch_aspace_t *aspace, vaddr_t vaddr, paddr_t paddr, size_t count, uint flags) 681{ 682 return arm64_mmu_map_aspace(aspace, vaddr, paddr, count, flags, true); 683} 684 685int arch_mmu_unmap(arch_aspace_t *aspace, vaddr_t vaddr, size_t count) 686{ 687 LTRACEF("vaddr 0x%lx count %zu\n", vaddr, count); 688 689 DEBUG_ASSERT(aspace); 690 DEBUG_ASSERT(aspace->tt_virt); 691 692 DEBUG_ASSERT(is_valid_vaddr(aspace, vaddr)); 693 694 if (!is_valid_vaddr(aspace, vaddr)) 695 return ERR_OUT_OF_RANGE; 696 697 vaddr = adjusted_vaddr(aspace, vaddr); 698 699 DEBUG_ASSERT(IS_PAGE_ALIGNED(vaddr)); 700 if (!IS_PAGE_ALIGNED(vaddr)) 701 return ERR_INVALID_ARGS; 702 703 int ret; 704 if (aspace->flags & ARCH_ASPACE_FLAG_KERNEL) { 705 ret = arm64_mmu_unmap(vaddr, count * PAGE_SIZE, 706 ~0UL << MMU_KERNEL_SIZE_SHIFT, MMU_KERNEL_SIZE_SHIFT, 707 MMU_KERNEL_TOP_SHIFT, MMU_KERNEL_PAGE_SIZE_SHIFT, 708 aspace->tt_virt, 709 MMU_ARM64_GLOBAL_ASID); 710 } else { 711 asid_t asid = arch_mmu_asid(aspace); 712 ret = arm64_mmu_unmap(vaddr, count * PAGE_SIZE, 713 0, MMU_USER_SIZE_SHIFT, 714 MMU_USER_TOP_SHIFT, MMU_USER_PAGE_SIZE_SHIFT, 715 aspace->tt_virt, asid); 716 arm64_tlbflush_if_asid_changed(aspace, asid); 717 } 718 719 return ret; 720} 721 722status_t arch_mmu_init_aspace(arch_aspace_t *aspace, vaddr_t base, size_t size, uint flags) 723{ 724 LTRACEF("aspace %p, base 0x%lx, size 0x%zx, flags 0x%x\n", aspace, base, size, flags); 725 726 DEBUG_ASSERT(aspace); 727 728 /* validate that the base + size is sane and doesn't wrap */ 729 DEBUG_ASSERT(size > PAGE_SIZE); 730 DEBUG_ASSERT(wrap_check(base, size)); 731 732 aspace->flags = flags; 733 if (flags & ARCH_ASPACE_FLAG_KERNEL) { 734 /* at the moment we can only deal with address spaces as globally defined */ 735 DEBUG_ASSERT(base == ~0UL << MMU_KERNEL_SIZE_SHIFT); 736 DEBUG_ASSERT(size == 1UL << MMU_KERNEL_SIZE_SHIFT); 737 738 aspace->base = base; 739 aspace->size = size; 740 aspace->tt_virt = arm64_kernel_translation_table; 741 aspace->tt_phys = vaddr_to_paddr(aspace->tt_virt); 742 } else { 743 size_t page_table_size = MMU_USER_PAGE_TABLE_ENTRIES_TOP * sizeof(pte_t); 744 //DEBUG_ASSERT(base >= 0); 745 DEBUG_ASSERT(base + size <= 1UL << MMU_USER_SIZE_SHIFT); 746 747 aspace->base = base; 748 aspace->size = size; 749 750 pte_t *va = memalign(page_table_size, page_table_size); 751 if (!va) 752 return ERR_NO_MEMORY; 753 754 aspace->tt_virt = va; 755 aspace->tt_phys = vaddr_to_paddr(aspace->tt_virt); 756 757 /* zero the top level translation table */ 758 memset(aspace->tt_virt, 0, page_table_size); 759 } 760 761 LTRACEF("tt_phys 0x%lx tt_virt %p\n", aspace->tt_phys, aspace->tt_virt); 762 763 return NO_ERROR; 764} 765 766status_t arch_mmu_destroy_aspace(arch_aspace_t *aspace) 767{ 768 LTRACEF("aspace %p\n", aspace); 769 770 DEBUG_ASSERT(aspace); 771 DEBUG_ASSERT((aspace->flags & ARCH_ASPACE_FLAG_KERNEL) == 0); 772 773 // XXX make sure it's not mapped 774 775 free(aspace->tt_virt); 776 777 return NO_ERROR; 778} 779 780void arch_mmu_context_switch(arch_aspace_t *aspace) 781{ 782 bool flush_tlb; 783 784 if (TRACE_CONTEXT_SWITCH) 785 TRACEF("aspace %p\n", aspace); 786 787 flush_tlb = vmm_asid_activate(aspace, ARM64_ASID_BITS); 788 789 uint64_t tcr; 790 uint64_t ttbr; 791 if (aspace) { 792 DEBUG_ASSERT((aspace->flags & ARCH_ASPACE_FLAG_KERNEL) == 0); 793 794 tcr = MMU_TCR_FLAGS_USER; 795 ttbr = (arch_mmu_asid(aspace) << 48) | aspace->tt_phys; 796 ARM64_WRITE_SYSREG(ttbr0_el1, ttbr); 797 798 } else { 799 tcr = MMU_TCR_FLAGS_KERNEL; 800 } 801 802#if KERNEL_PAC_ENABLED 803 /* If TBI0 is set, also set TBID0. 804 * TBID0 is RES0 if FEAT_PAuth is not supported, so this must be 805 * conditional. 806 */ 807 if ((tcr & MMU_TCR_TBI0) != 0 && arch_pac_address_supported()) { 808 tcr |= MMU_TCR_TBID0; 809 } 810#endif 811 812 if (TRACE_CONTEXT_SWITCH) { 813 if (aspace) { 814 TRACEF("ttbr 0x%" PRIx64 ", tcr 0x%" PRIx64 "\n", ttbr, tcr); 815 } else { 816 TRACEF("tcr 0x%" PRIx64 "\n", tcr); 817 } 818 } 819 820 ARM64_WRITE_SYSREG(tcr_el1, tcr); /* TODO: only needed when switching between kernel and user threads */ 821 822 if (flush_tlb) { 823 ARM64_TLBI_NOADDR(vmalle1); 824 DSB; 825 } 826} 827#endif /* EARLY_MMU */ 828