1 // SPDX-License-Identifier: GPL-2.0-only
5  *  Copyright (C) 1995-2005 Russell King
35 #include <asm/mach/map.h>
47  * zero-initialized data and COW.
53  * The pmd table for the upper-most set of pages.
144 	int i, selected = -1;  in early_cachepolicy()
155 	if (selected == -1)  in early_cachepolicy()
239 	[MT_DEVICE_NONSHARED] = { /* ARMv6 non-shared device */
436 			pr_warn("Forcing write-allocate cache policy for SMP\n");  in build_mem_type_table()
447 	 * Pre-ARMv5 CPUs don't have TEX bits.  Pre-ARMv6 CPUs or those  in build_mem_type_table()
459 	 * "update-able on write" bit on ARM610).  However, Xscale and  in build_mem_type_table()
482 			 * Mark device regions on ARMv6+ as execute-never  in build_mem_type_table()
496 			 * - shared device is SXCB=1100  in build_mem_type_table()
497 			 * - nonshared device is SXCB=0100  in build_mem_type_table()
498 			 * - write combine device mem is SXCB=0001  in build_mem_type_table()
507 			 * - shared device is TEXCB=00101  in build_mem_type_table()
508 			 * - nonshared device is TEXCB=01000  in build_mem_type_table()
509 			 * - write combine device mem is TEXCB=00100  in build_mem_type_table()
518 			 * - shared device is TEXCB=00001  in build_mem_type_table()
519 			 * - nonshared device is TEXCB=01000  in build_mem_type_table()
520 			 * - write combine device mem is TEXCB=00100  in build_mem_type_table()
535 	 * Now deal with the memory-type mappings  in build_mem_type_table()
538 	vecs_pgprot = kern_pgprot = user_pgprot = cp->pte;  in build_mem_type_table()
544 	 * r/o, kernel r/w to map the vectors page.  in build_mem_type_table()
551 	 * in the Short-descriptor translation table format descriptors.  in build_mem_type_table()
597 	 * Non-cacheable Normal - intended for memory areas that must  in build_mem_type_table()
602 			/* Non-cacheable Normal is XCB = 001 */  in build_mem_type_table()
606 			/* For both ARMv6 and non-TEX-remapping ARMv7 */  in build_mem_type_table()
646 	mem_types[MT_MEMORY_RWX].prot_sect |= ecc_mask | cp->pmd;  in build_mem_type_table()
648 	mem_types[MT_MEMORY_RW].prot_sect |= ecc_mask | cp->pmd;  in build_mem_type_table()
652 	mem_types[MT_ROM].prot_sect |= cp->pmd;  in build_mem_type_table()
654 	switch (cp->pmd) {  in build_mem_type_table()
664 		ecc_mask ? "ECC enabled, " : "", cp->policy);  in build_mem_type_table()
668 		if (t->prot_l1)  in build_mem_type_table()
669 			t->prot_l1 |= PMD_DOMAIN(t->domain);  in build_mem_type_table()
670 		if (t->prot_sect)  in build_mem_type_table()
671 			t->prot_sect |= PMD_DOMAIN(t->domain);  in build_mem_type_table()
681 	else if (file->f_flags & O_SYNC)  in phys_mem_access_prot()
734 	pte_t *pte = arm_pte_alloc(pmd, addr, type->prot_l1, alloc);  in alloc_init_pte()
736 		set_pte_ext(pte, pfn_pte(pfn, __pgprot(type->prot_pte)),  in alloc_init_pte()
756 	 * (See arch/arm/include/asm/pgtable-2level.h)  in __map_init_section()
762 		*pmd = __pmd(phys | type->prot_sect | (ng ? PMD_SECT_nG : 0));  in __map_init_section()
779 		 * With LPAE, we must loop over to map  in alloc_init_pmd()
785 		 * Try a section mapping - addr, next and phys must all be  in alloc_init_pmd()
788 		if (type->prot_sect &&  in alloc_init_pmd()
796 		phys += next - addr;  in alloc_init_pmd()
812 		phys += next - addr;  in alloc_init_pud()
827 		phys += next - addr;  in alloc_init_p4d()
841 	addr = md->virtual;  in create_36bit_mapping()
842 	phys = __pfn_to_phys(md->pfn);  in create_36bit_mapping()
843 	length = PAGE_ALIGN(md->length);  in create_36bit_mapping()
847 		       (long long)__pfn_to_phys((u64)md->pfn), addr);  in create_36bit_mapping()
857 	if (type->domain) {  in create_36bit_mapping()
859 		       (long long)__pfn_to_phys((u64)md->pfn), addr);  in create_36bit_mapping()
863 	if ((addr | length | __pfn_to_phys(md->pfn)) & ~SUPERSECTION_MASK) {  in create_36bit_mapping()
865 		       (long long)__pfn_to_phys((u64)md->pfn), addr);  in create_36bit_mapping()
873 	phys |= (((md->pfn >> (32 - PAGE_SHIFT)) & 0xF) << 20);  in create_36bit_mapping()
884 			*pmd++ = __pmd(phys | type->prot_sect | PMD_SECT_SUPER |  in create_36bit_mapping()
903 	type = &mem_types[md->type];  in __create_mapping()
907 	 * Catch 36-bit addresses  in __create_mapping()
909 	if (md->pfn >= 0x100000) {  in __create_mapping()
915 	addr = md->virtual & PAGE_MASK;  in __create_mapping()
916 	phys = __pfn_to_phys(md->pfn);  in __create_mapping()
917 	length = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK));  in __create_mapping()
919 	if (type->prot_l1 == 0 && ((addr | phys | length) & ~SECTION_MASK)) {  in __create_mapping()
920 		pr_warn("BUG: map for 0x%08llx at 0x%08lx can not be mapped using pages, ignoring.\n",  in __create_mapping()
921 			(long long)__pfn_to_phys(md->pfn), addr);  in __create_mapping()
932 		phys += next - addr;  in __create_mapping()
946 	if (md->virtual != vectors_base() && md->virtual < TASK_SIZE) {  in create_mapping()
948 			(long long)__pfn_to_phys((u64)md->pfn), md->virtual);  in create_mapping()
952 	if (md->type == MT_DEVICE &&  in create_mapping()
953 	    md->virtual >= PAGE_OFFSET && md->virtual < FIXADDR_START &&  in create_mapping()
954 	    (md->virtual < VMALLOC_START || md->virtual >= VMALLOC_END)) {  in create_mapping()
956 			(long long)__pfn_to_phys((u64)md->pfn), md->virtual);  in create_mapping()
969 	p4d = p4d_alloc(mm, pgd_offset(mm, md->virtual), md->virtual);  in create_mapping_late()
972 	pud = pud_alloc(mm, p4d, md->virtual);  in create_mapping_late()
986 	struct vm_struct *vm;  in iotable_init()  local
997 	for (md = io_desc; nr; md++, nr--) {  in iotable_init()
1000 		vm = &svm->vm;  in iotable_init()
1001 		vm->addr = (void *)(md->virtual & PAGE_MASK);  in iotable_init()
1002 		vm->size = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK));  in iotable_init()
1003 		vm->phys_addr = __pfn_to_phys(md->pfn);  in iotable_init()
1004 		vm->flags = VM_IOREMAP | VM_ARM_STATIC_MAPPING;  in iotable_init()
1005 		vm->flags |= VM_ARM_MTYPE(md->type);  in iotable_init()
1006 		vm->caller = iotable_init;  in iotable_init()
1014 	struct vm_struct *vm;  in vm_reserve_area_early()  local
1022 	vm = &svm->vm;  in vm_reserve_area_early()
1023 	vm->addr = (void *)addr;  in vm_reserve_area_early()
1024 	vm->size = size;  in vm_reserve_area_early()
1025 	vm->flags = VM_IOREMAP | VM_ARM_EMPTY_MAPPING;  in vm_reserve_area_early()
1026 	vm->caller = caller;  in vm_reserve_area_early()
1034  * (see definition in include/asm/pgtable-2level.h).  However a call to
1041  * Let's avoid the issue by inserting dummy vm entries covering the unused
1053 	struct vm_struct *vm;  in fill_pmd_gaps()  local
1058 		vm = &svm->vm;  in fill_pmd_gaps()
1059 		addr = (unsigned long)vm->addr;  in fill_pmd_gaps()
1064 		 * Check if this vm starts on an odd section boundary.  in fill_pmd_gaps()
1075 		 * Then check if this vm ends on an odd section boundary.  in fill_pmd_gaps()
1079 		addr += vm->size;  in fill_pmd_gaps()
1086 		/* no need to look at any vm entry until we hit the next PMD */  in fill_pmd_gaps()
1087 		next = (addr + PMD_SIZE - 1) & PMD_MASK;  in fill_pmd_gaps()
1113 	struct map_desc map;  in debug_ll_io_init()  local
1115 	debug_ll_addr(&map.pfn, &map.virtual);  in debug_ll_io_init()
1116 	if (!map.pfn || !map.virtual)  in debug_ll_io_init()
1118 	map.pfn = __phys_to_pfn(map.pfn);  in debug_ll_io_init()
1119 	map.virtual &= PAGE_MASK;  in debug_ll_io_init()
1120 	map.length = PAGE_SIZE;  in debug_ll_io_init()
1121 	map.type = MT_DEVICE;  in debug_ll_io_init()
1122 	iotable_init(&map, 1);  in debug_ll_io_init()
1131  * area - the default is 240m.
1143 	if (vmalloc_reserve > VMALLOC_END - (PAGE_OFFSET + SZ_32M)) {  in early_vmalloc()
1144 		vmalloc_reserve = VMALLOC_END - (PAGE_OFFSET + SZ_32M);  in early_vmalloc()
1149 	vmalloc_min = (void *)(VMALLOC_END - vmalloc_reserve);  in early_vmalloc()
1164 	 * not affected by wrap-arounds when sizeof(phys_addr_t) == 4.  in adjust_lowmem_bounds()
1169 	vmalloc_limit = (u64)(uintptr_t)vmalloc_min - PAGE_OFFSET + PHYS_OFFSET;  in adjust_lowmem_bounds()
1179 			len = round_up(block_start, PMD_SIZE) - block_start;  in adjust_lowmem_bounds()
1199 			 * Find the first non-pmd-aligned page, and point  in adjust_lowmem_bounds()
1201 			 * limit down to be pmd-aligned, which happens at the  in adjust_lowmem_bounds()
1205 			 * bank can be non-pmd-aligned. The only exception is  in adjust_lowmem_bounds()
1206 			 * that the start of the bank 0 must be section-  in adjust_lowmem_bounds()
1223 	high_memory = __va(arm_lowmem_limit - 1) + 1;  in adjust_lowmem_bounds()
1239 			pr_notice("Ignoring RAM at %pa-%pa\n",  in adjust_lowmem_bounds()
1243 			memblock_remove(memblock_limit, end - memblock_limit);  in adjust_lowmem_bounds()
1268 	 * are using a thumb-compiled kernel, there there will be 8MB more  in prepare_page_table()
1279 	/* The XIP kernel is mapped in the module area -- skip over it */  in prepare_page_table()
1280 	addr = ((unsigned long)_exiprom + PMD_SIZE - 1) & PMD_MASK;  in prepare_page_table()
1323 	 * precious DMA-able memory...  in arm_mm_memblock_reserve()
1325 	memblock_reserve(PHYS_OFFSET, __pa(swapper_pg_dir) - PHYS_OFFSET);  in arm_mm_memblock_reserve()
1338 	struct map_desc map;  in devicemaps_init()  local
1356 		/* create a read-only mapping of the device tree */  in devicemaps_init()
1357 		map.pfn = __phys_to_pfn(__atags_pointer & SECTION_MASK);  in devicemaps_init()
1358 		map.virtual = FDT_FIXED_BASE;  in devicemaps_init()
1359 		map.length = FDT_FIXED_SIZE;  in devicemaps_init()
1360 		map.type = MT_ROM;  in devicemaps_init()
1361 		create_mapping(&map);  in devicemaps_init()
1365 	 * Map the kernel if it is XIP.  in devicemaps_init()
1369 	map.pfn = __phys_to_pfn(CONFIG_XIP_PHYS_ADDR & SECTION_MASK);  in devicemaps_init()
1370 	map.virtual = MODULES_VADDR;  in devicemaps_init()
1371 	map.length = ((unsigned long)_exiprom - map.virtual + ~SECTION_MASK) & SECTION_MASK;  in devicemaps_init()
1372 	map.type = MT_ROM;  in devicemaps_init()
1373 	create_mapping(&map);  in devicemaps_init()
1377 	 * Map the cache flushing regions.  in devicemaps_init()
1380 	map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS);  in devicemaps_init()
1381 	map.virtual = FLUSH_BASE;  in devicemaps_init()
1382 	map.length = SZ_1M;  in devicemaps_init()
1383 	map.type = MT_CACHECLEAN;  in devicemaps_init()
1384 	create_mapping(&map);  in devicemaps_init()
1387 	map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS + SZ_1M);  in devicemaps_init()
1388 	map.virtual = FLUSH_BASE_MINICACHE;  in devicemaps_init()
1389 	map.length = SZ_1M;  in devicemaps_init()
1390 	map.type = MT_MINICLEAN;  in devicemaps_init()
1391 	create_mapping(&map);  in devicemaps_init()
1395 	 * Create a mapping for the machine vectors at the high-vectors  in devicemaps_init()
1396 	 * location (0xffff0000).  If we aren't using high-vectors, also  in devicemaps_init()
1397 	 * create a mapping at the low-vectors virtual address.  in devicemaps_init()
1399 	map.pfn = __phys_to_pfn(virt_to_phys(vectors));  in devicemaps_init()
1400 	map.virtual = 0xffff0000;  in devicemaps_init()
1401 	map.length = PAGE_SIZE;  in devicemaps_init()
1403 	map.type = MT_HIGH_VECTORS;  in devicemaps_init()
1405 	map.type = MT_LOW_VECTORS;  in devicemaps_init()
1407 	create_mapping(&map);  in devicemaps_init()
1410 		map.virtual = 0;  in devicemaps_init()
1411 		map.length = PAGE_SIZE * 2;  in devicemaps_init()
1412 		map.type = MT_LOW_VECTORS;  in devicemaps_init()
1413 		create_mapping(&map);  in devicemaps_init()
1416 	/* Now create a kernel read-only mapping */  in devicemaps_init()
1417 	map.pfn += 1;  in devicemaps_init()
1418 	map.virtual = 0xffff0000 + PAGE_SIZE;  in devicemaps_init()
1419 	map.length = PAGE_SIZE;  in devicemaps_init()
1420 	map.type = MT_LOW_VECTORS;  in devicemaps_init()
1421 	create_mapping(&map);  in devicemaps_init()
1424 	 * Ask the machine support to map in the statically mapped devices.  in devicemaps_init()
1426 	if (mdesc->map_io)  in devicemaps_init()
1427 		mdesc->map_io();  in devicemaps_init()
1438 	 * any write-allocated cache lines in the vector page are written  in devicemaps_init()
1466 	/* Map all the lowmem memory banks. */  in map_lowmem()
1468 		struct map_desc map;  in map_lowmem()  local
1476 			map.pfn = __phys_to_pfn(start);  in map_lowmem()
1477 			map.virtual = __phys_to_virt(start);  in map_lowmem()
1478 			map.length = end - start;  in map_lowmem()
1479 			map.type = MT_MEMORY_RWX;  in map_lowmem()
1481 			create_mapping(&map);  in map_lowmem()
1483 			map.pfn = __phys_to_pfn(start);  in map_lowmem()
1484 			map.virtual = __phys_to_virt(start);  in map_lowmem()
1485 			map.length = end - start;  in map_lowmem()
1486 			map.type = MT_MEMORY_RW;  in map_lowmem()
1488 			create_mapping(&map);  in map_lowmem()
1492 				map.pfn = __phys_to_pfn(start);  in map_lowmem()
1493 				map.virtual = __phys_to_virt(start);  in map_lowmem()
1494 				map.length = kernel_x_start - start;  in map_lowmem()
1495 				map.type = MT_MEMORY_RW;  in map_lowmem()
1497 				create_mapping(&map);  in map_lowmem()
1500 			map.pfn = __phys_to_pfn(kernel_x_start);  in map_lowmem()
1501 			map.virtual = __phys_to_virt(kernel_x_start);  in map_lowmem()
1502 			map.length = kernel_x_end - kernel_x_start;  in map_lowmem()
1503 			map.type = MT_MEMORY_RWX;  in map_lowmem()
1505 			create_mapping(&map);  in map_lowmem()
1508 				map.pfn = __phys_to_pfn(kernel_x_end);  in map_lowmem()
1509 				map.virtual = __phys_to_virt(kernel_x_end);  in map_lowmem()
1510 				map.length = end - kernel_x_end;  in map_lowmem()
1511 				map.type = MT_MEMORY_RW;  in map_lowmem()
1513 				create_mapping(&map);  in map_lowmem()
1534 	if (!mdesc->pv_fixup)  in early_paging_init()
1537 	offset = mdesc->pv_fixup();  in early_paging_init()
1554 	/* Re-set the phys pfn offset, and the pv offset */  in early_paging_init()
1560 		(&__pv_table_end - &__pv_table_begin) << 2);  in early_paging_init()
1579 	 * Fixup the page tables - this must be in the idmap region as  in early_paging_init()
1586 	/* Re-enable the caches and cacheable TLB walks */  in early_paging_init()
1597 	if (!mdesc->pv_fixup)  in early_paging_init()
1600 	offset = mdesc->pv_fixup();  in early_paging_init()
1615 	unsigned long va = fix_to_virt(__end_of_permanent_fixed_addresses - 1);  in early_fixmap_shutdown()
1623 		struct map_desc map;  in early_fixmap_shutdown()  local
1625 		map.virtual = fix_to_virt(i);  in early_fixmap_shutdown()
1626 		pte = pte_offset_early_fixmap(pmd_off_k(map.virtual), map.virtual);  in early_fixmap_shutdown()
1633 		map.pfn = pte_pfn(*pte);  in early_fixmap_shutdown()
1634 		map.type = MT_DEVICE;  in early_fixmap_shutdown()
1635 		map.length = PAGE_SIZE;  in early_fixmap_shutdown()
1637 		create_mapping(&map);  in early_fixmap_shutdown()