• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Page table handling routines for radix page table.
4  *
5  * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
6  */
7 
8 #define pr_fmt(fmt) "radix-mmu: " fmt
9 
10 #include <linux/io.h>
11 #include <linux/kernel.h>
12 #include <linux/sched/mm.h>
13 #include <linux/memblock.h>
14 #include <linux/of.h>
15 #include <linux/of_fdt.h>
16 #include <linux/mm.h>
17 #include <linux/hugetlb.h>
18 #include <linux/string_helpers.h>
19 #include <linux/memory.h>
20 
21 #include <asm/pgalloc.h>
22 #include <asm/mmu_context.h>
23 #include <asm/dma.h>
24 #include <asm/machdep.h>
25 #include <asm/mmu.h>
26 #include <asm/firmware.h>
27 #include <asm/powernv.h>
28 #include <asm/sections.h>
29 #include <asm/smp.h>
30 #include <asm/trace.h>
31 #include <asm/uaccess.h>
32 #include <asm/ultravisor.h>
33 
34 #include <trace/events/thp.h>
35 
36 unsigned int mmu_pid_bits;
37 unsigned int mmu_base_pid;
38 unsigned long radix_mem_block_size __ro_after_init;
39 
early_alloc_pgtable(unsigned long size,int nid,unsigned long region_start,unsigned long region_end)40 static __ref void *early_alloc_pgtable(unsigned long size, int nid,
41 			unsigned long region_start, unsigned long region_end)
42 {
43 	phys_addr_t min_addr = MEMBLOCK_LOW_LIMIT;
44 	phys_addr_t max_addr = MEMBLOCK_ALLOC_ANYWHERE;
45 	void *ptr;
46 
47 	if (region_start)
48 		min_addr = region_start;
49 	if (region_end)
50 		max_addr = region_end;
51 
52 	ptr = memblock_alloc_try_nid(size, size, min_addr, max_addr, nid);
53 
54 	if (!ptr)
55 		panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa max_addr=%pa\n",
56 		      __func__, size, size, nid, &min_addr, &max_addr);
57 
58 	return ptr;
59 }
60 
61 /*
62  * When allocating pud or pmd pointers, we allocate a complete page
63  * of PAGE_SIZE rather than PUD_TABLE_SIZE or PMD_TABLE_SIZE. This
64  * is to ensure that the page obtained from the memblock allocator
65  * can be completely used as page table page and can be freed
66  * correctly when the page table entries are removed.
67  */
early_map_kernel_page(unsigned long ea,unsigned long pa,pgprot_t flags,unsigned int map_page_size,int nid,unsigned long region_start,unsigned long region_end)68 static int early_map_kernel_page(unsigned long ea, unsigned long pa,
69 			  pgprot_t flags,
70 			  unsigned int map_page_size,
71 			  int nid,
72 			  unsigned long region_start, unsigned long region_end)
73 {
74 	unsigned long pfn = pa >> PAGE_SHIFT;
75 	pgd_t *pgdp;
76 	p4d_t *p4dp;
77 	pud_t *pudp;
78 	pmd_t *pmdp;
79 	pte_t *ptep;
80 
81 	pgdp = pgd_offset_k(ea);
82 	p4dp = p4d_offset(pgdp, ea);
83 	if (p4d_none(*p4dp)) {
84 		pudp = early_alloc_pgtable(PAGE_SIZE, nid,
85 					   region_start, region_end);
86 		p4d_populate(&init_mm, p4dp, pudp);
87 	}
88 	pudp = pud_offset(p4dp, ea);
89 	if (map_page_size == PUD_SIZE) {
90 		ptep = (pte_t *)pudp;
91 		goto set_the_pte;
92 	}
93 	if (pud_none(*pudp)) {
94 		pmdp = early_alloc_pgtable(PAGE_SIZE, nid, region_start,
95 					   region_end);
96 		pud_populate(&init_mm, pudp, pmdp);
97 	}
98 	pmdp = pmd_offset(pudp, ea);
99 	if (map_page_size == PMD_SIZE) {
100 		ptep = pmdp_ptep(pmdp);
101 		goto set_the_pte;
102 	}
103 	if (!pmd_present(*pmdp)) {
104 		ptep = early_alloc_pgtable(PAGE_SIZE, nid,
105 						region_start, region_end);
106 		pmd_populate_kernel(&init_mm, pmdp, ptep);
107 	}
108 	ptep = pte_offset_kernel(pmdp, ea);
109 
110 set_the_pte:
111 	set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
112 	asm volatile("ptesync": : :"memory");
113 	return 0;
114 }
115 
116 /*
117  * nid, region_start, and region_end are hints to try to place the page
118  * table memory in the same node or region.
119  */
__map_kernel_page(unsigned long ea,unsigned long pa,pgprot_t flags,unsigned int map_page_size,int nid,unsigned long region_start,unsigned long region_end)120 static int __map_kernel_page(unsigned long ea, unsigned long pa,
121 			  pgprot_t flags,
122 			  unsigned int map_page_size,
123 			  int nid,
124 			  unsigned long region_start, unsigned long region_end)
125 {
126 	unsigned long pfn = pa >> PAGE_SHIFT;
127 	pgd_t *pgdp;
128 	p4d_t *p4dp;
129 	pud_t *pudp;
130 	pmd_t *pmdp;
131 	pte_t *ptep;
132 	/*
133 	 * Make sure task size is correct as per the max adddr
134 	 */
135 	BUILD_BUG_ON(TASK_SIZE_USER64 > RADIX_PGTABLE_RANGE);
136 
137 #ifdef CONFIG_PPC_64K_PAGES
138 	BUILD_BUG_ON(RADIX_KERN_MAP_SIZE != (1UL << MAX_EA_BITS_PER_CONTEXT));
139 #endif
140 
141 	if (unlikely(!slab_is_available()))
142 		return early_map_kernel_page(ea, pa, flags, map_page_size,
143 						nid, region_start, region_end);
144 
145 	/*
146 	 * Should make page table allocation functions be able to take a
147 	 * node, so we can place kernel page tables on the right nodes after
148 	 * boot.
149 	 */
150 	pgdp = pgd_offset_k(ea);
151 	p4dp = p4d_offset(pgdp, ea);
152 	pudp = pud_alloc(&init_mm, p4dp, ea);
153 	if (!pudp)
154 		return -ENOMEM;
155 	if (map_page_size == PUD_SIZE) {
156 		ptep = (pte_t *)pudp;
157 		goto set_the_pte;
158 	}
159 	pmdp = pmd_alloc(&init_mm, pudp, ea);
160 	if (!pmdp)
161 		return -ENOMEM;
162 	if (map_page_size == PMD_SIZE) {
163 		ptep = pmdp_ptep(pmdp);
164 		goto set_the_pte;
165 	}
166 	ptep = pte_alloc_kernel(pmdp, ea);
167 	if (!ptep)
168 		return -ENOMEM;
169 
170 set_the_pte:
171 	set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
172 	asm volatile("ptesync": : :"memory");
173 	return 0;
174 }
175 
radix__map_kernel_page(unsigned long ea,unsigned long pa,pgprot_t flags,unsigned int map_page_size)176 int radix__map_kernel_page(unsigned long ea, unsigned long pa,
177 			  pgprot_t flags,
178 			  unsigned int map_page_size)
179 {
180 	return __map_kernel_page(ea, pa, flags, map_page_size, -1, 0, 0);
181 }
182 
183 #ifdef CONFIG_STRICT_KERNEL_RWX
radix__change_memory_range(unsigned long start,unsigned long end,unsigned long clear)184 static void radix__change_memory_range(unsigned long start, unsigned long end,
185 				       unsigned long clear)
186 {
187 	unsigned long idx;
188 	pgd_t *pgdp;
189 	p4d_t *p4dp;
190 	pud_t *pudp;
191 	pmd_t *pmdp;
192 	pte_t *ptep;
193 
194 	start = ALIGN_DOWN(start, PAGE_SIZE);
195 	end = PAGE_ALIGN(end); // aligns up
196 
197 	pr_debug("Changing flags on range %lx-%lx removing 0x%lx\n",
198 		 start, end, clear);
199 
200 	for (idx = start; idx < end; idx += PAGE_SIZE) {
201 		pgdp = pgd_offset_k(idx);
202 		p4dp = p4d_offset(pgdp, idx);
203 		pudp = pud_alloc(&init_mm, p4dp, idx);
204 		if (!pudp)
205 			continue;
206 		if (pud_is_leaf(*pudp)) {
207 			ptep = (pte_t *)pudp;
208 			goto update_the_pte;
209 		}
210 		pmdp = pmd_alloc(&init_mm, pudp, idx);
211 		if (!pmdp)
212 			continue;
213 		if (pmd_is_leaf(*pmdp)) {
214 			ptep = pmdp_ptep(pmdp);
215 			goto update_the_pte;
216 		}
217 		ptep = pte_alloc_kernel(pmdp, idx);
218 		if (!ptep)
219 			continue;
220 update_the_pte:
221 		radix__pte_update(&init_mm, idx, ptep, clear, 0, 0);
222 	}
223 
224 	radix__flush_tlb_kernel_range(start, end);
225 }
226 
radix__mark_rodata_ro(void)227 void radix__mark_rodata_ro(void)
228 {
229 	unsigned long start, end;
230 
231 	start = (unsigned long)_stext;
232 	end = (unsigned long)__init_begin;
233 
234 	radix__change_memory_range(start, end, _PAGE_WRITE);
235 
236 	for (start = PAGE_OFFSET; start < (unsigned long)_stext; start += PAGE_SIZE) {
237 		end = start + PAGE_SIZE;
238 		if (overlaps_interrupt_vector_text(start, end))
239 			radix__change_memory_range(start, end, _PAGE_WRITE);
240 		else
241 			break;
242 	}
243 }
244 
radix__mark_initmem_nx(void)245 void radix__mark_initmem_nx(void)
246 {
247 	unsigned long start = (unsigned long)__init_begin;
248 	unsigned long end = (unsigned long)__init_end;
249 
250 	radix__change_memory_range(start, end, _PAGE_EXEC);
251 }
252 #endif /* CONFIG_STRICT_KERNEL_RWX */
253 
254 static inline void __meminit
print_mapping(unsigned long start,unsigned long end,unsigned long size,bool exec)255 print_mapping(unsigned long start, unsigned long end, unsigned long size, bool exec)
256 {
257 	char buf[10];
258 
259 	if (end <= start)
260 		return;
261 
262 	string_get_size(size, 1, STRING_UNITS_2, buf, sizeof(buf));
263 
264 	pr_info("Mapped 0x%016lx-0x%016lx with %s pages%s\n", start, end, buf,
265 		exec ? " (exec)" : "");
266 }
267 
next_boundary(unsigned long addr,unsigned long end)268 static unsigned long next_boundary(unsigned long addr, unsigned long end)
269 {
270 #ifdef CONFIG_STRICT_KERNEL_RWX
271 	unsigned long stext_phys;
272 
273 	stext_phys = __pa_symbol(_stext);
274 
275 	// Relocatable kernel running at non-zero real address
276 	if (stext_phys != 0) {
277 		// The end of interrupts code at zero is a rodata boundary
278 		unsigned long end_intr = __pa_symbol(__end_interrupts) - stext_phys;
279 		if (addr < end_intr)
280 			return end_intr;
281 
282 		// Start of relocated kernel text is a rodata boundary
283 		if (addr < stext_phys)
284 			return stext_phys;
285 	}
286 
287 	if (addr < __pa_symbol(__srwx_boundary))
288 		return __pa_symbol(__srwx_boundary);
289 #endif
290 	return end;
291 }
292 
create_physical_mapping(unsigned long start,unsigned long end,unsigned long max_mapping_size,int nid,pgprot_t _prot)293 static int __meminit create_physical_mapping(unsigned long start,
294 					     unsigned long end,
295 					     unsigned long max_mapping_size,
296 					     int nid, pgprot_t _prot)
297 {
298 	unsigned long vaddr, addr, mapping_size = 0;
299 	bool prev_exec, exec = false;
300 	pgprot_t prot;
301 	int psize;
302 
303 	start = ALIGN(start, PAGE_SIZE);
304 	end   = ALIGN_DOWN(end, PAGE_SIZE);
305 	for (addr = start; addr < end; addr += mapping_size) {
306 		unsigned long gap, previous_size;
307 		int rc;
308 
309 		gap = next_boundary(addr, end) - addr;
310 		if (gap > max_mapping_size)
311 			gap = max_mapping_size;
312 		previous_size = mapping_size;
313 		prev_exec = exec;
314 
315 		if (IS_ALIGNED(addr, PUD_SIZE) && gap >= PUD_SIZE &&
316 		    mmu_psize_defs[MMU_PAGE_1G].shift) {
317 			mapping_size = PUD_SIZE;
318 			psize = MMU_PAGE_1G;
319 		} else if (IS_ALIGNED(addr, PMD_SIZE) && gap >= PMD_SIZE &&
320 			   mmu_psize_defs[MMU_PAGE_2M].shift) {
321 			mapping_size = PMD_SIZE;
322 			psize = MMU_PAGE_2M;
323 		} else {
324 			mapping_size = PAGE_SIZE;
325 			psize = mmu_virtual_psize;
326 		}
327 
328 		vaddr = (unsigned long)__va(addr);
329 
330 		if (overlaps_kernel_text(vaddr, vaddr + mapping_size) ||
331 		    overlaps_interrupt_vector_text(vaddr, vaddr + mapping_size)) {
332 			prot = PAGE_KERNEL_X;
333 			exec = true;
334 		} else {
335 			prot = _prot;
336 			exec = false;
337 		}
338 
339 		if (mapping_size != previous_size || exec != prev_exec) {
340 			print_mapping(start, addr, previous_size, prev_exec);
341 			start = addr;
342 		}
343 
344 		rc = __map_kernel_page(vaddr, addr, prot, mapping_size, nid, start, end);
345 		if (rc)
346 			return rc;
347 
348 		update_page_count(psize, 1);
349 	}
350 
351 	print_mapping(start, addr, mapping_size, exec);
352 	return 0;
353 }
354 
radix_init_pgtable(void)355 static void __init radix_init_pgtable(void)
356 {
357 	unsigned long rts_field;
358 	phys_addr_t start, end;
359 	u64 i;
360 
361 	/* We don't support slb for radix */
362 	mmu_slb_size = 0;
363 
364 	/*
365 	 * Create the linear mapping
366 	 */
367 	for_each_mem_range(i, &start, &end) {
368 		/*
369 		 * The memblock allocator  is up at this point, so the
370 		 * page tables will be allocated within the range. No
371 		 * need or a node (which we don't have yet).
372 		 */
373 
374 		if (end >= RADIX_VMALLOC_START) {
375 			pr_warn("Outside the supported range\n");
376 			continue;
377 		}
378 
379 		WARN_ON(create_physical_mapping(start, end,
380 						radix_mem_block_size,
381 						-1, PAGE_KERNEL));
382 	}
383 
384 	/* Find out how many PID bits are supported */
385 	if (!cpu_has_feature(CPU_FTR_HVMODE) &&
386 			cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) {
387 		/*
388 		 * Older versions of KVM on these machines perfer if the
389 		 * guest only uses the low 19 PID bits.
390 		 */
391 		if (!mmu_pid_bits)
392 			mmu_pid_bits = 19;
393 	} else {
394 		if (!mmu_pid_bits)
395 			mmu_pid_bits = 20;
396 	}
397 	mmu_base_pid = 1;
398 
399 	/*
400 	 * Allocate Partition table and process table for the
401 	 * host.
402 	 */
403 	BUG_ON(PRTB_SIZE_SHIFT > 36);
404 	process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT, -1, 0, 0);
405 	/*
406 	 * Fill in the process table.
407 	 */
408 	rts_field = radix__get_tree_size();
409 	process_tb->prtb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE);
410 
411 	/*
412 	 * The init_mm context is given the first available (non-zero) PID,
413 	 * which is the "guard PID" and contains no page table. PIDR should
414 	 * never be set to zero because that duplicates the kernel address
415 	 * space at the 0x0... offset (quadrant 0)!
416 	 *
417 	 * An arbitrary PID that may later be allocated by the PID allocator
418 	 * for userspace processes must not be used either, because that
419 	 * would cause stale user mappings for that PID on CPUs outside of
420 	 * the TLB invalidation scheme (because it won't be in mm_cpumask).
421 	 *
422 	 * So permanently carve out one PID for the purpose of a guard PID.
423 	 */
424 	init_mm.context.id = mmu_base_pid;
425 	mmu_base_pid++;
426 }
427 
radix_init_partition_table(void)428 static void __init radix_init_partition_table(void)
429 {
430 	unsigned long rts_field, dw0, dw1;
431 
432 	mmu_partition_table_init();
433 	rts_field = radix__get_tree_size();
434 	dw0 = rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE | PATB_HR;
435 	dw1 = __pa(process_tb) | (PRTB_SIZE_SHIFT - 12) | PATB_GR;
436 	mmu_partition_table_set_entry(0, dw0, dw1, false);
437 
438 	pr_info("Initializing Radix MMU\n");
439 }
440 
get_idx_from_shift(unsigned int shift)441 static int __init get_idx_from_shift(unsigned int shift)
442 {
443 	int idx = -1;
444 
445 	switch (shift) {
446 	case 0xc:
447 		idx = MMU_PAGE_4K;
448 		break;
449 	case 0x10:
450 		idx = MMU_PAGE_64K;
451 		break;
452 	case 0x15:
453 		idx = MMU_PAGE_2M;
454 		break;
455 	case 0x1e:
456 		idx = MMU_PAGE_1G;
457 		break;
458 	}
459 	return idx;
460 }
461 
radix_dt_scan_page_sizes(unsigned long node,const char * uname,int depth,void * data)462 static int __init radix_dt_scan_page_sizes(unsigned long node,
463 					   const char *uname, int depth,
464 					   void *data)
465 {
466 	int size = 0;
467 	int shift, idx;
468 	unsigned int ap;
469 	const __be32 *prop;
470 	const char *type = of_get_flat_dt_prop(node, "device_type", NULL);
471 
472 	/* We are scanning "cpu" nodes only */
473 	if (type == NULL || strcmp(type, "cpu") != 0)
474 		return 0;
475 
476 	/* Find MMU PID size */
477 	prop = of_get_flat_dt_prop(node, "ibm,mmu-pid-bits", &size);
478 	if (prop && size == 4)
479 		mmu_pid_bits = be32_to_cpup(prop);
480 
481 	/* Grab page size encodings */
482 	prop = of_get_flat_dt_prop(node, "ibm,processor-radix-AP-encodings", &size);
483 	if (!prop)
484 		return 0;
485 
486 	pr_info("Page sizes from device-tree:\n");
487 	for (; size >= 4; size -= 4, ++prop) {
488 
489 		struct mmu_psize_def *def;
490 
491 		/* top 3 bit is AP encoding */
492 		shift = be32_to_cpu(prop[0]) & ~(0xe << 28);
493 		ap = be32_to_cpu(prop[0]) >> 29;
494 		pr_info("Page size shift = %d AP=0x%x\n", shift, ap);
495 
496 		idx = get_idx_from_shift(shift);
497 		if (idx < 0)
498 			continue;
499 
500 		def = &mmu_psize_defs[idx];
501 		def->shift = shift;
502 		def->ap  = ap;
503 		def->h_rpt_pgsize = psize_to_rpti_pgsize(idx);
504 	}
505 
506 	/* needed ? */
507 	cur_cpu_spec->mmu_features &= ~MMU_FTR_NO_SLBIE_B;
508 	return 1;
509 }
510 
511 #ifdef CONFIG_MEMORY_HOTPLUG
probe_memory_block_size(unsigned long node,const char * uname,int depth,void * data)512 static int __init probe_memory_block_size(unsigned long node, const char *uname, int
513 					  depth, void *data)
514 {
515 	unsigned long *mem_block_size = (unsigned long *)data;
516 	const __be32 *prop;
517 	int len;
518 
519 	if (depth != 1)
520 		return 0;
521 
522 	if (strcmp(uname, "ibm,dynamic-reconfiguration-memory"))
523 		return 0;
524 
525 	prop = of_get_flat_dt_prop(node, "ibm,lmb-size", &len);
526 
527 	if (!prop || len < dt_root_size_cells * sizeof(__be32))
528 		/*
529 		 * Nothing in the device tree
530 		 */
531 		*mem_block_size = MIN_MEMORY_BLOCK_SIZE;
532 	else
533 		*mem_block_size = of_read_number(prop, dt_root_size_cells);
534 	return 1;
535 }
536 
radix_memory_block_size(void)537 static unsigned long radix_memory_block_size(void)
538 {
539 	unsigned long mem_block_size = MIN_MEMORY_BLOCK_SIZE;
540 
541 	/*
542 	 * OPAL firmware feature is set by now. Hence we are ok
543 	 * to test OPAL feature.
544 	 */
545 	if (firmware_has_feature(FW_FEATURE_OPAL))
546 		mem_block_size = 1UL * 1024 * 1024 * 1024;
547 	else
548 		of_scan_flat_dt(probe_memory_block_size, &mem_block_size);
549 
550 	return mem_block_size;
551 }
552 
553 #else   /* CONFIG_MEMORY_HOTPLUG */
554 
radix_memory_block_size(void)555 static unsigned long radix_memory_block_size(void)
556 {
557 	return 1UL * 1024 * 1024 * 1024;
558 }
559 
560 #endif /* CONFIG_MEMORY_HOTPLUG */
561 
562 
radix__early_init_devtree(void)563 void __init radix__early_init_devtree(void)
564 {
565 	int rc;
566 
567 	/*
568 	 * Try to find the available page sizes in the device-tree
569 	 */
570 	rc = of_scan_flat_dt(radix_dt_scan_page_sizes, NULL);
571 	if (!rc) {
572 		/*
573 		 * No page size details found in device tree.
574 		 * Let's assume we have page 4k and 64k support
575 		 */
576 		mmu_psize_defs[MMU_PAGE_4K].shift = 12;
577 		mmu_psize_defs[MMU_PAGE_4K].ap = 0x0;
578 		mmu_psize_defs[MMU_PAGE_4K].h_rpt_pgsize =
579 			psize_to_rpti_pgsize(MMU_PAGE_4K);
580 
581 		mmu_psize_defs[MMU_PAGE_64K].shift = 16;
582 		mmu_psize_defs[MMU_PAGE_64K].ap = 0x5;
583 		mmu_psize_defs[MMU_PAGE_64K].h_rpt_pgsize =
584 			psize_to_rpti_pgsize(MMU_PAGE_64K);
585 	}
586 
587 	/*
588 	 * Max mapping size used when mapping pages. We don't use
589 	 * ppc_md.memory_block_size() here because this get called
590 	 * early and we don't have machine probe called yet. Also
591 	 * the pseries implementation only check for ibm,lmb-size.
592 	 * All hypervisor supporting radix do expose that device
593 	 * tree node.
594 	 */
595 	radix_mem_block_size = radix_memory_block_size();
596 	return;
597 }
598 
radix_init_amor(void)599 static void radix_init_amor(void)
600 {
601 	/*
602 	* In HV mode, we init AMOR (Authority Mask Override Register) so that
603 	* the hypervisor and guest can setup IAMR (Instruction Authority Mask
604 	* Register), enable key 0 and set it to 1.
605 	*
606 	* AMOR = 0b1100 .... 0000 (Mask for key 0 is 11)
607 	*/
608 	mtspr(SPRN_AMOR, (3ul << 62));
609 }
610 
radix__early_init_mmu(void)611 void __init radix__early_init_mmu(void)
612 {
613 	unsigned long lpcr;
614 
615 #ifdef CONFIG_PPC_64K_PAGES
616 	/* PAGE_SIZE mappings */
617 	mmu_virtual_psize = MMU_PAGE_64K;
618 #else
619 	mmu_virtual_psize = MMU_PAGE_4K;
620 #endif
621 
622 #ifdef CONFIG_SPARSEMEM_VMEMMAP
623 	/* vmemmap mapping */
624 	if (mmu_psize_defs[MMU_PAGE_2M].shift) {
625 		/*
626 		 * map vmemmap using 2M if available
627 		 */
628 		mmu_vmemmap_psize = MMU_PAGE_2M;
629 	} else
630 		mmu_vmemmap_psize = mmu_virtual_psize;
631 #endif
632 	/*
633 	 * initialize page table size
634 	 */
635 	__pte_index_size = RADIX_PTE_INDEX_SIZE;
636 	__pmd_index_size = RADIX_PMD_INDEX_SIZE;
637 	__pud_index_size = RADIX_PUD_INDEX_SIZE;
638 	__pgd_index_size = RADIX_PGD_INDEX_SIZE;
639 	__pud_cache_index = RADIX_PUD_INDEX_SIZE;
640 	__pte_table_size = RADIX_PTE_TABLE_SIZE;
641 	__pmd_table_size = RADIX_PMD_TABLE_SIZE;
642 	__pud_table_size = RADIX_PUD_TABLE_SIZE;
643 	__pgd_table_size = RADIX_PGD_TABLE_SIZE;
644 
645 	__pmd_val_bits = RADIX_PMD_VAL_BITS;
646 	__pud_val_bits = RADIX_PUD_VAL_BITS;
647 	__pgd_val_bits = RADIX_PGD_VAL_BITS;
648 
649 	__kernel_virt_start = RADIX_KERN_VIRT_START;
650 	__vmalloc_start = RADIX_VMALLOC_START;
651 	__vmalloc_end = RADIX_VMALLOC_END;
652 	__kernel_io_start = RADIX_KERN_IO_START;
653 	__kernel_io_end = RADIX_KERN_IO_END;
654 	vmemmap = (struct page *)RADIX_VMEMMAP_START;
655 	ioremap_bot = IOREMAP_BASE;
656 
657 #ifdef CONFIG_PCI
658 	pci_io_base = ISA_IO_BASE;
659 #endif
660 	__pte_frag_nr = RADIX_PTE_FRAG_NR;
661 	__pte_frag_size_shift = RADIX_PTE_FRAG_SIZE_SHIFT;
662 	__pmd_frag_nr = RADIX_PMD_FRAG_NR;
663 	__pmd_frag_size_shift = RADIX_PMD_FRAG_SIZE_SHIFT;
664 
665 	radix_init_pgtable();
666 
667 	if (!firmware_has_feature(FW_FEATURE_LPAR)) {
668 		lpcr = mfspr(SPRN_LPCR);
669 		mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
670 		radix_init_partition_table();
671 		radix_init_amor();
672 	} else {
673 		radix_init_pseries();
674 	}
675 
676 	memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
677 
678 	/* Switch to the guard PID before turning on MMU */
679 	radix__switch_mmu_context(NULL, &init_mm);
680 	tlbiel_all();
681 }
682 
radix__early_init_mmu_secondary(void)683 void radix__early_init_mmu_secondary(void)
684 {
685 	unsigned long lpcr;
686 	/*
687 	 * update partition table control register and UPRT
688 	 */
689 	if (!firmware_has_feature(FW_FEATURE_LPAR)) {
690 		lpcr = mfspr(SPRN_LPCR);
691 		mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
692 
693 		set_ptcr_when_no_uv(__pa(partition_tb) |
694 				    (PATB_SIZE_SHIFT - 12));
695 
696 		radix_init_amor();
697 	}
698 
699 	radix__switch_mmu_context(NULL, &init_mm);
700 	tlbiel_all();
701 
702 	/* Make sure userspace can't change the AMR */
703 	mtspr(SPRN_UAMOR, 0);
704 }
705 
706 /* Called during kexec sequence with MMU off */
radix__mmu_cleanup_all(void)707 notrace void radix__mmu_cleanup_all(void)
708 {
709 	unsigned long lpcr;
710 
711 	if (!firmware_has_feature(FW_FEATURE_LPAR)) {
712 		lpcr = mfspr(SPRN_LPCR);
713 		mtspr(SPRN_LPCR, lpcr & ~LPCR_UPRT);
714 		set_ptcr_when_no_uv(0);
715 		powernv_set_nmmu_ptcr(0);
716 		radix__flush_tlb_all();
717 	}
718 }
719 
720 #ifdef CONFIG_MEMORY_HOTPLUG
free_pte_table(pte_t * pte_start,pmd_t * pmd)721 static void free_pte_table(pte_t *pte_start, pmd_t *pmd)
722 {
723 	pte_t *pte;
724 	int i;
725 
726 	for (i = 0; i < PTRS_PER_PTE; i++) {
727 		pte = pte_start + i;
728 		if (!pte_none(*pte))
729 			return;
730 	}
731 
732 	pte_free_kernel(&init_mm, pte_start);
733 	pmd_clear(pmd);
734 }
735 
free_pmd_table(pmd_t * pmd_start,pud_t * pud)736 static void free_pmd_table(pmd_t *pmd_start, pud_t *pud)
737 {
738 	pmd_t *pmd;
739 	int i;
740 
741 	for (i = 0; i < PTRS_PER_PMD; i++) {
742 		pmd = pmd_start + i;
743 		if (!pmd_none(*pmd))
744 			return;
745 	}
746 
747 	pmd_free(&init_mm, pmd_start);
748 	pud_clear(pud);
749 }
750 
free_pud_table(pud_t * pud_start,p4d_t * p4d)751 static void free_pud_table(pud_t *pud_start, p4d_t *p4d)
752 {
753 	pud_t *pud;
754 	int i;
755 
756 	for (i = 0; i < PTRS_PER_PUD; i++) {
757 		pud = pud_start + i;
758 		if (!pud_none(*pud))
759 			return;
760 	}
761 
762 	pud_free(&init_mm, pud_start);
763 	p4d_clear(p4d);
764 }
765 
remove_pte_table(pte_t * pte_start,unsigned long addr,unsigned long end,bool direct)766 static void remove_pte_table(pte_t *pte_start, unsigned long addr,
767 			     unsigned long end, bool direct)
768 {
769 	unsigned long next, pages = 0;
770 	pte_t *pte;
771 
772 	pte = pte_start + pte_index(addr);
773 	for (; addr < end; addr = next, pte++) {
774 		next = (addr + PAGE_SIZE) & PAGE_MASK;
775 		if (next > end)
776 			next = end;
777 
778 		if (!pte_present(*pte))
779 			continue;
780 
781 		if (!PAGE_ALIGNED(addr) || !PAGE_ALIGNED(next)) {
782 			/*
783 			 * The vmemmap_free() and remove_section_mapping()
784 			 * codepaths call us with aligned addresses.
785 			 */
786 			WARN_ONCE(1, "%s: unaligned range\n", __func__);
787 			continue;
788 		}
789 
790 		pte_clear(&init_mm, addr, pte);
791 		pages++;
792 	}
793 	if (direct)
794 		update_page_count(mmu_virtual_psize, -pages);
795 }
796 
remove_pmd_table(pmd_t * pmd_start,unsigned long addr,unsigned long end,bool direct)797 static void __meminit remove_pmd_table(pmd_t *pmd_start, unsigned long addr,
798 				       unsigned long end, bool direct)
799 {
800 	unsigned long next, pages = 0;
801 	pte_t *pte_base;
802 	pmd_t *pmd;
803 
804 	pmd = pmd_start + pmd_index(addr);
805 	for (; addr < end; addr = next, pmd++) {
806 		next = pmd_addr_end(addr, end);
807 
808 		if (!pmd_present(*pmd))
809 			continue;
810 
811 		if (pmd_is_leaf(*pmd)) {
812 			if (!IS_ALIGNED(addr, PMD_SIZE) ||
813 			    !IS_ALIGNED(next, PMD_SIZE)) {
814 				WARN_ONCE(1, "%s: unaligned range\n", __func__);
815 				continue;
816 			}
817 			pte_clear(&init_mm, addr, (pte_t *)pmd);
818 			pages++;
819 			continue;
820 		}
821 
822 		pte_base = (pte_t *)pmd_page_vaddr(*pmd);
823 		remove_pte_table(pte_base, addr, next, direct);
824 		free_pte_table(pte_base, pmd);
825 	}
826 	if (direct)
827 		update_page_count(MMU_PAGE_2M, -pages);
828 }
829 
remove_pud_table(pud_t * pud_start,unsigned long addr,unsigned long end,bool direct)830 static void __meminit remove_pud_table(pud_t *pud_start, unsigned long addr,
831 				       unsigned long end, bool direct)
832 {
833 	unsigned long next, pages = 0;
834 	pmd_t *pmd_base;
835 	pud_t *pud;
836 
837 	pud = pud_start + pud_index(addr);
838 	for (; addr < end; addr = next, pud++) {
839 		next = pud_addr_end(addr, end);
840 
841 		if (!pud_present(*pud))
842 			continue;
843 
844 		if (pud_is_leaf(*pud)) {
845 			if (!IS_ALIGNED(addr, PUD_SIZE) ||
846 			    !IS_ALIGNED(next, PUD_SIZE)) {
847 				WARN_ONCE(1, "%s: unaligned range\n", __func__);
848 				continue;
849 			}
850 			pte_clear(&init_mm, addr, (pte_t *)pud);
851 			pages++;
852 			continue;
853 		}
854 
855 		pmd_base = pud_pgtable(*pud);
856 		remove_pmd_table(pmd_base, addr, next, direct);
857 		free_pmd_table(pmd_base, pud);
858 	}
859 	if (direct)
860 		update_page_count(MMU_PAGE_1G, -pages);
861 }
862 
remove_pagetable(unsigned long start,unsigned long end,bool direct)863 static void __meminit remove_pagetable(unsigned long start, unsigned long end,
864 				       bool direct)
865 {
866 	unsigned long addr, next;
867 	pud_t *pud_base;
868 	pgd_t *pgd;
869 	p4d_t *p4d;
870 
871 	spin_lock(&init_mm.page_table_lock);
872 
873 	for (addr = start; addr < end; addr = next) {
874 		next = pgd_addr_end(addr, end);
875 
876 		pgd = pgd_offset_k(addr);
877 		p4d = p4d_offset(pgd, addr);
878 		if (!p4d_present(*p4d))
879 			continue;
880 
881 		if (p4d_is_leaf(*p4d)) {
882 			if (!IS_ALIGNED(addr, P4D_SIZE) ||
883 			    !IS_ALIGNED(next, P4D_SIZE)) {
884 				WARN_ONCE(1, "%s: unaligned range\n", __func__);
885 				continue;
886 			}
887 
888 			pte_clear(&init_mm, addr, (pte_t *)pgd);
889 			continue;
890 		}
891 
892 		pud_base = p4d_pgtable(*p4d);
893 		remove_pud_table(pud_base, addr, next, direct);
894 		free_pud_table(pud_base, p4d);
895 	}
896 
897 	spin_unlock(&init_mm.page_table_lock);
898 	radix__flush_tlb_kernel_range(start, end);
899 }
900 
radix__create_section_mapping(unsigned long start,unsigned long end,int nid,pgprot_t prot)901 int __meminit radix__create_section_mapping(unsigned long start,
902 					    unsigned long end, int nid,
903 					    pgprot_t prot)
904 {
905 	if (end >= RADIX_VMALLOC_START) {
906 		pr_warn("Outside the supported range\n");
907 		return -1;
908 	}
909 
910 	return create_physical_mapping(__pa(start), __pa(end),
911 				       radix_mem_block_size, nid, prot);
912 }
913 
radix__remove_section_mapping(unsigned long start,unsigned long end)914 int __meminit radix__remove_section_mapping(unsigned long start, unsigned long end)
915 {
916 	remove_pagetable(start, end, true);
917 	return 0;
918 }
919 #endif /* CONFIG_MEMORY_HOTPLUG */
920 
921 #ifdef CONFIG_SPARSEMEM_VMEMMAP
__map_kernel_page_nid(unsigned long ea,unsigned long pa,pgprot_t flags,unsigned int map_page_size,int nid)922 static int __map_kernel_page_nid(unsigned long ea, unsigned long pa,
923 				 pgprot_t flags, unsigned int map_page_size,
924 				 int nid)
925 {
926 	return __map_kernel_page(ea, pa, flags, map_page_size, nid, 0, 0);
927 }
928 
radix__vmemmap_create_mapping(unsigned long start,unsigned long page_size,unsigned long phys)929 int __meminit radix__vmemmap_create_mapping(unsigned long start,
930 				      unsigned long page_size,
931 				      unsigned long phys)
932 {
933 	/* Create a PTE encoding */
934 	unsigned long flags = _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_KERNEL_RW;
935 	int nid = early_pfn_to_nid(phys >> PAGE_SHIFT);
936 	int ret;
937 
938 	if ((start + page_size) >= RADIX_VMEMMAP_END) {
939 		pr_warn("Outside the supported range\n");
940 		return -1;
941 	}
942 
943 	ret = __map_kernel_page_nid(start, phys, __pgprot(flags), page_size, nid);
944 	BUG_ON(ret);
945 
946 	return 0;
947 }
948 
949 #ifdef CONFIG_MEMORY_HOTPLUG
radix__vmemmap_remove_mapping(unsigned long start,unsigned long page_size)950 void __meminit radix__vmemmap_remove_mapping(unsigned long start, unsigned long page_size)
951 {
952 	remove_pagetable(start, start + page_size, false);
953 }
954 #endif
955 #endif
956 
957 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
958 
radix__pmd_hugepage_update(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp,unsigned long clr,unsigned long set)959 unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
960 				  pmd_t *pmdp, unsigned long clr,
961 				  unsigned long set)
962 {
963 	unsigned long old;
964 
965 #ifdef CONFIG_DEBUG_VM
966 	WARN_ON(!radix__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp));
967 	assert_spin_locked(pmd_lockptr(mm, pmdp));
968 #endif
969 
970 	old = radix__pte_update(mm, addr, (pte_t *)pmdp, clr, set, 1);
971 	trace_hugepage_update(addr, old, clr, set);
972 
973 	return old;
974 }
975 
radix__pmdp_collapse_flush(struct vm_area_struct * vma,unsigned long address,pmd_t * pmdp)976 pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
977 			pmd_t *pmdp)
978 
979 {
980 	pmd_t pmd;
981 
982 	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
983 	VM_BUG_ON(radix__pmd_trans_huge(*pmdp));
984 	VM_BUG_ON(pmd_devmap(*pmdp));
985 	/*
986 	 * khugepaged calls this for normal pmd
987 	 */
988 	pmd = *pmdp;
989 	pmd_clear(pmdp);
990 
991 	radix__flush_tlb_collapsed_pmd(vma->vm_mm, address);
992 
993 	return pmd;
994 }
995 
996 /*
997  * For us pgtable_t is pte_t *. Inorder to save the deposisted
998  * page table, we consider the allocated page table as a list
999  * head. On withdraw we need to make sure we zero out the used
1000  * list_head memory area.
1001  */
radix__pgtable_trans_huge_deposit(struct mm_struct * mm,pmd_t * pmdp,pgtable_t pgtable)1002 void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
1003 				 pgtable_t pgtable)
1004 {
1005 	struct list_head *lh = (struct list_head *) pgtable;
1006 
1007 	assert_spin_locked(pmd_lockptr(mm, pmdp));
1008 
1009 	/* FIFO */
1010 	if (!pmd_huge_pte(mm, pmdp))
1011 		INIT_LIST_HEAD(lh);
1012 	else
1013 		list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
1014 	pmd_huge_pte(mm, pmdp) = pgtable;
1015 }
1016 
radix__pgtable_trans_huge_withdraw(struct mm_struct * mm,pmd_t * pmdp)1017 pgtable_t radix__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
1018 {
1019 	pte_t *ptep;
1020 	pgtable_t pgtable;
1021 	struct list_head *lh;
1022 
1023 	assert_spin_locked(pmd_lockptr(mm, pmdp));
1024 
1025 	/* FIFO */
1026 	pgtable = pmd_huge_pte(mm, pmdp);
1027 	lh = (struct list_head *) pgtable;
1028 	if (list_empty(lh))
1029 		pmd_huge_pte(mm, pmdp) = NULL;
1030 	else {
1031 		pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
1032 		list_del(lh);
1033 	}
1034 	ptep = (pte_t *) pgtable;
1035 	*ptep = __pte(0);
1036 	ptep++;
1037 	*ptep = __pte(0);
1038 	return pgtable;
1039 }
1040 
radix__pmdp_huge_get_and_clear(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp)1041 pmd_t radix__pmdp_huge_get_and_clear(struct mm_struct *mm,
1042 				     unsigned long addr, pmd_t *pmdp)
1043 {
1044 	pmd_t old_pmd;
1045 	unsigned long old;
1046 
1047 	old = radix__pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
1048 	old_pmd = __pmd(old);
1049 	return old_pmd;
1050 }
1051 
1052 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1053 
radix__ptep_set_access_flags(struct vm_area_struct * vma,pte_t * ptep,pte_t entry,unsigned long address,int psize)1054 void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep,
1055 				  pte_t entry, unsigned long address, int psize)
1056 {
1057 	struct mm_struct *mm = vma->vm_mm;
1058 	unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_SOFT_DIRTY |
1059 					      _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC);
1060 
1061 	unsigned long change = pte_val(entry) ^ pte_val(*ptep);
1062 	/*
1063 	 * To avoid NMMU hang while relaxing access, we need mark
1064 	 * the pte invalid in between.
1065 	 */
1066 	if ((change & _PAGE_RW) && atomic_read(&mm->context.copros) > 0) {
1067 		unsigned long old_pte, new_pte;
1068 
1069 		old_pte = __radix_pte_update(ptep, _PAGE_PRESENT, _PAGE_INVALID);
1070 		/*
1071 		 * new value of pte
1072 		 */
1073 		new_pte = old_pte | set;
1074 		radix__flush_tlb_page_psize(mm, address, psize);
1075 		__radix_pte_update(ptep, _PAGE_INVALID, new_pte);
1076 	} else {
1077 		__radix_pte_update(ptep, 0, set);
1078 		/*
1079 		 * Book3S does not require a TLB flush when relaxing access
1080 		 * restrictions when the address space is not attached to a
1081 		 * NMMU, because the core MMU will reload the pte after taking
1082 		 * an access fault, which is defined by the architecture.
1083 		 */
1084 	}
1085 	/* See ptesync comment in radix__set_pte_at */
1086 }
1087 
radix__ptep_modify_prot_commit(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep,pte_t old_pte,pte_t pte)1088 void radix__ptep_modify_prot_commit(struct vm_area_struct *vma,
1089 				    unsigned long addr, pte_t *ptep,
1090 				    pte_t old_pte, pte_t pte)
1091 {
1092 	struct mm_struct *mm = vma->vm_mm;
1093 
1094 	/*
1095 	 * To avoid NMMU hang while relaxing access we need to flush the tlb before
1096 	 * we set the new value. We need to do this only for radix, because hash
1097 	 * translation does flush when updating the linux pte.
1098 	 */
1099 	if (is_pte_rw_upgrade(pte_val(old_pte), pte_val(pte)) &&
1100 	    (atomic_read(&mm->context.copros) > 0))
1101 		radix__flush_tlb_page(vma, addr);
1102 
1103 	set_pte_at(mm, addr, ptep, pte);
1104 }
1105 
pud_set_huge(pud_t * pud,phys_addr_t addr,pgprot_t prot)1106 int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
1107 {
1108 	pte_t *ptep = (pte_t *)pud;
1109 	pte_t new_pud = pfn_pte(__phys_to_pfn(addr), prot);
1110 
1111 	if (!radix_enabled())
1112 		return 0;
1113 
1114 	set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pud);
1115 
1116 	return 1;
1117 }
1118 
pud_clear_huge(pud_t * pud)1119 int pud_clear_huge(pud_t *pud)
1120 {
1121 	if (pud_is_leaf(*pud)) {
1122 		pud_clear(pud);
1123 		return 1;
1124 	}
1125 
1126 	return 0;
1127 }
1128 
pud_free_pmd_page(pud_t * pud,unsigned long addr)1129 int pud_free_pmd_page(pud_t *pud, unsigned long addr)
1130 {
1131 	pmd_t *pmd;
1132 	int i;
1133 
1134 	pmd = pud_pgtable(*pud);
1135 	pud_clear(pud);
1136 
1137 	flush_tlb_kernel_range(addr, addr + PUD_SIZE);
1138 
1139 	for (i = 0; i < PTRS_PER_PMD; i++) {
1140 		if (!pmd_none(pmd[i])) {
1141 			pte_t *pte;
1142 			pte = (pte_t *)pmd_page_vaddr(pmd[i]);
1143 
1144 			pte_free_kernel(&init_mm, pte);
1145 		}
1146 	}
1147 
1148 	pmd_free(&init_mm, pmd);
1149 
1150 	return 1;
1151 }
1152 
pmd_set_huge(pmd_t * pmd,phys_addr_t addr,pgprot_t prot)1153 int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
1154 {
1155 	pte_t *ptep = (pte_t *)pmd;
1156 	pte_t new_pmd = pfn_pte(__phys_to_pfn(addr), prot);
1157 
1158 	if (!radix_enabled())
1159 		return 0;
1160 
1161 	set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pmd);
1162 
1163 	return 1;
1164 }
1165 
pmd_clear_huge(pmd_t * pmd)1166 int pmd_clear_huge(pmd_t *pmd)
1167 {
1168 	if (pmd_is_leaf(*pmd)) {
1169 		pmd_clear(pmd);
1170 		return 1;
1171 	}
1172 
1173 	return 0;
1174 }
1175 
pmd_free_pte_page(pmd_t * pmd,unsigned long addr)1176 int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
1177 {
1178 	pte_t *pte;
1179 
1180 	pte = (pte_t *)pmd_page_vaddr(*pmd);
1181 	pmd_clear(pmd);
1182 
1183 	flush_tlb_kernel_range(addr, addr + PMD_SIZE);
1184 
1185 	pte_free_kernel(&init_mm, pte);
1186 
1187 	return 1;
1188 }
1189