1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2005,2006,2007,2008,2009,2010 Imagination Technologies
4 *
5 */
6
7 #include <linux/export.h>
8 #include <linux/mm.h>
9 #include <linux/swap.h>
10 #include <linux/init.h>
11 #include <linux/bootmem.h>
12 #include <linux/pagemap.h>
13 #include <linux/percpu.h>
14 #include <linux/memblock.h>
15 #include <linux/initrd.h>
16 #include <linux/sched/task.h>
17
18 #include <asm/setup.h>
19 #include <asm/page.h>
20 #include <asm/pgalloc.h>
21 #include <asm/mmu.h>
22 #include <asm/mmu_context.h>
23 #include <asm/sections.h>
24 #include <asm/tlb.h>
25 #include <asm/user_gateway.h>
26 #include <asm/mmzone.h>
27 #include <asm/fixmap.h>
28
29 unsigned long pfn_base;
30 EXPORT_SYMBOL(pfn_base);
31
32 pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_data;
33
34 unsigned long empty_zero_page;
35 EXPORT_SYMBOL(empty_zero_page);
36
37 extern char __user_gateway_start;
38 extern char __user_gateway_end;
39
40 void *gateway_page;
41
42 /*
43 * Insert the gateway page into a set of page tables, creating the
44 * page tables if necessary.
45 */
insert_gateway_page(pgd_t * pgd,unsigned long address)46 static void insert_gateway_page(pgd_t *pgd, unsigned long address)
47 {
48 pud_t *pud;
49 pmd_t *pmd;
50 pte_t *pte;
51
52 BUG_ON(!pgd_present(*pgd));
53
54 pud = pud_offset(pgd, address);
55 BUG_ON(!pud_present(*pud));
56
57 pmd = pmd_offset(pud, address);
58 if (!pmd_present(*pmd)) {
59 pte = alloc_bootmem_pages(PAGE_SIZE);
60 set_pmd(pmd, __pmd(_PAGE_TABLE | __pa(pte)));
61 }
62
63 pte = pte_offset_kernel(pmd, address);
64 set_pte(pte, pfn_pte(__pa(gateway_page) >> PAGE_SHIFT, PAGE_READONLY));
65 }
66
67 /* Alloc and map a page in a known location accessible to userspace. */
user_gateway_init(void)68 static void __init user_gateway_init(void)
69 {
70 unsigned long address = USER_GATEWAY_PAGE;
71 int offset = pgd_index(address);
72 pgd_t *pgd;
73
74 gateway_page = alloc_bootmem_pages(PAGE_SIZE);
75
76 pgd = swapper_pg_dir + offset;
77 insert_gateway_page(pgd, address);
78
79 #ifdef CONFIG_METAG_META12
80 /*
81 * Insert the gateway page into our current page tables even
82 * though we've already inserted it into our reference page
83 * table (swapper_pg_dir). This is because with a META1 mmu we
84 * copy just the user address range and not the gateway page
85 * entry on context switch, see switch_mmu().
86 */
87 pgd = (pgd_t *)mmu_get_base() + offset;
88 insert_gateway_page(pgd, address);
89 #endif /* CONFIG_METAG_META12 */
90
91 BUG_ON((&__user_gateway_end - &__user_gateway_start) > PAGE_SIZE);
92
93 gateway_page += (address & ~PAGE_MASK);
94
95 memcpy(gateway_page, &__user_gateway_start,
96 &__user_gateway_end - &__user_gateway_start);
97
98 /*
99 * We don't need to flush the TLB here, there should be no mapping
100 * present at boot for this address and only valid mappings are in
101 * the TLB (apart from on Meta 1.x, but those cached invalid
102 * mappings should be impossible to hit here).
103 *
104 * We don't flush the code cache here even though we have written
105 * code through the data cache and they may not be coherent. At
106 * this point we assume there is no stale data in the code cache
107 * for this address so there is no need to flush.
108 */
109 }
110
allocate_pgdat(unsigned int nid)111 static void __init allocate_pgdat(unsigned int nid)
112 {
113 unsigned long start_pfn, end_pfn;
114 #ifdef CONFIG_NEED_MULTIPLE_NODES
115 unsigned long phys;
116 #endif
117
118 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
119
120 #ifdef CONFIG_NEED_MULTIPLE_NODES
121 phys = __memblock_alloc_base(sizeof(struct pglist_data),
122 SMP_CACHE_BYTES, end_pfn << PAGE_SHIFT);
123 /* Retry with all of system memory */
124 if (!phys)
125 phys = __memblock_alloc_base(sizeof(struct pglist_data),
126 SMP_CACHE_BYTES,
127 memblock_end_of_DRAM());
128 if (!phys)
129 panic("Can't allocate pgdat for node %d\n", nid);
130
131 NODE_DATA(nid) = __va(phys);
132 memset(NODE_DATA(nid), 0, sizeof(struct pglist_data));
133
134 NODE_DATA(nid)->bdata = &bootmem_node_data[nid];
135 #endif
136
137 NODE_DATA(nid)->node_start_pfn = start_pfn;
138 NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn;
139 }
140
bootmem_init_one_node(unsigned int nid)141 static void __init bootmem_init_one_node(unsigned int nid)
142 {
143 unsigned long total_pages, paddr;
144 unsigned long end_pfn;
145 struct pglist_data *p;
146
147 p = NODE_DATA(nid);
148
149 /* Nothing to do.. */
150 if (!p->node_spanned_pages)
151 return;
152
153 end_pfn = pgdat_end_pfn(p);
154 #ifdef CONFIG_HIGHMEM
155 if (end_pfn > max_low_pfn)
156 end_pfn = max_low_pfn;
157 #endif
158
159 total_pages = bootmem_bootmap_pages(end_pfn - p->node_start_pfn);
160
161 paddr = memblock_alloc(total_pages << PAGE_SHIFT, PAGE_SIZE);
162 if (!paddr)
163 panic("Can't allocate bootmap for nid[%d]\n", nid);
164
165 init_bootmem_node(p, paddr >> PAGE_SHIFT, p->node_start_pfn, end_pfn);
166
167 free_bootmem_with_active_regions(nid, end_pfn);
168
169 /*
170 * XXX Handle initial reservations for the system memory node
171 * only for the moment, we'll refactor this later for handling
172 * reservations in other nodes.
173 */
174 if (nid == 0) {
175 struct memblock_region *reg;
176
177 /* Reserve the sections we're already using. */
178 for_each_memblock(reserved, reg) {
179 unsigned long size = reg->size;
180
181 #ifdef CONFIG_HIGHMEM
182 /* ...but not highmem */
183 if (PFN_DOWN(reg->base) >= highstart_pfn)
184 continue;
185
186 if (PFN_UP(reg->base + size) > highstart_pfn)
187 size = (highstart_pfn - PFN_DOWN(reg->base))
188 << PAGE_SHIFT;
189 #endif
190
191 reserve_bootmem(reg->base, size, BOOTMEM_DEFAULT);
192 }
193 }
194
195 sparse_memory_present_with_active_regions(nid);
196 }
197
do_init_bootmem(void)198 static void __init do_init_bootmem(void)
199 {
200 struct memblock_region *reg;
201 int i;
202
203 /* Add active regions with valid PFNs. */
204 for_each_memblock(memory, reg) {
205 unsigned long start_pfn, end_pfn;
206 start_pfn = memblock_region_memory_base_pfn(reg);
207 end_pfn = memblock_region_memory_end_pfn(reg);
208 memblock_set_node(PFN_PHYS(start_pfn),
209 PFN_PHYS(end_pfn - start_pfn),
210 &memblock.memory, 0);
211 }
212
213 /* All of system RAM sits in node 0 for the non-NUMA case */
214 allocate_pgdat(0);
215 node_set_online(0);
216
217 soc_mem_setup();
218
219 for_each_online_node(i)
220 bootmem_init_one_node(i);
221
222 sparse_init();
223 }
224
225 extern char _heap_start[];
226
init_and_reserve_mem(void)227 static void __init init_and_reserve_mem(void)
228 {
229 unsigned long start_pfn, heap_start;
230 u64 base = min_low_pfn << PAGE_SHIFT;
231 u64 size = (max_low_pfn << PAGE_SHIFT) - base;
232
233 heap_start = (unsigned long) &_heap_start;
234
235 memblock_add(base, size);
236
237 /*
238 * Partially used pages are not usable - thus
239 * we are rounding upwards:
240 */
241 start_pfn = PFN_UP(__pa(heap_start));
242
243 /*
244 * Reserve the kernel text.
245 */
246 memblock_reserve(base, (PFN_PHYS(start_pfn) + PAGE_SIZE - 1) - base);
247
248 #ifdef CONFIG_HIGHMEM
249 /*
250 * Add & reserve highmem, so page structures are initialised.
251 */
252 base = highstart_pfn << PAGE_SHIFT;
253 size = (highend_pfn << PAGE_SHIFT) - base;
254 if (size) {
255 memblock_add(base, size);
256 memblock_reserve(base, size);
257 }
258 #endif
259 }
260
261 #ifdef CONFIG_HIGHMEM
262 /*
263 * Ensure we have allocated page tables in swapper_pg_dir for the
264 * fixed mappings range from 'start' to 'end'.
265 */
allocate_pgtables(unsigned long start,unsigned long end)266 static void __init allocate_pgtables(unsigned long start, unsigned long end)
267 {
268 pgd_t *pgd;
269 pmd_t *pmd;
270 pte_t *pte;
271 int i, j;
272 unsigned long vaddr;
273
274 vaddr = start;
275 i = pgd_index(vaddr);
276 j = pmd_index(vaddr);
277 pgd = swapper_pg_dir + i;
278
279 for ( ; (i < PTRS_PER_PGD) && (vaddr != end); pgd++, i++) {
280 pmd = (pmd_t *)pgd;
281 for (; (j < PTRS_PER_PMD) && (vaddr != end); pmd++, j++) {
282 vaddr += PMD_SIZE;
283
284 if (!pmd_none(*pmd))
285 continue;
286
287 pte = (pte_t *)alloc_bootmem_low_pages(PAGE_SIZE);
288 pmd_populate_kernel(&init_mm, pmd, pte);
289 }
290 j = 0;
291 }
292 }
293
fixedrange_init(void)294 static void __init fixedrange_init(void)
295 {
296 unsigned long vaddr, end;
297 pgd_t *pgd;
298 pud_t *pud;
299 pmd_t *pmd;
300 pte_t *pte;
301
302 /*
303 * Fixed mappings:
304 */
305 vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
306 end = (FIXADDR_TOP + PMD_SIZE - 1) & PMD_MASK;
307 allocate_pgtables(vaddr, end);
308
309 /*
310 * Permanent kmaps:
311 */
312 vaddr = PKMAP_BASE;
313 allocate_pgtables(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP);
314
315 pgd = swapper_pg_dir + pgd_index(vaddr);
316 pud = pud_offset(pgd, vaddr);
317 pmd = pmd_offset(pud, vaddr);
318 pte = pte_offset_kernel(pmd, vaddr);
319 pkmap_page_table = pte;
320 }
321 #endif /* CONFIG_HIGHMEM */
322
323 /*
324 * paging_init() continues the virtual memory environment setup which
325 * was begun by the code in arch/metag/kernel/setup.c.
326 */
paging_init(unsigned long mem_end)327 void __init paging_init(unsigned long mem_end)
328 {
329 unsigned long max_zone_pfns[MAX_NR_ZONES];
330 int nid;
331
332 init_and_reserve_mem();
333
334 memblock_allow_resize();
335
336 memblock_dump_all();
337
338 nodes_clear(node_online_map);
339
340 init_new_context(&init_task, &init_mm);
341
342 memset(swapper_pg_dir, 0, sizeof(swapper_pg_dir));
343
344 do_init_bootmem();
345 mmu_init(mem_end);
346
347 #ifdef CONFIG_HIGHMEM
348 fixedrange_init();
349 kmap_init();
350 #endif
351
352 /* Initialize the zero page to a bootmem page, already zeroed. */
353 empty_zero_page = (unsigned long)alloc_bootmem_pages(PAGE_SIZE);
354
355 user_gateway_init();
356
357 memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
358
359 for_each_online_node(nid) {
360 pg_data_t *pgdat = NODE_DATA(nid);
361 unsigned long low, start_pfn;
362
363 start_pfn = pgdat->bdata->node_min_pfn;
364 low = pgdat->bdata->node_low_pfn;
365
366 if (max_zone_pfns[ZONE_NORMAL] < low)
367 max_zone_pfns[ZONE_NORMAL] = low;
368
369 #ifdef CONFIG_HIGHMEM
370 max_zone_pfns[ZONE_HIGHMEM] = highend_pfn;
371 #endif
372 pr_info("Node %u: start_pfn = 0x%lx, low = 0x%lx\n",
373 nid, start_pfn, low);
374 }
375
376 free_area_init_nodes(max_zone_pfns);
377 }
378
mem_init(void)379 void __init mem_init(void)
380 {
381 #ifdef CONFIG_HIGHMEM
382 unsigned long tmp;
383
384 /*
385 * Explicitly reset zone->managed_pages because highmem pages are
386 * freed before calling free_all_bootmem();
387 */
388 reset_all_zones_managed_pages();
389 for (tmp = highstart_pfn; tmp < highend_pfn; tmp++)
390 free_highmem_page(pfn_to_page(tmp));
391 #endif /* CONFIG_HIGHMEM */
392
393 free_all_bootmem();
394 mem_init_print_info(NULL);
395 }
396
free_initmem(void)397 void free_initmem(void)
398 {
399 free_initmem_default(POISON_FREE_INITMEM);
400 }
401
402 #ifdef CONFIG_BLK_DEV_INITRD
free_initrd_mem(unsigned long start,unsigned long end)403 void free_initrd_mem(unsigned long start, unsigned long end)
404 {
405 free_reserved_area((void *)start, (void *)end, POISON_FREE_INITMEM,
406 "initrd");
407 }
408 #endif
409