57 #define NODEDATA_ALIGN(addr, node)						\  argument
59 	     (((node)*PERCPU_PAGE_SIZE) & (MAX_NODE_ALIGN_OFFSET - 1)))
75 				  int node)  in build_node_maps()  argument
78 	struct bootmem_data *bdp = &bootmem_node_data[node];  in build_node_maps()
102 static int __meminit early_nr_cpus_node(int node)  in early_nr_cpus_node()  argument
107 		if (node == node_cpuid[cpu].nid)  in early_nr_cpus_node()
117 static unsigned long __meminit compute_pernodesize(int node)  in compute_pernodesize()  argument
121 	cpus = early_nr_cpus_node(node);  in compute_pernodesize()
123 	pernodesize += node * L1_CACHE_BYTES;  in compute_pernodesize()
140 static void *per_cpu_node_setup(void *cpu_data, int node)  in per_cpu_node_setup()  argument
148 		if (node != node_cpuid[cpu].nid)  in per_cpu_node_setup()
193 	int node, prev_node, unit, nr_units, rc;  in setup_per_cpu_areas()  local
209 	for_each_node(node)  in setup_per_cpu_areas()
211 			if (node == node_cpuid[cpu].nid)  in setup_per_cpu_areas()
238 		node = node_cpuid[cpu].nid;  in setup_per_cpu_areas()
240 		if (node == prev_node) {  in setup_per_cpu_areas()
244 		prev_node = node;  in setup_per_cpu_areas()
266 static void __init fill_pernode(int node, unsigned long pernode,  in fill_pernode()  argument
270 	int cpus = early_nr_cpus_node(node);  in fill_pernode()
271 	struct bootmem_data *bdp = &bootmem_node_data[node];  in fill_pernode()
273 	mem_data[node].pernode_addr = pernode;  in fill_pernode()
274 	mem_data[node].pernode_size = pernodesize;  in fill_pernode()
279 	pernode += node * L1_CACHE_BYTES;  in fill_pernode()
281 	pgdat_list[node] = __va(pernode);  in fill_pernode()
284 	mem_data[node].node_data = __va(pernode);  in fill_pernode()
287 	pgdat_list[node]->bdata = bdp;  in fill_pernode()
290 	cpu_data = per_cpu_node_setup(cpu_data, node);  in fill_pernode()
324 				     int node)  in find_pernode_space()  argument
328 	struct bootmem_data *bdp = &bootmem_node_data[node];  in find_pernode_space()
344 	if (mem_data[node].pernode_addr)  in find_pernode_space()
351 	pernodesize = compute_pernodesize(node);  in find_pernode_space()
352 	pernode = NODEDATA_ALIGN(start, node);  in find_pernode_space()
356 		fill_pernode(node, pernode, pernodesize);  in find_pernode_space()
373 				    int node)  in free_node_bootmem()  argument
375 	free_bootmem_node(pgdat_list[node], start, len);  in free_node_bootmem()
391 	int node;  in reserve_pernode_space()  local
393 	for_each_online_node(node) {  in reserve_pernode_space()
394 		pg_data_t *pdp = pgdat_list[node];  in reserve_pernode_space()
396 		if (node_isset(node, memory_less_mask))  in reserve_pernode_space()
408 		size = mem_data[node].pernode_size;  in reserve_pernode_space()
409 		base = __pa(mem_data[node].pernode_addr);  in reserve_pernode_space()
417 	int node;  in scatter_node_data()  local
427 	for_each_node(node) {  in scatter_node_data()
428 		if (pgdat_list[node]) {  in scatter_node_data()
429 			dst = LOCAL_DATA_ADDR(pgdat_list[node])->pg_data_ptrs;  in scatter_node_data()
445 	int cpu, node;  in initialize_pernode_data()  local
452 		node = node_cpuid[cpu].nid;  in initialize_pernode_data()
454 			mem_data[node].node_data;  in initialize_pernode_data()
460 		node = node_cpuid[cpu].nid;  in initialize_pernode_data()
463 		cpu0_cpu_info->node_data = mem_data[node].node_data;  in initialize_pernode_data()
479 	int bestnode = -1, node, anynode = 0;  in memory_less_node_alloc()  local
481 	for_each_online_node(node) {  in memory_less_node_alloc()
482 		if (node_isset(node, memory_less_mask))  in memory_less_node_alloc()
484 		else if (node_distance(nid, node) < best) {  in memory_less_node_alloc()
485 			best = node_distance(nid, node);  in memory_less_node_alloc()
486 			bestnode = node;  in memory_less_node_alloc()
488 		anynode = node;  in memory_less_node_alloc()
508 	int node;  in memory_less_nodes()  local
510 	for_each_node_mask(node, memory_less_mask) {  in memory_less_nodes()
511 		pernodesize = compute_pernodesize(node);  in memory_less_nodes()
512 		pernode = memory_less_node_alloc(node, pernodesize);  in memory_less_nodes()
513 		fill_pernode(node, __pa(pernode), pernodesize);  in memory_less_nodes()
527 	int node;  in find_memory()  local
545 	for_each_online_node(node)  in find_memory()
546 		if (bootmem_node_data[node].node_low_pfn) {  in find_memory()
547 			node_clear(node, memory_less_mask);  in find_memory()
548 			mem_data[node].min_pfn = ~0UL;  in find_memory()
557 	for (node = MAX_NUMNODES - 1; node >= 0; node--) {  in find_memory()
561 		if (!node_online(node))  in find_memory()
563 		else if (node_isset(node, memory_less_mask))  in find_memory()
566 		bdp = &bootmem_node_data[node];  in find_memory()
567 		pernode = mem_data[node].pernode_addr;  in find_memory()
568 		pernodesize = mem_data[node].pernode_size;  in find_memory()
571 		init_bootmem_node(pgdat_list[node],  in find_memory()
667 static __init int count_node_pages(unsigned long start, unsigned long len, int node)  in count_node_pages()  argument
673 		mem_data[node].num_dma_physpages +=  in count_node_pages()
678 	mem_data[node].max_pfn = max(mem_data[node].max_pfn,  in count_node_pages()
680 	mem_data[node].min_pfn = min(mem_data[node].min_pfn,  in count_node_pages()
697 	int node;  in paging_init()  local
715 	for_each_online_node(node) {  in paging_init()
716 		pfn_offset = mem_data[node].min_pfn;  in paging_init()
719 		NODE_DATA(node)->node_mem_map = vmem_map + pfn_offset;  in paging_init()
721 		if (mem_data[node].max_pfn > max_pfn)  in paging_init()
722 			max_pfn = mem_data[node].max_pfn;  in paging_init()
756 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)  in vmemmap_populate()  argument
758 	return vmemmap_populate_basepages(start, end, node);  in vmemmap_populate()