354 static unsigned long find_smallest_section_pfn(int nid, struct zone *zone,  in find_smallest_section_pfn()  argument
365 		if (zone != page_zone(pfn_to_page(start_pfn)))  in find_smallest_section_pfn()
375 static unsigned long find_biggest_section_pfn(int nid, struct zone *zone,  in find_biggest_section_pfn()  argument
390 		if (zone != page_zone(pfn_to_page(pfn)))  in find_biggest_section_pfn()
399 static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,  in shrink_zone_span()  argument
403 	int nid = zone_to_nid(zone);  in shrink_zone_span()
405 	zone_span_writelock(zone);  in shrink_zone_span()
406 	if (zone->zone_start_pfn == start_pfn) {  in shrink_zone_span()
413 		pfn = find_smallest_section_pfn(nid, zone, end_pfn,  in shrink_zone_span()
414 						zone_end_pfn(zone));  in shrink_zone_span()
416 			zone->spanned_pages = zone_end_pfn(zone) - pfn;  in shrink_zone_span()
417 			zone->zone_start_pfn = pfn;  in shrink_zone_span()
419 			zone->zone_start_pfn = 0;  in shrink_zone_span()
420 			zone->spanned_pages = 0;  in shrink_zone_span()
422 	} else if (zone_end_pfn(zone) == end_pfn) {  in shrink_zone_span()
429 		pfn = find_biggest_section_pfn(nid, zone, zone->zone_start_pfn,  in shrink_zone_span()
432 			zone->spanned_pages = pfn - zone->zone_start_pfn + 1;  in shrink_zone_span()
434 			zone->zone_start_pfn = 0;  in shrink_zone_span()
435 			zone->spanned_pages = 0;  in shrink_zone_span()
438 	zone_span_writeunlock(zone);  in shrink_zone_span()
444 	struct zone *zone;  in update_pgdat_span()  local
446 	for (zone = pgdat->node_zones;  in update_pgdat_span()
447 	     zone < pgdat->node_zones + MAX_NR_ZONES; zone++) {  in update_pgdat_span()
448 		unsigned long zone_end_pfn = zone->zone_start_pfn +  in update_pgdat_span()
449 					     zone->spanned_pages;  in update_pgdat_span()
452 		if (!zone->spanned_pages)  in update_pgdat_span()
455 			node_start_pfn = zone->zone_start_pfn;  in update_pgdat_span()
462 		if (zone->zone_start_pfn < node_start_pfn)  in update_pgdat_span()
463 			node_start_pfn = zone->zone_start_pfn;  in update_pgdat_span()
470 void __ref remove_pfn_range_from_zone(struct zone *zone,  in remove_pfn_range_from_zone()  argument
475 	struct pglist_data *pgdat = zone->zone_pgdat;  in remove_pfn_range_from_zone()
495 	if (zone_idx(zone) == ZONE_DEVICE)  in remove_pfn_range_from_zone()
499 	clear_zone_contiguous(zone);  in remove_pfn_range_from_zone()
501 	pgdat_resize_lock(zone->zone_pgdat, &flags);  in remove_pfn_range_from_zone()
502 	shrink_zone_span(zone, start_pfn, start_pfn + nr_pages);  in remove_pfn_range_from_zone()
504 	pgdat_resize_unlock(zone->zone_pgdat, &flags);  in remove_pfn_range_from_zone()
506 	set_zone_contiguous(zone);  in remove_pfn_range_from_zone()
629 	struct zone *zone, struct memory_notify *arg)  in node_states_check_changes_online()  argument
631 	int nid = zone_to_nid(zone);  in node_states_check_changes_online()
639 	if (zone_idx(zone) <= ZONE_NORMAL && !node_state(nid, N_NORMAL_MEMORY))  in node_states_check_changes_online()
642 	if (zone_idx(zone) <= ZONE_HIGHMEM && !node_state(nid, N_HIGH_MEMORY))  in node_states_check_changes_online()
659 static void __meminit resize_zone_range(struct zone *zone, unsigned long start_pfn,  in resize_zone_range()  argument
662 	unsigned long old_end_pfn = zone_end_pfn(zone);  in resize_zone_range()
664 	if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn)  in resize_zone_range()
665 		zone->zone_start_pfn = start_pfn;  in resize_zone_range()
667 	zone->spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - zone->zone_start_pfn;  in resize_zone_range()
690 void __ref move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn,  in move_pfn_range_to_zone()  argument
694 	struct pglist_data *pgdat = zone->zone_pgdat;  in move_pfn_range_to_zone()
698 	clear_zone_contiguous(zone);  in move_pfn_range_to_zone()
702 	zone_span_writelock(zone);  in move_pfn_range_to_zone()
703 	if (zone_is_empty(zone))  in move_pfn_range_to_zone()
704 		init_currently_empty_zone(zone, start_pfn, nr_pages);  in move_pfn_range_to_zone()
705 	resize_zone_range(zone, start_pfn, nr_pages);  in move_pfn_range_to_zone()
706 	zone_span_writeunlock(zone);  in move_pfn_range_to_zone()
716 	memmap_init_zone(nr_pages, nid, zone_idx(zone), start_pfn, 0,  in move_pfn_range_to_zone()
719 	set_zone_contiguous(zone);  in move_pfn_range_to_zone()
727 static struct zone *default_kernel_zone_for_pfn(int nid, unsigned long start_pfn,  in default_kernel_zone_for_pfn()
734 		struct zone *zone = &pgdat->node_zones[zid];  in default_kernel_zone_for_pfn()  local
736 		if (zone_intersects(zone, start_pfn, nr_pages))  in default_kernel_zone_for_pfn()
737 			return zone;  in default_kernel_zone_for_pfn()
743 static inline struct zone *default_zone_for_pfn(int nid, unsigned long start_pfn,  in default_zone_for_pfn()
746 	struct zone *kernel_zone = default_kernel_zone_for_pfn(nid, start_pfn,  in default_zone_for_pfn()
748 	struct zone *movable_zone = &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];  in default_zone_for_pfn()
767 struct zone *zone_for_pfn_range(int online_type, int nid,  in zone_for_pfn_range()
783 	struct zone *zone;  in online_pages()  local
796 	zone = zone_for_pfn_range(online_type, nid, pfn, nr_pages);  in online_pages()
797 	move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_ISOLATE);  in online_pages()
801 	node_states_check_changes_online(nr_pages, zone, &arg);  in online_pages()
812 	spin_lock_irqsave(&zone->lock, flags);  in online_pages()
813 	zone->nr_isolate_pageblock += nr_pages / pageblock_nr_pages;  in online_pages()
814 	spin_unlock_irqrestore(&zone->lock, flags);  in online_pages()
821 	if (!populated_zone(zone)) {  in online_pages()
823 		setup_zone_pageset(zone);  in online_pages()
827 	zone->present_pages += nr_pages;  in online_pages()
829 	pgdat_resize_lock(zone->zone_pgdat, &flags);  in online_pages()
830 	zone->zone_pgdat->node_present_pages += nr_pages;  in online_pages()
831 	pgdat_resize_unlock(zone->zone_pgdat, &flags);  in online_pages()
836 	zone_pcp_update(zone);  in online_pages()
847 	shuffle_zone(zone);  in online_pages()
865 	remove_pfn_range_from_zone(zone, pfn, nr_pages);  in online_pages()
873 	struct zone *z;  in reset_node_present_pages()
1233 struct zone *test_pages_in_a_zone(unsigned long start_pfn,  in test_pages_in_a_zone()
1237 	struct zone *zone = NULL;  in test_pages_in_a_zone()  local
1256 			if (zone && !zone_spans_pfn(zone, pfn + i))  in test_pages_in_a_zone()
1259 			if (zone && page_zone(page) != zone)  in test_pages_in_a_zone()
1261 			zone = page_zone(page);  in test_pages_in_a_zone()
1265 	return zone;  in test_pages_in_a_zone()
1428 		struct zone *zone, struct memory_notify *arg)  in node_states_check_changes_offline()  argument
1430 	struct pglist_data *pgdat = zone->zone_pgdat;  in node_states_check_changes_offline()
1448 	if (zone_idx(zone) <= ZONE_NORMAL && nr_pages >= present_pages)  in node_states_check_changes_offline()
1449 		arg->status_change_nid_normal = zone_to_nid(zone);  in node_states_check_changes_offline()
1461 	if (zone_idx(zone) <= ZONE_HIGHMEM && nr_pages >= present_pages)  in node_states_check_changes_offline()
1462 		arg->status_change_nid_high = zone_to_nid(zone);  in node_states_check_changes_offline()
1478 		arg->status_change_nid = zone_to_nid(zone);  in node_states_check_changes_offline()
1507 	struct zone *zone;  in offline_pages()  local
1537 	zone = test_pages_in_a_zone(start_pfn, end_pfn);  in offline_pages()
1538 	if (!zone) {  in offline_pages()
1543 	node = zone_to_nid(zone);  in offline_pages()
1555 	drain_all_pages(zone);  in offline_pages()
1559 	node_states_check_changes_offline(nr_pages, zone, &arg);  in offline_pages()
1618 			drain_all_pages(zone);  in offline_pages()
1630 	spin_lock_irqsave(&zone->lock, flags);  in offline_pages()
1631 	zone->nr_isolate_pageblock -= nr_pages / pageblock_nr_pages;  in offline_pages()
1632 	spin_unlock_irqrestore(&zone->lock, flags);  in offline_pages()
1637 	zone->present_pages -= nr_pages;  in offline_pages()
1639 	pgdat_resize_lock(zone->zone_pgdat, &flags);  in offline_pages()
1640 	zone->zone_pgdat->node_present_pages -= nr_pages;  in offline_pages()
1641 	pgdat_resize_unlock(zone->zone_pgdat, &flags);  in offline_pages()
1645 	if (!populated_zone(zone)) {  in offline_pages()
1646 		zone_pcp_reset(zone);  in offline_pages()
1649 		zone_pcp_update(zone);  in offline_pages()
1660 	remove_pfn_range_from_zone(zone, start_pfn, nr_pages);  in offline_pages()