165 void defer_compaction(struct zone *zone, int order)  in defer_compaction()  argument
167 	zone->compact_considered = 0;  in defer_compaction()
168 	zone->compact_defer_shift++;  in defer_compaction()
170 	if (order < zone->compact_order_failed)  in defer_compaction()
171 		zone->compact_order_failed = order;  in defer_compaction()
173 	if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)  in defer_compaction()
174 		zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;  in defer_compaction()
176 	trace_mm_compaction_defer_compaction(zone, order);  in defer_compaction()
180 bool compaction_deferred(struct zone *zone, int order)  in compaction_deferred()  argument
182 	unsigned long defer_limit = 1UL << zone->compact_defer_shift;  in compaction_deferred()
184 	if (order < zone->compact_order_failed)  in compaction_deferred()
188 	if (++zone->compact_considered >= defer_limit) {  in compaction_deferred()
189 		zone->compact_considered = defer_limit;  in compaction_deferred()
193 	trace_mm_compaction_deferred(zone, order);  in compaction_deferred()
203 void compaction_defer_reset(struct zone *zone, int order,  in compaction_defer_reset()  argument
207 		zone->compact_considered = 0;  in compaction_defer_reset()
208 		zone->compact_defer_shift = 0;  in compaction_defer_reset()
210 	if (order >= zone->compact_order_failed)  in compaction_defer_reset()
211 		zone->compact_order_failed = order + 1;  in compaction_defer_reset()
213 	trace_mm_compaction_defer_reset(zone, order);  in compaction_defer_reset()
217 bool compaction_restarting(struct zone *zone, int order)  in compaction_restarting()  argument
219 	if (order < zone->compact_order_failed)  in compaction_restarting()
222 	return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&  in compaction_restarting()
223 		zone->compact_considered >= 1UL << zone->compact_defer_shift;  in compaction_restarting()
236 static void reset_cached_positions(struct zone *zone)  in reset_cached_positions()  argument
238 	zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;  in reset_cached_positions()
239 	zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;  in reset_cached_positions()
240 	zone->compact_cached_free_pfn =  in reset_cached_positions()
241 				pageblock_start_pfn(zone_end_pfn(zone) - 1);  in reset_cached_positions()
263 __reset_isolation_pfn(struct zone *zone, unsigned long pfn, bool check_source,  in __reset_isolation_pfn()  argument
273 	if (zone != page_zone(page))  in __reset_isolation_pfn()
295 	block_pfn = max(block_pfn, zone->zone_start_pfn);  in __reset_isolation_pfn()
304 	block_pfn = min(block_pfn, zone_end_pfn(zone) - 1);  in __reset_isolation_pfn()
339 static void __reset_isolation_suitable(struct zone *zone)  in __reset_isolation_suitable()  argument
341 	unsigned long migrate_pfn = zone->zone_start_pfn;  in __reset_isolation_suitable()
342 	unsigned long free_pfn = zone_end_pfn(zone) - 1;  in __reset_isolation_suitable()
348 	if (!zone->compact_blockskip_flush)  in __reset_isolation_suitable()
351 	zone->compact_blockskip_flush = false;  in __reset_isolation_suitable()
364 		if (__reset_isolation_pfn(zone, migrate_pfn, true, source_set) &&  in __reset_isolation_suitable()
368 			zone->compact_init_migrate_pfn = reset_migrate;  in __reset_isolation_suitable()
369 			zone->compact_cached_migrate_pfn[0] = reset_migrate;  in __reset_isolation_suitable()
370 			zone->compact_cached_migrate_pfn[1] = reset_migrate;  in __reset_isolation_suitable()
374 		if (__reset_isolation_pfn(zone, free_pfn, free_set, true) &&  in __reset_isolation_suitable()
378 			zone->compact_init_free_pfn = reset_free;  in __reset_isolation_suitable()
379 			zone->compact_cached_free_pfn = reset_free;  in __reset_isolation_suitable()
385 		zone->compact_cached_migrate_pfn[0] = migrate_pfn;  in __reset_isolation_suitable()
386 		zone->compact_cached_migrate_pfn[1] = migrate_pfn;  in __reset_isolation_suitable()
387 		zone->compact_cached_free_pfn = free_pfn;  in __reset_isolation_suitable()
396 		struct zone *zone = &pgdat->node_zones[zoneid];  in reset_isolation_suitable()  local
397 		if (!populated_zone(zone))  in reset_isolation_suitable()
401 		if (zone->compact_blockskip_flush)  in reset_isolation_suitable()
402 			__reset_isolation_suitable(zone);  in reset_isolation_suitable()
431 	struct zone *zone = cc->zone;  in update_cached_migrate()  local
439 	if (pfn > zone->compact_cached_migrate_pfn[0])  in update_cached_migrate()
440 		zone->compact_cached_migrate_pfn[0] = pfn;  in update_cached_migrate()
442 	    pfn > zone->compact_cached_migrate_pfn[1])  in update_cached_migrate()
443 		zone->compact_cached_migrate_pfn[1] = pfn;  in update_cached_migrate()
453 	struct zone *zone = cc->zone;  in update_pageblock_skip()  local
464 	if (pfn < zone->compact_cached_free_pfn)  in update_pageblock_skip()
465 		zone->compact_cached_free_pfn = pfn;  in update_pageblock_skip()
589 		    && compact_unlock_should_abort(&cc->zone->lock, flags,  in isolate_freepages_block()
624 			locked = compact_lock_irqsave(&cc->zone->lock,  in isolate_freepages_block()
661 		spin_unlock_irqrestore(&cc->zone->lock, flags);  in isolate_freepages_block()
713 	if (block_start_pfn < cc->zone->zone_start_pfn)  in isolate_freepages_range()
714 		block_start_pfn = cc->zone->zone_start_pfn;  in isolate_freepages_range()
737 					block_end_pfn, cc->zone))  in isolate_freepages_range()
833 	pg_data_t *pgdat = cc->zone->zone_pgdat;  in isolate_migratepages_block()
1149 	if (block_start_pfn < cc->zone->zone_start_pfn)  in isolate_migratepages_range()
1150 		block_start_pfn = cc->zone->zone_start_pfn;  in isolate_migratepages_range()
1160 					block_end_pfn, cc->zone))  in isolate_migratepages_range()
1292 	start_pfn = max(pageblock_start_pfn(pfn), cc->zone->zone_start_pfn);  in fast_isolate_around()
1293 	end_pfn = min(pageblock_end_pfn(pfn), zone_end_pfn(cc->zone));  in fast_isolate_around()
1295 	page = pageblock_pfn_to_page(start_pfn, end_pfn, cc->zone);  in fast_isolate_around()
1346 	if (cc->free_pfn >= cc->zone->compact_init_free_pfn) {  in fast_isolate_freepages()
1371 		struct free_area *area = &cc->zone->free_area[order];  in fast_isolate_freepages()
1381 		spin_lock_irqsave(&cc->zone->lock, flags);  in fast_isolate_freepages()
1392 					      cc->zone->zone_start_pfn);  in fast_isolate_freepages()
1438 		spin_unlock_irqrestore(&cc->zone->lock, flags);  in fast_isolate_freepages()
1463 						    zone_end_pfn(cc->zone)),  in fast_isolate_freepages()
1464 						cc->zone);  in fast_isolate_freepages()
1471 	if (highest && highest >= cc->zone->compact_cached_free_pfn) {  in fast_isolate_freepages()
1473 		cc->zone->compact_cached_free_pfn = highest;  in fast_isolate_freepages()
1491 	struct zone *zone = cc->zone;  in isolate_freepages()  local
1519 						zone_end_pfn(zone));  in isolate_freepages()
1542 									zone);  in isolate_freepages()
1704 	if (pfn != cc->zone->zone_start_pfn && pfn != pageblock_start_pfn(pfn))  in fast_find_migrateblock()
1731 	if (cc->migrate_pfn != cc->zone->zone_start_pfn)  in fast_find_migrateblock()
1738 		struct free_area *area = &cc->zone->free_area[order];  in fast_find_migrateblock()
1746 		spin_lock_irqsave(&cc->zone->lock, flags);  in fast_find_migrateblock()
1772 				if (pfn < cc->zone->zone_start_pfn)  in fast_find_migrateblock()
1773 					pfn = cc->zone->zone_start_pfn;  in fast_find_migrateblock()
1780 		spin_unlock_irqrestore(&cc->zone->lock, flags);  in fast_find_migrateblock()
1819 	if (block_start_pfn < cc->zone->zone_start_pfn)  in isolate_migratepages()
1820 		block_start_pfn = cc->zone->zone_start_pfn;  in isolate_migratepages()
1851 						block_end_pfn, cc->zone);  in isolate_migratepages()
1918 static unsigned int fragmentation_score_zone(struct zone *zone)  in fragmentation_score_zone()  argument
1920 	return extfrag_for_order(zone, COMPACTION_HPAGE_ORDER);  in fragmentation_score_zone()
1933 static unsigned int fragmentation_score_zone_weighted(struct zone *zone)  in fragmentation_score_zone_weighted()  argument
1937 	score = zone->present_pages * fragmentation_score_zone(zone);  in fragmentation_score_zone_weighted()
1938 	return div64_ul(score, zone->zone_pgdat->node_present_pages + 1);  in fragmentation_score_zone_weighted()
1954 		struct zone *zone;  in fragmentation_score_node()  local
1956 		zone = &pgdat->node_zones[zoneid];  in fragmentation_score_node()
1957 		score += fragmentation_score_zone_weighted(zone);  in fragmentation_score_node()
1997 		reset_cached_positions(cc->zone);  in __compact_finished()
2006 			cc->zone->compact_blockskip_flush = true;  in __compact_finished()
2018 		pgdat = cc->zone->zone_pgdat;  in __compact_finished()
2022 		score = fragmentation_score_zone(cc->zone);  in __compact_finished()
2048 		struct free_area *area = &cc->zone->free_area[order];  in __compact_finished()
2104 	trace_mm_compaction_finished(cc->zone, cc->order, ret);  in compact_finished()
2118 static enum compact_result __compaction_suitable(struct zone *zone, int order,  in __compaction_suitable()  argument
2128 	watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);  in __compaction_suitable()
2133 	if (zone_watermark_ok(zone, order, watermark, highest_zoneidx,  in __compaction_suitable()
2152 				low_wmark_pages(zone) : min_wmark_pages(zone);  in __compaction_suitable()
2154 	if (!__zone_watermark_ok(zone, 0, watermark, highest_zoneidx,  in __compaction_suitable()
2161 enum compact_result compaction_suitable(struct zone *zone, int order,  in compaction_suitable()  argument
2168 	ret = __compaction_suitable(zone, order, alloc_flags, highest_zoneidx,  in compaction_suitable()
2169 				    zone_page_state(zone, NR_FREE_PAGES));  in compaction_suitable()
2187 		fragindex = fragmentation_index(zone, order);  in compaction_suitable()
2192 	trace_mm_compaction_suitable(zone, order, ret);  in compaction_suitable()
2202 	struct zone *zone;  in compaction_zonelist_suitable()  local
2209 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,  in compaction_zonelist_suitable()
2220 		available = zone_reclaimable_pages(zone) / order;  in compaction_zonelist_suitable()
2221 		available += zone_page_state_snapshot(zone, NR_FREE_PAGES);  in compaction_zonelist_suitable()
2222 		compact_result = __compaction_suitable(zone, order, alloc_flags,  in compaction_zonelist_suitable()
2235 	unsigned long start_pfn = cc->zone->zone_start_pfn;  in compact_zone()
2236 	unsigned long end_pfn = zone_end_pfn(cc->zone);  in compact_zone()
2253 	ret = compaction_suitable(cc->zone, cc->order, cc->alloc_flags,  in compact_zone()
2266 	if (compaction_restarting(cc->zone, cc->order))  in compact_zone()
2267 		__reset_isolation_suitable(cc->zone);  in compact_zone()
2280 		cc->migrate_pfn = cc->zone->compact_cached_migrate_pfn[sync];  in compact_zone()
2281 		cc->free_pfn = cc->zone->compact_cached_free_pfn;  in compact_zone()
2284 			cc->zone->compact_cached_free_pfn = cc->free_pfn;  in compact_zone()
2288 			cc->zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;  in compact_zone()
2289 			cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;  in compact_zone()
2292 		if (cc->migrate_pfn <= cc->zone->compact_init_migrate_pfn)  in compact_zone()
2307 		cc->zone->compact_cached_migrate_pfn[0] == cc->zone->compact_cached_migrate_pfn[1];  in compact_zone()
2341 				cc->zone->compact_cached_migrate_pfn[1] =  in compact_zone()
2342 					cc->zone->compact_cached_migrate_pfn[0];  in compact_zone()
2402 				lru_add_drain_cpu_zone(cc->zone);  in compact_zone()
2431 		if (free_pfn > cc->zone->compact_cached_free_pfn)  in compact_zone()
2432 			cc->zone->compact_cached_free_pfn = free_pfn;  in compact_zone()
2444 static enum compact_result compact_zone_order(struct zone *zone, int order,  in compact_zone_order()  argument
2454 		.zone = zone,  in compact_zone_order()
2512 	struct zone *zone;  in try_to_compact_pages()  local
2525 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,  in try_to_compact_pages()
2530 					&& compaction_deferred(zone, order)) {  in try_to_compact_pages()
2535 		status = compact_zone_order(zone, order, gfp_mask, prio,  in try_to_compact_pages()
2547 			compaction_defer_reset(zone, order, false);  in try_to_compact_pages()
2559 			defer_compaction(zone, order);  in try_to_compact_pages()
2586 	struct zone *zone;  in proactive_compact_node()  local
2597 		zone = &pgdat->node_zones[zoneid];  in proactive_compact_node()
2598 		if (!populated_zone(zone))  in proactive_compact_node()
2601 		cc.zone = zone;  in proactive_compact_node()
2615 	struct zone *zone;  in compact_node()  local
2627 		zone = &pgdat->node_zones[zoneid];  in compact_node()
2628 		if (!populated_zone(zone))  in compact_node()
2631 		cc.zone = zone;  in compact_node()
2737 	struct zone *zone;  in kcompactd_node_suitable()  local
2741 		zone = &pgdat->node_zones[zoneid];  in kcompactd_node_suitable()
2743 		if (!populated_zone(zone))  in kcompactd_node_suitable()
2746 		if (compaction_suitable(zone, pgdat->kcompactd_max_order, 0,  in kcompactd_node_suitable()
2761 	struct zone *zone;  in kcompactd_do_work()  local
2777 		zone = &pgdat->node_zones[zoneid];  in kcompactd_do_work()
2778 		if (!populated_zone(zone))  in kcompactd_do_work()
2781 		if (compaction_deferred(zone, cc.order))  in kcompactd_do_work()
2784 		if (compaction_suitable(zone, cc.order, 0, zoneid) !=  in kcompactd_do_work()
2791 		cc.zone = zone;  in kcompactd_do_work()
2795 			compaction_defer_reset(zone, cc.order, false);  in kcompactd_do_work()
2803 			drain_all_pages(zone);  in kcompactd_do_work()
2809 			defer_compaction(zone, cc.order);  in kcompactd_do_work()