Lines Matching +full:processor +full:- +full:intensive
1 // SPDX-License-Identifier: GPL-2.0
34 #include <linux/backing-dev.h>
134 if ((_page)->lru.prev != _base) { \
137 prev = lru_to_page(&(_page->lru)); \
138 prefetch(&prev->_field); \
148 if ((_page)->lru.prev != _base) { \
151 prev = lru_to_page(&(_page->lru)); \
152 prefetchw(&prev->_field); \
175 * We allow subsystems to populate their shrinker-related
192 int id, ret = -ENOMEM; in prealloc_memcg_shrinker()
208 shrinker->id = id; in prealloc_memcg_shrinker()
217 int id = shrinker->id; in unregister_memcg_shrinker()
239 return !sc->target_mem_cgroup; in global_reclaim()
243 * sane_reclaim - is the usual dirty throttling mechanism operational?
257 struct mem_cgroup *memcg = sc->target_mem_cgroup; in sane_reclaim()
277 mn = mem_cgroup_nodeinfo(memcg, pgdat->node_id); in set_memcg_congestion()
278 WRITE_ONCE(mn->congested, congested); in set_memcg_congestion()
286 mn = mem_cgroup_nodeinfo(memcg, pgdat->node_id); in memcg_congested()
287 return READ_ONCE(mn->congested); in memcg_congested()
333 * lruvec_lru_size - Returns the number of pages on the given LRU list.
349 struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid]; in lruvec_lru_size()
358 size = zone_page_state(&lruvec_pgdat(lruvec)->node_zones[zid], in lruvec_lru_size()
360 lru_size -= min(size, lru_size); in lruvec_lru_size()
372 size_t size = sizeof(*shrinker->nr_deferred); in prealloc_shrinker()
374 if (shrinker->flags & SHRINKER_NUMA_AWARE) in prealloc_shrinker()
377 shrinker->nr_deferred = kzalloc(size, GFP_KERNEL); in prealloc_shrinker()
378 if (!shrinker->nr_deferred) in prealloc_shrinker()
379 return -ENOMEM; in prealloc_shrinker()
381 if (shrinker->flags & SHRINKER_MEMCG_AWARE) { in prealloc_shrinker()
389 kfree(shrinker->nr_deferred); in prealloc_shrinker()
390 shrinker->nr_deferred = NULL; in prealloc_shrinker()
391 return -ENOMEM; in prealloc_shrinker()
396 if (!shrinker->nr_deferred) in free_prealloced_shrinker()
399 if (shrinker->flags & SHRINKER_MEMCG_AWARE) in free_prealloced_shrinker()
402 kfree(shrinker->nr_deferred); in free_prealloced_shrinker()
403 shrinker->nr_deferred = NULL; in free_prealloced_shrinker()
409 list_add_tail(&shrinker->list, &shrinker_list); in register_shrinker_prepared()
411 if (shrinker->flags & SHRINKER_MEMCG_AWARE) in register_shrinker_prepared()
412 idr_replace(&shrinker_idr, shrinker, shrinker->id); in register_shrinker_prepared()
433 if (!shrinker->nr_deferred) in unregister_shrinker()
435 if (shrinker->flags & SHRINKER_MEMCG_AWARE) in unregister_shrinker()
438 list_del(&shrinker->list); in unregister_shrinker()
440 kfree(shrinker->nr_deferred); in unregister_shrinker()
441 shrinker->nr_deferred = NULL; in unregister_shrinker()
456 int nid = shrinkctl->nid; in do_shrink_slab()
457 long batch_size = shrinker->batch ? shrinker->batch in do_shrink_slab()
461 if (!(shrinker->flags & SHRINKER_NUMA_AWARE)) in do_shrink_slab()
464 freeable = shrinker->count_objects(shrinker, shrinkctl); in do_shrink_slab()
473 nr = atomic_long_xchg(&shrinker->nr_deferred[nid], 0); in do_shrink_slab()
478 do_div(delta, shrinker->seeks); in do_shrink_slab()
483 shrinker->scan_objects, total_scan); in do_shrink_slab()
492 * shrinkers to return -1 all the time. This results in a large in do_shrink_slab()
535 shrinkctl->nr_to_scan = nr_to_scan; in do_shrink_slab()
536 shrinkctl->nr_scanned = nr_to_scan; in do_shrink_slab()
537 ret = shrinker->scan_objects(shrinker, shrinkctl); in do_shrink_slab()
542 count_vm_events(SLABS_SCANNED, shrinkctl->nr_scanned); in do_shrink_slab()
543 total_scan -= shrinkctl->nr_scanned; in do_shrink_slab()
544 scanned += shrinkctl->nr_scanned; in do_shrink_slab()
550 next_deferred -= scanned; in do_shrink_slab()
560 &shrinker->nr_deferred[nid]); in do_shrink_slab()
562 new_nr = atomic_long_read(&shrinker->nr_deferred[nid]); in do_shrink_slab()
582 map = rcu_dereference_protected(memcg->nodeinfo[nid]->shrinker_map, in shrink_slab_memcg()
587 for_each_set_bit(i, map->map, shrinker_nr_max) { in shrink_slab_memcg()
598 clear_bit(i, map->map); in shrink_slab_memcg()
604 clear_bit(i, map->map); in shrink_slab_memcg()
647 * shrink_slab - shrink slab caches
661 * @priority is sc->priority, we take the number of objects and >> by priority
742 * optional buffer heads at page->private. in is_page_cache_freeable()
746 return page_count(page) - page_has_private(page) == 1 + radix_pins; in is_page_cache_freeable()
751 if (current->flags & PF_SWAPWRITE) in may_write_to_inode()
755 if (inode_to_bdi(inode) == current->backing_dev_info) in may_write_to_inode()
762 * -ENOSPC. We need to propagate that into the address_space for a subsequent
795 * Calls ->writepage().
802 * will be non-blocking. To prevent this allocation from being in pageout()
821 * page->mapping == NULL while being dirty with clean buffers. in pageout()
832 if (mapping->a_ops->writepage == NULL) in pageout()
834 if (!may_write_to_inode(mapping->host, sc)) in pageout()
848 res = mapping->a_ops->writepage(page, &wbc); in pageout()
881 xa_lock_irqsave(&mapping->i_pages, flags); in __remove_mapping()
901 * escape unnoticed. The smp_rmb is needed to ensure the page->flags in __remove_mapping()
902 * load is not satisfied before that of page->_refcount. in __remove_mapping()
923 xa_unlock_irqrestore(&mapping->i_pages, flags); in __remove_mapping()
929 freepage = mapping->a_ops->freepage; in __remove_mapping()
950 xa_unlock_irqrestore(&mapping->i_pages, flags); in __remove_mapping()
959 xa_unlock_irqrestore(&mapping->i_pages, flags); in __remove_mapping()
964 * Attempt to detach a locked page from its ->mapping. If it is dirty or if
984 * putback_lru_page - put previously isolated page onto appropriate LRU list
1011 referenced_ptes = page_referenced(page, 1, sc->target_mem_cgroup, in page_check_references()
1045 * Activate file-backed executable pages after first usage. in page_check_references()
1086 if (mapping && mapping->a_ops->is_dirty_writeback) in page_check_dirty_writeback()
1087 mapping->a_ops->is_dirty_writeback(page, dirty, writeback); in page_check_dirty_writeback()
1124 list_del(&page->lru); in shrink_page_list()
1131 sc->nr_scanned++; in shrink_page_list()
1136 if (!sc->may_unmap && page_mapped(page)) in shrink_page_list()
1142 sc->nr_scanned++; in shrink_page_list()
1144 may_enter_fs = (sc->gfp_mask & __GFP_FS) || in shrink_page_list()
1145 (PageSwapCache(page) && (sc->gfp_mask & __GFP_IO)); in shrink_page_list()
1168 inode_write_congested(mapping->host)) || in shrink_page_list()
1218 test_bit(PGDAT_WRITEBACK, &pgdat->flags)) { in shrink_page_list()
1226 * This is slightly racy - end_page_writeback() in shrink_page_list()
1229 * as PageReadahead - but that does not matter in shrink_page_list()
1245 list_add_tail(&page->lru, page_list); in shrink_page_list()
1271 if (!(sc->gfp_mask & __GFP_IO)) in shrink_page_list()
1331 * injecting inefficient single-page IO into in shrink_page_list()
1340 !test_bit(PGDAT_DIRTY, &pgdat->flags))) { in shrink_page_list()
1357 if (!sc->may_writepage) in shrink_page_list()
1378 * A synchronous write - probably a ramdisk. Go in shrink_page_list()
1403 * drop the buffers and mark the page clean - it can be freed. in shrink_page_list()
1405 * Rarely, pages can have buffers and no ->mapping. These are in shrink_page_list()
1413 if (!try_to_release_page(page, sc->gfp_mask)) in shrink_page_list()
1449 * from pagecache). Can use non-atomic bitops now (and in shrink_page_list()
1465 list_add(&page->lru, &free_pages); in shrink_page_list()
1482 list_add(&page->lru, &ret_pages); in shrink_page_list()
1494 stat->nr_dirty = nr_dirty; in shrink_page_list()
1495 stat->nr_congested = nr_congested; in shrink_page_list()
1496 stat->nr_unqueued_dirty = nr_unqueued_dirty; in shrink_page_list()
1497 stat->nr_writeback = nr_writeback; in shrink_page_list()
1498 stat->nr_immediate = nr_immediate; in shrink_page_list()
1499 stat->nr_activate = pgactivate; in shrink_page_list()
1500 stat->nr_ref_keep = nr_ref_keep; in shrink_page_list()
1501 stat->nr_unmap_fail = nr_unmap_fail; in shrink_page_list()
1522 list_move(&page->lru, &clean_pages); in reclaim_clean_pages_from_list()
1526 ret = shrink_page_list(&clean_pages, zone->zone_pgdat, &sc, in reclaim_clean_pages_from_list()
1529 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, -ret); in reclaim_clean_pages_from_list()
1541 * returns 0 on success, -ve errno on failure.
1545 int ret = -EINVAL; in __isolate_lru_page()
1555 ret = -EBUSY; in __isolate_lru_page()
1560 * blocking - clean pages for the most part. in __isolate_lru_page()
1576 * ->migratepage callback are possible to migrate in __isolate_lru_page()
1587 migrate_dirty = !mapping || mapping->a_ops->migratepage; in __isolate_lru_page()
1600 * sure the page is not being freed elsewhere -- the in __isolate_lru_page()
1624 __update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]); in update_lru_sizes()
1626 mem_cgroup_update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]); in update_lru_sizes()
1637 * For pagecache intensive workloads, this function is the hottest
1657 struct list_head *src = &lruvec->lists[lru]; in isolate_lru_pages()
1676 if (page_zonenum(page) > sc->reclaim_idx) { in isolate_lru_pages()
1677 list_move(&page->lru, &pages_skipped); in isolate_lru_pages()
1694 list_move(&page->lru, dst); in isolate_lru_pages()
1697 case -EBUSY: in isolate_lru_pages()
1699 list_move(&page->lru, src); in isolate_lru_pages()
1727 trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan, in isolate_lru_pages()
1734 * isolate_lru_page - tries to isolate a page from its LRU list
1741 * Returns -EBUSY if the page was not on an LRU list.
1761 int ret = -EBUSY; in isolate_lru_page()
1771 lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat); in isolate_lru_page()
1812 * won't get blocked by normal direct-reclaimers, forming a circular in too_many_isolated()
1815 if ((sc->gfp_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) in too_many_isolated()
1824 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat; in putback_inactive_pages()
1836 list_del(&page->lru); in putback_inactive_pages()
1838 spin_unlock_irq(&pgdat->lru_lock); in putback_inactive_pages()
1840 spin_lock_irq(&pgdat->lru_lock); in putback_inactive_pages()
1853 reclaim_stat->recent_rotated[file] += numpages; in putback_inactive_pages()
1861 spin_unlock_irq(&pgdat->lru_lock); in putback_inactive_pages()
1864 spin_lock_irq(&pgdat->lru_lock); in putback_inactive_pages()
1866 list_add(&page->lru, &pages_to_free); in putback_inactive_pages()
1877 * If a kernel thread (such as nfsd for loop-back mounts) services
1884 return !(current->flags & PF_LESS_THROTTLE) || in current_may_throttle()
1885 current->backing_dev_info == NULL || in current_may_throttle()
1886 bdi_write_congested(current->backing_dev_info); in current_may_throttle()
1905 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat; in shrink_inactive_list()
1923 if (!sc->may_unmap) in shrink_inactive_list()
1926 spin_lock_irq(&pgdat->lru_lock); in shrink_inactive_list()
1932 reclaim_stat->recent_scanned[file] += nr_taken; in shrink_inactive_list()
1945 spin_unlock_irq(&pgdat->lru_lock); in shrink_inactive_list()
1953 spin_lock_irq(&pgdat->lru_lock); in shrink_inactive_list()
1969 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); in shrink_inactive_list()
1971 spin_unlock_irq(&pgdat->lru_lock); in shrink_inactive_list()
1990 sc->nr.dirty += stat.nr_dirty; in shrink_inactive_list()
1991 sc->nr.congested += stat.nr_congested; in shrink_inactive_list()
1992 sc->nr.unqueued_dirty += stat.nr_unqueued_dirty; in shrink_inactive_list()
1993 sc->nr.writeback += stat.nr_writeback; in shrink_inactive_list()
1994 sc->nr.immediate += stat.nr_immediate; in shrink_inactive_list()
1995 sc->nr.taken += nr_taken; in shrink_inactive_list()
1997 sc->nr.file_taken += nr_taken; in shrink_inactive_list()
1999 trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id, in shrink_inactive_list()
2000 nr_scanned, nr_reclaimed, &stat, sc->priority, file); in shrink_inactive_list()
2015 * It is safe to rely on PG_active against the non-LRU pages in here because
2016 * nobody will play with that bit on a non-LRU page.
2018 * The downside is that we have to touch page->_refcount against each page.
2019 * But we had to alter page->flags anyway.
2043 list_move(&page->lru, &lruvec->lists[lru]); in move_active_pages_to_lru()
2051 spin_unlock_irq(&pgdat->lru_lock); in move_active_pages_to_lru()
2054 spin_lock_irq(&pgdat->lru_lock); in move_active_pages_to_lru()
2056 list_add(&page->lru, pages_to_free); in move_active_pages_to_lru()
2083 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat; in shrink_active_list()
2092 if (!sc->may_unmap) in shrink_active_list()
2095 spin_lock_irq(&pgdat->lru_lock); in shrink_active_list()
2101 reclaim_stat->recent_scanned[file] += nr_taken; in shrink_active_list()
2106 spin_unlock_irq(&pgdat->lru_lock); in shrink_active_list()
2111 list_del(&page->lru); in shrink_active_list()
2126 if (page_referenced(page, 0, sc->target_mem_cgroup, in shrink_active_list()
2130 * Identify referenced, file-backed active pages and in shrink_active_list()
2134 * are not likely to be evicted by use-once streaming in shrink_active_list()
2139 list_add(&page->lru, &l_active); in shrink_active_list()
2144 ClearPageActive(page); /* we are de-activating */ in shrink_active_list()
2145 list_add(&page->lru, &l_inactive); in shrink_active_list()
2151 spin_lock_irq(&pgdat->lru_lock); in shrink_active_list()
2154 * even though only some of them are actually re-activated. This in shrink_active_list()
2158 reclaim_stat->recent_rotated[file] += nr_rotated; in shrink_active_list()
2161 nr_deactivate = move_active_pages_to_lru(lruvec, &l_inactive, &l_hold, lru - LRU_ACTIVE); in shrink_active_list()
2162 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); in shrink_active_list()
2163 spin_unlock_irq(&pgdat->lru_lock); in shrink_active_list()
2167 trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate, in shrink_active_list()
2168 nr_deactivate, nr_rotated, sc->priority, file); in shrink_active_list()
2176 * to the established workingset on the scan-resistant active list,
2190 * -------------------------------------
2217 inactive = lruvec_lru_size(lruvec, inactive_lru, sc->reclaim_idx); in inactive_list_is_low()
2218 active = lruvec_lru_size(lruvec, active_lru, sc->reclaim_idx); in inactive_list_is_low()
2226 if (file && lruvec->refaults != refaults) { in inactive_list_is_low()
2229 gb = (inactive + active) >> (30 - PAGE_SHIFT); in inactive_list_is_low()
2237 trace_mm_vmscan_inactive_list_is_low(pgdat->node_id, sc->reclaim_idx, in inactive_list_is_low()
2278 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat; in get_scan_count()
2289 if (!sc->may_swap || mem_cgroup_get_nr_swap_pages(memcg) <= 0) { in get_scan_count()
2311 if (!sc->priority && swappiness) { in get_scan_count()
2331 pgdatfree = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); in get_scan_count()
2336 struct zone *zone = &pgdat->node_zones[z]; in get_scan_count()
2350 lruvec_lru_size(lruvec, LRU_INACTIVE_ANON, sc->reclaim_idx) in get_scan_count()
2351 >> sc->priority) { in get_scan_count()
2368 lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, sc->reclaim_idx) >> sc->priority) { in get_scan_count()
2380 file_prio = 200 - anon_prio; in get_scan_count()
2399 spin_lock_irq(&pgdat->lru_lock); in get_scan_count()
2400 if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) { in get_scan_count()
2401 reclaim_stat->recent_scanned[0] /= 2; in get_scan_count()
2402 reclaim_stat->recent_rotated[0] /= 2; in get_scan_count()
2405 if (unlikely(reclaim_stat->recent_scanned[1] > file / 4)) { in get_scan_count()
2406 reclaim_stat->recent_scanned[1] /= 2; in get_scan_count()
2407 reclaim_stat->recent_rotated[1] /= 2; in get_scan_count()
2415 ap = anon_prio * (reclaim_stat->recent_scanned[0] + 1); in get_scan_count()
2416 ap /= reclaim_stat->recent_rotated[0] + 1; in get_scan_count()
2418 fp = file_prio * (reclaim_stat->recent_scanned[1] + 1); in get_scan_count()
2419 fp /= reclaim_stat->recent_rotated[1] + 1; in get_scan_count()
2420 spin_unlock_irq(&pgdat->lru_lock); in get_scan_count()
2432 size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx); in get_scan_count()
2433 scan = size >> sc->priority; in get_scan_count()
2451 * round-off error. in get_scan_count()
2477 * This is a basic per-node page freer. Used by both kswapd and direct reclaim.
2488 unsigned long nr_to_reclaim = sc->nr_to_reclaim; in shrink_node_memcg()
2509 sc->priority == DEF_PRIORITY); in shrink_node_memcg()
2520 nr[lru] -= nr_to_scan; in shrink_node_memcg()
2572 nr_scanned = targets[lru] - nr[lru]; in shrink_node_memcg()
2573 nr[lru] = targets[lru] * (100 - percentage) / 100; in shrink_node_memcg()
2574 nr[lru] -= min(nr[lru], nr_scanned); in shrink_node_memcg()
2577 nr_scanned = targets[lru] - nr[lru]; in shrink_node_memcg()
2578 nr[lru] = targets[lru] * (100 - percentage) / 100; in shrink_node_memcg()
2579 nr[lru] -= min(nr[lru], nr_scanned); in shrink_node_memcg()
2584 sc->nr_reclaimed += nr_reclaimed; in shrink_node_memcg()
2598 if (IS_ENABLED(CONFIG_COMPACTION) && sc->order && in in_reclaim_compaction()
2599 (sc->order > PAGE_ALLOC_COSTLY_ORDER || in in_reclaim_compaction()
2600 sc->priority < DEF_PRIORITY - 2)) in in_reclaim_compaction()
2607 * Reclaim/compaction is used for high-order allocation requests. It reclaims
2608 * order-0 pages before compacting the zone. should_continue_reclaim() returns
2627 if (sc->gfp_mask & __GFP_RETRY_MAYFAIL) { in should_continue_reclaim()
2638 * For non-__GFP_RETRY_MAYFAIL allocations which can presumably in should_continue_reclaim()
2653 pages_for_compaction = compact_gap(sc->order); in should_continue_reclaim()
2657 if (sc->nr_reclaimed < pages_for_compaction && in should_continue_reclaim()
2662 for (z = 0; z <= sc->reclaim_idx; z++) { in should_continue_reclaim()
2663 struct zone *zone = &pgdat->node_zones[z]; in should_continue_reclaim()
2667 switch (compaction_suitable(zone, sc->order, 0, sc->reclaim_idx)) { in should_continue_reclaim()
2681 return test_bit(PGDAT_CONGESTED, &pgdat->flags) || in pgdat_memcg_congested()
2687 struct reclaim_state *reclaim_state = current->reclaim_state; in shrink_node()
2692 struct mem_cgroup *root = sc->target_mem_cgroup; in shrink_node()
2695 .priority = sc->priority, in shrink_node()
2700 memset(&sc->nr, 0, sizeof(sc->nr)); in shrink_node()
2702 nr_reclaimed = sc->nr_reclaimed; in shrink_node()
2703 nr_scanned = sc->nr_scanned; in shrink_node()
2712 * This loop can become CPU-bound when target memcgs in shrink_node()
2713 * aren't eligible for reclaim - either because they in shrink_node()
2733 if (!sc->memcg_low_reclaim) { in shrink_node()
2734 sc->memcg_low_skipped = 1; in shrink_node()
2743 reclaimed = sc->nr_reclaimed; in shrink_node()
2744 scanned = sc->nr_scanned; in shrink_node()
2748 shrink_slab(sc->gfp_mask, pgdat->node_id, in shrink_node()
2749 memcg, sc->priority); in shrink_node()
2752 vmpressure(sc->gfp_mask, memcg, false, in shrink_node()
2753 sc->nr_scanned - scanned, in shrink_node()
2754 sc->nr_reclaimed - reclaimed); in shrink_node()
2767 sc->nr_reclaimed >= sc->nr_to_reclaim) { in shrink_node()
2774 sc->nr_reclaimed += reclaim_state->reclaimed_slab; in shrink_node()
2775 reclaim_state->reclaimed_slab = 0; in shrink_node()
2779 vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true, in shrink_node()
2780 sc->nr_scanned - nr_scanned, in shrink_node()
2781 sc->nr_reclaimed - nr_reclaimed); in shrink_node()
2783 if (sc->nr_reclaimed - nr_reclaimed) in shrink_node()
2789 * it implies that the long-lived page allocation rate in shrink_node()
2804 if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken) in shrink_node()
2805 set_bit(PGDAT_WRITEBACK, &pgdat->flags); in shrink_node()
2812 if (sc->nr.dirty && sc->nr.dirty == sc->nr.congested) in shrink_node()
2813 set_bit(PGDAT_CONGESTED, &pgdat->flags); in shrink_node()
2816 if (sc->nr.unqueued_dirty == sc->nr.file_taken) in shrink_node()
2817 set_bit(PGDAT_DIRTY, &pgdat->flags); in shrink_node()
2825 if (sc->nr.immediate) in shrink_node()
2834 sc->nr.dirty && sc->nr.dirty == sc->nr.congested) in shrink_node()
2843 if (!sc->hibernation_mode && !current_is_kswapd() && in shrink_node()
2847 } while (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed, in shrink_node()
2848 sc->nr_scanned - nr_scanned, sc)); in shrink_node()
2857 pgdat->kswapd_failures = 0; in shrink_node()
2863 * Returns true if compaction should go ahead for a costly-order request, or
2872 suitable = compaction_suitable(zone, sc->order, 0, sc->reclaim_idx); in compaction_ready()
2889 watermark = high_wmark_pages(zone) + compact_gap(sc->order); in compaction_ready()
2891 return zone_watermark_ok_safe(zone, 0, watermark, sc->reclaim_idx); in compaction_ready()
2895 * This is the direct reclaim path, for page-allocating processes. We only
2916 orig_mask = sc->gfp_mask; in shrink_zones()
2918 sc->gfp_mask |= __GFP_HIGHMEM; in shrink_zones()
2919 sc->reclaim_idx = gfp_zone(sc->gfp_mask); in shrink_zones()
2923 sc->reclaim_idx, sc->nodemask) { in shrink_zones()
2937 * non-zero order, only frequent costly order in shrink_zones()
2943 sc->order > PAGE_ALLOC_COSTLY_ORDER && in shrink_zones()
2945 sc->compaction_ready = true; in shrink_zones()
2955 if (zone->zone_pgdat == last_pgdat) in shrink_zones()
2965 nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone->zone_pgdat, in shrink_zones()
2966 sc->order, sc->gfp_mask, in shrink_zones()
2968 sc->nr_reclaimed += nr_soft_reclaimed; in shrink_zones()
2969 sc->nr_scanned += nr_soft_scanned; in shrink_zones()
2974 if (zone->zone_pgdat == last_pgdat) in shrink_zones()
2976 last_pgdat = zone->zone_pgdat; in shrink_zones()
2977 shrink_node(zone->zone_pgdat, sc); in shrink_zones()
2984 sc->gfp_mask = orig_mask; in shrink_zones()
2998 lruvec->refaults = refaults; in snapshot_refaults()
3009 * high - the zone may be full of dirty or under-writeback pages, which this
3021 int initial_priority = sc->priority; in do_try_to_free_pages()
3029 __count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1); in do_try_to_free_pages()
3032 vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup, in do_try_to_free_pages()
3033 sc->priority); in do_try_to_free_pages()
3034 sc->nr_scanned = 0; in do_try_to_free_pages()
3037 if (sc->nr_reclaimed >= sc->nr_to_reclaim) in do_try_to_free_pages()
3040 if (sc->compaction_ready) in do_try_to_free_pages()
3047 if (sc->priority < DEF_PRIORITY - 2) in do_try_to_free_pages()
3048 sc->may_writepage = 1; in do_try_to_free_pages()
3049 } while (--sc->priority >= 0); in do_try_to_free_pages()
3052 for_each_zone_zonelist_nodemask(zone, z, zonelist, sc->reclaim_idx, in do_try_to_free_pages()
3053 sc->nodemask) { in do_try_to_free_pages()
3054 if (zone->zone_pgdat == last_pgdat) in do_try_to_free_pages()
3056 last_pgdat = zone->zone_pgdat; in do_try_to_free_pages()
3057 snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat); in do_try_to_free_pages()
3058 set_memcg_congestion(last_pgdat, sc->target_mem_cgroup, false); in do_try_to_free_pages()
3063 if (sc->nr_reclaimed) in do_try_to_free_pages()
3064 return sc->nr_reclaimed; in do_try_to_free_pages()
3067 if (sc->compaction_ready) in do_try_to_free_pages()
3071 if (sc->memcg_low_skipped) { in do_try_to_free_pages()
3072 sc->priority = initial_priority; in do_try_to_free_pages()
3073 sc->memcg_low_reclaim = 1; in do_try_to_free_pages()
3074 sc->memcg_low_skipped = 0; in do_try_to_free_pages()
3089 if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES) in allow_direct_reclaim()
3093 zone = &pgdat->node_zones[i]; in allow_direct_reclaim()
3111 if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) { in allow_direct_reclaim()
3112 if (READ_ONCE(pgdat->kswapd_classzone_idx) > ZONE_NORMAL) in allow_direct_reclaim()
3113 WRITE_ONCE(pgdat->kswapd_classzone_idx, ZONE_NORMAL); in allow_direct_reclaim()
3115 wake_up_interruptible(&pgdat->kswapd_wait); in allow_direct_reclaim()
3144 if (current->flags & PF_KTHREAD) in throttle_direct_reclaim()
3174 pgdat = zone->zone_pgdat; in throttle_direct_reclaim()
3196 wait_event_interruptible_timeout(pgdat->pfmemalloc_wait, in throttle_direct_reclaim()
3203 wait_event_killable(zone->zone_pgdat->pfmemalloc_wait, in throttle_direct_reclaim()
3270 .reclaim_idx = MAX_NR_ZONES - 1, in mem_cgroup_shrink_node()
3311 .reclaim_idx = MAX_NR_ZONES - 1, in try_to_free_mem_cgroup_pages()
3326 zonelist = &NODE_DATA(nid)->node_zonelists[ZONELIST_FALLBACK]; in try_to_free_mem_cgroup_pages()
3370 unsigned long mark = -1; in pgdat_balanced()
3374 zone = pgdat->node_zones + i; in pgdat_balanced()
3386 * need balancing by definition. This can happen if a zone-restricted in pgdat_balanced()
3389 if (mark == -1) in pgdat_balanced()
3398 clear_bit(PGDAT_CONGESTED, &pgdat->flags); in clear_pgdat_congested()
3399 clear_bit(PGDAT_DIRTY, &pgdat->flags); in clear_pgdat_congested()
3400 clear_bit(PGDAT_WRITEBACK, &pgdat->flags); in clear_pgdat_congested()
3424 if (waitqueue_active(&pgdat->pfmemalloc_wait)) in prepare_kswapd_sleep()
3425 wake_up_all(&pgdat->pfmemalloc_wait); in prepare_kswapd_sleep()
3428 if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES) in prepare_kswapd_sleep()
3454 sc->nr_to_reclaim = 0; in kswapd_shrink_node()
3455 for (z = 0; z <= sc->reclaim_idx; z++) { in kswapd_shrink_node()
3456 zone = pgdat->node_zones + z; in kswapd_shrink_node()
3460 sc->nr_to_reclaim += max(high_wmark_pages(zone), SWAP_CLUSTER_MAX); in kswapd_shrink_node()
3471 * high-order allocations. If twice the allocation size has been in kswapd_shrink_node()
3472 * reclaimed then recheck watermarks only at order-0 to prevent in kswapd_shrink_node()
3473 * excessive reclaim. Assume that a process requested a high-order in kswapd_shrink_node()
3476 if (sc->order && sc->nr_reclaimed >= compact_gap(sc->order)) in kswapd_shrink_node()
3477 sc->order = 0; in kswapd_shrink_node()
3479 return sc->nr_scanned >= sc->nr_to_reclaim; in kswapd_shrink_node()
3489 * kswapd scans the zones in the highmem->normal->dma direction. It skips
3524 * purpose -- on 64-bit systems it is expected that in balance_pgdat()
3525 * buffer_heads are stripped during active rotation. On 32-bit in balance_pgdat()
3532 for (i = MAX_NR_ZONES - 1; i >= 0; i--) { in balance_pgdat()
3533 zone = pgdat->node_zones + i; in balance_pgdat()
3562 if (sc.priority < DEF_PRIORITY - 2) in balance_pgdat()
3585 if (waitqueue_active(&pgdat->pfmemalloc_wait) && in balance_pgdat()
3587 wake_up_all(&pgdat->pfmemalloc_wait); in balance_pgdat()
3600 nr_reclaimed = sc.nr_reclaimed - nr_reclaimed; in balance_pgdat()
3602 sc.priority--; in balance_pgdat()
3606 pgdat->kswapd_failures++; in balance_pgdat()
3621 * The pgdat->kswapd_classzone_idx is used to pass the highest zone index to be
3630 enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_classzone_idx); in kswapd_classzone_idx()
3644 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE); in kswapd_try_to_sleep()
3676 WRITE_ONCE(pgdat->kswapd_classzone_idx, in kswapd_try_to_sleep()
3679 if (READ_ONCE(pgdat->kswapd_order) < reclaim_order) in kswapd_try_to_sleep()
3680 WRITE_ONCE(pgdat->kswapd_order, reclaim_order); in kswapd_try_to_sleep()
3683 finish_wait(&pgdat->kswapd_wait, &wait); in kswapd_try_to_sleep()
3684 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE); in kswapd_try_to_sleep()
3693 trace_mm_vmscan_kswapd_sleep(pgdat->node_id); in kswapd_try_to_sleep()
3700 * per-cpu vmstat threshold while kswapd is awake and restore in kswapd_try_to_sleep()
3715 finish_wait(&pgdat->kswapd_wait, &wait); in kswapd_try_to_sleep()
3728 * If there are applications that are active memory-allocators
3734 unsigned int classzone_idx = MAX_NR_ZONES - 1; in kswapd()
3741 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); in kswapd()
3745 current->reclaim_state = &reclaim_state; in kswapd()
3759 tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD; in kswapd()
3762 WRITE_ONCE(pgdat->kswapd_order, 0); in kswapd()
3763 WRITE_ONCE(pgdat->kswapd_classzone_idx, MAX_NR_ZONES); in kswapd()
3767 alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order); in kswapd()
3775 alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order); in kswapd()
3777 WRITE_ONCE(pgdat->kswapd_order, 0); in kswapd()
3778 WRITE_ONCE(pgdat->kswapd_classzone_idx, MAX_NR_ZONES); in kswapd()
3792 * Reclaim begins at the requested order but if a high-order in kswapd()
3794 * order-0. If that happens, kswapd will consider sleeping in kswapd()
3799 trace_mm_vmscan_kswapd_wake(pgdat->node_id, classzone_idx, in kswapd()
3806 tsk->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD); in kswapd()
3807 current->reclaim_state = NULL; in kswapd()
3813 * A zone is low on free memory or too fragmented for high-order memory. If
3831 pgdat = zone->zone_pgdat; in wakeup_kswapd()
3832 curr_idx = READ_ONCE(pgdat->kswapd_classzone_idx); in wakeup_kswapd()
3835 WRITE_ONCE(pgdat->kswapd_classzone_idx, classzone_idx); in wakeup_kswapd()
3837 if (READ_ONCE(pgdat->kswapd_order) < order) in wakeup_kswapd()
3838 WRITE_ONCE(pgdat->kswapd_order, order); in wakeup_kswapd()
3840 if (!waitqueue_active(&pgdat->kswapd_wait)) in wakeup_kswapd()
3844 if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES || in wakeup_kswapd()
3848 * fragmented for high-order allocations. Wake up kcompactd in wakeup_kswapd()
3858 trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, classzone_idx, order, in wakeup_kswapd()
3860 wake_up_interruptible(&pgdat->kswapd_wait); in wakeup_kswapd()
3865 * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of
3878 .reclaim_idx = MAX_NR_ZONES - 1, in shrink_all_memory()
3893 p->reclaim_state = &reclaim_state; in shrink_all_memory()
3897 p->reclaim_state = NULL; in shrink_all_memory()
3917 mask = cpumask_of_node(pgdat->node_id); in kswapd_cpu_online()
3921 set_cpus_allowed_ptr(pgdat->kswapd, mask); in kswapd_cpu_online()
3927 * This kswapd start function will be called by init and node-hot-add.
3928 * On node-hot-add, kswapd will moved to proper cpus if cpus are hot-added.
3935 if (pgdat->kswapd) in kswapd_run()
3938 pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid); in kswapd_run()
3939 if (IS_ERR(pgdat->kswapd)) { in kswapd_run()
3943 ret = PTR_ERR(pgdat->kswapd); in kswapd_run()
3944 pgdat->kswapd = NULL; in kswapd_run()
3955 struct task_struct *kswapd = NODE_DATA(nid)->kswapd; in kswapd_stop()
3959 NODE_DATA(nid)->kswapd = NULL; in kswapd_stop()
3983 * If non-zero call node_reclaim when the number of free pages falls below
4023 return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0; in node_unmapped_file_pages()
4051 return nr_pagecache_reclaimable - delta; in node_pagecache_reclaimable()
4083 p->flags |= PF_SWAPWRITE; in __node_reclaim()
4085 p->reclaim_state = &reclaim_state; in __node_reclaim()
4087 if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages) { in __node_reclaim()
4094 } while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0); in __node_reclaim()
4097 p->reclaim_state = NULL; in __node_reclaim()
4098 current->flags &= ~PF_SWAPWRITE; in __node_reclaim()
4118 if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages && in node_reclaim()
4119 node_page_state(pgdat, NR_SLAB_RECLAIMABLE) <= pgdat->min_slab_pages) in node_reclaim()
4125 if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC)) in node_reclaim()
4130 * have associated processors. This will favor the local processor in node_reclaim()
4134 if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id()) in node_reclaim()
4137 if (test_and_set_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags)) in node_reclaim()
4141 clear_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags); in node_reclaim()
4151 * page_evictable - test whether a page is evictable
4154 * Test whether page is evictable--i.e., should be placed on active/inactive
4175 * check_move_unevictable_pages - check pages for evictability and move to appropriate zone lru list
4198 spin_unlock_irq(&pgdat->lru_lock); in check_move_unevictable_pages()
4200 spin_lock_irq(&pgdat->lru_lock); in check_move_unevictable_pages()
4221 spin_unlock_irq(&pgdat->lru_lock); in check_move_unevictable_pages()