Lines Matching refs:gfp_mask
3343 static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) in __should_fail_alloc_page() argument
3347 if (gfp_mask & __GFP_NOFAIL) in __should_fail_alloc_page()
3349 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) in __should_fail_alloc_page()
3352 (gfp_mask & __GFP_DIRECT_RECLAIM)) in __should_fail_alloc_page()
3383 static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) in __should_fail_alloc_page() argument
3390 static noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) in should_fail_alloc_page() argument
3392 return __should_fail_alloc_page(gfp_mask, order); in should_fail_alloc_page()
3548 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) in alloc_flags_nofragment() argument
3552 if (gfp_mask & __GFP_KSWAPD_RECLAIM) in alloc_flags_nofragment()
3581 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, in get_page_from_freelist() argument
3603 !__cpuset_zone_allowed(zone, gfp_mask)) in get_page_from_freelist()
3674 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); in get_page_from_freelist()
3694 gfp_mask, alloc_flags, ac->migratetype); in get_page_from_freelist()
3696 prep_new_page(page, order, gfp_mask, alloc_flags); in get_page_from_freelist()
3729 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) in warn_alloc_show_mem() argument
3738 if (!(gfp_mask & __GFP_NOMEMALLOC)) in warn_alloc_show_mem()
3742 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) in warn_alloc_show_mem()
3748 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) in warn_alloc() argument
3754 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs)) in warn_alloc()
3761 current->comm, &vaf, gfp_mask, &gfp_mask, in warn_alloc()
3768 warn_alloc_show_mem(gfp_mask, nodemask); in warn_alloc()
3772 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order, in __alloc_pages_cpuset_fallback() argument
3778 page = get_page_from_freelist(gfp_mask, order, in __alloc_pages_cpuset_fallback()
3785 page = get_page_from_freelist(gfp_mask, order, in __alloc_pages_cpuset_fallback()
3792 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, in __alloc_pages_may_oom() argument
3799 .gfp_mask = gfp_mask, in __alloc_pages_may_oom()
3823 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & in __alloc_pages_may_oom()
3841 if (gfp_mask & __GFP_RETRY_MAYFAIL) in __alloc_pages_may_oom()
3859 if (gfp_mask & __GFP_THISNODE) in __alloc_pages_may_oom()
3863 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { in __alloc_pages_may_oom()
3870 if (gfp_mask & __GFP_NOFAIL) in __alloc_pages_may_oom()
3871 page = __alloc_pages_cpuset_fallback(gfp_mask, order, in __alloc_pages_may_oom()
3888 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, in __alloc_pages_direct_compact() argument
3902 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, in __alloc_pages_direct_compact()
3916 prep_new_page(page, order, gfp_mask, alloc_flags); in __alloc_pages_direct_compact()
3920 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_direct_compact()
4021 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, in __alloc_pages_direct_compact() argument
4061 static bool __need_fs_reclaim(gfp_t gfp_mask) in __need_fs_reclaim() argument
4063 gfp_mask = current_gfp_context(gfp_mask); in __need_fs_reclaim()
4066 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) in __need_fs_reclaim()
4074 if (!(gfp_mask & __GFP_FS)) in __need_fs_reclaim()
4077 if (gfp_mask & __GFP_NOLOCKDEP) in __need_fs_reclaim()
4093 void fs_reclaim_acquire(gfp_t gfp_mask) in fs_reclaim_acquire() argument
4095 if (__need_fs_reclaim(gfp_mask)) in fs_reclaim_acquire()
4100 void fs_reclaim_release(gfp_t gfp_mask) in fs_reclaim_release() argument
4102 if (__need_fs_reclaim(gfp_mask)) in fs_reclaim_release()
4110 __perform_reclaim(gfp_t gfp_mask, unsigned int order, in __perform_reclaim() argument
4122 fs_reclaim_acquire(gfp_mask); in __perform_reclaim()
4125 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, in __perform_reclaim()
4129 fs_reclaim_release(gfp_mask); in __perform_reclaim()
4139 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, in __alloc_pages_direct_reclaim() argument
4146 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); in __alloc_pages_direct_reclaim()
4151 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_direct_reclaim()
4168 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, in wake_all_kswapds() argument
4179 wakeup_kswapd(zone, gfp_mask, order, high_zoneidx); in wake_all_kswapds()
4185 gfp_to_alloc_flags(gfp_t gfp_mask) in gfp_to_alloc_flags() argument
4198 alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH); in gfp_to_alloc_flags()
4200 if (gfp_mask & __GFP_ATOMIC) { in gfp_to_alloc_flags()
4205 if (!(gfp_mask & __GFP_NOMEMALLOC)) in gfp_to_alloc_flags()
4215 if (gfp_mask & __GFP_KSWAPD_RECLAIM) in gfp_to_alloc_flags()
4219 if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE) in gfp_to_alloc_flags()
4244 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask) in __gfp_pfmemalloc_flags() argument
4246 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) in __gfp_pfmemalloc_flags()
4248 if (gfp_mask & __GFP_MEMALLOC) in __gfp_pfmemalloc_flags()
4262 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) in gfp_pfmemalloc_allowed() argument
4264 return !!__gfp_pfmemalloc_flags(gfp_mask); in gfp_pfmemalloc_allowed()
4278 should_reclaim_retry(gfp_t gfp_mask, unsigned order, in should_reclaim_retry() argument
4402 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, in __alloc_pages_slowpath() argument
4405 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; in __alloc_pages_slowpath()
4421 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == in __alloc_pages_slowpath()
4423 gfp_mask &= ~__GFP_ATOMIC; in __alloc_pages_slowpath()
4436 alloc_flags = gfp_to_alloc_flags(gfp_mask); in __alloc_pages_slowpath()
4450 wake_all_kswapds(order, gfp_mask, ac); in __alloc_pages_slowpath()
4456 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_slowpath()
4472 && !gfp_pfmemalloc_allowed(gfp_mask)) { in __alloc_pages_slowpath()
4473 page = __alloc_pages_direct_compact(gfp_mask, order, in __alloc_pages_slowpath()
4480 if (order >= pageblock_order && (gfp_mask & __GFP_IO) && in __alloc_pages_slowpath()
4481 !(gfp_mask & __GFP_RETRY_MAYFAIL)) { in __alloc_pages_slowpath()
4508 if (costly_order && (gfp_mask & __GFP_NORETRY)) { in __alloc_pages_slowpath()
4532 wake_all_kswapds(order, gfp_mask, ac); in __alloc_pages_slowpath()
4534 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); in __alloc_pages_slowpath()
4550 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_slowpath()
4563 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, in __alloc_pages_slowpath()
4569 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, in __alloc_pages_slowpath()
4575 if (gfp_mask & __GFP_NORETRY) in __alloc_pages_slowpath()
4582 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL)) in __alloc_pages_slowpath()
4585 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, in __alloc_pages_slowpath()
4607 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); in __alloc_pages_slowpath()
4614 (gfp_mask & __GFP_NOMEMALLOC))) in __alloc_pages_slowpath()
4632 if (gfp_mask & __GFP_NOFAIL) { in __alloc_pages_slowpath()
4661 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac); in __alloc_pages_slowpath()
4669 warn_alloc(gfp_mask, ac->nodemask, in __alloc_pages_slowpath()
4675 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, in prepare_alloc_pages() argument
4680 ac->high_zoneidx = gfp_zone(gfp_mask); in prepare_alloc_pages()
4681 ac->zonelist = node_zonelist(preferred_nid, gfp_mask); in prepare_alloc_pages()
4683 ac->migratetype = gfpflags_to_migratetype(gfp_mask); in prepare_alloc_pages()
4693 fs_reclaim_acquire(gfp_mask); in prepare_alloc_pages()
4694 fs_reclaim_release(gfp_mask); in prepare_alloc_pages()
4696 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM); in prepare_alloc_pages()
4698 if (should_fail_alloc_page(gfp_mask, order)) in prepare_alloc_pages()
4708 static inline void finalise_ac(gfp_t gfp_mask, struct alloc_context *ac) in finalise_ac() argument
4711 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); in finalise_ac()
4726 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid, in __alloc_pages_nodemask() argument
4739 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN)); in __alloc_pages_nodemask()
4743 gfp_mask &= gfp_allowed_mask; in __alloc_pages_nodemask()
4744 alloc_mask = gfp_mask; in __alloc_pages_nodemask()
4745 if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags)) in __alloc_pages_nodemask()
4748 finalise_ac(gfp_mask, &ac); in __alloc_pages_nodemask()
4754 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp_mask); in __alloc_pages_nodemask()
4767 alloc_mask = current_gfp_context(gfp_mask); in __alloc_pages_nodemask()
4780 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page && in __alloc_pages_nodemask()
4781 unlikely(__memcg_kmem_charge(page, gfp_mask, order) != 0)) { in __alloc_pages_nodemask()
4797 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) in __get_free_pages() argument
4801 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order); in __get_free_pages()
4808 unsigned long get_zeroed_page(gfp_t gfp_mask) in get_zeroed_page() argument
4810 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); in get_zeroed_page()
4851 gfp_t gfp_mask) in __page_frag_cache_refill() argument
4854 gfp_t gfp = gfp_mask; in __page_frag_cache_refill()
4857 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | in __page_frag_cache_refill()
4859 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, in __page_frag_cache_refill()
4881 unsigned int fragsz, gfp_t gfp_mask) in page_frag_alloc() argument
4889 page = __page_frag_cache_refill(nc, gfp_mask); in page_frag_alloc()
4977 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) in alloc_pages_exact() argument
4982 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) in alloc_pages_exact()
4983 gfp_mask &= ~__GFP_COMP; in alloc_pages_exact()
4985 addr = __get_free_pages(gfp_mask, order); in alloc_pages_exact()
5002 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) in alloc_pages_exact_nid() argument
5007 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) in alloc_pages_exact_nid()
5008 gfp_mask &= ~__GFP_COMP; in alloc_pages_exact_nid()
5010 p = alloc_pages_node(nid, gfp_mask, order); in alloc_pages_exact_nid()
8388 unsigned migratetype, gfp_t gfp_mask) in alloc_contig_range() argument
8401 .gfp_mask = current_gfp_context(gfp_mask), in alloc_contig_range()