Lines Matching refs:gfp_mask
3548 static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) in __should_fail_alloc_page() argument
3552 if (gfp_mask & __GFP_NOFAIL) in __should_fail_alloc_page()
3554 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) in __should_fail_alloc_page()
3557 (gfp_mask & __GFP_DIRECT_RECLAIM)) in __should_fail_alloc_page()
3588 static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) in __should_fail_alloc_page() argument
3595 noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) in should_fail_alloc_page() argument
3597 return __should_fail_alloc_page(gfp_mask, order); in should_fail_alloc_page()
3703 unsigned int alloc_flags, gfp_t gfp_mask) in zone_watermark_fast() argument
3735 if (unlikely(!order && (gfp_mask & __GFP_ATOMIC) && z->watermark_boost in zone_watermark_fast()
3779 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) in alloc_flags_nofragment() argument
3787 alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM); in alloc_flags_nofragment()
3810 static inline unsigned int current_alloc_flags(gfp_t gfp_mask, in current_alloc_flags() argument
3817 gfp_migratetype(gfp_mask) == get_cma_migratetype()) in current_alloc_flags()
3829 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, in get_page_from_freelist() argument
3851 !__cpuset_zone_allowed(zone, gfp_mask)) in get_page_from_freelist()
3901 gfp_mask)) { in get_page_from_freelist()
3923 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); in get_page_from_freelist()
3943 gfp_mask, alloc_flags, ac->migratetype); in get_page_from_freelist()
3945 prep_new_page(page, order, gfp_mask, alloc_flags); in get_page_from_freelist()
3978 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) in warn_alloc_show_mem() argument
3987 if (!(gfp_mask & __GFP_NOMEMALLOC)) in warn_alloc_show_mem()
3991 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) in warn_alloc_show_mem()
3997 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) in warn_alloc() argument
4003 if ((gfp_mask & __GFP_NOWARN) || in warn_alloc()
4005 ((gfp_mask & __GFP_DMA) && !has_managed_dma())) in warn_alloc()
4012 current->comm, &vaf, gfp_mask, &gfp_mask, in warn_alloc()
4019 warn_alloc_show_mem(gfp_mask, nodemask); in warn_alloc()
4023 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order, in __alloc_pages_cpuset_fallback() argument
4029 page = get_page_from_freelist(gfp_mask, order, in __alloc_pages_cpuset_fallback()
4036 page = get_page_from_freelist(gfp_mask, order, in __alloc_pages_cpuset_fallback()
4043 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, in __alloc_pages_may_oom() argument
4050 .gfp_mask = gfp_mask, in __alloc_pages_may_oom()
4074 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & in __alloc_pages_may_oom()
4094 if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE)) in __alloc_pages_may_oom()
4112 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { in __alloc_pages_may_oom()
4119 if (gfp_mask & __GFP_NOFAIL) in __alloc_pages_may_oom()
4120 page = __alloc_pages_cpuset_fallback(gfp_mask, order, in __alloc_pages_may_oom()
4137 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, in __alloc_pages_direct_compact() argument
4151 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, in __alloc_pages_direct_compact()
4165 prep_new_page(page, order, gfp_mask, alloc_flags); in __alloc_pages_direct_compact()
4169 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_direct_compact()
4270 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, in __alloc_pages_direct_compact() argument
4310 static bool __need_fs_reclaim(gfp_t gfp_mask) in __need_fs_reclaim() argument
4312 gfp_mask = current_gfp_context(gfp_mask); in __need_fs_reclaim()
4315 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) in __need_fs_reclaim()
4323 if (!(gfp_mask & __GFP_FS)) in __need_fs_reclaim()
4326 if (gfp_mask & __GFP_NOLOCKDEP) in __need_fs_reclaim()
4342 void fs_reclaim_acquire(gfp_t gfp_mask) in fs_reclaim_acquire() argument
4344 if (__need_fs_reclaim(gfp_mask)) in fs_reclaim_acquire()
4349 void fs_reclaim_release(gfp_t gfp_mask) in fs_reclaim_release() argument
4351 if (__need_fs_reclaim(gfp_mask)) in fs_reclaim_release()
4383 __perform_reclaim(gfp_t gfp_mask, unsigned int order, in __perform_reclaim() argument
4394 fs_reclaim_acquire(gfp_mask); in __perform_reclaim()
4397 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, in __perform_reclaim()
4401 fs_reclaim_release(gfp_mask); in __perform_reclaim()
4411 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, in __alloc_pages_direct_reclaim() argument
4418 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); in __alloc_pages_direct_reclaim()
4423 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_direct_reclaim()
4446 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, in wake_all_kswapds() argument
4457 wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx); in wake_all_kswapds()
4463 gfp_to_alloc_flags(gfp_t gfp_mask) in gfp_to_alloc_flags() argument
4482 (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM)); in gfp_to_alloc_flags()
4484 if (gfp_mask & __GFP_ATOMIC) { in gfp_to_alloc_flags()
4489 if (!(gfp_mask & __GFP_NOMEMALLOC)) in gfp_to_alloc_flags()
4499 alloc_flags = current_alloc_flags(gfp_mask, alloc_flags); in gfp_to_alloc_flags()
4523 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask) in __gfp_pfmemalloc_flags() argument
4525 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) in __gfp_pfmemalloc_flags()
4527 if (gfp_mask & __GFP_MEMALLOC) in __gfp_pfmemalloc_flags()
4541 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) in gfp_pfmemalloc_allowed() argument
4543 return !!__gfp_pfmemalloc_flags(gfp_mask); in gfp_pfmemalloc_allowed()
4557 should_reclaim_retry(gfp_t gfp_mask, unsigned order, in should_reclaim_retry() argument
4681 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, in __alloc_pages_slowpath() argument
4684 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; in __alloc_pages_slowpath()
4704 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == in __alloc_pages_slowpath()
4706 gfp_mask &= ~__GFP_ATOMIC; in __alloc_pages_slowpath()
4720 alloc_flags = gfp_to_alloc_flags(gfp_mask); in __alloc_pages_slowpath()
4734 wake_all_kswapds(order, gfp_mask, ac); in __alloc_pages_slowpath()
4740 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_slowpath()
4756 && !gfp_pfmemalloc_allowed(gfp_mask)) { in __alloc_pages_slowpath()
4757 page = __alloc_pages_direct_compact(gfp_mask, order, in __alloc_pages_slowpath()
4768 if (costly_order && (gfp_mask & __GFP_NORETRY)) { in __alloc_pages_slowpath()
4802 wake_all_kswapds(order, gfp_mask, ac); in __alloc_pages_slowpath()
4804 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); in __alloc_pages_slowpath()
4806 alloc_flags = current_alloc_flags(gfp_mask, reserve_flags); in __alloc_pages_slowpath()
4820 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_slowpath()
4836 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, in __alloc_pages_slowpath()
4845 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, in __alloc_pages_slowpath()
4851 if (gfp_mask & __GFP_NORETRY) in __alloc_pages_slowpath()
4858 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL)) in __alloc_pages_slowpath()
4861 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, in __alloc_pages_slowpath()
4887 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); in __alloc_pages_slowpath()
4894 (gfp_mask & __GFP_NOMEMALLOC))) in __alloc_pages_slowpath()
4916 if (gfp_mask & __GFP_NOFAIL) { in __alloc_pages_slowpath()
4945 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac); in __alloc_pages_slowpath()
4953 warn_alloc(gfp_mask, ac->nodemask, in __alloc_pages_slowpath()
4959 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, in prepare_alloc_pages() argument
4964 ac->highest_zoneidx = gfp_zone(gfp_mask); in prepare_alloc_pages()
4965 ac->zonelist = node_zonelist(preferred_nid, gfp_mask); in prepare_alloc_pages()
4967 ac->migratetype = gfp_migratetype(gfp_mask); in prepare_alloc_pages()
4981 fs_reclaim_acquire(gfp_mask); in prepare_alloc_pages()
4982 fs_reclaim_release(gfp_mask); in prepare_alloc_pages()
4984 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM); in prepare_alloc_pages()
4987 if (gfp_mask & __GFP_KSWAPD_RECLAIM) in prepare_alloc_pages()
4991 if (should_fail_alloc_page(gfp_mask, order)) in prepare_alloc_pages()
4994 *alloc_flags = current_alloc_flags(gfp_mask, *alloc_flags); in prepare_alloc_pages()
4997 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); in prepare_alloc_pages()
5014 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid, in __alloc_pages_nodemask() argument
5027 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN)); in __alloc_pages_nodemask()
5031 gfp_mask &= gfp_allowed_mask; in __alloc_pages_nodemask()
5032 alloc_mask = gfp_mask; in __alloc_pages_nodemask()
5033 if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags)) in __alloc_pages_nodemask()
5040 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp_mask); in __alloc_pages_nodemask()
5053 alloc_mask = current_gfp_context(gfp_mask); in __alloc_pages_nodemask()
5065 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page && in __alloc_pages_nodemask()
5066 unlikely(__memcg_kmem_charge_page(page, gfp_mask, order) != 0)) { in __alloc_pages_nodemask()
5082 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) in __get_free_pages() argument
5086 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order); in __get_free_pages()
5093 unsigned long get_zeroed_page(gfp_t gfp_mask) in get_zeroed_page() argument
5095 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); in get_zeroed_page()
5139 gfp_t gfp_mask) in __page_frag_cache_refill() argument
5142 gfp_t gfp = gfp_mask; in __page_frag_cache_refill()
5145 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | in __page_frag_cache_refill()
5147 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, in __page_frag_cache_refill()
5194 unsigned int fragsz, gfp_t gfp_mask) in page_frag_alloc() argument
5202 page = __page_frag_cache_refill(nc, gfp_mask); in page_frag_alloc()
5315 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) in alloc_pages_exact() argument
5320 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) in alloc_pages_exact()
5321 gfp_mask &= ~__GFP_COMP; in alloc_pages_exact()
5323 addr = __get_free_pages(gfp_mask, order); in alloc_pages_exact()
5340 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) in alloc_pages_exact_nid() argument
5345 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) in alloc_pages_exact_nid()
5346 gfp_mask &= ~__GFP_COMP; in alloc_pages_exact_nid()
5348 p = alloc_pages_node(nid, gfp_mask, order); in alloc_pages_exact_nid()
8580 .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL, in __alloc_contig_migrate_range()
8640 unsigned migratetype, gfp_t gfp_mask) in alloc_contig_range() argument
8653 .gfp_mask = current_gfp_context(gfp_mask), in alloc_contig_range()
8773 unsigned long nr_pages, gfp_t gfp_mask) in __alloc_contig_pages() argument
8778 gfp_mask); in __alloc_contig_pages()
8836 struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask, in alloc_contig_pages() argument
8844 zonelist = node_zonelist(nid, gfp_mask); in alloc_contig_pages()
8846 gfp_zone(gfp_mask), nodemask) { in alloc_contig_pages()
8861 gfp_mask); in alloc_contig_pages()