Lines Matching refs:gfp_mask
3687 static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) in __should_fail_alloc_page() argument
3691 if (gfp_mask & __GFP_NOFAIL) in __should_fail_alloc_page()
3693 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) in __should_fail_alloc_page()
3696 (gfp_mask & __GFP_DIRECT_RECLAIM)) in __should_fail_alloc_page()
3727 static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) in __should_fail_alloc_page() argument
3734 noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) in should_fail_alloc_page() argument
3736 return __should_fail_alloc_page(gfp_mask, order); in should_fail_alloc_page()
3851 unsigned int alloc_flags, gfp_t gfp_mask) in zone_watermark_fast() argument
3883 if (unlikely(!order && (gfp_mask & __GFP_ATOMIC) && z->watermark_boost in zone_watermark_fast()
3928 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) in alloc_flags_nofragment() argument
3936 alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM); in alloc_flags_nofragment()
3959 static inline unsigned int current_alloc_flags(gfp_t gfp_mask, in current_alloc_flags() argument
3966 gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE && in current_alloc_flags()
3967 gfp_mask & __GFP_CMA) in current_alloc_flags()
3979 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, in get_page_from_freelist() argument
4001 !__cpuset_zone_allowed(zone, gfp_mask)) in get_page_from_freelist()
4051 gfp_mask)) { in get_page_from_freelist()
4073 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); in get_page_from_freelist()
4093 gfp_mask, alloc_flags, ac->migratetype); in get_page_from_freelist()
4095 prep_new_page(page, order, gfp_mask, alloc_flags); in get_page_from_freelist()
4128 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) in warn_alloc_show_mem() argument
4137 if (!(gfp_mask & __GFP_NOMEMALLOC)) in warn_alloc_show_mem()
4141 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) in warn_alloc_show_mem()
4147 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) in warn_alloc() argument
4153 if ((gfp_mask & __GFP_NOWARN) || in warn_alloc()
4155 ((gfp_mask & __GFP_DMA) && !has_managed_dma())) in warn_alloc()
4162 current->comm, &vaf, gfp_mask, &gfp_mask, in warn_alloc()
4169 warn_alloc_show_mem(gfp_mask, nodemask); in warn_alloc()
4173 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order, in __alloc_pages_cpuset_fallback() argument
4179 page = get_page_from_freelist(gfp_mask, order, in __alloc_pages_cpuset_fallback()
4186 page = get_page_from_freelist(gfp_mask, order, in __alloc_pages_cpuset_fallback()
4193 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, in __alloc_pages_may_oom() argument
4200 .gfp_mask = gfp_mask, in __alloc_pages_may_oom()
4224 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & in __alloc_pages_may_oom()
4244 if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE)) in __alloc_pages_may_oom()
4262 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { in __alloc_pages_may_oom()
4269 if (gfp_mask & __GFP_NOFAIL) in __alloc_pages_may_oom()
4270 page = __alloc_pages_cpuset_fallback(gfp_mask, order, in __alloc_pages_may_oom()
4287 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, in __alloc_pages_direct_compact() argument
4301 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, in __alloc_pages_direct_compact()
4315 prep_new_page(page, order, gfp_mask, alloc_flags); in __alloc_pages_direct_compact()
4319 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_direct_compact()
4420 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, in __alloc_pages_direct_compact() argument
4460 static bool __need_fs_reclaim(gfp_t gfp_mask) in __need_fs_reclaim() argument
4462 gfp_mask = current_gfp_context(gfp_mask); in __need_fs_reclaim()
4465 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) in __need_fs_reclaim()
4473 if (!(gfp_mask & __GFP_FS)) in __need_fs_reclaim()
4476 if (gfp_mask & __GFP_NOLOCKDEP) in __need_fs_reclaim()
4492 void fs_reclaim_acquire(gfp_t gfp_mask) in fs_reclaim_acquire() argument
4494 if (__need_fs_reclaim(gfp_mask)) in fs_reclaim_acquire()
4499 void fs_reclaim_release(gfp_t gfp_mask) in fs_reclaim_release() argument
4501 if (__need_fs_reclaim(gfp_mask)) in fs_reclaim_release()
4533 __perform_reclaim(gfp_t gfp_mask, unsigned int order, in __perform_reclaim() argument
4543 fs_reclaim_acquire(gfp_mask); in __perform_reclaim()
4546 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, in __perform_reclaim()
4550 fs_reclaim_release(gfp_mask); in __perform_reclaim()
4559 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, in __alloc_pages_direct_reclaim() argument
4569 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); in __alloc_pages_direct_reclaim()
4574 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_direct_reclaim()
4583 trace_android_vh_drain_all_pages_bypass(gfp_mask, order, in __alloc_pages_direct_reclaim()
4596 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, in wake_all_kswapds() argument
4607 wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx); in wake_all_kswapds()
4613 gfp_to_alloc_flags(gfp_t gfp_mask) in gfp_to_alloc_flags() argument
4632 (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM)); in gfp_to_alloc_flags()
4634 if (gfp_mask & __GFP_ATOMIC) { in gfp_to_alloc_flags()
4639 if (!(gfp_mask & __GFP_NOMEMALLOC)) in gfp_to_alloc_flags()
4649 alloc_flags = current_alloc_flags(gfp_mask, alloc_flags); in gfp_to_alloc_flags()
4673 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask) in __gfp_pfmemalloc_flags() argument
4675 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) in __gfp_pfmemalloc_flags()
4677 if (gfp_mask & __GFP_MEMALLOC) in __gfp_pfmemalloc_flags()
4691 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) in gfp_pfmemalloc_allowed() argument
4693 return !!__gfp_pfmemalloc_flags(gfp_mask); in gfp_pfmemalloc_allowed()
4707 should_reclaim_retry(gfp_t gfp_mask, unsigned order, in should_reclaim_retry() argument
4831 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, in __alloc_pages_slowpath() argument
4834 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; in __alloc_pages_slowpath()
4849 trace_android_vh_alloc_pages_slowpath_begin(gfp_mask, order, &vh_record); in __alloc_pages_slowpath()
4854 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == in __alloc_pages_slowpath()
4856 gfp_mask &= ~__GFP_ATOMIC; in __alloc_pages_slowpath()
4870 alloc_flags = gfp_to_alloc_flags(gfp_mask); in __alloc_pages_slowpath()
4884 wake_all_kswapds(order, gfp_mask, ac); in __alloc_pages_slowpath()
4890 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_slowpath()
4906 && !gfp_pfmemalloc_allowed(gfp_mask)) { in __alloc_pages_slowpath()
4907 page = __alloc_pages_direct_compact(gfp_mask, order, in __alloc_pages_slowpath()
4918 if (costly_order && (gfp_mask & __GFP_NORETRY)) { in __alloc_pages_slowpath()
4952 wake_all_kswapds(order, gfp_mask, ac); in __alloc_pages_slowpath()
4954 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); in __alloc_pages_slowpath()
4956 alloc_flags = current_alloc_flags(gfp_mask, reserve_flags); in __alloc_pages_slowpath()
4970 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_slowpath()
4982 trace_android_vh_alloc_pages_reclaim_bypass(gfp_mask, order, in __alloc_pages_slowpath()
4988 trace_android_vh_should_alloc_pages_retry(gfp_mask, order, in __alloc_pages_slowpath()
4995 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, in __alloc_pages_slowpath()
5001 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, in __alloc_pages_slowpath()
5007 if (gfp_mask & __GFP_NORETRY) in __alloc_pages_slowpath()
5014 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL)) in __alloc_pages_slowpath()
5017 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, in __alloc_pages_slowpath()
5043 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); in __alloc_pages_slowpath()
5050 (gfp_mask & __GFP_NOMEMALLOC))) in __alloc_pages_slowpath()
5072 if (gfp_mask & __GFP_NOFAIL) { in __alloc_pages_slowpath()
5101 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac); in __alloc_pages_slowpath()
5109 trace_android_vh_alloc_pages_failure_bypass(gfp_mask, order, in __alloc_pages_slowpath()
5114 warn_alloc(gfp_mask, ac->nodemask, in __alloc_pages_slowpath()
5117 trace_android_vh_alloc_pages_slowpath_end(gfp_mask, order, vh_record); in __alloc_pages_slowpath()
5121 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, in prepare_alloc_pages() argument
5126 ac->highest_zoneidx = gfp_zone(gfp_mask); in prepare_alloc_pages()
5127 ac->zonelist = node_zonelist(preferred_nid, gfp_mask); in prepare_alloc_pages()
5129 ac->migratetype = gfp_migratetype(gfp_mask); in prepare_alloc_pages()
5143 fs_reclaim_acquire(gfp_mask); in prepare_alloc_pages()
5144 fs_reclaim_release(gfp_mask); in prepare_alloc_pages()
5146 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM); in prepare_alloc_pages()
5148 if (should_fail_alloc_page(gfp_mask, order)) in prepare_alloc_pages()
5151 *alloc_flags = current_alloc_flags(gfp_mask, *alloc_flags); in prepare_alloc_pages()
5154 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); in prepare_alloc_pages()
5171 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid, in __alloc_pages_nodemask() argument
5184 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN)); in __alloc_pages_nodemask()
5188 gfp_mask &= gfp_allowed_mask; in __alloc_pages_nodemask()
5189 alloc_mask = gfp_mask; in __alloc_pages_nodemask()
5190 if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags)) in __alloc_pages_nodemask()
5197 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp_mask); in __alloc_pages_nodemask()
5210 alloc_mask = current_gfp_context(gfp_mask); in __alloc_pages_nodemask()
5222 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page && in __alloc_pages_nodemask()
5223 unlikely(__memcg_kmem_charge_page(page, gfp_mask, order) != 0)) { in __alloc_pages_nodemask()
5239 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) in __get_free_pages() argument
5243 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order); in __get_free_pages()
5250 unsigned long get_zeroed_page(gfp_t gfp_mask) in get_zeroed_page() argument
5252 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); in get_zeroed_page()
5300 gfp_t gfp_mask) in __page_frag_cache_refill() argument
5303 gfp_t gfp = gfp_mask; in __page_frag_cache_refill()
5306 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | in __page_frag_cache_refill()
5308 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, in __page_frag_cache_refill()
5330 unsigned int fragsz, gfp_t gfp_mask) in page_frag_alloc() argument
5338 page = __page_frag_cache_refill(nc, gfp_mask); in page_frag_alloc()
5443 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) in alloc_pages_exact() argument
5448 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) in alloc_pages_exact()
5449 gfp_mask &= ~__GFP_COMP; in alloc_pages_exact()
5451 addr = __get_free_pages(gfp_mask, order); in alloc_pages_exact()
5468 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) in alloc_pages_exact_nid() argument
5473 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) in alloc_pages_exact_nid()
5474 gfp_mask &= ~__GFP_COMP; in alloc_pages_exact_nid()
5476 p = alloc_pages_node(nid, gfp_mask, order); in alloc_pages_exact_nid()
8754 .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL, in __alloc_contig_migrate_range()
8836 unsigned migratetype, gfp_t gfp_mask, in alloc_contig_range() argument
8848 .mode = gfp_mask & __GFP_NORETRY ? MIGRATE_ASYNC : MIGRATE_SYNC, in alloc_contig_range()
8851 .gfp_mask = current_gfp_context(gfp_mask), in alloc_contig_range()
8904 if (ret && (ret != -EBUSY || (gfp_mask & __GFP_NORETRY))) in alloc_contig_range()
8978 unsigned long nr_pages, gfp_t gfp_mask) in __alloc_contig_pages() argument
8984 gfp_mask, &dummy); in __alloc_contig_pages()
9042 struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask, in alloc_contig_pages() argument
9050 zonelist = node_zonelist(nid, gfp_mask); in alloc_contig_pages()
9052 gfp_zone(gfp_mask), nodemask) { in alloc_contig_pages()
9067 gfp_mask); in alloc_contig_pages()