1 /* SPDX-License-Identifier: GPL-2.0
12  * uses one frame per-page, but it can fallback on the
19  * The API keeps track of in-flight pages, in order to let API users know
22  * the page to a page_pool-aware object like skbs marked with
27  * release the DMA mapping and in-flight state accounting.
40  * Drivers that wish to harvest page pool stats and report them to users
42  * struct page_pool_stats call page_pool_get_stats to get stats for the specified pool.
44 bool page_pool_get_stats(struct page_pool *pool,
64  * page_pool_dev_alloc_pages() - allocate a page.
65  * @pool:	pool from which to allocate
69 static inline struct page *page_pool_dev_alloc_pages(struct page_pool *pool)  in page_pool_dev_alloc_pages()  argument
73 	return page_pool_alloc_pages(pool, gfp);  in page_pool_dev_alloc_pages()
76 static inline struct page *page_pool_dev_alloc_frag(struct page_pool *pool,  in page_pool_dev_alloc_frag()  argument
82 	return page_pool_alloc_frag(pool, offset, size, gfp);  in page_pool_dev_alloc_frag()
86  * page_pool_get_dma_dir() - Retrieve the stored DMA direction.
87  * @pool:	pool from which page was allocated
89  * Get the stored dma direction. A driver might decide to store this locally
93 inline enum dma_data_direction page_pool_get_dma_dir(struct page_pool *pool)  in page_pool_get_dma_dir()  argument
95 	return pool->p.dma_dir;  in page_pool_get_dma_dir()
99  * either by updating skb->data or via DMA mappings for the device.
101  * holding page references and we can't reliably destroy or sync DMA mappings
110 	atomic_long_set(&page->pp_frag_count, nr);  in page_pool_fragment_page()
126 	if (atomic_long_read(&page->pp_frag_count) == nr)  in page_pool_defrag_page()
129 	ret = atomic_long_sub_return(nr, &page->pp_frag_count);  in page_pool_defrag_page()
134 static inline bool page_pool_is_last_frag(struct page_pool *pool,  in page_pool_is_last_frag()  argument
138 	return !(pool->p.flags & PP_FLAG_PAGE_FRAG) ||  in page_pool_is_last_frag()
143  * page_pool_put_page() - release a reference to a page pool page
144  * @pool:	pool from which page was allocated
151  * the allocator owns the page and will try to recycle it in one of the pool
155 static inline void page_pool_put_page(struct page_pool *pool,  in page_pool_put_page()  argument
160 	/* When page_pool isn't compiled-in, net/core/xdp.c doesn't  in page_pool_put_page()
164 	if (!page_pool_is_last_frag(pool, page))  in page_pool_put_page()
167 	page_pool_put_defragged_page(pool, page, dma_sync_size, allow_direct);  in page_pool_put_page()
172  * page_pool_put_full_page() - release a reference on a page pool page
173  * @pool:	pool from which page was allocated
177  * Similar to page_pool_put_page(), but will DMA sync the entire memory area
180 static inline void page_pool_put_full_page(struct page_pool *pool,  in page_pool_put_full_page()  argument
183 	page_pool_put_page(pool, page, -1, allow_direct);  in page_pool_put_full_page()
187  * page_pool_recycle_direct() - release a reference on a page pool page
188  * @pool:	pool from which page was allocated
192  * (e.g NAPI), since it will recycle the page directly into the pool fast cache.
194 static inline void page_pool_recycle_direct(struct page_pool *pool,  in page_pool_recycle_direct()  argument
197 	page_pool_put_full_page(pool, page, true);  in page_pool_recycle_direct()
204  * page_pool_get_dma_addr() - Retrieve the stored DMA address.
205  * @page:	page allocated from a page pool
207  * Fetch the DMA address of the page. The page pool to which the page belongs
212 	dma_addr_t ret = page->dma_addr;  in page_pool_get_dma_addr()
215 		ret |= (dma_addr_t)page->dma_addr_upper << 16 << 16;  in page_pool_get_dma_addr()
222 	page->dma_addr = addr;  in page_pool_set_dma_addr()
224 		page->dma_addr_upper = upper_32_bits(addr);  in page_pool_set_dma_addr()
227 static inline bool page_pool_put(struct page_pool *pool)  in page_pool_put()  argument
229 	return refcount_dec_and_test(&pool->user_cnt);  in page_pool_put()
232 static inline void page_pool_nid_changed(struct page_pool *pool, int new_nid)  in page_pool_nid_changed()  argument
234 	if (unlikely(pool->p.nid != new_nid))  in page_pool_nid_changed()
235 		page_pool_update_nid(pool, new_nid);  in page_pool_nid_changed()