31 #define sg_dma_address(sg)	((sg)->dma_address)  argument
34 #define sg_dma_len(sg)		((sg)->dma_length)  argument
36 #define sg_dma_len(sg)		((sg)->length)  argument
77 static inline unsigned int __sg_flags(struct scatterlist *sg)  in __sg_flags()  argument
79 	return sg->page_link & SG_PAGE_LINK_MASK;  in __sg_flags()
82 static inline struct scatterlist *sg_chain_ptr(struct scatterlist *sg)  in sg_chain_ptr()  argument
84 	return (struct scatterlist *)(sg->page_link & ~SG_PAGE_LINK_MASK);  in sg_chain_ptr()
87 static inline bool sg_is_chain(struct scatterlist *sg)  in sg_is_chain()  argument
89 	return __sg_flags(sg) & SG_CHAIN;  in sg_is_chain()
92 static inline bool sg_is_last(struct scatterlist *sg)  in sg_is_last()  argument
94 	return __sg_flags(sg) & SG_END;  in sg_is_last()
107 static inline void sg_assign_page(struct scatterlist *sg, struct page *page)  in sg_assign_page()  argument
109 	unsigned long page_link = sg->page_link & (SG_CHAIN | SG_END);  in sg_assign_page()
117 	BUG_ON(sg_is_chain(sg));  in sg_assign_page()
119 	sg->page_link = page_link | (unsigned long) page;  in sg_assign_page()
136 static inline void sg_set_page(struct scatterlist *sg, struct page *page,  in sg_set_page()  argument
139 	sg_assign_page(sg, page);  in sg_set_page()
140 	sg->offset = offset;  in sg_set_page()
141 	sg->length = len;  in sg_set_page()
158 static inline void sg_set_folio(struct scatterlist *sg, struct folio *folio,  in sg_set_folio()  argument
163 	sg_assign_page(sg, &folio->page);  in sg_set_folio()
164 	sg->offset = offset;  in sg_set_folio()
165 	sg->length = len;  in sg_set_folio()
168 static inline struct page *sg_page(struct scatterlist *sg)  in sg_page()  argument
171 	BUG_ON(sg_is_chain(sg));  in sg_page()
173 	return (struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK);  in sg_page()
183 static inline void sg_set_buf(struct scatterlist *sg, const void *buf,  in sg_set_buf()  argument
189 	sg_set_page(sg, virt_to_page(buf), buflen, offset_in_page(buf));  in sg_set_buf()
195 #define for_each_sg(sglist, sg, nr, __i)	\  argument
196 	for (__i = 0, sg = (sglist); __i < (nr); __i++, sg = sg_next(sg))
201 #define for_each_sgtable_sg(sgt, sg, i)		\  argument
202 	for_each_sg((sgt)->sgl, sg, (sgt)->orig_nents, i)
209 #define for_each_sgtable_dma_sg(sgt, sg, i)	\  argument
210 	for_each_sg((sgt)->sgl, sg, (sgt)->nents, i)
253 static inline void sg_mark_end(struct scatterlist *sg)  in sg_mark_end()  argument
258 	sg->page_link |= SG_END;  in sg_mark_end()
259 	sg->page_link &= ~SG_CHAIN;  in sg_mark_end()
270 static inline void sg_unmark_end(struct scatterlist *sg)  in sg_unmark_end()  argument
272 	sg->page_link &= ~SG_END;  in sg_unmark_end()
295 static inline bool sg_dma_is_bus_address(struct scatterlist *sg)  in sg_dma_is_bus_address()  argument
297 	return sg->dma_flags & SG_DMA_BUS_ADDRESS;  in sg_dma_is_bus_address()
310 static inline void sg_dma_mark_bus_address(struct scatterlist *sg)  in sg_dma_mark_bus_address()  argument
312 	sg->dma_flags |= SG_DMA_BUS_ADDRESS;  in sg_dma_mark_bus_address()
322 static inline void sg_dma_unmark_bus_address(struct scatterlist *sg)  in sg_dma_unmark_bus_address()  argument
324 	sg->dma_flags &= ~SG_DMA_BUS_ADDRESS;  in sg_dma_unmark_bus_address()
337 static inline bool sg_dma_is_swiotlb(struct scatterlist *sg)  in sg_dma_is_swiotlb()  argument
339 	return sg->dma_flags & SG_DMA_SWIOTLB;  in sg_dma_is_swiotlb()
350 static inline void sg_dma_mark_swiotlb(struct scatterlist *sg)  in sg_dma_mark_swiotlb()  argument
352 	sg->dma_flags |= SG_DMA_SWIOTLB;  in sg_dma_mark_swiotlb()
357 static inline bool sg_dma_is_bus_address(struct scatterlist *sg)  in sg_dma_is_bus_address()  argument
361 static inline void sg_dma_mark_bus_address(struct scatterlist *sg)  in sg_dma_mark_bus_address()  argument
364 static inline void sg_dma_unmark_bus_address(struct scatterlist *sg)  in sg_dma_unmark_bus_address()  argument
367 static inline bool sg_dma_is_swiotlb(struct scatterlist *sg)  in sg_dma_is_swiotlb()  argument
371 static inline void sg_dma_mark_swiotlb(struct scatterlist *sg)  in sg_dma_mark_swiotlb()  argument
387 static inline dma_addr_t sg_phys(struct scatterlist *sg)  in sg_phys()  argument
389 	return page_to_phys(sg_page(sg)) + sg->offset;  in sg_phys()
402 static inline void *sg_virt(struct scatterlist *sg)  in sg_virt()  argument
404 	return page_address(sg_page(sg)) + sg->offset;  in sg_virt()
419 int sg_nents(struct scatterlist *sg);
420 int sg_nents_for_len(struct scatterlist *sg, u64 len);
551 	struct scatterlist	*sg;		/* sg holding the page */  member
582 	return nth_page(sg_page(piter->sg), piter->sg_pgoffset);  in sg_page_iter_page()
593 	return sg_dma_address(dma_iter->base.sg) +  in sg_page_iter_dma_address()