612 static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,  in swiotlb_alloc_tlb()  argument
621 	if (!gfpflags_allow_blocking(gfp) && dev && force_dma_unencrypted(dev)) {  in swiotlb_alloc_tlb()
627 		return dma_alloc_from_pool(dev, bytes, &vaddr, gfp,  in swiotlb_alloc_tlb()
683 static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,  in swiotlb_alloc_pool()  argument
705 	while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp))) {  in swiotlb_alloc_pool()
777 struct io_tlb_pool *__swiotlb_find_pool(struct device *dev, phys_addr_t paddr)  in __swiotlb_find_pool()  argument
779 	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;  in __swiotlb_find_pool()
788 	list_for_each_entry_rcu(pool, &dev->dma_io_tlb_pools, node) {  in __swiotlb_find_pool()
804 static void swiotlb_del_pool(struct device *dev, struct io_tlb_pool *pool)  in swiotlb_del_pool()  argument
808 	spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);  in swiotlb_del_pool()
810 	spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags);  in swiotlb_del_pool()
821 void swiotlb_dev_init(struct device *dev)  in swiotlb_dev_init()  argument
823 	dev->dma_io_tlb_mem = &io_tlb_default_mem;  in swiotlb_dev_init()
825 	INIT_LIST_HEAD(&dev->dma_io_tlb_pools);  in swiotlb_dev_init()
826 	spin_lock_init(&dev->dma_io_tlb_lock);  in swiotlb_dev_init()
827 	dev->dma_uses_io_tlb = false;  in swiotlb_dev_init()
850 static unsigned int swiotlb_align_offset(struct device *dev,  in swiotlb_align_offset()  argument
853 	return addr & dma_get_min_align_mask(dev) &  in swiotlb_align_offset()
860 static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size,  in swiotlb_bounce()  argument
884 		     swiotlb_align_offset(dev, 0, orig_addr);  in swiotlb_bounce()
890 		dev_WARN_ONCE(dev, 1,  in swiotlb_bounce()
1017 static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool,  in swiotlb_search_pool_area()  argument
1022 	unsigned long boundary_mask = dma_get_seg_boundary(dev);  in swiotlb_search_pool_area()
1024 		phys_to_dma_unencrypted(dev, pool->start) & boundary_mask;  in swiotlb_search_pool_area()
1026 	unsigned int iotlb_align_mask = dma_get_min_align_mask(dev);  in swiotlb_search_pool_area()
1028 	unsigned int offset = swiotlb_align_offset(dev, 0, orig_addr);  in swiotlb_search_pool_area()
1120 	inc_used_and_hiwater(dev->dma_io_tlb_mem, nslots);  in swiotlb_search_pool_area()
1142 static int swiotlb_search_area(struct device *dev, int start_cpu,  in swiotlb_search_area()  argument
1146 	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;  in swiotlb_search_area()
1156 		index = swiotlb_search_pool_area(dev, pool, area_index,  in swiotlb_search_area()
1182 static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,  in swiotlb_find_slots()  argument
1186 	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;  in swiotlb_find_slots()
1199 		index = swiotlb_search_area(dev, cpu, i, orig_addr, alloc_size,  in swiotlb_find_slots()
1211 	phys_limit = min_not_zero(*dev->dma_mask, dev->bus_dma_limit);  in swiotlb_find_slots()
1212 	pool = swiotlb_alloc_pool(dev, nslabs, nslabs, 1, phys_limit,  in swiotlb_find_slots()
1217 	index = swiotlb_search_pool_area(dev, pool, 0, orig_addr,  in swiotlb_find_slots()
1225 	spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);  in swiotlb_find_slots()
1226 	list_add_rcu(&pool->node, &dev->dma_io_tlb_pools);  in swiotlb_find_slots()
1227 	spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags);  in swiotlb_find_slots()
1231 	WRITE_ONCE(dev->dma_uses_io_tlb, true);  in swiotlb_find_slots()
1258 static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,  in swiotlb_find_slots()  argument
1266 	*retpool = pool = &dev->dma_io_tlb_mem->defpool;  in swiotlb_find_slots()
1269 		index = swiotlb_search_pool_area(dev, pool, i, orig_addr,  in swiotlb_find_slots()
1370 phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,  in swiotlb_tbl_map_single()  argument
1374 	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;  in swiotlb_tbl_map_single()
1384 		dev_warn_ratelimited(dev,  in swiotlb_tbl_map_single()
1399 	dev_WARN_ONCE(dev, alloc_align_mask > ~PAGE_MASK,  in swiotlb_tbl_map_single()
1402 	offset = swiotlb_align_offset(dev, alloc_align_mask, orig_addr);  in swiotlb_tbl_map_single()
1404 	index = swiotlb_find_slots(dev, orig_addr, size, alloc_align_mask, &pool);  in swiotlb_tbl_map_single()
1407 			dev_warn_ratelimited(dev,  in swiotlb_tbl_map_single()
1417 	dma_reset_need_sync(dev);  in swiotlb_tbl_map_single()
1440 	swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_TO_DEVICE, pool);  in swiotlb_tbl_map_single()
1444 static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr,  in swiotlb_release_slots()  argument
1448 	unsigned int offset = swiotlb_align_offset(dev, 0, tlb_addr);  in swiotlb_release_slots()
1495 	dec_used(dev->dma_io_tlb_mem, nslots);  in swiotlb_release_slots()
1511 static bool swiotlb_del_transient(struct device *dev, phys_addr_t tlb_addr,  in swiotlb_del_transient()  argument
1517 	dec_used(dev->dma_io_tlb_mem, pool->nslabs);  in swiotlb_del_transient()
1518 	swiotlb_del_pool(dev, pool);  in swiotlb_del_transient()
1519 	dec_transient_used(dev->dma_io_tlb_mem, pool->nslabs);  in swiotlb_del_transient()
1525 static inline bool swiotlb_del_transient(struct device *dev,  in swiotlb_del_transient()  argument
1536 void __swiotlb_tbl_unmap_single(struct device *dev, phys_addr_t tlb_addr,  in __swiotlb_tbl_unmap_single()  argument
1545 		swiotlb_bounce(dev, tlb_addr, mapping_size,  in __swiotlb_tbl_unmap_single()
1548 	if (swiotlb_del_transient(dev, tlb_addr, pool))  in __swiotlb_tbl_unmap_single()
1550 	swiotlb_release_slots(dev, tlb_addr, pool);  in __swiotlb_tbl_unmap_single()
1553 void __swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,  in __swiotlb_sync_single_for_device()  argument
1558 		swiotlb_bounce(dev, tlb_addr, size, DMA_TO_DEVICE, pool);  in __swiotlb_sync_single_for_device()
1563 void __swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,  in __swiotlb_sync_single_for_cpu()  argument
1568 		swiotlb_bounce(dev, tlb_addr, size, DMA_FROM_DEVICE, pool);  in __swiotlb_sync_single_for_cpu()
1577 dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,  in swiotlb_map()  argument
1583 	trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size);  in swiotlb_map()
1585 	swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, 0, dir, attrs);  in swiotlb_map()
1590 	dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);  in swiotlb_map()
1591 	if (unlikely(!dma_capable(dev, dma_addr, size, true))) {  in swiotlb_map()
1592 		__swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir,  in swiotlb_map()
1594 			swiotlb_find_pool(dev, swiotlb_addr));  in swiotlb_map()
1595 		dev_WARN_ONCE(dev, 1,  in swiotlb_map()
1597 			&dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);  in swiotlb_map()
1601 	if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))  in swiotlb_map()
1606 size_t swiotlb_max_mapping_size(struct device *dev)  in swiotlb_max_mapping_size()  argument
1608 	int min_align_mask = dma_get_min_align_mask(dev);  in swiotlb_max_mapping_size()
1630 bool is_swiotlb_active(struct device *dev)  in is_swiotlb_active()  argument
1632 	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;  in is_swiotlb_active()
1752 struct page *swiotlb_alloc(struct device *dev, size_t size)  in swiotlb_alloc()  argument
1754 	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;  in swiotlb_alloc()
1764 	index = swiotlb_find_slots(dev, 0, size, align, &pool);  in swiotlb_alloc()
1770 		dev_WARN_ONCE(dev, 1, "Cannot allocate pages from non page-aligned swiotlb addr 0x%pa.\n",  in swiotlb_alloc()
1772 		swiotlb_release_slots(dev, tlb_addr, pool);  in swiotlb_alloc()
1779 bool swiotlb_free(struct device *dev, struct page *page, size_t size)  in swiotlb_free()  argument
1784 	pool = swiotlb_find_pool(dev, tlb_addr);  in swiotlb_free()
1788 	swiotlb_release_slots(dev, tlb_addr, pool);  in swiotlb_free()
1794 				    struct device *dev)  in rmem_swiotlb_device_init()  argument
1803 		dev_err(dev, "Restricted DMA pool must be accessible within the linear mapping.");  in rmem_swiotlb_device_init()
1851 	dev->dma_io_tlb_mem = mem;  in rmem_swiotlb_device_init()
1857 					struct device *dev)  in rmem_swiotlb_device_release()  argument
1859 	dev->dma_io_tlb_mem = &io_tlb_default_mem;  in rmem_swiotlb_device_release()