21 static int zpci_refresh_global(struct zpci_dev *zdev)  in zpci_refresh_global()  argument
23 	return zpci_refresh_trans((u64) zdev->fh << 32, zdev->start_dma,  in zpci_refresh_global()
24 				  zdev->iommu_pages * PAGE_SIZE);  in zpci_refresh_global()
117 static void dma_update_cpu_trans(struct zpci_dev *zdev, void *page_addr,  in dma_update_cpu_trans()  argument
122 	entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr);  in dma_update_cpu_trans()
142 static int dma_update_trans(struct zpci_dev *zdev, unsigned long pa,  in dma_update_trans()  argument
154 	spin_lock_irqsave(&zdev->dma_table_lock, irq_flags);  in dma_update_trans()
155 	if (!zdev->dma_table)  in dma_update_trans()
159 		dma_update_cpu_trans(zdev, page_addr, dma_addr, flags);  in dma_update_trans()
171 	if (!zdev->tlb_refresh &&  in dma_update_trans()
176 	rc = zpci_refresh_trans((u64) zdev->fh << 32, start_dma_addr,  in dma_update_trans()
180 	spin_unlock_irqrestore(&zdev->dma_table_lock, irq_flags);  in dma_update_trans()
196 static void dma_cleanup_tables(struct zpci_dev *zdev)  in dma_cleanup_tables()  argument
201 	if (!zdev || !zdev->dma_table)  in dma_cleanup_tables()
204 	table = zdev->dma_table;  in dma_cleanup_tables()
210 	zdev->dma_table = NULL;  in dma_cleanup_tables()
213 static unsigned long __dma_alloc_iommu(struct zpci_dev *zdev,  in __dma_alloc_iommu()  argument
218 	boundary_size = ALIGN(dma_get_seg_boundary(&zdev->pdev->dev) + 1,  in __dma_alloc_iommu()
220 	return iommu_area_alloc(zdev->iommu_bitmap, zdev->iommu_pages,  in __dma_alloc_iommu()
224 static unsigned long dma_alloc_iommu(struct zpci_dev *zdev, int size)  in dma_alloc_iommu()  argument
229 	spin_lock_irqsave(&zdev->iommu_bitmap_lock, flags);  in dma_alloc_iommu()
230 	offset = __dma_alloc_iommu(zdev, zdev->next_bit, size);  in dma_alloc_iommu()
233 		offset = __dma_alloc_iommu(zdev, 0, size);  in dma_alloc_iommu()
238 		zdev->next_bit = offset + size;  in dma_alloc_iommu()
239 		if (!zdev->tlb_refresh && !s390_iommu_strict && wrap)  in dma_alloc_iommu()
241 			zpci_refresh_global(zdev);  in dma_alloc_iommu()
243 	spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, flags);  in dma_alloc_iommu()
247 static void dma_free_iommu(struct zpci_dev *zdev, unsigned long offset, int size)  in dma_free_iommu()  argument
251 	spin_lock_irqsave(&zdev->iommu_bitmap_lock, flags);  in dma_free_iommu()
252 	if (!zdev->iommu_bitmap)  in dma_free_iommu()
254 	bitmap_clear(zdev->iommu_bitmap, offset, size);  in dma_free_iommu()
259 	if (!s390_iommu_strict && offset >= zdev->next_bit)  in dma_free_iommu()
260 		zdev->next_bit = offset + size;  in dma_free_iommu()
262 	spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, flags);  in dma_free_iommu()
280 	struct zpci_dev *zdev = get_zdev(to_pci_dev(dev));  in s390_dma_map_pages()  local
288 	iommu_page_index = dma_alloc_iommu(zdev, nr_pages);  in s390_dma_map_pages()
295 	dma_addr = zdev->start_dma + iommu_page_index * PAGE_SIZE;  in s390_dma_map_pages()
296 	if (dma_addr + size > zdev->end_dma)  in s390_dma_map_pages()
302 	if (!dma_update_trans(zdev, pa, dma_addr, size, flags)) {  in s390_dma_map_pages()
303 		atomic64_add(nr_pages, &zdev->mapped_pages);  in s390_dma_map_pages()
308 	dma_free_iommu(zdev, iommu_page_index, nr_pages);  in s390_dma_map_pages()
319 	struct zpci_dev *zdev = get_zdev(to_pci_dev(dev));  in s390_dma_unmap_pages()  local
325 	if (dma_update_trans(zdev, 0, dma_addr, npages * PAGE_SIZE,  in s390_dma_unmap_pages()
331 	atomic64_add(npages, &zdev->unmapped_pages);  in s390_dma_unmap_pages()
332 	iommu_page_index = (dma_addr - zdev->start_dma) >> PAGE_SHIFT;  in s390_dma_unmap_pages()
333 	dma_free_iommu(zdev, iommu_page_index, npages);  in s390_dma_unmap_pages()
340 	struct zpci_dev *zdev = get_zdev(to_pci_dev(dev));  in s390_dma_alloc()  local
360 	atomic64_add(size / PAGE_SIZE, &zdev->allocated_pages);  in s390_dma_alloc()
370 	struct zpci_dev *zdev = get_zdev(to_pci_dev(dev));  in s390_dma_free()  local
373 	atomic64_sub(size / PAGE_SIZE, &zdev->allocated_pages);  in s390_dma_free()
425 int zpci_dma_init_device(struct zpci_dev *zdev)  in zpci_dma_init_device()  argument
429 	spin_lock_init(&zdev->iommu_bitmap_lock);  in zpci_dma_init_device()
430 	spin_lock_init(&zdev->dma_table_lock);  in zpci_dma_init_device()
432 	zdev->dma_table = dma_alloc_cpu_table();  in zpci_dma_init_device()
433 	if (!zdev->dma_table) {  in zpci_dma_init_device()
438 	zdev->iommu_size = (unsigned long) high_memory - PAGE_OFFSET;  in zpci_dma_init_device()
439 	zdev->iommu_pages = zdev->iommu_size >> PAGE_SHIFT;  in zpci_dma_init_device()
440 	zdev->iommu_bitmap = vzalloc(zdev->iommu_pages / 8);  in zpci_dma_init_device()
441 	if (!zdev->iommu_bitmap) {  in zpci_dma_init_device()
446 	rc = zpci_register_ioat(zdev,  in zpci_dma_init_device()
448 				zdev->start_dma + PAGE_OFFSET,  in zpci_dma_init_device()
449 				zdev->start_dma + zdev->iommu_size - 1,  in zpci_dma_init_device()
450 				(u64) zdev->dma_table);  in zpci_dma_init_device()
456 	dma_free_cpu_table(zdev->dma_table);  in zpci_dma_init_device()
461 void zpci_dma_exit_device(struct zpci_dev *zdev)  in zpci_dma_exit_device()  argument
463 	zpci_unregister_ioat(zdev, 0);  in zpci_dma_exit_device()
464 	dma_cleanup_tables(zdev);  in zpci_dma_exit_device()
465 	vfree(zdev->iommu_bitmap);  in zpci_dma_exit_device()
466 	zdev->iommu_bitmap = NULL;  in zpci_dma_exit_device()
467 	zdev->next_bit = 0;  in zpci_dma_exit_device()