20 static int vduse_iotlb_add_range(struct vduse_iova_domain *domain,  in vduse_iotlb_add_range()  argument
35 	ret = vhost_iotlb_add_range_ctx(domain->iotlb, start, last,  in vduse_iotlb_add_range()
45 static void vduse_iotlb_del_range(struct vduse_iova_domain *domain,  in vduse_iotlb_del_range()  argument
51 	while ((map = vhost_iotlb_itree_first(domain->iotlb, start, last))) {  in vduse_iotlb_del_range()
55 		vhost_iotlb_map_free(domain->iotlb, map);  in vduse_iotlb_del_range()
59 int vduse_domain_set_map(struct vduse_iova_domain *domain,  in vduse_domain_set_map()  argument
67 	spin_lock(&domain->iotlb_lock);  in vduse_domain_set_map()
68 	vduse_iotlb_del_range(domain, start, last);  in vduse_domain_set_map()
73 		ret = vduse_iotlb_add_range(domain, map->start, map->last,  in vduse_domain_set_map()
80 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_set_map()
84 	vduse_iotlb_del_range(domain, start, last);  in vduse_domain_set_map()
85 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_set_map()
89 void vduse_domain_clear_map(struct vduse_iova_domain *domain,  in vduse_domain_clear_map()  argument
95 	spin_lock(&domain->iotlb_lock);  in vduse_domain_clear_map()
98 		vduse_iotlb_del_range(domain, map->start, map->last);  in vduse_domain_clear_map()
100 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_clear_map()
103 static int vduse_domain_map_bounce_page(struct vduse_iova_domain *domain,  in vduse_domain_map_bounce_page()  argument
110 		map = &domain->bounce_maps[iova >> PAGE_SHIFT];  in vduse_domain_map_bounce_page()
123 static void vduse_domain_unmap_bounce_page(struct vduse_iova_domain *domain,  in vduse_domain_unmap_bounce_page()  argument
130 		map = &domain->bounce_maps[iova >> PAGE_SHIFT];  in vduse_domain_unmap_bounce_page()
161 static void vduse_domain_bounce(struct vduse_iova_domain *domain,  in vduse_domain_bounce()  argument
170 	if (iova >= domain->bounce_size)  in vduse_domain_bounce()
174 		map = &domain->bounce_maps[iova >> PAGE_SHIFT];  in vduse_domain_bounce()
190 vduse_domain_get_coherent_page(struct vduse_iova_domain *domain, u64 iova)  in vduse_domain_get_coherent_page()  argument
197 	spin_lock(&domain->iotlb_lock);  in vduse_domain_get_coherent_page()
198 	map = vhost_iotlb_itree_first(domain->iotlb, start, last);  in vduse_domain_get_coherent_page()
205 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_get_coherent_page()
211 vduse_domain_get_bounce_page(struct vduse_iova_domain *domain, u64 iova)  in vduse_domain_get_bounce_page()  argument
216 	spin_lock(&domain->iotlb_lock);  in vduse_domain_get_bounce_page()
217 	map = &domain->bounce_maps[iova >> PAGE_SHIFT];  in vduse_domain_get_bounce_page()
224 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_get_bounce_page()
230 vduse_domain_free_bounce_pages(struct vduse_iova_domain *domain)  in vduse_domain_free_bounce_pages()  argument
235 	bounce_pfns = domain->bounce_size >> PAGE_SHIFT;  in vduse_domain_free_bounce_pages()
238 		map = &domain->bounce_maps[pfn];  in vduse_domain_free_bounce_pages()
250 void vduse_domain_reset_bounce_map(struct vduse_iova_domain *domain)  in vduse_domain_reset_bounce_map()  argument
252 	if (!domain->bounce_map)  in vduse_domain_reset_bounce_map()
255 	spin_lock(&domain->iotlb_lock);  in vduse_domain_reset_bounce_map()
256 	if (!domain->bounce_map)  in vduse_domain_reset_bounce_map()
259 	vduse_iotlb_del_range(domain, 0, domain->bounce_size - 1);  in vduse_domain_reset_bounce_map()
260 	domain->bounce_map = 0;  in vduse_domain_reset_bounce_map()
262 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_reset_bounce_map()
265 static int vduse_domain_init_bounce_map(struct vduse_iova_domain *domain)  in vduse_domain_init_bounce_map()  argument
269 	if (domain->bounce_map)  in vduse_domain_init_bounce_map()
272 	spin_lock(&domain->iotlb_lock);  in vduse_domain_init_bounce_map()
273 	if (domain->bounce_map)  in vduse_domain_init_bounce_map()
276 	ret = vduse_iotlb_add_range(domain, 0, domain->bounce_size - 1,  in vduse_domain_init_bounce_map()
277 				    0, VHOST_MAP_RW, domain->file, 0);  in vduse_domain_init_bounce_map()
281 	domain->bounce_map = 1;  in vduse_domain_init_bounce_map()
283 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_init_bounce_map()
317 dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain,  in vduse_domain_map_page()  argument
322 	struct iova_domain *iovad = &domain->stream_iovad;  in vduse_domain_map_page()
323 	unsigned long limit = domain->bounce_size - 1;  in vduse_domain_map_page()
330 	if (vduse_domain_init_bounce_map(domain))  in vduse_domain_map_page()
333 	if (vduse_domain_map_bounce_page(domain, (u64)iova, (u64)size, pa))  in vduse_domain_map_page()
337 		vduse_domain_bounce(domain, iova, size, DMA_TO_DEVICE);  in vduse_domain_map_page()
345 void vduse_domain_unmap_page(struct vduse_iova_domain *domain,  in vduse_domain_unmap_page()  argument
349 	struct iova_domain *iovad = &domain->stream_iovad;  in vduse_domain_unmap_page()
352 		vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE);  in vduse_domain_unmap_page()
354 	vduse_domain_unmap_bounce_page(domain, (u64)dma_addr, (u64)size);  in vduse_domain_unmap_page()
358 void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain,  in vduse_domain_alloc_coherent()  argument
362 	struct iova_domain *iovad = &domain->consistent_iovad;  in vduse_domain_alloc_coherent()
363 	unsigned long limit = domain->iova_limit;  in vduse_domain_alloc_coherent()
370 	spin_lock(&domain->iotlb_lock);  in vduse_domain_alloc_coherent()
371 	if (vduse_iotlb_add_range(domain, (u64)iova, (u64)iova + size - 1,  in vduse_domain_alloc_coherent()
373 				  domain->file, (u64)iova)) {  in vduse_domain_alloc_coherent()
374 		spin_unlock(&domain->iotlb_lock);  in vduse_domain_alloc_coherent()
377 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_alloc_coherent()
392 void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size,  in vduse_domain_free_coherent()  argument
396 	struct iova_domain *iovad = &domain->consistent_iovad;  in vduse_domain_free_coherent()
401 	spin_lock(&domain->iotlb_lock);  in vduse_domain_free_coherent()
402 	map = vhost_iotlb_itree_first(domain->iotlb, (u64)dma_addr,  in vduse_domain_free_coherent()
405 		spin_unlock(&domain->iotlb_lock);  in vduse_domain_free_coherent()
412 	vhost_iotlb_map_free(domain->iotlb, map);  in vduse_domain_free_coherent()
413 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_free_coherent()
421 	struct vduse_iova_domain *domain = vmf->vma->vm_private_data;  in vduse_domain_mmap_fault()  local
425 	if (!domain)  in vduse_domain_mmap_fault()
428 	if (iova < domain->bounce_size)  in vduse_domain_mmap_fault()
429 		page = vduse_domain_get_bounce_page(domain, iova);  in vduse_domain_mmap_fault()
431 		page = vduse_domain_get_coherent_page(domain, iova);  in vduse_domain_mmap_fault()
447 	struct vduse_iova_domain *domain = file->private_data;  in vduse_domain_mmap()  local
450 	vma->vm_private_data = domain;  in vduse_domain_mmap()
458 	struct vduse_iova_domain *domain = file->private_data;  in vduse_domain_release()  local
460 	spin_lock(&domain->iotlb_lock);  in vduse_domain_release()
461 	vduse_iotlb_del_range(domain, 0, ULLONG_MAX);  in vduse_domain_release()
462 	vduse_domain_free_bounce_pages(domain);  in vduse_domain_release()
463 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_release()
464 	put_iova_domain(&domain->stream_iovad);  in vduse_domain_release()
465 	put_iova_domain(&domain->consistent_iovad);  in vduse_domain_release()
466 	vhost_iotlb_free(domain->iotlb);  in vduse_domain_release()
467 	vfree(domain->bounce_maps);  in vduse_domain_release()
468 	kfree(domain);  in vduse_domain_release()
479 void vduse_domain_destroy(struct vduse_iova_domain *domain)  in vduse_domain_destroy()  argument
481 	fput(domain->file);  in vduse_domain_destroy()
487 	struct vduse_iova_domain *domain;  in vduse_domain_create()  local
496 	domain = kzalloc(sizeof(*domain), GFP_KERNEL);  in vduse_domain_create()
497 	if (!domain)  in vduse_domain_create()
500 	domain->iotlb = vhost_iotlb_alloc(0, 0);  in vduse_domain_create()
501 	if (!domain->iotlb)  in vduse_domain_create()
504 	domain->iova_limit = iova_limit;  in vduse_domain_create()
505 	domain->bounce_size = PAGE_ALIGN(bounce_size);  in vduse_domain_create()
506 	domain->bounce_maps = vzalloc(bounce_pfns *  in vduse_domain_create()
508 	if (!domain->bounce_maps)  in vduse_domain_create()
512 		map = &domain->bounce_maps[pfn];  in vduse_domain_create()
516 				domain, O_RDWR);  in vduse_domain_create()
520 	domain->file = file;  in vduse_domain_create()
521 	spin_lock_init(&domain->iotlb_lock);  in vduse_domain_create()
522 	init_iova_domain(&domain->stream_iovad,  in vduse_domain_create()
524 	init_iova_domain(&domain->consistent_iovad,  in vduse_domain_create()
527 	return domain;  in vduse_domain_create()
529 	vfree(domain->bounce_maps);  in vduse_domain_create()
531 	vhost_iotlb_free(domain->iotlb);  in vduse_domain_create()
533 	kfree(domain);  in vduse_domain_create()