180 	struct hl_userptr *userptr;  in dma_map_host_va()  local
183 	userptr = kzalloc(sizeof(*userptr), GFP_KERNEL);  in dma_map_host_va()
184 	if (!userptr) {  in dma_map_host_va()
189 	rc = hl_pin_host_memory(hdev, addr, size, userptr);  in dma_map_host_va()
195 	rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl,  in dma_map_host_va()
196 					userptr->sgt->nents, DMA_BIDIRECTIONAL);  in dma_map_host_va()
202 	userptr->dma_mapped = true;  in dma_map_host_va()
203 	userptr->dir = DMA_BIDIRECTIONAL;  in dma_map_host_va()
204 	userptr->vm_type = VM_TYPE_USERPTR;  in dma_map_host_va()
206 	*p_userptr = userptr;  in dma_map_host_va()
211 	hl_unpin_host_memory(hdev, userptr);  in dma_map_host_va()
213 	kfree(userptr);  in dma_map_host_va()
229 				struct hl_userptr *userptr)  in dma_unmap_host_va()  argument
231 	hl_unpin_host_memory(hdev, userptr);  in dma_unmap_host_va()
232 	kfree(userptr);  in dma_unmap_host_va()
634 				struct hl_userptr *userptr,  in init_phys_pg_pack_from_userptr()  argument
651 	phys_pg_pack->vm_type = userptr->vm_type;  in init_phys_pg_pack_from_userptr()
663 	for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) {  in init_phys_pg_pack_from_userptr()
692 	for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) {  in init_phys_pg_pack_from_userptr()
852 	struct hl_userptr *userptr = NULL;  in map_device_va()  local
870 		rc = dma_map_host_va(hdev, addr, size, &userptr);  in map_device_va()
876 		rc = init_phys_pg_pack_from_userptr(ctx, userptr,  in map_device_va()
885 		vm_type = (enum vm_type_t *) userptr;  in map_device_va()
1016 		dma_unmap_host_va(hdev, userptr);  in map_device_va()
1037 	struct hl_userptr *userptr = NULL;  in unmap_device_va()  local
1064 		userptr = hnode->ptr;  in unmap_device_va()
1065 		rc = init_phys_pg_pack_from_userptr(ctx, userptr,  in unmap_device_va()
1144 		dma_unmap_host_va(hdev, userptr);  in unmap_device_va()
1285 				struct hl_userptr *userptr)  in get_user_memory()  argument
1294 	userptr->vec = frame_vector_create(npages);  in get_user_memory()
1295 	if (!userptr->vec) {  in get_user_memory()
1301 				userptr->vec);  in get_user_memory()
1312 	if (frame_vector_to_pages(userptr->vec) < 0) {  in get_user_memory()
1319 	rc = sg_alloc_table_from_pages(userptr->sgt,  in get_user_memory()
1320 					frame_vector_pages(userptr->vec),  in get_user_memory()
1330 	put_vaddr_frames(userptr->vec);  in get_user_memory()
1332 	frame_vector_destroy(userptr->vec);  in get_user_memory()
1348 					struct hl_userptr *userptr)  in hl_pin_host_memory()  argument
1375 	userptr->sgt = kzalloc(sizeof(*userptr->sgt), GFP_ATOMIC);  in hl_pin_host_memory()
1376 	if (!userptr->sgt)  in hl_pin_host_memory()
1384 	userptr->size = size;  in hl_pin_host_memory()
1385 	userptr->addr = addr;  in hl_pin_host_memory()
1386 	userptr->dma_mapped = false;  in hl_pin_host_memory()
1387 	INIT_LIST_HEAD(&userptr->job_node);  in hl_pin_host_memory()
1390 				userptr);  in hl_pin_host_memory()
1398 	hl_debugfs_add_userptr(hdev, userptr);  in hl_pin_host_memory()
1403 	kfree(userptr->sgt);  in hl_pin_host_memory()
1416 void hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr)  in hl_unpin_host_memory()  argument
1420 	hl_debugfs_remove_userptr(hdev, userptr);  in hl_unpin_host_memory()
1422 	if (userptr->dma_mapped)  in hl_unpin_host_memory()
1423 		hdev->asic_funcs->hl_dma_unmap_sg(hdev, userptr->sgt->sgl,  in hl_unpin_host_memory()
1424 							userptr->sgt->nents,  in hl_unpin_host_memory()
1425 							userptr->dir);  in hl_unpin_host_memory()
1427 	pages = frame_vector_pages(userptr->vec);  in hl_unpin_host_memory()
1431 		for (i = 0; i < frame_vector_count(userptr->vec); i++)  in hl_unpin_host_memory()
1434 	put_vaddr_frames(userptr->vec);  in hl_unpin_host_memory()
1435 	frame_vector_destroy(userptr->vec);  in hl_unpin_host_memory()
1437 	list_del(&userptr->job_node);  in hl_unpin_host_memory()
1439 	sg_free_table(userptr->sgt);  in hl_unpin_host_memory()
1440 	kfree(userptr->sgt);  in hl_unpin_host_memory()
1456 	struct hl_userptr *userptr, *tmp;  in hl_userptr_delete_list()  local
1458 	list_for_each_entry_safe(userptr, tmp, userptr_list, job_node) {  in hl_userptr_delete_list()
1459 		hl_unpin_host_memory(hdev, userptr);  in hl_userptr_delete_list()
1460 		kfree(userptr);  in hl_userptr_delete_list()
1479 				struct hl_userptr **userptr)  in hl_userptr_is_pinned()  argument
1481 	list_for_each_entry((*userptr), userptr_list, job_node) {  in hl_userptr_is_pinned()
1482 		if ((addr == (*userptr)->addr) && (size == (*userptr)->size))  in hl_userptr_is_pinned()