23 static void tee_shm_release(struct tee_shm *shm)  in tee_shm_release()  argument
25 	struct tee_device *teedev = shm->teedev;  in tee_shm_release()
28 	idr_remove(&teedev->idr, shm->id);  in tee_shm_release()
29 	if (shm->ctx)  in tee_shm_release()
30 		list_del(&shm->link);  in tee_shm_release()
33 	if (shm->flags & TEE_SHM_POOL) {  in tee_shm_release()
36 		if (shm->flags & TEE_SHM_DMA_BUF)  in tee_shm_release()
41 		poolm->ops->free(poolm, shm);  in tee_shm_release()
42 	} else if (shm->flags & TEE_SHM_REGISTER) {  in tee_shm_release()
44 		int rc = teedev->desc->ops->shm_unregister(shm->ctx, shm);  in tee_shm_release()
48 				"unregister shm %p failed: %d", shm, rc);  in tee_shm_release()
50 		for (n = 0; n < shm->num_pages; n++)  in tee_shm_release()
51 			put_page(shm->pages[n]);  in tee_shm_release()
53 		kfree(shm->pages);  in tee_shm_release()
56 	if (shm->ctx)  in tee_shm_release()
57 		teedev_ctx_put(shm->ctx);  in tee_shm_release()
59 	kfree(shm);  in tee_shm_release()
78 	struct tee_shm *shm = dmabuf->priv;  in tee_shm_op_release()  local
80 	tee_shm_release(shm);  in tee_shm_op_release()
95 	struct tee_shm *shm = dmabuf->priv;  in tee_shm_op_mmap()  local
99 	if (shm->flags & TEE_SHM_REGISTER)  in tee_shm_op_mmap()
102 	return remap_pfn_range(vma, vma->vm_start, shm->paddr >> PAGE_SHIFT,  in tee_shm_op_mmap()
120 	struct tee_shm *shm;  in __tee_shm_alloc()  local
149 	shm = kzalloc(sizeof(*shm), GFP_KERNEL);  in __tee_shm_alloc()
150 	if (!shm) {  in __tee_shm_alloc()
155 	shm->flags = flags | TEE_SHM_POOL;  in __tee_shm_alloc()
156 	shm->teedev = teedev;  in __tee_shm_alloc()
157 	shm->ctx = ctx;  in __tee_shm_alloc()
163 	rc = poolm->ops->alloc(poolm, shm, size);  in __tee_shm_alloc()
170 	shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);  in __tee_shm_alloc()
172 	if (shm->id < 0) {  in __tee_shm_alloc()
173 		ret = ERR_PTR(shm->id);  in __tee_shm_alloc()
181 		exp_info.size = shm->size;  in __tee_shm_alloc()
183 		exp_info.priv = shm;  in __tee_shm_alloc()
185 		shm->dmabuf = dma_buf_export(&exp_info);  in __tee_shm_alloc()
186 		if (IS_ERR(shm->dmabuf)) {  in __tee_shm_alloc()
187 			ret = ERR_CAST(shm->dmabuf);  in __tee_shm_alloc()
195 		list_add_tail(&shm->link, &ctx->list_shm);  in __tee_shm_alloc()
199 	return shm;  in __tee_shm_alloc()
202 	idr_remove(&teedev->idr, shm->id);  in __tee_shm_alloc()
205 	poolm->ops->free(poolm, shm);  in __tee_shm_alloc()
207 	kfree(shm);  in __tee_shm_alloc()
242 	struct tee_shm *shm;  in tee_shm_register()  local
262 	shm = kzalloc(sizeof(*shm), GFP_KERNEL);  in tee_shm_register()
263 	if (!shm) {  in tee_shm_register()
268 	shm->flags = flags | TEE_SHM_REGISTER;  in tee_shm_register()
269 	shm->teedev = teedev;  in tee_shm_register()
270 	shm->ctx = ctx;  in tee_shm_register()
271 	shm->id = -1;  in tee_shm_register()
273 	shm->offset = addr - start;  in tee_shm_register()
274 	shm->size = length;  in tee_shm_register()
276 	shm->pages = kcalloc(num_pages, sizeof(*shm->pages), GFP_KERNEL);  in tee_shm_register()
277 	if (!shm->pages) {  in tee_shm_register()
282 	rc = get_user_pages_fast(start, num_pages, 1, shm->pages);  in tee_shm_register()
284 		shm->num_pages = rc;  in tee_shm_register()
293 	shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);  in tee_shm_register()
296 	if (shm->id < 0) {  in tee_shm_register()
297 		ret = ERR_PTR(shm->id);  in tee_shm_register()
301 	rc = teedev->desc->ops->shm_register(ctx, shm, shm->pages,  in tee_shm_register()
302 					     shm->num_pages, start);  in tee_shm_register()
312 		exp_info.size = shm->size;  in tee_shm_register()
314 		exp_info.priv = shm;  in tee_shm_register()
316 		shm->dmabuf = dma_buf_export(&exp_info);  in tee_shm_register()
317 		if (IS_ERR(shm->dmabuf)) {  in tee_shm_register()
318 			ret = ERR_CAST(shm->dmabuf);  in tee_shm_register()
319 			teedev->desc->ops->shm_unregister(ctx, shm);  in tee_shm_register()
325 	list_add_tail(&shm->link, &ctx->list_shm);  in tee_shm_register()
328 	return shm;  in tee_shm_register()
330 	if (shm) {  in tee_shm_register()
333 		if (shm->id >= 0) {  in tee_shm_register()
335 			idr_remove(&teedev->idr, shm->id);  in tee_shm_register()
338 		if (shm->pages) {  in tee_shm_register()
339 			for (n = 0; n < shm->num_pages; n++)  in tee_shm_register()
340 				put_page(shm->pages[n]);  in tee_shm_register()
341 			kfree(shm->pages);  in tee_shm_register()
344 	kfree(shm);  in tee_shm_register()
356 int tee_shm_get_fd(struct tee_shm *shm)  in tee_shm_get_fd()  argument
360 	if (!(shm->flags & TEE_SHM_DMA_BUF))  in tee_shm_get_fd()
363 	fd = dma_buf_fd(shm->dmabuf, O_CLOEXEC);  in tee_shm_get_fd()
365 		get_dma_buf(shm->dmabuf);  in tee_shm_get_fd()
373 void tee_shm_free(struct tee_shm *shm)  in tee_shm_free()  argument
382 	if (shm->flags & TEE_SHM_DMA_BUF)  in tee_shm_free()
383 		dma_buf_put(shm->dmabuf);  in tee_shm_free()
385 		tee_shm_release(shm);  in tee_shm_free()
396 int tee_shm_va2pa(struct tee_shm *shm, void *va, phys_addr_t *pa)  in tee_shm_va2pa()  argument
398 	if (!(shm->flags & TEE_SHM_MAPPED))  in tee_shm_va2pa()
401 	if ((char *)va < (char *)shm->kaddr)  in tee_shm_va2pa()
403 	if ((char *)va >= ((char *)shm->kaddr + shm->size))  in tee_shm_va2pa()
407 			shm, (unsigned long)va - (unsigned long)shm->kaddr, pa);  in tee_shm_va2pa()
418 int tee_shm_pa2va(struct tee_shm *shm, phys_addr_t pa, void **va)  in tee_shm_pa2va()  argument
420 	if (!(shm->flags & TEE_SHM_MAPPED))  in tee_shm_pa2va()
423 	if (pa < shm->paddr)  in tee_shm_pa2va()
425 	if (pa >= (shm->paddr + shm->size))  in tee_shm_pa2va()
429 		void *v = tee_shm_get_va(shm, pa - shm->paddr);  in tee_shm_pa2va()
446 void *tee_shm_get_va(struct tee_shm *shm, size_t offs)  in tee_shm_get_va()  argument
448 	if (!(shm->flags & TEE_SHM_MAPPED))  in tee_shm_get_va()
450 	if (offs >= shm->size)  in tee_shm_get_va()
452 	return (char *)shm->kaddr + offs;  in tee_shm_get_va()
464 int tee_shm_get_pa(struct tee_shm *shm, size_t offs, phys_addr_t *pa)  in tee_shm_get_pa()  argument
466 	if (offs >= shm->size)  in tee_shm_get_pa()
469 		*pa = shm->paddr + offs;  in tee_shm_get_pa()
484 	struct tee_shm *shm;  in tee_shm_get_from_id()  local
491 	shm = idr_find(&teedev->idr, id);  in tee_shm_get_from_id()
492 	if (!shm || shm->ctx != ctx)  in tee_shm_get_from_id()
493 		shm = ERR_PTR(-EINVAL);  in tee_shm_get_from_id()
494 	else if (shm->flags & TEE_SHM_DMA_BUF)  in tee_shm_get_from_id()
495 		get_dma_buf(shm->dmabuf);  in tee_shm_get_from_id()
497 	return shm;  in tee_shm_get_from_id()
505 void tee_shm_put(struct tee_shm *shm)  in tee_shm_put()  argument
507 	if (shm->flags & TEE_SHM_DMA_BUF)  in tee_shm_put()
508 		dma_buf_put(shm->dmabuf);  in tee_shm_put()