16 static void release_registered_pages(struct tee_shm *shm)  in release_registered_pages()  argument
18 	if (shm->pages) {  in release_registered_pages()
19 		if (shm->flags & TEE_SHM_USER_MAPPED) {  in release_registered_pages()
20 			unpin_user_pages(shm->pages, shm->num_pages);  in release_registered_pages()
24 			for (n = 0; n < shm->num_pages; n++)  in release_registered_pages()
25 				put_page(shm->pages[n]);  in release_registered_pages()
28 		kfree(shm->pages);  in release_registered_pages()
32 static void tee_shm_release(struct tee_device *teedev, struct tee_shm *shm)  in tee_shm_release()  argument
34 	if (shm->flags & TEE_SHM_POOL) {  in tee_shm_release()
37 		if (shm->flags & TEE_SHM_DMA_BUF)  in tee_shm_release()
42 		poolm->ops->free(poolm, shm);  in tee_shm_release()
43 	} else if (shm->flags & TEE_SHM_REGISTER) {  in tee_shm_release()
44 		int rc = teedev->desc->ops->shm_unregister(shm->ctx, shm);  in tee_shm_release()
48 				"unregister shm %p failed: %d", shm, rc);  in tee_shm_release()
50 		release_registered_pages(shm);  in tee_shm_release()
53 	teedev_ctx_put(shm->ctx);  in tee_shm_release()
55 	kfree(shm);  in tee_shm_release()
64 	struct tee_shm *shm;  in tee_shm_alloc()  local
88 	shm = kzalloc(sizeof(*shm), GFP_KERNEL);  in tee_shm_alloc()
89 	if (!shm) {  in tee_shm_alloc()
94 	refcount_set(&shm->refcount, 1);  in tee_shm_alloc()
95 	shm->flags = flags | TEE_SHM_POOL;  in tee_shm_alloc()
96 	shm->ctx = ctx;  in tee_shm_alloc()
102 	rc = poolm->ops->alloc(poolm, shm, size);  in tee_shm_alloc()
110 		shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);  in tee_shm_alloc()
112 		if (shm->id < 0) {  in tee_shm_alloc()
113 			ret = ERR_PTR(shm->id);  in tee_shm_alloc()
120 	return shm;  in tee_shm_alloc()
122 	poolm->ops->free(poolm, shm);  in tee_shm_alloc()
124 	kfree(shm);  in tee_shm_alloc()
155 	struct tee_shm *shm;  in tee_shm_register()  local
175 	shm = kzalloc(sizeof(*shm), GFP_KERNEL);  in tee_shm_register()
176 	if (!shm) {  in tee_shm_register()
181 	refcount_set(&shm->refcount, 1);  in tee_shm_register()
182 	shm->flags = flags | TEE_SHM_REGISTER;  in tee_shm_register()
183 	shm->ctx = ctx;  in tee_shm_register()
184 	shm->id = -1;  in tee_shm_register()
187 	shm->offset = addr - start;  in tee_shm_register()
188 	shm->size = length;  in tee_shm_register()
190 	shm->pages = kcalloc(num_pages, sizeof(*shm->pages), GFP_KERNEL);  in tee_shm_register()
191 	if (!shm->pages) {  in tee_shm_register()
198 					 shm->pages);  in tee_shm_register()
214 		rc = get_kernel_pages(kiov, num_pages, 0, shm->pages);  in tee_shm_register()
218 		shm->num_pages = rc;  in tee_shm_register()
227 	shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);  in tee_shm_register()
230 	if (shm->id < 0) {  in tee_shm_register()
231 		ret = ERR_PTR(shm->id);  in tee_shm_register()
235 	rc = teedev->desc->ops->shm_register(ctx, shm, shm->pages,  in tee_shm_register()
236 					     shm->num_pages, start);  in tee_shm_register()
242 	return shm;  in tee_shm_register()
244 	if (shm) {  in tee_shm_register()
245 		if (shm->id >= 0) {  in tee_shm_register()
247 			idr_remove(&teedev->idr, shm->id);  in tee_shm_register()
250 		release_registered_pages(shm);  in tee_shm_register()
252 	kfree(shm);  in tee_shm_register()
267 	struct tee_shm *shm = filp->private_data;  in tee_shm_fop_mmap()  local
271 	if (shm->flags & TEE_SHM_USER_MAPPED)  in tee_shm_fop_mmap()
275 	if (vma->vm_pgoff + vma_pages(vma) > shm->size >> PAGE_SHIFT)  in tee_shm_fop_mmap()
278 	return remap_pfn_range(vma, vma->vm_start, shm->paddr >> PAGE_SHIFT,  in tee_shm_fop_mmap()
293 int tee_shm_get_fd(struct tee_shm *shm)  in tee_shm_get_fd()  argument
297 	if (!(shm->flags & TEE_SHM_DMA_BUF))  in tee_shm_get_fd()
301 	refcount_inc(&shm->refcount);  in tee_shm_get_fd()
302 	fd = anon_inode_getfd("tee_shm", &tee_shm_fops, shm, O_RDWR);  in tee_shm_get_fd()
304 		tee_shm_put(shm);  in tee_shm_get_fd()
312 void tee_shm_free(struct tee_shm *shm)  in tee_shm_free()  argument
314 	tee_shm_put(shm);  in tee_shm_free()
325 int tee_shm_va2pa(struct tee_shm *shm, void *va, phys_addr_t *pa)  in tee_shm_va2pa()  argument
327 	if (!(shm->flags & TEE_SHM_MAPPED))  in tee_shm_va2pa()
330 	if ((char *)va < (char *)shm->kaddr)  in tee_shm_va2pa()
332 	if ((char *)va >= ((char *)shm->kaddr + shm->size))  in tee_shm_va2pa()
336 			shm, (unsigned long)va - (unsigned long)shm->kaddr, pa);  in tee_shm_va2pa()
347 int tee_shm_pa2va(struct tee_shm *shm, phys_addr_t pa, void **va)  in tee_shm_pa2va()  argument
349 	if (!(shm->flags & TEE_SHM_MAPPED))  in tee_shm_pa2va()
352 	if (pa < shm->paddr)  in tee_shm_pa2va()
354 	if (pa >= (shm->paddr + shm->size))  in tee_shm_pa2va()
358 		void *v = tee_shm_get_va(shm, pa - shm->paddr);  in tee_shm_pa2va()
375 void *tee_shm_get_va(struct tee_shm *shm, size_t offs)  in tee_shm_get_va()  argument
377 	if (!(shm->flags & TEE_SHM_MAPPED))  in tee_shm_get_va()
379 	if (offs >= shm->size)  in tee_shm_get_va()
381 	return (char *)shm->kaddr + offs;  in tee_shm_get_va()
393 int tee_shm_get_pa(struct tee_shm *shm, size_t offs, phys_addr_t *pa)  in tee_shm_get_pa()  argument
395 	if (offs >= shm->size)  in tee_shm_get_pa()
398 		*pa = shm->paddr + offs;  in tee_shm_get_pa()
413 	struct tee_shm *shm;  in tee_shm_get_from_id()  local
420 	shm = idr_find(&teedev->idr, id);  in tee_shm_get_from_id()
426 	if (!shm || shm->ctx != ctx)  in tee_shm_get_from_id()
427 		shm = ERR_PTR(-EINVAL);  in tee_shm_get_from_id()
429 		refcount_inc(&shm->refcount);  in tee_shm_get_from_id()
431 	return shm;  in tee_shm_get_from_id()
439 void tee_shm_put(struct tee_shm *shm)  in tee_shm_put()  argument
441 	struct tee_device *teedev = shm->ctx->teedev;  in tee_shm_put()
445 	if (refcount_dec_and_test(&shm->refcount)) {  in tee_shm_put()
452 		if (shm->flags & TEE_SHM_DMA_BUF)  in tee_shm_put()
453 			idr_remove(&teedev->idr, shm->id);  in tee_shm_put()
459 		tee_shm_release(teedev, shm);  in tee_shm_put()