1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Framework for userspace DMA-BUF allocations
4 *
5 * Copyright (C) 2011 Google, Inc.
6 * Copyright (C) 2019 Linaro Ltd.
7 */
8
9 #include <linux/cdev.h>
10 #include <linux/device.h>
11 #include <linux/dma-buf.h>
12 #include <linux/dma-heap.h>
13 #include <linux/err.h>
14 #include <linux/list.h>
15 #include <linux/nospec.h>
16 #include <linux/page_size_compat.h>
17 #include <linux/syscalls.h>
18 #include <linux/uaccess.h>
19 #include <linux/xarray.h>
20 #include <uapi/linux/dma-heap.h>
21 #include <trace/hooks/dmabuf.h>
22
23 #define DEVNAME "dma_heap"
24
25 #define NUM_HEAP_MINORS 128
26
27 /**
28 * struct dma_heap - represents a dmabuf heap in the system
29 * @name: used for debugging/device-node name
30 * @ops: ops struct for this heap
31 * @priv: private data for this heap
32 * @heap_devt heap device node
33 * @list list head connecting to list of heaps
34 * @heap_cdev heap char device
35 * @heap_dev heap device struct
36 *
37 * Represents a heap of memory from which buffers can be made.
38 */
39 struct dma_heap {
40 const char *name;
41 const struct dma_heap_ops *ops;
42 void *priv;
43 dev_t heap_devt;
44 struct list_head list;
45 struct cdev heap_cdev;
46 struct kref refcount;
47 struct device *heap_dev;
48 };
49
50 static LIST_HEAD(heap_list);
51 static DEFINE_MUTEX(heap_list_lock);
52 static dev_t dma_heap_devt;
53 static struct class *dma_heap_class;
54 static DEFINE_XARRAY_ALLOC(dma_heap_minors);
55
dma_heap_find(const char * name)56 struct dma_heap *dma_heap_find(const char *name)
57 {
58 struct dma_heap *h;
59
60 mutex_lock(&heap_list_lock);
61 list_for_each_entry(h, &heap_list, list) {
62 if (!strcmp(h->name, name)) {
63 kref_get(&h->refcount);
64 mutex_unlock(&heap_list_lock);
65 return h;
66 }
67 }
68 mutex_unlock(&heap_list_lock);
69 return NULL;
70 }
71 EXPORT_SYMBOL_GPL(dma_heap_find);
72
73
dma_heap_buffer_free(struct dma_buf * dmabuf)74 void dma_heap_buffer_free(struct dma_buf *dmabuf)
75 {
76 dma_buf_put(dmabuf);
77 }
78 EXPORT_SYMBOL_GPL(dma_heap_buffer_free);
79
dma_heap_buffer_alloc(struct dma_heap * heap,size_t len,u32 fd_flags,u64 heap_flags)80 struct dma_buf *dma_heap_buffer_alloc(struct dma_heap *heap, size_t len,
81 u32 fd_flags,
82 u64 heap_flags)
83 {
84 struct dma_buf *dma_buf;
85
86 if (fd_flags & ~DMA_HEAP_VALID_FD_FLAGS)
87 return ERR_PTR(-EINVAL);
88
89 if (heap_flags & ~DMA_HEAP_VALID_HEAP_FLAGS)
90 return ERR_PTR(-EINVAL);
91 /*
92 * Allocations from all heaps have to begin
93 * and end on page boundaries.
94 */
95 len = __PAGE_ALIGN(len);
96 if (!len)
97 return ERR_PTR(-EINVAL);
98
99 trace_android_vh_dma_heap_buffer_alloc_start(heap->name, len,
100 fd_flags, heap_flags);
101 dma_buf = heap->ops->allocate(heap, len, fd_flags, heap_flags);
102 trace_android_vh_dma_heap_buffer_alloc_end(heap->name, len);
103
104 return dma_buf;
105 }
106 EXPORT_SYMBOL_GPL(dma_heap_buffer_alloc);
107
dma_heap_bufferfd_alloc(struct dma_heap * heap,size_t len,u32 fd_flags,u64 heap_flags)108 int dma_heap_bufferfd_alloc(struct dma_heap *heap, size_t len,
109 u32 fd_flags,
110 u64 heap_flags)
111 {
112 struct dma_buf *dmabuf;
113 int fd;
114
115 dmabuf = dma_heap_buffer_alloc(heap, len, fd_flags, heap_flags);
116
117 if (IS_ERR(dmabuf))
118 return PTR_ERR(dmabuf);
119
120 fd = dma_buf_fd(dmabuf, fd_flags);
121 if (fd < 0) {
122 dma_buf_put(dmabuf);
123 /* just return, as put will call release and that will free */
124 }
125 return fd;
126
127 }
128 EXPORT_SYMBOL_GPL(dma_heap_bufferfd_alloc);
129
dma_heap_open(struct inode * inode,struct file * file)130 static int dma_heap_open(struct inode *inode, struct file *file)
131 {
132 struct dma_heap *heap;
133
134 heap = xa_load(&dma_heap_minors, iminor(inode));
135 if (!heap) {
136 pr_err("dma_heap: minor %d unknown.\n", iminor(inode));
137 return -ENODEV;
138 }
139
140 /* instance data as context */
141 file->private_data = heap;
142 nonseekable_open(inode, file);
143
144 return 0;
145 }
146
dma_heap_ioctl_allocate(struct file * file,void * data)147 static long dma_heap_ioctl_allocate(struct file *file, void *data)
148 {
149 struct dma_heap_allocation_data *heap_allocation = data;
150 struct dma_heap *heap = file->private_data;
151 int fd;
152
153 if (heap_allocation->fd)
154 return -EINVAL;
155
156 fd = dma_heap_bufferfd_alloc(heap, heap_allocation->len,
157 heap_allocation->fd_flags,
158 heap_allocation->heap_flags);
159 if (fd < 0)
160 return fd;
161
162 heap_allocation->fd = fd;
163
164 return 0;
165 }
166
167 static unsigned int dma_heap_ioctl_cmds[] = {
168 DMA_HEAP_IOCTL_ALLOC,
169 };
170
dma_heap_ioctl(struct file * file,unsigned int ucmd,unsigned long arg)171 static long dma_heap_ioctl(struct file *file, unsigned int ucmd,
172 unsigned long arg)
173 {
174 char stack_kdata[128];
175 char *kdata = stack_kdata;
176 unsigned int kcmd;
177 unsigned int in_size, out_size, drv_size, ksize;
178 int nr = _IOC_NR(ucmd);
179 int ret = 0;
180
181 if (nr >= ARRAY_SIZE(dma_heap_ioctl_cmds))
182 return -EINVAL;
183
184 nr = array_index_nospec(nr, ARRAY_SIZE(dma_heap_ioctl_cmds));
185 /* Get the kernel ioctl cmd that matches */
186 kcmd = dma_heap_ioctl_cmds[nr];
187
188 /* Figure out the delta between user cmd size and kernel cmd size */
189 drv_size = _IOC_SIZE(kcmd);
190 out_size = _IOC_SIZE(ucmd);
191 in_size = out_size;
192 if ((ucmd & kcmd & IOC_IN) == 0)
193 in_size = 0;
194 if ((ucmd & kcmd & IOC_OUT) == 0)
195 out_size = 0;
196 ksize = max(max(in_size, out_size), drv_size);
197
198 /* If necessary, allocate buffer for ioctl argument */
199 if (ksize > sizeof(stack_kdata)) {
200 kdata = kmalloc(ksize, GFP_KERNEL);
201 if (!kdata)
202 return -ENOMEM;
203 }
204
205 if (copy_from_user(kdata, (void __user *)arg, in_size) != 0) {
206 ret = -EFAULT;
207 goto err;
208 }
209
210 /* zero out any difference between the kernel/user structure size */
211 if (ksize > in_size)
212 memset(kdata + in_size, 0, ksize - in_size);
213
214 switch (kcmd) {
215 case DMA_HEAP_IOCTL_ALLOC:
216 ret = dma_heap_ioctl_allocate(file, kdata);
217 break;
218 default:
219 ret = -ENOTTY;
220 goto err;
221 }
222
223 if (copy_to_user((void __user *)arg, kdata, out_size) != 0)
224 ret = -EFAULT;
225 err:
226 if (kdata != stack_kdata)
227 kfree(kdata);
228 return ret;
229 }
230
231 static const struct file_operations dma_heap_fops = {
232 .owner = THIS_MODULE,
233 .open = dma_heap_open,
234 .unlocked_ioctl = dma_heap_ioctl,
235 #ifdef CONFIG_COMPAT
236 .compat_ioctl = dma_heap_ioctl,
237 #endif
238 };
239
240 /**
241 * dma_heap_get_drvdata - get per-heap driver data
242 * @heap: DMA-Heap to retrieve private data for
243 *
244 * Returns:
245 * The per-heap data for the heap.
246 */
dma_heap_get_drvdata(struct dma_heap * heap)247 void *dma_heap_get_drvdata(struct dma_heap *heap)
248 {
249 return heap->priv;
250 }
251 EXPORT_SYMBOL_GPL(dma_heap_get_drvdata);
252
dma_heap_release(struct kref * ref)253 static void dma_heap_release(struct kref *ref)
254 {
255 struct dma_heap *heap = container_of(ref, struct dma_heap, refcount);
256 int minor = MINOR(heap->heap_devt);
257
258 /* Note, we already holding the heap_list_lock here */
259 list_del(&heap->list);
260
261 device_destroy(dma_heap_class, heap->heap_devt);
262 cdev_del(&heap->heap_cdev);
263 xa_erase(&dma_heap_minors, minor);
264
265 kfree(heap);
266 }
267
dma_heap_put(struct dma_heap * h)268 void dma_heap_put(struct dma_heap *h)
269 {
270 /*
271 * Take the heap_list_lock now to avoid racing with code
272 * scanning the list and then taking a kref.
273 */
274 mutex_lock(&heap_list_lock);
275 kref_put(&h->refcount, dma_heap_release);
276 mutex_unlock(&heap_list_lock);
277 }
278 EXPORT_SYMBOL_GPL(dma_heap_put);
279
280 /**
281 * dma_heap_get_dev() - get device struct for the heap
282 * @heap: DMA-Heap to retrieve device struct from
283 *
284 * Returns:
285 * The device struct for the heap.
286 */
dma_heap_get_dev(struct dma_heap * heap)287 struct device *dma_heap_get_dev(struct dma_heap *heap)
288 {
289 return heap->heap_dev;
290 }
291 EXPORT_SYMBOL_GPL(dma_heap_get_dev);
292
293 /**
294 * dma_heap_get_name - get heap name
295 * @heap: DMA-Heap to retrieve the name of
296 *
297 * Returns:
298 * The char* for the heap name.
299 */
dma_heap_get_name(struct dma_heap * heap)300 const char *dma_heap_get_name(struct dma_heap *heap)
301 {
302 return heap->name;
303 }
304 EXPORT_SYMBOL_GPL(dma_heap_get_name);
305
306 /**
307 * dma_heap_add - adds a heap to dmabuf heaps
308 * @exp_info: information needed to register this heap
309 */
dma_heap_add(const struct dma_heap_export_info * exp_info)310 struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info)
311 {
312 struct dma_heap *heap, *h, *err_ret;
313 unsigned int minor;
314 int ret;
315
316 if (!exp_info->name || !strcmp(exp_info->name, "")) {
317 pr_err("dma_heap: Cannot add heap without a name\n");
318 return ERR_PTR(-EINVAL);
319 }
320
321 if (!exp_info->ops || !exp_info->ops->allocate) {
322 pr_err("dma_heap: Cannot add heap with invalid ops struct\n");
323 return ERR_PTR(-EINVAL);
324 }
325
326 heap = kzalloc(sizeof(*heap), GFP_KERNEL);
327 if (!heap)
328 return ERR_PTR(-ENOMEM);
329
330 kref_init(&heap->refcount);
331 heap->name = exp_info->name;
332 heap->ops = exp_info->ops;
333 heap->priv = exp_info->priv;
334
335 /* Find unused minor number */
336 ret = xa_alloc(&dma_heap_minors, &minor, heap,
337 XA_LIMIT(0, NUM_HEAP_MINORS - 1), GFP_KERNEL);
338 if (ret < 0) {
339 pr_err("dma_heap: Unable to get minor number for heap\n");
340 err_ret = ERR_PTR(ret);
341 goto err0;
342 }
343
344 /* Create device */
345 heap->heap_devt = MKDEV(MAJOR(dma_heap_devt), minor);
346
347 cdev_init(&heap->heap_cdev, &dma_heap_fops);
348 ret = cdev_add(&heap->heap_cdev, heap->heap_devt, 1);
349 if (ret < 0) {
350 pr_err("dma_heap: Unable to add char device\n");
351 err_ret = ERR_PTR(ret);
352 goto err1;
353 }
354
355 heap->heap_dev = device_create(dma_heap_class,
356 NULL,
357 heap->heap_devt,
358 NULL,
359 heap->name);
360 if (IS_ERR(heap->heap_dev)) {
361 pr_err("dma_heap: Unable to create device\n");
362 err_ret = ERR_CAST(heap->heap_dev);
363 goto err2;
364 }
365
366 /* Make sure it doesn't disappear on us */
367 heap->heap_dev = get_device(heap->heap_dev);
368
369 mutex_lock(&heap_list_lock);
370 /* check the name is unique */
371 list_for_each_entry(h, &heap_list, list) {
372 if (!strcmp(h->name, exp_info->name)) {
373 mutex_unlock(&heap_list_lock);
374 pr_err("dma_heap: Already registered heap named %s\n",
375 exp_info->name);
376 err_ret = ERR_PTR(-EINVAL);
377 put_device(heap->heap_dev);
378 goto err3;
379 }
380 }
381
382 /* Add heap to the list */
383 list_add(&heap->list, &heap_list);
384 mutex_unlock(&heap_list_lock);
385
386 return heap;
387
388 err3:
389 device_destroy(dma_heap_class, heap->heap_devt);
390 err2:
391 cdev_del(&heap->heap_cdev);
392 err1:
393 xa_erase(&dma_heap_minors, minor);
394 err0:
395 kfree(heap);
396 return err_ret;
397 }
398 EXPORT_SYMBOL_GPL(dma_heap_add);
399
dma_heap_devnode(const struct device * dev,umode_t * mode)400 static char *dma_heap_devnode(const struct device *dev, umode_t *mode)
401 {
402 return kasprintf(GFP_KERNEL, "dma_heap/%s", dev_name(dev));
403 }
404
dma_heap_try_get_pool_size_kb(void)405 long dma_heap_try_get_pool_size_kb(void)
406 {
407 struct dma_heap *heap;
408 u64 total_pool_size = 0;
409
410 if (!mutex_trylock(&heap_list_lock))
411 return -1;
412
413 list_for_each_entry(heap, &heap_list, list) {
414 if (heap->ops->get_pool_size)
415 total_pool_size += heap->ops->get_pool_size(heap);
416 }
417 mutex_unlock(&heap_list_lock);
418
419 return (long)(total_pool_size / 1024);
420 }
421 EXPORT_SYMBOL_GPL(dma_heap_try_get_pool_size_kb);
422
total_pools_kb_show(struct kobject * kobj,struct kobj_attribute * attr,char * buf)423 static ssize_t total_pools_kb_show(struct kobject *kobj,
424 struct kobj_attribute *attr, char *buf)
425 {
426 struct dma_heap *heap;
427 u64 total_pool_size = 0;
428
429 mutex_lock(&heap_list_lock);
430 list_for_each_entry(heap, &heap_list, list) {
431 if (heap->ops->get_pool_size)
432 total_pool_size += heap->ops->get_pool_size(heap);
433 }
434 mutex_unlock(&heap_list_lock);
435
436 return sysfs_emit(buf, "%llu\n", total_pool_size / 1024);
437 }
438
439 static struct kobj_attribute total_pools_kb_attr =
440 __ATTR_RO(total_pools_kb);
441
442 static struct attribute *dma_heap_sysfs_attrs[] = {
443 &total_pools_kb_attr.attr,
444 NULL,
445 };
446
447 ATTRIBUTE_GROUPS(dma_heap_sysfs);
448
449 static struct kobject *dma_heap_kobject;
450
dma_heap_sysfs_setup(void)451 static int dma_heap_sysfs_setup(void)
452 {
453 int ret;
454
455 dma_heap_kobject = kobject_create_and_add("dma_heap", kernel_kobj);
456 if (!dma_heap_kobject)
457 return -ENOMEM;
458
459 ret = sysfs_create_groups(dma_heap_kobject, dma_heap_sysfs_groups);
460 if (ret) {
461 kobject_put(dma_heap_kobject);
462 return ret;
463 }
464
465 return 0;
466 }
467
dma_heap_sysfs_teardown(void)468 static void dma_heap_sysfs_teardown(void)
469 {
470 kobject_put(dma_heap_kobject);
471 }
472
dma_heap_init(void)473 static int dma_heap_init(void)
474 {
475 int ret;
476
477 ret = dma_heap_sysfs_setup();
478 if (ret)
479 return ret;
480
481 ret = alloc_chrdev_region(&dma_heap_devt, 0, NUM_HEAP_MINORS, DEVNAME);
482 if (ret)
483 goto err_chrdev;
484
485 dma_heap_class = class_create(DEVNAME);
486 if (IS_ERR(dma_heap_class)) {
487 unregister_chrdev_region(dma_heap_devt, NUM_HEAP_MINORS);
488 ret = PTR_ERR(dma_heap_class);
489 goto err_class;
490 }
491 dma_heap_class->devnode = dma_heap_devnode;
492
493 return 0;
494
495 err_class:
496 unregister_chrdev_region(dma_heap_devt, NUM_HEAP_MINORS);
497 err_chrdev:
498 dma_heap_sysfs_teardown();
499 return ret;
500 }
501 subsys_initcall(dma_heap_init);
502