Lines Matching +full:dma +full:- +full:mem
1 // SPDX-License-Identifier: GPL-2.0
3 * Coherent per-device memory handling.
10 #include <linux/dma-mapping.h>
27 if (dev && dev->dma_mem) in dev_get_coherent_memory()
28 return dev->dma_mem; in dev_get_coherent_memory()
33 struct dma_coherent_mem * mem) in dma_get_device_base() argument
35 if (mem->use_dev_dma_pfn_offset) in dma_get_device_base()
36 return (mem->pfn_base - dev->dma_pfn_offset) << PAGE_SHIFT; in dma_get_device_base()
38 return mem->device_base; in dma_get_device_base()
43 struct dma_coherent_mem **mem) in dma_init_coherent_memory() argument
52 ret = -EINVAL; in dma_init_coherent_memory()
58 ret = -EINVAL; in dma_init_coherent_memory()
63 ret = -ENOMEM; in dma_init_coherent_memory()
66 dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL); in dma_init_coherent_memory()
67 if (!dma_mem->bitmap) { in dma_init_coherent_memory()
68 ret = -ENOMEM; in dma_init_coherent_memory()
72 dma_mem->virt_base = mem_base; in dma_init_coherent_memory()
73 dma_mem->device_base = device_addr; in dma_init_coherent_memory()
74 dma_mem->pfn_base = PFN_DOWN(phys_addr); in dma_init_coherent_memory()
75 dma_mem->size = pages; in dma_init_coherent_memory()
76 dma_mem->flags = flags; in dma_init_coherent_memory()
77 spin_lock_init(&dma_mem->spinlock); in dma_init_coherent_memory()
79 *mem = dma_mem; in dma_init_coherent_memory()
89 static void dma_release_coherent_memory(struct dma_coherent_mem *mem) in dma_release_coherent_memory() argument
91 if (!mem) in dma_release_coherent_memory()
94 memunmap(mem->virt_base); in dma_release_coherent_memory()
95 kfree(mem->bitmap); in dma_release_coherent_memory()
96 kfree(mem); in dma_release_coherent_memory()
100 struct dma_coherent_mem *mem) in dma_assign_coherent_memory() argument
103 return -ENODEV; in dma_assign_coherent_memory()
105 if (dev->dma_mem) in dma_assign_coherent_memory()
106 return -EBUSY; in dma_assign_coherent_memory()
108 dev->dma_mem = mem; in dma_assign_coherent_memory()
115 struct dma_coherent_mem *mem; in dma_declare_coherent_memory() local
118 ret = dma_init_coherent_memory(phys_addr, device_addr, size, flags, &mem); in dma_declare_coherent_memory()
122 ret = dma_assign_coherent_memory(dev, mem); in dma_declare_coherent_memory()
124 dma_release_coherent_memory(mem); in dma_declare_coherent_memory()
131 struct dma_coherent_mem *mem = dev->dma_mem; in dma_release_declared_memory() local
133 if (!mem) in dma_release_declared_memory()
135 dma_release_coherent_memory(mem); in dma_release_declared_memory()
136 dev->dma_mem = NULL; in dma_release_declared_memory()
143 struct dma_coherent_mem *mem = dev->dma_mem; in dma_mark_declared_memory_occupied() local
149 if (!mem) in dma_mark_declared_memory_occupied()
150 return ERR_PTR(-EINVAL); in dma_mark_declared_memory_occupied()
152 spin_lock_irqsave(&mem->spinlock, flags); in dma_mark_declared_memory_occupied()
153 pos = PFN_DOWN(device_addr - dma_get_device_base(dev, mem)); in dma_mark_declared_memory_occupied()
154 err = bitmap_allocate_region(mem->bitmap, pos, get_order(size)); in dma_mark_declared_memory_occupied()
155 spin_unlock_irqrestore(&mem->spinlock, flags); in dma_mark_declared_memory_occupied()
159 return mem->virt_base + (pos << PAGE_SHIFT); in dma_mark_declared_memory_occupied()
163 static void *__dma_alloc_from_coherent(struct dma_coherent_mem *mem, in __dma_alloc_from_coherent() argument
171 spin_lock_irqsave(&mem->spinlock, flags); in __dma_alloc_from_coherent()
173 if (unlikely(size > (mem->size << PAGE_SHIFT))) in __dma_alloc_from_coherent()
176 pageno = bitmap_find_free_region(mem->bitmap, mem->size, order); in __dma_alloc_from_coherent()
183 *dma_handle = mem->device_base + (pageno << PAGE_SHIFT); in __dma_alloc_from_coherent()
184 ret = mem->virt_base + (pageno << PAGE_SHIFT); in __dma_alloc_from_coherent()
185 spin_unlock_irqrestore(&mem->spinlock, flags); in __dma_alloc_from_coherent()
189 spin_unlock_irqrestore(&mem->spinlock, flags); in __dma_alloc_from_coherent()
194 * dma_alloc_from_dev_coherent() - allocate memory from device coherent pool
197 * @dma_handle: This will be filled with the correct dma handle
201 * This function should be only called from per-arch dma_alloc_coherent()
202 * to support allocation from per-device coherent memory pools.
210 struct dma_coherent_mem *mem = dev_get_coherent_memory(dev); in dma_alloc_from_dev_coherent() local
212 if (!mem) in dma_alloc_from_dev_coherent()
215 *ret = __dma_alloc_from_coherent(mem, size, dma_handle); in dma_alloc_from_dev_coherent()
221 * per-device area, try to fall back to generic memory if the in dma_alloc_from_dev_coherent()
224 return mem->flags & DMA_MEMORY_EXCLUSIVE; in dma_alloc_from_dev_coherent()
237 static int __dma_release_from_coherent(struct dma_coherent_mem *mem, in __dma_release_from_coherent() argument
240 if (mem && vaddr >= mem->virt_base && vaddr < in __dma_release_from_coherent()
241 (mem->virt_base + (mem->size << PAGE_SHIFT))) { in __dma_release_from_coherent()
242 int page = (vaddr - mem->virt_base) >> PAGE_SHIFT; in __dma_release_from_coherent()
245 spin_lock_irqsave(&mem->spinlock, flags); in __dma_release_from_coherent()
246 bitmap_release_region(mem->bitmap, page, order); in __dma_release_from_coherent()
247 spin_unlock_irqrestore(&mem->spinlock, flags); in __dma_release_from_coherent()
254 * dma_release_from_dev_coherent() - free memory to device coherent memory pool
259 * This checks whether the memory was allocated from the per-device
267 struct dma_coherent_mem *mem = dev_get_coherent_memory(dev); in dma_release_from_dev_coherent() local
269 return __dma_release_from_coherent(mem, order, vaddr); in dma_release_from_dev_coherent()
282 static int __dma_mmap_from_coherent(struct dma_coherent_mem *mem, in __dma_mmap_from_coherent() argument
285 if (mem && vaddr >= mem->virt_base && vaddr + size <= in __dma_mmap_from_coherent()
286 (mem->virt_base + (mem->size << PAGE_SHIFT))) { in __dma_mmap_from_coherent()
287 unsigned long off = vma->vm_pgoff; in __dma_mmap_from_coherent()
288 int start = (vaddr - mem->virt_base) >> PAGE_SHIFT; in __dma_mmap_from_coherent()
292 *ret = -ENXIO; in __dma_mmap_from_coherent()
293 if (off < count && user_count <= count - off) { in __dma_mmap_from_coherent()
294 unsigned long pfn = mem->pfn_base + start + off; in __dma_mmap_from_coherent()
295 *ret = remap_pfn_range(vma, vma->vm_start, pfn, in __dma_mmap_from_coherent()
297 vma->vm_page_prot); in __dma_mmap_from_coherent()
305 * dma_mmap_from_dev_coherent() - mmap memory from the device coherent pool
312 * This checks whether the memory was allocated from the per-device
322 struct dma_coherent_mem *mem = dev_get_coherent_memory(dev); in dma_mmap_from_dev_coherent() local
324 return __dma_mmap_from_coherent(mem, vma, vaddr, size, ret); in dma_mmap_from_dev_coherent()
350 struct dma_coherent_mem *mem = rmem->priv; in rmem_dma_device_init() local
353 if (!mem) { in rmem_dma_device_init()
354 ret = dma_init_coherent_memory(rmem->base, rmem->base, in rmem_dma_device_init()
355 rmem->size, in rmem_dma_device_init()
356 DMA_MEMORY_EXCLUSIVE, &mem); in rmem_dma_device_init()
358 pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %ld MiB\n", in rmem_dma_device_init()
359 &rmem->base, (unsigned long)rmem->size / SZ_1M); in rmem_dma_device_init()
363 mem->use_dev_dma_pfn_offset = true; in rmem_dma_device_init()
364 rmem->priv = mem; in rmem_dma_device_init()
365 dma_assign_coherent_memory(dev, mem); in rmem_dma_device_init()
373 dev->dma_mem = NULL; in rmem_dma_device_release()
383 unsigned long node = rmem->fdt_node; in rmem_dma_setup()
386 return -EINVAL; in rmem_dma_setup()
389 if (!of_get_flat_dt_prop(node, "no-map", NULL)) { in rmem_dma_setup()
390 pr_err("Reserved memory: regions without no-map are not yet supported\n"); in rmem_dma_setup()
391 return -EINVAL; in rmem_dma_setup()
394 if (of_get_flat_dt_prop(node, "linux,dma-default", NULL)) { in rmem_dma_setup()
396 "Reserved memory: region for default DMA coherent area is redefined\n"); in rmem_dma_setup()
401 rmem->ops = &rmem_dma_ops; in rmem_dma_setup()
402 pr_info("Reserved memory: created DMA memory pool at %pa, size %ld MiB\n", in rmem_dma_setup()
403 &rmem->base, (unsigned long)rmem->size / SZ_1M); in rmem_dma_setup()
413 return -ENOMEM; in dma_init_reserved_memory()
415 ops = dma_reserved_default_memory->ops; in dma_init_reserved_memory()
421 ret = ops->device_init(dma_reserved_default_memory, NULL); in dma_init_reserved_memory()
424 dma_coherent_default_memory = dma_reserved_default_memory->priv; in dma_init_reserved_memory()
425 pr_info("DMA: default coherent area is set\n"); in dma_init_reserved_memory()
433 RESERVEDMEM_OF_DECLARE(dma, "shared-dma-pool", rmem_dma_setup);