31 					     struct dma_coherent_mem * mem)  in dma_get_device_base()  argument
33 	if (mem->use_dev_dma_pfn_offset)  in dma_get_device_base()
34 		return phys_to_dma(dev, PFN_PHYS(mem->pfn_base));  in dma_get_device_base()
35 	return mem->device_base;  in dma_get_device_base()
77 static void _dma_release_coherent_memory(struct dma_coherent_mem *mem)  in _dma_release_coherent_memory()  argument
79 	if (!mem)  in _dma_release_coherent_memory()
82 	memunmap(mem->virt_base);  in _dma_release_coherent_memory()
83 	bitmap_free(mem->bitmap);  in _dma_release_coherent_memory()
84 	kfree(mem);  in _dma_release_coherent_memory()
88 				      struct dma_coherent_mem *mem)  in dma_assign_coherent_memory()  argument
96 	dev->dma_mem = mem;  in dma_assign_coherent_memory()
120 	struct dma_coherent_mem *mem;  in dma_declare_coherent_memory()  local
123 	mem = dma_init_coherent_memory(phys_addr, device_addr, size, false);  in dma_declare_coherent_memory()
124 	if (IS_ERR(mem))  in dma_declare_coherent_memory()
125 		return PTR_ERR(mem);  in dma_declare_coherent_memory()
127 	ret = dma_assign_coherent_memory(dev, mem);  in dma_declare_coherent_memory()
129 		_dma_release_coherent_memory(mem);  in dma_declare_coherent_memory()
142 				       struct dma_coherent_mem *mem,  in __dma_alloc_from_coherent()  argument
150 	spin_lock_irqsave(&mem->spinlock, flags);  in __dma_alloc_from_coherent()
152 	if (unlikely(size > ((dma_addr_t)mem->size << PAGE_SHIFT)))  in __dma_alloc_from_coherent()
155 	pageno = bitmap_find_free_region(mem->bitmap, mem->size, order);  in __dma_alloc_from_coherent()
162 	*dma_handle = dma_get_device_base(dev, mem) +  in __dma_alloc_from_coherent()
164 	ret = mem->virt_base + ((dma_addr_t)pageno << PAGE_SHIFT);  in __dma_alloc_from_coherent()
165 	spin_unlock_irqrestore(&mem->spinlock, flags);  in __dma_alloc_from_coherent()
169 	spin_unlock_irqrestore(&mem->spinlock, flags);  in __dma_alloc_from_coherent()
190 	struct dma_coherent_mem *mem = dev_get_coherent_memory(dev);  in dma_alloc_from_dev_coherent()  local
192 	if (!mem)  in dma_alloc_from_dev_coherent()
195 	*ret = __dma_alloc_from_coherent(dev, mem, size, dma_handle);  in dma_alloc_from_dev_coherent()
199 static int __dma_release_from_coherent(struct dma_coherent_mem *mem,  in __dma_release_from_coherent()  argument
202 	if (mem && vaddr >= mem->virt_base && vaddr <  in __dma_release_from_coherent()
203 		   (mem->virt_base + ((dma_addr_t)mem->size << PAGE_SHIFT))) {  in __dma_release_from_coherent()
204 		int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;  in __dma_release_from_coherent()
207 		spin_lock_irqsave(&mem->spinlock, flags);  in __dma_release_from_coherent()
208 		bitmap_release_region(mem->bitmap, page, order);  in __dma_release_from_coherent()
209 		spin_unlock_irqrestore(&mem->spinlock, flags);  in __dma_release_from_coherent()
229 	struct dma_coherent_mem *mem = dev_get_coherent_memory(dev);  in dma_release_from_dev_coherent()  local
231 	return __dma_release_from_coherent(mem, order, vaddr);  in dma_release_from_dev_coherent()
234 static int __dma_mmap_from_coherent(struct dma_coherent_mem *mem,  in __dma_mmap_from_coherent()  argument
237 	if (mem && vaddr >= mem->virt_base && vaddr + size <=  in __dma_mmap_from_coherent()
238 		   (mem->virt_base + ((dma_addr_t)mem->size << PAGE_SHIFT))) {  in __dma_mmap_from_coherent()
240 		int start = (vaddr - mem->virt_base) >> PAGE_SHIFT;  in __dma_mmap_from_coherent()
246 			unsigned long pfn = mem->pfn_base + start + off;  in __dma_mmap_from_coherent()
274 	struct dma_coherent_mem *mem = dev_get_coherent_memory(dev);  in dma_mmap_from_dev_coherent()  local
276 	return __dma_mmap_from_coherent(mem, vma, vaddr, size, ret);  in dma_mmap_from_dev_coherent()
313 	struct dma_coherent_mem *mem;  in dma_init_global_coherent()  local
315 	mem = dma_init_coherent_memory(phys_addr, phys_addr, size, true);  in dma_init_global_coherent()
316 	if (IS_ERR(mem))  in dma_init_global_coherent()
317 		return PTR_ERR(mem);  in dma_init_global_coherent()
318 	dma_coherent_default_memory = mem;  in dma_init_global_coherent()
339 		struct dma_coherent_mem *mem;  in rmem_dma_device_init()  local
341 		mem = dma_init_coherent_memory(rmem->base, rmem->base,  in rmem_dma_device_init()
343 		if (IS_ERR(mem))  in rmem_dma_device_init()
344 			return PTR_ERR(mem);  in rmem_dma_device_init()
345 		rmem->priv = mem;  in rmem_dma_device_init()