1 /*
2 * DMA coherent memory allocation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the
6 * Free Software Foundation; either version 2 of the License, or (at your
7 * option) any later version.
8 *
9 * Copyright (C) 2002 - 2005 Tensilica Inc.
10 * Copyright (C) 2015 Cadence Design Systems Inc.
11 *
12 * Based on version for i386.
13 *
14 * Chris Zankel <chris@zankel.net>
15 * Joe Taylor <joe@tensilica.com, joetylr@yahoo.com>
16 */
17
18 #include <linux/dma-contiguous.h>
19 #include <linux/gfp.h>
20 #include <linux/highmem.h>
21 #include <linux/mm.h>
22 #include <linux/module.h>
23 #include <linux/pci.h>
24 #include <linux/string.h>
25 #include <linux/types.h>
26 #include <asm/cacheflush.h>
27 #include <asm/io.h>
28
dma_cache_sync(struct device * dev,void * vaddr,size_t size,enum dma_data_direction dir)29 void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
30 enum dma_data_direction dir)
31 {
32 switch (dir) {
33 case DMA_BIDIRECTIONAL:
34 __flush_invalidate_dcache_range((unsigned long)vaddr, size);
35 break;
36
37 case DMA_FROM_DEVICE:
38 __invalidate_dcache_range((unsigned long)vaddr, size);
39 break;
40
41 case DMA_TO_DEVICE:
42 __flush_dcache_range((unsigned long)vaddr, size);
43 break;
44
45 case DMA_NONE:
46 BUG();
47 break;
48 }
49 }
50 EXPORT_SYMBOL(dma_cache_sync);
51
do_cache_op(dma_addr_t dma_handle,size_t size,void (* fn)(unsigned long,unsigned long))52 static void do_cache_op(dma_addr_t dma_handle, size_t size,
53 void (*fn)(unsigned long, unsigned long))
54 {
55 unsigned long off = dma_handle & (PAGE_SIZE - 1);
56 unsigned long pfn = PFN_DOWN(dma_handle);
57 struct page *page = pfn_to_page(pfn);
58
59 if (!PageHighMem(page))
60 fn((unsigned long)bus_to_virt(dma_handle), size);
61 else
62 while (size > 0) {
63 size_t sz = min_t(size_t, size, PAGE_SIZE - off);
64 void *vaddr = kmap_atomic(page);
65
66 fn((unsigned long)vaddr + off, sz);
67 kunmap_atomic(vaddr);
68 off = 0;
69 ++page;
70 size -= sz;
71 }
72 }
73
xtensa_sync_single_for_cpu(struct device * dev,dma_addr_t dma_handle,size_t size,enum dma_data_direction dir)74 static void xtensa_sync_single_for_cpu(struct device *dev,
75 dma_addr_t dma_handle, size_t size,
76 enum dma_data_direction dir)
77 {
78 switch (dir) {
79 case DMA_BIDIRECTIONAL:
80 case DMA_FROM_DEVICE:
81 do_cache_op(dma_handle, size, __invalidate_dcache_range);
82 break;
83
84 case DMA_NONE:
85 BUG();
86 break;
87
88 default:
89 break;
90 }
91 }
92
xtensa_sync_single_for_device(struct device * dev,dma_addr_t dma_handle,size_t size,enum dma_data_direction dir)93 static void xtensa_sync_single_for_device(struct device *dev,
94 dma_addr_t dma_handle, size_t size,
95 enum dma_data_direction dir)
96 {
97 switch (dir) {
98 case DMA_BIDIRECTIONAL:
99 case DMA_TO_DEVICE:
100 if (XCHAL_DCACHE_IS_WRITEBACK)
101 do_cache_op(dma_handle, size, __flush_dcache_range);
102 break;
103
104 case DMA_NONE:
105 BUG();
106 break;
107
108 default:
109 break;
110 }
111 }
112
xtensa_sync_sg_for_cpu(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir)113 static void xtensa_sync_sg_for_cpu(struct device *dev,
114 struct scatterlist *sg, int nents,
115 enum dma_data_direction dir)
116 {
117 struct scatterlist *s;
118 int i;
119
120 for_each_sg(sg, s, nents, i) {
121 xtensa_sync_single_for_cpu(dev, sg_dma_address(s),
122 sg_dma_len(s), dir);
123 }
124 }
125
xtensa_sync_sg_for_device(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir)126 static void xtensa_sync_sg_for_device(struct device *dev,
127 struct scatterlist *sg, int nents,
128 enum dma_data_direction dir)
129 {
130 struct scatterlist *s;
131 int i;
132
133 for_each_sg(sg, s, nents, i) {
134 xtensa_sync_single_for_device(dev, sg_dma_address(s),
135 sg_dma_len(s), dir);
136 }
137 }
138
139 /*
140 * Note: We assume that the full memory space is always mapped to 'kseg'
141 * Otherwise we have to use page attributes (not implemented).
142 */
143
xtensa_dma_alloc(struct device * dev,size_t size,dma_addr_t * handle,gfp_t flag,unsigned long attrs)144 static void *xtensa_dma_alloc(struct device *dev, size_t size,
145 dma_addr_t *handle, gfp_t flag,
146 unsigned long attrs)
147 {
148 unsigned long ret;
149 unsigned long uncached = 0;
150 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
151 struct page *page = NULL;
152
153 /* ignore region speicifiers */
154
155 flag &= ~(__GFP_DMA | __GFP_HIGHMEM);
156
157 if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff))
158 flag |= GFP_DMA;
159
160 if (gfpflags_allow_blocking(flag))
161 page = dma_alloc_from_contiguous(dev, count, get_order(size),
162 flag);
163
164 if (!page)
165 page = alloc_pages(flag, get_order(size));
166
167 if (!page)
168 return NULL;
169
170 ret = (unsigned long)page_address(page);
171
172 /* We currently don't support coherent memory outside KSEG */
173
174 BUG_ON(ret < XCHAL_KSEG_CACHED_VADDR ||
175 ret > XCHAL_KSEG_CACHED_VADDR + XCHAL_KSEG_SIZE - 1);
176
177 uncached = ret + XCHAL_KSEG_BYPASS_VADDR - XCHAL_KSEG_CACHED_VADDR;
178 *handle = virt_to_bus((void *)ret);
179 __invalidate_dcache_range(ret, size);
180
181 return (void *)uncached;
182 }
183
xtensa_dma_free(struct device * dev,size_t size,void * vaddr,dma_addr_t dma_handle,unsigned long attrs)184 static void xtensa_dma_free(struct device *dev, size_t size, void *vaddr,
185 dma_addr_t dma_handle, unsigned long attrs)
186 {
187 unsigned long addr = (unsigned long)vaddr +
188 XCHAL_KSEG_CACHED_VADDR - XCHAL_KSEG_BYPASS_VADDR;
189 struct page *page = virt_to_page(addr);
190 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
191
192 BUG_ON(addr < XCHAL_KSEG_CACHED_VADDR ||
193 addr > XCHAL_KSEG_CACHED_VADDR + XCHAL_KSEG_SIZE - 1);
194
195 if (!dma_release_from_contiguous(dev, page, count))
196 __free_pages(page, get_order(size));
197 }
198
xtensa_map_page(struct device * dev,struct page * page,unsigned long offset,size_t size,enum dma_data_direction dir,unsigned long attrs)199 static dma_addr_t xtensa_map_page(struct device *dev, struct page *page,
200 unsigned long offset, size_t size,
201 enum dma_data_direction dir,
202 unsigned long attrs)
203 {
204 dma_addr_t dma_handle = page_to_phys(page) + offset;
205
206 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
207 xtensa_sync_single_for_device(dev, dma_handle, size, dir);
208
209 return dma_handle;
210 }
211
xtensa_unmap_page(struct device * dev,dma_addr_t dma_handle,size_t size,enum dma_data_direction dir,unsigned long attrs)212 static void xtensa_unmap_page(struct device *dev, dma_addr_t dma_handle,
213 size_t size, enum dma_data_direction dir,
214 unsigned long attrs)
215 {
216 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
217 xtensa_sync_single_for_cpu(dev, dma_handle, size, dir);
218 }
219
xtensa_map_sg(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir,unsigned long attrs)220 static int xtensa_map_sg(struct device *dev, struct scatterlist *sg,
221 int nents, enum dma_data_direction dir,
222 unsigned long attrs)
223 {
224 struct scatterlist *s;
225 int i;
226
227 for_each_sg(sg, s, nents, i) {
228 s->dma_address = xtensa_map_page(dev, sg_page(s), s->offset,
229 s->length, dir, attrs);
230 }
231 return nents;
232 }
233
xtensa_unmap_sg(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir,unsigned long attrs)234 static void xtensa_unmap_sg(struct device *dev,
235 struct scatterlist *sg, int nents,
236 enum dma_data_direction dir,
237 unsigned long attrs)
238 {
239 struct scatterlist *s;
240 int i;
241
242 for_each_sg(sg, s, nents, i) {
243 xtensa_unmap_page(dev, sg_dma_address(s),
244 sg_dma_len(s), dir, attrs);
245 }
246 }
247
xtensa_dma_mapping_error(struct device * dev,dma_addr_t dma_addr)248 int xtensa_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
249 {
250 return 0;
251 }
252
253 const struct dma_map_ops xtensa_dma_map_ops = {
254 .alloc = xtensa_dma_alloc,
255 .free = xtensa_dma_free,
256 .map_page = xtensa_map_page,
257 .unmap_page = xtensa_unmap_page,
258 .map_sg = xtensa_map_sg,
259 .unmap_sg = xtensa_unmap_sg,
260 .sync_single_for_cpu = xtensa_sync_single_for_cpu,
261 .sync_single_for_device = xtensa_sync_single_for_device,
262 .sync_sg_for_cpu = xtensa_sync_sg_for_cpu,
263 .sync_sg_for_device = xtensa_sync_sg_for_device,
264 .mapping_error = xtensa_dma_mapping_error,
265 };
266 EXPORT_SYMBOL(xtensa_dma_map_ops);
267
268 #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
269
xtensa_dma_init(void)270 static int __init xtensa_dma_init(void)
271 {
272 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
273 return 0;
274 }
275 fs_initcall(xtensa_dma_init);
276