• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright 2020 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Christian König
23  */
24 
25 #include <linux/dma-buf-map.h>
26 #include <linux/io-mapping.h>
27 #include <linux/scatterlist.h>
28 
29 #include <drm/ttm/ttm_resource.h>
30 #include <drm/ttm/ttm_bo_driver.h>
31 
ttm_resource_init(struct ttm_buffer_object * bo,const struct ttm_place * place,struct ttm_resource * res)32 void ttm_resource_init(struct ttm_buffer_object *bo,
33                        const struct ttm_place *place,
34                        struct ttm_resource *res)
35 {
36 	res->start = 0;
37 	res->num_pages = PFN_UP(bo->base.size);
38 	res->mem_type = place->mem_type;
39 	res->placement = place->flags;
40 	res->bus.addr = NULL;
41 	res->bus.offset = 0;
42 	res->bus.is_iomem = false;
43 	res->bus.caching = ttm_cached;
44 }
45 EXPORT_SYMBOL(ttm_resource_init);
46 
ttm_resource_alloc(struct ttm_buffer_object * bo,const struct ttm_place * place,struct ttm_resource ** res_ptr)47 int ttm_resource_alloc(struct ttm_buffer_object *bo,
48 		       const struct ttm_place *place,
49 		       struct ttm_resource **res_ptr)
50 {
51 	struct ttm_resource_manager *man =
52 		ttm_manager_type(bo->bdev, place->mem_type);
53 
54 	return man->func->alloc(man, bo, place, res_ptr);
55 }
56 
ttm_resource_free(struct ttm_buffer_object * bo,struct ttm_resource ** res)57 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
58 {
59 	struct ttm_resource_manager *man;
60 
61 	if (!*res)
62 		return;
63 
64 	man = ttm_manager_type(bo->bdev, (*res)->mem_type);
65 	man->func->free(man, *res);
66 	*res = NULL;
67 }
68 EXPORT_SYMBOL(ttm_resource_free);
69 
70 /**
71  * ttm_resource_manager_init
72  *
73  * @man: memory manager object to init
74  * @p_size: size managed area in pages.
75  *
76  * Initialise core parts of a manager object.
77  */
ttm_resource_manager_init(struct ttm_resource_manager * man,unsigned long p_size)78 void ttm_resource_manager_init(struct ttm_resource_manager *man,
79 			       unsigned long p_size)
80 {
81 	unsigned i;
82 
83 	spin_lock_init(&man->move_lock);
84 	man->size = p_size;
85 
86 	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
87 		INIT_LIST_HEAD(&man->lru[i]);
88 	man->move = NULL;
89 }
90 EXPORT_SYMBOL(ttm_resource_manager_init);
91 
92 /*
93  * ttm_resource_manager_evict_all
94  *
95  * @bdev - device to use
96  * @man - manager to use
97  *
98  * Evict all the objects out of a memory manager until it is empty.
99  * Part of memory manager cleanup sequence.
100  */
ttm_resource_manager_evict_all(struct ttm_device * bdev,struct ttm_resource_manager * man)101 int ttm_resource_manager_evict_all(struct ttm_device *bdev,
102 				   struct ttm_resource_manager *man)
103 {
104 	struct ttm_operation_ctx ctx = {
105 		.interruptible = false,
106 		.no_wait_gpu = false,
107 		.force_alloc = true
108 	};
109 	struct dma_fence *fence;
110 	int ret;
111 	unsigned i;
112 
113 	/*
114 	 * Can't use standard list traversal since we're unlocking.
115 	 */
116 
117 	spin_lock(&bdev->lru_lock);
118 	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
119 		while (!list_empty(&man->lru[i])) {
120 			spin_unlock(&bdev->lru_lock);
121 			ret = ttm_mem_evict_first(bdev, man, NULL, &ctx,
122 						  NULL);
123 			if (ret)
124 				return ret;
125 			spin_lock(&bdev->lru_lock);
126 		}
127 	}
128 	spin_unlock(&bdev->lru_lock);
129 
130 	spin_lock(&man->move_lock);
131 	fence = dma_fence_get(man->move);
132 	spin_unlock(&man->move_lock);
133 
134 	if (fence) {
135 		ret = dma_fence_wait(fence, false);
136 		dma_fence_put(fence);
137 		if (ret)
138 			return ret;
139 	}
140 
141 	return 0;
142 }
143 EXPORT_SYMBOL(ttm_resource_manager_evict_all);
144 
145 /**
146  * ttm_resource_manager_debug
147  *
148  * @man: manager type to dump.
149  * @p: printer to use for debug.
150  */
ttm_resource_manager_debug(struct ttm_resource_manager * man,struct drm_printer * p)151 void ttm_resource_manager_debug(struct ttm_resource_manager *man,
152 				struct drm_printer *p)
153 {
154 	drm_printf(p, "  use_type: %d\n", man->use_type);
155 	drm_printf(p, "  use_tt: %d\n", man->use_tt);
156 	drm_printf(p, "  size: %llu\n", man->size);
157 	if (man->func->debug)
158 		man->func->debug(man, p);
159 }
160 EXPORT_SYMBOL(ttm_resource_manager_debug);
161 
ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter * iter,struct dma_buf_map * dmap,pgoff_t i)162 static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
163 					  struct dma_buf_map *dmap,
164 					  pgoff_t i)
165 {
166 	struct ttm_kmap_iter_iomap *iter_io =
167 		container_of(iter, typeof(*iter_io), base);
168 	void __iomem *addr;
169 
170 retry:
171 	while (i >= iter_io->cache.end) {
172 		iter_io->cache.sg = iter_io->cache.sg ?
173 			sg_next(iter_io->cache.sg) : iter_io->st->sgl;
174 		iter_io->cache.i = iter_io->cache.end;
175 		iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
176 			PAGE_SHIFT;
177 		iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
178 			iter_io->start;
179 	}
180 
181 	if (i < iter_io->cache.i) {
182 		iter_io->cache.end = 0;
183 		iter_io->cache.sg = NULL;
184 		goto retry;
185 	}
186 
187 	addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
188 				       (((resource_size_t)i - iter_io->cache.i)
189 					<< PAGE_SHIFT));
190 	dma_buf_map_set_vaddr_iomem(dmap, addr);
191 }
192 
ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter * iter,struct dma_buf_map * map)193 static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
194 					    struct dma_buf_map *map)
195 {
196 	io_mapping_unmap_local(map->vaddr_iomem);
197 }
198 
199 static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
200 	.map_local =  ttm_kmap_iter_iomap_map_local,
201 	.unmap_local = ttm_kmap_iter_iomap_unmap_local,
202 	.maps_tt = false,
203 };
204 
205 /**
206  * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
207  * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
208  * @iomap: The struct io_mapping representing the underlying linear io_memory.
209  * @st: sg_table into @iomap, representing the memory of the struct
210  * ttm_resource.
211  * @start: Offset that needs to be subtracted from @st to make
212  * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
213  *
214  * Return: Pointer to the embedded struct ttm_kmap_iter.
215  */
216 struct ttm_kmap_iter *
ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap * iter_io,struct io_mapping * iomap,struct sg_table * st,resource_size_t start)217 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
218 			 struct io_mapping *iomap,
219 			 struct sg_table *st,
220 			 resource_size_t start)
221 {
222 	iter_io->base.ops = &ttm_kmap_iter_io_ops;
223 	iter_io->iomap = iomap;
224 	iter_io->st = st;
225 	iter_io->start = start;
226 	memset(&iter_io->cache, 0, sizeof(iter_io->cache));
227 
228 	return &iter_io->base;
229 }
230 EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
231 
232 /**
233  * DOC: Linear io iterator
234  *
235  * This code should die in the not too near future. Best would be if we could
236  * make io-mapping use memremap for all io memory, and have memremap
237  * implement a kmap_local functionality. We could then strip a huge amount of
238  * code. These linear io iterators are implemented to mimic old functionality,
239  * and they don't use kmap_local semantics at all internally. Rather ioremap or
240  * friends, and at least on 32-bit they add global TLB flushes and points
241  * of failure.
242  */
243 
ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter * iter,struct dma_buf_map * dmap,pgoff_t i)244 static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
245 					      struct dma_buf_map *dmap,
246 					      pgoff_t i)
247 {
248 	struct ttm_kmap_iter_linear_io *iter_io =
249 		container_of(iter, typeof(*iter_io), base);
250 
251 	*dmap = iter_io->dmap;
252 	dma_buf_map_incr(dmap, i * PAGE_SIZE);
253 }
254 
255 static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
256 	.map_local =  ttm_kmap_iter_linear_io_map_local,
257 	.maps_tt = false,
258 };
259 
260 /**
261  * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
262  * @iter_io: The iterator to initialize
263  * @bdev: The TTM device
264  * @mem: The ttm resource representing the iomap.
265  *
266  * This function is for internal TTM use only. It sets up a memcpy kmap iterator
267  * pointing at a linear chunk of io memory.
268  *
269  * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
270  * failure.
271  */
272 struct ttm_kmap_iter *
ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io * iter_io,struct ttm_device * bdev,struct ttm_resource * mem)273 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
274 			     struct ttm_device *bdev,
275 			     struct ttm_resource *mem)
276 {
277 	int ret;
278 
279 	ret = ttm_mem_io_reserve(bdev, mem);
280 	if (ret)
281 		goto out_err;
282 	if (!mem->bus.is_iomem) {
283 		ret = -EINVAL;
284 		goto out_io_free;
285 	}
286 
287 	if (mem->bus.addr) {
288 		dma_buf_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
289 		iter_io->needs_unmap = false;
290 	} else {
291 		size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT;
292 
293 		iter_io->needs_unmap = true;
294 		memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
295 		if (mem->bus.caching == ttm_write_combined)
296 			dma_buf_map_set_vaddr_iomem(&iter_io->dmap,
297 						    ioremap_wc(mem->bus.offset,
298 							       bus_size));
299 		else if (mem->bus.caching == ttm_cached)
300 			dma_buf_map_set_vaddr(&iter_io->dmap,
301 					      memremap(mem->bus.offset, bus_size,
302 						       MEMREMAP_WB |
303 						       MEMREMAP_WT |
304 						       MEMREMAP_WC));
305 
306 		/* If uncached requested or if mapping cached or wc failed */
307 		if (dma_buf_map_is_null(&iter_io->dmap))
308 			dma_buf_map_set_vaddr_iomem(&iter_io->dmap,
309 						    ioremap(mem->bus.offset,
310 							    bus_size));
311 
312 		if (dma_buf_map_is_null(&iter_io->dmap)) {
313 			ret = -ENOMEM;
314 			goto out_io_free;
315 		}
316 	}
317 
318 	iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
319 	return &iter_io->base;
320 
321 out_io_free:
322 	ttm_mem_io_free(bdev, mem);
323 out_err:
324 	return ERR_PTR(ret);
325 }
326 
327 /**
328  * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
329  * @iter_io: The iterator to initialize
330  * @bdev: The TTM device
331  * @mem: The ttm resource representing the iomap.
332  *
333  * This function is for internal TTM use only. It cleans up a memcpy kmap
334  * iterator initialized by ttm_kmap_iter_linear_io_init.
335  */
336 void
ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io * iter_io,struct ttm_device * bdev,struct ttm_resource * mem)337 ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
338 			     struct ttm_device *bdev,
339 			     struct ttm_resource *mem)
340 {
341 	if (iter_io->needs_unmap && dma_buf_map_is_set(&iter_io->dmap)) {
342 		if (iter_io->dmap.is_iomem)
343 			iounmap(iter_io->dmap.vaddr_iomem);
344 		else
345 			memunmap(iter_io->dmap.vaddr);
346 	}
347 
348 	ttm_mem_io_free(bdev, mem);
349 }
350