1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2020-2023 Intel Corporation
4 */
5
6 #include <linux/dma-buf.h>
7 #include <linux/highmem.h>
8 #include <linux/module.h>
9 #include <linux/set_memory.h>
10 #include <linux/xarray.h>
11
12 #include <drm/drm_cache.h>
13 #include <drm/drm_debugfs.h>
14 #include <drm/drm_file.h>
15 #include <drm/drm_utils.h>
16
17 #include "ivpu_drv.h"
18 #include "ivpu_gem.h"
19 #include "ivpu_hw.h"
20 #include "ivpu_mmu.h"
21 #include "ivpu_mmu_context.h"
22
23 static const struct drm_gem_object_funcs ivpu_gem_funcs;
24
ivpu_dbg_bo(struct ivpu_device * vdev,struct ivpu_bo * bo,const char * action)25 static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action)
26 {
27 ivpu_dbg(vdev, BO,
28 "%6s: bo %8p vpu_addr %9llx size %8zu ctx %d has_pages %d dma_mapped %d mmu_mapped %d wc %d imported %d\n",
29 action, bo, bo->vpu_addr, ivpu_bo_size(bo), bo->ctx_id,
30 (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc,
31 (bool)bo->base.base.import_attach);
32 }
33
ivpu_bo_lock(struct ivpu_bo * bo)34 static inline int ivpu_bo_lock(struct ivpu_bo *bo)
35 {
36 return dma_resv_lock(bo->base.base.resv, NULL);
37 }
38
ivpu_bo_unlock(struct ivpu_bo * bo)39 static inline void ivpu_bo_unlock(struct ivpu_bo *bo)
40 {
41 dma_resv_unlock(bo->base.base.resv);
42 }
43
44 /*
45 * ivpu_bo_pin() - pin the backing physical pages and map them to VPU.
46 *
47 * This function pins physical memory pages, then maps the physical pages
48 * to IOMMU address space and finally updates the VPU MMU page tables
49 * to allow the VPU to translate VPU address to IOMMU address.
50 */
ivpu_bo_pin(struct ivpu_bo * bo)51 int __must_check ivpu_bo_pin(struct ivpu_bo *bo)
52 {
53 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
54 struct sg_table *sgt;
55 int ret = 0;
56
57 ivpu_dbg_bo(vdev, bo, "pin");
58
59 sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
60 if (IS_ERR(sgt)) {
61 ret = PTR_ERR(sgt);
62 ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);
63 return ret;
64 }
65
66 ivpu_bo_lock(bo);
67
68 if (!bo->mmu_mapped) {
69 drm_WARN_ON(&vdev->drm, !bo->ctx);
70 ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt,
71 ivpu_bo_is_snooped(bo));
72 if (ret) {
73 ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);
74 goto unlock;
75 }
76 bo->mmu_mapped = true;
77 }
78
79 unlock:
80 ivpu_bo_unlock(bo);
81
82 return ret;
83 }
84
85 static int
ivpu_bo_alloc_vpu_addr(struct ivpu_bo * bo,struct ivpu_mmu_context * ctx,const struct ivpu_addr_range * range)86 ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx,
87 const struct ivpu_addr_range *range)
88 {
89 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
90 int idx, ret;
91
92 if (!drm_dev_enter(&vdev->drm, &idx))
93 return -ENODEV;
94
95 ivpu_bo_lock(bo);
96
97 ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node);
98 if (!ret) {
99 bo->ctx = ctx;
100 bo->vpu_addr = bo->mm_node.start;
101 } else {
102 ivpu_err(vdev, "Failed to add BO to context %u: %d\n", ctx->id, ret);
103 }
104
105 ivpu_bo_unlock(bo);
106
107 drm_dev_exit(idx);
108
109 return ret;
110 }
111
ivpu_bo_unbind_locked(struct ivpu_bo * bo)112 static void ivpu_bo_unbind_locked(struct ivpu_bo *bo)
113 {
114 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
115
116 lockdep_assert(dma_resv_held(bo->base.base.resv) || !kref_read(&bo->base.base.refcount));
117
118 if (bo->mmu_mapped) {
119 drm_WARN_ON(&vdev->drm, !bo->ctx);
120 drm_WARN_ON(&vdev->drm, !bo->vpu_addr);
121 drm_WARN_ON(&vdev->drm, !bo->base.sgt);
122 ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt);
123 bo->mmu_mapped = false;
124 }
125
126 if (bo->ctx) {
127 ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node);
128 bo->ctx = NULL;
129 }
130
131 if (bo->base.base.import_attach)
132 return;
133
134 if (bo->base.sgt) {
135 dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0);
136 sg_free_table(bo->base.sgt);
137 kfree(bo->base.sgt);
138 bo->base.sgt = NULL;
139 }
140 }
141
ivpu_bo_unbind_all_bos_from_context(struct ivpu_device * vdev,struct ivpu_mmu_context * ctx)142 void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
143 {
144 struct ivpu_bo *bo;
145
146 if (drm_WARN_ON(&vdev->drm, !ctx))
147 return;
148
149 mutex_lock(&vdev->bo_list_lock);
150 list_for_each_entry(bo, &vdev->bo_list, bo_list_node) {
151 ivpu_bo_lock(bo);
152 if (bo->ctx == ctx) {
153 ivpu_dbg_bo(vdev, bo, "unbind");
154 ivpu_bo_unbind_locked(bo);
155 }
156 ivpu_bo_unlock(bo);
157 }
158 mutex_unlock(&vdev->bo_list_lock);
159 }
160
ivpu_gem_create_object(struct drm_device * dev,size_t size)161 struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size)
162 {
163 struct ivpu_bo *bo;
164
165 if (size == 0 || !PAGE_ALIGNED(size))
166 return ERR_PTR(-EINVAL);
167
168 bo = kzalloc(sizeof(*bo), GFP_KERNEL);
169 if (!bo)
170 return ERR_PTR(-ENOMEM);
171
172 bo->base.base.funcs = &ivpu_gem_funcs;
173 bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */
174
175 INIT_LIST_HEAD(&bo->bo_list_node);
176
177 return &bo->base.base;
178 }
179
ivpu_bo_alloc(struct ivpu_device * vdev,u64 size,u32 flags,u32 ctx_id)180 static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags, u32 ctx_id)
181 {
182 struct drm_gem_shmem_object *shmem;
183 struct ivpu_bo *bo;
184
185 switch (flags & DRM_IVPU_BO_CACHE_MASK) {
186 case DRM_IVPU_BO_CACHED:
187 case DRM_IVPU_BO_WC:
188 break;
189 default:
190 return ERR_PTR(-EINVAL);
191 }
192
193 shmem = drm_gem_shmem_create(&vdev->drm, size);
194 if (IS_ERR(shmem))
195 return ERR_CAST(shmem);
196
197 bo = to_ivpu_bo(&shmem->base);
198 bo->ctx_id = ctx_id;
199 bo->base.map_wc = flags & DRM_IVPU_BO_WC;
200 bo->flags = flags;
201
202 mutex_lock(&vdev->bo_list_lock);
203 list_add_tail(&bo->bo_list_node, &vdev->bo_list);
204 mutex_unlock(&vdev->bo_list_lock);
205
206 ivpu_dbg_bo(vdev, bo, "alloc");
207
208 return bo;
209 }
210
ivpu_gem_bo_open(struct drm_gem_object * obj,struct drm_file * file)211 static int ivpu_gem_bo_open(struct drm_gem_object *obj, struct drm_file *file)
212 {
213 struct ivpu_file_priv *file_priv = file->driver_priv;
214 struct ivpu_device *vdev = file_priv->vdev;
215 struct ivpu_bo *bo = to_ivpu_bo(obj);
216 struct ivpu_addr_range *range;
217
218 if (bo->ctx) {
219 ivpu_warn(vdev, "Can't add BO to ctx %u: already in ctx %u\n",
220 file_priv->ctx.id, bo->ctx->id);
221 return -EALREADY;
222 }
223
224 if (bo->flags & DRM_IVPU_BO_SHAVE_MEM)
225 range = &vdev->hw->ranges.shave;
226 else if (bo->flags & DRM_IVPU_BO_DMA_MEM)
227 range = &vdev->hw->ranges.dma;
228 else
229 range = &vdev->hw->ranges.user;
230
231 return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range);
232 }
233
ivpu_gem_bo_free(struct drm_gem_object * obj)234 static void ivpu_gem_bo_free(struct drm_gem_object *obj)
235 {
236 struct ivpu_device *vdev = to_ivpu_device(obj->dev);
237 struct ivpu_bo *bo = to_ivpu_bo(obj);
238
239 ivpu_dbg_bo(vdev, bo, "free");
240
241 mutex_lock(&vdev->bo_list_lock);
242 list_del(&bo->bo_list_node);
243 mutex_unlock(&vdev->bo_list_lock);
244
245 drm_WARN_ON(&vdev->drm, !drm_gem_is_imported(&bo->base.base) &&
246 !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ));
247 drm_WARN_ON(&vdev->drm, ivpu_bo_size(bo) == 0);
248 drm_WARN_ON(&vdev->drm, bo->base.vaddr);
249
250 ivpu_bo_unbind_locked(bo);
251 drm_WARN_ON(&vdev->drm, bo->mmu_mapped);
252 drm_WARN_ON(&vdev->drm, bo->ctx);
253
254 drm_WARN_ON(obj->dev, bo->base.pages_use_count > 1);
255 drm_gem_shmem_free(&bo->base);
256 }
257
258 static const struct drm_gem_object_funcs ivpu_gem_funcs = {
259 .free = ivpu_gem_bo_free,
260 .open = ivpu_gem_bo_open,
261 .print_info = drm_gem_shmem_object_print_info,
262 .pin = drm_gem_shmem_object_pin,
263 .unpin = drm_gem_shmem_object_unpin,
264 .get_sg_table = drm_gem_shmem_object_get_sg_table,
265 .vmap = drm_gem_shmem_object_vmap,
266 .vunmap = drm_gem_shmem_object_vunmap,
267 .mmap = drm_gem_shmem_object_mmap,
268 .vm_ops = &drm_gem_shmem_vm_ops,
269 };
270
ivpu_bo_create_ioctl(struct drm_device * dev,void * data,struct drm_file * file)271 int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
272 {
273 struct ivpu_file_priv *file_priv = file->driver_priv;
274 struct ivpu_device *vdev = file_priv->vdev;
275 struct drm_ivpu_bo_create *args = data;
276 u64 size = PAGE_ALIGN(args->size);
277 struct ivpu_bo *bo;
278 int ret;
279
280 if (args->flags & ~DRM_IVPU_BO_FLAGS)
281 return -EINVAL;
282
283 if (size == 0)
284 return -EINVAL;
285
286 bo = ivpu_bo_alloc(vdev, size, args->flags, file_priv->ctx.id);
287 if (IS_ERR(bo)) {
288 ivpu_err(vdev, "Failed to allocate BO: %pe (ctx %u size %llu flags 0x%x)",
289 bo, file_priv->ctx.id, args->size, args->flags);
290 return PTR_ERR(bo);
291 }
292
293 ret = drm_gem_handle_create(file, &bo->base.base, &args->handle);
294 if (ret)
295 ivpu_err(vdev, "Failed to create handle for BO: %pe (ctx %u size %llu flags 0x%x)",
296 bo, file_priv->ctx.id, args->size, args->flags);
297 else
298 args->vpu_addr = bo->vpu_addr;
299
300 drm_gem_object_put(&bo->base.base);
301
302 return ret;
303 }
304
305 struct ivpu_bo *
ivpu_bo_create(struct ivpu_device * vdev,struct ivpu_mmu_context * ctx,struct ivpu_addr_range * range,u64 size,u32 flags)306 ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
307 struct ivpu_addr_range *range, u64 size, u32 flags)
308 {
309 struct iosys_map map;
310 struct ivpu_bo *bo;
311 int ret;
312
313 if (drm_WARN_ON(&vdev->drm, !range))
314 return NULL;
315
316 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start));
317 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end));
318 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size));
319
320 bo = ivpu_bo_alloc(vdev, size, flags, IVPU_GLOBAL_CONTEXT_MMU_SSID);
321 if (IS_ERR(bo)) {
322 ivpu_err(vdev, "Failed to allocate BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)",
323 bo, range->start, size, flags);
324 return NULL;
325 }
326
327 ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range);
328 if (ret)
329 goto err_put;
330
331 ret = ivpu_bo_pin(bo);
332 if (ret)
333 goto err_put;
334
335 if (flags & DRM_IVPU_BO_MAPPABLE) {
336 ivpu_bo_lock(bo);
337 ret = drm_gem_shmem_vmap(&bo->base, &map);
338 ivpu_bo_unlock(bo);
339
340 if (ret)
341 goto err_put;
342 }
343
344 return bo;
345
346 err_put:
347 drm_gem_object_put(&bo->base.base);
348 return NULL;
349 }
350
ivpu_bo_create_global(struct ivpu_device * vdev,u64 size,u32 flags)351 struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags)
352 {
353 return ivpu_bo_create(vdev, &vdev->gctx, &vdev->hw->ranges.global, size, flags);
354 }
355
ivpu_bo_free(struct ivpu_bo * bo)356 void ivpu_bo_free(struct ivpu_bo *bo)
357 {
358 struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr);
359
360 if (bo->flags & DRM_IVPU_BO_MAPPABLE) {
361 ivpu_bo_lock(bo);
362 drm_gem_shmem_vunmap(&bo->base, &map);
363 ivpu_bo_unlock(bo);
364 }
365
366 drm_gem_object_put(&bo->base.base);
367 }
368
ivpu_bo_info_ioctl(struct drm_device * dev,void * data,struct drm_file * file)369 int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
370 {
371 struct drm_ivpu_bo_info *args = data;
372 struct drm_gem_object *obj;
373 struct ivpu_bo *bo;
374 int ret = 0;
375
376 obj = drm_gem_object_lookup(file, args->handle);
377 if (!obj)
378 return -ENOENT;
379
380 bo = to_ivpu_bo(obj);
381
382 ivpu_bo_lock(bo);
383 args->flags = bo->flags;
384 args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node);
385 args->vpu_addr = bo->vpu_addr;
386 args->size = obj->size;
387 ivpu_bo_unlock(bo);
388
389 drm_gem_object_put(obj);
390 return ret;
391 }
392
ivpu_bo_wait_ioctl(struct drm_device * dev,void * data,struct drm_file * file)393 int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
394 {
395 struct drm_ivpu_bo_wait *args = data;
396 struct drm_gem_object *obj;
397 unsigned long timeout;
398 long ret;
399
400 timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);
401
402 obj = drm_gem_object_lookup(file, args->handle);
403 if (!obj)
404 return -EINVAL;
405
406 ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout);
407 if (ret == 0) {
408 ret = -ETIMEDOUT;
409 } else if (ret > 0) {
410 ret = 0;
411 args->job_status = to_ivpu_bo(obj)->job_status;
412 }
413
414 drm_gem_object_put(obj);
415
416 return ret;
417 }
418
ivpu_bo_print_info(struct ivpu_bo * bo,struct drm_printer * p)419 static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p)
420 {
421 ivpu_bo_lock(bo);
422
423 drm_printf(p, "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u",
424 bo, bo->ctx_id, bo->vpu_addr, bo->base.base.size,
425 bo->flags, kref_read(&bo->base.base.refcount));
426
427 if (bo->base.pages)
428 drm_printf(p, " has_pages");
429
430 if (bo->mmu_mapped)
431 drm_printf(p, " mmu_mapped");
432
433 if (bo->base.base.import_attach)
434 drm_printf(p, " imported");
435
436 drm_printf(p, "\n");
437
438 ivpu_bo_unlock(bo);
439 }
440
ivpu_bo_list(struct drm_device * dev,struct drm_printer * p)441 void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p)
442 {
443 struct ivpu_device *vdev = to_ivpu_device(dev);
444 struct ivpu_bo *bo;
445
446 drm_printf(p, "%-9s %-3s %-14s %-10s %-10s %-4s %s\n",
447 "bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs");
448
449 mutex_lock(&vdev->bo_list_lock);
450 list_for_each_entry(bo, &vdev->bo_list, bo_list_node)
451 ivpu_bo_print_info(bo, p);
452 mutex_unlock(&vdev->bo_list_lock);
453 }
454
ivpu_bo_list_print(struct drm_device * dev)455 void ivpu_bo_list_print(struct drm_device *dev)
456 {
457 struct drm_printer p = drm_info_printer(dev->dev);
458
459 ivpu_bo_list(dev, &p);
460 }
461