1 /*
2 * Copyright 2008 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 *
24 * Authors:
25 * Jerome Glisse <glisse@freedesktop.org>
26 */
27
28 #include <linux/file.h>
29 #include <linux/pagemap.h>
30 #include <linux/sync_file.h>
31 #include <linux/dma-buf.h>
32
33 #include <drm/amdgpu_drm.h>
34 #include <drm/drm_syncobj.h>
35 #include "amdgpu.h"
36 #include "amdgpu_trace.h"
37 #include "amdgpu_gmc.h"
38 #include "amdgpu_gem.h"
39 #include "amdgpu_ras.h"
40
amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser * p,struct drm_amdgpu_cs_chunk_fence * data,uint32_t * offset)41 static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p,
42 struct drm_amdgpu_cs_chunk_fence *data,
43 uint32_t *offset)
44 {
45 struct drm_gem_object *gobj;
46 struct amdgpu_bo *bo;
47 unsigned long size;
48
49 gobj = drm_gem_object_lookup(p->filp, data->handle);
50 if (gobj == NULL)
51 return -EINVAL;
52
53 bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
54 p->uf_entry.priority = 0;
55 p->uf_entry.tv.bo = &bo->tbo;
56 /* One for TTM and one for the CS job */
57 p->uf_entry.tv.num_shared = 2;
58
59 drm_gem_object_put(gobj);
60
61 size = amdgpu_bo_size(bo);
62 if (size != PAGE_SIZE || data->offset > (size - 8))
63 return -EINVAL;
64
65 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
66 return -EINVAL;
67
68 *offset = data->offset;
69 return 0;
70 }
71
amdgpu_cs_bo_handles_chunk(struct amdgpu_cs_parser * p,struct drm_amdgpu_bo_list_in * data)72 static int amdgpu_cs_bo_handles_chunk(struct amdgpu_cs_parser *p,
73 struct drm_amdgpu_bo_list_in *data)
74 {
75 int r;
76 struct drm_amdgpu_bo_list_entry *info = NULL;
77
78 r = amdgpu_bo_create_list_entry_array(data, &info);
79 if (r)
80 return r;
81
82 r = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number,
83 &p->bo_list);
84 if (r)
85 goto error_free;
86
87 kvfree(info);
88 return 0;
89
90 error_free:
91 kvfree(info);
92
93 return r;
94 }
95
amdgpu_cs_parser_init(struct amdgpu_cs_parser * p,union drm_amdgpu_cs * cs)96 static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs *cs)
97 {
98 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
99 struct amdgpu_vm *vm = &fpriv->vm;
100 uint64_t *chunk_array_user;
101 uint64_t *chunk_array;
102 unsigned size, num_ibs = 0;
103 uint32_t uf_offset = 0;
104 int i;
105 int ret;
106
107 if (cs->in.num_chunks == 0)
108 return -EINVAL;
109
110 chunk_array = kvmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL);
111 if (!chunk_array)
112 return -ENOMEM;
113
114 p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id);
115 if (!p->ctx) {
116 ret = -EINVAL;
117 goto free_chunk;
118 }
119
120 mutex_lock(&p->ctx->lock);
121
122 /* skip guilty context job */
123 if (atomic_read(&p->ctx->guilty) == 1) {
124 ret = -ECANCELED;
125 goto free_chunk;
126 }
127
128 /* get chunks */
129 chunk_array_user = u64_to_user_ptr(cs->in.chunks);
130 if (copy_from_user(chunk_array, chunk_array_user,
131 sizeof(uint64_t)*cs->in.num_chunks)) {
132 ret = -EFAULT;
133 goto free_chunk;
134 }
135
136 p->nchunks = cs->in.num_chunks;
137 p->chunks = kvmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk),
138 GFP_KERNEL);
139 if (!p->chunks) {
140 ret = -ENOMEM;
141 goto free_chunk;
142 }
143
144 for (i = 0; i < p->nchunks; i++) {
145 struct drm_amdgpu_cs_chunk __user *chunk_ptr = NULL;
146 struct drm_amdgpu_cs_chunk user_chunk;
147 uint32_t __user *cdata;
148
149 chunk_ptr = u64_to_user_ptr(chunk_array[i]);
150 if (copy_from_user(&user_chunk, chunk_ptr,
151 sizeof(struct drm_amdgpu_cs_chunk))) {
152 ret = -EFAULT;
153 i--;
154 goto free_partial_kdata;
155 }
156 p->chunks[i].chunk_id = user_chunk.chunk_id;
157 p->chunks[i].length_dw = user_chunk.length_dw;
158
159 size = p->chunks[i].length_dw;
160 cdata = u64_to_user_ptr(user_chunk.chunk_data);
161
162 p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t), GFP_KERNEL);
163 if (p->chunks[i].kdata == NULL) {
164 ret = -ENOMEM;
165 i--;
166 goto free_partial_kdata;
167 }
168 size *= sizeof(uint32_t);
169 if (copy_from_user(p->chunks[i].kdata, cdata, size)) {
170 ret = -EFAULT;
171 goto free_partial_kdata;
172 }
173
174 switch (p->chunks[i].chunk_id) {
175 case AMDGPU_CHUNK_ID_IB:
176 ++num_ibs;
177 break;
178
179 case AMDGPU_CHUNK_ID_FENCE:
180 size = sizeof(struct drm_amdgpu_cs_chunk_fence);
181 if (p->chunks[i].length_dw * sizeof(uint32_t) < size) {
182 ret = -EINVAL;
183 goto free_partial_kdata;
184 }
185
186 ret = amdgpu_cs_user_fence_chunk(p, p->chunks[i].kdata,
187 &uf_offset);
188 if (ret)
189 goto free_partial_kdata;
190
191 break;
192
193 case AMDGPU_CHUNK_ID_BO_HANDLES:
194 size = sizeof(struct drm_amdgpu_bo_list_in);
195 if (p->chunks[i].length_dw * sizeof(uint32_t) < size) {
196 ret = -EINVAL;
197 goto free_partial_kdata;
198 }
199
200 ret = amdgpu_cs_bo_handles_chunk(p, p->chunks[i].kdata);
201 if (ret)
202 goto free_partial_kdata;
203
204 break;
205
206 case AMDGPU_CHUNK_ID_DEPENDENCIES:
207 case AMDGPU_CHUNK_ID_SYNCOBJ_IN:
208 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:
209 case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES:
210 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT:
211 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL:
212 break;
213
214 default:
215 ret = -EINVAL;
216 goto free_partial_kdata;
217 }
218 }
219
220 ret = amdgpu_job_alloc(p->adev, num_ibs, &p->job, vm);
221 if (ret)
222 goto free_all_kdata;
223
224 if (p->ctx->vram_lost_counter != p->job->vram_lost_counter) {
225 ret = -ECANCELED;
226 goto free_all_kdata;
227 }
228
229 if (p->uf_entry.tv.bo)
230 p->job->uf_addr = uf_offset;
231 kvfree(chunk_array);
232
233 /* Use this opportunity to fill in task info for the vm */
234 amdgpu_vm_set_task_info(vm);
235
236 return 0;
237
238 free_all_kdata:
239 i = p->nchunks - 1;
240 free_partial_kdata:
241 for (; i >= 0; i--)
242 kvfree(p->chunks[i].kdata);
243 kvfree(p->chunks);
244 p->chunks = NULL;
245 p->nchunks = 0;
246 free_chunk:
247 kvfree(chunk_array);
248
249 return ret;
250 }
251
252 /* Convert microseconds to bytes. */
us_to_bytes(struct amdgpu_device * adev,s64 us)253 static u64 us_to_bytes(struct amdgpu_device *adev, s64 us)
254 {
255 if (us <= 0 || !adev->mm_stats.log2_max_MBps)
256 return 0;
257
258 /* Since accum_us is incremented by a million per second, just
259 * multiply it by the number of MB/s to get the number of bytes.
260 */
261 return us << adev->mm_stats.log2_max_MBps;
262 }
263
bytes_to_us(struct amdgpu_device * adev,u64 bytes)264 static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes)
265 {
266 if (!adev->mm_stats.log2_max_MBps)
267 return 0;
268
269 return bytes >> adev->mm_stats.log2_max_MBps;
270 }
271
272 /* Returns how many bytes TTM can move right now. If no bytes can be moved,
273 * it returns 0. If it returns non-zero, it's OK to move at least one buffer,
274 * which means it can go over the threshold once. If that happens, the driver
275 * will be in debt and no other buffer migrations can be done until that debt
276 * is repaid.
277 *
278 * This approach allows moving a buffer of any size (it's important to allow
279 * that).
280 *
281 * The currency is simply time in microseconds and it increases as the clock
282 * ticks. The accumulated microseconds (us) are converted to bytes and
283 * returned.
284 */
amdgpu_cs_get_threshold_for_moves(struct amdgpu_device * adev,u64 * max_bytes,u64 * max_vis_bytes)285 static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev,
286 u64 *max_bytes,
287 u64 *max_vis_bytes)
288 {
289 s64 time_us, increment_us;
290 u64 free_vram, total_vram, used_vram;
291 struct ttm_resource_manager *vram_man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
292 /* Allow a maximum of 200 accumulated ms. This is basically per-IB
293 * throttling.
294 *
295 * It means that in order to get full max MBps, at least 5 IBs per
296 * second must be submitted and not more than 200ms apart from each
297 * other.
298 */
299 const s64 us_upper_bound = 200000;
300
301 if (!adev->mm_stats.log2_max_MBps) {
302 *max_bytes = 0;
303 *max_vis_bytes = 0;
304 return;
305 }
306
307 total_vram = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size);
308 used_vram = amdgpu_vram_mgr_usage(vram_man);
309 free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram;
310
311 spin_lock(&adev->mm_stats.lock);
312
313 /* Increase the amount of accumulated us. */
314 time_us = ktime_to_us(ktime_get());
315 increment_us = time_us - adev->mm_stats.last_update_us;
316 adev->mm_stats.last_update_us = time_us;
317 adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us,
318 us_upper_bound);
319
320 /* This prevents the short period of low performance when the VRAM
321 * usage is low and the driver is in debt or doesn't have enough
322 * accumulated us to fill VRAM quickly.
323 *
324 * The situation can occur in these cases:
325 * - a lot of VRAM is freed by userspace
326 * - the presence of a big buffer causes a lot of evictions
327 * (solution: split buffers into smaller ones)
328 *
329 * If 128 MB or 1/8th of VRAM is free, start filling it now by setting
330 * accum_us to a positive number.
331 */
332 if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) {
333 s64 min_us;
334
335 /* Be more aggresive on dGPUs. Try to fill a portion of free
336 * VRAM now.
337 */
338 if (!(adev->flags & AMD_IS_APU))
339 min_us = bytes_to_us(adev, free_vram / 4);
340 else
341 min_us = 0; /* Reset accum_us on APUs. */
342
343 adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us);
344 }
345
346 /* This is set to 0 if the driver is in debt to disallow (optional)
347 * buffer moves.
348 */
349 *max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us);
350
351 /* Do the same for visible VRAM if half of it is free */
352 if (!amdgpu_gmc_vram_full_visible(&adev->gmc)) {
353 u64 total_vis_vram = adev->gmc.visible_vram_size;
354 u64 used_vis_vram =
355 amdgpu_vram_mgr_vis_usage(vram_man);
356
357 if (used_vis_vram < total_vis_vram) {
358 u64 free_vis_vram = total_vis_vram - used_vis_vram;
359 adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis +
360 increment_us, us_upper_bound);
361
362 if (free_vis_vram >= total_vis_vram / 2)
363 adev->mm_stats.accum_us_vis =
364 max(bytes_to_us(adev, free_vis_vram / 2),
365 adev->mm_stats.accum_us_vis);
366 }
367
368 *max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis);
369 } else {
370 *max_vis_bytes = 0;
371 }
372
373 spin_unlock(&adev->mm_stats.lock);
374 }
375
376 /* Report how many bytes have really been moved for the last command
377 * submission. This can result in a debt that can stop buffer migrations
378 * temporarily.
379 */
amdgpu_cs_report_moved_bytes(struct amdgpu_device * adev,u64 num_bytes,u64 num_vis_bytes)380 void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,
381 u64 num_vis_bytes)
382 {
383 spin_lock(&adev->mm_stats.lock);
384 adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes);
385 adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes);
386 spin_unlock(&adev->mm_stats.lock);
387 }
388
amdgpu_cs_bo_validate(void * param,struct amdgpu_bo * bo)389 static int amdgpu_cs_bo_validate(void *param, struct amdgpu_bo *bo)
390 {
391 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
392 struct amdgpu_cs_parser *p = param;
393 struct ttm_operation_ctx ctx = {
394 .interruptible = true,
395 .no_wait_gpu = false,
396 .resv = bo->tbo.base.resv
397 };
398 uint32_t domain;
399 int r;
400
401 if (bo->tbo.pin_count)
402 return 0;
403
404 /* Don't move this buffer if we have depleted our allowance
405 * to move it. Don't move anything if the threshold is zero.
406 */
407 if (p->bytes_moved < p->bytes_moved_threshold &&
408 (!bo->tbo.base.dma_buf ||
409 list_empty(&bo->tbo.base.dma_buf->attachments))) {
410 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
411 (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) {
412 /* And don't move a CPU_ACCESS_REQUIRED BO to limited
413 * visible VRAM if we've depleted our allowance to do
414 * that.
415 */
416 if (p->bytes_moved_vis < p->bytes_moved_vis_threshold)
417 domain = bo->preferred_domains;
418 else
419 domain = bo->allowed_domains;
420 } else {
421 domain = bo->preferred_domains;
422 }
423 } else {
424 domain = bo->allowed_domains;
425 }
426
427 retry:
428 amdgpu_bo_placement_from_domain(bo, domain);
429 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
430
431 p->bytes_moved += ctx.bytes_moved;
432 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
433 amdgpu_bo_in_cpu_visible_vram(bo))
434 p->bytes_moved_vis += ctx.bytes_moved;
435
436 if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
437 domain = bo->allowed_domains;
438 goto retry;
439 }
440
441 return r;
442 }
443
amdgpu_cs_list_validate(struct amdgpu_cs_parser * p,struct list_head * validated)444 static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
445 struct list_head *validated)
446 {
447 struct ttm_operation_ctx ctx = { true, false };
448 struct amdgpu_bo_list_entry *lobj;
449 int r;
450
451 list_for_each_entry(lobj, validated, tv.head) {
452 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(lobj->tv.bo);
453 struct mm_struct *usermm;
454
455 usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm);
456 if (usermm && usermm != current->mm)
457 return -EPERM;
458
459 if (amdgpu_ttm_tt_is_userptr(bo->tbo.ttm) &&
460 lobj->user_invalidated && lobj->user_pages) {
461 amdgpu_bo_placement_from_domain(bo,
462 AMDGPU_GEM_DOMAIN_CPU);
463 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
464 if (r)
465 return r;
466
467 amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm,
468 lobj->user_pages);
469 }
470
471 r = amdgpu_cs_bo_validate(p, bo);
472 if (r)
473 return r;
474
475 kvfree(lobj->user_pages);
476 lobj->user_pages = NULL;
477 }
478 return 0;
479 }
480
amdgpu_cs_parser_bos(struct amdgpu_cs_parser * p,union drm_amdgpu_cs * cs)481 static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
482 union drm_amdgpu_cs *cs)
483 {
484 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
485 struct amdgpu_vm *vm = &fpriv->vm;
486 struct amdgpu_bo_list_entry *e;
487 struct list_head duplicates;
488 struct amdgpu_bo *gds;
489 struct amdgpu_bo *gws;
490 struct amdgpu_bo *oa;
491 int r;
492
493 INIT_LIST_HEAD(&p->validated);
494
495 /* p->bo_list could already be assigned if AMDGPU_CHUNK_ID_BO_HANDLES is present */
496 if (cs->in.bo_list_handle) {
497 if (p->bo_list)
498 return -EINVAL;
499
500 r = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle,
501 &p->bo_list);
502 if (r)
503 return r;
504 } else if (!p->bo_list) {
505 /* Create a empty bo_list when no handle is provided */
506 r = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0,
507 &p->bo_list);
508 if (r)
509 return r;
510 }
511
512 /* One for TTM and one for the CS job */
513 amdgpu_bo_list_for_each_entry(e, p->bo_list)
514 e->tv.num_shared = 2;
515
516 amdgpu_bo_list_get_list(p->bo_list, &p->validated);
517
518 INIT_LIST_HEAD(&duplicates);
519 amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd);
520
521 if (p->uf_entry.tv.bo && !ttm_to_amdgpu_bo(p->uf_entry.tv.bo)->parent)
522 list_add(&p->uf_entry.tv.head, &p->validated);
523
524 /* Get userptr backing pages. If pages are updated after registered
525 * in amdgpu_gem_userptr_ioctl(), amdgpu_cs_list_validate() will do
526 * amdgpu_ttm_backend_bind() to flush and invalidate new pages
527 */
528 amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
529 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
530 bool userpage_invalidated = false;
531 int i;
532
533 e->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages,
534 sizeof(struct page *),
535 GFP_KERNEL | __GFP_ZERO);
536 if (!e->user_pages) {
537 DRM_ERROR("kvmalloc_array failure\n");
538 return -ENOMEM;
539 }
540
541 r = amdgpu_ttm_tt_get_user_pages(bo, e->user_pages);
542 if (r) {
543 kvfree(e->user_pages);
544 e->user_pages = NULL;
545 return r;
546 }
547
548 for (i = 0; i < bo->tbo.ttm->num_pages; i++) {
549 if (bo->tbo.ttm->pages[i] != e->user_pages[i]) {
550 userpage_invalidated = true;
551 break;
552 }
553 }
554 e->user_invalidated = userpage_invalidated;
555 }
556
557 r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true,
558 &duplicates);
559 if (unlikely(r != 0)) {
560 if (r != -ERESTARTSYS)
561 DRM_ERROR("ttm_eu_reserve_buffers failed.\n");
562 goto out;
563 }
564
565 amdgpu_bo_list_for_each_entry(e, p->bo_list) {
566 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
567
568 e->bo_va = amdgpu_vm_bo_find(vm, bo);
569
570 if (bo->tbo.base.dma_buf && !amdgpu_bo_explicit_sync(bo)) {
571 e->chain = dma_fence_chain_alloc();
572 if (!e->chain) {
573 r = -ENOMEM;
574 goto error_validate;
575 }
576 }
577 }
578
579 amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold,
580 &p->bytes_moved_vis_threshold);
581 p->bytes_moved = 0;
582 p->bytes_moved_vis = 0;
583
584 r = amdgpu_vm_validate_pt_bos(p->adev, &fpriv->vm,
585 amdgpu_cs_bo_validate, p);
586 if (r) {
587 DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n");
588 goto error_validate;
589 }
590
591 r = amdgpu_cs_list_validate(p, &duplicates);
592 if (r)
593 goto error_validate;
594
595 r = amdgpu_cs_list_validate(p, &p->validated);
596 if (r)
597 goto error_validate;
598
599 amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
600 p->bytes_moved_vis);
601
602 gds = p->bo_list->gds_obj;
603 gws = p->bo_list->gws_obj;
604 oa = p->bo_list->oa_obj;
605
606 if (gds) {
607 p->job->gds_base = amdgpu_bo_gpu_offset(gds) >> PAGE_SHIFT;
608 p->job->gds_size = amdgpu_bo_size(gds) >> PAGE_SHIFT;
609 }
610 if (gws) {
611 p->job->gws_base = amdgpu_bo_gpu_offset(gws) >> PAGE_SHIFT;
612 p->job->gws_size = amdgpu_bo_size(gws) >> PAGE_SHIFT;
613 }
614 if (oa) {
615 p->job->oa_base = amdgpu_bo_gpu_offset(oa) >> PAGE_SHIFT;
616 p->job->oa_size = amdgpu_bo_size(oa) >> PAGE_SHIFT;
617 }
618
619 if (!r && p->uf_entry.tv.bo) {
620 struct amdgpu_bo *uf = ttm_to_amdgpu_bo(p->uf_entry.tv.bo);
621
622 r = amdgpu_ttm_alloc_gart(&uf->tbo);
623 p->job->uf_addr += amdgpu_bo_gpu_offset(uf);
624 }
625
626 error_validate:
627 if (r) {
628 amdgpu_bo_list_for_each_entry(e, p->bo_list) {
629 dma_fence_chain_free(e->chain);
630 e->chain = NULL;
631 }
632 ttm_eu_backoff_reservation(&p->ticket, &p->validated);
633 }
634 out:
635 return r;
636 }
637
amdgpu_cs_sync_rings(struct amdgpu_cs_parser * p)638 static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
639 {
640 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
641 struct amdgpu_bo_list_entry *e;
642 int r;
643
644 list_for_each_entry(e, &p->validated, tv.head) {
645 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
646 struct dma_resv *resv = bo->tbo.base.resv;
647 enum amdgpu_sync_mode sync_mode;
648
649 sync_mode = amdgpu_bo_explicit_sync(bo) ?
650 AMDGPU_SYNC_EXPLICIT : AMDGPU_SYNC_NE_OWNER;
651 r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, sync_mode,
652 &fpriv->vm);
653 if (r)
654 return r;
655 }
656 return 0;
657 }
658
659 /**
660 * amdgpu_cs_parser_fini() - clean parser states
661 * @parser: parser structure holding parsing context.
662 * @error: error number
663 * @backoff: indicator to backoff the reservation
664 *
665 * If error is set then unvalidate buffer, otherwise just free memory
666 * used by parsing context.
667 **/
amdgpu_cs_parser_fini(struct amdgpu_cs_parser * parser,int error,bool backoff)668 static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error,
669 bool backoff)
670 {
671 unsigned i;
672
673 if (error && backoff) {
674 struct amdgpu_bo_list_entry *e;
675
676 amdgpu_bo_list_for_each_entry(e, parser->bo_list) {
677 dma_fence_chain_free(e->chain);
678 e->chain = NULL;
679 }
680
681 ttm_eu_backoff_reservation(&parser->ticket,
682 &parser->validated);
683 }
684
685 for (i = 0; i < parser->num_post_deps; i++) {
686 drm_syncobj_put(parser->post_deps[i].syncobj);
687 kfree(parser->post_deps[i].chain);
688 }
689 kfree(parser->post_deps);
690
691 dma_fence_put(parser->fence);
692
693 if (parser->ctx) {
694 mutex_unlock(&parser->ctx->lock);
695 amdgpu_ctx_put(parser->ctx);
696 }
697 if (parser->bo_list)
698 amdgpu_bo_list_put(parser->bo_list);
699
700 for (i = 0; i < parser->nchunks; i++)
701 kvfree(parser->chunks[i].kdata);
702 kvfree(parser->chunks);
703 if (parser->job)
704 amdgpu_job_free(parser->job);
705 if (parser->uf_entry.tv.bo) {
706 struct amdgpu_bo *uf = ttm_to_amdgpu_bo(parser->uf_entry.tv.bo);
707
708 amdgpu_bo_unref(&uf);
709 }
710 }
711
amdgpu_cs_vm_handling(struct amdgpu_cs_parser * p)712 static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
713 {
714 struct amdgpu_ring *ring = to_amdgpu_ring(p->entity->rq->sched);
715 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
716 struct amdgpu_device *adev = p->adev;
717 struct amdgpu_vm *vm = &fpriv->vm;
718 struct amdgpu_bo_list_entry *e;
719 struct amdgpu_bo_va *bo_va;
720 struct amdgpu_bo *bo;
721 int r;
722
723 /* Only for UVD/VCE VM emulation */
724 if (ring->funcs->parse_cs || ring->funcs->patch_cs_in_place) {
725 unsigned i, j;
726
727 for (i = 0, j = 0; i < p->nchunks && j < p->job->num_ibs; i++) {
728 struct drm_amdgpu_cs_chunk_ib *chunk_ib;
729 struct amdgpu_bo_va_mapping *m;
730 struct amdgpu_bo *aobj = NULL;
731 struct amdgpu_cs_chunk *chunk;
732 uint64_t offset, va_start;
733 struct amdgpu_ib *ib;
734 uint8_t *kptr;
735
736 chunk = &p->chunks[i];
737 ib = &p->job->ibs[j];
738 chunk_ib = chunk->kdata;
739
740 if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
741 continue;
742
743 va_start = chunk_ib->va_start & AMDGPU_GMC_HOLE_MASK;
744 r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m);
745 if (r) {
746 DRM_ERROR("IB va_start is invalid\n");
747 return r;
748 }
749
750 if ((va_start + chunk_ib->ib_bytes) >
751 (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
752 DRM_ERROR("IB va_start+ib_bytes is invalid\n");
753 return -EINVAL;
754 }
755
756 /* the IB should be reserved at this point */
757 r = amdgpu_bo_kmap(aobj, (void **)&kptr);
758 if (r) {
759 return r;
760 }
761
762 offset = m->start * AMDGPU_GPU_PAGE_SIZE;
763 kptr += va_start - offset;
764
765 if (ring->funcs->parse_cs) {
766 memcpy(ib->ptr, kptr, chunk_ib->ib_bytes);
767 amdgpu_bo_kunmap(aobj);
768
769 r = amdgpu_ring_parse_cs(ring, p, j);
770 if (r)
771 return r;
772 } else {
773 ib->ptr = (uint32_t *)kptr;
774 r = amdgpu_ring_patch_cs_in_place(ring, p, j);
775 amdgpu_bo_kunmap(aobj);
776 if (r)
777 return r;
778 }
779
780 j++;
781 }
782 }
783
784 if (!p->job->vm)
785 return amdgpu_cs_sync_rings(p);
786
787
788 r = amdgpu_vm_clear_freed(adev, vm, NULL);
789 if (r)
790 return r;
791
792 r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false, NULL);
793 if (r)
794 return r;
795
796 r = amdgpu_sync_vm_fence(&p->job->sync, fpriv->prt_va->last_pt_update);
797 if (r)
798 return r;
799
800 if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) {
801 bo_va = fpriv->csa_va;
802 BUG_ON(!bo_va);
803 r = amdgpu_vm_bo_update(adev, bo_va, false, NULL);
804 if (r)
805 return r;
806
807 r = amdgpu_sync_vm_fence(&p->job->sync, bo_va->last_pt_update);
808 if (r)
809 return r;
810 }
811
812 amdgpu_bo_list_for_each_entry(e, p->bo_list) {
813 /* ignore duplicates */
814 bo = ttm_to_amdgpu_bo(e->tv.bo);
815 if (!bo)
816 continue;
817
818 bo_va = e->bo_va;
819 if (bo_va == NULL)
820 continue;
821
822 r = amdgpu_vm_bo_update(adev, bo_va, false, NULL);
823 if (r)
824 return r;
825
826 r = amdgpu_sync_vm_fence(&p->job->sync, bo_va->last_pt_update);
827 if (r)
828 return r;
829 }
830
831 r = amdgpu_vm_handle_moved(adev, vm);
832 if (r)
833 return r;
834
835 r = amdgpu_vm_update_pdes(adev, vm, false);
836 if (r)
837 return r;
838
839 r = amdgpu_sync_vm_fence(&p->job->sync, vm->last_update);
840 if (r)
841 return r;
842
843 p->job->vm_pd_addr = amdgpu_gmc_pd_addr(vm->root.bo);
844
845 if (amdgpu_vm_debug) {
846 /* Invalidate all BOs to test for userspace bugs */
847 amdgpu_bo_list_for_each_entry(e, p->bo_list) {
848 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
849
850 /* ignore duplicates */
851 if (!bo)
852 continue;
853
854 amdgpu_vm_bo_invalidate(adev, bo, false);
855 }
856 }
857
858 return amdgpu_cs_sync_rings(p);
859 }
860
amdgpu_cs_ib_fill(struct amdgpu_device * adev,struct amdgpu_cs_parser * parser)861 static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
862 struct amdgpu_cs_parser *parser)
863 {
864 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
865 struct amdgpu_vm *vm = &fpriv->vm;
866 int r, ce_preempt = 0, de_preempt = 0;
867 struct amdgpu_ring *ring;
868 int i, j;
869
870 for (i = 0, j = 0; i < parser->nchunks && j < parser->job->num_ibs; i++) {
871 struct amdgpu_cs_chunk *chunk;
872 struct amdgpu_ib *ib;
873 struct drm_amdgpu_cs_chunk_ib *chunk_ib;
874 struct drm_sched_entity *entity;
875
876 chunk = &parser->chunks[i];
877 ib = &parser->job->ibs[j];
878 chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;
879
880 if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
881 continue;
882
883 if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX &&
884 (amdgpu_mcbp || amdgpu_sriov_vf(adev))) {
885 if (chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
886 if (chunk_ib->flags & AMDGPU_IB_FLAG_CE)
887 ce_preempt++;
888 else
889 de_preempt++;
890 }
891
892 /* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */
893 if (ce_preempt > 1 || de_preempt > 1)
894 return -EINVAL;
895 }
896
897 r = amdgpu_ctx_get_entity(parser->ctx, chunk_ib->ip_type,
898 chunk_ib->ip_instance, chunk_ib->ring,
899 &entity);
900 if (r)
901 return r;
902
903 if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE)
904 parser->job->preamble_status |=
905 AMDGPU_PREAMBLE_IB_PRESENT;
906
907 if (parser->entity && parser->entity != entity)
908 return -EINVAL;
909
910 /* Return if there is no run queue associated with this entity.
911 * Possibly because of disabled HW IP*/
912 if (entity->rq == NULL)
913 return -EINVAL;
914
915 parser->entity = entity;
916
917 ring = to_amdgpu_ring(entity->rq->sched);
918 r = amdgpu_ib_get(adev, vm, ring->funcs->parse_cs ?
919 chunk_ib->ib_bytes : 0,
920 AMDGPU_IB_POOL_DELAYED, ib);
921 if (r) {
922 DRM_ERROR("Failed to get ib !\n");
923 return r;
924 }
925
926 ib->gpu_addr = chunk_ib->va_start;
927 ib->length_dw = chunk_ib->ib_bytes / 4;
928 ib->flags = chunk_ib->flags;
929
930 j++;
931 }
932
933 /* MM engine doesn't support user fences */
934 ring = to_amdgpu_ring(parser->entity->rq->sched);
935 if (parser->job->uf_addr && ring->funcs->no_user_fence)
936 return -EINVAL;
937
938 return amdgpu_ctx_wait_prev_fence(parser->ctx, parser->entity);
939 }
940
amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser * p,struct amdgpu_cs_chunk * chunk)941 static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p,
942 struct amdgpu_cs_chunk *chunk)
943 {
944 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
945 unsigned num_deps;
946 int i, r;
947 struct drm_amdgpu_cs_chunk_dep *deps;
948
949 deps = (struct drm_amdgpu_cs_chunk_dep *)chunk->kdata;
950 num_deps = chunk->length_dw * 4 /
951 sizeof(struct drm_amdgpu_cs_chunk_dep);
952
953 for (i = 0; i < num_deps; ++i) {
954 struct amdgpu_ctx *ctx;
955 struct drm_sched_entity *entity;
956 struct dma_fence *fence;
957
958 ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id);
959 if (ctx == NULL)
960 return -EINVAL;
961
962 r = amdgpu_ctx_get_entity(ctx, deps[i].ip_type,
963 deps[i].ip_instance,
964 deps[i].ring, &entity);
965 if (r) {
966 amdgpu_ctx_put(ctx);
967 return r;
968 }
969
970 fence = amdgpu_ctx_get_fence(ctx, entity, deps[i].handle);
971 amdgpu_ctx_put(ctx);
972
973 if (IS_ERR(fence))
974 return PTR_ERR(fence);
975 else if (!fence)
976 continue;
977
978 if (chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) {
979 struct drm_sched_fence *s_fence;
980 struct dma_fence *old = fence;
981
982 s_fence = to_drm_sched_fence(fence);
983 fence = dma_fence_get(&s_fence->scheduled);
984 dma_fence_put(old);
985 }
986
987 r = amdgpu_sync_fence(&p->job->sync, fence);
988 dma_fence_put(fence);
989 if (r)
990 return r;
991 }
992 return 0;
993 }
994
amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser * p,uint32_t handle,u64 point,u64 flags)995 static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser *p,
996 uint32_t handle, u64 point,
997 u64 flags)
998 {
999 struct dma_fence *fence;
1000 int r;
1001
1002 r = drm_syncobj_find_fence(p->filp, handle, point, flags, &fence);
1003 if (r) {
1004 DRM_ERROR("syncobj %u failed to find fence @ %llu (%d)!\n",
1005 handle, point, r);
1006 return r;
1007 }
1008
1009 r = amdgpu_sync_fence(&p->job->sync, fence);
1010 dma_fence_put(fence);
1011
1012 return r;
1013 }
1014
amdgpu_cs_process_syncobj_in_dep(struct amdgpu_cs_parser * p,struct amdgpu_cs_chunk * chunk)1015 static int amdgpu_cs_process_syncobj_in_dep(struct amdgpu_cs_parser *p,
1016 struct amdgpu_cs_chunk *chunk)
1017 {
1018 struct drm_amdgpu_cs_chunk_sem *deps;
1019 unsigned num_deps;
1020 int i, r;
1021
1022 deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata;
1023 num_deps = chunk->length_dw * 4 /
1024 sizeof(struct drm_amdgpu_cs_chunk_sem);
1025 for (i = 0; i < num_deps; ++i) {
1026 r = amdgpu_syncobj_lookup_and_add_to_sync(p, deps[i].handle,
1027 0, 0);
1028 if (r)
1029 return r;
1030 }
1031
1032 return 0;
1033 }
1034
1035
amdgpu_cs_process_syncobj_timeline_in_dep(struct amdgpu_cs_parser * p,struct amdgpu_cs_chunk * chunk)1036 static int amdgpu_cs_process_syncobj_timeline_in_dep(struct amdgpu_cs_parser *p,
1037 struct amdgpu_cs_chunk *chunk)
1038 {
1039 struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps;
1040 unsigned num_deps;
1041 int i, r;
1042
1043 syncobj_deps = (struct drm_amdgpu_cs_chunk_syncobj *)chunk->kdata;
1044 num_deps = chunk->length_dw * 4 /
1045 sizeof(struct drm_amdgpu_cs_chunk_syncobj);
1046 for (i = 0; i < num_deps; ++i) {
1047 r = amdgpu_syncobj_lookup_and_add_to_sync(p,
1048 syncobj_deps[i].handle,
1049 syncobj_deps[i].point,
1050 syncobj_deps[i].flags);
1051 if (r)
1052 return r;
1053 }
1054
1055 return 0;
1056 }
1057
amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser * p,struct amdgpu_cs_chunk * chunk)1058 static int amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser *p,
1059 struct amdgpu_cs_chunk *chunk)
1060 {
1061 struct drm_amdgpu_cs_chunk_sem *deps;
1062 unsigned num_deps;
1063 int i;
1064
1065 deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata;
1066 num_deps = chunk->length_dw * 4 /
1067 sizeof(struct drm_amdgpu_cs_chunk_sem);
1068
1069 if (p->post_deps)
1070 return -EINVAL;
1071
1072 p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps),
1073 GFP_KERNEL);
1074 p->num_post_deps = 0;
1075
1076 if (!p->post_deps)
1077 return -ENOMEM;
1078
1079
1080 for (i = 0; i < num_deps; ++i) {
1081 p->post_deps[i].syncobj =
1082 drm_syncobj_find(p->filp, deps[i].handle);
1083 if (!p->post_deps[i].syncobj)
1084 return -EINVAL;
1085 p->post_deps[i].chain = NULL;
1086 p->post_deps[i].point = 0;
1087 p->num_post_deps++;
1088 }
1089
1090 return 0;
1091 }
1092
1093
amdgpu_cs_process_syncobj_timeline_out_dep(struct amdgpu_cs_parser * p,struct amdgpu_cs_chunk * chunk)1094 static int amdgpu_cs_process_syncobj_timeline_out_dep(struct amdgpu_cs_parser *p,
1095 struct amdgpu_cs_chunk *chunk)
1096 {
1097 struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps;
1098 unsigned num_deps;
1099 int i;
1100
1101 syncobj_deps = (struct drm_amdgpu_cs_chunk_syncobj *)chunk->kdata;
1102 num_deps = chunk->length_dw * 4 /
1103 sizeof(struct drm_amdgpu_cs_chunk_syncobj);
1104
1105 if (p->post_deps)
1106 return -EINVAL;
1107
1108 p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps),
1109 GFP_KERNEL);
1110 p->num_post_deps = 0;
1111
1112 if (!p->post_deps)
1113 return -ENOMEM;
1114
1115 for (i = 0; i < num_deps; ++i) {
1116 struct amdgpu_cs_post_dep *dep = &p->post_deps[i];
1117
1118 dep->chain = NULL;
1119 if (syncobj_deps[i].point) {
1120 dep->chain = dma_fence_chain_alloc();
1121 if (!dep->chain)
1122 return -ENOMEM;
1123 }
1124
1125 dep->syncobj = drm_syncobj_find(p->filp,
1126 syncobj_deps[i].handle);
1127 if (!dep->syncobj) {
1128 dma_fence_chain_free(dep->chain);
1129 return -EINVAL;
1130 }
1131 dep->point = syncobj_deps[i].point;
1132 p->num_post_deps++;
1133 }
1134
1135 return 0;
1136 }
1137
amdgpu_cs_dependencies(struct amdgpu_device * adev,struct amdgpu_cs_parser * p)1138 static int amdgpu_cs_dependencies(struct amdgpu_device *adev,
1139 struct amdgpu_cs_parser *p)
1140 {
1141 int i, r;
1142
1143 for (i = 0; i < p->nchunks; ++i) {
1144 struct amdgpu_cs_chunk *chunk;
1145
1146 chunk = &p->chunks[i];
1147
1148 switch (chunk->chunk_id) {
1149 case AMDGPU_CHUNK_ID_DEPENDENCIES:
1150 case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES:
1151 r = amdgpu_cs_process_fence_dep(p, chunk);
1152 if (r)
1153 return r;
1154 break;
1155 case AMDGPU_CHUNK_ID_SYNCOBJ_IN:
1156 r = amdgpu_cs_process_syncobj_in_dep(p, chunk);
1157 if (r)
1158 return r;
1159 break;
1160 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:
1161 r = amdgpu_cs_process_syncobj_out_dep(p, chunk);
1162 if (r)
1163 return r;
1164 break;
1165 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT:
1166 r = amdgpu_cs_process_syncobj_timeline_in_dep(p, chunk);
1167 if (r)
1168 return r;
1169 break;
1170 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL:
1171 r = amdgpu_cs_process_syncobj_timeline_out_dep(p, chunk);
1172 if (r)
1173 return r;
1174 break;
1175 }
1176 }
1177
1178 return 0;
1179 }
1180
amdgpu_cs_post_dependencies(struct amdgpu_cs_parser * p)1181 static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p)
1182 {
1183 int i;
1184
1185 for (i = 0; i < p->num_post_deps; ++i) {
1186 if (p->post_deps[i].chain && p->post_deps[i].point) {
1187 drm_syncobj_add_point(p->post_deps[i].syncobj,
1188 p->post_deps[i].chain,
1189 p->fence, p->post_deps[i].point);
1190 p->post_deps[i].chain = NULL;
1191 } else {
1192 drm_syncobj_replace_fence(p->post_deps[i].syncobj,
1193 p->fence);
1194 }
1195 }
1196 }
1197
amdgpu_cs_submit(struct amdgpu_cs_parser * p,union drm_amdgpu_cs * cs)1198 static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
1199 union drm_amdgpu_cs *cs)
1200 {
1201 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
1202 struct drm_sched_entity *entity = p->entity;
1203 struct amdgpu_bo_list_entry *e;
1204 struct amdgpu_job *job;
1205 uint64_t seq;
1206 int r;
1207
1208 job = p->job;
1209 p->job = NULL;
1210
1211 r = drm_sched_job_init(&job->base, entity, &fpriv->vm);
1212 if (r)
1213 goto error_unlock;
1214
1215 /* No memory allocation is allowed while holding the notifier lock.
1216 * The lock is held until amdgpu_cs_submit is finished and fence is
1217 * added to BOs.
1218 */
1219 mutex_lock(&p->adev->notifier_lock);
1220
1221 /* If userptr are invalidated after amdgpu_cs_parser_bos(), return
1222 * -EAGAIN, drmIoctl in libdrm will restart the amdgpu_cs_ioctl.
1223 */
1224 amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
1225 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
1226
1227 r |= !amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
1228 }
1229 if (r) {
1230 r = -EAGAIN;
1231 goto error_abort;
1232 }
1233
1234 p->fence = dma_fence_get(&job->base.s_fence->finished);
1235
1236 amdgpu_ctx_add_fence(p->ctx, entity, p->fence, &seq);
1237 amdgpu_cs_post_dependencies(p);
1238
1239 if ((job->preamble_status & AMDGPU_PREAMBLE_IB_PRESENT) &&
1240 !p->ctx->preamble_presented) {
1241 job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST;
1242 p->ctx->preamble_presented = true;
1243 }
1244
1245 cs->out.handle = seq;
1246 job->uf_sequence = seq;
1247
1248 amdgpu_job_free_resources(job);
1249
1250 trace_amdgpu_cs_ioctl(job);
1251 amdgpu_vm_bo_trace_cs(&fpriv->vm, &p->ticket);
1252 drm_sched_entity_push_job(&job->base, entity);
1253
1254 amdgpu_vm_move_to_lru_tail(p->adev, &fpriv->vm);
1255
1256 amdgpu_bo_list_for_each_entry(e, p->bo_list) {
1257 struct dma_resv *resv = e->tv.bo->base.resv;
1258 struct dma_fence_chain *chain = e->chain;
1259
1260 if (!chain)
1261 continue;
1262
1263 /*
1264 * Work around dma_resv shortcommings by wrapping up the
1265 * submission in a dma_fence_chain and add it as exclusive
1266 * fence, but first add the submission as shared fence to make
1267 * sure that shared fences never signal before the exclusive
1268 * one.
1269 */
1270 dma_fence_chain_init(chain, dma_resv_excl_fence(resv),
1271 dma_fence_get(p->fence), 1);
1272
1273 dma_resv_add_shared_fence(resv, p->fence);
1274 rcu_assign_pointer(resv->fence_excl, &chain->base);
1275 e->chain = NULL;
1276 }
1277
1278 ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence);
1279 mutex_unlock(&p->adev->notifier_lock);
1280
1281 return 0;
1282
1283 error_abort:
1284 drm_sched_job_cleanup(&job->base);
1285 mutex_unlock(&p->adev->notifier_lock);
1286
1287 error_unlock:
1288 amdgpu_job_free(job);
1289 return r;
1290 }
1291
trace_amdgpu_cs_ibs(struct amdgpu_cs_parser * parser)1292 static void trace_amdgpu_cs_ibs(struct amdgpu_cs_parser *parser)
1293 {
1294 int i;
1295
1296 if (!trace_amdgpu_cs_enabled())
1297 return;
1298
1299 for (i = 0; i < parser->job->num_ibs; i++)
1300 trace_amdgpu_cs(parser, i);
1301 }
1302
amdgpu_cs_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)1303 int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
1304 {
1305 struct amdgpu_device *adev = drm_to_adev(dev);
1306 union drm_amdgpu_cs *cs = data;
1307 struct amdgpu_cs_parser parser = {};
1308 bool reserved_buffers = false;
1309 int r;
1310
1311 if (amdgpu_ras_intr_triggered())
1312 return -EHWPOISON;
1313
1314 if (!adev->accel_working)
1315 return -EBUSY;
1316
1317 parser.adev = adev;
1318 parser.filp = filp;
1319
1320 r = amdgpu_cs_parser_init(&parser, data);
1321 if (r) {
1322 if (printk_ratelimit())
1323 DRM_ERROR("Failed to initialize parser %d!\n", r);
1324 goto out;
1325 }
1326
1327 r = amdgpu_cs_ib_fill(adev, &parser);
1328 if (r)
1329 goto out;
1330
1331 r = amdgpu_cs_dependencies(adev, &parser);
1332 if (r) {
1333 DRM_ERROR("Failed in the dependencies handling %d!\n", r);
1334 goto out;
1335 }
1336
1337 r = amdgpu_cs_parser_bos(&parser, data);
1338 if (r) {
1339 if (r == -ENOMEM)
1340 DRM_ERROR("Not enough memory for command submission!\n");
1341 else if (r != -ERESTARTSYS && r != -EAGAIN)
1342 DRM_ERROR("Failed to process the buffer list %d!\n", r);
1343 goto out;
1344 }
1345
1346 reserved_buffers = true;
1347
1348 trace_amdgpu_cs_ibs(&parser);
1349
1350 r = amdgpu_cs_vm_handling(&parser);
1351 if (r)
1352 goto out;
1353
1354 r = amdgpu_cs_submit(&parser, cs);
1355
1356 out:
1357 amdgpu_cs_parser_fini(&parser, r, reserved_buffers);
1358
1359 return r;
1360 }
1361
1362 /**
1363 * amdgpu_cs_wait_ioctl - wait for a command submission to finish
1364 *
1365 * @dev: drm device
1366 * @data: data from userspace
1367 * @filp: file private
1368 *
1369 * Wait for the command submission identified by handle to finish.
1370 */
amdgpu_cs_wait_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)1371 int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
1372 struct drm_file *filp)
1373 {
1374 union drm_amdgpu_wait_cs *wait = data;
1375 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout);
1376 struct drm_sched_entity *entity;
1377 struct amdgpu_ctx *ctx;
1378 struct dma_fence *fence;
1379 long r;
1380
1381 ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id);
1382 if (ctx == NULL)
1383 return -EINVAL;
1384
1385 r = amdgpu_ctx_get_entity(ctx, wait->in.ip_type, wait->in.ip_instance,
1386 wait->in.ring, &entity);
1387 if (r) {
1388 amdgpu_ctx_put(ctx);
1389 return r;
1390 }
1391
1392 fence = amdgpu_ctx_get_fence(ctx, entity, wait->in.handle);
1393 if (IS_ERR(fence))
1394 r = PTR_ERR(fence);
1395 else if (fence) {
1396 r = dma_fence_wait_timeout(fence, true, timeout);
1397 if (r > 0 && fence->error)
1398 r = fence->error;
1399 dma_fence_put(fence);
1400 } else
1401 r = 1;
1402
1403 amdgpu_ctx_put(ctx);
1404 if (r < 0)
1405 return r;
1406
1407 memset(wait, 0, sizeof(*wait));
1408 wait->out.status = (r == 0);
1409
1410 return 0;
1411 }
1412
1413 /**
1414 * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence
1415 *
1416 * @adev: amdgpu device
1417 * @filp: file private
1418 * @user: drm_amdgpu_fence copied from user space
1419 */
amdgpu_cs_get_fence(struct amdgpu_device * adev,struct drm_file * filp,struct drm_amdgpu_fence * user)1420 static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev,
1421 struct drm_file *filp,
1422 struct drm_amdgpu_fence *user)
1423 {
1424 struct drm_sched_entity *entity;
1425 struct amdgpu_ctx *ctx;
1426 struct dma_fence *fence;
1427 int r;
1428
1429 ctx = amdgpu_ctx_get(filp->driver_priv, user->ctx_id);
1430 if (ctx == NULL)
1431 return ERR_PTR(-EINVAL);
1432
1433 r = amdgpu_ctx_get_entity(ctx, user->ip_type, user->ip_instance,
1434 user->ring, &entity);
1435 if (r) {
1436 amdgpu_ctx_put(ctx);
1437 return ERR_PTR(r);
1438 }
1439
1440 fence = amdgpu_ctx_get_fence(ctx, entity, user->seq_no);
1441 amdgpu_ctx_put(ctx);
1442
1443 return fence;
1444 }
1445
amdgpu_cs_fence_to_handle_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)1446 int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data,
1447 struct drm_file *filp)
1448 {
1449 struct amdgpu_device *adev = drm_to_adev(dev);
1450 union drm_amdgpu_fence_to_handle *info = data;
1451 struct dma_fence *fence;
1452 struct drm_syncobj *syncobj;
1453 struct sync_file *sync_file;
1454 int fd, r;
1455
1456 fence = amdgpu_cs_get_fence(adev, filp, &info->in.fence);
1457 if (IS_ERR(fence))
1458 return PTR_ERR(fence);
1459
1460 if (!fence)
1461 fence = dma_fence_get_stub();
1462
1463 switch (info->in.what) {
1464 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ:
1465 r = drm_syncobj_create(&syncobj, 0, fence);
1466 dma_fence_put(fence);
1467 if (r)
1468 return r;
1469 r = drm_syncobj_get_handle(filp, syncobj, &info->out.handle);
1470 drm_syncobj_put(syncobj);
1471 return r;
1472
1473 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD:
1474 r = drm_syncobj_create(&syncobj, 0, fence);
1475 dma_fence_put(fence);
1476 if (r)
1477 return r;
1478 r = drm_syncobj_get_fd(syncobj, (int *)&info->out.handle);
1479 drm_syncobj_put(syncobj);
1480 return r;
1481
1482 case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD:
1483 fd = get_unused_fd_flags(O_CLOEXEC);
1484 if (fd < 0) {
1485 dma_fence_put(fence);
1486 return fd;
1487 }
1488
1489 sync_file = sync_file_create(fence);
1490 dma_fence_put(fence);
1491 if (!sync_file) {
1492 put_unused_fd(fd);
1493 return -ENOMEM;
1494 }
1495
1496 fd_install(fd, sync_file->file);
1497 info->out.handle = fd;
1498 return 0;
1499
1500 default:
1501 dma_fence_put(fence);
1502 return -EINVAL;
1503 }
1504 }
1505
1506 /**
1507 * amdgpu_cs_wait_all_fences - wait on all fences to signal
1508 *
1509 * @adev: amdgpu device
1510 * @filp: file private
1511 * @wait: wait parameters
1512 * @fences: array of drm_amdgpu_fence
1513 */
amdgpu_cs_wait_all_fences(struct amdgpu_device * adev,struct drm_file * filp,union drm_amdgpu_wait_fences * wait,struct drm_amdgpu_fence * fences)1514 static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev,
1515 struct drm_file *filp,
1516 union drm_amdgpu_wait_fences *wait,
1517 struct drm_amdgpu_fence *fences)
1518 {
1519 uint32_t fence_count = wait->in.fence_count;
1520 unsigned int i;
1521 long r = 1;
1522
1523 for (i = 0; i < fence_count; i++) {
1524 struct dma_fence *fence;
1525 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns);
1526
1527 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]);
1528 if (IS_ERR(fence))
1529 return PTR_ERR(fence);
1530 else if (!fence)
1531 continue;
1532
1533 r = dma_fence_wait_timeout(fence, true, timeout);
1534 if (r > 0 && fence->error)
1535 r = fence->error;
1536
1537 dma_fence_put(fence);
1538 if (r < 0)
1539 return r;
1540
1541 if (r == 0)
1542 break;
1543 }
1544
1545 memset(wait, 0, sizeof(*wait));
1546 wait->out.status = (r > 0);
1547
1548 return 0;
1549 }
1550
1551 /**
1552 * amdgpu_cs_wait_any_fence - wait on any fence to signal
1553 *
1554 * @adev: amdgpu device
1555 * @filp: file private
1556 * @wait: wait parameters
1557 * @fences: array of drm_amdgpu_fence
1558 */
amdgpu_cs_wait_any_fence(struct amdgpu_device * adev,struct drm_file * filp,union drm_amdgpu_wait_fences * wait,struct drm_amdgpu_fence * fences)1559 static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev,
1560 struct drm_file *filp,
1561 union drm_amdgpu_wait_fences *wait,
1562 struct drm_amdgpu_fence *fences)
1563 {
1564 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns);
1565 uint32_t fence_count = wait->in.fence_count;
1566 uint32_t first = ~0;
1567 struct dma_fence **array;
1568 unsigned int i;
1569 long r;
1570
1571 /* Prepare the fence array */
1572 array = kcalloc(fence_count, sizeof(struct dma_fence *), GFP_KERNEL);
1573
1574 if (array == NULL)
1575 return -ENOMEM;
1576
1577 for (i = 0; i < fence_count; i++) {
1578 struct dma_fence *fence;
1579
1580 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]);
1581 if (IS_ERR(fence)) {
1582 r = PTR_ERR(fence);
1583 goto err_free_fence_array;
1584 } else if (fence) {
1585 array[i] = fence;
1586 } else { /* NULL, the fence has been already signaled */
1587 r = 1;
1588 first = i;
1589 goto out;
1590 }
1591 }
1592
1593 r = dma_fence_wait_any_timeout(array, fence_count, true, timeout,
1594 &first);
1595 if (r < 0)
1596 goto err_free_fence_array;
1597
1598 out:
1599 memset(wait, 0, sizeof(*wait));
1600 wait->out.status = (r > 0);
1601 wait->out.first_signaled = first;
1602
1603 if (first < fence_count && array[first])
1604 r = array[first]->error;
1605 else
1606 r = 0;
1607
1608 err_free_fence_array:
1609 for (i = 0; i < fence_count; i++)
1610 dma_fence_put(array[i]);
1611 kfree(array);
1612
1613 return r;
1614 }
1615
1616 /**
1617 * amdgpu_cs_wait_fences_ioctl - wait for multiple command submissions to finish
1618 *
1619 * @dev: drm device
1620 * @data: data from userspace
1621 * @filp: file private
1622 */
amdgpu_cs_wait_fences_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)1623 int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
1624 struct drm_file *filp)
1625 {
1626 struct amdgpu_device *adev = drm_to_adev(dev);
1627 union drm_amdgpu_wait_fences *wait = data;
1628 uint32_t fence_count = wait->in.fence_count;
1629 struct drm_amdgpu_fence *fences_user;
1630 struct drm_amdgpu_fence *fences;
1631 int r;
1632
1633 /* Get the fences from userspace */
1634 fences = kmalloc_array(fence_count, sizeof(struct drm_amdgpu_fence),
1635 GFP_KERNEL);
1636 if (fences == NULL)
1637 return -ENOMEM;
1638
1639 fences_user = u64_to_user_ptr(wait->in.fences);
1640 if (copy_from_user(fences, fences_user,
1641 sizeof(struct drm_amdgpu_fence) * fence_count)) {
1642 r = -EFAULT;
1643 goto err_free_fences;
1644 }
1645
1646 if (wait->in.wait_all)
1647 r = amdgpu_cs_wait_all_fences(adev, filp, wait, fences);
1648 else
1649 r = amdgpu_cs_wait_any_fence(adev, filp, wait, fences);
1650
1651 err_free_fences:
1652 kfree(fences);
1653
1654 return r;
1655 }
1656
1657 /**
1658 * amdgpu_cs_find_mapping - find bo_va for VM address
1659 *
1660 * @parser: command submission parser context
1661 * @addr: VM address
1662 * @bo: resulting BO of the mapping found
1663 * @map: Placeholder to return found BO mapping
1664 *
1665 * Search the buffer objects in the command submission context for a certain
1666 * virtual memory address. Returns allocation structure when found, NULL
1667 * otherwise.
1668 */
amdgpu_cs_find_mapping(struct amdgpu_cs_parser * parser,uint64_t addr,struct amdgpu_bo ** bo,struct amdgpu_bo_va_mapping ** map)1669 int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
1670 uint64_t addr, struct amdgpu_bo **bo,
1671 struct amdgpu_bo_va_mapping **map)
1672 {
1673 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
1674 struct ttm_operation_ctx ctx = { false, false };
1675 struct amdgpu_vm *vm = &fpriv->vm;
1676 struct amdgpu_bo_va_mapping *mapping;
1677 int r;
1678
1679 addr /= AMDGPU_GPU_PAGE_SIZE;
1680
1681 mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);
1682 if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo)
1683 return -EINVAL;
1684
1685 *bo = mapping->bo_va->base.bo;
1686 *map = mapping;
1687
1688 /* Double check that the BO is reserved by this CS */
1689 if (dma_resv_locking_ctx((*bo)->tbo.base.resv) != &parser->ticket)
1690 return -EINVAL;
1691
1692 if (!((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)) {
1693 (*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1694 amdgpu_bo_placement_from_domain(*bo, (*bo)->allowed_domains);
1695 r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, &ctx);
1696 if (r)
1697 return r;
1698 }
1699
1700 return amdgpu_ttm_alloc_gart(&(*bo)->tbo);
1701 }
1702