1 /*
2 * Copyright 2013 Advanced Micro Devices, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 * Authors: Christian König <christian.koenig@amd.com>
26 */
27
28 #include <linux/firmware.h>
29 #include <linux/module.h>
30 #include <drm/drmP.h>
31 #include <drm/drm.h>
32
33 #include "amdgpu.h"
34 #include "amdgpu_pm.h"
35 #include "amdgpu_vce.h"
36 #include "cikd.h"
37
38 /* 1 second timeout */
39 #define VCE_IDLE_TIMEOUT msecs_to_jiffies(1000)
40
41 /* Firmware Names */
42 #ifdef CONFIG_DRM_AMDGPU_CIK
43 #define FIRMWARE_BONAIRE "radeon/bonaire_vce.bin"
44 #define FIRMWARE_KABINI "radeon/kabini_vce.bin"
45 #define FIRMWARE_KAVERI "radeon/kaveri_vce.bin"
46 #define FIRMWARE_HAWAII "radeon/hawaii_vce.bin"
47 #define FIRMWARE_MULLINS "radeon/mullins_vce.bin"
48 #endif
49 #define FIRMWARE_TONGA "amdgpu/tonga_vce.bin"
50 #define FIRMWARE_CARRIZO "amdgpu/carrizo_vce.bin"
51 #define FIRMWARE_FIJI "amdgpu/fiji_vce.bin"
52 #define FIRMWARE_STONEY "amdgpu/stoney_vce.bin"
53 #define FIRMWARE_POLARIS10 "amdgpu/polaris10_vce.bin"
54 #define FIRMWARE_POLARIS11 "amdgpu/polaris11_vce.bin"
55 #define FIRMWARE_POLARIS12 "amdgpu/polaris12_vce.bin"
56
57 #define FIRMWARE_VEGA10 "amdgpu/vega10_vce.bin"
58
59 #ifdef CONFIG_DRM_AMDGPU_CIK
60 MODULE_FIRMWARE(FIRMWARE_BONAIRE);
61 MODULE_FIRMWARE(FIRMWARE_KABINI);
62 MODULE_FIRMWARE(FIRMWARE_KAVERI);
63 MODULE_FIRMWARE(FIRMWARE_HAWAII);
64 MODULE_FIRMWARE(FIRMWARE_MULLINS);
65 #endif
66 MODULE_FIRMWARE(FIRMWARE_TONGA);
67 MODULE_FIRMWARE(FIRMWARE_CARRIZO);
68 MODULE_FIRMWARE(FIRMWARE_FIJI);
69 MODULE_FIRMWARE(FIRMWARE_STONEY);
70 MODULE_FIRMWARE(FIRMWARE_POLARIS10);
71 MODULE_FIRMWARE(FIRMWARE_POLARIS11);
72 MODULE_FIRMWARE(FIRMWARE_POLARIS12);
73
74 MODULE_FIRMWARE(FIRMWARE_VEGA10);
75
76 static void amdgpu_vce_idle_work_handler(struct work_struct *work);
77
78 /**
79 * amdgpu_vce_init - allocate memory, load vce firmware
80 *
81 * @adev: amdgpu_device pointer
82 *
83 * First step to get VCE online, allocate memory and load the firmware
84 */
amdgpu_vce_sw_init(struct amdgpu_device * adev,unsigned long size)85 int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size)
86 {
87 struct amdgpu_ring *ring;
88 struct amd_sched_rq *rq;
89 const char *fw_name;
90 const struct common_firmware_header *hdr;
91 unsigned ucode_version, version_major, version_minor, binary_id;
92 int i, r;
93
94 switch (adev->asic_type) {
95 #ifdef CONFIG_DRM_AMDGPU_CIK
96 case CHIP_BONAIRE:
97 fw_name = FIRMWARE_BONAIRE;
98 break;
99 case CHIP_KAVERI:
100 fw_name = FIRMWARE_KAVERI;
101 break;
102 case CHIP_KABINI:
103 fw_name = FIRMWARE_KABINI;
104 break;
105 case CHIP_HAWAII:
106 fw_name = FIRMWARE_HAWAII;
107 break;
108 case CHIP_MULLINS:
109 fw_name = FIRMWARE_MULLINS;
110 break;
111 #endif
112 case CHIP_TONGA:
113 fw_name = FIRMWARE_TONGA;
114 break;
115 case CHIP_CARRIZO:
116 fw_name = FIRMWARE_CARRIZO;
117 break;
118 case CHIP_FIJI:
119 fw_name = FIRMWARE_FIJI;
120 break;
121 case CHIP_STONEY:
122 fw_name = FIRMWARE_STONEY;
123 break;
124 case CHIP_POLARIS10:
125 fw_name = FIRMWARE_POLARIS10;
126 break;
127 case CHIP_POLARIS11:
128 fw_name = FIRMWARE_POLARIS11;
129 break;
130 case CHIP_VEGA10:
131 fw_name = FIRMWARE_VEGA10;
132 break;
133 case CHIP_POLARIS12:
134 fw_name = FIRMWARE_POLARIS12;
135 break;
136
137 default:
138 return -EINVAL;
139 }
140
141 r = request_firmware(&adev->vce.fw, fw_name, adev->dev);
142 if (r) {
143 dev_err(adev->dev, "amdgpu_vce: Can't load firmware \"%s\"\n",
144 fw_name);
145 return r;
146 }
147
148 r = amdgpu_ucode_validate(adev->vce.fw);
149 if (r) {
150 dev_err(adev->dev, "amdgpu_vce: Can't validate firmware \"%s\"\n",
151 fw_name);
152 release_firmware(adev->vce.fw);
153 adev->vce.fw = NULL;
154 return r;
155 }
156
157 hdr = (const struct common_firmware_header *)adev->vce.fw->data;
158
159 ucode_version = le32_to_cpu(hdr->ucode_version);
160 version_major = (ucode_version >> 20) & 0xfff;
161 version_minor = (ucode_version >> 8) & 0xfff;
162 binary_id = ucode_version & 0xff;
163 DRM_INFO("Found VCE firmware Version: %hhd.%hhd Binary ID: %hhd\n",
164 version_major, version_minor, binary_id);
165 adev->vce.fw_version = ((version_major << 24) | (version_minor << 16) |
166 (binary_id << 8));
167
168 r = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE,
169 AMDGPU_GEM_DOMAIN_VRAM, &adev->vce.vcpu_bo,
170 &adev->vce.gpu_addr, &adev->vce.cpu_addr);
171 if (r) {
172 dev_err(adev->dev, "(%d) failed to allocate VCE bo\n", r);
173 return r;
174 }
175
176 ring = &adev->vce.ring[0];
177 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL];
178 r = amd_sched_entity_init(&ring->sched, &adev->vce.entity,
179 rq, amdgpu_sched_jobs);
180 if (r != 0) {
181 DRM_ERROR("Failed setting up VCE run queue.\n");
182 return r;
183 }
184
185 for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) {
186 atomic_set(&adev->vce.handles[i], 0);
187 adev->vce.filp[i] = NULL;
188 }
189
190 INIT_DELAYED_WORK(&adev->vce.idle_work, amdgpu_vce_idle_work_handler);
191 mutex_init(&adev->vce.idle_mutex);
192
193 return 0;
194 }
195
196 /**
197 * amdgpu_vce_fini - free memory
198 *
199 * @adev: amdgpu_device pointer
200 *
201 * Last step on VCE teardown, free firmware memory
202 */
amdgpu_vce_sw_fini(struct amdgpu_device * adev)203 int amdgpu_vce_sw_fini(struct amdgpu_device *adev)
204 {
205 unsigned i;
206
207 if (adev->vce.vcpu_bo == NULL)
208 return 0;
209
210 amd_sched_entity_fini(&adev->vce.ring[0].sched, &adev->vce.entity);
211
212 amdgpu_bo_free_kernel(&adev->vce.vcpu_bo, &adev->vce.gpu_addr,
213 (void **)&adev->vce.cpu_addr);
214
215 for (i = 0; i < adev->vce.num_rings; i++)
216 amdgpu_ring_fini(&adev->vce.ring[i]);
217
218 release_firmware(adev->vce.fw);
219 mutex_destroy(&adev->vce.idle_mutex);
220
221 return 0;
222 }
223
224 /**
225 * amdgpu_vce_suspend - unpin VCE fw memory
226 *
227 * @adev: amdgpu_device pointer
228 *
229 */
amdgpu_vce_suspend(struct amdgpu_device * adev)230 int amdgpu_vce_suspend(struct amdgpu_device *adev)
231 {
232 int i;
233
234 cancel_delayed_work_sync(&adev->vce.idle_work);
235
236 if (adev->vce.vcpu_bo == NULL)
237 return 0;
238
239 for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i)
240 if (atomic_read(&adev->vce.handles[i]))
241 break;
242
243 if (i == AMDGPU_MAX_VCE_HANDLES)
244 return 0;
245
246 /* TODO: suspending running encoding sessions isn't supported */
247 return -EINVAL;
248 }
249
250 /**
251 * amdgpu_vce_resume - pin VCE fw memory
252 *
253 * @adev: amdgpu_device pointer
254 *
255 */
amdgpu_vce_resume(struct amdgpu_device * adev)256 int amdgpu_vce_resume(struct amdgpu_device *adev)
257 {
258 void *cpu_addr;
259 const struct common_firmware_header *hdr;
260 unsigned offset;
261 int r;
262
263 if (adev->vce.vcpu_bo == NULL)
264 return -EINVAL;
265
266 r = amdgpu_bo_reserve(adev->vce.vcpu_bo, false);
267 if (r) {
268 dev_err(adev->dev, "(%d) failed to reserve VCE bo\n", r);
269 return r;
270 }
271
272 r = amdgpu_bo_kmap(adev->vce.vcpu_bo, &cpu_addr);
273 if (r) {
274 amdgpu_bo_unreserve(adev->vce.vcpu_bo);
275 dev_err(adev->dev, "(%d) VCE map failed\n", r);
276 return r;
277 }
278
279 hdr = (const struct common_firmware_header *)adev->vce.fw->data;
280 offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
281 memcpy_toio(cpu_addr, adev->vce.fw->data + offset,
282 adev->vce.fw->size - offset);
283
284 amdgpu_bo_kunmap(adev->vce.vcpu_bo);
285
286 amdgpu_bo_unreserve(adev->vce.vcpu_bo);
287
288 return 0;
289 }
290
291 /**
292 * amdgpu_vce_idle_work_handler - power off VCE
293 *
294 * @work: pointer to work structure
295 *
296 * power of VCE when it's not used any more
297 */
amdgpu_vce_idle_work_handler(struct work_struct * work)298 static void amdgpu_vce_idle_work_handler(struct work_struct *work)
299 {
300 struct amdgpu_device *adev =
301 container_of(work, struct amdgpu_device, vce.idle_work.work);
302 unsigned i, count = 0;
303
304 if (amdgpu_sriov_vf(adev))
305 return;
306
307 for (i = 0; i < adev->vce.num_rings; i++)
308 count += amdgpu_fence_count_emitted(&adev->vce.ring[i]);
309
310 if (count == 0) {
311 if (adev->pm.dpm_enabled) {
312 amdgpu_dpm_enable_vce(adev, false);
313 } else {
314 amdgpu_asic_set_vce_clocks(adev, 0, 0);
315 amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
316 AMD_PG_STATE_GATE);
317 amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
318 AMD_CG_STATE_GATE);
319 }
320 } else {
321 schedule_delayed_work(&adev->vce.idle_work, VCE_IDLE_TIMEOUT);
322 }
323 }
324
325 /**
326 * amdgpu_vce_ring_begin_use - power up VCE
327 *
328 * @ring: amdgpu ring
329 *
330 * Make sure VCE is powerd up when we want to use it
331 */
amdgpu_vce_ring_begin_use(struct amdgpu_ring * ring)332 void amdgpu_vce_ring_begin_use(struct amdgpu_ring *ring)
333 {
334 struct amdgpu_device *adev = ring->adev;
335 bool set_clocks;
336
337 if (amdgpu_sriov_vf(adev))
338 return;
339
340 mutex_lock(&adev->vce.idle_mutex);
341 set_clocks = !cancel_delayed_work_sync(&adev->vce.idle_work);
342 if (set_clocks) {
343 if (adev->pm.dpm_enabled) {
344 amdgpu_dpm_enable_vce(adev, true);
345 } else {
346 amdgpu_asic_set_vce_clocks(adev, 53300, 40000);
347 amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
348 AMD_CG_STATE_UNGATE);
349 amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
350 AMD_PG_STATE_UNGATE);
351
352 }
353 }
354 mutex_unlock(&adev->vce.idle_mutex);
355 }
356
357 /**
358 * amdgpu_vce_ring_end_use - power VCE down
359 *
360 * @ring: amdgpu ring
361 *
362 * Schedule work to power VCE down again
363 */
amdgpu_vce_ring_end_use(struct amdgpu_ring * ring)364 void amdgpu_vce_ring_end_use(struct amdgpu_ring *ring)
365 {
366 schedule_delayed_work(&ring->adev->vce.idle_work, VCE_IDLE_TIMEOUT);
367 }
368
369 /**
370 * amdgpu_vce_free_handles - free still open VCE handles
371 *
372 * @adev: amdgpu_device pointer
373 * @filp: drm file pointer
374 *
375 * Close all VCE handles still open by this file pointer
376 */
amdgpu_vce_free_handles(struct amdgpu_device * adev,struct drm_file * filp)377 void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp)
378 {
379 struct amdgpu_ring *ring = &adev->vce.ring[0];
380 int i, r;
381 for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) {
382 uint32_t handle = atomic_read(&adev->vce.handles[i]);
383
384 if (!handle || adev->vce.filp[i] != filp)
385 continue;
386
387 r = amdgpu_vce_get_destroy_msg(ring, handle, false, NULL);
388 if (r)
389 DRM_ERROR("Error destroying VCE handle (%d)!\n", r);
390
391 adev->vce.filp[i] = NULL;
392 atomic_set(&adev->vce.handles[i], 0);
393 }
394 }
395
396 /**
397 * amdgpu_vce_get_create_msg - generate a VCE create msg
398 *
399 * @adev: amdgpu_device pointer
400 * @ring: ring we should submit the msg to
401 * @handle: VCE session handle to use
402 * @fence: optional fence to return
403 *
404 * Open up a stream for HW test
405 */
amdgpu_vce_get_create_msg(struct amdgpu_ring * ring,uint32_t handle,struct dma_fence ** fence)406 int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
407 struct dma_fence **fence)
408 {
409 const unsigned ib_size_dw = 1024;
410 struct amdgpu_job *job;
411 struct amdgpu_ib *ib;
412 struct dma_fence *f = NULL;
413 uint64_t dummy;
414 int i, r;
415
416 r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job);
417 if (r)
418 return r;
419
420 ib = &job->ibs[0];
421
422 dummy = ib->gpu_addr + 1024;
423
424 /* stitch together an VCE create msg */
425 ib->length_dw = 0;
426 ib->ptr[ib->length_dw++] = 0x0000000c; /* len */
427 ib->ptr[ib->length_dw++] = 0x00000001; /* session cmd */
428 ib->ptr[ib->length_dw++] = handle;
429
430 if ((ring->adev->vce.fw_version >> 24) >= 52)
431 ib->ptr[ib->length_dw++] = 0x00000040; /* len */
432 else
433 ib->ptr[ib->length_dw++] = 0x00000030; /* len */
434 ib->ptr[ib->length_dw++] = 0x01000001; /* create cmd */
435 ib->ptr[ib->length_dw++] = 0x00000000;
436 ib->ptr[ib->length_dw++] = 0x00000042;
437 ib->ptr[ib->length_dw++] = 0x0000000a;
438 ib->ptr[ib->length_dw++] = 0x00000001;
439 ib->ptr[ib->length_dw++] = 0x00000080;
440 ib->ptr[ib->length_dw++] = 0x00000060;
441 ib->ptr[ib->length_dw++] = 0x00000100;
442 ib->ptr[ib->length_dw++] = 0x00000100;
443 ib->ptr[ib->length_dw++] = 0x0000000c;
444 ib->ptr[ib->length_dw++] = 0x00000000;
445 if ((ring->adev->vce.fw_version >> 24) >= 52) {
446 ib->ptr[ib->length_dw++] = 0x00000000;
447 ib->ptr[ib->length_dw++] = 0x00000000;
448 ib->ptr[ib->length_dw++] = 0x00000000;
449 ib->ptr[ib->length_dw++] = 0x00000000;
450 }
451
452 ib->ptr[ib->length_dw++] = 0x00000014; /* len */
453 ib->ptr[ib->length_dw++] = 0x05000005; /* feedback buffer */
454 ib->ptr[ib->length_dw++] = upper_32_bits(dummy);
455 ib->ptr[ib->length_dw++] = dummy;
456 ib->ptr[ib->length_dw++] = 0x00000001;
457
458 for (i = ib->length_dw; i < ib_size_dw; ++i)
459 ib->ptr[i] = 0x0;
460
461 r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
462 job->fence = dma_fence_get(f);
463 if (r)
464 goto err;
465
466 amdgpu_job_free(job);
467 if (fence)
468 *fence = dma_fence_get(f);
469 dma_fence_put(f);
470 return 0;
471
472 err:
473 amdgpu_job_free(job);
474 return r;
475 }
476
477 /**
478 * amdgpu_vce_get_destroy_msg - generate a VCE destroy msg
479 *
480 * @adev: amdgpu_device pointer
481 * @ring: ring we should submit the msg to
482 * @handle: VCE session handle to use
483 * @fence: optional fence to return
484 *
485 * Close up a stream for HW test or if userspace failed to do so
486 */
amdgpu_vce_get_destroy_msg(struct amdgpu_ring * ring,uint32_t handle,bool direct,struct dma_fence ** fence)487 int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
488 bool direct, struct dma_fence **fence)
489 {
490 const unsigned ib_size_dw = 1024;
491 struct amdgpu_job *job;
492 struct amdgpu_ib *ib;
493 struct dma_fence *f = NULL;
494 int i, r;
495
496 r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job);
497 if (r)
498 return r;
499
500 ib = &job->ibs[0];
501
502 /* stitch together an VCE destroy msg */
503 ib->length_dw = 0;
504 ib->ptr[ib->length_dw++] = 0x0000000c; /* len */
505 ib->ptr[ib->length_dw++] = 0x00000001; /* session cmd */
506 ib->ptr[ib->length_dw++] = handle;
507
508 ib->ptr[ib->length_dw++] = 0x00000020; /* len */
509 ib->ptr[ib->length_dw++] = 0x00000002; /* task info */
510 ib->ptr[ib->length_dw++] = 0xffffffff; /* next task info, set to 0xffffffff if no */
511 ib->ptr[ib->length_dw++] = 0x00000001; /* destroy session */
512 ib->ptr[ib->length_dw++] = 0x00000000;
513 ib->ptr[ib->length_dw++] = 0x00000000;
514 ib->ptr[ib->length_dw++] = 0xffffffff; /* feedback is not needed, set to 0xffffffff and firmware will not output feedback */
515 ib->ptr[ib->length_dw++] = 0x00000000;
516
517 ib->ptr[ib->length_dw++] = 0x00000008; /* len */
518 ib->ptr[ib->length_dw++] = 0x02000001; /* destroy cmd */
519
520 for (i = ib->length_dw; i < ib_size_dw; ++i)
521 ib->ptr[i] = 0x0;
522
523 if (direct) {
524 r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
525 job->fence = dma_fence_get(f);
526 if (r)
527 goto err;
528
529 amdgpu_job_free(job);
530 } else {
531 r = amdgpu_job_submit(job, ring, &ring->adev->vce.entity,
532 AMDGPU_FENCE_OWNER_UNDEFINED, &f);
533 if (r)
534 goto err;
535 }
536
537 if (fence)
538 *fence = dma_fence_get(f);
539 dma_fence_put(f);
540 return 0;
541
542 err:
543 amdgpu_job_free(job);
544 return r;
545 }
546
547 /**
548 * amdgpu_vce_cs_reloc - command submission relocation
549 *
550 * @p: parser context
551 * @lo: address of lower dword
552 * @hi: address of higher dword
553 * @size: minimum size
554 *
555 * Patch relocation inside command stream with real buffer address
556 */
amdgpu_vce_cs_reloc(struct amdgpu_cs_parser * p,uint32_t ib_idx,int lo,int hi,unsigned size,uint32_t index)557 static int amdgpu_vce_cs_reloc(struct amdgpu_cs_parser *p, uint32_t ib_idx,
558 int lo, int hi, unsigned size, uint32_t index)
559 {
560 struct amdgpu_bo_va_mapping *mapping;
561 struct amdgpu_bo *bo;
562 uint64_t addr;
563
564 if (index == 0xffffffff)
565 index = 0;
566
567 addr = ((uint64_t)amdgpu_get_ib_value(p, ib_idx, lo)) |
568 ((uint64_t)amdgpu_get_ib_value(p, ib_idx, hi)) << 32;
569 addr += ((uint64_t)size) * ((uint64_t)index);
570
571 mapping = amdgpu_cs_find_mapping(p, addr, &bo);
572 if (mapping == NULL) {
573 DRM_ERROR("Can't find BO for addr 0x%010Lx %d %d %d %d\n",
574 addr, lo, hi, size, index);
575 return -EINVAL;
576 }
577
578 if ((addr + (uint64_t)size) >
579 (mapping->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
580 DRM_ERROR("BO to small for addr 0x%010Lx %d %d\n",
581 addr, lo, hi);
582 return -EINVAL;
583 }
584
585 addr -= mapping->start * AMDGPU_GPU_PAGE_SIZE;
586 addr += amdgpu_bo_gpu_offset(bo);
587 addr -= ((uint64_t)size) * ((uint64_t)index);
588
589 amdgpu_set_ib_value(p, ib_idx, lo, lower_32_bits(addr));
590 amdgpu_set_ib_value(p, ib_idx, hi, upper_32_bits(addr));
591
592 return 0;
593 }
594
595 /**
596 * amdgpu_vce_validate_handle - validate stream handle
597 *
598 * @p: parser context
599 * @handle: handle to validate
600 * @allocated: allocated a new handle?
601 *
602 * Validates the handle and return the found session index or -EINVAL
603 * we we don't have another free session index.
604 */
amdgpu_vce_validate_handle(struct amdgpu_cs_parser * p,uint32_t handle,uint32_t * allocated)605 static int amdgpu_vce_validate_handle(struct amdgpu_cs_parser *p,
606 uint32_t handle, uint32_t *allocated)
607 {
608 unsigned i;
609
610 /* validate the handle */
611 for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) {
612 if (atomic_read(&p->adev->vce.handles[i]) == handle) {
613 if (p->adev->vce.filp[i] != p->filp) {
614 DRM_ERROR("VCE handle collision detected!\n");
615 return -EINVAL;
616 }
617 return i;
618 }
619 }
620
621 /* handle not found try to alloc a new one */
622 for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) {
623 if (!atomic_cmpxchg(&p->adev->vce.handles[i], 0, handle)) {
624 p->adev->vce.filp[i] = p->filp;
625 p->adev->vce.img_size[i] = 0;
626 *allocated |= 1 << i;
627 return i;
628 }
629 }
630
631 DRM_ERROR("No more free VCE handles!\n");
632 return -EINVAL;
633 }
634
635 /**
636 * amdgpu_vce_cs_parse - parse and validate the command stream
637 *
638 * @p: parser context
639 *
640 */
amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser * p,uint32_t ib_idx)641 int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
642 {
643 struct amdgpu_ib *ib = &p->job->ibs[ib_idx];
644 unsigned fb_idx = 0, bs_idx = 0;
645 int session_idx = -1;
646 uint32_t destroyed = 0;
647 uint32_t created = 0;
648 uint32_t allocated = 0;
649 uint32_t tmp, handle = 0;
650 uint32_t *size = &tmp;
651 int i, r = 0, idx = 0;
652
653 p->job->vm = NULL;
654 ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo);
655
656 r = amdgpu_cs_sysvm_access_required(p);
657 if (r)
658 return r;
659
660 while (idx < ib->length_dw) {
661 uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx);
662 uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1);
663
664 if ((len < 8) || (len & 3)) {
665 DRM_ERROR("invalid VCE command length (%d)!\n", len);
666 r = -EINVAL;
667 goto out;
668 }
669
670 switch (cmd) {
671 case 0x00000001: /* session */
672 handle = amdgpu_get_ib_value(p, ib_idx, idx + 2);
673 session_idx = amdgpu_vce_validate_handle(p, handle,
674 &allocated);
675 if (session_idx < 0) {
676 r = session_idx;
677 goto out;
678 }
679 size = &p->adev->vce.img_size[session_idx];
680 break;
681
682 case 0x00000002: /* task info */
683 fb_idx = amdgpu_get_ib_value(p, ib_idx, idx + 6);
684 bs_idx = amdgpu_get_ib_value(p, ib_idx, idx + 7);
685 break;
686
687 case 0x01000001: /* create */
688 created |= 1 << session_idx;
689 if (destroyed & (1 << session_idx)) {
690 destroyed &= ~(1 << session_idx);
691 allocated |= 1 << session_idx;
692
693 } else if (!(allocated & (1 << session_idx))) {
694 DRM_ERROR("Handle already in use!\n");
695 r = -EINVAL;
696 goto out;
697 }
698
699 *size = amdgpu_get_ib_value(p, ib_idx, idx + 8) *
700 amdgpu_get_ib_value(p, ib_idx, idx + 10) *
701 8 * 3 / 2;
702 break;
703
704 case 0x04000001: /* config extension */
705 case 0x04000002: /* pic control */
706 case 0x04000005: /* rate control */
707 case 0x04000007: /* motion estimation */
708 case 0x04000008: /* rdo */
709 case 0x04000009: /* vui */
710 case 0x05000002: /* auxiliary buffer */
711 case 0x05000009: /* clock table */
712 break;
713
714 case 0x0500000c: /* hw config */
715 switch (p->adev->asic_type) {
716 #ifdef CONFIG_DRM_AMDGPU_CIK
717 case CHIP_KAVERI:
718 case CHIP_MULLINS:
719 #endif
720 case CHIP_CARRIZO:
721 break;
722 default:
723 r = -EINVAL;
724 goto out;
725 }
726 break;
727
728 case 0x03000001: /* encode */
729 r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 10, idx + 9,
730 *size, 0);
731 if (r)
732 goto out;
733
734 r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 12, idx + 11,
735 *size / 3, 0);
736 if (r)
737 goto out;
738 break;
739
740 case 0x02000001: /* destroy */
741 destroyed |= 1 << session_idx;
742 break;
743
744 case 0x05000001: /* context buffer */
745 r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3, idx + 2,
746 *size * 2, 0);
747 if (r)
748 goto out;
749 break;
750
751 case 0x05000004: /* video bitstream buffer */
752 tmp = amdgpu_get_ib_value(p, ib_idx, idx + 4);
753 r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3, idx + 2,
754 tmp, bs_idx);
755 if (r)
756 goto out;
757 break;
758
759 case 0x05000005: /* feedback buffer */
760 r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3, idx + 2,
761 4096, fb_idx);
762 if (r)
763 goto out;
764 break;
765
766 default:
767 DRM_ERROR("invalid VCE command (0x%x)!\n", cmd);
768 r = -EINVAL;
769 goto out;
770 }
771
772 if (session_idx == -1) {
773 DRM_ERROR("no session command at start of IB\n");
774 r = -EINVAL;
775 goto out;
776 }
777
778 idx += len / 4;
779 }
780
781 if (allocated & ~created) {
782 DRM_ERROR("New session without create command!\n");
783 r = -ENOENT;
784 }
785
786 out:
787 if (!r) {
788 /* No error, free all destroyed handle slots */
789 tmp = destroyed;
790 } else {
791 /* Error during parsing, free all allocated handle slots */
792 tmp = allocated;
793 }
794
795 for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i)
796 if (tmp & (1 << i))
797 atomic_set(&p->adev->vce.handles[i], 0);
798
799 return r;
800 }
801
802 /**
803 * amdgpu_vce_cs_parse_vm - parse the command stream in VM mode
804 *
805 * @p: parser context
806 *
807 */
amdgpu_vce_ring_parse_cs_vm(struct amdgpu_cs_parser * p,uint32_t ib_idx)808 int amdgpu_vce_ring_parse_cs_vm(struct amdgpu_cs_parser *p, uint32_t ib_idx)
809 {
810 struct amdgpu_ib *ib = &p->job->ibs[ib_idx];
811 int session_idx = -1;
812 uint32_t destroyed = 0;
813 uint32_t created = 0;
814 uint32_t allocated = 0;
815 uint32_t tmp, handle = 0;
816 int i, r = 0, idx = 0;
817
818 while (idx < ib->length_dw) {
819 uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx);
820 uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1);
821
822 if ((len < 8) || (len & 3)) {
823 DRM_ERROR("invalid VCE command length (%d)!\n", len);
824 r = -EINVAL;
825 goto out;
826 }
827
828 switch (cmd) {
829 case 0x00000001: /* session */
830 handle = amdgpu_get_ib_value(p, ib_idx, idx + 2);
831 session_idx = amdgpu_vce_validate_handle(p, handle,
832 &allocated);
833 if (session_idx < 0) {
834 r = session_idx;
835 goto out;
836 }
837 break;
838
839 case 0x01000001: /* create */
840 created |= 1 << session_idx;
841 if (destroyed & (1 << session_idx)) {
842 destroyed &= ~(1 << session_idx);
843 allocated |= 1 << session_idx;
844
845 } else if (!(allocated & (1 << session_idx))) {
846 DRM_ERROR("Handle already in use!\n");
847 r = -EINVAL;
848 goto out;
849 }
850
851 break;
852
853 case 0x02000001: /* destroy */
854 destroyed |= 1 << session_idx;
855 break;
856
857 default:
858 break;
859 }
860
861 if (session_idx == -1) {
862 DRM_ERROR("no session command at start of IB\n");
863 r = -EINVAL;
864 goto out;
865 }
866
867 idx += len / 4;
868 }
869
870 if (allocated & ~created) {
871 DRM_ERROR("New session without create command!\n");
872 r = -ENOENT;
873 }
874
875 out:
876 if (!r) {
877 /* No error, free all destroyed handle slots */
878 tmp = destroyed;
879 amdgpu_ib_free(p->adev, ib, NULL);
880 } else {
881 /* Error during parsing, free all allocated handle slots */
882 tmp = allocated;
883 }
884
885 for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i)
886 if (tmp & (1 << i))
887 atomic_set(&p->adev->vce.handles[i], 0);
888
889 return r;
890 }
891
892 /**
893 * amdgpu_vce_ring_emit_ib - execute indirect buffer
894 *
895 * @ring: engine to use
896 * @ib: the IB to execute
897 *
898 */
amdgpu_vce_ring_emit_ib(struct amdgpu_ring * ring,struct amdgpu_ib * ib,unsigned vm_id,bool ctx_switch)899 void amdgpu_vce_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib,
900 unsigned vm_id, bool ctx_switch)
901 {
902 amdgpu_ring_write(ring, VCE_CMD_IB);
903 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
904 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
905 amdgpu_ring_write(ring, ib->length_dw);
906 }
907
908 /**
909 * amdgpu_vce_ring_emit_fence - add a fence command to the ring
910 *
911 * @ring: engine to use
912 * @fence: the fence
913 *
914 */
amdgpu_vce_ring_emit_fence(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned flags)915 void amdgpu_vce_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
916 unsigned flags)
917 {
918 WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
919
920 amdgpu_ring_write(ring, VCE_CMD_FENCE);
921 amdgpu_ring_write(ring, addr);
922 amdgpu_ring_write(ring, upper_32_bits(addr));
923 amdgpu_ring_write(ring, seq);
924 amdgpu_ring_write(ring, VCE_CMD_TRAP);
925 amdgpu_ring_write(ring, VCE_CMD_END);
926 }
927
928 /**
929 * amdgpu_vce_ring_test_ring - test if VCE ring is working
930 *
931 * @ring: the engine to test on
932 *
933 */
amdgpu_vce_ring_test_ring(struct amdgpu_ring * ring)934 int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring)
935 {
936 struct amdgpu_device *adev = ring->adev;
937 uint32_t rptr = amdgpu_ring_get_rptr(ring);
938 unsigned i;
939 int r, timeout = adev->usec_timeout;
940
941 /* skip ring test for sriov*/
942 if (amdgpu_sriov_vf(adev))
943 return 0;
944
945 r = amdgpu_ring_alloc(ring, 16);
946 if (r) {
947 DRM_ERROR("amdgpu: vce failed to lock ring %d (%d).\n",
948 ring->idx, r);
949 return r;
950 }
951 amdgpu_ring_write(ring, VCE_CMD_END);
952 amdgpu_ring_commit(ring);
953
954 for (i = 0; i < timeout; i++) {
955 if (amdgpu_ring_get_rptr(ring) != rptr)
956 break;
957 DRM_UDELAY(1);
958 }
959
960 if (i < timeout) {
961 DRM_INFO("ring test on %d succeeded in %d usecs\n",
962 ring->idx, i);
963 } else {
964 DRM_ERROR("amdgpu: ring %d test failed\n",
965 ring->idx);
966 r = -ETIMEDOUT;
967 }
968
969 return r;
970 }
971
972 /**
973 * amdgpu_vce_ring_test_ib - test if VCE IBs are working
974 *
975 * @ring: the engine to test on
976 *
977 */
amdgpu_vce_ring_test_ib(struct amdgpu_ring * ring,long timeout)978 int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout)
979 {
980 struct dma_fence *fence = NULL;
981 long r;
982
983 /* skip vce ring1/2 ib test for now, since it's not reliable */
984 if (ring != &ring->adev->vce.ring[0])
985 return 0;
986
987 r = amdgpu_vce_get_create_msg(ring, 1, NULL);
988 if (r) {
989 DRM_ERROR("amdgpu: failed to get create msg (%ld).\n", r);
990 goto error;
991 }
992
993 r = amdgpu_vce_get_destroy_msg(ring, 1, true, &fence);
994 if (r) {
995 DRM_ERROR("amdgpu: failed to get destroy ib (%ld).\n", r);
996 goto error;
997 }
998
999 r = dma_fence_wait_timeout(fence, false, timeout);
1000 if (r == 0) {
1001 DRM_ERROR("amdgpu: IB test timed out.\n");
1002 r = -ETIMEDOUT;
1003 } else if (r < 0) {
1004 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
1005 } else {
1006 DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
1007 r = 0;
1008 }
1009 error:
1010 dma_fence_put(fence);
1011 return r;
1012 }
1013