1 /*
2 * Copyright 2019 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/firmware.h>
25 #include <drm/drm_exec.h>
26
27 #include "amdgpu_mes.h"
28 #include "amdgpu.h"
29 #include "soc15_common.h"
30 #include "amdgpu_mes_ctx.h"
31
32 #define AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS 1024
33 #define AMDGPU_ONE_DOORBELL_SIZE 8
34
amdgpu_mes_doorbell_process_slice(struct amdgpu_device * adev)35 int amdgpu_mes_doorbell_process_slice(struct amdgpu_device *adev)
36 {
37 return roundup(AMDGPU_ONE_DOORBELL_SIZE *
38 AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS,
39 PAGE_SIZE);
40 }
41
amdgpu_mes_kernel_doorbell_get(struct amdgpu_device * adev,struct amdgpu_mes_process * process,int ip_type,uint64_t * doorbell_index)42 static int amdgpu_mes_kernel_doorbell_get(struct amdgpu_device *adev,
43 struct amdgpu_mes_process *process,
44 int ip_type, uint64_t *doorbell_index)
45 {
46 unsigned int offset, found;
47 struct amdgpu_mes *mes = &adev->mes;
48
49 if (ip_type == AMDGPU_RING_TYPE_SDMA)
50 offset = adev->doorbell_index.sdma_engine[0];
51 else
52 offset = 0;
53
54 found = find_next_zero_bit(mes->doorbell_bitmap, mes->num_mes_dbs, offset);
55 if (found >= mes->num_mes_dbs) {
56 DRM_WARN("No doorbell available\n");
57 return -ENOSPC;
58 }
59
60 set_bit(found, mes->doorbell_bitmap);
61
62 /* Get the absolute doorbell index on BAR */
63 *doorbell_index = mes->db_start_dw_offset + found * 2;
64 return 0;
65 }
66
amdgpu_mes_kernel_doorbell_free(struct amdgpu_device * adev,struct amdgpu_mes_process * process,uint32_t doorbell_index)67 static void amdgpu_mes_kernel_doorbell_free(struct amdgpu_device *adev,
68 struct amdgpu_mes_process *process,
69 uint32_t doorbell_index)
70 {
71 unsigned int old, rel_index;
72 struct amdgpu_mes *mes = &adev->mes;
73
74 /* Find the relative index of the doorbell in this object */
75 rel_index = (doorbell_index - mes->db_start_dw_offset) / 2;
76 old = test_and_clear_bit(rel_index, mes->doorbell_bitmap);
77 WARN_ON(!old);
78 }
79
amdgpu_mes_doorbell_init(struct amdgpu_device * adev)80 static int amdgpu_mes_doorbell_init(struct amdgpu_device *adev)
81 {
82 int i;
83 struct amdgpu_mes *mes = &adev->mes;
84
85 /* Bitmap for dynamic allocation of kernel doorbells */
86 mes->doorbell_bitmap = bitmap_zalloc(PAGE_SIZE / sizeof(u32), GFP_KERNEL);
87 if (!mes->doorbell_bitmap) {
88 DRM_ERROR("Failed to allocate MES doorbell bitmap\n");
89 return -ENOMEM;
90 }
91
92 mes->num_mes_dbs = PAGE_SIZE / AMDGPU_ONE_DOORBELL_SIZE;
93 for (i = 0; i < AMDGPU_MES_PRIORITY_NUM_LEVELS; i++) {
94 adev->mes.aggregated_doorbells[i] = mes->db_start_dw_offset + i * 2;
95 set_bit(i, mes->doorbell_bitmap);
96 }
97
98 return 0;
99 }
100
amdgpu_mes_doorbell_free(struct amdgpu_device * adev)101 static void amdgpu_mes_doorbell_free(struct amdgpu_device *adev)
102 {
103 bitmap_free(adev->mes.doorbell_bitmap);
104 }
105
amdgpu_mes_init(struct amdgpu_device * adev)106 int amdgpu_mes_init(struct amdgpu_device *adev)
107 {
108 int i, r;
109
110 adev->mes.adev = adev;
111
112 idr_init(&adev->mes.pasid_idr);
113 idr_init(&adev->mes.gang_id_idr);
114 idr_init(&adev->mes.queue_id_idr);
115 ida_init(&adev->mes.doorbell_ida);
116 spin_lock_init(&adev->mes.queue_id_lock);
117 spin_lock_init(&adev->mes.ring_lock);
118 mutex_init(&adev->mes.mutex_hidden);
119
120 adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK;
121 adev->mes.vmid_mask_mmhub = 0xffffff00;
122 adev->mes.vmid_mask_gfxhub = 0xffffff00;
123
124 for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) {
125 /* use only 1st MEC pipes */
126 if (i >= 4)
127 continue;
128 adev->mes.compute_hqd_mask[i] = 0xc;
129 }
130
131 for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++)
132 adev->mes.gfx_hqd_mask[i] = i ? 0 : 0xfffffffe;
133
134 for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++) {
135 if (adev->ip_versions[SDMA0_HWIP][0] < IP_VERSION(6, 0, 0))
136 adev->mes.sdma_hqd_mask[i] = i ? 0 : 0x3fc;
137 /* zero sdma_hqd_mask for non-existent engine */
138 else if (adev->sdma.num_instances == 1)
139 adev->mes.sdma_hqd_mask[i] = i ? 0 : 0xfc;
140 else
141 adev->mes.sdma_hqd_mask[i] = 0xfc;
142 }
143
144 r = amdgpu_device_wb_get(adev, &adev->mes.sch_ctx_offs);
145 if (r) {
146 dev_err(adev->dev,
147 "(%d) ring trail_fence_offs wb alloc failed\n", r);
148 goto error_ids;
149 }
150 adev->mes.sch_ctx_gpu_addr =
151 adev->wb.gpu_addr + (adev->mes.sch_ctx_offs * 4);
152 adev->mes.sch_ctx_ptr =
153 (uint64_t *)&adev->wb.wb[adev->mes.sch_ctx_offs];
154
155 r = amdgpu_device_wb_get(adev, &adev->mes.query_status_fence_offs);
156 if (r) {
157 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);
158 dev_err(adev->dev,
159 "(%d) query_status_fence_offs wb alloc failed\n", r);
160 goto error_ids;
161 }
162 adev->mes.query_status_fence_gpu_addr =
163 adev->wb.gpu_addr + (adev->mes.query_status_fence_offs * 4);
164 adev->mes.query_status_fence_ptr =
165 (uint64_t *)&adev->wb.wb[adev->mes.query_status_fence_offs];
166
167 r = amdgpu_device_wb_get(adev, &adev->mes.read_val_offs);
168 if (r) {
169 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);
170 amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs);
171 dev_err(adev->dev,
172 "(%d) read_val_offs alloc failed\n", r);
173 goto error_ids;
174 }
175 adev->mes.read_val_gpu_addr =
176 adev->wb.gpu_addr + (adev->mes.read_val_offs * 4);
177 adev->mes.read_val_ptr =
178 (uint32_t *)&adev->wb.wb[adev->mes.read_val_offs];
179
180 r = amdgpu_mes_doorbell_init(adev);
181 if (r)
182 goto error;
183
184 return 0;
185
186 error:
187 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);
188 amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs);
189 amdgpu_device_wb_free(adev, adev->mes.read_val_offs);
190 error_ids:
191 idr_destroy(&adev->mes.pasid_idr);
192 idr_destroy(&adev->mes.gang_id_idr);
193 idr_destroy(&adev->mes.queue_id_idr);
194 ida_destroy(&adev->mes.doorbell_ida);
195 mutex_destroy(&adev->mes.mutex_hidden);
196 return r;
197 }
198
amdgpu_mes_fini(struct amdgpu_device * adev)199 void amdgpu_mes_fini(struct amdgpu_device *adev)
200 {
201 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);
202 amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs);
203 amdgpu_device_wb_free(adev, adev->mes.read_val_offs);
204 amdgpu_mes_doorbell_free(adev);
205
206 idr_destroy(&adev->mes.pasid_idr);
207 idr_destroy(&adev->mes.gang_id_idr);
208 idr_destroy(&adev->mes.queue_id_idr);
209 ida_destroy(&adev->mes.doorbell_ida);
210 mutex_destroy(&adev->mes.mutex_hidden);
211 }
212
amdgpu_mes_queue_free_mqd(struct amdgpu_mes_queue * q)213 static void amdgpu_mes_queue_free_mqd(struct amdgpu_mes_queue *q)
214 {
215 amdgpu_bo_free_kernel(&q->mqd_obj,
216 &q->mqd_gpu_addr,
217 &q->mqd_cpu_ptr);
218 }
219
amdgpu_mes_create_process(struct amdgpu_device * adev,int pasid,struct amdgpu_vm * vm)220 int amdgpu_mes_create_process(struct amdgpu_device *adev, int pasid,
221 struct amdgpu_vm *vm)
222 {
223 struct amdgpu_mes_process *process;
224 int r;
225
226 /* allocate the mes process buffer */
227 process = kzalloc(sizeof(struct amdgpu_mes_process), GFP_KERNEL);
228 if (!process) {
229 DRM_ERROR("no more memory to create mes process\n");
230 return -ENOMEM;
231 }
232
233 /* allocate the process context bo and map it */
234 r = amdgpu_bo_create_kernel(adev, AMDGPU_MES_PROC_CTX_SIZE, PAGE_SIZE,
235 AMDGPU_GEM_DOMAIN_GTT,
236 &process->proc_ctx_bo,
237 &process->proc_ctx_gpu_addr,
238 &process->proc_ctx_cpu_ptr);
239 if (r) {
240 DRM_ERROR("failed to allocate process context bo\n");
241 goto clean_up_memory;
242 }
243 memset(process->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE);
244
245 /*
246 * Avoid taking any other locks under MES lock to avoid circular
247 * lock dependencies.
248 */
249 amdgpu_mes_lock(&adev->mes);
250
251 /* add the mes process to idr list */
252 r = idr_alloc(&adev->mes.pasid_idr, process, pasid, pasid + 1,
253 GFP_KERNEL);
254 if (r < 0) {
255 DRM_ERROR("failed to lock pasid=%d\n", pasid);
256 goto clean_up_ctx;
257 }
258
259 INIT_LIST_HEAD(&process->gang_list);
260 process->vm = vm;
261 process->pasid = pasid;
262 process->process_quantum = adev->mes.default_process_quantum;
263 process->pd_gpu_addr = amdgpu_bo_gpu_offset(vm->root.bo);
264
265 amdgpu_mes_unlock(&adev->mes);
266 return 0;
267
268 clean_up_ctx:
269 amdgpu_mes_unlock(&adev->mes);
270 amdgpu_bo_free_kernel(&process->proc_ctx_bo,
271 &process->proc_ctx_gpu_addr,
272 &process->proc_ctx_cpu_ptr);
273 clean_up_memory:
274 kfree(process);
275 return r;
276 }
277
amdgpu_mes_destroy_process(struct amdgpu_device * adev,int pasid)278 void amdgpu_mes_destroy_process(struct amdgpu_device *adev, int pasid)
279 {
280 struct amdgpu_mes_process *process;
281 struct amdgpu_mes_gang *gang, *tmp1;
282 struct amdgpu_mes_queue *queue, *tmp2;
283 struct mes_remove_queue_input queue_input;
284 unsigned long flags;
285 int r;
286
287 /*
288 * Avoid taking any other locks under MES lock to avoid circular
289 * lock dependencies.
290 */
291 amdgpu_mes_lock(&adev->mes);
292
293 process = idr_find(&adev->mes.pasid_idr, pasid);
294 if (!process) {
295 DRM_WARN("pasid %d doesn't exist\n", pasid);
296 amdgpu_mes_unlock(&adev->mes);
297 return;
298 }
299
300 /* Remove all queues from hardware */
301 list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) {
302 list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) {
303 spin_lock_irqsave(&adev->mes.queue_id_lock, flags);
304 idr_remove(&adev->mes.queue_id_idr, queue->queue_id);
305 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);
306
307 queue_input.doorbell_offset = queue->doorbell_off;
308 queue_input.gang_context_addr = gang->gang_ctx_gpu_addr;
309
310 r = adev->mes.funcs->remove_hw_queue(&adev->mes,
311 &queue_input);
312 if (r)
313 DRM_WARN("failed to remove hardware queue\n");
314 }
315
316 idr_remove(&adev->mes.gang_id_idr, gang->gang_id);
317 }
318
319 idr_remove(&adev->mes.pasid_idr, pasid);
320 amdgpu_mes_unlock(&adev->mes);
321
322 /* free all memory allocated by the process */
323 list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) {
324 /* free all queues in the gang */
325 list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) {
326 amdgpu_mes_queue_free_mqd(queue);
327 list_del(&queue->list);
328 kfree(queue);
329 }
330 amdgpu_bo_free_kernel(&gang->gang_ctx_bo,
331 &gang->gang_ctx_gpu_addr,
332 &gang->gang_ctx_cpu_ptr);
333 list_del(&gang->list);
334 kfree(gang);
335
336 }
337 amdgpu_bo_free_kernel(&process->proc_ctx_bo,
338 &process->proc_ctx_gpu_addr,
339 &process->proc_ctx_cpu_ptr);
340 kfree(process);
341 }
342
amdgpu_mes_add_gang(struct amdgpu_device * adev,int pasid,struct amdgpu_mes_gang_properties * gprops,int * gang_id)343 int amdgpu_mes_add_gang(struct amdgpu_device *adev, int pasid,
344 struct amdgpu_mes_gang_properties *gprops,
345 int *gang_id)
346 {
347 struct amdgpu_mes_process *process;
348 struct amdgpu_mes_gang *gang;
349 int r;
350
351 /* allocate the mes gang buffer */
352 gang = kzalloc(sizeof(struct amdgpu_mes_gang), GFP_KERNEL);
353 if (!gang) {
354 return -ENOMEM;
355 }
356
357 /* allocate the gang context bo and map it to cpu space */
358 r = amdgpu_bo_create_kernel(adev, AMDGPU_MES_GANG_CTX_SIZE, PAGE_SIZE,
359 AMDGPU_GEM_DOMAIN_GTT,
360 &gang->gang_ctx_bo,
361 &gang->gang_ctx_gpu_addr,
362 &gang->gang_ctx_cpu_ptr);
363 if (r) {
364 DRM_ERROR("failed to allocate process context bo\n");
365 goto clean_up_mem;
366 }
367 memset(gang->gang_ctx_cpu_ptr, 0, AMDGPU_MES_GANG_CTX_SIZE);
368
369 /*
370 * Avoid taking any other locks under MES lock to avoid circular
371 * lock dependencies.
372 */
373 amdgpu_mes_lock(&adev->mes);
374
375 process = idr_find(&adev->mes.pasid_idr, pasid);
376 if (!process) {
377 DRM_ERROR("pasid %d doesn't exist\n", pasid);
378 r = -EINVAL;
379 goto clean_up_ctx;
380 }
381
382 /* add the mes gang to idr list */
383 r = idr_alloc(&adev->mes.gang_id_idr, gang, 1, 0,
384 GFP_KERNEL);
385 if (r < 0) {
386 DRM_ERROR("failed to allocate idr for gang\n");
387 goto clean_up_ctx;
388 }
389
390 gang->gang_id = r;
391 *gang_id = r;
392
393 INIT_LIST_HEAD(&gang->queue_list);
394 gang->process = process;
395 gang->priority = gprops->priority;
396 gang->gang_quantum = gprops->gang_quantum ?
397 gprops->gang_quantum : adev->mes.default_gang_quantum;
398 gang->global_priority_level = gprops->global_priority_level;
399 gang->inprocess_gang_priority = gprops->inprocess_gang_priority;
400 list_add_tail(&gang->list, &process->gang_list);
401
402 amdgpu_mes_unlock(&adev->mes);
403 return 0;
404
405 clean_up_ctx:
406 amdgpu_mes_unlock(&adev->mes);
407 amdgpu_bo_free_kernel(&gang->gang_ctx_bo,
408 &gang->gang_ctx_gpu_addr,
409 &gang->gang_ctx_cpu_ptr);
410 clean_up_mem:
411 kfree(gang);
412 return r;
413 }
414
amdgpu_mes_remove_gang(struct amdgpu_device * adev,int gang_id)415 int amdgpu_mes_remove_gang(struct amdgpu_device *adev, int gang_id)
416 {
417 struct amdgpu_mes_gang *gang;
418
419 /*
420 * Avoid taking any other locks under MES lock to avoid circular
421 * lock dependencies.
422 */
423 amdgpu_mes_lock(&adev->mes);
424
425 gang = idr_find(&adev->mes.gang_id_idr, gang_id);
426 if (!gang) {
427 DRM_ERROR("gang id %d doesn't exist\n", gang_id);
428 amdgpu_mes_unlock(&adev->mes);
429 return -EINVAL;
430 }
431
432 if (!list_empty(&gang->queue_list)) {
433 DRM_ERROR("queue list is not empty\n");
434 amdgpu_mes_unlock(&adev->mes);
435 return -EBUSY;
436 }
437
438 idr_remove(&adev->mes.gang_id_idr, gang->gang_id);
439 list_del(&gang->list);
440 amdgpu_mes_unlock(&adev->mes);
441
442 amdgpu_bo_free_kernel(&gang->gang_ctx_bo,
443 &gang->gang_ctx_gpu_addr,
444 &gang->gang_ctx_cpu_ptr);
445
446 kfree(gang);
447
448 return 0;
449 }
450
amdgpu_mes_suspend(struct amdgpu_device * adev)451 int amdgpu_mes_suspend(struct amdgpu_device *adev)
452 {
453 struct idr *idp;
454 struct amdgpu_mes_process *process;
455 struct amdgpu_mes_gang *gang;
456 struct mes_suspend_gang_input input;
457 int r, pasid;
458
459 /*
460 * Avoid taking any other locks under MES lock to avoid circular
461 * lock dependencies.
462 */
463 amdgpu_mes_lock(&adev->mes);
464
465 idp = &adev->mes.pasid_idr;
466
467 idr_for_each_entry(idp, process, pasid) {
468 list_for_each_entry(gang, &process->gang_list, list) {
469 r = adev->mes.funcs->suspend_gang(&adev->mes, &input);
470 if (r)
471 DRM_ERROR("failed to suspend pasid %d gangid %d",
472 pasid, gang->gang_id);
473 }
474 }
475
476 amdgpu_mes_unlock(&adev->mes);
477 return 0;
478 }
479
amdgpu_mes_resume(struct amdgpu_device * adev)480 int amdgpu_mes_resume(struct amdgpu_device *adev)
481 {
482 struct idr *idp;
483 struct amdgpu_mes_process *process;
484 struct amdgpu_mes_gang *gang;
485 struct mes_resume_gang_input input;
486 int r, pasid;
487
488 /*
489 * Avoid taking any other locks under MES lock to avoid circular
490 * lock dependencies.
491 */
492 amdgpu_mes_lock(&adev->mes);
493
494 idp = &adev->mes.pasid_idr;
495
496 idr_for_each_entry(idp, process, pasid) {
497 list_for_each_entry(gang, &process->gang_list, list) {
498 r = adev->mes.funcs->resume_gang(&adev->mes, &input);
499 if (r)
500 DRM_ERROR("failed to resume pasid %d gangid %d",
501 pasid, gang->gang_id);
502 }
503 }
504
505 amdgpu_mes_unlock(&adev->mes);
506 return 0;
507 }
508
amdgpu_mes_queue_alloc_mqd(struct amdgpu_device * adev,struct amdgpu_mes_queue * q,struct amdgpu_mes_queue_properties * p)509 static int amdgpu_mes_queue_alloc_mqd(struct amdgpu_device *adev,
510 struct amdgpu_mes_queue *q,
511 struct amdgpu_mes_queue_properties *p)
512 {
513 struct amdgpu_mqd *mqd_mgr = &adev->mqds[p->queue_type];
514 u32 mqd_size = mqd_mgr->mqd_size;
515 int r;
516
517 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
518 AMDGPU_GEM_DOMAIN_GTT,
519 &q->mqd_obj,
520 &q->mqd_gpu_addr, &q->mqd_cpu_ptr);
521 if (r) {
522 dev_warn(adev->dev, "failed to create queue mqd bo (%d)", r);
523 return r;
524 }
525 memset(q->mqd_cpu_ptr, 0, mqd_size);
526
527 r = amdgpu_bo_reserve(q->mqd_obj, false);
528 if (unlikely(r != 0))
529 goto clean_up;
530
531 return 0;
532
533 clean_up:
534 amdgpu_bo_free_kernel(&q->mqd_obj,
535 &q->mqd_gpu_addr,
536 &q->mqd_cpu_ptr);
537 return r;
538 }
539
amdgpu_mes_queue_init_mqd(struct amdgpu_device * adev,struct amdgpu_mes_queue * q,struct amdgpu_mes_queue_properties * p)540 static void amdgpu_mes_queue_init_mqd(struct amdgpu_device *adev,
541 struct amdgpu_mes_queue *q,
542 struct amdgpu_mes_queue_properties *p)
543 {
544 struct amdgpu_mqd *mqd_mgr = &adev->mqds[p->queue_type];
545 struct amdgpu_mqd_prop mqd_prop = {0};
546
547 mqd_prop.mqd_gpu_addr = q->mqd_gpu_addr;
548 mqd_prop.hqd_base_gpu_addr = p->hqd_base_gpu_addr;
549 mqd_prop.rptr_gpu_addr = p->rptr_gpu_addr;
550 mqd_prop.wptr_gpu_addr = p->wptr_gpu_addr;
551 mqd_prop.queue_size = p->queue_size;
552 mqd_prop.use_doorbell = true;
553 mqd_prop.doorbell_index = p->doorbell_off;
554 mqd_prop.eop_gpu_addr = p->eop_gpu_addr;
555 mqd_prop.hqd_pipe_priority = p->hqd_pipe_priority;
556 mqd_prop.hqd_queue_priority = p->hqd_queue_priority;
557 mqd_prop.hqd_active = false;
558
559 if (p->queue_type == AMDGPU_RING_TYPE_GFX ||
560 p->queue_type == AMDGPU_RING_TYPE_COMPUTE) {
561 mutex_lock(&adev->srbm_mutex);
562 amdgpu_gfx_select_me_pipe_q(adev, p->ring->me, p->ring->pipe, 0, 0, 0);
563 }
564
565 mqd_mgr->init_mqd(adev, q->mqd_cpu_ptr, &mqd_prop);
566
567 if (p->queue_type == AMDGPU_RING_TYPE_GFX ||
568 p->queue_type == AMDGPU_RING_TYPE_COMPUTE) {
569 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
570 mutex_unlock(&adev->srbm_mutex);
571 }
572
573 amdgpu_bo_unreserve(q->mqd_obj);
574 }
575
amdgpu_mes_add_hw_queue(struct amdgpu_device * adev,int gang_id,struct amdgpu_mes_queue_properties * qprops,int * queue_id)576 int amdgpu_mes_add_hw_queue(struct amdgpu_device *adev, int gang_id,
577 struct amdgpu_mes_queue_properties *qprops,
578 int *queue_id)
579 {
580 struct amdgpu_mes_queue *queue;
581 struct amdgpu_mes_gang *gang;
582 struct mes_add_queue_input queue_input;
583 unsigned long flags;
584 int r;
585
586 memset(&queue_input, 0, sizeof(struct mes_add_queue_input));
587
588 /* allocate the mes queue buffer */
589 queue = kzalloc(sizeof(struct amdgpu_mes_queue), GFP_KERNEL);
590 if (!queue) {
591 DRM_ERROR("Failed to allocate memory for queue\n");
592 return -ENOMEM;
593 }
594
595 /* Allocate the queue mqd */
596 r = amdgpu_mes_queue_alloc_mqd(adev, queue, qprops);
597 if (r)
598 goto clean_up_memory;
599
600 /*
601 * Avoid taking any other locks under MES lock to avoid circular
602 * lock dependencies.
603 */
604 amdgpu_mes_lock(&adev->mes);
605
606 gang = idr_find(&adev->mes.gang_id_idr, gang_id);
607 if (!gang) {
608 DRM_ERROR("gang id %d doesn't exist\n", gang_id);
609 r = -EINVAL;
610 goto clean_up_mqd;
611 }
612
613 /* add the mes gang to idr list */
614 spin_lock_irqsave(&adev->mes.queue_id_lock, flags);
615 r = idr_alloc(&adev->mes.queue_id_idr, queue, 1, 0,
616 GFP_ATOMIC);
617 if (r < 0) {
618 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);
619 goto clean_up_mqd;
620 }
621 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);
622 *queue_id = queue->queue_id = r;
623
624 /* allocate a doorbell index for the queue */
625 r = amdgpu_mes_kernel_doorbell_get(adev, gang->process,
626 qprops->queue_type,
627 &qprops->doorbell_off);
628 if (r)
629 goto clean_up_queue_id;
630
631 /* initialize the queue mqd */
632 amdgpu_mes_queue_init_mqd(adev, queue, qprops);
633
634 /* add hw queue to mes */
635 queue_input.process_id = gang->process->pasid;
636
637 queue_input.page_table_base_addr =
638 adev->vm_manager.vram_base_offset + gang->process->pd_gpu_addr -
639 adev->gmc.vram_start;
640
641 queue_input.process_va_start = 0;
642 queue_input.process_va_end =
643 (adev->vm_manager.max_pfn - 1) << AMDGPU_GPU_PAGE_SHIFT;
644 queue_input.process_quantum = gang->process->process_quantum;
645 queue_input.process_context_addr = gang->process->proc_ctx_gpu_addr;
646 queue_input.gang_quantum = gang->gang_quantum;
647 queue_input.gang_context_addr = gang->gang_ctx_gpu_addr;
648 queue_input.inprocess_gang_priority = gang->inprocess_gang_priority;
649 queue_input.gang_global_priority_level = gang->global_priority_level;
650 queue_input.doorbell_offset = qprops->doorbell_off;
651 queue_input.mqd_addr = queue->mqd_gpu_addr;
652 queue_input.wptr_addr = qprops->wptr_gpu_addr;
653 queue_input.wptr_mc_addr = qprops->wptr_mc_addr;
654 queue_input.queue_type = qprops->queue_type;
655 queue_input.paging = qprops->paging;
656 queue_input.is_kfd_process = 0;
657
658 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
659 if (r) {
660 DRM_ERROR("failed to add hardware queue to MES, doorbell=0x%llx\n",
661 qprops->doorbell_off);
662 goto clean_up_doorbell;
663 }
664
665 DRM_DEBUG("MES hw queue was added, pasid=%d, gang id=%d, "
666 "queue type=%d, doorbell=0x%llx\n",
667 gang->process->pasid, gang_id, qprops->queue_type,
668 qprops->doorbell_off);
669
670 queue->ring = qprops->ring;
671 queue->doorbell_off = qprops->doorbell_off;
672 queue->wptr_gpu_addr = qprops->wptr_gpu_addr;
673 queue->queue_type = qprops->queue_type;
674 queue->paging = qprops->paging;
675 queue->gang = gang;
676 queue->ring->mqd_ptr = queue->mqd_cpu_ptr;
677 list_add_tail(&queue->list, &gang->queue_list);
678
679 amdgpu_mes_unlock(&adev->mes);
680 return 0;
681
682 clean_up_doorbell:
683 amdgpu_mes_kernel_doorbell_free(adev, gang->process,
684 qprops->doorbell_off);
685 clean_up_queue_id:
686 spin_lock_irqsave(&adev->mes.queue_id_lock, flags);
687 idr_remove(&adev->mes.queue_id_idr, queue->queue_id);
688 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);
689 clean_up_mqd:
690 amdgpu_mes_unlock(&adev->mes);
691 amdgpu_mes_queue_free_mqd(queue);
692 clean_up_memory:
693 kfree(queue);
694 return r;
695 }
696
amdgpu_mes_remove_hw_queue(struct amdgpu_device * adev,int queue_id)697 int amdgpu_mes_remove_hw_queue(struct amdgpu_device *adev, int queue_id)
698 {
699 unsigned long flags;
700 struct amdgpu_mes_queue *queue;
701 struct amdgpu_mes_gang *gang;
702 struct mes_remove_queue_input queue_input;
703 int r;
704
705 /*
706 * Avoid taking any other locks under MES lock to avoid circular
707 * lock dependencies.
708 */
709 amdgpu_mes_lock(&adev->mes);
710
711 /* remove the mes gang from idr list */
712 spin_lock_irqsave(&adev->mes.queue_id_lock, flags);
713
714 queue = idr_find(&adev->mes.queue_id_idr, queue_id);
715 if (!queue) {
716 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);
717 amdgpu_mes_unlock(&adev->mes);
718 DRM_ERROR("queue id %d doesn't exist\n", queue_id);
719 return -EINVAL;
720 }
721
722 idr_remove(&adev->mes.queue_id_idr, queue_id);
723 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);
724
725 DRM_DEBUG("try to remove queue, doorbell off = 0x%llx\n",
726 queue->doorbell_off);
727
728 gang = queue->gang;
729 queue_input.doorbell_offset = queue->doorbell_off;
730 queue_input.gang_context_addr = gang->gang_ctx_gpu_addr;
731
732 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);
733 if (r)
734 DRM_ERROR("failed to remove hardware queue, queue id = %d\n",
735 queue_id);
736
737 list_del(&queue->list);
738 amdgpu_mes_kernel_doorbell_free(adev, gang->process,
739 queue->doorbell_off);
740 amdgpu_mes_unlock(&adev->mes);
741
742 amdgpu_mes_queue_free_mqd(queue);
743 kfree(queue);
744 return 0;
745 }
746
amdgpu_mes_unmap_legacy_queue(struct amdgpu_device * adev,struct amdgpu_ring * ring,enum amdgpu_unmap_queues_action action,u64 gpu_addr,u64 seq)747 int amdgpu_mes_unmap_legacy_queue(struct amdgpu_device *adev,
748 struct amdgpu_ring *ring,
749 enum amdgpu_unmap_queues_action action,
750 u64 gpu_addr, u64 seq)
751 {
752 struct mes_unmap_legacy_queue_input queue_input;
753 int r;
754
755 queue_input.action = action;
756 queue_input.queue_type = ring->funcs->type;
757 queue_input.doorbell_offset = ring->doorbell_index;
758 queue_input.pipe_id = ring->pipe;
759 queue_input.queue_id = ring->queue;
760 queue_input.trail_fence_addr = gpu_addr;
761 queue_input.trail_fence_data = seq;
762
763 r = adev->mes.funcs->unmap_legacy_queue(&adev->mes, &queue_input);
764 if (r)
765 DRM_ERROR("failed to unmap legacy queue\n");
766
767 return r;
768 }
769
amdgpu_mes_rreg(struct amdgpu_device * adev,uint32_t reg)770 uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg)
771 {
772 struct mes_misc_op_input op_input;
773 int r, val = 0;
774
775 op_input.op = MES_MISC_OP_READ_REG;
776 op_input.read_reg.reg_offset = reg;
777 op_input.read_reg.buffer_addr = adev->mes.read_val_gpu_addr;
778
779 if (!adev->mes.funcs->misc_op) {
780 DRM_ERROR("mes rreg is not supported!\n");
781 goto error;
782 }
783
784 r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
785 if (r)
786 DRM_ERROR("failed to read reg (0x%x)\n", reg);
787 else
788 val = *(adev->mes.read_val_ptr);
789
790 error:
791 return val;
792 }
793
amdgpu_mes_wreg(struct amdgpu_device * adev,uint32_t reg,uint32_t val)794 int amdgpu_mes_wreg(struct amdgpu_device *adev,
795 uint32_t reg, uint32_t val)
796 {
797 struct mes_misc_op_input op_input;
798 int r;
799
800 op_input.op = MES_MISC_OP_WRITE_REG;
801 op_input.write_reg.reg_offset = reg;
802 op_input.write_reg.reg_value = val;
803
804 if (!adev->mes.funcs->misc_op) {
805 DRM_ERROR("mes wreg is not supported!\n");
806 r = -EINVAL;
807 goto error;
808 }
809
810 r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
811 if (r)
812 DRM_ERROR("failed to write reg (0x%x)\n", reg);
813
814 error:
815 return r;
816 }
817
amdgpu_mes_reg_write_reg_wait(struct amdgpu_device * adev,uint32_t reg0,uint32_t reg1,uint32_t ref,uint32_t mask)818 int amdgpu_mes_reg_write_reg_wait(struct amdgpu_device *adev,
819 uint32_t reg0, uint32_t reg1,
820 uint32_t ref, uint32_t mask)
821 {
822 struct mes_misc_op_input op_input;
823 int r;
824
825 op_input.op = MES_MISC_OP_WRM_REG_WR_WAIT;
826 op_input.wrm_reg.reg0 = reg0;
827 op_input.wrm_reg.reg1 = reg1;
828 op_input.wrm_reg.ref = ref;
829 op_input.wrm_reg.mask = mask;
830
831 if (!adev->mes.funcs->misc_op) {
832 DRM_ERROR("mes reg_write_reg_wait is not supported!\n");
833 r = -EINVAL;
834 goto error;
835 }
836
837 r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
838 if (r)
839 DRM_ERROR("failed to reg_write_reg_wait\n");
840
841 error:
842 return r;
843 }
844
amdgpu_mes_reg_wait(struct amdgpu_device * adev,uint32_t reg,uint32_t val,uint32_t mask)845 int amdgpu_mes_reg_wait(struct amdgpu_device *adev, uint32_t reg,
846 uint32_t val, uint32_t mask)
847 {
848 struct mes_misc_op_input op_input;
849 int r;
850
851 op_input.op = MES_MISC_OP_WRM_REG_WAIT;
852 op_input.wrm_reg.reg0 = reg;
853 op_input.wrm_reg.ref = val;
854 op_input.wrm_reg.mask = mask;
855
856 if (!adev->mes.funcs->misc_op) {
857 DRM_ERROR("mes reg wait is not supported!\n");
858 r = -EINVAL;
859 goto error;
860 }
861
862 r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
863 if (r)
864 DRM_ERROR("failed to reg_write_reg_wait\n");
865
866 error:
867 return r;
868 }
869
amdgpu_mes_set_shader_debugger(struct amdgpu_device * adev,uint64_t process_context_addr,uint32_t spi_gdbg_per_vmid_cntl,const uint32_t * tcp_watch_cntl,uint32_t flags,bool trap_en)870 int amdgpu_mes_set_shader_debugger(struct amdgpu_device *adev,
871 uint64_t process_context_addr,
872 uint32_t spi_gdbg_per_vmid_cntl,
873 const uint32_t *tcp_watch_cntl,
874 uint32_t flags,
875 bool trap_en)
876 {
877 struct mes_misc_op_input op_input = {0};
878 int r;
879
880 if (!adev->mes.funcs->misc_op) {
881 DRM_ERROR("mes set shader debugger is not supported!\n");
882 return -EINVAL;
883 }
884
885 op_input.op = MES_MISC_OP_SET_SHADER_DEBUGGER;
886 op_input.set_shader_debugger.process_context_addr = process_context_addr;
887 op_input.set_shader_debugger.flags.u32all = flags;
888
889 /* use amdgpu mes_flush_shader_debugger instead */
890 if (op_input.set_shader_debugger.flags.process_ctx_flush)
891 return -EINVAL;
892
893 op_input.set_shader_debugger.spi_gdbg_per_vmid_cntl = spi_gdbg_per_vmid_cntl;
894 memcpy(op_input.set_shader_debugger.tcp_watch_cntl, tcp_watch_cntl,
895 sizeof(op_input.set_shader_debugger.tcp_watch_cntl));
896
897 if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >>
898 AMDGPU_MES_API_VERSION_SHIFT) >= 14)
899 op_input.set_shader_debugger.trap_en = trap_en;
900
901 amdgpu_mes_lock(&adev->mes);
902
903 r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
904 if (r)
905 DRM_ERROR("failed to set_shader_debugger\n");
906
907 amdgpu_mes_unlock(&adev->mes);
908
909 return r;
910 }
911
amdgpu_mes_flush_shader_debugger(struct amdgpu_device * adev,uint64_t process_context_addr)912 int amdgpu_mes_flush_shader_debugger(struct amdgpu_device *adev,
913 uint64_t process_context_addr)
914 {
915 struct mes_misc_op_input op_input = {0};
916 int r;
917
918 if (!adev->mes.funcs->misc_op) {
919 DRM_ERROR("mes flush shader debugger is not supported!\n");
920 return -EINVAL;
921 }
922
923 op_input.op = MES_MISC_OP_SET_SHADER_DEBUGGER;
924 op_input.set_shader_debugger.process_context_addr = process_context_addr;
925 op_input.set_shader_debugger.flags.process_ctx_flush = true;
926
927 amdgpu_mes_lock(&adev->mes);
928
929 r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
930 if (r)
931 DRM_ERROR("failed to set_shader_debugger\n");
932
933 amdgpu_mes_unlock(&adev->mes);
934
935 return r;
936 }
937
938 static void
amdgpu_mes_ring_to_queue_props(struct amdgpu_device * adev,struct amdgpu_ring * ring,struct amdgpu_mes_queue_properties * props)939 amdgpu_mes_ring_to_queue_props(struct amdgpu_device *adev,
940 struct amdgpu_ring *ring,
941 struct amdgpu_mes_queue_properties *props)
942 {
943 props->queue_type = ring->funcs->type;
944 props->hqd_base_gpu_addr = ring->gpu_addr;
945 props->rptr_gpu_addr = ring->rptr_gpu_addr;
946 props->wptr_gpu_addr = ring->wptr_gpu_addr;
947 props->wptr_mc_addr =
948 ring->mes_ctx->meta_data_mc_addr + ring->wptr_offs;
949 props->queue_size = ring->ring_size;
950 props->eop_gpu_addr = ring->eop_gpu_addr;
951 props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL;
952 props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM;
953 props->paging = false;
954 props->ring = ring;
955 }
956
957 #define DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(_eng) \
958 do { \
959 if (id_offs < AMDGPU_MES_CTX_MAX_OFFS) \
960 return offsetof(struct amdgpu_mes_ctx_meta_data, \
961 _eng[ring->idx].slots[id_offs]); \
962 else if (id_offs == AMDGPU_MES_CTX_RING_OFFS) \
963 return offsetof(struct amdgpu_mes_ctx_meta_data, \
964 _eng[ring->idx].ring); \
965 else if (id_offs == AMDGPU_MES_CTX_IB_OFFS) \
966 return offsetof(struct amdgpu_mes_ctx_meta_data, \
967 _eng[ring->idx].ib); \
968 else if (id_offs == AMDGPU_MES_CTX_PADDING_OFFS) \
969 return offsetof(struct amdgpu_mes_ctx_meta_data, \
970 _eng[ring->idx].padding); \
971 } while(0)
972
amdgpu_mes_ctx_get_offs(struct amdgpu_ring * ring,unsigned int id_offs)973 int amdgpu_mes_ctx_get_offs(struct amdgpu_ring *ring, unsigned int id_offs)
974 {
975 switch (ring->funcs->type) {
976 case AMDGPU_RING_TYPE_GFX:
977 DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(gfx);
978 break;
979 case AMDGPU_RING_TYPE_COMPUTE:
980 DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(compute);
981 break;
982 case AMDGPU_RING_TYPE_SDMA:
983 DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(sdma);
984 break;
985 default:
986 break;
987 }
988
989 WARN_ON(1);
990 return -EINVAL;
991 }
992
amdgpu_mes_add_ring(struct amdgpu_device * adev,int gang_id,int queue_type,int idx,struct amdgpu_mes_ctx_data * ctx_data,struct amdgpu_ring ** out)993 int amdgpu_mes_add_ring(struct amdgpu_device *adev, int gang_id,
994 int queue_type, int idx,
995 struct amdgpu_mes_ctx_data *ctx_data,
996 struct amdgpu_ring **out)
997 {
998 struct amdgpu_ring *ring;
999 struct amdgpu_mes_gang *gang;
1000 struct amdgpu_mes_queue_properties qprops = {0};
1001 int r, queue_id, pasid;
1002
1003 /*
1004 * Avoid taking any other locks under MES lock to avoid circular
1005 * lock dependencies.
1006 */
1007 amdgpu_mes_lock(&adev->mes);
1008 gang = idr_find(&adev->mes.gang_id_idr, gang_id);
1009 if (!gang) {
1010 DRM_ERROR("gang id %d doesn't exist\n", gang_id);
1011 amdgpu_mes_unlock(&adev->mes);
1012 return -EINVAL;
1013 }
1014 pasid = gang->process->pasid;
1015
1016 ring = kzalloc(sizeof(struct amdgpu_ring), GFP_KERNEL);
1017 if (!ring) {
1018 amdgpu_mes_unlock(&adev->mes);
1019 return -ENOMEM;
1020 }
1021
1022 ring->ring_obj = NULL;
1023 ring->use_doorbell = true;
1024 ring->is_mes_queue = true;
1025 ring->mes_ctx = ctx_data;
1026 ring->idx = idx;
1027 ring->no_scheduler = true;
1028
1029 if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
1030 int offset = offsetof(struct amdgpu_mes_ctx_meta_data,
1031 compute[ring->idx].mec_hpd);
1032 ring->eop_gpu_addr =
1033 amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
1034 }
1035
1036 switch (queue_type) {
1037 case AMDGPU_RING_TYPE_GFX:
1038 ring->funcs = adev->gfx.gfx_ring[0].funcs;
1039 ring->me = adev->gfx.gfx_ring[0].me;
1040 ring->pipe = adev->gfx.gfx_ring[0].pipe;
1041 break;
1042 case AMDGPU_RING_TYPE_COMPUTE:
1043 ring->funcs = adev->gfx.compute_ring[0].funcs;
1044 ring->me = adev->gfx.compute_ring[0].me;
1045 ring->pipe = adev->gfx.compute_ring[0].pipe;
1046 break;
1047 case AMDGPU_RING_TYPE_SDMA:
1048 ring->funcs = adev->sdma.instance[0].ring.funcs;
1049 break;
1050 default:
1051 BUG();
1052 }
1053
1054 r = amdgpu_ring_init(adev, ring, 1024, NULL, 0,
1055 AMDGPU_RING_PRIO_DEFAULT, NULL);
1056 if (r)
1057 goto clean_up_memory;
1058
1059 amdgpu_mes_ring_to_queue_props(adev, ring, &qprops);
1060
1061 dma_fence_wait(gang->process->vm->last_update, false);
1062 dma_fence_wait(ctx_data->meta_data_va->last_pt_update, false);
1063 amdgpu_mes_unlock(&adev->mes);
1064
1065 r = amdgpu_mes_add_hw_queue(adev, gang_id, &qprops, &queue_id);
1066 if (r)
1067 goto clean_up_ring;
1068
1069 ring->hw_queue_id = queue_id;
1070 ring->doorbell_index = qprops.doorbell_off;
1071
1072 if (queue_type == AMDGPU_RING_TYPE_GFX)
1073 sprintf(ring->name, "gfx_%d.%d.%d", pasid, gang_id, queue_id);
1074 else if (queue_type == AMDGPU_RING_TYPE_COMPUTE)
1075 sprintf(ring->name, "compute_%d.%d.%d", pasid, gang_id,
1076 queue_id);
1077 else if (queue_type == AMDGPU_RING_TYPE_SDMA)
1078 sprintf(ring->name, "sdma_%d.%d.%d", pasid, gang_id,
1079 queue_id);
1080 else
1081 BUG();
1082
1083 *out = ring;
1084 return 0;
1085
1086 clean_up_ring:
1087 amdgpu_ring_fini(ring);
1088 clean_up_memory:
1089 kfree(ring);
1090 amdgpu_mes_unlock(&adev->mes);
1091 return r;
1092 }
1093
amdgpu_mes_remove_ring(struct amdgpu_device * adev,struct amdgpu_ring * ring)1094 void amdgpu_mes_remove_ring(struct amdgpu_device *adev,
1095 struct amdgpu_ring *ring)
1096 {
1097 if (!ring)
1098 return;
1099
1100 amdgpu_mes_remove_hw_queue(adev, ring->hw_queue_id);
1101 del_timer_sync(&ring->fence_drv.fallback_timer);
1102 amdgpu_ring_fini(ring);
1103 kfree(ring);
1104 }
1105
amdgpu_mes_get_aggregated_doorbell_index(struct amdgpu_device * adev,enum amdgpu_mes_priority_level prio)1106 uint32_t amdgpu_mes_get_aggregated_doorbell_index(struct amdgpu_device *adev,
1107 enum amdgpu_mes_priority_level prio)
1108 {
1109 return adev->mes.aggregated_doorbells[prio];
1110 }
1111
amdgpu_mes_ctx_alloc_meta_data(struct amdgpu_device * adev,struct amdgpu_mes_ctx_data * ctx_data)1112 int amdgpu_mes_ctx_alloc_meta_data(struct amdgpu_device *adev,
1113 struct amdgpu_mes_ctx_data *ctx_data)
1114 {
1115 int r;
1116
1117 r = amdgpu_bo_create_kernel(adev,
1118 sizeof(struct amdgpu_mes_ctx_meta_data),
1119 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1120 &ctx_data->meta_data_obj,
1121 &ctx_data->meta_data_mc_addr,
1122 &ctx_data->meta_data_ptr);
1123 if (r) {
1124 dev_warn(adev->dev, "(%d) create CTX bo failed\n", r);
1125 return r;
1126 }
1127
1128 if (!ctx_data->meta_data_obj)
1129 return -ENOMEM;
1130
1131 memset(ctx_data->meta_data_ptr, 0,
1132 sizeof(struct amdgpu_mes_ctx_meta_data));
1133
1134 return 0;
1135 }
1136
amdgpu_mes_ctx_free_meta_data(struct amdgpu_mes_ctx_data * ctx_data)1137 void amdgpu_mes_ctx_free_meta_data(struct amdgpu_mes_ctx_data *ctx_data)
1138 {
1139 if (ctx_data->meta_data_obj)
1140 amdgpu_bo_free_kernel(&ctx_data->meta_data_obj,
1141 &ctx_data->meta_data_mc_addr,
1142 &ctx_data->meta_data_ptr);
1143 }
1144
amdgpu_mes_ctx_map_meta_data(struct amdgpu_device * adev,struct amdgpu_vm * vm,struct amdgpu_mes_ctx_data * ctx_data)1145 int amdgpu_mes_ctx_map_meta_data(struct amdgpu_device *adev,
1146 struct amdgpu_vm *vm,
1147 struct amdgpu_mes_ctx_data *ctx_data)
1148 {
1149 struct amdgpu_bo_va *bo_va;
1150 struct amdgpu_sync sync;
1151 struct drm_exec exec;
1152 int r;
1153
1154 amdgpu_sync_create(&sync);
1155
1156 drm_exec_init(&exec, 0);
1157 drm_exec_until_all_locked(&exec) {
1158 r = drm_exec_lock_obj(&exec,
1159 &ctx_data->meta_data_obj->tbo.base);
1160 drm_exec_retry_on_contention(&exec);
1161 if (unlikely(r))
1162 goto error_fini_exec;
1163
1164 r = amdgpu_vm_lock_pd(vm, &exec, 0);
1165 drm_exec_retry_on_contention(&exec);
1166 if (unlikely(r))
1167 goto error_fini_exec;
1168 }
1169
1170 bo_va = amdgpu_vm_bo_add(adev, vm, ctx_data->meta_data_obj);
1171 if (!bo_va) {
1172 DRM_ERROR("failed to create bo_va for meta data BO\n");
1173 r = -ENOMEM;
1174 goto error_fini_exec;
1175 }
1176
1177 r = amdgpu_vm_bo_map(adev, bo_va, ctx_data->meta_data_gpu_addr, 0,
1178 sizeof(struct amdgpu_mes_ctx_meta_data),
1179 AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE |
1180 AMDGPU_PTE_EXECUTABLE);
1181
1182 if (r) {
1183 DRM_ERROR("failed to do bo_map on meta data, err=%d\n", r);
1184 goto error_del_bo_va;
1185 }
1186
1187 r = amdgpu_vm_bo_update(adev, bo_va, false);
1188 if (r) {
1189 DRM_ERROR("failed to do vm_bo_update on meta data\n");
1190 goto error_del_bo_va;
1191 }
1192 amdgpu_sync_fence(&sync, bo_va->last_pt_update);
1193
1194 r = amdgpu_vm_update_pdes(adev, vm, false);
1195 if (r) {
1196 DRM_ERROR("failed to update pdes on meta data\n");
1197 goto error_del_bo_va;
1198 }
1199 amdgpu_sync_fence(&sync, vm->last_update);
1200
1201 amdgpu_sync_wait(&sync, false);
1202 drm_exec_fini(&exec);
1203
1204 amdgpu_sync_free(&sync);
1205 ctx_data->meta_data_va = bo_va;
1206 return 0;
1207
1208 error_del_bo_va:
1209 amdgpu_vm_bo_del(adev, bo_va);
1210
1211 error_fini_exec:
1212 drm_exec_fini(&exec);
1213 amdgpu_sync_free(&sync);
1214 return r;
1215 }
1216
amdgpu_mes_ctx_unmap_meta_data(struct amdgpu_device * adev,struct amdgpu_mes_ctx_data * ctx_data)1217 int amdgpu_mes_ctx_unmap_meta_data(struct amdgpu_device *adev,
1218 struct amdgpu_mes_ctx_data *ctx_data)
1219 {
1220 struct amdgpu_bo_va *bo_va = ctx_data->meta_data_va;
1221 struct amdgpu_bo *bo = ctx_data->meta_data_obj;
1222 struct amdgpu_vm *vm = bo_va->base.vm;
1223 struct dma_fence *fence;
1224 struct drm_exec exec;
1225 long r;
1226
1227 drm_exec_init(&exec, 0);
1228 drm_exec_until_all_locked(&exec) {
1229 r = drm_exec_lock_obj(&exec,
1230 &ctx_data->meta_data_obj->tbo.base);
1231 drm_exec_retry_on_contention(&exec);
1232 if (unlikely(r))
1233 goto out_unlock;
1234
1235 r = amdgpu_vm_lock_pd(vm, &exec, 0);
1236 drm_exec_retry_on_contention(&exec);
1237 if (unlikely(r))
1238 goto out_unlock;
1239 }
1240
1241 amdgpu_vm_bo_del(adev, bo_va);
1242 if (!amdgpu_vm_ready(vm))
1243 goto out_unlock;
1244
1245 r = dma_resv_get_singleton(bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP,
1246 &fence);
1247 if (r)
1248 goto out_unlock;
1249 if (fence) {
1250 amdgpu_bo_fence(bo, fence, true);
1251 fence = NULL;
1252 }
1253
1254 r = amdgpu_vm_clear_freed(adev, vm, &fence);
1255 if (r || !fence)
1256 goto out_unlock;
1257
1258 dma_fence_wait(fence, false);
1259 amdgpu_bo_fence(bo, fence, true);
1260 dma_fence_put(fence);
1261
1262 out_unlock:
1263 if (unlikely(r < 0))
1264 dev_err(adev->dev, "failed to clear page tables (%ld)\n", r);
1265 drm_exec_fini(&exec);
1266
1267 return r;
1268 }
1269
amdgpu_mes_test_create_gang_and_queues(struct amdgpu_device * adev,int pasid,int * gang_id,int queue_type,int num_queue,struct amdgpu_ring ** added_rings,struct amdgpu_mes_ctx_data * ctx_data)1270 static int amdgpu_mes_test_create_gang_and_queues(struct amdgpu_device *adev,
1271 int pasid, int *gang_id,
1272 int queue_type, int num_queue,
1273 struct amdgpu_ring **added_rings,
1274 struct amdgpu_mes_ctx_data *ctx_data)
1275 {
1276 struct amdgpu_ring *ring;
1277 struct amdgpu_mes_gang_properties gprops = {0};
1278 int r, j;
1279
1280 /* create a gang for the process */
1281 gprops.priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
1282 gprops.gang_quantum = adev->mes.default_gang_quantum;
1283 gprops.inprocess_gang_priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
1284 gprops.priority_level = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
1285 gprops.global_priority_level = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
1286
1287 r = amdgpu_mes_add_gang(adev, pasid, &gprops, gang_id);
1288 if (r) {
1289 DRM_ERROR("failed to add gang\n");
1290 return r;
1291 }
1292
1293 /* create queues for the gang */
1294 for (j = 0; j < num_queue; j++) {
1295 r = amdgpu_mes_add_ring(adev, *gang_id, queue_type, j,
1296 ctx_data, &ring);
1297 if (r) {
1298 DRM_ERROR("failed to add ring\n");
1299 break;
1300 }
1301
1302 DRM_INFO("ring %s was added\n", ring->name);
1303 added_rings[j] = ring;
1304 }
1305
1306 return 0;
1307 }
1308
amdgpu_mes_test_queues(struct amdgpu_ring ** added_rings)1309 static int amdgpu_mes_test_queues(struct amdgpu_ring **added_rings)
1310 {
1311 struct amdgpu_ring *ring;
1312 int i, r;
1313
1314 for (i = 0; i < AMDGPU_MES_CTX_MAX_RINGS; i++) {
1315 ring = added_rings[i];
1316 if (!ring)
1317 continue;
1318
1319 r = amdgpu_ring_test_helper(ring);
1320 if (r)
1321 return r;
1322
1323 r = amdgpu_ring_test_ib(ring, 1000 * 10);
1324 if (r) {
1325 DRM_DEV_ERROR(ring->adev->dev,
1326 "ring %s ib test failed (%d)\n",
1327 ring->name, r);
1328 return r;
1329 } else
1330 DRM_INFO("ring %s ib test pass\n", ring->name);
1331 }
1332
1333 return 0;
1334 }
1335
amdgpu_mes_self_test(struct amdgpu_device * adev)1336 int amdgpu_mes_self_test(struct amdgpu_device *adev)
1337 {
1338 struct amdgpu_vm *vm = NULL;
1339 struct amdgpu_mes_ctx_data ctx_data = {0};
1340 struct amdgpu_ring *added_rings[AMDGPU_MES_CTX_MAX_RINGS] = { NULL };
1341 int gang_ids[3] = {0};
1342 int queue_types[][2] = { { AMDGPU_RING_TYPE_GFX, 1 },
1343 { AMDGPU_RING_TYPE_COMPUTE, 1 },
1344 { AMDGPU_RING_TYPE_SDMA, 1} };
1345 int i, r, pasid, k = 0;
1346
1347 pasid = amdgpu_pasid_alloc(16);
1348 if (pasid < 0) {
1349 dev_warn(adev->dev, "No more PASIDs available!");
1350 pasid = 0;
1351 }
1352
1353 vm = kzalloc(sizeof(*vm), GFP_KERNEL);
1354 if (!vm) {
1355 r = -ENOMEM;
1356 goto error_pasid;
1357 }
1358
1359 r = amdgpu_vm_init(adev, vm, -1);
1360 if (r) {
1361 DRM_ERROR("failed to initialize vm\n");
1362 goto error_pasid;
1363 }
1364
1365 r = amdgpu_mes_ctx_alloc_meta_data(adev, &ctx_data);
1366 if (r) {
1367 DRM_ERROR("failed to alloc ctx meta data\n");
1368 goto error_fini;
1369 }
1370
1371 ctx_data.meta_data_gpu_addr = AMDGPU_VA_RESERVED_SIZE;
1372 r = amdgpu_mes_ctx_map_meta_data(adev, vm, &ctx_data);
1373 if (r) {
1374 DRM_ERROR("failed to map ctx meta data\n");
1375 goto error_vm;
1376 }
1377
1378 r = amdgpu_mes_create_process(adev, pasid, vm);
1379 if (r) {
1380 DRM_ERROR("failed to create MES process\n");
1381 goto error_vm;
1382 }
1383
1384 for (i = 0; i < ARRAY_SIZE(queue_types); i++) {
1385 /* On GFX v10.3, fw hasn't supported to map sdma queue. */
1386 if (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(10, 3, 0) &&
1387 adev->ip_versions[GC_HWIP][0] < IP_VERSION(11, 0, 0) &&
1388 queue_types[i][0] == AMDGPU_RING_TYPE_SDMA)
1389 continue;
1390
1391 r = amdgpu_mes_test_create_gang_and_queues(adev, pasid,
1392 &gang_ids[i],
1393 queue_types[i][0],
1394 queue_types[i][1],
1395 &added_rings[k],
1396 &ctx_data);
1397 if (r)
1398 goto error_queues;
1399
1400 k += queue_types[i][1];
1401 }
1402
1403 /* start ring test and ib test for MES queues */
1404 amdgpu_mes_test_queues(added_rings);
1405
1406 error_queues:
1407 /* remove all queues */
1408 for (i = 0; i < ARRAY_SIZE(added_rings); i++) {
1409 if (!added_rings[i])
1410 continue;
1411 amdgpu_mes_remove_ring(adev, added_rings[i]);
1412 }
1413
1414 for (i = 0; i < ARRAY_SIZE(gang_ids); i++) {
1415 if (!gang_ids[i])
1416 continue;
1417 amdgpu_mes_remove_gang(adev, gang_ids[i]);
1418 }
1419
1420 amdgpu_mes_destroy_process(adev, pasid);
1421
1422 error_vm:
1423 amdgpu_mes_ctx_unmap_meta_data(adev, &ctx_data);
1424
1425 error_fini:
1426 amdgpu_vm_fini(adev, vm);
1427
1428 error_pasid:
1429 if (pasid)
1430 amdgpu_pasid_free(pasid);
1431
1432 amdgpu_mes_ctx_free_meta_data(&ctx_data);
1433 kfree(vm);
1434 return 0;
1435 }
1436
amdgpu_mes_init_microcode(struct amdgpu_device * adev,int pipe)1437 int amdgpu_mes_init_microcode(struct amdgpu_device *adev, int pipe)
1438 {
1439 const struct mes_firmware_header_v1_0 *mes_hdr;
1440 struct amdgpu_firmware_info *info;
1441 char ucode_prefix[30];
1442 char fw_name[40];
1443 bool need_retry = false;
1444 int r;
1445
1446 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix,
1447 sizeof(ucode_prefix));
1448 if (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(11, 0, 0)) {
1449 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes%s.bin",
1450 ucode_prefix,
1451 pipe == AMDGPU_MES_SCHED_PIPE ? "_2" : "1");
1452 need_retry = true;
1453 } else {
1454 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes%s.bin",
1455 ucode_prefix,
1456 pipe == AMDGPU_MES_SCHED_PIPE ? "" : "1");
1457 }
1458
1459 r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe], fw_name);
1460 if (r && need_retry && pipe == AMDGPU_MES_SCHED_PIPE) {
1461 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes.bin",
1462 ucode_prefix);
1463 DRM_INFO("try to fall back to %s\n", fw_name);
1464 r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe],
1465 fw_name);
1466 }
1467
1468 if (r)
1469 goto out;
1470
1471 mes_hdr = (const struct mes_firmware_header_v1_0 *)
1472 adev->mes.fw[pipe]->data;
1473 adev->mes.uc_start_addr[pipe] =
1474 le32_to_cpu(mes_hdr->mes_uc_start_addr_lo) |
1475 ((uint64_t)(le32_to_cpu(mes_hdr->mes_uc_start_addr_hi)) << 32);
1476 adev->mes.data_start_addr[pipe] =
1477 le32_to_cpu(mes_hdr->mes_data_start_addr_lo) |
1478 ((uint64_t)(le32_to_cpu(mes_hdr->mes_data_start_addr_hi)) << 32);
1479
1480 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1481 int ucode, ucode_data;
1482
1483 if (pipe == AMDGPU_MES_SCHED_PIPE) {
1484 ucode = AMDGPU_UCODE_ID_CP_MES;
1485 ucode_data = AMDGPU_UCODE_ID_CP_MES_DATA;
1486 } else {
1487 ucode = AMDGPU_UCODE_ID_CP_MES1;
1488 ucode_data = AMDGPU_UCODE_ID_CP_MES1_DATA;
1489 }
1490
1491 info = &adev->firmware.ucode[ucode];
1492 info->ucode_id = ucode;
1493 info->fw = adev->mes.fw[pipe];
1494 adev->firmware.fw_size +=
1495 ALIGN(le32_to_cpu(mes_hdr->mes_ucode_size_bytes),
1496 PAGE_SIZE);
1497
1498 info = &adev->firmware.ucode[ucode_data];
1499 info->ucode_id = ucode_data;
1500 info->fw = adev->mes.fw[pipe];
1501 adev->firmware.fw_size +=
1502 ALIGN(le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes),
1503 PAGE_SIZE);
1504 }
1505
1506 return 0;
1507 out:
1508 amdgpu_ucode_release(&adev->mes.fw[pipe]);
1509 return r;
1510 }
1511