1 /*
2 * Copyright 2019 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/firmware.h>
25 #include <linux/module.h>
26 #include "amdgpu.h"
27 #include "soc15_common.h"
28 #include "soc21.h"
29 #include "gc/gc_11_0_0_offset.h"
30 #include "gc/gc_11_0_0_sh_mask.h"
31 #include "gc/gc_11_0_0_default.h"
32 #include "v11_structs.h"
33 #include "mes_v11_api_def.h"
34
35 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes.bin");
36 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes_2.bin");
37 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes1.bin");
38 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes.bin");
39 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes_2.bin");
40 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes1.bin");
41 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes.bin");
42 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes_2.bin");
43 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes1.bin");
44 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes.bin");
45 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes_2.bin");
46 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes1.bin");
47 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mes.bin");
48 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mes_2.bin");
49 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mes1.bin");
50
51 static int mes_v11_0_hw_fini(void *handle);
52 static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev);
53 static int mes_v11_0_kiq_hw_fini(struct amdgpu_device *adev);
54
55 #define MES_EOP_SIZE 2048
56
mes_v11_0_ring_set_wptr(struct amdgpu_ring * ring)57 static void mes_v11_0_ring_set_wptr(struct amdgpu_ring *ring)
58 {
59 struct amdgpu_device *adev = ring->adev;
60
61 if (ring->use_doorbell) {
62 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
63 ring->wptr);
64 WDOORBELL64(ring->doorbell_index, ring->wptr);
65 } else {
66 BUG();
67 }
68 }
69
mes_v11_0_ring_get_rptr(struct amdgpu_ring * ring)70 static u64 mes_v11_0_ring_get_rptr(struct amdgpu_ring *ring)
71 {
72 return *ring->rptr_cpu_addr;
73 }
74
mes_v11_0_ring_get_wptr(struct amdgpu_ring * ring)75 static u64 mes_v11_0_ring_get_wptr(struct amdgpu_ring *ring)
76 {
77 u64 wptr;
78
79 if (ring->use_doorbell)
80 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
81 else
82 BUG();
83 return wptr;
84 }
85
86 static const struct amdgpu_ring_funcs mes_v11_0_ring_funcs = {
87 .type = AMDGPU_RING_TYPE_MES,
88 .align_mask = 1,
89 .nop = 0,
90 .support_64bit_ptrs = true,
91 .get_rptr = mes_v11_0_ring_get_rptr,
92 .get_wptr = mes_v11_0_ring_get_wptr,
93 .set_wptr = mes_v11_0_ring_set_wptr,
94 .insert_nop = amdgpu_ring_insert_nop,
95 };
96
mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes * mes,void * pkt,int size,int api_status_off)97 static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
98 void *pkt, int size,
99 int api_status_off)
100 {
101 int ndw = size / 4;
102 signed long r;
103 union MESAPI__ADD_QUEUE *x_pkt = pkt;
104 struct MES_API_STATUS *api_status;
105 struct amdgpu_device *adev = mes->adev;
106 struct amdgpu_ring *ring = &mes->ring;
107 unsigned long flags;
108 signed long timeout = adev->usec_timeout;
109
110 if (amdgpu_emu_mode) {
111 timeout *= 100;
112 } else if (amdgpu_sriov_vf(adev)) {
113 /* Worst case in sriov where all other 15 VF timeout, each VF needs about 600ms */
114 timeout = 15 * 600 * 1000;
115 }
116 BUG_ON(size % 4 != 0);
117
118 spin_lock_irqsave(&mes->ring_lock, flags);
119 if (amdgpu_ring_alloc(ring, ndw)) {
120 spin_unlock_irqrestore(&mes->ring_lock, flags);
121 return -ENOMEM;
122 }
123
124 api_status = (struct MES_API_STATUS *)((char *)pkt + api_status_off);
125 api_status->api_completion_fence_addr = mes->ring.fence_drv.gpu_addr;
126 api_status->api_completion_fence_value = ++mes->ring.fence_drv.sync_seq;
127
128 amdgpu_ring_write_multiple(ring, pkt, ndw);
129 amdgpu_ring_commit(ring);
130 spin_unlock_irqrestore(&mes->ring_lock, flags);
131
132 DRM_DEBUG("MES msg=%d was emitted\n", x_pkt->header.opcode);
133
134 r = amdgpu_fence_wait_polling(ring, ring->fence_drv.sync_seq,
135 timeout);
136 if (r < 1) {
137 DRM_ERROR("MES failed to response msg=%d\n",
138 x_pkt->header.opcode);
139 return -ETIMEDOUT;
140 }
141
142 return 0;
143 }
144
convert_to_mes_queue_type(int queue_type)145 static int convert_to_mes_queue_type(int queue_type)
146 {
147 if (queue_type == AMDGPU_RING_TYPE_GFX)
148 return MES_QUEUE_TYPE_GFX;
149 else if (queue_type == AMDGPU_RING_TYPE_COMPUTE)
150 return MES_QUEUE_TYPE_COMPUTE;
151 else if (queue_type == AMDGPU_RING_TYPE_SDMA)
152 return MES_QUEUE_TYPE_SDMA;
153 else
154 BUG();
155 return -1;
156 }
157
mes_v11_0_add_hw_queue(struct amdgpu_mes * mes,struct mes_add_queue_input * input)158 static int mes_v11_0_add_hw_queue(struct amdgpu_mes *mes,
159 struct mes_add_queue_input *input)
160 {
161 struct amdgpu_device *adev = mes->adev;
162 union MESAPI__ADD_QUEUE mes_add_queue_pkt;
163 struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0];
164 uint32_t vm_cntx_cntl = hub->vm_cntx_cntl;
165
166 memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));
167
168 mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
169 mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;
170 mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
171
172 mes_add_queue_pkt.process_id = input->process_id;
173 mes_add_queue_pkt.page_table_base_addr = input->page_table_base_addr;
174 mes_add_queue_pkt.process_va_start = input->process_va_start;
175 mes_add_queue_pkt.process_va_end = input->process_va_end;
176 mes_add_queue_pkt.process_quantum = input->process_quantum;
177 mes_add_queue_pkt.process_context_addr = input->process_context_addr;
178 mes_add_queue_pkt.gang_quantum = input->gang_quantum;
179 mes_add_queue_pkt.gang_context_addr = input->gang_context_addr;
180 mes_add_queue_pkt.inprocess_gang_priority =
181 input->inprocess_gang_priority;
182 mes_add_queue_pkt.gang_global_priority_level =
183 input->gang_global_priority_level;
184 mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;
185 mes_add_queue_pkt.mqd_addr = input->mqd_addr;
186
187 if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >>
188 AMDGPU_MES_API_VERSION_SHIFT) >= 2)
189 mes_add_queue_pkt.wptr_addr = input->wptr_mc_addr;
190 else
191 mes_add_queue_pkt.wptr_addr = input->wptr_addr;
192
193 mes_add_queue_pkt.queue_type =
194 convert_to_mes_queue_type(input->queue_type);
195 mes_add_queue_pkt.paging = input->paging;
196 mes_add_queue_pkt.vm_context_cntl = vm_cntx_cntl;
197 mes_add_queue_pkt.gws_base = input->gws_base;
198 mes_add_queue_pkt.gws_size = input->gws_size;
199 mes_add_queue_pkt.trap_handler_addr = input->tba_addr;
200 mes_add_queue_pkt.tma_addr = input->tma_addr;
201 mes_add_queue_pkt.is_kfd_process = input->is_kfd_process;
202
203 /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
204 mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
205 mes_add_queue_pkt.gds_size = input->queue_size;
206
207 if (!(((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 4) &&
208 (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(11, 0, 0)) &&
209 (adev->ip_versions[GC_HWIP][0] <= IP_VERSION(11, 0, 3))))
210 mes_add_queue_pkt.trap_en = 1;
211
212 /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
213 mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
214 mes_add_queue_pkt.gds_size = input->queue_size;
215
216 return mes_v11_0_submit_pkt_and_poll_completion(mes,
217 &mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
218 offsetof(union MESAPI__ADD_QUEUE, api_status));
219 }
220
mes_v11_0_remove_hw_queue(struct amdgpu_mes * mes,struct mes_remove_queue_input * input)221 static int mes_v11_0_remove_hw_queue(struct amdgpu_mes *mes,
222 struct mes_remove_queue_input *input)
223 {
224 union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
225
226 memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
227
228 mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
229 mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
230 mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
231
232 mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
233 mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr;
234
235 return mes_v11_0_submit_pkt_and_poll_completion(mes,
236 &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
237 offsetof(union MESAPI__REMOVE_QUEUE, api_status));
238 }
239
mes_v11_0_unmap_legacy_queue(struct amdgpu_mes * mes,struct mes_unmap_legacy_queue_input * input)240 static int mes_v11_0_unmap_legacy_queue(struct amdgpu_mes *mes,
241 struct mes_unmap_legacy_queue_input *input)
242 {
243 union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
244
245 memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
246
247 mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
248 mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
249 mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
250
251 mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
252 mes_remove_queue_pkt.gang_context_addr = 0;
253
254 mes_remove_queue_pkt.pipe_id = input->pipe_id;
255 mes_remove_queue_pkt.queue_id = input->queue_id;
256
257 if (input->action == PREEMPT_QUEUES_NO_UNMAP) {
258 mes_remove_queue_pkt.preempt_legacy_gfx_queue = 1;
259 mes_remove_queue_pkt.tf_addr = input->trail_fence_addr;
260 mes_remove_queue_pkt.tf_data =
261 lower_32_bits(input->trail_fence_data);
262 } else {
263 mes_remove_queue_pkt.unmap_legacy_queue = 1;
264 mes_remove_queue_pkt.queue_type =
265 convert_to_mes_queue_type(input->queue_type);
266 }
267
268 return mes_v11_0_submit_pkt_and_poll_completion(mes,
269 &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
270 offsetof(union MESAPI__REMOVE_QUEUE, api_status));
271 }
272
mes_v11_0_suspend_gang(struct amdgpu_mes * mes,struct mes_suspend_gang_input * input)273 static int mes_v11_0_suspend_gang(struct amdgpu_mes *mes,
274 struct mes_suspend_gang_input *input)
275 {
276 return 0;
277 }
278
mes_v11_0_resume_gang(struct amdgpu_mes * mes,struct mes_resume_gang_input * input)279 static int mes_v11_0_resume_gang(struct amdgpu_mes *mes,
280 struct mes_resume_gang_input *input)
281 {
282 return 0;
283 }
284
mes_v11_0_query_sched_status(struct amdgpu_mes * mes)285 static int mes_v11_0_query_sched_status(struct amdgpu_mes *mes)
286 {
287 union MESAPI__QUERY_MES_STATUS mes_status_pkt;
288
289 memset(&mes_status_pkt, 0, sizeof(mes_status_pkt));
290
291 mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;
292 mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;
293 mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
294
295 return mes_v11_0_submit_pkt_and_poll_completion(mes,
296 &mes_status_pkt, sizeof(mes_status_pkt),
297 offsetof(union MESAPI__QUERY_MES_STATUS, api_status));
298 }
299
mes_v11_0_misc_op(struct amdgpu_mes * mes,struct mes_misc_op_input * input)300 static int mes_v11_0_misc_op(struct amdgpu_mes *mes,
301 struct mes_misc_op_input *input)
302 {
303 union MESAPI__MISC misc_pkt;
304
305 memset(&misc_pkt, 0, sizeof(misc_pkt));
306
307 misc_pkt.header.type = MES_API_TYPE_SCHEDULER;
308 misc_pkt.header.opcode = MES_SCH_API_MISC;
309 misc_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
310
311 switch (input->op) {
312 case MES_MISC_OP_READ_REG:
313 misc_pkt.opcode = MESAPI_MISC__READ_REG;
314 misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset;
315 misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr;
316 break;
317 case MES_MISC_OP_WRITE_REG:
318 misc_pkt.opcode = MESAPI_MISC__WRITE_REG;
319 misc_pkt.write_reg.reg_offset = input->write_reg.reg_offset;
320 misc_pkt.write_reg.reg_value = input->write_reg.reg_value;
321 break;
322 case MES_MISC_OP_WRM_REG_WAIT:
323 misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
324 misc_pkt.wait_reg_mem.op = WRM_OPERATION__WAIT_REG_MEM;
325 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
326 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
327 misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
328 misc_pkt.wait_reg_mem.reg_offset2 = 0;
329 break;
330 case MES_MISC_OP_WRM_REG_WR_WAIT:
331 misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
332 misc_pkt.wait_reg_mem.op = WRM_OPERATION__WR_WAIT_WR_REG;
333 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
334 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
335 misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
336 misc_pkt.wait_reg_mem.reg_offset2 = input->wrm_reg.reg1;
337 break;
338 default:
339 DRM_ERROR("unsupported misc op (%d) \n", input->op);
340 return -EINVAL;
341 }
342
343 return mes_v11_0_submit_pkt_and_poll_completion(mes,
344 &misc_pkt, sizeof(misc_pkt),
345 offsetof(union MESAPI__MISC, api_status));
346 }
347
mes_v11_0_set_hw_resources(struct amdgpu_mes * mes)348 static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)
349 {
350 int i;
351 struct amdgpu_device *adev = mes->adev;
352 union MESAPI_SET_HW_RESOURCES mes_set_hw_res_pkt;
353
354 memset(&mes_set_hw_res_pkt, 0, sizeof(mes_set_hw_res_pkt));
355
356 mes_set_hw_res_pkt.header.type = MES_API_TYPE_SCHEDULER;
357 mes_set_hw_res_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC;
358 mes_set_hw_res_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
359
360 mes_set_hw_res_pkt.vmid_mask_mmhub = mes->vmid_mask_mmhub;
361 mes_set_hw_res_pkt.vmid_mask_gfxhub = mes->vmid_mask_gfxhub;
362 mes_set_hw_res_pkt.gds_size = adev->gds.gds_size;
363 mes_set_hw_res_pkt.paging_vmid = 0;
364 mes_set_hw_res_pkt.g_sch_ctx_gpu_mc_ptr = mes->sch_ctx_gpu_addr;
365 mes_set_hw_res_pkt.query_status_fence_gpu_mc_ptr =
366 mes->query_status_fence_gpu_addr;
367
368 for (i = 0; i < MAX_COMPUTE_PIPES; i++)
369 mes_set_hw_res_pkt.compute_hqd_mask[i] =
370 mes->compute_hqd_mask[i];
371
372 for (i = 0; i < MAX_GFX_PIPES; i++)
373 mes_set_hw_res_pkt.gfx_hqd_mask[i] = mes->gfx_hqd_mask[i];
374
375 for (i = 0; i < MAX_SDMA_PIPES; i++)
376 mes_set_hw_res_pkt.sdma_hqd_mask[i] = mes->sdma_hqd_mask[i];
377
378 for (i = 0; i < AMD_PRIORITY_NUM_LEVELS; i++)
379 mes_set_hw_res_pkt.aggregated_doorbells[i] =
380 mes->aggregated_doorbells[i];
381
382 for (i = 0; i < 5; i++) {
383 mes_set_hw_res_pkt.gc_base[i] = adev->reg_offset[GC_HWIP][0][i];
384 mes_set_hw_res_pkt.mmhub_base[i] =
385 adev->reg_offset[MMHUB_HWIP][0][i];
386 mes_set_hw_res_pkt.osssys_base[i] =
387 adev->reg_offset[OSSSYS_HWIP][0][i];
388 }
389
390 mes_set_hw_res_pkt.disable_reset = 1;
391 mes_set_hw_res_pkt.disable_mes_log = 1;
392 mes_set_hw_res_pkt.use_different_vmid_compute = 1;
393 mes_set_hw_res_pkt.enable_reg_active_poll = 1;
394 mes_set_hw_res_pkt.oversubscription_timer = 50;
395
396 return mes_v11_0_submit_pkt_and_poll_completion(mes,
397 &mes_set_hw_res_pkt, sizeof(mes_set_hw_res_pkt),
398 offsetof(union MESAPI_SET_HW_RESOURCES, api_status));
399 }
400
mes_v11_0_init_aggregated_doorbell(struct amdgpu_mes * mes)401 static void mes_v11_0_init_aggregated_doorbell(struct amdgpu_mes *mes)
402 {
403 struct amdgpu_device *adev = mes->adev;
404 uint32_t data;
405
406 data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL1);
407 data &= ~(CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET_MASK |
408 CP_MES_DOORBELL_CONTROL1__DOORBELL_EN_MASK |
409 CP_MES_DOORBELL_CONTROL1__DOORBELL_HIT_MASK);
410 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_LOW] <<
411 CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET__SHIFT;
412 data |= 1 << CP_MES_DOORBELL_CONTROL1__DOORBELL_EN__SHIFT;
413 WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL1, data);
414
415 data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL2);
416 data &= ~(CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET_MASK |
417 CP_MES_DOORBELL_CONTROL2__DOORBELL_EN_MASK |
418 CP_MES_DOORBELL_CONTROL2__DOORBELL_HIT_MASK);
419 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_NORMAL] <<
420 CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET__SHIFT;
421 data |= 1 << CP_MES_DOORBELL_CONTROL2__DOORBELL_EN__SHIFT;
422 WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL2, data);
423
424 data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL3);
425 data &= ~(CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET_MASK |
426 CP_MES_DOORBELL_CONTROL3__DOORBELL_EN_MASK |
427 CP_MES_DOORBELL_CONTROL3__DOORBELL_HIT_MASK);
428 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_MEDIUM] <<
429 CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET__SHIFT;
430 data |= 1 << CP_MES_DOORBELL_CONTROL3__DOORBELL_EN__SHIFT;
431 WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL3, data);
432
433 data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL4);
434 data &= ~(CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET_MASK |
435 CP_MES_DOORBELL_CONTROL4__DOORBELL_EN_MASK |
436 CP_MES_DOORBELL_CONTROL4__DOORBELL_HIT_MASK);
437 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_HIGH] <<
438 CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET__SHIFT;
439 data |= 1 << CP_MES_DOORBELL_CONTROL4__DOORBELL_EN__SHIFT;
440 WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL4, data);
441
442 data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL5);
443 data &= ~(CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET_MASK |
444 CP_MES_DOORBELL_CONTROL5__DOORBELL_EN_MASK |
445 CP_MES_DOORBELL_CONTROL5__DOORBELL_HIT_MASK);
446 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_REALTIME] <<
447 CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET__SHIFT;
448 data |= 1 << CP_MES_DOORBELL_CONTROL5__DOORBELL_EN__SHIFT;
449 WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL5, data);
450
451 data = 1 << CP_HQD_GFX_CONTROL__DB_UPDATED_MSG_EN__SHIFT;
452 WREG32_SOC15(GC, 0, regCP_HQD_GFX_CONTROL, data);
453 }
454
455 static const struct amdgpu_mes_funcs mes_v11_0_funcs = {
456 .add_hw_queue = mes_v11_0_add_hw_queue,
457 .remove_hw_queue = mes_v11_0_remove_hw_queue,
458 .unmap_legacy_queue = mes_v11_0_unmap_legacy_queue,
459 .suspend_gang = mes_v11_0_suspend_gang,
460 .resume_gang = mes_v11_0_resume_gang,
461 .misc_op = mes_v11_0_misc_op,
462 };
463
mes_v11_0_allocate_ucode_buffer(struct amdgpu_device * adev,enum admgpu_mes_pipe pipe)464 static int mes_v11_0_allocate_ucode_buffer(struct amdgpu_device *adev,
465 enum admgpu_mes_pipe pipe)
466 {
467 int r;
468 const struct mes_firmware_header_v1_0 *mes_hdr;
469 const __le32 *fw_data;
470 unsigned fw_size;
471
472 mes_hdr = (const struct mes_firmware_header_v1_0 *)
473 adev->mes.fw[pipe]->data;
474
475 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
476 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
477 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
478
479 r = amdgpu_bo_create_reserved(adev, fw_size,
480 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
481 &adev->mes.ucode_fw_obj[pipe],
482 &adev->mes.ucode_fw_gpu_addr[pipe],
483 (void **)&adev->mes.ucode_fw_ptr[pipe]);
484 if (r) {
485 dev_err(adev->dev, "(%d) failed to create mes fw bo\n", r);
486 return r;
487 }
488
489 memcpy(adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size);
490
491 amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[pipe]);
492 amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[pipe]);
493
494 return 0;
495 }
496
mes_v11_0_allocate_ucode_data_buffer(struct amdgpu_device * adev,enum admgpu_mes_pipe pipe)497 static int mes_v11_0_allocate_ucode_data_buffer(struct amdgpu_device *adev,
498 enum admgpu_mes_pipe pipe)
499 {
500 int r;
501 const struct mes_firmware_header_v1_0 *mes_hdr;
502 const __le32 *fw_data;
503 unsigned fw_size;
504
505 mes_hdr = (const struct mes_firmware_header_v1_0 *)
506 adev->mes.fw[pipe]->data;
507
508 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
509 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
510 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
511
512 r = amdgpu_bo_create_reserved(adev, fw_size,
513 64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
514 &adev->mes.data_fw_obj[pipe],
515 &adev->mes.data_fw_gpu_addr[pipe],
516 (void **)&adev->mes.data_fw_ptr[pipe]);
517 if (r) {
518 dev_err(adev->dev, "(%d) failed to create mes data fw bo\n", r);
519 return r;
520 }
521
522 memcpy(adev->mes.data_fw_ptr[pipe], fw_data, fw_size);
523
524 amdgpu_bo_kunmap(adev->mes.data_fw_obj[pipe]);
525 amdgpu_bo_unreserve(adev->mes.data_fw_obj[pipe]);
526
527 return 0;
528 }
529
mes_v11_0_free_ucode_buffers(struct amdgpu_device * adev,enum admgpu_mes_pipe pipe)530 static void mes_v11_0_free_ucode_buffers(struct amdgpu_device *adev,
531 enum admgpu_mes_pipe pipe)
532 {
533 amdgpu_bo_free_kernel(&adev->mes.data_fw_obj[pipe],
534 &adev->mes.data_fw_gpu_addr[pipe],
535 (void **)&adev->mes.data_fw_ptr[pipe]);
536
537 amdgpu_bo_free_kernel(&adev->mes.ucode_fw_obj[pipe],
538 &adev->mes.ucode_fw_gpu_addr[pipe],
539 (void **)&adev->mes.ucode_fw_ptr[pipe]);
540 }
541
mes_v11_0_enable(struct amdgpu_device * adev,bool enable)542 static void mes_v11_0_enable(struct amdgpu_device *adev, bool enable)
543 {
544 uint64_t ucode_addr;
545 uint32_t pipe, data = 0;
546
547 if (enable) {
548 data = RREG32_SOC15(GC, 0, regCP_MES_CNTL);
549 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
550 data = REG_SET_FIELD(data, CP_MES_CNTL,
551 MES_PIPE1_RESET, adev->enable_mes_kiq ? 1 : 0);
552 WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
553
554 mutex_lock(&adev->srbm_mutex);
555 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
556 if (!adev->enable_mes_kiq &&
557 pipe == AMDGPU_MES_KIQ_PIPE)
558 continue;
559
560 soc21_grbm_select(adev, 3, pipe, 0, 0);
561
562 ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
563 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START,
564 lower_32_bits(ucode_addr));
565 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI,
566 upper_32_bits(ucode_addr));
567 }
568 soc21_grbm_select(adev, 0, 0, 0, 0);
569 mutex_unlock(&adev->srbm_mutex);
570
571 /* unhalt MES and activate pipe0 */
572 data = REG_SET_FIELD(0, CP_MES_CNTL, MES_PIPE0_ACTIVE, 1);
573 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE,
574 adev->enable_mes_kiq ? 1 : 0);
575 WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
576
577 if (amdgpu_emu_mode)
578 msleep(100);
579 else
580 udelay(50);
581 } else {
582 data = RREG32_SOC15(GC, 0, regCP_MES_CNTL);
583 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_ACTIVE, 0);
584 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, 0);
585 data = REG_SET_FIELD(data, CP_MES_CNTL,
586 MES_INVALIDATE_ICACHE, 1);
587 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
588 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_RESET,
589 adev->enable_mes_kiq ? 1 : 0);
590 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_HALT, 1);
591 WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
592 }
593 }
594
595 /* This function is for backdoor MES firmware */
mes_v11_0_load_microcode(struct amdgpu_device * adev,enum admgpu_mes_pipe pipe,bool prime_icache)596 static int mes_v11_0_load_microcode(struct amdgpu_device *adev,
597 enum admgpu_mes_pipe pipe, bool prime_icache)
598 {
599 int r;
600 uint32_t data;
601 uint64_t ucode_addr;
602
603 mes_v11_0_enable(adev, false);
604
605 if (!adev->mes.fw[pipe])
606 return -EINVAL;
607
608 r = mes_v11_0_allocate_ucode_buffer(adev, pipe);
609 if (r)
610 return r;
611
612 r = mes_v11_0_allocate_ucode_data_buffer(adev, pipe);
613 if (r) {
614 mes_v11_0_free_ucode_buffers(adev, pipe);
615 return r;
616 }
617
618 mutex_lock(&adev->srbm_mutex);
619 /* me=3, pipe=0, queue=0 */
620 soc21_grbm_select(adev, 3, pipe, 0, 0);
621
622 WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_CNTL, 0);
623
624 /* set ucode start address */
625 ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
626 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START,
627 lower_32_bits(ucode_addr));
628 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI,
629 upper_32_bits(ucode_addr));
630
631 /* set ucode fimrware address */
632 WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_LO,
633 lower_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));
634 WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_HI,
635 upper_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));
636
637 /* set ucode instruction cache boundary to 2M-1 */
638 WREG32_SOC15(GC, 0, regCP_MES_MIBOUND_LO, 0x1FFFFF);
639
640 /* set ucode data firmware address */
641 WREG32_SOC15(GC, 0, regCP_MES_MDBASE_LO,
642 lower_32_bits(adev->mes.data_fw_gpu_addr[pipe]));
643 WREG32_SOC15(GC, 0, regCP_MES_MDBASE_HI,
644 upper_32_bits(adev->mes.data_fw_gpu_addr[pipe]));
645
646 /* Set 0x3FFFF (256K-1) to CP_MES_MDBOUND_LO */
647 WREG32_SOC15(GC, 0, regCP_MES_MDBOUND_LO, 0x3FFFF);
648
649 if (prime_icache) {
650 /* invalidate ICACHE */
651 data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL);
652 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 0);
653 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, INVALIDATE_CACHE, 1);
654 WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data);
655
656 /* prime the ICACHE. */
657 data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL);
658 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 1);
659 WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data);
660 }
661
662 soc21_grbm_select(adev, 0, 0, 0, 0);
663 mutex_unlock(&adev->srbm_mutex);
664
665 return 0;
666 }
667
mes_v11_0_allocate_eop_buf(struct amdgpu_device * adev,enum admgpu_mes_pipe pipe)668 static int mes_v11_0_allocate_eop_buf(struct amdgpu_device *adev,
669 enum admgpu_mes_pipe pipe)
670 {
671 int r;
672 u32 *eop;
673
674 r = amdgpu_bo_create_reserved(adev, MES_EOP_SIZE, PAGE_SIZE,
675 AMDGPU_GEM_DOMAIN_GTT,
676 &adev->mes.eop_gpu_obj[pipe],
677 &adev->mes.eop_gpu_addr[pipe],
678 (void **)&eop);
679 if (r) {
680 dev_warn(adev->dev, "(%d) create EOP bo failed\n", r);
681 return r;
682 }
683
684 memset(eop, 0,
685 adev->mes.eop_gpu_obj[pipe]->tbo.base.size);
686
687 amdgpu_bo_kunmap(adev->mes.eop_gpu_obj[pipe]);
688 amdgpu_bo_unreserve(adev->mes.eop_gpu_obj[pipe]);
689
690 return 0;
691 }
692
mes_v11_0_mqd_init(struct amdgpu_ring * ring)693 static int mes_v11_0_mqd_init(struct amdgpu_ring *ring)
694 {
695 struct v11_compute_mqd *mqd = ring->mqd_ptr;
696 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
697 uint32_t tmp;
698
699 mqd->header = 0xC0310800;
700 mqd->compute_pipelinestat_enable = 0x00000001;
701 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
702 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
703 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
704 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
705 mqd->compute_misc_reserved = 0x00000007;
706
707 eop_base_addr = ring->eop_gpu_addr >> 8;
708
709 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
710 tmp = regCP_HQD_EOP_CONTROL_DEFAULT;
711 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
712 (order_base_2(MES_EOP_SIZE / 4) - 1));
713
714 mqd->cp_hqd_eop_base_addr_lo = lower_32_bits(eop_base_addr);
715 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
716 mqd->cp_hqd_eop_control = tmp;
717
718 /* disable the queue if it's active */
719 ring->wptr = 0;
720 mqd->cp_hqd_pq_rptr = 0;
721 mqd->cp_hqd_pq_wptr_lo = 0;
722 mqd->cp_hqd_pq_wptr_hi = 0;
723
724 /* set the pointer to the MQD */
725 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
726 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
727
728 /* set MQD vmid to 0 */
729 tmp = regCP_MQD_CONTROL_DEFAULT;
730 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
731 mqd->cp_mqd_control = tmp;
732
733 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
734 hqd_gpu_addr = ring->gpu_addr >> 8;
735 mqd->cp_hqd_pq_base_lo = lower_32_bits(hqd_gpu_addr);
736 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
737
738 /* set the wb address whether it's enabled or not */
739 wb_gpu_addr = ring->rptr_gpu_addr;
740 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
741 mqd->cp_hqd_pq_rptr_report_addr_hi =
742 upper_32_bits(wb_gpu_addr) & 0xffff;
743
744 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
745 wb_gpu_addr = ring->wptr_gpu_addr;
746 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffff8;
747 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
748
749 /* set up the HQD, this is similar to CP_RB0_CNTL */
750 tmp = regCP_HQD_PQ_CONTROL_DEFAULT;
751 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
752 (order_base_2(ring->ring_size / 4) - 1));
753 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
754 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
755 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1);
756 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
757 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
758 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
759 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, NO_UPDATE_RPTR, 1);
760 mqd->cp_hqd_pq_control = tmp;
761
762 /* enable doorbell */
763 tmp = 0;
764 if (ring->use_doorbell) {
765 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
766 DOORBELL_OFFSET, ring->doorbell_index);
767 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
768 DOORBELL_EN, 1);
769 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
770 DOORBELL_SOURCE, 0);
771 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
772 DOORBELL_HIT, 0);
773 }
774 else
775 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
776 DOORBELL_EN, 0);
777 mqd->cp_hqd_pq_doorbell_control = tmp;
778
779 mqd->cp_hqd_vmid = 0;
780 /* activate the queue */
781 mqd->cp_hqd_active = 1;
782
783 tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT;
784 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE,
785 PRELOAD_SIZE, 0x55);
786 mqd->cp_hqd_persistent_state = tmp;
787
788 mqd->cp_hqd_ib_control = regCP_HQD_IB_CONTROL_DEFAULT;
789 mqd->cp_hqd_iq_timer = regCP_HQD_IQ_TIMER_DEFAULT;
790 mqd->cp_hqd_quantum = regCP_HQD_QUANTUM_DEFAULT;
791
792 return 0;
793 }
794
mes_v11_0_queue_init_register(struct amdgpu_ring * ring)795 static void mes_v11_0_queue_init_register(struct amdgpu_ring *ring)
796 {
797 struct v11_compute_mqd *mqd = ring->mqd_ptr;
798 struct amdgpu_device *adev = ring->adev;
799 uint32_t data = 0;
800
801 mutex_lock(&adev->srbm_mutex);
802 soc21_grbm_select(adev, 3, ring->pipe, 0, 0);
803
804 /* set CP_HQD_VMID.VMID = 0. */
805 data = RREG32_SOC15(GC, 0, regCP_HQD_VMID);
806 data = REG_SET_FIELD(data, CP_HQD_VMID, VMID, 0);
807 WREG32_SOC15(GC, 0, regCP_HQD_VMID, data);
808
809 /* set CP_HQD_PQ_DOORBELL_CONTROL.DOORBELL_EN=0 */
810 data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
811 data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
812 DOORBELL_EN, 0);
813 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data);
814
815 /* set CP_MQD_BASE_ADDR/HI with the MQD base address */
816 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr_lo);
817 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi);
818
819 /* set CP_MQD_CONTROL.VMID=0 */
820 data = RREG32_SOC15(GC, 0, regCP_MQD_CONTROL);
821 data = REG_SET_FIELD(data, CP_MQD_CONTROL, VMID, 0);
822 WREG32_SOC15(GC, 0, regCP_MQD_CONTROL, 0);
823
824 /* set CP_HQD_PQ_BASE/HI with the ring buffer base address */
825 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE, mqd->cp_hqd_pq_base_lo);
826 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI, mqd->cp_hqd_pq_base_hi);
827
828 /* set CP_HQD_PQ_RPTR_REPORT_ADDR/HI */
829 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,
830 mqd->cp_hqd_pq_rptr_report_addr_lo);
831 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
832 mqd->cp_hqd_pq_rptr_report_addr_hi);
833
834 /* set CP_HQD_PQ_CONTROL */
835 WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL, mqd->cp_hqd_pq_control);
836
837 /* set CP_HQD_PQ_WPTR_POLL_ADDR/HI */
838 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,
839 mqd->cp_hqd_pq_wptr_poll_addr_lo);
840 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
841 mqd->cp_hqd_pq_wptr_poll_addr_hi);
842
843 /* set CP_HQD_PQ_DOORBELL_CONTROL */
844 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
845 mqd->cp_hqd_pq_doorbell_control);
846
847 /* set CP_HQD_PERSISTENT_STATE.PRELOAD_SIZE=0x53 */
848 WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE, mqd->cp_hqd_persistent_state);
849
850 /* set CP_HQD_ACTIVE.ACTIVE=1 */
851 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, mqd->cp_hqd_active);
852
853 soc21_grbm_select(adev, 0, 0, 0, 0);
854 mutex_unlock(&adev->srbm_mutex);
855 }
856
mes_v11_0_kiq_enable_queue(struct amdgpu_device * adev)857 static int mes_v11_0_kiq_enable_queue(struct amdgpu_device *adev)
858 {
859 struct amdgpu_kiq *kiq = &adev->gfx.kiq;
860 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
861 int r;
862
863 if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
864 return -EINVAL;
865
866 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
867 if (r) {
868 DRM_ERROR("Failed to lock KIQ (%d).\n", r);
869 return r;
870 }
871
872 kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring);
873
874 r = amdgpu_ring_test_ring(kiq_ring);
875 if (r) {
876 DRM_ERROR("kfq enable failed\n");
877 kiq_ring->sched.ready = false;
878 }
879 return r;
880 }
881
mes_v11_0_queue_init(struct amdgpu_device * adev,enum admgpu_mes_pipe pipe)882 static int mes_v11_0_queue_init(struct amdgpu_device *adev,
883 enum admgpu_mes_pipe pipe)
884 {
885 struct amdgpu_ring *ring;
886 int r;
887
888 if (pipe == AMDGPU_MES_KIQ_PIPE)
889 ring = &adev->gfx.kiq.ring;
890 else if (pipe == AMDGPU_MES_SCHED_PIPE)
891 ring = &adev->mes.ring;
892 else
893 BUG();
894
895 if ((pipe == AMDGPU_MES_SCHED_PIPE) &&
896 (amdgpu_in_reset(adev) || adev->in_suspend)) {
897 *(ring->wptr_cpu_addr) = 0;
898 *(ring->rptr_cpu_addr) = 0;
899 amdgpu_ring_clear_ring(ring);
900 }
901
902 r = mes_v11_0_mqd_init(ring);
903 if (r)
904 return r;
905
906 if (pipe == AMDGPU_MES_SCHED_PIPE) {
907 r = mes_v11_0_kiq_enable_queue(adev);
908 if (r)
909 return r;
910 } else {
911 mes_v11_0_queue_init_register(ring);
912 }
913
914 /* get MES scheduler/KIQ versions */
915 mutex_lock(&adev->srbm_mutex);
916 soc21_grbm_select(adev, 3, pipe, 0, 0);
917
918 if (pipe == AMDGPU_MES_SCHED_PIPE)
919 adev->mes.sched_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);
920 else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq)
921 adev->mes.kiq_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);
922
923 soc21_grbm_select(adev, 0, 0, 0, 0);
924 mutex_unlock(&adev->srbm_mutex);
925
926 return 0;
927 }
928
mes_v11_0_ring_init(struct amdgpu_device * adev)929 static int mes_v11_0_ring_init(struct amdgpu_device *adev)
930 {
931 struct amdgpu_ring *ring;
932
933 ring = &adev->mes.ring;
934
935 ring->funcs = &mes_v11_0_ring_funcs;
936
937 ring->me = 3;
938 ring->pipe = 0;
939 ring->queue = 0;
940
941 ring->ring_obj = NULL;
942 ring->use_doorbell = true;
943 ring->doorbell_index = adev->doorbell_index.mes_ring0 << 1;
944 ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_SCHED_PIPE];
945 ring->no_scheduler = true;
946 sprintf(ring->name, "mes_%d.%d.%d", ring->me, ring->pipe, ring->queue);
947
948 return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
949 AMDGPU_RING_PRIO_DEFAULT, NULL);
950 }
951
mes_v11_0_kiq_ring_init(struct amdgpu_device * adev)952 static int mes_v11_0_kiq_ring_init(struct amdgpu_device *adev)
953 {
954 struct amdgpu_ring *ring;
955
956 spin_lock_init(&adev->gfx.kiq.ring_lock);
957
958 ring = &adev->gfx.kiq.ring;
959
960 ring->me = 3;
961 ring->pipe = 1;
962 ring->queue = 0;
963
964 ring->adev = NULL;
965 ring->ring_obj = NULL;
966 ring->use_doorbell = true;
967 ring->doorbell_index = adev->doorbell_index.mes_ring1 << 1;
968 ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_KIQ_PIPE];
969 ring->no_scheduler = true;
970 sprintf(ring->name, "mes_kiq_%d.%d.%d",
971 ring->me, ring->pipe, ring->queue);
972
973 return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
974 AMDGPU_RING_PRIO_DEFAULT, NULL);
975 }
976
mes_v11_0_mqd_sw_init(struct amdgpu_device * adev,enum admgpu_mes_pipe pipe)977 static int mes_v11_0_mqd_sw_init(struct amdgpu_device *adev,
978 enum admgpu_mes_pipe pipe)
979 {
980 int r, mqd_size = sizeof(struct v11_compute_mqd);
981 struct amdgpu_ring *ring;
982
983 if (pipe == AMDGPU_MES_KIQ_PIPE)
984 ring = &adev->gfx.kiq.ring;
985 else if (pipe == AMDGPU_MES_SCHED_PIPE)
986 ring = &adev->mes.ring;
987 else
988 BUG();
989
990 if (ring->mqd_obj)
991 return 0;
992
993 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
994 AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
995 &ring->mqd_gpu_addr, &ring->mqd_ptr);
996 if (r) {
997 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
998 return r;
999 }
1000
1001 memset(ring->mqd_ptr, 0, mqd_size);
1002
1003 /* prepare MQD backup */
1004 adev->mes.mqd_backup[pipe] = kmalloc(mqd_size, GFP_KERNEL);
1005 if (!adev->mes.mqd_backup[pipe])
1006 dev_warn(adev->dev,
1007 "no memory to create MQD backup for ring %s\n",
1008 ring->name);
1009
1010 return 0;
1011 }
1012
mes_v11_0_sw_init(void * handle)1013 static int mes_v11_0_sw_init(void *handle)
1014 {
1015 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1016 int pipe, r;
1017
1018 adev->mes.adev = adev;
1019 adev->mes.funcs = &mes_v11_0_funcs;
1020 adev->mes.kiq_hw_init = &mes_v11_0_kiq_hw_init;
1021 adev->mes.kiq_hw_fini = &mes_v11_0_kiq_hw_fini;
1022
1023 r = amdgpu_mes_init(adev);
1024 if (r)
1025 return r;
1026
1027 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1028 if (!adev->enable_mes_kiq && pipe == AMDGPU_MES_KIQ_PIPE)
1029 continue;
1030
1031 r = mes_v11_0_allocate_eop_buf(adev, pipe);
1032 if (r)
1033 return r;
1034
1035 r = mes_v11_0_mqd_sw_init(adev, pipe);
1036 if (r)
1037 return r;
1038 }
1039
1040 if (adev->enable_mes_kiq) {
1041 r = mes_v11_0_kiq_ring_init(adev);
1042 if (r)
1043 return r;
1044 }
1045
1046 r = mes_v11_0_ring_init(adev);
1047 if (r)
1048 return r;
1049
1050 return 0;
1051 }
1052
mes_v11_0_sw_fini(void * handle)1053 static int mes_v11_0_sw_fini(void *handle)
1054 {
1055 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1056 int pipe;
1057
1058 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);
1059 amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs);
1060
1061 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1062 kfree(adev->mes.mqd_backup[pipe]);
1063
1064 amdgpu_bo_free_kernel(&adev->mes.eop_gpu_obj[pipe],
1065 &adev->mes.eop_gpu_addr[pipe],
1066 NULL);
1067 amdgpu_ucode_release(&adev->mes.fw[pipe]);
1068 }
1069
1070 amdgpu_bo_free_kernel(&adev->gfx.kiq.ring.mqd_obj,
1071 &adev->gfx.kiq.ring.mqd_gpu_addr,
1072 &adev->gfx.kiq.ring.mqd_ptr);
1073
1074 amdgpu_bo_free_kernel(&adev->mes.ring.mqd_obj,
1075 &adev->mes.ring.mqd_gpu_addr,
1076 &adev->mes.ring.mqd_ptr);
1077
1078 amdgpu_ring_fini(&adev->gfx.kiq.ring);
1079 amdgpu_ring_fini(&adev->mes.ring);
1080
1081 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1082 mes_v11_0_free_ucode_buffers(adev, AMDGPU_MES_KIQ_PIPE);
1083 mes_v11_0_free_ucode_buffers(adev, AMDGPU_MES_SCHED_PIPE);
1084 }
1085
1086 amdgpu_mes_fini(adev);
1087 return 0;
1088 }
1089
mes_v11_0_kiq_dequeue_sched(struct amdgpu_device * adev)1090 static void mes_v11_0_kiq_dequeue_sched(struct amdgpu_device *adev)
1091 {
1092 uint32_t data;
1093 int i;
1094
1095 mutex_lock(&adev->srbm_mutex);
1096 soc21_grbm_select(adev, 3, AMDGPU_MES_SCHED_PIPE, 0, 0);
1097
1098 /* disable the queue if it's active */
1099 if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {
1100 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1);
1101 for (i = 0; i < adev->usec_timeout; i++) {
1102 if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
1103 break;
1104 udelay(1);
1105 }
1106 }
1107 data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
1108 data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1109 DOORBELL_EN, 0);
1110 data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1111 DOORBELL_HIT, 1);
1112 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data);
1113
1114 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 0);
1115
1116 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 0);
1117 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 0);
1118 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR, 0);
1119
1120 soc21_grbm_select(adev, 0, 0, 0, 0);
1121 mutex_unlock(&adev->srbm_mutex);
1122
1123 adev->mes.ring.sched.ready = false;
1124 }
1125
mes_v11_0_kiq_setting(struct amdgpu_ring * ring)1126 static void mes_v11_0_kiq_setting(struct amdgpu_ring *ring)
1127 {
1128 uint32_t tmp;
1129 struct amdgpu_device *adev = ring->adev;
1130
1131 /* tell RLC which is KIQ queue */
1132 tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
1133 tmp &= 0xffffff00;
1134 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
1135 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
1136 tmp |= 0x80;
1137 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
1138 }
1139
mes_v11_0_kiq_hw_init(struct amdgpu_device * adev)1140 static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev)
1141 {
1142 int r = 0;
1143
1144 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1145
1146 r = mes_v11_0_load_microcode(adev, AMDGPU_MES_SCHED_PIPE, false);
1147 if (r) {
1148 DRM_ERROR("failed to load MES fw, r=%d\n", r);
1149 return r;
1150 }
1151
1152 r = mes_v11_0_load_microcode(adev, AMDGPU_MES_KIQ_PIPE, true);
1153 if (r) {
1154 DRM_ERROR("failed to load MES kiq fw, r=%d\n", r);
1155 return r;
1156 }
1157
1158 }
1159
1160 mes_v11_0_enable(adev, true);
1161
1162 mes_v11_0_kiq_setting(&adev->gfx.kiq.ring);
1163
1164 r = mes_v11_0_queue_init(adev, AMDGPU_MES_KIQ_PIPE);
1165 if (r)
1166 goto failure;
1167
1168 return r;
1169
1170 failure:
1171 mes_v11_0_hw_fini(adev);
1172 return r;
1173 }
1174
mes_v11_0_kiq_hw_fini(struct amdgpu_device * adev)1175 static int mes_v11_0_kiq_hw_fini(struct amdgpu_device *adev)
1176 {
1177 if (adev->mes.ring.sched.ready)
1178 mes_v11_0_kiq_dequeue_sched(adev);
1179
1180 mes_v11_0_enable(adev, false);
1181 return 0;
1182 }
1183
mes_v11_0_hw_init(void * handle)1184 static int mes_v11_0_hw_init(void *handle)
1185 {
1186 int r;
1187 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1188
1189 if (!adev->enable_mes_kiq) {
1190 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1191 r = mes_v11_0_load_microcode(adev,
1192 AMDGPU_MES_SCHED_PIPE, true);
1193 if (r) {
1194 DRM_ERROR("failed to MES fw, r=%d\n", r);
1195 return r;
1196 }
1197 }
1198
1199 mes_v11_0_enable(adev, true);
1200 }
1201
1202 r = mes_v11_0_queue_init(adev, AMDGPU_MES_SCHED_PIPE);
1203 if (r)
1204 goto failure;
1205
1206 r = mes_v11_0_set_hw_resources(&adev->mes);
1207 if (r)
1208 goto failure;
1209
1210 mes_v11_0_init_aggregated_doorbell(&adev->mes);
1211
1212 r = mes_v11_0_query_sched_status(&adev->mes);
1213 if (r) {
1214 DRM_ERROR("MES is busy\n");
1215 goto failure;
1216 }
1217
1218 /*
1219 * Disable KIQ ring usage from the driver once MES is enabled.
1220 * MES uses KIQ ring exclusively so driver cannot access KIQ ring
1221 * with MES enabled.
1222 */
1223 adev->gfx.kiq.ring.sched.ready = false;
1224 adev->mes.ring.sched.ready = true;
1225
1226 return 0;
1227
1228 failure:
1229 mes_v11_0_hw_fini(adev);
1230 return r;
1231 }
1232
mes_v11_0_hw_fini(void * handle)1233 static int mes_v11_0_hw_fini(void *handle)
1234 {
1235 return 0;
1236 }
1237
mes_v11_0_suspend(void * handle)1238 static int mes_v11_0_suspend(void *handle)
1239 {
1240 int r;
1241 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1242
1243 r = amdgpu_mes_suspend(adev);
1244 if (r)
1245 return r;
1246
1247 return mes_v11_0_hw_fini(adev);
1248 }
1249
mes_v11_0_resume(void * handle)1250 static int mes_v11_0_resume(void *handle)
1251 {
1252 int r;
1253 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1254
1255 r = mes_v11_0_hw_init(adev);
1256 if (r)
1257 return r;
1258
1259 return amdgpu_mes_resume(adev);
1260 }
1261
mes_v11_0_early_init(void * handle)1262 static int mes_v11_0_early_init(void *handle)
1263 {
1264 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1265 int pipe, r;
1266
1267 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1268 if (!adev->enable_mes_kiq && pipe == AMDGPU_MES_KIQ_PIPE)
1269 continue;
1270 r = amdgpu_mes_init_microcode(adev, pipe);
1271 if (r)
1272 return r;
1273 }
1274
1275 return 0;
1276 }
1277
mes_v11_0_late_init(void * handle)1278 static int mes_v11_0_late_init(void *handle)
1279 {
1280 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1281
1282 /* it's only intended for use in mes_self_test case, not for s0ix and reset */
1283 if (!amdgpu_in_reset(adev) && !adev->in_s0ix && !adev->in_suspend &&
1284 (adev->ip_versions[GC_HWIP][0] != IP_VERSION(11, 0, 3)))
1285 amdgpu_mes_self_test(adev);
1286
1287 return 0;
1288 }
1289
1290 static const struct amd_ip_funcs mes_v11_0_ip_funcs = {
1291 .name = "mes_v11_0",
1292 .early_init = mes_v11_0_early_init,
1293 .late_init = mes_v11_0_late_init,
1294 .sw_init = mes_v11_0_sw_init,
1295 .sw_fini = mes_v11_0_sw_fini,
1296 .hw_init = mes_v11_0_hw_init,
1297 .hw_fini = mes_v11_0_hw_fini,
1298 .suspend = mes_v11_0_suspend,
1299 .resume = mes_v11_0_resume,
1300 };
1301
1302 const struct amdgpu_ip_block_version mes_v11_0_ip_block = {
1303 .type = AMD_IP_BLOCK_TYPE_MES,
1304 .major = 11,
1305 .minor = 0,
1306 .rev = 0,
1307 .funcs = &mes_v11_0_ip_funcs,
1308 };
1309