1 /*
2 * Copyright 2021 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23 #include <linux/delay.h>
24 #include <linux/kernel.h>
25 #include <linux/firmware.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28 #include "amdgpu.h"
29 #include "amdgpu_gfx.h"
30 #include "amdgpu_psp.h"
31 #include "amdgpu_smu.h"
32 #include "amdgpu_atomfirmware.h"
33 #include "imu_v11_0.h"
34 #include "soc21.h"
35 #include "nvd.h"
36
37 #include "gc/gc_11_0_0_offset.h"
38 #include "gc/gc_11_0_0_sh_mask.h"
39 #include "smuio/smuio_13_0_6_offset.h"
40 #include "smuio/smuio_13_0_6_sh_mask.h"
41 #include "navi10_enum.h"
42 #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h"
43
44 #include "soc15.h"
45 #include "soc15d.h"
46 #include "clearstate_gfx11.h"
47 #include "v11_structs.h"
48 #include "gfx_v11_0.h"
49 #include "gfx_v11_0_3.h"
50 #include "nbio_v4_3.h"
51 #include "mes_v11_0.h"
52
53 #define GFX11_NUM_GFX_RINGS 1
54 #define GFX11_MEC_HPD_SIZE 2048
55
56 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
57 #define RLC_PG_DELAY_3_DEFAULT_GC_11_0_1 0x1388
58
59 #define regCGTT_WD_CLK_CTRL 0x5086
60 #define regCGTT_WD_CLK_CTRL_BASE_IDX 1
61 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1 0x4e7e
62 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1_BASE_IDX 1
63 #define regPC_CONFIG_CNTL_1 0x194d
64 #define regPC_CONFIG_CNTL_1_BASE_IDX 1
65
66 #define regCP_GFX_MQD_CONTROL_DEFAULT 0x00000100
67 #define regCP_GFX_HQD_VMID_DEFAULT 0x00000000
68 #define regCP_GFX_HQD_QUEUE_PRIORITY_DEFAULT 0x00000000
69 #define regCP_GFX_HQD_QUANTUM_DEFAULT 0x00000a01
70 #define regCP_GFX_HQD_CNTL_DEFAULT 0x00a00000
71 #define regCP_RB_DOORBELL_CONTROL_DEFAULT 0x00000000
72 #define regCP_GFX_HQD_RPTR_DEFAULT 0x00000000
73
74 #define regCP_HQD_EOP_CONTROL_DEFAULT 0x00000006
75 #define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0x00000000
76 #define regCP_MQD_CONTROL_DEFAULT 0x00000100
77 #define regCP_HQD_PQ_CONTROL_DEFAULT 0x00308509
78 #define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0x00000000
79 #define regCP_HQD_PQ_RPTR_DEFAULT 0x00000000
80 #define regCP_HQD_PERSISTENT_STATE_DEFAULT 0x0be05501
81 #define regCP_HQD_IB_CONTROL_DEFAULT 0x00300000
82
83 MODULE_FIRMWARE("amdgpu/gc_11_0_0_pfp.bin");
84 MODULE_FIRMWARE("amdgpu/gc_11_0_0_me.bin");
85 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mec.bin");
86 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc.bin");
87 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc_kicker.bin");
88 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc_1.bin");
89 MODULE_FIRMWARE("amdgpu/gc_11_0_0_toc.bin");
90 MODULE_FIRMWARE("amdgpu/gc_11_0_1_pfp.bin");
91 MODULE_FIRMWARE("amdgpu/gc_11_0_1_me.bin");
92 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mec.bin");
93 MODULE_FIRMWARE("amdgpu/gc_11_0_1_rlc.bin");
94 MODULE_FIRMWARE("amdgpu/gc_11_0_2_pfp.bin");
95 MODULE_FIRMWARE("amdgpu/gc_11_0_2_me.bin");
96 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mec.bin");
97 MODULE_FIRMWARE("amdgpu/gc_11_0_2_rlc.bin");
98 MODULE_FIRMWARE("amdgpu/gc_11_0_3_pfp.bin");
99 MODULE_FIRMWARE("amdgpu/gc_11_0_3_me.bin");
100 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mec.bin");
101 MODULE_FIRMWARE("amdgpu/gc_11_0_3_rlc.bin");
102 MODULE_FIRMWARE("amdgpu/gc_11_0_4_pfp.bin");
103 MODULE_FIRMWARE("amdgpu/gc_11_0_4_me.bin");
104 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mec.bin");
105 MODULE_FIRMWARE("amdgpu/gc_11_0_4_rlc.bin");
106 MODULE_FIRMWARE("amdgpu/gc_11_5_0_pfp.bin");
107 MODULE_FIRMWARE("amdgpu/gc_11_5_0_me.bin");
108 MODULE_FIRMWARE("amdgpu/gc_11_5_0_mec.bin");
109 MODULE_FIRMWARE("amdgpu/gc_11_5_0_rlc.bin");
110 MODULE_FIRMWARE("amdgpu/gc_11_5_1_pfp.bin");
111 MODULE_FIRMWARE("amdgpu/gc_11_5_1_me.bin");
112 MODULE_FIRMWARE("amdgpu/gc_11_5_1_mec.bin");
113 MODULE_FIRMWARE("amdgpu/gc_11_5_1_rlc.bin");
114 MODULE_FIRMWARE("amdgpu/gc_11_5_2_pfp.bin");
115 MODULE_FIRMWARE("amdgpu/gc_11_5_2_me.bin");
116 MODULE_FIRMWARE("amdgpu/gc_11_5_2_mec.bin");
117 MODULE_FIRMWARE("amdgpu/gc_11_5_2_rlc.bin");
118
119 static const struct amdgpu_hwip_reg_entry gc_reg_list_11_0[] = {
120 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS),
121 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2),
122 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS3),
123 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT1),
124 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT2),
125 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT3),
126 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STALLED_STAT1),
127 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STALLED_STAT1),
128 SOC15_REG_ENTRY_STR(GC, 0, regCP_BUSY_STAT),
129 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT),
130 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT),
131 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT2),
132 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT2),
133 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STATUS),
134 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_ERROR),
135 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HPD_STATUS0),
136 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_BASE),
137 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR),
138 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR),
139 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_BASE),
140 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_RPTR),
141 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_WPTR),
142 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_BASE),
143 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_RPTR),
144 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_WPTR),
145 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ),
146 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_CMD_BUFSZ),
147 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO),
148 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI),
149 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ),
150 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_LO),
151 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_HI),
152 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BUFSZ),
153 SOC15_REG_ENTRY_STR(GC, 0, regCPF_UTCL1_STATUS),
154 SOC15_REG_ENTRY_STR(GC, 0, regCPC_UTCL1_STATUS),
155 SOC15_REG_ENTRY_STR(GC, 0, regCPG_UTCL1_STATUS),
156 SOC15_REG_ENTRY_STR(GC, 0, regGDS_PROTECTION_FAULT),
157 SOC15_REG_ENTRY_STR(GC, 0, regGDS_VM_PROTECTION_FAULT),
158 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS),
159 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS_2),
160 SOC15_REG_ENTRY_STR(GC, 0, regPA_CL_CNTL_STATUS),
161 SOC15_REG_ENTRY_STR(GC, 0, regRLC_UTCL1_STATUS),
162 SOC15_REG_ENTRY_STR(GC, 0, regRMI_UTCL1_STATUS),
163 SOC15_REG_ENTRY_STR(GC, 0, regSQC_CACHES),
164 SOC15_REG_ENTRY_STR(GC, 0, regSQG_STATUS),
165 SOC15_REG_ENTRY_STR(GC, 0, regWD_UTCL1_STATUS),
166 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL),
167 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS),
168 SOC15_REG_ENTRY_STR(GC, 0, regCP_DEBUG),
169 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_CNTL),
170 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_CNTL),
171 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC1_INSTR_PNTR),
172 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_DEBUG_INTERRUPT_INSTR_PNTR),
173 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_INSTR_PNTR),
174 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_INSTR_PNTR),
175 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_INSTR_PNTR),
176 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STATUS),
177 /* cp header registers */
178 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
179 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
180 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
181 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
182 /* SE status registers */
183 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE0),
184 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE1),
185 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE2),
186 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE3),
187 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE4),
188 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE5)
189 };
190
191 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_11[] = {
192 /* compute registers */
193 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_VMID),
194 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PERSISTENT_STATE),
195 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PIPE_PRIORITY),
196 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUEUE_PRIORITY),
197 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUANTUM),
198 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE),
199 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE_HI),
200 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_RPTR),
201 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR),
202 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI),
203 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL),
204 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_CONTROL),
205 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR),
206 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR_HI),
207 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_RPTR),
208 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_CONTROL),
209 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_REQUEST),
210 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR),
211 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI),
212 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_CONTROL),
213 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_RPTR),
214 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR),
215 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_EVENTS),
216 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_LO),
217 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_HI),
218 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_CONTROL),
219 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_OFFSET),
220 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_SIZE),
221 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_WG_STATE_OFFSET),
222 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_SIZE),
223 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GDS_RESOURCE_STATE),
224 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ERROR),
225 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR_MEM),
226 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_LO),
227 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_HI),
228 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_OFFSET),
229 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT),
230 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_WG_STATE_OFFSET),
231 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_STATUS)
232 };
233
234 static const struct amdgpu_hwip_reg_entry gc_gfx_queue_reg_list_11[] = {
235 /* gfx queue registers */
236 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_ACTIVE),
237 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_VMID),
238 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY),
239 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUANTUM),
240 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE),
241 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE_HI),
242 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_OFFSET),
243 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CNTL),
244 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CSMD_RPTR),
245 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR),
246 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR_HI),
247 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST),
248 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_MAPPED),
249 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUE_MGR_CONTROL),
250 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_CONTROL0),
251 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_STATUS0),
252 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR),
253 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR_HI),
254 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO),
255 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI),
256 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR),
257 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO),
258 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI),
259 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ),
260 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ)
261 };
262
263 static const struct soc15_reg_golden golden_settings_gc_11_0[] = {
264 SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL, 0x20000000, 0x20000000)
265 };
266
267 static const struct soc15_reg_golden golden_settings_gc_11_0_1[] =
268 {
269 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_GS_NGG_CLK_CTRL, 0x9fff8fff, 0x00000010),
270 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_WD_CLK_CTRL, 0xffff8fff, 0x00000010),
271 SOC15_REG_GOLDEN_VALUE(GC, 0, regCPF_GCR_CNTL, 0x0007ffff, 0x0000c200),
272 SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2C_CTRL3, 0xffff001b, 0x00f01988),
273 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_CL_ENHANCE, 0xf0ffffff, 0x00880007),
274 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_ENHANCE_3, 0xfffffffd, 0x00000008),
275 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_VRS_SURFACE_CNTL_1, 0xfff891ff, 0x55480100),
276 SOC15_REG_GOLDEN_VALUE(GC, 0, regTA_CNTL_AUX, 0xf7f7ffff, 0x01030000),
277 SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL2, 0xfcffffff, 0x0000000a)
278 };
279
280 #define DEFAULT_SH_MEM_CONFIG \
281 ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \
282 (SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \
283 (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT))
284
285 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev);
286 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev);
287 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev);
288 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev);
289 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev);
290 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev);
291 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev);
292 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,
293 struct amdgpu_cu_info *cu_info);
294 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev);
295 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
296 u32 sh_num, u32 instance, int xcc_id);
297 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev);
298
299 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume);
300 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure);
301 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
302 uint32_t val);
303 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);
304 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
305 uint16_t pasid, uint32_t flush_type,
306 bool all_hub, uint8_t dst_sel);
307 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
308 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
309 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev,
310 bool enable);
311
gfx11_kiq_set_resources(struct amdgpu_ring * kiq_ring,uint64_t queue_mask)312 static void gfx11_kiq_set_resources(struct amdgpu_ring *kiq_ring, uint64_t queue_mask)
313 {
314 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
315 amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) |
316 PACKET3_SET_RESOURCES_UNMAP_LATENTY(0xa) | /* unmap_latency: 0xa (~ 1s) */
317 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */
318 amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */
319 amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */
320 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
321 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
322 amdgpu_ring_write(kiq_ring, 0); /* oac mask */
323 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
324 }
325
gfx11_kiq_map_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring)326 static void gfx11_kiq_map_queues(struct amdgpu_ring *kiq_ring,
327 struct amdgpu_ring *ring)
328 {
329 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
330 uint64_t wptr_addr = ring->wptr_gpu_addr;
331 uint32_t me = 0, eng_sel = 0;
332
333 switch (ring->funcs->type) {
334 case AMDGPU_RING_TYPE_COMPUTE:
335 me = 1;
336 eng_sel = 0;
337 break;
338 case AMDGPU_RING_TYPE_GFX:
339 me = 0;
340 eng_sel = 4;
341 break;
342 case AMDGPU_RING_TYPE_MES:
343 me = 2;
344 eng_sel = 5;
345 break;
346 default:
347 WARN_ON(1);
348 }
349
350 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
351 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
352 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
353 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
354 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
355 PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
356 PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
357 PACKET3_MAP_QUEUES_ME((me)) |
358 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
359 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */
360 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
361 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
362 amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
363 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
364 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
365 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
366 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
367 }
368
gfx11_kiq_unmap_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,enum amdgpu_unmap_queues_action action,u64 gpu_addr,u64 seq)369 static void gfx11_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
370 struct amdgpu_ring *ring,
371 enum amdgpu_unmap_queues_action action,
372 u64 gpu_addr, u64 seq)
373 {
374 struct amdgpu_device *adev = kiq_ring->adev;
375 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
376
377 if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) {
378 amdgpu_mes_unmap_legacy_queue(adev, ring, action, gpu_addr, seq);
379 return;
380 }
381
382 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
383 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
384 PACKET3_UNMAP_QUEUES_ACTION(action) |
385 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
386 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
387 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
388 amdgpu_ring_write(kiq_ring,
389 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
390
391 if (action == PREEMPT_QUEUES_NO_UNMAP) {
392 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
393 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
394 amdgpu_ring_write(kiq_ring, seq);
395 } else {
396 amdgpu_ring_write(kiq_ring, 0);
397 amdgpu_ring_write(kiq_ring, 0);
398 amdgpu_ring_write(kiq_ring, 0);
399 }
400 }
401
gfx11_kiq_query_status(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,u64 addr,u64 seq)402 static void gfx11_kiq_query_status(struct amdgpu_ring *kiq_ring,
403 struct amdgpu_ring *ring,
404 u64 addr,
405 u64 seq)
406 {
407 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
408
409 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
410 amdgpu_ring_write(kiq_ring,
411 PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
412 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
413 PACKET3_QUERY_STATUS_COMMAND(2));
414 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
415 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
416 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
417 amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
418 amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
419 amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
420 amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
421 }
422
gfx11_kiq_invalidate_tlbs(struct amdgpu_ring * kiq_ring,uint16_t pasid,uint32_t flush_type,bool all_hub)423 static void gfx11_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
424 uint16_t pasid, uint32_t flush_type,
425 bool all_hub)
426 {
427 gfx_v11_0_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1);
428 }
429
430 static const struct kiq_pm4_funcs gfx_v11_0_kiq_pm4_funcs = {
431 .kiq_set_resources = gfx11_kiq_set_resources,
432 .kiq_map_queues = gfx11_kiq_map_queues,
433 .kiq_unmap_queues = gfx11_kiq_unmap_queues,
434 .kiq_query_status = gfx11_kiq_query_status,
435 .kiq_invalidate_tlbs = gfx11_kiq_invalidate_tlbs,
436 .set_resources_size = 8,
437 .map_queues_size = 7,
438 .unmap_queues_size = 6,
439 .query_status_size = 7,
440 .invalidate_tlbs_size = 2,
441 };
442
gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device * adev)443 static void gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
444 {
445 adev->gfx.kiq[0].pmf = &gfx_v11_0_kiq_pm4_funcs;
446 }
447
gfx_v11_0_init_golden_registers(struct amdgpu_device * adev)448 static void gfx_v11_0_init_golden_registers(struct amdgpu_device *adev)
449 {
450 if (amdgpu_sriov_vf(adev))
451 return;
452
453 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
454 case IP_VERSION(11, 0, 1):
455 case IP_VERSION(11, 0, 4):
456 soc15_program_register_sequence(adev,
457 golden_settings_gc_11_0_1,
458 (const u32)ARRAY_SIZE(golden_settings_gc_11_0_1));
459 break;
460 default:
461 break;
462 }
463 soc15_program_register_sequence(adev,
464 golden_settings_gc_11_0,
465 (const u32)ARRAY_SIZE(golden_settings_gc_11_0));
466
467 }
468
gfx_v11_0_write_data_to_reg(struct amdgpu_ring * ring,int eng_sel,bool wc,uint32_t reg,uint32_t val)469 static void gfx_v11_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
470 bool wc, uint32_t reg, uint32_t val)
471 {
472 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
473 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
474 WRITE_DATA_DST_SEL(0) | (wc ? WR_CONFIRM : 0));
475 amdgpu_ring_write(ring, reg);
476 amdgpu_ring_write(ring, 0);
477 amdgpu_ring_write(ring, val);
478 }
479
gfx_v11_0_wait_reg_mem(struct amdgpu_ring * ring,int eng_sel,int mem_space,int opt,uint32_t addr0,uint32_t addr1,uint32_t ref,uint32_t mask,uint32_t inv)480 static void gfx_v11_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
481 int mem_space, int opt, uint32_t addr0,
482 uint32_t addr1, uint32_t ref, uint32_t mask,
483 uint32_t inv)
484 {
485 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
486 amdgpu_ring_write(ring,
487 /* memory (1) or register (0) */
488 (WAIT_REG_MEM_MEM_SPACE(mem_space) |
489 WAIT_REG_MEM_OPERATION(opt) | /* wait */
490 WAIT_REG_MEM_FUNCTION(3) | /* equal */
491 WAIT_REG_MEM_ENGINE(eng_sel)));
492
493 if (mem_space)
494 BUG_ON(addr0 & 0x3); /* Dword align */
495 amdgpu_ring_write(ring, addr0);
496 amdgpu_ring_write(ring, addr1);
497 amdgpu_ring_write(ring, ref);
498 amdgpu_ring_write(ring, mask);
499 amdgpu_ring_write(ring, inv); /* poll interval */
500 }
501
gfx_v11_ring_insert_nop(struct amdgpu_ring * ring,uint32_t num_nop)502 static void gfx_v11_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop)
503 {
504 int i;
505
506 /* Header itself is a NOP packet */
507 if (num_nop == 1) {
508 amdgpu_ring_write(ring, ring->funcs->nop);
509 return;
510 }
511
512 /* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/
513 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe)));
514
515 /* Header is at index 0, followed by num_nops - 1 NOP packet's */
516 for (i = 1; i < num_nop; i++)
517 amdgpu_ring_write(ring, ring->funcs->nop);
518 }
519
gfx_v11_0_ring_test_ring(struct amdgpu_ring * ring)520 static int gfx_v11_0_ring_test_ring(struct amdgpu_ring *ring)
521 {
522 struct amdgpu_device *adev = ring->adev;
523 uint32_t scratch = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
524 uint32_t tmp = 0;
525 unsigned i;
526 int r;
527
528 WREG32(scratch, 0xCAFEDEAD);
529 r = amdgpu_ring_alloc(ring, 5);
530 if (r) {
531 DRM_ERROR("amdgpu: cp failed to lock ring %d (%d).\n",
532 ring->idx, r);
533 return r;
534 }
535
536 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) {
537 gfx_v11_0_ring_emit_wreg(ring, scratch, 0xDEADBEEF);
538 } else {
539 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
540 amdgpu_ring_write(ring, scratch -
541 PACKET3_SET_UCONFIG_REG_START);
542 amdgpu_ring_write(ring, 0xDEADBEEF);
543 }
544 amdgpu_ring_commit(ring);
545
546 for (i = 0; i < adev->usec_timeout; i++) {
547 tmp = RREG32(scratch);
548 if (tmp == 0xDEADBEEF)
549 break;
550 if (amdgpu_emu_mode == 1)
551 msleep(1);
552 else
553 udelay(1);
554 }
555
556 if (i >= adev->usec_timeout)
557 r = -ETIMEDOUT;
558 return r;
559 }
560
gfx_v11_0_ring_test_ib(struct amdgpu_ring * ring,long timeout)561 static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
562 {
563 struct amdgpu_device *adev = ring->adev;
564 struct amdgpu_ib ib;
565 struct dma_fence *f = NULL;
566 unsigned index;
567 uint64_t gpu_addr;
568 volatile uint32_t *cpu_ptr;
569 long r;
570
571 /* MES KIQ fw hasn't indirect buffer support for now */
572 if (adev->enable_mes_kiq &&
573 ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
574 return 0;
575
576 memset(&ib, 0, sizeof(ib));
577
578 if (ring->is_mes_queue) {
579 uint32_t padding, offset;
580
581 offset = amdgpu_mes_ctx_get_offs(ring, AMDGPU_MES_CTX_IB_OFFS);
582 padding = amdgpu_mes_ctx_get_offs(ring,
583 AMDGPU_MES_CTX_PADDING_OFFS);
584
585 ib.gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
586 ib.ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
587
588 gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, padding);
589 cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, padding);
590 *cpu_ptr = cpu_to_le32(0xCAFEDEAD);
591 } else {
592 r = amdgpu_device_wb_get(adev, &index);
593 if (r)
594 return r;
595
596 gpu_addr = adev->wb.gpu_addr + (index * 4);
597 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
598 cpu_ptr = &adev->wb.wb[index];
599
600 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);
601 if (r) {
602 DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
603 goto err1;
604 }
605 }
606
607 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
608 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
609 ib.ptr[2] = lower_32_bits(gpu_addr);
610 ib.ptr[3] = upper_32_bits(gpu_addr);
611 ib.ptr[4] = 0xDEADBEEF;
612 ib.length_dw = 5;
613
614 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
615 if (r)
616 goto err2;
617
618 r = dma_fence_wait_timeout(f, false, timeout);
619 if (r == 0) {
620 r = -ETIMEDOUT;
621 goto err2;
622 } else if (r < 0) {
623 goto err2;
624 }
625
626 if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF)
627 r = 0;
628 else
629 r = -EINVAL;
630 err2:
631 if (!ring->is_mes_queue)
632 amdgpu_ib_free(adev, &ib, NULL);
633 dma_fence_put(f);
634 err1:
635 if (!ring->is_mes_queue)
636 amdgpu_device_wb_free(adev, index);
637 return r;
638 }
639
gfx_v11_0_free_microcode(struct amdgpu_device * adev)640 static void gfx_v11_0_free_microcode(struct amdgpu_device *adev)
641 {
642 amdgpu_ucode_release(&adev->gfx.pfp_fw);
643 amdgpu_ucode_release(&adev->gfx.me_fw);
644 amdgpu_ucode_release(&adev->gfx.rlc_fw);
645 amdgpu_ucode_release(&adev->gfx.mec_fw);
646
647 kfree(adev->gfx.rlc.register_list_format);
648 }
649
gfx_v11_0_init_toc_microcode(struct amdgpu_device * adev,const char * ucode_prefix)650 static int gfx_v11_0_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix)
651 {
652 const struct psp_firmware_header_v1_0 *toc_hdr;
653 int err = 0;
654
655 err = amdgpu_ucode_request(adev, &adev->psp.toc_fw,
656 "amdgpu/%s_toc.bin", ucode_prefix);
657 if (err)
658 goto out;
659
660 toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
661 adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
662 adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
663 adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
664 adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
665 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
666 return 0;
667 out:
668 amdgpu_ucode_release(&adev->psp.toc_fw);
669 return err;
670 }
671
gfx_v11_0_check_fw_cp_gfx_shadow(struct amdgpu_device * adev)672 static void gfx_v11_0_check_fw_cp_gfx_shadow(struct amdgpu_device *adev)
673 {
674 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
675 case IP_VERSION(11, 0, 0):
676 case IP_VERSION(11, 0, 2):
677 case IP_VERSION(11, 0, 3):
678 if ((adev->gfx.me_fw_version >= 1505) &&
679 (adev->gfx.pfp_fw_version >= 1600) &&
680 (adev->gfx.mec_fw_version >= 512)) {
681 if (amdgpu_sriov_vf(adev))
682 adev->gfx.cp_gfx_shadow = true;
683 else
684 adev->gfx.cp_gfx_shadow = false;
685 }
686 break;
687 default:
688 adev->gfx.cp_gfx_shadow = false;
689 break;
690 }
691 }
692
gfx_v11_0_init_microcode(struct amdgpu_device * adev)693 static int gfx_v11_0_init_microcode(struct amdgpu_device *adev)
694 {
695 char ucode_prefix[25];
696 int err;
697 const struct rlc_firmware_header_v2_0 *rlc_hdr;
698 uint16_t version_major;
699 uint16_t version_minor;
700
701 DRM_DEBUG("\n");
702
703 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
704 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,
705 "amdgpu/%s_pfp.bin", ucode_prefix);
706 if (err)
707 goto out;
708 /* check pfp fw hdr version to decide if enable rs64 for gfx11.*/
709 adev->gfx.rs64_enable = amdgpu_ucode_hdr_version(
710 (union amdgpu_firmware_header *)
711 adev->gfx.pfp_fw->data, 2, 0);
712 if (adev->gfx.rs64_enable) {
713 dev_info(adev->dev, "CP RS64 enable\n");
714 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP);
715 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK);
716 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK);
717 } else {
718 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP);
719 }
720
721 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,
722 "amdgpu/%s_me.bin", ucode_prefix);
723 if (err)
724 goto out;
725 if (adev->gfx.rs64_enable) {
726 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME);
727 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK);
728 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK);
729 } else {
730 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME);
731 }
732
733 if (!amdgpu_sriov_vf(adev)) {
734 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 0) &&
735 adev->pdev->revision == 0xCE)
736 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
737 "amdgpu/gc_11_0_0_rlc_1.bin");
738 else if (amdgpu_is_kicker_fw(adev))
739 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
740 "amdgpu/%s_rlc_kicker.bin", ucode_prefix);
741 else
742 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
743 "amdgpu/%s_rlc.bin", ucode_prefix);
744 if (err)
745 goto out;
746 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
747 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
748 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
749 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
750 if (err)
751 goto out;
752 }
753
754 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
755 "amdgpu/%s_mec.bin", ucode_prefix);
756 if (err)
757 goto out;
758 if (adev->gfx.rs64_enable) {
759 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC);
760 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK);
761 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK);
762 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK);
763 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK);
764 } else {
765 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
766 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);
767 }
768
769 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
770 err = gfx_v11_0_init_toc_microcode(adev, ucode_prefix);
771
772 /* only one MEC for gfx 11.0.0. */
773 adev->gfx.mec2_fw = NULL;
774
775 gfx_v11_0_check_fw_cp_gfx_shadow(adev);
776
777 if (adev->gfx.imu.funcs && adev->gfx.imu.funcs->init_microcode) {
778 err = adev->gfx.imu.funcs->init_microcode(adev);
779 if (err)
780 DRM_ERROR("Failed to init imu firmware!\n");
781 return err;
782 }
783
784 out:
785 if (err) {
786 amdgpu_ucode_release(&adev->gfx.pfp_fw);
787 amdgpu_ucode_release(&adev->gfx.me_fw);
788 amdgpu_ucode_release(&adev->gfx.rlc_fw);
789 amdgpu_ucode_release(&adev->gfx.mec_fw);
790 }
791
792 return err;
793 }
794
gfx_v11_0_get_csb_size(struct amdgpu_device * adev)795 static u32 gfx_v11_0_get_csb_size(struct amdgpu_device *adev)
796 {
797 u32 count = 0;
798 const struct cs_section_def *sect = NULL;
799 const struct cs_extent_def *ext = NULL;
800
801 /* begin clear state */
802 count += 2;
803 /* context control state */
804 count += 3;
805
806 for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {
807 for (ext = sect->section; ext->extent != NULL; ++ext) {
808 if (sect->id == SECT_CONTEXT)
809 count += 2 + ext->reg_count;
810 else
811 return 0;
812 }
813 }
814
815 /* set PA_SC_TILE_STEERING_OVERRIDE */
816 count += 3;
817 /* end clear state */
818 count += 2;
819 /* clear state */
820 count += 2;
821
822 return count;
823 }
824
gfx_v11_0_get_csb_buffer(struct amdgpu_device * adev,volatile u32 * buffer)825 static void gfx_v11_0_get_csb_buffer(struct amdgpu_device *adev,
826 volatile u32 *buffer)
827 {
828 u32 count = 0, i;
829 const struct cs_section_def *sect = NULL;
830 const struct cs_extent_def *ext = NULL;
831 int ctx_reg_offset;
832
833 if (adev->gfx.rlc.cs_data == NULL)
834 return;
835 if (buffer == NULL)
836 return;
837
838 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
839 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
840
841 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
842 buffer[count++] = cpu_to_le32(0x80000000);
843 buffer[count++] = cpu_to_le32(0x80000000);
844
845 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
846 for (ext = sect->section; ext->extent != NULL; ++ext) {
847 if (sect->id == SECT_CONTEXT) {
848 buffer[count++] =
849 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
850 buffer[count++] = cpu_to_le32(ext->reg_index -
851 PACKET3_SET_CONTEXT_REG_START);
852 for (i = 0; i < ext->reg_count; i++)
853 buffer[count++] = cpu_to_le32(ext->extent[i]);
854 } else {
855 return;
856 }
857 }
858 }
859
860 ctx_reg_offset =
861 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
862 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
863 buffer[count++] = cpu_to_le32(ctx_reg_offset);
864 buffer[count++] = cpu_to_le32(adev->gfx.config.pa_sc_tile_steering_override);
865
866 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
867 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
868
869 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
870 buffer[count++] = cpu_to_le32(0);
871 }
872
gfx_v11_0_rlc_fini(struct amdgpu_device * adev)873 static void gfx_v11_0_rlc_fini(struct amdgpu_device *adev)
874 {
875 /* clear state block */
876 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
877 &adev->gfx.rlc.clear_state_gpu_addr,
878 (void **)&adev->gfx.rlc.cs_ptr);
879
880 /* jump table block */
881 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
882 &adev->gfx.rlc.cp_table_gpu_addr,
883 (void **)&adev->gfx.rlc.cp_table_ptr);
884 }
885
gfx_v11_0_init_rlcg_reg_access_ctrl(struct amdgpu_device * adev)886 static void gfx_v11_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
887 {
888 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
889
890 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];
891 reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
892 reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG1);
893 reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG2);
894 reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG3);
895 reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_CNTL);
896 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX);
897 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, regRLC_SPARE_INT_0);
898 adev->gfx.rlc.rlcg_reg_access_supported = true;
899 }
900
gfx_v11_0_rlc_init(struct amdgpu_device * adev)901 static int gfx_v11_0_rlc_init(struct amdgpu_device *adev)
902 {
903 const struct cs_section_def *cs_data;
904 int r;
905
906 adev->gfx.rlc.cs_data = gfx11_cs_data;
907
908 cs_data = adev->gfx.rlc.cs_data;
909
910 if (cs_data) {
911 /* init clear state block */
912 r = amdgpu_gfx_rlc_init_csb(adev);
913 if (r)
914 return r;
915 }
916
917 /* init spm vmid with 0xf */
918 if (adev->gfx.rlc.funcs->update_spm_vmid)
919 adev->gfx.rlc.funcs->update_spm_vmid(adev, NULL, 0xf);
920
921 return 0;
922 }
923
gfx_v11_0_mec_fini(struct amdgpu_device * adev)924 static void gfx_v11_0_mec_fini(struct amdgpu_device *adev)
925 {
926 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
927 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
928 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL);
929 }
930
gfx_v11_0_me_init(struct amdgpu_device * adev)931 static void gfx_v11_0_me_init(struct amdgpu_device *adev)
932 {
933 bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
934
935 amdgpu_gfx_graphics_queue_acquire(adev);
936 }
937
gfx_v11_0_mec_init(struct amdgpu_device * adev)938 static int gfx_v11_0_mec_init(struct amdgpu_device *adev)
939 {
940 int r;
941 u32 *hpd;
942 size_t mec_hpd_size;
943
944 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
945
946 /* take ownership of the relevant compute queues */
947 amdgpu_gfx_compute_queue_acquire(adev);
948 mec_hpd_size = adev->gfx.num_compute_rings * GFX11_MEC_HPD_SIZE;
949
950 if (mec_hpd_size) {
951 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
952 AMDGPU_GEM_DOMAIN_GTT,
953 &adev->gfx.mec.hpd_eop_obj,
954 &adev->gfx.mec.hpd_eop_gpu_addr,
955 (void **)&hpd);
956 if (r) {
957 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
958 gfx_v11_0_mec_fini(adev);
959 return r;
960 }
961
962 memset(hpd, 0, mec_hpd_size);
963
964 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
965 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
966 }
967
968 return 0;
969 }
970
wave_read_ind(struct amdgpu_device * adev,uint32_t wave,uint32_t address)971 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address)
972 {
973 WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
974 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
975 (address << SQ_IND_INDEX__INDEX__SHIFT));
976 return RREG32_SOC15(GC, 0, regSQ_IND_DATA);
977 }
978
wave_read_regs(struct amdgpu_device * adev,uint32_t wave,uint32_t thread,uint32_t regno,uint32_t num,uint32_t * out)979 static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave,
980 uint32_t thread, uint32_t regno,
981 uint32_t num, uint32_t *out)
982 {
983 WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
984 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
985 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
986 (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) |
987 (SQ_IND_INDEX__AUTO_INCR_MASK));
988 while (num--)
989 *(out++) = RREG32_SOC15(GC, 0, regSQ_IND_DATA);
990 }
991
gfx_v11_0_read_wave_data(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t * dst,int * no_fields)992 static void gfx_v11_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
993 {
994 /* in gfx11 the SIMD_ID is specified as part of the INSTANCE
995 * field when performing a select_se_sh so it should be
996 * zero here */
997 WARN_ON(simd != 0);
998
999 /* type 3 wave data */
1000 dst[(*no_fields)++] = 3;
1001 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATUS);
1002 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_LO);
1003 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_HI);
1004 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_LO);
1005 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_HI);
1006 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID1);
1007 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID2);
1008 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_GPR_ALLOC);
1009 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_LDS_ALLOC);
1010 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_TRAPSTS);
1011 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS);
1012 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2);
1013 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1);
1014 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0);
1015 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_MODE);
1016 }
1017
gfx_v11_0_read_wave_sgprs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t start,uint32_t size,uint32_t * dst)1018 static void gfx_v11_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1019 uint32_t wave, uint32_t start,
1020 uint32_t size, uint32_t *dst)
1021 {
1022 WARN_ON(simd != 0);
1023
1024 wave_read_regs(
1025 adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size,
1026 dst);
1027 }
1028
gfx_v11_0_read_wave_vgprs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t thread,uint32_t start,uint32_t size,uint32_t * dst)1029 static void gfx_v11_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1030 uint32_t wave, uint32_t thread,
1031 uint32_t start, uint32_t size,
1032 uint32_t *dst)
1033 {
1034 wave_read_regs(
1035 adev, wave, thread,
1036 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
1037 }
1038
gfx_v11_0_select_me_pipe_q(struct amdgpu_device * adev,u32 me,u32 pipe,u32 q,u32 vm,u32 xcc_id)1039 static void gfx_v11_0_select_me_pipe_q(struct amdgpu_device *adev,
1040 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
1041 {
1042 soc21_grbm_select(adev, me, pipe, q, vm);
1043 }
1044
1045 /* all sizes are in bytes */
1046 #define MQD_SHADOW_BASE_SIZE 73728
1047 #define MQD_SHADOW_BASE_ALIGNMENT 256
1048 #define MQD_FWWORKAREA_SIZE 484
1049 #define MQD_FWWORKAREA_ALIGNMENT 256
1050
gfx_v11_0_get_gfx_shadow_info(struct amdgpu_device * adev,struct amdgpu_gfx_shadow_info * shadow_info)1051 static int gfx_v11_0_get_gfx_shadow_info(struct amdgpu_device *adev,
1052 struct amdgpu_gfx_shadow_info *shadow_info)
1053 {
1054 if (adev->gfx.cp_gfx_shadow) {
1055 shadow_info->shadow_size = MQD_SHADOW_BASE_SIZE;
1056 shadow_info->shadow_alignment = MQD_SHADOW_BASE_ALIGNMENT;
1057 shadow_info->csa_size = MQD_FWWORKAREA_SIZE;
1058 shadow_info->csa_alignment = MQD_FWWORKAREA_ALIGNMENT;
1059 return 0;
1060 } else {
1061 memset(shadow_info, 0, sizeof(struct amdgpu_gfx_shadow_info));
1062 return -ENOTSUPP;
1063 }
1064 }
1065
1066 static const struct amdgpu_gfx_funcs gfx_v11_0_gfx_funcs = {
1067 .get_gpu_clock_counter = &gfx_v11_0_get_gpu_clock_counter,
1068 .select_se_sh = &gfx_v11_0_select_se_sh,
1069 .read_wave_data = &gfx_v11_0_read_wave_data,
1070 .read_wave_sgprs = &gfx_v11_0_read_wave_sgprs,
1071 .read_wave_vgprs = &gfx_v11_0_read_wave_vgprs,
1072 .select_me_pipe_q = &gfx_v11_0_select_me_pipe_q,
1073 .update_perfmon_mgcg = &gfx_v11_0_update_perf_clk,
1074 .get_gfx_shadow_info = &gfx_v11_0_get_gfx_shadow_info,
1075 };
1076
gfx_v11_0_gpu_early_init(struct amdgpu_device * adev)1077 static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev)
1078 {
1079 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1080 case IP_VERSION(11, 0, 0):
1081 case IP_VERSION(11, 0, 2):
1082 adev->gfx.config.max_hw_contexts = 8;
1083 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1084 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1085 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
1086 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1087 break;
1088 case IP_VERSION(11, 0, 3):
1089 adev->gfx.ras = &gfx_v11_0_3_ras;
1090 adev->gfx.config.max_hw_contexts = 8;
1091 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1092 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1093 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
1094 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1095 break;
1096 case IP_VERSION(11, 0, 1):
1097 case IP_VERSION(11, 0, 4):
1098 case IP_VERSION(11, 5, 0):
1099 case IP_VERSION(11, 5, 1):
1100 case IP_VERSION(11, 5, 2):
1101 adev->gfx.config.max_hw_contexts = 8;
1102 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1103 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1104 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
1105 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x300;
1106 break;
1107 default:
1108 BUG();
1109 break;
1110 }
1111
1112 return 0;
1113 }
1114
gfx_v11_0_gfx_ring_init(struct amdgpu_device * adev,int ring_id,int me,int pipe,int queue)1115 static int gfx_v11_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,
1116 int me, int pipe, int queue)
1117 {
1118 struct amdgpu_ring *ring;
1119 unsigned int irq_type;
1120 unsigned int hw_prio;
1121
1122 ring = &adev->gfx.gfx_ring[ring_id];
1123
1124 ring->me = me;
1125 ring->pipe = pipe;
1126 ring->queue = queue;
1127
1128 ring->ring_obj = NULL;
1129 ring->use_doorbell = true;
1130
1131 if (!ring_id)
1132 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
1133 else
1134 ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1;
1135 ring->vm_hub = AMDGPU_GFXHUB(0);
1136 sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1137
1138 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe;
1139 hw_prio = amdgpu_gfx_is_high_priority_graphics_queue(adev, ring) ?
1140 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
1141 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
1142 hw_prio, NULL);
1143 }
1144
gfx_v11_0_compute_ring_init(struct amdgpu_device * adev,int ring_id,int mec,int pipe,int queue)1145 static int gfx_v11_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
1146 int mec, int pipe, int queue)
1147 {
1148 int r;
1149 unsigned irq_type;
1150 struct amdgpu_ring *ring;
1151 unsigned int hw_prio;
1152
1153 ring = &adev->gfx.compute_ring[ring_id];
1154
1155 /* mec0 is me1 */
1156 ring->me = mec + 1;
1157 ring->pipe = pipe;
1158 ring->queue = queue;
1159
1160 ring->ring_obj = NULL;
1161 ring->use_doorbell = true;
1162 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
1163 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
1164 + (ring_id * GFX11_MEC_HPD_SIZE);
1165 ring->vm_hub = AMDGPU_GFXHUB(0);
1166 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1167
1168 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
1169 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
1170 + ring->pipe;
1171 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
1172 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
1173 /* type-2 packets are deprecated on MEC, use type-3 instead */
1174 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
1175 hw_prio, NULL);
1176 if (r)
1177 return r;
1178
1179 return 0;
1180 }
1181
1182 static struct {
1183 SOC21_FIRMWARE_ID id;
1184 unsigned int offset;
1185 unsigned int size;
1186 } rlc_autoload_info[SOC21_FIRMWARE_ID_MAX];
1187
gfx_v11_0_parse_rlc_toc(struct amdgpu_device * adev,void * rlc_toc)1188 static void gfx_v11_0_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc)
1189 {
1190 RLC_TABLE_OF_CONTENT *ucode = rlc_toc;
1191
1192 while (ucode && (ucode->id > SOC21_FIRMWARE_ID_INVALID) &&
1193 (ucode->id < SOC21_FIRMWARE_ID_MAX)) {
1194 rlc_autoload_info[ucode->id].id = ucode->id;
1195 rlc_autoload_info[ucode->id].offset = ucode->offset * 4;
1196 rlc_autoload_info[ucode->id].size = ucode->size * 4;
1197
1198 ucode++;
1199 }
1200 }
1201
gfx_v11_0_calc_toc_total_size(struct amdgpu_device * adev)1202 static uint32_t gfx_v11_0_calc_toc_total_size(struct amdgpu_device *adev)
1203 {
1204 uint32_t total_size = 0;
1205 SOC21_FIRMWARE_ID id;
1206
1207 gfx_v11_0_parse_rlc_toc(adev, adev->psp.toc.start_addr);
1208
1209 for (id = SOC21_FIRMWARE_ID_RLC_G_UCODE; id < SOC21_FIRMWARE_ID_MAX; id++)
1210 total_size += rlc_autoload_info[id].size;
1211
1212 /* In case the offset in rlc toc ucode is aligned */
1213 if (total_size < rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset)
1214 total_size = rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset +
1215 rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].size;
1216
1217 return total_size;
1218 }
1219
gfx_v11_0_rlc_autoload_buffer_init(struct amdgpu_device * adev)1220 static int gfx_v11_0_rlc_autoload_buffer_init(struct amdgpu_device *adev)
1221 {
1222 int r;
1223 uint32_t total_size;
1224
1225 total_size = gfx_v11_0_calc_toc_total_size(adev);
1226
1227 r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024,
1228 AMDGPU_GEM_DOMAIN_VRAM |
1229 AMDGPU_GEM_DOMAIN_GTT,
1230 &adev->gfx.rlc.rlc_autoload_bo,
1231 &adev->gfx.rlc.rlc_autoload_gpu_addr,
1232 (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1233
1234 if (r) {
1235 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r);
1236 return r;
1237 }
1238
1239 return 0;
1240 }
1241
gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device * adev,SOC21_FIRMWARE_ID id,const void * fw_data,uint32_t fw_size,uint32_t * fw_autoload_mask)1242 static void gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev,
1243 SOC21_FIRMWARE_ID id,
1244 const void *fw_data,
1245 uint32_t fw_size,
1246 uint32_t *fw_autoload_mask)
1247 {
1248 uint32_t toc_offset;
1249 uint32_t toc_fw_size;
1250 char *ptr = adev->gfx.rlc.rlc_autoload_ptr;
1251
1252 if (id <= SOC21_FIRMWARE_ID_INVALID || id >= SOC21_FIRMWARE_ID_MAX)
1253 return;
1254
1255 toc_offset = rlc_autoload_info[id].offset;
1256 toc_fw_size = rlc_autoload_info[id].size;
1257
1258 if (fw_size == 0)
1259 fw_size = toc_fw_size;
1260
1261 if (fw_size > toc_fw_size)
1262 fw_size = toc_fw_size;
1263
1264 memcpy(ptr + toc_offset, fw_data, fw_size);
1265
1266 if (fw_size < toc_fw_size)
1267 memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size);
1268
1269 if ((id != SOC21_FIRMWARE_ID_RS64_PFP) && (id != SOC21_FIRMWARE_ID_RS64_ME))
1270 *(uint64_t *)fw_autoload_mask |= 1ULL << id;
1271 }
1272
gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device * adev,uint32_t * fw_autoload_mask)1273 static void gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev,
1274 uint32_t *fw_autoload_mask)
1275 {
1276 void *data;
1277 uint32_t size;
1278 uint64_t *toc_ptr;
1279
1280 *(uint64_t *)fw_autoload_mask |= 0x1;
1281
1282 DRM_DEBUG("rlc autoload enabled fw: 0x%llx\n", *(uint64_t *)fw_autoload_mask);
1283
1284 data = adev->psp.toc.start_addr;
1285 size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_TOC].size;
1286
1287 toc_ptr = (uint64_t *)data + size / 8 - 1;
1288 *toc_ptr = *(uint64_t *)fw_autoload_mask;
1289
1290 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_TOC,
1291 data, size, fw_autoload_mask);
1292 }
1293
gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device * adev,uint32_t * fw_autoload_mask)1294 static void gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev,
1295 uint32_t *fw_autoload_mask)
1296 {
1297 const __le32 *fw_data;
1298 uint32_t fw_size;
1299 const struct gfx_firmware_header_v1_0 *cp_hdr;
1300 const struct gfx_firmware_header_v2_0 *cpv2_hdr;
1301 const struct rlc_firmware_header_v2_0 *rlc_hdr;
1302 const struct rlc_firmware_header_v2_2 *rlcv22_hdr;
1303 uint16_t version_major, version_minor;
1304
1305 if (adev->gfx.rs64_enable) {
1306 /* pfp ucode */
1307 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1308 adev->gfx.pfp_fw->data;
1309 /* instruction */
1310 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1311 le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1312 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1313 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP,
1314 fw_data, fw_size, fw_autoload_mask);
1315 /* data */
1316 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1317 le32_to_cpu(cpv2_hdr->data_offset_bytes));
1318 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1319 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK,
1320 fw_data, fw_size, fw_autoload_mask);
1321 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P1_STACK,
1322 fw_data, fw_size, fw_autoload_mask);
1323 /* me ucode */
1324 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1325 adev->gfx.me_fw->data;
1326 /* instruction */
1327 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1328 le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1329 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1330 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME,
1331 fw_data, fw_size, fw_autoload_mask);
1332 /* data */
1333 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1334 le32_to_cpu(cpv2_hdr->data_offset_bytes));
1335 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1336 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P0_STACK,
1337 fw_data, fw_size, fw_autoload_mask);
1338 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P1_STACK,
1339 fw_data, fw_size, fw_autoload_mask);
1340 /* mec ucode */
1341 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1342 adev->gfx.mec_fw->data;
1343 /* instruction */
1344 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1345 le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1346 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1347 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC,
1348 fw_data, fw_size, fw_autoload_mask);
1349 /* data */
1350 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1351 le32_to_cpu(cpv2_hdr->data_offset_bytes));
1352 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1353 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK,
1354 fw_data, fw_size, fw_autoload_mask);
1355 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P1_STACK,
1356 fw_data, fw_size, fw_autoload_mask);
1357 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P2_STACK,
1358 fw_data, fw_size, fw_autoload_mask);
1359 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P3_STACK,
1360 fw_data, fw_size, fw_autoload_mask);
1361 } else {
1362 /* pfp ucode */
1363 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1364 adev->gfx.pfp_fw->data;
1365 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1366 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1367 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1368 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_PFP,
1369 fw_data, fw_size, fw_autoload_mask);
1370
1371 /* me ucode */
1372 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1373 adev->gfx.me_fw->data;
1374 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1375 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1376 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1377 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_ME,
1378 fw_data, fw_size, fw_autoload_mask);
1379
1380 /* mec ucode */
1381 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1382 adev->gfx.mec_fw->data;
1383 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1384 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1385 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
1386 cp_hdr->jt_size * 4;
1387 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_MEC,
1388 fw_data, fw_size, fw_autoload_mask);
1389 }
1390
1391 /* rlc ucode */
1392 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)
1393 adev->gfx.rlc_fw->data;
1394 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1395 le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes));
1396 fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes);
1397 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_G_UCODE,
1398 fw_data, fw_size, fw_autoload_mask);
1399
1400 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1401 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1402 if (version_major == 2) {
1403 if (version_minor >= 2) {
1404 rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1405
1406 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1407 le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes));
1408 fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes);
1409 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_UCODE,
1410 fw_data, fw_size, fw_autoload_mask);
1411
1412 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1413 le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes));
1414 fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes);
1415 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_DRAM_BOOT,
1416 fw_data, fw_size, fw_autoload_mask);
1417 }
1418 }
1419 }
1420
gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device * adev,uint32_t * fw_autoload_mask)1421 static void gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev,
1422 uint32_t *fw_autoload_mask)
1423 {
1424 const __le32 *fw_data;
1425 uint32_t fw_size;
1426 const struct sdma_firmware_header_v2_0 *sdma_hdr;
1427
1428 sdma_hdr = (const struct sdma_firmware_header_v2_0 *)
1429 adev->sdma.instance[0].fw->data;
1430 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1431 le32_to_cpu(sdma_hdr->header.ucode_array_offset_bytes));
1432 fw_size = le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes);
1433
1434 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1435 SOC21_FIRMWARE_ID_SDMA_UCODE_TH0, fw_data, fw_size, fw_autoload_mask);
1436
1437 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1438 le32_to_cpu(sdma_hdr->ctl_ucode_offset));
1439 fw_size = le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes);
1440
1441 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1442 SOC21_FIRMWARE_ID_SDMA_UCODE_TH1, fw_data, fw_size, fw_autoload_mask);
1443 }
1444
gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device * adev,uint32_t * fw_autoload_mask)1445 static void gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev,
1446 uint32_t *fw_autoload_mask)
1447 {
1448 const __le32 *fw_data;
1449 unsigned fw_size;
1450 const struct mes_firmware_header_v1_0 *mes_hdr;
1451 int pipe, ucode_id, data_id;
1452
1453 for (pipe = 0; pipe < 2; pipe++) {
1454 if (pipe==0) {
1455 ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P0;
1456 data_id = SOC21_FIRMWARE_ID_RS64_MES_P0_STACK;
1457 } else {
1458 ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P1;
1459 data_id = SOC21_FIRMWARE_ID_RS64_MES_P1_STACK;
1460 }
1461
1462 mes_hdr = (const struct mes_firmware_header_v1_0 *)
1463 adev->mes.fw[pipe]->data;
1464
1465 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1466 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
1467 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
1468
1469 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1470 ucode_id, fw_data, fw_size, fw_autoload_mask);
1471
1472 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1473 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
1474 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
1475
1476 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1477 data_id, fw_data, fw_size, fw_autoload_mask);
1478 }
1479 }
1480
gfx_v11_0_rlc_backdoor_autoload_enable(struct amdgpu_device * adev)1481 static int gfx_v11_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
1482 {
1483 uint32_t rlc_g_offset, rlc_g_size;
1484 uint64_t gpu_addr;
1485 uint32_t autoload_fw_id[2];
1486
1487 memset(autoload_fw_id, 0, sizeof(uint32_t) * 2);
1488
1489 /* RLC autoload sequence 2: copy ucode */
1490 gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(adev, autoload_fw_id);
1491 gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(adev, autoload_fw_id);
1492 gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(adev, autoload_fw_id);
1493 gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(adev, autoload_fw_id);
1494
1495 rlc_g_offset = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].offset;
1496 rlc_g_size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].size;
1497 gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset;
1498
1499 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, upper_32_bits(gpu_addr));
1500 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, lower_32_bits(gpu_addr));
1501
1502 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_SIZE, rlc_g_size);
1503
1504 /* RLC autoload sequence 3: load IMU fw */
1505 if (adev->gfx.imu.funcs->load_microcode)
1506 adev->gfx.imu.funcs->load_microcode(adev);
1507 /* RLC autoload sequence 4 init IMU fw */
1508 if (adev->gfx.imu.funcs->setup_imu)
1509 adev->gfx.imu.funcs->setup_imu(adev);
1510 if (adev->gfx.imu.funcs->start_imu)
1511 adev->gfx.imu.funcs->start_imu(adev);
1512
1513 /* RLC autoload sequence 5 disable gpa mode */
1514 gfx_v11_0_disable_gpa_mode(adev);
1515
1516 return 0;
1517 }
1518
gfx_v11_0_alloc_ip_dump(struct amdgpu_device * adev)1519 static void gfx_v11_0_alloc_ip_dump(struct amdgpu_device *adev)
1520 {
1521 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0);
1522 uint32_t *ptr;
1523 uint32_t inst;
1524
1525 ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL);
1526 if (!ptr) {
1527 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n");
1528 adev->gfx.ip_dump_core = NULL;
1529 } else {
1530 adev->gfx.ip_dump_core = ptr;
1531 }
1532
1533 /* Allocate memory for compute queue registers for all the instances */
1534 reg_count = ARRAY_SIZE(gc_cp_reg_list_11);
1535 inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
1536 adev->gfx.mec.num_queue_per_pipe;
1537
1538 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);
1539 if (!ptr) {
1540 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n");
1541 adev->gfx.ip_dump_compute_queues = NULL;
1542 } else {
1543 adev->gfx.ip_dump_compute_queues = ptr;
1544 }
1545
1546 /* Allocate memory for gfx queue registers for all the instances */
1547 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11);
1548 inst = adev->gfx.me.num_me * adev->gfx.me.num_pipe_per_me *
1549 adev->gfx.me.num_queue_per_pipe;
1550
1551 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);
1552 if (!ptr) {
1553 DRM_ERROR("Failed to allocate memory for GFX Queues IP Dump\n");
1554 adev->gfx.ip_dump_gfx_queues = NULL;
1555 } else {
1556 adev->gfx.ip_dump_gfx_queues = ptr;
1557 }
1558 }
1559
gfx_v11_0_sw_init(void * handle)1560 static int gfx_v11_0_sw_init(void *handle)
1561 {
1562 int i, j, k, r, ring_id = 0;
1563 int xcc_id = 0;
1564 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1565
1566 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1567 case IP_VERSION(11, 0, 0):
1568 case IP_VERSION(11, 0, 2):
1569 case IP_VERSION(11, 0, 3):
1570 adev->gfx.me.num_me = 1;
1571 adev->gfx.me.num_pipe_per_me = 1;
1572 adev->gfx.me.num_queue_per_pipe = 1;
1573 adev->gfx.mec.num_mec = 1;
1574 adev->gfx.mec.num_pipe_per_mec = 4;
1575 adev->gfx.mec.num_queue_per_pipe = 4;
1576 break;
1577 case IP_VERSION(11, 0, 1):
1578 case IP_VERSION(11, 0, 4):
1579 case IP_VERSION(11, 5, 0):
1580 case IP_VERSION(11, 5, 1):
1581 case IP_VERSION(11, 5, 2):
1582 adev->gfx.me.num_me = 1;
1583 adev->gfx.me.num_pipe_per_me = 1;
1584 adev->gfx.me.num_queue_per_pipe = 1;
1585 adev->gfx.mec.num_mec = 1;
1586 adev->gfx.mec.num_pipe_per_mec = 4;
1587 adev->gfx.mec.num_queue_per_pipe = 4;
1588 break;
1589 default:
1590 adev->gfx.me.num_me = 1;
1591 adev->gfx.me.num_pipe_per_me = 1;
1592 adev->gfx.me.num_queue_per_pipe = 1;
1593 adev->gfx.mec.num_mec = 1;
1594 adev->gfx.mec.num_pipe_per_mec = 4;
1595 adev->gfx.mec.num_queue_per_pipe = 8;
1596 break;
1597 }
1598
1599 /* Enable CG flag in one VF mode for enabling RLC safe mode enter/exit */
1600 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3) &&
1601 amdgpu_sriov_is_pp_one_vf(adev))
1602 adev->cg_flags = AMD_CG_SUPPORT_GFX_CGCG;
1603
1604 /* EOP Event */
1605 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1606 GFX_11_0_0__SRCID__CP_EOP_INTERRUPT,
1607 &adev->gfx.eop_irq);
1608 if (r)
1609 return r;
1610
1611 /* Bad opcode Event */
1612 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1613 GFX_11_0_0__SRCID__CP_BAD_OPCODE_ERROR,
1614 &adev->gfx.bad_op_irq);
1615 if (r)
1616 return r;
1617
1618 /* Privileged reg */
1619 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1620 GFX_11_0_0__SRCID__CP_PRIV_REG_FAULT,
1621 &adev->gfx.priv_reg_irq);
1622 if (r)
1623 return r;
1624
1625 /* Privileged inst */
1626 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1627 GFX_11_0_0__SRCID__CP_PRIV_INSTR_FAULT,
1628 &adev->gfx.priv_inst_irq);
1629 if (r)
1630 return r;
1631
1632 /* FED error */
1633 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX,
1634 GFX_11_0_0__SRCID__RLC_GC_FED_INTERRUPT,
1635 &adev->gfx.rlc_gc_fed_irq);
1636 if (r)
1637 return r;
1638
1639 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1640
1641 gfx_v11_0_me_init(adev);
1642
1643 r = gfx_v11_0_rlc_init(adev);
1644 if (r) {
1645 DRM_ERROR("Failed to init rlc BOs!\n");
1646 return r;
1647 }
1648
1649 r = gfx_v11_0_mec_init(adev);
1650 if (r) {
1651 DRM_ERROR("Failed to init MEC BOs!\n");
1652 return r;
1653 }
1654
1655 /* set up the gfx ring */
1656 for (i = 0; i < adev->gfx.me.num_me; i++) {
1657 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
1658 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
1659 if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
1660 continue;
1661
1662 r = gfx_v11_0_gfx_ring_init(adev, ring_id,
1663 i, k, j);
1664 if (r)
1665 return r;
1666 ring_id++;
1667 }
1668 }
1669 }
1670
1671 ring_id = 0;
1672 /* set up the compute queues - allocate horizontally across pipes */
1673 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
1674 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
1675 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
1676 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
1677 k, j))
1678 continue;
1679
1680 r = gfx_v11_0_compute_ring_init(adev, ring_id,
1681 i, k, j);
1682 if (r)
1683 return r;
1684
1685 ring_id++;
1686 }
1687 }
1688 }
1689
1690 if (!adev->enable_mes_kiq) {
1691 r = amdgpu_gfx_kiq_init(adev, GFX11_MEC_HPD_SIZE, 0);
1692 if (r) {
1693 DRM_ERROR("Failed to init KIQ BOs!\n");
1694 return r;
1695 }
1696
1697 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
1698 if (r)
1699 return r;
1700 }
1701
1702 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v11_compute_mqd), 0);
1703 if (r)
1704 return r;
1705
1706 /* allocate visible FB for rlc auto-loading fw */
1707 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1708 r = gfx_v11_0_rlc_autoload_buffer_init(adev);
1709 if (r)
1710 return r;
1711 }
1712
1713 r = gfx_v11_0_gpu_early_init(adev);
1714 if (r)
1715 return r;
1716
1717 if (amdgpu_gfx_ras_sw_init(adev)) {
1718 dev_err(adev->dev, "Failed to initialize gfx ras block!\n");
1719 return -EINVAL;
1720 }
1721
1722 gfx_v11_0_alloc_ip_dump(adev);
1723
1724 return 0;
1725 }
1726
gfx_v11_0_pfp_fini(struct amdgpu_device * adev)1727 static void gfx_v11_0_pfp_fini(struct amdgpu_device *adev)
1728 {
1729 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj,
1730 &adev->gfx.pfp.pfp_fw_gpu_addr,
1731 (void **)&adev->gfx.pfp.pfp_fw_ptr);
1732
1733 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_data_obj,
1734 &adev->gfx.pfp.pfp_fw_data_gpu_addr,
1735 (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
1736 }
1737
gfx_v11_0_me_fini(struct amdgpu_device * adev)1738 static void gfx_v11_0_me_fini(struct amdgpu_device *adev)
1739 {
1740 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
1741 &adev->gfx.me.me_fw_gpu_addr,
1742 (void **)&adev->gfx.me.me_fw_ptr);
1743
1744 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_data_obj,
1745 &adev->gfx.me.me_fw_data_gpu_addr,
1746 (void **)&adev->gfx.me.me_fw_data_ptr);
1747 }
1748
gfx_v11_0_rlc_autoload_buffer_fini(struct amdgpu_device * adev)1749 static void gfx_v11_0_rlc_autoload_buffer_fini(struct amdgpu_device *adev)
1750 {
1751 amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo,
1752 &adev->gfx.rlc.rlc_autoload_gpu_addr,
1753 (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1754 }
1755
gfx_v11_0_sw_fini(void * handle)1756 static int gfx_v11_0_sw_fini(void *handle)
1757 {
1758 int i;
1759 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1760
1761 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
1762 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
1763 for (i = 0; i < adev->gfx.num_compute_rings; i++)
1764 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
1765
1766 amdgpu_gfx_mqd_sw_fini(adev, 0);
1767
1768 if (!adev->enable_mes_kiq) {
1769 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);
1770 amdgpu_gfx_kiq_fini(adev, 0);
1771 }
1772
1773 gfx_v11_0_pfp_fini(adev);
1774 gfx_v11_0_me_fini(adev);
1775 gfx_v11_0_rlc_fini(adev);
1776 gfx_v11_0_mec_fini(adev);
1777
1778 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1779 gfx_v11_0_rlc_autoload_buffer_fini(adev);
1780
1781 gfx_v11_0_free_microcode(adev);
1782
1783 kfree(adev->gfx.ip_dump_core);
1784 kfree(adev->gfx.ip_dump_compute_queues);
1785 kfree(adev->gfx.ip_dump_gfx_queues);
1786
1787 return 0;
1788 }
1789
gfx_v11_0_select_se_sh(struct amdgpu_device * adev,u32 se_num,u32 sh_num,u32 instance,int xcc_id)1790 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
1791 u32 sh_num, u32 instance, int xcc_id)
1792 {
1793 u32 data;
1794
1795 if (instance == 0xffffffff)
1796 data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
1797 INSTANCE_BROADCAST_WRITES, 1);
1798 else
1799 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX,
1800 instance);
1801
1802 if (se_num == 0xffffffff)
1803 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES,
1804 1);
1805 else
1806 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
1807
1808 if (sh_num == 0xffffffff)
1809 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES,
1810 1);
1811 else
1812 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num);
1813
1814 WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX, data);
1815 }
1816
gfx_v11_0_get_sa_active_bitmap(struct amdgpu_device * adev)1817 static u32 gfx_v11_0_get_sa_active_bitmap(struct amdgpu_device *adev)
1818 {
1819 u32 gc_disabled_sa_mask, gc_user_disabled_sa_mask, sa_mask;
1820
1821 gc_disabled_sa_mask = RREG32_SOC15(GC, 0, regCC_GC_SA_UNIT_DISABLE);
1822 gc_disabled_sa_mask = REG_GET_FIELD(gc_disabled_sa_mask,
1823 CC_GC_SA_UNIT_DISABLE,
1824 SA_DISABLE);
1825 gc_user_disabled_sa_mask = RREG32_SOC15(GC, 0, regGC_USER_SA_UNIT_DISABLE);
1826 gc_user_disabled_sa_mask = REG_GET_FIELD(gc_user_disabled_sa_mask,
1827 GC_USER_SA_UNIT_DISABLE,
1828 SA_DISABLE);
1829 sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se *
1830 adev->gfx.config.max_shader_engines);
1831
1832 return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask));
1833 }
1834
gfx_v11_0_get_rb_active_bitmap(struct amdgpu_device * adev)1835 static u32 gfx_v11_0_get_rb_active_bitmap(struct amdgpu_device *adev)
1836 {
1837 u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask;
1838 u32 rb_mask;
1839
1840 gc_disabled_rb_mask = RREG32_SOC15(GC, 0, regCC_RB_BACKEND_DISABLE);
1841 gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask,
1842 CC_RB_BACKEND_DISABLE,
1843 BACKEND_DISABLE);
1844 gc_user_disabled_rb_mask = RREG32_SOC15(GC, 0, regGC_USER_RB_BACKEND_DISABLE);
1845 gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask,
1846 GC_USER_RB_BACKEND_DISABLE,
1847 BACKEND_DISABLE);
1848 rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se *
1849 adev->gfx.config.max_shader_engines);
1850
1851 return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask));
1852 }
1853
gfx_v11_0_setup_rb(struct amdgpu_device * adev)1854 static void gfx_v11_0_setup_rb(struct amdgpu_device *adev)
1855 {
1856 u32 rb_bitmap_width_per_sa;
1857 u32 max_sa;
1858 u32 active_sa_bitmap;
1859 u32 global_active_rb_bitmap;
1860 u32 active_rb_bitmap = 0;
1861 u32 i;
1862
1863 /* query sa bitmap from SA_UNIT_DISABLE registers */
1864 active_sa_bitmap = gfx_v11_0_get_sa_active_bitmap(adev);
1865 /* query rb bitmap from RB_BACKEND_DISABLE registers */
1866 global_active_rb_bitmap = gfx_v11_0_get_rb_active_bitmap(adev);
1867
1868 /* generate active rb bitmap according to active sa bitmap */
1869 max_sa = adev->gfx.config.max_shader_engines *
1870 adev->gfx.config.max_sh_per_se;
1871 rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se /
1872 adev->gfx.config.max_sh_per_se;
1873 for (i = 0; i < max_sa; i++) {
1874 if (active_sa_bitmap & (1 << i))
1875 active_rb_bitmap |= (0x3 << (i * rb_bitmap_width_per_sa));
1876 }
1877
1878 active_rb_bitmap &= global_active_rb_bitmap;
1879 adev->gfx.config.backend_enable_mask = active_rb_bitmap;
1880 adev->gfx.config.num_rbs = hweight32(active_rb_bitmap);
1881 }
1882
1883 #define DEFAULT_SH_MEM_BASES (0x6000)
1884 #define LDS_APP_BASE 0x1
1885 #define SCRATCH_APP_BASE 0x2
1886
gfx_v11_0_init_compute_vmid(struct amdgpu_device * adev)1887 static void gfx_v11_0_init_compute_vmid(struct amdgpu_device *adev)
1888 {
1889 int i;
1890 uint32_t sh_mem_bases;
1891 uint32_t data;
1892
1893 /*
1894 * Configure apertures:
1895 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
1896 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
1897 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
1898 */
1899 sh_mem_bases = (LDS_APP_BASE << SH_MEM_BASES__SHARED_BASE__SHIFT) |
1900 SCRATCH_APP_BASE;
1901
1902 mutex_lock(&adev->srbm_mutex);
1903 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1904 soc21_grbm_select(adev, 0, 0, 0, i);
1905 /* CP and shaders */
1906 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1907 WREG32_SOC15(GC, 0, regSH_MEM_BASES, sh_mem_bases);
1908
1909 /* Enable trap for each kfd vmid. */
1910 data = RREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL);
1911 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
1912 WREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL, data);
1913 }
1914 soc21_grbm_select(adev, 0, 0, 0, 0);
1915 mutex_unlock(&adev->srbm_mutex);
1916
1917 /* Initialize all compute VMIDs to have no GDS, GWS, or OA
1918 acccess. These should be enabled by FW for target VMIDs. */
1919 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1920 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * i, 0);
1921 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * i, 0);
1922 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, i, 0);
1923 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, i, 0);
1924 }
1925 }
1926
gfx_v11_0_init_gds_vmid(struct amdgpu_device * adev)1927 static void gfx_v11_0_init_gds_vmid(struct amdgpu_device *adev)
1928 {
1929 int vmid;
1930
1931 /*
1932 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
1933 * access. Compute VMIDs should be enabled by FW for target VMIDs,
1934 * the driver can enable them for graphics. VMID0 should maintain
1935 * access so that HWS firmware can save/restore entries.
1936 */
1937 for (vmid = 1; vmid < 16; vmid++) {
1938 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * vmid, 0);
1939 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * vmid, 0);
1940 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, vmid, 0);
1941 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, vmid, 0);
1942 }
1943 }
1944
gfx_v11_0_tcp_harvest(struct amdgpu_device * adev)1945 static void gfx_v11_0_tcp_harvest(struct amdgpu_device *adev)
1946 {
1947 /* TODO: harvest feature to be added later. */
1948 }
1949
gfx_v11_0_get_tcc_info(struct amdgpu_device * adev)1950 static void gfx_v11_0_get_tcc_info(struct amdgpu_device *adev)
1951 {
1952 /* TCCs are global (not instanced). */
1953 uint32_t tcc_disable = RREG32_SOC15(GC, 0, regCGTS_TCC_DISABLE) |
1954 RREG32_SOC15(GC, 0, regCGTS_USER_TCC_DISABLE);
1955
1956 adev->gfx.config.tcc_disabled_mask =
1957 REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, TCC_DISABLE) |
1958 (REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, HI_TCC_DISABLE) << 16);
1959 }
1960
gfx_v11_0_constants_init(struct amdgpu_device * adev)1961 static void gfx_v11_0_constants_init(struct amdgpu_device *adev)
1962 {
1963 u32 tmp;
1964 int i;
1965
1966 if (!amdgpu_sriov_vf(adev))
1967 WREG32_FIELD15_PREREG(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
1968
1969 gfx_v11_0_setup_rb(adev);
1970 gfx_v11_0_get_cu_info(adev, &adev->gfx.cu_info);
1971 gfx_v11_0_get_tcc_info(adev);
1972 adev->gfx.config.pa_sc_tile_steering_override = 0;
1973
1974 /* Set whether texture coordinate truncation is conformant. */
1975 tmp = RREG32_SOC15(GC, 0, regTA_CNTL2);
1976 adev->gfx.config.ta_cntl2_truncate_coord_mode =
1977 REG_GET_FIELD(tmp, TA_CNTL2, TRUNCATE_COORD_MODE);
1978
1979 /* XXX SH_MEM regs */
1980 /* where to put LDS, scratch, GPUVM in FSA64 space */
1981 mutex_lock(&adev->srbm_mutex);
1982 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
1983 soc21_grbm_select(adev, 0, 0, 0, i);
1984 /* CP and shaders */
1985 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1986 if (i != 0) {
1987 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
1988 (adev->gmc.private_aperture_start >> 48));
1989 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
1990 (adev->gmc.shared_aperture_start >> 48));
1991 WREG32_SOC15(GC, 0, regSH_MEM_BASES, tmp);
1992 }
1993 }
1994 soc21_grbm_select(adev, 0, 0, 0, 0);
1995
1996 mutex_unlock(&adev->srbm_mutex);
1997
1998 gfx_v11_0_init_compute_vmid(adev);
1999 gfx_v11_0_init_gds_vmid(adev);
2000 }
2001
gfx_v11_0_get_cpg_int_cntl(struct amdgpu_device * adev,int me,int pipe)2002 static u32 gfx_v11_0_get_cpg_int_cntl(struct amdgpu_device *adev,
2003 int me, int pipe)
2004 {
2005 if (me != 0)
2006 return 0;
2007
2008 switch (pipe) {
2009 case 0:
2010 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0);
2011 case 1:
2012 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1);
2013 default:
2014 return 0;
2015 }
2016 }
2017
gfx_v11_0_get_cpc_int_cntl(struct amdgpu_device * adev,int me,int pipe)2018 static u32 gfx_v11_0_get_cpc_int_cntl(struct amdgpu_device *adev,
2019 int me, int pipe)
2020 {
2021 /*
2022 * amdgpu controls only the first MEC. That's why this function only
2023 * handles the setting of interrupts for this specific MEC. All other
2024 * pipes' interrupts are set by amdkfd.
2025 */
2026 if (me != 1)
2027 return 0;
2028
2029 switch (pipe) {
2030 case 0:
2031 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
2032 case 1:
2033 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL);
2034 case 2:
2035 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL);
2036 case 3:
2037 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL);
2038 default:
2039 return 0;
2040 }
2041 }
2042
gfx_v11_0_enable_gui_idle_interrupt(struct amdgpu_device * adev,bool enable)2043 static void gfx_v11_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2044 bool enable)
2045 {
2046 u32 tmp, cp_int_cntl_reg;
2047 int i, j;
2048
2049 if (amdgpu_sriov_vf(adev))
2050 return;
2051
2052 for (i = 0; i < adev->gfx.me.num_me; i++) {
2053 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
2054 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
2055
2056 if (cp_int_cntl_reg) {
2057 tmp = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
2058 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE,
2059 enable ? 1 : 0);
2060 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE,
2061 enable ? 1 : 0);
2062 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE,
2063 enable ? 1 : 0);
2064 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE,
2065 enable ? 1 : 0);
2066 WREG32_SOC15_IP(GC, cp_int_cntl_reg, tmp);
2067 }
2068 }
2069 }
2070 }
2071
gfx_v11_0_init_csb(struct amdgpu_device * adev)2072 static int gfx_v11_0_init_csb(struct amdgpu_device *adev)
2073 {
2074 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2075
2076 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_HI,
2077 adev->gfx.rlc.clear_state_gpu_addr >> 32);
2078 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_LO,
2079 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2080 WREG32_SOC15(GC, 0, regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size);
2081
2082 return 0;
2083 }
2084
gfx_v11_0_rlc_stop(struct amdgpu_device * adev)2085 static void gfx_v11_0_rlc_stop(struct amdgpu_device *adev)
2086 {
2087 u32 tmp = RREG32_SOC15(GC, 0, regRLC_CNTL);
2088
2089 tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
2090 WREG32_SOC15(GC, 0, regRLC_CNTL, tmp);
2091 }
2092
gfx_v11_0_rlc_reset(struct amdgpu_device * adev)2093 static void gfx_v11_0_rlc_reset(struct amdgpu_device *adev)
2094 {
2095 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
2096 udelay(50);
2097 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
2098 udelay(50);
2099 }
2100
gfx_v11_0_rlc_smu_handshake_cntl(struct amdgpu_device * adev,bool enable)2101 static void gfx_v11_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev,
2102 bool enable)
2103 {
2104 uint32_t rlc_pg_cntl;
2105
2106 rlc_pg_cntl = RREG32_SOC15(GC, 0, regRLC_PG_CNTL);
2107
2108 if (!enable) {
2109 /* RLC_PG_CNTL[23] = 0 (default)
2110 * RLC will wait for handshake acks with SMU
2111 * GFXOFF will be enabled
2112 * RLC_PG_CNTL[23] = 1
2113 * RLC will not issue any message to SMU
2114 * hence no handshake between SMU & RLC
2115 * GFXOFF will be disabled
2116 */
2117 rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
2118 } else
2119 rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
2120 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, rlc_pg_cntl);
2121 }
2122
gfx_v11_0_rlc_start(struct amdgpu_device * adev)2123 static void gfx_v11_0_rlc_start(struct amdgpu_device *adev)
2124 {
2125 /* TODO: enable rlc & smu handshake until smu
2126 * and gfxoff feature works as expected */
2127 if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK))
2128 gfx_v11_0_rlc_smu_handshake_cntl(adev, false);
2129
2130 WREG32_FIELD15_PREREG(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
2131 udelay(50);
2132 }
2133
gfx_v11_0_rlc_enable_srm(struct amdgpu_device * adev)2134 static void gfx_v11_0_rlc_enable_srm(struct amdgpu_device *adev)
2135 {
2136 uint32_t tmp;
2137
2138 /* enable Save Restore Machine */
2139 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL));
2140 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2141 tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK;
2142 WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL), tmp);
2143 }
2144
gfx_v11_0_load_rlcg_microcode(struct amdgpu_device * adev)2145 static void gfx_v11_0_load_rlcg_microcode(struct amdgpu_device *adev)
2146 {
2147 const struct rlc_firmware_header_v2_0 *hdr;
2148 const __le32 *fw_data;
2149 unsigned i, fw_size;
2150
2151 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2152 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2153 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2154 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
2155
2156 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR,
2157 RLCG_UCODE_LOADING_START_ADDRESS);
2158
2159 for (i = 0; i < fw_size; i++)
2160 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_DATA,
2161 le32_to_cpup(fw_data++));
2162
2163 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
2164 }
2165
gfx_v11_0_load_rlc_iram_dram_microcode(struct amdgpu_device * adev)2166 static void gfx_v11_0_load_rlc_iram_dram_microcode(struct amdgpu_device *adev)
2167 {
2168 const struct rlc_firmware_header_v2_2 *hdr;
2169 const __le32 *fw_data;
2170 unsigned i, fw_size;
2171 u32 tmp;
2172
2173 hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
2174
2175 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2176 le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes));
2177 fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4;
2178
2179 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, 0);
2180
2181 for (i = 0; i < fw_size; i++) {
2182 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2183 msleep(1);
2184 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_DATA,
2185 le32_to_cpup(fw_data++));
2186 }
2187
2188 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
2189
2190 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2191 le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes));
2192 fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4;
2193
2194 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_ADDR, 0);
2195 for (i = 0; i < fw_size; i++) {
2196 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2197 msleep(1);
2198 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_DATA,
2199 le32_to_cpup(fw_data++));
2200 }
2201
2202 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
2203
2204 tmp = RREG32_SOC15(GC, 0, regRLC_LX6_CNTL);
2205 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1);
2206 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0);
2207 WREG32_SOC15(GC, 0, regRLC_LX6_CNTL, tmp);
2208 }
2209
gfx_v11_0_load_rlcp_rlcv_microcode(struct amdgpu_device * adev)2210 static void gfx_v11_0_load_rlcp_rlcv_microcode(struct amdgpu_device *adev)
2211 {
2212 const struct rlc_firmware_header_v2_3 *hdr;
2213 const __le32 *fw_data;
2214 unsigned i, fw_size;
2215 u32 tmp;
2216
2217 hdr = (const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data;
2218
2219 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2220 le32_to_cpu(hdr->rlcp_ucode_offset_bytes));
2221 fw_size = le32_to_cpu(hdr->rlcp_ucode_size_bytes) / 4;
2222
2223 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, 0);
2224
2225 for (i = 0; i < fw_size; i++) {
2226 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2227 msleep(1);
2228 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_DATA,
2229 le32_to_cpup(fw_data++));
2230 }
2231
2232 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, adev->gfx.rlc_fw_version);
2233
2234 tmp = RREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE);
2235 tmp = REG_SET_FIELD(tmp, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1);
2236 WREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE, tmp);
2237
2238 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2239 le32_to_cpu(hdr->rlcv_ucode_offset_bytes));
2240 fw_size = le32_to_cpu(hdr->rlcv_ucode_size_bytes) / 4;
2241
2242 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, 0);
2243
2244 for (i = 0; i < fw_size; i++) {
2245 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2246 msleep(1);
2247 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_DATA,
2248 le32_to_cpup(fw_data++));
2249 }
2250
2251 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, adev->gfx.rlc_fw_version);
2252
2253 tmp = RREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL);
2254 tmp = REG_SET_FIELD(tmp, RLC_GPU_IOV_F32_CNTL, ENABLE, 1);
2255 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL, tmp);
2256 }
2257
gfx_v11_0_rlc_load_microcode(struct amdgpu_device * adev)2258 static int gfx_v11_0_rlc_load_microcode(struct amdgpu_device *adev)
2259 {
2260 const struct rlc_firmware_header_v2_0 *hdr;
2261 uint16_t version_major;
2262 uint16_t version_minor;
2263
2264 if (!adev->gfx.rlc_fw)
2265 return -EINVAL;
2266
2267 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2268 amdgpu_ucode_print_rlc_hdr(&hdr->header);
2269
2270 version_major = le16_to_cpu(hdr->header.header_version_major);
2271 version_minor = le16_to_cpu(hdr->header.header_version_minor);
2272
2273 if (version_major == 2) {
2274 gfx_v11_0_load_rlcg_microcode(adev);
2275 if (amdgpu_dpm == 1) {
2276 if (version_minor >= 2)
2277 gfx_v11_0_load_rlc_iram_dram_microcode(adev);
2278 if (version_minor == 3)
2279 gfx_v11_0_load_rlcp_rlcv_microcode(adev);
2280 }
2281
2282 return 0;
2283 }
2284
2285 return -EINVAL;
2286 }
2287
gfx_v11_0_rlc_resume(struct amdgpu_device * adev)2288 static int gfx_v11_0_rlc_resume(struct amdgpu_device *adev)
2289 {
2290 int r;
2291
2292 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
2293 gfx_v11_0_init_csb(adev);
2294
2295 if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */
2296 gfx_v11_0_rlc_enable_srm(adev);
2297 } else {
2298 if (amdgpu_sriov_vf(adev)) {
2299 gfx_v11_0_init_csb(adev);
2300 return 0;
2301 }
2302
2303 adev->gfx.rlc.funcs->stop(adev);
2304
2305 /* disable CG */
2306 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, 0);
2307
2308 /* disable PG */
2309 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, 0);
2310
2311 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
2312 /* legacy rlc firmware loading */
2313 r = gfx_v11_0_rlc_load_microcode(adev);
2314 if (r)
2315 return r;
2316 }
2317
2318 gfx_v11_0_init_csb(adev);
2319
2320 adev->gfx.rlc.funcs->start(adev);
2321 }
2322 return 0;
2323 }
2324
gfx_v11_0_config_me_cache(struct amdgpu_device * adev,uint64_t addr)2325 static int gfx_v11_0_config_me_cache(struct amdgpu_device *adev, uint64_t addr)
2326 {
2327 uint32_t usec_timeout = 50000; /* wait for 50ms */
2328 uint32_t tmp;
2329 int i;
2330
2331 /* Trigger an invalidation of the L1 instruction caches */
2332 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2333 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2334 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2335
2336 /* Wait for invalidation complete */
2337 for (i = 0; i < usec_timeout; i++) {
2338 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2339 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2340 INVALIDATE_CACHE_COMPLETE))
2341 break;
2342 udelay(1);
2343 }
2344
2345 if (i >= usec_timeout) {
2346 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2347 return -EINVAL;
2348 }
2349
2350 if (amdgpu_emu_mode == 1)
2351 amdgpu_device_flush_hdp(adev, NULL);
2352
2353 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2354 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2355 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2356 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2357 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2358 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2359
2360 /* Program me ucode address into intruction cache address register */
2361 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2362 lower_32_bits(addr) & 0xFFFFF000);
2363 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2364 upper_32_bits(addr));
2365
2366 return 0;
2367 }
2368
gfx_v11_0_config_pfp_cache(struct amdgpu_device * adev,uint64_t addr)2369 static int gfx_v11_0_config_pfp_cache(struct amdgpu_device *adev, uint64_t addr)
2370 {
2371 uint32_t usec_timeout = 50000; /* wait for 50ms */
2372 uint32_t tmp;
2373 int i;
2374
2375 /* Trigger an invalidation of the L1 instruction caches */
2376 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2377 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2378 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2379
2380 /* Wait for invalidation complete */
2381 for (i = 0; i < usec_timeout; i++) {
2382 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2383 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2384 INVALIDATE_CACHE_COMPLETE))
2385 break;
2386 udelay(1);
2387 }
2388
2389 if (i >= usec_timeout) {
2390 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2391 return -EINVAL;
2392 }
2393
2394 if (amdgpu_emu_mode == 1)
2395 amdgpu_device_flush_hdp(adev, NULL);
2396
2397 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2398 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2399 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2400 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2401 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2402 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2403
2404 /* Program pfp ucode address into intruction cache address register */
2405 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2406 lower_32_bits(addr) & 0xFFFFF000);
2407 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2408 upper_32_bits(addr));
2409
2410 return 0;
2411 }
2412
gfx_v11_0_config_mec_cache(struct amdgpu_device * adev,uint64_t addr)2413 static int gfx_v11_0_config_mec_cache(struct amdgpu_device *adev, uint64_t addr)
2414 {
2415 uint32_t usec_timeout = 50000; /* wait for 50ms */
2416 uint32_t tmp;
2417 int i;
2418
2419 /* Trigger an invalidation of the L1 instruction caches */
2420 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2421 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2422
2423 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
2424
2425 /* Wait for invalidation complete */
2426 for (i = 0; i < usec_timeout; i++) {
2427 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2428 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2429 INVALIDATE_CACHE_COMPLETE))
2430 break;
2431 udelay(1);
2432 }
2433
2434 if (i >= usec_timeout) {
2435 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2436 return -EINVAL;
2437 }
2438
2439 if (amdgpu_emu_mode == 1)
2440 amdgpu_device_flush_hdp(adev, NULL);
2441
2442 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2443 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2444 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2445 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2446 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2447
2448 /* Program mec1 ucode address into intruction cache address register */
2449 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO,
2450 lower_32_bits(addr) & 0xFFFFF000);
2451 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2452 upper_32_bits(addr));
2453
2454 return 0;
2455 }
2456
gfx_v11_0_config_pfp_cache_rs64(struct amdgpu_device * adev,uint64_t addr,uint64_t addr2)2457 static int gfx_v11_0_config_pfp_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2458 {
2459 uint32_t usec_timeout = 50000; /* wait for 50ms */
2460 uint32_t tmp;
2461 unsigned i, pipe_id;
2462 const struct gfx_firmware_header_v2_0 *pfp_hdr;
2463
2464 pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2465 adev->gfx.pfp_fw->data;
2466
2467 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2468 lower_32_bits(addr));
2469 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2470 upper_32_bits(addr));
2471
2472 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2473 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2474 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2475 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2476 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2477
2478 /*
2479 * Programming any of the CP_PFP_IC_BASE registers
2480 * forces invalidation of the ME L1 I$. Wait for the
2481 * invalidation complete
2482 */
2483 for (i = 0; i < usec_timeout; i++) {
2484 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2485 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2486 INVALIDATE_CACHE_COMPLETE))
2487 break;
2488 udelay(1);
2489 }
2490
2491 if (i >= usec_timeout) {
2492 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2493 return -EINVAL;
2494 }
2495
2496 /* Prime the L1 instruction caches */
2497 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2498 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
2499 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2500 /* Waiting for cache primed*/
2501 for (i = 0; i < usec_timeout; i++) {
2502 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2503 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2504 ICACHE_PRIMED))
2505 break;
2506 udelay(1);
2507 }
2508
2509 if (i >= usec_timeout) {
2510 dev_err(adev->dev, "failed to prime instruction cache\n");
2511 return -EINVAL;
2512 }
2513
2514 mutex_lock(&adev->srbm_mutex);
2515 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2516 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2517 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2518 (pfp_hdr->ucode_start_addr_hi << 30) |
2519 (pfp_hdr->ucode_start_addr_lo >> 2));
2520 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2521 pfp_hdr->ucode_start_addr_hi >> 2);
2522
2523 /*
2524 * Program CP_ME_CNTL to reset given PIPE to take
2525 * effect of CP_PFP_PRGRM_CNTR_START.
2526 */
2527 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2528 if (pipe_id == 0)
2529 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2530 PFP_PIPE0_RESET, 1);
2531 else
2532 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2533 PFP_PIPE1_RESET, 1);
2534 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2535
2536 /* Clear pfp pipe0 reset bit. */
2537 if (pipe_id == 0)
2538 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2539 PFP_PIPE0_RESET, 0);
2540 else
2541 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2542 PFP_PIPE1_RESET, 0);
2543 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2544
2545 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
2546 lower_32_bits(addr2));
2547 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
2548 upper_32_bits(addr2));
2549 }
2550 soc21_grbm_select(adev, 0, 0, 0, 0);
2551 mutex_unlock(&adev->srbm_mutex);
2552
2553 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2554 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2555 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2556 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2557
2558 /* Invalidate the data caches */
2559 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2560 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2561 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2562
2563 for (i = 0; i < usec_timeout; i++) {
2564 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2565 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2566 INVALIDATE_DCACHE_COMPLETE))
2567 break;
2568 udelay(1);
2569 }
2570
2571 if (i >= usec_timeout) {
2572 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2573 return -EINVAL;
2574 }
2575
2576 return 0;
2577 }
2578
gfx_v11_0_config_me_cache_rs64(struct amdgpu_device * adev,uint64_t addr,uint64_t addr2)2579 static int gfx_v11_0_config_me_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2580 {
2581 uint32_t usec_timeout = 50000; /* wait for 50ms */
2582 uint32_t tmp;
2583 unsigned i, pipe_id;
2584 const struct gfx_firmware_header_v2_0 *me_hdr;
2585
2586 me_hdr = (const struct gfx_firmware_header_v2_0 *)
2587 adev->gfx.me_fw->data;
2588
2589 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2590 lower_32_bits(addr));
2591 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2592 upper_32_bits(addr));
2593
2594 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2595 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2596 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2597 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2598 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2599
2600 /*
2601 * Programming any of the CP_ME_IC_BASE registers
2602 * forces invalidation of the ME L1 I$. Wait for the
2603 * invalidation complete
2604 */
2605 for (i = 0; i < usec_timeout; i++) {
2606 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2607 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2608 INVALIDATE_CACHE_COMPLETE))
2609 break;
2610 udelay(1);
2611 }
2612
2613 if (i >= usec_timeout) {
2614 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2615 return -EINVAL;
2616 }
2617
2618 /* Prime the instruction caches */
2619 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2620 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
2621 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2622
2623 /* Waiting for instruction cache primed*/
2624 for (i = 0; i < usec_timeout; i++) {
2625 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2626 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2627 ICACHE_PRIMED))
2628 break;
2629 udelay(1);
2630 }
2631
2632 if (i >= usec_timeout) {
2633 dev_err(adev->dev, "failed to prime instruction cache\n");
2634 return -EINVAL;
2635 }
2636
2637 mutex_lock(&adev->srbm_mutex);
2638 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2639 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2640 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
2641 (me_hdr->ucode_start_addr_hi << 30) |
2642 (me_hdr->ucode_start_addr_lo >> 2) );
2643 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
2644 me_hdr->ucode_start_addr_hi>>2);
2645
2646 /*
2647 * Program CP_ME_CNTL to reset given PIPE to take
2648 * effect of CP_PFP_PRGRM_CNTR_START.
2649 */
2650 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2651 if (pipe_id == 0)
2652 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2653 ME_PIPE0_RESET, 1);
2654 else
2655 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2656 ME_PIPE1_RESET, 1);
2657 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2658
2659 /* Clear pfp pipe0 reset bit. */
2660 if (pipe_id == 0)
2661 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2662 ME_PIPE0_RESET, 0);
2663 else
2664 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2665 ME_PIPE1_RESET, 0);
2666 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2667
2668 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
2669 lower_32_bits(addr2));
2670 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
2671 upper_32_bits(addr2));
2672 }
2673 soc21_grbm_select(adev, 0, 0, 0, 0);
2674 mutex_unlock(&adev->srbm_mutex);
2675
2676 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2677 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2678 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2679 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2680
2681 /* Invalidate the data caches */
2682 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2683 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2684 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2685
2686 for (i = 0; i < usec_timeout; i++) {
2687 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2688 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2689 INVALIDATE_DCACHE_COMPLETE))
2690 break;
2691 udelay(1);
2692 }
2693
2694 if (i >= usec_timeout) {
2695 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2696 return -EINVAL;
2697 }
2698
2699 return 0;
2700 }
2701
gfx_v11_0_config_mec_cache_rs64(struct amdgpu_device * adev,uint64_t addr,uint64_t addr2)2702 static int gfx_v11_0_config_mec_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2703 {
2704 uint32_t usec_timeout = 50000; /* wait for 50ms */
2705 uint32_t tmp;
2706 unsigned i;
2707 const struct gfx_firmware_header_v2_0 *mec_hdr;
2708
2709 mec_hdr = (const struct gfx_firmware_header_v2_0 *)
2710 adev->gfx.mec_fw->data;
2711
2712 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2713 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
2714 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2715 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2716 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2717
2718 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
2719 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
2720 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
2721 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
2722
2723 mutex_lock(&adev->srbm_mutex);
2724 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
2725 soc21_grbm_select(adev, 1, i, 0, 0);
2726
2727 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, addr2);
2728 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
2729 upper_32_bits(addr2));
2730
2731 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
2732 mec_hdr->ucode_start_addr_lo >> 2 |
2733 mec_hdr->ucode_start_addr_hi << 30);
2734 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
2735 mec_hdr->ucode_start_addr_hi >> 2);
2736
2737 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, addr);
2738 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2739 upper_32_bits(addr));
2740 }
2741 mutex_unlock(&adev->srbm_mutex);
2742 soc21_grbm_select(adev, 0, 0, 0, 0);
2743
2744 /* Trigger an invalidation of the L1 instruction caches */
2745 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2746 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2747 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
2748
2749 /* Wait for invalidation complete */
2750 for (i = 0; i < usec_timeout; i++) {
2751 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2752 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
2753 INVALIDATE_DCACHE_COMPLETE))
2754 break;
2755 udelay(1);
2756 }
2757
2758 if (i >= usec_timeout) {
2759 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2760 return -EINVAL;
2761 }
2762
2763 /* Trigger an invalidation of the L1 instruction caches */
2764 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2765 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2766 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
2767
2768 /* Wait for invalidation complete */
2769 for (i = 0; i < usec_timeout; i++) {
2770 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2771 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2772 INVALIDATE_CACHE_COMPLETE))
2773 break;
2774 udelay(1);
2775 }
2776
2777 if (i >= usec_timeout) {
2778 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2779 return -EINVAL;
2780 }
2781
2782 return 0;
2783 }
2784
gfx_v11_0_config_gfx_rs64(struct amdgpu_device * adev)2785 static void gfx_v11_0_config_gfx_rs64(struct amdgpu_device *adev)
2786 {
2787 const struct gfx_firmware_header_v2_0 *pfp_hdr;
2788 const struct gfx_firmware_header_v2_0 *me_hdr;
2789 const struct gfx_firmware_header_v2_0 *mec_hdr;
2790 uint32_t pipe_id, tmp;
2791
2792 mec_hdr = (const struct gfx_firmware_header_v2_0 *)
2793 adev->gfx.mec_fw->data;
2794 me_hdr = (const struct gfx_firmware_header_v2_0 *)
2795 adev->gfx.me_fw->data;
2796 pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2797 adev->gfx.pfp_fw->data;
2798
2799 /* config pfp program start addr */
2800 for (pipe_id = 0; pipe_id < 2; pipe_id++) {
2801 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2802 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2803 (pfp_hdr->ucode_start_addr_hi << 30) |
2804 (pfp_hdr->ucode_start_addr_lo >> 2));
2805 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2806 pfp_hdr->ucode_start_addr_hi >> 2);
2807 }
2808 soc21_grbm_select(adev, 0, 0, 0, 0);
2809
2810 /* reset pfp pipe */
2811 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2812 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 1);
2813 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 1);
2814 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2815
2816 /* clear pfp pipe reset */
2817 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 0);
2818 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 0);
2819 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2820
2821 /* config me program start addr */
2822 for (pipe_id = 0; pipe_id < 2; pipe_id++) {
2823 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2824 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
2825 (me_hdr->ucode_start_addr_hi << 30) |
2826 (me_hdr->ucode_start_addr_lo >> 2) );
2827 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
2828 me_hdr->ucode_start_addr_hi>>2);
2829 }
2830 soc21_grbm_select(adev, 0, 0, 0, 0);
2831
2832 /* reset me pipe */
2833 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2834 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 1);
2835 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 1);
2836 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2837
2838 /* clear me pipe reset */
2839 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 0);
2840 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 0);
2841 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2842
2843 /* config mec program start addr */
2844 for (pipe_id = 0; pipe_id < 4; pipe_id++) {
2845 soc21_grbm_select(adev, 1, pipe_id, 0, 0);
2846 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
2847 mec_hdr->ucode_start_addr_lo >> 2 |
2848 mec_hdr->ucode_start_addr_hi << 30);
2849 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
2850 mec_hdr->ucode_start_addr_hi >> 2);
2851 }
2852 soc21_grbm_select(adev, 0, 0, 0, 0);
2853
2854 /* reset mec pipe */
2855 tmp = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
2856 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1);
2857 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1);
2858 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1);
2859 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1);
2860 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
2861
2862 /* clear mec pipe reset */
2863 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0);
2864 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0);
2865 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0);
2866 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0);
2867 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
2868 }
2869
gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device * adev)2870 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev)
2871 {
2872 uint32_t cp_status;
2873 uint32_t bootload_status;
2874 int i, r;
2875 uint64_t addr, addr2;
2876
2877 for (i = 0; i < adev->usec_timeout; i++) {
2878 cp_status = RREG32_SOC15(GC, 0, regCP_STAT);
2879
2880 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
2881 IP_VERSION(11, 0, 1) ||
2882 amdgpu_ip_version(adev, GC_HWIP, 0) ==
2883 IP_VERSION(11, 0, 4) ||
2884 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 0) ||
2885 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 1) ||
2886 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 2))
2887 bootload_status = RREG32_SOC15(GC, 0,
2888 regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1);
2889 else
2890 bootload_status = RREG32_SOC15(GC, 0, regRLC_RLCS_BOOTLOAD_STATUS);
2891
2892 if ((cp_status == 0) &&
2893 (REG_GET_FIELD(bootload_status,
2894 RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) {
2895 break;
2896 }
2897 udelay(1);
2898 }
2899
2900 if (i >= adev->usec_timeout) {
2901 dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n");
2902 return -ETIMEDOUT;
2903 }
2904
2905 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
2906 if (adev->gfx.rs64_enable) {
2907 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2908 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME].offset;
2909 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2910 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME_P0_STACK].offset;
2911 r = gfx_v11_0_config_me_cache_rs64(adev, addr, addr2);
2912 if (r)
2913 return r;
2914 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2915 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP].offset;
2916 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2917 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK].offset;
2918 r = gfx_v11_0_config_pfp_cache_rs64(adev, addr, addr2);
2919 if (r)
2920 return r;
2921 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2922 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC].offset;
2923 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2924 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK].offset;
2925 r = gfx_v11_0_config_mec_cache_rs64(adev, addr, addr2);
2926 if (r)
2927 return r;
2928 } else {
2929 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2930 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_ME].offset;
2931 r = gfx_v11_0_config_me_cache(adev, addr);
2932 if (r)
2933 return r;
2934 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2935 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_PFP].offset;
2936 r = gfx_v11_0_config_pfp_cache(adev, addr);
2937 if (r)
2938 return r;
2939 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2940 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_MEC].offset;
2941 r = gfx_v11_0_config_mec_cache(adev, addr);
2942 if (r)
2943 return r;
2944 }
2945 }
2946
2947 return 0;
2948 }
2949
gfx_v11_0_cp_gfx_enable(struct amdgpu_device * adev,bool enable)2950 static int gfx_v11_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
2951 {
2952 int i;
2953 u32 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2954
2955 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
2956 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
2957 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2958
2959 for (i = 0; i < adev->usec_timeout; i++) {
2960 if (RREG32_SOC15(GC, 0, regCP_STAT) == 0)
2961 break;
2962 udelay(1);
2963 }
2964
2965 if (i >= adev->usec_timeout)
2966 DRM_ERROR("failed to %s cp gfx\n", enable ? "unhalt" : "halt");
2967
2968 return 0;
2969 }
2970
gfx_v11_0_cp_gfx_load_pfp_microcode(struct amdgpu_device * adev)2971 static int gfx_v11_0_cp_gfx_load_pfp_microcode(struct amdgpu_device *adev)
2972 {
2973 int r;
2974 const struct gfx_firmware_header_v1_0 *pfp_hdr;
2975 const __le32 *fw_data;
2976 unsigned i, fw_size;
2977
2978 pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
2979 adev->gfx.pfp_fw->data;
2980
2981 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
2982
2983 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
2984 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
2985 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes);
2986
2987 r = amdgpu_bo_create_reserved(adev, pfp_hdr->header.ucode_size_bytes,
2988 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
2989 &adev->gfx.pfp.pfp_fw_obj,
2990 &adev->gfx.pfp.pfp_fw_gpu_addr,
2991 (void **)&adev->gfx.pfp.pfp_fw_ptr);
2992 if (r) {
2993 dev_err(adev->dev, "(%d) failed to create pfp fw bo\n", r);
2994 gfx_v11_0_pfp_fini(adev);
2995 return r;
2996 }
2997
2998 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_data, fw_size);
2999
3000 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
3001 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
3002
3003 gfx_v11_0_config_pfp_cache(adev, adev->gfx.pfp.pfp_fw_gpu_addr);
3004
3005 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, 0);
3006
3007 for (i = 0; i < pfp_hdr->jt_size; i++)
3008 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_DATA,
3009 le32_to_cpup(fw_data + pfp_hdr->jt_offset + i));
3010
3011 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3012
3013 return 0;
3014 }
3015
gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device * adev)3016 static int gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev)
3017 {
3018 int r;
3019 const struct gfx_firmware_header_v2_0 *pfp_hdr;
3020 const __le32 *fw_ucode, *fw_data;
3021 unsigned i, pipe_id, fw_ucode_size, fw_data_size;
3022 uint32_t tmp;
3023 uint32_t usec_timeout = 50000; /* wait for 50ms */
3024
3025 pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
3026 adev->gfx.pfp_fw->data;
3027
3028 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3029
3030 /* instruction */
3031 fw_ucode = (const __le32 *)(adev->gfx.pfp_fw->data +
3032 le32_to_cpu(pfp_hdr->ucode_offset_bytes));
3033 fw_ucode_size = le32_to_cpu(pfp_hdr->ucode_size_bytes);
3034 /* data */
3035 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
3036 le32_to_cpu(pfp_hdr->data_offset_bytes));
3037 fw_data_size = le32_to_cpu(pfp_hdr->data_size_bytes);
3038
3039 /* 64kb align */
3040 r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3041 64 * 1024,
3042 AMDGPU_GEM_DOMAIN_VRAM |
3043 AMDGPU_GEM_DOMAIN_GTT,
3044 &adev->gfx.pfp.pfp_fw_obj,
3045 &adev->gfx.pfp.pfp_fw_gpu_addr,
3046 (void **)&adev->gfx.pfp.pfp_fw_ptr);
3047 if (r) {
3048 dev_err(adev->dev, "(%d) failed to create pfp ucode fw bo\n", r);
3049 gfx_v11_0_pfp_fini(adev);
3050 return r;
3051 }
3052
3053 r = amdgpu_bo_create_reserved(adev, fw_data_size,
3054 64 * 1024,
3055 AMDGPU_GEM_DOMAIN_VRAM |
3056 AMDGPU_GEM_DOMAIN_GTT,
3057 &adev->gfx.pfp.pfp_fw_data_obj,
3058 &adev->gfx.pfp.pfp_fw_data_gpu_addr,
3059 (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
3060 if (r) {
3061 dev_err(adev->dev, "(%d) failed to create pfp data fw bo\n", r);
3062 gfx_v11_0_pfp_fini(adev);
3063 return r;
3064 }
3065
3066 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_ucode, fw_ucode_size);
3067 memcpy(adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size);
3068
3069 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
3070 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_data_obj);
3071 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
3072 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_data_obj);
3073
3074 if (amdgpu_emu_mode == 1)
3075 amdgpu_device_flush_hdp(adev, NULL);
3076
3077 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
3078 lower_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
3079 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
3080 upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
3081
3082 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
3083 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
3084 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
3085 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
3086 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
3087
3088 /*
3089 * Programming any of the CP_PFP_IC_BASE registers
3090 * forces invalidation of the ME L1 I$. Wait for the
3091 * invalidation complete
3092 */
3093 for (i = 0; i < usec_timeout; i++) {
3094 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
3095 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
3096 INVALIDATE_CACHE_COMPLETE))
3097 break;
3098 udelay(1);
3099 }
3100
3101 if (i >= usec_timeout) {
3102 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3103 return -EINVAL;
3104 }
3105
3106 /* Prime the L1 instruction caches */
3107 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
3108 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
3109 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
3110 /* Waiting for cache primed*/
3111 for (i = 0; i < usec_timeout; i++) {
3112 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
3113 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
3114 ICACHE_PRIMED))
3115 break;
3116 udelay(1);
3117 }
3118
3119 if (i >= usec_timeout) {
3120 dev_err(adev->dev, "failed to prime instruction cache\n");
3121 return -EINVAL;
3122 }
3123
3124 mutex_lock(&adev->srbm_mutex);
3125 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
3126 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
3127 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
3128 (pfp_hdr->ucode_start_addr_hi << 30) |
3129 (pfp_hdr->ucode_start_addr_lo >> 2) );
3130 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
3131 pfp_hdr->ucode_start_addr_hi>>2);
3132
3133 /*
3134 * Program CP_ME_CNTL to reset given PIPE to take
3135 * effect of CP_PFP_PRGRM_CNTR_START.
3136 */
3137 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
3138 if (pipe_id == 0)
3139 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3140 PFP_PIPE0_RESET, 1);
3141 else
3142 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3143 PFP_PIPE1_RESET, 1);
3144 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3145
3146 /* Clear pfp pipe0 reset bit. */
3147 if (pipe_id == 0)
3148 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3149 PFP_PIPE0_RESET, 0);
3150 else
3151 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3152 PFP_PIPE1_RESET, 0);
3153 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3154
3155 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
3156 lower_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
3157 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
3158 upper_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
3159 }
3160 soc21_grbm_select(adev, 0, 0, 0, 0);
3161 mutex_unlock(&adev->srbm_mutex);
3162
3163 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
3164 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
3165 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
3166 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
3167
3168 /* Invalidate the data caches */
3169 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3170 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3171 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
3172
3173 for (i = 0; i < usec_timeout; i++) {
3174 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3175 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
3176 INVALIDATE_DCACHE_COMPLETE))
3177 break;
3178 udelay(1);
3179 }
3180
3181 if (i >= usec_timeout) {
3182 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
3183 return -EINVAL;
3184 }
3185
3186 return 0;
3187 }
3188
gfx_v11_0_cp_gfx_load_me_microcode(struct amdgpu_device * adev)3189 static int gfx_v11_0_cp_gfx_load_me_microcode(struct amdgpu_device *adev)
3190 {
3191 int r;
3192 const struct gfx_firmware_header_v1_0 *me_hdr;
3193 const __le32 *fw_data;
3194 unsigned i, fw_size;
3195
3196 me_hdr = (const struct gfx_firmware_header_v1_0 *)
3197 adev->gfx.me_fw->data;
3198
3199 amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3200
3201 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
3202 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3203 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes);
3204
3205 r = amdgpu_bo_create_reserved(adev, me_hdr->header.ucode_size_bytes,
3206 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
3207 &adev->gfx.me.me_fw_obj,
3208 &adev->gfx.me.me_fw_gpu_addr,
3209 (void **)&adev->gfx.me.me_fw_ptr);
3210 if (r) {
3211 dev_err(adev->dev, "(%d) failed to create me fw bo\n", r);
3212 gfx_v11_0_me_fini(adev);
3213 return r;
3214 }
3215
3216 memcpy(adev->gfx.me.me_fw_ptr, fw_data, fw_size);
3217
3218 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
3219 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
3220
3221 gfx_v11_0_config_me_cache(adev, adev->gfx.me.me_fw_gpu_addr);
3222
3223 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, 0);
3224
3225 for (i = 0; i < me_hdr->jt_size; i++)
3226 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_DATA,
3227 le32_to_cpup(fw_data + me_hdr->jt_offset + i));
3228
3229 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, adev->gfx.me_fw_version);
3230
3231 return 0;
3232 }
3233
gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device * adev)3234 static int gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev)
3235 {
3236 int r;
3237 const struct gfx_firmware_header_v2_0 *me_hdr;
3238 const __le32 *fw_ucode, *fw_data;
3239 unsigned i, pipe_id, fw_ucode_size, fw_data_size;
3240 uint32_t tmp;
3241 uint32_t usec_timeout = 50000; /* wait for 50ms */
3242
3243 me_hdr = (const struct gfx_firmware_header_v2_0 *)
3244 adev->gfx.me_fw->data;
3245
3246 amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3247
3248 /* instruction */
3249 fw_ucode = (const __le32 *)(adev->gfx.me_fw->data +
3250 le32_to_cpu(me_hdr->ucode_offset_bytes));
3251 fw_ucode_size = le32_to_cpu(me_hdr->ucode_size_bytes);
3252 /* data */
3253 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
3254 le32_to_cpu(me_hdr->data_offset_bytes));
3255 fw_data_size = le32_to_cpu(me_hdr->data_size_bytes);
3256
3257 /* 64kb align*/
3258 r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3259 64 * 1024,
3260 AMDGPU_GEM_DOMAIN_VRAM |
3261 AMDGPU_GEM_DOMAIN_GTT,
3262 &adev->gfx.me.me_fw_obj,
3263 &adev->gfx.me.me_fw_gpu_addr,
3264 (void **)&adev->gfx.me.me_fw_ptr);
3265 if (r) {
3266 dev_err(adev->dev, "(%d) failed to create me ucode bo\n", r);
3267 gfx_v11_0_me_fini(adev);
3268 return r;
3269 }
3270
3271 r = amdgpu_bo_create_reserved(adev, fw_data_size,
3272 64 * 1024,
3273 AMDGPU_GEM_DOMAIN_VRAM |
3274 AMDGPU_GEM_DOMAIN_GTT,
3275 &adev->gfx.me.me_fw_data_obj,
3276 &adev->gfx.me.me_fw_data_gpu_addr,
3277 (void **)&adev->gfx.me.me_fw_data_ptr);
3278 if (r) {
3279 dev_err(adev->dev, "(%d) failed to create me data bo\n", r);
3280 gfx_v11_0_pfp_fini(adev);
3281 return r;
3282 }
3283
3284 memcpy(adev->gfx.me.me_fw_ptr, fw_ucode, fw_ucode_size);
3285 memcpy(adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size);
3286
3287 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
3288 amdgpu_bo_kunmap(adev->gfx.me.me_fw_data_obj);
3289 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
3290 amdgpu_bo_unreserve(adev->gfx.me.me_fw_data_obj);
3291
3292 if (amdgpu_emu_mode == 1)
3293 amdgpu_device_flush_hdp(adev, NULL);
3294
3295 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
3296 lower_32_bits(adev->gfx.me.me_fw_gpu_addr));
3297 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
3298 upper_32_bits(adev->gfx.me.me_fw_gpu_addr));
3299
3300 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
3301 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
3302 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
3303 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
3304 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
3305
3306 /*
3307 * Programming any of the CP_ME_IC_BASE registers
3308 * forces invalidation of the ME L1 I$. Wait for the
3309 * invalidation complete
3310 */
3311 for (i = 0; i < usec_timeout; i++) {
3312 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3313 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
3314 INVALIDATE_CACHE_COMPLETE))
3315 break;
3316 udelay(1);
3317 }
3318
3319 if (i >= usec_timeout) {
3320 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3321 return -EINVAL;
3322 }
3323
3324 /* Prime the instruction caches */
3325 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3326 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
3327 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
3328
3329 /* Waiting for instruction cache primed*/
3330 for (i = 0; i < usec_timeout; i++) {
3331 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3332 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
3333 ICACHE_PRIMED))
3334 break;
3335 udelay(1);
3336 }
3337
3338 if (i >= usec_timeout) {
3339 dev_err(adev->dev, "failed to prime instruction cache\n");
3340 return -EINVAL;
3341 }
3342
3343 mutex_lock(&adev->srbm_mutex);
3344 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
3345 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
3346 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
3347 (me_hdr->ucode_start_addr_hi << 30) |
3348 (me_hdr->ucode_start_addr_lo >> 2) );
3349 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
3350 me_hdr->ucode_start_addr_hi>>2);
3351
3352 /*
3353 * Program CP_ME_CNTL to reset given PIPE to take
3354 * effect of CP_PFP_PRGRM_CNTR_START.
3355 */
3356 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
3357 if (pipe_id == 0)
3358 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3359 ME_PIPE0_RESET, 1);
3360 else
3361 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3362 ME_PIPE1_RESET, 1);
3363 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3364
3365 /* Clear pfp pipe0 reset bit. */
3366 if (pipe_id == 0)
3367 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3368 ME_PIPE0_RESET, 0);
3369 else
3370 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3371 ME_PIPE1_RESET, 0);
3372 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3373
3374 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
3375 lower_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
3376 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
3377 upper_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
3378 }
3379 soc21_grbm_select(adev, 0, 0, 0, 0);
3380 mutex_unlock(&adev->srbm_mutex);
3381
3382 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
3383 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
3384 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
3385 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
3386
3387 /* Invalidate the data caches */
3388 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3389 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3390 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
3391
3392 for (i = 0; i < usec_timeout; i++) {
3393 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3394 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
3395 INVALIDATE_DCACHE_COMPLETE))
3396 break;
3397 udelay(1);
3398 }
3399
3400 if (i >= usec_timeout) {
3401 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
3402 return -EINVAL;
3403 }
3404
3405 return 0;
3406 }
3407
gfx_v11_0_cp_gfx_load_microcode(struct amdgpu_device * adev)3408 static int gfx_v11_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3409 {
3410 int r;
3411
3412 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw)
3413 return -EINVAL;
3414
3415 gfx_v11_0_cp_gfx_enable(adev, false);
3416
3417 if (adev->gfx.rs64_enable)
3418 r = gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(adev);
3419 else
3420 r = gfx_v11_0_cp_gfx_load_pfp_microcode(adev);
3421 if (r) {
3422 dev_err(adev->dev, "(%d) failed to load pfp fw\n", r);
3423 return r;
3424 }
3425
3426 if (adev->gfx.rs64_enable)
3427 r = gfx_v11_0_cp_gfx_load_me_microcode_rs64(adev);
3428 else
3429 r = gfx_v11_0_cp_gfx_load_me_microcode(adev);
3430 if (r) {
3431 dev_err(adev->dev, "(%d) failed to load me fw\n", r);
3432 return r;
3433 }
3434
3435 return 0;
3436 }
3437
gfx_v11_0_cp_gfx_start(struct amdgpu_device * adev)3438 static int gfx_v11_0_cp_gfx_start(struct amdgpu_device *adev)
3439 {
3440 struct amdgpu_ring *ring;
3441 const struct cs_section_def *sect = NULL;
3442 const struct cs_extent_def *ext = NULL;
3443 int r, i;
3444 int ctx_reg_offset;
3445
3446 /* init the CP */
3447 WREG32_SOC15(GC, 0, regCP_MAX_CONTEXT,
3448 adev->gfx.config.max_hw_contexts - 1);
3449 WREG32_SOC15(GC, 0, regCP_DEVICE_ID, 1);
3450
3451 if (!amdgpu_async_gfx_ring)
3452 gfx_v11_0_cp_gfx_enable(adev, true);
3453
3454 ring = &adev->gfx.gfx_ring[0];
3455 r = amdgpu_ring_alloc(ring, gfx_v11_0_get_csb_size(adev));
3456 if (r) {
3457 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3458 return r;
3459 }
3460
3461 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3462 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3463
3464 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3465 amdgpu_ring_write(ring, 0x80000000);
3466 amdgpu_ring_write(ring, 0x80000000);
3467
3468 for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {
3469 for (ext = sect->section; ext->extent != NULL; ++ext) {
3470 if (sect->id == SECT_CONTEXT) {
3471 amdgpu_ring_write(ring,
3472 PACKET3(PACKET3_SET_CONTEXT_REG,
3473 ext->reg_count));
3474 amdgpu_ring_write(ring, ext->reg_index -
3475 PACKET3_SET_CONTEXT_REG_START);
3476 for (i = 0; i < ext->reg_count; i++)
3477 amdgpu_ring_write(ring, ext->extent[i]);
3478 }
3479 }
3480 }
3481
3482 ctx_reg_offset =
3483 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
3484 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
3485 amdgpu_ring_write(ring, ctx_reg_offset);
3486 amdgpu_ring_write(ring, adev->gfx.config.pa_sc_tile_steering_override);
3487
3488 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3489 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3490
3491 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3492 amdgpu_ring_write(ring, 0);
3493
3494 amdgpu_ring_commit(ring);
3495
3496 /* submit cs packet to copy state 0 to next available state */
3497 if (adev->gfx.num_gfx_rings > 1) {
3498 /* maximum supported gfx ring is 2 */
3499 ring = &adev->gfx.gfx_ring[1];
3500 r = amdgpu_ring_alloc(ring, 2);
3501 if (r) {
3502 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3503 return r;
3504 }
3505
3506 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3507 amdgpu_ring_write(ring, 0);
3508
3509 amdgpu_ring_commit(ring);
3510 }
3511 return 0;
3512 }
3513
gfx_v11_0_cp_gfx_switch_pipe(struct amdgpu_device * adev,CP_PIPE_ID pipe)3514 static void gfx_v11_0_cp_gfx_switch_pipe(struct amdgpu_device *adev,
3515 CP_PIPE_ID pipe)
3516 {
3517 u32 tmp;
3518
3519 tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL);
3520 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, pipe);
3521
3522 WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp);
3523 }
3524
gfx_v11_0_cp_gfx_set_doorbell(struct amdgpu_device * adev,struct amdgpu_ring * ring)3525 static void gfx_v11_0_cp_gfx_set_doorbell(struct amdgpu_device *adev,
3526 struct amdgpu_ring *ring)
3527 {
3528 u32 tmp;
3529
3530 tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
3531 if (ring->use_doorbell) {
3532 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3533 DOORBELL_OFFSET, ring->doorbell_index);
3534 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3535 DOORBELL_EN, 1);
3536 } else {
3537 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3538 DOORBELL_EN, 0);
3539 }
3540 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, tmp);
3541
3542 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3543 DOORBELL_RANGE_LOWER, ring->doorbell_index);
3544 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, tmp);
3545
3546 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
3547 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3548 }
3549
gfx_v11_0_cp_gfx_resume(struct amdgpu_device * adev)3550 static int gfx_v11_0_cp_gfx_resume(struct amdgpu_device *adev)
3551 {
3552 struct amdgpu_ring *ring;
3553 u32 tmp;
3554 u32 rb_bufsz;
3555 u64 rb_addr, rptr_addr, wptr_gpu_addr;
3556
3557 /* Set the write pointer delay */
3558 WREG32_SOC15(GC, 0, regCP_RB_WPTR_DELAY, 0);
3559
3560 /* set the RB to use vmid 0 */
3561 WREG32_SOC15(GC, 0, regCP_RB_VMID, 0);
3562
3563 /* Init gfx ring 0 for pipe 0 */
3564 mutex_lock(&adev->srbm_mutex);
3565 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
3566
3567 /* Set ring buffer size */
3568 ring = &adev->gfx.gfx_ring[0];
3569 rb_bufsz = order_base_2(ring->ring_size / 8);
3570 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3571 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3572 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
3573
3574 /* Initialize the ring buffer's write pointers */
3575 ring->wptr = 0;
3576 WREG32_SOC15(GC, 0, regCP_RB0_WPTR, lower_32_bits(ring->wptr));
3577 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3578
3579 /* set the wb address wether it's enabled or not */
3580 rptr_addr = ring->rptr_gpu_addr;
3581 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3582 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
3583 CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3584
3585 wptr_gpu_addr = ring->wptr_gpu_addr;
3586 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
3587 lower_32_bits(wptr_gpu_addr));
3588 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
3589 upper_32_bits(wptr_gpu_addr));
3590
3591 mdelay(1);
3592 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
3593
3594 rb_addr = ring->gpu_addr >> 8;
3595 WREG32_SOC15(GC, 0, regCP_RB0_BASE, rb_addr);
3596 WREG32_SOC15(GC, 0, regCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3597
3598 WREG32_SOC15(GC, 0, regCP_RB_ACTIVE, 1);
3599
3600 gfx_v11_0_cp_gfx_set_doorbell(adev, ring);
3601 mutex_unlock(&adev->srbm_mutex);
3602
3603 /* Init gfx ring 1 for pipe 1 */
3604 if (adev->gfx.num_gfx_rings > 1) {
3605 mutex_lock(&adev->srbm_mutex);
3606 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID1);
3607 /* maximum supported gfx ring is 2 */
3608 ring = &adev->gfx.gfx_ring[1];
3609 rb_bufsz = order_base_2(ring->ring_size / 8);
3610 tmp = REG_SET_FIELD(0, CP_RB1_CNTL, RB_BUFSZ, rb_bufsz);
3611 tmp = REG_SET_FIELD(tmp, CP_RB1_CNTL, RB_BLKSZ, rb_bufsz - 2);
3612 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp);
3613 /* Initialize the ring buffer's write pointers */
3614 ring->wptr = 0;
3615 WREG32_SOC15(GC, 0, regCP_RB1_WPTR, lower_32_bits(ring->wptr));
3616 WREG32_SOC15(GC, 0, regCP_RB1_WPTR_HI, upper_32_bits(ring->wptr));
3617 /* Set the wb address wether it's enabled or not */
3618 rptr_addr = ring->rptr_gpu_addr;
3619 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr));
3620 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
3621 CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3622 wptr_gpu_addr = ring->wptr_gpu_addr;
3623 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
3624 lower_32_bits(wptr_gpu_addr));
3625 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
3626 upper_32_bits(wptr_gpu_addr));
3627
3628 mdelay(1);
3629 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp);
3630
3631 rb_addr = ring->gpu_addr >> 8;
3632 WREG32_SOC15(GC, 0, regCP_RB1_BASE, rb_addr);
3633 WREG32_SOC15(GC, 0, regCP_RB1_BASE_HI, upper_32_bits(rb_addr));
3634 WREG32_SOC15(GC, 0, regCP_RB1_ACTIVE, 1);
3635
3636 gfx_v11_0_cp_gfx_set_doorbell(adev, ring);
3637 mutex_unlock(&adev->srbm_mutex);
3638 }
3639 /* Switch to pipe 0 */
3640 mutex_lock(&adev->srbm_mutex);
3641 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
3642 mutex_unlock(&adev->srbm_mutex);
3643
3644 /* start the ring */
3645 gfx_v11_0_cp_gfx_start(adev);
3646
3647 return 0;
3648 }
3649
gfx_v11_0_cp_compute_enable(struct amdgpu_device * adev,bool enable)3650 static void gfx_v11_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3651 {
3652 u32 data;
3653
3654 if (adev->gfx.rs64_enable) {
3655 data = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
3656 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE,
3657 enable ? 0 : 1);
3658 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET,
3659 enable ? 0 : 1);
3660 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET,
3661 enable ? 0 : 1);
3662 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET,
3663 enable ? 0 : 1);
3664 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET,
3665 enable ? 0 : 1);
3666 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE,
3667 enable ? 1 : 0);
3668 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE,
3669 enable ? 1 : 0);
3670 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE,
3671 enable ? 1 : 0);
3672 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE,
3673 enable ? 1 : 0);
3674 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT,
3675 enable ? 0 : 1);
3676 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, data);
3677 } else {
3678 data = RREG32_SOC15(GC, 0, regCP_MEC_CNTL);
3679
3680 if (enable) {
3681 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 0);
3682 if (!adev->enable_mes_kiq)
3683 data = REG_SET_FIELD(data, CP_MEC_CNTL,
3684 MEC_ME2_HALT, 0);
3685 } else {
3686 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 1);
3687 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME2_HALT, 1);
3688 }
3689 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, data);
3690 }
3691
3692 udelay(50);
3693 }
3694
gfx_v11_0_cp_compute_load_microcode(struct amdgpu_device * adev)3695 static int gfx_v11_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3696 {
3697 const struct gfx_firmware_header_v1_0 *mec_hdr;
3698 const __le32 *fw_data;
3699 unsigned i, fw_size;
3700 u32 *fw = NULL;
3701 int r;
3702
3703 if (!adev->gfx.mec_fw)
3704 return -EINVAL;
3705
3706 gfx_v11_0_cp_compute_enable(adev, false);
3707
3708 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3709 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3710
3711 fw_data = (const __le32 *)
3712 (adev->gfx.mec_fw->data +
3713 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3714 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
3715
3716 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
3717 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
3718 &adev->gfx.mec.mec_fw_obj,
3719 &adev->gfx.mec.mec_fw_gpu_addr,
3720 (void **)&fw);
3721 if (r) {
3722 dev_err(adev->dev, "(%d) failed to create mec fw bo\n", r);
3723 gfx_v11_0_mec_fini(adev);
3724 return r;
3725 }
3726
3727 memcpy(fw, fw_data, fw_size);
3728
3729 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
3730 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
3731
3732 gfx_v11_0_config_mec_cache(adev, adev->gfx.mec.mec_fw_gpu_addr);
3733
3734 /* MEC1 */
3735 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, 0);
3736
3737 for (i = 0; i < mec_hdr->jt_size; i++)
3738 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_DATA,
3739 le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3740
3741 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version);
3742
3743 return 0;
3744 }
3745
gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device * adev)3746 static int gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev)
3747 {
3748 const struct gfx_firmware_header_v2_0 *mec_hdr;
3749 const __le32 *fw_ucode, *fw_data;
3750 u32 tmp, fw_ucode_size, fw_data_size;
3751 u32 i, usec_timeout = 50000; /* Wait for 50 ms */
3752 u32 *fw_ucode_ptr, *fw_data_ptr;
3753 int r;
3754
3755 if (!adev->gfx.mec_fw)
3756 return -EINVAL;
3757
3758 gfx_v11_0_cp_compute_enable(adev, false);
3759
3760 mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
3761 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3762
3763 fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data +
3764 le32_to_cpu(mec_hdr->ucode_offset_bytes));
3765 fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes);
3766
3767 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
3768 le32_to_cpu(mec_hdr->data_offset_bytes));
3769 fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes);
3770
3771 r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3772 64 * 1024,
3773 AMDGPU_GEM_DOMAIN_VRAM |
3774 AMDGPU_GEM_DOMAIN_GTT,
3775 &adev->gfx.mec.mec_fw_obj,
3776 &adev->gfx.mec.mec_fw_gpu_addr,
3777 (void **)&fw_ucode_ptr);
3778 if (r) {
3779 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
3780 gfx_v11_0_mec_fini(adev);
3781 return r;
3782 }
3783
3784 r = amdgpu_bo_create_reserved(adev, fw_data_size,
3785 64 * 1024,
3786 AMDGPU_GEM_DOMAIN_VRAM |
3787 AMDGPU_GEM_DOMAIN_GTT,
3788 &adev->gfx.mec.mec_fw_data_obj,
3789 &adev->gfx.mec.mec_fw_data_gpu_addr,
3790 (void **)&fw_data_ptr);
3791 if (r) {
3792 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
3793 gfx_v11_0_mec_fini(adev);
3794 return r;
3795 }
3796
3797 memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size);
3798 memcpy(fw_data_ptr, fw_data, fw_data_size);
3799
3800 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
3801 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj);
3802 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
3803 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj);
3804
3805 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
3806 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3807 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
3808 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3809 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
3810
3811 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
3812 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
3813 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
3814 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
3815
3816 mutex_lock(&adev->srbm_mutex);
3817 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
3818 soc21_grbm_select(adev, 1, i, 0, 0);
3819
3820 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, adev->gfx.mec.mec_fw_data_gpu_addr);
3821 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
3822 upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr));
3823
3824 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
3825 mec_hdr->ucode_start_addr_lo >> 2 |
3826 mec_hdr->ucode_start_addr_hi << 30);
3827 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
3828 mec_hdr->ucode_start_addr_hi >> 2);
3829
3830 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, adev->gfx.mec.mec_fw_gpu_addr);
3831 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
3832 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3833 }
3834 mutex_unlock(&adev->srbm_mutex);
3835 soc21_grbm_select(adev, 0, 0, 0, 0);
3836
3837 /* Trigger an invalidation of the L1 instruction caches */
3838 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
3839 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3840 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
3841
3842 /* Wait for invalidation complete */
3843 for (i = 0; i < usec_timeout; i++) {
3844 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
3845 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
3846 INVALIDATE_DCACHE_COMPLETE))
3847 break;
3848 udelay(1);
3849 }
3850
3851 if (i >= usec_timeout) {
3852 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3853 return -EINVAL;
3854 }
3855
3856 /* Trigger an invalidation of the L1 instruction caches */
3857 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
3858 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
3859 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
3860
3861 /* Wait for invalidation complete */
3862 for (i = 0; i < usec_timeout; i++) {
3863 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
3864 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
3865 INVALIDATE_CACHE_COMPLETE))
3866 break;
3867 udelay(1);
3868 }
3869
3870 if (i >= usec_timeout) {
3871 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3872 return -EINVAL;
3873 }
3874
3875 return 0;
3876 }
3877
gfx_v11_0_kiq_setting(struct amdgpu_ring * ring)3878 static void gfx_v11_0_kiq_setting(struct amdgpu_ring *ring)
3879 {
3880 uint32_t tmp;
3881 struct amdgpu_device *adev = ring->adev;
3882
3883 /* tell RLC which is KIQ queue */
3884 tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
3885 tmp &= 0xffffff00;
3886 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3887 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
3888 tmp |= 0x80;
3889 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
3890 }
3891
gfx_v11_0_cp_set_doorbell_range(struct amdgpu_device * adev)3892 static void gfx_v11_0_cp_set_doorbell_range(struct amdgpu_device *adev)
3893 {
3894 /* set graphics engine doorbell range */
3895 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER,
3896 (adev->doorbell_index.gfx_ring0 * 2) << 2);
3897 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
3898 (adev->doorbell_index.gfx_userqueue_end * 2) << 2);
3899
3900 /* set compute engine doorbell range */
3901 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
3902 (adev->doorbell_index.kiq * 2) << 2);
3903 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
3904 (adev->doorbell_index.userqueue_end * 2) << 2);
3905 }
3906
gfx_v11_0_gfx_mqd_set_priority(struct amdgpu_device * adev,struct v11_gfx_mqd * mqd,struct amdgpu_mqd_prop * prop)3907 static void gfx_v11_0_gfx_mqd_set_priority(struct amdgpu_device *adev,
3908 struct v11_gfx_mqd *mqd,
3909 struct amdgpu_mqd_prop *prop)
3910 {
3911 bool priority = 0;
3912 u32 tmp;
3913
3914 /* set up default queue priority level
3915 * 0x0 = low priority, 0x1 = high priority
3916 */
3917 if (prop->hqd_pipe_priority == AMDGPU_GFX_PIPE_PRIO_HIGH)
3918 priority = 1;
3919
3920 tmp = regCP_GFX_HQD_QUEUE_PRIORITY_DEFAULT;
3921 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUEUE_PRIORITY, PRIORITY_LEVEL, priority);
3922 mqd->cp_gfx_hqd_queue_priority = tmp;
3923 }
3924
gfx_v11_0_gfx_mqd_init(struct amdgpu_device * adev,void * m,struct amdgpu_mqd_prop * prop)3925 static int gfx_v11_0_gfx_mqd_init(struct amdgpu_device *adev, void *m,
3926 struct amdgpu_mqd_prop *prop)
3927 {
3928 struct v11_gfx_mqd *mqd = m;
3929 uint64_t hqd_gpu_addr, wb_gpu_addr;
3930 uint32_t tmp;
3931 uint32_t rb_bufsz;
3932
3933 /* set up gfx hqd wptr */
3934 mqd->cp_gfx_hqd_wptr = 0;
3935 mqd->cp_gfx_hqd_wptr_hi = 0;
3936
3937 /* set the pointer to the MQD */
3938 mqd->cp_mqd_base_addr = prop->mqd_gpu_addr & 0xfffffffc;
3939 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
3940
3941 /* set up mqd control */
3942 tmp = regCP_GFX_MQD_CONTROL_DEFAULT;
3943 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, VMID, 0);
3944 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, PRIV_STATE, 1);
3945 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, CACHE_POLICY, 0);
3946 mqd->cp_gfx_mqd_control = tmp;
3947
3948 /* set up gfx_hqd_vimd with 0x0 to indicate the ring buffer's vmid */
3949 tmp = regCP_GFX_HQD_VMID_DEFAULT;
3950 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_VMID, VMID, 0);
3951 mqd->cp_gfx_hqd_vmid = 0;
3952
3953 /* set up gfx queue priority */
3954 gfx_v11_0_gfx_mqd_set_priority(adev, mqd, prop);
3955
3956 /* set up time quantum */
3957 tmp = regCP_GFX_HQD_QUANTUM_DEFAULT;
3958 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUANTUM, QUANTUM_EN, 1);
3959 mqd->cp_gfx_hqd_quantum = tmp;
3960
3961 /* set up gfx hqd base. this is similar as CP_RB_BASE */
3962 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
3963 mqd->cp_gfx_hqd_base = hqd_gpu_addr;
3964 mqd->cp_gfx_hqd_base_hi = upper_32_bits(hqd_gpu_addr);
3965
3966 /* set up hqd_rptr_addr/_hi, similar as CP_RB_RPTR */
3967 wb_gpu_addr = prop->rptr_gpu_addr;
3968 mqd->cp_gfx_hqd_rptr_addr = wb_gpu_addr & 0xfffffffc;
3969 mqd->cp_gfx_hqd_rptr_addr_hi =
3970 upper_32_bits(wb_gpu_addr) & 0xffff;
3971
3972 /* set up rb_wptr_poll addr */
3973 wb_gpu_addr = prop->wptr_gpu_addr;
3974 mqd->cp_rb_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3975 mqd->cp_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3976
3977 /* set up the gfx_hqd_control, similar as CP_RB0_CNTL */
3978 rb_bufsz = order_base_2(prop->queue_size / 4) - 1;
3979 tmp = regCP_GFX_HQD_CNTL_DEFAULT;
3980 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz);
3981 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2);
3982 #ifdef __BIG_ENDIAN
3983 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1);
3984 #endif
3985 mqd->cp_gfx_hqd_cntl = tmp;
3986
3987 /* set up cp_doorbell_control */
3988 tmp = regCP_RB_DOORBELL_CONTROL_DEFAULT;
3989 if (prop->use_doorbell) {
3990 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3991 DOORBELL_OFFSET, prop->doorbell_index);
3992 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3993 DOORBELL_EN, 1);
3994 } else
3995 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3996 DOORBELL_EN, 0);
3997 mqd->cp_rb_doorbell_control = tmp;
3998
3999 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4000 mqd->cp_gfx_hqd_rptr = regCP_GFX_HQD_RPTR_DEFAULT;
4001
4002 /* active the queue */
4003 mqd->cp_gfx_hqd_active = 1;
4004
4005 return 0;
4006 }
4007
gfx_v11_0_kgq_init_queue(struct amdgpu_ring * ring,bool reset)4008 static int gfx_v11_0_kgq_init_queue(struct amdgpu_ring *ring, bool reset)
4009 {
4010 struct amdgpu_device *adev = ring->adev;
4011 struct v11_gfx_mqd *mqd = ring->mqd_ptr;
4012 int mqd_idx = ring - &adev->gfx.gfx_ring[0];
4013
4014 if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) {
4015 memset((void *)mqd, 0, sizeof(*mqd));
4016 mutex_lock(&adev->srbm_mutex);
4017 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4018 amdgpu_ring_init_mqd(ring);
4019 soc21_grbm_select(adev, 0, 0, 0, 0);
4020 mutex_unlock(&adev->srbm_mutex);
4021 if (adev->gfx.me.mqd_backup[mqd_idx])
4022 memcpy_fromio(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
4023 } else {
4024 /* restore mqd with the backup copy */
4025 if (adev->gfx.me.mqd_backup[mqd_idx])
4026 memcpy_toio(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd));
4027 /* reset the ring */
4028 ring->wptr = 0;
4029 *ring->wptr_cpu_addr = 0;
4030 amdgpu_ring_clear_ring(ring);
4031 }
4032
4033 return 0;
4034 }
4035
gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device * adev)4036 static int gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev)
4037 {
4038 int r, i;
4039 struct amdgpu_ring *ring;
4040
4041 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4042 ring = &adev->gfx.gfx_ring[i];
4043
4044 r = amdgpu_bo_reserve(ring->mqd_obj, false);
4045 if (unlikely(r != 0))
4046 return r;
4047
4048 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
4049 if (!r) {
4050 r = gfx_v11_0_kgq_init_queue(ring, false);
4051 amdgpu_bo_kunmap(ring->mqd_obj);
4052 ring->mqd_ptr = NULL;
4053 }
4054 amdgpu_bo_unreserve(ring->mqd_obj);
4055 if (r)
4056 return r;
4057 }
4058
4059 r = amdgpu_gfx_enable_kgq(adev, 0);
4060 if (r)
4061 return r;
4062
4063 return gfx_v11_0_cp_gfx_start(adev);
4064 }
4065
gfx_v11_0_compute_mqd_init(struct amdgpu_device * adev,void * m,struct amdgpu_mqd_prop * prop)4066 static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
4067 struct amdgpu_mqd_prop *prop)
4068 {
4069 struct v11_compute_mqd *mqd = m;
4070 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
4071 uint32_t tmp;
4072
4073 mqd->header = 0xC0310800;
4074 mqd->compute_pipelinestat_enable = 0x00000001;
4075 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
4076 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
4077 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
4078 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
4079 mqd->compute_misc_reserved = 0x00000007;
4080
4081 eop_base_addr = prop->eop_gpu_addr >> 8;
4082 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
4083 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
4084
4085 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4086 tmp = regCP_HQD_EOP_CONTROL_DEFAULT;
4087 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
4088 (order_base_2(GFX11_MEC_HPD_SIZE / 4) - 1));
4089
4090 mqd->cp_hqd_eop_control = tmp;
4091
4092 /* enable doorbell? */
4093 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT;
4094
4095 if (prop->use_doorbell) {
4096 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4097 DOORBELL_OFFSET, prop->doorbell_index);
4098 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4099 DOORBELL_EN, 1);
4100 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4101 DOORBELL_SOURCE, 0);
4102 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4103 DOORBELL_HIT, 0);
4104 } else {
4105 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4106 DOORBELL_EN, 0);
4107 }
4108
4109 mqd->cp_hqd_pq_doorbell_control = tmp;
4110
4111 /* disable the queue if it's active */
4112 mqd->cp_hqd_dequeue_request = 0;
4113 mqd->cp_hqd_pq_rptr = 0;
4114 mqd->cp_hqd_pq_wptr_lo = 0;
4115 mqd->cp_hqd_pq_wptr_hi = 0;
4116
4117 /* set the pointer to the MQD */
4118 mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc;
4119 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
4120
4121 /* set MQD vmid to 0 */
4122 tmp = regCP_MQD_CONTROL_DEFAULT;
4123 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
4124 mqd->cp_mqd_control = tmp;
4125
4126 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4127 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
4128 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
4129 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4130
4131 /* set up the HQD, this is similar to CP_RB0_CNTL */
4132 tmp = regCP_HQD_PQ_CONTROL_DEFAULT;
4133 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
4134 (order_base_2(prop->queue_size / 4) - 1));
4135 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
4136 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
4137 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1);
4138 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH,
4139 prop->allow_tunneling);
4140 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
4141 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
4142 mqd->cp_hqd_pq_control = tmp;
4143
4144 /* set the wb address whether it's enabled or not */
4145 wb_gpu_addr = prop->rptr_gpu_addr;
4146 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
4147 mqd->cp_hqd_pq_rptr_report_addr_hi =
4148 upper_32_bits(wb_gpu_addr) & 0xffff;
4149
4150 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4151 wb_gpu_addr = prop->wptr_gpu_addr;
4152 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
4153 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4154
4155 tmp = 0;
4156 /* enable the doorbell if requested */
4157 if (prop->use_doorbell) {
4158 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT;
4159 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4160 DOORBELL_OFFSET, prop->doorbell_index);
4161
4162 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4163 DOORBELL_EN, 1);
4164 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4165 DOORBELL_SOURCE, 0);
4166 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4167 DOORBELL_HIT, 0);
4168 }
4169
4170 mqd->cp_hqd_pq_doorbell_control = tmp;
4171
4172 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4173 mqd->cp_hqd_pq_rptr = regCP_HQD_PQ_RPTR_DEFAULT;
4174
4175 /* set the vmid for the queue */
4176 mqd->cp_hqd_vmid = 0;
4177
4178 tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT;
4179 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x55);
4180 mqd->cp_hqd_persistent_state = tmp;
4181
4182 /* set MIN_IB_AVAIL_SIZE */
4183 tmp = regCP_HQD_IB_CONTROL_DEFAULT;
4184 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
4185 mqd->cp_hqd_ib_control = tmp;
4186
4187 /* set static priority for a compute queue/ring */
4188 mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
4189 mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;
4190
4191 mqd->cp_hqd_active = prop->hqd_active;
4192
4193 return 0;
4194 }
4195
gfx_v11_0_kiq_init_register(struct amdgpu_ring * ring)4196 static int gfx_v11_0_kiq_init_register(struct amdgpu_ring *ring)
4197 {
4198 struct amdgpu_device *adev = ring->adev;
4199 struct v11_compute_mqd *mqd = ring->mqd_ptr;
4200 int j;
4201
4202 /* inactivate the queue */
4203 if (amdgpu_sriov_vf(adev))
4204 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0);
4205
4206 /* disable wptr polling */
4207 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4208
4209 /* write the EOP addr */
4210 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR,
4211 mqd->cp_hqd_eop_base_addr_lo);
4212 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI,
4213 mqd->cp_hqd_eop_base_addr_hi);
4214
4215 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4216 WREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL,
4217 mqd->cp_hqd_eop_control);
4218
4219 /* enable doorbell? */
4220 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
4221 mqd->cp_hqd_pq_doorbell_control);
4222
4223 /* disable the queue if it's active */
4224 if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {
4225 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1);
4226 for (j = 0; j < adev->usec_timeout; j++) {
4227 if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
4228 break;
4229 udelay(1);
4230 }
4231 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST,
4232 mqd->cp_hqd_dequeue_request);
4233 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR,
4234 mqd->cp_hqd_pq_rptr);
4235 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
4236 mqd->cp_hqd_pq_wptr_lo);
4237 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
4238 mqd->cp_hqd_pq_wptr_hi);
4239 }
4240
4241 /* set the pointer to the MQD */
4242 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR,
4243 mqd->cp_mqd_base_addr_lo);
4244 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI,
4245 mqd->cp_mqd_base_addr_hi);
4246
4247 /* set MQD vmid to 0 */
4248 WREG32_SOC15(GC, 0, regCP_MQD_CONTROL,
4249 mqd->cp_mqd_control);
4250
4251 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4252 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE,
4253 mqd->cp_hqd_pq_base_lo);
4254 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI,
4255 mqd->cp_hqd_pq_base_hi);
4256
4257 /* set up the HQD, this is similar to CP_RB0_CNTL */
4258 WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL,
4259 mqd->cp_hqd_pq_control);
4260
4261 /* set the wb address whether it's enabled or not */
4262 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,
4263 mqd->cp_hqd_pq_rptr_report_addr_lo);
4264 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4265 mqd->cp_hqd_pq_rptr_report_addr_hi);
4266
4267 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4268 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,
4269 mqd->cp_hqd_pq_wptr_poll_addr_lo);
4270 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
4271 mqd->cp_hqd_pq_wptr_poll_addr_hi);
4272
4273 /* enable the doorbell if requested */
4274 if (ring->use_doorbell) {
4275 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
4276 (adev->doorbell_index.kiq * 2) << 2);
4277 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
4278 (adev->doorbell_index.userqueue_end * 2) << 2);
4279 }
4280
4281 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
4282 mqd->cp_hqd_pq_doorbell_control);
4283
4284 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4285 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
4286 mqd->cp_hqd_pq_wptr_lo);
4287 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
4288 mqd->cp_hqd_pq_wptr_hi);
4289
4290 /* set the vmid for the queue */
4291 WREG32_SOC15(GC, 0, regCP_HQD_VMID, mqd->cp_hqd_vmid);
4292
4293 WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE,
4294 mqd->cp_hqd_persistent_state);
4295
4296 /* activate the queue */
4297 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE,
4298 mqd->cp_hqd_active);
4299
4300 if (ring->use_doorbell)
4301 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
4302
4303 return 0;
4304 }
4305
gfx_v11_0_kiq_init_queue(struct amdgpu_ring * ring)4306 static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring)
4307 {
4308 struct amdgpu_device *adev = ring->adev;
4309 struct v11_compute_mqd *mqd = ring->mqd_ptr;
4310
4311 gfx_v11_0_kiq_setting(ring);
4312
4313 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
4314 /* reset MQD to a clean status */
4315 if (adev->gfx.kiq[0].mqd_backup)
4316 memcpy_toio(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(*mqd));
4317
4318 /* reset ring buffer */
4319 ring->wptr = 0;
4320 amdgpu_ring_clear_ring(ring);
4321
4322 mutex_lock(&adev->srbm_mutex);
4323 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4324 gfx_v11_0_kiq_init_register(ring);
4325 soc21_grbm_select(adev, 0, 0, 0, 0);
4326 mutex_unlock(&adev->srbm_mutex);
4327 } else {
4328 memset((void *)mqd, 0, sizeof(*mqd));
4329 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
4330 amdgpu_ring_clear_ring(ring);
4331 mutex_lock(&adev->srbm_mutex);
4332 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4333 amdgpu_ring_init_mqd(ring);
4334 gfx_v11_0_kiq_init_register(ring);
4335 soc21_grbm_select(adev, 0, 0, 0, 0);
4336 mutex_unlock(&adev->srbm_mutex);
4337
4338 if (adev->gfx.kiq[0].mqd_backup)
4339 memcpy_fromio(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(*mqd));
4340 }
4341
4342 return 0;
4343 }
4344
gfx_v11_0_kcq_init_queue(struct amdgpu_ring * ring,bool reset)4345 static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring, bool reset)
4346 {
4347 struct amdgpu_device *adev = ring->adev;
4348 struct v11_compute_mqd *mqd = ring->mqd_ptr;
4349 int mqd_idx = ring - &adev->gfx.compute_ring[0];
4350
4351 if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) {
4352 memset((void *)mqd, 0, sizeof(*mqd));
4353 mutex_lock(&adev->srbm_mutex);
4354 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4355 amdgpu_ring_init_mqd(ring);
4356 soc21_grbm_select(adev, 0, 0, 0, 0);
4357 mutex_unlock(&adev->srbm_mutex);
4358
4359 if (adev->gfx.mec.mqd_backup[mqd_idx])
4360 memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
4361 } else {
4362 /* restore MQD to a clean status */
4363 if (adev->gfx.mec.mqd_backup[mqd_idx])
4364 memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
4365 /* reset ring buffer */
4366 ring->wptr = 0;
4367 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
4368 amdgpu_ring_clear_ring(ring);
4369 }
4370
4371 return 0;
4372 }
4373
gfx_v11_0_kiq_resume(struct amdgpu_device * adev)4374 static int gfx_v11_0_kiq_resume(struct amdgpu_device *adev)
4375 {
4376 struct amdgpu_ring *ring;
4377 int r;
4378
4379 ring = &adev->gfx.kiq[0].ring;
4380
4381 r = amdgpu_bo_reserve(ring->mqd_obj, false);
4382 if (unlikely(r != 0))
4383 return r;
4384
4385 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
4386 if (unlikely(r != 0)) {
4387 amdgpu_bo_unreserve(ring->mqd_obj);
4388 return r;
4389 }
4390
4391 gfx_v11_0_kiq_init_queue(ring);
4392 amdgpu_bo_kunmap(ring->mqd_obj);
4393 ring->mqd_ptr = NULL;
4394 amdgpu_bo_unreserve(ring->mqd_obj);
4395 ring->sched.ready = true;
4396 return 0;
4397 }
4398
gfx_v11_0_kcq_resume(struct amdgpu_device * adev)4399 static int gfx_v11_0_kcq_resume(struct amdgpu_device *adev)
4400 {
4401 struct amdgpu_ring *ring = NULL;
4402 int r = 0, i;
4403
4404 if (!amdgpu_async_gfx_ring)
4405 gfx_v11_0_cp_compute_enable(adev, true);
4406
4407 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4408 ring = &adev->gfx.compute_ring[i];
4409
4410 r = amdgpu_bo_reserve(ring->mqd_obj, false);
4411 if (unlikely(r != 0))
4412 goto done;
4413 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
4414 if (!r) {
4415 r = gfx_v11_0_kcq_init_queue(ring, false);
4416 amdgpu_bo_kunmap(ring->mqd_obj);
4417 ring->mqd_ptr = NULL;
4418 }
4419 amdgpu_bo_unreserve(ring->mqd_obj);
4420 if (r)
4421 goto done;
4422 }
4423
4424 r = amdgpu_gfx_enable_kcq(adev, 0);
4425 done:
4426 return r;
4427 }
4428
gfx_v11_0_cp_resume(struct amdgpu_device * adev)4429 static int gfx_v11_0_cp_resume(struct amdgpu_device *adev)
4430 {
4431 int r, i;
4432 struct amdgpu_ring *ring;
4433
4434 if (!(adev->flags & AMD_IS_APU))
4435 gfx_v11_0_enable_gui_idle_interrupt(adev, false);
4436
4437 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4438 /* legacy firmware loading */
4439 r = gfx_v11_0_cp_gfx_load_microcode(adev);
4440 if (r)
4441 return r;
4442
4443 if (adev->gfx.rs64_enable)
4444 r = gfx_v11_0_cp_compute_load_microcode_rs64(adev);
4445 else
4446 r = gfx_v11_0_cp_compute_load_microcode(adev);
4447 if (r)
4448 return r;
4449 }
4450
4451 gfx_v11_0_cp_set_doorbell_range(adev);
4452
4453 if (amdgpu_async_gfx_ring) {
4454 gfx_v11_0_cp_compute_enable(adev, true);
4455 gfx_v11_0_cp_gfx_enable(adev, true);
4456 }
4457
4458 if (adev->enable_mes_kiq && adev->mes.kiq_hw_init)
4459 r = amdgpu_mes_kiq_hw_init(adev);
4460 else
4461 r = gfx_v11_0_kiq_resume(adev);
4462 if (r)
4463 return r;
4464
4465 r = gfx_v11_0_kcq_resume(adev);
4466 if (r)
4467 return r;
4468
4469 if (!amdgpu_async_gfx_ring) {
4470 r = gfx_v11_0_cp_gfx_resume(adev);
4471 if (r)
4472 return r;
4473 } else {
4474 r = gfx_v11_0_cp_async_gfx_ring_resume(adev);
4475 if (r)
4476 return r;
4477 }
4478
4479 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4480 ring = &adev->gfx.gfx_ring[i];
4481 r = amdgpu_ring_test_helper(ring);
4482 if (r)
4483 return r;
4484 }
4485
4486 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4487 ring = &adev->gfx.compute_ring[i];
4488 r = amdgpu_ring_test_helper(ring);
4489 if (r)
4490 return r;
4491 }
4492
4493 return 0;
4494 }
4495
gfx_v11_0_cp_enable(struct amdgpu_device * adev,bool enable)4496 static void gfx_v11_0_cp_enable(struct amdgpu_device *adev, bool enable)
4497 {
4498 gfx_v11_0_cp_gfx_enable(adev, enable);
4499 gfx_v11_0_cp_compute_enable(adev, enable);
4500 }
4501
gfx_v11_0_gfxhub_enable(struct amdgpu_device * adev)4502 static int gfx_v11_0_gfxhub_enable(struct amdgpu_device *adev)
4503 {
4504 int r;
4505 bool value;
4506
4507 r = adev->gfxhub.funcs->gart_enable(adev);
4508 if (r)
4509 return r;
4510
4511 amdgpu_device_flush_hdp(adev, NULL);
4512
4513 value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ?
4514 false : true;
4515
4516 adev->gfxhub.funcs->set_fault_enable_default(adev, value);
4517 /* TODO investigate why this and the hdp flush above is needed,
4518 * are we missing a flush somewhere else? */
4519 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0);
4520
4521 return 0;
4522 }
4523
gfx_v11_0_select_cp_fw_arch(struct amdgpu_device * adev)4524 static void gfx_v11_0_select_cp_fw_arch(struct amdgpu_device *adev)
4525 {
4526 u32 tmp;
4527
4528 /* select RS64 */
4529 if (adev->gfx.rs64_enable) {
4530 tmp = RREG32_SOC15(GC, 0, regCP_GFX_CNTL);
4531 tmp = REG_SET_FIELD(tmp, CP_GFX_CNTL, ENGINE_SEL, 1);
4532 WREG32_SOC15(GC, 0, regCP_GFX_CNTL, tmp);
4533
4534 tmp = RREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL);
4535 tmp = REG_SET_FIELD(tmp, CP_MEC_ISA_CNTL, ISA_MODE, 1);
4536 WREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL, tmp);
4537 }
4538
4539 if (amdgpu_emu_mode == 1)
4540 msleep(100);
4541 }
4542
get_gb_addr_config(struct amdgpu_device * adev)4543 static int get_gb_addr_config(struct amdgpu_device * adev)
4544 {
4545 u32 gb_addr_config;
4546
4547 gb_addr_config = RREG32_SOC15(GC, 0, regGB_ADDR_CONFIG);
4548 if (gb_addr_config == 0)
4549 return -EINVAL;
4550
4551 adev->gfx.config.gb_addr_config_fields.num_pkrs =
4552 1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG, NUM_PKRS);
4553
4554 adev->gfx.config.gb_addr_config = gb_addr_config;
4555
4556 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
4557 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4558 GB_ADDR_CONFIG, NUM_PIPES);
4559
4560 adev->gfx.config.max_tile_pipes =
4561 adev->gfx.config.gb_addr_config_fields.num_pipes;
4562
4563 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
4564 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4565 GB_ADDR_CONFIG, MAX_COMPRESSED_FRAGS);
4566 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
4567 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4568 GB_ADDR_CONFIG, NUM_RB_PER_SE);
4569 adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
4570 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4571 GB_ADDR_CONFIG, NUM_SHADER_ENGINES);
4572 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
4573 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4574 GB_ADDR_CONFIG, PIPE_INTERLEAVE_SIZE));
4575
4576 return 0;
4577 }
4578
gfx_v11_0_disable_gpa_mode(struct amdgpu_device * adev)4579 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev)
4580 {
4581 uint32_t data;
4582
4583 data = RREG32_SOC15(GC, 0, regCPC_PSP_DEBUG);
4584 data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK;
4585 WREG32_SOC15(GC, 0, regCPC_PSP_DEBUG, data);
4586
4587 data = RREG32_SOC15(GC, 0, regCPG_PSP_DEBUG);
4588 data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK;
4589 WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data);
4590 }
4591
gfx_v11_0_hw_init(void * handle)4592 static int gfx_v11_0_hw_init(void *handle)
4593 {
4594 int r;
4595 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4596
4597 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
4598 if (adev->gfx.imu.funcs) {
4599 /* RLC autoload sequence 1: Program rlc ram */
4600 if (adev->gfx.imu.funcs->program_rlc_ram)
4601 adev->gfx.imu.funcs->program_rlc_ram(adev);
4602 /* rlc autoload firmware */
4603 r = gfx_v11_0_rlc_backdoor_autoload_enable(adev);
4604 if (r)
4605 return r;
4606 }
4607 } else {
4608 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4609 if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
4610 if (adev->gfx.imu.funcs->load_microcode)
4611 adev->gfx.imu.funcs->load_microcode(adev);
4612 if (adev->gfx.imu.funcs->setup_imu)
4613 adev->gfx.imu.funcs->setup_imu(adev);
4614 if (adev->gfx.imu.funcs->start_imu)
4615 adev->gfx.imu.funcs->start_imu(adev);
4616 }
4617
4618 /* disable gpa mode in backdoor loading */
4619 gfx_v11_0_disable_gpa_mode(adev);
4620 }
4621 }
4622
4623 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) ||
4624 (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
4625 r = gfx_v11_0_wait_for_rlc_autoload_complete(adev);
4626 if (r) {
4627 dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r);
4628 return r;
4629 }
4630 }
4631
4632 adev->gfx.is_poweron = true;
4633
4634 if(get_gb_addr_config(adev))
4635 DRM_WARN("Invalid gb_addr_config !\n");
4636
4637 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP &&
4638 adev->gfx.rs64_enable)
4639 gfx_v11_0_config_gfx_rs64(adev);
4640
4641 r = gfx_v11_0_gfxhub_enable(adev);
4642 if (r)
4643 return r;
4644
4645 if (!amdgpu_emu_mode)
4646 gfx_v11_0_init_golden_registers(adev);
4647
4648 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) ||
4649 (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO && amdgpu_dpm == 1)) {
4650 /**
4651 * For gfx 11, rlc firmware loading relies on smu firmware is
4652 * loaded firstly, so in direct type, it has to load smc ucode
4653 * here before rlc.
4654 */
4655 r = amdgpu_pm_load_smu_firmware(adev, NULL);
4656 if (r)
4657 return r;
4658 }
4659
4660 gfx_v11_0_constants_init(adev);
4661
4662 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
4663 gfx_v11_0_select_cp_fw_arch(adev);
4664
4665 if (adev->nbio.funcs->gc_doorbell_init)
4666 adev->nbio.funcs->gc_doorbell_init(adev);
4667
4668 r = gfx_v11_0_rlc_resume(adev);
4669 if (r)
4670 return r;
4671
4672 /*
4673 * init golden registers and rlc resume may override some registers,
4674 * reconfig them here
4675 */
4676 gfx_v11_0_tcp_harvest(adev);
4677
4678 r = gfx_v11_0_cp_resume(adev);
4679 if (r)
4680 return r;
4681
4682 /* get IMU version from HW if it's not set */
4683 if (!adev->gfx.imu_fw_version)
4684 adev->gfx.imu_fw_version = RREG32_SOC15(GC, 0, regGFX_IMU_SCRATCH_0);
4685
4686 return r;
4687 }
4688
gfx_v11_0_hw_fini(void * handle)4689 static int gfx_v11_0_hw_fini(void *handle)
4690 {
4691 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4692
4693 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4694 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4695 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
4696
4697 if (!adev->no_hw_access) {
4698 if (amdgpu_async_gfx_ring) {
4699 if (amdgpu_gfx_disable_kgq(adev, 0))
4700 DRM_ERROR("KGQ disable failed\n");
4701 }
4702
4703 if (amdgpu_gfx_disable_kcq(adev, 0))
4704 DRM_ERROR("KCQ disable failed\n");
4705
4706 amdgpu_mes_kiq_hw_fini(adev);
4707 }
4708
4709 if (amdgpu_sriov_vf(adev))
4710 /* Remove the steps disabling CPG and clearing KIQ position,
4711 * so that CP could perform IDLE-SAVE during switch. Those
4712 * steps are necessary to avoid a DMAR error in gfx9 but it is
4713 * not reproduced on gfx11.
4714 */
4715 return 0;
4716
4717 gfx_v11_0_cp_enable(adev, false);
4718 gfx_v11_0_enable_gui_idle_interrupt(adev, false);
4719
4720 adev->gfxhub.funcs->gart_disable(adev);
4721
4722 adev->gfx.is_poweron = false;
4723
4724 return 0;
4725 }
4726
gfx_v11_0_suspend(void * handle)4727 static int gfx_v11_0_suspend(void *handle)
4728 {
4729 return gfx_v11_0_hw_fini(handle);
4730 }
4731
gfx_v11_0_resume(void * handle)4732 static int gfx_v11_0_resume(void *handle)
4733 {
4734 return gfx_v11_0_hw_init(handle);
4735 }
4736
gfx_v11_0_is_idle(void * handle)4737 static bool gfx_v11_0_is_idle(void *handle)
4738 {
4739 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4740
4741 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, regGRBM_STATUS),
4742 GRBM_STATUS, GUI_ACTIVE))
4743 return false;
4744 else
4745 return true;
4746 }
4747
gfx_v11_0_wait_for_idle(void * handle)4748 static int gfx_v11_0_wait_for_idle(void *handle)
4749 {
4750 unsigned i;
4751 u32 tmp;
4752 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4753
4754 for (i = 0; i < adev->usec_timeout; i++) {
4755 /* read MC_STATUS */
4756 tmp = RREG32_SOC15(GC, 0, regGRBM_STATUS) &
4757 GRBM_STATUS__GUI_ACTIVE_MASK;
4758
4759 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE))
4760 return 0;
4761 udelay(1);
4762 }
4763 return -ETIMEDOUT;
4764 }
4765
gfx_v11_0_request_gfx_index_mutex(struct amdgpu_device * adev,bool req)4766 int gfx_v11_0_request_gfx_index_mutex(struct amdgpu_device *adev,
4767 bool req)
4768 {
4769 u32 i, tmp, val;
4770
4771 for (i = 0; i < adev->usec_timeout; i++) {
4772 /* Request with MeId=2, PipeId=0 */
4773 tmp = REG_SET_FIELD(0, CP_GFX_INDEX_MUTEX, REQUEST, req);
4774 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, CLIENTID, 4);
4775 WREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX, tmp);
4776
4777 val = RREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX);
4778 if (req) {
4779 if (val == tmp)
4780 break;
4781 } else {
4782 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX,
4783 REQUEST, 1);
4784
4785 /* unlocked or locked by firmware */
4786 if (val != tmp)
4787 break;
4788 }
4789 udelay(1);
4790 }
4791
4792 if (i >= adev->usec_timeout)
4793 return -EINVAL;
4794
4795 return 0;
4796 }
4797
gfx_v11_0_soft_reset(void * handle)4798 static int gfx_v11_0_soft_reset(void *handle)
4799 {
4800 u32 grbm_soft_reset = 0;
4801 u32 tmp;
4802 int r, i, j, k;
4803 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4804
4805 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
4806
4807 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
4808 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 0);
4809 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 0);
4810 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 0);
4811 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 0);
4812 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp);
4813
4814 mutex_lock(&adev->srbm_mutex);
4815 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
4816 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
4817 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
4818 soc21_grbm_select(adev, i, k, j, 0);
4819
4820 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2);
4821 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1);
4822 }
4823 }
4824 }
4825 for (i = 0; i < adev->gfx.me.num_me; ++i) {
4826 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
4827 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
4828 soc21_grbm_select(adev, i, k, j, 0);
4829
4830 WREG32_SOC15(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST, 0x1);
4831 }
4832 }
4833 }
4834 soc21_grbm_select(adev, 0, 0, 0, 0);
4835 mutex_unlock(&adev->srbm_mutex);
4836
4837 /* Try to acquire the gfx mutex before access to CP_VMID_RESET */
4838 mutex_lock(&adev->gfx.reset_sem_mutex);
4839 r = gfx_v11_0_request_gfx_index_mutex(adev, true);
4840 if (r) {
4841 mutex_unlock(&adev->gfx.reset_sem_mutex);
4842 DRM_ERROR("Failed to acquire the gfx mutex during soft reset\n");
4843 return r;
4844 }
4845
4846 WREG32_SOC15(GC, 0, regCP_VMID_RESET, 0xfffffffe);
4847
4848 // Read CP_VMID_RESET register three times.
4849 // to get sufficient time for GFX_HQD_ACTIVE reach 0
4850 RREG32_SOC15(GC, 0, regCP_VMID_RESET);
4851 RREG32_SOC15(GC, 0, regCP_VMID_RESET);
4852 RREG32_SOC15(GC, 0, regCP_VMID_RESET);
4853
4854 /* release the gfx mutex */
4855 r = gfx_v11_0_request_gfx_index_mutex(adev, false);
4856 mutex_unlock(&adev->gfx.reset_sem_mutex);
4857 if (r) {
4858 DRM_ERROR("Failed to release the gfx mutex during soft reset\n");
4859 return r;
4860 }
4861
4862 for (i = 0; i < adev->usec_timeout; i++) {
4863 if (!RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) &&
4864 !RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE))
4865 break;
4866 udelay(1);
4867 }
4868 if (i >= adev->usec_timeout) {
4869 printk("Failed to wait all pipes clean\n");
4870 return -EINVAL;
4871 }
4872
4873 /********** trigger soft reset ***********/
4874 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
4875 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4876 SOFT_RESET_CP, 1);
4877 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4878 SOFT_RESET_GFX, 1);
4879 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4880 SOFT_RESET_CPF, 1);
4881 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4882 SOFT_RESET_CPC, 1);
4883 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4884 SOFT_RESET_CPG, 1);
4885 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset);
4886 /********** exit soft reset ***********/
4887 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
4888 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4889 SOFT_RESET_CP, 0);
4890 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4891 SOFT_RESET_GFX, 0);
4892 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4893 SOFT_RESET_CPF, 0);
4894 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4895 SOFT_RESET_CPC, 0);
4896 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4897 SOFT_RESET_CPG, 0);
4898 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset);
4899
4900 tmp = RREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL);
4901 tmp = REG_SET_FIELD(tmp, CP_SOFT_RESET_CNTL, CMP_HQD_REG_RESET, 0x1);
4902 WREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL, tmp);
4903
4904 WREG32_SOC15(GC, 0, regCP_ME_CNTL, 0x0);
4905 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, 0x0);
4906
4907 for (i = 0; i < adev->usec_timeout; i++) {
4908 if (!RREG32_SOC15(GC, 0, regCP_VMID_RESET))
4909 break;
4910 udelay(1);
4911 }
4912 if (i >= adev->usec_timeout) {
4913 printk("Failed to wait CP_VMID_RESET to 0\n");
4914 return -EINVAL;
4915 }
4916
4917 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
4918 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
4919 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
4920 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
4921 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
4922 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp);
4923
4924 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
4925
4926 return gfx_v11_0_cp_resume(adev);
4927 }
4928
gfx_v11_0_check_soft_reset(void * handle)4929 static bool gfx_v11_0_check_soft_reset(void *handle)
4930 {
4931 int i, r;
4932 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4933 struct amdgpu_ring *ring;
4934 long tmo = msecs_to_jiffies(1000);
4935
4936 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4937 ring = &adev->gfx.gfx_ring[i];
4938 r = amdgpu_ring_test_ib(ring, tmo);
4939 if (r)
4940 return true;
4941 }
4942
4943 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4944 ring = &adev->gfx.compute_ring[i];
4945 r = amdgpu_ring_test_ib(ring, tmo);
4946 if (r)
4947 return true;
4948 }
4949
4950 return false;
4951 }
4952
gfx_v11_0_post_soft_reset(void * handle)4953 static int gfx_v11_0_post_soft_reset(void *handle)
4954 {
4955 /**
4956 * GFX soft reset will impact MES, need resume MES when do GFX soft reset
4957 */
4958 return amdgpu_mes_resume((struct amdgpu_device *)handle);
4959 }
4960
gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device * adev)4961 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4962 {
4963 uint64_t clock;
4964 uint64_t clock_counter_lo, clock_counter_hi_pre, clock_counter_hi_after;
4965
4966 if (amdgpu_sriov_vf(adev)) {
4967 amdgpu_gfx_off_ctrl(adev, false);
4968 mutex_lock(&adev->gfx.gpu_clock_mutex);
4969 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI);
4970 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO);
4971 clock_counter_hi_after = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI);
4972 if (clock_counter_hi_pre != clock_counter_hi_after)
4973 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO);
4974 mutex_unlock(&adev->gfx.gpu_clock_mutex);
4975 amdgpu_gfx_off_ctrl(adev, true);
4976 } else {
4977 preempt_disable();
4978 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER);
4979 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER);
4980 clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER);
4981 if (clock_counter_hi_pre != clock_counter_hi_after)
4982 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER);
4983 preempt_enable();
4984 }
4985 clock = clock_counter_lo | (clock_counter_hi_after << 32ULL);
4986
4987 return clock;
4988 }
4989
gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring * ring,uint32_t vmid,uint32_t gds_base,uint32_t gds_size,uint32_t gws_base,uint32_t gws_size,uint32_t oa_base,uint32_t oa_size)4990 static void gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4991 uint32_t vmid,
4992 uint32_t gds_base, uint32_t gds_size,
4993 uint32_t gws_base, uint32_t gws_size,
4994 uint32_t oa_base, uint32_t oa_size)
4995 {
4996 struct amdgpu_device *adev = ring->adev;
4997
4998 /* GDS Base */
4999 gfx_v11_0_write_data_to_reg(ring, 0, false,
5000 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_BASE) + 2 * vmid,
5001 gds_base);
5002
5003 /* GDS Size */
5004 gfx_v11_0_write_data_to_reg(ring, 0, false,
5005 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_SIZE) + 2 * vmid,
5006 gds_size);
5007
5008 /* GWS */
5009 gfx_v11_0_write_data_to_reg(ring, 0, false,
5010 SOC15_REG_OFFSET(GC, 0, regGDS_GWS_VMID0) + vmid,
5011 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
5012
5013 /* OA */
5014 gfx_v11_0_write_data_to_reg(ring, 0, false,
5015 SOC15_REG_OFFSET(GC, 0, regGDS_OA_VMID0) + vmid,
5016 (1 << (oa_size + oa_base)) - (1 << oa_base));
5017 }
5018
gfx_v11_0_early_init(void * handle)5019 static int gfx_v11_0_early_init(void *handle)
5020 {
5021 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5022
5023 adev->gfx.funcs = &gfx_v11_0_gfx_funcs;
5024
5025 adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS;
5026 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
5027 AMDGPU_MAX_COMPUTE_RINGS);
5028
5029 gfx_v11_0_set_kiq_pm4_funcs(adev);
5030 gfx_v11_0_set_ring_funcs(adev);
5031 gfx_v11_0_set_irq_funcs(adev);
5032 gfx_v11_0_set_gds_init(adev);
5033 gfx_v11_0_set_rlc_funcs(adev);
5034 gfx_v11_0_set_mqd_funcs(adev);
5035 gfx_v11_0_set_imu_funcs(adev);
5036
5037 gfx_v11_0_init_rlcg_reg_access_ctrl(adev);
5038
5039 return gfx_v11_0_init_microcode(adev);
5040 }
5041
gfx_v11_0_late_init(void * handle)5042 static int gfx_v11_0_late_init(void *handle)
5043 {
5044 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5045 int r;
5046
5047 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
5048 if (r)
5049 return r;
5050
5051 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
5052 if (r)
5053 return r;
5054
5055 r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
5056 if (r)
5057 return r;
5058 return 0;
5059 }
5060
gfx_v11_0_is_rlc_enabled(struct amdgpu_device * adev)5061 static bool gfx_v11_0_is_rlc_enabled(struct amdgpu_device *adev)
5062 {
5063 uint32_t rlc_cntl;
5064
5065 /* if RLC is not enabled, do nothing */
5066 rlc_cntl = RREG32_SOC15(GC, 0, regRLC_CNTL);
5067 return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false;
5068 }
5069
gfx_v11_0_set_safe_mode(struct amdgpu_device * adev,int xcc_id)5070 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
5071 {
5072 uint32_t data;
5073 unsigned i;
5074
5075 data = RLC_SAFE_MODE__CMD_MASK;
5076 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
5077
5078 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, data);
5079
5080 /* wait for RLC_SAFE_MODE */
5081 for (i = 0; i < adev->usec_timeout; i++) {
5082 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, regRLC_SAFE_MODE),
5083 RLC_SAFE_MODE, CMD))
5084 break;
5085 udelay(1);
5086 }
5087 }
5088
gfx_v11_0_unset_safe_mode(struct amdgpu_device * adev,int xcc_id)5089 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
5090 {
5091 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK);
5092 }
5093
gfx_v11_0_update_perf_clk(struct amdgpu_device * adev,bool enable)5094 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev,
5095 bool enable)
5096 {
5097 uint32_t def, data;
5098
5099 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK))
5100 return;
5101
5102 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5103
5104 if (enable)
5105 data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
5106 else
5107 data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
5108
5109 if (def != data)
5110 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5111 }
5112
gfx_v11_0_update_sram_fgcg(struct amdgpu_device * adev,bool enable)5113 static void gfx_v11_0_update_sram_fgcg(struct amdgpu_device *adev,
5114 bool enable)
5115 {
5116 uint32_t def, data;
5117
5118 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG))
5119 return;
5120
5121 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5122
5123 if (enable)
5124 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
5125 else
5126 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
5127
5128 if (def != data)
5129 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5130 }
5131
gfx_v11_0_update_repeater_fgcg(struct amdgpu_device * adev,bool enable)5132 static void gfx_v11_0_update_repeater_fgcg(struct amdgpu_device *adev,
5133 bool enable)
5134 {
5135 uint32_t def, data;
5136
5137 if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG))
5138 return;
5139
5140 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5141
5142 if (enable)
5143 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK;
5144 else
5145 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK;
5146
5147 if (def != data)
5148 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5149 }
5150
gfx_v11_0_update_medium_grain_clock_gating(struct amdgpu_device * adev,bool enable)5151 static void gfx_v11_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
5152 bool enable)
5153 {
5154 uint32_t data, def;
5155
5156 if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)))
5157 return;
5158
5159 /* It is disabled by HW by default */
5160 if (enable) {
5161 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5162 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
5163 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5164
5165 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
5166 RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
5167 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
5168
5169 if (def != data)
5170 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5171 }
5172 } else {
5173 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5174 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5175
5176 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
5177 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
5178 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
5179
5180 if (def != data)
5181 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5182 }
5183 }
5184 }
5185
gfx_v11_0_update_coarse_grain_clock_gating(struct amdgpu_device * adev,bool enable)5186 static void gfx_v11_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5187 bool enable)
5188 {
5189 uint32_t def, data;
5190
5191 if (!(adev->cg_flags &
5192 (AMD_CG_SUPPORT_GFX_CGCG |
5193 AMD_CG_SUPPORT_GFX_CGLS |
5194 AMD_CG_SUPPORT_GFX_3D_CGCG |
5195 AMD_CG_SUPPORT_GFX_3D_CGLS)))
5196 return;
5197
5198 if (enable) {
5199 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5200
5201 /* unset CGCG override */
5202 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
5203 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
5204 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5205 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5206 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG ||
5207 adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5208 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
5209
5210 /* update CGCG override bits */
5211 if (def != data)
5212 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5213
5214 /* enable cgcg FSM(0x0000363F) */
5215 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5216
5217 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
5218 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK;
5219 data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5220 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5221 }
5222
5223 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5224 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK;
5225 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5226 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5227 }
5228
5229 if (def != data)
5230 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
5231
5232 /* Program RLC_CGCG_CGLS_CTRL_3D */
5233 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5234
5235 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {
5236 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD_MASK;
5237 data |= (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5238 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5239 }
5240
5241 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {
5242 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY_MASK;
5243 data |= (0xf << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5244 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5245 }
5246
5247 if (def != data)
5248 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
5249
5250 /* set IDLE_POLL_COUNT(0x00900100) */
5251 def = data = RREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL);
5252
5253 data &= ~(CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK | CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK);
5254 data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5255 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5256
5257 if (def != data)
5258 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL, data);
5259
5260 data = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
5261 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
5262 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
5263 data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
5264 data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
5265 WREG32_SOC15(GC, 0, regCP_INT_CNTL, data);
5266
5267 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
5268 data = REG_SET_FIELD(data, SDMA0_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
5269 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
5270
5271 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
5272 if (adev->sdma.num_instances > 1) {
5273 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
5274 data = REG_SET_FIELD(data, SDMA1_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
5275 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
5276 }
5277 } else {
5278 /* Program RLC_CGCG_CGLS_CTRL */
5279 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5280
5281 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
5282 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5283
5284 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5285 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5286
5287 if (def != data)
5288 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
5289
5290 /* Program RLC_CGCG_CGLS_CTRL_3D */
5291 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5292
5293 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
5294 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5295 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5296 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5297
5298 if (def != data)
5299 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
5300
5301 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
5302 data &= ~SDMA0_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
5303 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
5304
5305 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
5306 if (adev->sdma.num_instances > 1) {
5307 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
5308 data &= ~SDMA1_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
5309 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
5310 }
5311 }
5312 }
5313
gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device * adev,bool enable)5314 static int gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5315 bool enable)
5316 {
5317 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5318
5319 gfx_v11_0_update_coarse_grain_clock_gating(adev, enable);
5320
5321 gfx_v11_0_update_medium_grain_clock_gating(adev, enable);
5322
5323 gfx_v11_0_update_repeater_fgcg(adev, enable);
5324
5325 gfx_v11_0_update_sram_fgcg(adev, enable);
5326
5327 gfx_v11_0_update_perf_clk(adev, enable);
5328
5329 if (adev->cg_flags &
5330 (AMD_CG_SUPPORT_GFX_MGCG |
5331 AMD_CG_SUPPORT_GFX_CGLS |
5332 AMD_CG_SUPPORT_GFX_CGCG |
5333 AMD_CG_SUPPORT_GFX_3D_CGCG |
5334 AMD_CG_SUPPORT_GFX_3D_CGLS))
5335 gfx_v11_0_enable_gui_idle_interrupt(adev, enable);
5336
5337 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5338
5339 return 0;
5340 }
5341
gfx_v11_0_update_spm_vmid(struct amdgpu_device * adev,struct amdgpu_ring * ring,unsigned vmid)5342 static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned vmid)
5343 {
5344 u32 reg, pre_data, data;
5345
5346 amdgpu_gfx_off_ctrl(adev, false);
5347 reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL);
5348 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev))
5349 pre_data = RREG32_NO_KIQ(reg);
5350 else
5351 pre_data = RREG32(reg);
5352
5353 data = pre_data & (~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK);
5354 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
5355
5356 if (pre_data != data) {
5357 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) {
5358 WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data);
5359 } else
5360 WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data);
5361 }
5362 amdgpu_gfx_off_ctrl(adev, true);
5363
5364 if (ring
5365 && amdgpu_sriov_is_pp_one_vf(adev)
5366 && (pre_data != data)
5367 && ((ring->funcs->type == AMDGPU_RING_TYPE_GFX)
5368 || (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) {
5369 amdgpu_ring_emit_wreg(ring, reg, data);
5370 }
5371 }
5372
5373 static const struct amdgpu_rlc_funcs gfx_v11_0_rlc_funcs = {
5374 .is_rlc_enabled = gfx_v11_0_is_rlc_enabled,
5375 .set_safe_mode = gfx_v11_0_set_safe_mode,
5376 .unset_safe_mode = gfx_v11_0_unset_safe_mode,
5377 .init = gfx_v11_0_rlc_init,
5378 .get_csb_size = gfx_v11_0_get_csb_size,
5379 .get_csb_buffer = gfx_v11_0_get_csb_buffer,
5380 .resume = gfx_v11_0_rlc_resume,
5381 .stop = gfx_v11_0_rlc_stop,
5382 .reset = gfx_v11_0_rlc_reset,
5383 .start = gfx_v11_0_rlc_start,
5384 .update_spm_vmid = gfx_v11_0_update_spm_vmid,
5385 };
5386
gfx_v11_cntl_power_gating(struct amdgpu_device * adev,bool enable)5387 static void gfx_v11_cntl_power_gating(struct amdgpu_device *adev, bool enable)
5388 {
5389 u32 data = RREG32_SOC15(GC, 0, regRLC_PG_CNTL);
5390
5391 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG))
5392 data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
5393 else
5394 data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
5395
5396 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, data);
5397
5398 // Program RLC_PG_DELAY3 for CGPG hysteresis
5399 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) {
5400 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5401 case IP_VERSION(11, 0, 1):
5402 case IP_VERSION(11, 0, 4):
5403 case IP_VERSION(11, 5, 0):
5404 case IP_VERSION(11, 5, 1):
5405 case IP_VERSION(11, 5, 2):
5406 WREG32_SOC15(GC, 0, regRLC_PG_DELAY_3, RLC_PG_DELAY_3_DEFAULT_GC_11_0_1);
5407 break;
5408 default:
5409 break;
5410 }
5411 }
5412 }
5413
gfx_v11_cntl_pg(struct amdgpu_device * adev,bool enable)5414 static void gfx_v11_cntl_pg(struct amdgpu_device *adev, bool enable)
5415 {
5416 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5417
5418 gfx_v11_cntl_power_gating(adev, enable);
5419
5420 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5421 }
5422
gfx_v11_0_set_powergating_state(void * handle,enum amd_powergating_state state)5423 static int gfx_v11_0_set_powergating_state(void *handle,
5424 enum amd_powergating_state state)
5425 {
5426 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5427 bool enable = (state == AMD_PG_STATE_GATE);
5428
5429 if (amdgpu_sriov_vf(adev))
5430 return 0;
5431
5432 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5433 case IP_VERSION(11, 0, 0):
5434 case IP_VERSION(11, 0, 2):
5435 case IP_VERSION(11, 0, 3):
5436 amdgpu_gfx_off_ctrl(adev, enable);
5437 break;
5438 case IP_VERSION(11, 0, 1):
5439 case IP_VERSION(11, 0, 4):
5440 case IP_VERSION(11, 5, 0):
5441 case IP_VERSION(11, 5, 1):
5442 case IP_VERSION(11, 5, 2):
5443 if (!enable)
5444 amdgpu_gfx_off_ctrl(adev, false);
5445
5446 gfx_v11_cntl_pg(adev, enable);
5447
5448 if (enable)
5449 amdgpu_gfx_off_ctrl(adev, true);
5450
5451 break;
5452 default:
5453 break;
5454 }
5455
5456 return 0;
5457 }
5458
gfx_v11_0_set_clockgating_state(void * handle,enum amd_clockgating_state state)5459 static int gfx_v11_0_set_clockgating_state(void *handle,
5460 enum amd_clockgating_state state)
5461 {
5462 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5463
5464 if (amdgpu_sriov_vf(adev))
5465 return 0;
5466
5467 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5468 case IP_VERSION(11, 0, 0):
5469 case IP_VERSION(11, 0, 1):
5470 case IP_VERSION(11, 0, 2):
5471 case IP_VERSION(11, 0, 3):
5472 case IP_VERSION(11, 0, 4):
5473 case IP_VERSION(11, 5, 0):
5474 case IP_VERSION(11, 5, 1):
5475 case IP_VERSION(11, 5, 2):
5476 gfx_v11_0_update_gfx_clock_gating(adev,
5477 state == AMD_CG_STATE_GATE);
5478 break;
5479 default:
5480 break;
5481 }
5482
5483 return 0;
5484 }
5485
gfx_v11_0_get_clockgating_state(void * handle,u64 * flags)5486 static void gfx_v11_0_get_clockgating_state(void *handle, u64 *flags)
5487 {
5488 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5489 int data;
5490
5491 /* AMD_CG_SUPPORT_GFX_MGCG */
5492 data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5493 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5494 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5495
5496 /* AMD_CG_SUPPORT_REPEATER_FGCG */
5497 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK))
5498 *flags |= AMD_CG_SUPPORT_REPEATER_FGCG;
5499
5500 /* AMD_CG_SUPPORT_GFX_FGCG */
5501 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK))
5502 *flags |= AMD_CG_SUPPORT_GFX_FGCG;
5503
5504 /* AMD_CG_SUPPORT_GFX_PERF_CLK */
5505 if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK))
5506 *flags |= AMD_CG_SUPPORT_GFX_PERF_CLK;
5507
5508 /* AMD_CG_SUPPORT_GFX_CGCG */
5509 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5510 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5511 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5512
5513 /* AMD_CG_SUPPORT_GFX_CGLS */
5514 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5515 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5516
5517 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5518 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5519 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5520 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5521
5522 /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5523 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5524 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5525 }
5526
gfx_v11_0_ring_get_rptr_gfx(struct amdgpu_ring * ring)5527 static u64 gfx_v11_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5528 {
5529 /* gfx11 is 32bit rptr*/
5530 return *(uint32_t *)ring->rptr_cpu_addr;
5531 }
5532
gfx_v11_0_ring_get_wptr_gfx(struct amdgpu_ring * ring)5533 static u64 gfx_v11_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5534 {
5535 struct amdgpu_device *adev = ring->adev;
5536 u64 wptr;
5537
5538 /* XXX check if swapping is necessary on BE */
5539 if (ring->use_doorbell) {
5540 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5541 } else {
5542 wptr = RREG32_SOC15(GC, 0, regCP_RB0_WPTR);
5543 wptr += (u64)RREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI) << 32;
5544 }
5545
5546 return wptr;
5547 }
5548
gfx_v11_0_ring_set_wptr_gfx(struct amdgpu_ring * ring)5549 static void gfx_v11_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5550 {
5551 struct amdgpu_device *adev = ring->adev;
5552
5553 if (ring->use_doorbell) {
5554 /* XXX check if swapping is necessary on BE */
5555 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
5556 ring->wptr);
5557 WDOORBELL64(ring->doorbell_index, ring->wptr);
5558 } else {
5559 WREG32_SOC15(GC, 0, regCP_RB0_WPTR,
5560 lower_32_bits(ring->wptr));
5561 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI,
5562 upper_32_bits(ring->wptr));
5563 }
5564 }
5565
gfx_v11_0_ring_get_rptr_compute(struct amdgpu_ring * ring)5566 static u64 gfx_v11_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5567 {
5568 /* gfx11 hardware is 32bit rptr */
5569 return *(uint32_t *)ring->rptr_cpu_addr;
5570 }
5571
gfx_v11_0_ring_get_wptr_compute(struct amdgpu_ring * ring)5572 static u64 gfx_v11_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5573 {
5574 u64 wptr;
5575
5576 /* XXX check if swapping is necessary on BE */
5577 if (ring->use_doorbell)
5578 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5579 else
5580 BUG();
5581 return wptr;
5582 }
5583
gfx_v11_0_ring_set_wptr_compute(struct amdgpu_ring * ring)5584 static void gfx_v11_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5585 {
5586 struct amdgpu_device *adev = ring->adev;
5587
5588 /* XXX check if swapping is necessary on BE */
5589 if (ring->use_doorbell) {
5590 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
5591 ring->wptr);
5592 WDOORBELL64(ring->doorbell_index, ring->wptr);
5593 } else {
5594 BUG(); /* only DOORBELL method supported on gfx11 now */
5595 }
5596 }
5597
gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring * ring)5598 static void gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5599 {
5600 struct amdgpu_device *adev = ring->adev;
5601 u32 ref_and_mask, reg_mem_engine;
5602 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5603
5604 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5605 switch (ring->me) {
5606 case 1:
5607 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5608 break;
5609 case 2:
5610 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5611 break;
5612 default:
5613 return;
5614 }
5615 reg_mem_engine = 0;
5616 } else {
5617 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0 << ring->pipe;
5618 reg_mem_engine = 1; /* pfp */
5619 }
5620
5621 gfx_v11_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5622 adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5623 adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5624 ref_and_mask, ref_and_mask, 0x20);
5625 }
5626
gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)5627 static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5628 struct amdgpu_job *job,
5629 struct amdgpu_ib *ib,
5630 uint32_t flags)
5631 {
5632 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5633 u32 header, control = 0;
5634
5635 BUG_ON(ib->flags & AMDGPU_IB_FLAG_CE);
5636
5637 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5638
5639 control |= ib->length_dw | (vmid << 24);
5640
5641 if (ring->adev->gfx.mcbp && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
5642 control |= INDIRECT_BUFFER_PRE_ENB(1);
5643
5644 if (flags & AMDGPU_IB_PREEMPTED)
5645 control |= INDIRECT_BUFFER_PRE_RESUME(1);
5646
5647 if (vmid)
5648 gfx_v11_0_ring_emit_de_meta(ring,
5649 (!amdgpu_sriov_vf(ring->adev) && flags & AMDGPU_IB_PREEMPTED) ? true : false);
5650 }
5651
5652 if (ring->is_mes_queue)
5653 /* inherit vmid from mqd */
5654 control |= 0x400000;
5655
5656 amdgpu_ring_write(ring, header);
5657 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5658 amdgpu_ring_write(ring,
5659 #ifdef __BIG_ENDIAN
5660 (2 << 0) |
5661 #endif
5662 lower_32_bits(ib->gpu_addr));
5663 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5664 amdgpu_ring_write(ring, control);
5665 }
5666
gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)5667 static void gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5668 struct amdgpu_job *job,
5669 struct amdgpu_ib *ib,
5670 uint32_t flags)
5671 {
5672 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5673 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5674
5675 if (ring->is_mes_queue)
5676 /* inherit vmid from mqd */
5677 control |= 0x40000000;
5678
5679 /* Currently, there is a high possibility to get wave ID mismatch
5680 * between ME and GDS, leading to a hw deadlock, because ME generates
5681 * different wave IDs than the GDS expects. This situation happens
5682 * randomly when at least 5 compute pipes use GDS ordered append.
5683 * The wave IDs generated by ME are also wrong after suspend/resume.
5684 * Those are probably bugs somewhere else in the kernel driver.
5685 *
5686 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5687 * GDS to 0 for this ring (me/pipe).
5688 */
5689 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5690 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5691 amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID);
5692 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5693 }
5694
5695 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5696 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5697 amdgpu_ring_write(ring,
5698 #ifdef __BIG_ENDIAN
5699 (2 << 0) |
5700 #endif
5701 lower_32_bits(ib->gpu_addr));
5702 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5703 amdgpu_ring_write(ring, control);
5704 }
5705
gfx_v11_0_ring_emit_fence(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned flags)5706 static void gfx_v11_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5707 u64 seq, unsigned flags)
5708 {
5709 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5710 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5711
5712 /* RELEASE_MEM - flush caches, send int */
5713 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5714 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ |
5715 PACKET3_RELEASE_MEM_GCR_GL2_WB |
5716 PACKET3_RELEASE_MEM_GCR_GLM_INV | /* must be set with GLM_WB */
5717 PACKET3_RELEASE_MEM_GCR_GLM_WB |
5718 PACKET3_RELEASE_MEM_CACHE_POLICY(3) |
5719 PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5720 PACKET3_RELEASE_MEM_EVENT_INDEX(5)));
5721 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) |
5722 PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0)));
5723
5724 /*
5725 * the address should be Qword aligned if 64bit write, Dword
5726 * aligned if only send 32bit data low (discard data high)
5727 */
5728 if (write64bit)
5729 BUG_ON(addr & 0x7);
5730 else
5731 BUG_ON(addr & 0x3);
5732 amdgpu_ring_write(ring, lower_32_bits(addr));
5733 amdgpu_ring_write(ring, upper_32_bits(addr));
5734 amdgpu_ring_write(ring, lower_32_bits(seq));
5735 amdgpu_ring_write(ring, upper_32_bits(seq));
5736 amdgpu_ring_write(ring, ring->is_mes_queue ?
5737 (ring->hw_queue_id | AMDGPU_FENCE_MES_QUEUE_FLAG) : 0);
5738 }
5739
gfx_v11_0_ring_emit_pipeline_sync(struct amdgpu_ring * ring)5740 static void gfx_v11_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5741 {
5742 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5743 uint32_t seq = ring->fence_drv.sync_seq;
5744 uint64_t addr = ring->fence_drv.gpu_addr;
5745
5746 gfx_v11_0_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr),
5747 upper_32_bits(addr), seq, 0xffffffff, 4);
5748 }
5749
gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring * ring,uint16_t pasid,uint32_t flush_type,bool all_hub,uint8_t dst_sel)5750 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
5751 uint16_t pasid, uint32_t flush_type,
5752 bool all_hub, uint8_t dst_sel)
5753 {
5754 amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
5755 amdgpu_ring_write(ring,
5756 PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) |
5757 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
5758 PACKET3_INVALIDATE_TLBS_PASID(pasid) |
5759 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
5760 }
5761
gfx_v11_0_ring_emit_vm_flush(struct amdgpu_ring * ring,unsigned vmid,uint64_t pd_addr)5762 static void gfx_v11_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5763 unsigned vmid, uint64_t pd_addr)
5764 {
5765 if (ring->is_mes_queue)
5766 gfx_v11_0_ring_invalidate_tlbs(ring, 0, 0, false, 0);
5767 else
5768 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5769
5770 /* compute doesn't have PFP */
5771 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5772 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5773 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5774 amdgpu_ring_write(ring, 0x0);
5775 }
5776
5777 /* Make sure that we can't skip the SET_Q_MODE packets when the VM
5778 * changed in any way.
5779 */
5780 ring->set_q_mode_offs = 0;
5781 ring->set_q_mode_ptr = NULL;
5782 }
5783
gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned int flags)5784 static void gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5785 u64 seq, unsigned int flags)
5786 {
5787 struct amdgpu_device *adev = ring->adev;
5788
5789 /* we only allocate 32bit for each seq wb address */
5790 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5791
5792 /* write fence seq to the "addr" */
5793 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5794 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5795 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5796 amdgpu_ring_write(ring, lower_32_bits(addr));
5797 amdgpu_ring_write(ring, upper_32_bits(addr));
5798 amdgpu_ring_write(ring, lower_32_bits(seq));
5799
5800 if (flags & AMDGPU_FENCE_FLAG_INT) {
5801 /* set register to trigger INT */
5802 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5803 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5804 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5805 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, regCPC_INT_STATUS));
5806 amdgpu_ring_write(ring, 0);
5807 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5808 }
5809 }
5810
gfx_v11_0_ring_emit_cntxcntl(struct amdgpu_ring * ring,uint32_t flags)5811 static void gfx_v11_0_ring_emit_cntxcntl(struct amdgpu_ring *ring,
5812 uint32_t flags)
5813 {
5814 uint32_t dw2 = 0;
5815
5816 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5817 if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5818 /* set load_global_config & load_global_uconfig */
5819 dw2 |= 0x8001;
5820 /* set load_cs_sh_regs */
5821 dw2 |= 0x01000000;
5822 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5823 dw2 |= 0x10002;
5824 }
5825
5826 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5827 amdgpu_ring_write(ring, dw2);
5828 amdgpu_ring_write(ring, 0);
5829 }
5830
gfx_v11_0_ring_emit_init_cond_exec(struct amdgpu_ring * ring,uint64_t addr)5831 static unsigned gfx_v11_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
5832 uint64_t addr)
5833 {
5834 unsigned ret;
5835
5836 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5837 amdgpu_ring_write(ring, lower_32_bits(addr));
5838 amdgpu_ring_write(ring, upper_32_bits(addr));
5839 /* discard following DWs if *cond_exec_gpu_addr==0 */
5840 amdgpu_ring_write(ring, 0);
5841 ret = ring->wptr & ring->buf_mask;
5842 /* patch dummy value later */
5843 amdgpu_ring_write(ring, 0);
5844
5845 return ret;
5846 }
5847
gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring * ring,u64 shadow_va,u64 csa_va,u64 gds_va,bool init_shadow,int vmid)5848 static void gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring *ring,
5849 u64 shadow_va, u64 csa_va,
5850 u64 gds_va, bool init_shadow,
5851 int vmid)
5852 {
5853 struct amdgpu_device *adev = ring->adev;
5854 unsigned int offs, end;
5855
5856 if (!adev->gfx.cp_gfx_shadow || !ring->ring_obj)
5857 return;
5858
5859 /*
5860 * The logic here isn't easy to understand because we need to keep state
5861 * accross multiple executions of the function as well as between the
5862 * CPU and GPU. The general idea is that the newly written GPU command
5863 * has a condition on the previous one and only executed if really
5864 * necessary.
5865 */
5866
5867 /*
5868 * The dw in the NOP controls if the next SET_Q_MODE packet should be
5869 * executed or not. Reserve 64bits just to be on the save side.
5870 */
5871 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, 1));
5872 offs = ring->wptr & ring->buf_mask;
5873
5874 /*
5875 * We start with skipping the prefix SET_Q_MODE and always executing
5876 * the postfix SET_Q_MODE packet. This is changed below with a
5877 * WRITE_DATA command when the postfix executed.
5878 */
5879 amdgpu_ring_write(ring, shadow_va ? 1 : 0);
5880 amdgpu_ring_write(ring, 0);
5881
5882 if (ring->set_q_mode_offs) {
5883 uint64_t addr;
5884
5885 addr = amdgpu_bo_gpu_offset(ring->ring_obj);
5886 addr += ring->set_q_mode_offs << 2;
5887 end = gfx_v11_0_ring_emit_init_cond_exec(ring, addr);
5888 }
5889
5890 /*
5891 * When the postfix SET_Q_MODE packet executes we need to make sure that the
5892 * next prefix SET_Q_MODE packet executes as well.
5893 */
5894 if (!shadow_va) {
5895 uint64_t addr;
5896
5897 addr = amdgpu_bo_gpu_offset(ring->ring_obj);
5898 addr += offs << 2;
5899 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5900 amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM);
5901 amdgpu_ring_write(ring, lower_32_bits(addr));
5902 amdgpu_ring_write(ring, upper_32_bits(addr));
5903 amdgpu_ring_write(ring, 0x1);
5904 }
5905
5906 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_Q_PREEMPTION_MODE, 7));
5907 amdgpu_ring_write(ring, lower_32_bits(shadow_va));
5908 amdgpu_ring_write(ring, upper_32_bits(shadow_va));
5909 amdgpu_ring_write(ring, lower_32_bits(gds_va));
5910 amdgpu_ring_write(ring, upper_32_bits(gds_va));
5911 amdgpu_ring_write(ring, lower_32_bits(csa_va));
5912 amdgpu_ring_write(ring, upper_32_bits(csa_va));
5913 amdgpu_ring_write(ring, shadow_va ?
5914 PACKET3_SET_Q_PREEMPTION_MODE_IB_VMID(vmid) : 0);
5915 amdgpu_ring_write(ring, init_shadow ?
5916 PACKET3_SET_Q_PREEMPTION_MODE_INIT_SHADOW_MEM : 0);
5917
5918 if (ring->set_q_mode_offs)
5919 amdgpu_ring_patch_cond_exec(ring, end);
5920
5921 if (shadow_va) {
5922 uint64_t token = shadow_va ^ csa_va ^ gds_va ^ vmid;
5923
5924 /*
5925 * If the tokens match try to skip the last postfix SET_Q_MODE
5926 * packet to avoid saving/restoring the state all the time.
5927 */
5928 if (ring->set_q_mode_ptr && ring->set_q_mode_token == token)
5929 *ring->set_q_mode_ptr = 0;
5930
5931 ring->set_q_mode_token = token;
5932 } else {
5933 ring->set_q_mode_ptr = &ring->ring[ring->set_q_mode_offs];
5934 }
5935
5936 ring->set_q_mode_offs = offs;
5937 }
5938
gfx_v11_0_ring_preempt_ib(struct amdgpu_ring * ring)5939 static int gfx_v11_0_ring_preempt_ib(struct amdgpu_ring *ring)
5940 {
5941 int i, r = 0;
5942 struct amdgpu_device *adev = ring->adev;
5943 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
5944 struct amdgpu_ring *kiq_ring = &kiq->ring;
5945 unsigned long flags;
5946
5947 if (adev->enable_mes)
5948 return -EINVAL;
5949
5950 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
5951 return -EINVAL;
5952
5953 spin_lock_irqsave(&kiq->ring_lock, flags);
5954
5955 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
5956 spin_unlock_irqrestore(&kiq->ring_lock, flags);
5957 return -ENOMEM;
5958 }
5959
5960 /* assert preemption condition */
5961 amdgpu_ring_set_preempt_cond_exec(ring, false);
5962
5963 /* assert IB preemption, emit the trailing fence */
5964 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
5965 ring->trail_fence_gpu_addr,
5966 ++ring->trail_seq);
5967 amdgpu_ring_commit(kiq_ring);
5968
5969 spin_unlock_irqrestore(&kiq->ring_lock, flags);
5970
5971 /* poll the trailing fence */
5972 for (i = 0; i < adev->usec_timeout; i++) {
5973 if (ring->trail_seq ==
5974 le32_to_cpu(*(ring->trail_fence_cpu_addr)))
5975 break;
5976 udelay(1);
5977 }
5978
5979 if (i >= adev->usec_timeout) {
5980 r = -EINVAL;
5981 DRM_ERROR("ring %d failed to preempt ib\n", ring->idx);
5982 }
5983
5984 /* deassert preemption condition */
5985 amdgpu_ring_set_preempt_cond_exec(ring, true);
5986 return r;
5987 }
5988
gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring * ring,bool resume)5989 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume)
5990 {
5991 struct amdgpu_device *adev = ring->adev;
5992 struct v10_de_ib_state de_payload = {0};
5993 uint64_t offset, gds_addr, de_payload_gpu_addr;
5994 void *de_payload_cpu_addr;
5995 int cnt;
5996
5997 if (ring->is_mes_queue) {
5998 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5999 gfx[0].gfx_meta_data) +
6000 offsetof(struct v10_gfx_meta_data, de_payload);
6001 de_payload_gpu_addr =
6002 amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
6003 de_payload_cpu_addr =
6004 amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
6005
6006 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
6007 gfx[0].gds_backup) +
6008 offsetof(struct v10_gfx_meta_data, de_payload);
6009 gds_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
6010 } else {
6011 offset = offsetof(struct v10_gfx_meta_data, de_payload);
6012 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
6013 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
6014
6015 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +
6016 AMDGPU_CSA_SIZE - adev->gds.gds_size,
6017 PAGE_SIZE);
6018 }
6019
6020 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
6021 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
6022
6023 cnt = (sizeof(de_payload) >> 2) + 4 - 2;
6024 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
6025 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
6026 WRITE_DATA_DST_SEL(8) |
6027 WR_CONFIRM) |
6028 WRITE_DATA_CACHE_POLICY(0));
6029 amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));
6030 amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));
6031
6032 if (resume)
6033 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr,
6034 sizeof(de_payload) >> 2);
6035 else
6036 amdgpu_ring_write_multiple(ring, (void *)&de_payload,
6037 sizeof(de_payload) >> 2);
6038 }
6039
gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring * ring,bool start,bool secure)6040 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
6041 bool secure)
6042 {
6043 uint32_t v = secure ? FRAME_TMZ : 0;
6044
6045 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
6046 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
6047 }
6048
gfx_v11_0_ring_emit_rreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t reg_val_offs)6049 static void gfx_v11_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
6050 uint32_t reg_val_offs)
6051 {
6052 struct amdgpu_device *adev = ring->adev;
6053
6054 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
6055 amdgpu_ring_write(ring, 0 | /* src: register*/
6056 (5 << 8) | /* dst: memory */
6057 (1 << 20)); /* write confirm */
6058 amdgpu_ring_write(ring, reg);
6059 amdgpu_ring_write(ring, 0);
6060 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
6061 reg_val_offs * 4));
6062 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
6063 reg_val_offs * 4));
6064 }
6065
gfx_v11_0_ring_emit_wreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t val)6066 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
6067 uint32_t val)
6068 {
6069 uint32_t cmd = 0;
6070
6071 switch (ring->funcs->type) {
6072 case AMDGPU_RING_TYPE_GFX:
6073 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
6074 break;
6075 case AMDGPU_RING_TYPE_KIQ:
6076 cmd = (1 << 16); /* no inc addr */
6077 break;
6078 default:
6079 cmd = WR_CONFIRM;
6080 break;
6081 }
6082 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6083 amdgpu_ring_write(ring, cmd);
6084 amdgpu_ring_write(ring, reg);
6085 amdgpu_ring_write(ring, 0);
6086 amdgpu_ring_write(ring, val);
6087 }
6088
gfx_v11_0_ring_emit_reg_wait(struct amdgpu_ring * ring,uint32_t reg,uint32_t val,uint32_t mask)6089 static void gfx_v11_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
6090 uint32_t val, uint32_t mask)
6091 {
6092 gfx_v11_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
6093 }
6094
gfx_v11_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring * ring,uint32_t reg0,uint32_t reg1,uint32_t ref,uint32_t mask)6095 static void gfx_v11_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
6096 uint32_t reg0, uint32_t reg1,
6097 uint32_t ref, uint32_t mask)
6098 {
6099 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
6100
6101 gfx_v11_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
6102 ref, mask, 0x20);
6103 }
6104
gfx_v11_0_ring_soft_recovery(struct amdgpu_ring * ring,unsigned vmid)6105 static void gfx_v11_0_ring_soft_recovery(struct amdgpu_ring *ring,
6106 unsigned vmid)
6107 {
6108 struct amdgpu_device *adev = ring->adev;
6109 uint32_t value = 0;
6110
6111 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
6112 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
6113 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
6114 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
6115 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
6116 WREG32_SOC15(GC, 0, regSQ_CMD, value);
6117 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
6118 }
6119
6120 static void
gfx_v11_0_set_gfx_eop_interrupt_state(struct amdgpu_device * adev,uint32_t me,uint32_t pipe,enum amdgpu_interrupt_state state)6121 gfx_v11_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
6122 uint32_t me, uint32_t pipe,
6123 enum amdgpu_interrupt_state state)
6124 {
6125 uint32_t cp_int_cntl, cp_int_cntl_reg;
6126
6127 if (!me) {
6128 switch (pipe) {
6129 case 0:
6130 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0);
6131 break;
6132 case 1:
6133 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1);
6134 break;
6135 default:
6136 DRM_DEBUG("invalid pipe %d\n", pipe);
6137 return;
6138 }
6139 } else {
6140 DRM_DEBUG("invalid me %d\n", me);
6141 return;
6142 }
6143
6144 switch (state) {
6145 case AMDGPU_IRQ_STATE_DISABLE:
6146 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6147 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6148 TIME_STAMP_INT_ENABLE, 0);
6149 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6150 GENERIC0_INT_ENABLE, 0);
6151 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6152 break;
6153 case AMDGPU_IRQ_STATE_ENABLE:
6154 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6155 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6156 TIME_STAMP_INT_ENABLE, 1);
6157 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6158 GENERIC0_INT_ENABLE, 1);
6159 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6160 break;
6161 default:
6162 break;
6163 }
6164 }
6165
gfx_v11_0_set_compute_eop_interrupt_state(struct amdgpu_device * adev,int me,int pipe,enum amdgpu_interrupt_state state)6166 static void gfx_v11_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
6167 int me, int pipe,
6168 enum amdgpu_interrupt_state state)
6169 {
6170 u32 mec_int_cntl, mec_int_cntl_reg;
6171
6172 /*
6173 * amdgpu controls only the first MEC. That's why this function only
6174 * handles the setting of interrupts for this specific MEC. All other
6175 * pipes' interrupts are set by amdkfd.
6176 */
6177
6178 if (me == 1) {
6179 switch (pipe) {
6180 case 0:
6181 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
6182 break;
6183 case 1:
6184 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL);
6185 break;
6186 case 2:
6187 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL);
6188 break;
6189 case 3:
6190 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL);
6191 break;
6192 default:
6193 DRM_DEBUG("invalid pipe %d\n", pipe);
6194 return;
6195 }
6196 } else {
6197 DRM_DEBUG("invalid me %d\n", me);
6198 return;
6199 }
6200
6201 switch (state) {
6202 case AMDGPU_IRQ_STATE_DISABLE:
6203 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
6204 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6205 TIME_STAMP_INT_ENABLE, 0);
6206 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6207 GENERIC0_INT_ENABLE, 0);
6208 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
6209 break;
6210 case AMDGPU_IRQ_STATE_ENABLE:
6211 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
6212 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6213 TIME_STAMP_INT_ENABLE, 1);
6214 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6215 GENERIC0_INT_ENABLE, 1);
6216 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
6217 break;
6218 default:
6219 break;
6220 }
6221 }
6222
gfx_v11_0_set_eop_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * src,unsigned type,enum amdgpu_interrupt_state state)6223 static int gfx_v11_0_set_eop_interrupt_state(struct amdgpu_device *adev,
6224 struct amdgpu_irq_src *src,
6225 unsigned type,
6226 enum amdgpu_interrupt_state state)
6227 {
6228 switch (type) {
6229 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
6230 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 0, state);
6231 break;
6232 case AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP:
6233 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 1, state);
6234 break;
6235 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
6236 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
6237 break;
6238 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
6239 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
6240 break;
6241 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
6242 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
6243 break;
6244 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
6245 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
6246 break;
6247 default:
6248 break;
6249 }
6250 return 0;
6251 }
6252
gfx_v11_0_eop_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6253 static int gfx_v11_0_eop_irq(struct amdgpu_device *adev,
6254 struct amdgpu_irq_src *source,
6255 struct amdgpu_iv_entry *entry)
6256 {
6257 int i;
6258 u8 me_id, pipe_id, queue_id;
6259 struct amdgpu_ring *ring;
6260 uint32_t mes_queue_id = entry->src_data[0];
6261
6262 DRM_DEBUG("IH: CP EOP\n");
6263
6264 if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) {
6265 struct amdgpu_mes_queue *queue;
6266
6267 mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK;
6268
6269 spin_lock(&adev->mes.queue_id_lock);
6270 queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id);
6271 if (queue) {
6272 DRM_DEBUG("process mes queue id = %d\n", mes_queue_id);
6273 amdgpu_fence_process(queue->ring);
6274 }
6275 spin_unlock(&adev->mes.queue_id_lock);
6276 } else {
6277 me_id = (entry->ring_id & 0x0c) >> 2;
6278 pipe_id = (entry->ring_id & 0x03) >> 0;
6279 queue_id = (entry->ring_id & 0x70) >> 4;
6280
6281 switch (me_id) {
6282 case 0:
6283 if (pipe_id == 0)
6284 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
6285 else
6286 amdgpu_fence_process(&adev->gfx.gfx_ring[1]);
6287 break;
6288 case 1:
6289 case 2:
6290 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6291 ring = &adev->gfx.compute_ring[i];
6292 /* Per-queue interrupt is supported for MEC starting from VI.
6293 * The interrupt can only be enabled/disabled per pipe instead
6294 * of per queue.
6295 */
6296 if ((ring->me == me_id) &&
6297 (ring->pipe == pipe_id) &&
6298 (ring->queue == queue_id))
6299 amdgpu_fence_process(ring);
6300 }
6301 break;
6302 }
6303 }
6304
6305 return 0;
6306 }
6307
gfx_v11_0_set_priv_reg_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int type,enum amdgpu_interrupt_state state)6308 static int gfx_v11_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
6309 struct amdgpu_irq_src *source,
6310 unsigned int type,
6311 enum amdgpu_interrupt_state state)
6312 {
6313 u32 cp_int_cntl_reg, cp_int_cntl;
6314 int i, j;
6315
6316 switch (state) {
6317 case AMDGPU_IRQ_STATE_DISABLE:
6318 case AMDGPU_IRQ_STATE_ENABLE:
6319 for (i = 0; i < adev->gfx.me.num_me; i++) {
6320 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6321 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
6322
6323 if (cp_int_cntl_reg) {
6324 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6325 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6326 PRIV_REG_INT_ENABLE,
6327 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6328 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6329 }
6330 }
6331 }
6332 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6333 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6334 /* MECs start at 1 */
6335 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j);
6336
6337 if (cp_int_cntl_reg) {
6338 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6339 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6340 PRIV_REG_INT_ENABLE,
6341 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6342 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6343 }
6344 }
6345 }
6346 break;
6347 default:
6348 break;
6349 }
6350
6351 return 0;
6352 }
6353
gfx_v11_0_set_bad_op_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)6354 static int gfx_v11_0_set_bad_op_fault_state(struct amdgpu_device *adev,
6355 struct amdgpu_irq_src *source,
6356 unsigned type,
6357 enum amdgpu_interrupt_state state)
6358 {
6359 u32 cp_int_cntl_reg, cp_int_cntl;
6360 int i, j;
6361
6362 switch (state) {
6363 case AMDGPU_IRQ_STATE_DISABLE:
6364 case AMDGPU_IRQ_STATE_ENABLE:
6365 for (i = 0; i < adev->gfx.me.num_me; i++) {
6366 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6367 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
6368
6369 if (cp_int_cntl_reg) {
6370 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6371 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6372 OPCODE_ERROR_INT_ENABLE,
6373 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6374 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6375 }
6376 }
6377 }
6378 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6379 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6380 /* MECs start at 1 */
6381 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j);
6382
6383 if (cp_int_cntl_reg) {
6384 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6385 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6386 OPCODE_ERROR_INT_ENABLE,
6387 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6388 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6389 }
6390 }
6391 }
6392 break;
6393 default:
6394 break;
6395 }
6396 return 0;
6397 }
6398
gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int type,enum amdgpu_interrupt_state state)6399 static int gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
6400 struct amdgpu_irq_src *source,
6401 unsigned int type,
6402 enum amdgpu_interrupt_state state)
6403 {
6404 u32 cp_int_cntl_reg, cp_int_cntl;
6405 int i, j;
6406
6407 switch (state) {
6408 case AMDGPU_IRQ_STATE_DISABLE:
6409 case AMDGPU_IRQ_STATE_ENABLE:
6410 for (i = 0; i < adev->gfx.me.num_me; i++) {
6411 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6412 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
6413
6414 if (cp_int_cntl_reg) {
6415 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6416 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6417 PRIV_INSTR_INT_ENABLE,
6418 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6419 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6420 }
6421 }
6422 }
6423 break;
6424 default:
6425 break;
6426 }
6427
6428 return 0;
6429 }
6430
gfx_v11_0_handle_priv_fault(struct amdgpu_device * adev,struct amdgpu_iv_entry * entry)6431 static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev,
6432 struct amdgpu_iv_entry *entry)
6433 {
6434 u8 me_id, pipe_id, queue_id;
6435 struct amdgpu_ring *ring;
6436 int i;
6437
6438 me_id = (entry->ring_id & 0x0c) >> 2;
6439 pipe_id = (entry->ring_id & 0x03) >> 0;
6440 queue_id = (entry->ring_id & 0x70) >> 4;
6441
6442 switch (me_id) {
6443 case 0:
6444 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
6445 ring = &adev->gfx.gfx_ring[i];
6446 if (ring->me == me_id && ring->pipe == pipe_id &&
6447 ring->queue == queue_id)
6448 drm_sched_fault(&ring->sched);
6449 }
6450 break;
6451 case 1:
6452 case 2:
6453 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6454 ring = &adev->gfx.compute_ring[i];
6455 if (ring->me == me_id && ring->pipe == pipe_id &&
6456 ring->queue == queue_id)
6457 drm_sched_fault(&ring->sched);
6458 }
6459 break;
6460 default:
6461 BUG();
6462 break;
6463 }
6464 }
6465
gfx_v11_0_priv_reg_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6466 static int gfx_v11_0_priv_reg_irq(struct amdgpu_device *adev,
6467 struct amdgpu_irq_src *source,
6468 struct amdgpu_iv_entry *entry)
6469 {
6470 DRM_ERROR("Illegal register access in command stream\n");
6471 gfx_v11_0_handle_priv_fault(adev, entry);
6472 return 0;
6473 }
6474
gfx_v11_0_bad_op_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6475 static int gfx_v11_0_bad_op_irq(struct amdgpu_device *adev,
6476 struct amdgpu_irq_src *source,
6477 struct amdgpu_iv_entry *entry)
6478 {
6479 DRM_ERROR("Illegal opcode in command stream \n");
6480 gfx_v11_0_handle_priv_fault(adev, entry);
6481 return 0;
6482 }
6483
gfx_v11_0_priv_inst_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6484 static int gfx_v11_0_priv_inst_irq(struct amdgpu_device *adev,
6485 struct amdgpu_irq_src *source,
6486 struct amdgpu_iv_entry *entry)
6487 {
6488 DRM_ERROR("Illegal instruction in command stream\n");
6489 gfx_v11_0_handle_priv_fault(adev, entry);
6490 return 0;
6491 }
6492
gfx_v11_0_rlc_gc_fed_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6493 static int gfx_v11_0_rlc_gc_fed_irq(struct amdgpu_device *adev,
6494 struct amdgpu_irq_src *source,
6495 struct amdgpu_iv_entry *entry)
6496 {
6497 if (adev->gfx.ras && adev->gfx.ras->rlc_gc_fed_irq)
6498 return adev->gfx.ras->rlc_gc_fed_irq(adev, source, entry);
6499
6500 return 0;
6501 }
6502
6503 #if 0
6504 static int gfx_v11_0_kiq_set_interrupt_state(struct amdgpu_device *adev,
6505 struct amdgpu_irq_src *src,
6506 unsigned int type,
6507 enum amdgpu_interrupt_state state)
6508 {
6509 uint32_t tmp, target;
6510 struct amdgpu_ring *ring = &(adev->gfx.kiq[0].ring);
6511
6512 target = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
6513 target += ring->pipe;
6514
6515 switch (type) {
6516 case AMDGPU_CP_KIQ_IRQ_DRIVER0:
6517 if (state == AMDGPU_IRQ_STATE_DISABLE) {
6518 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
6519 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6520 GENERIC2_INT_ENABLE, 0);
6521 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
6522
6523 tmp = RREG32_SOC15_IP(GC, target);
6524 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL,
6525 GENERIC2_INT_ENABLE, 0);
6526 WREG32_SOC15_IP(GC, target, tmp);
6527 } else {
6528 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
6529 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6530 GENERIC2_INT_ENABLE, 1);
6531 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
6532
6533 tmp = RREG32_SOC15_IP(GC, target);
6534 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL,
6535 GENERIC2_INT_ENABLE, 1);
6536 WREG32_SOC15_IP(GC, target, tmp);
6537 }
6538 break;
6539 default:
6540 BUG(); /* kiq only support GENERIC2_INT now */
6541 break;
6542 }
6543 return 0;
6544 }
6545 #endif
6546
gfx_v11_0_emit_mem_sync(struct amdgpu_ring * ring)6547 static void gfx_v11_0_emit_mem_sync(struct amdgpu_ring *ring)
6548 {
6549 const unsigned int gcr_cntl =
6550 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) |
6551 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) |
6552 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(1) |
6553 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(1) |
6554 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(1) |
6555 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) |
6556 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) |
6557 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1);
6558
6559 /* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */
6560 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6));
6561 amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */
6562 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */
6563 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */
6564 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
6565 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */
6566 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
6567 amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */
6568 }
6569
gfx_v11_0_reset_kgq(struct amdgpu_ring * ring,unsigned int vmid)6570 static int gfx_v11_0_reset_kgq(struct amdgpu_ring *ring, unsigned int vmid)
6571 {
6572 struct amdgpu_device *adev = ring->adev;
6573 int r;
6574
6575 if (amdgpu_sriov_vf(adev))
6576 return -EINVAL;
6577
6578 r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, false);
6579 if (r)
6580 return r;
6581
6582 r = amdgpu_bo_reserve(ring->mqd_obj, false);
6583 if (unlikely(r != 0)) {
6584 dev_err(adev->dev, "fail to resv mqd_obj\n");
6585 return r;
6586 }
6587 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
6588 if (!r) {
6589 r = gfx_v11_0_kgq_init_queue(ring, true);
6590 amdgpu_bo_kunmap(ring->mqd_obj);
6591 ring->mqd_ptr = NULL;
6592 }
6593 amdgpu_bo_unreserve(ring->mqd_obj);
6594 if (r) {
6595 dev_err(adev->dev, "fail to unresv mqd_obj\n");
6596 return r;
6597 }
6598
6599 r = amdgpu_mes_map_legacy_queue(adev, ring);
6600 if (r) {
6601 dev_err(adev->dev, "failed to remap kgq\n");
6602 return r;
6603 }
6604
6605 return amdgpu_ring_test_ring(ring);
6606 }
6607
gfx_v11_0_reset_kcq(struct amdgpu_ring * ring,unsigned int vmid)6608 static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring, unsigned int vmid)
6609 {
6610 struct amdgpu_device *adev = ring->adev;
6611 int i, r = 0;
6612
6613 if (amdgpu_sriov_vf(adev))
6614 return -EINVAL;
6615
6616 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
6617 mutex_lock(&adev->srbm_mutex);
6618 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
6619 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2);
6620 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1);
6621
6622 /* make sure dequeue is complete*/
6623 for (i = 0; i < adev->usec_timeout; i++) {
6624 if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
6625 break;
6626 udelay(1);
6627 }
6628 if (i >= adev->usec_timeout)
6629 r = -ETIMEDOUT;
6630 soc21_grbm_select(adev, 0, 0, 0, 0);
6631 mutex_unlock(&adev->srbm_mutex);
6632 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
6633 if (r) {
6634 dev_err(adev->dev, "fail to wait on hqd deactivate\n");
6635 return r;
6636 }
6637
6638 r = amdgpu_bo_reserve(ring->mqd_obj, false);
6639 if (unlikely(r != 0)) {
6640 dev_err(adev->dev, "fail to resv mqd_obj\n");
6641 return r;
6642 }
6643 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
6644 if (!r) {
6645 r = gfx_v11_0_kcq_init_queue(ring, true);
6646 amdgpu_bo_kunmap(ring->mqd_obj);
6647 ring->mqd_ptr = NULL;
6648 }
6649 amdgpu_bo_unreserve(ring->mqd_obj);
6650 if (r) {
6651 dev_err(adev->dev, "fail to unresv mqd_obj\n");
6652 return r;
6653 }
6654 r = amdgpu_mes_map_legacy_queue(adev, ring);
6655 if (r) {
6656 dev_err(adev->dev, "failed to remap kcq\n");
6657 return r;
6658 }
6659
6660 return amdgpu_ring_test_ring(ring);
6661 }
6662
gfx_v11_ip_print(void * handle,struct drm_printer * p)6663 static void gfx_v11_ip_print(void *handle, struct drm_printer *p)
6664 {
6665 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
6666 uint32_t i, j, k, reg, index = 0;
6667 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0);
6668
6669 if (!adev->gfx.ip_dump_core)
6670 return;
6671
6672 for (i = 0; i < reg_count; i++)
6673 drm_printf(p, "%-50s \t 0x%08x\n",
6674 gc_reg_list_11_0[i].reg_name,
6675 adev->gfx.ip_dump_core[i]);
6676
6677 /* print compute queue registers for all instances */
6678 if (!adev->gfx.ip_dump_compute_queues)
6679 return;
6680
6681 reg_count = ARRAY_SIZE(gc_cp_reg_list_11);
6682 drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n",
6683 adev->gfx.mec.num_mec,
6684 adev->gfx.mec.num_pipe_per_mec,
6685 adev->gfx.mec.num_queue_per_pipe);
6686
6687 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6688 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6689 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
6690 drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k);
6691 for (reg = 0; reg < reg_count; reg++) {
6692 drm_printf(p, "%-50s \t 0x%08x\n",
6693 gc_cp_reg_list_11[reg].reg_name,
6694 adev->gfx.ip_dump_compute_queues[index + reg]);
6695 }
6696 index += reg_count;
6697 }
6698 }
6699 }
6700
6701 /* print gfx queue registers for all instances */
6702 if (!adev->gfx.ip_dump_gfx_queues)
6703 return;
6704
6705 index = 0;
6706 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11);
6707 drm_printf(p, "\nnum_me: %d num_pipe: %d num_queue: %d\n",
6708 adev->gfx.me.num_me,
6709 adev->gfx.me.num_pipe_per_me,
6710 adev->gfx.me.num_queue_per_pipe);
6711
6712 for (i = 0; i < adev->gfx.me.num_me; i++) {
6713 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6714 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) {
6715 drm_printf(p, "\nme %d, pipe %d, queue %d\n", i, j, k);
6716 for (reg = 0; reg < reg_count; reg++) {
6717 drm_printf(p, "%-50s \t 0x%08x\n",
6718 gc_gfx_queue_reg_list_11[reg].reg_name,
6719 adev->gfx.ip_dump_gfx_queues[index + reg]);
6720 }
6721 index += reg_count;
6722 }
6723 }
6724 }
6725 }
6726
gfx_v11_ip_dump(void * handle)6727 static void gfx_v11_ip_dump(void *handle)
6728 {
6729 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
6730 uint32_t i, j, k, reg, index = 0;
6731 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0);
6732
6733 if (!adev->gfx.ip_dump_core)
6734 return;
6735
6736 amdgpu_gfx_off_ctrl(adev, false);
6737 for (i = 0; i < reg_count; i++)
6738 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_11_0[i]));
6739 amdgpu_gfx_off_ctrl(adev, true);
6740
6741 /* dump compute queue registers for all instances */
6742 if (!adev->gfx.ip_dump_compute_queues)
6743 return;
6744
6745 reg_count = ARRAY_SIZE(gc_cp_reg_list_11);
6746 amdgpu_gfx_off_ctrl(adev, false);
6747 mutex_lock(&adev->srbm_mutex);
6748 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6749 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6750 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
6751 /* ME0 is for GFX so start from 1 for CP */
6752 soc21_grbm_select(adev, adev->gfx.me.num_me + i, j, k, 0);
6753 for (reg = 0; reg < reg_count; reg++) {
6754 adev->gfx.ip_dump_compute_queues[index + reg] =
6755 RREG32(SOC15_REG_ENTRY_OFFSET(
6756 gc_cp_reg_list_11[reg]));
6757 }
6758 index += reg_count;
6759 }
6760 }
6761 }
6762 soc21_grbm_select(adev, 0, 0, 0, 0);
6763 mutex_unlock(&adev->srbm_mutex);
6764 amdgpu_gfx_off_ctrl(adev, true);
6765
6766 /* dump gfx queue registers for all instances */
6767 if (!adev->gfx.ip_dump_gfx_queues)
6768 return;
6769
6770 index = 0;
6771 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11);
6772 amdgpu_gfx_off_ctrl(adev, false);
6773 mutex_lock(&adev->srbm_mutex);
6774 for (i = 0; i < adev->gfx.me.num_me; i++) {
6775 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6776 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) {
6777 soc21_grbm_select(adev, i, j, k, 0);
6778
6779 for (reg = 0; reg < reg_count; reg++) {
6780 adev->gfx.ip_dump_gfx_queues[index + reg] =
6781 RREG32(SOC15_REG_ENTRY_OFFSET(
6782 gc_gfx_queue_reg_list_11[reg]));
6783 }
6784 index += reg_count;
6785 }
6786 }
6787 }
6788 soc21_grbm_select(adev, 0, 0, 0, 0);
6789 mutex_unlock(&adev->srbm_mutex);
6790 amdgpu_gfx_off_ctrl(adev, true);
6791 }
6792
6793 static const struct amd_ip_funcs gfx_v11_0_ip_funcs = {
6794 .name = "gfx_v11_0",
6795 .early_init = gfx_v11_0_early_init,
6796 .late_init = gfx_v11_0_late_init,
6797 .sw_init = gfx_v11_0_sw_init,
6798 .sw_fini = gfx_v11_0_sw_fini,
6799 .hw_init = gfx_v11_0_hw_init,
6800 .hw_fini = gfx_v11_0_hw_fini,
6801 .suspend = gfx_v11_0_suspend,
6802 .resume = gfx_v11_0_resume,
6803 .is_idle = gfx_v11_0_is_idle,
6804 .wait_for_idle = gfx_v11_0_wait_for_idle,
6805 .soft_reset = gfx_v11_0_soft_reset,
6806 .check_soft_reset = gfx_v11_0_check_soft_reset,
6807 .post_soft_reset = gfx_v11_0_post_soft_reset,
6808 .set_clockgating_state = gfx_v11_0_set_clockgating_state,
6809 .set_powergating_state = gfx_v11_0_set_powergating_state,
6810 .get_clockgating_state = gfx_v11_0_get_clockgating_state,
6811 .dump_ip_state = gfx_v11_ip_dump,
6812 .print_ip_state = gfx_v11_ip_print,
6813 };
6814
6815 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = {
6816 .type = AMDGPU_RING_TYPE_GFX,
6817 .align_mask = 0xff,
6818 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6819 .support_64bit_ptrs = true,
6820 .secure_submission_supported = true,
6821 .get_rptr = gfx_v11_0_ring_get_rptr_gfx,
6822 .get_wptr = gfx_v11_0_ring_get_wptr_gfx,
6823 .set_wptr = gfx_v11_0_ring_set_wptr_gfx,
6824 .emit_frame_size = /* totally 247 maximum if 16 IBs */
6825 5 + /* update_spm_vmid */
6826 5 + /* COND_EXEC */
6827 22 + /* SET_Q_PREEMPTION_MODE */
6828 7 + /* PIPELINE_SYNC */
6829 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6830 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6831 4 + /* VM_FLUSH */
6832 8 + /* FENCE for VM_FLUSH */
6833 20 + /* GDS switch */
6834 5 + /* COND_EXEC */
6835 7 + /* HDP_flush */
6836 4 + /* VGT_flush */
6837 31 + /* DE_META */
6838 3 + /* CNTX_CTRL */
6839 5 + /* HDP_INVL */
6840 22 + /* SET_Q_PREEMPTION_MODE */
6841 8 + 8 + /* FENCE x2 */
6842 8, /* gfx_v11_0_emit_mem_sync */
6843 .emit_ib_size = 4, /* gfx_v11_0_ring_emit_ib_gfx */
6844 .emit_ib = gfx_v11_0_ring_emit_ib_gfx,
6845 .emit_fence = gfx_v11_0_ring_emit_fence,
6846 .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync,
6847 .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush,
6848 .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch,
6849 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush,
6850 .test_ring = gfx_v11_0_ring_test_ring,
6851 .test_ib = gfx_v11_0_ring_test_ib,
6852 .insert_nop = gfx_v11_ring_insert_nop,
6853 .pad_ib = amdgpu_ring_generic_pad_ib,
6854 .emit_cntxcntl = gfx_v11_0_ring_emit_cntxcntl,
6855 .emit_gfx_shadow = gfx_v11_0_ring_emit_gfx_shadow,
6856 .init_cond_exec = gfx_v11_0_ring_emit_init_cond_exec,
6857 .preempt_ib = gfx_v11_0_ring_preempt_ib,
6858 .emit_frame_cntl = gfx_v11_0_ring_emit_frame_cntl,
6859 .emit_wreg = gfx_v11_0_ring_emit_wreg,
6860 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6861 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6862 .soft_recovery = gfx_v11_0_ring_soft_recovery,
6863 .emit_mem_sync = gfx_v11_0_emit_mem_sync,
6864 .reset = gfx_v11_0_reset_kgq,
6865 };
6866
6867 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_compute = {
6868 .type = AMDGPU_RING_TYPE_COMPUTE,
6869 .align_mask = 0xff,
6870 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6871 .support_64bit_ptrs = true,
6872 .get_rptr = gfx_v11_0_ring_get_rptr_compute,
6873 .get_wptr = gfx_v11_0_ring_get_wptr_compute,
6874 .set_wptr = gfx_v11_0_ring_set_wptr_compute,
6875 .emit_frame_size =
6876 5 + /* update_spm_vmid */
6877 20 + /* gfx_v11_0_ring_emit_gds_switch */
6878 7 + /* gfx_v11_0_ring_emit_hdp_flush */
6879 5 + /* hdp invalidate */
6880 7 + /* gfx_v11_0_ring_emit_pipeline_sync */
6881 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6882 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6883 2 + /* gfx_v11_0_ring_emit_vm_flush */
6884 8 + 8 + 8 + /* gfx_v11_0_ring_emit_fence x3 for user fence, vm fence */
6885 8, /* gfx_v11_0_emit_mem_sync */
6886 .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */
6887 .emit_ib = gfx_v11_0_ring_emit_ib_compute,
6888 .emit_fence = gfx_v11_0_ring_emit_fence,
6889 .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync,
6890 .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush,
6891 .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch,
6892 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush,
6893 .test_ring = gfx_v11_0_ring_test_ring,
6894 .test_ib = gfx_v11_0_ring_test_ib,
6895 .insert_nop = gfx_v11_ring_insert_nop,
6896 .pad_ib = amdgpu_ring_generic_pad_ib,
6897 .emit_wreg = gfx_v11_0_ring_emit_wreg,
6898 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6899 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6900 .soft_recovery = gfx_v11_0_ring_soft_recovery,
6901 .emit_mem_sync = gfx_v11_0_emit_mem_sync,
6902 .reset = gfx_v11_0_reset_kcq,
6903 };
6904
6905 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_kiq = {
6906 .type = AMDGPU_RING_TYPE_KIQ,
6907 .align_mask = 0xff,
6908 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6909 .support_64bit_ptrs = true,
6910 .get_rptr = gfx_v11_0_ring_get_rptr_compute,
6911 .get_wptr = gfx_v11_0_ring_get_wptr_compute,
6912 .set_wptr = gfx_v11_0_ring_set_wptr_compute,
6913 .emit_frame_size =
6914 20 + /* gfx_v11_0_ring_emit_gds_switch */
6915 7 + /* gfx_v11_0_ring_emit_hdp_flush */
6916 5 + /*hdp invalidate */
6917 7 + /* gfx_v11_0_ring_emit_pipeline_sync */
6918 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6919 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6920 8 + 8 + 8, /* gfx_v11_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6921 .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */
6922 .emit_ib = gfx_v11_0_ring_emit_ib_compute,
6923 .emit_fence = gfx_v11_0_ring_emit_fence_kiq,
6924 .test_ring = gfx_v11_0_ring_test_ring,
6925 .test_ib = gfx_v11_0_ring_test_ib,
6926 .insert_nop = amdgpu_ring_insert_nop,
6927 .pad_ib = amdgpu_ring_generic_pad_ib,
6928 .emit_rreg = gfx_v11_0_ring_emit_rreg,
6929 .emit_wreg = gfx_v11_0_ring_emit_wreg,
6930 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6931 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6932 };
6933
gfx_v11_0_set_ring_funcs(struct amdgpu_device * adev)6934 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev)
6935 {
6936 int i;
6937
6938 adev->gfx.kiq[0].ring.funcs = &gfx_v11_0_ring_funcs_kiq;
6939
6940 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6941 adev->gfx.gfx_ring[i].funcs = &gfx_v11_0_ring_funcs_gfx;
6942
6943 for (i = 0; i < adev->gfx.num_compute_rings; i++)
6944 adev->gfx.compute_ring[i].funcs = &gfx_v11_0_ring_funcs_compute;
6945 }
6946
6947 static const struct amdgpu_irq_src_funcs gfx_v11_0_eop_irq_funcs = {
6948 .set = gfx_v11_0_set_eop_interrupt_state,
6949 .process = gfx_v11_0_eop_irq,
6950 };
6951
6952 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_reg_irq_funcs = {
6953 .set = gfx_v11_0_set_priv_reg_fault_state,
6954 .process = gfx_v11_0_priv_reg_irq,
6955 };
6956
6957 static const struct amdgpu_irq_src_funcs gfx_v11_0_bad_op_irq_funcs = {
6958 .set = gfx_v11_0_set_bad_op_fault_state,
6959 .process = gfx_v11_0_bad_op_irq,
6960 };
6961
6962 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_inst_irq_funcs = {
6963 .set = gfx_v11_0_set_priv_inst_fault_state,
6964 .process = gfx_v11_0_priv_inst_irq,
6965 };
6966
6967 static const struct amdgpu_irq_src_funcs gfx_v11_0_rlc_gc_fed_irq_funcs = {
6968 .process = gfx_v11_0_rlc_gc_fed_irq,
6969 };
6970
gfx_v11_0_set_irq_funcs(struct amdgpu_device * adev)6971 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev)
6972 {
6973 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
6974 adev->gfx.eop_irq.funcs = &gfx_v11_0_eop_irq_funcs;
6975
6976 adev->gfx.priv_reg_irq.num_types = 1;
6977 adev->gfx.priv_reg_irq.funcs = &gfx_v11_0_priv_reg_irq_funcs;
6978
6979 adev->gfx.bad_op_irq.num_types = 1;
6980 adev->gfx.bad_op_irq.funcs = &gfx_v11_0_bad_op_irq_funcs;
6981
6982 adev->gfx.priv_inst_irq.num_types = 1;
6983 adev->gfx.priv_inst_irq.funcs = &gfx_v11_0_priv_inst_irq_funcs;
6984
6985 adev->gfx.rlc_gc_fed_irq.num_types = 1; /* 0x80 FED error */
6986 adev->gfx.rlc_gc_fed_irq.funcs = &gfx_v11_0_rlc_gc_fed_irq_funcs;
6987
6988 }
6989
gfx_v11_0_set_imu_funcs(struct amdgpu_device * adev)6990 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev)
6991 {
6992 if (adev->flags & AMD_IS_APU)
6993 adev->gfx.imu.mode = MISSION_MODE;
6994 else
6995 adev->gfx.imu.mode = DEBUG_MODE;
6996
6997 adev->gfx.imu.funcs = &gfx_v11_0_imu_funcs;
6998 }
6999
gfx_v11_0_set_rlc_funcs(struct amdgpu_device * adev)7000 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev)
7001 {
7002 adev->gfx.rlc.funcs = &gfx_v11_0_rlc_funcs;
7003 }
7004
gfx_v11_0_set_gds_init(struct amdgpu_device * adev)7005 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev)
7006 {
7007 unsigned total_cu = adev->gfx.config.max_cu_per_sh *
7008 adev->gfx.config.max_sh_per_se *
7009 adev->gfx.config.max_shader_engines;
7010
7011 adev->gds.gds_size = 0x1000;
7012 adev->gds.gds_compute_max_wave_id = total_cu * 32 - 1;
7013 adev->gds.gws_size = 64;
7014 adev->gds.oa_size = 16;
7015 }
7016
gfx_v11_0_set_mqd_funcs(struct amdgpu_device * adev)7017 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev)
7018 {
7019 /* set gfx eng mqd */
7020 adev->mqds[AMDGPU_HW_IP_GFX].mqd_size =
7021 sizeof(struct v11_gfx_mqd);
7022 adev->mqds[AMDGPU_HW_IP_GFX].init_mqd =
7023 gfx_v11_0_gfx_mqd_init;
7024 /* set compute eng mqd */
7025 adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size =
7026 sizeof(struct v11_compute_mqd);
7027 adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd =
7028 gfx_v11_0_compute_mqd_init;
7029 }
7030
gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device * adev,u32 bitmap)7031 static void gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev,
7032 u32 bitmap)
7033 {
7034 u32 data;
7035
7036 if (!bitmap)
7037 return;
7038
7039 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
7040 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
7041
7042 WREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG, data);
7043 }
7044
gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device * adev)7045 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev)
7046 {
7047 u32 data, wgp_bitmask;
7048 data = RREG32_SOC15(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG);
7049 data |= RREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG);
7050
7051 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
7052 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
7053
7054 wgp_bitmask =
7055 amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1);
7056
7057 return (~data) & wgp_bitmask;
7058 }
7059
gfx_v11_0_get_cu_active_bitmap_per_sh(struct amdgpu_device * adev)7060 static u32 gfx_v11_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev)
7061 {
7062 u32 wgp_idx, wgp_active_bitmap;
7063 u32 cu_bitmap_per_wgp, cu_active_bitmap;
7064
7065 wgp_active_bitmap = gfx_v11_0_get_wgp_active_bitmap_per_sh(adev);
7066 cu_active_bitmap = 0;
7067
7068 for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) {
7069 /* if there is one WGP enabled, it means 2 CUs will be enabled */
7070 cu_bitmap_per_wgp = 3 << (2 * wgp_idx);
7071 if (wgp_active_bitmap & (1 << wgp_idx))
7072 cu_active_bitmap |= cu_bitmap_per_wgp;
7073 }
7074
7075 return cu_active_bitmap;
7076 }
7077
gfx_v11_0_get_cu_info(struct amdgpu_device * adev,struct amdgpu_cu_info * cu_info)7078 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,
7079 struct amdgpu_cu_info *cu_info)
7080 {
7081 int i, j, k, counter, active_cu_number = 0;
7082 u32 mask, bitmap;
7083 unsigned disable_masks[8 * 2];
7084
7085 if (!adev || !cu_info)
7086 return -EINVAL;
7087
7088 amdgpu_gfx_parse_disable_cu(disable_masks, 8, 2);
7089
7090 mutex_lock(&adev->grbm_idx_mutex);
7091 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7092 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7093 bitmap = i * adev->gfx.config.max_sh_per_se + j;
7094 if (!((gfx_v11_0_get_sa_active_bitmap(adev) >> bitmap) & 1))
7095 continue;
7096 mask = 1;
7097 counter = 0;
7098 gfx_v11_0_select_se_sh(adev, i, j, 0xffffffff, 0);
7099 if (i < 8 && j < 2)
7100 gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(
7101 adev, disable_masks[i * 2 + j]);
7102 bitmap = gfx_v11_0_get_cu_active_bitmap_per_sh(adev);
7103
7104 /**
7105 * GFX11 could support more than 4 SEs, while the bitmap
7106 * in cu_info struct is 4x4 and ioctl interface struct
7107 * drm_amdgpu_info_device should keep stable.
7108 * So we use last two columns of bitmap to store cu mask for
7109 * SEs 4 to 7, the layout of the bitmap is as below:
7110 * SE0: {SH0,SH1} --> {bitmap[0][0], bitmap[0][1]}
7111 * SE1: {SH0,SH1} --> {bitmap[1][0], bitmap[1][1]}
7112 * SE2: {SH0,SH1} --> {bitmap[2][0], bitmap[2][1]}
7113 * SE3: {SH0,SH1} --> {bitmap[3][0], bitmap[3][1]}
7114 * SE4: {SH0,SH1} --> {bitmap[0][2], bitmap[0][3]}
7115 * SE5: {SH0,SH1} --> {bitmap[1][2], bitmap[1][3]}
7116 * SE6: {SH0,SH1} --> {bitmap[2][2], bitmap[2][3]}
7117 * SE7: {SH0,SH1} --> {bitmap[3][2], bitmap[3][3]}
7118 */
7119 cu_info->bitmap[0][i % 4][j + (i / 4) * 2] = bitmap;
7120
7121 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
7122 if (bitmap & mask)
7123 counter++;
7124
7125 mask <<= 1;
7126 }
7127 active_cu_number += counter;
7128 }
7129 }
7130 gfx_v11_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7131 mutex_unlock(&adev->grbm_idx_mutex);
7132
7133 cu_info->number = active_cu_number;
7134 cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7135
7136 return 0;
7137 }
7138
7139 const struct amdgpu_ip_block_version gfx_v11_0_ip_block =
7140 {
7141 .type = AMD_IP_BLOCK_TYPE_GFX,
7142 .major = 11,
7143 .minor = 0,
7144 .rev = 0,
7145 .funcs = &gfx_v11_0_ip_funcs,
7146 };
7147