1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2016-2017 The Linux Foundation. All rights reserved.
3 */
4
5 #include <linux/kernel.h>
6 #include <linux/types.h>
7 #include <linux/cpumask.h>
8 #include <linux/qcom_scm.h>
9 #include <linux/pm_opp.h>
10 #include <linux/nvmem-consumer.h>
11 #include <linux/slab.h>
12 #include "msm_gem.h"
13 #include "msm_mmu.h"
14 #include "a5xx_gpu.h"
15
16 extern bool hang_debug;
17 static void a5xx_dump(struct msm_gpu *gpu);
18
19 #define GPU_PAS_ID 13
20
update_shadow_rptr(struct msm_gpu * gpu,struct msm_ringbuffer * ring)21 static void update_shadow_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
22 {
23 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
24 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
25
26 if (a5xx_gpu->has_whereami) {
27 OUT_PKT7(ring, CP_WHERE_AM_I, 2);
28 OUT_RING(ring, lower_32_bits(shadowptr(a5xx_gpu, ring)));
29 OUT_RING(ring, upper_32_bits(shadowptr(a5xx_gpu, ring)));
30 }
31 }
32
a5xx_flush(struct msm_gpu * gpu,struct msm_ringbuffer * ring,bool sync)33 void a5xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring,
34 bool sync)
35 {
36 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
37 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
38 uint32_t wptr;
39 unsigned long flags;
40
41 /*
42 * Most flush operations need to issue a WHERE_AM_I opcode to sync up
43 * the rptr shadow
44 */
45 if (sync)
46 update_shadow_rptr(gpu, ring);
47
48 spin_lock_irqsave(&ring->preempt_lock, flags);
49
50 /* Copy the shadow to the actual register */
51 ring->cur = ring->next;
52
53 /* Make sure to wrap wptr if we need to */
54 wptr = get_wptr(ring);
55
56 spin_unlock_irqrestore(&ring->preempt_lock, flags);
57
58 /* Make sure everything is posted before making a decision */
59 mb();
60
61 /* Update HW if this is the current ring and we are not in preempt */
62 if (a5xx_gpu->cur_ring == ring && !a5xx_in_preempt(a5xx_gpu))
63 gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);
64 }
65
a5xx_submit_in_rb(struct msm_gpu * gpu,struct msm_gem_submit * submit)66 static void a5xx_submit_in_rb(struct msm_gpu *gpu, struct msm_gem_submit *submit)
67 {
68 struct msm_ringbuffer *ring = submit->ring;
69 struct msm_gem_object *obj;
70 uint32_t *ptr, dwords;
71 unsigned int i;
72
73 for (i = 0; i < submit->nr_cmds; i++) {
74 switch (submit->cmd[i].type) {
75 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
76 break;
77 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
78 if (gpu->cur_ctx_seqno == submit->queue->ctx->seqno)
79 break;
80 fallthrough;
81 case MSM_SUBMIT_CMD_BUF:
82 /* copy commands into RB: */
83 obj = submit->bos[submit->cmd[i].idx].obj;
84 dwords = submit->cmd[i].size;
85
86 ptr = msm_gem_get_vaddr(&obj->base);
87
88 /* _get_vaddr() shouldn't fail at this point,
89 * since we've already mapped it once in
90 * submit_reloc()
91 */
92 if (WARN_ON(IS_ERR_OR_NULL(ptr)))
93 return;
94
95 for (i = 0; i < dwords; i++) {
96 /* normally the OUT_PKTn() would wait
97 * for space for the packet. But since
98 * we just OUT_RING() the whole thing,
99 * need to call adreno_wait_ring()
100 * ourself:
101 */
102 adreno_wait_ring(ring, 1);
103 OUT_RING(ring, ptr[i]);
104 }
105
106 msm_gem_put_vaddr(&obj->base);
107
108 break;
109 }
110 }
111
112 a5xx_flush(gpu, ring, true);
113 a5xx_preempt_trigger(gpu);
114
115 /* we might not necessarily have a cmd from userspace to
116 * trigger an event to know that submit has completed, so
117 * do this manually:
118 */
119 a5xx_idle(gpu, ring);
120 ring->memptrs->fence = submit->seqno;
121 msm_gpu_retire(gpu);
122 }
123
a5xx_submit(struct msm_gpu * gpu,struct msm_gem_submit * submit)124 static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
125 {
126 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
127 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
128 struct msm_ringbuffer *ring = submit->ring;
129 unsigned int i, ibs = 0;
130
131 if (IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) && submit->in_rb) {
132 gpu->cur_ctx_seqno = 0;
133 a5xx_submit_in_rb(gpu, submit);
134 return;
135 }
136
137 OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1);
138 OUT_RING(ring, 0x02);
139
140 /* Turn off protected mode to write to special registers */
141 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
142 OUT_RING(ring, 0);
143
144 /* Set the save preemption record for the ring/command */
145 OUT_PKT4(ring, REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 2);
146 OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[submit->ring->id]));
147 OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[submit->ring->id]));
148
149 /* Turn back on protected mode */
150 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
151 OUT_RING(ring, 1);
152
153 /* Enable local preemption for finegrain preemption */
154 OUT_PKT7(ring, CP_PREEMPT_ENABLE_LOCAL, 1);
155 OUT_RING(ring, 0x1);
156
157 /* Allow CP_CONTEXT_SWITCH_YIELD packets in the IB2 */
158 OUT_PKT7(ring, CP_YIELD_ENABLE, 1);
159 OUT_RING(ring, 0x02);
160
161 /* Submit the commands */
162 for (i = 0; i < submit->nr_cmds; i++) {
163 switch (submit->cmd[i].type) {
164 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
165 break;
166 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
167 if (gpu->cur_ctx_seqno == submit->queue->ctx->seqno)
168 break;
169 fallthrough;
170 case MSM_SUBMIT_CMD_BUF:
171 OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
172 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
173 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
174 OUT_RING(ring, submit->cmd[i].size);
175 ibs++;
176 break;
177 }
178
179 /*
180 * Periodically update shadow-wptr if needed, so that we
181 * can see partial progress of submits with large # of
182 * cmds.. otherwise we could needlessly stall waiting for
183 * ringbuffer state, simply due to looking at a shadow
184 * rptr value that has not been updated
185 */
186 if ((ibs % 32) == 0)
187 update_shadow_rptr(gpu, ring);
188 }
189
190 /*
191 * Write the render mode to NULL (0) to indicate to the CP that the IBs
192 * are done rendering - otherwise a lucky preemption would start
193 * replaying from the last checkpoint
194 */
195 OUT_PKT7(ring, CP_SET_RENDER_MODE, 5);
196 OUT_RING(ring, 0);
197 OUT_RING(ring, 0);
198 OUT_RING(ring, 0);
199 OUT_RING(ring, 0);
200 OUT_RING(ring, 0);
201
202 /* Turn off IB level preemptions */
203 OUT_PKT7(ring, CP_YIELD_ENABLE, 1);
204 OUT_RING(ring, 0x01);
205
206 /* Write the fence to the scratch register */
207 OUT_PKT4(ring, REG_A5XX_CP_SCRATCH_REG(2), 1);
208 OUT_RING(ring, submit->seqno);
209
210 /*
211 * Execute a CACHE_FLUSH_TS event. This will ensure that the
212 * timestamp is written to the memory and then triggers the interrupt
213 */
214 OUT_PKT7(ring, CP_EVENT_WRITE, 4);
215 OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(CACHE_FLUSH_TS) |
216 CP_EVENT_WRITE_0_IRQ);
217 OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence)));
218 OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence)));
219 OUT_RING(ring, submit->seqno);
220
221 /* Yield the floor on command completion */
222 OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4);
223 /*
224 * If dword[2:1] are non zero, they specify an address for the CP to
225 * write the value of dword[3] to on preemption complete. Write 0 to
226 * skip the write
227 */
228 OUT_RING(ring, 0x00);
229 OUT_RING(ring, 0x00);
230 /* Data value - not used if the address above is 0 */
231 OUT_RING(ring, 0x01);
232 /* Set bit 0 to trigger an interrupt on preempt complete */
233 OUT_RING(ring, 0x01);
234
235 /* A WHERE_AM_I packet is not needed after a YIELD */
236 a5xx_flush(gpu, ring, false);
237
238 /* Check to see if we need to start preemption */
239 a5xx_preempt_trigger(gpu);
240 }
241
242 static const struct adreno_five_hwcg_regs {
243 u32 offset;
244 u32 value;
245 } a5xx_hwcg[] = {
246 {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
247 {REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222},
248 {REG_A5XX_RBBM_CLOCK_CNTL_SP2, 0x02222222},
249 {REG_A5XX_RBBM_CLOCK_CNTL_SP3, 0x02222222},
250 {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
251 {REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220},
252 {REG_A5XX_RBBM_CLOCK_CNTL2_SP2, 0x02222220},
253 {REG_A5XX_RBBM_CLOCK_CNTL2_SP3, 0x02222220},
254 {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
255 {REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF},
256 {REG_A5XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF},
257 {REG_A5XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF},
258 {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
259 {REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
260 {REG_A5XX_RBBM_CLOCK_DELAY_SP2, 0x00000080},
261 {REG_A5XX_RBBM_CLOCK_DELAY_SP3, 0x00000080},
262 {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
263 {REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222},
264 {REG_A5XX_RBBM_CLOCK_CNTL_TP2, 0x22222222},
265 {REG_A5XX_RBBM_CLOCK_CNTL_TP3, 0x22222222},
266 {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
267 {REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
268 {REG_A5XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222},
269 {REG_A5XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222},
270 {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222},
271 {REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222},
272 {REG_A5XX_RBBM_CLOCK_CNTL3_TP2, 0x00002222},
273 {REG_A5XX_RBBM_CLOCK_CNTL3_TP3, 0x00002222},
274 {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
275 {REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
276 {REG_A5XX_RBBM_CLOCK_HYST_TP2, 0x77777777},
277 {REG_A5XX_RBBM_CLOCK_HYST_TP3, 0x77777777},
278 {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
279 {REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
280 {REG_A5XX_RBBM_CLOCK_HYST2_TP2, 0x77777777},
281 {REG_A5XX_RBBM_CLOCK_HYST2_TP3, 0x77777777},
282 {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777},
283 {REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777},
284 {REG_A5XX_RBBM_CLOCK_HYST3_TP2, 0x00007777},
285 {REG_A5XX_RBBM_CLOCK_HYST3_TP3, 0x00007777},
286 {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
287 {REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
288 {REG_A5XX_RBBM_CLOCK_DELAY_TP2, 0x11111111},
289 {REG_A5XX_RBBM_CLOCK_DELAY_TP3, 0x11111111},
290 {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
291 {REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
292 {REG_A5XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111},
293 {REG_A5XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111},
294 {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111},
295 {REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111},
296 {REG_A5XX_RBBM_CLOCK_DELAY3_TP2, 0x00001111},
297 {REG_A5XX_RBBM_CLOCK_DELAY3_TP3, 0x00001111},
298 {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
299 {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
300 {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
301 {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
302 {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444},
303 {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
304 {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
305 {REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
306 {REG_A5XX_RBBM_CLOCK_CNTL_RB2, 0x22222222},
307 {REG_A5XX_RBBM_CLOCK_CNTL_RB3, 0x22222222},
308 {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222},
309 {REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222},
310 {REG_A5XX_RBBM_CLOCK_CNTL2_RB2, 0x00222222},
311 {REG_A5XX_RBBM_CLOCK_CNTL2_RB3, 0x00222222},
312 {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220},
313 {REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220},
314 {REG_A5XX_RBBM_CLOCK_CNTL_CCU2, 0x00022220},
315 {REG_A5XX_RBBM_CLOCK_CNTL_CCU3, 0x00022220},
316 {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222},
317 {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555},
318 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404},
319 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404},
320 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU2, 0x04040404},
321 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU3, 0x04040404},
322 {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044},
323 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002},
324 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002},
325 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2, 0x00000002},
326 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3, 0x00000002},
327 {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011},
328 {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
329 {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
330 {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
331 {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
332 {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
333 {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
334 {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
335 {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
336 {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
337 {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}
338 }, a50x_hwcg[] = {
339 {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
340 {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
341 {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
342 {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
343 {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
344 {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
345 {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222},
346 {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
347 {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
348 {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777},
349 {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
350 {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
351 {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111},
352 {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
353 {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
354 {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
355 {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
356 {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00FFFFF4},
357 {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
358 {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
359 {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222},
360 {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220},
361 {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222},
362 {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555},
363 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404},
364 {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044},
365 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002},
366 {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011},
367 {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
368 {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
369 {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
370 {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
371 {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
372 {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
373 {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
374 {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
375 {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
376 {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
377 }, a512_hwcg[] = {
378 {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
379 {REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222},
380 {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
381 {REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220},
382 {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
383 {REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF},
384 {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
385 {REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
386 {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
387 {REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222},
388 {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
389 {REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
390 {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222},
391 {REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222},
392 {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
393 {REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
394 {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
395 {REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
396 {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777},
397 {REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777},
398 {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
399 {REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
400 {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
401 {REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
402 {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111},
403 {REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111},
404 {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
405 {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
406 {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
407 {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
408 {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444},
409 {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
410 {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
411 {REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
412 {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222},
413 {REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222},
414 {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220},
415 {REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220},
416 {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222},
417 {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555},
418 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404},
419 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404},
420 {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044},
421 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002},
422 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002},
423 {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011},
424 {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
425 {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
426 {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
427 {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
428 {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
429 {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
430 {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
431 {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
432 {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
433 {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
434 };
435
a5xx_set_hwcg(struct msm_gpu * gpu,bool state)436 void a5xx_set_hwcg(struct msm_gpu *gpu, bool state)
437 {
438 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
439 const struct adreno_five_hwcg_regs *regs;
440 unsigned int i, sz;
441
442 if (adreno_is_a506(adreno_gpu) || adreno_is_a508(adreno_gpu)) {
443 regs = a50x_hwcg;
444 sz = ARRAY_SIZE(a50x_hwcg);
445 } else if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu)) {
446 regs = a512_hwcg;
447 sz = ARRAY_SIZE(a512_hwcg);
448 } else {
449 regs = a5xx_hwcg;
450 sz = ARRAY_SIZE(a5xx_hwcg);
451 }
452
453 for (i = 0; i < sz; i++)
454 gpu_write(gpu, regs[i].offset,
455 state ? regs[i].value : 0);
456
457 if (adreno_is_a540(adreno_gpu)) {
458 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_DELAY_GPMU, state ? 0x00000770 : 0);
459 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_HYST_GPMU, state ? 0x00000004 : 0);
460 }
461
462 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, state ? 0xAAA8AA00 : 0);
463 gpu_write(gpu, REG_A5XX_RBBM_ISDB_CNT, state ? 0x182 : 0x180);
464 }
465
a5xx_me_init(struct msm_gpu * gpu)466 static int a5xx_me_init(struct msm_gpu *gpu)
467 {
468 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
469 struct msm_ringbuffer *ring = gpu->rb[0];
470
471 OUT_PKT7(ring, CP_ME_INIT, 8);
472
473 OUT_RING(ring, 0x0000002F);
474
475 /* Enable multiple hardware contexts */
476 OUT_RING(ring, 0x00000003);
477
478 /* Enable error detection */
479 OUT_RING(ring, 0x20000000);
480
481 /* Don't enable header dump */
482 OUT_RING(ring, 0x00000000);
483 OUT_RING(ring, 0x00000000);
484
485 /* Specify workarounds for various microcode issues */
486 if (adreno_is_a506(adreno_gpu) || adreno_is_a530(adreno_gpu)) {
487 /* Workaround for token end syncs
488 * Force a WFI after every direct-render 3D mode draw and every
489 * 2D mode 3 draw
490 */
491 OUT_RING(ring, 0x0000000B);
492 } else if (adreno_is_a510(adreno_gpu)) {
493 /* Workaround for token and syncs */
494 OUT_RING(ring, 0x00000001);
495 } else {
496 /* No workarounds enabled */
497 OUT_RING(ring, 0x00000000);
498 }
499
500 OUT_RING(ring, 0x00000000);
501 OUT_RING(ring, 0x00000000);
502
503 a5xx_flush(gpu, ring, true);
504 return a5xx_idle(gpu, ring) ? 0 : -EINVAL;
505 }
506
a5xx_preempt_start(struct msm_gpu * gpu)507 static int a5xx_preempt_start(struct msm_gpu *gpu)
508 {
509 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
510 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
511 struct msm_ringbuffer *ring = gpu->rb[0];
512
513 if (gpu->nr_rings == 1)
514 return 0;
515
516 /* Turn off protected mode to write to special registers */
517 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
518 OUT_RING(ring, 0);
519
520 /* Set the save preemption record for the ring/command */
521 OUT_PKT4(ring, REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 2);
522 OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[ring->id]));
523 OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[ring->id]));
524
525 /* Turn back on protected mode */
526 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
527 OUT_RING(ring, 1);
528
529 OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1);
530 OUT_RING(ring, 0x00);
531
532 OUT_PKT7(ring, CP_PREEMPT_ENABLE_LOCAL, 1);
533 OUT_RING(ring, 0x01);
534
535 OUT_PKT7(ring, CP_YIELD_ENABLE, 1);
536 OUT_RING(ring, 0x01);
537
538 /* Yield the floor on command completion */
539 OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4);
540 OUT_RING(ring, 0x00);
541 OUT_RING(ring, 0x00);
542 OUT_RING(ring, 0x01);
543 OUT_RING(ring, 0x01);
544
545 /* The WHERE_AMI_I packet is not needed after a YIELD is issued */
546 a5xx_flush(gpu, ring, false);
547
548 return a5xx_idle(gpu, ring) ? 0 : -EINVAL;
549 }
550
a5xx_ucode_check_version(struct a5xx_gpu * a5xx_gpu,struct drm_gem_object * obj)551 static void a5xx_ucode_check_version(struct a5xx_gpu *a5xx_gpu,
552 struct drm_gem_object *obj)
553 {
554 u32 *buf = msm_gem_get_vaddr(obj);
555
556 if (IS_ERR(buf))
557 return;
558
559 /*
560 * If the lowest nibble is 0xa that is an indication that this microcode
561 * has been patched. The actual version is in dword [3] but we only care
562 * about the patchlevel which is the lowest nibble of dword [3]
563 */
564 if (((buf[0] & 0xf) == 0xa) && (buf[2] & 0xf) >= 1)
565 a5xx_gpu->has_whereami = true;
566
567 msm_gem_put_vaddr(obj);
568 }
569
a5xx_ucode_init(struct msm_gpu * gpu)570 static int a5xx_ucode_init(struct msm_gpu *gpu)
571 {
572 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
573 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
574 int ret;
575
576 if (!a5xx_gpu->pm4_bo) {
577 a5xx_gpu->pm4_bo = adreno_fw_create_bo(gpu,
578 adreno_gpu->fw[ADRENO_FW_PM4], &a5xx_gpu->pm4_iova);
579
580
581 if (IS_ERR(a5xx_gpu->pm4_bo)) {
582 ret = PTR_ERR(a5xx_gpu->pm4_bo);
583 a5xx_gpu->pm4_bo = NULL;
584 DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PM4: %d\n",
585 ret);
586 return ret;
587 }
588
589 msm_gem_object_set_name(a5xx_gpu->pm4_bo, "pm4fw");
590 }
591
592 if (!a5xx_gpu->pfp_bo) {
593 a5xx_gpu->pfp_bo = adreno_fw_create_bo(gpu,
594 adreno_gpu->fw[ADRENO_FW_PFP], &a5xx_gpu->pfp_iova);
595
596 if (IS_ERR(a5xx_gpu->pfp_bo)) {
597 ret = PTR_ERR(a5xx_gpu->pfp_bo);
598 a5xx_gpu->pfp_bo = NULL;
599 DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PFP: %d\n",
600 ret);
601 return ret;
602 }
603
604 msm_gem_object_set_name(a5xx_gpu->pfp_bo, "pfpfw");
605 a5xx_ucode_check_version(a5xx_gpu, a5xx_gpu->pfp_bo);
606 }
607
608 gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO, a5xx_gpu->pm4_iova);
609
610 gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO, a5xx_gpu->pfp_iova);
611
612 return 0;
613 }
614
615 #define SCM_GPU_ZAP_SHADER_RESUME 0
616
a5xx_zap_shader_resume(struct msm_gpu * gpu)617 static int a5xx_zap_shader_resume(struct msm_gpu *gpu)
618 {
619 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
620 int ret;
621
622 /*
623 * Adreno 506 have CPZ Retention feature and doesn't require
624 * to resume zap shader
625 */
626 if (adreno_is_a506(adreno_gpu))
627 return 0;
628
629 ret = qcom_scm_set_remote_state(SCM_GPU_ZAP_SHADER_RESUME, GPU_PAS_ID);
630 if (ret)
631 DRM_ERROR("%s: zap-shader resume failed: %d\n",
632 gpu->name, ret);
633
634 return ret;
635 }
636
a5xx_zap_shader_init(struct msm_gpu * gpu)637 static int a5xx_zap_shader_init(struct msm_gpu *gpu)
638 {
639 static bool loaded;
640 int ret;
641
642 /*
643 * If the zap shader is already loaded into memory we just need to kick
644 * the remote processor to reinitialize it
645 */
646 if (loaded)
647 return a5xx_zap_shader_resume(gpu);
648
649 ret = adreno_zap_shader_load(gpu, GPU_PAS_ID);
650
651 loaded = !ret;
652 return ret;
653 }
654
655 #define A5XX_INT_MASK (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
656 A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
657 A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
658 A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
659 A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
660 A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW | \
661 A5XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
662 A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT | \
663 A5XX_RBBM_INT_0_MASK_CP_SW | \
664 A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
665 A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
666 A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
667
a5xx_hw_init(struct msm_gpu * gpu)668 static int a5xx_hw_init(struct msm_gpu *gpu)
669 {
670 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
671 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
672 u32 regbit;
673 int ret;
674
675 gpu_write(gpu, REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003);
676
677 if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu) ||
678 adreno_is_a540(adreno_gpu))
679 gpu_write(gpu, REG_A5XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009);
680
681 /* Make all blocks contribute to the GPU BUSY perf counter */
682 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF);
683
684 /* Enable RBBM error reporting bits */
685 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL0, 0x00000001);
686
687 if (adreno_gpu->info->quirks & ADRENO_QUIRK_FAULT_DETECT_MASK) {
688 /*
689 * Mask out the activity signals from RB1-3 to avoid false
690 * positives
691 */
692
693 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11,
694 0xF0000000);
695 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12,
696 0xFFFFFFFF);
697 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13,
698 0xFFFFFFFF);
699 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14,
700 0xFFFFFFFF);
701 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15,
702 0xFFFFFFFF);
703 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16,
704 0xFFFFFFFF);
705 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17,
706 0xFFFFFFFF);
707 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18,
708 0xFFFFFFFF);
709 }
710
711 /* Enable fault detection */
712 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_INT_CNTL,
713 (1 << 30) | 0xFFFF);
714
715 /* Turn on performance counters */
716 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_CNTL, 0x01);
717
718 /* Select CP0 to always count cycles */
719 gpu_write(gpu, REG_A5XX_CP_PERFCTR_CP_SEL_0, PERF_CP_ALWAYS_COUNT);
720
721 /* Select RBBM0 to countable 6 to get the busy status for devfreq */
722 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_SEL_0, 6);
723
724 /* Increase VFD cache access so LRZ and other data gets evicted less */
725 gpu_write(gpu, REG_A5XX_UCHE_CACHE_WAYS, 0x02);
726
727 /* Disable L2 bypass in the UCHE */
728 gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_LO, 0xFFFF0000);
729 gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_HI, 0x0001FFFF);
730 gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_LO, 0xFFFF0000);
731 gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_HI, 0x0001FFFF);
732
733 /* Set the GMEM VA range (0 to gpu->gmem) */
734 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_LO, 0x00100000);
735 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_HI, 0x00000000);
736 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_LO,
737 0x00100000 + adreno_gpu->gmem - 1);
738 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x00000000);
739
740 if (adreno_is_a506(adreno_gpu) || adreno_is_a508(adreno_gpu) ||
741 adreno_is_a510(adreno_gpu)) {
742 gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x20);
743 if (adreno_is_a506(adreno_gpu) || adreno_is_a508(adreno_gpu))
744 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400);
745 else
746 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x20);
747 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x40000030);
748 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x20100D0A);
749 } else {
750 gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40);
751 if (adreno_is_a530(adreno_gpu))
752 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40);
753 else
754 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400);
755 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060);
756 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16);
757 }
758
759 if (adreno_is_a506(adreno_gpu) || adreno_is_a508(adreno_gpu))
760 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL,
761 (0x100 << 11 | 0x100 << 22));
762 else if (adreno_is_a509(adreno_gpu) || adreno_is_a510(adreno_gpu) ||
763 adreno_is_a512(adreno_gpu))
764 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL,
765 (0x200 << 11 | 0x200 << 22));
766 else
767 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL,
768 (0x400 << 11 | 0x300 << 22));
769
770 if (adreno_gpu->info->quirks & ADRENO_QUIRK_TWO_PASS_USE_WFI)
771 gpu_rmw(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8));
772
773 /*
774 * Disable the RB sampler datapath DP2 clock gating optimization
775 * for 1-SP GPUs, as it is enabled by default.
776 */
777 if (adreno_is_a506(adreno_gpu) || adreno_is_a508(adreno_gpu) ||
778 adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu))
779 gpu_rmw(gpu, REG_A5XX_RB_DBG_ECO_CNTL, 0, (1 << 9));
780
781 /* Disable UCHE global filter as SP can invalidate/flush independently */
782 gpu_write(gpu, REG_A5XX_UCHE_MODE_CNTL, BIT(29));
783
784 /* Enable USE_RETENTION_FLOPS */
785 gpu_write(gpu, REG_A5XX_CP_CHICKEN_DBG, 0x02000000);
786
787 /* Enable ME/PFP split notification */
788 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF);
789
790 /*
791 * In A5x, CCU can send context_done event of a particular context to
792 * UCHE which ultimately reaches CP even when there is valid
793 * transaction of that context inside CCU. This can let CP to program
794 * config registers, which will make the "valid transaction" inside
795 * CCU to be interpreted differently. This can cause gpu fault. This
796 * bug is fixed in latest A510 revision. To enable this bug fix -
797 * bit[11] of RB_DBG_ECO_CNTL need to be set to 0, default is 1
798 * (disable). For older A510 version this bit is unused.
799 */
800 if (adreno_is_a510(adreno_gpu))
801 gpu_rmw(gpu, REG_A5XX_RB_DBG_ECO_CNTL, (1 << 11), 0);
802
803 /* Enable HWCG */
804 a5xx_set_hwcg(gpu, true);
805
806 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL2, 0x0000003F);
807
808 /* Set the highest bank bit */
809 if (adreno_is_a540(adreno_gpu) || adreno_is_a530(adreno_gpu))
810 regbit = 2;
811 else
812 regbit = 1;
813
814 gpu_write(gpu, REG_A5XX_TPL1_MODE_CNTL, regbit << 7);
815 gpu_write(gpu, REG_A5XX_RB_MODE_CNTL, regbit << 1);
816
817 if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu) ||
818 adreno_is_a540(adreno_gpu))
819 gpu_write(gpu, REG_A5XX_UCHE_DBG_ECO_CNTL_2, regbit);
820
821 /* Disable All flat shading optimization (ALLFLATOPTDIS) */
822 gpu_rmw(gpu, REG_A5XX_VPC_DBG_ECO_CNTL, 0, (1 << 10));
823
824 /* Protect registers from the CP */
825 gpu_write(gpu, REG_A5XX_CP_PROTECT_CNTL, 0x00000007);
826
827 /* RBBM */
828 gpu_write(gpu, REG_A5XX_CP_PROTECT(0), ADRENO_PROTECT_RW(0x04, 4));
829 gpu_write(gpu, REG_A5XX_CP_PROTECT(1), ADRENO_PROTECT_RW(0x08, 8));
830 gpu_write(gpu, REG_A5XX_CP_PROTECT(2), ADRENO_PROTECT_RW(0x10, 16));
831 gpu_write(gpu, REG_A5XX_CP_PROTECT(3), ADRENO_PROTECT_RW(0x20, 32));
832 gpu_write(gpu, REG_A5XX_CP_PROTECT(4), ADRENO_PROTECT_RW(0x40, 64));
833 gpu_write(gpu, REG_A5XX_CP_PROTECT(5), ADRENO_PROTECT_RW(0x80, 64));
834
835 /* Content protect */
836 gpu_write(gpu, REG_A5XX_CP_PROTECT(6),
837 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
838 16));
839 gpu_write(gpu, REG_A5XX_CP_PROTECT(7),
840 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TRUST_CNTL, 2));
841
842 /* CP */
843 gpu_write(gpu, REG_A5XX_CP_PROTECT(8), ADRENO_PROTECT_RW(0x800, 64));
844 gpu_write(gpu, REG_A5XX_CP_PROTECT(9), ADRENO_PROTECT_RW(0x840, 8));
845 gpu_write(gpu, REG_A5XX_CP_PROTECT(10), ADRENO_PROTECT_RW(0x880, 32));
846 gpu_write(gpu, REG_A5XX_CP_PROTECT(11), ADRENO_PROTECT_RW(0xAA0, 1));
847
848 /* RB */
849 gpu_write(gpu, REG_A5XX_CP_PROTECT(12), ADRENO_PROTECT_RW(0xCC0, 1));
850 gpu_write(gpu, REG_A5XX_CP_PROTECT(13), ADRENO_PROTECT_RW(0xCF0, 2));
851
852 /* VPC */
853 gpu_write(gpu, REG_A5XX_CP_PROTECT(14), ADRENO_PROTECT_RW(0xE68, 8));
854 gpu_write(gpu, REG_A5XX_CP_PROTECT(15), ADRENO_PROTECT_RW(0xE70, 16));
855
856 /* UCHE */
857 gpu_write(gpu, REG_A5XX_CP_PROTECT(16), ADRENO_PROTECT_RW(0xE80, 16));
858
859 /* SMMU */
860 gpu_write(gpu, REG_A5XX_CP_PROTECT(17),
861 ADRENO_PROTECT_RW(0x10000, 0x8000));
862
863 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_CNTL, 0);
864 /*
865 * Disable the trusted memory range - we don't actually supported secure
866 * memory rendering at this point in time and we don't want to block off
867 * part of the virtual memory space.
868 */
869 gpu_write64(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO, 0x00000000);
870 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
871
872 /* Put the GPU into 64 bit by default */
873 gpu_write(gpu, REG_A5XX_CP_ADDR_MODE_CNTL, 0x1);
874 gpu_write(gpu, REG_A5XX_VSC_ADDR_MODE_CNTL, 0x1);
875 gpu_write(gpu, REG_A5XX_GRAS_ADDR_MODE_CNTL, 0x1);
876 gpu_write(gpu, REG_A5XX_RB_ADDR_MODE_CNTL, 0x1);
877 gpu_write(gpu, REG_A5XX_PC_ADDR_MODE_CNTL, 0x1);
878 gpu_write(gpu, REG_A5XX_HLSQ_ADDR_MODE_CNTL, 0x1);
879 gpu_write(gpu, REG_A5XX_VFD_ADDR_MODE_CNTL, 0x1);
880 gpu_write(gpu, REG_A5XX_VPC_ADDR_MODE_CNTL, 0x1);
881 gpu_write(gpu, REG_A5XX_UCHE_ADDR_MODE_CNTL, 0x1);
882 gpu_write(gpu, REG_A5XX_SP_ADDR_MODE_CNTL, 0x1);
883 gpu_write(gpu, REG_A5XX_TPL1_ADDR_MODE_CNTL, 0x1);
884 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1);
885
886 /*
887 * VPC corner case with local memory load kill leads to corrupt
888 * internal state. Normal Disable does not work for all a5x chips.
889 * So do the following setting to disable it.
890 */
891 if (adreno_gpu->info->quirks & ADRENO_QUIRK_LMLOADKILL_DISABLE) {
892 gpu_rmw(gpu, REG_A5XX_VPC_DBG_ECO_CNTL, 0, BIT(23));
893 gpu_rmw(gpu, REG_A5XX_HLSQ_DBG_ECO_CNTL, BIT(18), 0);
894 }
895
896 ret = adreno_hw_init(gpu);
897 if (ret)
898 return ret;
899
900 if (adreno_is_a530(adreno_gpu) || adreno_is_a540(adreno_gpu))
901 a5xx_gpmu_ucode_init(gpu);
902
903 ret = a5xx_ucode_init(gpu);
904 if (ret)
905 return ret;
906
907 /* Set the ringbuffer address */
908 gpu_write64(gpu, REG_A5XX_CP_RB_BASE, gpu->rb[0]->iova);
909
910 /*
911 * If the microcode supports the WHERE_AM_I opcode then we can use that
912 * in lieu of the RPTR shadow and enable preemption. Otherwise, we
913 * can't safely use the RPTR shadow or preemption. In either case, the
914 * RPTR shadow should be disabled in hardware.
915 */
916 gpu_write(gpu, REG_A5XX_CP_RB_CNTL,
917 MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE);
918
919 /* Create a privileged buffer for the RPTR shadow */
920 if (a5xx_gpu->has_whereami) {
921 if (!a5xx_gpu->shadow_bo) {
922 a5xx_gpu->shadow = msm_gem_kernel_new(gpu->dev,
923 sizeof(u32) * gpu->nr_rings,
924 MSM_BO_WC | MSM_BO_MAP_PRIV,
925 gpu->aspace, &a5xx_gpu->shadow_bo,
926 &a5xx_gpu->shadow_iova);
927
928 if (IS_ERR(a5xx_gpu->shadow))
929 return PTR_ERR(a5xx_gpu->shadow);
930
931 msm_gem_object_set_name(a5xx_gpu->shadow_bo, "shadow");
932 }
933
934 gpu_write64(gpu, REG_A5XX_CP_RB_RPTR_ADDR,
935 shadowptr(a5xx_gpu, gpu->rb[0]));
936 } else if (gpu->nr_rings > 1) {
937 /* Disable preemption if WHERE_AM_I isn't available */
938 a5xx_preempt_fini(gpu);
939 gpu->nr_rings = 1;
940 }
941
942 a5xx_preempt_hw_init(gpu);
943
944 /* Disable the interrupts through the initial bringup stage */
945 gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, A5XX_INT_MASK);
946
947 /* Clear ME_HALT to start the micro engine */
948 gpu_write(gpu, REG_A5XX_CP_PFP_ME_CNTL, 0);
949 ret = a5xx_me_init(gpu);
950 if (ret)
951 return ret;
952
953 ret = a5xx_power_init(gpu);
954 if (ret)
955 return ret;
956
957 /*
958 * Send a pipeline event stat to get misbehaving counters to start
959 * ticking correctly
960 */
961 if (adreno_is_a530(adreno_gpu)) {
962 OUT_PKT7(gpu->rb[0], CP_EVENT_WRITE, 1);
963 OUT_RING(gpu->rb[0], CP_EVENT_WRITE_0_EVENT(STAT_EVENT));
964
965 a5xx_flush(gpu, gpu->rb[0], true);
966 if (!a5xx_idle(gpu, gpu->rb[0]))
967 return -EINVAL;
968 }
969
970 /*
971 * If the chip that we are using does support loading one, then
972 * try to load a zap shader into the secure world. If successful
973 * we can use the CP to switch out of secure mode. If not then we
974 * have no resource but to try to switch ourselves out manually. If we
975 * guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will
976 * be blocked and a permissions violation will soon follow.
977 */
978 ret = a5xx_zap_shader_init(gpu);
979 if (!ret) {
980 OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1);
981 OUT_RING(gpu->rb[0], 0x00000000);
982
983 a5xx_flush(gpu, gpu->rb[0], true);
984 if (!a5xx_idle(gpu, gpu->rb[0]))
985 return -EINVAL;
986 } else if (ret == -ENODEV) {
987 /*
988 * This device does not use zap shader (but print a warning
989 * just in case someone got their dt wrong.. hopefully they
990 * have a debug UART to realize the error of their ways...
991 * if you mess this up you are about to crash horribly)
992 */
993 dev_warn_once(gpu->dev->dev,
994 "Zap shader not enabled - using SECVID_TRUST_CNTL instead\n");
995 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TRUST_CNTL, 0x0);
996 } else {
997 return ret;
998 }
999
1000 /* Last step - yield the ringbuffer */
1001 a5xx_preempt_start(gpu);
1002
1003 return 0;
1004 }
1005
a5xx_recover(struct msm_gpu * gpu)1006 static void a5xx_recover(struct msm_gpu *gpu)
1007 {
1008 int i;
1009
1010 adreno_dump_info(gpu);
1011
1012 for (i = 0; i < 8; i++) {
1013 printk("CP_SCRATCH_REG%d: %u\n", i,
1014 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(i)));
1015 }
1016
1017 if (hang_debug)
1018 a5xx_dump(gpu);
1019
1020 gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 1);
1021 gpu_read(gpu, REG_A5XX_RBBM_SW_RESET_CMD);
1022 gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 0);
1023 adreno_recover(gpu);
1024 }
1025
a5xx_destroy(struct msm_gpu * gpu)1026 static void a5xx_destroy(struct msm_gpu *gpu)
1027 {
1028 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1029 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1030
1031 DBG("%s", gpu->name);
1032
1033 a5xx_preempt_fini(gpu);
1034
1035 if (a5xx_gpu->pm4_bo) {
1036 msm_gem_unpin_iova(a5xx_gpu->pm4_bo, gpu->aspace);
1037 drm_gem_object_put(a5xx_gpu->pm4_bo);
1038 }
1039
1040 if (a5xx_gpu->pfp_bo) {
1041 msm_gem_unpin_iova(a5xx_gpu->pfp_bo, gpu->aspace);
1042 drm_gem_object_put(a5xx_gpu->pfp_bo);
1043 }
1044
1045 if (a5xx_gpu->gpmu_bo) {
1046 msm_gem_unpin_iova(a5xx_gpu->gpmu_bo, gpu->aspace);
1047 drm_gem_object_put(a5xx_gpu->gpmu_bo);
1048 }
1049
1050 if (a5xx_gpu->shadow_bo) {
1051 msm_gem_unpin_iova(a5xx_gpu->shadow_bo, gpu->aspace);
1052 drm_gem_object_put(a5xx_gpu->shadow_bo);
1053 }
1054
1055 adreno_gpu_cleanup(adreno_gpu);
1056 kfree(a5xx_gpu);
1057 }
1058
_a5xx_check_idle(struct msm_gpu * gpu)1059 static inline bool _a5xx_check_idle(struct msm_gpu *gpu)
1060 {
1061 if (gpu_read(gpu, REG_A5XX_RBBM_STATUS) & ~A5XX_RBBM_STATUS_HI_BUSY)
1062 return false;
1063
1064 /*
1065 * Nearly every abnormality ends up pausing the GPU and triggering a
1066 * fault so we can safely just watch for this one interrupt to fire
1067 */
1068 return !(gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS) &
1069 A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT);
1070 }
1071
a5xx_idle(struct msm_gpu * gpu,struct msm_ringbuffer * ring)1072 bool a5xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
1073 {
1074 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1075 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1076
1077 if (ring != a5xx_gpu->cur_ring) {
1078 WARN(1, "Tried to idle a non-current ringbuffer\n");
1079 return false;
1080 }
1081
1082 /* wait for CP to drain ringbuffer: */
1083 if (!adreno_idle(gpu, ring))
1084 return false;
1085
1086 if (spin_until(_a5xx_check_idle(gpu))) {
1087 DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n",
1088 gpu->name, __builtin_return_address(0),
1089 gpu_read(gpu, REG_A5XX_RBBM_STATUS),
1090 gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS),
1091 gpu_read(gpu, REG_A5XX_CP_RB_RPTR),
1092 gpu_read(gpu, REG_A5XX_CP_RB_WPTR));
1093 return false;
1094 }
1095
1096 return true;
1097 }
1098
a5xx_fault_handler(void * arg,unsigned long iova,int flags,void * data)1099 static int a5xx_fault_handler(void *arg, unsigned long iova, int flags, void *data)
1100 {
1101 struct msm_gpu *gpu = arg;
1102 pr_warn_ratelimited("*** gpu fault: iova=%08lx, flags=%d (%u,%u,%u,%u)\n",
1103 iova, flags,
1104 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(4)),
1105 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(5)),
1106 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(6)),
1107 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(7)));
1108
1109 return 0;
1110 }
1111
a5xx_cp_err_irq(struct msm_gpu * gpu)1112 static void a5xx_cp_err_irq(struct msm_gpu *gpu)
1113 {
1114 u32 status = gpu_read(gpu, REG_A5XX_CP_INTERRUPT_STATUS);
1115
1116 if (status & A5XX_CP_INT_CP_OPCODE_ERROR) {
1117 u32 val;
1118
1119 gpu_write(gpu, REG_A5XX_CP_PFP_STAT_ADDR, 0);
1120
1121 /*
1122 * REG_A5XX_CP_PFP_STAT_DATA is indexed, and we want index 1 so
1123 * read it twice
1124 */
1125
1126 gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
1127 val = gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
1128
1129 dev_err_ratelimited(gpu->dev->dev, "CP | opcode error | possible opcode=0x%8.8X\n",
1130 val);
1131 }
1132
1133 if (status & A5XX_CP_INT_CP_HW_FAULT_ERROR)
1134 dev_err_ratelimited(gpu->dev->dev, "CP | HW fault | status=0x%8.8X\n",
1135 gpu_read(gpu, REG_A5XX_CP_HW_FAULT));
1136
1137 if (status & A5XX_CP_INT_CP_DMA_ERROR)
1138 dev_err_ratelimited(gpu->dev->dev, "CP | DMA error\n");
1139
1140 if (status & A5XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) {
1141 u32 val = gpu_read(gpu, REG_A5XX_CP_PROTECT_STATUS);
1142
1143 dev_err_ratelimited(gpu->dev->dev,
1144 "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n",
1145 val & (1 << 24) ? "WRITE" : "READ",
1146 (val & 0xFFFFF) >> 2, val);
1147 }
1148
1149 if (status & A5XX_CP_INT_CP_AHB_ERROR) {
1150 u32 status = gpu_read(gpu, REG_A5XX_CP_AHB_FAULT);
1151 const char *access[16] = { "reserved", "reserved",
1152 "timestamp lo", "timestamp hi", "pfp read", "pfp write",
1153 "", "", "me read", "me write", "", "", "crashdump read",
1154 "crashdump write" };
1155
1156 dev_err_ratelimited(gpu->dev->dev,
1157 "CP | AHB error | addr=%X access=%s error=%d | status=0x%8.8X\n",
1158 status & 0xFFFFF, access[(status >> 24) & 0xF],
1159 (status & (1 << 31)), status);
1160 }
1161 }
1162
a5xx_rbbm_err_irq(struct msm_gpu * gpu,u32 status)1163 static void a5xx_rbbm_err_irq(struct msm_gpu *gpu, u32 status)
1164 {
1165 if (status & A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR) {
1166 u32 val = gpu_read(gpu, REG_A5XX_RBBM_AHB_ERROR_STATUS);
1167
1168 dev_err_ratelimited(gpu->dev->dev,
1169 "RBBM | AHB bus error | %s | addr=0x%X | ports=0x%X:0x%X\n",
1170 val & (1 << 28) ? "WRITE" : "READ",
1171 (val & 0xFFFFF) >> 2, (val >> 20) & 0x3,
1172 (val >> 24) & 0xF);
1173
1174 /* Clear the error */
1175 gpu_write(gpu, REG_A5XX_RBBM_AHB_CMD, (1 << 4));
1176
1177 /* Clear the interrupt */
1178 gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,
1179 A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR);
1180 }
1181
1182 if (status & A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT)
1183 dev_err_ratelimited(gpu->dev->dev, "RBBM | AHB transfer timeout\n");
1184
1185 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT)
1186 dev_err_ratelimited(gpu->dev->dev, "RBBM | ME master split | status=0x%X\n",
1187 gpu_read(gpu, REG_A5XX_RBBM_AHB_ME_SPLIT_STATUS));
1188
1189 if (status & A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT)
1190 dev_err_ratelimited(gpu->dev->dev, "RBBM | PFP master split | status=0x%X\n",
1191 gpu_read(gpu, REG_A5XX_RBBM_AHB_PFP_SPLIT_STATUS));
1192
1193 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT)
1194 dev_err_ratelimited(gpu->dev->dev, "RBBM | ETS master split | status=0x%X\n",
1195 gpu_read(gpu, REG_A5XX_RBBM_AHB_ETS_SPLIT_STATUS));
1196
1197 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
1198 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB ASYNC overflow\n");
1199
1200 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
1201 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB bus overflow\n");
1202 }
1203
a5xx_uche_err_irq(struct msm_gpu * gpu)1204 static void a5xx_uche_err_irq(struct msm_gpu *gpu)
1205 {
1206 uint64_t addr = (uint64_t) gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_HI);
1207
1208 addr |= gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_LO);
1209
1210 dev_err_ratelimited(gpu->dev->dev, "UCHE | Out of bounds access | addr=0x%llX\n",
1211 addr);
1212 }
1213
a5xx_gpmu_err_irq(struct msm_gpu * gpu)1214 static void a5xx_gpmu_err_irq(struct msm_gpu *gpu)
1215 {
1216 dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n");
1217 }
1218
a5xx_fault_detect_irq(struct msm_gpu * gpu)1219 static void a5xx_fault_detect_irq(struct msm_gpu *gpu)
1220 {
1221 struct drm_device *dev = gpu->dev;
1222 struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);
1223
1224 /*
1225 * If stalled on SMMU fault, we could trip the GPU's hang detection,
1226 * but the fault handler will trigger the devcore dump, and we want
1227 * to otherwise resume normally rather than killing the submit, so
1228 * just bail.
1229 */
1230 if (gpu_read(gpu, REG_A5XX_RBBM_STATUS3) & BIT(24))
1231 return;
1232
1233 DRM_DEV_ERROR(dev->dev, "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n",
1234 ring ? ring->id : -1, ring ? ring->fctx->last_fence : 0,
1235 gpu_read(gpu, REG_A5XX_RBBM_STATUS),
1236 gpu_read(gpu, REG_A5XX_CP_RB_RPTR),
1237 gpu_read(gpu, REG_A5XX_CP_RB_WPTR),
1238 gpu_read64(gpu, REG_A5XX_CP_IB1_BASE),
1239 gpu_read(gpu, REG_A5XX_CP_IB1_BUFSZ),
1240 gpu_read64(gpu, REG_A5XX_CP_IB2_BASE),
1241 gpu_read(gpu, REG_A5XX_CP_IB2_BUFSZ));
1242
1243 /* Turn off the hangcheck timer to keep it from bothering us */
1244 del_timer(&gpu->hangcheck_timer);
1245
1246 kthread_queue_work(gpu->worker, &gpu->recover_work);
1247 }
1248
1249 #define RBBM_ERROR_MASK \
1250 (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
1251 A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
1252 A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
1253 A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
1254 A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
1255 A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
1256
a5xx_irq(struct msm_gpu * gpu)1257 static irqreturn_t a5xx_irq(struct msm_gpu *gpu)
1258 {
1259 struct msm_drm_private *priv = gpu->dev->dev_private;
1260 u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS);
1261
1262 /*
1263 * Clear all the interrupts except RBBM_AHB_ERROR - if we clear it
1264 * before the source is cleared the interrupt will storm.
1265 */
1266 gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,
1267 status & ~A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR);
1268
1269 if (priv->disable_err_irq) {
1270 status &= A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS |
1271 A5XX_RBBM_INT_0_MASK_CP_SW;
1272 }
1273
1274 /* Pass status to a5xx_rbbm_err_irq because we've already cleared it */
1275 if (status & RBBM_ERROR_MASK)
1276 a5xx_rbbm_err_irq(gpu, status);
1277
1278 if (status & A5XX_RBBM_INT_0_MASK_CP_HW_ERROR)
1279 a5xx_cp_err_irq(gpu);
1280
1281 if (status & A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT)
1282 a5xx_fault_detect_irq(gpu);
1283
1284 if (status & A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
1285 a5xx_uche_err_irq(gpu);
1286
1287 if (status & A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
1288 a5xx_gpmu_err_irq(gpu);
1289
1290 if (status & A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS) {
1291 a5xx_preempt_trigger(gpu);
1292 msm_gpu_retire(gpu);
1293 }
1294
1295 if (status & A5XX_RBBM_INT_0_MASK_CP_SW)
1296 a5xx_preempt_irq(gpu);
1297
1298 return IRQ_HANDLED;
1299 }
1300
1301 static const u32 a5xx_registers[] = {
1302 0x0000, 0x0002, 0x0004, 0x0020, 0x0022, 0x0026, 0x0029, 0x002B,
1303 0x002E, 0x0035, 0x0038, 0x0042, 0x0044, 0x0044, 0x0047, 0x0095,
1304 0x0097, 0x00BB, 0x03A0, 0x0464, 0x0469, 0x046F, 0x04D2, 0x04D3,
1305 0x04E0, 0x0533, 0x0540, 0x0555, 0x0800, 0x081A, 0x081F, 0x0841,
1306 0x0860, 0x0860, 0x0880, 0x08A0, 0x0B00, 0x0B12, 0x0B15, 0x0B28,
1307 0x0B78, 0x0B7F, 0x0BB0, 0x0BBD, 0x0BC0, 0x0BC6, 0x0BD0, 0x0C53,
1308 0x0C60, 0x0C61, 0x0C80, 0x0C82, 0x0C84, 0x0C85, 0x0C90, 0x0C98,
1309 0x0CA0, 0x0CA0, 0x0CB0, 0x0CB2, 0x2180, 0x2185, 0x2580, 0x2585,
1310 0x0CC1, 0x0CC1, 0x0CC4, 0x0CC7, 0x0CCC, 0x0CCC, 0x0CD0, 0x0CD8,
1311 0x0CE0, 0x0CE5, 0x0CE8, 0x0CE8, 0x0CEC, 0x0CF1, 0x0CFB, 0x0D0E,
1312 0x2100, 0x211E, 0x2140, 0x2145, 0x2500, 0x251E, 0x2540, 0x2545,
1313 0x0D10, 0x0D17, 0x0D20, 0x0D23, 0x0D30, 0x0D30, 0x20C0, 0x20C0,
1314 0x24C0, 0x24C0, 0x0E40, 0x0E43, 0x0E4A, 0x0E4A, 0x0E50, 0x0E57,
1315 0x0E60, 0x0E7C, 0x0E80, 0x0E8E, 0x0E90, 0x0E96, 0x0EA0, 0x0EA8,
1316 0x0EB0, 0x0EB2, 0xE140, 0xE147, 0xE150, 0xE187, 0xE1A0, 0xE1A9,
1317 0xE1B0, 0xE1B6, 0xE1C0, 0xE1C7, 0xE1D0, 0xE1D1, 0xE200, 0xE201,
1318 0xE210, 0xE21C, 0xE240, 0xE268, 0xE000, 0xE006, 0xE010, 0xE09A,
1319 0xE0A0, 0xE0A4, 0xE0AA, 0xE0EB, 0xE100, 0xE105, 0xE380, 0xE38F,
1320 0xE3B0, 0xE3B0, 0xE400, 0xE405, 0xE408, 0xE4E9, 0xE4F0, 0xE4F0,
1321 0xE280, 0xE280, 0xE282, 0xE2A3, 0xE2A5, 0xE2C2, 0xE940, 0xE947,
1322 0xE950, 0xE987, 0xE9A0, 0xE9A9, 0xE9B0, 0xE9B6, 0xE9C0, 0xE9C7,
1323 0xE9D0, 0xE9D1, 0xEA00, 0xEA01, 0xEA10, 0xEA1C, 0xEA40, 0xEA68,
1324 0xE800, 0xE806, 0xE810, 0xE89A, 0xE8A0, 0xE8A4, 0xE8AA, 0xE8EB,
1325 0xE900, 0xE905, 0xEB80, 0xEB8F, 0xEBB0, 0xEBB0, 0xEC00, 0xEC05,
1326 0xEC08, 0xECE9, 0xECF0, 0xECF0, 0xEA80, 0xEA80, 0xEA82, 0xEAA3,
1327 0xEAA5, 0xEAC2, 0xA800, 0xA800, 0xA820, 0xA828, 0xA840, 0xA87D,
1328 0XA880, 0xA88D, 0xA890, 0xA8A3, 0xA8D0, 0xA8D8, 0xA8E0, 0xA8F5,
1329 0xAC60, 0xAC60, ~0,
1330 };
1331
a5xx_dump(struct msm_gpu * gpu)1332 static void a5xx_dump(struct msm_gpu *gpu)
1333 {
1334 DRM_DEV_INFO(gpu->dev->dev, "status: %08x\n",
1335 gpu_read(gpu, REG_A5XX_RBBM_STATUS));
1336 adreno_dump(gpu);
1337 }
1338
a5xx_pm_resume(struct msm_gpu * gpu)1339 static int a5xx_pm_resume(struct msm_gpu *gpu)
1340 {
1341 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1342 int ret;
1343
1344 /* Turn on the core power */
1345 ret = msm_gpu_pm_resume(gpu);
1346 if (ret)
1347 return ret;
1348
1349 /* Adreno 506, 508, 509, 510, 512 needs manual RBBM sus/res control */
1350 if (!(adreno_is_a530(adreno_gpu) || adreno_is_a540(adreno_gpu))) {
1351 /* Halt the sp_input_clk at HM level */
1352 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0x00000055);
1353 a5xx_set_hwcg(gpu, true);
1354 /* Turn on sp_input_clk at HM level */
1355 gpu_rmw(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0xff, 0);
1356 return 0;
1357 }
1358
1359 /* Turn the RBCCU domain first to limit the chances of voltage droop */
1360 gpu_write(gpu, REG_A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000);
1361
1362 /* Wait 3 usecs before polling */
1363 udelay(3);
1364
1365 ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS,
1366 (1 << 20), (1 << 20));
1367 if (ret) {
1368 DRM_ERROR("%s: timeout waiting for RBCCU GDSC enable: %X\n",
1369 gpu->name,
1370 gpu_read(gpu, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS));
1371 return ret;
1372 }
1373
1374 /* Turn on the SP domain */
1375 gpu_write(gpu, REG_A5XX_GPMU_SP_POWER_CNTL, 0x778000);
1376 ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_SP_PWR_CLK_STATUS,
1377 (1 << 20), (1 << 20));
1378 if (ret)
1379 DRM_ERROR("%s: timeout waiting for SP GDSC enable\n",
1380 gpu->name);
1381
1382 return ret;
1383 }
1384
a5xx_pm_suspend(struct msm_gpu * gpu)1385 static int a5xx_pm_suspend(struct msm_gpu *gpu)
1386 {
1387 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1388 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1389 u32 mask = 0xf;
1390 int i, ret;
1391
1392 /* A506, A508, A510 have 3 XIN ports in VBIF */
1393 if (adreno_is_a506(adreno_gpu) || adreno_is_a508(adreno_gpu) ||
1394 adreno_is_a510(adreno_gpu))
1395 mask = 0x7;
1396
1397 /* Clear the VBIF pipe before shutting down */
1398 gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, mask);
1399 spin_until((gpu_read(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL1) &
1400 mask) == mask);
1401
1402 gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0);
1403
1404 /*
1405 * Reset the VBIF before power collapse to avoid issue with FIFO
1406 * entries on Adreno A510 and A530 (the others will tend to lock up)
1407 */
1408 if (adreno_is_a510(adreno_gpu) || adreno_is_a530(adreno_gpu)) {
1409 gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x003C0000);
1410 gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x00000000);
1411 }
1412
1413 ret = msm_gpu_pm_suspend(gpu);
1414 if (ret)
1415 return ret;
1416
1417 if (a5xx_gpu->has_whereami)
1418 for (i = 0; i < gpu->nr_rings; i++)
1419 a5xx_gpu->shadow[i] = 0;
1420
1421 return 0;
1422 }
1423
a5xx_get_timestamp(struct msm_gpu * gpu,uint64_t * value)1424 static int a5xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
1425 {
1426 *value = gpu_read64(gpu, REG_A5XX_RBBM_ALWAYSON_COUNTER_LO);
1427
1428 return 0;
1429 }
1430
1431 struct a5xx_crashdumper {
1432 void *ptr;
1433 struct drm_gem_object *bo;
1434 u64 iova;
1435 };
1436
1437 struct a5xx_gpu_state {
1438 struct msm_gpu_state base;
1439 u32 *hlsqregs;
1440 };
1441
a5xx_crashdumper_init(struct msm_gpu * gpu,struct a5xx_crashdumper * dumper)1442 static int a5xx_crashdumper_init(struct msm_gpu *gpu,
1443 struct a5xx_crashdumper *dumper)
1444 {
1445 dumper->ptr = msm_gem_kernel_new(gpu->dev,
1446 SZ_1M, MSM_BO_WC, gpu->aspace,
1447 &dumper->bo, &dumper->iova);
1448
1449 if (!IS_ERR(dumper->ptr))
1450 msm_gem_object_set_name(dumper->bo, "crashdump");
1451
1452 return PTR_ERR_OR_ZERO(dumper->ptr);
1453 }
1454
a5xx_crashdumper_run(struct msm_gpu * gpu,struct a5xx_crashdumper * dumper)1455 static int a5xx_crashdumper_run(struct msm_gpu *gpu,
1456 struct a5xx_crashdumper *dumper)
1457 {
1458 u32 val;
1459
1460 if (IS_ERR_OR_NULL(dumper->ptr))
1461 return -EINVAL;
1462
1463 gpu_write64(gpu, REG_A5XX_CP_CRASH_SCRIPT_BASE_LO, dumper->iova);
1464
1465 gpu_write(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, 1);
1466
1467 return gpu_poll_timeout(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, val,
1468 val & 0x04, 100, 10000);
1469 }
1470
1471 /*
1472 * These are a list of the registers that need to be read through the HLSQ
1473 * aperture through the crashdumper. These are not nominally accessible from
1474 * the CPU on a secure platform.
1475 */
1476 static const struct {
1477 u32 type;
1478 u32 regoffset;
1479 u32 count;
1480 } a5xx_hlsq_aperture_regs[] = {
1481 { 0x35, 0xe00, 0x32 }, /* HSLQ non-context */
1482 { 0x31, 0x2080, 0x1 }, /* HLSQ 2D context 0 */
1483 { 0x33, 0x2480, 0x1 }, /* HLSQ 2D context 1 */
1484 { 0x32, 0xe780, 0x62 }, /* HLSQ 3D context 0 */
1485 { 0x34, 0xef80, 0x62 }, /* HLSQ 3D context 1 */
1486 { 0x3f, 0x0ec0, 0x40 }, /* SP non-context */
1487 { 0x3d, 0x2040, 0x1 }, /* SP 2D context 0 */
1488 { 0x3b, 0x2440, 0x1 }, /* SP 2D context 1 */
1489 { 0x3e, 0xe580, 0x170 }, /* SP 3D context 0 */
1490 { 0x3c, 0xed80, 0x170 }, /* SP 3D context 1 */
1491 { 0x3a, 0x0f00, 0x1c }, /* TP non-context */
1492 { 0x38, 0x2000, 0xa }, /* TP 2D context 0 */
1493 { 0x36, 0x2400, 0xa }, /* TP 2D context 1 */
1494 { 0x39, 0xe700, 0x80 }, /* TP 3D context 0 */
1495 { 0x37, 0xef00, 0x80 }, /* TP 3D context 1 */
1496 };
1497
a5xx_gpu_state_get_hlsq_regs(struct msm_gpu * gpu,struct a5xx_gpu_state * a5xx_state)1498 static void a5xx_gpu_state_get_hlsq_regs(struct msm_gpu *gpu,
1499 struct a5xx_gpu_state *a5xx_state)
1500 {
1501 struct a5xx_crashdumper dumper = { 0 };
1502 u32 offset, count = 0;
1503 u64 *ptr;
1504 int i;
1505
1506 if (a5xx_crashdumper_init(gpu, &dumper))
1507 return;
1508
1509 /* The script will be written at offset 0 */
1510 ptr = dumper.ptr;
1511
1512 /* Start writing the data at offset 256k */
1513 offset = dumper.iova + (256 * SZ_1K);
1514
1515 /* Count how many additional registers to get from the HLSQ aperture */
1516 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++)
1517 count += a5xx_hlsq_aperture_regs[i].count;
1518
1519 a5xx_state->hlsqregs = kcalloc(count, sizeof(u32), GFP_KERNEL);
1520 if (!a5xx_state->hlsqregs)
1521 return;
1522
1523 /* Build the crashdump script */
1524 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) {
1525 u32 type = a5xx_hlsq_aperture_regs[i].type;
1526 u32 c = a5xx_hlsq_aperture_regs[i].count;
1527
1528 /* Write the register to select the desired bank */
1529 *ptr++ = ((u64) type << 8);
1530 *ptr++ = (((u64) REG_A5XX_HLSQ_DBG_READ_SEL) << 44) |
1531 (1 << 21) | 1;
1532
1533 *ptr++ = offset;
1534 *ptr++ = (((u64) REG_A5XX_HLSQ_DBG_AHB_READ_APERTURE) << 44)
1535 | c;
1536
1537 offset += c * sizeof(u32);
1538 }
1539
1540 /* Write two zeros to close off the script */
1541 *ptr++ = 0;
1542 *ptr++ = 0;
1543
1544 if (a5xx_crashdumper_run(gpu, &dumper)) {
1545 kfree(a5xx_state->hlsqregs);
1546 msm_gem_kernel_put(dumper.bo, gpu->aspace);
1547 return;
1548 }
1549
1550 /* Copy the data from the crashdumper to the state */
1551 memcpy(a5xx_state->hlsqregs, dumper.ptr + (256 * SZ_1K),
1552 count * sizeof(u32));
1553
1554 msm_gem_kernel_put(dumper.bo, gpu->aspace);
1555 }
1556
a5xx_gpu_state_get(struct msm_gpu * gpu)1557 static struct msm_gpu_state *a5xx_gpu_state_get(struct msm_gpu *gpu)
1558 {
1559 struct a5xx_gpu_state *a5xx_state = kzalloc(sizeof(*a5xx_state),
1560 GFP_KERNEL);
1561 bool stalled = !!(gpu_read(gpu, REG_A5XX_RBBM_STATUS3) & BIT(24));
1562
1563 if (!a5xx_state)
1564 return ERR_PTR(-ENOMEM);
1565
1566 /* Temporarily disable hardware clock gating before reading the hw */
1567 a5xx_set_hwcg(gpu, false);
1568
1569 /* First get the generic state from the adreno core */
1570 adreno_gpu_state_get(gpu, &(a5xx_state->base));
1571
1572 a5xx_state->base.rbbm_status = gpu_read(gpu, REG_A5XX_RBBM_STATUS);
1573
1574 /*
1575 * Get the HLSQ regs with the help of the crashdumper, but only if
1576 * we are not stalled in an iommu fault (in which case the crashdumper
1577 * would not have access to memory)
1578 */
1579 if (!stalled)
1580 a5xx_gpu_state_get_hlsq_regs(gpu, a5xx_state);
1581
1582 a5xx_set_hwcg(gpu, true);
1583
1584 return &a5xx_state->base;
1585 }
1586
a5xx_gpu_state_destroy(struct kref * kref)1587 static void a5xx_gpu_state_destroy(struct kref *kref)
1588 {
1589 struct msm_gpu_state *state = container_of(kref,
1590 struct msm_gpu_state, ref);
1591 struct a5xx_gpu_state *a5xx_state = container_of(state,
1592 struct a5xx_gpu_state, base);
1593
1594 kfree(a5xx_state->hlsqregs);
1595
1596 adreno_gpu_state_destroy(state);
1597 kfree(a5xx_state);
1598 }
1599
a5xx_gpu_state_put(struct msm_gpu_state * state)1600 static int a5xx_gpu_state_put(struct msm_gpu_state *state)
1601 {
1602 if (IS_ERR_OR_NULL(state))
1603 return 1;
1604
1605 return kref_put(&state->ref, a5xx_gpu_state_destroy);
1606 }
1607
1608
1609 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
a5xx_show(struct msm_gpu * gpu,struct msm_gpu_state * state,struct drm_printer * p)1610 static void a5xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
1611 struct drm_printer *p)
1612 {
1613 int i, j;
1614 u32 pos = 0;
1615 struct a5xx_gpu_state *a5xx_state = container_of(state,
1616 struct a5xx_gpu_state, base);
1617
1618 if (IS_ERR_OR_NULL(state))
1619 return;
1620
1621 adreno_show(gpu, state, p);
1622
1623 /* Dump the additional a5xx HLSQ registers */
1624 if (!a5xx_state->hlsqregs)
1625 return;
1626
1627 drm_printf(p, "registers-hlsq:\n");
1628
1629 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) {
1630 u32 o = a5xx_hlsq_aperture_regs[i].regoffset;
1631 u32 c = a5xx_hlsq_aperture_regs[i].count;
1632
1633 for (j = 0; j < c; j++, pos++, o++) {
1634 /*
1635 * To keep the crashdump simple we pull the entire range
1636 * for each register type but not all of the registers
1637 * in the range are valid. Fortunately invalid registers
1638 * stick out like a sore thumb with a value of
1639 * 0xdeadbeef
1640 */
1641 if (a5xx_state->hlsqregs[pos] == 0xdeadbeef)
1642 continue;
1643
1644 drm_printf(p, " - { offset: 0x%04x, value: 0x%08x }\n",
1645 o << 2, a5xx_state->hlsqregs[pos]);
1646 }
1647 }
1648 }
1649 #endif
1650
a5xx_active_ring(struct msm_gpu * gpu)1651 static struct msm_ringbuffer *a5xx_active_ring(struct msm_gpu *gpu)
1652 {
1653 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1654 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1655
1656 return a5xx_gpu->cur_ring;
1657 }
1658
a5xx_gpu_busy(struct msm_gpu * gpu,unsigned long * out_sample_rate)1659 static u64 a5xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)
1660 {
1661 u64 busy_cycles;
1662
1663 busy_cycles = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_0_LO);
1664 *out_sample_rate = clk_get_rate(gpu->core_clk);
1665
1666 return busy_cycles;
1667 }
1668
a5xx_get_rptr(struct msm_gpu * gpu,struct msm_ringbuffer * ring)1669 static uint32_t a5xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
1670 {
1671 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1672 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1673
1674 if (a5xx_gpu->has_whereami)
1675 return a5xx_gpu->shadow[ring->id];
1676
1677 return ring->memptrs->rptr = gpu_read(gpu, REG_A5XX_CP_RB_RPTR);
1678 }
1679
1680 static const struct adreno_gpu_funcs funcs = {
1681 .base = {
1682 .get_param = adreno_get_param,
1683 .set_param = adreno_set_param,
1684 .hw_init = a5xx_hw_init,
1685 .pm_suspend = a5xx_pm_suspend,
1686 .pm_resume = a5xx_pm_resume,
1687 .recover = a5xx_recover,
1688 .submit = a5xx_submit,
1689 .active_ring = a5xx_active_ring,
1690 .irq = a5xx_irq,
1691 .destroy = a5xx_destroy,
1692 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
1693 .show = a5xx_show,
1694 #endif
1695 #if defined(CONFIG_DEBUG_FS)
1696 .debugfs_init = a5xx_debugfs_init,
1697 #endif
1698 .gpu_busy = a5xx_gpu_busy,
1699 .gpu_state_get = a5xx_gpu_state_get,
1700 .gpu_state_put = a5xx_gpu_state_put,
1701 .create_address_space = adreno_iommu_create_address_space,
1702 .get_rptr = a5xx_get_rptr,
1703 },
1704 .get_timestamp = a5xx_get_timestamp,
1705 };
1706
check_speed_bin(struct device * dev)1707 static void check_speed_bin(struct device *dev)
1708 {
1709 struct nvmem_cell *cell;
1710 u32 val;
1711
1712 /*
1713 * If the OPP table specifies a opp-supported-hw property then we have
1714 * to set something with dev_pm_opp_set_supported_hw() or the table
1715 * doesn't get populated so pick an arbitrary value that should
1716 * ensure the default frequencies are selected but not conflict with any
1717 * actual bins
1718 */
1719 val = 0x80;
1720
1721 cell = nvmem_cell_get(dev, "speed_bin");
1722
1723 if (!IS_ERR(cell)) {
1724 void *buf = nvmem_cell_read(cell, NULL);
1725
1726 if (!IS_ERR(buf)) {
1727 u8 bin = *((u8 *) buf);
1728
1729 val = (1 << bin);
1730 kfree(buf);
1731 }
1732
1733 nvmem_cell_put(cell);
1734 }
1735
1736 devm_pm_opp_set_supported_hw(dev, &val, 1);
1737 }
1738
a5xx_gpu_init(struct drm_device * dev)1739 struct msm_gpu *a5xx_gpu_init(struct drm_device *dev)
1740 {
1741 struct msm_drm_private *priv = dev->dev_private;
1742 struct platform_device *pdev = priv->gpu_pdev;
1743 struct adreno_platform_config *config = pdev->dev.platform_data;
1744 struct a5xx_gpu *a5xx_gpu = NULL;
1745 struct adreno_gpu *adreno_gpu;
1746 struct msm_gpu *gpu;
1747 unsigned int nr_rings;
1748 int ret;
1749
1750 if (!pdev) {
1751 DRM_DEV_ERROR(dev->dev, "No A5XX device is defined\n");
1752 return ERR_PTR(-ENXIO);
1753 }
1754
1755 a5xx_gpu = kzalloc(sizeof(*a5xx_gpu), GFP_KERNEL);
1756 if (!a5xx_gpu)
1757 return ERR_PTR(-ENOMEM);
1758
1759 adreno_gpu = &a5xx_gpu->base;
1760 gpu = &adreno_gpu->base;
1761
1762 adreno_gpu->registers = a5xx_registers;
1763
1764 a5xx_gpu->lm_leakage = 0x4E001A;
1765
1766 check_speed_bin(&pdev->dev);
1767
1768 nr_rings = 4;
1769
1770 if (adreno_cmp_rev(ADRENO_REV(5, 1, 0, ANY_ID), config->rev))
1771 nr_rings = 1;
1772
1773 ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, nr_rings);
1774 if (ret) {
1775 a5xx_destroy(&(a5xx_gpu->base.base));
1776 return ERR_PTR(ret);
1777 }
1778
1779 if (gpu->aspace)
1780 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, a5xx_fault_handler);
1781
1782 /* Set up the preemption specific bits and pieces for each ringbuffer */
1783 a5xx_preempt_init(gpu);
1784
1785 return gpu;
1786 }
1787