1 /*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2014-2018 Intel Corporation
5 */
6
7 #include "i915_drv.h"
8 #include "intel_context.h"
9 #include "intel_engine_pm.h"
10 #include "intel_gt.h"
11 #include "intel_ring.h"
12 #include "intel_workarounds.h"
13
14 /**
15 * DOC: Hardware workarounds
16 *
17 * This file is intended as a central place to implement most [1]_ of the
18 * required workarounds for hardware to work as originally intended. They fall
19 * in five basic categories depending on how/when they are applied:
20 *
21 * - Workarounds that touch registers that are saved/restored to/from the HW
22 * context image. The list is emitted (via Load Register Immediate commands)
23 * everytime a new context is created.
24 * - GT workarounds. The list of these WAs is applied whenever these registers
25 * revert to default values (on GPU reset, suspend/resume [2]_, etc..).
26 * - Display workarounds. The list is applied during display clock-gating
27 * initialization.
28 * - Workarounds that whitelist a privileged register, so that UMDs can manage
29 * them directly. This is just a special case of a MMMIO workaround (as we
30 * write the list of these to/be-whitelisted registers to some special HW
31 * registers).
32 * - Workaround batchbuffers, that get executed automatically by the hardware
33 * on every HW context restore.
34 *
35 * .. [1] Please notice that there are other WAs that, due to their nature,
36 * cannot be applied from a central place. Those are peppered around the rest
37 * of the code, as needed.
38 *
39 * .. [2] Technically, some registers are powercontext saved & restored, so they
40 * survive a suspend/resume. In practice, writing them again is not too
41 * costly and simplifies things. We can revisit this in the future.
42 *
43 * Layout
44 * ~~~~~~
45 *
46 * Keep things in this file ordered by WA type, as per the above (context, GT,
47 * display, register whitelist, batchbuffer). Then, inside each type, keep the
48 * following order:
49 *
50 * - Infrastructure functions and macros
51 * - WAs per platform in standard gen/chrono order
52 * - Public functions to init or apply the given workaround type.
53 */
54
55 /*
56 * KBL revision ID ordering is bizarre; higher revision ID's map to lower
57 * steppings in some cases. So rather than test against the revision ID
58 * directly, let's map that into our own range of increasing ID's that we
59 * can test against in a regular manner.
60 */
61
62 const struct i915_rev_steppings kbl_revids[] = {
63 [0] = { .gt_stepping = KBL_REVID_A0, .disp_stepping = KBL_REVID_A0 },
64 [1] = { .gt_stepping = KBL_REVID_B0, .disp_stepping = KBL_REVID_B0 },
65 [2] = { .gt_stepping = KBL_REVID_C0, .disp_stepping = KBL_REVID_B0 },
66 [3] = { .gt_stepping = KBL_REVID_D0, .disp_stepping = KBL_REVID_B0 },
67 [4] = { .gt_stepping = KBL_REVID_F0, .disp_stepping = KBL_REVID_C0 },
68 [5] = { .gt_stepping = KBL_REVID_C0, .disp_stepping = KBL_REVID_B1 },
69 [6] = { .gt_stepping = KBL_REVID_D1, .disp_stepping = KBL_REVID_B1 },
70 [7] = { .gt_stepping = KBL_REVID_G0, .disp_stepping = KBL_REVID_C0 },
71 };
72
73 const struct i915_rev_steppings tgl_uy_revids[] = {
74 [0] = { .gt_stepping = TGL_REVID_A0, .disp_stepping = TGL_REVID_A0 },
75 [1] = { .gt_stepping = TGL_REVID_B0, .disp_stepping = TGL_REVID_C0 },
76 [2] = { .gt_stepping = TGL_REVID_B1, .disp_stepping = TGL_REVID_C0 },
77 [3] = { .gt_stepping = TGL_REVID_C0, .disp_stepping = TGL_REVID_D0 },
78 };
79
80 /* Same GT stepping between tgl_uy_revids and tgl_revids don't mean the same HW */
81 const struct i915_rev_steppings tgl_revids[] = {
82 [0] = { .gt_stepping = TGL_REVID_A0, .disp_stepping = TGL_REVID_B0 },
83 [1] = { .gt_stepping = TGL_REVID_B0, .disp_stepping = TGL_REVID_D0 },
84 };
85
wa_init_start(struct i915_wa_list * wal,const char * name,const char * engine_name)86 static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name)
87 {
88 wal->name = name;
89 wal->engine_name = engine_name;
90 }
91
92 #define WA_LIST_CHUNK (1 << 4)
93
wa_init_finish(struct i915_wa_list * wal)94 static void wa_init_finish(struct i915_wa_list *wal)
95 {
96 /* Trim unused entries. */
97 if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) {
98 struct i915_wa *list = kmemdup(wal->list,
99 wal->count * sizeof(*list),
100 GFP_KERNEL);
101
102 if (list) {
103 kfree(wal->list);
104 wal->list = list;
105 }
106 }
107
108 if (!wal->count)
109 return;
110
111 DRM_DEBUG_DRIVER("Initialized %u %s workarounds on %s\n",
112 wal->wa_count, wal->name, wal->engine_name);
113 }
114
_wa_add(struct i915_wa_list * wal,const struct i915_wa * wa)115 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
116 {
117 unsigned int addr = i915_mmio_reg_offset(wa->reg);
118 unsigned int start = 0, end = wal->count;
119 const unsigned int grow = WA_LIST_CHUNK;
120 struct i915_wa *wa_;
121
122 GEM_BUG_ON(!is_power_of_2(grow));
123
124 if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
125 struct i915_wa *list;
126
127 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa),
128 GFP_KERNEL);
129 if (!list) {
130 DRM_ERROR("No space for workaround init!\n");
131 return;
132 }
133
134 if (wal->list) {
135 memcpy(list, wal->list, sizeof(*wa) * wal->count);
136 kfree(wal->list);
137 }
138
139 wal->list = list;
140 }
141
142 while (start < end) {
143 unsigned int mid = start + (end - start) / 2;
144
145 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
146 start = mid + 1;
147 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
148 end = mid;
149 } else {
150 wa_ = &wal->list[mid];
151
152 if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) {
153 DRM_ERROR("Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n",
154 i915_mmio_reg_offset(wa_->reg),
155 wa_->clr, wa_->set);
156
157 wa_->set &= ~wa->clr;
158 }
159
160 wal->wa_count++;
161 wa_->set |= wa->set;
162 wa_->clr |= wa->clr;
163 wa_->read |= wa->read;
164 return;
165 }
166 }
167
168 wal->wa_count++;
169 wa_ = &wal->list[wal->count++];
170 *wa_ = *wa;
171
172 while (wa_-- > wal->list) {
173 GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==
174 i915_mmio_reg_offset(wa_[1].reg));
175 if (i915_mmio_reg_offset(wa_[1].reg) >
176 i915_mmio_reg_offset(wa_[0].reg))
177 break;
178
179 swap(wa_[1], wa_[0]);
180 }
181 }
182
wa_add(struct i915_wa_list * wal,i915_reg_t reg,u32 clear,u32 set,u32 read_mask)183 static void wa_add(struct i915_wa_list *wal, i915_reg_t reg,
184 u32 clear, u32 set, u32 read_mask)
185 {
186 struct i915_wa wa = {
187 .reg = reg,
188 .clr = clear,
189 .set = set,
190 .read = read_mask,
191 };
192
193 _wa_add(wal, &wa);
194 }
195
196 static void
wa_write_masked_or(struct i915_wa_list * wal,i915_reg_t reg,u32 clear,u32 set)197 wa_write_masked_or(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set)
198 {
199 wa_add(wal, reg, clear, set, clear);
200 }
201
202 static void
wa_write(struct i915_wa_list * wal,i915_reg_t reg,u32 set)203 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
204 {
205 wa_write_masked_or(wal, reg, ~0, set);
206 }
207
208 static void
wa_write_or(struct i915_wa_list * wal,i915_reg_t reg,u32 set)209 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
210 {
211 wa_write_masked_or(wal, reg, set, set);
212 }
213
214 static void
wa_write_clr(struct i915_wa_list * wal,i915_reg_t reg,u32 clr)215 wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr)
216 {
217 wa_write_masked_or(wal, reg, clr, 0);
218 }
219
220 static void
wa_masked_en(struct i915_wa_list * wal,i915_reg_t reg,u32 val)221 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
222 {
223 wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val);
224 }
225
226 static void
wa_masked_dis(struct i915_wa_list * wal,i915_reg_t reg,u32 val)227 wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
228 {
229 wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val);
230 }
231
232 #define WA_SET_BIT_MASKED(addr, mask) \
233 wa_masked_en(wal, (addr), (mask))
234
235 #define WA_CLR_BIT_MASKED(addr, mask) \
236 wa_masked_dis(wal, (addr), (mask))
237
238 #define WA_SET_FIELD_MASKED(addr, mask, value) \
239 wa_write_masked_or(wal, (addr), 0, _MASKED_FIELD((mask), (value)))
240
gen6_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)241 static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine,
242 struct i915_wa_list *wal)
243 {
244 WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
245 }
246
gen7_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)247 static void gen7_ctx_workarounds_init(struct intel_engine_cs *engine,
248 struct i915_wa_list *wal)
249 {
250 WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
251 }
252
gen8_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)253 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine,
254 struct i915_wa_list *wal)
255 {
256 WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
257
258 /* WaDisableAsyncFlipPerfMode:bdw,chv */
259 WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
260
261 /* WaDisablePartialInstShootdown:bdw,chv */
262 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
263 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
264
265 /* Use Force Non-Coherent whenever executing a 3D context. This is a
266 * workaround for for a possible hang in the unlikely event a TLB
267 * invalidation occurs during a PSD flush.
268 */
269 /* WaForceEnableNonCoherent:bdw,chv */
270 /* WaHdcDisableFetchWhenMasked:bdw,chv */
271 WA_SET_BIT_MASKED(HDC_CHICKEN0,
272 HDC_DONOT_FETCH_MEM_WHEN_MASKED |
273 HDC_FORCE_NON_COHERENT);
274
275 /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
276 * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
277 * polygons in the same 8x4 pixel/sample area to be processed without
278 * stalling waiting for the earlier ones to write to Hierarchical Z
279 * buffer."
280 *
281 * This optimization is off by default for BDW and CHV; turn it on.
282 */
283 WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
284
285 /* Wa4x4STCOptimizationDisable:bdw,chv */
286 WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
287
288 /*
289 * BSpec recommends 8x4 when MSAA is used,
290 * however in practice 16x4 seems fastest.
291 *
292 * Note that PS/WM thread counts depend on the WIZ hashing
293 * disable bit, which we don't touch here, but it's good
294 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
295 */
296 WA_SET_FIELD_MASKED(GEN7_GT_MODE,
297 GEN6_WIZ_HASHING_MASK,
298 GEN6_WIZ_HASHING_16x4);
299 }
300
bdw_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)301 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine,
302 struct i915_wa_list *wal)
303 {
304 struct drm_i915_private *i915 = engine->i915;
305
306 gen8_ctx_workarounds_init(engine, wal);
307
308 /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
309 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
310
311 /* WaDisableDopClockGating:bdw
312 *
313 * Also see the related UCGTCL1 write in bdw_init_clock_gating()
314 * to disable EUTC clock gating.
315 */
316 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
317 DOP_CLOCK_GATING_DISABLE);
318
319 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
320 GEN8_SAMPLER_POWER_BYPASS_DIS);
321
322 WA_SET_BIT_MASKED(HDC_CHICKEN0,
323 /* WaForceContextSaveRestoreNonCoherent:bdw */
324 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
325 /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
326 (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
327 }
328
chv_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)329 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine,
330 struct i915_wa_list *wal)
331 {
332 gen8_ctx_workarounds_init(engine, wal);
333
334 /* WaDisableThreadStallDopClockGating:chv */
335 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
336
337 /* Improve HiZ throughput on CHV. */
338 WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
339 }
340
gen9_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)341 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine,
342 struct i915_wa_list *wal)
343 {
344 struct drm_i915_private *i915 = engine->i915;
345
346 if (HAS_LLC(i915)) {
347 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
348 *
349 * Must match Display Engine. See
350 * WaCompressedResourceDisplayNewHashMode.
351 */
352 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
353 GEN9_PBE_COMPRESSED_HASH_SELECTION);
354 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
355 GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
356 }
357
358 /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
359 /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
360 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
361 FLOW_CONTROL_ENABLE |
362 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
363
364 /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
365 /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
366 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
367 GEN9_ENABLE_YV12_BUGFIX |
368 GEN9_ENABLE_GPGPU_PREEMPTION);
369
370 /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
371 /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
372 WA_SET_BIT_MASKED(CACHE_MODE_1,
373 GEN8_4x4_STC_OPTIMIZATION_DISABLE |
374 GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
375
376 /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
377 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
378 GEN9_CCS_TLB_PREFETCH_ENABLE);
379
380 /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
381 WA_SET_BIT_MASKED(HDC_CHICKEN0,
382 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
383 HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
384
385 /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
386 * both tied to WaForceContextSaveRestoreNonCoherent
387 * in some hsds for skl. We keep the tie for all gen9. The
388 * documentation is a bit hazy and so we want to get common behaviour,
389 * even though there is no clear evidence we would need both on kbl/bxt.
390 * This area has been source of system hangs so we play it safe
391 * and mimic the skl regardless of what bspec says.
392 *
393 * Use Force Non-Coherent whenever executing a 3D context. This
394 * is a workaround for a possible hang in the unlikely event
395 * a TLB invalidation occurs during a PSD flush.
396 */
397
398 /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
399 WA_SET_BIT_MASKED(HDC_CHICKEN0,
400 HDC_FORCE_NON_COHERENT);
401
402 /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
403 if (IS_SKYLAKE(i915) ||
404 IS_KABYLAKE(i915) ||
405 IS_COFFEELAKE(i915) ||
406 IS_COMETLAKE(i915))
407 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
408 GEN8_SAMPLER_POWER_BYPASS_DIS);
409
410 /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
411 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
412
413 /*
414 * Supporting preemption with fine-granularity requires changes in the
415 * batch buffer programming. Since we can't break old userspace, we
416 * need to set our default preemption level to safe value. Userspace is
417 * still able to use more fine-grained preemption levels, since in
418 * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
419 * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
420 * not real HW workarounds, but merely a way to start using preemption
421 * while maintaining old contract with userspace.
422 */
423
424 /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
425 WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
426
427 /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
428 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
429 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
430 GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
431
432 /* WaClearHIZ_WM_CHICKEN3:bxt,glk */
433 if (IS_GEN9_LP(i915))
434 WA_SET_BIT_MASKED(GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
435 }
436
skl_tune_iz_hashing(struct intel_engine_cs * engine,struct i915_wa_list * wal)437 static void skl_tune_iz_hashing(struct intel_engine_cs *engine,
438 struct i915_wa_list *wal)
439 {
440 struct intel_gt *gt = engine->gt;
441 u8 vals[3] = { 0, 0, 0 };
442 unsigned int i;
443
444 for (i = 0; i < 3; i++) {
445 u8 ss;
446
447 /*
448 * Only consider slices where one, and only one, subslice has 7
449 * EUs
450 */
451 if (!is_power_of_2(gt->info.sseu.subslice_7eu[i]))
452 continue;
453
454 /*
455 * subslice_7eu[i] != 0 (because of the check above) and
456 * ss_max == 4 (maximum number of subslices possible per slice)
457 *
458 * -> 0 <= ss <= 3;
459 */
460 ss = ffs(gt->info.sseu.subslice_7eu[i]) - 1;
461 vals[i] = 3 - ss;
462 }
463
464 if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
465 return;
466
467 /* Tune IZ hashing. See intel_device_info_runtime_init() */
468 WA_SET_FIELD_MASKED(GEN7_GT_MODE,
469 GEN9_IZ_HASHING_MASK(2) |
470 GEN9_IZ_HASHING_MASK(1) |
471 GEN9_IZ_HASHING_MASK(0),
472 GEN9_IZ_HASHING(2, vals[2]) |
473 GEN9_IZ_HASHING(1, vals[1]) |
474 GEN9_IZ_HASHING(0, vals[0]));
475 }
476
skl_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)477 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine,
478 struct i915_wa_list *wal)
479 {
480 gen9_ctx_workarounds_init(engine, wal);
481 skl_tune_iz_hashing(engine, wal);
482 }
483
bxt_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)484 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine,
485 struct i915_wa_list *wal)
486 {
487 gen9_ctx_workarounds_init(engine, wal);
488
489 /* WaDisableThreadStallDopClockGating:bxt */
490 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
491 STALL_DOP_GATING_DISABLE);
492
493 /* WaToEnableHwFixForPushConstHWBug:bxt */
494 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
495 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
496 }
497
kbl_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)498 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine,
499 struct i915_wa_list *wal)
500 {
501 struct drm_i915_private *i915 = engine->i915;
502
503 gen9_ctx_workarounds_init(engine, wal);
504
505 /* WaToEnableHwFixForPushConstHWBug:kbl */
506 if (IS_KBL_GT_REVID(i915, KBL_REVID_C0, REVID_FOREVER))
507 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
508 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
509
510 /* WaDisableSbeCacheDispatchPortSharing:kbl */
511 WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
512 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
513 }
514
glk_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)515 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine,
516 struct i915_wa_list *wal)
517 {
518 gen9_ctx_workarounds_init(engine, wal);
519
520 /* WaToEnableHwFixForPushConstHWBug:glk */
521 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
522 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
523 }
524
cfl_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)525 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine,
526 struct i915_wa_list *wal)
527 {
528 gen9_ctx_workarounds_init(engine, wal);
529
530 /* WaToEnableHwFixForPushConstHWBug:cfl */
531 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
532 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
533
534 /* WaDisableSbeCacheDispatchPortSharing:cfl */
535 WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
536 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
537 }
538
cnl_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)539 static void cnl_ctx_workarounds_init(struct intel_engine_cs *engine,
540 struct i915_wa_list *wal)
541 {
542 /* WaForceContextSaveRestoreNonCoherent:cnl */
543 WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0,
544 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
545
546 /* WaDisableReplayBufferBankArbitrationOptimization:cnl */
547 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
548 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
549
550 /* WaPushConstantDereferenceHoldDisable:cnl */
551 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
552
553 /* FtrEnableFastAnisoL1BankingFix:cnl */
554 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
555
556 /* WaDisable3DMidCmdPreemption:cnl */
557 WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
558
559 /* WaDisableGPGPUMidCmdPreemption:cnl */
560 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
561 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
562 GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
563
564 /* WaDisableEarlyEOT:cnl */
565 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT);
566 }
567
icl_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)568 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
569 struct i915_wa_list *wal)
570 {
571 struct drm_i915_private *i915 = engine->i915;
572
573 /* WaDisableBankHangMode:icl */
574 wa_write(wal,
575 GEN8_L3CNTLREG,
576 intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) |
577 GEN8_ERRDETBCTRL);
578
579 /* Wa_1604370585:icl (pre-prod)
580 * Formerly known as WaPushConstantDereferenceHoldDisable
581 */
582 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
583 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
584 PUSH_CONSTANT_DEREF_DISABLE);
585
586 /* WaForceEnableNonCoherent:icl
587 * This is not the same workaround as in early Gen9 platforms, where
588 * lacking this could cause system hangs, but coherency performance
589 * overhead is high and only a few compute workloads really need it
590 * (the register is whitelisted in hardware now, so UMDs can opt in
591 * for coherency if they have a good reason).
592 */
593 WA_SET_BIT_MASKED(ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
594
595 /* Wa_2006611047:icl (pre-prod)
596 * Formerly known as WaDisableImprovedTdlClkGating
597 */
598 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
599 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
600 GEN11_TDL_CLOCK_GATING_FIX_DISABLE);
601
602 /* Wa_2006665173:icl (pre-prod) */
603 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
604 WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
605 GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC);
606
607 /* WaEnableFloatBlendOptimization:icl */
608 wa_write_masked_or(wal,
609 GEN10_CACHE_MODE_SS,
610 0, /* write-only, so skip validation */
611 _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE));
612
613 /* WaDisableGPGPUMidThreadPreemption:icl */
614 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
615 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
616 GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
617
618 /* allow headerless messages for preemptible GPGPU context */
619 WA_SET_BIT_MASKED(GEN10_SAMPLER_MODE,
620 GEN11_SAMPLER_ENABLE_HEADLESS_MSG);
621
622 /* Wa_1604278689:icl,ehl */
623 wa_write(wal, IVB_FBC_RT_BASE, 0xFFFFFFFF & ~ILK_FBC_RT_VALID);
624 wa_write_masked_or(wal, IVB_FBC_RT_BASE_UPPER,
625 0, /* write-only register; skip validation */
626 0xFFFFFFFF);
627
628 /* Wa_1406306137:icl,ehl */
629 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU);
630 }
631
gen12_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)632 static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine,
633 struct i915_wa_list *wal)
634 {
635 /*
636 * Wa_1409142259:tgl
637 * Wa_1409347922:tgl
638 * Wa_1409252684:tgl
639 * Wa_1409217633:tgl
640 * Wa_1409207793:tgl
641 * Wa_1409178076:tgl
642 * Wa_1408979724:tgl
643 * Wa_14010443199:rkl
644 * Wa_14010698770:rkl
645 */
646 WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
647 GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
648
649 /* WaDisableGPGPUMidThreadPreemption:gen12 */
650 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
651 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
652 GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
653 }
654
tgl_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)655 static void tgl_ctx_workarounds_init(struct intel_engine_cs *engine,
656 struct i915_wa_list *wal)
657 {
658 gen12_ctx_workarounds_init(engine, wal);
659
660 /*
661 * Wa_1604555607:tgl,rkl
662 *
663 * Note that the implementation of this workaround is further modified
664 * according to the FF_MODE2 guidance given by Wa_1608008084:gen12.
665 * FF_MODE2 register will return the wrong value when read. The default
666 * value for this register is zero for all fields and there are no bit
667 * masks. So instead of doing a RMW we should just write the GS Timer
668 * and TDS timer values for Wa_1604555607 and Wa_16011163337.
669 */
670 wa_add(wal,
671 FF_MODE2,
672 FF_MODE2_GS_TIMER_MASK | FF_MODE2_TDS_TIMER_MASK,
673 FF_MODE2_GS_TIMER_224 | FF_MODE2_TDS_TIMER_128,
674 0);
675 }
676
677 static void
__intel_engine_init_ctx_wa(struct intel_engine_cs * engine,struct i915_wa_list * wal,const char * name)678 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
679 struct i915_wa_list *wal,
680 const char *name)
681 {
682 struct drm_i915_private *i915 = engine->i915;
683
684 if (engine->class != RENDER_CLASS)
685 return;
686
687 wa_init_start(wal, name, engine->name);
688
689 if (IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915))
690 tgl_ctx_workarounds_init(engine, wal);
691 else if (IS_GEN(i915, 12))
692 gen12_ctx_workarounds_init(engine, wal);
693 else if (IS_GEN(i915, 11))
694 icl_ctx_workarounds_init(engine, wal);
695 else if (IS_CANNONLAKE(i915))
696 cnl_ctx_workarounds_init(engine, wal);
697 else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
698 cfl_ctx_workarounds_init(engine, wal);
699 else if (IS_GEMINILAKE(i915))
700 glk_ctx_workarounds_init(engine, wal);
701 else if (IS_KABYLAKE(i915))
702 kbl_ctx_workarounds_init(engine, wal);
703 else if (IS_BROXTON(i915))
704 bxt_ctx_workarounds_init(engine, wal);
705 else if (IS_SKYLAKE(i915))
706 skl_ctx_workarounds_init(engine, wal);
707 else if (IS_CHERRYVIEW(i915))
708 chv_ctx_workarounds_init(engine, wal);
709 else if (IS_BROADWELL(i915))
710 bdw_ctx_workarounds_init(engine, wal);
711 else if (IS_GEN(i915, 7))
712 gen7_ctx_workarounds_init(engine, wal);
713 else if (IS_GEN(i915, 6))
714 gen6_ctx_workarounds_init(engine, wal);
715 else if (INTEL_GEN(i915) < 8)
716 return;
717 else
718 MISSING_CASE(INTEL_GEN(i915));
719
720 wa_init_finish(wal);
721 }
722
intel_engine_init_ctx_wa(struct intel_engine_cs * engine)723 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
724 {
725 __intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context");
726 }
727
intel_engine_emit_ctx_wa(struct i915_request * rq)728 int intel_engine_emit_ctx_wa(struct i915_request *rq)
729 {
730 struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
731 struct i915_wa *wa;
732 unsigned int i;
733 u32 *cs;
734 int ret;
735
736 if (wal->count == 0)
737 return 0;
738
739 ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
740 if (ret)
741 return ret;
742
743 cs = intel_ring_begin(rq, (wal->count * 2 + 2));
744 if (IS_ERR(cs))
745 return PTR_ERR(cs);
746
747 *cs++ = MI_LOAD_REGISTER_IMM(wal->count);
748 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
749 *cs++ = i915_mmio_reg_offset(wa->reg);
750 *cs++ = wa->set;
751 }
752 *cs++ = MI_NOOP;
753
754 intel_ring_advance(rq, cs);
755
756 ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
757 if (ret)
758 return ret;
759
760 return 0;
761 }
762
763 static void
gen4_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)764 gen4_gt_workarounds_init(struct drm_i915_private *i915,
765 struct i915_wa_list *wal)
766 {
767 /* WaDisable_RenderCache_OperationalFlush:gen4,ilk */
768 wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
769 }
770
771 static void
g4x_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)772 g4x_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
773 {
774 gen4_gt_workarounds_init(i915, wal);
775
776 /* WaDisableRenderCachePipelinedFlush:g4x,ilk */
777 wa_masked_en(wal, CACHE_MODE_0, CM0_PIPELINED_RENDER_FLUSH_DISABLE);
778 }
779
780 static void
ilk_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)781 ilk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
782 {
783 g4x_gt_workarounds_init(i915, wal);
784
785 wa_masked_en(wal, _3D_CHICKEN2, _3D_CHICKEN2_WM_READ_PIPELINED);
786 }
787
788 static void
snb_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)789 snb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
790 {
791 /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
792 wa_masked_en(wal,
793 _3D_CHICKEN,
794 _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB);
795
796 /* WaDisable_RenderCache_OperationalFlush:snb */
797 wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
798
799 /*
800 * BSpec recommends 8x4 when MSAA is used,
801 * however in practice 16x4 seems fastest.
802 *
803 * Note that PS/WM thread counts depend on the WIZ hashing
804 * disable bit, which we don't touch here, but it's good
805 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
806 */
807 wa_add(wal,
808 GEN6_GT_MODE, 0,
809 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
810 GEN6_WIZ_HASHING_16x4);
811
812 wa_masked_dis(wal, CACHE_MODE_0, CM0_STC_EVICT_DISABLE_LRA_SNB);
813
814 wa_masked_en(wal,
815 _3D_CHICKEN3,
816 /* WaStripsFansDisableFastClipPerformanceFix:snb */
817 _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL |
818 /*
819 * Bspec says:
820 * "This bit must be set if 3DSTATE_CLIP clip mode is set
821 * to normal and 3DSTATE_SF number of SF output attributes
822 * is more than 16."
823 */
824 _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH);
825 }
826
827 static void
ivb_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)828 ivb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
829 {
830 /* WaDisableEarlyCull:ivb */
831 wa_masked_en(wal, _3D_CHICKEN3, _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
832
833 /* WaDisablePSDDualDispatchEnable:ivb */
834 if (IS_IVB_GT1(i915))
835 wa_masked_en(wal,
836 GEN7_HALF_SLICE_CHICKEN1,
837 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
838
839 /* WaDisable_RenderCache_OperationalFlush:ivb */
840 wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
841
842 /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
843 wa_masked_dis(wal,
844 GEN7_COMMON_SLICE_CHICKEN1,
845 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
846
847 /* WaApplyL3ControlAndL3ChickenMode:ivb */
848 wa_write(wal, GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL);
849 wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
850
851 /* WaForceL3Serialization:ivb */
852 wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
853
854 /*
855 * WaVSThreadDispatchOverride:ivb,vlv
856 *
857 * This actually overrides the dispatch
858 * mode for all thread types.
859 */
860 wa_write_masked_or(wal, GEN7_FF_THREAD_MODE,
861 GEN7_FF_SCHED_MASK,
862 GEN7_FF_TS_SCHED_HW |
863 GEN7_FF_VS_SCHED_HW |
864 GEN7_FF_DS_SCHED_HW);
865
866 if (0) { /* causes HiZ corruption on ivb:gt1 */
867 /* enable HiZ Raw Stall Optimization */
868 wa_masked_dis(wal, CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
869 }
870
871 /* WaDisable4x2SubspanOptimization:ivb */
872 wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
873
874 /*
875 * BSpec recommends 8x4 when MSAA is used,
876 * however in practice 16x4 seems fastest.
877 *
878 * Note that PS/WM thread counts depend on the WIZ hashing
879 * disable bit, which we don't touch here, but it's good
880 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
881 */
882 wa_add(wal, GEN7_GT_MODE, 0,
883 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
884 GEN6_WIZ_HASHING_16x4);
885 }
886
887 static void
vlv_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)888 vlv_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
889 {
890 /* WaDisableEarlyCull:vlv */
891 wa_masked_en(wal, _3D_CHICKEN3, _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
892
893 /* WaPsdDispatchEnable:vlv */
894 /* WaDisablePSDDualDispatchEnable:vlv */
895 wa_masked_en(wal,
896 GEN7_HALF_SLICE_CHICKEN1,
897 GEN7_MAX_PS_THREAD_DEP |
898 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
899
900 /* WaDisable_RenderCache_OperationalFlush:vlv */
901 wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
902
903 /* WaForceL3Serialization:vlv */
904 wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
905
906 /*
907 * WaVSThreadDispatchOverride:ivb,vlv
908 *
909 * This actually overrides the dispatch
910 * mode for all thread types.
911 */
912 wa_write_masked_or(wal,
913 GEN7_FF_THREAD_MODE,
914 GEN7_FF_SCHED_MASK,
915 GEN7_FF_TS_SCHED_HW |
916 GEN7_FF_VS_SCHED_HW |
917 GEN7_FF_DS_SCHED_HW);
918
919 /*
920 * BSpec says this must be set, even though
921 * WaDisable4x2SubspanOptimization isn't listed for VLV.
922 */
923 wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
924
925 /*
926 * BSpec recommends 8x4 when MSAA is used,
927 * however in practice 16x4 seems fastest.
928 *
929 * Note that PS/WM thread counts depend on the WIZ hashing
930 * disable bit, which we don't touch here, but it's good
931 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
932 */
933 wa_add(wal, GEN7_GT_MODE, 0,
934 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
935 GEN6_WIZ_HASHING_16x4);
936
937 /*
938 * WaIncreaseL3CreditsForVLVB0:vlv
939 * This is the hardware default actually.
940 */
941 wa_write(wal, GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
942 }
943
944 static void
hsw_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)945 hsw_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
946 {
947 /* L3 caching of data atomics doesn't work -- disable it. */
948 wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
949
950 wa_add(wal,
951 HSW_ROW_CHICKEN3, 0,
952 _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE),
953 0 /* XXX does this reg exist? */);
954
955 /* WaVSRefCountFullforceMissDisable:hsw */
956 wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME);
957
958 wa_masked_dis(wal,
959 CACHE_MODE_0_GEN7,
960 /* WaDisable_RenderCache_OperationalFlush:hsw */
961 RC_OP_FLUSH_ENABLE |
962 /* enable HiZ Raw Stall Optimization */
963 HIZ_RAW_STALL_OPT_DISABLE);
964
965 /* WaDisable4x2SubspanOptimization:hsw */
966 wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
967
968 /*
969 * BSpec recommends 8x4 when MSAA is used,
970 * however in practice 16x4 seems fastest.
971 *
972 * Note that PS/WM thread counts depend on the WIZ hashing
973 * disable bit, which we don't touch here, but it's good
974 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
975 */
976 wa_add(wal, GEN7_GT_MODE, 0,
977 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
978 GEN6_WIZ_HASHING_16x4);
979
980 /* WaSampleCChickenBitEnable:hsw */
981 wa_masked_en(wal, HALF_SLICE_CHICKEN3, HSW_SAMPLE_C_PERFORMANCE);
982 }
983
984 static void
gen9_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)985 gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
986 {
987 /* WaDisableKillLogic:bxt,skl,kbl */
988 if (!IS_COFFEELAKE(i915) && !IS_COMETLAKE(i915))
989 wa_write_or(wal,
990 GAM_ECOCHK,
991 ECOCHK_DIS_TLB);
992
993 if (HAS_LLC(i915)) {
994 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
995 *
996 * Must match Display Engine. See
997 * WaCompressedResourceDisplayNewHashMode.
998 */
999 wa_write_or(wal,
1000 MMCD_MISC_CTRL,
1001 MMCD_PCLA | MMCD_HOTSPOT_EN);
1002 }
1003
1004 /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
1005 wa_write_or(wal,
1006 GAM_ECOCHK,
1007 BDW_DISABLE_HDC_INVALIDATION);
1008 }
1009
1010 static void
skl_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)1011 skl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1012 {
1013 gen9_gt_workarounds_init(i915, wal);
1014
1015 /* WaDisableGafsUnitClkGating:skl */
1016 wa_write_or(wal,
1017 GEN7_UCGCTL4,
1018 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1019
1020 /* WaInPlaceDecompressionHang:skl */
1021 if (IS_SKL_REVID(i915, SKL_REVID_H0, REVID_FOREVER))
1022 wa_write_or(wal,
1023 GEN9_GAMT_ECO_REG_RW_IA,
1024 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1025 }
1026
1027 static void
bxt_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)1028 bxt_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1029 {
1030 gen9_gt_workarounds_init(i915, wal);
1031
1032 /* WaInPlaceDecompressionHang:bxt */
1033 wa_write_or(wal,
1034 GEN9_GAMT_ECO_REG_RW_IA,
1035 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1036 }
1037
1038 static void
kbl_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)1039 kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1040 {
1041 gen9_gt_workarounds_init(i915, wal);
1042
1043 /* WaDisableDynamicCreditSharing:kbl */
1044 if (IS_KBL_GT_REVID(i915, 0, KBL_REVID_B0))
1045 wa_write_or(wal,
1046 GAMT_CHKN_BIT_REG,
1047 GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
1048
1049 /* WaDisableGafsUnitClkGating:kbl */
1050 wa_write_or(wal,
1051 GEN7_UCGCTL4,
1052 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1053
1054 /* WaInPlaceDecompressionHang:kbl */
1055 wa_write_or(wal,
1056 GEN9_GAMT_ECO_REG_RW_IA,
1057 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1058 }
1059
1060 static void
glk_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)1061 glk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1062 {
1063 gen9_gt_workarounds_init(i915, wal);
1064 }
1065
1066 static void
cfl_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)1067 cfl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1068 {
1069 gen9_gt_workarounds_init(i915, wal);
1070
1071 /* WaDisableGafsUnitClkGating:cfl */
1072 wa_write_or(wal,
1073 GEN7_UCGCTL4,
1074 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1075
1076 /* WaInPlaceDecompressionHang:cfl */
1077 wa_write_or(wal,
1078 GEN9_GAMT_ECO_REG_RW_IA,
1079 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1080 }
1081
1082 static void
wa_init_mcr(struct drm_i915_private * i915,struct i915_wa_list * wal)1083 wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal)
1084 {
1085 const struct sseu_dev_info *sseu = &i915->gt.info.sseu;
1086 unsigned int slice, subslice;
1087 u32 l3_en, mcr, mcr_mask;
1088
1089 GEM_BUG_ON(INTEL_GEN(i915) < 10);
1090
1091 /*
1092 * WaProgramMgsrForL3BankSpecificMmioReads: cnl,icl
1093 * L3Banks could be fused off in single slice scenario. If that is
1094 * the case, we might need to program MCR select to a valid L3Bank
1095 * by default, to make sure we correctly read certain registers
1096 * later on (in the range 0xB100 - 0xB3FF).
1097 *
1098 * WaProgramMgsrForCorrectSliceSpecificMmioReads:cnl,icl
1099 * Before any MMIO read into slice/subslice specific registers, MCR
1100 * packet control register needs to be programmed to point to any
1101 * enabled s/ss pair. Otherwise, incorrect values will be returned.
1102 * This means each subsequent MMIO read will be forwarded to an
1103 * specific s/ss combination, but this is OK since these registers
1104 * are consistent across s/ss in almost all cases. In the rare
1105 * occasions, such as INSTDONE, where this value is dependent
1106 * on s/ss combo, the read should be done with read_subslice_reg.
1107 *
1108 * Since GEN8_MCR_SELECTOR contains dual-purpose bits which select both
1109 * to which subslice, or to which L3 bank, the respective mmio reads
1110 * will go, we have to find a common index which works for both
1111 * accesses.
1112 *
1113 * Case where we cannot find a common index fortunately should not
1114 * happen in production hardware, so we only emit a warning instead of
1115 * implementing something more complex that requires checking the range
1116 * of every MMIO read.
1117 */
1118
1119 if (INTEL_GEN(i915) >= 10 && is_power_of_2(sseu->slice_mask)) {
1120 u32 l3_fuse =
1121 intel_uncore_read(&i915->uncore, GEN10_MIRROR_FUSE3) &
1122 GEN10_L3BANK_MASK;
1123
1124 drm_dbg(&i915->drm, "L3 fuse = %x\n", l3_fuse);
1125 l3_en = ~(l3_fuse << GEN10_L3BANK_PAIR_COUNT | l3_fuse);
1126 } else {
1127 l3_en = ~0;
1128 }
1129
1130 slice = fls(sseu->slice_mask) - 1;
1131 subslice = fls(l3_en & intel_sseu_get_subslices(sseu, slice));
1132 if (!subslice) {
1133 drm_warn(&i915->drm,
1134 "No common index found between subslice mask %x and L3 bank mask %x!\n",
1135 intel_sseu_get_subslices(sseu, slice), l3_en);
1136 subslice = fls(l3_en);
1137 drm_WARN_ON(&i915->drm, !subslice);
1138 }
1139 subslice--;
1140
1141 if (INTEL_GEN(i915) >= 11) {
1142 mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice);
1143 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
1144 } else {
1145 mcr = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
1146 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
1147 }
1148
1149 drm_dbg(&i915->drm, "MCR slice/subslice = %x\n", mcr);
1150
1151 wa_write_masked_or(wal, GEN8_MCR_SELECTOR, mcr_mask, mcr);
1152 }
1153
1154 static void
cnl_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)1155 cnl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1156 {
1157 wa_init_mcr(i915, wal);
1158
1159 /* WaInPlaceDecompressionHang:cnl */
1160 wa_write_or(wal,
1161 GEN9_GAMT_ECO_REG_RW_IA,
1162 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1163 }
1164
1165 static void
icl_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)1166 icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1167 {
1168 wa_init_mcr(i915, wal);
1169
1170 /* WaInPlaceDecompressionHang:icl */
1171 wa_write_or(wal,
1172 GEN9_GAMT_ECO_REG_RW_IA,
1173 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1174
1175 /* WaModifyGamTlbPartitioning:icl */
1176 wa_write_masked_or(wal,
1177 GEN11_GACB_PERF_CTRL,
1178 GEN11_HASH_CTRL_MASK,
1179 GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
1180
1181 /* Wa_1405766107:icl
1182 * Formerly known as WaCL2SFHalfMaxAlloc
1183 */
1184 wa_write_or(wal,
1185 GEN11_LSN_UNSLCVC,
1186 GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
1187 GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
1188
1189 /* Wa_220166154:icl
1190 * Formerly known as WaDisCtxReload
1191 */
1192 wa_write_or(wal,
1193 GEN8_GAMW_ECO_DEV_RW_IA,
1194 GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
1195
1196 /* Wa_1405779004:icl (pre-prod) */
1197 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
1198 wa_write_or(wal,
1199 SLICE_UNIT_LEVEL_CLKGATE,
1200 MSCUNIT_CLKGATE_DIS);
1201
1202 /* Wa_1406838659:icl (pre-prod) */
1203 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1204 wa_write_or(wal,
1205 INF_UNIT_LEVEL_CLKGATE,
1206 CGPSF_CLKGATE_DIS);
1207
1208 /* Wa_1406463099:icl
1209 * Formerly known as WaGamTlbPendError
1210 */
1211 wa_write_or(wal,
1212 GAMT_CHKN_BIT_REG,
1213 GAMT_CHKN_DISABLE_L3_COH_PIPE);
1214
1215 /*
1216 * Wa_1408615072:icl,ehl (vsunit)
1217 * Wa_1407596294:icl,ehl (hsunit)
1218 */
1219 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1220 VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS);
1221
1222 /* Wa_1407352427:icl,ehl */
1223 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1224 PSDUNIT_CLKGATE_DIS);
1225
1226 /* Wa_1406680159:icl,ehl */
1227 wa_write_or(wal,
1228 SUBSLICE_UNIT_LEVEL_CLKGATE,
1229 GWUNIT_CLKGATE_DIS);
1230
1231 /* Wa_1607087056:icl,ehl,jsl */
1232 if (IS_ICELAKE(i915) ||
1233 IS_EHL_REVID(i915, EHL_REVID_A0, EHL_REVID_A0)) {
1234 wa_write_or(wal,
1235 SLICE_UNIT_LEVEL_CLKGATE,
1236 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1237 }
1238 }
1239
1240 static void
gen12_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)1241 gen12_gt_workarounds_init(struct drm_i915_private *i915,
1242 struct i915_wa_list *wal)
1243 {
1244 wa_init_mcr(i915, wal);
1245 }
1246
1247 static void
tgl_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)1248 tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1249 {
1250 gen12_gt_workarounds_init(i915, wal);
1251
1252 /* Wa_1409420604:tgl */
1253 if (IS_TGL_UY_GT_REVID(i915, TGL_REVID_A0, TGL_REVID_A0))
1254 wa_write_or(wal,
1255 SUBSLICE_UNIT_LEVEL_CLKGATE2,
1256 CPSSUNIT_CLKGATE_DIS);
1257
1258 /* Wa_1607087056:tgl also know as BUG:1409180338 */
1259 if (IS_TGL_UY_GT_REVID(i915, TGL_REVID_A0, TGL_REVID_A0))
1260 wa_write_or(wal,
1261 SLICE_UNIT_LEVEL_CLKGATE,
1262 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1263 }
1264
1265 static void
gt_init_workarounds(struct drm_i915_private * i915,struct i915_wa_list * wal)1266 gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal)
1267 {
1268 if (IS_TIGERLAKE(i915))
1269 tgl_gt_workarounds_init(i915, wal);
1270 else if (IS_GEN(i915, 12))
1271 gen12_gt_workarounds_init(i915, wal);
1272 else if (IS_GEN(i915, 11))
1273 icl_gt_workarounds_init(i915, wal);
1274 else if (IS_CANNONLAKE(i915))
1275 cnl_gt_workarounds_init(i915, wal);
1276 else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
1277 cfl_gt_workarounds_init(i915, wal);
1278 else if (IS_GEMINILAKE(i915))
1279 glk_gt_workarounds_init(i915, wal);
1280 else if (IS_KABYLAKE(i915))
1281 kbl_gt_workarounds_init(i915, wal);
1282 else if (IS_BROXTON(i915))
1283 bxt_gt_workarounds_init(i915, wal);
1284 else if (IS_SKYLAKE(i915))
1285 skl_gt_workarounds_init(i915, wal);
1286 else if (IS_HASWELL(i915))
1287 hsw_gt_workarounds_init(i915, wal);
1288 else if (IS_VALLEYVIEW(i915))
1289 vlv_gt_workarounds_init(i915, wal);
1290 else if (IS_IVYBRIDGE(i915))
1291 ivb_gt_workarounds_init(i915, wal);
1292 else if (IS_GEN(i915, 6))
1293 snb_gt_workarounds_init(i915, wal);
1294 else if (IS_GEN(i915, 5))
1295 ilk_gt_workarounds_init(i915, wal);
1296 else if (IS_G4X(i915))
1297 g4x_gt_workarounds_init(i915, wal);
1298 else if (IS_GEN(i915, 4))
1299 gen4_gt_workarounds_init(i915, wal);
1300 else if (INTEL_GEN(i915) <= 8)
1301 return;
1302 else
1303 MISSING_CASE(INTEL_GEN(i915));
1304 }
1305
intel_gt_init_workarounds(struct drm_i915_private * i915)1306 void intel_gt_init_workarounds(struct drm_i915_private *i915)
1307 {
1308 struct i915_wa_list *wal = &i915->gt_wa_list;
1309
1310 wa_init_start(wal, "GT", "global");
1311 gt_init_workarounds(i915, wal);
1312 wa_init_finish(wal);
1313 }
1314
1315 static enum forcewake_domains
wal_get_fw_for_rmw(struct intel_uncore * uncore,const struct i915_wa_list * wal)1316 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1317 {
1318 enum forcewake_domains fw = 0;
1319 struct i915_wa *wa;
1320 unsigned int i;
1321
1322 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1323 fw |= intel_uncore_forcewake_for_reg(uncore,
1324 wa->reg,
1325 FW_REG_READ |
1326 FW_REG_WRITE);
1327
1328 return fw;
1329 }
1330
1331 static bool
wa_verify(const struct i915_wa * wa,u32 cur,const char * name,const char * from)1332 wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from)
1333 {
1334 if ((cur ^ wa->set) & wa->read) {
1335 DRM_ERROR("%s workaround lost on %s! (%x=%x/%x, expected %x)\n",
1336 name, from, i915_mmio_reg_offset(wa->reg),
1337 cur, cur & wa->read, wa->set);
1338
1339 return false;
1340 }
1341
1342 return true;
1343 }
1344
1345 static void
wa_list_apply(struct intel_uncore * uncore,const struct i915_wa_list * wal)1346 wa_list_apply(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1347 {
1348 enum forcewake_domains fw;
1349 unsigned long flags;
1350 struct i915_wa *wa;
1351 unsigned int i;
1352
1353 if (!wal->count)
1354 return;
1355
1356 fw = wal_get_fw_for_rmw(uncore, wal);
1357
1358 spin_lock_irqsave(&uncore->lock, flags);
1359 intel_uncore_forcewake_get__locked(uncore, fw);
1360
1361 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1362 if (wa->clr)
1363 intel_uncore_rmw_fw(uncore, wa->reg, wa->clr, wa->set);
1364 else
1365 intel_uncore_write_fw(uncore, wa->reg, wa->set);
1366 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
1367 wa_verify(wa,
1368 intel_uncore_read_fw(uncore, wa->reg),
1369 wal->name, "application");
1370 }
1371
1372 intel_uncore_forcewake_put__locked(uncore, fw);
1373 spin_unlock_irqrestore(&uncore->lock, flags);
1374 }
1375
intel_gt_apply_workarounds(struct intel_gt * gt)1376 void intel_gt_apply_workarounds(struct intel_gt *gt)
1377 {
1378 wa_list_apply(gt->uncore, >->i915->gt_wa_list);
1379 }
1380
wa_list_verify(struct intel_uncore * uncore,const struct i915_wa_list * wal,const char * from)1381 static bool wa_list_verify(struct intel_uncore *uncore,
1382 const struct i915_wa_list *wal,
1383 const char *from)
1384 {
1385 struct i915_wa *wa;
1386 unsigned int i;
1387 bool ok = true;
1388
1389 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1390 ok &= wa_verify(wa,
1391 intel_uncore_read(uncore, wa->reg),
1392 wal->name, from);
1393
1394 return ok;
1395 }
1396
intel_gt_verify_workarounds(struct intel_gt * gt,const char * from)1397 bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from)
1398 {
1399 return wa_list_verify(gt->uncore, >->i915->gt_wa_list, from);
1400 }
1401
is_nonpriv_flags_valid(u32 flags)1402 static inline bool is_nonpriv_flags_valid(u32 flags)
1403 {
1404 /* Check only valid flag bits are set */
1405 if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID)
1406 return false;
1407
1408 /* NB: Only 3 out of 4 enum values are valid for access field */
1409 if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
1410 RING_FORCE_TO_NONPRIV_ACCESS_INVALID)
1411 return false;
1412
1413 return true;
1414 }
1415
1416 static void
whitelist_reg_ext(struct i915_wa_list * wal,i915_reg_t reg,u32 flags)1417 whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags)
1418 {
1419 struct i915_wa wa = {
1420 .reg = reg
1421 };
1422
1423 if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1424 return;
1425
1426 if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1427 return;
1428
1429 wa.reg.reg |= flags;
1430 _wa_add(wal, &wa);
1431 }
1432
1433 static void
whitelist_reg(struct i915_wa_list * wal,i915_reg_t reg)1434 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
1435 {
1436 whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
1437 }
1438
gen9_whitelist_build(struct i915_wa_list * w)1439 static void gen9_whitelist_build(struct i915_wa_list *w)
1440 {
1441 /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1442 whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
1443
1444 /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1445 whitelist_reg(w, GEN8_CS_CHICKEN1);
1446
1447 /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1448 whitelist_reg(w, GEN8_HDC_CHICKEN1);
1449
1450 /* WaSendPushConstantsFromMMIO:skl,bxt */
1451 whitelist_reg(w, COMMON_SLICE_CHICKEN2);
1452 }
1453
skl_whitelist_build(struct intel_engine_cs * engine)1454 static void skl_whitelist_build(struct intel_engine_cs *engine)
1455 {
1456 struct i915_wa_list *w = &engine->whitelist;
1457
1458 if (engine->class != RENDER_CLASS)
1459 return;
1460
1461 gen9_whitelist_build(w);
1462
1463 /* WaDisableLSQCROPERFforOCL:skl */
1464 whitelist_reg(w, GEN8_L3SQCREG4);
1465 }
1466
bxt_whitelist_build(struct intel_engine_cs * engine)1467 static void bxt_whitelist_build(struct intel_engine_cs *engine)
1468 {
1469 if (engine->class != RENDER_CLASS)
1470 return;
1471
1472 gen9_whitelist_build(&engine->whitelist);
1473 }
1474
kbl_whitelist_build(struct intel_engine_cs * engine)1475 static void kbl_whitelist_build(struct intel_engine_cs *engine)
1476 {
1477 struct i915_wa_list *w = &engine->whitelist;
1478
1479 if (engine->class != RENDER_CLASS)
1480 return;
1481
1482 gen9_whitelist_build(w);
1483
1484 /* WaDisableLSQCROPERFforOCL:kbl */
1485 whitelist_reg(w, GEN8_L3SQCREG4);
1486 }
1487
glk_whitelist_build(struct intel_engine_cs * engine)1488 static void glk_whitelist_build(struct intel_engine_cs *engine)
1489 {
1490 struct i915_wa_list *w = &engine->whitelist;
1491
1492 if (engine->class != RENDER_CLASS)
1493 return;
1494
1495 gen9_whitelist_build(w);
1496
1497 /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
1498 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1499 }
1500
cfl_whitelist_build(struct intel_engine_cs * engine)1501 static void cfl_whitelist_build(struct intel_engine_cs *engine)
1502 {
1503 struct i915_wa_list *w = &engine->whitelist;
1504
1505 if (engine->class != RENDER_CLASS)
1506 return;
1507
1508 gen9_whitelist_build(w);
1509
1510 /*
1511 * WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml
1512 *
1513 * This covers 4 register which are next to one another :
1514 * - PS_INVOCATION_COUNT
1515 * - PS_INVOCATION_COUNT_UDW
1516 * - PS_DEPTH_COUNT
1517 * - PS_DEPTH_COUNT_UDW
1518 */
1519 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1520 RING_FORCE_TO_NONPRIV_ACCESS_RD |
1521 RING_FORCE_TO_NONPRIV_RANGE_4);
1522 }
1523
cml_whitelist_build(struct intel_engine_cs * engine)1524 static void cml_whitelist_build(struct intel_engine_cs *engine)
1525 {
1526 struct i915_wa_list *w = &engine->whitelist;
1527
1528 if (engine->class != RENDER_CLASS)
1529 whitelist_reg_ext(w,
1530 RING_CTX_TIMESTAMP(engine->mmio_base),
1531 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1532
1533 cfl_whitelist_build(engine);
1534 }
1535
cnl_whitelist_build(struct intel_engine_cs * engine)1536 static void cnl_whitelist_build(struct intel_engine_cs *engine)
1537 {
1538 struct i915_wa_list *w = &engine->whitelist;
1539
1540 if (engine->class != RENDER_CLASS)
1541 return;
1542
1543 /* WaEnablePreemptionGranularityControlByUMD:cnl */
1544 whitelist_reg(w, GEN8_CS_CHICKEN1);
1545 }
1546
icl_whitelist_build(struct intel_engine_cs * engine)1547 static void icl_whitelist_build(struct intel_engine_cs *engine)
1548 {
1549 struct i915_wa_list *w = &engine->whitelist;
1550
1551 switch (engine->class) {
1552 case RENDER_CLASS:
1553 /* WaAllowUMDToModifyHalfSliceChicken7:icl */
1554 whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7);
1555
1556 /* WaAllowUMDToModifySamplerMode:icl */
1557 whitelist_reg(w, GEN10_SAMPLER_MODE);
1558
1559 /* WaEnableStateCacheRedirectToCS:icl */
1560 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1561
1562 /*
1563 * WaAllowPMDepthAndInvocationCountAccessFromUMD:icl
1564 *
1565 * This covers 4 register which are next to one another :
1566 * - PS_INVOCATION_COUNT
1567 * - PS_INVOCATION_COUNT_UDW
1568 * - PS_DEPTH_COUNT
1569 * - PS_DEPTH_COUNT_UDW
1570 */
1571 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1572 RING_FORCE_TO_NONPRIV_ACCESS_RD |
1573 RING_FORCE_TO_NONPRIV_RANGE_4);
1574 break;
1575
1576 case VIDEO_DECODE_CLASS:
1577 /* hucStatusRegOffset */
1578 whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base),
1579 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1580 /* hucUKernelHdrInfoRegOffset */
1581 whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base),
1582 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1583 /* hucStatus2RegOffset */
1584 whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base),
1585 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1586 whitelist_reg_ext(w,
1587 RING_CTX_TIMESTAMP(engine->mmio_base),
1588 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1589 break;
1590
1591 default:
1592 whitelist_reg_ext(w,
1593 RING_CTX_TIMESTAMP(engine->mmio_base),
1594 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1595 break;
1596 }
1597 }
1598
tgl_whitelist_build(struct intel_engine_cs * engine)1599 static void tgl_whitelist_build(struct intel_engine_cs *engine)
1600 {
1601 struct i915_wa_list *w = &engine->whitelist;
1602
1603 switch (engine->class) {
1604 case RENDER_CLASS:
1605 /*
1606 * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl
1607 * Wa_1408556865:tgl
1608 *
1609 * This covers 4 registers which are next to one another :
1610 * - PS_INVOCATION_COUNT
1611 * - PS_INVOCATION_COUNT_UDW
1612 * - PS_DEPTH_COUNT
1613 * - PS_DEPTH_COUNT_UDW
1614 */
1615 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1616 RING_FORCE_TO_NONPRIV_ACCESS_RD |
1617 RING_FORCE_TO_NONPRIV_RANGE_4);
1618
1619 /* Wa_1808121037:tgl */
1620 whitelist_reg(w, GEN7_COMMON_SLICE_CHICKEN1);
1621
1622 /* Wa_1806527549:tgl */
1623 whitelist_reg(w, HIZ_CHICKEN);
1624 break;
1625 default:
1626 whitelist_reg_ext(w,
1627 RING_CTX_TIMESTAMP(engine->mmio_base),
1628 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1629 break;
1630 }
1631 }
1632
intel_engine_init_whitelist(struct intel_engine_cs * engine)1633 void intel_engine_init_whitelist(struct intel_engine_cs *engine)
1634 {
1635 struct drm_i915_private *i915 = engine->i915;
1636 struct i915_wa_list *w = &engine->whitelist;
1637
1638 wa_init_start(w, "whitelist", engine->name);
1639
1640 if (IS_GEN(i915, 12))
1641 tgl_whitelist_build(engine);
1642 else if (IS_GEN(i915, 11))
1643 icl_whitelist_build(engine);
1644 else if (IS_CANNONLAKE(i915))
1645 cnl_whitelist_build(engine);
1646 else if (IS_COMETLAKE(i915))
1647 cml_whitelist_build(engine);
1648 else if (IS_COFFEELAKE(i915))
1649 cfl_whitelist_build(engine);
1650 else if (IS_GEMINILAKE(i915))
1651 glk_whitelist_build(engine);
1652 else if (IS_KABYLAKE(i915))
1653 kbl_whitelist_build(engine);
1654 else if (IS_BROXTON(i915))
1655 bxt_whitelist_build(engine);
1656 else if (IS_SKYLAKE(i915))
1657 skl_whitelist_build(engine);
1658 else if (INTEL_GEN(i915) <= 8)
1659 return;
1660 else
1661 MISSING_CASE(INTEL_GEN(i915));
1662
1663 wa_init_finish(w);
1664 }
1665
intel_engine_apply_whitelist(struct intel_engine_cs * engine)1666 void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
1667 {
1668 const struct i915_wa_list *wal = &engine->whitelist;
1669 struct intel_uncore *uncore = engine->uncore;
1670 const u32 base = engine->mmio_base;
1671 struct i915_wa *wa;
1672 unsigned int i;
1673
1674 if (!wal->count)
1675 return;
1676
1677 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1678 intel_uncore_write(uncore,
1679 RING_FORCE_TO_NONPRIV(base, i),
1680 i915_mmio_reg_offset(wa->reg));
1681
1682 /* And clear the rest just in case of garbage */
1683 for (; i < RING_MAX_NONPRIV_SLOTS; i++)
1684 intel_uncore_write(uncore,
1685 RING_FORCE_TO_NONPRIV(base, i),
1686 i915_mmio_reg_offset(RING_NOPID(base)));
1687 }
1688
1689 static void
rcs_engine_wa_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)1690 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1691 {
1692 struct drm_i915_private *i915 = engine->i915;
1693
1694 if (IS_TGL_UY_GT_REVID(i915, TGL_REVID_A0, TGL_REVID_A0)) {
1695 /*
1696 * Wa_1607138336:tgl
1697 * Wa_1607063988:tgl
1698 */
1699 wa_write_or(wal,
1700 GEN9_CTX_PREEMPT_REG,
1701 GEN12_DISABLE_POSH_BUSY_FF_DOP_CG);
1702
1703 /*
1704 * Wa_1606679103:tgl
1705 * (see also Wa_1606682166:icl)
1706 */
1707 wa_write_or(wal,
1708 GEN7_SARCHKMD,
1709 GEN7_DISABLE_SAMPLER_PREFETCH);
1710
1711 /* Wa_1408615072:tgl */
1712 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1713 VSUNIT_CLKGATE_DIS_TGL);
1714 }
1715
1716 if (IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1717 /* Wa_1606931601:tgl,rkl */
1718 wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ);
1719
1720 /* Wa_1409804808:tgl,rkl */
1721 wa_masked_en(wal, GEN7_ROW_CHICKEN2,
1722 GEN12_PUSH_CONST_DEREF_HOLD_DIS);
1723
1724 /*
1725 * Wa_1409085225:tgl
1726 * Wa_14010229206:tgl,rkl
1727 */
1728 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH);
1729
1730 /*
1731 * Wa_1407928979:tgl A*
1732 * Wa_18011464164:tgl B0+
1733 * Wa_22010931296:tgl B0+
1734 * Wa_14010919138:rkl,tgl
1735 */
1736 wa_write_or(wal, GEN7_FF_THREAD_MODE,
1737 GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1738
1739 /*
1740 * Wa_1607030317:tgl
1741 * Wa_1607186500:tgl
1742 * Wa_1607297627:tgl,rkl there are multiple entries for this
1743 * WA in the BSpec; some indicate this is an A0-only WA,
1744 * others indicate it applies to all steppings.
1745 */
1746 wa_masked_en(wal,
1747 GEN6_RC_SLEEP_PSMI_CONTROL,
1748 GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE |
1749 GEN8_RC_SEMA_IDLE_MSG_DISABLE);
1750
1751 /*
1752 * Wa_1606700617:tgl
1753 * Wa_22010271021:tgl,rkl
1754 */
1755 wa_masked_en(wal,
1756 GEN9_CS_DEBUG_MODE1,
1757 FF_DOP_CLOCK_GATE_DISABLE);
1758 }
1759
1760 if (IS_GEN(i915, 12)) {
1761 /* Wa_1406941453:gen12 */
1762 wa_masked_en(wal,
1763 GEN10_SAMPLER_MODE,
1764 ENABLE_SMALLPL);
1765 }
1766
1767 if (IS_GEN(i915, 11)) {
1768 /* This is not an Wa. Enable for better image quality */
1769 wa_masked_en(wal,
1770 _3D_CHICKEN3,
1771 _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
1772
1773 /* WaPipelineFlushCoherentLines:icl */
1774 wa_write_or(wal,
1775 GEN8_L3SQCREG4,
1776 GEN8_LQSC_FLUSH_COHERENT_LINES);
1777
1778 /*
1779 * Wa_1405543622:icl
1780 * Formerly known as WaGAPZPriorityScheme
1781 */
1782 wa_write_or(wal,
1783 GEN8_GARBCNTL,
1784 GEN11_ARBITRATION_PRIO_ORDER_MASK);
1785
1786 /*
1787 * Wa_1604223664:icl
1788 * Formerly known as WaL3BankAddressHashing
1789 */
1790 wa_write_masked_or(wal,
1791 GEN8_GARBCNTL,
1792 GEN11_HASH_CTRL_EXCL_MASK,
1793 GEN11_HASH_CTRL_EXCL_BIT0);
1794 wa_write_masked_or(wal,
1795 GEN11_GLBLINVL,
1796 GEN11_BANK_HASH_ADDR_EXCL_MASK,
1797 GEN11_BANK_HASH_ADDR_EXCL_BIT0);
1798
1799 /*
1800 * Wa_1405733216:icl
1801 * Formerly known as WaDisableCleanEvicts
1802 */
1803 wa_write_or(wal,
1804 GEN8_L3SQCREG4,
1805 GEN11_LQSC_CLEAN_EVICT_DISABLE);
1806
1807 /* WaForwardProgressSoftReset:icl */
1808 wa_write_or(wal,
1809 GEN10_SCRATCH_LNCF2,
1810 PMFLUSHDONE_LNICRSDROP |
1811 PMFLUSH_GAPL3UNBLOCK |
1812 PMFLUSHDONE_LNEBLK);
1813
1814 /* Wa_1406609255:icl (pre-prod) */
1815 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1816 wa_write_or(wal,
1817 GEN7_SARCHKMD,
1818 GEN7_DISABLE_DEMAND_PREFETCH);
1819
1820 /* Wa_1606682166:icl */
1821 wa_write_or(wal,
1822 GEN7_SARCHKMD,
1823 GEN7_DISABLE_SAMPLER_PREFETCH);
1824
1825 /* Wa_1409178092:icl */
1826 wa_write_masked_or(wal,
1827 GEN11_SCRATCH2,
1828 GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE,
1829 0);
1830
1831 /* WaEnable32PlaneMode:icl */
1832 wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS,
1833 GEN11_ENABLE_32_PLANE_MODE);
1834
1835 /*
1836 * Wa_1408767742:icl[a2..forever],ehl[all]
1837 * Wa_1605460711:icl[a0..c0]
1838 */
1839 wa_write_or(wal,
1840 GEN7_FF_THREAD_MODE,
1841 GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1842
1843 /* Wa_22010271021:ehl */
1844 if (IS_ELKHARTLAKE(i915))
1845 wa_masked_en(wal,
1846 GEN9_CS_DEBUG_MODE1,
1847 FF_DOP_CLOCK_GATE_DISABLE);
1848 }
1849
1850 if (IS_GEN_RANGE(i915, 9, 12)) {
1851 /* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl,tgl */
1852 wa_masked_en(wal,
1853 GEN7_FF_SLICE_CS_CHICKEN1,
1854 GEN9_FFSC_PERCTX_PREEMPT_CTRL);
1855 }
1856
1857 if (IS_SKYLAKE(i915) ||
1858 IS_KABYLAKE(i915) ||
1859 IS_COFFEELAKE(i915) ||
1860 IS_COMETLAKE(i915)) {
1861 /* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
1862 wa_write_or(wal,
1863 GEN8_GARBCNTL,
1864 GEN9_GAPS_TSV_CREDIT_DISABLE);
1865 }
1866
1867 if (IS_BROXTON(i915)) {
1868 /* WaDisablePooledEuLoadBalancingFix:bxt */
1869 wa_masked_en(wal,
1870 FF_SLICE_CS_CHICKEN2,
1871 GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
1872 }
1873
1874 if (IS_GEN(i915, 9)) {
1875 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
1876 wa_masked_en(wal,
1877 GEN9_CSFE_CHICKEN1_RCS,
1878 GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
1879
1880 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
1881 wa_write_or(wal,
1882 BDW_SCRATCH1,
1883 GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
1884
1885 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
1886 if (IS_GEN9_LP(i915))
1887 wa_write_masked_or(wal,
1888 GEN8_L3SQCREG1,
1889 L3_PRIO_CREDITS_MASK,
1890 L3_GENERAL_PRIO_CREDITS(62) |
1891 L3_HIGH_PRIO_CREDITS(2));
1892
1893 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
1894 wa_write_or(wal,
1895 GEN8_L3SQCREG4,
1896 GEN8_LQSC_FLUSH_COHERENT_LINES);
1897 }
1898
1899 if (IS_GEN(i915, 7))
1900 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
1901 wa_masked_en(wal,
1902 GFX_MODE_GEN7,
1903 GFX_TLB_INVALIDATE_EXPLICIT | GFX_REPLAY_MODE);
1904
1905 if (IS_GEN_RANGE(i915, 6, 7))
1906 /*
1907 * We need to disable the AsyncFlip performance optimisations in
1908 * order to use MI_WAIT_FOR_EVENT within the CS. It should
1909 * already be programmed to '1' on all products.
1910 *
1911 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
1912 */
1913 wa_masked_en(wal,
1914 MI_MODE,
1915 ASYNC_FLIP_PERF_DISABLE);
1916
1917 if (IS_GEN(i915, 6)) {
1918 /*
1919 * Required for the hardware to program scanline values for
1920 * waiting
1921 * WaEnableFlushTlbInvalidationMode:snb
1922 */
1923 wa_masked_en(wal,
1924 GFX_MODE,
1925 GFX_TLB_INVALIDATE_EXPLICIT);
1926
1927 /*
1928 * From the Sandybridge PRM, volume 1 part 3, page 24:
1929 * "If this bit is set, STCunit will have LRA as replacement
1930 * policy. [...] This bit must be reset. LRA replacement
1931 * policy is not supported."
1932 */
1933 wa_masked_dis(wal,
1934 CACHE_MODE_0,
1935 CM0_STC_EVICT_DISABLE_LRA_SNB);
1936 }
1937
1938 if (IS_GEN_RANGE(i915, 4, 6))
1939 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
1940 wa_add(wal, MI_MODE,
1941 0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH),
1942 /* XXX bit doesn't stick on Broadwater */
1943 IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH);
1944
1945 if (IS_GEN(i915, 4))
1946 /*
1947 * Disable CONSTANT_BUFFER before it is loaded from the context
1948 * image. For as it is loaded, it is executed and the stored
1949 * address may no longer be valid, leading to a GPU hang.
1950 *
1951 * This imposes the requirement that userspace reload their
1952 * CONSTANT_BUFFER on every batch, fortunately a requirement
1953 * they are already accustomed to from before contexts were
1954 * enabled.
1955 */
1956 wa_add(wal, ECOSKPD,
1957 0, _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE),
1958 0 /* XXX bit doesn't stick on Broadwater */);
1959 }
1960
1961 static void
xcs_engine_wa_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)1962 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1963 {
1964 struct drm_i915_private *i915 = engine->i915;
1965
1966 /* WaKBLVECSSemaphoreWaitPoll:kbl */
1967 if (IS_KBL_GT_REVID(i915, KBL_REVID_A0, KBL_REVID_E0)) {
1968 wa_write(wal,
1969 RING_SEMA_WAIT_POLL(engine->mmio_base),
1970 1);
1971 }
1972 }
1973
1974 static void
engine_init_workarounds(struct intel_engine_cs * engine,struct i915_wa_list * wal)1975 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1976 {
1977 if (I915_SELFTEST_ONLY(INTEL_GEN(engine->i915) < 4))
1978 return;
1979
1980 if (engine->class == RENDER_CLASS)
1981 rcs_engine_wa_init(engine, wal);
1982 else
1983 xcs_engine_wa_init(engine, wal);
1984 }
1985
intel_engine_init_workarounds(struct intel_engine_cs * engine)1986 void intel_engine_init_workarounds(struct intel_engine_cs *engine)
1987 {
1988 struct i915_wa_list *wal = &engine->wa_list;
1989
1990 if (INTEL_GEN(engine->i915) < 4)
1991 return;
1992
1993 wa_init_start(wal, "engine", engine->name);
1994 engine_init_workarounds(engine, wal);
1995 wa_init_finish(wal);
1996 }
1997
intel_engine_apply_workarounds(struct intel_engine_cs * engine)1998 void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
1999 {
2000 wa_list_apply(engine->uncore, &engine->wa_list);
2001 }
2002
2003 static struct i915_vma *
create_scratch(struct i915_address_space * vm,int count)2004 create_scratch(struct i915_address_space *vm, int count)
2005 {
2006 struct drm_i915_gem_object *obj;
2007 struct i915_vma *vma;
2008 unsigned int size;
2009 int err;
2010
2011 size = round_up(count * sizeof(u32), PAGE_SIZE);
2012 obj = i915_gem_object_create_internal(vm->i915, size);
2013 if (IS_ERR(obj))
2014 return ERR_CAST(obj);
2015
2016 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
2017
2018 vma = i915_vma_instance(obj, vm, NULL);
2019 if (IS_ERR(vma)) {
2020 err = PTR_ERR(vma);
2021 goto err_obj;
2022 }
2023
2024 err = i915_vma_pin(vma, 0, 0,
2025 i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
2026 if (err)
2027 goto err_obj;
2028
2029 return vma;
2030
2031 err_obj:
2032 i915_gem_object_put(obj);
2033 return ERR_PTR(err);
2034 }
2035
2036 static const struct {
2037 u32 start;
2038 u32 end;
2039 } mcr_ranges_gen8[] = {
2040 { .start = 0x5500, .end = 0x55ff },
2041 { .start = 0x7000, .end = 0x7fff },
2042 { .start = 0x9400, .end = 0x97ff },
2043 { .start = 0xb000, .end = 0xb3ff },
2044 { .start = 0xe000, .end = 0xe7ff },
2045 {},
2046 };
2047
mcr_range(struct drm_i915_private * i915,u32 offset)2048 static bool mcr_range(struct drm_i915_private *i915, u32 offset)
2049 {
2050 int i;
2051
2052 if (INTEL_GEN(i915) < 8)
2053 return false;
2054
2055 /*
2056 * Registers in these ranges are affected by the MCR selector
2057 * which only controls CPU initiated MMIO. Routing does not
2058 * work for CS access so we cannot verify them on this path.
2059 */
2060 for (i = 0; mcr_ranges_gen8[i].start; i++)
2061 if (offset >= mcr_ranges_gen8[i].start &&
2062 offset <= mcr_ranges_gen8[i].end)
2063 return true;
2064
2065 return false;
2066 }
2067
2068 static int
wa_list_srm(struct i915_request * rq,const struct i915_wa_list * wal,struct i915_vma * vma)2069 wa_list_srm(struct i915_request *rq,
2070 const struct i915_wa_list *wal,
2071 struct i915_vma *vma)
2072 {
2073 struct drm_i915_private *i915 = rq->engine->i915;
2074 unsigned int i, count = 0;
2075 const struct i915_wa *wa;
2076 u32 srm, *cs;
2077
2078 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
2079 if (INTEL_GEN(i915) >= 8)
2080 srm++;
2081
2082 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2083 if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg)))
2084 count++;
2085 }
2086
2087 cs = intel_ring_begin(rq, 4 * count);
2088 if (IS_ERR(cs))
2089 return PTR_ERR(cs);
2090
2091 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2092 u32 offset = i915_mmio_reg_offset(wa->reg);
2093
2094 if (mcr_range(i915, offset))
2095 continue;
2096
2097 *cs++ = srm;
2098 *cs++ = offset;
2099 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
2100 *cs++ = 0;
2101 }
2102 intel_ring_advance(rq, cs);
2103
2104 return 0;
2105 }
2106
engine_wa_list_verify(struct intel_context * ce,const struct i915_wa_list * const wal,const char * from)2107 static int engine_wa_list_verify(struct intel_context *ce,
2108 const struct i915_wa_list * const wal,
2109 const char *from)
2110 {
2111 const struct i915_wa *wa;
2112 struct i915_request *rq;
2113 struct i915_vma *vma;
2114 struct i915_gem_ww_ctx ww;
2115 unsigned int i;
2116 u32 *results;
2117 int err;
2118
2119 if (!wal->count)
2120 return 0;
2121
2122 vma = create_scratch(&ce->engine->gt->ggtt->vm, wal->count);
2123 if (IS_ERR(vma))
2124 return PTR_ERR(vma);
2125
2126 intel_engine_pm_get(ce->engine);
2127 i915_gem_ww_ctx_init(&ww, false);
2128 retry:
2129 err = i915_gem_object_lock(vma->obj, &ww);
2130 if (err == 0)
2131 err = intel_context_pin_ww(ce, &ww);
2132 if (err)
2133 goto err_pm;
2134
2135 rq = i915_request_create(ce);
2136 if (IS_ERR(rq)) {
2137 err = PTR_ERR(rq);
2138 goto err_unpin;
2139 }
2140
2141 err = i915_request_await_object(rq, vma->obj, true);
2142 if (err == 0)
2143 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
2144 if (err == 0)
2145 err = wa_list_srm(rq, wal, vma);
2146
2147 i915_request_get(rq);
2148 if (err)
2149 i915_request_set_error_once(rq, err);
2150 i915_request_add(rq);
2151
2152 if (err)
2153 goto err_rq;
2154
2155 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2156 err = -ETIME;
2157 goto err_rq;
2158 }
2159
2160 results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
2161 if (IS_ERR(results)) {
2162 err = PTR_ERR(results);
2163 goto err_rq;
2164 }
2165
2166 err = 0;
2167 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2168 if (mcr_range(rq->engine->i915, i915_mmio_reg_offset(wa->reg)))
2169 continue;
2170
2171 if (!wa_verify(wa, results[i], wal->name, from))
2172 err = -ENXIO;
2173 }
2174
2175 i915_gem_object_unpin_map(vma->obj);
2176
2177 err_rq:
2178 i915_request_put(rq);
2179 err_unpin:
2180 intel_context_unpin(ce);
2181 err_pm:
2182 if (err == -EDEADLK) {
2183 err = i915_gem_ww_ctx_backoff(&ww);
2184 if (!err)
2185 goto retry;
2186 }
2187 i915_gem_ww_ctx_fini(&ww);
2188 intel_engine_pm_put(ce->engine);
2189 i915_vma_unpin(vma);
2190 i915_vma_put(vma);
2191 return err;
2192 }
2193
intel_engine_verify_workarounds(struct intel_engine_cs * engine,const char * from)2194 int intel_engine_verify_workarounds(struct intel_engine_cs *engine,
2195 const char *from)
2196 {
2197 return engine_wa_list_verify(engine->kernel_context,
2198 &engine->wa_list,
2199 from);
2200 }
2201
2202 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2203 #include "selftest_workarounds.c"
2204 #endif
2205