1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2014-2018 Intel Corporation
4 */
5
6 #include "i915_drv.h"
7 #include "intel_context.h"
8 #include "intel_engine_pm.h"
9 #include "intel_gpu_commands.h"
10 #include "intel_gt.h"
11 #include "intel_ring.h"
12 #include "intel_workarounds.h"
13
14 /**
15 * DOC: Hardware workarounds
16 *
17 * This file is intended as a central place to implement most [1]_ of the
18 * required workarounds for hardware to work as originally intended. They fall
19 * in five basic categories depending on how/when they are applied:
20 *
21 * - Workarounds that touch registers that are saved/restored to/from the HW
22 * context image. The list is emitted (via Load Register Immediate commands)
23 * everytime a new context is created.
24 * - GT workarounds. The list of these WAs is applied whenever these registers
25 * revert to default values (on GPU reset, suspend/resume [2]_, etc..).
26 * - Display workarounds. The list is applied during display clock-gating
27 * initialization.
28 * - Workarounds that whitelist a privileged register, so that UMDs can manage
29 * them directly. This is just a special case of a MMMIO workaround (as we
30 * write the list of these to/be-whitelisted registers to some special HW
31 * registers).
32 * - Workaround batchbuffers, that get executed automatically by the hardware
33 * on every HW context restore.
34 *
35 * .. [1] Please notice that there are other WAs that, due to their nature,
36 * cannot be applied from a central place. Those are peppered around the rest
37 * of the code, as needed.
38 *
39 * .. [2] Technically, some registers are powercontext saved & restored, so they
40 * survive a suspend/resume. In practice, writing them again is not too
41 * costly and simplifies things. We can revisit this in the future.
42 *
43 * Layout
44 * ~~~~~~
45 *
46 * Keep things in this file ordered by WA type, as per the above (context, GT,
47 * display, register whitelist, batchbuffer). Then, inside each type, keep the
48 * following order:
49 *
50 * - Infrastructure functions and macros
51 * - WAs per platform in standard gen/chrono order
52 * - Public functions to init or apply the given workaround type.
53 */
54
wa_init_start(struct i915_wa_list * wal,const char * name,const char * engine_name)55 static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name)
56 {
57 wal->name = name;
58 wal->engine_name = engine_name;
59 }
60
61 #define WA_LIST_CHUNK (1 << 4)
62
wa_init_finish(struct i915_wa_list * wal)63 static void wa_init_finish(struct i915_wa_list *wal)
64 {
65 /* Trim unused entries. */
66 if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) {
67 struct i915_wa *list = kmemdup(wal->list,
68 wal->count * sizeof(*list),
69 GFP_KERNEL);
70
71 if (list) {
72 kfree(wal->list);
73 wal->list = list;
74 }
75 }
76
77 if (!wal->count)
78 return;
79
80 DRM_DEBUG_DRIVER("Initialized %u %s workarounds on %s\n",
81 wal->wa_count, wal->name, wal->engine_name);
82 }
83
_wa_add(struct i915_wa_list * wal,const struct i915_wa * wa)84 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
85 {
86 unsigned int addr = i915_mmio_reg_offset(wa->reg);
87 unsigned int start = 0, end = wal->count;
88 const unsigned int grow = WA_LIST_CHUNK;
89 struct i915_wa *wa_;
90
91 GEM_BUG_ON(!is_power_of_2(grow));
92
93 if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
94 struct i915_wa *list;
95
96 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa),
97 GFP_KERNEL);
98 if (!list) {
99 DRM_ERROR("No space for workaround init!\n");
100 return;
101 }
102
103 if (wal->list) {
104 memcpy(list, wal->list, sizeof(*wa) * wal->count);
105 kfree(wal->list);
106 }
107
108 wal->list = list;
109 }
110
111 while (start < end) {
112 unsigned int mid = start + (end - start) / 2;
113
114 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
115 start = mid + 1;
116 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
117 end = mid;
118 } else {
119 wa_ = &wal->list[mid];
120
121 if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) {
122 DRM_ERROR("Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n",
123 i915_mmio_reg_offset(wa_->reg),
124 wa_->clr, wa_->set);
125
126 wa_->set &= ~wa->clr;
127 }
128
129 wal->wa_count++;
130 wa_->set |= wa->set;
131 wa_->clr |= wa->clr;
132 wa_->read |= wa->read;
133 return;
134 }
135 }
136
137 wal->wa_count++;
138 wa_ = &wal->list[wal->count++];
139 *wa_ = *wa;
140
141 while (wa_-- > wal->list) {
142 GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==
143 i915_mmio_reg_offset(wa_[1].reg));
144 if (i915_mmio_reg_offset(wa_[1].reg) >
145 i915_mmio_reg_offset(wa_[0].reg))
146 break;
147
148 swap(wa_[1], wa_[0]);
149 }
150 }
151
wa_add(struct i915_wa_list * wal,i915_reg_t reg,u32 clear,u32 set,u32 read_mask,bool masked_reg)152 static void wa_add(struct i915_wa_list *wal, i915_reg_t reg,
153 u32 clear, u32 set, u32 read_mask, bool masked_reg)
154 {
155 struct i915_wa wa = {
156 .reg = reg,
157 .clr = clear,
158 .set = set,
159 .read = read_mask,
160 .masked_reg = masked_reg,
161 };
162
163 _wa_add(wal, &wa);
164 }
165
166 static void
wa_write_clr_set(struct i915_wa_list * wal,i915_reg_t reg,u32 clear,u32 set)167 wa_write_clr_set(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set)
168 {
169 wa_add(wal, reg, clear, set, clear, false);
170 }
171
172 static void
wa_write(struct i915_wa_list * wal,i915_reg_t reg,u32 set)173 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
174 {
175 wa_write_clr_set(wal, reg, ~0, set);
176 }
177
178 static void
wa_write_or(struct i915_wa_list * wal,i915_reg_t reg,u32 set)179 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
180 {
181 wa_write_clr_set(wal, reg, set, set);
182 }
183
184 static void
wa_write_clr(struct i915_wa_list * wal,i915_reg_t reg,u32 clr)185 wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr)
186 {
187 wa_write_clr_set(wal, reg, clr, 0);
188 }
189
190 /*
191 * WA operations on "masked register". A masked register has the upper 16 bits
192 * documented as "masked" in b-spec. Its purpose is to allow writing to just a
193 * portion of the register without a rmw: you simply write in the upper 16 bits
194 * the mask of bits you are going to modify.
195 *
196 * The wa_masked_* family of functions already does the necessary operations to
197 * calculate the mask based on the parameters passed, so user only has to
198 * provide the lower 16 bits of that register.
199 */
200
201 static void
wa_masked_en(struct i915_wa_list * wal,i915_reg_t reg,u32 val)202 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
203 {
204 wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val, true);
205 }
206
207 static void
wa_masked_dis(struct i915_wa_list * wal,i915_reg_t reg,u32 val)208 wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
209 {
210 wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val, true);
211 }
212
213 static void
wa_masked_field_set(struct i915_wa_list * wal,i915_reg_t reg,u32 mask,u32 val)214 wa_masked_field_set(struct i915_wa_list *wal, i915_reg_t reg,
215 u32 mask, u32 val)
216 {
217 wa_add(wal, reg, 0, _MASKED_FIELD(mask, val), mask, true);
218 }
219
gen6_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)220 static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine,
221 struct i915_wa_list *wal)
222 {
223 wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
224 }
225
gen7_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)226 static void gen7_ctx_workarounds_init(struct intel_engine_cs *engine,
227 struct i915_wa_list *wal)
228 {
229 wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
230 }
231
gen8_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)232 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine,
233 struct i915_wa_list *wal)
234 {
235 wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
236
237 /* WaDisableAsyncFlipPerfMode:bdw,chv */
238 wa_masked_en(wal, MI_MODE, ASYNC_FLIP_PERF_DISABLE);
239
240 /* WaDisablePartialInstShootdown:bdw,chv */
241 wa_masked_en(wal, GEN8_ROW_CHICKEN,
242 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
243
244 /* Use Force Non-Coherent whenever executing a 3D context. This is a
245 * workaround for a possible hang in the unlikely event a TLB
246 * invalidation occurs during a PSD flush.
247 */
248 /* WaForceEnableNonCoherent:bdw,chv */
249 /* WaHdcDisableFetchWhenMasked:bdw,chv */
250 wa_masked_en(wal, HDC_CHICKEN0,
251 HDC_DONOT_FETCH_MEM_WHEN_MASKED |
252 HDC_FORCE_NON_COHERENT);
253
254 /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
255 * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
256 * polygons in the same 8x4 pixel/sample area to be processed without
257 * stalling waiting for the earlier ones to write to Hierarchical Z
258 * buffer."
259 *
260 * This optimization is off by default for BDW and CHV; turn it on.
261 */
262 wa_masked_dis(wal, CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
263
264 /* Wa4x4STCOptimizationDisable:bdw,chv */
265 wa_masked_en(wal, CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
266
267 /*
268 * BSpec recommends 8x4 when MSAA is used,
269 * however in practice 16x4 seems fastest.
270 *
271 * Note that PS/WM thread counts depend on the WIZ hashing
272 * disable bit, which we don't touch here, but it's good
273 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
274 */
275 wa_masked_field_set(wal, GEN7_GT_MODE,
276 GEN6_WIZ_HASHING_MASK,
277 GEN6_WIZ_HASHING_16x4);
278 }
279
bdw_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)280 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine,
281 struct i915_wa_list *wal)
282 {
283 struct drm_i915_private *i915 = engine->i915;
284
285 gen8_ctx_workarounds_init(engine, wal);
286
287 /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
288 wa_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
289
290 /* WaDisableDopClockGating:bdw
291 *
292 * Also see the related UCGTCL1 write in bdw_init_clock_gating()
293 * to disable EUTC clock gating.
294 */
295 wa_masked_en(wal, GEN7_ROW_CHICKEN2,
296 DOP_CLOCK_GATING_DISABLE);
297
298 wa_masked_en(wal, HALF_SLICE_CHICKEN3,
299 GEN8_SAMPLER_POWER_BYPASS_DIS);
300
301 wa_masked_en(wal, HDC_CHICKEN0,
302 /* WaForceContextSaveRestoreNonCoherent:bdw */
303 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
304 /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
305 (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
306 }
307
chv_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)308 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine,
309 struct i915_wa_list *wal)
310 {
311 gen8_ctx_workarounds_init(engine, wal);
312
313 /* WaDisableThreadStallDopClockGating:chv */
314 wa_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
315
316 /* Improve HiZ throughput on CHV. */
317 wa_masked_en(wal, HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
318 }
319
gen9_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)320 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine,
321 struct i915_wa_list *wal)
322 {
323 struct drm_i915_private *i915 = engine->i915;
324
325 if (HAS_LLC(i915)) {
326 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
327 *
328 * Must match Display Engine. See
329 * WaCompressedResourceDisplayNewHashMode.
330 */
331 wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
332 GEN9_PBE_COMPRESSED_HASH_SELECTION);
333 wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7,
334 GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
335 }
336
337 /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
338 /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
339 wa_masked_en(wal, GEN8_ROW_CHICKEN,
340 FLOW_CONTROL_ENABLE |
341 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
342
343 /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
344 /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
345 wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7,
346 GEN9_ENABLE_YV12_BUGFIX |
347 GEN9_ENABLE_GPGPU_PREEMPTION);
348
349 /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
350 /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
351 wa_masked_en(wal, CACHE_MODE_1,
352 GEN8_4x4_STC_OPTIMIZATION_DISABLE |
353 GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
354
355 /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
356 wa_masked_dis(wal, GEN9_HALF_SLICE_CHICKEN5,
357 GEN9_CCS_TLB_PREFETCH_ENABLE);
358
359 /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
360 wa_masked_en(wal, HDC_CHICKEN0,
361 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
362 HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
363
364 /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
365 * both tied to WaForceContextSaveRestoreNonCoherent
366 * in some hsds for skl. We keep the tie for all gen9. The
367 * documentation is a bit hazy and so we want to get common behaviour,
368 * even though there is no clear evidence we would need both on kbl/bxt.
369 * This area has been source of system hangs so we play it safe
370 * and mimic the skl regardless of what bspec says.
371 *
372 * Use Force Non-Coherent whenever executing a 3D context. This
373 * is a workaround for a possible hang in the unlikely event
374 * a TLB invalidation occurs during a PSD flush.
375 */
376
377 /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
378 wa_masked_en(wal, HDC_CHICKEN0,
379 HDC_FORCE_NON_COHERENT);
380
381 /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
382 if (IS_SKYLAKE(i915) ||
383 IS_KABYLAKE(i915) ||
384 IS_COFFEELAKE(i915) ||
385 IS_COMETLAKE(i915))
386 wa_masked_en(wal, HALF_SLICE_CHICKEN3,
387 GEN8_SAMPLER_POWER_BYPASS_DIS);
388
389 /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
390 wa_masked_en(wal, HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
391
392 /*
393 * Supporting preemption with fine-granularity requires changes in the
394 * batch buffer programming. Since we can't break old userspace, we
395 * need to set our default preemption level to safe value. Userspace is
396 * still able to use more fine-grained preemption levels, since in
397 * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
398 * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
399 * not real HW workarounds, but merely a way to start using preemption
400 * while maintaining old contract with userspace.
401 */
402
403 /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
404 wa_masked_dis(wal, GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
405
406 /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
407 wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
408 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
409 GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
410
411 /* WaClearHIZ_WM_CHICKEN3:bxt,glk */
412 if (IS_GEN9_LP(i915))
413 wa_masked_en(wal, GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
414 }
415
skl_tune_iz_hashing(struct intel_engine_cs * engine,struct i915_wa_list * wal)416 static void skl_tune_iz_hashing(struct intel_engine_cs *engine,
417 struct i915_wa_list *wal)
418 {
419 struct intel_gt *gt = engine->gt;
420 u8 vals[3] = { 0, 0, 0 };
421 unsigned int i;
422
423 for (i = 0; i < 3; i++) {
424 u8 ss;
425
426 /*
427 * Only consider slices where one, and only one, subslice has 7
428 * EUs
429 */
430 if (!is_power_of_2(gt->info.sseu.subslice_7eu[i]))
431 continue;
432
433 /*
434 * subslice_7eu[i] != 0 (because of the check above) and
435 * ss_max == 4 (maximum number of subslices possible per slice)
436 *
437 * -> 0 <= ss <= 3;
438 */
439 ss = ffs(gt->info.sseu.subslice_7eu[i]) - 1;
440 vals[i] = 3 - ss;
441 }
442
443 if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
444 return;
445
446 /* Tune IZ hashing. See intel_device_info_runtime_init() */
447 wa_masked_field_set(wal, GEN7_GT_MODE,
448 GEN9_IZ_HASHING_MASK(2) |
449 GEN9_IZ_HASHING_MASK(1) |
450 GEN9_IZ_HASHING_MASK(0),
451 GEN9_IZ_HASHING(2, vals[2]) |
452 GEN9_IZ_HASHING(1, vals[1]) |
453 GEN9_IZ_HASHING(0, vals[0]));
454 }
455
skl_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)456 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine,
457 struct i915_wa_list *wal)
458 {
459 gen9_ctx_workarounds_init(engine, wal);
460 skl_tune_iz_hashing(engine, wal);
461 }
462
bxt_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)463 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine,
464 struct i915_wa_list *wal)
465 {
466 gen9_ctx_workarounds_init(engine, wal);
467
468 /* WaDisableThreadStallDopClockGating:bxt */
469 wa_masked_en(wal, GEN8_ROW_CHICKEN,
470 STALL_DOP_GATING_DISABLE);
471
472 /* WaToEnableHwFixForPushConstHWBug:bxt */
473 wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
474 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
475 }
476
kbl_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)477 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine,
478 struct i915_wa_list *wal)
479 {
480 struct drm_i915_private *i915 = engine->i915;
481
482 gen9_ctx_workarounds_init(engine, wal);
483
484 /* WaToEnableHwFixForPushConstHWBug:kbl */
485 if (IS_KBL_GT_STEP(i915, STEP_C0, STEP_FOREVER))
486 wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
487 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
488
489 /* WaDisableSbeCacheDispatchPortSharing:kbl */
490 wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1,
491 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
492 }
493
glk_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)494 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine,
495 struct i915_wa_list *wal)
496 {
497 gen9_ctx_workarounds_init(engine, wal);
498
499 /* WaToEnableHwFixForPushConstHWBug:glk */
500 wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
501 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
502 }
503
cfl_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)504 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine,
505 struct i915_wa_list *wal)
506 {
507 gen9_ctx_workarounds_init(engine, wal);
508
509 /* WaToEnableHwFixForPushConstHWBug:cfl */
510 wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
511 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
512
513 /* WaDisableSbeCacheDispatchPortSharing:cfl */
514 wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1,
515 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
516 }
517
icl_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)518 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
519 struct i915_wa_list *wal)
520 {
521 /* Wa_1406697149 (WaDisableBankHangMode:icl) */
522 wa_write(wal,
523 GEN8_L3CNTLREG,
524 intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) |
525 GEN8_ERRDETBCTRL);
526
527 /* WaForceEnableNonCoherent:icl
528 * This is not the same workaround as in early Gen9 platforms, where
529 * lacking this could cause system hangs, but coherency performance
530 * overhead is high and only a few compute workloads really need it
531 * (the register is whitelisted in hardware now, so UMDs can opt in
532 * for coherency if they have a good reason).
533 */
534 wa_masked_en(wal, ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
535
536 /* WaEnableFloatBlendOptimization:icl */
537 wa_add(wal, GEN10_CACHE_MODE_SS, 0,
538 _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE),
539 0 /* write-only, so skip validation */,
540 true);
541
542 /* WaDisableGPGPUMidThreadPreemption:icl */
543 wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
544 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
545 GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
546
547 /* allow headerless messages for preemptible GPGPU context */
548 wa_masked_en(wal, GEN10_SAMPLER_MODE,
549 GEN11_SAMPLER_ENABLE_HEADLESS_MSG);
550
551 /* Wa_1604278689:icl,ehl */
552 wa_write(wal, IVB_FBC_RT_BASE, 0xFFFFFFFF & ~ILK_FBC_RT_VALID);
553 wa_write_clr_set(wal, IVB_FBC_RT_BASE_UPPER,
554 0, /* write-only register; skip validation */
555 0xFFFFFFFF);
556
557 /* Wa_1406306137:icl,ehl */
558 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU);
559 }
560
561 /*
562 * These settings aren't actually workarounds, but general tuning settings that
563 * need to be programmed on several platforms.
564 */
gen12_ctx_gt_tuning_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)565 static void gen12_ctx_gt_tuning_init(struct intel_engine_cs *engine,
566 struct i915_wa_list *wal)
567 {
568 /*
569 * Although some platforms refer to it as Wa_1604555607, we need to
570 * program it even on those that don't explicitly list that
571 * workaround.
572 *
573 * Note that the programming of this register is further modified
574 * according to the FF_MODE2 guidance given by Wa_1608008084:gen12.
575 * Wa_1608008084 tells us the FF_MODE2 register will return the wrong
576 * value when read. The default value for this register is zero for all
577 * fields and there are no bit masks. So instead of doing a RMW we
578 * should just write TDS timer value. For the same reason read
579 * verification is ignored.
580 */
581 wa_add(wal,
582 FF_MODE2,
583 FF_MODE2_TDS_TIMER_MASK,
584 FF_MODE2_TDS_TIMER_128,
585 0, false);
586 }
587
gen12_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)588 static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine,
589 struct i915_wa_list *wal)
590 {
591 gen12_ctx_gt_tuning_init(engine, wal);
592
593 /*
594 * Wa_1409142259:tgl,dg1,adl-p
595 * Wa_1409347922:tgl,dg1,adl-p
596 * Wa_1409252684:tgl,dg1,adl-p
597 * Wa_1409217633:tgl,dg1,adl-p
598 * Wa_1409207793:tgl,dg1,adl-p
599 * Wa_1409178076:tgl,dg1,adl-p
600 * Wa_1408979724:tgl,dg1,adl-p
601 * Wa_14010443199:tgl,rkl,dg1,adl-p
602 * Wa_14010698770:tgl,rkl,dg1,adl-s,adl-p
603 * Wa_1409342910:tgl,rkl,dg1,adl-s,adl-p
604 */
605 wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3,
606 GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
607
608 /* WaDisableGPGPUMidThreadPreemption:gen12 */
609 wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
610 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
611 GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
612
613 /*
614 * Wa_16011163337
615 *
616 * Like in gen12_ctx_gt_tuning_init(), read verification is ignored due
617 * to Wa_1608008084.
618 */
619 wa_add(wal,
620 FF_MODE2,
621 FF_MODE2_GS_TIMER_MASK,
622 FF_MODE2_GS_TIMER_224,
623 0, false);
624 }
625
dg1_ctx_workarounds_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)626 static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine,
627 struct i915_wa_list *wal)
628 {
629 gen12_ctx_workarounds_init(engine, wal);
630
631 /* Wa_1409044764 */
632 wa_masked_dis(wal, GEN11_COMMON_SLICE_CHICKEN3,
633 DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN);
634
635 /* Wa_22010493298 */
636 wa_masked_en(wal, HIZ_CHICKEN,
637 DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE);
638 }
639
640 static void
__intel_engine_init_ctx_wa(struct intel_engine_cs * engine,struct i915_wa_list * wal,const char * name)641 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
642 struct i915_wa_list *wal,
643 const char *name)
644 {
645 struct drm_i915_private *i915 = engine->i915;
646
647 if (engine->class != RENDER_CLASS)
648 return;
649
650 wa_init_start(wal, name, engine->name);
651
652 if (IS_DG1(i915))
653 dg1_ctx_workarounds_init(engine, wal);
654 else if (GRAPHICS_VER(i915) == 12)
655 gen12_ctx_workarounds_init(engine, wal);
656 else if (GRAPHICS_VER(i915) == 11)
657 icl_ctx_workarounds_init(engine, wal);
658 else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
659 cfl_ctx_workarounds_init(engine, wal);
660 else if (IS_GEMINILAKE(i915))
661 glk_ctx_workarounds_init(engine, wal);
662 else if (IS_KABYLAKE(i915))
663 kbl_ctx_workarounds_init(engine, wal);
664 else if (IS_BROXTON(i915))
665 bxt_ctx_workarounds_init(engine, wal);
666 else if (IS_SKYLAKE(i915))
667 skl_ctx_workarounds_init(engine, wal);
668 else if (IS_CHERRYVIEW(i915))
669 chv_ctx_workarounds_init(engine, wal);
670 else if (IS_BROADWELL(i915))
671 bdw_ctx_workarounds_init(engine, wal);
672 else if (GRAPHICS_VER(i915) == 7)
673 gen7_ctx_workarounds_init(engine, wal);
674 else if (GRAPHICS_VER(i915) == 6)
675 gen6_ctx_workarounds_init(engine, wal);
676 else if (GRAPHICS_VER(i915) < 8)
677 ;
678 else
679 MISSING_CASE(GRAPHICS_VER(i915));
680
681 wa_init_finish(wal);
682 }
683
intel_engine_init_ctx_wa(struct intel_engine_cs * engine)684 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
685 {
686 __intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context");
687 }
688
intel_engine_emit_ctx_wa(struct i915_request * rq)689 int intel_engine_emit_ctx_wa(struct i915_request *rq)
690 {
691 struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
692 struct i915_wa *wa;
693 unsigned int i;
694 u32 *cs;
695 int ret;
696
697 if (wal->count == 0)
698 return 0;
699
700 ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
701 if (ret)
702 return ret;
703
704 cs = intel_ring_begin(rq, (wal->count * 2 + 2));
705 if (IS_ERR(cs))
706 return PTR_ERR(cs);
707
708 *cs++ = MI_LOAD_REGISTER_IMM(wal->count);
709 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
710 *cs++ = i915_mmio_reg_offset(wa->reg);
711 *cs++ = wa->set;
712 }
713 *cs++ = MI_NOOP;
714
715 intel_ring_advance(rq, cs);
716
717 ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
718 if (ret)
719 return ret;
720
721 return 0;
722 }
723
724 static void
gen4_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)725 gen4_gt_workarounds_init(struct drm_i915_private *i915,
726 struct i915_wa_list *wal)
727 {
728 /* WaDisable_RenderCache_OperationalFlush:gen4,ilk */
729 wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
730 }
731
732 static void
g4x_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)733 g4x_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
734 {
735 gen4_gt_workarounds_init(i915, wal);
736
737 /* WaDisableRenderCachePipelinedFlush:g4x,ilk */
738 wa_masked_en(wal, CACHE_MODE_0, CM0_PIPELINED_RENDER_FLUSH_DISABLE);
739 }
740
741 static void
ilk_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)742 ilk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
743 {
744 g4x_gt_workarounds_init(i915, wal);
745
746 wa_masked_en(wal, _3D_CHICKEN2, _3D_CHICKEN2_WM_READ_PIPELINED);
747 }
748
749 static void
snb_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)750 snb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
751 {
752 }
753
754 static void
ivb_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)755 ivb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
756 {
757 /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
758 wa_masked_dis(wal,
759 GEN7_COMMON_SLICE_CHICKEN1,
760 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
761
762 /* WaApplyL3ControlAndL3ChickenMode:ivb */
763 wa_write(wal, GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL);
764 wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
765
766 /* WaForceL3Serialization:ivb */
767 wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
768 }
769
770 static void
vlv_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)771 vlv_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
772 {
773 /* WaForceL3Serialization:vlv */
774 wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
775
776 /*
777 * WaIncreaseL3CreditsForVLVB0:vlv
778 * This is the hardware default actually.
779 */
780 wa_write(wal, GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
781 }
782
783 static void
hsw_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)784 hsw_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
785 {
786 /* L3 caching of data atomics doesn't work -- disable it. */
787 wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
788
789 wa_add(wal,
790 HSW_ROW_CHICKEN3, 0,
791 _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE),
792 0 /* XXX does this reg exist? */, true);
793
794 /* WaVSRefCountFullforceMissDisable:hsw */
795 wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME);
796 }
797
798 static void
gen9_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)799 gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
800 {
801 /* WaDisableKillLogic:bxt,skl,kbl */
802 if (!IS_COFFEELAKE(i915) && !IS_COMETLAKE(i915))
803 wa_write_or(wal,
804 GAM_ECOCHK,
805 ECOCHK_DIS_TLB);
806
807 if (HAS_LLC(i915)) {
808 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
809 *
810 * Must match Display Engine. See
811 * WaCompressedResourceDisplayNewHashMode.
812 */
813 wa_write_or(wal,
814 MMCD_MISC_CTRL,
815 MMCD_PCLA | MMCD_HOTSPOT_EN);
816 }
817
818 /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
819 wa_write_or(wal,
820 GAM_ECOCHK,
821 BDW_DISABLE_HDC_INVALIDATION);
822 }
823
824 static void
skl_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)825 skl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
826 {
827 gen9_gt_workarounds_init(i915, wal);
828
829 /* WaDisableGafsUnitClkGating:skl */
830 wa_write_or(wal,
831 GEN7_UCGCTL4,
832 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
833
834 /* WaInPlaceDecompressionHang:skl */
835 if (IS_SKL_GT_STEP(i915, STEP_A0, STEP_H0))
836 wa_write_or(wal,
837 GEN9_GAMT_ECO_REG_RW_IA,
838 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
839 }
840
841 static void
kbl_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)842 kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
843 {
844 gen9_gt_workarounds_init(i915, wal);
845
846 /* WaDisableDynamicCreditSharing:kbl */
847 if (IS_KBL_GT_STEP(i915, 0, STEP_C0))
848 wa_write_or(wal,
849 GAMT_CHKN_BIT_REG,
850 GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
851
852 /* WaDisableGafsUnitClkGating:kbl */
853 wa_write_or(wal,
854 GEN7_UCGCTL4,
855 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
856
857 /* WaInPlaceDecompressionHang:kbl */
858 wa_write_or(wal,
859 GEN9_GAMT_ECO_REG_RW_IA,
860 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
861 }
862
863 static void
glk_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)864 glk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
865 {
866 gen9_gt_workarounds_init(i915, wal);
867 }
868
869 static void
cfl_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)870 cfl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
871 {
872 gen9_gt_workarounds_init(i915, wal);
873
874 /* WaDisableGafsUnitClkGating:cfl */
875 wa_write_or(wal,
876 GEN7_UCGCTL4,
877 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
878
879 /* WaInPlaceDecompressionHang:cfl */
880 wa_write_or(wal,
881 GEN9_GAMT_ECO_REG_RW_IA,
882 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
883 }
884
__set_mcr_steering(struct i915_wa_list * wal,i915_reg_t steering_reg,unsigned int slice,unsigned int subslice)885 static void __set_mcr_steering(struct i915_wa_list *wal,
886 i915_reg_t steering_reg,
887 unsigned int slice, unsigned int subslice)
888 {
889 u32 mcr, mcr_mask;
890
891 mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice);
892 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
893
894 wa_write_clr_set(wal, steering_reg, mcr_mask, mcr);
895 }
896
__add_mcr_wa(struct drm_i915_private * i915,struct i915_wa_list * wal,unsigned int slice,unsigned int subslice)897 static void __add_mcr_wa(struct drm_i915_private *i915, struct i915_wa_list *wal,
898 unsigned int slice, unsigned int subslice)
899 {
900 drm_dbg(&i915->drm, "MCR slice=0x%x, subslice=0x%x\n", slice, subslice);
901
902 __set_mcr_steering(wal, GEN8_MCR_SELECTOR, slice, subslice);
903 }
904
905 static void
icl_wa_init_mcr(struct drm_i915_private * i915,struct i915_wa_list * wal)906 icl_wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal)
907 {
908 const struct sseu_dev_info *sseu = &i915->gt.info.sseu;
909 unsigned int slice, subslice;
910
911 GEM_BUG_ON(GRAPHICS_VER(i915) < 11);
912 GEM_BUG_ON(hweight8(sseu->slice_mask) > 1);
913 slice = 0;
914
915 /*
916 * Although a platform may have subslices, we need to always steer
917 * reads to the lowest instance that isn't fused off. When Render
918 * Power Gating is enabled, grabbing forcewake will only power up a
919 * single subslice (the "minconfig") if there isn't a real workload
920 * that needs to be run; this means that if we steer register reads to
921 * one of the higher subslices, we run the risk of reading back 0's or
922 * random garbage.
923 */
924 subslice = __ffs(intel_sseu_get_subslices(sseu, slice));
925
926 /*
927 * If the subslice we picked above also steers us to a valid L3 bank,
928 * then we can just rely on the default steering and won't need to
929 * worry about explicitly re-steering L3BANK reads later.
930 */
931 if (i915->gt.info.l3bank_mask & BIT(subslice))
932 i915->gt.steering_table[L3BANK] = NULL;
933
934 __add_mcr_wa(i915, wal, slice, subslice);
935 }
936
937 static void
xehp_init_mcr(struct intel_gt * gt,struct i915_wa_list * wal)938 xehp_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
939 {
940 struct drm_i915_private *i915 = gt->i915;
941 const struct sseu_dev_info *sseu = >->info.sseu;
942 unsigned long slice, subslice = 0, slice_mask = 0;
943 u64 dss_mask = 0;
944 u32 lncf_mask = 0;
945 int i;
946
947 /*
948 * On Xe_HP the steering increases in complexity. There are now several
949 * more units that require steering and we're not guaranteed to be able
950 * to find a common setting for all of them. These are:
951 * - GSLICE (fusable)
952 * - DSS (sub-unit within gslice; fusable)
953 * - L3 Bank (fusable)
954 * - MSLICE (fusable)
955 * - LNCF (sub-unit within mslice; always present if mslice is present)
956 *
957 * We'll do our default/implicit steering based on GSLICE (in the
958 * sliceid field) and DSS (in the subsliceid field). If we can
959 * find overlap between the valid MSLICE and/or LNCF values with
960 * a suitable GSLICE, then we can just re-use the default value and
961 * skip and explicit steering at runtime.
962 *
963 * We only need to look for overlap between GSLICE/MSLICE/LNCF to find
964 * a valid sliceid value. DSS steering is the only type of steering
965 * that utilizes the 'subsliceid' bits.
966 *
967 * Also note that, even though the steering domain is called "GSlice"
968 * and it is encoded in the register using the gslice format, the spec
969 * says that the combined (geometry | compute) fuse should be used to
970 * select the steering.
971 */
972
973 /* Find the potential gslice candidates */
974 dss_mask = intel_sseu_get_subslices(sseu, 0);
975 slice_mask = intel_slicemask_from_dssmask(dss_mask, GEN_DSS_PER_GSLICE);
976
977 /*
978 * Find the potential LNCF candidates. Either LNCF within a valid
979 * mslice is fine.
980 */
981 for_each_set_bit(i, >->info.mslice_mask, GEN12_MAX_MSLICES)
982 lncf_mask |= (0x3 << (i * 2));
983
984 /*
985 * Are there any sliceid values that work for both GSLICE and LNCF
986 * steering?
987 */
988 if (slice_mask & lncf_mask) {
989 slice_mask &= lncf_mask;
990 gt->steering_table[LNCF] = NULL;
991 }
992
993 /* How about sliceid values that also work for MSLICE steering? */
994 if (slice_mask & gt->info.mslice_mask) {
995 slice_mask &= gt->info.mslice_mask;
996 gt->steering_table[MSLICE] = NULL;
997 }
998
999 slice = __ffs(slice_mask);
1000 subslice = __ffs(dss_mask >> (slice * GEN_DSS_PER_GSLICE));
1001 WARN_ON(subslice > GEN_DSS_PER_GSLICE);
1002 WARN_ON(dss_mask >> (slice * GEN_DSS_PER_GSLICE) == 0);
1003
1004 __add_mcr_wa(i915, wal, slice, subslice);
1005
1006 /*
1007 * SQIDI ranges are special because they use different steering
1008 * registers than everything else we work with. On XeHP SDV and
1009 * DG2-G10, any value in the steering registers will work fine since
1010 * all instances are present, but DG2-G11 only has SQIDI instances at
1011 * ID's 2 and 3, so we need to steer to one of those. For simplicity
1012 * we'll just steer to a hardcoded "2" since that value will work
1013 * everywhere.
1014 */
1015 __set_mcr_steering(wal, MCFG_MCR_SELECTOR, 0, 2);
1016 __set_mcr_steering(wal, SF_MCR_SELECTOR, 0, 2);
1017 }
1018
1019 static void
icl_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)1020 icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1021 {
1022 icl_wa_init_mcr(i915, wal);
1023
1024 /* WaModifyGamTlbPartitioning:icl */
1025 wa_write_clr_set(wal,
1026 GEN11_GACB_PERF_CTRL,
1027 GEN11_HASH_CTRL_MASK,
1028 GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
1029
1030 /* Wa_1405766107:icl
1031 * Formerly known as WaCL2SFHalfMaxAlloc
1032 */
1033 wa_write_or(wal,
1034 GEN11_LSN_UNSLCVC,
1035 GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
1036 GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
1037
1038 /* Wa_220166154:icl
1039 * Formerly known as WaDisCtxReload
1040 */
1041 wa_write_or(wal,
1042 GEN8_GAMW_ECO_DEV_RW_IA,
1043 GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
1044
1045 /* Wa_1406463099:icl
1046 * Formerly known as WaGamTlbPendError
1047 */
1048 wa_write_or(wal,
1049 GAMT_CHKN_BIT_REG,
1050 GAMT_CHKN_DISABLE_L3_COH_PIPE);
1051
1052 /*
1053 * Wa_1408615072:icl,ehl (vsunit)
1054 * Wa_1407596294:icl,ehl (hsunit)
1055 */
1056 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1057 VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS);
1058
1059 /* Wa_1407352427:icl,ehl */
1060 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1061 PSDUNIT_CLKGATE_DIS);
1062
1063 /* Wa_1406680159:icl,ehl */
1064 wa_write_or(wal,
1065 SUBSLICE_UNIT_LEVEL_CLKGATE,
1066 GWUNIT_CLKGATE_DIS);
1067
1068 /* Wa_1607087056:icl,ehl,jsl */
1069 if (IS_ICELAKE(i915) ||
1070 IS_JSL_EHL_GT_STEP(i915, STEP_A0, STEP_B0))
1071 wa_write_or(wal,
1072 SLICE_UNIT_LEVEL_CLKGATE,
1073 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1074
1075 /*
1076 * This is not a documented workaround, but rather an optimization
1077 * to reduce sampler power.
1078 */
1079 wa_write_clr(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE);
1080 }
1081
1082 /*
1083 * Though there are per-engine instances of these registers,
1084 * they retain their value through engine resets and should
1085 * only be provided on the GT workaround list rather than
1086 * the engine-specific workaround list.
1087 */
1088 static void
wa_14011060649(struct drm_i915_private * i915,struct i915_wa_list * wal)1089 wa_14011060649(struct drm_i915_private *i915, struct i915_wa_list *wal)
1090 {
1091 struct intel_engine_cs *engine;
1092 struct intel_gt *gt = &i915->gt;
1093 int id;
1094
1095 for_each_engine(engine, gt, id) {
1096 if (engine->class != VIDEO_DECODE_CLASS ||
1097 (engine->instance % 2))
1098 continue;
1099
1100 wa_write_or(wal, VDBOX_CGCTL3F10(engine->mmio_base),
1101 IECPUNIT_CLKGATE_DIS);
1102 }
1103 }
1104
1105 static void
gen12_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)1106 gen12_gt_workarounds_init(struct drm_i915_private *i915,
1107 struct i915_wa_list *wal)
1108 {
1109 icl_wa_init_mcr(i915, wal);
1110
1111 /* Wa_14011060649:tgl,rkl,dg1,adl-s,adl-p */
1112 wa_14011060649(i915, wal);
1113
1114 /* Wa_14011059788:tgl,rkl,adl-s,dg1,adl-p */
1115 wa_write_or(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE);
1116 }
1117
1118 static void
tgl_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)1119 tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1120 {
1121 gen12_gt_workarounds_init(i915, wal);
1122
1123 /* Wa_1409420604:tgl */
1124 if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0))
1125 wa_write_or(wal,
1126 SUBSLICE_UNIT_LEVEL_CLKGATE2,
1127 CPSSUNIT_CLKGATE_DIS);
1128
1129 /* Wa_1607087056:tgl also know as BUG:1409180338 */
1130 if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0))
1131 wa_write_or(wal,
1132 SLICE_UNIT_LEVEL_CLKGATE,
1133 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1134
1135 /* Wa_1408615072:tgl[a0] */
1136 if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0))
1137 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1138 VSUNIT_CLKGATE_DIS_TGL);
1139 }
1140
1141 static void
dg1_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)1142 dg1_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1143 {
1144 gen12_gt_workarounds_init(i915, wal);
1145
1146 /* Wa_1607087056:dg1 */
1147 if (IS_DG1_GT_STEP(i915, STEP_A0, STEP_B0))
1148 wa_write_or(wal,
1149 SLICE_UNIT_LEVEL_CLKGATE,
1150 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1151
1152 /* Wa_1409420604:dg1 */
1153 if (IS_DG1(i915))
1154 wa_write_or(wal,
1155 SUBSLICE_UNIT_LEVEL_CLKGATE2,
1156 CPSSUNIT_CLKGATE_DIS);
1157
1158 /* Wa_1408615072:dg1 */
1159 /* Empirical testing shows this register is unaffected by engine reset. */
1160 if (IS_DG1(i915))
1161 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1162 VSUNIT_CLKGATE_DIS_TGL);
1163 }
1164
1165 static void
xehpsdv_gt_workarounds_init(struct drm_i915_private * i915,struct i915_wa_list * wal)1166 xehpsdv_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1167 {
1168 xehp_init_mcr(&i915->gt, wal);
1169 }
1170
1171 static void
gt_init_workarounds(struct drm_i915_private * i915,struct i915_wa_list * wal)1172 gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal)
1173 {
1174 if (IS_XEHPSDV(i915))
1175 xehpsdv_gt_workarounds_init(i915, wal);
1176 else if (IS_DG1(i915))
1177 dg1_gt_workarounds_init(i915, wal);
1178 else if (IS_TIGERLAKE(i915))
1179 tgl_gt_workarounds_init(i915, wal);
1180 else if (GRAPHICS_VER(i915) == 12)
1181 gen12_gt_workarounds_init(i915, wal);
1182 else if (GRAPHICS_VER(i915) == 11)
1183 icl_gt_workarounds_init(i915, wal);
1184 else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
1185 cfl_gt_workarounds_init(i915, wal);
1186 else if (IS_GEMINILAKE(i915))
1187 glk_gt_workarounds_init(i915, wal);
1188 else if (IS_KABYLAKE(i915))
1189 kbl_gt_workarounds_init(i915, wal);
1190 else if (IS_BROXTON(i915))
1191 gen9_gt_workarounds_init(i915, wal);
1192 else if (IS_SKYLAKE(i915))
1193 skl_gt_workarounds_init(i915, wal);
1194 else if (IS_HASWELL(i915))
1195 hsw_gt_workarounds_init(i915, wal);
1196 else if (IS_VALLEYVIEW(i915))
1197 vlv_gt_workarounds_init(i915, wal);
1198 else if (IS_IVYBRIDGE(i915))
1199 ivb_gt_workarounds_init(i915, wal);
1200 else if (GRAPHICS_VER(i915) == 6)
1201 snb_gt_workarounds_init(i915, wal);
1202 else if (GRAPHICS_VER(i915) == 5)
1203 ilk_gt_workarounds_init(i915, wal);
1204 else if (IS_G4X(i915))
1205 g4x_gt_workarounds_init(i915, wal);
1206 else if (GRAPHICS_VER(i915) == 4)
1207 gen4_gt_workarounds_init(i915, wal);
1208 else if (GRAPHICS_VER(i915) <= 8)
1209 ;
1210 else
1211 MISSING_CASE(GRAPHICS_VER(i915));
1212 }
1213
intel_gt_init_workarounds(struct drm_i915_private * i915)1214 void intel_gt_init_workarounds(struct drm_i915_private *i915)
1215 {
1216 struct i915_wa_list *wal = &i915->gt_wa_list;
1217
1218 wa_init_start(wal, "GT", "global");
1219 gt_init_workarounds(i915, wal);
1220 wa_init_finish(wal);
1221 }
1222
1223 static enum forcewake_domains
wal_get_fw_for_rmw(struct intel_uncore * uncore,const struct i915_wa_list * wal)1224 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1225 {
1226 enum forcewake_domains fw = 0;
1227 struct i915_wa *wa;
1228 unsigned int i;
1229
1230 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1231 fw |= intel_uncore_forcewake_for_reg(uncore,
1232 wa->reg,
1233 FW_REG_READ |
1234 FW_REG_WRITE);
1235
1236 return fw;
1237 }
1238
1239 static bool
wa_verify(const struct i915_wa * wa,u32 cur,const char * name,const char * from)1240 wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from)
1241 {
1242 if ((cur ^ wa->set) & wa->read) {
1243 DRM_ERROR("%s workaround lost on %s! (reg[%x]=0x%x, relevant bits were 0x%x vs expected 0x%x)\n",
1244 name, from, i915_mmio_reg_offset(wa->reg),
1245 cur, cur & wa->read, wa->set & wa->read);
1246
1247 return false;
1248 }
1249
1250 return true;
1251 }
1252
1253 static void
wa_list_apply(struct intel_gt * gt,const struct i915_wa_list * wal)1254 wa_list_apply(struct intel_gt *gt, const struct i915_wa_list *wal)
1255 {
1256 struct intel_uncore *uncore = gt->uncore;
1257 enum forcewake_domains fw;
1258 unsigned long flags;
1259 struct i915_wa *wa;
1260 unsigned int i;
1261
1262 if (!wal->count)
1263 return;
1264
1265 fw = wal_get_fw_for_rmw(uncore, wal);
1266
1267 spin_lock_irqsave(&uncore->lock, flags);
1268 intel_uncore_forcewake_get__locked(uncore, fw);
1269
1270 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1271 u32 val, old = 0;
1272
1273 /* open-coded rmw due to steering */
1274 old = wa->clr ? intel_gt_read_register_fw(gt, wa->reg) : 0;
1275 val = (old & ~wa->clr) | wa->set;
1276 if (val != old || !wa->clr)
1277 intel_uncore_write_fw(uncore, wa->reg, val);
1278
1279 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
1280 wa_verify(wa, intel_gt_read_register_fw(gt, wa->reg),
1281 wal->name, "application");
1282 }
1283
1284 intel_uncore_forcewake_put__locked(uncore, fw);
1285 spin_unlock_irqrestore(&uncore->lock, flags);
1286 }
1287
intel_gt_apply_workarounds(struct intel_gt * gt)1288 void intel_gt_apply_workarounds(struct intel_gt *gt)
1289 {
1290 wa_list_apply(gt, >->i915->gt_wa_list);
1291 }
1292
wa_list_verify(struct intel_gt * gt,const struct i915_wa_list * wal,const char * from)1293 static bool wa_list_verify(struct intel_gt *gt,
1294 const struct i915_wa_list *wal,
1295 const char *from)
1296 {
1297 struct intel_uncore *uncore = gt->uncore;
1298 struct i915_wa *wa;
1299 enum forcewake_domains fw;
1300 unsigned long flags;
1301 unsigned int i;
1302 bool ok = true;
1303
1304 fw = wal_get_fw_for_rmw(uncore, wal);
1305
1306 spin_lock_irqsave(&uncore->lock, flags);
1307 intel_uncore_forcewake_get__locked(uncore, fw);
1308
1309 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1310 ok &= wa_verify(wa,
1311 intel_gt_read_register_fw(gt, wa->reg),
1312 wal->name, from);
1313
1314 intel_uncore_forcewake_put__locked(uncore, fw);
1315 spin_unlock_irqrestore(&uncore->lock, flags);
1316
1317 return ok;
1318 }
1319
intel_gt_verify_workarounds(struct intel_gt * gt,const char * from)1320 bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from)
1321 {
1322 return wa_list_verify(gt, >->i915->gt_wa_list, from);
1323 }
1324
1325 __maybe_unused
is_nonpriv_flags_valid(u32 flags)1326 static bool is_nonpriv_flags_valid(u32 flags)
1327 {
1328 /* Check only valid flag bits are set */
1329 if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID)
1330 return false;
1331
1332 /* NB: Only 3 out of 4 enum values are valid for access field */
1333 if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
1334 RING_FORCE_TO_NONPRIV_ACCESS_INVALID)
1335 return false;
1336
1337 return true;
1338 }
1339
1340 static void
whitelist_reg_ext(struct i915_wa_list * wal,i915_reg_t reg,u32 flags)1341 whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags)
1342 {
1343 struct i915_wa wa = {
1344 .reg = reg
1345 };
1346
1347 if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1348 return;
1349
1350 if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1351 return;
1352
1353 wa.reg.reg |= flags;
1354 _wa_add(wal, &wa);
1355 }
1356
1357 static void
whitelist_reg(struct i915_wa_list * wal,i915_reg_t reg)1358 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
1359 {
1360 whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
1361 }
1362
gen9_whitelist_build(struct i915_wa_list * w)1363 static void gen9_whitelist_build(struct i915_wa_list *w)
1364 {
1365 /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1366 whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
1367
1368 /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1369 whitelist_reg(w, GEN8_CS_CHICKEN1);
1370
1371 /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1372 whitelist_reg(w, GEN8_HDC_CHICKEN1);
1373
1374 /* WaSendPushConstantsFromMMIO:skl,bxt */
1375 whitelist_reg(w, COMMON_SLICE_CHICKEN2);
1376 }
1377
skl_whitelist_build(struct intel_engine_cs * engine)1378 static void skl_whitelist_build(struct intel_engine_cs *engine)
1379 {
1380 struct i915_wa_list *w = &engine->whitelist;
1381
1382 if (engine->class != RENDER_CLASS)
1383 return;
1384
1385 gen9_whitelist_build(w);
1386
1387 /* WaDisableLSQCROPERFforOCL:skl */
1388 whitelist_reg(w, GEN8_L3SQCREG4);
1389 }
1390
bxt_whitelist_build(struct intel_engine_cs * engine)1391 static void bxt_whitelist_build(struct intel_engine_cs *engine)
1392 {
1393 if (engine->class != RENDER_CLASS)
1394 return;
1395
1396 gen9_whitelist_build(&engine->whitelist);
1397 }
1398
kbl_whitelist_build(struct intel_engine_cs * engine)1399 static void kbl_whitelist_build(struct intel_engine_cs *engine)
1400 {
1401 struct i915_wa_list *w = &engine->whitelist;
1402
1403 if (engine->class != RENDER_CLASS)
1404 return;
1405
1406 gen9_whitelist_build(w);
1407
1408 /* WaDisableLSQCROPERFforOCL:kbl */
1409 whitelist_reg(w, GEN8_L3SQCREG4);
1410 }
1411
glk_whitelist_build(struct intel_engine_cs * engine)1412 static void glk_whitelist_build(struct intel_engine_cs *engine)
1413 {
1414 struct i915_wa_list *w = &engine->whitelist;
1415
1416 if (engine->class != RENDER_CLASS)
1417 return;
1418
1419 gen9_whitelist_build(w);
1420
1421 /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
1422 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1423 }
1424
cfl_whitelist_build(struct intel_engine_cs * engine)1425 static void cfl_whitelist_build(struct intel_engine_cs *engine)
1426 {
1427 struct i915_wa_list *w = &engine->whitelist;
1428
1429 if (engine->class != RENDER_CLASS)
1430 return;
1431
1432 gen9_whitelist_build(w);
1433
1434 /*
1435 * WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml
1436 *
1437 * This covers 4 register which are next to one another :
1438 * - PS_INVOCATION_COUNT
1439 * - PS_INVOCATION_COUNT_UDW
1440 * - PS_DEPTH_COUNT
1441 * - PS_DEPTH_COUNT_UDW
1442 */
1443 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1444 RING_FORCE_TO_NONPRIV_ACCESS_RD |
1445 RING_FORCE_TO_NONPRIV_RANGE_4);
1446 }
1447
cml_whitelist_build(struct intel_engine_cs * engine)1448 static void cml_whitelist_build(struct intel_engine_cs *engine)
1449 {
1450 struct i915_wa_list *w = &engine->whitelist;
1451
1452 if (engine->class != RENDER_CLASS)
1453 whitelist_reg_ext(w,
1454 RING_CTX_TIMESTAMP(engine->mmio_base),
1455 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1456
1457 cfl_whitelist_build(engine);
1458 }
1459
icl_whitelist_build(struct intel_engine_cs * engine)1460 static void icl_whitelist_build(struct intel_engine_cs *engine)
1461 {
1462 struct i915_wa_list *w = &engine->whitelist;
1463
1464 switch (engine->class) {
1465 case RENDER_CLASS:
1466 /* WaAllowUMDToModifyHalfSliceChicken7:icl */
1467 whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7);
1468
1469 /* WaAllowUMDToModifySamplerMode:icl */
1470 whitelist_reg(w, GEN10_SAMPLER_MODE);
1471
1472 /* WaEnableStateCacheRedirectToCS:icl */
1473 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1474
1475 /*
1476 * WaAllowPMDepthAndInvocationCountAccessFromUMD:icl
1477 *
1478 * This covers 4 register which are next to one another :
1479 * - PS_INVOCATION_COUNT
1480 * - PS_INVOCATION_COUNT_UDW
1481 * - PS_DEPTH_COUNT
1482 * - PS_DEPTH_COUNT_UDW
1483 */
1484 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1485 RING_FORCE_TO_NONPRIV_ACCESS_RD |
1486 RING_FORCE_TO_NONPRIV_RANGE_4);
1487 break;
1488
1489 case VIDEO_DECODE_CLASS:
1490 /* hucStatusRegOffset */
1491 whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base),
1492 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1493 /* hucUKernelHdrInfoRegOffset */
1494 whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base),
1495 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1496 /* hucStatus2RegOffset */
1497 whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base),
1498 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1499 whitelist_reg_ext(w,
1500 RING_CTX_TIMESTAMP(engine->mmio_base),
1501 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1502 break;
1503
1504 default:
1505 whitelist_reg_ext(w,
1506 RING_CTX_TIMESTAMP(engine->mmio_base),
1507 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1508 break;
1509 }
1510 }
1511
tgl_whitelist_build(struct intel_engine_cs * engine)1512 static void tgl_whitelist_build(struct intel_engine_cs *engine)
1513 {
1514 struct i915_wa_list *w = &engine->whitelist;
1515
1516 switch (engine->class) {
1517 case RENDER_CLASS:
1518 /*
1519 * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl
1520 * Wa_1408556865:tgl
1521 *
1522 * This covers 4 registers which are next to one another :
1523 * - PS_INVOCATION_COUNT
1524 * - PS_INVOCATION_COUNT_UDW
1525 * - PS_DEPTH_COUNT
1526 * - PS_DEPTH_COUNT_UDW
1527 */
1528 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1529 RING_FORCE_TO_NONPRIV_ACCESS_RD |
1530 RING_FORCE_TO_NONPRIV_RANGE_4);
1531
1532 /* Wa_1808121037:tgl */
1533 whitelist_reg(w, GEN7_COMMON_SLICE_CHICKEN1);
1534
1535 /* Wa_1806527549:tgl */
1536 whitelist_reg(w, HIZ_CHICKEN);
1537 break;
1538 default:
1539 whitelist_reg_ext(w,
1540 RING_CTX_TIMESTAMP(engine->mmio_base),
1541 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1542 break;
1543 }
1544 }
1545
dg1_whitelist_build(struct intel_engine_cs * engine)1546 static void dg1_whitelist_build(struct intel_engine_cs *engine)
1547 {
1548 struct i915_wa_list *w = &engine->whitelist;
1549
1550 tgl_whitelist_build(engine);
1551
1552 /* GEN:BUG:1409280441:dg1 */
1553 if (IS_DG1_GT_STEP(engine->i915, STEP_A0, STEP_B0) &&
1554 (engine->class == RENDER_CLASS ||
1555 engine->class == COPY_ENGINE_CLASS))
1556 whitelist_reg_ext(w, RING_ID(engine->mmio_base),
1557 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1558 }
1559
intel_engine_init_whitelist(struct intel_engine_cs * engine)1560 void intel_engine_init_whitelist(struct intel_engine_cs *engine)
1561 {
1562 struct drm_i915_private *i915 = engine->i915;
1563 struct i915_wa_list *w = &engine->whitelist;
1564
1565 wa_init_start(w, "whitelist", engine->name);
1566
1567 if (IS_DG1(i915))
1568 dg1_whitelist_build(engine);
1569 else if (GRAPHICS_VER(i915) == 12)
1570 tgl_whitelist_build(engine);
1571 else if (GRAPHICS_VER(i915) == 11)
1572 icl_whitelist_build(engine);
1573 else if (IS_COMETLAKE(i915))
1574 cml_whitelist_build(engine);
1575 else if (IS_COFFEELAKE(i915))
1576 cfl_whitelist_build(engine);
1577 else if (IS_GEMINILAKE(i915))
1578 glk_whitelist_build(engine);
1579 else if (IS_KABYLAKE(i915))
1580 kbl_whitelist_build(engine);
1581 else if (IS_BROXTON(i915))
1582 bxt_whitelist_build(engine);
1583 else if (IS_SKYLAKE(i915))
1584 skl_whitelist_build(engine);
1585 else if (GRAPHICS_VER(i915) <= 8)
1586 ;
1587 else
1588 MISSING_CASE(GRAPHICS_VER(i915));
1589
1590 wa_init_finish(w);
1591 }
1592
intel_engine_apply_whitelist(struct intel_engine_cs * engine)1593 void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
1594 {
1595 const struct i915_wa_list *wal = &engine->whitelist;
1596 struct intel_uncore *uncore = engine->uncore;
1597 const u32 base = engine->mmio_base;
1598 struct i915_wa *wa;
1599 unsigned int i;
1600
1601 if (!wal->count)
1602 return;
1603
1604 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1605 intel_uncore_write(uncore,
1606 RING_FORCE_TO_NONPRIV(base, i),
1607 i915_mmio_reg_offset(wa->reg));
1608
1609 /* And clear the rest just in case of garbage */
1610 for (; i < RING_MAX_NONPRIV_SLOTS; i++)
1611 intel_uncore_write(uncore,
1612 RING_FORCE_TO_NONPRIV(base, i),
1613 i915_mmio_reg_offset(RING_NOPID(base)));
1614 }
1615
1616 static void
rcs_engine_wa_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)1617 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1618 {
1619 struct drm_i915_private *i915 = engine->i915;
1620
1621 if (IS_DG1_GT_STEP(i915, STEP_A0, STEP_B0) ||
1622 IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0)) {
1623 /*
1624 * Wa_1607138336:tgl[a0],dg1[a0]
1625 * Wa_1607063988:tgl[a0],dg1[a0]
1626 */
1627 wa_write_or(wal,
1628 GEN9_CTX_PREEMPT_REG,
1629 GEN12_DISABLE_POSH_BUSY_FF_DOP_CG);
1630 }
1631
1632 if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0)) {
1633 /*
1634 * Wa_1606679103:tgl
1635 * (see also Wa_1606682166:icl)
1636 */
1637 wa_write_or(wal,
1638 GEN7_SARCHKMD,
1639 GEN7_DISABLE_SAMPLER_PREFETCH);
1640 }
1641
1642 if (IS_ALDERLAKE_P(i915) || IS_ALDERLAKE_S(i915) || IS_DG1(i915) ||
1643 IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1644 /* Wa_1606931601:tgl,rkl,dg1,adl-s,adl-p */
1645 wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ);
1646
1647 /*
1648 * Wa_1407928979:tgl A*
1649 * Wa_18011464164:tgl[B0+],dg1[B0+]
1650 * Wa_22010931296:tgl[B0+],dg1[B0+]
1651 * Wa_14010919138:rkl,dg1,adl-s,adl-p
1652 */
1653 wa_write_or(wal, GEN7_FF_THREAD_MODE,
1654 GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1655
1656 /*
1657 * Wa_1606700617:tgl,dg1,adl-p
1658 * Wa_22010271021:tgl,rkl,dg1,adl-s,adl-p
1659 * Wa_14010826681:tgl,dg1,rkl,adl-p
1660 */
1661 wa_masked_en(wal,
1662 GEN9_CS_DEBUG_MODE1,
1663 FF_DOP_CLOCK_GATE_DISABLE);
1664 }
1665
1666 if (IS_ALDERLAKE_P(i915) || IS_ALDERLAKE_S(i915) ||
1667 IS_DG1_GT_STEP(i915, STEP_A0, STEP_B0) ||
1668 IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1669 /* Wa_1409804808:tgl,rkl,dg1[a0],adl-s,adl-p */
1670 wa_masked_en(wal, GEN7_ROW_CHICKEN2,
1671 GEN12_PUSH_CONST_DEREF_HOLD_DIS);
1672
1673 /*
1674 * Wa_1409085225:tgl
1675 * Wa_14010229206:tgl,rkl,dg1[a0],adl-s,adl-p
1676 */
1677 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH);
1678 }
1679
1680
1681 if (IS_DG1_GT_STEP(i915, STEP_A0, STEP_B0) ||
1682 IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1683 /*
1684 * Wa_1607030317:tgl
1685 * Wa_1607186500:tgl
1686 * Wa_1607297627:tgl,rkl,dg1[a0]
1687 *
1688 * On TGL and RKL there are multiple entries for this WA in the
1689 * BSpec; some indicate this is an A0-only WA, others indicate
1690 * it applies to all steppings so we trust the "all steppings."
1691 * For DG1 this only applies to A0.
1692 */
1693 wa_masked_en(wal,
1694 GEN6_RC_SLEEP_PSMI_CONTROL,
1695 GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE |
1696 GEN8_RC_SEMA_IDLE_MSG_DISABLE);
1697 }
1698
1699 if (IS_DG1(i915) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915) ||
1700 IS_ALDERLAKE_S(i915) || IS_ALDERLAKE_P(i915)) {
1701 /* Wa_1406941453:tgl,rkl,dg1,adl-s,adl-p */
1702 wa_masked_en(wal,
1703 GEN10_SAMPLER_MODE,
1704 ENABLE_SMALLPL);
1705 }
1706
1707 if (GRAPHICS_VER(i915) == 11) {
1708 /* This is not an Wa. Enable for better image quality */
1709 wa_masked_en(wal,
1710 _3D_CHICKEN3,
1711 _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
1712
1713 /*
1714 * Wa_1405543622:icl
1715 * Formerly known as WaGAPZPriorityScheme
1716 */
1717 wa_write_or(wal,
1718 GEN8_GARBCNTL,
1719 GEN11_ARBITRATION_PRIO_ORDER_MASK);
1720
1721 /*
1722 * Wa_1604223664:icl
1723 * Formerly known as WaL3BankAddressHashing
1724 */
1725 wa_write_clr_set(wal,
1726 GEN8_GARBCNTL,
1727 GEN11_HASH_CTRL_EXCL_MASK,
1728 GEN11_HASH_CTRL_EXCL_BIT0);
1729 wa_write_clr_set(wal,
1730 GEN11_GLBLINVL,
1731 GEN11_BANK_HASH_ADDR_EXCL_MASK,
1732 GEN11_BANK_HASH_ADDR_EXCL_BIT0);
1733
1734 /*
1735 * Wa_1405733216:icl
1736 * Formerly known as WaDisableCleanEvicts
1737 */
1738 wa_write_or(wal,
1739 GEN8_L3SQCREG4,
1740 GEN11_LQSC_CLEAN_EVICT_DISABLE);
1741
1742 /* Wa_1606682166:icl */
1743 wa_write_or(wal,
1744 GEN7_SARCHKMD,
1745 GEN7_DISABLE_SAMPLER_PREFETCH);
1746
1747 /* Wa_1409178092:icl */
1748 wa_write_clr_set(wal,
1749 GEN11_SCRATCH2,
1750 GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE,
1751 0);
1752
1753 /* WaEnable32PlaneMode:icl */
1754 wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS,
1755 GEN11_ENABLE_32_PLANE_MODE);
1756
1757 /*
1758 * Wa_1408767742:icl[a2..forever],ehl[all]
1759 * Wa_1605460711:icl[a0..c0]
1760 */
1761 wa_write_or(wal,
1762 GEN7_FF_THREAD_MODE,
1763 GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1764
1765 /* Wa_22010271021 */
1766 wa_masked_en(wal,
1767 GEN9_CS_DEBUG_MODE1,
1768 FF_DOP_CLOCK_GATE_DISABLE);
1769 }
1770
1771 if (IS_GRAPHICS_VER(i915, 9, 12)) {
1772 /* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl,tgl */
1773 wa_masked_en(wal,
1774 GEN7_FF_SLICE_CS_CHICKEN1,
1775 GEN9_FFSC_PERCTX_PREEMPT_CTRL);
1776 }
1777
1778 if (IS_SKYLAKE(i915) ||
1779 IS_KABYLAKE(i915) ||
1780 IS_COFFEELAKE(i915) ||
1781 IS_COMETLAKE(i915)) {
1782 /* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
1783 wa_write_or(wal,
1784 GEN8_GARBCNTL,
1785 GEN9_GAPS_TSV_CREDIT_DISABLE);
1786 }
1787
1788 if (IS_BROXTON(i915)) {
1789 /* WaDisablePooledEuLoadBalancingFix:bxt */
1790 wa_masked_en(wal,
1791 FF_SLICE_CS_CHICKEN2,
1792 GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
1793 }
1794
1795 if (GRAPHICS_VER(i915) == 9) {
1796 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
1797 wa_masked_en(wal,
1798 GEN9_CSFE_CHICKEN1_RCS,
1799 GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
1800
1801 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
1802 wa_write_or(wal,
1803 BDW_SCRATCH1,
1804 GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
1805
1806 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
1807 if (IS_GEN9_LP(i915))
1808 wa_write_clr_set(wal,
1809 GEN8_L3SQCREG1,
1810 L3_PRIO_CREDITS_MASK,
1811 L3_GENERAL_PRIO_CREDITS(62) |
1812 L3_HIGH_PRIO_CREDITS(2));
1813
1814 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
1815 wa_write_or(wal,
1816 GEN8_L3SQCREG4,
1817 GEN8_LQSC_FLUSH_COHERENT_LINES);
1818
1819 /* Disable atomics in L3 to prevent unrecoverable hangs */
1820 wa_write_clr_set(wal, GEN9_SCRATCH_LNCF1,
1821 GEN9_LNCF_NONIA_COHERENT_ATOMICS_ENABLE, 0);
1822 wa_write_clr_set(wal, GEN8_L3SQCREG4,
1823 GEN8_LQSQ_NONIA_COHERENT_ATOMICS_ENABLE, 0);
1824 wa_write_clr_set(wal, GEN9_SCRATCH1,
1825 EVICTION_PERF_FIX_ENABLE, 0);
1826 }
1827
1828 if (IS_HASWELL(i915)) {
1829 /* WaSampleCChickenBitEnable:hsw */
1830 wa_masked_en(wal,
1831 HALF_SLICE_CHICKEN3, HSW_SAMPLE_C_PERFORMANCE);
1832
1833 wa_masked_dis(wal,
1834 CACHE_MODE_0_GEN7,
1835 /* enable HiZ Raw Stall Optimization */
1836 HIZ_RAW_STALL_OPT_DISABLE);
1837 }
1838
1839 if (IS_VALLEYVIEW(i915)) {
1840 /* WaDisableEarlyCull:vlv */
1841 wa_masked_en(wal,
1842 _3D_CHICKEN3,
1843 _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
1844
1845 /*
1846 * WaVSThreadDispatchOverride:ivb,vlv
1847 *
1848 * This actually overrides the dispatch
1849 * mode for all thread types.
1850 */
1851 wa_write_clr_set(wal,
1852 GEN7_FF_THREAD_MODE,
1853 GEN7_FF_SCHED_MASK,
1854 GEN7_FF_TS_SCHED_HW |
1855 GEN7_FF_VS_SCHED_HW |
1856 GEN7_FF_DS_SCHED_HW);
1857
1858 /* WaPsdDispatchEnable:vlv */
1859 /* WaDisablePSDDualDispatchEnable:vlv */
1860 wa_masked_en(wal,
1861 GEN7_HALF_SLICE_CHICKEN1,
1862 GEN7_MAX_PS_THREAD_DEP |
1863 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
1864 }
1865
1866 if (IS_IVYBRIDGE(i915)) {
1867 /* WaDisableEarlyCull:ivb */
1868 wa_masked_en(wal,
1869 _3D_CHICKEN3,
1870 _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
1871
1872 if (0) { /* causes HiZ corruption on ivb:gt1 */
1873 /* enable HiZ Raw Stall Optimization */
1874 wa_masked_dis(wal,
1875 CACHE_MODE_0_GEN7,
1876 HIZ_RAW_STALL_OPT_DISABLE);
1877 }
1878
1879 /*
1880 * WaVSThreadDispatchOverride:ivb,vlv
1881 *
1882 * This actually overrides the dispatch
1883 * mode for all thread types.
1884 */
1885 wa_write_clr_set(wal,
1886 GEN7_FF_THREAD_MODE,
1887 GEN7_FF_SCHED_MASK,
1888 GEN7_FF_TS_SCHED_HW |
1889 GEN7_FF_VS_SCHED_HW |
1890 GEN7_FF_DS_SCHED_HW);
1891
1892 /* WaDisablePSDDualDispatchEnable:ivb */
1893 if (IS_IVB_GT1(i915))
1894 wa_masked_en(wal,
1895 GEN7_HALF_SLICE_CHICKEN1,
1896 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
1897 }
1898
1899 if (GRAPHICS_VER(i915) == 7) {
1900 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
1901 wa_masked_en(wal,
1902 GFX_MODE_GEN7,
1903 GFX_TLB_INVALIDATE_EXPLICIT | GFX_REPLAY_MODE);
1904
1905 /* WaDisable_RenderCache_OperationalFlush:ivb,vlv,hsw */
1906 wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
1907
1908 /*
1909 * BSpec says this must be set, even though
1910 * WaDisable4x2SubspanOptimization:ivb,hsw
1911 * WaDisable4x2SubspanOptimization isn't listed for VLV.
1912 */
1913 wa_masked_en(wal,
1914 CACHE_MODE_1,
1915 PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
1916
1917 /*
1918 * BSpec recommends 8x4 when MSAA is used,
1919 * however in practice 16x4 seems fastest.
1920 *
1921 * Note that PS/WM thread counts depend on the WIZ hashing
1922 * disable bit, which we don't touch here, but it's good
1923 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
1924 */
1925 wa_masked_field_set(wal,
1926 GEN7_GT_MODE,
1927 GEN6_WIZ_HASHING_MASK,
1928 GEN6_WIZ_HASHING_16x4);
1929 }
1930
1931 if (IS_GRAPHICS_VER(i915, 6, 7))
1932 /*
1933 * We need to disable the AsyncFlip performance optimisations in
1934 * order to use MI_WAIT_FOR_EVENT within the CS. It should
1935 * already be programmed to '1' on all products.
1936 *
1937 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
1938 */
1939 wa_masked_en(wal,
1940 MI_MODE,
1941 ASYNC_FLIP_PERF_DISABLE);
1942
1943 if (GRAPHICS_VER(i915) == 6) {
1944 /*
1945 * Required for the hardware to program scanline values for
1946 * waiting
1947 * WaEnableFlushTlbInvalidationMode:snb
1948 */
1949 wa_masked_en(wal,
1950 GFX_MODE,
1951 GFX_TLB_INVALIDATE_EXPLICIT);
1952
1953 /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
1954 wa_masked_en(wal,
1955 _3D_CHICKEN,
1956 _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB);
1957
1958 wa_masked_en(wal,
1959 _3D_CHICKEN3,
1960 /* WaStripsFansDisableFastClipPerformanceFix:snb */
1961 _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL |
1962 /*
1963 * Bspec says:
1964 * "This bit must be set if 3DSTATE_CLIP clip mode is set
1965 * to normal and 3DSTATE_SF number of SF output attributes
1966 * is more than 16."
1967 */
1968 _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH);
1969
1970 /*
1971 * BSpec recommends 8x4 when MSAA is used,
1972 * however in practice 16x4 seems fastest.
1973 *
1974 * Note that PS/WM thread counts depend on the WIZ hashing
1975 * disable bit, which we don't touch here, but it's good
1976 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
1977 */
1978 wa_masked_field_set(wal,
1979 GEN6_GT_MODE,
1980 GEN6_WIZ_HASHING_MASK,
1981 GEN6_WIZ_HASHING_16x4);
1982
1983 /* WaDisable_RenderCache_OperationalFlush:snb */
1984 wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
1985
1986 /*
1987 * From the Sandybridge PRM, volume 1 part 3, page 24:
1988 * "If this bit is set, STCunit will have LRA as replacement
1989 * policy. [...] This bit must be reset. LRA replacement
1990 * policy is not supported."
1991 */
1992 wa_masked_dis(wal,
1993 CACHE_MODE_0,
1994 CM0_STC_EVICT_DISABLE_LRA_SNB);
1995 }
1996
1997 if (IS_GRAPHICS_VER(i915, 4, 6))
1998 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
1999 wa_add(wal, MI_MODE,
2000 0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH),
2001 /* XXX bit doesn't stick on Broadwater */
2002 IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH, true);
2003
2004 if (GRAPHICS_VER(i915) == 4)
2005 /*
2006 * Disable CONSTANT_BUFFER before it is loaded from the context
2007 * image. For as it is loaded, it is executed and the stored
2008 * address may no longer be valid, leading to a GPU hang.
2009 *
2010 * This imposes the requirement that userspace reload their
2011 * CONSTANT_BUFFER on every batch, fortunately a requirement
2012 * they are already accustomed to from before contexts were
2013 * enabled.
2014 */
2015 wa_add(wal, ECOSKPD,
2016 0, _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE),
2017 0 /* XXX bit doesn't stick on Broadwater */,
2018 true);
2019 }
2020
2021 static void
xcs_engine_wa_init(struct intel_engine_cs * engine,struct i915_wa_list * wal)2022 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2023 {
2024 struct drm_i915_private *i915 = engine->i915;
2025
2026 /* WaKBLVECSSemaphoreWaitPoll:kbl */
2027 if (IS_KBL_GT_STEP(i915, STEP_A0, STEP_F0)) {
2028 wa_write(wal,
2029 RING_SEMA_WAIT_POLL(engine->mmio_base),
2030 1);
2031 }
2032 }
2033
2034 static void
engine_init_workarounds(struct intel_engine_cs * engine,struct i915_wa_list * wal)2035 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2036 {
2037 if (I915_SELFTEST_ONLY(GRAPHICS_VER(engine->i915) < 4))
2038 return;
2039
2040 if (engine->class == RENDER_CLASS)
2041 rcs_engine_wa_init(engine, wal);
2042 else
2043 xcs_engine_wa_init(engine, wal);
2044 }
2045
intel_engine_init_workarounds(struct intel_engine_cs * engine)2046 void intel_engine_init_workarounds(struct intel_engine_cs *engine)
2047 {
2048 struct i915_wa_list *wal = &engine->wa_list;
2049
2050 if (GRAPHICS_VER(engine->i915) < 4)
2051 return;
2052
2053 wa_init_start(wal, "engine", engine->name);
2054 engine_init_workarounds(engine, wal);
2055 wa_init_finish(wal);
2056 }
2057
intel_engine_apply_workarounds(struct intel_engine_cs * engine)2058 void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
2059 {
2060 wa_list_apply(engine->gt, &engine->wa_list);
2061 }
2062
2063 struct mcr_range {
2064 u32 start;
2065 u32 end;
2066 };
2067
2068 static const struct mcr_range mcr_ranges_gen8[] = {
2069 { .start = 0x5500, .end = 0x55ff },
2070 { .start = 0x7000, .end = 0x7fff },
2071 { .start = 0x9400, .end = 0x97ff },
2072 { .start = 0xb000, .end = 0xb3ff },
2073 { .start = 0xe000, .end = 0xe7ff },
2074 {},
2075 };
2076
2077 static const struct mcr_range mcr_ranges_gen12[] = {
2078 { .start = 0x8150, .end = 0x815f },
2079 { .start = 0x9520, .end = 0x955f },
2080 { .start = 0xb100, .end = 0xb3ff },
2081 { .start = 0xde80, .end = 0xe8ff },
2082 { .start = 0x24a00, .end = 0x24a7f },
2083 {},
2084 };
2085
2086 static const struct mcr_range mcr_ranges_xehp[] = {
2087 { .start = 0x4000, .end = 0x4aff },
2088 { .start = 0x5200, .end = 0x52ff },
2089 { .start = 0x5400, .end = 0x7fff },
2090 { .start = 0x8140, .end = 0x815f },
2091 { .start = 0x8c80, .end = 0x8dff },
2092 { .start = 0x94d0, .end = 0x955f },
2093 { .start = 0x9680, .end = 0x96ff },
2094 { .start = 0xb000, .end = 0xb3ff },
2095 { .start = 0xc800, .end = 0xcfff },
2096 { .start = 0xd800, .end = 0xd8ff },
2097 { .start = 0xdc00, .end = 0xffff },
2098 { .start = 0x17000, .end = 0x17fff },
2099 { .start = 0x24a00, .end = 0x24a7f },
2100 {},
2101 };
2102
mcr_range(struct drm_i915_private * i915,u32 offset)2103 static bool mcr_range(struct drm_i915_private *i915, u32 offset)
2104 {
2105 const struct mcr_range *mcr_ranges;
2106 int i;
2107
2108 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50))
2109 mcr_ranges = mcr_ranges_xehp;
2110 else if (GRAPHICS_VER(i915) >= 12)
2111 mcr_ranges = mcr_ranges_gen12;
2112 else if (GRAPHICS_VER(i915) >= 8)
2113 mcr_ranges = mcr_ranges_gen8;
2114 else
2115 return false;
2116
2117 /*
2118 * Registers in these ranges are affected by the MCR selector
2119 * which only controls CPU initiated MMIO. Routing does not
2120 * work for CS access so we cannot verify them on this path.
2121 */
2122 for (i = 0; mcr_ranges[i].start; i++)
2123 if (offset >= mcr_ranges[i].start &&
2124 offset <= mcr_ranges[i].end)
2125 return true;
2126
2127 return false;
2128 }
2129
2130 static int
wa_list_srm(struct i915_request * rq,const struct i915_wa_list * wal,struct i915_vma * vma)2131 wa_list_srm(struct i915_request *rq,
2132 const struct i915_wa_list *wal,
2133 struct i915_vma *vma)
2134 {
2135 struct drm_i915_private *i915 = rq->engine->i915;
2136 unsigned int i, count = 0;
2137 const struct i915_wa *wa;
2138 u32 srm, *cs;
2139
2140 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
2141 if (GRAPHICS_VER(i915) >= 8)
2142 srm++;
2143
2144 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2145 if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg)))
2146 count++;
2147 }
2148
2149 cs = intel_ring_begin(rq, 4 * count);
2150 if (IS_ERR(cs))
2151 return PTR_ERR(cs);
2152
2153 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2154 u32 offset = i915_mmio_reg_offset(wa->reg);
2155
2156 if (mcr_range(i915, offset))
2157 continue;
2158
2159 *cs++ = srm;
2160 *cs++ = offset;
2161 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
2162 *cs++ = 0;
2163 }
2164 intel_ring_advance(rq, cs);
2165
2166 return 0;
2167 }
2168
engine_wa_list_verify(struct intel_context * ce,const struct i915_wa_list * const wal,const char * from)2169 static int engine_wa_list_verify(struct intel_context *ce,
2170 const struct i915_wa_list * const wal,
2171 const char *from)
2172 {
2173 const struct i915_wa *wa;
2174 struct i915_request *rq;
2175 struct i915_vma *vma;
2176 struct i915_gem_ww_ctx ww;
2177 unsigned int i;
2178 u32 *results;
2179 int err;
2180
2181 if (!wal->count)
2182 return 0;
2183
2184 vma = __vm_create_scratch_for_read(&ce->engine->gt->ggtt->vm,
2185 wal->count * sizeof(u32));
2186 if (IS_ERR(vma))
2187 return PTR_ERR(vma);
2188
2189 intel_engine_pm_get(ce->engine);
2190 i915_gem_ww_ctx_init(&ww, false);
2191 retry:
2192 err = i915_gem_object_lock(vma->obj, &ww);
2193 if (err == 0)
2194 err = intel_context_pin_ww(ce, &ww);
2195 if (err)
2196 goto err_pm;
2197
2198 err = i915_vma_pin_ww(vma, &ww, 0, 0,
2199 i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
2200 if (err)
2201 goto err_unpin;
2202
2203 rq = i915_request_create(ce);
2204 if (IS_ERR(rq)) {
2205 err = PTR_ERR(rq);
2206 goto err_vma;
2207 }
2208
2209 err = i915_request_await_object(rq, vma->obj, true);
2210 if (err == 0)
2211 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
2212 if (err == 0)
2213 err = wa_list_srm(rq, wal, vma);
2214
2215 i915_request_get(rq);
2216 if (err)
2217 i915_request_set_error_once(rq, err);
2218 i915_request_add(rq);
2219
2220 if (err)
2221 goto err_rq;
2222
2223 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2224 err = -ETIME;
2225 goto err_rq;
2226 }
2227
2228 results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
2229 if (IS_ERR(results)) {
2230 err = PTR_ERR(results);
2231 goto err_rq;
2232 }
2233
2234 err = 0;
2235 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2236 if (mcr_range(rq->engine->i915, i915_mmio_reg_offset(wa->reg)))
2237 continue;
2238
2239 if (!wa_verify(wa, results[i], wal->name, from))
2240 err = -ENXIO;
2241 }
2242
2243 i915_gem_object_unpin_map(vma->obj);
2244
2245 err_rq:
2246 i915_request_put(rq);
2247 err_vma:
2248 i915_vma_unpin(vma);
2249 err_unpin:
2250 intel_context_unpin(ce);
2251 err_pm:
2252 if (err == -EDEADLK) {
2253 err = i915_gem_ww_ctx_backoff(&ww);
2254 if (!err)
2255 goto retry;
2256 }
2257 i915_gem_ww_ctx_fini(&ww);
2258 intel_engine_pm_put(ce->engine);
2259 i915_vma_put(vma);
2260 return err;
2261 }
2262
intel_engine_verify_workarounds(struct intel_engine_cs * engine,const char * from)2263 int intel_engine_verify_workarounds(struct intel_engine_cs *engine,
2264 const char *from)
2265 {
2266 return engine_wa_list_verify(engine->kernel_context,
2267 &engine->wa_list,
2268 from);
2269 }
2270
2271 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2272 #include "selftest_workarounds.c"
2273 #endif
2274