1 /*
2 * Copyright (c) 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions: *
10 * The above copyright notice and this permission notice (including the next
11 * paragraph) shall be included in all copies or substantial portions of the
12 * Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20 * SOFTWARE.
21 */
22
23 #include "i915_drv.h"
24
25 #include "intel_engine.h"
26 #include "intel_gt.h"
27 #include "intel_mocs.h"
28 #include "intel_lrc.h"
29 #include "intel_ring.h"
30
31 /* structures required */
32 struct drm_i915_mocs_entry {
33 u32 control_value;
34 u16 l3cc_value;
35 u16 used;
36 };
37
38 struct drm_i915_mocs_table {
39 unsigned int size;
40 unsigned int n_entries;
41 const struct drm_i915_mocs_entry *table;
42 };
43
44 /* Defines for the tables (XXX_MOCS_0 - XXX_MOCS_63) */
45 #define _LE_CACHEABILITY(value) ((value) << 0)
46 #define _LE_TGT_CACHE(value) ((value) << 2)
47 #define LE_LRUM(value) ((value) << 4)
48 #define LE_AOM(value) ((value) << 6)
49 #define LE_RSC(value) ((value) << 7)
50 #define LE_SCC(value) ((value) << 8)
51 #define LE_PFM(value) ((value) << 11)
52 #define LE_SCF(value) ((value) << 14)
53 #define LE_COS(value) ((value) << 15)
54 #define LE_SSE(value) ((value) << 17)
55
56 /* Defines for the tables (LNCFMOCS0 - LNCFMOCS31) - two entries per word */
57 #define L3_ESC(value) ((value) << 0)
58 #define L3_SCC(value) ((value) << 1)
59 #define _L3_CACHEABILITY(value) ((value) << 4)
60
61 /* Helper defines */
62 #define GEN9_NUM_MOCS_ENTRIES 64 /* 63-64 are reserved, but configured. */
63
64 /* (e)LLC caching options */
65 /*
66 * Note: LE_0_PAGETABLE works only up to Gen11; for newer gens it means
67 * the same as LE_UC
68 */
69 #define LE_0_PAGETABLE _LE_CACHEABILITY(0)
70 #define LE_1_UC _LE_CACHEABILITY(1)
71 #define LE_2_WT _LE_CACHEABILITY(2)
72 #define LE_3_WB _LE_CACHEABILITY(3)
73
74 /* Target cache */
75 #define LE_TC_0_PAGETABLE _LE_TGT_CACHE(0)
76 #define LE_TC_1_LLC _LE_TGT_CACHE(1)
77 #define LE_TC_2_LLC_ELLC _LE_TGT_CACHE(2)
78 #define LE_TC_3_LLC_ELLC_ALT _LE_TGT_CACHE(3)
79
80 /* L3 caching options */
81 #define L3_0_DIRECT _L3_CACHEABILITY(0)
82 #define L3_1_UC _L3_CACHEABILITY(1)
83 #define L3_2_RESERVED _L3_CACHEABILITY(2)
84 #define L3_3_WB _L3_CACHEABILITY(3)
85
86 #define MOCS_ENTRY(__idx, __control_value, __l3cc_value) \
87 [__idx] = { \
88 .control_value = __control_value, \
89 .l3cc_value = __l3cc_value, \
90 .used = 1, \
91 }
92
93 /*
94 * MOCS tables
95 *
96 * These are the MOCS tables that are programmed across all the rings.
97 * The control value is programmed to all the rings that support the
98 * MOCS registers. While the l3cc_values are only programmed to the
99 * LNCFCMOCS0 - LNCFCMOCS32 registers.
100 *
101 * These tables are intended to be kept reasonably consistent across
102 * HW platforms, and for ICL+, be identical across OSes. To achieve
103 * that, for Icelake and above, list of entries is published as part
104 * of bspec.
105 *
106 * Entries not part of the following tables are undefined as far as
107 * userspace is concerned and shouldn't be relied upon. For Gen < 12
108 * they will be initialized to PTE. Gen >= 12 onwards don't have a setting for
109 * PTE and will be initialized to an invalid value.
110 *
111 * The last two entries are reserved by the hardware. For ICL+ they
112 * should be initialized according to bspec and never used, for older
113 * platforms they should never be written to.
114 *
115 * NOTE: These tables are part of bspec and defined as part of hardware
116 * interface for ICL+. For older platforms, they are part of kernel
117 * ABI. It is expected that, for specific hardware platform, existing
118 * entries will remain constant and the table will only be updated by
119 * adding new entries, filling unused positions.
120 */
121 #define GEN9_MOCS_ENTRIES \
122 MOCS_ENTRY(I915_MOCS_UNCACHED, \
123 LE_1_UC | LE_TC_2_LLC_ELLC, \
124 L3_1_UC), \
125 MOCS_ENTRY(I915_MOCS_PTE, \
126 LE_0_PAGETABLE | LE_TC_2_LLC_ELLC | LE_LRUM(3), \
127 L3_3_WB)
128
129 static const struct drm_i915_mocs_entry skl_mocs_table[] = {
130 GEN9_MOCS_ENTRIES,
131 MOCS_ENTRY(I915_MOCS_CACHED,
132 LE_3_WB | LE_TC_2_LLC_ELLC | LE_LRUM(3),
133 L3_3_WB),
134
135 /*
136 * mocs:63
137 * - used by the L3 for all of its evictions.
138 * Thus it is expected to allow LLC cacheability to enable coherent
139 * flows to be maintained.
140 * - used to force L3 uncachable cycles.
141 * Thus it is expected to make the surface L3 uncacheable.
142 */
143 MOCS_ENTRY(63,
144 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
145 L3_1_UC)
146 };
147
148 /* NOTE: the LE_TGT_CACHE is not used on Broxton */
149 static const struct drm_i915_mocs_entry broxton_mocs_table[] = {
150 GEN9_MOCS_ENTRIES,
151 MOCS_ENTRY(I915_MOCS_CACHED,
152 LE_1_UC | LE_TC_2_LLC_ELLC | LE_LRUM(3),
153 L3_3_WB)
154 };
155
156 #define GEN11_MOCS_ENTRIES \
157 /* Entries 0 and 1 are defined per-platform */ \
158 /* Base - L3 + LLC */ \
159 MOCS_ENTRY(2, \
160 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
161 L3_3_WB), \
162 /* Base - Uncached */ \
163 MOCS_ENTRY(3, \
164 LE_1_UC | LE_TC_1_LLC, \
165 L3_1_UC), \
166 /* Base - L3 */ \
167 MOCS_ENTRY(4, \
168 LE_1_UC | LE_TC_1_LLC, \
169 L3_3_WB), \
170 /* Base - LLC */ \
171 MOCS_ENTRY(5, \
172 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
173 L3_1_UC), \
174 /* Age 0 - LLC */ \
175 MOCS_ENTRY(6, \
176 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), \
177 L3_1_UC), \
178 /* Age 0 - L3 + LLC */ \
179 MOCS_ENTRY(7, \
180 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), \
181 L3_3_WB), \
182 /* Age: Don't Chg. - LLC */ \
183 MOCS_ENTRY(8, \
184 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), \
185 L3_1_UC), \
186 /* Age: Don't Chg. - L3 + LLC */ \
187 MOCS_ENTRY(9, \
188 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), \
189 L3_3_WB), \
190 /* No AOM - LLC */ \
191 MOCS_ENTRY(10, \
192 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), \
193 L3_1_UC), \
194 /* No AOM - L3 + LLC */ \
195 MOCS_ENTRY(11, \
196 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), \
197 L3_3_WB), \
198 /* No AOM; Age 0 - LLC */ \
199 MOCS_ENTRY(12, \
200 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), \
201 L3_1_UC), \
202 /* No AOM; Age 0 - L3 + LLC */ \
203 MOCS_ENTRY(13, \
204 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), \
205 L3_3_WB), \
206 /* No AOM; Age:DC - LLC */ \
207 MOCS_ENTRY(14, \
208 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \
209 L3_1_UC), \
210 /* No AOM; Age:DC - L3 + LLC */ \
211 MOCS_ENTRY(15, \
212 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \
213 L3_3_WB), \
214 /* Self-Snoop - L3 + LLC */ \
215 MOCS_ENTRY(18, \
216 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SSE(3), \
217 L3_3_WB), \
218 /* Skip Caching - L3 + LLC(12.5%) */ \
219 MOCS_ENTRY(19, \
220 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(7), \
221 L3_3_WB), \
222 /* Skip Caching - L3 + LLC(25%) */ \
223 MOCS_ENTRY(20, \
224 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(3), \
225 L3_3_WB), \
226 /* Skip Caching - L3 + LLC(50%) */ \
227 MOCS_ENTRY(21, \
228 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(1), \
229 L3_3_WB), \
230 /* Skip Caching - L3 + LLC(75%) */ \
231 MOCS_ENTRY(22, \
232 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(3), \
233 L3_3_WB), \
234 /* Skip Caching - L3 + LLC(87.5%) */ \
235 MOCS_ENTRY(23, \
236 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(7), \
237 L3_3_WB), \
238 /* HW Reserved - SW program but never use */ \
239 MOCS_ENTRY(62, \
240 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
241 L3_1_UC), \
242 /* HW Reserved - SW program but never use */ \
243 MOCS_ENTRY(63, \
244 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
245 L3_1_UC)
246
247 static const struct drm_i915_mocs_entry tgl_mocs_table[] = {
248 /*
249 * NOTE:
250 * Reserved and unspecified MOCS indices have been set to (L3 + LCC).
251 * These reserved entries should never be used, they may be changed
252 * to low performant variants with better coherency in the future if
253 * more entries are needed. We are programming index I915_MOCS_PTE(1)
254 * only, __init_mocs_table() take care to program unused index with
255 * this entry.
256 */
257 MOCS_ENTRY(I915_MOCS_PTE,
258 LE_0_PAGETABLE | LE_TC_0_PAGETABLE,
259 L3_1_UC),
260 GEN11_MOCS_ENTRIES,
261
262 /* Implicitly enable L1 - HDC:L1 + L3 + LLC */
263 MOCS_ENTRY(48,
264 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
265 L3_3_WB),
266 /* Implicitly enable L1 - HDC:L1 + L3 */
267 MOCS_ENTRY(49,
268 LE_1_UC | LE_TC_1_LLC,
269 L3_3_WB),
270 /* Implicitly enable L1 - HDC:L1 + LLC */
271 MOCS_ENTRY(50,
272 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
273 L3_1_UC),
274 /* Implicitly enable L1 - HDC:L1 */
275 MOCS_ENTRY(51,
276 LE_1_UC | LE_TC_1_LLC,
277 L3_1_UC),
278 /* HW Special Case (CCS) */
279 MOCS_ENTRY(60,
280 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
281 L3_1_UC),
282 /* HW Special Case (Displayable) */
283 MOCS_ENTRY(61,
284 LE_1_UC | LE_TC_1_LLC,
285 L3_3_WB),
286 };
287
288 static const struct drm_i915_mocs_entry icl_mocs_table[] = {
289 /* Base - Uncached (Deprecated) */
290 MOCS_ENTRY(I915_MOCS_UNCACHED,
291 LE_1_UC | LE_TC_1_LLC,
292 L3_1_UC),
293 /* Base - L3 + LeCC:PAT (Deprecated) */
294 MOCS_ENTRY(I915_MOCS_PTE,
295 LE_0_PAGETABLE | LE_TC_1_LLC,
296 L3_3_WB),
297
298 GEN11_MOCS_ENTRIES
299 };
300
301 enum {
302 HAS_GLOBAL_MOCS = BIT(0),
303 HAS_ENGINE_MOCS = BIT(1),
304 HAS_RENDER_L3CC = BIT(2),
305 };
306
has_l3cc(const struct drm_i915_private * i915)307 static bool has_l3cc(const struct drm_i915_private *i915)
308 {
309 return true;
310 }
311
has_global_mocs(const struct drm_i915_private * i915)312 static bool has_global_mocs(const struct drm_i915_private *i915)
313 {
314 return HAS_GLOBAL_MOCS_REGISTERS(i915);
315 }
316
has_mocs(const struct drm_i915_private * i915)317 static bool has_mocs(const struct drm_i915_private *i915)
318 {
319 return !IS_DGFX(i915);
320 }
321
get_mocs_settings(const struct drm_i915_private * i915,struct drm_i915_mocs_table * table)322 static unsigned int get_mocs_settings(const struct drm_i915_private *i915,
323 struct drm_i915_mocs_table *table)
324 {
325 unsigned int flags;
326
327 if (INTEL_GEN(i915) >= 12) {
328 table->size = ARRAY_SIZE(tgl_mocs_table);
329 table->table = tgl_mocs_table;
330 table->n_entries = GEN9_NUM_MOCS_ENTRIES;
331 } else if (IS_GEN(i915, 11)) {
332 table->size = ARRAY_SIZE(icl_mocs_table);
333 table->table = icl_mocs_table;
334 table->n_entries = GEN9_NUM_MOCS_ENTRIES;
335 } else if (IS_GEN9_BC(i915) || IS_CANNONLAKE(i915)) {
336 table->size = ARRAY_SIZE(skl_mocs_table);
337 table->n_entries = GEN9_NUM_MOCS_ENTRIES;
338 table->table = skl_mocs_table;
339 } else if (IS_GEN9_LP(i915)) {
340 table->size = ARRAY_SIZE(broxton_mocs_table);
341 table->n_entries = GEN9_NUM_MOCS_ENTRIES;
342 table->table = broxton_mocs_table;
343 } else {
344 drm_WARN_ONCE(&i915->drm, INTEL_GEN(i915) >= 9,
345 "Platform that should have a MOCS table does not.\n");
346 return 0;
347 }
348
349 if (GEM_DEBUG_WARN_ON(table->size > table->n_entries))
350 return 0;
351
352 /* WaDisableSkipCaching:skl,bxt,kbl,glk */
353 if (IS_GEN(i915, 9)) {
354 int i;
355
356 for (i = 0; i < table->size; i++)
357 if (GEM_DEBUG_WARN_ON(table->table[i].l3cc_value &
358 (L3_ESC(1) | L3_SCC(0x7))))
359 return 0;
360 }
361
362 flags = 0;
363 if (has_mocs(i915)) {
364 if (has_global_mocs(i915))
365 flags |= HAS_GLOBAL_MOCS;
366 else
367 flags |= HAS_ENGINE_MOCS;
368 }
369 if (has_l3cc(i915))
370 flags |= HAS_RENDER_L3CC;
371
372 return flags;
373 }
374
375 /*
376 * Get control_value from MOCS entry taking into account when it's not used:
377 * I915_MOCS_PTE's value is returned in this case.
378 */
get_entry_control(const struct drm_i915_mocs_table * table,unsigned int index)379 static u32 get_entry_control(const struct drm_i915_mocs_table *table,
380 unsigned int index)
381 {
382 if (index < table->size && table->table[index].used)
383 return table->table[index].control_value;
384
385 return table->table[I915_MOCS_PTE].control_value;
386 }
387
388 #define for_each_mocs(mocs, t, i) \
389 for (i = 0; \
390 i < (t)->n_entries ? (mocs = get_entry_control((t), i)), 1 : 0;\
391 i++)
392
__init_mocs_table(struct intel_uncore * uncore,const struct drm_i915_mocs_table * table,u32 addr)393 static void __init_mocs_table(struct intel_uncore *uncore,
394 const struct drm_i915_mocs_table *table,
395 u32 addr)
396 {
397 unsigned int i;
398 u32 mocs;
399
400 for_each_mocs(mocs, table, i)
401 intel_uncore_write_fw(uncore, _MMIO(addr + i * 4), mocs);
402 }
403
mocs_offset(const struct intel_engine_cs * engine)404 static u32 mocs_offset(const struct intel_engine_cs *engine)
405 {
406 static const u32 offset[] = {
407 [RCS0] = __GEN9_RCS0_MOCS0,
408 [VCS0] = __GEN9_VCS0_MOCS0,
409 [VCS1] = __GEN9_VCS1_MOCS0,
410 [VECS0] = __GEN9_VECS0_MOCS0,
411 [BCS0] = __GEN9_BCS0_MOCS0,
412 [VCS2] = __GEN11_VCS2_MOCS0,
413 };
414
415 GEM_BUG_ON(engine->id >= ARRAY_SIZE(offset));
416 return offset[engine->id];
417 }
418
init_mocs_table(struct intel_engine_cs * engine,const struct drm_i915_mocs_table * table)419 static void init_mocs_table(struct intel_engine_cs *engine,
420 const struct drm_i915_mocs_table *table)
421 {
422 __init_mocs_table(engine->uncore, table, mocs_offset(engine));
423 }
424
425 /*
426 * Get l3cc_value from MOCS entry taking into account when it's not used:
427 * I915_MOCS_PTE's value is returned in this case.
428 */
get_entry_l3cc(const struct drm_i915_mocs_table * table,unsigned int index)429 static u16 get_entry_l3cc(const struct drm_i915_mocs_table *table,
430 unsigned int index)
431 {
432 if (index < table->size && table->table[index].used)
433 return table->table[index].l3cc_value;
434
435 return table->table[I915_MOCS_PTE].l3cc_value;
436 }
437
l3cc_combine(u16 low,u16 high)438 static inline u32 l3cc_combine(u16 low, u16 high)
439 {
440 return low | (u32)high << 16;
441 }
442
443 #define for_each_l3cc(l3cc, t, i) \
444 for (i = 0; \
445 i < ((t)->n_entries + 1) / 2 ? \
446 (l3cc = l3cc_combine(get_entry_l3cc((t), 2 * i), \
447 get_entry_l3cc((t), 2 * i + 1))), 1 : \
448 0; \
449 i++)
450
init_l3cc_table(struct intel_engine_cs * engine,const struct drm_i915_mocs_table * table)451 static void init_l3cc_table(struct intel_engine_cs *engine,
452 const struct drm_i915_mocs_table *table)
453 {
454 struct intel_uncore *uncore = engine->uncore;
455 unsigned int i;
456 u32 l3cc;
457
458 for_each_l3cc(l3cc, table, i)
459 intel_uncore_write_fw(uncore, GEN9_LNCFCMOCS(i), l3cc);
460 }
461
intel_mocs_init_engine(struct intel_engine_cs * engine)462 void intel_mocs_init_engine(struct intel_engine_cs *engine)
463 {
464 struct drm_i915_mocs_table table;
465 unsigned int flags;
466
467 /* Called under a blanket forcewake */
468 assert_forcewakes_active(engine->uncore, FORCEWAKE_ALL);
469
470 flags = get_mocs_settings(engine->i915, &table);
471 if (!flags)
472 return;
473
474 /* Platforms with global MOCS do not need per-engine initialization. */
475 if (flags & HAS_ENGINE_MOCS)
476 init_mocs_table(engine, &table);
477
478 if (flags & HAS_RENDER_L3CC && engine->class == RENDER_CLASS)
479 init_l3cc_table(engine, &table);
480 }
481
global_mocs_offset(void)482 static u32 global_mocs_offset(void)
483 {
484 return i915_mmio_reg_offset(GEN12_GLOBAL_MOCS(0));
485 }
486
intel_mocs_init(struct intel_gt * gt)487 void intel_mocs_init(struct intel_gt *gt)
488 {
489 struct drm_i915_mocs_table table;
490 unsigned int flags;
491
492 /*
493 * LLC and eDRAM control values are not applicable to dgfx
494 */
495 flags = get_mocs_settings(gt->i915, &table);
496 if (flags & HAS_GLOBAL_MOCS)
497 __init_mocs_table(gt->uncore, &table, global_mocs_offset());
498 }
499
500 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
501 #include "selftest_mocs.c"
502 #endif
503