1 /*
2 * Copyright © 2017 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23 #ifndef IRIS_CONTEXT_H
24 #define IRIS_CONTEXT_H
25
26 #include "pipe/p_context.h"
27 #include "pipe/p_state.h"
28 #include "util/perf/u_trace.h"
29 #include "util/set.h"
30 #include "util/slab.h"
31 #include "util/u_debug.h"
32 #include "util/macros.h"
33 #include "util/u_threaded_context.h"
34 #include "intel/blorp/blorp.h"
35 #include "intel/dev/intel_debug.h"
36 #include "intel/common/intel_l3_config.h"
37 #include "intel/compiler/brw_compiler.h"
38 #include "intel/ds/intel_driver_ds.h"
39 #include "iris_batch.h"
40 #include "iris_binder.h"
41 #include "iris_fence.h"
42 #include "iris_resource.h"
43 #include "iris_screen.h"
44
45 struct iris_bo;
46 struct iris_context;
47 struct blorp_batch;
48 struct blorp_params;
49
50 #define IRIS_MAX_TEXTURE_BUFFER_SIZE (1 << 27)
51 #define IRIS_MAX_TEXTURE_SAMPLERS 32
52 /* IRIS_MAX_ABOS and IRIS_MAX_SSBOS must be the same. */
53 #define IRIS_MAX_ABOS 16
54 #define IRIS_MAX_SSBOS 16
55 #define IRIS_MAX_VIEWPORTS 16
56 #define IRIS_MAX_CLIP_PLANES 8
57 #define IRIS_MAX_GLOBAL_BINDINGS 32
58
59 enum iris_param_domain {
60 BRW_PARAM_DOMAIN_BUILTIN = 0,
61 BRW_PARAM_DOMAIN_IMAGE,
62 };
63
64 enum {
65 DRI_CONF_BO_REUSE_DISABLED,
66 DRI_CONF_BO_REUSE_ALL
67 };
68
69 #define BRW_PARAM(domain, val) (BRW_PARAM_DOMAIN_##domain << 24 | (val))
70 #define BRW_PARAM_DOMAIN(param) ((uint32_t)(param) >> 24)
71 #define BRW_PARAM_VALUE(param) ((uint32_t)(param) & 0x00ffffff)
72 #define BRW_PARAM_IMAGE(idx, offset) BRW_PARAM(IMAGE, ((idx) << 8) | (offset))
73 #define BRW_PARAM_IMAGE_IDX(value) (BRW_PARAM_VALUE(value) >> 8)
74 #define BRW_PARAM_IMAGE_OFFSET(value)(BRW_PARAM_VALUE(value) & 0xf)
75
76 /**
77 * Dirty flags. When state changes, we flag some combination of these
78 * to indicate that particular GPU commands need to be re-emitted.
79 *
80 * Each bit typically corresponds to a single 3DSTATE_* command packet, but
81 * in rare cases they map to a group of related packets that need to be
82 * emitted together.
83 *
84 * See iris_upload_render_state().
85 */
86 #define IRIS_DIRTY_COLOR_CALC_STATE (1ull << 0)
87 #define IRIS_DIRTY_POLYGON_STIPPLE (1ull << 1)
88 #define IRIS_DIRTY_SCISSOR_RECT (1ull << 2)
89 #define IRIS_DIRTY_WM_DEPTH_STENCIL (1ull << 3)
90 #define IRIS_DIRTY_CC_VIEWPORT (1ull << 4)
91 #define IRIS_DIRTY_SF_CL_VIEWPORT (1ull << 5)
92 #define IRIS_DIRTY_PS_BLEND (1ull << 6)
93 #define IRIS_DIRTY_BLEND_STATE (1ull << 7)
94 #define IRIS_DIRTY_RASTER (1ull << 8)
95 #define IRIS_DIRTY_CLIP (1ull << 9)
96 #define IRIS_DIRTY_SBE (1ull << 10)
97 #define IRIS_DIRTY_LINE_STIPPLE (1ull << 11)
98 #define IRIS_DIRTY_VERTEX_ELEMENTS (1ull << 12)
99 #define IRIS_DIRTY_MULTISAMPLE (1ull << 13)
100 #define IRIS_DIRTY_VERTEX_BUFFERS (1ull << 14)
101 #define IRIS_DIRTY_SAMPLE_MASK (1ull << 15)
102 #define IRIS_DIRTY_URB (1ull << 16)
103 #define IRIS_DIRTY_DEPTH_BUFFER (1ull << 17)
104 #define IRIS_DIRTY_WM (1ull << 18)
105 #define IRIS_DIRTY_SO_BUFFERS (1ull << 19)
106 #define IRIS_DIRTY_SO_DECL_LIST (1ull << 20)
107 #define IRIS_DIRTY_STREAMOUT (1ull << 21)
108 #define IRIS_DIRTY_VF_SGVS (1ull << 22)
109 #define IRIS_DIRTY_VF (1ull << 23)
110 #define IRIS_DIRTY_VF_TOPOLOGY (1ull << 24)
111 #define IRIS_DIRTY_RENDER_RESOLVES_AND_FLUSHES (1ull << 25)
112 #define IRIS_DIRTY_COMPUTE_RESOLVES_AND_FLUSHES (1ull << 26)
113 #define IRIS_DIRTY_VF_STATISTICS (1ull << 27)
114 #define IRIS_DIRTY_PMA_FIX (1ull << 28)
115 #define IRIS_DIRTY_DEPTH_BOUNDS (1ull << 29)
116 #define IRIS_DIRTY_RENDER_BUFFER (1ull << 30)
117 #define IRIS_DIRTY_STENCIL_REF (1ull << 31)
118 #define IRIS_DIRTY_VERTEX_BUFFER_FLUSHES (1ull << 32)
119 #define IRIS_DIRTY_RENDER_MISC_BUFFER_FLUSHES (1ull << 33)
120 #define IRIS_DIRTY_COMPUTE_MISC_BUFFER_FLUSHES (1ull << 34)
121 #define IRIS_DIRTY_VFG (1ull << 35)
122
123 #define IRIS_ALL_DIRTY_FOR_COMPUTE (IRIS_DIRTY_COMPUTE_RESOLVES_AND_FLUSHES | \
124 IRIS_DIRTY_COMPUTE_MISC_BUFFER_FLUSHES)
125
126 #define IRIS_ALL_DIRTY_FOR_RENDER (~IRIS_ALL_DIRTY_FOR_COMPUTE)
127
128 /**
129 * Per-stage dirty flags. When state changes, we flag some combination of
130 * these to indicate that particular GPU commands need to be re-emitted.
131 * Unlike the IRIS_DIRTY_* flags these are shader stage-specific and can be
132 * indexed by shifting the mask by the shader stage index.
133 *
134 * See iris_upload_render_state().
135 */
136 #define IRIS_STAGE_DIRTY_SAMPLER_STATES_VS (1ull << 0)
137 #define IRIS_STAGE_DIRTY_SAMPLER_STATES_TCS (1ull << 1)
138 #define IRIS_STAGE_DIRTY_SAMPLER_STATES_TES (1ull << 2)
139 #define IRIS_STAGE_DIRTY_SAMPLER_STATES_GS (1ull << 3)
140 #define IRIS_STAGE_DIRTY_SAMPLER_STATES_PS (1ull << 4)
141 #define IRIS_STAGE_DIRTY_SAMPLER_STATES_CS (1ull << 5)
142 #define IRIS_STAGE_DIRTY_UNCOMPILED_VS (1ull << 6)
143 #define IRIS_STAGE_DIRTY_UNCOMPILED_TCS (1ull << 7)
144 #define IRIS_STAGE_DIRTY_UNCOMPILED_TES (1ull << 8)
145 #define IRIS_STAGE_DIRTY_UNCOMPILED_GS (1ull << 9)
146 #define IRIS_STAGE_DIRTY_UNCOMPILED_FS (1ull << 10)
147 #define IRIS_STAGE_DIRTY_UNCOMPILED_CS (1ull << 11)
148 #define IRIS_STAGE_DIRTY_VS (1ull << 12)
149 #define IRIS_STAGE_DIRTY_TCS (1ull << 13)
150 #define IRIS_STAGE_DIRTY_TES (1ull << 14)
151 #define IRIS_STAGE_DIRTY_GS (1ull << 15)
152 #define IRIS_STAGE_DIRTY_FS (1ull << 16)
153 #define IRIS_STAGE_DIRTY_CS (1ull << 17)
154 #define IRIS_SHIFT_FOR_STAGE_DIRTY_CONSTANTS 18
155 #define IRIS_STAGE_DIRTY_CONSTANTS_VS (1ull << 18)
156 #define IRIS_STAGE_DIRTY_CONSTANTS_TCS (1ull << 19)
157 #define IRIS_STAGE_DIRTY_CONSTANTS_TES (1ull << 20)
158 #define IRIS_STAGE_DIRTY_CONSTANTS_GS (1ull << 21)
159 #define IRIS_STAGE_DIRTY_CONSTANTS_FS (1ull << 22)
160 #define IRIS_STAGE_DIRTY_CONSTANTS_CS (1ull << 23)
161 #define IRIS_SHIFT_FOR_STAGE_DIRTY_BINDINGS 24
162 #define IRIS_STAGE_DIRTY_BINDINGS_VS (1ull << 24)
163 #define IRIS_STAGE_DIRTY_BINDINGS_TCS (1ull << 25)
164 #define IRIS_STAGE_DIRTY_BINDINGS_TES (1ull << 26)
165 #define IRIS_STAGE_DIRTY_BINDINGS_GS (1ull << 27)
166 #define IRIS_STAGE_DIRTY_BINDINGS_FS (1ull << 28)
167 #define IRIS_STAGE_DIRTY_BINDINGS_CS (1ull << 29)
168
169 #define IRIS_ALL_STAGE_DIRTY_FOR_COMPUTE (IRIS_STAGE_DIRTY_CS | \
170 IRIS_STAGE_DIRTY_SAMPLER_STATES_CS | \
171 IRIS_STAGE_DIRTY_UNCOMPILED_CS | \
172 IRIS_STAGE_DIRTY_CONSTANTS_CS | \
173 IRIS_STAGE_DIRTY_BINDINGS_CS)
174
175 #define IRIS_ALL_STAGE_DIRTY_FOR_RENDER (~IRIS_ALL_STAGE_DIRTY_FOR_COMPUTE)
176
177 #define IRIS_ALL_STAGE_DIRTY_BINDINGS_FOR_RENDER (IRIS_STAGE_DIRTY_BINDINGS_VS | \
178 IRIS_STAGE_DIRTY_BINDINGS_TCS | \
179 IRIS_STAGE_DIRTY_BINDINGS_TES | \
180 IRIS_STAGE_DIRTY_BINDINGS_GS | \
181 IRIS_STAGE_DIRTY_BINDINGS_FS)
182
183 #define IRIS_ALL_STAGE_DIRTY_BINDINGS (IRIS_ALL_STAGE_DIRTY_BINDINGS_FOR_RENDER | \
184 IRIS_STAGE_DIRTY_BINDINGS_CS)
185
186 /**
187 * Non-orthogonal state (NOS) dependency flags.
188 *
189 * Shader programs may depend on non-orthogonal state. These flags are
190 * used to indicate that a shader's key depends on the state provided by
191 * a certain Gallium CSO. Changing any CSOs marked as a dependency will
192 * cause the driver to re-compute the shader key, possibly triggering a
193 * shader recompile.
194 */
195 enum iris_nos_dep {
196 IRIS_NOS_FRAMEBUFFER,
197 IRIS_NOS_DEPTH_STENCIL_ALPHA,
198 IRIS_NOS_RASTERIZER,
199 IRIS_NOS_BLEND,
200 IRIS_NOS_LAST_VUE_MAP,
201
202 IRIS_NOS_COUNT,
203 };
204
205 /** @{
206 *
207 * Program cache keys for state based recompiles.
208 */
209
210 /* Provide explicit padding for each member, to ensure that the compiler
211 * initializes every bit in the shader cache keys. The keys will be compared
212 * with memcmp.
213 */
214 PRAGMA_DIAGNOSTIC_PUSH
215 PRAGMA_DIAGNOSTIC_ERROR(-Wpadded)
216
217 /**
218 * Note, we need to take care to have padding explicitly declared
219 * for key since we will directly memcmp the whole struct.
220 */
221 struct iris_base_prog_key {
222 unsigned program_string_id;
223 bool limit_trig_input_range;
224 unsigned padding:24;
225 };
226
227 struct iris_vue_prog_key {
228 struct iris_base_prog_key base;
229
230 unsigned nr_userclip_plane_consts:4;
231 unsigned padding:28;
232 };
233
234 struct iris_vs_prog_key {
235 struct iris_vue_prog_key vue;
236 };
237
238 struct iris_tcs_prog_key {
239 struct iris_vue_prog_key vue;
240
241 enum tess_primitive_mode _tes_primitive_mode;
242
243 uint8_t input_vertices;
244
245 bool quads_workaround;
246 unsigned padding:16;
247
248 /** A bitfield of per-patch outputs written. */
249 uint32_t patch_outputs_written;
250
251 /** A bitfield of per-vertex outputs written. */
252 uint64_t outputs_written;
253 };
254
255 struct iris_tes_prog_key {
256 struct iris_vue_prog_key vue;
257
258 /** A bitfield of per-patch inputs read. */
259 uint32_t patch_inputs_read;
260
261 /** A bitfield of per-vertex inputs read. */
262 uint64_t inputs_read;
263 };
264
265 struct iris_gs_prog_key {
266 struct iris_vue_prog_key vue;
267 };
268
269 struct iris_fs_prog_key {
270 struct iris_base_prog_key base;
271
272 uint64_t input_slots_valid;
273 uint8_t color_outputs_valid;
274
275 unsigned nr_color_regions:5;
276 bool flat_shade:1;
277 bool alpha_test_replicate_alpha:1;
278 bool alpha_to_coverage:1;
279 bool clamp_fragment_color:1;
280 bool persample_interp:1;
281 bool multisample_fbo:1;
282 bool force_dual_color_blend:1;
283 bool coherent_fb_fetch:1;
284 uint64_t padding:43;
285 };
286
287 struct iris_cs_prog_key {
288 struct iris_base_prog_key base;
289 };
290
291 union iris_any_prog_key {
292 struct iris_base_prog_key base;
293 struct iris_vue_prog_key vue;
294 struct iris_vs_prog_key vs;
295 struct iris_tcs_prog_key tcs;
296 struct iris_tes_prog_key tes;
297 struct iris_gs_prog_key gs;
298 struct iris_fs_prog_key fs;
299 struct iris_cs_prog_key cs;
300 };
301
302 /* Restore the pack alignment to default. */
303 PRAGMA_DIAGNOSTIC_POP
304
305 /** @} */
306
307 struct iris_depth_stencil_alpha_state;
308
309 /**
310 * Cache IDs for the in-memory program cache (ice->shaders.cache).
311 */
312 enum iris_program_cache_id {
313 IRIS_CACHE_VS = MESA_SHADER_VERTEX,
314 IRIS_CACHE_TCS = MESA_SHADER_TESS_CTRL,
315 IRIS_CACHE_TES = MESA_SHADER_TESS_EVAL,
316 IRIS_CACHE_GS = MESA_SHADER_GEOMETRY,
317 IRIS_CACHE_FS = MESA_SHADER_FRAGMENT,
318 IRIS_CACHE_CS = MESA_SHADER_COMPUTE,
319 IRIS_CACHE_BLORP,
320 };
321
322 /** @{
323 *
324 * Defines for PIPE_CONTROL operations, which trigger cache flushes,
325 * synchronization, pipelined memory writes, and so on.
326 *
327 * The bits here are not the actual hardware values. The actual fields
328 * move between various generations, so we just have flags for each
329 * potential operation, and use genxml to encode the actual packet.
330 */
331 enum pipe_control_flags
332 {
333 PIPE_CONTROL_FLUSH_LLC = (1 << 1),
334 PIPE_CONTROL_LRI_POST_SYNC_OP = (1 << 2),
335 PIPE_CONTROL_STORE_DATA_INDEX = (1 << 3),
336 PIPE_CONTROL_CS_STALL = (1 << 4),
337 PIPE_CONTROL_GLOBAL_SNAPSHOT_COUNT_RESET = (1 << 5),
338 PIPE_CONTROL_SYNC_GFDT = (1 << 6),
339 PIPE_CONTROL_TLB_INVALIDATE = (1 << 7),
340 PIPE_CONTROL_MEDIA_STATE_CLEAR = (1 << 8),
341 PIPE_CONTROL_WRITE_IMMEDIATE = (1 << 9),
342 PIPE_CONTROL_WRITE_DEPTH_COUNT = (1 << 10),
343 PIPE_CONTROL_WRITE_TIMESTAMP = (1 << 11),
344 PIPE_CONTROL_DEPTH_STALL = (1 << 12),
345 PIPE_CONTROL_RENDER_TARGET_FLUSH = (1 << 13),
346 PIPE_CONTROL_INSTRUCTION_INVALIDATE = (1 << 14),
347 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE = (1 << 15),
348 PIPE_CONTROL_INDIRECT_STATE_POINTERS_DISABLE = (1 << 16),
349 PIPE_CONTROL_NOTIFY_ENABLE = (1 << 17),
350 PIPE_CONTROL_FLUSH_ENABLE = (1 << 18),
351 PIPE_CONTROL_DATA_CACHE_FLUSH = (1 << 19),
352 PIPE_CONTROL_VF_CACHE_INVALIDATE = (1 << 20),
353 PIPE_CONTROL_CONST_CACHE_INVALIDATE = (1 << 21),
354 PIPE_CONTROL_STATE_CACHE_INVALIDATE = (1 << 22),
355 PIPE_CONTROL_STALL_AT_SCOREBOARD = (1 << 23),
356 PIPE_CONTROL_DEPTH_CACHE_FLUSH = (1 << 24),
357 PIPE_CONTROL_TILE_CACHE_FLUSH = (1 << 25),
358 PIPE_CONTROL_FLUSH_HDC = (1 << 26),
359 PIPE_CONTROL_PSS_STALL_SYNC = (1 << 27),
360 PIPE_CONTROL_L3_READ_ONLY_CACHE_INVALIDATE = (1 << 28),
361 };
362
363 #define PIPE_CONTROL_CACHE_FLUSH_BITS \
364 (PIPE_CONTROL_DEPTH_CACHE_FLUSH | \
365 PIPE_CONTROL_DATA_CACHE_FLUSH | \
366 PIPE_CONTROL_TILE_CACHE_FLUSH | \
367 PIPE_CONTROL_FLUSH_HDC | \
368 PIPE_CONTROL_RENDER_TARGET_FLUSH)
369
370 #define PIPE_CONTROL_CACHE_INVALIDATE_BITS \
371 (PIPE_CONTROL_STATE_CACHE_INVALIDATE | \
372 PIPE_CONTROL_CONST_CACHE_INVALIDATE | \
373 PIPE_CONTROL_VF_CACHE_INVALIDATE | \
374 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \
375 PIPE_CONTROL_INSTRUCTION_INVALIDATE)
376
377 #define PIPE_CONTROL_L3_RO_INVALIDATE_BITS \
378 (PIPE_CONTROL_L3_READ_ONLY_CACHE_INVALIDATE | \
379 PIPE_CONTROL_CONST_CACHE_INVALIDATE)
380
381 enum iris_predicate_state {
382 /* The first two states are used if we can determine whether to draw
383 * without having to look at the values in the query object buffer. This
384 * will happen if there is no conditional render in progress, if the query
385 * object is already completed or if something else has already added
386 * samples to the preliminary result.
387 */
388 IRIS_PREDICATE_STATE_RENDER,
389 IRIS_PREDICATE_STATE_DONT_RENDER,
390
391 /* In this case whether to draw or not depends on the result of an
392 * MI_PREDICATE command so the predicate enable bit needs to be checked.
393 */
394 IRIS_PREDICATE_STATE_USE_BIT,
395 };
396
397 /** @} */
398
399 /**
400 * An uncompiled, API-facing shader. This is the Gallium CSO for shaders.
401 * It primarily contains the NIR for the shader.
402 *
403 * Each API-facing shader can be compiled into multiple shader variants,
404 * based on non-orthogonal state dependencies, recorded in the shader key.
405 *
406 * See iris_compiled_shader, which represents a compiled shader variant.
407 */
408 struct iris_uncompiled_shader {
409 struct pipe_reference ref;
410
411 /**
412 * NIR for the shader.
413 *
414 * Even for shaders that originate as TGSI, this pointer will be non-NULL.
415 */
416 struct nir_shader *nir;
417
418 struct pipe_stream_output_info stream_output;
419
420 /* A SHA1 of the serialized NIR for the disk cache. */
421 unsigned char nir_sha1[20];
422
423 unsigned program_id;
424
425 /** Bitfield of (1 << IRIS_NOS_*) flags. */
426 unsigned nos;
427
428 /** Have any shader variants been compiled yet? */
429 bool compiled_once;
430
431 /* Whether shader uses atomic operations. */
432 bool uses_atomic_load_store;
433
434 /** Size (in bytes) of the kernel input data */
435 unsigned kernel_input_size;
436
437 /** Size (in bytes) of the local (shared) data passed as kernel inputs */
438 unsigned kernel_shared_size;
439
440 /** List of iris_compiled_shader variants */
441 struct list_head variants;
442
443 /** Lock for the variants list */
444 simple_mtx_t lock;
445
446 /** For parallel shader compiles */
447 struct util_queue_fence ready;
448 };
449
450 enum iris_surface_group {
451 IRIS_SURFACE_GROUP_RENDER_TARGET,
452 IRIS_SURFACE_GROUP_RENDER_TARGET_READ,
453 IRIS_SURFACE_GROUP_CS_WORK_GROUPS,
454 IRIS_SURFACE_GROUP_TEXTURE,
455 IRIS_SURFACE_GROUP_IMAGE,
456 IRIS_SURFACE_GROUP_UBO,
457 IRIS_SURFACE_GROUP_SSBO,
458
459 IRIS_SURFACE_GROUP_COUNT,
460 };
461
462 enum {
463 /* Invalid value for a binding table index. */
464 IRIS_SURFACE_NOT_USED = 0xa0a0a0a0,
465 };
466
467 struct iris_binding_table {
468 uint32_t size_bytes;
469
470 /** Number of surfaces in each group, before compacting. */
471 uint32_t sizes[IRIS_SURFACE_GROUP_COUNT];
472
473 /** Initial offset of each group. */
474 uint32_t offsets[IRIS_SURFACE_GROUP_COUNT];
475
476 /** Mask of surfaces used in each group. */
477 uint64_t used_mask[IRIS_SURFACE_GROUP_COUNT];
478 };
479
480 /**
481 * A compiled shader variant, containing a pointer to the GPU assembly,
482 * as well as program data and other packets needed by state upload.
483 *
484 * There can be several iris_compiled_shader variants per API-level shader
485 * (iris_uncompiled_shader), due to state-based recompiles (brw_*_prog_key).
486 */
487 struct iris_compiled_shader {
488 struct pipe_reference ref;
489
490 /** Link in the iris_uncompiled_shader::variants list */
491 struct list_head link;
492
493 /** Key for this variant (but not for BLORP programs) */
494 union iris_any_prog_key key;
495
496 /**
497 * Is the variant fully compiled and ready?
498 *
499 * Variants are added to \c iris_uncompiled_shader::variants before
500 * compilation actually occurs. This signals that compilation has
501 * completed.
502 */
503 struct util_queue_fence ready;
504
505 /** Variant is ready, but compilation failed. */
506 bool compilation_failed;
507
508 /** Reference to the uploaded assembly. */
509 struct iris_state_ref assembly;
510
511 /** Pointer to the assembly in the BO's map. */
512 void *map;
513
514 /** The program data (owned by the program cache hash table) */
515 struct brw_stage_prog_data *prog_data;
516
517 /** A list of system values to be uploaded as uniforms. */
518 enum brw_param_builtin *system_values;
519 unsigned num_system_values;
520
521 /** Size (in bytes) of the kernel input data */
522 unsigned kernel_input_size;
523
524 /** Number of constbufs expected by the shader. */
525 unsigned num_cbufs;
526
527 /**
528 * Derived 3DSTATE_STREAMOUT and 3DSTATE_SO_DECL_LIST packets
529 * (the VUE-based information for transform feedback outputs).
530 */
531 uint32_t *streamout;
532
533 struct iris_binding_table bt;
534
535 /**
536 * Shader packets and other data derived from prog_data. These must be
537 * completely determined from prog_data.
538 */
539 uint8_t derived_data[0];
540 };
541
542 /**
543 * API context state that is replicated per shader stage.
544 */
545 struct iris_shader_state {
546 /** Uniform Buffers */
547 struct pipe_shader_buffer constbuf[PIPE_MAX_CONSTANT_BUFFERS];
548 struct iris_state_ref constbuf_surf_state[PIPE_MAX_CONSTANT_BUFFERS];
549
550 bool sysvals_need_upload;
551
552 /** Shader Storage Buffers */
553 struct pipe_shader_buffer ssbo[PIPE_MAX_SHADER_BUFFERS];
554 struct iris_state_ref ssbo_surf_state[PIPE_MAX_SHADER_BUFFERS];
555
556 /** Shader Storage Images (image load store) */
557 struct iris_image_view image[PIPE_MAX_SHADER_IMAGES];
558
559 struct iris_state_ref sampler_table;
560 struct iris_sampler_state *samplers[IRIS_MAX_TEXTURE_SAMPLERS];
561 struct iris_sampler_view *textures[IRIS_MAX_TEXTURE_SAMPLERS];
562
563 /** Bitfield of which constant buffers are bound (non-null). */
564 uint32_t bound_cbufs;
565 uint32_t dirty_cbufs;
566
567 /** Bitfield of which image views are bound (non-null). */
568 uint32_t bound_image_views;
569
570 /** Bitfield of which sampler views are bound (non-null). */
571 uint32_t bound_sampler_views;
572
573 /** Bitfield of which shader storage buffers are bound (non-null). */
574 uint32_t bound_ssbos;
575
576 /** Bitfield of which shader storage buffers are writable. */
577 uint32_t writable_ssbos;
578 };
579
580 /**
581 * Gallium CSO for stream output (transform feedback) targets.
582 */
583 struct iris_stream_output_target {
584 struct pipe_stream_output_target base;
585
586 /** Storage holding the offset where we're writing in the buffer */
587 struct iris_state_ref offset;
588
589 /** Stride (bytes-per-vertex) during this transform feedback operation */
590 uint16_t stride;
591
592 /** Does the next 3DSTATE_SO_BUFFER need to zero the offsets? */
593 bool zero_offset;
594 };
595
596 /**
597 * The API context (derived from pipe_context).
598 *
599 * Most driver state is tracked here.
600 */
601 struct iris_context {
602 struct pipe_context ctx;
603 struct threaded_context *thrctx;
604
605 /** A debug callback for KHR_debug output. */
606 struct util_debug_callback dbg;
607
608 /** A device reset status callback for notifying that the GPU is hosed. */
609 struct pipe_device_reset_callback reset;
610
611 /** A set of dmabuf resources dirtied beyond their default aux-states. */
612 struct set *dirty_dmabufs;
613
614 /** Slab allocator for iris_transfer_map objects. */
615 struct slab_child_pool transfer_pool;
616
617 /** Slab allocator for threaded_context's iris_transfer_map objects */
618 struct slab_child_pool transfer_pool_unsync;
619
620 struct blorp_context blorp;
621
622 struct iris_batch batches[IRIS_BATCH_COUNT];
623
624 struct u_upload_mgr *query_buffer_uploader;
625
626 struct intel_ds_device ds;
627
628 struct {
629 struct {
630 /**
631 * Either the value of BaseVertex for indexed draw calls or the value
632 * of the argument <first> for non-indexed draw calls.
633 */
634 int firstvertex;
635 int baseinstance;
636 } params;
637
638 /**
639 * Are the above values the ones stored in the draw_params buffer?
640 * If so, we can compare them against new values to see if anything
641 * changed. If not, we need to assume they changed.
642 */
643 bool params_valid;
644
645 /**
646 * Resource and offset that stores draw_parameters from the indirect
647 * buffer or to the buffer that stures the previous values for non
648 * indirect draws.
649 */
650 struct iris_state_ref draw_params;
651
652 struct {
653 /**
654 * The value of DrawID. This always comes in from it's own vertex
655 * buffer since it's not part of the indirect draw parameters.
656 */
657 int drawid;
658
659 /**
660 * Stores if an indexed or non-indexed draw (~0/0). Useful to
661 * calculate BaseVertex as an AND of firstvertex and is_indexed_draw.
662 */
663 int is_indexed_draw;
664 } derived_params;
665
666 /**
667 * Resource and offset used for GL_ARB_shader_draw_parameters which
668 * contains parameters that are not present in the indirect buffer as
669 * drawid and is_indexed_draw. They will go in their own vertex element.
670 */
671 struct iris_state_ref derived_draw_params;
672 } draw;
673
674 struct {
675 struct iris_uncompiled_shader *uncompiled[MESA_SHADER_STAGES];
676 struct iris_compiled_shader *prog[MESA_SHADER_STAGES];
677 struct iris_compiled_shader *last_vue_shader;
678 struct {
679 unsigned size[4];
680 unsigned entries[4];
681 unsigned start[4];
682 bool constrained;
683 } urb;
684
685 /** Uploader for shader assembly from the driver thread */
686 struct u_upload_mgr *uploader_driver;
687 /** Uploader for shader assembly from the threaded context */
688 struct u_upload_mgr *uploader_unsync;
689 struct hash_table *cache;
690
691 /** Is a GS or TES outputting points or lines? */
692 bool output_topology_is_points_or_lines;
693
694 /**
695 * Scratch buffers for various sizes and stages.
696 *
697 * Indexed by the "Per-Thread Scratch Space" field's 4-bit encoding,
698 * and shader stage.
699 */
700 struct iris_bo *scratch_bos[1 << 4][MESA_SHADER_STAGES];
701
702 /**
703 * Scratch buffer surface states on Gfx12.5+
704 */
705 struct iris_state_ref scratch_surfs[1 << 4];
706 } shaders;
707
708 struct intel_perf_context *perf_ctx;
709
710 /** Frame number for debug prints */
711 uint32_t frame;
712
713 struct {
714 uint64_t dirty;
715 uint64_t stage_dirty;
716 uint64_t stage_dirty_for_nos[IRIS_NOS_COUNT];
717
718 unsigned num_viewports;
719 unsigned sample_mask;
720 struct iris_blend_state *cso_blend;
721 struct iris_rasterizer_state *cso_rast;
722 struct iris_depth_stencil_alpha_state *cso_zsa;
723 struct iris_vertex_element_state *cso_vertex_elements;
724 struct pipe_blend_color blend_color;
725 struct pipe_poly_stipple poly_stipple;
726 struct pipe_viewport_state viewports[IRIS_MAX_VIEWPORTS];
727 struct pipe_scissor_state scissors[IRIS_MAX_VIEWPORTS];
728 struct pipe_stencil_ref stencil_ref;
729 struct pipe_framebuffer_state framebuffer;
730 struct pipe_clip_state clip_planes;
731
732 float default_outer_level[4];
733 float default_inner_level[2];
734
735 /** Bitfield of which vertex buffers are bound (non-null). */
736 uint64_t bound_vertex_buffers;
737
738 uint8_t patch_vertices;
739 bool primitive_restart;
740 unsigned cut_index;
741 enum pipe_prim_type prim_mode:8;
742 bool prim_is_points_or_lines;
743 uint8_t vertices_per_patch;
744
745 bool window_space_position;
746
747 /** The last compute group size */
748 uint32_t last_block[3];
749
750 /** The last compute grid size */
751 uint32_t last_grid[3];
752 /** The last compute grid dimensions */
753 uint32_t last_grid_dim;
754 /** Reference to the BO containing the compute grid size */
755 struct iris_state_ref grid_size;
756 /** Reference to the SURFACE_STATE for the compute grid resource */
757 struct iris_state_ref grid_surf_state;
758
759 /**
760 * Array of aux usages for drawing, altered to account for any
761 * self-dependencies from resources bound for sampling and rendering.
762 */
763 enum isl_aux_usage draw_aux_usage[BRW_MAX_DRAW_BUFFERS];
764
765 /** Aux usage of the fb's depth buffer (which may or may not exist). */
766 enum isl_aux_usage hiz_usage;
767
768 enum intel_urb_deref_block_size urb_deref_block_size;
769
770 /** Are depth writes enabled? (Depth buffer may or may not exist.) */
771 bool depth_writes_enabled;
772
773 /** Are stencil writes enabled? (Stencil buffer may or may not exist.) */
774 bool stencil_writes_enabled;
775
776 /** GenX-specific current state */
777 struct iris_genx_state *genx;
778
779 struct iris_shader_state shaders[MESA_SHADER_STAGES];
780
781 /** Do vertex shader uses shader draw parameters ? */
782 bool vs_uses_draw_params;
783 bool vs_uses_derived_draw_params;
784 bool vs_needs_sgvs_element;
785
786 /** Do vertex shader uses edge flag ? */
787 bool vs_needs_edge_flag;
788
789 /** Do any samplers need border color? One bit per shader stage. */
790 uint8_t need_border_colors;
791
792 /** Global resource bindings */
793 struct pipe_resource *global_bindings[IRIS_MAX_GLOBAL_BINDINGS];
794
795 struct pipe_stream_output_target *so_target[PIPE_MAX_SO_BUFFERS];
796 bool streamout_active;
797
798 bool statistics_counters_enabled;
799
800 /** Current conditional rendering mode */
801 enum iris_predicate_state predicate;
802
803 /**
804 * Query BO with a MI_PREDICATE_RESULT snapshot calculated on the
805 * render context that needs to be uploaded to the compute context.
806 */
807 struct iris_bo *compute_predicate;
808
809 /** Is a PIPE_QUERY_PRIMITIVES_GENERATED query active? */
810 bool prims_generated_query_active;
811
812 /** 3DSTATE_STREAMOUT and 3DSTATE_SO_DECL_LIST packets */
813 uint32_t *streamout;
814
815 /** The SURFACE_STATE for a 1x1x1 null surface. */
816 struct iris_state_ref unbound_tex;
817
818 /** The SURFACE_STATE for a framebuffer-sized null surface. */
819 struct iris_state_ref null_fb;
820
821 struct u_upload_mgr *surface_uploader;
822 struct u_upload_mgr *bindless_uploader;
823 struct u_upload_mgr *dynamic_uploader;
824
825 struct iris_binder binder;
826
827 /** The high 16-bits of the last VBO/index buffer addresses */
828 uint16_t last_vbo_high_bits[33];
829 uint16_t last_index_bo_high_bits;
830
831 /**
832 * Resources containing streamed state which our render context
833 * currently points to. Used to re-add these to the validation
834 * list when we start a new batch and haven't resubmitted commands.
835 */
836 struct {
837 struct pipe_resource *cc_vp;
838 struct pipe_resource *sf_cl_vp;
839 struct pipe_resource *color_calc;
840 struct pipe_resource *scissor;
841 struct pipe_resource *blend;
842 struct pipe_resource *index_buffer;
843 struct pipe_resource *cs_thread_ids;
844 struct pipe_resource *cs_desc;
845 } last_res;
846
847 /** Records the size of variable-length state for INTEL_DEBUG=bat */
848 struct hash_table_u64 *sizes;
849
850 /** Last rendering scale argument provided to genX(emit_hashing_mode). */
851 unsigned current_hash_scale;
852
853 /** Resource holding the pixel pipe hashing tables. */
854 struct pipe_resource *pixel_hashing_tables;
855 } state;
856 };
857
858 #define perf_debug(dbg, ...) do { \
859 if (INTEL_DEBUG(DEBUG_PERF)) \
860 dbg_printf(__VA_ARGS__); \
861 if (unlikely(dbg)) \
862 util_debug_message(dbg, PERF_INFO, __VA_ARGS__); \
863 } while(0)
864
865 struct pipe_context *
866 iris_create_context(struct pipe_screen *screen, void *priv, unsigned flags);
867 void iris_destroy_context(struct pipe_context *ctx);
868
869 void iris_lost_context_state(struct iris_batch *batch);
870
871 void iris_mark_dirty_dmabuf(struct iris_context *ice,
872 struct pipe_resource *res);
873 void iris_flush_dirty_dmabufs(struct iris_context *ice);
874
875 void iris_init_blit_functions(struct pipe_context *ctx);
876 void iris_init_clear_functions(struct pipe_context *ctx);
877 void iris_init_program_functions(struct pipe_context *ctx);
878 void iris_init_screen_program_functions(struct pipe_screen *pscreen);
879 void iris_init_resource_functions(struct pipe_context *ctx);
880 void iris_init_perfquery_functions(struct pipe_context *ctx);
881 void iris_update_compiled_shaders(struct iris_context *ice);
882 void iris_update_compiled_compute_shader(struct iris_context *ice);
883 void iris_fill_cs_push_const_buffer(struct brw_cs_prog_data *cs_prog_data,
884 unsigned threads,
885 uint32_t *dst);
886
887
888 /* iris_blit.c */
889 void iris_blorp_surf_for_resource(struct isl_device *isl_dev,
890 struct blorp_surf *surf,
891 struct pipe_resource *p_res,
892 enum isl_aux_usage aux_usage,
893 unsigned level,
894 bool is_render_target);
895 void iris_copy_region(struct blorp_context *blorp,
896 struct iris_batch *batch,
897 struct pipe_resource *dst,
898 unsigned dst_level,
899 unsigned dstx, unsigned dsty, unsigned dstz,
900 struct pipe_resource *src,
901 unsigned src_level,
902 const struct pipe_box *src_box);
903
904 static inline enum blorp_batch_flags
iris_blorp_flags_for_batch(struct iris_batch * batch)905 iris_blorp_flags_for_batch(struct iris_batch *batch)
906 {
907 if (batch->name == IRIS_BATCH_COMPUTE)
908 return BLORP_BATCH_USE_COMPUTE;
909
910 if (batch->name == IRIS_BATCH_BLITTER)
911 return BLORP_BATCH_USE_BLITTER;
912
913 return 0;
914 }
915
916 /* iris_draw.c */
917
918 void iris_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info *info,
919 unsigned drawid_offset,
920 const struct pipe_draw_indirect_info *indirect,
921 const struct pipe_draw_start_count_bias *draws,
922 unsigned num_draws);
923 void iris_launch_grid(struct pipe_context *, const struct pipe_grid_info *);
924
925 /* iris_pipe_control.c */
926
927 void iris_emit_pipe_control_flush(struct iris_batch *batch,
928 const char *reason, uint32_t flags);
929 void iris_emit_pipe_control_write(struct iris_batch *batch,
930 const char *reason, uint32_t flags,
931 struct iris_bo *bo, uint32_t offset,
932 uint64_t imm);
933 void iris_emit_end_of_pipe_sync(struct iris_batch *batch,
934 const char *reason, uint32_t flags);
935 void iris_emit_buffer_barrier_for(struct iris_batch *batch,
936 struct iris_bo *bo,
937 enum iris_domain access);
938 void iris_flush_all_caches(struct iris_batch *batch);
939
940 #define iris_handle_always_flush_cache(batch) \
941 if (unlikely(batch->screen->driconf.always_flush_cache)) \
942 iris_flush_all_caches(batch);
943
944 void iris_init_flush_functions(struct pipe_context *ctx);
945
946 /* iris_program.c */
947 void iris_upload_ubo_ssbo_surf_state(struct iris_context *ice,
948 struct pipe_shader_buffer *buf,
949 struct iris_state_ref *surf_state,
950 isl_surf_usage_flags_t usage);
951 const struct shader_info *iris_get_shader_info(const struct iris_context *ice,
952 gl_shader_stage stage);
953 struct iris_bo *iris_get_scratch_space(struct iris_context *ice,
954 unsigned per_thread_scratch,
955 gl_shader_stage stage);
956 const struct iris_state_ref *iris_get_scratch_surf(struct iris_context *ice,
957 unsigned per_thread_scratch);
958 uint32_t iris_group_index_to_bti(const struct iris_binding_table *bt,
959 enum iris_surface_group group,
960 uint32_t index);
961 uint32_t iris_bti_to_group_index(const struct iris_binding_table *bt,
962 enum iris_surface_group group,
963 uint32_t bti);
964
965 /* iris_disk_cache.c */
966
967 void iris_disk_cache_store(struct disk_cache *cache,
968 const struct iris_uncompiled_shader *ish,
969 const struct iris_compiled_shader *shader,
970 const void *prog_key,
971 uint32_t prog_key_size);
972 bool
973 iris_disk_cache_retrieve(struct iris_screen *screen,
974 struct u_upload_mgr *uploader,
975 struct iris_uncompiled_shader *ish,
976 struct iris_compiled_shader *shader,
977 const void *prog_key,
978 uint32_t prog_key_size);
979
980 /* iris_program_cache.c */
981
982 void iris_init_program_cache(struct iris_context *ice);
983 void iris_destroy_program_cache(struct iris_context *ice);
984 struct iris_compiled_shader *iris_find_cached_shader(struct iris_context *ice,
985 enum iris_program_cache_id,
986 uint32_t key_size,
987 const void *key);
988
989 struct iris_compiled_shader *iris_create_shader_variant(const struct iris_screen *,
990 void *mem_ctx,
991 enum iris_program_cache_id cache_id,
992 uint32_t key_size,
993 const void *key);
994
995 void iris_finalize_program(struct iris_compiled_shader *shader,
996 struct brw_stage_prog_data *prog_data,
997 uint32_t *streamout,
998 enum brw_param_builtin *system_values,
999 unsigned num_system_values,
1000 unsigned kernel_input_size,
1001 unsigned num_cbufs,
1002 const struct iris_binding_table *bt);
1003
1004 void iris_upload_shader(struct iris_screen *screen,
1005 struct iris_uncompiled_shader *,
1006 struct iris_compiled_shader *,
1007 struct hash_table *driver_ht,
1008 struct u_upload_mgr *uploader,
1009 enum iris_program_cache_id,
1010 uint32_t key_size,
1011 const void *key,
1012 const void *assembly);
1013 void iris_delete_shader_variant(struct iris_compiled_shader *shader);
1014
1015 void iris_destroy_shader_state(struct pipe_context *ctx, void *state);
1016
1017 static inline void
iris_uncompiled_shader_reference(struct pipe_context * ctx,struct iris_uncompiled_shader ** dst,struct iris_uncompiled_shader * src)1018 iris_uncompiled_shader_reference(struct pipe_context *ctx,
1019 struct iris_uncompiled_shader **dst,
1020 struct iris_uncompiled_shader *src)
1021 {
1022 if (*dst == src)
1023 return;
1024
1025 struct iris_uncompiled_shader *old_dst = *dst;
1026
1027 if (pipe_reference(old_dst != NULL ? &old_dst->ref : NULL,
1028 src != NULL ? &src->ref : NULL)) {
1029 iris_destroy_shader_state(ctx, *dst);
1030 }
1031
1032 *dst = src;
1033 }
1034
1035 static inline void
iris_shader_variant_reference(struct iris_compiled_shader ** dst,struct iris_compiled_shader * src)1036 iris_shader_variant_reference(struct iris_compiled_shader **dst,
1037 struct iris_compiled_shader *src)
1038 {
1039 struct iris_compiled_shader *old_dst = *dst;
1040
1041 if (pipe_reference(old_dst ? &old_dst->ref: NULL, src ? &src->ref : NULL))
1042 iris_delete_shader_variant(old_dst);
1043
1044 *dst = src;
1045 }
1046
1047 bool iris_blorp_lookup_shader(struct blorp_batch *blorp_batch,
1048 const void *key,
1049 uint32_t key_size,
1050 uint32_t *kernel_out,
1051 void *prog_data_out);
1052 bool iris_blorp_upload_shader(struct blorp_batch *blorp_batch, uint32_t stage,
1053 const void *key, uint32_t key_size,
1054 const void *kernel, uint32_t kernel_size,
1055 const struct brw_stage_prog_data *prog_data,
1056 uint32_t prog_data_size,
1057 uint32_t *kernel_out,
1058 void *prog_data_out);
1059
1060 /* iris_resolve.c */
1061
1062 void iris_predraw_resolve_inputs(struct iris_context *ice,
1063 struct iris_batch *batch,
1064 bool *draw_aux_buffer_disabled,
1065 gl_shader_stage stage,
1066 bool consider_framebuffer);
1067 void iris_predraw_resolve_framebuffer(struct iris_context *ice,
1068 struct iris_batch *batch,
1069 bool *draw_aux_buffer_disabled);
1070 void iris_predraw_flush_buffers(struct iris_context *ice,
1071 struct iris_batch *batch,
1072 gl_shader_stage stage);
1073 void iris_postdraw_update_resolve_tracking(struct iris_context *ice,
1074 struct iris_batch *batch);
1075 void iris_cache_flush_for_render(struct iris_batch *batch,
1076 struct iris_bo *bo,
1077 enum isl_aux_usage aux_usage);
1078 int iris_get_driver_query_info(struct pipe_screen *pscreen, unsigned index,
1079 struct pipe_driver_query_info *info);
1080 int iris_get_driver_query_group_info(struct pipe_screen *pscreen,
1081 unsigned index,
1082 struct pipe_driver_query_group_info *info);
1083
1084 /* iris_state.c */
1085 void gfx9_toggle_preemption(struct iris_context *ice,
1086 struct iris_batch *batch,
1087 const struct pipe_draw_info *draw);
1088
1089
1090
1091 #ifdef genX
1092 # include "iris_genx_protos.h"
1093 #else
1094 # define genX(x) gfx4_##x
1095 # include "iris_genx_protos.h"
1096 # undef genX
1097 # define genX(x) gfx5_##x
1098 # include "iris_genx_protos.h"
1099 # undef genX
1100 # define genX(x) gfx6_##x
1101 # include "iris_genx_protos.h"
1102 # undef genX
1103 # define genX(x) gfx7_##x
1104 # include "iris_genx_protos.h"
1105 # undef genX
1106 # define genX(x) gfx75_##x
1107 # include "iris_genx_protos.h"
1108 # undef genX
1109 # define genX(x) gfx8_##x
1110 # include "iris_genx_protos.h"
1111 # undef genX
1112 # define genX(x) gfx9_##x
1113 # include "iris_genx_protos.h"
1114 # undef genX
1115 # define genX(x) gfx11_##x
1116 # include "iris_genx_protos.h"
1117 # undef genX
1118 # define genX(x) gfx12_##x
1119 # include "iris_genx_protos.h"
1120 # undef genX
1121 # define genX(x) gfx125_##x
1122 # include "iris_genx_protos.h"
1123 # undef genX
1124 #endif
1125
1126 #endif
1127