1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #ifndef BLORP_GENX_EXEC_ELK_H
25 #define BLORP_GENX_EXEC_ELK_H
26
27 #if GFX_VER > 8
28 #error "ELK doesn't support Gfx > 8."
29 #endif
30
31 #include "blorp_priv.h"
32 #include "dev/intel_device_info.h"
33 #include "common/intel_sample_positions.h"
34 #include "common/intel_l3_config.h"
35 #include "genxml/gen_macros.h"
36 #include "intel/compiler/elk/elk_compiler.h"
37
38 /**
39 * This file provides the blorp pipeline setup and execution functionality.
40 * It defines the following function:
41 *
42 * static void
43 * blorp_exec(struct blorp_context *blorp, void *batch_data,
44 * const struct blorp_params *params);
45 *
46 * It is the job of whoever includes this header to wrap this in something
47 * to get an externally visible symbol.
48 *
49 * In order for the blorp_exec function to work, the driver must provide
50 * implementations of the following static helper functions.
51 */
52
53 static void *
54 blorp_emit_dwords(struct blorp_batch *batch, unsigned n);
55
56 static uint64_t
57 blorp_emit_reloc(struct blorp_batch *batch,
58 void *location, struct blorp_address address, uint32_t delta);
59
60 static void
61 blorp_measure_start(struct blorp_batch *batch,
62 const struct blorp_params *params);
63
64 static void
65 blorp_measure_end(struct blorp_batch *batch,
66 const struct blorp_params *params);
67
68 static void *
69 blorp_alloc_dynamic_state(struct blorp_batch *batch,
70 uint32_t size,
71 uint32_t alignment,
72 uint32_t *offset);
73
74 UNUSED static void *
75 blorp_alloc_general_state(struct blorp_batch *batch,
76 uint32_t size,
77 uint32_t alignment,
78 uint32_t *offset);
79
80 static void *
81 blorp_alloc_vertex_buffer(struct blorp_batch *batch, uint32_t size,
82 struct blorp_address *addr);
83 static void
84 blorp_vf_invalidate_for_vb_48b_transitions(struct blorp_batch *batch,
85 const struct blorp_address *addrs,
86 uint32_t *sizes,
87 unsigned num_vbs);
88
89 UNUSED static struct blorp_address
90 blorp_get_workaround_address(struct blorp_batch *batch);
91
92 static bool
93 blorp_alloc_binding_table(struct blorp_batch *batch, unsigned num_entries,
94 unsigned state_size, unsigned state_alignment,
95 uint32_t *bt_offset, uint32_t *surface_offsets,
96 void **surface_maps);
97
98 static uint32_t
99 blorp_binding_table_offset_to_pointer(struct blorp_batch *batch,
100 uint32_t offset);
101
102 static void
103 blorp_flush_range(struct blorp_batch *batch, void *start, size_t size);
104
105 static void
106 blorp_surface_reloc(struct blorp_batch *batch, uint32_t ss_offset,
107 struct blorp_address address, uint32_t delta);
108
109 static uint64_t
110 blorp_get_surface_address(struct blorp_batch *batch,
111 struct blorp_address address);
112
113 #if GFX_VER >= 7
114 static struct blorp_address
115 blorp_get_surface_base_address(struct blorp_batch *batch);
116 #endif
117
118 #if GFX_VER >= 7
119 static const struct intel_l3_config *
120 blorp_get_l3_config(struct blorp_batch *batch);
121 #endif
122
123 static void
124 blorp_pre_emit_urb_config(struct blorp_batch *batch,
125 struct intel_urb_config *urb_config);
126
127 static void
128 blorp_emit_urb_config(struct blorp_batch *batch,
129 struct intel_urb_config *urb_config);
130
131 static void
132 blorp_emit_pipeline(struct blorp_batch *batch,
133 const struct blorp_params *params);
134
135 static void
136 blorp_emit_pre_draw(struct blorp_batch *batch,
137 const struct blorp_params *params);
138 static void
139 blorp_emit_post_draw(struct blorp_batch *batch,
140 const struct blorp_params *params);
141
142 static inline unsigned
elk_blorp_get_urb_length(const struct elk_wm_prog_data * prog_data)143 elk_blorp_get_urb_length(const struct elk_wm_prog_data *prog_data)
144 {
145 if (prog_data == NULL)
146 return 1;
147
148 /* From the BSpec: 3D Pipeline - Strips and Fans - 3DSTATE_SBE
149 *
150 * read_length = ceiling((max_source_attr+1)/2)
151 */
152 return MAX2((prog_data->num_varying_inputs + 1) / 2, 1);
153 }
154
155 /***** BEGIN blorp_exec implementation ******/
156
157 static uint64_t
_blorp_combine_address(struct blorp_batch * batch,void * location,struct blorp_address address,uint32_t delta)158 _blorp_combine_address(struct blorp_batch *batch, void *location,
159 struct blorp_address address, uint32_t delta)
160 {
161 if (address.buffer == NULL) {
162 return address.offset + delta;
163 } else {
164 return blorp_emit_reloc(batch, location, address, delta);
165 }
166 }
167
168 #define __gen_address_type struct blorp_address
169 #define __gen_user_data struct blorp_batch
170 #define __gen_combine_address _blorp_combine_address
171
172 #include "genxml/genX_pack.h"
173 #include "common/intel_genX_state_elk.h"
174
175 #define _blorp_cmd_length(cmd) cmd ## _length
176 #define _blorp_cmd_length_bias(cmd) cmd ## _length_bias
177 #define _blorp_cmd_header(cmd) cmd ## _header
178 #define _blorp_cmd_pack(cmd) cmd ## _pack
179
180 #define blorp_emit(batch, cmd, name) \
181 for (struct cmd name = { _blorp_cmd_header(cmd) }, \
182 *_dst = blorp_emit_dwords(batch, _blorp_cmd_length(cmd)); \
183 __builtin_expect(_dst != NULL, 1); \
184 _blorp_cmd_pack(cmd)(batch, (void *)_dst, &name), \
185 _dst = NULL)
186
187 #define blorp_emitn(batch, cmd, n, ...) ({ \
188 uint32_t *_dw = blorp_emit_dwords(batch, n); \
189 if (_dw) { \
190 struct cmd template = { \
191 _blorp_cmd_header(cmd), \
192 .DWordLength = n - _blorp_cmd_length_bias(cmd), \
193 __VA_ARGS__ \
194 }; \
195 _blorp_cmd_pack(cmd)(batch, _dw, &template); \
196 } \
197 _dw ? _dw + 1 : NULL; /* Array starts at dw[1] */ \
198 })
199
200 #define STRUCT_ZERO(S) ({ struct S t; memset(&t, 0, sizeof(t)); t; })
201
202 #define blorp_emit_dynamic(batch, state, name, align, offset) \
203 for (struct state name = STRUCT_ZERO(state), \
204 *_dst = blorp_alloc_dynamic_state(batch, \
205 _blorp_cmd_length(state) * 4, \
206 align, offset); \
207 __builtin_expect(_dst != NULL, 1); \
208 _blorp_cmd_pack(state)(batch, (void *)_dst, &name), \
209 blorp_flush_range(batch, _dst, _blorp_cmd_length(state) * 4), \
210 _dst = NULL)
211
212 /* 3DSTATE_URB
213 * 3DSTATE_URB_VS
214 * 3DSTATE_URB_HS
215 * 3DSTATE_URB_DS
216 * 3DSTATE_URB_GS
217 *
218 * Assign the entire URB to the VS. Even though the VS disabled, URB space
219 * is still needed because the clipper loads the VUE's from the URB. From
220 * the Sandybridge PRM, Volume 2, Part 1, Section 3DSTATE,
221 * Dword 1.15:0 "VS Number of URB Entries":
222 * This field is always used (even if VS Function Enable is DISABLED).
223 *
224 * The warning below appears in the PRM (Section 3DSTATE_URB), but we can
225 * safely ignore it because this batch contains only one draw call.
226 * Because of URB corruption caused by allocating a previous GS unit
227 * URB entry to the VS unit, software is required to send a “GS NULL
228 * Fence” (Send URB fence with VS URB size == 1 and GS URB size == 0)
229 * plus a dummy DRAW call before any case where VS will be taking over
230 * GS URB space.
231 *
232 * If the 3DSTATE_URB_VS is emitted, than the others must be also.
233 * From the Ivybridge PRM, Volume 2 Part 1, section 1.7.1 3DSTATE_URB_VS:
234 *
235 * 3DSTATE_URB_HS, 3DSTATE_URB_DS, and 3DSTATE_URB_GS must also be
236 * programmed in order for the programming of this state to be
237 * valid.
238 */
239 static void
emit_urb_config(struct blorp_batch * batch,const struct blorp_params * params,UNUSED enum intel_urb_deref_block_size * deref_block_size)240 emit_urb_config(struct blorp_batch *batch,
241 const struct blorp_params *params,
242 UNUSED enum intel_urb_deref_block_size *deref_block_size)
243 {
244 /* Once vertex fetcher has written full VUE entries with complete
245 * header the space requirement is as follows per vertex (in bytes):
246 *
247 * Header Position Program constants
248 * +--------+------------+-------------------+
249 * | 16 | 16 | n x 16 |
250 * +--------+------------+-------------------+
251 *
252 * where 'n' stands for number of varying inputs expressed as vec4s.
253 */
254 struct elk_wm_prog_data *wm_prog_data = params->wm_prog_data;
255 const unsigned num_varyings =
256 wm_prog_data ? wm_prog_data->num_varying_inputs : 0;
257 const unsigned total_needed = 16 + 16 + num_varyings * 16;
258
259 /* The URB size is expressed in units of 64 bytes (512 bits) */
260 const unsigned vs_entry_size = DIV_ROUND_UP(total_needed, 64);
261
262 ASSERTED struct elk_sf_prog_data *sf_prog_data = params->sf_prog_data;
263 ASSERTED const unsigned sf_entry_size =
264 sf_prog_data ? sf_prog_data->urb_entry_size : 0;
265
266 #if GFX_VER >= 7
267 assert(sf_entry_size == 0);
268
269 struct intel_urb_config urb_cfg = {
270 .size = { vs_entry_size, 1, 1, 1 },
271 };
272
273 bool constrained;
274 intel_get_urb_config(batch->blorp->compiler->elk->devinfo,
275 blorp_get_l3_config(batch),
276 false, false, &urb_cfg,
277 deref_block_size, &constrained);
278
279 /* Tell drivers about the config. */
280 blorp_pre_emit_urb_config(batch, &urb_cfg);
281
282 #if GFX_VERx10 == 70
283 /* From the IVB PRM Vol. 2, Part 1, Section 3.2.1:
284 *
285 * "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth stall
286 * needs to be sent just prior to any 3DSTATE_VS, 3DSTATE_URB_VS,
287 * 3DSTATE_CONSTANT_VS, 3DSTATE_BINDING_TABLE_POINTER_VS,
288 * 3DSTATE_SAMPLER_STATE_POINTER_VS command. Only one PIPE_CONTROL
289 * needs to be sent before any combination of VS associated 3DSTATE."
290 */
291 blorp_emit(batch, GENX(PIPE_CONTROL), pc) {
292 pc.DepthStallEnable = true;
293 pc.PostSyncOperation = WriteImmediateData;
294 pc.Address = blorp_get_workaround_address(batch);
295 }
296 #endif
297
298 for (int i = 0; i <= MESA_SHADER_GEOMETRY; i++) {
299 blorp_emit(batch, GENX(3DSTATE_URB_VS), urb) {
300 urb._3DCommandSubOpcode += i;
301 urb.VSURBStartingAddress = urb_cfg.start[i];
302 urb.VSURBEntryAllocationSize = urb_cfg.size[i] - 1;
303 urb.VSNumberofURBEntries = urb_cfg.entries[i];
304 }
305 }
306
307 #else /* GFX_VER < 7 */
308 struct intel_urb_config urb_cfg = {
309 .size = { vs_entry_size, 0, 0, 0, sf_entry_size, },
310 };
311 blorp_emit_urb_config(batch, &urb_cfg);
312 #endif
313 }
314
315 #if GFX_VER >= 7
316 static void
317 blorp_emit_memcpy(struct blorp_batch *batch,
318 struct blorp_address dst,
319 struct blorp_address src,
320 uint32_t size);
321 #endif
322
323 static void
blorp_emit_vertex_data(struct blorp_batch * batch,const struct blorp_params * params,struct blorp_address * addr,uint32_t * size)324 blorp_emit_vertex_data(struct blorp_batch *batch,
325 const struct blorp_params *params,
326 struct blorp_address *addr,
327 uint32_t *size)
328 {
329 const float vertices[] = {
330 /* v0 */ (float)params->x1, (float)params->y1, params->z,
331 /* v1 */ (float)params->x0, (float)params->y1, params->z,
332 /* v2 */ (float)params->x0, (float)params->y0, params->z,
333 };
334
335 void *data = blorp_alloc_vertex_buffer(batch, sizeof(vertices), addr);
336 memcpy(data, vertices, sizeof(vertices));
337 *size = sizeof(vertices);
338 blorp_flush_range(batch, data, *size);
339 }
340
341 static void
blorp_emit_input_varying_data(struct blorp_batch * batch,const struct blorp_params * params,struct blorp_address * addr,uint32_t * size)342 blorp_emit_input_varying_data(struct blorp_batch *batch,
343 const struct blorp_params *params,
344 struct blorp_address *addr,
345 uint32_t *size)
346 {
347 const unsigned vec4_size_in_bytes = 4 * sizeof(float);
348 const unsigned max_num_varyings =
349 DIV_ROUND_UP(sizeof(params->wm_inputs), vec4_size_in_bytes);
350 struct elk_wm_prog_data *wm_prog_data = params->wm_prog_data;
351 const unsigned num_varyings =
352 wm_prog_data ? wm_prog_data->num_varying_inputs : 0;
353
354 *size = 16 + num_varyings * vec4_size_in_bytes;
355
356 const uint32_t *const inputs_src = (const uint32_t *)¶ms->wm_inputs;
357 void *data = blorp_alloc_vertex_buffer(batch, *size, addr);
358 uint32_t *inputs = data;
359
360 /* Copy in the VS inputs */
361 assert(sizeof(params->vs_inputs) == 16);
362 memcpy(inputs, ¶ms->vs_inputs, sizeof(params->vs_inputs));
363 inputs += 4;
364
365 if (params->wm_prog_data) {
366 /* Walk over the attribute slots, determine if the attribute is used by
367 * the program and when necessary copy the values from the input storage
368 * to the vertex data buffer.
369 */
370 for (unsigned i = 0; i < max_num_varyings; i++) {
371 const gl_varying_slot attr = VARYING_SLOT_VAR0 + i;
372
373 const int input_index = wm_prog_data->urb_setup[attr];
374 if (input_index < 0)
375 continue;
376
377 memcpy(inputs, inputs_src + i * 4, vec4_size_in_bytes);
378
379 inputs += 4;
380 }
381 }
382
383 blorp_flush_range(batch, data, *size);
384
385 if (params->dst_clear_color_as_input) {
386 #if GFX_VER >= 7
387 /* In this case, the clear color isn't known statically and instead
388 * comes in through an indirect which we have to copy into the vertex
389 * buffer before we execute the 3DPRIMITIVE. We already copied the
390 * value of params->wm_inputs.clear_color into the vertex buffer in the
391 * loop above. Now we emit code to stomp it from the GPU with the
392 * actual clear color value.
393 */
394 assert(num_varyings == 1);
395
396 /* The clear color is the first thing after the header */
397 struct blorp_address clear_color_input_addr = *addr;
398 clear_color_input_addr.offset += 16;
399
400 const unsigned clear_color_size = batch->blorp->isl_dev->ss.clear_value_size;
401 blorp_emit_memcpy(batch, clear_color_input_addr,
402 params->dst.clear_color_addr,
403 clear_color_size);
404 #else
405 unreachable("MCS partial resolve is not a thing on SNB and earlier");
406 #endif
407 }
408 }
409
410 static void
blorp_fill_vertex_buffer_state(struct GENX (VERTEX_BUFFER_STATE)* vb,unsigned idx,struct blorp_address addr,uint32_t size,uint32_t stride)411 blorp_fill_vertex_buffer_state(struct GENX(VERTEX_BUFFER_STATE) *vb,
412 unsigned idx,
413 struct blorp_address addr, uint32_t size,
414 uint32_t stride)
415 {
416 vb[idx].VertexBufferIndex = idx;
417 vb[idx].BufferStartingAddress = addr;
418 vb[idx].BufferPitch = stride;
419
420 #if GFX_VER >= 6
421 vb[idx].MOCS = addr.mocs;
422 #endif
423
424 #if GFX_VER >= 7
425 vb[idx].AddressModifyEnable = true;
426 #endif
427
428 #if GFX_VER >= 8
429 vb[idx].BufferSize = size;
430 #elif GFX_VER >= 5
431 vb[idx].BufferAccessType = stride > 0 ? VERTEXDATA : INSTANCEDATA;
432 vb[idx].EndAddress = vb[idx].BufferStartingAddress;
433 vb[idx].EndAddress.offset += size - 1;
434 #elif GFX_VER == 4
435 vb[idx].BufferAccessType = stride > 0 ? VERTEXDATA : INSTANCEDATA;
436 vb[idx].MaxIndex = stride > 0 ? size / stride : 0;
437 #endif
438 }
439
440 static void
blorp_emit_vertex_buffers(struct blorp_batch * batch,const struct blorp_params * params)441 blorp_emit_vertex_buffers(struct blorp_batch *batch,
442 const struct blorp_params *params)
443 {
444 struct GENX(VERTEX_BUFFER_STATE) vb[2] = {};
445 const uint32_t num_vbs = ARRAY_SIZE(vb);
446
447 struct blorp_address addrs[2] = {};
448 uint32_t sizes[2];
449 blorp_emit_vertex_data(batch, params, &addrs[0], &sizes[0]);
450 blorp_fill_vertex_buffer_state(vb, 0, addrs[0], sizes[0],
451 3 * sizeof(float));
452
453 blorp_emit_input_varying_data(batch, params, &addrs[1], &sizes[1]);
454 blorp_fill_vertex_buffer_state(vb, 1, addrs[1], sizes[1], 0);
455
456 blorp_vf_invalidate_for_vb_48b_transitions(batch, addrs, sizes, num_vbs);
457
458 const unsigned num_dwords = 1 + num_vbs * GENX(VERTEX_BUFFER_STATE_length);
459 uint32_t *dw = blorp_emitn(batch, GENX(3DSTATE_VERTEX_BUFFERS), num_dwords);
460 if (!dw)
461 return;
462
463 for (unsigned i = 0; i < num_vbs; i++) {
464 GENX(VERTEX_BUFFER_STATE_pack)(batch, dw, &vb[i]);
465 dw += GENX(VERTEX_BUFFER_STATE_length);
466 }
467 }
468
469 static void
blorp_emit_vertex_elements(struct blorp_batch * batch,const struct blorp_params * params)470 blorp_emit_vertex_elements(struct blorp_batch *batch,
471 const struct blorp_params *params)
472 {
473 struct elk_wm_prog_data *wm_prog_data = params->wm_prog_data;
474 const unsigned num_varyings =
475 wm_prog_data ? wm_prog_data->num_varying_inputs : 0;
476 bool need_ndc = batch->blorp->compiler->elk->devinfo->ver <= 5;
477 const unsigned num_elements = 2 + need_ndc + num_varyings;
478
479 struct GENX(VERTEX_ELEMENT_STATE) ve[num_elements];
480 memset(ve, 0, num_elements * sizeof(*ve));
481
482 /* Setup VBO for the rectangle primitive..
483 *
484 * A rectangle primitive (3DPRIM_RECTLIST) consists of only three
485 * vertices. The vertices reside in screen space with DirectX
486 * coordinates (that is, (0, 0) is the upper left corner).
487 *
488 * v2 ------ implied
489 * | |
490 * | |
491 * v1 ----- v0
492 *
493 * Since the VS is disabled, the clipper loads each VUE directly from
494 * the URB. This is controlled by the 3DSTATE_VERTEX_BUFFERS and
495 * 3DSTATE_VERTEX_ELEMENTS packets below. The VUE contents are as follows:
496 * dw0: Reserved, MBZ.
497 * dw1: Render Target Array Index. Below vertex fetcher gets programmed
498 * to assign this with primitive instance identifier which will be
499 * used for layered clears. All other renders have only one instance
500 * and therefore the value will be effectively zero.
501 * dw2: Viewport Index. The HiZ op disables viewport mapping and
502 * scissoring, so set the dword to 0.
503 * dw3: Point Width: The HiZ op does not emit the POINTLIST primitive,
504 * so set the dword to 0.
505 * dw4: Vertex Position X.
506 * dw5: Vertex Position Y.
507 * dw6: Vertex Position Z.
508 * dw7: Vertex Position W.
509 *
510 * dw8: Flat vertex input 0
511 * dw9: Flat vertex input 1
512 * ...
513 * dwn: Flat vertex input n - 8
514 *
515 * For details, see the Sandybridge PRM, Volume 2, Part 1, Section 1.5.1
516 * "Vertex URB Entry (VUE) Formats".
517 *
518 * Only vertex position X and Y are going to be variable, Z is fixed to
519 * zero and W to one. Header words dw0,2,3 are zero. There is no need to
520 * include the fixed values in the vertex buffer. Vertex fetcher can be
521 * instructed to fill vertex elements with constant values of one and zero
522 * instead of reading them from the buffer.
523 * Flat inputs are program constants that are not interpolated. Moreover
524 * their values will be the same between vertices.
525 *
526 * See the vertex element setup below.
527 */
528 unsigned slot = 0;
529
530 ve[slot] = (struct GENX(VERTEX_ELEMENT_STATE)) {
531 .VertexBufferIndex = 1,
532 .Valid = true,
533 .SourceElementFormat = ISL_FORMAT_R32G32B32A32_FLOAT,
534 .SourceElementOffset = 0,
535 .Component0Control = VFCOMP_STORE_SRC,
536
537 /* From Gfx8 onwards hardware is no more instructed to overwrite
538 * components using an element specifier. Instead one has separate
539 * 3DSTATE_VF_SGVS (System Generated Value Setup) state packet for it.
540 */
541 #if GFX_VER >= 8
542 .Component1Control = VFCOMP_STORE_0,
543 #elif GFX_VER >= 5
544 .Component1Control = VFCOMP_STORE_IID,
545 #else
546 .Component1Control = VFCOMP_STORE_0,
547 #endif
548 .Component2Control = VFCOMP_STORE_0,
549 .Component3Control = VFCOMP_STORE_0,
550 #if GFX_VER <= 5
551 .DestinationElementOffset = slot * 4,
552 #endif
553 };
554 slot++;
555
556 #if GFX_VER <= 5
557 /* On Iron Lake and earlier, a native device coordinates version of the
558 * position goes right after the normal VUE header and before position.
559 * Since w == 1 for all of our coordinates, this is just a copy of the
560 * position.
561 */
562 ve[slot] = (struct GENX(VERTEX_ELEMENT_STATE)) {
563 .VertexBufferIndex = 0,
564 .Valid = true,
565 .SourceElementFormat = ISL_FORMAT_R32G32B32_FLOAT,
566 .SourceElementOffset = 0,
567 .Component0Control = VFCOMP_STORE_SRC,
568 .Component1Control = VFCOMP_STORE_SRC,
569 .Component2Control = VFCOMP_STORE_SRC,
570 .Component3Control = VFCOMP_STORE_1_FP,
571 .DestinationElementOffset = slot * 4,
572 };
573 slot++;
574 #endif
575
576 ve[slot] = (struct GENX(VERTEX_ELEMENT_STATE)) {
577 .VertexBufferIndex = 0,
578 .Valid = true,
579 .SourceElementFormat = ISL_FORMAT_R32G32B32_FLOAT,
580 .SourceElementOffset = 0,
581 .Component0Control = VFCOMP_STORE_SRC,
582 .Component1Control = VFCOMP_STORE_SRC,
583 .Component2Control = VFCOMP_STORE_SRC,
584 .Component3Control = VFCOMP_STORE_1_FP,
585 #if GFX_VER <= 5
586 .DestinationElementOffset = slot * 4,
587 #endif
588 };
589 slot++;
590
591 for (unsigned i = 0; i < num_varyings; ++i) {
592 ve[slot] = (struct GENX(VERTEX_ELEMENT_STATE)) {
593 .VertexBufferIndex = 1,
594 .Valid = true,
595 .SourceElementFormat = ISL_FORMAT_R32G32B32A32_FLOAT,
596 .SourceElementOffset = 16 + i * 4 * sizeof(float),
597 .Component0Control = VFCOMP_STORE_SRC,
598 .Component1Control = VFCOMP_STORE_SRC,
599 .Component2Control = VFCOMP_STORE_SRC,
600 .Component3Control = VFCOMP_STORE_SRC,
601 #if GFX_VER <= 5
602 .DestinationElementOffset = slot * 4,
603 #endif
604 };
605 slot++;
606 }
607
608 const unsigned num_dwords =
609 1 + GENX(VERTEX_ELEMENT_STATE_length) * num_elements;
610 uint32_t *dw = blorp_emitn(batch, GENX(3DSTATE_VERTEX_ELEMENTS), num_dwords);
611 if (!dw)
612 return;
613
614 for (unsigned i = 0; i < num_elements; i++) {
615 GENX(VERTEX_ELEMENT_STATE_pack)(batch, dw, &ve[i]);
616 dw += GENX(VERTEX_ELEMENT_STATE_length);
617 }
618
619 blorp_emit(batch, GENX(3DSTATE_VF_STATISTICS), vf) {
620 vf.StatisticsEnable = false;
621 }
622
623 #if GFX_VER >= 8
624 /* Overwrite Render Target Array Index (2nd dword) in the VUE header with
625 * primitive instance identifier. This is used for layered clears.
626 */
627 blorp_emit(batch, GENX(3DSTATE_VF_SGVS), sgvs) {
628 sgvs.InstanceIDEnable = true;
629 sgvs.InstanceIDComponentNumber = COMP_1;
630 sgvs.InstanceIDElementOffset = 0;
631 }
632
633 for (unsigned i = 0; i < num_elements; i++) {
634 blorp_emit(batch, GENX(3DSTATE_VF_INSTANCING), vf) {
635 vf.VertexElementIndex = i;
636 vf.InstancingEnable = false;
637 }
638 }
639
640 blorp_emit(batch, GENX(3DSTATE_VF_TOPOLOGY), topo) {
641 topo.PrimitiveTopologyType = _3DPRIM_RECTLIST;
642 }
643 #endif
644 }
645
646 /* 3DSTATE_VIEWPORT_STATE_POINTERS */
647 static uint32_t
blorp_emit_cc_viewport(struct blorp_batch * batch)648 blorp_emit_cc_viewport(struct blorp_batch *batch)
649 {
650 uint32_t cc_vp_offset;
651 blorp_emit_dynamic(batch, GENX(CC_VIEWPORT), vp, 32, &cc_vp_offset) {
652 vp.MinimumDepth = batch->blorp->config.use_unrestricted_depth_range ?
653 -FLT_MAX : 0.0;
654 vp.MaximumDepth = batch->blorp->config.use_unrestricted_depth_range ?
655 FLT_MAX : 1.0;
656 }
657
658 #if GFX_VER >= 7
659 blorp_emit(batch, GENX(3DSTATE_VIEWPORT_STATE_POINTERS_CC), vsp) {
660 vsp.CCViewportPointer = cc_vp_offset;
661 }
662 #elif GFX_VER == 6
663 blorp_emit(batch, GENX(3DSTATE_VIEWPORT_STATE_POINTERS), vsp) {
664 vsp.CCViewportStateChange = true;
665 vsp.PointertoCC_VIEWPORT = cc_vp_offset;
666 }
667 #endif
668
669 return cc_vp_offset;
670 }
671
672 static uint32_t
blorp_emit_sampler_state(struct blorp_batch * batch)673 blorp_emit_sampler_state(struct blorp_batch *batch)
674 {
675 uint32_t offset;
676 blorp_emit_dynamic(batch, GENX(SAMPLER_STATE), sampler, 32, &offset) {
677 sampler.MipModeFilter = MIPFILTER_NONE;
678 sampler.MagModeFilter = MAPFILTER_LINEAR;
679 sampler.MinModeFilter = MAPFILTER_LINEAR;
680 sampler.MinLOD = 0;
681 sampler.MaxLOD = 0;
682 sampler.TCXAddressControlMode = TCM_CLAMP;
683 sampler.TCYAddressControlMode = TCM_CLAMP;
684 sampler.TCZAddressControlMode = TCM_CLAMP;
685 sampler.MaximumAnisotropy = RATIO21;
686 sampler.RAddressMinFilterRoundingEnable = true;
687 sampler.RAddressMagFilterRoundingEnable = true;
688 sampler.VAddressMinFilterRoundingEnable = true;
689 sampler.VAddressMagFilterRoundingEnable = true;
690 sampler.UAddressMinFilterRoundingEnable = true;
691 sampler.UAddressMagFilterRoundingEnable = true;
692 #if GFX_VER > 6
693 sampler.NonnormalizedCoordinateEnable = true;
694 #endif
695 }
696
697 return offset;
698 }
699
700 UNUSED static uint32_t
blorp_emit_sampler_state_ps(struct blorp_batch * batch)701 blorp_emit_sampler_state_ps(struct blorp_batch *batch)
702 {
703 uint32_t offset = blorp_emit_sampler_state(batch);
704
705 #if GFX_VER >= 7
706 blorp_emit(batch, GENX(3DSTATE_SAMPLER_STATE_POINTERS_PS), ssp) {
707 ssp.PointertoPSSamplerState = offset;
708 }
709 #elif GFX_VER == 6
710 blorp_emit(batch, GENX(3DSTATE_SAMPLER_STATE_POINTERS), ssp) {
711 ssp.VSSamplerStateChange = true;
712 ssp.GSSamplerStateChange = true;
713 ssp.PSSamplerStateChange = true;
714 ssp.PointertoPSSamplerState = offset;
715 }
716 #endif
717
718 return offset;
719 }
720
721 /* What follows is the code for setting up a "pipeline" on Sandy Bridge and
722 * later hardware. This file will be included by i965 for gfx4-5 as well, so
723 * this code is guarded by GFX_VER >= 6.
724 */
725 #if GFX_VER >= 6
726
727 static void
blorp_emit_vs_config(struct blorp_batch * batch,const struct blorp_params * params)728 blorp_emit_vs_config(struct blorp_batch *batch,
729 const struct blorp_params *params)
730 {
731 struct elk_vs_prog_data *vs_prog_data = params->vs_prog_data;
732
733 blorp_emit(batch, GENX(3DSTATE_VS), vs) {
734 if (vs_prog_data) {
735 vs.Enable = true;
736
737 vs.KernelStartPointer = params->vs_prog_kernel;
738
739 vs.DispatchGRFStartRegisterForURBData =
740 vs_prog_data->base.base.dispatch_grf_start_reg;
741 vs.VertexURBEntryReadLength =
742 vs_prog_data->base.urb_read_length;
743 vs.VertexURBEntryReadOffset = 0;
744
745 vs.MaximumNumberofThreads =
746 batch->blorp->isl_dev->info->max_vs_threads - 1;
747
748 assert(GFX_VER < 8 ||
749 vs_prog_data->base.dispatch_mode == INTEL_DISPATCH_MODE_SIMD8);
750 #if GFX_VER >= 8 && GFX_VER < 20
751 vs.SIMD8DispatchEnable = true;
752 #endif
753 }
754 }
755 }
756
757 static void
blorp_emit_sf_config(struct blorp_batch * batch,const struct blorp_params * params,UNUSED enum intel_urb_deref_block_size urb_deref_block_size)758 blorp_emit_sf_config(struct blorp_batch *batch,
759 const struct blorp_params *params,
760 UNUSED enum intel_urb_deref_block_size urb_deref_block_size)
761 {
762 const struct elk_wm_prog_data *prog_data = params->wm_prog_data;
763
764 /* 3DSTATE_SF
765 *
766 * Disable ViewportTransformEnable (dw2.1)
767 *
768 * From the SandyBridge PRM, Volume 2, Part 1, Section 1.3, "3D
769 * Primitives Overview":
770 * RECTLIST: Viewport Mapping must be DISABLED (as is typical with the
771 * use of screen- space coordinates).
772 *
773 * A solid rectangle must be rendered, so set FrontFaceFillMode (dw2.4:3)
774 * and BackFaceFillMode (dw2.5:6) to SOLID(0).
775 *
776 * From the Sandy Bridge PRM, Volume 2, Part 1, Section
777 * 6.4.1.1 3DSTATE_SF, Field FrontFaceFillMode:
778 * SOLID: Any triangle or rectangle object found to be front-facing
779 * is rendered as a solid object. This setting is required when
780 * (rendering rectangle (RECTLIST) objects.
781 */
782
783 #if GFX_VER >= 8
784
785 blorp_emit(batch, GENX(3DSTATE_SF), sf) {
786 }
787
788 blorp_emit(batch, GENX(3DSTATE_RASTER), raster) {
789 raster.CullMode = CULLMODE_NONE;
790 }
791
792 blorp_emit(batch, GENX(3DSTATE_SBE), sbe) {
793 sbe.VertexURBEntryReadOffset = 1;
794 if (prog_data) {
795 sbe.NumberofSFOutputAttributes = prog_data->num_varying_inputs;
796 sbe.VertexURBEntryReadLength = elk_blorp_get_urb_length(prog_data);
797 sbe.ConstantInterpolationEnable = prog_data->flat_inputs;
798 } else {
799 sbe.NumberofSFOutputAttributes = 0;
800 sbe.VertexURBEntryReadLength = 1;
801 }
802 sbe.ForceVertexURBEntryReadLength = true;
803 sbe.ForceVertexURBEntryReadOffset = true;
804 }
805
806 #elif GFX_VER >= 7
807
808 blorp_emit(batch, GENX(3DSTATE_SF), sf) {
809 sf.FrontFaceFillMode = FILL_MODE_SOLID;
810 sf.BackFaceFillMode = FILL_MODE_SOLID;
811
812 sf.MultisampleRasterizationMode = params->num_samples > 1 ?
813 MSRASTMODE_ON_PATTERN : MSRASTMODE_OFF_PIXEL;
814
815 #if GFX_VER == 7
816 sf.DepthBufferSurfaceFormat = params->depth_format;
817 #endif
818 }
819
820 blorp_emit(batch, GENX(3DSTATE_SBE), sbe) {
821 sbe.VertexURBEntryReadOffset = 1;
822 if (prog_data) {
823 sbe.NumberofSFOutputAttributes = prog_data->num_varying_inputs;
824 sbe.VertexURBEntryReadLength = elk_blorp_get_urb_length(prog_data);
825 sbe.ConstantInterpolationEnable = prog_data->flat_inputs;
826 } else {
827 sbe.NumberofSFOutputAttributes = 0;
828 sbe.VertexURBEntryReadLength = 1;
829 }
830 }
831
832 #else /* GFX_VER <= 6 */
833
834 blorp_emit(batch, GENX(3DSTATE_SF), sf) {
835 sf.FrontFaceFillMode = FILL_MODE_SOLID;
836 sf.BackFaceFillMode = FILL_MODE_SOLID;
837
838 sf.MultisampleRasterizationMode = params->num_samples > 1 ?
839 MSRASTMODE_ON_PATTERN : MSRASTMODE_OFF_PIXEL;
840
841 sf.VertexURBEntryReadOffset = 1;
842 if (prog_data) {
843 sf.NumberofSFOutputAttributes = prog_data->num_varying_inputs;
844 sf.VertexURBEntryReadLength = elk_blorp_get_urb_length(prog_data);
845 sf.ConstantInterpolationEnable = prog_data->flat_inputs;
846 } else {
847 sf.NumberofSFOutputAttributes = 0;
848 sf.VertexURBEntryReadLength = 1;
849 }
850 }
851
852 #endif /* GFX_VER */
853 }
854
855 static void
blorp_emit_ps_config(struct blorp_batch * batch,const struct blorp_params * params)856 blorp_emit_ps_config(struct blorp_batch *batch,
857 const struct blorp_params *params)
858 {
859 const struct elk_wm_prog_data *prog_data = params->wm_prog_data;
860
861 /* Even when thread dispatch is disabled, max threads (dw5.25:31) must be
862 * nonzero to prevent the GPU from hanging. While the documentation doesn't
863 * mention this explicitly, it notes that the valid range for the field is
864 * [1,39] = [2,40] threads, which excludes zero.
865 *
866 * To be safe (and to minimize extraneous code) we go ahead and fully
867 * configure the WM state whether or not there is a WM program.
868 */
869
870 #if GFX_VER >= 8
871 const struct intel_device_info *devinfo = batch->blorp->compiler->elk->devinfo;
872
873 blorp_emit(batch, GENX(3DSTATE_WM), wm);
874
875 blorp_emit(batch, GENX(3DSTATE_PS), ps) {
876 if (params->src.enabled) {
877 ps.SamplerCount = 1; /* Up to 4 samplers */
878 ps.BindingTableEntryCount = 2;
879 } else {
880 ps.BindingTableEntryCount = 1;
881 }
882
883 /* SAMPLER_STATE prefetching is broken on Gfx11 - Wa_1606682166 */
884 if (GFX_VER == 11)
885 ps.SamplerCount = 0;
886
887 /* 3DSTATE_PS expects the number of threads per PSD, which is always 64
888 * for pre Gfx11 and 128 for gfx11+; On gfx11+ If a programmed value is
889 * k, it implies 2(k+1) threads. It implicitly scales for different GT
890 * levels (which have some # of PSDs).
891 *
892 * In Gfx8 the format is U8-2 whereas in Gfx9+ it is U9-1.
893 */
894 ps.MaximumNumberofThreadsPerPSD =
895 devinfo->max_threads_per_psd - (GFX_VER == 8 ? 2 : 1);
896
897 switch (params->fast_clear_op) {
898 case ISL_AUX_OP_NONE:
899 break;
900 case ISL_AUX_OP_FULL_RESOLVE:
901 ps.RenderTargetResolveEnable = true;
902 break;
903 case ISL_AUX_OP_FAST_CLEAR:
904 ps.RenderTargetFastClearEnable = true;
905 break;
906 default:
907 unreachable("Invalid fast clear op");
908 }
909
910 /* The RENDER_SURFACE_STATE page for TGL says:
911 *
912 * For an 8 bpp surface with NUM_MULTISAMPLES = 1, Surface Width not
913 * multiple of 64 pixels and more than 1 mip level in the view, Fast
914 * Clear is not supported when AUX_CCS_E is set in this field.
915 *
916 * The granularity of a fast-clear or ambiguate operation is likely one
917 * CCS element. For an 8 bpp primary surface, this maps to 32px x 4rows.
918 * Due to the surface layout parameters, if LOD0's width isn't a
919 * multiple of 64px, LOD1 and LOD2+ will share CCS elements. Assert that
920 * these operations aren't occurring on these LODs.
921 *
922 * We don't explicitly check for TGL+ because the restriction is
923 * technically applicable to all hardware. Platforms prior to TGL don't
924 * support CCS on 8 bpp surfaces. So, these unaligned fast clear
925 * operations shouldn't be occurring prior to TGL as well.
926 */
927 if (isl_format_get_layout(params->dst.surf.format)->bpb == 8 &&
928 params->dst.surf.logical_level0_px.width % 64 != 0 &&
929 params->dst.surf.levels >= 3 &&
930 params->dst.view.base_level >= 1) {
931 assert(params->num_samples == 1);
932 assert(!ps.RenderTargetFastClearEnable);
933 }
934
935 if (prog_data) {
936 intel_set_ps_dispatch_state(&ps, devinfo, prog_data,
937 params->num_samples,
938 0 /* msaa_flags */);
939
940 ps.DispatchGRFStartRegisterForConstantSetupData0 =
941 elk_wm_prog_data_dispatch_grf_start_reg(prog_data, ps, 0);
942 ps.DispatchGRFStartRegisterForConstantSetupData1 =
943 elk_wm_prog_data_dispatch_grf_start_reg(prog_data, ps, 1);
944 ps.DispatchGRFStartRegisterForConstantSetupData2 =
945 elk_wm_prog_data_dispatch_grf_start_reg(prog_data, ps, 2);
946
947 ps.KernelStartPointer0 = params->wm_prog_kernel +
948 elk_wm_prog_data_prog_offset(prog_data, ps, 0);
949 ps.KernelStartPointer1 = params->wm_prog_kernel +
950 elk_wm_prog_data_prog_offset(prog_data, ps, 1);
951 ps.KernelStartPointer2 = params->wm_prog_kernel +
952 elk_wm_prog_data_prog_offset(prog_data, ps, 2);
953 }
954 }
955
956 blorp_emit(batch, GENX(3DSTATE_PS_EXTRA), psx) {
957 if (prog_data) {
958 psx.PixelShaderValid = true;
959 psx.AttributeEnable = prog_data->num_varying_inputs > 0;
960 psx.PixelShaderIsPerSample = prog_data->persample_dispatch;
961 psx.PixelShaderComputedDepthMode = prog_data->computed_depth_mode;
962 }
963
964 if (params->src.enabled)
965 psx.PixelShaderKillsPixel = true;
966 }
967
968 #elif GFX_VER >= 7
969 const struct intel_device_info *devinfo = batch->blorp->compiler->elk->devinfo;
970
971 blorp_emit(batch, GENX(3DSTATE_WM), wm) {
972 switch (params->hiz_op) {
973 case ISL_AUX_OP_FAST_CLEAR:
974 wm.DepthBufferClear = true;
975 break;
976 case ISL_AUX_OP_FULL_RESOLVE:
977 wm.DepthBufferResolveEnable = true;
978 break;
979 case ISL_AUX_OP_AMBIGUATE:
980 wm.HierarchicalDepthBufferResolveEnable = true;
981 break;
982 case ISL_AUX_OP_NONE:
983 break;
984 default:
985 unreachable("not reached");
986 }
987
988 if (prog_data) {
989 wm.ThreadDispatchEnable = true;
990 wm.PixelShaderComputedDepthMode = prog_data->computed_depth_mode;
991 }
992
993 if (params->src.enabled)
994 wm.PixelShaderKillsPixel = true;
995
996 if (params->num_samples > 1) {
997 wm.MultisampleRasterizationMode = MSRASTMODE_ON_PATTERN;
998 wm.MultisampleDispatchMode =
999 (prog_data && prog_data->persample_dispatch) ?
1000 MSDISPMODE_PERSAMPLE : MSDISPMODE_PERPIXEL;
1001 } else {
1002 wm.MultisampleRasterizationMode = MSRASTMODE_OFF_PIXEL;
1003 wm.MultisampleDispatchMode = MSDISPMODE_PERSAMPLE;
1004 }
1005 }
1006
1007 blorp_emit(batch, GENX(3DSTATE_PS), ps) {
1008 ps.MaximumNumberofThreads =
1009 batch->blorp->isl_dev->info->max_wm_threads - 1;
1010
1011 #if GFX_VERx10 == 75
1012 ps.SampleMask = 1;
1013 #endif
1014
1015 if (prog_data) {
1016 intel_set_ps_dispatch_state(&ps, devinfo, prog_data,
1017 params->num_samples,
1018 0 /* msaa_flags */);
1019
1020 ps.DispatchGRFStartRegisterForConstantSetupData0 =
1021 elk_wm_prog_data_dispatch_grf_start_reg(prog_data, ps, 0);
1022 ps.DispatchGRFStartRegisterForConstantSetupData1 =
1023 elk_wm_prog_data_dispatch_grf_start_reg(prog_data, ps, 1);
1024 ps.DispatchGRFStartRegisterForConstantSetupData2 =
1025 elk_wm_prog_data_dispatch_grf_start_reg(prog_data, ps, 2);
1026
1027 ps.KernelStartPointer0 = params->wm_prog_kernel +
1028 elk_wm_prog_data_prog_offset(prog_data, ps, 0);
1029 ps.KernelStartPointer1 = params->wm_prog_kernel +
1030 elk_wm_prog_data_prog_offset(prog_data, ps, 1);
1031 ps.KernelStartPointer2 = params->wm_prog_kernel +
1032 elk_wm_prog_data_prog_offset(prog_data, ps, 2);
1033
1034 ps.AttributeEnable = prog_data->num_varying_inputs > 0;
1035 } else {
1036 /* Gfx7 hardware gets angry if we don't enable at least one dispatch
1037 * mode, so just enable 16-pixel dispatch if we don't have a program.
1038 */
1039 ps._16PixelDispatchEnable = true;
1040 }
1041
1042 if (params->src.enabled)
1043 ps.SamplerCount = 1; /* Up to 4 samplers */
1044
1045 switch (params->fast_clear_op) {
1046 case ISL_AUX_OP_NONE:
1047 break;
1048 case ISL_AUX_OP_FULL_RESOLVE:
1049 ps.RenderTargetResolveEnable = true;
1050 break;
1051 case ISL_AUX_OP_FAST_CLEAR:
1052 ps.RenderTargetFastClearEnable = true;
1053 break;
1054 default:
1055 unreachable("Invalid fast clear op");
1056 }
1057 }
1058
1059 #else /* GFX_VER <= 6 */
1060
1061 blorp_emit(batch, GENX(3DSTATE_WM), wm) {
1062 wm.MaximumNumberofThreads =
1063 batch->blorp->isl_dev->info->max_wm_threads - 1;
1064
1065 switch (params->hiz_op) {
1066 case ISL_AUX_OP_FAST_CLEAR:
1067 wm.DepthBufferClear = true;
1068 break;
1069 case ISL_AUX_OP_FULL_RESOLVE:
1070 wm.DepthBufferResolveEnable = true;
1071 break;
1072 case ISL_AUX_OP_AMBIGUATE:
1073 wm.HierarchicalDepthBufferResolveEnable = true;
1074 break;
1075 case ISL_AUX_OP_NONE:
1076 break;
1077 default:
1078 unreachable("not reached");
1079 }
1080
1081 if (prog_data) {
1082 wm.ThreadDispatchEnable = true;
1083
1084 wm._8PixelDispatchEnable = prog_data->dispatch_8;
1085 wm._16PixelDispatchEnable = prog_data->dispatch_16;
1086 wm._32PixelDispatchEnable = prog_data->dispatch_32;
1087
1088 wm.DispatchGRFStartRegisterForConstantSetupData0 =
1089 elk_wm_prog_data_dispatch_grf_start_reg(prog_data, wm, 0);
1090 wm.DispatchGRFStartRegisterForConstantSetupData1 =
1091 elk_wm_prog_data_dispatch_grf_start_reg(prog_data, wm, 1);
1092 wm.DispatchGRFStartRegisterForConstantSetupData2 =
1093 elk_wm_prog_data_dispatch_grf_start_reg(prog_data, wm, 2);
1094
1095 wm.KernelStartPointer0 = params->wm_prog_kernel +
1096 elk_wm_prog_data_prog_offset(prog_data, wm, 0);
1097 wm.KernelStartPointer1 = params->wm_prog_kernel +
1098 elk_wm_prog_data_prog_offset(prog_data, wm, 1);
1099 wm.KernelStartPointer2 = params->wm_prog_kernel +
1100 elk_wm_prog_data_prog_offset(prog_data, wm, 2);
1101
1102 wm.NumberofSFOutputAttributes = prog_data->num_varying_inputs;
1103 }
1104
1105 if (params->src.enabled) {
1106 wm.SamplerCount = 1; /* Up to 4 samplers */
1107 wm.PixelShaderKillsPixel = true; /* TODO: temporarily smash on */
1108 }
1109
1110 if (params->num_samples > 1) {
1111 wm.MultisampleRasterizationMode = MSRASTMODE_ON_PATTERN;
1112 wm.MultisampleDispatchMode =
1113 (prog_data && prog_data->persample_dispatch) ?
1114 MSDISPMODE_PERSAMPLE : MSDISPMODE_PERPIXEL;
1115 } else {
1116 wm.MultisampleRasterizationMode = MSRASTMODE_OFF_PIXEL;
1117 wm.MultisampleDispatchMode = MSDISPMODE_PERSAMPLE;
1118 }
1119 }
1120
1121 #endif /* GFX_VER */
1122 }
1123
1124 static uint32_t
blorp_emit_blend_state(struct blorp_batch * batch,const struct blorp_params * params)1125 blorp_emit_blend_state(struct blorp_batch *batch,
1126 const struct blorp_params *params)
1127 {
1128 struct GENX(BLEND_STATE) blend = { };
1129
1130 uint32_t offset;
1131 int size = GENX(BLEND_STATE_length) * 4;
1132 size += GENX(BLEND_STATE_ENTRY_length) * 4 * params->num_draw_buffers;
1133 uint32_t *state = blorp_alloc_dynamic_state(batch, size, 64, &offset);
1134 uint32_t *pos = state;
1135
1136 GENX(BLEND_STATE_pack)(NULL, pos, &blend);
1137 pos += GENX(BLEND_STATE_length);
1138
1139 for (unsigned i = 0; i < params->num_draw_buffers; ++i) {
1140 struct GENX(BLEND_STATE_ENTRY) entry = {
1141 .PreBlendColorClampEnable = true,
1142 .PostBlendColorClampEnable = true,
1143 .ColorClampRange = COLORCLAMP_RTFORMAT,
1144
1145 .WriteDisableRed = params->color_write_disable & 1,
1146 .WriteDisableGreen = params->color_write_disable & 2,
1147 .WriteDisableBlue = params->color_write_disable & 4,
1148 .WriteDisableAlpha = params->color_write_disable & 8,
1149 };
1150 GENX(BLEND_STATE_ENTRY_pack)(NULL, pos, &entry);
1151 pos += GENX(BLEND_STATE_ENTRY_length);
1152 }
1153
1154 blorp_flush_range(batch, state, size);
1155
1156 #if GFX_VER >= 7
1157 blorp_emit(batch, GENX(3DSTATE_BLEND_STATE_POINTERS), sp) {
1158 sp.BlendStatePointer = offset;
1159 #if GFX_VER >= 8
1160 sp.BlendStatePointerValid = true;
1161 #endif
1162 }
1163 #endif
1164
1165 #if GFX_VER >= 8
1166 blorp_emit(batch, GENX(3DSTATE_PS_BLEND), ps_blend) {
1167 ps_blend.HasWriteableRT = true;
1168 }
1169 #endif
1170
1171 return offset;
1172 }
1173
1174 static uint32_t
blorp_emit_color_calc_state(struct blorp_batch * batch,UNUSED const struct blorp_params * params)1175 blorp_emit_color_calc_state(struct blorp_batch *batch,
1176 UNUSED const struct blorp_params *params)
1177 {
1178 uint32_t offset;
1179 blorp_emit_dynamic(batch, GENX(COLOR_CALC_STATE), cc, 64, &offset) {
1180 #if GFX_VER <= 8
1181 cc.StencilReferenceValue = params->stencil_ref;
1182 #endif
1183 }
1184
1185 #if GFX_VER >= 7
1186 blorp_emit(batch, GENX(3DSTATE_CC_STATE_POINTERS), sp) {
1187 sp.ColorCalcStatePointer = offset;
1188 #if GFX_VER >= 8
1189 sp.ColorCalcStatePointerValid = true;
1190 #endif
1191 }
1192 #endif
1193
1194 return offset;
1195 }
1196
1197 static uint32_t
blorp_emit_depth_stencil_state(struct blorp_batch * batch,const struct blorp_params * params)1198 blorp_emit_depth_stencil_state(struct blorp_batch *batch,
1199 const struct blorp_params *params)
1200 {
1201 #if GFX_VER >= 8
1202 struct GENX(3DSTATE_WM_DEPTH_STENCIL) ds = {
1203 GENX(3DSTATE_WM_DEPTH_STENCIL_header),
1204 };
1205 #else
1206 struct GENX(DEPTH_STENCIL_STATE) ds = { 0 };
1207 #endif
1208
1209 if (params->depth.enabled) {
1210 ds.DepthBufferWriteEnable = true;
1211
1212 switch (params->hiz_op) {
1213 /* See the following sections of the Sandy Bridge PRM, Volume 2, Part1:
1214 * - 7.5.3.1 Depth Buffer Clear
1215 * - 7.5.3.2 Depth Buffer Resolve
1216 * - 7.5.3.3 Hierarchical Depth Buffer Resolve
1217 */
1218 case ISL_AUX_OP_FULL_RESOLVE:
1219 ds.DepthTestEnable = true;
1220 ds.DepthTestFunction = COMPAREFUNCTION_NEVER;
1221 break;
1222
1223 case ISL_AUX_OP_NONE:
1224 case ISL_AUX_OP_FAST_CLEAR:
1225 case ISL_AUX_OP_AMBIGUATE:
1226 ds.DepthTestEnable = false;
1227 break;
1228 case ISL_AUX_OP_PARTIAL_RESOLVE:
1229 unreachable("Invalid HIZ op");
1230 }
1231 }
1232
1233 if (params->stencil.enabled) {
1234 ds.StencilBufferWriteEnable = true;
1235 ds.StencilTestEnable = true;
1236 ds.DoubleSidedStencilEnable = false;
1237
1238 ds.StencilTestFunction = COMPAREFUNCTION_ALWAYS;
1239 ds.StencilPassDepthPassOp = STENCILOP_REPLACE;
1240
1241 ds.StencilWriteMask = params->stencil_mask;
1242 }
1243
1244 #if GFX_VER >= 8
1245 uint32_t offset = 0;
1246 uint32_t *dw = blorp_emit_dwords(batch,
1247 GENX(3DSTATE_WM_DEPTH_STENCIL_length));
1248 if (!dw)
1249 return 0;
1250
1251 GENX(3DSTATE_WM_DEPTH_STENCIL_pack)(NULL, dw, &ds);
1252 #else
1253 uint32_t offset;
1254 void *state = blorp_alloc_dynamic_state(batch,
1255 GENX(DEPTH_STENCIL_STATE_length) * 4,
1256 64, &offset);
1257 GENX(DEPTH_STENCIL_STATE_pack)(NULL, state, &ds);
1258 blorp_flush_range(batch, state, GENX(DEPTH_STENCIL_STATE_length) * 4);
1259 #endif
1260
1261 #if GFX_VER == 7
1262 blorp_emit(batch, GENX(3DSTATE_DEPTH_STENCIL_STATE_POINTERS), sp) {
1263 sp.PointertoDEPTH_STENCIL_STATE = offset;
1264 }
1265 #endif
1266
1267 return offset;
1268 }
1269
1270 static void
blorp_emit_3dstate_multisample(struct blorp_batch * batch,const struct blorp_params * params)1271 blorp_emit_3dstate_multisample(struct blorp_batch *batch,
1272 const struct blorp_params *params)
1273 {
1274 blorp_emit(batch, GENX(3DSTATE_MULTISAMPLE), ms) {
1275 ms.NumberofMultisamples = __builtin_ffs(params->num_samples) - 1;
1276 ms.PixelLocation = CENTER;
1277 #if GFX_VER >= 7 && GFX_VER < 8
1278 switch (params->num_samples) {
1279 case 1:
1280 INTEL_SAMPLE_POS_1X(ms.Sample);
1281 break;
1282 case 2:
1283 INTEL_SAMPLE_POS_2X(ms.Sample);
1284 break;
1285 case 4:
1286 INTEL_SAMPLE_POS_4X(ms.Sample);
1287 break;
1288 case 8:
1289 INTEL_SAMPLE_POS_8X(ms.Sample);
1290 break;
1291 default:
1292 break;
1293 }
1294 #elif GFX_VER < 7
1295 INTEL_SAMPLE_POS_4X(ms.Sample);
1296 #endif
1297 }
1298 }
1299
1300 static void
blorp_emit_pipeline(struct blorp_batch * batch,const struct blorp_params * params)1301 blorp_emit_pipeline(struct blorp_batch *batch,
1302 const struct blorp_params *params)
1303 {
1304 uint32_t blend_state_offset = 0;
1305 uint32_t color_calc_state_offset;
1306 uint32_t depth_stencil_state_offset;
1307
1308 enum intel_urb_deref_block_size urb_deref_block_size;
1309 emit_urb_config(batch, params, &urb_deref_block_size);
1310
1311 if (params->wm_prog_data) {
1312 blend_state_offset = blorp_emit_blend_state(batch, params);
1313 }
1314 color_calc_state_offset = blorp_emit_color_calc_state(batch, params);
1315 depth_stencil_state_offset = blorp_emit_depth_stencil_state(batch, params);
1316
1317 #if GFX_VER == 6
1318 /* 3DSTATE_CC_STATE_POINTERS
1319 *
1320 * The pointer offsets are relative to
1321 * CMD_STATE_BASE_ADDRESS.DynamicStateBaseAddress.
1322 *
1323 * The HiZ op doesn't use BLEND_STATE or COLOR_CALC_STATE.
1324 *
1325 * The dynamic state emit helpers emit their own STATE_POINTERS packets on
1326 * gfx7+. However, on gfx6 and earlier, they're all lumpped together in
1327 * one CC_STATE_POINTERS packet so we have to emit that here.
1328 */
1329 blorp_emit(batch, GENX(3DSTATE_CC_STATE_POINTERS), cc) {
1330 cc.BLEND_STATEChange = params->wm_prog_data ? true : false;
1331 cc.ColorCalcStatePointerValid = true;
1332 cc.DEPTH_STENCIL_STATEChange = true;
1333 cc.PointertoBLEND_STATE = blend_state_offset;
1334 cc.ColorCalcStatePointer = color_calc_state_offset;
1335 cc.PointertoDEPTH_STENCIL_STATE = depth_stencil_state_offset;
1336 }
1337 #else
1338 (void)blend_state_offset;
1339 (void)color_calc_state_offset;
1340 (void)depth_stencil_state_offset;
1341 #endif
1342
1343 UNUSED uint32_t mocs = isl_mocs(batch->blorp->isl_dev, 0, false);
1344
1345 #if GFX_VER == 7
1346 #define CONSTANT_MOCS xs.ConstantBody.MOCS = mocs
1347 #else
1348 #define CONSTANT_MOCS
1349 #endif
1350 blorp_emit(batch, GENX(3DSTATE_CONSTANT_VS), xs) { CONSTANT_MOCS; }
1351 #if GFX_VER >= 7
1352 blorp_emit(batch, GENX(3DSTATE_CONSTANT_HS), xs) { CONSTANT_MOCS; }
1353 blorp_emit(batch, GENX(3DSTATE_CONSTANT_DS), xs) { CONSTANT_MOCS; }
1354 #endif
1355 blorp_emit(batch, GENX(3DSTATE_CONSTANT_GS), xs) { CONSTANT_MOCS; }
1356 blorp_emit(batch, GENX(3DSTATE_CONSTANT_PS), xs) { CONSTANT_MOCS; }
1357 #undef CONSTANT_MOCS
1358
1359 if (params->src.enabled)
1360 blorp_emit_sampler_state_ps(batch);
1361
1362 blorp_emit_3dstate_multisample(batch, params);
1363
1364 blorp_emit(batch, GENX(3DSTATE_SAMPLE_MASK), mask) {
1365 mask.SampleMask = (1 << params->num_samples) - 1;
1366 }
1367
1368 /* From the BSpec, 3D Pipeline > Geometry > Vertex Shader > State,
1369 * 3DSTATE_VS, Dword 5.0 "VS Function Enable":
1370 *
1371 * [DevSNB] A pipeline flush must be programmed prior to a
1372 * 3DSTATE_VS command that causes the VS Function Enable to
1373 * toggle. Pipeline flush can be executed by sending a PIPE_CONTROL
1374 * command with CS stall bit set and a post sync operation.
1375 *
1376 * We've already done one at the start of the BLORP operation.
1377 */
1378 blorp_emit_vs_config(batch, params);
1379 #if GFX_VER >= 7
1380 blorp_emit(batch, GENX(3DSTATE_HS), hs);
1381 blorp_emit(batch, GENX(3DSTATE_TE), te);
1382 blorp_emit(batch, GENX(3DSTATE_DS), DS);
1383 blorp_emit(batch, GENX(3DSTATE_STREAMOUT), so);
1384 #endif
1385 blorp_emit(batch, GENX(3DSTATE_GS), gs);
1386
1387 blorp_emit(batch, GENX(3DSTATE_CLIP), clip) {
1388 clip.PerspectiveDivideDisable = true;
1389 }
1390
1391 blorp_emit_sf_config(batch, params, urb_deref_block_size);
1392 blorp_emit_ps_config(batch, params);
1393
1394 blorp_emit_cc_viewport(batch);
1395 }
1396
1397 /******** This is the end of the pipeline setup code ********/
1398
1399 #endif /* GFX_VER >= 6 */
1400
1401 #if GFX_VER >= 7
1402 static void
blorp_emit_memcpy(struct blorp_batch * batch,struct blorp_address dst,struct blorp_address src,uint32_t size)1403 blorp_emit_memcpy(struct blorp_batch *batch,
1404 struct blorp_address dst,
1405 struct blorp_address src,
1406 uint32_t size)
1407 {
1408 assert(size % 4 == 0);
1409
1410 for (unsigned dw = 0; dw < size; dw += 4) {
1411 #if GFX_VER >= 8
1412 blorp_emit(batch, GENX(MI_COPY_MEM_MEM), cp) {
1413 cp.DestinationMemoryAddress = dst;
1414 cp.SourceMemoryAddress = src;
1415 }
1416 #else
1417 /* IVB does not have a general purpose register for command streamer
1418 * commands. Therefore, we use an alternate temporary register.
1419 */
1420 #define BLORP_TEMP_REG 0x2440 /* GFX7_3DPRIM_BASE_VERTEX */
1421 blorp_emit(batch, GENX(MI_LOAD_REGISTER_MEM), load) {
1422 load.RegisterAddress = BLORP_TEMP_REG;
1423 load.MemoryAddress = src;
1424 }
1425 blorp_emit(batch, GENX(MI_STORE_REGISTER_MEM), store) {
1426 store.RegisterAddress = BLORP_TEMP_REG;
1427 store.MemoryAddress = dst;
1428 }
1429 #undef BLORP_TEMP_REG
1430 #endif
1431 dst.offset += 4;
1432 src.offset += 4;
1433 }
1434 }
1435 #endif
1436
1437 static void
blorp_emit_surface_state(struct blorp_batch * batch,const struct blorp_surface_info * surface,UNUSED enum isl_aux_op aux_op,void * state,uint32_t state_offset,uint8_t color_write_disable,bool is_render_target)1438 blorp_emit_surface_state(struct blorp_batch *batch,
1439 const struct blorp_surface_info *surface,
1440 UNUSED enum isl_aux_op aux_op,
1441 void *state, uint32_t state_offset,
1442 uint8_t color_write_disable,
1443 bool is_render_target)
1444 {
1445 const struct isl_device *isl_dev = batch->blorp->isl_dev;
1446 struct isl_surf surf = surface->surf;
1447
1448 if (surf.dim == ISL_SURF_DIM_1D &&
1449 surf.dim_layout == ISL_DIM_LAYOUT_GFX4_2D) {
1450 assert(surf.logical_level0_px.height == 1);
1451 surf.dim = ISL_SURF_DIM_2D;
1452 }
1453
1454 if (isl_aux_usage_has_hiz(surface->aux_usage)) {
1455 /* BLORP doesn't render with depth so we can't use HiZ */
1456 assert(!is_render_target);
1457 /* We can't reinterpret HiZ */
1458 assert(surface->surf.format == surface->view.format);
1459 }
1460
1461 enum isl_aux_usage aux_usage = surface->aux_usage;
1462
1463 /* On gfx12, implicit CCS has no aux buffer */
1464 bool use_aux_address = (aux_usage != ISL_AUX_USAGE_NONE) &&
1465 (surface->aux_addr.buffer != NULL);
1466
1467 isl_channel_mask_t write_disable_mask = 0;
1468 if (is_render_target && GFX_VER <= 5) {
1469 if (color_write_disable & BITFIELD_BIT(0))
1470 write_disable_mask |= ISL_CHANNEL_RED_BIT;
1471 if (color_write_disable & BITFIELD_BIT(1))
1472 write_disable_mask |= ISL_CHANNEL_GREEN_BIT;
1473 if (color_write_disable & BITFIELD_BIT(2))
1474 write_disable_mask |= ISL_CHANNEL_BLUE_BIT;
1475 if (color_write_disable & BITFIELD_BIT(3))
1476 write_disable_mask |= ISL_CHANNEL_ALPHA_BIT;
1477 }
1478
1479 isl_surf_fill_state(batch->blorp->isl_dev, state,
1480 .surf = &surf, .view = &surface->view,
1481 .aux_surf = &surface->aux_surf, .aux_usage = aux_usage,
1482 .address =
1483 blorp_get_surface_address(batch, surface->addr),
1484 .aux_address = !use_aux_address ? 0 :
1485 blorp_get_surface_address(batch, surface->aux_addr),
1486 .mocs = surface->addr.mocs,
1487 .clear_color = surface->clear_color,
1488 .write_disables = write_disable_mask);
1489
1490 blorp_surface_reloc(batch, state_offset + isl_dev->ss.addr_offset,
1491 surface->addr, 0);
1492
1493 if (use_aux_address) {
1494 /* On gfx7 and prior, the bottom 12 bits of the MCS base address are
1495 * used to store other information. This should be ok, however, because
1496 * surface buffer addresses are always 4K page alinged.
1497 */
1498 assert((surface->aux_addr.offset & 0xfff) == 0);
1499 uint32_t *aux_addr = state + isl_dev->ss.aux_addr_offset;
1500 blorp_surface_reloc(batch, state_offset + isl_dev->ss.aux_addr_offset,
1501 surface->aux_addr, *aux_addr);
1502 }
1503
1504 if (aux_usage != ISL_AUX_USAGE_NONE && surface->clear_color_addr.buffer) {
1505 #if GFX_VER >= 7
1506 /* Fast clears just whack the AUX surface and don't actually use the
1507 * clear color for anything. We can avoid the MI memcpy on that case.
1508 */
1509 if (aux_op != ISL_AUX_OP_FAST_CLEAR) {
1510 struct blorp_address dst_addr = blorp_get_surface_base_address(batch);
1511 dst_addr.offset += state_offset + isl_dev->ss.clear_value_offset;
1512 blorp_emit_memcpy(batch, dst_addr, surface->clear_color_addr,
1513 isl_dev->ss.clear_value_size);
1514 }
1515 #else
1516 unreachable("Fast clears are only supported on gfx7+");
1517 #endif
1518 }
1519
1520 blorp_flush_range(batch, state, GENX(RENDER_SURFACE_STATE_length) * 4);
1521 }
1522
1523 static void
blorp_emit_null_surface_state(struct blorp_batch * batch,const struct blorp_surface_info * surface,uint32_t * state)1524 blorp_emit_null_surface_state(struct blorp_batch *batch,
1525 const struct blorp_surface_info *surface,
1526 uint32_t *state)
1527 {
1528 struct GENX(RENDER_SURFACE_STATE) ss = {
1529 .SurfaceType = SURFTYPE_NULL,
1530 .SurfaceFormat = ISL_FORMAT_R8G8B8A8_UNORM,
1531 .Width = surface->surf.logical_level0_px.width - 1,
1532 .Height = surface->surf.logical_level0_px.height - 1,
1533 .MIPCountLOD = surface->view.base_level,
1534 .MinimumArrayElement = surface->view.base_array_layer,
1535 .Depth = surface->view.array_len - 1,
1536 .RenderTargetViewExtent = surface->view.array_len - 1,
1537 #if GFX_VER >= 6
1538 .NumberofMultisamples = ffs(surface->surf.samples) - 1,
1539 .MOCS = isl_mocs(batch->blorp->isl_dev, 0, false),
1540 #endif
1541
1542 #if GFX_VER >= 7
1543 .SurfaceArray = surface->surf.dim != ISL_SURF_DIM_3D,
1544 #endif
1545
1546 #if GFX_VER >= 8
1547 .TileMode = YMAJOR,
1548 #else
1549 .TiledSurface = true,
1550 #endif
1551 };
1552
1553 GENX(RENDER_SURFACE_STATE_pack)(NULL, state, &ss);
1554
1555 blorp_flush_range(batch, state, GENX(RENDER_SURFACE_STATE_length) * 4);
1556 }
1557
1558 static uint32_t
blorp_setup_binding_table(struct blorp_batch * batch,const struct blorp_params * params)1559 blorp_setup_binding_table(struct blorp_batch *batch,
1560 const struct blorp_params *params)
1561 {
1562 const struct isl_device *isl_dev = batch->blorp->isl_dev;
1563 uint32_t surface_offsets[2], bind_offset = 0;
1564 void *surface_maps[2];
1565
1566 if (params->use_pre_baked_binding_table) {
1567 bind_offset = params->pre_baked_binding_table_offset;
1568 } else {
1569 unsigned num_surfaces = 1 + params->src.enabled;
1570 if (!blorp_alloc_binding_table(batch, num_surfaces,
1571 isl_dev->ss.size, isl_dev->ss.align,
1572 &bind_offset, surface_offsets, surface_maps))
1573 return 0;
1574
1575 if (params->dst.enabled) {
1576 blorp_emit_surface_state(batch, ¶ms->dst,
1577 params->fast_clear_op,
1578 surface_maps[BLORP_RENDERBUFFER_BT_INDEX],
1579 surface_offsets[BLORP_RENDERBUFFER_BT_INDEX],
1580 params->color_write_disable, true);
1581 } else {
1582 assert(params->depth.enabled || params->stencil.enabled);
1583 const struct blorp_surface_info *surface =
1584 params->depth.enabled ? ¶ms->depth : ¶ms->stencil;
1585 blorp_emit_null_surface_state(batch, surface,
1586 surface_maps[BLORP_RENDERBUFFER_BT_INDEX]);
1587 }
1588
1589 if (params->src.enabled) {
1590 blorp_emit_surface_state(batch, ¶ms->src,
1591 params->fast_clear_op,
1592 surface_maps[BLORP_TEXTURE_BT_INDEX],
1593 surface_offsets[BLORP_TEXTURE_BT_INDEX],
1594 0, false);
1595 }
1596 }
1597
1598 return bind_offset;
1599 }
1600
1601 static void
blorp_emit_btp(struct blorp_batch * batch,uint32_t bind_offset)1602 blorp_emit_btp(struct blorp_batch *batch, uint32_t bind_offset)
1603 {
1604 #if GFX_VER >= 7
1605 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS_VS), bt);
1606 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS_HS), bt);
1607 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS_DS), bt);
1608 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS_GS), bt);
1609
1610 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS_PS), bt) {
1611 bt.PointertoPSBindingTable =
1612 blorp_binding_table_offset_to_pointer(batch, bind_offset);
1613 }
1614 #elif GFX_VER >= 6
1615 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS), bt) {
1616 bt.PSBindingTableChange = true;
1617 bt.PointertoPSBindingTable =
1618 blorp_binding_table_offset_to_pointer(batch, bind_offset);
1619 }
1620 #else
1621 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS), bt) {
1622 bt.PointertoPSBindingTable =
1623 blorp_binding_table_offset_to_pointer(batch, bind_offset);
1624 }
1625 #endif
1626 }
1627
1628 static void
blorp_emit_depth_stencil_config(struct blorp_batch * batch,const struct blorp_params * params)1629 blorp_emit_depth_stencil_config(struct blorp_batch *batch,
1630 const struct blorp_params *params)
1631 {
1632 const struct isl_device *isl_dev = batch->blorp->isl_dev;
1633
1634 uint32_t *dw = blorp_emit_dwords(batch, isl_dev->ds.size / 4);
1635 if (dw == NULL)
1636 return;
1637
1638 struct isl_depth_stencil_hiz_emit_info info = { };
1639
1640 if (params->depth.enabled) {
1641 info.view = ¶ms->depth.view;
1642 info.mocs = params->depth.addr.mocs;
1643 } else if (params->stencil.enabled) {
1644 info.view = ¶ms->stencil.view;
1645 info.mocs = params->stencil.addr.mocs;
1646 } else {
1647 info.mocs = isl_mocs(isl_dev, 0, false);
1648 }
1649
1650 if (params->depth.enabled) {
1651 info.depth_surf = ¶ms->depth.surf;
1652
1653 info.depth_address =
1654 blorp_emit_reloc(batch, dw + isl_dev->ds.depth_offset / 4,
1655 params->depth.addr, 0);
1656
1657 info.hiz_usage = params->depth.aux_usage;
1658 if (isl_aux_usage_has_hiz(info.hiz_usage)) {
1659 info.hiz_surf = ¶ms->depth.aux_surf;
1660
1661 struct blorp_address hiz_address = params->depth.aux_addr;
1662 #if GFX_VER == 6
1663 /* Sandy bridge hardware does not technically support mipmapped HiZ.
1664 * However, we have a special layout that allows us to make it work
1665 * anyway by manually offsetting to the specified miplevel.
1666 */
1667 assert(info.hiz_surf->dim_layout == ISL_DIM_LAYOUT_GFX6_STENCIL_HIZ);
1668 uint64_t offset_B;
1669 isl_surf_get_image_offset_B_tile_sa(info.hiz_surf,
1670 info.view->base_level, 0, 0,
1671 &offset_B, NULL, NULL);
1672 hiz_address.offset += offset_B;
1673 #endif
1674
1675 info.hiz_address =
1676 blorp_emit_reloc(batch, dw + isl_dev->ds.hiz_offset / 4,
1677 hiz_address, 0);
1678
1679 info.depth_clear_value = params->depth.clear_color.f32[0];
1680 }
1681 }
1682
1683 if (params->stencil.enabled) {
1684 info.stencil_surf = ¶ms->stencil.surf;
1685
1686 info.stencil_aux_usage = params->stencil.aux_usage;
1687 struct blorp_address stencil_address = params->stencil.addr;
1688 #if GFX_VER == 6
1689 /* Sandy bridge hardware does not technically support mipmapped stencil.
1690 * However, we have a special layout that allows us to make it work
1691 * anyway by manually offsetting to the specified miplevel.
1692 */
1693 assert(info.stencil_surf->dim_layout == ISL_DIM_LAYOUT_GFX6_STENCIL_HIZ);
1694 uint64_t offset_B;
1695 isl_surf_get_image_offset_B_tile_sa(info.stencil_surf,
1696 info.view->base_level, 0, 0,
1697 &offset_B, NULL, NULL);
1698 stencil_address.offset += offset_B;
1699 #endif
1700
1701 info.stencil_address =
1702 blorp_emit_reloc(batch, dw + isl_dev->ds.stencil_offset / 4,
1703 stencil_address, 0);
1704 }
1705
1706 isl_emit_depth_stencil_hiz_s(isl_dev, dw, &info);
1707 }
1708
1709 #if GFX_VER >= 8
1710 /* Emits the Optimized HiZ sequence specified in the BDW+ PRMs. The
1711 * depth/stencil buffer extents are ignored to handle APIs which perform
1712 * clearing operations without such information.
1713 * */
1714 static void
blorp_emit_gfx8_hiz_op(struct blorp_batch * batch,const struct blorp_params * params)1715 blorp_emit_gfx8_hiz_op(struct blorp_batch *batch,
1716 const struct blorp_params *params)
1717 {
1718 /* We should be performing an operation on a depth or stencil buffer.
1719 */
1720 assert(params->depth.enabled || params->stencil.enabled);
1721
1722 blorp_measure_start(batch, params);
1723
1724 /* The stencil buffer should only be enabled if a fast clear operation is
1725 * requested.
1726 */
1727 if (params->stencil.enabled)
1728 assert(params->hiz_op == ISL_AUX_OP_FAST_CLEAR);
1729
1730 /* From the BDW PRM Volume 2, 3DSTATE_WM_HZ_OP:
1731 *
1732 * 3DSTATE_MULTISAMPLE packet must be used prior to this packet to change
1733 * the Number of Multisamples. This packet must not be used to change
1734 * Number of Multisamples in a rendering sequence.
1735 *
1736 * Since HIZ may be the first thing in a batch buffer, play safe and always
1737 * emit 3DSTATE_MULTISAMPLE.
1738 */
1739 blorp_emit_3dstate_multisample(batch, params);
1740
1741 /* From the BDW PRM Volume 7, Depth Buffer Clear:
1742 *
1743 * The clear value must be between the min and max depth values
1744 * (inclusive) defined in the CC_VIEWPORT. If the depth buffer format is
1745 * D32_FLOAT, then +/-DENORM values are also allowed.
1746 *
1747 * Set the bounds to match our hardware limits, [0.0, 1.0].
1748 */
1749 if (params->depth.enabled && params->hiz_op == ISL_AUX_OP_FAST_CLEAR) {
1750 assert(params->depth.clear_color.f32[0] >= 0.0f);
1751 assert(params->depth.clear_color.f32[0] <= 1.0f);
1752 blorp_emit_cc_viewport(batch);
1753 }
1754
1755 /* According to the SKL PRM formula for WM_INT::ThreadDispatchEnable, the
1756 * 3DSTATE_WM::ForceThreadDispatchEnable field can force WM thread dispatch
1757 * even when WM_HZ_OP is active. However, WM thread dispatch is normally
1758 * disabled for HiZ ops and it appears that force-enabling it can lead to
1759 * GPU hangs on at least Skylake. Since we don't know the current state of
1760 * the 3DSTATE_WM packet, just emit a dummy one prior to 3DSTATE_WM_HZ_OP.
1761 */
1762 blorp_emit(batch, GENX(3DSTATE_WM), wm);
1763
1764 /* If we can't alter the depth stencil config and multiple layers are
1765 * involved, the HiZ op will fail. This is because the op requires that a
1766 * new config is emitted for each additional layer.
1767 */
1768 if (batch->flags & BLORP_BATCH_NO_EMIT_DEPTH_STENCIL) {
1769 assert(params->num_layers <= 1);
1770 } else {
1771 blorp_emit_depth_stencil_config(batch, params);
1772 }
1773
1774 /* TODO - If we ever start using 3DSTATE_WM_HZ_OP::StencilBufferResolveEnable
1775 * we need to implement required steps, flushes documented in Wa_1605967699.
1776 */
1777 blorp_emit(batch, GENX(3DSTATE_WM_HZ_OP), hzp) {
1778 switch (params->hiz_op) {
1779 case ISL_AUX_OP_FAST_CLEAR:
1780 hzp.StencilBufferClearEnable = params->stencil.enabled;
1781 hzp.DepthBufferClearEnable = params->depth.enabled;
1782 hzp.StencilClearValue = params->stencil_ref;
1783 hzp.FullSurfaceDepthandStencilClear = params->full_surface_hiz_op;
1784 break;
1785 case ISL_AUX_OP_FULL_RESOLVE:
1786 assert(params->full_surface_hiz_op);
1787 hzp.DepthBufferResolveEnable = true;
1788 break;
1789 case ISL_AUX_OP_AMBIGUATE:
1790 assert(params->full_surface_hiz_op);
1791 hzp.HierarchicalDepthBufferResolveEnable = true;
1792 break;
1793 case ISL_AUX_OP_PARTIAL_RESOLVE:
1794 case ISL_AUX_OP_NONE:
1795 unreachable("Invalid HIZ op");
1796 }
1797
1798 hzp.NumberofMultisamples = ffs(params->num_samples) - 1;
1799 hzp.SampleMask = 0xFFFF;
1800
1801 /* Due to a hardware issue, this bit MBZ */
1802 assert(hzp.ScissorRectangleEnable == false);
1803
1804 /* Contrary to the HW docs both fields are inclusive */
1805 hzp.ClearRectangleXMin = params->x0;
1806 hzp.ClearRectangleYMin = params->y0;
1807
1808 /* Contrary to the HW docs both fields are exclusive */
1809 hzp.ClearRectangleXMax = params->x1;
1810 hzp.ClearRectangleYMax = params->y1;
1811 }
1812
1813 /* PIPE_CONTROL w/ all bits clear except for “Post-Sync Operation” must set
1814 * to “Write Immediate Data” enabled.
1815 */
1816 blorp_emit(batch, GENX(PIPE_CONTROL), pc) {
1817 pc.PostSyncOperation = WriteImmediateData;
1818 pc.Address = blorp_get_workaround_address(batch);
1819 }
1820
1821 blorp_emit(batch, GENX(3DSTATE_WM_HZ_OP), hzp);
1822
1823 blorp_measure_end(batch, params);
1824 }
1825 #endif
1826
1827 static void
blorp_update_clear_color(UNUSED struct blorp_batch * batch,const struct blorp_surface_info * info)1828 blorp_update_clear_color(UNUSED struct blorp_batch *batch,
1829 const struct blorp_surface_info *info)
1830 {
1831 assert(info->clear_color_addr.buffer != NULL);
1832 #if GFX_VER >= 7
1833 blorp_emit(batch, GENX(MI_STORE_DATA_IMM), sdi) {
1834 sdi.Address = info->clear_color_addr;
1835 sdi.ImmediateData = ISL_CHANNEL_SELECT_RED << 25 |
1836 ISL_CHANNEL_SELECT_GREEN << 22 |
1837 ISL_CHANNEL_SELECT_BLUE << 19 |
1838 ISL_CHANNEL_SELECT_ALPHA << 16;
1839 if (isl_format_has_int_channel(info->view.format)) {
1840 for (unsigned i = 0; i < 4; i++) {
1841 assert(info->clear_color.u32[i] == 0 ||
1842 info->clear_color.u32[i] == 1);
1843 }
1844 sdi.ImmediateData |= (info->clear_color.u32[0] != 0) << 31;
1845 sdi.ImmediateData |= (info->clear_color.u32[1] != 0) << 30;
1846 sdi.ImmediateData |= (info->clear_color.u32[2] != 0) << 29;
1847 sdi.ImmediateData |= (info->clear_color.u32[3] != 0) << 28;
1848 } else {
1849 for (unsigned i = 0; i < 4; i++) {
1850 assert(info->clear_color.f32[i] == 0.0f ||
1851 info->clear_color.f32[i] == 1.0f);
1852 }
1853 sdi.ImmediateData |= (info->clear_color.f32[0] != 0.0f) << 31;
1854 sdi.ImmediateData |= (info->clear_color.f32[1] != 0.0f) << 30;
1855 sdi.ImmediateData |= (info->clear_color.f32[2] != 0.0f) << 29;
1856 sdi.ImmediateData |= (info->clear_color.f32[3] != 0.0f) << 28;
1857 }
1858 }
1859 #endif
1860 }
1861
1862 static bool
blorp_uses_bti_rt_writes(const struct blorp_batch * batch,const struct blorp_params * params)1863 blorp_uses_bti_rt_writes(const struct blorp_batch *batch, const struct blorp_params *params)
1864 {
1865 if (batch->flags & (BLORP_BATCH_USE_BLITTER | BLORP_BATCH_USE_COMPUTE))
1866 return false;
1867
1868 /* HIZ clears use WM_HZ ops rather than a clear shader using RT writes. */
1869 return params->hiz_op == ISL_AUX_OP_NONE;
1870 }
1871
1872 static void
blorp_exec_3d(struct blorp_batch * batch,const struct blorp_params * params)1873 blorp_exec_3d(struct blorp_batch *batch, const struct blorp_params *params)
1874 {
1875 if (!(batch->flags & BLORP_BATCH_NO_UPDATE_CLEAR_COLOR)) {
1876 if (params->fast_clear_op == ISL_AUX_OP_FAST_CLEAR &&
1877 params->dst.clear_color_addr.buffer != NULL) {
1878 blorp_update_clear_color(batch, ¶ms->dst);
1879 }
1880
1881 if (params->hiz_op == ISL_AUX_OP_FAST_CLEAR &&
1882 params->depth.clear_color_addr.buffer != NULL) {
1883 blorp_update_clear_color(batch, ¶ms->depth);
1884 }
1885 }
1886
1887 #if GFX_VER >= 8
1888 if (params->hiz_op != ISL_AUX_OP_NONE) {
1889 blorp_emit_gfx8_hiz_op(batch, params);
1890 return;
1891 }
1892 #endif
1893
1894 blorp_emit_vertex_buffers(batch, params);
1895 blorp_emit_vertex_elements(batch, params);
1896
1897 blorp_emit_pipeline(batch, params);
1898
1899 blorp_emit_btp(batch, blorp_setup_binding_table(batch, params));
1900
1901 if (!(batch->flags & BLORP_BATCH_NO_EMIT_DEPTH_STENCIL))
1902 blorp_emit_depth_stencil_config(batch, params);
1903
1904 const UNUSED bool use_tbimr = false;
1905 blorp_emit_pre_draw(batch, params);
1906 blorp_emit(batch, GENX(3DPRIMITIVE), prim) {
1907 prim.VertexAccessType = SEQUENTIAL;
1908 prim.PrimitiveTopologyType = _3DPRIM_RECTLIST;
1909 #if GFX_VER >= 7
1910 prim.PredicateEnable = batch->flags & BLORP_BATCH_PREDICATE_ENABLE;
1911 #endif
1912 prim.VertexCountPerInstance = 3;
1913 prim.InstanceCount = params->num_layers;
1914 }
1915 blorp_emit_post_draw(batch, params);
1916 }
1917
1918 #if GFX_VER >= 7
1919
1920 static void
blorp_get_compute_push_const(struct blorp_batch * batch,const struct blorp_params * params,uint32_t threads,uint32_t * state_offset,unsigned * state_size)1921 blorp_get_compute_push_const(struct blorp_batch *batch,
1922 const struct blorp_params *params,
1923 uint32_t threads,
1924 uint32_t *state_offset,
1925 unsigned *state_size)
1926 {
1927 const struct elk_cs_prog_data *cs_prog_data = params->cs_prog_data;
1928 const unsigned push_const_size =
1929 ALIGN(elk_cs_push_const_total_size(cs_prog_data, threads), 64);
1930 assert(cs_prog_data->push.cross_thread.size +
1931 cs_prog_data->push.per_thread.size == sizeof(params->wm_inputs));
1932
1933 if (push_const_size == 0) {
1934 *state_offset = 0;
1935 *state_size = 0;
1936 return;
1937 }
1938
1939 uint32_t push_const_offset;
1940 uint32_t *push_const =
1941 blorp_alloc_dynamic_state(batch, push_const_size, 64,
1942 &push_const_offset);
1943 memset(push_const, 0x0, push_const_size);
1944
1945 void *dst = push_const;
1946 const void *src = (char *)¶ms->wm_inputs;
1947
1948 if (cs_prog_data->push.cross_thread.size > 0) {
1949 memcpy(dst, src, cs_prog_data->push.cross_thread.size);
1950 dst += cs_prog_data->push.cross_thread.size;
1951 src += cs_prog_data->push.cross_thread.size;
1952 }
1953
1954 if (cs_prog_data->push.per_thread.size > 0) {
1955 for (unsigned t = 0; t < threads; t++) {
1956 memcpy(dst, src, (cs_prog_data->push.per_thread.dwords - 1) * 4);
1957
1958 uint32_t *subgroup_id = dst + cs_prog_data->push.per_thread.size - 4;
1959 *subgroup_id = t;
1960
1961 dst += cs_prog_data->push.per_thread.size;
1962 }
1963 }
1964
1965 *state_offset = push_const_offset;
1966 *state_size = push_const_size;
1967 }
1968
1969 #endif /* GFX_VER >= 7 */
1970
1971 static void
blorp_exec_compute(struct blorp_batch * batch,const struct blorp_params * params)1972 blorp_exec_compute(struct blorp_batch *batch, const struct blorp_params *params)
1973 {
1974 assert(!(batch->flags & BLORP_BATCH_NO_UPDATE_CLEAR_COLOR));
1975 assert(!(batch->flags & BLORP_BATCH_PREDICATE_ENABLE));
1976 assert(params->hiz_op == ISL_AUX_OP_NONE);
1977
1978 blorp_measure_start(batch, params);
1979
1980 #if GFX_VER >= 7
1981 const struct intel_device_info *devinfo = batch->blorp->compiler->elk->devinfo;
1982 const struct elk_cs_prog_data *cs_prog_data = params->cs_prog_data;
1983 const struct elk_stage_prog_data *prog_data = &cs_prog_data->base;
1984 const struct intel_cs_dispatch_info dispatch =
1985 elk_cs_get_dispatch_info(devinfo, cs_prog_data, NULL);
1986
1987 uint32_t group_x0 = params->x0 / cs_prog_data->local_size[0];
1988 uint32_t group_y0 = params->y0 / cs_prog_data->local_size[1];
1989 uint32_t group_z0 = params->dst.z_offset;
1990 uint32_t group_x1 = DIV_ROUND_UP(params->x1, cs_prog_data->local_size[0]);
1991 uint32_t group_y1 = DIV_ROUND_UP(params->y1, cs_prog_data->local_size[1]);
1992 assert(params->num_layers >= 1);
1993 uint32_t group_z1 = params->dst.z_offset + params->num_layers;
1994 assert(cs_prog_data->local_size[2] == 1);
1995
1996 #endif /* GFX_VER >= 7 */
1997
1998 #if GFX_VER >= 7
1999
2000 /* The MEDIA_VFE_STATE documentation for Gfx8+ says:
2001 *
2002 * "A stalling PIPE_CONTROL is required before MEDIA_VFE_STATE unless
2003 * the only bits that are changed are scoreboard related: Scoreboard
2004 * Enable, Scoreboard Type, Scoreboard Mask, Scoreboard * Delta. For
2005 * these scoreboard related states, a MEDIA_STATE_FLUSH is sufficient."
2006 *
2007 * Earlier generations say "MI_FLUSH" instead of "stalling PIPE_CONTROL",
2008 * but MI_FLUSH isn't really a thing, so we assume they meant PIPE_CONTROL.
2009 */
2010 blorp_emit(batch, GENX(PIPE_CONTROL), pc) {
2011 pc.CommandStreamerStallEnable = true;
2012 pc.StallAtPixelScoreboard = true;
2013 }
2014
2015 blorp_emit(batch, GENX(MEDIA_VFE_STATE), vfe) {
2016 assert(prog_data->total_scratch == 0);
2017 vfe.MaximumNumberofThreads =
2018 devinfo->max_cs_threads * devinfo->subslice_total - 1;
2019 vfe.NumberofURBEntries = GFX_VER >= 8 ? 2 : 0;
2020 vfe.ResetGatewayTimer =
2021 Resettingrelativetimerandlatchingtheglobaltimestamp;
2022 vfe.BypassGatewayControl = BypassingOpenGatewayCloseGatewayprotocol;
2023 #if GFX_VER == 7
2024 vfe.GPGPUMode = true;
2025 #endif
2026 vfe.URBEntryAllocationSize = GFX_VER >= 8 ? 2 : 0;
2027
2028 const uint32_t vfe_curbe_allocation =
2029 ALIGN(cs_prog_data->push.per_thread.regs * dispatch.threads +
2030 cs_prog_data->push.cross_thread.regs, 2);
2031 vfe.CURBEAllocationSize = vfe_curbe_allocation;
2032 }
2033
2034 uint32_t push_const_offset;
2035 unsigned push_const_size;
2036 blorp_get_compute_push_const(batch, params, dispatch.threads,
2037 &push_const_offset, &push_const_size);
2038
2039 blorp_emit(batch, GENX(MEDIA_CURBE_LOAD), curbe) {
2040 curbe.CURBETotalDataLength = push_const_size;
2041 curbe.CURBEDataStartAddress = push_const_offset;
2042 }
2043
2044 uint32_t surfaces_offset = blorp_setup_binding_table(batch, params);
2045
2046 uint32_t samplers_offset =
2047 params->src.enabled ? blorp_emit_sampler_state(batch) : 0;
2048
2049 struct GENX(INTERFACE_DESCRIPTOR_DATA) idd = {
2050 .KernelStartPointer = params->cs_prog_kernel,
2051 .SamplerStatePointer = samplers_offset,
2052 .SamplerCount = params->src.enabled ? 1 : 0,
2053 .BindingTableEntryCount = params->src.enabled ? 2 : 1,
2054 .BindingTablePointer = surfaces_offset,
2055 .ConstantURBEntryReadLength = cs_prog_data->push.per_thread.regs,
2056 .NumberofThreadsinGPGPUThreadGroup = dispatch.threads,
2057 .SharedLocalMemorySize = elk_encode_slm_size(GFX_VER,
2058 prog_data->total_shared),
2059 .BarrierEnable = cs_prog_data->uses_barrier,
2060 #if GFX_VER >= 8 || GFX_VERx10 == 75
2061 .CrossThreadConstantDataReadLength =
2062 cs_prog_data->push.cross_thread.regs,
2063 #endif
2064 };
2065
2066 uint32_t idd_offset;
2067 uint32_t size = GENX(INTERFACE_DESCRIPTOR_DATA_length) * sizeof(uint32_t);
2068 void *state = blorp_alloc_dynamic_state(batch, size, 64, &idd_offset);
2069 GENX(INTERFACE_DESCRIPTOR_DATA_pack)(NULL, state, &idd);
2070
2071 blorp_emit(batch, GENX(MEDIA_INTERFACE_DESCRIPTOR_LOAD), mid) {
2072 mid.InterfaceDescriptorTotalLength = size;
2073 mid.InterfaceDescriptorDataStartAddress = idd_offset;
2074 }
2075
2076 blorp_emit(batch, GENX(GPGPU_WALKER), ggw) {
2077 ggw.SIMDSize = dispatch.simd_size / 16;
2078 ggw.ThreadDepthCounterMaximum = 0;
2079 ggw.ThreadHeightCounterMaximum = 0;
2080 ggw.ThreadWidthCounterMaximum = dispatch.threads - 1;
2081 ggw.ThreadGroupIDStartingX = group_x0;
2082 ggw.ThreadGroupIDStartingY = group_y0;
2083 #if GFX_VER >= 8
2084 ggw.ThreadGroupIDStartingResumeZ = group_z0;
2085 #else
2086 ggw.ThreadGroupIDStartingZ = group_z0;
2087 #endif
2088 ggw.ThreadGroupIDXDimension = group_x1;
2089 ggw.ThreadGroupIDYDimension = group_y1;
2090 ggw.ThreadGroupIDZDimension = group_z1;
2091 ggw.RightExecutionMask = dispatch.right_mask;
2092 ggw.BottomExecutionMask = 0xffffffff;
2093 }
2094
2095 #else /* GFX_VER >= 7 */
2096
2097 unreachable("Compute blorp is not supported on SNB and earlier");
2098
2099 #endif /* GFX_VER >= 7 */
2100
2101 blorp_measure_end(batch, params);
2102 }
2103
2104 /**
2105 * \brief Execute a blit or render pass operation.
2106 *
2107 * To execute the operation, this function manually constructs and emits a
2108 * batch to draw a rectangle primitive. The batchbuffer is flushed before
2109 * constructing and after emitting the batch.
2110 *
2111 * This function alters no GL state.
2112 */
2113 static void
blorp_exec(struct blorp_batch * batch,const struct blorp_params * params)2114 blorp_exec(struct blorp_batch *batch, const struct blorp_params *params)
2115 {
2116 /* Not supported in Gfx versions that use Elk. */
2117 assert((batch->flags & BLORP_BATCH_USE_BLITTER) == 0);
2118
2119 if (batch->flags & BLORP_BATCH_USE_COMPUTE) {
2120 blorp_exec_compute(batch, params);
2121 } else {
2122 blorp_exec_3d(batch, params);
2123 }
2124 }
2125
2126 #endif /* BLORP_GENX_EXEC_ELK_H */
2127