1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #ifndef BLORP_GENX_EXEC_H
25 #define BLORP_GENX_EXEC_H
26
27 #include "blorp_priv.h"
28 #include "dev/gen_device_info.h"
29 #include "common/gen_sample_positions.h"
30 #include "common/gen_l3_config.h"
31 #include "genxml/gen_macros.h"
32
33 /**
34 * This file provides the blorp pipeline setup and execution functionality.
35 * It defines the following function:
36 *
37 * static void
38 * blorp_exec(struct blorp_context *blorp, void *batch_data,
39 * const struct blorp_params *params);
40 *
41 * It is the job of whoever includes this header to wrap this in something
42 * to get an externally visible symbol.
43 *
44 * In order for the blorp_exec function to work, the driver must provide
45 * implementations of the following static helper functions.
46 */
47
48 static void *
49 blorp_emit_dwords(struct blorp_batch *batch, unsigned n);
50
51 static uint64_t
52 blorp_emit_reloc(struct blorp_batch *batch,
53 void *location, struct blorp_address address, uint32_t delta);
54
55 static void *
56 blorp_alloc_dynamic_state(struct blorp_batch *batch,
57 uint32_t size,
58 uint32_t alignment,
59 uint32_t *offset);
60 static void *
61 blorp_alloc_vertex_buffer(struct blorp_batch *batch, uint32_t size,
62 struct blorp_address *addr);
63 static void
64 blorp_vf_invalidate_for_vb_48b_transitions(struct blorp_batch *batch,
65 const struct blorp_address *addrs,
66 uint32_t *sizes,
67 unsigned num_vbs);
68
69 UNUSED static struct blorp_address
70 blorp_get_workaround_address(struct blorp_batch *batch);
71
72 static void
73 blorp_alloc_binding_table(struct blorp_batch *batch, unsigned num_entries,
74 unsigned state_size, unsigned state_alignment,
75 uint32_t *bt_offset, uint32_t *surface_offsets,
76 void **surface_maps);
77
78 static void
79 blorp_flush_range(struct blorp_batch *batch, void *start, size_t size);
80
81 static void
82 blorp_surface_reloc(struct blorp_batch *batch, uint32_t ss_offset,
83 struct blorp_address address, uint32_t delta);
84
85 static uint64_t
86 blorp_get_surface_address(struct blorp_batch *batch,
87 struct blorp_address address);
88
89 #if GEN_GEN >= 7 && GEN_GEN < 10
90 static struct blorp_address
91 blorp_get_surface_base_address(struct blorp_batch *batch);
92 #endif
93
94 #if GEN_GEN >= 7
95 static const struct gen_l3_config *
96 blorp_get_l3_config(struct blorp_batch *batch);
97 # else
98 static void
99 blorp_emit_urb_config(struct blorp_batch *batch,
100 unsigned vs_entry_size, unsigned sf_entry_size);
101 #endif
102
103 static void
104 blorp_emit_pipeline(struct blorp_batch *batch,
105 const struct blorp_params *params);
106
107 /***** BEGIN blorp_exec implementation ******/
108
109 static uint64_t
_blorp_combine_address(struct blorp_batch * batch,void * location,struct blorp_address address,uint32_t delta)110 _blorp_combine_address(struct blorp_batch *batch, void *location,
111 struct blorp_address address, uint32_t delta)
112 {
113 if (address.buffer == NULL) {
114 return address.offset + delta;
115 } else {
116 return blorp_emit_reloc(batch, location, address, delta);
117 }
118 }
119
120 #define __gen_address_type struct blorp_address
121 #define __gen_user_data struct blorp_batch
122 #define __gen_combine_address _blorp_combine_address
123
124 #include "genxml/genX_pack.h"
125
126 #define _blorp_cmd_length(cmd) cmd ## _length
127 #define _blorp_cmd_length_bias(cmd) cmd ## _length_bias
128 #define _blorp_cmd_header(cmd) cmd ## _header
129 #define _blorp_cmd_pack(cmd) cmd ## _pack
130
131 #define blorp_emit(batch, cmd, name) \
132 for (struct cmd name = { _blorp_cmd_header(cmd) }, \
133 *_dst = blorp_emit_dwords(batch, _blorp_cmd_length(cmd)); \
134 __builtin_expect(_dst != NULL, 1); \
135 _blorp_cmd_pack(cmd)(batch, (void *)_dst, &name), \
136 _dst = NULL)
137
138 #define blorp_emitn(batch, cmd, n, ...) ({ \
139 uint32_t *_dw = blorp_emit_dwords(batch, n); \
140 if (_dw) { \
141 struct cmd template = { \
142 _blorp_cmd_header(cmd), \
143 .DWordLength = n - _blorp_cmd_length_bias(cmd), \
144 __VA_ARGS__ \
145 }; \
146 _blorp_cmd_pack(cmd)(batch, _dw, &template); \
147 } \
148 _dw ? _dw + 1 : NULL; /* Array starts at dw[1] */ \
149 })
150
151 #define STRUCT_ZERO(S) ({ struct S t; memset(&t, 0, sizeof(t)); t; })
152
153 #define blorp_emit_dynamic(batch, state, name, align, offset) \
154 for (struct state name = STRUCT_ZERO(state), \
155 *_dst = blorp_alloc_dynamic_state(batch, \
156 _blorp_cmd_length(state) * 4, \
157 align, offset); \
158 __builtin_expect(_dst != NULL, 1); \
159 _blorp_cmd_pack(state)(batch, (void *)_dst, &name), \
160 blorp_flush_range(batch, _dst, _blorp_cmd_length(state) * 4), \
161 _dst = NULL)
162
163 /* 3DSTATE_URB
164 * 3DSTATE_URB_VS
165 * 3DSTATE_URB_HS
166 * 3DSTATE_URB_DS
167 * 3DSTATE_URB_GS
168 *
169 * Assign the entire URB to the VS. Even though the VS disabled, URB space
170 * is still needed because the clipper loads the VUE's from the URB. From
171 * the Sandybridge PRM, Volume 2, Part 1, Section 3DSTATE,
172 * Dword 1.15:0 "VS Number of URB Entries":
173 * This field is always used (even if VS Function Enable is DISABLED).
174 *
175 * The warning below appears in the PRM (Section 3DSTATE_URB), but we can
176 * safely ignore it because this batch contains only one draw call.
177 * Because of URB corruption caused by allocating a previous GS unit
178 * URB entry to the VS unit, software is required to send a “GS NULL
179 * Fence” (Send URB fence with VS URB size == 1 and GS URB size == 0)
180 * plus a dummy DRAW call before any case where VS will be taking over
181 * GS URB space.
182 *
183 * If the 3DSTATE_URB_VS is emitted, than the others must be also.
184 * From the Ivybridge PRM, Volume 2 Part 1, section 1.7.1 3DSTATE_URB_VS:
185 *
186 * 3DSTATE_URB_HS, 3DSTATE_URB_DS, and 3DSTATE_URB_GS must also be
187 * programmed in order for the programming of this state to be
188 * valid.
189 */
190 static void
emit_urb_config(struct blorp_batch * batch,const struct blorp_params * params,UNUSED enum gen_urb_deref_block_size * deref_block_size)191 emit_urb_config(struct blorp_batch *batch,
192 const struct blorp_params *params,
193 UNUSED enum gen_urb_deref_block_size *deref_block_size)
194 {
195 /* Once vertex fetcher has written full VUE entries with complete
196 * header the space requirement is as follows per vertex (in bytes):
197 *
198 * Header Position Program constants
199 * +--------+------------+-------------------+
200 * | 16 | 16 | n x 16 |
201 * +--------+------------+-------------------+
202 *
203 * where 'n' stands for number of varying inputs expressed as vec4s.
204 */
205 const unsigned num_varyings =
206 params->wm_prog_data ? params->wm_prog_data->num_varying_inputs : 0;
207 const unsigned total_needed = 16 + 16 + num_varyings * 16;
208
209 /* The URB size is expressed in units of 64 bytes (512 bits) */
210 const unsigned vs_entry_size = DIV_ROUND_UP(total_needed, 64);
211
212 ASSERTED const unsigned sf_entry_size =
213 params->sf_prog_data ? params->sf_prog_data->urb_entry_size : 0;
214
215 #if GEN_GEN >= 7
216 assert(sf_entry_size == 0);
217 const unsigned entry_size[4] = { vs_entry_size, 1, 1, 1 };
218
219 unsigned entries[4], start[4];
220 gen_get_urb_config(batch->blorp->compiler->devinfo,
221 blorp_get_l3_config(batch),
222 false, false, entry_size,
223 entries, start, deref_block_size);
224
225 #if GEN_GEN == 7 && !GEN_IS_HASWELL
226 /* From the IVB PRM Vol. 2, Part 1, Section 3.2.1:
227 *
228 * "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth stall
229 * needs to be sent just prior to any 3DSTATE_VS, 3DSTATE_URB_VS,
230 * 3DSTATE_CONSTANT_VS, 3DSTATE_BINDING_TABLE_POINTER_VS,
231 * 3DSTATE_SAMPLER_STATE_POINTER_VS command. Only one PIPE_CONTROL
232 * needs to be sent before any combination of VS associated 3DSTATE."
233 */
234 blorp_emit(batch, GENX(PIPE_CONTROL), pc) {
235 pc.DepthStallEnable = true;
236 pc.PostSyncOperation = WriteImmediateData;
237 pc.Address = blorp_get_workaround_address(batch);
238 }
239 #endif
240
241 for (int i = 0; i <= MESA_SHADER_GEOMETRY; i++) {
242 blorp_emit(batch, GENX(3DSTATE_URB_VS), urb) {
243 urb._3DCommandSubOpcode += i;
244 urb.VSURBStartingAddress = start[i];
245 urb.VSURBEntryAllocationSize = entry_size[i] - 1;
246 urb.VSNumberofURBEntries = entries[i];
247 }
248 }
249 #else /* GEN_GEN < 7 */
250 blorp_emit_urb_config(batch, vs_entry_size, sf_entry_size);
251 #endif
252 }
253
254 #if GEN_GEN >= 7
255 static void
256 blorp_emit_memcpy(struct blorp_batch *batch,
257 struct blorp_address dst,
258 struct blorp_address src,
259 uint32_t size);
260 #endif
261
262 static void
blorp_emit_vertex_data(struct blorp_batch * batch,const struct blorp_params * params,struct blorp_address * addr,uint32_t * size)263 blorp_emit_vertex_data(struct blorp_batch *batch,
264 const struct blorp_params *params,
265 struct blorp_address *addr,
266 uint32_t *size)
267 {
268 const float vertices[] = {
269 /* v0 */ (float)params->x1, (float)params->y1, params->z,
270 /* v1 */ (float)params->x0, (float)params->y1, params->z,
271 /* v2 */ (float)params->x0, (float)params->y0, params->z,
272 };
273
274 void *data = blorp_alloc_vertex_buffer(batch, sizeof(vertices), addr);
275 memcpy(data, vertices, sizeof(vertices));
276 *size = sizeof(vertices);
277 blorp_flush_range(batch, data, *size);
278 }
279
280 static void
blorp_emit_input_varying_data(struct blorp_batch * batch,const struct blorp_params * params,struct blorp_address * addr,uint32_t * size)281 blorp_emit_input_varying_data(struct blorp_batch *batch,
282 const struct blorp_params *params,
283 struct blorp_address *addr,
284 uint32_t *size)
285 {
286 const unsigned vec4_size_in_bytes = 4 * sizeof(float);
287 const unsigned max_num_varyings =
288 DIV_ROUND_UP(sizeof(params->wm_inputs), vec4_size_in_bytes);
289 const unsigned num_varyings =
290 params->wm_prog_data ? params->wm_prog_data->num_varying_inputs : 0;
291
292 *size = 16 + num_varyings * vec4_size_in_bytes;
293
294 const uint32_t *const inputs_src = (const uint32_t *)¶ms->wm_inputs;
295 void *data = blorp_alloc_vertex_buffer(batch, *size, addr);
296 uint32_t *inputs = data;
297
298 /* Copy in the VS inputs */
299 assert(sizeof(params->vs_inputs) == 16);
300 memcpy(inputs, ¶ms->vs_inputs, sizeof(params->vs_inputs));
301 inputs += 4;
302
303 if (params->wm_prog_data) {
304 /* Walk over the attribute slots, determine if the attribute is used by
305 * the program and when necessary copy the values from the input storage
306 * to the vertex data buffer.
307 */
308 for (unsigned i = 0; i < max_num_varyings; i++) {
309 const gl_varying_slot attr = VARYING_SLOT_VAR0 + i;
310
311 const int input_index = params->wm_prog_data->urb_setup[attr];
312 if (input_index < 0)
313 continue;
314
315 memcpy(inputs, inputs_src + i * 4, vec4_size_in_bytes);
316
317 inputs += 4;
318 }
319 }
320
321 blorp_flush_range(batch, data, *size);
322
323 if (params->dst_clear_color_as_input) {
324 #if GEN_GEN >= 7
325 /* In this case, the clear color isn't known statically and instead
326 * comes in through an indirect which we have to copy into the vertex
327 * buffer before we execute the 3DPRIMITIVE. We already copied the
328 * value of params->wm_inputs.clear_color into the vertex buffer in the
329 * loop above. Now we emit code to stomp it from the GPU with the
330 * actual clear color value.
331 */
332 assert(num_varyings == 1);
333
334 /* The clear color is the first thing after the header */
335 struct blorp_address clear_color_input_addr = *addr;
336 clear_color_input_addr.offset += 16;
337
338 const unsigned clear_color_size =
339 GEN_GEN < 10 ? batch->blorp->isl_dev->ss.clear_value_size : 4 * 4;
340 blorp_emit_memcpy(batch, clear_color_input_addr,
341 params->dst.clear_color_addr,
342 clear_color_size);
343 #else
344 unreachable("MCS partial resolve is not a thing on SNB and earlier");
345 #endif
346 }
347 }
348
349 static void
blorp_fill_vertex_buffer_state(struct GENX (VERTEX_BUFFER_STATE)* vb,unsigned idx,struct blorp_address addr,uint32_t size,uint32_t stride)350 blorp_fill_vertex_buffer_state(struct GENX(VERTEX_BUFFER_STATE) *vb,
351 unsigned idx,
352 struct blorp_address addr, uint32_t size,
353 uint32_t stride)
354 {
355 vb[idx].VertexBufferIndex = idx;
356 vb[idx].BufferStartingAddress = addr;
357 vb[idx].BufferPitch = stride;
358
359 #if GEN_GEN >= 6
360 vb[idx].MOCS = addr.mocs;
361 #endif
362
363 #if GEN_GEN >= 7
364 vb[idx].AddressModifyEnable = true;
365 #endif
366
367 #if GEN_GEN >= 8
368 vb[idx].BufferSize = size;
369 #elif GEN_GEN >= 5
370 vb[idx].BufferAccessType = stride > 0 ? VERTEXDATA : INSTANCEDATA;
371 vb[idx].EndAddress = vb[idx].BufferStartingAddress;
372 vb[idx].EndAddress.offset += size - 1;
373 #elif GEN_GEN == 4
374 vb[idx].BufferAccessType = stride > 0 ? VERTEXDATA : INSTANCEDATA;
375 vb[idx].MaxIndex = stride > 0 ? size / stride : 0;
376 #endif
377 }
378
379 static void
blorp_emit_vertex_buffers(struct blorp_batch * batch,const struct blorp_params * params)380 blorp_emit_vertex_buffers(struct blorp_batch *batch,
381 const struct blorp_params *params)
382 {
383 struct GENX(VERTEX_BUFFER_STATE) vb[3];
384 uint32_t num_vbs = 2;
385 memset(vb, 0, sizeof(vb));
386
387 struct blorp_address addrs[2] = {};
388 uint32_t sizes[2];
389 blorp_emit_vertex_data(batch, params, &addrs[0], &sizes[0]);
390 blorp_fill_vertex_buffer_state(vb, 0, addrs[0], sizes[0],
391 3 * sizeof(float));
392
393 blorp_emit_input_varying_data(batch, params, &addrs[1], &sizes[1]);
394 blorp_fill_vertex_buffer_state(vb, 1, addrs[1], sizes[1], 0);
395
396 blorp_vf_invalidate_for_vb_48b_transitions(batch, addrs, sizes, num_vbs);
397
398 const unsigned num_dwords = 1 + num_vbs * GENX(VERTEX_BUFFER_STATE_length);
399 uint32_t *dw = blorp_emitn(batch, GENX(3DSTATE_VERTEX_BUFFERS), num_dwords);
400 if (!dw)
401 return;
402
403 for (unsigned i = 0; i < num_vbs; i++) {
404 GENX(VERTEX_BUFFER_STATE_pack)(batch, dw, &vb[i]);
405 dw += GENX(VERTEX_BUFFER_STATE_length);
406 }
407 }
408
409 static void
blorp_emit_vertex_elements(struct blorp_batch * batch,const struct blorp_params * params)410 blorp_emit_vertex_elements(struct blorp_batch *batch,
411 const struct blorp_params *params)
412 {
413 const unsigned num_varyings =
414 params->wm_prog_data ? params->wm_prog_data->num_varying_inputs : 0;
415 bool need_ndc = batch->blorp->compiler->devinfo->gen <= 5;
416 const unsigned num_elements = 2 + need_ndc + num_varyings;
417
418 struct GENX(VERTEX_ELEMENT_STATE) ve[num_elements];
419 memset(ve, 0, num_elements * sizeof(*ve));
420
421 /* Setup VBO for the rectangle primitive..
422 *
423 * A rectangle primitive (3DPRIM_RECTLIST) consists of only three
424 * vertices. The vertices reside in screen space with DirectX
425 * coordinates (that is, (0, 0) is the upper left corner).
426 *
427 * v2 ------ implied
428 * | |
429 * | |
430 * v1 ----- v0
431 *
432 * Since the VS is disabled, the clipper loads each VUE directly from
433 * the URB. This is controlled by the 3DSTATE_VERTEX_BUFFERS and
434 * 3DSTATE_VERTEX_ELEMENTS packets below. The VUE contents are as follows:
435 * dw0: Reserved, MBZ.
436 * dw1: Render Target Array Index. Below vertex fetcher gets programmed
437 * to assign this with primitive instance identifier which will be
438 * used for layered clears. All other renders have only one instance
439 * and therefore the value will be effectively zero.
440 * dw2: Viewport Index. The HiZ op disables viewport mapping and
441 * scissoring, so set the dword to 0.
442 * dw3: Point Width: The HiZ op does not emit the POINTLIST primitive,
443 * so set the dword to 0.
444 * dw4: Vertex Position X.
445 * dw5: Vertex Position Y.
446 * dw6: Vertex Position Z.
447 * dw7: Vertex Position W.
448 *
449 * dw8: Flat vertex input 0
450 * dw9: Flat vertex input 1
451 * ...
452 * dwn: Flat vertex input n - 8
453 *
454 * For details, see the Sandybridge PRM, Volume 2, Part 1, Section 1.5.1
455 * "Vertex URB Entry (VUE) Formats".
456 *
457 * Only vertex position X and Y are going to be variable, Z is fixed to
458 * zero and W to one. Header words dw0,2,3 are zero. There is no need to
459 * include the fixed values in the vertex buffer. Vertex fetcher can be
460 * instructed to fill vertex elements with constant values of one and zero
461 * instead of reading them from the buffer.
462 * Flat inputs are program constants that are not interpolated. Moreover
463 * their values will be the same between vertices.
464 *
465 * See the vertex element setup below.
466 */
467 unsigned slot = 0;
468
469 ve[slot] = (struct GENX(VERTEX_ELEMENT_STATE)) {
470 .VertexBufferIndex = 1,
471 .Valid = true,
472 .SourceElementFormat = ISL_FORMAT_R32G32B32A32_FLOAT,
473 .SourceElementOffset = 0,
474 .Component0Control = VFCOMP_STORE_SRC,
475
476 /* From Gen8 onwards hardware is no more instructed to overwrite
477 * components using an element specifier. Instead one has separate
478 * 3DSTATE_VF_SGVS (System Generated Value Setup) state packet for it.
479 */
480 #if GEN_GEN >= 8
481 .Component1Control = VFCOMP_STORE_0,
482 #elif GEN_GEN >= 5
483 .Component1Control = VFCOMP_STORE_IID,
484 #else
485 .Component1Control = VFCOMP_STORE_0,
486 #endif
487 .Component2Control = VFCOMP_STORE_0,
488 .Component3Control = VFCOMP_STORE_0,
489 #if GEN_GEN <= 5
490 .DestinationElementOffset = slot * 4,
491 #endif
492 };
493 slot++;
494
495 #if GEN_GEN <= 5
496 /* On Iron Lake and earlier, a native device coordinates version of the
497 * position goes right after the normal VUE header and before position.
498 * Since w == 1 for all of our coordinates, this is just a copy of the
499 * position.
500 */
501 ve[slot] = (struct GENX(VERTEX_ELEMENT_STATE)) {
502 .VertexBufferIndex = 0,
503 .Valid = true,
504 .SourceElementFormat = ISL_FORMAT_R32G32B32_FLOAT,
505 .SourceElementOffset = 0,
506 .Component0Control = VFCOMP_STORE_SRC,
507 .Component1Control = VFCOMP_STORE_SRC,
508 .Component2Control = VFCOMP_STORE_SRC,
509 .Component3Control = VFCOMP_STORE_1_FP,
510 .DestinationElementOffset = slot * 4,
511 };
512 slot++;
513 #endif
514
515 ve[slot] = (struct GENX(VERTEX_ELEMENT_STATE)) {
516 .VertexBufferIndex = 0,
517 .Valid = true,
518 .SourceElementFormat = ISL_FORMAT_R32G32B32_FLOAT,
519 .SourceElementOffset = 0,
520 .Component0Control = VFCOMP_STORE_SRC,
521 .Component1Control = VFCOMP_STORE_SRC,
522 .Component2Control = VFCOMP_STORE_SRC,
523 .Component3Control = VFCOMP_STORE_1_FP,
524 #if GEN_GEN <= 5
525 .DestinationElementOffset = slot * 4,
526 #endif
527 };
528 slot++;
529
530 for (unsigned i = 0; i < num_varyings; ++i) {
531 ve[slot] = (struct GENX(VERTEX_ELEMENT_STATE)) {
532 .VertexBufferIndex = 1,
533 .Valid = true,
534 .SourceElementFormat = ISL_FORMAT_R32G32B32A32_FLOAT,
535 .SourceElementOffset = 16 + i * 4 * sizeof(float),
536 .Component0Control = VFCOMP_STORE_SRC,
537 .Component1Control = VFCOMP_STORE_SRC,
538 .Component2Control = VFCOMP_STORE_SRC,
539 .Component3Control = VFCOMP_STORE_SRC,
540 #if GEN_GEN <= 5
541 .DestinationElementOffset = slot * 4,
542 #endif
543 };
544 slot++;
545 }
546
547 const unsigned num_dwords =
548 1 + GENX(VERTEX_ELEMENT_STATE_length) * num_elements;
549 uint32_t *dw = blorp_emitn(batch, GENX(3DSTATE_VERTEX_ELEMENTS), num_dwords);
550 if (!dw)
551 return;
552
553 for (unsigned i = 0; i < num_elements; i++) {
554 GENX(VERTEX_ELEMENT_STATE_pack)(batch, dw, &ve[i]);
555 dw += GENX(VERTEX_ELEMENT_STATE_length);
556 }
557
558 blorp_emit(batch, GENX(3DSTATE_VF_STATISTICS), vf) {
559 vf.StatisticsEnable = false;
560 }
561
562 #if GEN_GEN >= 8
563 /* Overwrite Render Target Array Index (2nd dword) in the VUE header with
564 * primitive instance identifier. This is used for layered clears.
565 */
566 blorp_emit(batch, GENX(3DSTATE_VF_SGVS), sgvs) {
567 sgvs.InstanceIDEnable = true;
568 sgvs.InstanceIDComponentNumber = COMP_1;
569 sgvs.InstanceIDElementOffset = 0;
570 }
571
572 for (unsigned i = 0; i < num_elements; i++) {
573 blorp_emit(batch, GENX(3DSTATE_VF_INSTANCING), vf) {
574 vf.VertexElementIndex = i;
575 vf.InstancingEnable = false;
576 }
577 }
578
579 blorp_emit(batch, GENX(3DSTATE_VF_TOPOLOGY), topo) {
580 topo.PrimitiveTopologyType = _3DPRIM_RECTLIST;
581 }
582 #endif
583 }
584
585 /* 3DSTATE_VIEWPORT_STATE_POINTERS */
586 static uint32_t
blorp_emit_cc_viewport(struct blorp_batch * batch)587 blorp_emit_cc_viewport(struct blorp_batch *batch)
588 {
589 uint32_t cc_vp_offset;
590 blorp_emit_dynamic(batch, GENX(CC_VIEWPORT), vp, 32, &cc_vp_offset) {
591 vp.MinimumDepth = 0.0;
592 vp.MaximumDepth = 1.0;
593 }
594
595 #if GEN_GEN >= 7
596 blorp_emit(batch, GENX(3DSTATE_VIEWPORT_STATE_POINTERS_CC), vsp) {
597 vsp.CCViewportPointer = cc_vp_offset;
598 }
599 #elif GEN_GEN == 6
600 blorp_emit(batch, GENX(3DSTATE_VIEWPORT_STATE_POINTERS), vsp) {
601 vsp.CCViewportStateChange = true;
602 vsp.PointertoCC_VIEWPORT = cc_vp_offset;
603 }
604 #endif
605
606 return cc_vp_offset;
607 }
608
609 static uint32_t
blorp_emit_sampler_state(struct blorp_batch * batch)610 blorp_emit_sampler_state(struct blorp_batch *batch)
611 {
612 uint32_t offset;
613 blorp_emit_dynamic(batch, GENX(SAMPLER_STATE), sampler, 32, &offset) {
614 sampler.MipModeFilter = MIPFILTER_NONE;
615 sampler.MagModeFilter = MAPFILTER_LINEAR;
616 sampler.MinModeFilter = MAPFILTER_LINEAR;
617 sampler.MinLOD = 0;
618 sampler.MaxLOD = 0;
619 sampler.TCXAddressControlMode = TCM_CLAMP;
620 sampler.TCYAddressControlMode = TCM_CLAMP;
621 sampler.TCZAddressControlMode = TCM_CLAMP;
622 sampler.MaximumAnisotropy = RATIO21;
623 sampler.RAddressMinFilterRoundingEnable = true;
624 sampler.RAddressMagFilterRoundingEnable = true;
625 sampler.VAddressMinFilterRoundingEnable = true;
626 sampler.VAddressMagFilterRoundingEnable = true;
627 sampler.UAddressMinFilterRoundingEnable = true;
628 sampler.UAddressMagFilterRoundingEnable = true;
629 #if GEN_GEN > 6
630 sampler.NonnormalizedCoordinateEnable = true;
631 #endif
632 }
633
634 #if GEN_GEN >= 7
635 blorp_emit(batch, GENX(3DSTATE_SAMPLER_STATE_POINTERS_PS), ssp) {
636 ssp.PointertoPSSamplerState = offset;
637 }
638 #elif GEN_GEN == 6
639 blorp_emit(batch, GENX(3DSTATE_SAMPLER_STATE_POINTERS), ssp) {
640 ssp.VSSamplerStateChange = true;
641 ssp.GSSamplerStateChange = true;
642 ssp.PSSamplerStateChange = true;
643 ssp.PointertoPSSamplerState = offset;
644 }
645 #endif
646
647 return offset;
648 }
649
650 /* What follows is the code for setting up a "pipeline" on Sandy Bridge and
651 * later hardware. This file will be included by i965 for gen4-5 as well, so
652 * this code is guarded by GEN_GEN >= 6.
653 */
654 #if GEN_GEN >= 6
655
656 static void
blorp_emit_vs_config(struct blorp_batch * batch,const struct blorp_params * params)657 blorp_emit_vs_config(struct blorp_batch *batch,
658 const struct blorp_params *params)
659 {
660 struct brw_vs_prog_data *vs_prog_data = params->vs_prog_data;
661 assert(!vs_prog_data || GEN_GEN < 11 ||
662 vs_prog_data->base.dispatch_mode == DISPATCH_MODE_SIMD8);
663
664 blorp_emit(batch, GENX(3DSTATE_VS), vs) {
665 if (vs_prog_data) {
666 vs.Enable = true;
667
668 vs.KernelStartPointer = params->vs_prog_kernel;
669
670 vs.DispatchGRFStartRegisterForURBData =
671 vs_prog_data->base.base.dispatch_grf_start_reg;
672 vs.VertexURBEntryReadLength =
673 vs_prog_data->base.urb_read_length;
674 vs.VertexURBEntryReadOffset = 0;
675
676 vs.MaximumNumberofThreads =
677 batch->blorp->isl_dev->info->max_vs_threads - 1;
678
679 #if GEN_GEN >= 8
680 vs.SIMD8DispatchEnable =
681 vs_prog_data->base.dispatch_mode == DISPATCH_MODE_SIMD8;
682 #endif
683 }
684 }
685 }
686
687 static void
blorp_emit_sf_config(struct blorp_batch * batch,const struct blorp_params * params,UNUSED enum gen_urb_deref_block_size urb_deref_block_size)688 blorp_emit_sf_config(struct blorp_batch *batch,
689 const struct blorp_params *params,
690 UNUSED enum gen_urb_deref_block_size urb_deref_block_size)
691 {
692 const struct brw_wm_prog_data *prog_data = params->wm_prog_data;
693
694 /* 3DSTATE_SF
695 *
696 * Disable ViewportTransformEnable (dw2.1)
697 *
698 * From the SandyBridge PRM, Volume 2, Part 1, Section 1.3, "3D
699 * Primitives Overview":
700 * RECTLIST: Viewport Mapping must be DISABLED (as is typical with the
701 * use of screen- space coordinates).
702 *
703 * A solid rectangle must be rendered, so set FrontFaceFillMode (dw2.4:3)
704 * and BackFaceFillMode (dw2.5:6) to SOLID(0).
705 *
706 * From the Sandy Bridge PRM, Volume 2, Part 1, Section
707 * 6.4.1.1 3DSTATE_SF, Field FrontFaceFillMode:
708 * SOLID: Any triangle or rectangle object found to be front-facing
709 * is rendered as a solid object. This setting is required when
710 * (rendering rectangle (RECTLIST) objects.
711 */
712
713 #if GEN_GEN >= 8
714
715 blorp_emit(batch, GENX(3DSTATE_SF), sf) {
716 #if GEN_GEN >= 12
717 sf.DerefBlockSize = urb_deref_block_size;
718 #endif
719 }
720
721 blorp_emit(batch, GENX(3DSTATE_RASTER), raster) {
722 raster.CullMode = CULLMODE_NONE;
723 }
724
725 blorp_emit(batch, GENX(3DSTATE_SBE), sbe) {
726 sbe.VertexURBEntryReadOffset = 1;
727 if (prog_data) {
728 sbe.NumberofSFOutputAttributes = prog_data->num_varying_inputs;
729 sbe.VertexURBEntryReadLength = brw_blorp_get_urb_length(prog_data);
730 sbe.ConstantInterpolationEnable = prog_data->flat_inputs;
731 } else {
732 sbe.NumberofSFOutputAttributes = 0;
733 sbe.VertexURBEntryReadLength = 1;
734 }
735 sbe.ForceVertexURBEntryReadLength = true;
736 sbe.ForceVertexURBEntryReadOffset = true;
737
738 #if GEN_GEN >= 9
739 for (unsigned i = 0; i < 32; i++)
740 sbe.AttributeActiveComponentFormat[i] = ACF_XYZW;
741 #endif
742 }
743
744 #elif GEN_GEN >= 7
745
746 blorp_emit(batch, GENX(3DSTATE_SF), sf) {
747 sf.FrontFaceFillMode = FILL_MODE_SOLID;
748 sf.BackFaceFillMode = FILL_MODE_SOLID;
749
750 sf.MultisampleRasterizationMode = params->num_samples > 1 ?
751 MSRASTMODE_ON_PATTERN : MSRASTMODE_OFF_PIXEL;
752
753 #if GEN_GEN == 7
754 sf.DepthBufferSurfaceFormat = params->depth_format;
755 #endif
756 }
757
758 blorp_emit(batch, GENX(3DSTATE_SBE), sbe) {
759 sbe.VertexURBEntryReadOffset = 1;
760 if (prog_data) {
761 sbe.NumberofSFOutputAttributes = prog_data->num_varying_inputs;
762 sbe.VertexURBEntryReadLength = brw_blorp_get_urb_length(prog_data);
763 sbe.ConstantInterpolationEnable = prog_data->flat_inputs;
764 } else {
765 sbe.NumberofSFOutputAttributes = 0;
766 sbe.VertexURBEntryReadLength = 1;
767 }
768 }
769
770 #else /* GEN_GEN <= 6 */
771
772 blorp_emit(batch, GENX(3DSTATE_SF), sf) {
773 sf.FrontFaceFillMode = FILL_MODE_SOLID;
774 sf.BackFaceFillMode = FILL_MODE_SOLID;
775
776 sf.MultisampleRasterizationMode = params->num_samples > 1 ?
777 MSRASTMODE_ON_PATTERN : MSRASTMODE_OFF_PIXEL;
778
779 sf.VertexURBEntryReadOffset = 1;
780 if (prog_data) {
781 sf.NumberofSFOutputAttributes = prog_data->num_varying_inputs;
782 sf.VertexURBEntryReadLength = brw_blorp_get_urb_length(prog_data);
783 sf.ConstantInterpolationEnable = prog_data->flat_inputs;
784 } else {
785 sf.NumberofSFOutputAttributes = 0;
786 sf.VertexURBEntryReadLength = 1;
787 }
788 }
789
790 #endif /* GEN_GEN */
791 }
792
793 static void
blorp_emit_ps_config(struct blorp_batch * batch,const struct blorp_params * params)794 blorp_emit_ps_config(struct blorp_batch *batch,
795 const struct blorp_params *params)
796 {
797 const struct brw_wm_prog_data *prog_data = params->wm_prog_data;
798
799 /* Even when thread dispatch is disabled, max threads (dw5.25:31) must be
800 * nonzero to prevent the GPU from hanging. While the documentation doesn't
801 * mention this explicitly, it notes that the valid range for the field is
802 * [1,39] = [2,40] threads, which excludes zero.
803 *
804 * To be safe (and to minimize extraneous code) we go ahead and fully
805 * configure the WM state whether or not there is a WM program.
806 */
807
808 #if GEN_GEN >= 8
809
810 blorp_emit(batch, GENX(3DSTATE_WM), wm);
811
812 blorp_emit(batch, GENX(3DSTATE_PS), ps) {
813 if (params->src.enabled) {
814 ps.SamplerCount = 1; /* Up to 4 samplers */
815 ps.BindingTableEntryCount = 2;
816 } else {
817 ps.BindingTableEntryCount = 1;
818 }
819
820 /* SAMPLER_STATE prefetching is broken on Gen11 - WA_1606682166 */
821 if (GEN_GEN == 11)
822 ps.SamplerCount = 0;
823
824 if (prog_data) {
825 ps._8PixelDispatchEnable = prog_data->dispatch_8;
826 ps._16PixelDispatchEnable = prog_data->dispatch_16;
827 ps._32PixelDispatchEnable = prog_data->dispatch_32;
828
829 /* From the Sky Lake PRM 3DSTATE_PS::32 Pixel Dispatch Enable:
830 *
831 * "When NUM_MULTISAMPLES = 16 or FORCE_SAMPLE_COUNT = 16, SIMD32
832 * Dispatch must not be enabled for PER_PIXEL dispatch mode."
833 *
834 * Since 16x MSAA is first introduced on SKL, we don't need to apply
835 * the workaround on any older hardware.
836 */
837 if (GEN_GEN >= 9 && !prog_data->persample_dispatch &&
838 params->num_samples == 16) {
839 assert(ps._8PixelDispatchEnable || ps._16PixelDispatchEnable);
840 ps._32PixelDispatchEnable = false;
841 }
842
843 ps.DispatchGRFStartRegisterForConstantSetupData0 =
844 brw_wm_prog_data_dispatch_grf_start_reg(prog_data, ps, 0);
845 ps.DispatchGRFStartRegisterForConstantSetupData1 =
846 brw_wm_prog_data_dispatch_grf_start_reg(prog_data, ps, 1);
847 ps.DispatchGRFStartRegisterForConstantSetupData2 =
848 brw_wm_prog_data_dispatch_grf_start_reg(prog_data, ps, 2);
849
850 ps.KernelStartPointer0 = params->wm_prog_kernel +
851 brw_wm_prog_data_prog_offset(prog_data, ps, 0);
852 ps.KernelStartPointer1 = params->wm_prog_kernel +
853 brw_wm_prog_data_prog_offset(prog_data, ps, 1);
854 ps.KernelStartPointer2 = params->wm_prog_kernel +
855 brw_wm_prog_data_prog_offset(prog_data, ps, 2);
856 }
857
858 /* 3DSTATE_PS expects the number of threads per PSD, which is always 64
859 * for pre Gen11 and 128 for gen11+; On gen11+ If a programmed value is
860 * k, it implies 2(k+1) threads. It implicitly scales for different GT
861 * levels (which have some # of PSDs).
862 *
863 * In Gen8 the format is U8-2 whereas in Gen9+ it is U9-1.
864 */
865 if (GEN_GEN >= 9)
866 ps.MaximumNumberofThreadsPerPSD = 64 - 1;
867 else
868 ps.MaximumNumberofThreadsPerPSD = 64 - 2;
869
870 switch (params->fast_clear_op) {
871 case ISL_AUX_OP_NONE:
872 break;
873 #if GEN_GEN >= 10
874 case ISL_AUX_OP_AMBIGUATE:
875 ps.RenderTargetFastClearEnable = true;
876 ps.RenderTargetResolveType = FAST_CLEAR_0;
877 break;
878 #endif
879 #if GEN_GEN >= 9
880 case ISL_AUX_OP_PARTIAL_RESOLVE:
881 ps.RenderTargetResolveType = RESOLVE_PARTIAL;
882 break;
883 case ISL_AUX_OP_FULL_RESOLVE:
884 ps.RenderTargetResolveType = RESOLVE_FULL;
885 break;
886 #else
887 case ISL_AUX_OP_FULL_RESOLVE:
888 ps.RenderTargetResolveEnable = true;
889 break;
890 #endif
891 case ISL_AUX_OP_FAST_CLEAR:
892 ps.RenderTargetFastClearEnable = true;
893 break;
894 default:
895 unreachable("Invalid fast clear op");
896 }
897 }
898
899 blorp_emit(batch, GENX(3DSTATE_PS_EXTRA), psx) {
900 if (prog_data) {
901 psx.PixelShaderValid = true;
902 psx.AttributeEnable = prog_data->num_varying_inputs > 0;
903 psx.PixelShaderIsPerSample = prog_data->persample_dispatch;
904 psx.PixelShaderComputedDepthMode = prog_data->computed_depth_mode;
905 #if GEN_GEN >= 9
906 psx.PixelShaderComputesStencil = prog_data->computed_stencil;
907 #endif
908 }
909
910 if (params->src.enabled)
911 psx.PixelShaderKillsPixel = true;
912 }
913
914 #elif GEN_GEN >= 7
915
916 blorp_emit(batch, GENX(3DSTATE_WM), wm) {
917 switch (params->hiz_op) {
918 case ISL_AUX_OP_FAST_CLEAR:
919 wm.DepthBufferClear = true;
920 break;
921 case ISL_AUX_OP_FULL_RESOLVE:
922 wm.DepthBufferResolveEnable = true;
923 break;
924 case ISL_AUX_OP_AMBIGUATE:
925 wm.HierarchicalDepthBufferResolveEnable = true;
926 break;
927 case ISL_AUX_OP_NONE:
928 break;
929 default:
930 unreachable("not reached");
931 }
932
933 if (prog_data) {
934 wm.ThreadDispatchEnable = true;
935 wm.PixelShaderComputedDepthMode = prog_data->computed_depth_mode;
936 }
937
938 if (params->src.enabled)
939 wm.PixelShaderKillsPixel = true;
940
941 if (params->num_samples > 1) {
942 wm.MultisampleRasterizationMode = MSRASTMODE_ON_PATTERN;
943 wm.MultisampleDispatchMode =
944 (prog_data && prog_data->persample_dispatch) ?
945 MSDISPMODE_PERSAMPLE : MSDISPMODE_PERPIXEL;
946 } else {
947 wm.MultisampleRasterizationMode = MSRASTMODE_OFF_PIXEL;
948 wm.MultisampleDispatchMode = MSDISPMODE_PERSAMPLE;
949 }
950 }
951
952 blorp_emit(batch, GENX(3DSTATE_PS), ps) {
953 ps.MaximumNumberofThreads =
954 batch->blorp->isl_dev->info->max_wm_threads - 1;
955
956 #if GEN_IS_HASWELL
957 ps.SampleMask = 1;
958 #endif
959
960 if (prog_data) {
961 ps._8PixelDispatchEnable = prog_data->dispatch_8;
962 ps._16PixelDispatchEnable = prog_data->dispatch_16;
963 ps._32PixelDispatchEnable = prog_data->dispatch_32;
964
965 ps.DispatchGRFStartRegisterForConstantSetupData0 =
966 brw_wm_prog_data_dispatch_grf_start_reg(prog_data, ps, 0);
967 ps.DispatchGRFStartRegisterForConstantSetupData1 =
968 brw_wm_prog_data_dispatch_grf_start_reg(prog_data, ps, 1);
969 ps.DispatchGRFStartRegisterForConstantSetupData2 =
970 brw_wm_prog_data_dispatch_grf_start_reg(prog_data, ps, 2);
971
972 ps.KernelStartPointer0 = params->wm_prog_kernel +
973 brw_wm_prog_data_prog_offset(prog_data, ps, 0);
974 ps.KernelStartPointer1 = params->wm_prog_kernel +
975 brw_wm_prog_data_prog_offset(prog_data, ps, 1);
976 ps.KernelStartPointer2 = params->wm_prog_kernel +
977 brw_wm_prog_data_prog_offset(prog_data, ps, 2);
978
979 ps.AttributeEnable = prog_data->num_varying_inputs > 0;
980 } else {
981 /* Gen7 hardware gets angry if we don't enable at least one dispatch
982 * mode, so just enable 16-pixel dispatch if we don't have a program.
983 */
984 ps._16PixelDispatchEnable = true;
985 }
986
987 if (params->src.enabled)
988 ps.SamplerCount = 1; /* Up to 4 samplers */
989
990 switch (params->fast_clear_op) {
991 case ISL_AUX_OP_NONE:
992 break;
993 case ISL_AUX_OP_FULL_RESOLVE:
994 ps.RenderTargetResolveEnable = true;
995 break;
996 case ISL_AUX_OP_FAST_CLEAR:
997 ps.RenderTargetFastClearEnable = true;
998 break;
999 default:
1000 unreachable("Invalid fast clear op");
1001 }
1002 }
1003
1004 #else /* GEN_GEN <= 6 */
1005
1006 blorp_emit(batch, GENX(3DSTATE_WM), wm) {
1007 wm.MaximumNumberofThreads =
1008 batch->blorp->isl_dev->info->max_wm_threads - 1;
1009
1010 switch (params->hiz_op) {
1011 case ISL_AUX_OP_FAST_CLEAR:
1012 wm.DepthBufferClear = true;
1013 break;
1014 case ISL_AUX_OP_FULL_RESOLVE:
1015 wm.DepthBufferResolveEnable = true;
1016 break;
1017 case ISL_AUX_OP_AMBIGUATE:
1018 wm.HierarchicalDepthBufferResolveEnable = true;
1019 break;
1020 case ISL_AUX_OP_NONE:
1021 break;
1022 default:
1023 unreachable("not reached");
1024 }
1025
1026 if (prog_data) {
1027 wm.ThreadDispatchEnable = true;
1028
1029 wm._8PixelDispatchEnable = prog_data->dispatch_8;
1030 wm._16PixelDispatchEnable = prog_data->dispatch_16;
1031 wm._32PixelDispatchEnable = prog_data->dispatch_32;
1032
1033 wm.DispatchGRFStartRegisterForConstantSetupData0 =
1034 brw_wm_prog_data_dispatch_grf_start_reg(prog_data, wm, 0);
1035 wm.DispatchGRFStartRegisterForConstantSetupData1 =
1036 brw_wm_prog_data_dispatch_grf_start_reg(prog_data, wm, 1);
1037 wm.DispatchGRFStartRegisterForConstantSetupData2 =
1038 brw_wm_prog_data_dispatch_grf_start_reg(prog_data, wm, 2);
1039
1040 wm.KernelStartPointer0 = params->wm_prog_kernel +
1041 brw_wm_prog_data_prog_offset(prog_data, wm, 0);
1042 wm.KernelStartPointer1 = params->wm_prog_kernel +
1043 brw_wm_prog_data_prog_offset(prog_data, wm, 1);
1044 wm.KernelStartPointer2 = params->wm_prog_kernel +
1045 brw_wm_prog_data_prog_offset(prog_data, wm, 2);
1046
1047 wm.NumberofSFOutputAttributes = prog_data->num_varying_inputs;
1048 }
1049
1050 if (params->src.enabled) {
1051 wm.SamplerCount = 1; /* Up to 4 samplers */
1052 wm.PixelShaderKillsPixel = true; /* TODO: temporarily smash on */
1053 }
1054
1055 if (params->num_samples > 1) {
1056 wm.MultisampleRasterizationMode = MSRASTMODE_ON_PATTERN;
1057 wm.MultisampleDispatchMode =
1058 (prog_data && prog_data->persample_dispatch) ?
1059 MSDISPMODE_PERSAMPLE : MSDISPMODE_PERPIXEL;
1060 } else {
1061 wm.MultisampleRasterizationMode = MSRASTMODE_OFF_PIXEL;
1062 wm.MultisampleDispatchMode = MSDISPMODE_PERSAMPLE;
1063 }
1064 }
1065
1066 #endif /* GEN_GEN */
1067 }
1068
1069 static uint32_t
blorp_emit_blend_state(struct blorp_batch * batch,const struct blorp_params * params)1070 blorp_emit_blend_state(struct blorp_batch *batch,
1071 const struct blorp_params *params)
1072 {
1073 struct GENX(BLEND_STATE) blend;
1074 memset(&blend, 0, sizeof(blend));
1075
1076 uint32_t offset;
1077 int size = GENX(BLEND_STATE_length) * 4;
1078 size += GENX(BLEND_STATE_ENTRY_length) * 4 * params->num_draw_buffers;
1079 uint32_t *state = blorp_alloc_dynamic_state(batch, size, 64, &offset);
1080 uint32_t *pos = state;
1081
1082 GENX(BLEND_STATE_pack)(NULL, pos, &blend);
1083 pos += GENX(BLEND_STATE_length);
1084
1085 for (unsigned i = 0; i < params->num_draw_buffers; ++i) {
1086 struct GENX(BLEND_STATE_ENTRY) entry = {
1087 .PreBlendColorClampEnable = true,
1088 .PostBlendColorClampEnable = true,
1089 .ColorClampRange = COLORCLAMP_RTFORMAT,
1090
1091 .WriteDisableRed = params->color_write_disable[0],
1092 .WriteDisableGreen = params->color_write_disable[1],
1093 .WriteDisableBlue = params->color_write_disable[2],
1094 .WriteDisableAlpha = params->color_write_disable[3],
1095 };
1096 GENX(BLEND_STATE_ENTRY_pack)(NULL, pos, &entry);
1097 pos += GENX(BLEND_STATE_ENTRY_length);
1098 }
1099
1100 blorp_flush_range(batch, state, size);
1101
1102 #if GEN_GEN >= 7
1103 blorp_emit(batch, GENX(3DSTATE_BLEND_STATE_POINTERS), sp) {
1104 sp.BlendStatePointer = offset;
1105 #if GEN_GEN >= 8
1106 sp.BlendStatePointerValid = true;
1107 #endif
1108 }
1109 #endif
1110
1111 #if GEN_GEN >= 8
1112 blorp_emit(batch, GENX(3DSTATE_PS_BLEND), ps_blend) {
1113 ps_blend.HasWriteableRT = true;
1114 }
1115 #endif
1116
1117 return offset;
1118 }
1119
1120 static uint32_t
blorp_emit_color_calc_state(struct blorp_batch * batch,UNUSED const struct blorp_params * params)1121 blorp_emit_color_calc_state(struct blorp_batch *batch,
1122 UNUSED const struct blorp_params *params)
1123 {
1124 uint32_t offset;
1125 blorp_emit_dynamic(batch, GENX(COLOR_CALC_STATE), cc, 64, &offset) {
1126 #if GEN_GEN <= 8
1127 cc.StencilReferenceValue = params->stencil_ref;
1128 #endif
1129 }
1130
1131 #if GEN_GEN >= 7
1132 blorp_emit(batch, GENX(3DSTATE_CC_STATE_POINTERS), sp) {
1133 sp.ColorCalcStatePointer = offset;
1134 #if GEN_GEN >= 8
1135 sp.ColorCalcStatePointerValid = true;
1136 #endif
1137 }
1138 #endif
1139
1140 return offset;
1141 }
1142
1143 static uint32_t
blorp_emit_depth_stencil_state(struct blorp_batch * batch,const struct blorp_params * params)1144 blorp_emit_depth_stencil_state(struct blorp_batch *batch,
1145 const struct blorp_params *params)
1146 {
1147 #if GEN_GEN >= 8
1148 struct GENX(3DSTATE_WM_DEPTH_STENCIL) ds = {
1149 GENX(3DSTATE_WM_DEPTH_STENCIL_header),
1150 };
1151 #else
1152 struct GENX(DEPTH_STENCIL_STATE) ds = { 0 };
1153 #endif
1154
1155 if (params->depth.enabled) {
1156 ds.DepthBufferWriteEnable = true;
1157
1158 switch (params->hiz_op) {
1159 /* See the following sections of the Sandy Bridge PRM, Volume 2, Part1:
1160 * - 7.5.3.1 Depth Buffer Clear
1161 * - 7.5.3.2 Depth Buffer Resolve
1162 * - 7.5.3.3 Hierarchical Depth Buffer Resolve
1163 */
1164 case ISL_AUX_OP_FULL_RESOLVE:
1165 ds.DepthTestEnable = true;
1166 ds.DepthTestFunction = COMPAREFUNCTION_NEVER;
1167 break;
1168
1169 case ISL_AUX_OP_NONE:
1170 case ISL_AUX_OP_FAST_CLEAR:
1171 case ISL_AUX_OP_AMBIGUATE:
1172 ds.DepthTestEnable = false;
1173 break;
1174 case ISL_AUX_OP_PARTIAL_RESOLVE:
1175 unreachable("Invalid HIZ op");
1176 }
1177 }
1178
1179 if (params->stencil.enabled) {
1180 ds.StencilBufferWriteEnable = true;
1181 ds.StencilTestEnable = true;
1182 ds.DoubleSidedStencilEnable = false;
1183
1184 ds.StencilTestFunction = COMPAREFUNCTION_ALWAYS;
1185 ds.StencilPassDepthPassOp = STENCILOP_REPLACE;
1186
1187 ds.StencilWriteMask = params->stencil_mask;
1188 #if GEN_GEN >= 9
1189 ds.StencilReferenceValue = params->stencil_ref;
1190 #endif
1191 }
1192
1193 #if GEN_GEN >= 8
1194 uint32_t offset = 0;
1195 uint32_t *dw = blorp_emit_dwords(batch,
1196 GENX(3DSTATE_WM_DEPTH_STENCIL_length));
1197 if (!dw)
1198 return 0;
1199
1200 GENX(3DSTATE_WM_DEPTH_STENCIL_pack)(NULL, dw, &ds);
1201 #else
1202 uint32_t offset;
1203 void *state = blorp_alloc_dynamic_state(batch,
1204 GENX(DEPTH_STENCIL_STATE_length) * 4,
1205 64, &offset);
1206 GENX(DEPTH_STENCIL_STATE_pack)(NULL, state, &ds);
1207 blorp_flush_range(batch, state, GENX(DEPTH_STENCIL_STATE_length) * 4);
1208 #endif
1209
1210 #if GEN_GEN == 7
1211 blorp_emit(batch, GENX(3DSTATE_DEPTH_STENCIL_STATE_POINTERS), sp) {
1212 sp.PointertoDEPTH_STENCIL_STATE = offset;
1213 }
1214 #endif
1215
1216 return offset;
1217 }
1218
1219 static void
blorp_emit_3dstate_multisample(struct blorp_batch * batch,const struct blorp_params * params)1220 blorp_emit_3dstate_multisample(struct blorp_batch *batch,
1221 const struct blorp_params *params)
1222 {
1223 blorp_emit(batch, GENX(3DSTATE_MULTISAMPLE), ms) {
1224 ms.NumberofMultisamples = __builtin_ffs(params->num_samples) - 1;
1225
1226 #if GEN_GEN >= 8
1227 /* The PRM says that this bit is valid only for DX9:
1228 *
1229 * SW can choose to set this bit only for DX9 API. DX10/OGL API's
1230 * should not have any effect by setting or not setting this bit.
1231 */
1232 ms.PixelPositionOffsetEnable = false;
1233 #elif GEN_GEN >= 7
1234
1235 switch (params->num_samples) {
1236 case 1:
1237 GEN_SAMPLE_POS_1X(ms.Sample);
1238 break;
1239 case 2:
1240 GEN_SAMPLE_POS_2X(ms.Sample);
1241 break;
1242 case 4:
1243 GEN_SAMPLE_POS_4X(ms.Sample);
1244 break;
1245 case 8:
1246 GEN_SAMPLE_POS_8X(ms.Sample);
1247 break;
1248 default:
1249 break;
1250 }
1251 #else
1252 GEN_SAMPLE_POS_4X(ms.Sample);
1253 #endif
1254 ms.PixelLocation = CENTER;
1255 }
1256 }
1257
1258 static void
blorp_emit_pipeline(struct blorp_batch * batch,const struct blorp_params * params)1259 blorp_emit_pipeline(struct blorp_batch *batch,
1260 const struct blorp_params *params)
1261 {
1262 uint32_t blend_state_offset = 0;
1263 uint32_t color_calc_state_offset;
1264 uint32_t depth_stencil_state_offset;
1265
1266 enum gen_urb_deref_block_size urb_deref_block_size;
1267 emit_urb_config(batch, params, &urb_deref_block_size);
1268
1269 if (params->wm_prog_data) {
1270 blend_state_offset = blorp_emit_blend_state(batch, params);
1271 }
1272 color_calc_state_offset = blorp_emit_color_calc_state(batch, params);
1273 depth_stencil_state_offset = blorp_emit_depth_stencil_state(batch, params);
1274
1275 #if GEN_GEN == 6
1276 /* 3DSTATE_CC_STATE_POINTERS
1277 *
1278 * The pointer offsets are relative to
1279 * CMD_STATE_BASE_ADDRESS.DynamicStateBaseAddress.
1280 *
1281 * The HiZ op doesn't use BLEND_STATE or COLOR_CALC_STATE.
1282 *
1283 * The dynamic state emit helpers emit their own STATE_POINTERS packets on
1284 * gen7+. However, on gen6 and earlier, they're all lumpped together in
1285 * one CC_STATE_POINTERS packet so we have to emit that here.
1286 */
1287 blorp_emit(batch, GENX(3DSTATE_CC_STATE_POINTERS), cc) {
1288 cc.BLEND_STATEChange = true;
1289 cc.ColorCalcStatePointerValid = true;
1290 cc.DEPTH_STENCIL_STATEChange = true;
1291 cc.PointertoBLEND_STATE = blend_state_offset;
1292 cc.ColorCalcStatePointer = color_calc_state_offset;
1293 cc.PointertoDEPTH_STENCIL_STATE = depth_stencil_state_offset;
1294 }
1295 #else
1296 (void)blend_state_offset;
1297 (void)color_calc_state_offset;
1298 (void)depth_stencil_state_offset;
1299 #endif
1300
1301 #if GEN_GEN >= 12
1302 blorp_emit(batch, GENX(3DSTATE_CONSTANT_ALL), pc) {
1303 /* Update empty push constants for all stages (bitmask = 11111b) */
1304 pc.ShaderUpdateEnable = 0x1f;
1305 }
1306 #else
1307 blorp_emit(batch, GENX(3DSTATE_CONSTANT_VS), vs);
1308 #if GEN_GEN >= 7
1309 blorp_emit(batch, GENX(3DSTATE_CONSTANT_HS), hs);
1310 blorp_emit(batch, GENX(3DSTATE_CONSTANT_DS), DS);
1311 #endif
1312 blorp_emit(batch, GENX(3DSTATE_CONSTANT_GS), gs);
1313 blorp_emit(batch, GENX(3DSTATE_CONSTANT_PS), ps);
1314 #endif
1315
1316 if (params->src.enabled)
1317 blorp_emit_sampler_state(batch);
1318
1319 blorp_emit_3dstate_multisample(batch, params);
1320
1321 blorp_emit(batch, GENX(3DSTATE_SAMPLE_MASK), mask) {
1322 mask.SampleMask = (1 << params->num_samples) - 1;
1323 }
1324
1325 /* From the BSpec, 3D Pipeline > Geometry > Vertex Shader > State,
1326 * 3DSTATE_VS, Dword 5.0 "VS Function Enable":
1327 *
1328 * [DevSNB] A pipeline flush must be programmed prior to a
1329 * 3DSTATE_VS command that causes the VS Function Enable to
1330 * toggle. Pipeline flush can be executed by sending a PIPE_CONTROL
1331 * command with CS stall bit set and a post sync operation.
1332 *
1333 * We've already done one at the start of the BLORP operation.
1334 */
1335 blorp_emit_vs_config(batch, params);
1336 #if GEN_GEN >= 7
1337 blorp_emit(batch, GENX(3DSTATE_HS), hs);
1338 blorp_emit(batch, GENX(3DSTATE_TE), te);
1339 blorp_emit(batch, GENX(3DSTATE_DS), DS);
1340 blorp_emit(batch, GENX(3DSTATE_STREAMOUT), so);
1341 #endif
1342 blorp_emit(batch, GENX(3DSTATE_GS), gs);
1343
1344 blorp_emit(batch, GENX(3DSTATE_CLIP), clip) {
1345 clip.PerspectiveDivideDisable = true;
1346 }
1347
1348 blorp_emit_sf_config(batch, params, urb_deref_block_size);
1349 blorp_emit_ps_config(batch, params);
1350
1351 blorp_emit_cc_viewport(batch);
1352
1353 #if GEN_GEN >= 12
1354 /* Disable Primitive Replication. */
1355 blorp_emit(batch, GENX(3DSTATE_PRIMITIVE_REPLICATION), pr);
1356 #endif
1357 }
1358
1359 /******** This is the end of the pipeline setup code ********/
1360
1361 #endif /* GEN_GEN >= 6 */
1362
1363 #if GEN_GEN >= 7
1364 static void
blorp_emit_memcpy(struct blorp_batch * batch,struct blorp_address dst,struct blorp_address src,uint32_t size)1365 blorp_emit_memcpy(struct blorp_batch *batch,
1366 struct blorp_address dst,
1367 struct blorp_address src,
1368 uint32_t size)
1369 {
1370 assert(size % 4 == 0);
1371
1372 for (unsigned dw = 0; dw < size; dw += 4) {
1373 #if GEN_GEN >= 8
1374 blorp_emit(batch, GENX(MI_COPY_MEM_MEM), cp) {
1375 cp.DestinationMemoryAddress = dst;
1376 cp.SourceMemoryAddress = src;
1377 }
1378 #else
1379 /* IVB does not have a general purpose register for command streamer
1380 * commands. Therefore, we use an alternate temporary register.
1381 */
1382 #define BLORP_TEMP_REG 0x2440 /* GEN7_3DPRIM_BASE_VERTEX */
1383 blorp_emit(batch, GENX(MI_LOAD_REGISTER_MEM), load) {
1384 load.RegisterAddress = BLORP_TEMP_REG;
1385 load.MemoryAddress = src;
1386 }
1387 blorp_emit(batch, GENX(MI_STORE_REGISTER_MEM), store) {
1388 store.RegisterAddress = BLORP_TEMP_REG;
1389 store.MemoryAddress = dst;
1390 }
1391 #undef BLORP_TEMP_REG
1392 #endif
1393 dst.offset += 4;
1394 src.offset += 4;
1395 }
1396 }
1397 #endif
1398
1399 static void
blorp_emit_surface_state(struct blorp_batch * batch,const struct brw_blorp_surface_info * surface,UNUSED enum isl_aux_op aux_op,void * state,uint32_t state_offset,const bool color_write_disables[4],bool is_render_target)1400 blorp_emit_surface_state(struct blorp_batch *batch,
1401 const struct brw_blorp_surface_info *surface,
1402 UNUSED enum isl_aux_op aux_op,
1403 void *state, uint32_t state_offset,
1404 const bool color_write_disables[4],
1405 bool is_render_target)
1406 {
1407 const struct isl_device *isl_dev = batch->blorp->isl_dev;
1408 struct isl_surf surf = surface->surf;
1409
1410 if (surf.dim == ISL_SURF_DIM_1D &&
1411 surf.dim_layout == ISL_DIM_LAYOUT_GEN4_2D) {
1412 assert(surf.logical_level0_px.height == 1);
1413 surf.dim = ISL_SURF_DIM_2D;
1414 }
1415
1416 if (isl_aux_usage_has_hiz(surface->aux_usage)) {
1417 /* BLORP doesn't render with depth so we can't use HiZ */
1418 assert(!is_render_target);
1419 /* We can't reinterpret HiZ */
1420 assert(surface->surf.format == surface->view.format);
1421 }
1422 enum isl_aux_usage aux_usage = surface->aux_usage;
1423
1424 isl_channel_mask_t write_disable_mask = 0;
1425 if (is_render_target && GEN_GEN <= 5) {
1426 if (color_write_disables[0])
1427 write_disable_mask |= ISL_CHANNEL_RED_BIT;
1428 if (color_write_disables[1])
1429 write_disable_mask |= ISL_CHANNEL_GREEN_BIT;
1430 if (color_write_disables[2])
1431 write_disable_mask |= ISL_CHANNEL_BLUE_BIT;
1432 if (color_write_disables[3])
1433 write_disable_mask |= ISL_CHANNEL_ALPHA_BIT;
1434 }
1435
1436 const bool use_clear_address =
1437 GEN_GEN >= 10 && (surface->clear_color_addr.buffer != NULL);
1438
1439 isl_surf_fill_state(batch->blorp->isl_dev, state,
1440 .surf = &surf, .view = &surface->view,
1441 .aux_surf = &surface->aux_surf, .aux_usage = aux_usage,
1442 .address =
1443 blorp_get_surface_address(batch, surface->addr),
1444 .aux_address = aux_usage == ISL_AUX_USAGE_NONE ? 0 :
1445 blorp_get_surface_address(batch, surface->aux_addr),
1446 .clear_address = !use_clear_address ? 0 :
1447 blorp_get_surface_address(batch,
1448 surface->clear_color_addr),
1449 .mocs = surface->addr.mocs,
1450 .clear_color = surface->clear_color,
1451 .use_clear_address = use_clear_address,
1452 .write_disables = write_disable_mask);
1453
1454 blorp_surface_reloc(batch, state_offset + isl_dev->ss.addr_offset,
1455 surface->addr, 0);
1456
1457 if (aux_usage != ISL_AUX_USAGE_NONE) {
1458 /* On gen7 and prior, the bottom 12 bits of the MCS base address are
1459 * used to store other information. This should be ok, however, because
1460 * surface buffer addresses are always 4K page alinged.
1461 */
1462 assert((surface->aux_addr.offset & 0xfff) == 0);
1463 uint32_t *aux_addr = state + isl_dev->ss.aux_addr_offset;
1464 blorp_surface_reloc(batch, state_offset + isl_dev->ss.aux_addr_offset,
1465 surface->aux_addr, *aux_addr);
1466 }
1467
1468 if (aux_usage != ISL_AUX_USAGE_NONE && surface->clear_color_addr.buffer) {
1469 #if GEN_GEN >= 10
1470 assert((surface->clear_color_addr.offset & 0x3f) == 0);
1471 uint32_t *clear_addr = state + isl_dev->ss.clear_color_state_offset;
1472 blorp_surface_reloc(batch, state_offset +
1473 isl_dev->ss.clear_color_state_offset,
1474 surface->clear_color_addr, *clear_addr);
1475 #elif GEN_GEN >= 7
1476 /* Fast clears just whack the AUX surface and don't actually use the
1477 * clear color for anything. We can avoid the MI memcpy on that case.
1478 */
1479 if (aux_op != ISL_AUX_OP_FAST_CLEAR) {
1480 struct blorp_address dst_addr = blorp_get_surface_base_address(batch);
1481 dst_addr.offset += state_offset + isl_dev->ss.clear_value_offset;
1482 blorp_emit_memcpy(batch, dst_addr, surface->clear_color_addr,
1483 isl_dev->ss.clear_value_size);
1484 }
1485 #else
1486 unreachable("Fast clears are only supported on gen7+");
1487 #endif
1488 }
1489
1490 blorp_flush_range(batch, state, GENX(RENDER_SURFACE_STATE_length) * 4);
1491 }
1492
1493 static void
blorp_emit_null_surface_state(struct blorp_batch * batch,const struct brw_blorp_surface_info * surface,uint32_t * state)1494 blorp_emit_null_surface_state(struct blorp_batch *batch,
1495 const struct brw_blorp_surface_info *surface,
1496 uint32_t *state)
1497 {
1498 struct GENX(RENDER_SURFACE_STATE) ss = {
1499 .SurfaceType = SURFTYPE_NULL,
1500 .SurfaceFormat = ISL_FORMAT_R8G8B8A8_UNORM,
1501 .Width = surface->surf.logical_level0_px.width - 1,
1502 .Height = surface->surf.logical_level0_px.height - 1,
1503 .MIPCountLOD = surface->view.base_level,
1504 .MinimumArrayElement = surface->view.base_array_layer,
1505 .Depth = surface->view.array_len - 1,
1506 .RenderTargetViewExtent = surface->view.array_len - 1,
1507 #if GEN_GEN >= 6
1508 .NumberofMultisamples = ffs(surface->surf.samples) - 1,
1509 #endif
1510
1511 #if GEN_GEN >= 7
1512 .SurfaceArray = surface->surf.dim != ISL_SURF_DIM_3D,
1513 #endif
1514
1515 #if GEN_GEN >= 8
1516 .TileMode = YMAJOR,
1517 #else
1518 .TiledSurface = true,
1519 #endif
1520 };
1521
1522 GENX(RENDER_SURFACE_STATE_pack)(NULL, state, &ss);
1523
1524 blorp_flush_range(batch, state, GENX(RENDER_SURFACE_STATE_length) * 4);
1525 }
1526
1527 static void
blorp_emit_surface_states(struct blorp_batch * batch,const struct blorp_params * params)1528 blorp_emit_surface_states(struct blorp_batch *batch,
1529 const struct blorp_params *params)
1530 {
1531 const struct isl_device *isl_dev = batch->blorp->isl_dev;
1532 uint32_t bind_offset = 0, surface_offsets[2];
1533 void *surface_maps[2];
1534
1535 UNUSED bool has_indirect_clear_color = false;
1536 if (params->use_pre_baked_binding_table) {
1537 bind_offset = params->pre_baked_binding_table_offset;
1538 } else {
1539 unsigned num_surfaces = 1 + params->src.enabled;
1540 blorp_alloc_binding_table(batch, num_surfaces,
1541 isl_dev->ss.size, isl_dev->ss.align,
1542 &bind_offset, surface_offsets, surface_maps);
1543
1544 if (params->dst.enabled) {
1545 blorp_emit_surface_state(batch, ¶ms->dst,
1546 params->fast_clear_op,
1547 surface_maps[BLORP_RENDERBUFFER_BT_INDEX],
1548 surface_offsets[BLORP_RENDERBUFFER_BT_INDEX],
1549 params->color_write_disable, true);
1550 if (params->dst.clear_color_addr.buffer != NULL)
1551 has_indirect_clear_color = true;
1552 } else {
1553 assert(params->depth.enabled || params->stencil.enabled);
1554 const struct brw_blorp_surface_info *surface =
1555 params->depth.enabled ? ¶ms->depth : ¶ms->stencil;
1556 blorp_emit_null_surface_state(batch, surface,
1557 surface_maps[BLORP_RENDERBUFFER_BT_INDEX]);
1558 }
1559
1560 if (params->src.enabled) {
1561 blorp_emit_surface_state(batch, ¶ms->src,
1562 params->fast_clear_op,
1563 surface_maps[BLORP_TEXTURE_BT_INDEX],
1564 surface_offsets[BLORP_TEXTURE_BT_INDEX],
1565 NULL, false);
1566 if (params->src.clear_color_addr.buffer != NULL)
1567 has_indirect_clear_color = true;
1568 }
1569 }
1570
1571 #if GEN_GEN >= 7
1572 if (has_indirect_clear_color) {
1573 /* Updating a surface state object may require that the state cache be
1574 * invalidated. From the SKL PRM, Shared Functions -> State -> State
1575 * Caching:
1576 *
1577 * Whenever the RENDER_SURFACE_STATE object in memory pointed to by
1578 * the Binding Table Pointer (BTP) and Binding Table Index (BTI) is
1579 * modified [...], the L1 state cache must be invalidated to ensure
1580 * the new surface or sampler state is fetched from system memory.
1581 */
1582 blorp_emit(batch, GENX(PIPE_CONTROL), pipe) {
1583 pipe.StateCacheInvalidationEnable = true;
1584 #if GEN_GEN >= 12
1585 pipe.TileCacheFlushEnable = true;
1586 #endif
1587 }
1588 }
1589 #endif
1590
1591 #if GEN_GEN >= 7
1592 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS_VS), bt);
1593 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS_HS), bt);
1594 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS_DS), bt);
1595 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS_GS), bt);
1596
1597 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS_PS), bt) {
1598 bt.PointertoPSBindingTable = bind_offset;
1599 }
1600 #elif GEN_GEN >= 6
1601 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS), bt) {
1602 bt.PSBindingTableChange = true;
1603 bt.PointertoPSBindingTable = bind_offset;
1604 }
1605 #else
1606 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS), bt) {
1607 bt.PointertoPSBindingTable = bind_offset;
1608 }
1609 #endif
1610 }
1611
1612 static void
blorp_emit_depth_stencil_config(struct blorp_batch * batch,const struct blorp_params * params)1613 blorp_emit_depth_stencil_config(struct blorp_batch *batch,
1614 const struct blorp_params *params)
1615 {
1616 const struct isl_device *isl_dev = batch->blorp->isl_dev;
1617
1618 uint32_t *dw = blorp_emit_dwords(batch, isl_dev->ds.size / 4);
1619 if (dw == NULL)
1620 return;
1621
1622 struct isl_depth_stencil_hiz_emit_info info = { };
1623
1624 if (params->depth.enabled) {
1625 info.view = ¶ms->depth.view;
1626 info.mocs = params->depth.addr.mocs;
1627 } else if (params->stencil.enabled) {
1628 info.view = ¶ms->stencil.view;
1629 info.mocs = params->stencil.addr.mocs;
1630 }
1631
1632 if (params->depth.enabled) {
1633 info.depth_surf = ¶ms->depth.surf;
1634
1635 info.depth_address =
1636 blorp_emit_reloc(batch, dw + isl_dev->ds.depth_offset / 4,
1637 params->depth.addr, 0);
1638
1639 info.hiz_usage = params->depth.aux_usage;
1640 if (isl_aux_usage_has_hiz(info.hiz_usage)) {
1641 info.hiz_surf = ¶ms->depth.aux_surf;
1642
1643 struct blorp_address hiz_address = params->depth.aux_addr;
1644 #if GEN_GEN == 6
1645 /* Sandy bridge hardware does not technically support mipmapped HiZ.
1646 * However, we have a special layout that allows us to make it work
1647 * anyway by manually offsetting to the specified miplevel.
1648 */
1649 assert(info.hiz_surf->dim_layout == ISL_DIM_LAYOUT_GEN6_STENCIL_HIZ);
1650 uint32_t offset_B;
1651 isl_surf_get_image_offset_B_tile_sa(info.hiz_surf,
1652 info.view->base_level, 0, 0,
1653 &offset_B, NULL, NULL);
1654 hiz_address.offset += offset_B;
1655 #endif
1656
1657 info.hiz_address =
1658 blorp_emit_reloc(batch, dw + isl_dev->ds.hiz_offset / 4,
1659 hiz_address, 0);
1660
1661 info.depth_clear_value = params->depth.clear_color.f32[0];
1662 }
1663 }
1664
1665 if (params->stencil.enabled) {
1666 info.stencil_surf = ¶ms->stencil.surf;
1667
1668 info.stencil_aux_usage = params->stencil.aux_usage;
1669 struct blorp_address stencil_address = params->stencil.addr;
1670 #if GEN_GEN == 6
1671 /* Sandy bridge hardware does not technically support mipmapped stencil.
1672 * However, we have a special layout that allows us to make it work
1673 * anyway by manually offsetting to the specified miplevel.
1674 */
1675 assert(info.stencil_surf->dim_layout == ISL_DIM_LAYOUT_GEN6_STENCIL_HIZ);
1676 uint32_t offset_B;
1677 isl_surf_get_image_offset_B_tile_sa(info.stencil_surf,
1678 info.view->base_level, 0, 0,
1679 &offset_B, NULL, NULL);
1680 stencil_address.offset += offset_B;
1681 #endif
1682
1683 info.stencil_address =
1684 blorp_emit_reloc(batch, dw + isl_dev->ds.stencil_offset / 4,
1685 stencil_address, 0);
1686 }
1687
1688 isl_emit_depth_stencil_hiz_s(isl_dev, dw, &info);
1689
1690 #if GEN_GEN >= 12
1691 /* GEN:BUG:1408224581
1692 *
1693 * Workaround: Gen12LP Astep only An additional pipe control with
1694 * post-sync = store dword operation would be required.( w/a is to
1695 * have an additional pipe control after the stencil state whenever
1696 * the surface state bits of this state is changing).
1697 */
1698 blorp_emit(batch, GENX(PIPE_CONTROL), pc) {
1699 pc.PostSyncOperation = WriteImmediateData;
1700 pc.Address = blorp_get_workaround_address(batch);
1701 }
1702 #endif
1703 }
1704
1705 #if GEN_GEN >= 8
1706 /* Emits the Optimized HiZ sequence specified in the BDW+ PRMs. The
1707 * depth/stencil buffer extents are ignored to handle APIs which perform
1708 * clearing operations without such information.
1709 * */
1710 static void
blorp_emit_gen8_hiz_op(struct blorp_batch * batch,const struct blorp_params * params)1711 blorp_emit_gen8_hiz_op(struct blorp_batch *batch,
1712 const struct blorp_params *params)
1713 {
1714 /* We should be performing an operation on a depth or stencil buffer.
1715 */
1716 assert(params->depth.enabled || params->stencil.enabled);
1717
1718 /* The stencil buffer should only be enabled on GEN == 12, if a fast clear
1719 * or full resolve operation is requested. On rest of the GEN, if a fast
1720 * clear operation is requested.
1721 */
1722 if (params->stencil.enabled) {
1723 #if GEN_GEN >= 12
1724 assert(params->hiz_op == ISL_AUX_OP_FAST_CLEAR ||
1725 params->hiz_op == ISL_AUX_OP_FULL_RESOLVE);
1726 #else
1727 assert(params->hiz_op == ISL_AUX_OP_FAST_CLEAR);
1728 #endif
1729 }
1730
1731 /* From the BDW PRM Volume 2, 3DSTATE_WM_HZ_OP:
1732 *
1733 * 3DSTATE_MULTISAMPLE packet must be used prior to this packet to change
1734 * the Number of Multisamples. This packet must not be used to change
1735 * Number of Multisamples in a rendering sequence.
1736 *
1737 * Since HIZ may be the first thing in a batch buffer, play safe and always
1738 * emit 3DSTATE_MULTISAMPLE.
1739 */
1740 blorp_emit_3dstate_multisample(batch, params);
1741
1742 /* From the BDW PRM Volume 7, Depth Buffer Clear:
1743 *
1744 * The clear value must be between the min and max depth values
1745 * (inclusive) defined in the CC_VIEWPORT. If the depth buffer format is
1746 * D32_FLOAT, then +/-DENORM values are also allowed.
1747 *
1748 * Set the bounds to match our hardware limits, [0.0, 1.0].
1749 */
1750 if (params->depth.enabled && params->hiz_op == ISL_AUX_OP_FAST_CLEAR) {
1751 assert(params->depth.clear_color.f32[0] >= 0.0f);
1752 assert(params->depth.clear_color.f32[0] <= 1.0f);
1753 blorp_emit_cc_viewport(batch);
1754 }
1755
1756 if (GEN_GEN >= 12 && params->stencil.enabled &&
1757 params->hiz_op == ISL_AUX_OP_FULL_RESOLVE) {
1758 /* GEN:BUG:1605967699
1759 *
1760 * This workaround requires that the Force Thread Dispatch Enable flag
1761 * needs to be set to ForceOFF on the first WM_HZ_OP state cycle
1762 * (followed by a CS Stall):
1763 *
1764 * "Workaround: There is a potential software workaround for the
1765 * issue by doing these 2 steps 1) setting the force thread dispatch
1766 * enable(bits 20:19) in the 3dstate_WM_body state to be set to
1767 * Force_OFF (value of 1) along with the first WM_HZ_OP state cycle.
1768 * The second WM_HZ_OP state which is required by programming
1769 * sequencing to complete the HZ_OP operation can reprogram the
1770 * 3dstate_WM_body to set to NORMAL(value of 0)."
1771 */
1772 blorp_emit(batch, GENX(3DSTATE_WM), wm) {
1773 wm.ForceThreadDispatchEnable = ForceOff;
1774 }
1775 blorp_emit(batch, GENX(PIPE_CONTROL), pipe) {
1776 pipe.CommandStreamerStallEnable = true;
1777 }
1778 } else {
1779 /* According to the SKL PRM formula for WM_INT::ThreadDispatchEnable, the
1780 * 3DSTATE_WM::ForceThreadDispatchEnable field can force WM thread dispatch
1781 * even when WM_HZ_OP is active. However, WM thread dispatch is normally
1782 * disabled for HiZ ops and it appears that force-enabling it can lead to
1783 * GPU hangs on at least Skylake. Since we don't know the current state of
1784 * the 3DSTATE_WM packet, just emit a dummy one prior to 3DSTATE_WM_HZ_OP.
1785 */
1786 blorp_emit(batch, GENX(3DSTATE_WM), wm);
1787 }
1788
1789 /* If we can't alter the depth stencil config and multiple layers are
1790 * involved, the HiZ op will fail. This is because the op requires that a
1791 * new config is emitted for each additional layer.
1792 */
1793 if (batch->flags & BLORP_BATCH_NO_EMIT_DEPTH_STENCIL) {
1794 assert(params->num_layers <= 1);
1795 } else {
1796 blorp_emit_depth_stencil_config(batch, params);
1797 }
1798
1799 blorp_emit(batch, GENX(3DSTATE_WM_HZ_OP), hzp) {
1800 switch (params->hiz_op) {
1801 case ISL_AUX_OP_FAST_CLEAR:
1802 hzp.StencilBufferClearEnable = params->stencil.enabled;
1803 hzp.DepthBufferClearEnable = params->depth.enabled;
1804 hzp.StencilClearValue = params->stencil_ref;
1805 hzp.FullSurfaceDepthandStencilClear = params->full_surface_hiz_op;
1806 break;
1807 case ISL_AUX_OP_FULL_RESOLVE:
1808 assert(params->full_surface_hiz_op);
1809 hzp.DepthBufferResolveEnable = params->depth.enabled;
1810 #if GEN_GEN >= 12
1811 if (params->stencil.enabled) {
1812 assert(params->stencil.aux_usage == ISL_AUX_USAGE_STC_CCS);
1813 hzp.StencilBufferResolveEnable = true;
1814 }
1815 #endif
1816 break;
1817 case ISL_AUX_OP_AMBIGUATE:
1818 assert(params->full_surface_hiz_op);
1819 hzp.HierarchicalDepthBufferResolveEnable = true;
1820 break;
1821 case ISL_AUX_OP_PARTIAL_RESOLVE:
1822 case ISL_AUX_OP_NONE:
1823 unreachable("Invalid HIZ op");
1824 }
1825
1826 hzp.NumberofMultisamples = ffs(params->num_samples) - 1;
1827 hzp.SampleMask = 0xFFFF;
1828
1829 /* Due to a hardware issue, this bit MBZ */
1830 assert(hzp.ScissorRectangleEnable == false);
1831
1832 /* Contrary to the HW docs both fields are inclusive */
1833 hzp.ClearRectangleXMin = params->x0;
1834 hzp.ClearRectangleYMin = params->y0;
1835
1836 /* Contrary to the HW docs both fields are exclusive */
1837 hzp.ClearRectangleXMax = params->x1;
1838 hzp.ClearRectangleYMax = params->y1;
1839 }
1840
1841 /* PIPE_CONTROL w/ all bits clear except for “Post-Sync Operation” must set
1842 * to “Write Immediate Data” enabled.
1843 */
1844 blorp_emit(batch, GENX(PIPE_CONTROL), pc) {
1845 pc.PostSyncOperation = WriteImmediateData;
1846 pc.Address = blorp_get_workaround_address(batch);
1847 }
1848
1849
1850 if (GEN_GEN >= 12 && params->stencil.enabled &&
1851 params->hiz_op == ISL_AUX_OP_FULL_RESOLVE) {
1852 /* GEN:BUG:1605967699
1853 *
1854 * The second WM_HZ_OP state which is required by programming
1855 * sequencing to complete the HZ_OP operation can reprogram the
1856 * 3dstate_WM_body to set to NORMAL(value of 0)."
1857 */
1858 blorp_emit(batch, GENX(3DSTATE_WM), wm);
1859 }
1860
1861 blorp_emit(batch, GENX(3DSTATE_WM_HZ_OP), hzp);
1862 }
1863 #endif
1864
1865 static void
blorp_update_clear_color(UNUSED struct blorp_batch * batch,const struct brw_blorp_surface_info * info,enum isl_aux_op op)1866 blorp_update_clear_color(UNUSED struct blorp_batch *batch,
1867 const struct brw_blorp_surface_info *info,
1868 enum isl_aux_op op)
1869 {
1870 if (info->clear_color_addr.buffer && op == ISL_AUX_OP_FAST_CLEAR) {
1871 #if GEN_GEN == 11
1872 blorp_emit(batch, GENX(PIPE_CONTROL), pipe) {
1873 pipe.CommandStreamerStallEnable = true;
1874 }
1875
1876 /* 2 QWORDS */
1877 const unsigned inlinedata_dw = 2 * 2;
1878 const unsigned num_dwords = GENX(MI_ATOMIC_length) + inlinedata_dw;
1879
1880 struct blorp_address clear_addr = info->clear_color_addr;
1881 uint32_t *dw = blorp_emitn(batch, GENX(MI_ATOMIC), num_dwords,
1882 .DataSize = MI_ATOMIC_QWORD,
1883 .ATOMICOPCODE = MI_ATOMIC_OP_MOVE8B,
1884 .InlineData = true,
1885 .MemoryAddress = clear_addr);
1886 /* dw starts at dword 1, but we need to fill dwords 3 and 5 */
1887 dw[2] = info->clear_color.u32[0];
1888 dw[3] = 0;
1889 dw[4] = info->clear_color.u32[1];
1890 dw[5] = 0;
1891
1892 clear_addr.offset += 8;
1893 dw = blorp_emitn(batch, GENX(MI_ATOMIC), num_dwords,
1894 .DataSize = MI_ATOMIC_QWORD,
1895 .ATOMICOPCODE = MI_ATOMIC_OP_MOVE8B,
1896 .CSSTALL = true,
1897 .ReturnDataControl = true,
1898 .InlineData = true,
1899 .MemoryAddress = clear_addr);
1900 /* dw starts at dword 1, but we need to fill dwords 3 and 5 */
1901 dw[2] = info->clear_color.u32[2];
1902 dw[3] = 0;
1903 dw[4] = info->clear_color.u32[3];
1904 dw[5] = 0;
1905
1906 blorp_emit(batch, GENX(PIPE_CONTROL), pipe) {
1907 pipe.StateCacheInvalidationEnable = true;
1908 pipe.TextureCacheInvalidationEnable = true;
1909 }
1910 #elif GEN_GEN >= 9
1911
1912 /* According to GEN:BUG:2201730850, in the Clear Color Programming Note
1913 * under the Red channel, "Software shall write the converted Depth
1914 * Clear to this dword." The only depth formats listed under the red
1915 * channel are IEEE_FP and UNORM24_X8. These two requirements are
1916 * incompatible with the UNORM16 depth format, so just ignore that case
1917 * and simply perform the conversion for all depth formats.
1918 */
1919 union isl_color_value fixed_color = info->clear_color;
1920 if (GEN_GEN == 12 && isl_surf_usage_is_depth(info->surf.usage)) {
1921 isl_color_value_pack(&info->clear_color, info->surf.format,
1922 fixed_color.u32);
1923 }
1924
1925 for (int i = 0; i < 4; i++) {
1926 blorp_emit(batch, GENX(MI_STORE_DATA_IMM), sdi) {
1927 sdi.Address = info->clear_color_addr;
1928 sdi.Address.offset += i * 4;
1929 sdi.ImmediateData = fixed_color.u32[i];
1930 #if GEN_GEN >= 12
1931 if (i == 3)
1932 sdi.ForceWriteCompletionCheck = true;
1933 #endif
1934 }
1935 }
1936
1937 /* The RENDER_SURFACE_STATE::ClearColor field states that software should
1938 * write the converted depth value 16B after the clear address:
1939 *
1940 * 3D Sampler will always fetch clear depth from the location 16-bytes
1941 * above this address, where the clear depth, converted to native
1942 * surface format by software, will be stored.
1943 *
1944 */
1945 #if GEN_GEN >= 12
1946 if (isl_surf_usage_is_depth(info->surf.usage)) {
1947 blorp_emit(batch, GENX(MI_STORE_DATA_IMM), sdi) {
1948 sdi.Address = info->clear_color_addr;
1949 sdi.Address.offset += 4 * 4;
1950 sdi.ImmediateData = fixed_color.u32[0];
1951 sdi.ForceWriteCompletionCheck = true;
1952 }
1953 }
1954 #endif
1955
1956 #elif GEN_GEN >= 7
1957 blorp_emit(batch, GENX(MI_STORE_DATA_IMM), sdi) {
1958 sdi.Address = info->clear_color_addr;
1959 sdi.ImmediateData = ISL_CHANNEL_SELECT_RED << 25 |
1960 ISL_CHANNEL_SELECT_GREEN << 22 |
1961 ISL_CHANNEL_SELECT_BLUE << 19 |
1962 ISL_CHANNEL_SELECT_ALPHA << 16;
1963 if (isl_format_has_int_channel(info->view.format)) {
1964 for (unsigned i = 0; i < 4; i++) {
1965 assert(info->clear_color.u32[i] == 0 ||
1966 info->clear_color.u32[i] == 1);
1967 }
1968 sdi.ImmediateData |= (info->clear_color.u32[0] != 0) << 31;
1969 sdi.ImmediateData |= (info->clear_color.u32[1] != 0) << 30;
1970 sdi.ImmediateData |= (info->clear_color.u32[2] != 0) << 29;
1971 sdi.ImmediateData |= (info->clear_color.u32[3] != 0) << 28;
1972 } else {
1973 for (unsigned i = 0; i < 4; i++) {
1974 assert(info->clear_color.f32[i] == 0.0f ||
1975 info->clear_color.f32[i] == 1.0f);
1976 }
1977 sdi.ImmediateData |= (info->clear_color.f32[0] != 0.0f) << 31;
1978 sdi.ImmediateData |= (info->clear_color.f32[1] != 0.0f) << 30;
1979 sdi.ImmediateData |= (info->clear_color.f32[2] != 0.0f) << 29;
1980 sdi.ImmediateData |= (info->clear_color.f32[3] != 0.0f) << 28;
1981 }
1982 }
1983 #endif
1984 }
1985 }
1986
1987 /**
1988 * \brief Execute a blit or render pass operation.
1989 *
1990 * To execute the operation, this function manually constructs and emits a
1991 * batch to draw a rectangle primitive. The batchbuffer is flushed before
1992 * constructing and after emitting the batch.
1993 *
1994 * This function alters no GL state.
1995 */
1996 static void
blorp_exec(struct blorp_batch * batch,const struct blorp_params * params)1997 blorp_exec(struct blorp_batch *batch, const struct blorp_params *params)
1998 {
1999 if (!(batch->flags & BLORP_BATCH_NO_UPDATE_CLEAR_COLOR)) {
2000 blorp_update_clear_color(batch, ¶ms->dst, params->fast_clear_op);
2001 blorp_update_clear_color(batch, ¶ms->depth, params->hiz_op);
2002 }
2003
2004 #if GEN_GEN >= 8
2005 if (params->hiz_op != ISL_AUX_OP_NONE) {
2006 blorp_emit_gen8_hiz_op(batch, params);
2007 return;
2008 }
2009 #endif
2010
2011 blorp_emit_vertex_buffers(batch, params);
2012 blorp_emit_vertex_elements(batch, params);
2013
2014 blorp_emit_pipeline(batch, params);
2015
2016 blorp_emit_surface_states(batch, params);
2017
2018 if (!(batch->flags & BLORP_BATCH_NO_EMIT_DEPTH_STENCIL))
2019 blorp_emit_depth_stencil_config(batch, params);
2020
2021 blorp_emit(batch, GENX(3DPRIMITIVE), prim) {
2022 prim.VertexAccessType = SEQUENTIAL;
2023 prim.PrimitiveTopologyType = _3DPRIM_RECTLIST;
2024 #if GEN_GEN >= 7
2025 prim.PredicateEnable = batch->flags & BLORP_BATCH_PREDICATE_ENABLE;
2026 #endif
2027 prim.VertexCountPerInstance = 3;
2028 prim.InstanceCount = params->num_layers;
2029 }
2030 }
2031
2032 #endif /* BLORP_GENX_EXEC_H */
2033