1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * SPDX-License-Identifier: MIT
5 */
6
7 #include "si_build_pm4.h"
8 #include "si_query.h"
9 #include "si_shader_internal.h"
10 #include "sid.h"
11 #include "util/fast_idiv_by_const.h"
12 #include "util/format/u_format.h"
13 #include "util/format/u_format_s3tc.h"
14 #include "util/hash_table.h"
15 #include "util/u_dual_blend.h"
16 #include "util/u_helpers.h"
17 #include "util/u_memory.h"
18 #include "util/u_resource.h"
19 #include "util/u_upload_mgr.h"
20 #include "util/u_blend.h"
21
22 #include "ac_cmdbuf.h"
23 #include "ac_descriptors.h"
24 #include "ac_formats.h"
25 #include "gfx10_format_table.h"
26
27 /* 12.4 fixed-point */
si_pack_float_12p4(float x)28 static unsigned si_pack_float_12p4(float x)
29 {
30 return x <= 0 ? 0 : x >= 4096 ? 0xffff : x * 16;
31 }
32
33 /*
34 * Inferred framebuffer and blender state.
35 *
36 * CB_TARGET_MASK is emitted here to avoid a hang with dual source blending
37 * if there is not enough PS outputs.
38 */
si_emit_cb_render_state(struct si_context * sctx,unsigned index)39 static void si_emit_cb_render_state(struct si_context *sctx, unsigned index)
40 {
41 struct radeon_cmdbuf *cs = &sctx->gfx_cs;
42 struct si_state_blend *blend = sctx->queued.named.blend;
43 /* CB_COLORn_INFO.FORMAT=INVALID should disable unbound colorbuffers,
44 * but you never know. */
45 uint32_t cb_target_mask = sctx->framebuffer.colorbuf_enabled_4bit & blend->cb_target_mask;
46 unsigned i;
47
48 /* Avoid a hang that happens when dual source blending is enabled
49 * but there is not enough color outputs. This is undefined behavior,
50 * so disable color writes completely.
51 *
52 * Reproducible with Unigine Heaven 4.0 and drirc missing.
53 */
54 if (blend->dual_src_blend && sctx->shader.ps.cso &&
55 (sctx->shader.ps.cso->info.colors_written & 0x3) != 0x3)
56 cb_target_mask = 0;
57
58 /* GFX9: Flush DFSM when CB_TARGET_MASK changes.
59 * I think we don't have to do anything between IBs.
60 */
61 if (sctx->screen->dpbb_allowed && sctx->last_cb_target_mask != cb_target_mask &&
62 sctx->screen->pbb_context_states_per_bin > 1) {
63 sctx->last_cb_target_mask = cb_target_mask;
64
65 radeon_begin(cs);
66 radeon_event_write(V_028A90_BREAK_BATCH);
67 radeon_end();
68 }
69
70 uint32_t cb_dcc_control = 0;
71
72 if (sctx->gfx_level >= GFX8 && sctx->gfx_level < GFX12) {
73 /* DCC MSAA workaround.
74 * Alternatively, we can set CB_COLORi_DCC_CONTROL.OVERWRITE_-
75 * COMBINER_DISABLE, but that would be more complicated.
76 */
77 bool oc_disable =
78 blend->dcc_msaa_corruption_4bit & cb_target_mask && sctx->framebuffer.nr_samples >= 2;
79
80 if (sctx->gfx_level >= GFX11) {
81 cb_dcc_control =
82 S_028424_SAMPLE_MASK_TRACKER_DISABLE(oc_disable) |
83 S_028424_SAMPLE_MASK_TRACKER_WATERMARK(sctx->screen->info.has_dedicated_vram ? 0 : 15);
84 } else {
85 cb_dcc_control =
86 S_028424_OVERWRITE_COMBINER_MRT_SHARING_DISABLE(sctx->gfx_level <= GFX9) |
87 S_028424_OVERWRITE_COMBINER_WATERMARK(sctx->gfx_level >= GFX10 ? 6 : 4) |
88 S_028424_OVERWRITE_COMBINER_DISABLE(oc_disable) |
89 S_028424_DISABLE_CONSTANT_ENCODE_REG(sctx->gfx_level < GFX11 &&
90 sctx->screen->info.has_dcc_constant_encode);
91 }
92 }
93
94 uint32_t sx_ps_downconvert = 0;
95 uint32_t sx_blend_opt_epsilon = 0;
96 uint32_t sx_blend_opt_control = 0;
97
98 /* RB+ register settings. */
99 if (sctx->screen->info.rbplus_allowed) {
100 unsigned spi_shader_col_format =
101 sctx->shader.ps.cso ? sctx->shader.ps.current->key.ps.part.epilog.spi_shader_col_format
102 : 0;
103 unsigned num_cbufs = util_last_bit(sctx->framebuffer.colorbuf_enabled_4bit &
104 blend->cb_target_enabled_4bit) / 4;
105
106 for (i = 0; i < num_cbufs; i++) {
107 struct si_surface *surf = (struct si_surface *)sctx->framebuffer.state.cbufs[i];
108 unsigned format, swap, spi_format, colormask;
109 bool has_alpha, has_rgb;
110
111 if (!surf) {
112 /* If the color buffer is not set, the driver sets 32_R
113 * as the SPI color format, because the hw doesn't allow
114 * holes between color outputs, so also set this to
115 * enable RB+.
116 */
117 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_32_R << (i * 4);
118 continue;
119 }
120
121 format = sctx->gfx_level >= GFX11 ? G_028C70_FORMAT_GFX11(surf->cb.cb_color_info):
122 G_028C70_FORMAT_GFX6(surf->cb.cb_color_info);
123 swap = G_028C70_COMP_SWAP(surf->cb.cb_color_info);
124 spi_format = (spi_shader_col_format >> (i * 4)) & 0xf;
125 colormask = (cb_target_mask >> (i * 4)) & 0xf;
126
127 /* Set if RGB and A are present. */
128 has_alpha = !(sctx->gfx_level >= GFX11 ? G_028C74_FORCE_DST_ALPHA_1_GFX11(surf->cb.cb_color_attrib):
129 G_028C74_FORCE_DST_ALPHA_1_GFX6(surf->cb.cb_color_attrib));
130
131 if (format == V_028C70_COLOR_8 || format == V_028C70_COLOR_16 ||
132 format == V_028C70_COLOR_32)
133 has_rgb = !has_alpha;
134 else
135 has_rgb = true;
136
137 /* Check the colormask and export format. */
138 if (!(colormask & (PIPE_MASK_RGBA & ~PIPE_MASK_A)))
139 has_rgb = false;
140 if (!(colormask & PIPE_MASK_A))
141 has_alpha = false;
142
143 if (spi_format == V_028714_SPI_SHADER_ZERO) {
144 has_rgb = false;
145 has_alpha = false;
146 }
147
148 /* Disable value checking for disabled channels. */
149 if (!has_rgb)
150 sx_blend_opt_control |= S_02875C_MRT0_COLOR_OPT_DISABLE(1) << (i * 4);
151 if (!has_alpha)
152 sx_blend_opt_control |= S_02875C_MRT0_ALPHA_OPT_DISABLE(1) << (i * 4);
153
154 /* Enable down-conversion for 32bpp and smaller formats. */
155 switch (format) {
156 case V_028C70_COLOR_8:
157 case V_028C70_COLOR_8_8:
158 case V_028C70_COLOR_8_8_8_8:
159 /* For 1 and 2-channel formats, use the superset thereof. */
160 if (spi_format == V_028714_SPI_SHADER_FP16_ABGR ||
161 spi_format == V_028714_SPI_SHADER_UINT16_ABGR ||
162 spi_format == V_028714_SPI_SHADER_SINT16_ABGR) {
163 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_8_8_8_8 << (i * 4);
164 if (G_028C70_NUMBER_TYPE(surf->cb.cb_color_info) != V_028C70_NUMBER_SRGB)
165 sx_blend_opt_epsilon |= V_028758_8BIT_FORMAT_0_5 << (i * 4);
166 }
167 break;
168
169 case V_028C70_COLOR_5_6_5:
170 if (spi_format == V_028714_SPI_SHADER_FP16_ABGR) {
171 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_5_6_5 << (i * 4);
172 sx_blend_opt_epsilon |= V_028758_6BIT_FORMAT_0_5 << (i * 4);
173 }
174 break;
175
176 case V_028C70_COLOR_1_5_5_5:
177 if (spi_format == V_028714_SPI_SHADER_FP16_ABGR) {
178 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_1_5_5_5 << (i * 4);
179 sx_blend_opt_epsilon |= V_028758_5BIT_FORMAT_0_5 << (i * 4);
180 }
181 break;
182
183 case V_028C70_COLOR_4_4_4_4:
184 if (spi_format == V_028714_SPI_SHADER_FP16_ABGR) {
185 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_4_4_4_4 << (i * 4);
186 sx_blend_opt_epsilon |= V_028758_4BIT_FORMAT_0_5 << (i * 4);
187 }
188 break;
189
190 case V_028C70_COLOR_32:
191 if (swap == V_028C70_SWAP_STD && spi_format == V_028714_SPI_SHADER_32_R)
192 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_32_R << (i * 4);
193 else if (swap == V_028C70_SWAP_ALT_REV && spi_format == V_028714_SPI_SHADER_32_AR)
194 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_32_A << (i * 4);
195 break;
196
197 case V_028C70_COLOR_16:
198 case V_028C70_COLOR_16_16:
199 /* For 1-channel formats, use the superset thereof. */
200 if (spi_format == V_028714_SPI_SHADER_UNORM16_ABGR ||
201 spi_format == V_028714_SPI_SHADER_SNORM16_ABGR ||
202 spi_format == V_028714_SPI_SHADER_UINT16_ABGR ||
203 spi_format == V_028714_SPI_SHADER_SINT16_ABGR) {
204 if (swap == V_028C70_SWAP_STD || swap == V_028C70_SWAP_STD_REV)
205 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_16_16_GR << (i * 4);
206 else
207 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_16_16_AR << (i * 4);
208 }
209 break;
210
211 case V_028C70_COLOR_10_11_11:
212 if (spi_format == V_028714_SPI_SHADER_FP16_ABGR)
213 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_10_11_11 << (i * 4);
214 break;
215
216 case V_028C70_COLOR_2_10_10_10:
217 case V_028C70_COLOR_10_10_10_2:
218 if (spi_format == V_028714_SPI_SHADER_FP16_ABGR) {
219 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_2_10_10_10 << (i * 4);
220 sx_blend_opt_epsilon |= V_028758_10BIT_FORMAT_0_5 << (i * 4);
221 }
222 break;
223
224 case V_028C70_COLOR_5_9_9_9:
225 if (spi_format == V_028714_SPI_SHADER_FP16_ABGR)
226 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_9_9_9_E5 << (i * 4);
227 break;
228 }
229 }
230
231 /* If there are no color outputs, the first color export is
232 * always enabled as 32_R, so also set this to enable RB+.
233 */
234 if (!sx_ps_downconvert)
235 sx_ps_downconvert = V_028754_SX_RT_EXPORT_32_R;
236 }
237
238 if (sctx->gfx_level >= GFX12) {
239 /* GFX12 doesn't have CB_FDCC_CONTROL. */
240 assert(cb_dcc_control == 0);
241
242 radeon_begin(cs);
243 gfx12_begin_context_regs();
244 gfx12_opt_set_context_reg(R_028850_CB_TARGET_MASK, SI_TRACKED_CB_TARGET_MASK,
245 cb_target_mask);
246 gfx12_opt_set_context_reg(R_028754_SX_PS_DOWNCONVERT, SI_TRACKED_SX_PS_DOWNCONVERT,
247 sx_ps_downconvert);
248 gfx12_opt_set_context_reg(R_028758_SX_BLEND_OPT_EPSILON, SI_TRACKED_SX_BLEND_OPT_EPSILON,
249 sx_blend_opt_epsilon);
250 gfx12_opt_set_context_reg(R_02875C_SX_BLEND_OPT_CONTROL, SI_TRACKED_SX_BLEND_OPT_CONTROL,
251 sx_blend_opt_control);
252 gfx12_end_context_regs();
253 radeon_end(); /* don't track context rolls on GFX12 */
254 } else if (sctx->screen->info.has_set_context_pairs_packed) {
255 radeon_begin(cs);
256 gfx11_begin_packed_context_regs();
257 gfx11_opt_set_context_reg(R_028238_CB_TARGET_MASK, SI_TRACKED_CB_TARGET_MASK,
258 cb_target_mask);
259 gfx11_opt_set_context_reg(R_028424_CB_DCC_CONTROL, SI_TRACKED_CB_DCC_CONTROL,
260 cb_dcc_control);
261 gfx11_opt_set_context_reg(R_028754_SX_PS_DOWNCONVERT, SI_TRACKED_SX_PS_DOWNCONVERT,
262 sx_ps_downconvert);
263 gfx11_opt_set_context_reg(R_028758_SX_BLEND_OPT_EPSILON, SI_TRACKED_SX_BLEND_OPT_EPSILON,
264 sx_blend_opt_epsilon);
265 gfx11_opt_set_context_reg(R_02875C_SX_BLEND_OPT_CONTROL, SI_TRACKED_SX_BLEND_OPT_CONTROL,
266 sx_blend_opt_control);
267 gfx11_end_packed_context_regs();
268 radeon_end(); /* don't track context rolls on GFX11 */
269 } else {
270 radeon_begin(cs);
271 radeon_opt_set_context_reg(R_028238_CB_TARGET_MASK, SI_TRACKED_CB_TARGET_MASK,
272 cb_target_mask);
273 if (sctx->gfx_level >= GFX8) {
274 radeon_opt_set_context_reg(R_028424_CB_DCC_CONTROL, SI_TRACKED_CB_DCC_CONTROL,
275 cb_dcc_control);
276 }
277 if (sctx->screen->info.rbplus_allowed) {
278 radeon_opt_set_context_reg3(R_028754_SX_PS_DOWNCONVERT, SI_TRACKED_SX_PS_DOWNCONVERT,
279 sx_ps_downconvert, sx_blend_opt_epsilon, sx_blend_opt_control);
280 }
281 radeon_end_update_context_roll();
282 }
283 }
284
285 /*
286 * Blender functions
287 */
288
si_translate_blend_function(int blend_func)289 static uint32_t si_translate_blend_function(int blend_func)
290 {
291 switch (blend_func) {
292 case PIPE_BLEND_ADD:
293 return V_028780_COMB_DST_PLUS_SRC;
294 case PIPE_BLEND_SUBTRACT:
295 return V_028780_COMB_SRC_MINUS_DST;
296 case PIPE_BLEND_REVERSE_SUBTRACT:
297 return V_028780_COMB_DST_MINUS_SRC;
298 case PIPE_BLEND_MIN:
299 return V_028780_COMB_MIN_DST_SRC;
300 case PIPE_BLEND_MAX:
301 return V_028780_COMB_MAX_DST_SRC;
302 default:
303 PRINT_ERR("Unknown blend function %d\n", blend_func);
304 assert(0);
305 break;
306 }
307 return 0;
308 }
309
si_translate_blend_factor(enum amd_gfx_level gfx_level,int blend_fact)310 static uint32_t si_translate_blend_factor(enum amd_gfx_level gfx_level, int blend_fact)
311 {
312 switch (blend_fact) {
313 case PIPE_BLENDFACTOR_ONE:
314 return V_028780_BLEND_ONE;
315 case PIPE_BLENDFACTOR_SRC_COLOR:
316 return V_028780_BLEND_SRC_COLOR;
317 case PIPE_BLENDFACTOR_SRC_ALPHA:
318 return V_028780_BLEND_SRC_ALPHA;
319 case PIPE_BLENDFACTOR_DST_ALPHA:
320 return V_028780_BLEND_DST_ALPHA;
321 case PIPE_BLENDFACTOR_DST_COLOR:
322 return V_028780_BLEND_DST_COLOR;
323 case PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE:
324 return V_028780_BLEND_SRC_ALPHA_SATURATE;
325 case PIPE_BLENDFACTOR_CONST_COLOR:
326 return gfx_level >= GFX11 ? V_028780_BLEND_CONSTANT_COLOR_GFX11:
327 V_028780_BLEND_CONSTANT_COLOR_GFX6;
328 case PIPE_BLENDFACTOR_CONST_ALPHA:
329 return gfx_level >= GFX11 ? V_028780_BLEND_CONSTANT_ALPHA_GFX11 :
330 V_028780_BLEND_CONSTANT_ALPHA_GFX6;
331 case PIPE_BLENDFACTOR_ZERO:
332 return V_028780_BLEND_ZERO;
333 case PIPE_BLENDFACTOR_INV_SRC_COLOR:
334 return V_028780_BLEND_ONE_MINUS_SRC_COLOR;
335 case PIPE_BLENDFACTOR_INV_SRC_ALPHA:
336 return V_028780_BLEND_ONE_MINUS_SRC_ALPHA;
337 case PIPE_BLENDFACTOR_INV_DST_ALPHA:
338 return V_028780_BLEND_ONE_MINUS_DST_ALPHA;
339 case PIPE_BLENDFACTOR_INV_DST_COLOR:
340 return V_028780_BLEND_ONE_MINUS_DST_COLOR;
341 case PIPE_BLENDFACTOR_INV_CONST_COLOR:
342 return gfx_level >= GFX11 ? V_028780_BLEND_ONE_MINUS_CONSTANT_COLOR_GFX11:
343 V_028780_BLEND_ONE_MINUS_CONSTANT_COLOR_GFX6;
344 case PIPE_BLENDFACTOR_INV_CONST_ALPHA:
345 return gfx_level >= GFX11 ? V_028780_BLEND_ONE_MINUS_CONSTANT_ALPHA_GFX11:
346 V_028780_BLEND_ONE_MINUS_CONSTANT_ALPHA_GFX6;
347 case PIPE_BLENDFACTOR_SRC1_COLOR:
348 return gfx_level >= GFX11 ? V_028780_BLEND_SRC1_COLOR_GFX11:
349 V_028780_BLEND_SRC1_COLOR_GFX6;
350 case PIPE_BLENDFACTOR_SRC1_ALPHA:
351 return gfx_level >= GFX11 ? V_028780_BLEND_SRC1_ALPHA_GFX11:
352 V_028780_BLEND_SRC1_ALPHA_GFX6;
353 case PIPE_BLENDFACTOR_INV_SRC1_COLOR:
354 return gfx_level >= GFX11 ? V_028780_BLEND_INV_SRC1_COLOR_GFX11:
355 V_028780_BLEND_INV_SRC1_COLOR_GFX6;
356 case PIPE_BLENDFACTOR_INV_SRC1_ALPHA:
357 return gfx_level >= GFX11 ? V_028780_BLEND_INV_SRC1_ALPHA_GFX11:
358 V_028780_BLEND_INV_SRC1_ALPHA_GFX6;
359 default:
360 PRINT_ERR("Bad blend factor %d not supported!\n", blend_fact);
361 assert(0);
362 break;
363 }
364 return 0;
365 }
366
si_translate_blend_opt_function(int blend_func)367 static uint32_t si_translate_blend_opt_function(int blend_func)
368 {
369 switch (blend_func) {
370 case PIPE_BLEND_ADD:
371 return V_028760_OPT_COMB_ADD;
372 case PIPE_BLEND_SUBTRACT:
373 return V_028760_OPT_COMB_SUBTRACT;
374 case PIPE_BLEND_REVERSE_SUBTRACT:
375 return V_028760_OPT_COMB_REVSUBTRACT;
376 case PIPE_BLEND_MIN:
377 return V_028760_OPT_COMB_MIN;
378 case PIPE_BLEND_MAX:
379 return V_028760_OPT_COMB_MAX;
380 default:
381 return V_028760_OPT_COMB_BLEND_DISABLED;
382 }
383 }
384
si_translate_blend_opt_factor(int blend_fact,bool is_alpha)385 static uint32_t si_translate_blend_opt_factor(int blend_fact, bool is_alpha)
386 {
387 switch (blend_fact) {
388 case PIPE_BLENDFACTOR_ZERO:
389 return V_028760_BLEND_OPT_PRESERVE_NONE_IGNORE_ALL;
390 case PIPE_BLENDFACTOR_ONE:
391 return V_028760_BLEND_OPT_PRESERVE_ALL_IGNORE_NONE;
392 case PIPE_BLENDFACTOR_SRC_COLOR:
393 return is_alpha ? V_028760_BLEND_OPT_PRESERVE_A1_IGNORE_A0
394 : V_028760_BLEND_OPT_PRESERVE_C1_IGNORE_C0;
395 case PIPE_BLENDFACTOR_INV_SRC_COLOR:
396 return is_alpha ? V_028760_BLEND_OPT_PRESERVE_A0_IGNORE_A1
397 : V_028760_BLEND_OPT_PRESERVE_C0_IGNORE_C1;
398 case PIPE_BLENDFACTOR_SRC_ALPHA:
399 return V_028760_BLEND_OPT_PRESERVE_A1_IGNORE_A0;
400 case PIPE_BLENDFACTOR_INV_SRC_ALPHA:
401 return V_028760_BLEND_OPT_PRESERVE_A0_IGNORE_A1;
402 case PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE:
403 return is_alpha ? V_028760_BLEND_OPT_PRESERVE_ALL_IGNORE_NONE
404 : V_028760_BLEND_OPT_PRESERVE_NONE_IGNORE_A0;
405 default:
406 return V_028760_BLEND_OPT_PRESERVE_NONE_IGNORE_NONE;
407 }
408 }
409
si_blend_check_commutativity(struct si_screen * sscreen,struct si_state_blend * blend,enum pipe_blend_func func,enum pipe_blendfactor src,enum pipe_blendfactor dst,unsigned chanmask)410 static void si_blend_check_commutativity(struct si_screen *sscreen, struct si_state_blend *blend,
411 enum pipe_blend_func func, enum pipe_blendfactor src,
412 enum pipe_blendfactor dst, unsigned chanmask)
413 {
414 /* Src factor is allowed when it does not depend on Dst */
415 static const uint32_t src_allowed =
416 (1u << PIPE_BLENDFACTOR_ONE) | (1u << PIPE_BLENDFACTOR_SRC_COLOR) |
417 (1u << PIPE_BLENDFACTOR_SRC_ALPHA) | (1u << PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE) |
418 (1u << PIPE_BLENDFACTOR_CONST_COLOR) | (1u << PIPE_BLENDFACTOR_CONST_ALPHA) |
419 (1u << PIPE_BLENDFACTOR_SRC1_COLOR) | (1u << PIPE_BLENDFACTOR_SRC1_ALPHA) |
420 (1u << PIPE_BLENDFACTOR_ZERO) | (1u << PIPE_BLENDFACTOR_INV_SRC_COLOR) |
421 (1u << PIPE_BLENDFACTOR_INV_SRC_ALPHA) | (1u << PIPE_BLENDFACTOR_INV_CONST_COLOR) |
422 (1u << PIPE_BLENDFACTOR_INV_CONST_ALPHA) | (1u << PIPE_BLENDFACTOR_INV_SRC1_COLOR) |
423 (1u << PIPE_BLENDFACTOR_INV_SRC1_ALPHA);
424
425 if (dst == PIPE_BLENDFACTOR_ONE && (src_allowed & (1u << src)) &&
426 (func == PIPE_BLEND_MAX || func == PIPE_BLEND_MIN))
427 blend->commutative_4bit |= chanmask;
428 }
429
430 /**
431 * Get rid of DST in the blend factors by commuting the operands:
432 * func(src * DST, dst * 0) ---> func(src * 0, dst * SRC)
433 */
si_blend_remove_dst(unsigned * func,unsigned * src_factor,unsigned * dst_factor,unsigned expected_dst,unsigned replacement_src)434 static void si_blend_remove_dst(unsigned *func, unsigned *src_factor, unsigned *dst_factor,
435 unsigned expected_dst, unsigned replacement_src)
436 {
437 if (*src_factor == expected_dst && *dst_factor == PIPE_BLENDFACTOR_ZERO) {
438 *src_factor = PIPE_BLENDFACTOR_ZERO;
439 *dst_factor = replacement_src;
440
441 /* Commuting the operands requires reversing subtractions. */
442 if (*func == PIPE_BLEND_SUBTRACT)
443 *func = PIPE_BLEND_REVERSE_SUBTRACT;
444 else if (*func == PIPE_BLEND_REVERSE_SUBTRACT)
445 *func = PIPE_BLEND_SUBTRACT;
446 }
447 }
448
si_create_blend_state_mode(struct pipe_context * ctx,const struct pipe_blend_state * state,unsigned mode)449 static void *si_create_blend_state_mode(struct pipe_context *ctx,
450 const struct pipe_blend_state *state, unsigned mode)
451 {
452 struct si_context *sctx = (struct si_context *)ctx;
453 struct si_state_blend *blend = CALLOC_STRUCT(si_state_blend);
454 struct si_pm4_state *pm4 = &blend->pm4;
455 uint32_t sx_mrt_blend_opt[8] = {0};
456 uint32_t color_control = 0;
457 bool logicop_enable = state->logicop_enable && state->logicop_func != PIPE_LOGICOP_COPY;
458
459 if (!blend)
460 return NULL;
461
462 si_pm4_clear_state(pm4, sctx->screen, false);
463
464 blend->alpha_to_coverage = state->alpha_to_coverage;
465 blend->alpha_to_one = state->alpha_to_one;
466 blend->dual_src_blend = util_blend_state_is_dual(state, 0);
467 blend->logicop_enable = logicop_enable;
468 blend->allows_noop_optimization =
469 state->rt[0].rgb_func == PIPE_BLEND_ADD &&
470 state->rt[0].alpha_func == PIPE_BLEND_ADD &&
471 state->rt[0].rgb_src_factor == PIPE_BLENDFACTOR_DST_COLOR &&
472 state->rt[0].alpha_src_factor == PIPE_BLENDFACTOR_DST_COLOR &&
473 state->rt[0].rgb_dst_factor == PIPE_BLENDFACTOR_ZERO &&
474 state->rt[0].alpha_dst_factor == PIPE_BLENDFACTOR_ZERO &&
475 mode == V_028808_CB_NORMAL;
476
477 unsigned num_shader_outputs = state->max_rt + 1; /* estimate */
478 if (blend->dual_src_blend)
479 num_shader_outputs = MAX2(num_shader_outputs, 2);
480
481 if (logicop_enable) {
482 color_control |= S_028808_ROP3(state->logicop_func | (state->logicop_func << 4));
483 } else {
484 color_control |= S_028808_ROP3(0xcc);
485 }
486
487 unsigned db_alpha_to_mask;
488 if (state->alpha_to_coverage && state->alpha_to_coverage_dither) {
489 db_alpha_to_mask = S_028B70_ALPHA_TO_MASK_ENABLE(state->alpha_to_coverage) |
490 S_028B70_ALPHA_TO_MASK_OFFSET0(3) | S_028B70_ALPHA_TO_MASK_OFFSET1(1) |
491 S_028B70_ALPHA_TO_MASK_OFFSET2(0) | S_028B70_ALPHA_TO_MASK_OFFSET3(2) |
492 S_028B70_OFFSET_ROUND(1);
493 } else {
494 db_alpha_to_mask = S_028B70_ALPHA_TO_MASK_ENABLE(state->alpha_to_coverage) |
495 S_028B70_ALPHA_TO_MASK_OFFSET0(2) | S_028B70_ALPHA_TO_MASK_OFFSET1(2) |
496 S_028B70_ALPHA_TO_MASK_OFFSET2(2) | S_028B70_ALPHA_TO_MASK_OFFSET3(2) |
497 S_028B70_OFFSET_ROUND(0);
498 }
499
500 if (sctx->gfx_level >= GFX12)
501 ac_pm4_set_reg(&pm4->base, R_02807C_DB_ALPHA_TO_MASK, db_alpha_to_mask);
502 else
503 ac_pm4_set_reg(&pm4->base, R_028B70_DB_ALPHA_TO_MASK, db_alpha_to_mask);
504
505 blend->cb_target_mask = 0;
506 blend->cb_target_enabled_4bit = 0;
507
508 unsigned last_blend_cntl;
509
510 for (int i = 0; i < num_shader_outputs; i++) {
511 /* state->rt entries > 0 only written if independent blending */
512 const int j = state->independent_blend_enable ? i : 0;
513
514 unsigned eqRGB = state->rt[j].rgb_func;
515 unsigned srcRGB = state->rt[j].rgb_src_factor;
516 unsigned dstRGB = state->rt[j].rgb_dst_factor;
517 unsigned eqA = state->rt[j].alpha_func;
518 unsigned srcA = state->rt[j].alpha_src_factor;
519 unsigned dstA = state->rt[j].alpha_dst_factor;
520
521 unsigned srcRGB_opt, dstRGB_opt, srcA_opt, dstA_opt;
522 unsigned blend_cntl = 0;
523
524 sx_mrt_blend_opt[i] = S_028760_COLOR_COMB_FCN(V_028760_OPT_COMB_BLEND_DISABLED) |
525 S_028760_ALPHA_COMB_FCN(V_028760_OPT_COMB_BLEND_DISABLED);
526
527 /* Only set dual source blending for MRT0 to avoid a hang. */
528 if (i >= 1 && blend->dual_src_blend) {
529 if (i == 1) {
530 if (sctx->gfx_level >= GFX11)
531 blend_cntl = last_blend_cntl;
532 else
533 blend_cntl = S_028780_ENABLE(1);
534 }
535
536 ac_pm4_set_reg(&pm4->base, R_028780_CB_BLEND0_CONTROL + i * 4, blend_cntl);
537 continue;
538 }
539
540 /* Only addition and subtraction equations are supported with
541 * dual source blending.
542 */
543 if (blend->dual_src_blend && (eqRGB == PIPE_BLEND_MIN || eqRGB == PIPE_BLEND_MAX ||
544 eqA == PIPE_BLEND_MIN || eqA == PIPE_BLEND_MAX)) {
545 assert(!"Unsupported equation for dual source blending");
546 ac_pm4_set_reg(&pm4->base, R_028780_CB_BLEND0_CONTROL + i * 4, blend_cntl);
547 continue;
548 }
549
550 /* cb_render_state will disable unused ones */
551 blend->cb_target_mask |= (unsigned)state->rt[j].colormask << (4 * i);
552 if (state->rt[j].colormask)
553 blend->cb_target_enabled_4bit |= 0xf << (4 * i);
554
555 if (!state->rt[j].colormask || !state->rt[j].blend_enable) {
556 ac_pm4_set_reg(&pm4->base, R_028780_CB_BLEND0_CONTROL + i * 4, blend_cntl);
557 continue;
558 }
559
560 si_blend_check_commutativity(sctx->screen, blend, eqRGB, srcRGB, dstRGB, 0x7 << (4 * i));
561 si_blend_check_commutativity(sctx->screen, blend, eqA, srcA, dstA, 0x8 << (4 * i));
562
563 /* Blending optimizations for RB+.
564 * These transformations don't change the behavior.
565 *
566 * First, get rid of DST in the blend factors:
567 * func(src * DST, dst * 0) ---> func(src * 0, dst * SRC)
568 */
569 si_blend_remove_dst(&eqRGB, &srcRGB, &dstRGB, PIPE_BLENDFACTOR_DST_COLOR,
570 PIPE_BLENDFACTOR_SRC_COLOR);
571 si_blend_remove_dst(&eqA, &srcA, &dstA, PIPE_BLENDFACTOR_DST_COLOR,
572 PIPE_BLENDFACTOR_SRC_COLOR);
573 si_blend_remove_dst(&eqA, &srcA, &dstA, PIPE_BLENDFACTOR_DST_ALPHA,
574 PIPE_BLENDFACTOR_SRC_ALPHA);
575
576 /* Look up the ideal settings from tables. */
577 srcRGB_opt = si_translate_blend_opt_factor(srcRGB, false);
578 dstRGB_opt = si_translate_blend_opt_factor(dstRGB, false);
579 srcA_opt = si_translate_blend_opt_factor(srcA, true);
580 dstA_opt = si_translate_blend_opt_factor(dstA, true);
581
582 /* Handle interdependencies. */
583 if (util_blend_factor_uses_dest(srcRGB, false))
584 dstRGB_opt = V_028760_BLEND_OPT_PRESERVE_NONE_IGNORE_NONE;
585 if (util_blend_factor_uses_dest(srcA, false))
586 dstA_opt = V_028760_BLEND_OPT_PRESERVE_NONE_IGNORE_NONE;
587
588 if (srcRGB == PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE &&
589 (dstRGB == PIPE_BLENDFACTOR_ZERO || dstRGB == PIPE_BLENDFACTOR_SRC_ALPHA ||
590 dstRGB == PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE))
591 dstRGB_opt = V_028760_BLEND_OPT_PRESERVE_NONE_IGNORE_A0;
592
593 /* Set the final value. */
594 sx_mrt_blend_opt[i] = S_028760_COLOR_SRC_OPT(srcRGB_opt) |
595 S_028760_COLOR_DST_OPT(dstRGB_opt) |
596 S_028760_COLOR_COMB_FCN(si_translate_blend_opt_function(eqRGB)) |
597 S_028760_ALPHA_SRC_OPT(srcA_opt) | S_028760_ALPHA_DST_OPT(dstA_opt) |
598 S_028760_ALPHA_COMB_FCN(si_translate_blend_opt_function(eqA));
599
600 /* Alpha-to-coverage with blending enabled, depth writes enabled, and having no MRTZ export
601 * should disable SX blend optimizations.
602 *
603 * TODO: Add a piglit test for this. It should fail on gfx11 without this.
604 */
605 if (sctx->gfx_level >= GFX11 && state->alpha_to_coverage && i == 0) {
606 sx_mrt_blend_opt[0] = S_028760_COLOR_COMB_FCN(V_028760_OPT_COMB_NONE) |
607 S_028760_ALPHA_COMB_FCN(V_028760_OPT_COMB_NONE);
608 }
609
610 /* Set blend state. */
611 blend_cntl |= S_028780_ENABLE(1);
612 blend_cntl |= S_028780_COLOR_COMB_FCN(si_translate_blend_function(eqRGB));
613 blend_cntl |= S_028780_COLOR_SRCBLEND(si_translate_blend_factor(sctx->gfx_level, srcRGB));
614 blend_cntl |= S_028780_COLOR_DESTBLEND(si_translate_blend_factor(sctx->gfx_level, dstRGB));
615
616 if (srcA != srcRGB || dstA != dstRGB || eqA != eqRGB) {
617 blend_cntl |= S_028780_SEPARATE_ALPHA_BLEND(1);
618 blend_cntl |= S_028780_ALPHA_COMB_FCN(si_translate_blend_function(eqA));
619 blend_cntl |= S_028780_ALPHA_SRCBLEND(si_translate_blend_factor(sctx->gfx_level, srcA));
620 blend_cntl |= S_028780_ALPHA_DESTBLEND(si_translate_blend_factor(sctx->gfx_level, dstA));
621 }
622 ac_pm4_set_reg(&pm4->base, R_028780_CB_BLEND0_CONTROL + i * 4, blend_cntl);
623 last_blend_cntl = blend_cntl;
624
625 blend->blend_enable_4bit |= 0xfu << (i * 4);
626
627 if (sctx->gfx_level >= GFX8 && sctx->gfx_level <= GFX10)
628 blend->dcc_msaa_corruption_4bit |= 0xfu << (i * 4);
629
630 /* This is only important for formats without alpha. */
631 if (srcRGB == PIPE_BLENDFACTOR_SRC_ALPHA || dstRGB == PIPE_BLENDFACTOR_SRC_ALPHA ||
632 srcRGB == PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE ||
633 dstRGB == PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE ||
634 srcRGB == PIPE_BLENDFACTOR_INV_SRC_ALPHA || dstRGB == PIPE_BLENDFACTOR_INV_SRC_ALPHA)
635 blend->need_src_alpha_4bit |= 0xfu << (i * 4);
636 }
637
638 if (sctx->gfx_level >= GFX8 && sctx->gfx_level <= GFX10 && logicop_enable)
639 blend->dcc_msaa_corruption_4bit |= blend->cb_target_enabled_4bit;
640
641 if (blend->cb_target_mask) {
642 color_control |= S_028808_MODE(mode);
643 } else {
644 color_control |= S_028808_MODE(V_028808_CB_DISABLE);
645 }
646
647 if (sctx->screen->info.rbplus_allowed) {
648 /* Disable RB+ blend optimizations for dual source blending.
649 * Vulkan does this.
650 */
651 if (blend->dual_src_blend) {
652 for (int i = 0; i < num_shader_outputs; i++) {
653 sx_mrt_blend_opt[i] = S_028760_COLOR_COMB_FCN(V_028760_OPT_COMB_NONE) |
654 S_028760_ALPHA_COMB_FCN(V_028760_OPT_COMB_NONE);
655 }
656 }
657
658 for (int i = 0; i < num_shader_outputs; i++)
659 ac_pm4_set_reg(&pm4->base, R_028760_SX_MRT0_BLEND_OPT + i * 4, sx_mrt_blend_opt[i]);
660
661 /* RB+ doesn't work with dual source blending, logic op, and RESOLVE. */
662 if (blend->dual_src_blend || logicop_enable || mode == V_028808_CB_RESOLVE ||
663 /* Disabling RB+ improves blending performance in synthetic tests on GFX11. */
664 (sctx->gfx_level == GFX11 && blend->blend_enable_4bit))
665 color_control |= S_028808_DISABLE_DUAL_QUAD(1);
666 }
667
668 if (sctx->gfx_level >= GFX12)
669 ac_pm4_set_reg(&pm4->base, R_028858_CB_COLOR_CONTROL, color_control);
670 else
671 ac_pm4_set_reg(&pm4->base, R_028808_CB_COLOR_CONTROL, color_control);
672
673 ac_pm4_finalize(&pm4->base);
674 return blend;
675 }
676
si_create_blend_state(struct pipe_context * ctx,const struct pipe_blend_state * state)677 static void *si_create_blend_state(struct pipe_context *ctx, const struct pipe_blend_state *state)
678 {
679 return si_create_blend_state_mode(ctx, state, V_028808_CB_NORMAL);
680 }
681
si_check_blend_dst_sampler_noop(struct si_context * sctx)682 static bool si_check_blend_dst_sampler_noop(struct si_context *sctx)
683 {
684 if (sctx->framebuffer.state.nr_cbufs == 1) {
685 struct si_shader_selector *sel = sctx->shader.ps.cso;
686
687 if (unlikely(sel->info.writes_1_if_tex_is_1 == 0xff)) {
688 /* Wait for the shader to be ready. */
689 util_queue_fence_wait(&sel->ready);
690 assert(sel->nir_binary);
691
692 struct nir_shader *nir = si_deserialize_shader(sel);
693
694 /* Determine if this fragment shader always writes vec4(1) if a specific texture
695 * is all 1s.
696 */
697 float in[4] = { 1.0, 1.0, 1.0, 1.0 };
698 float out[4];
699 int texunit;
700 if (si_nir_is_output_const_if_tex_is_const(nir, in, out, &texunit) &&
701 !memcmp(in, out, 4 * sizeof(float))) {
702 sel->info.writes_1_if_tex_is_1 = 1 + texunit;
703 } else {
704 sel->info.writes_1_if_tex_is_1 = 0;
705 }
706
707 ralloc_free(nir);
708 }
709
710 if (sel->info.writes_1_if_tex_is_1 &&
711 sel->info.writes_1_if_tex_is_1 != 0xff) {
712 /* Now check if the texture is cleared to 1 */
713 int unit = sctx->shader.ps.cso->info.writes_1_if_tex_is_1 - 1;
714 struct si_samplers *samp = &sctx->samplers[PIPE_SHADER_FRAGMENT];
715 if ((1u << unit) & samp->enabled_mask) {
716 struct si_texture* tex = (struct si_texture*) samp->views[unit]->texture;
717 if (tex->is_depth &&
718 tex->depth_cleared_level_mask & BITFIELD_BIT(samp->views[unit]->u.tex.first_level) &&
719 tex->depth_clear_value[0] == 1) {
720 return false;
721 }
722 /* TODO: handle color textures */
723 }
724 }
725 }
726
727 return true;
728 }
729
si_draw_blend_dst_sampler_noop(struct pipe_context * ctx,const struct pipe_draw_info * info,unsigned drawid_offset,const struct pipe_draw_indirect_info * indirect,const struct pipe_draw_start_count_bias * draws,unsigned num_draws)730 static void si_draw_blend_dst_sampler_noop(struct pipe_context *ctx,
731 const struct pipe_draw_info *info,
732 unsigned drawid_offset,
733 const struct pipe_draw_indirect_info *indirect,
734 const struct pipe_draw_start_count_bias *draws,
735 unsigned num_draws) {
736 struct si_context *sctx = (struct si_context *)ctx;
737
738 if (!si_check_blend_dst_sampler_noop(sctx))
739 return;
740
741 sctx->real_draw_vbo(ctx, info, drawid_offset, indirect, draws, num_draws);
742 }
743
si_draw_vstate_blend_dst_sampler_noop(struct pipe_context * ctx,struct pipe_vertex_state * state,uint32_t partial_velem_mask,struct pipe_draw_vertex_state_info info,const struct pipe_draw_start_count_bias * draws,unsigned num_draws)744 static void si_draw_vstate_blend_dst_sampler_noop(struct pipe_context *ctx,
745 struct pipe_vertex_state *state,
746 uint32_t partial_velem_mask,
747 struct pipe_draw_vertex_state_info info,
748 const struct pipe_draw_start_count_bias *draws,
749 unsigned num_draws) {
750 struct si_context *sctx = (struct si_context *)ctx;
751
752 if (!si_check_blend_dst_sampler_noop(sctx))
753 return;
754
755 sctx->real_draw_vertex_state(ctx, state, partial_velem_mask, info, draws, num_draws);
756 }
757
si_bind_blend_state(struct pipe_context * ctx,void * state)758 static void si_bind_blend_state(struct pipe_context *ctx, void *state)
759 {
760 struct si_context *sctx = (struct si_context *)ctx;
761 struct si_state_blend *old_blend = sctx->queued.named.blend;
762 struct si_state_blend *blend = (struct si_state_blend *)state;
763
764 if (!blend)
765 blend = (struct si_state_blend *)sctx->noop_blend;
766
767 si_pm4_bind_state(sctx, blend, blend);
768
769 if (old_blend->cb_target_mask != blend->cb_target_mask ||
770 old_blend->dual_src_blend != blend->dual_src_blend ||
771 (old_blend->dcc_msaa_corruption_4bit != blend->dcc_msaa_corruption_4bit &&
772 sctx->framebuffer.has_dcc_msaa))
773 si_mark_atom_dirty(sctx, &sctx->atoms.s.cb_render_state);
774
775 if ((sctx->screen->info.has_export_conflict_bug &&
776 old_blend->blend_enable_4bit != blend->blend_enable_4bit) ||
777 (sctx->occlusion_query_mode == SI_OCCLUSION_QUERY_MODE_PRECISE_BOOLEAN &&
778 !!old_blend->cb_target_mask != !!blend->cb_target_enabled_4bit))
779 si_mark_atom_dirty(sctx, &sctx->atoms.s.db_render_state);
780
781 if (old_blend->cb_target_enabled_4bit != blend->cb_target_enabled_4bit ||
782 old_blend->alpha_to_coverage != blend->alpha_to_coverage ||
783 old_blend->alpha_to_one != blend->alpha_to_one ||
784 old_blend->dual_src_blend != blend->dual_src_blend ||
785 old_blend->blend_enable_4bit != blend->blend_enable_4bit ||
786 old_blend->need_src_alpha_4bit != blend->need_src_alpha_4bit)
787 si_ps_key_update_framebuffer_blend_dsa_rasterizer(sctx);
788
789 if (old_blend->cb_target_enabled_4bit != blend->cb_target_enabled_4bit ||
790 old_blend->alpha_to_coverage != blend->alpha_to_coverage)
791 si_update_ps_inputs_read_or_disabled(sctx);
792
793 if (sctx->screen->dpbb_allowed &&
794 (old_blend->alpha_to_coverage != blend->alpha_to_coverage ||
795 old_blend->blend_enable_4bit != blend->blend_enable_4bit ||
796 old_blend->cb_target_enabled_4bit != blend->cb_target_enabled_4bit))
797 si_mark_atom_dirty(sctx, &sctx->atoms.s.dpbb_state);
798
799 if (sctx->screen->info.has_out_of_order_rast &&
800 ((old_blend->blend_enable_4bit != blend->blend_enable_4bit ||
801 old_blend->cb_target_enabled_4bit != blend->cb_target_enabled_4bit ||
802 old_blend->commutative_4bit != blend->commutative_4bit ||
803 old_blend->logicop_enable != blend->logicop_enable)))
804 si_mark_atom_dirty(sctx, &sctx->atoms.s.msaa_config);
805
806 /* RB+ depth-only rendering. See the comment where we set rbplus_depth_only_opt for more
807 * information.
808 */
809 if (sctx->screen->info.rbplus_allowed &&
810 !!old_blend->cb_target_mask != !!blend->cb_target_mask) {
811 sctx->framebuffer.dirty_cbufs |= BITFIELD_BIT(0);
812 si_mark_atom_dirty(sctx, &sctx->atoms.s.framebuffer);
813 }
814
815 if (likely(!radeon_uses_secure_bos(sctx->ws))) {
816 if (unlikely(blend->allows_noop_optimization)) {
817 si_install_draw_wrapper(sctx, si_draw_blend_dst_sampler_noop,
818 si_draw_vstate_blend_dst_sampler_noop);
819 } else {
820 si_install_draw_wrapper(sctx, NULL, NULL);
821 }
822 }
823 }
824
si_delete_blend_state(struct pipe_context * ctx,void * state)825 static void si_delete_blend_state(struct pipe_context *ctx, void *state)
826 {
827 struct si_context *sctx = (struct si_context *)ctx;
828
829 if (sctx->queued.named.blend == state)
830 si_bind_blend_state(ctx, sctx->noop_blend);
831
832 si_pm4_free_state(sctx, (struct si_pm4_state*)state, SI_STATE_IDX(blend));
833 }
834
si_set_blend_color(struct pipe_context * ctx,const struct pipe_blend_color * state)835 static void si_set_blend_color(struct pipe_context *ctx, const struct pipe_blend_color *state)
836 {
837 struct si_context *sctx = (struct si_context *)ctx;
838 static const struct pipe_blend_color zeros;
839
840 sctx->blend_color = *state;
841 sctx->blend_color_any_nonzeros = memcmp(state, &zeros, sizeof(*state)) != 0;
842 si_mark_atom_dirty(sctx, &sctx->atoms.s.blend_color);
843 }
844
si_emit_blend_color(struct si_context * sctx,unsigned index)845 static void si_emit_blend_color(struct si_context *sctx, unsigned index)
846 {
847 struct radeon_cmdbuf *cs = &sctx->gfx_cs;
848
849 radeon_begin(cs);
850 radeon_set_context_reg_seq(R_028414_CB_BLEND_RED, 4);
851 radeon_emit_array((uint32_t *)sctx->blend_color.color, 4);
852 radeon_end();
853 }
854
855 /*
856 * Clipping
857 */
858
si_set_clip_state(struct pipe_context * ctx,const struct pipe_clip_state * state)859 static void si_set_clip_state(struct pipe_context *ctx, const struct pipe_clip_state *state)
860 {
861 struct si_context *sctx = (struct si_context *)ctx;
862 struct pipe_constant_buffer cb;
863 static const struct pipe_clip_state zeros;
864
865 if (memcmp(&sctx->clip_state, state, sizeof(*state)) == 0)
866 return;
867
868 sctx->clip_state = *state;
869 sctx->clip_state_any_nonzeros = memcmp(state, &zeros, sizeof(*state)) != 0;
870 si_mark_atom_dirty(sctx, &sctx->atoms.s.clip_state);
871
872 cb.buffer = NULL;
873 cb.user_buffer = state->ucp;
874 cb.buffer_offset = 0;
875 cb.buffer_size = 4 * 4 * 8;
876 si_set_internal_const_buffer(sctx, SI_VS_CONST_CLIP_PLANES, &cb);
877 }
878
si_emit_clip_state(struct si_context * sctx,unsigned index)879 static void si_emit_clip_state(struct si_context *sctx, unsigned index)
880 {
881 struct radeon_cmdbuf *cs = &sctx->gfx_cs;
882
883 radeon_begin(cs);
884 if (sctx->gfx_level >= GFX12)
885 radeon_set_context_reg_seq(R_0282D0_PA_CL_UCP_0_X, 6 * 4);
886 else
887 radeon_set_context_reg_seq(R_0285BC_PA_CL_UCP_0_X, 6 * 4);
888 radeon_emit_array((uint32_t *)sctx->clip_state.ucp, 6 * 4);
889 radeon_end();
890 }
891
si_emit_clip_regs(struct si_context * sctx,unsigned index)892 static void si_emit_clip_regs(struct si_context *sctx, unsigned index)
893 {
894 struct si_shader *vs = si_get_vs(sctx)->current;
895 struct si_shader_selector *vs_sel = vs->selector;
896 struct si_shader_info *info = &vs_sel->info;
897 struct si_state_rasterizer *rs = sctx->queued.named.rasterizer;
898 bool window_space = vs_sel->stage == MESA_SHADER_VERTEX ?
899 info->base.vs.window_space_position : 0;
900 unsigned clipdist_mask = vs_sel->info.clipdist_mask;
901 unsigned ucp_mask = clipdist_mask ? 0 : rs->clip_plane_enable & SI_USER_CLIP_PLANE_MASK;
902 unsigned culldist_mask = vs_sel->info.culldist_mask;
903
904 /* Clip distances on points have no effect, so need to be implemented
905 * as cull distances. This applies for the clipvertex case as well.
906 *
907 * Setting this for primitives other than points should have no adverse
908 * effects.
909 */
910 clipdist_mask &= rs->clip_plane_enable;
911 culldist_mask |= clipdist_mask;
912
913 unsigned pa_cl_cntl = S_02881C_BYPASS_VTX_RATE_COMBINER(sctx->gfx_level >= GFX10_3 &&
914 !sctx->screen->options.vrs2x2) |
915 S_02881C_BYPASS_PRIM_RATE_COMBINER(sctx->gfx_level >= GFX10_3) |
916 clipdist_mask | (culldist_mask << 8);
917
918 unsigned pa_cl_clip_cntl = rs->pa_cl_clip_cntl | ucp_mask |
919 S_028810_CLIP_DISABLE(window_space);
920 unsigned pa_cl_vs_out_cntl = pa_cl_cntl | vs->pa_cl_vs_out_cntl;
921
922 if (sctx->gfx_level >= GFX12) {
923 radeon_begin(&sctx->gfx_cs);
924 gfx12_begin_context_regs();
925 gfx12_opt_set_context_reg(R_028810_PA_CL_CLIP_CNTL, SI_TRACKED_PA_CL_CLIP_CNTL,
926 pa_cl_clip_cntl);
927 gfx12_opt_set_context_reg(R_028818_PA_CL_VS_OUT_CNTL, SI_TRACKED_PA_CL_VS_OUT_CNTL,
928 pa_cl_vs_out_cntl);
929 gfx12_end_context_regs();
930 radeon_end(); /* don't track context rolls on GFX12 */
931 } else if (sctx->screen->info.has_set_context_pairs_packed) {
932 radeon_begin(&sctx->gfx_cs);
933 gfx11_begin_packed_context_regs();
934 gfx11_opt_set_context_reg(R_028810_PA_CL_CLIP_CNTL, SI_TRACKED_PA_CL_CLIP_CNTL,
935 pa_cl_clip_cntl);
936 gfx11_opt_set_context_reg(R_02881C_PA_CL_VS_OUT_CNTL, SI_TRACKED_PA_CL_VS_OUT_CNTL,
937 pa_cl_vs_out_cntl);
938 gfx11_end_packed_context_regs();
939 radeon_end(); /* don't track context rolls on GFX11 */
940 } else {
941 radeon_begin(&sctx->gfx_cs);
942 radeon_opt_set_context_reg(R_028810_PA_CL_CLIP_CNTL, SI_TRACKED_PA_CL_CLIP_CNTL,
943 pa_cl_clip_cntl);
944 radeon_opt_set_context_reg(R_02881C_PA_CL_VS_OUT_CNTL, SI_TRACKED_PA_CL_VS_OUT_CNTL,
945 pa_cl_vs_out_cntl);
946 radeon_end_update_context_roll();
947 }
948 }
949
950 /*
951 * Rasterizer
952 */
953
si_translate_fill(uint32_t func)954 static uint32_t si_translate_fill(uint32_t func)
955 {
956 switch (func) {
957 case PIPE_POLYGON_MODE_FILL:
958 return V_028814_X_DRAW_TRIANGLES;
959 case PIPE_POLYGON_MODE_LINE:
960 return V_028814_X_DRAW_LINES;
961 case PIPE_POLYGON_MODE_POINT:
962 return V_028814_X_DRAW_POINTS;
963 default:
964 assert(0);
965 return V_028814_X_DRAW_POINTS;
966 }
967 }
968
si_create_rs_state(struct pipe_context * ctx,const struct pipe_rasterizer_state * state)969 static void *si_create_rs_state(struct pipe_context *ctx, const struct pipe_rasterizer_state *state)
970 {
971 struct si_screen *sscreen = ((struct si_context *)ctx)->screen;
972 struct si_state_rasterizer *rs = CALLOC_STRUCT(si_state_rasterizer);
973
974 if (!rs) {
975 return NULL;
976 }
977
978 rs->scissor_enable = state->scissor;
979 rs->clip_halfz = state->clip_halfz;
980 rs->two_side = state->light_twoside;
981 rs->multisample_enable = state->multisample;
982 rs->clip_plane_enable = state->clip_plane_enable;
983 rs->half_pixel_center = state->half_pixel_center;
984 rs->line_stipple_enable = state->line_stipple_enable;
985 rs->poly_stipple_enable = state->poly_stipple_enable;
986 rs->line_smooth = state->line_smooth;
987 rs->line_width = state->line_width;
988 rs->poly_smooth = state->poly_smooth;
989 rs->point_smooth = state->point_smooth;
990 rs->uses_poly_offset = state->offset_point || state->offset_line || state->offset_tri;
991 rs->clamp_fragment_color = state->clamp_fragment_color;
992 rs->clamp_vertex_color = state->clamp_vertex_color;
993 rs->flatshade = state->flatshade;
994 rs->flatshade_first = state->flatshade_first;
995 rs->sprite_coord_enable = state->sprite_coord_enable;
996 rs->rasterizer_discard = state->rasterizer_discard;
997 rs->bottom_edge_rule = state->bottom_edge_rule;
998 rs->polygon_mode_is_lines =
999 (state->fill_front == PIPE_POLYGON_MODE_LINE && !(state->cull_face & PIPE_FACE_FRONT)) ||
1000 (state->fill_back == PIPE_POLYGON_MODE_LINE && !(state->cull_face & PIPE_FACE_BACK));
1001 rs->polygon_mode_is_points =
1002 (state->fill_front == PIPE_POLYGON_MODE_POINT && !(state->cull_face & PIPE_FACE_FRONT)) ||
1003 (state->fill_back == PIPE_POLYGON_MODE_POINT && !(state->cull_face & PIPE_FACE_BACK));
1004 rs->pa_sc_line_stipple = state->line_stipple_enable ?
1005 S_028A0C_LINE_PATTERN(state->line_stipple_pattern) |
1006 S_028A0C_REPEAT_COUNT(state->line_stipple_factor) : 0;
1007 /* TODO: implement line stippling with perpendicular end caps. */
1008 /* Line width > 2 is an internal recommendation. */
1009 rs->perpendicular_end_caps = state->multisample &&
1010 state->line_width > 2 && !state->line_stipple_enable;
1011
1012 rs->pa_cl_clip_cntl = S_028810_DX_CLIP_SPACE_DEF(state->clip_halfz) |
1013 S_028810_ZCLIP_NEAR_DISABLE(!state->depth_clip_near) |
1014 S_028810_ZCLIP_FAR_DISABLE(!state->depth_clip_far) |
1015 S_028810_DX_RASTERIZATION_KILL(state->rasterizer_discard) |
1016 S_028810_DX_LINEAR_ATTR_CLIP_ENA(1);
1017
1018 rs->ngg_cull_flags_tris = SI_NGG_CULL_CLIP_PLANE_ENABLE(state->clip_plane_enable);
1019 rs->ngg_cull_flags_lines = (!rs->perpendicular_end_caps ? SI_NGG_CULL_SMALL_LINES_DIAMOND_EXIT : 0) |
1020 SI_NGG_CULL_CLIP_PLANE_ENABLE(state->clip_plane_enable);
1021
1022 if (!state->front_ccw) {
1023 rs->ngg_cull_front = state->cull_face & PIPE_FACE_FRONT || rs->rasterizer_discard;
1024 rs->ngg_cull_back = state->cull_face & PIPE_FACE_BACK || rs->rasterizer_discard;
1025 } else {
1026 rs->ngg_cull_front = state->cull_face & PIPE_FACE_BACK || rs->rasterizer_discard;
1027 rs->ngg_cull_back = state->cull_face & PIPE_FACE_FRONT || rs->rasterizer_discard;
1028 }
1029
1030 /* Force gl_FrontFacing to true or false if the other face is culled. */
1031 if (util_bitcount(state->cull_face) == 1) {
1032 if (state->cull_face & PIPE_FACE_FRONT)
1033 rs->force_front_face_input = -1;
1034 else
1035 rs->force_front_face_input = 1;
1036 }
1037
1038 rs->spi_interp_control_0 = S_0286D4_FLAT_SHADE_ENA(1) |
1039 S_0286D4_PNT_SPRITE_ENA(state->point_quad_rasterization) |
1040 S_0286D4_PNT_SPRITE_OVRD_X(V_0286D4_SPI_PNT_SPRITE_SEL_S) |
1041 S_0286D4_PNT_SPRITE_OVRD_Y(V_0286D4_SPI_PNT_SPRITE_SEL_T) |
1042 S_0286D4_PNT_SPRITE_OVRD_Z(V_0286D4_SPI_PNT_SPRITE_SEL_0) |
1043 S_0286D4_PNT_SPRITE_OVRD_W(V_0286D4_SPI_PNT_SPRITE_SEL_1) |
1044 S_0286D4_PNT_SPRITE_TOP_1(state->sprite_coord_mode !=
1045 PIPE_SPRITE_COORD_UPPER_LEFT);
1046
1047 /* point size 12.4 fixed point */
1048 float psize_min, psize_max;
1049 unsigned tmp = (unsigned)(state->point_size * 8.0);
1050 rs->pa_su_point_size = S_028A00_HEIGHT(tmp) | S_028A00_WIDTH(tmp);
1051
1052 if (state->point_size_per_vertex) {
1053 psize_min = util_get_min_point_size(state);
1054 psize_max = SI_MAX_POINT_SIZE;
1055 } else {
1056 /* Force the point size to be as if the vertex output was disabled. */
1057 psize_min = state->point_size;
1058 psize_max = state->point_size;
1059 }
1060 rs->max_point_size = psize_max;
1061
1062 /* Divide by two, because 0.5 = 1 pixel. */
1063 rs->pa_su_point_minmax = S_028A04_MIN_SIZE(si_pack_float_12p4(psize_min / 2)) |
1064 S_028A04_MAX_SIZE(si_pack_float_12p4(psize_max / 2));
1065 rs->pa_su_line_cntl = S_028A08_WIDTH(si_pack_float_12p4(state->line_width / 2));
1066
1067 rs->pa_sc_mode_cntl_0 = S_028A48_LINE_STIPPLE_ENABLE(state->line_stipple_enable) |
1068 S_028A48_MSAA_ENABLE(state->multisample || state->poly_smooth ||
1069 state->line_smooth) |
1070 S_028A48_VPORT_SCISSOR_ENABLE(1) |
1071 S_028A48_ALTERNATE_RBS_PER_TILE(sscreen->info.gfx_level >= GFX9);
1072
1073 bool polygon_mode_enabled =
1074 (state->fill_front != PIPE_POLYGON_MODE_FILL && !(state->cull_face & PIPE_FACE_FRONT)) ||
1075 (state->fill_back != PIPE_POLYGON_MODE_FILL && !(state->cull_face & PIPE_FACE_BACK));
1076
1077 rs->pa_su_sc_mode_cntl = S_028814_PROVOKING_VTX_LAST(!state->flatshade_first) |
1078 S_028814_CULL_FRONT((state->cull_face & PIPE_FACE_FRONT) ? 1 : 0) |
1079 S_028814_CULL_BACK((state->cull_face & PIPE_FACE_BACK) ? 1 : 0) |
1080 S_028814_FACE(!state->front_ccw) |
1081 S_028814_POLY_OFFSET_FRONT_ENABLE(util_get_offset(state, state->fill_front)) |
1082 S_028814_POLY_OFFSET_BACK_ENABLE(util_get_offset(state, state->fill_back)) |
1083 S_028814_POLY_OFFSET_PARA_ENABLE(state->offset_point || state->offset_line) |
1084 S_028814_POLY_MODE(polygon_mode_enabled) |
1085 S_028814_POLYMODE_FRONT_PTYPE(si_translate_fill(state->fill_front)) |
1086 S_028814_POLYMODE_BACK_PTYPE(si_translate_fill(state->fill_back)) |
1087 /* this must be set if POLY_MODE or PERPENDICULAR_ENDCAP_ENA is set */
1088 S_028814_KEEP_TOGETHER_ENABLE(sscreen->info.gfx_level >= GFX10 &&
1089 sscreen->info.gfx_level < GFX12 ?
1090 polygon_mode_enabled ||
1091 rs->perpendicular_end_caps : 0);
1092 if (sscreen->info.gfx_level >= GFX10) {
1093 rs->pa_cl_ngg_cntl = S_028838_INDEX_BUF_EDGE_FLAG_ENA(rs->polygon_mode_is_points ||
1094 rs->polygon_mode_is_lines) |
1095 S_028838_VERTEX_REUSE_DEPTH(sscreen->info.gfx_level >= GFX10_3 ? 30 : 0);
1096 }
1097
1098 if (state->bottom_edge_rule) {
1099 /* OpenGL windows should set this. */
1100 rs->pa_sc_edgerule = S_028230_ER_TRI(0xA) |
1101 S_028230_ER_POINT(0x5) |
1102 S_028230_ER_RECT(0x9) |
1103 S_028230_ER_LINE_LR(0x2A) |
1104 S_028230_ER_LINE_RL(0x2A) |
1105 S_028230_ER_LINE_TB(0xA) |
1106 S_028230_ER_LINE_BT(0xA);
1107 } else {
1108 /* OpenGL FBOs and Direct3D should set this. */
1109 rs->pa_sc_edgerule = S_028230_ER_TRI(0xA) |
1110 S_028230_ER_POINT(0x6) |
1111 S_028230_ER_RECT(0xA) |
1112 S_028230_ER_LINE_LR(0x19) |
1113 S_028230_ER_LINE_RL(0x25) |
1114 S_028230_ER_LINE_TB(0xA) |
1115 S_028230_ER_LINE_BT(0xA);
1116 }
1117
1118 if (rs->uses_poly_offset) {
1119 /* Calculate polygon offset states for 16-bit, 24-bit, and 32-bit zbuffers. */
1120 rs->pa_su_poly_offset_clamp = fui(state->offset_clamp);
1121 rs->pa_su_poly_offset_frontback_scale = fui(state->offset_scale * 16);
1122
1123 if (!state->offset_units_unscaled) {
1124 /* 16-bit zbuffer */
1125 rs->pa_su_poly_offset_db_fmt_cntl[0] = S_028B78_POLY_OFFSET_NEG_NUM_DB_BITS(-16);
1126 rs->pa_su_poly_offset_frontback_offset[0] = fui(state->offset_units * 4);
1127
1128 /* 24-bit zbuffer */
1129 rs->pa_su_poly_offset_db_fmt_cntl[1] = S_028B78_POLY_OFFSET_NEG_NUM_DB_BITS(-24);
1130 rs->pa_su_poly_offset_frontback_offset[1] = fui(state->offset_units * 2);
1131
1132 /* 32-bit zbuffer */
1133 rs->pa_su_poly_offset_db_fmt_cntl[2] = S_028B78_POLY_OFFSET_NEG_NUM_DB_BITS(-23) |
1134 S_028B78_POLY_OFFSET_DB_IS_FLOAT_FMT(1);
1135 rs->pa_su_poly_offset_frontback_offset[2] = fui(state->offset_units);
1136 } else {
1137 rs->pa_su_poly_offset_frontback_offset[0] = fui(state->offset_units);
1138 rs->pa_su_poly_offset_frontback_offset[1] = fui(state->offset_units);
1139 rs->pa_su_poly_offset_frontback_offset[2] = fui(state->offset_units);
1140 }
1141 }
1142
1143 return rs;
1144 }
1145
si_pm4_emit_rasterizer(struct si_context * sctx,unsigned index)1146 static void si_pm4_emit_rasterizer(struct si_context *sctx, unsigned index)
1147 {
1148 struct si_state_rasterizer *state = sctx->queued.named.rasterizer;
1149
1150 if (sctx->screen->info.gfx_level >= GFX12) {
1151 radeon_begin(&sctx->gfx_cs);
1152 gfx12_begin_context_regs();
1153 if (state->line_stipple_enable) {
1154 gfx12_opt_set_context_reg(R_028A0C_PA_SC_LINE_STIPPLE, SI_TRACKED_PA_SC_LINE_STIPPLE,
1155 state->pa_sc_line_stipple);
1156 }
1157
1158 gfx12_opt_set_context_reg(R_028644_SPI_INTERP_CONTROL_0, SI_TRACKED_SPI_INTERP_CONTROL_0,
1159 state->spi_interp_control_0);
1160 gfx12_opt_set_context_reg(R_028A00_PA_SU_POINT_SIZE, SI_TRACKED_PA_SU_POINT_SIZE,
1161 state->pa_su_point_size);
1162 gfx12_opt_set_context_reg(R_028A04_PA_SU_POINT_MINMAX, SI_TRACKED_PA_SU_POINT_MINMAX,
1163 state->pa_su_point_minmax);
1164 gfx12_opt_set_context_reg(R_028A08_PA_SU_LINE_CNTL, SI_TRACKED_PA_SU_LINE_CNTL,
1165 state->pa_su_line_cntl);
1166 gfx12_opt_set_context_reg(R_028A48_PA_SC_MODE_CNTL_0, SI_TRACKED_PA_SC_MODE_CNTL_0,
1167 state->pa_sc_mode_cntl_0);
1168 gfx12_opt_set_context_reg(R_02881C_PA_SU_SC_MODE_CNTL, SI_TRACKED_PA_SU_SC_MODE_CNTL,
1169 state->pa_su_sc_mode_cntl);
1170 gfx12_opt_set_context_reg(R_028838_PA_CL_NGG_CNTL, SI_TRACKED_PA_CL_NGG_CNTL,
1171 state->pa_cl_ngg_cntl);
1172 gfx12_opt_set_context_reg(R_028230_PA_SC_EDGERULE, SI_TRACKED_PA_SC_EDGERULE,
1173 state->pa_sc_edgerule);
1174
1175 if (state->uses_poly_offset && sctx->framebuffer.state.zsbuf) {
1176 unsigned db_format_index =
1177 ((struct si_surface *)sctx->framebuffer.state.zsbuf)->db_format_index;
1178
1179 gfx12_opt_set_context_reg(R_028B78_PA_SU_POLY_OFFSET_DB_FMT_CNTL,
1180 SI_TRACKED_PA_SU_POLY_OFFSET_DB_FMT_CNTL,
1181 state->pa_su_poly_offset_db_fmt_cntl[db_format_index]);
1182 gfx12_opt_set_context_reg(R_028B7C_PA_SU_POLY_OFFSET_CLAMP,
1183 SI_TRACKED_PA_SU_POLY_OFFSET_CLAMP,
1184 state->pa_su_poly_offset_clamp);
1185 gfx12_opt_set_context_reg(R_028B80_PA_SU_POLY_OFFSET_FRONT_SCALE,
1186 SI_TRACKED_PA_SU_POLY_OFFSET_FRONT_SCALE,
1187 state->pa_su_poly_offset_frontback_scale);
1188 gfx12_opt_set_context_reg(R_028B84_PA_SU_POLY_OFFSET_FRONT_OFFSET,
1189 SI_TRACKED_PA_SU_POLY_OFFSET_FRONT_OFFSET,
1190 state->pa_su_poly_offset_frontback_offset[db_format_index]);
1191 gfx12_opt_set_context_reg(R_028B88_PA_SU_POLY_OFFSET_BACK_SCALE,
1192 SI_TRACKED_PA_SU_POLY_OFFSET_BACK_SCALE,
1193 state->pa_su_poly_offset_frontback_scale);
1194 gfx12_opt_set_context_reg(R_028B8C_PA_SU_POLY_OFFSET_BACK_OFFSET,
1195 SI_TRACKED_PA_SU_POLY_OFFSET_BACK_OFFSET,
1196 state->pa_su_poly_offset_frontback_offset[db_format_index]);
1197 }
1198 gfx12_end_context_regs();
1199 radeon_end(); /* don't track context rolls on GFX12 */
1200 } else if (sctx->screen->info.has_set_context_pairs_packed) {
1201 radeon_begin(&sctx->gfx_cs);
1202 gfx11_begin_packed_context_regs();
1203 gfx11_opt_set_context_reg(R_0286D4_SPI_INTERP_CONTROL_0, SI_TRACKED_SPI_INTERP_CONTROL_0,
1204 state->spi_interp_control_0);
1205 gfx11_opt_set_context_reg(R_028A00_PA_SU_POINT_SIZE, SI_TRACKED_PA_SU_POINT_SIZE,
1206 state->pa_su_point_size);
1207 gfx11_opt_set_context_reg(R_028A04_PA_SU_POINT_MINMAX, SI_TRACKED_PA_SU_POINT_MINMAX,
1208 state->pa_su_point_minmax);
1209 gfx11_opt_set_context_reg(R_028A08_PA_SU_LINE_CNTL, SI_TRACKED_PA_SU_LINE_CNTL,
1210 state->pa_su_line_cntl);
1211 gfx11_opt_set_context_reg(R_028A48_PA_SC_MODE_CNTL_0, SI_TRACKED_PA_SC_MODE_CNTL_0,
1212 state->pa_sc_mode_cntl_0);
1213 gfx11_opt_set_context_reg(R_028814_PA_SU_SC_MODE_CNTL, SI_TRACKED_PA_SU_SC_MODE_CNTL,
1214 state->pa_su_sc_mode_cntl);
1215 gfx11_opt_set_context_reg(R_028838_PA_CL_NGG_CNTL, SI_TRACKED_PA_CL_NGG_CNTL,
1216 state->pa_cl_ngg_cntl);
1217 gfx11_opt_set_context_reg(R_028230_PA_SC_EDGERULE, SI_TRACKED_PA_SC_EDGERULE,
1218 state->pa_sc_edgerule);
1219
1220 if (state->uses_poly_offset && sctx->framebuffer.state.zsbuf) {
1221 unsigned db_format_index =
1222 ((struct si_surface *)sctx->framebuffer.state.zsbuf)->db_format_index;
1223
1224 gfx11_opt_set_context_reg(R_028B78_PA_SU_POLY_OFFSET_DB_FMT_CNTL,
1225 SI_TRACKED_PA_SU_POLY_OFFSET_DB_FMT_CNTL,
1226 state->pa_su_poly_offset_db_fmt_cntl[db_format_index]);
1227 gfx11_opt_set_context_reg(R_028B7C_PA_SU_POLY_OFFSET_CLAMP,
1228 SI_TRACKED_PA_SU_POLY_OFFSET_CLAMP,
1229 state->pa_su_poly_offset_clamp);
1230 gfx11_opt_set_context_reg(R_028B80_PA_SU_POLY_OFFSET_FRONT_SCALE,
1231 SI_TRACKED_PA_SU_POLY_OFFSET_FRONT_SCALE,
1232 state->pa_su_poly_offset_frontback_scale);
1233 gfx11_opt_set_context_reg(R_028B84_PA_SU_POLY_OFFSET_FRONT_OFFSET,
1234 SI_TRACKED_PA_SU_POLY_OFFSET_FRONT_OFFSET,
1235 state->pa_su_poly_offset_frontback_offset[db_format_index]);
1236 gfx11_opt_set_context_reg(R_028B88_PA_SU_POLY_OFFSET_BACK_SCALE,
1237 SI_TRACKED_PA_SU_POLY_OFFSET_BACK_SCALE,
1238 state->pa_su_poly_offset_frontback_scale);
1239 gfx11_opt_set_context_reg(R_028B8C_PA_SU_POLY_OFFSET_BACK_OFFSET,
1240 SI_TRACKED_PA_SU_POLY_OFFSET_BACK_OFFSET,
1241 state->pa_su_poly_offset_frontback_offset[db_format_index]);
1242 }
1243 gfx11_end_packed_context_regs();
1244 radeon_end(); /* don't track context rolls on GFX11 */
1245 } else {
1246 radeon_begin(&sctx->gfx_cs);
1247 radeon_opt_set_context_reg(R_0286D4_SPI_INTERP_CONTROL_0,
1248 SI_TRACKED_SPI_INTERP_CONTROL_0,
1249 state->spi_interp_control_0);
1250 radeon_opt_set_context_reg(R_028A00_PA_SU_POINT_SIZE, SI_TRACKED_PA_SU_POINT_SIZE,
1251 state->pa_su_point_size);
1252 radeon_opt_set_context_reg(R_028A04_PA_SU_POINT_MINMAX, SI_TRACKED_PA_SU_POINT_MINMAX,
1253 state->pa_su_point_minmax);
1254 radeon_opt_set_context_reg(R_028A08_PA_SU_LINE_CNTL, SI_TRACKED_PA_SU_LINE_CNTL,
1255 state->pa_su_line_cntl);
1256 radeon_opt_set_context_reg(R_028A48_PA_SC_MODE_CNTL_0, SI_TRACKED_PA_SC_MODE_CNTL_0,
1257 state->pa_sc_mode_cntl_0);
1258 radeon_opt_set_context_reg(R_028814_PA_SU_SC_MODE_CNTL,
1259 SI_TRACKED_PA_SU_SC_MODE_CNTL, state->pa_su_sc_mode_cntl);
1260 if (sctx->gfx_level >= GFX10) {
1261 radeon_opt_set_context_reg(R_028838_PA_CL_NGG_CNTL, SI_TRACKED_PA_CL_NGG_CNTL,
1262 state->pa_cl_ngg_cntl);
1263 }
1264 radeon_opt_set_context_reg(R_028230_PA_SC_EDGERULE, SI_TRACKED_PA_SC_EDGERULE,
1265 state->pa_sc_edgerule);
1266
1267 if (state->uses_poly_offset && sctx->framebuffer.state.zsbuf) {
1268 unsigned db_format_index =
1269 ((struct si_surface *)sctx->framebuffer.state.zsbuf)->db_format_index;
1270
1271 radeon_opt_set_context_reg6(R_028B78_PA_SU_POLY_OFFSET_DB_FMT_CNTL,
1272 SI_TRACKED_PA_SU_POLY_OFFSET_DB_FMT_CNTL,
1273 state->pa_su_poly_offset_db_fmt_cntl[db_format_index],
1274 state->pa_su_poly_offset_clamp,
1275 state->pa_su_poly_offset_frontback_scale,
1276 state->pa_su_poly_offset_frontback_offset[db_format_index],
1277 state->pa_su_poly_offset_frontback_scale,
1278 state->pa_su_poly_offset_frontback_offset[db_format_index]);
1279 }
1280 radeon_end_update_context_roll();
1281 }
1282
1283 sctx->emitted.named.rasterizer = state;
1284 }
1285
si_bind_rs_state(struct pipe_context * ctx,void * state)1286 static void si_bind_rs_state(struct pipe_context *ctx, void *state)
1287 {
1288 struct si_context *sctx = (struct si_context *)ctx;
1289 struct si_state_rasterizer *old_rs = (struct si_state_rasterizer *)sctx->queued.named.rasterizer;
1290 struct si_state_rasterizer *rs = (struct si_state_rasterizer *)state;
1291
1292 if (!rs)
1293 rs = (struct si_state_rasterizer *)sctx->discard_rasterizer_state;
1294
1295 if (old_rs->multisample_enable != rs->multisample_enable) {
1296 si_mark_atom_dirty(sctx, &sctx->atoms.s.msaa_config);
1297
1298 /* Update the small primitive filter workaround if necessary. */
1299 if (sctx->screen->info.has_small_prim_filter_sample_loc_bug && sctx->framebuffer.nr_samples > 1)
1300 si_mark_atom_dirty(sctx, &sctx->atoms.s.sample_locations);
1301
1302 /* NGG cull state uses multisample_enable. */
1303 if (sctx->screen->use_ngg_culling)
1304 si_mark_atom_dirty(sctx, &sctx->atoms.s.ngg_cull_state);
1305 }
1306
1307 if (old_rs->perpendicular_end_caps != rs->perpendicular_end_caps)
1308 si_mark_atom_dirty(sctx, &sctx->atoms.s.msaa_config);
1309
1310 if (sctx->screen->use_ngg_culling &&
1311 (old_rs->half_pixel_center != rs->half_pixel_center ||
1312 old_rs->line_width != rs->line_width))
1313 si_mark_atom_dirty(sctx, &sctx->atoms.s.ngg_cull_state);
1314
1315 SET_FIELD(sctx->current_vs_state, VS_STATE_CLAMP_VERTEX_COLOR, rs->clamp_vertex_color);
1316
1317 si_pm4_bind_state(sctx, rasterizer, rs);
1318 si_update_ngg_cull_face_state(sctx);
1319
1320 if (old_rs->scissor_enable != rs->scissor_enable)
1321 si_mark_atom_dirty(sctx, &sctx->atoms.s.scissors);
1322
1323 /* This never changes for OpenGL. */
1324 if (old_rs->half_pixel_center != rs->half_pixel_center)
1325 si_mark_atom_dirty(sctx, &sctx->atoms.s.guardband);
1326
1327 if (util_prim_is_lines(sctx->current_rast_prim))
1328 si_set_clip_discard_distance(sctx, rs->line_width);
1329 else if (sctx->current_rast_prim == MESA_PRIM_POINTS)
1330 si_set_clip_discard_distance(sctx, rs->max_point_size);
1331
1332 if (old_rs->clip_halfz != rs->clip_halfz)
1333 si_mark_atom_dirty(sctx, &sctx->atoms.s.viewports);
1334
1335 if (old_rs->clip_plane_enable != rs->clip_plane_enable ||
1336 old_rs->pa_cl_clip_cntl != rs->pa_cl_clip_cntl)
1337 si_mark_atom_dirty(sctx, &sctx->atoms.s.clip_regs);
1338
1339 if (old_rs->sprite_coord_enable != rs->sprite_coord_enable ||
1340 old_rs->flatshade != rs->flatshade)
1341 si_mark_atom_dirty(sctx, &sctx->atoms.s.spi_map);
1342
1343 if (sctx->screen->dpbb_allowed && (old_rs->bottom_edge_rule != rs->bottom_edge_rule))
1344 si_mark_atom_dirty(sctx, &sctx->atoms.s.dpbb_state);
1345
1346 if (old_rs->multisample_enable != rs->multisample_enable)
1347 si_ps_key_update_framebuffer_blend_dsa_rasterizer(sctx);
1348
1349 if (old_rs->flatshade != rs->flatshade ||
1350 old_rs->clamp_fragment_color != rs->clamp_fragment_color)
1351 si_ps_key_update_rasterizer(sctx);
1352
1353 if (old_rs->flatshade != rs->flatshade ||
1354 old_rs->multisample_enable != rs->multisample_enable)
1355 si_ps_key_update_framebuffer_rasterizer_sample_shading(sctx);
1356
1357 if (old_rs->rasterizer_discard != rs->rasterizer_discard ||
1358 old_rs->two_side != rs->two_side ||
1359 old_rs->poly_stipple_enable != rs->poly_stipple_enable ||
1360 old_rs->point_smooth != rs->point_smooth)
1361 si_update_ps_inputs_read_or_disabled(sctx);
1362
1363 if (old_rs->point_smooth != rs->point_smooth ||
1364 old_rs->line_smooth != rs->line_smooth ||
1365 old_rs->poly_smooth != rs->poly_smooth ||
1366 old_rs->polygon_mode_is_points != rs->polygon_mode_is_points ||
1367 old_rs->poly_stipple_enable != rs->poly_stipple_enable ||
1368 old_rs->two_side != rs->two_side ||
1369 old_rs->force_front_face_input != rs->force_front_face_input)
1370 si_vs_ps_key_update_rast_prim_smooth_stipple(sctx);
1371
1372 /* Used by si_get_vs_key_outputs in si_update_shaders: */
1373 if (old_rs->clip_plane_enable != rs->clip_plane_enable)
1374 sctx->do_update_shaders = true;
1375
1376 if (old_rs->line_smooth != rs->line_smooth ||
1377 old_rs->poly_smooth != rs->poly_smooth ||
1378 old_rs->point_smooth != rs->point_smooth ||
1379 old_rs->poly_stipple_enable != rs->poly_stipple_enable ||
1380 old_rs->flatshade != rs->flatshade)
1381 si_update_vrs_flat_shading(sctx);
1382
1383 if (old_rs->flatshade_first != rs->flatshade_first)
1384 si_update_ngg_sgpr_state_provoking_vtx(sctx, si_get_vs(sctx)->current, sctx->ngg);
1385 }
1386
si_delete_rs_state(struct pipe_context * ctx,void * state)1387 static void si_delete_rs_state(struct pipe_context *ctx, void *state)
1388 {
1389 struct si_context *sctx = (struct si_context *)ctx;
1390 struct si_state_rasterizer *rs = (struct si_state_rasterizer *)state;
1391
1392 if (sctx->queued.named.rasterizer == state)
1393 si_bind_rs_state(ctx, sctx->discard_rasterizer_state);
1394
1395 si_pm4_free_state(sctx, &rs->pm4, SI_STATE_IDX(rasterizer));
1396 }
1397
1398 /*
1399 * inferred state between dsa and stencil ref
1400 */
si_emit_stencil_ref(struct si_context * sctx,unsigned index)1401 static void si_emit_stencil_ref(struct si_context *sctx, unsigned index)
1402 {
1403 struct pipe_stencil_ref *ref = &sctx->stencil_ref.state;
1404
1405 if (sctx->gfx_level >= GFX12) {
1406 radeon_begin(&sctx->gfx_cs);
1407 radeon_set_context_reg(R_028088_DB_STENCIL_REF,
1408 S_028088_TESTVAL(ref->ref_value[0]) |
1409 S_028088_TESTVAL_BF(ref->ref_value[1]));
1410 radeon_end();
1411 } else {
1412 struct si_dsa_stencil_ref_part *dsa = &sctx->stencil_ref.dsa_part;
1413
1414 radeon_begin(&sctx->gfx_cs);
1415 radeon_set_context_reg_seq(R_028430_DB_STENCILREFMASK, 2);
1416 radeon_emit(S_028430_STENCILTESTVAL(ref->ref_value[0]) |
1417 S_028430_STENCILMASK(dsa->valuemask[0]) |
1418 S_028430_STENCILWRITEMASK(dsa->writemask[0]) |
1419 S_028430_STENCILOPVAL(1));
1420 radeon_emit(S_028434_STENCILTESTVAL_BF(ref->ref_value[1]) |
1421 S_028434_STENCILMASK_BF(dsa->valuemask[1]) |
1422 S_028434_STENCILWRITEMASK_BF(dsa->writemask[1]) |
1423 S_028434_STENCILOPVAL_BF(1));
1424 radeon_end();
1425 }
1426 }
1427
si_set_stencil_ref(struct pipe_context * ctx,const struct pipe_stencil_ref state)1428 static void si_set_stencil_ref(struct pipe_context *ctx, const struct pipe_stencil_ref state)
1429 {
1430 struct si_context *sctx = (struct si_context *)ctx;
1431
1432 if (memcmp(&sctx->stencil_ref.state, &state, sizeof(state)) == 0)
1433 return;
1434
1435 sctx->stencil_ref.state = state;
1436 si_mark_atom_dirty(sctx, &sctx->atoms.s.stencil_ref);
1437 }
1438
1439 /*
1440 * DSA
1441 */
1442
si_translate_stencil_op(int s_op)1443 static uint32_t si_translate_stencil_op(int s_op)
1444 {
1445 switch (s_op) {
1446 case PIPE_STENCIL_OP_KEEP:
1447 return V_02842C_STENCIL_KEEP;
1448 case PIPE_STENCIL_OP_ZERO:
1449 return V_02842C_STENCIL_ZERO;
1450 case PIPE_STENCIL_OP_REPLACE:
1451 return V_02842C_STENCIL_REPLACE_TEST;
1452 case PIPE_STENCIL_OP_INCR:
1453 return V_02842C_STENCIL_ADD_CLAMP;
1454 case PIPE_STENCIL_OP_DECR:
1455 return V_02842C_STENCIL_SUB_CLAMP;
1456 case PIPE_STENCIL_OP_INCR_WRAP:
1457 return V_02842C_STENCIL_ADD_WRAP;
1458 case PIPE_STENCIL_OP_DECR_WRAP:
1459 return V_02842C_STENCIL_SUB_WRAP;
1460 case PIPE_STENCIL_OP_INVERT:
1461 return V_02842C_STENCIL_INVERT;
1462 default:
1463 PRINT_ERR("Unknown stencil op %d", s_op);
1464 assert(0);
1465 break;
1466 }
1467 return 0;
1468 }
1469
si_order_invariant_stencil_op(enum pipe_stencil_op op)1470 static bool si_order_invariant_stencil_op(enum pipe_stencil_op op)
1471 {
1472 /* REPLACE is normally order invariant, except when the stencil
1473 * reference value is written by the fragment shader. Tracking this
1474 * interaction does not seem worth the effort, so be conservative. */
1475 return op != PIPE_STENCIL_OP_INCR && op != PIPE_STENCIL_OP_DECR && op != PIPE_STENCIL_OP_REPLACE;
1476 }
1477
1478 /* Compute whether, assuming Z writes are disabled, this stencil state is order
1479 * invariant in the sense that the set of passing fragments as well as the
1480 * final stencil buffer result does not depend on the order of fragments. */
si_order_invariant_stencil_state(const struct pipe_stencil_state * state)1481 static bool si_order_invariant_stencil_state(const struct pipe_stencil_state *state)
1482 {
1483 return !state->enabled || !state->writemask ||
1484 /* The following assumes that Z writes are disabled. */
1485 (state->func == PIPE_FUNC_ALWAYS && si_order_invariant_stencil_op(state->zpass_op) &&
1486 si_order_invariant_stencil_op(state->zfail_op)) ||
1487 (state->func == PIPE_FUNC_NEVER && si_order_invariant_stencil_op(state->fail_op));
1488 }
1489
si_create_dsa_state(struct pipe_context * ctx,const struct pipe_depth_stencil_alpha_state * state)1490 static void *si_create_dsa_state(struct pipe_context *ctx,
1491 const struct pipe_depth_stencil_alpha_state *state)
1492 {
1493 struct si_context *sctx = (struct si_context *)ctx;
1494 struct si_state_dsa *dsa = CALLOC_STRUCT(si_state_dsa);
1495 if (!dsa) {
1496 return NULL;
1497 }
1498
1499 dsa->stencil_ref.valuemask[0] = state->stencil[0].valuemask;
1500 dsa->stencil_ref.valuemask[1] = state->stencil[1].valuemask;
1501 dsa->stencil_ref.writemask[0] = state->stencil[0].writemask;
1502 dsa->stencil_ref.writemask[1] = state->stencil[1].writemask;
1503
1504 dsa->db_depth_control =
1505 S_028800_Z_ENABLE(state->depth_enabled) | S_028800_Z_WRITE_ENABLE(state->depth_writemask) |
1506 S_028800_ZFUNC(state->depth_func) | S_028800_DEPTH_BOUNDS_ENABLE(state->depth_bounds_test);
1507
1508 /* stencil */
1509 if (state->stencil[0].enabled) {
1510 dsa->db_depth_control |= S_028800_STENCIL_ENABLE(1);
1511 dsa->db_depth_control |= S_028800_STENCILFUNC(state->stencil[0].func);
1512 dsa->db_stencil_control |=
1513 S_02842C_STENCILFAIL(si_translate_stencil_op(state->stencil[0].fail_op));
1514 dsa->db_stencil_control |=
1515 S_02842C_STENCILZPASS(si_translate_stencil_op(state->stencil[0].zpass_op));
1516 dsa->db_stencil_control |=
1517 S_02842C_STENCILZFAIL(si_translate_stencil_op(state->stencil[0].zfail_op));
1518
1519 if (state->stencil[1].enabled) {
1520 dsa->db_depth_control |= S_028800_BACKFACE_ENABLE(1);
1521 dsa->db_depth_control |= S_028800_STENCILFUNC_BF(state->stencil[1].func);
1522 dsa->db_stencil_control |=
1523 S_02842C_STENCILFAIL_BF(si_translate_stencil_op(state->stencil[1].fail_op));
1524 dsa->db_stencil_control |=
1525 S_02842C_STENCILZPASS_BF(si_translate_stencil_op(state->stencil[1].zpass_op));
1526 dsa->db_stencil_control |=
1527 S_02842C_STENCILZFAIL_BF(si_translate_stencil_op(state->stencil[1].zfail_op));
1528 }
1529 }
1530
1531 dsa->db_depth_bounds_min = fui(state->depth_bounds_min);
1532 dsa->db_depth_bounds_max = fui(state->depth_bounds_max);
1533
1534 /* alpha */
1535 if (state->alpha_enabled) {
1536 dsa->alpha_func = state->alpha_func;
1537 dsa->spi_shader_user_data_ps_alpha_ref = fui(state->alpha_ref_value);
1538 } else {
1539 dsa->alpha_func = PIPE_FUNC_ALWAYS;
1540 }
1541
1542 dsa->depth_enabled = state->depth_enabled &&
1543 (state->depth_writemask || state->depth_func != PIPE_FUNC_ALWAYS);
1544 dsa->depth_write_enabled = state->depth_enabled && state->depth_writemask;
1545 dsa->stencil_enabled = state->stencil[0].enabled;
1546 dsa->stencil_write_enabled =
1547 (util_writes_stencil(&state->stencil[0]) || util_writes_stencil(&state->stencil[1]));
1548 dsa->db_can_write = dsa->depth_write_enabled || dsa->stencil_write_enabled;
1549 dsa->depth_bounds_enabled = state->depth_bounds_test;
1550
1551 if (sctx->gfx_level >= GFX12) {
1552 dsa->db_stencil_read_mask = S_028090_TESTMASK(state->stencil[0].valuemask) |
1553 S_028090_TESTMASK_BF(state->stencil[1].valuemask);
1554 dsa->db_stencil_write_mask = S_028094_WRITEMASK(state->stencil[0].writemask) |
1555 S_028094_WRITEMASK_BF(state->stencil[1].writemask);
1556
1557 bool force_s_valid = state->stencil[0].zpass_op != state->stencil[0].zfail_op ||
1558 (state->stencil[1].enabled &&
1559 state->stencil[1].zpass_op != state->stencil[1].zfail_op);
1560 dsa->db_render_override = S_02800C_FORCE_STENCIL_READ(1) |
1561 S_02800C_FORCE_STENCIL_VALID(force_s_valid);
1562 }
1563
1564 bool zfunc_is_ordered =
1565 state->depth_func == PIPE_FUNC_NEVER || state->depth_func == PIPE_FUNC_LESS ||
1566 state->depth_func == PIPE_FUNC_LEQUAL || state->depth_func == PIPE_FUNC_GREATER ||
1567 state->depth_func == PIPE_FUNC_GEQUAL;
1568
1569 bool nozwrite_and_order_invariant_stencil =
1570 !dsa->db_can_write ||
1571 (!dsa->depth_write_enabled && si_order_invariant_stencil_state(&state->stencil[0]) &&
1572 si_order_invariant_stencil_state(&state->stencil[1]));
1573
1574 dsa->order_invariance[1].zs =
1575 nozwrite_and_order_invariant_stencil || (!dsa->stencil_write_enabled && zfunc_is_ordered);
1576 dsa->order_invariance[0].zs = !dsa->depth_write_enabled || zfunc_is_ordered;
1577
1578 dsa->order_invariance[1].pass_set =
1579 nozwrite_and_order_invariant_stencil ||
1580 (!dsa->stencil_write_enabled &&
1581 (state->depth_func == PIPE_FUNC_ALWAYS || state->depth_func == PIPE_FUNC_NEVER));
1582 dsa->order_invariance[0].pass_set =
1583 !dsa->depth_write_enabled ||
1584 (state->depth_func == PIPE_FUNC_ALWAYS || state->depth_func == PIPE_FUNC_NEVER);
1585
1586 return dsa;
1587 }
1588
si_pm4_emit_dsa(struct si_context * sctx,unsigned index)1589 static void si_pm4_emit_dsa(struct si_context *sctx, unsigned index)
1590 {
1591 struct si_state_dsa *state = sctx->queued.named.dsa;
1592 assert(state && state != sctx->emitted.named.dsa);
1593
1594 if (sctx->gfx_level >= GFX12) {
1595 radeon_begin(&sctx->gfx_cs);
1596 gfx12_begin_context_regs();
1597 gfx12_opt_set_context_reg(R_02800C_DB_RENDER_OVERRIDE, SI_TRACKED_DB_RENDER_OVERRIDE,
1598 state->db_render_override);
1599 gfx12_opt_set_context_reg(R_028070_DB_DEPTH_CONTROL, SI_TRACKED_DB_DEPTH_CONTROL,
1600 state->db_depth_control);
1601 if (state->stencil_enabled) {
1602 gfx12_opt_set_context_reg(R_028074_DB_STENCIL_CONTROL, SI_TRACKED_DB_STENCIL_CONTROL,
1603 state->db_stencil_control);
1604 gfx12_opt_set_context_reg(R_028090_DB_STENCIL_READ_MASK, SI_TRACKED_DB_STENCIL_READ_MASK,
1605 state->db_stencil_read_mask);
1606 gfx12_opt_set_context_reg(R_028094_DB_STENCIL_WRITE_MASK, SI_TRACKED_DB_STENCIL_WRITE_MASK,
1607 state->db_stencil_write_mask);
1608 }
1609 if (state->depth_bounds_enabled) {
1610 gfx12_opt_set_context_reg(R_028050_DB_DEPTH_BOUNDS_MIN, SI_TRACKED_DB_DEPTH_BOUNDS_MIN,
1611 state->db_depth_bounds_min);
1612 gfx12_opt_set_context_reg(R_028054_DB_DEPTH_BOUNDS_MAX, SI_TRACKED_DB_DEPTH_BOUNDS_MAX,
1613 state->db_depth_bounds_max);
1614 }
1615 gfx12_end_context_regs();
1616 radeon_end(); /* don't track context rolls on GFX12 */
1617
1618 if (state->alpha_func != PIPE_FUNC_ALWAYS && state->alpha_func != PIPE_FUNC_NEVER) {
1619 gfx12_opt_push_gfx_sh_reg(R_00B030_SPI_SHADER_USER_DATA_PS_0 + SI_SGPR_ALPHA_REF * 4,
1620 SI_TRACKED_SPI_SHADER_USER_DATA_PS__ALPHA_REF,
1621 state->spi_shader_user_data_ps_alpha_ref);
1622 }
1623 } else if (sctx->screen->info.has_set_context_pairs_packed) {
1624 radeon_begin(&sctx->gfx_cs);
1625 gfx11_begin_packed_context_regs();
1626 gfx11_opt_set_context_reg(R_028800_DB_DEPTH_CONTROL, SI_TRACKED_DB_DEPTH_CONTROL,
1627 state->db_depth_control);
1628 if (state->stencil_enabled) {
1629 gfx11_opt_set_context_reg(R_02842C_DB_STENCIL_CONTROL, SI_TRACKED_DB_STENCIL_CONTROL,
1630 state->db_stencil_control);
1631 }
1632 if (state->depth_bounds_enabled) {
1633 gfx11_opt_set_context_reg(R_028020_DB_DEPTH_BOUNDS_MIN, SI_TRACKED_DB_DEPTH_BOUNDS_MIN,
1634 state->db_depth_bounds_min);
1635 gfx11_opt_set_context_reg(R_028024_DB_DEPTH_BOUNDS_MAX, SI_TRACKED_DB_DEPTH_BOUNDS_MAX,
1636 state->db_depth_bounds_max);
1637 }
1638 gfx11_end_packed_context_regs();
1639
1640 if (state->alpha_func != PIPE_FUNC_ALWAYS && state->alpha_func != PIPE_FUNC_NEVER) {
1641 if (sctx->screen->info.has_set_sh_pairs_packed) {
1642 gfx11_opt_push_gfx_sh_reg(R_00B030_SPI_SHADER_USER_DATA_PS_0 + SI_SGPR_ALPHA_REF * 4,
1643 SI_TRACKED_SPI_SHADER_USER_DATA_PS__ALPHA_REF,
1644 state->spi_shader_user_data_ps_alpha_ref);
1645 } else {
1646 radeon_opt_set_sh_reg(R_00B030_SPI_SHADER_USER_DATA_PS_0 + SI_SGPR_ALPHA_REF * 4,
1647 SI_TRACKED_SPI_SHADER_USER_DATA_PS__ALPHA_REF,
1648 state->spi_shader_user_data_ps_alpha_ref);
1649 }
1650 }
1651 radeon_end(); /* don't track context rolls on GFX11 */
1652 } else {
1653 radeon_begin(&sctx->gfx_cs);
1654 radeon_opt_set_context_reg(R_028800_DB_DEPTH_CONTROL, SI_TRACKED_DB_DEPTH_CONTROL,
1655 state->db_depth_control);
1656 if (state->stencil_enabled) {
1657 radeon_opt_set_context_reg(R_02842C_DB_STENCIL_CONTROL, SI_TRACKED_DB_STENCIL_CONTROL,
1658 state->db_stencil_control);
1659 }
1660 if (state->depth_bounds_enabled) {
1661 radeon_opt_set_context_reg2(R_028020_DB_DEPTH_BOUNDS_MIN,
1662 SI_TRACKED_DB_DEPTH_BOUNDS_MIN,
1663 state->db_depth_bounds_min,
1664 state->db_depth_bounds_max);
1665 }
1666 radeon_end_update_context_roll();
1667
1668 if (state->alpha_func != PIPE_FUNC_ALWAYS && state->alpha_func != PIPE_FUNC_NEVER) {
1669 radeon_begin(&sctx->gfx_cs);
1670 radeon_opt_set_sh_reg(R_00B030_SPI_SHADER_USER_DATA_PS_0 + SI_SGPR_ALPHA_REF * 4,
1671 SI_TRACKED_SPI_SHADER_USER_DATA_PS__ALPHA_REF,
1672 state->spi_shader_user_data_ps_alpha_ref);
1673 radeon_end();
1674 }
1675 }
1676
1677 sctx->emitted.named.dsa = state;
1678 }
1679
si_bind_dsa_state(struct pipe_context * ctx,void * state)1680 static void si_bind_dsa_state(struct pipe_context *ctx, void *state)
1681 {
1682 struct si_context *sctx = (struct si_context *)ctx;
1683 struct si_state_dsa *old_dsa = sctx->queued.named.dsa;
1684 struct si_state_dsa *dsa = state;
1685
1686 if (!dsa)
1687 dsa = (struct si_state_dsa *)sctx->noop_dsa;
1688
1689 si_pm4_bind_state(sctx, dsa, dsa);
1690
1691 /* Gfx12 doesn't need to combine a DSA state with a stencil ref state. */
1692 if (sctx->gfx_level < GFX12 &&
1693 memcmp(&dsa->stencil_ref, &sctx->stencil_ref.dsa_part,
1694 sizeof(struct si_dsa_stencil_ref_part)) != 0) {
1695 sctx->stencil_ref.dsa_part = dsa->stencil_ref;
1696 si_mark_atom_dirty(sctx, &sctx->atoms.s.stencil_ref);
1697 }
1698
1699 struct pipe_surface *zssurf = sctx->framebuffer.state.zsbuf;
1700 struct si_texture *zstex = (struct si_texture*)(zssurf ? zssurf->texture : NULL);
1701
1702 if (sctx->gfx_level == GFX12 && !sctx->screen->options.alt_hiz_logic &&
1703 sctx->framebuffer.has_stencil && dsa->stencil_enabled && !zstex->force_disable_hiz_his) {
1704 zstex->force_disable_hiz_his = true;
1705 si_mark_atom_dirty(sctx, &sctx->atoms.s.framebuffer);
1706
1707 if (sctx->framebuffer.has_hiz_his) {
1708 sctx->framebuffer.has_hiz_his = false;
1709 si_mark_atom_dirty(sctx, &sctx->atoms.s.msaa_config);
1710 }
1711 }
1712
1713 if (old_dsa->alpha_func != dsa->alpha_func) {
1714 si_ps_key_update_dsa(sctx);
1715 si_update_ps_inputs_read_or_disabled(sctx);
1716 sctx->do_update_shaders = true;
1717 }
1718
1719 if (old_dsa->depth_enabled != dsa->depth_enabled ||
1720 old_dsa->stencil_enabled != dsa->stencil_enabled) {
1721 si_ps_key_update_framebuffer_blend_dsa_rasterizer(sctx);
1722 sctx->do_update_shaders = true;
1723 }
1724
1725 if (sctx->occlusion_query_mode == SI_OCCLUSION_QUERY_MODE_PRECISE_BOOLEAN &&
1726 (old_dsa->depth_enabled != dsa->depth_enabled ||
1727 old_dsa->depth_write_enabled != dsa->depth_write_enabled))
1728 si_mark_atom_dirty(sctx, &sctx->atoms.s.db_render_state);
1729
1730 if (sctx->screen->dpbb_allowed && ((old_dsa->depth_enabled != dsa->depth_enabled ||
1731 old_dsa->stencil_enabled != dsa->stencil_enabled ||
1732 old_dsa->db_can_write != dsa->db_can_write)))
1733 si_mark_atom_dirty(sctx, &sctx->atoms.s.dpbb_state);
1734
1735 if (sctx->screen->info.has_out_of_order_rast &&
1736 (memcmp(old_dsa->order_invariance, dsa->order_invariance,
1737 sizeof(old_dsa->order_invariance))))
1738 si_mark_atom_dirty(sctx, &sctx->atoms.s.msaa_config);
1739 }
1740
si_delete_dsa_state(struct pipe_context * ctx,void * state)1741 static void si_delete_dsa_state(struct pipe_context *ctx, void *state)
1742 {
1743 struct si_context *sctx = (struct si_context *)ctx;
1744
1745 if (sctx->queued.named.dsa == state)
1746 si_bind_dsa_state(ctx, sctx->noop_dsa);
1747
1748 si_pm4_free_state(sctx, (struct si_pm4_state*)state, SI_STATE_IDX(dsa));
1749 }
1750
si_create_db_flush_dsa(struct si_context * sctx)1751 static void *si_create_db_flush_dsa(struct si_context *sctx)
1752 {
1753 struct pipe_depth_stencil_alpha_state dsa = {};
1754
1755 return sctx->b.create_depth_stencil_alpha_state(&sctx->b, &dsa);
1756 }
1757
1758 /* DB RENDER STATE */
1759
si_set_active_query_state(struct pipe_context * ctx,bool enable)1760 static void si_set_active_query_state(struct pipe_context *ctx, bool enable)
1761 {
1762 struct si_context *sctx = (struct si_context *)ctx;
1763
1764 /* Pipeline stat & streamout queries. */
1765 if (enable) {
1766 /* Disable pipeline stats if there are no active queries. */
1767 if (sctx->num_hw_pipestat_streamout_queries) {
1768 sctx->barrier_flags &= ~SI_BARRIER_EVENT_PIPELINESTAT_STOP;
1769 sctx->barrier_flags |= SI_BARRIER_EVENT_PIPELINESTAT_START;
1770 si_mark_atom_dirty(sctx, &sctx->atoms.s.barrier);
1771 }
1772 } else {
1773 if (sctx->num_hw_pipestat_streamout_queries) {
1774 sctx->barrier_flags &= ~SI_BARRIER_EVENT_PIPELINESTAT_START;
1775 sctx->barrier_flags |= SI_BARRIER_EVENT_PIPELINESTAT_STOP;
1776 si_mark_atom_dirty(sctx, &sctx->atoms.s.barrier);
1777 }
1778 }
1779
1780 /* Occlusion queries. */
1781 if (sctx->occlusion_queries_disabled != !enable) {
1782 sctx->occlusion_queries_disabled = !enable;
1783 si_mark_atom_dirty(sctx, &sctx->atoms.s.db_render_state);
1784 }
1785 }
1786
si_save_qbo_state(struct si_context * sctx,struct si_qbo_state * st)1787 void si_save_qbo_state(struct si_context *sctx, struct si_qbo_state *st)
1788 {
1789 si_get_pipe_constant_buffer(sctx, PIPE_SHADER_COMPUTE, 0, &st->saved_const0);
1790 }
1791
si_restore_qbo_state(struct si_context * sctx,struct si_qbo_state * st)1792 void si_restore_qbo_state(struct si_context *sctx, struct si_qbo_state *st)
1793 {
1794 sctx->b.set_constant_buffer(&sctx->b, PIPE_SHADER_COMPUTE, 0, true, &st->saved_const0);
1795 }
1796
si_emit_db_render_state(struct si_context * sctx,unsigned index)1797 static void si_emit_db_render_state(struct si_context *sctx, unsigned index)
1798 {
1799 unsigned db_shader_control = 0, db_render_control = 0, db_count_control = 0, vrs_override_cntl = 0;
1800
1801 /* DB_RENDER_CONTROL */
1802 /* Program OREO_MODE optimally for GFX11+. */
1803 if (sctx->gfx_level >= GFX11) {
1804 bool z_export = G_02880C_Z_EXPORT_ENABLE(sctx->ps_db_shader_control);
1805 db_render_control |= S_028000_OREO_MODE(z_export ? V_028000_OMODE_BLEND : V_028000_OMODE_O_THEN_B);
1806 }
1807
1808 if (sctx->gfx_level >= GFX12) {
1809 assert(!sctx->dbcb_depth_copy_enabled && !sctx->dbcb_stencil_copy_enabled);
1810 assert(!sctx->db_flush_depth_inplace && !sctx->db_flush_stencil_inplace);
1811 assert(!sctx->db_depth_clear && !sctx->db_stencil_clear);
1812 } else {
1813 if (sctx->dbcb_depth_copy_enabled || sctx->dbcb_stencil_copy_enabled) {
1814 db_render_control |= S_028000_DEPTH_COPY(sctx->dbcb_depth_copy_enabled) |
1815 S_028000_STENCIL_COPY(sctx->dbcb_stencil_copy_enabled) |
1816 S_028000_COPY_CENTROID(1) | S_028000_COPY_SAMPLE(sctx->dbcb_copy_sample);
1817 } else if (sctx->db_flush_depth_inplace || sctx->db_flush_stencil_inplace) {
1818 db_render_control |= S_028000_DEPTH_COMPRESS_DISABLE(sctx->db_flush_depth_inplace) |
1819 S_028000_STENCIL_COMPRESS_DISABLE(sctx->db_flush_stencil_inplace);
1820 } else {
1821 db_render_control |= S_028000_DEPTH_CLEAR_ENABLE(sctx->db_depth_clear) |
1822 S_028000_STENCIL_CLEAR_ENABLE(sctx->db_stencil_clear);
1823 }
1824
1825 if (sctx->gfx_level >= GFX11) {
1826 unsigned max_allowed_tiles_in_wave;
1827
1828 if (sctx->screen->info.has_dedicated_vram) {
1829 if (sctx->framebuffer.nr_samples == 8)
1830 max_allowed_tiles_in_wave = 6;
1831 else if (sctx->framebuffer.nr_samples == 4)
1832 max_allowed_tiles_in_wave = 13;
1833 else
1834 max_allowed_tiles_in_wave = 0;
1835 } else {
1836 if (sctx->framebuffer.nr_samples == 8)
1837 max_allowed_tiles_in_wave = 7;
1838 else if (sctx->framebuffer.nr_samples == 4)
1839 max_allowed_tiles_in_wave = 15;
1840 else
1841 max_allowed_tiles_in_wave = 0;
1842 }
1843
1844 db_render_control |= S_028000_MAX_ALLOWED_TILES_IN_WAVE(max_allowed_tiles_in_wave);
1845 }
1846 }
1847
1848 /* DB_COUNT_CONTROL (occlusion queries) */
1849 if (sctx->occlusion_query_mode == SI_OCCLUSION_QUERY_MODE_DISABLE ||
1850 sctx->occlusion_queries_disabled) {
1851 /* Occlusion queries disabled. */
1852 if (sctx->gfx_level >= GFX7)
1853 db_count_control |= S_028004_ZPASS_ENABLE(0);
1854 else
1855 db_count_control |= S_028004_ZPASS_INCREMENT_DISABLE(1);
1856 } else {
1857 /* Occlusion queries enabled. */
1858 if (sctx->gfx_level < GFX12)
1859 db_count_control |= S_028004_SAMPLE_RATE(sctx->framebuffer.log_samples);
1860
1861 if (sctx->gfx_level >= GFX7) {
1862 db_count_control |= S_028004_ZPASS_ENABLE(1) |
1863 S_028004_SLICE_EVEN_ENABLE(1) |
1864 S_028004_SLICE_ODD_ENABLE(1);
1865 }
1866
1867 if (sctx->occlusion_query_mode == SI_OCCLUSION_QUERY_MODE_PRECISE_INTEGER ||
1868 /* Boolean occlusion queries must set PERFECT_ZPASS_COUNTS for depth-only rendering
1869 * without depth writes or when depth testing is disabled. */
1870 (sctx->occlusion_query_mode == SI_OCCLUSION_QUERY_MODE_PRECISE_BOOLEAN &&
1871 (!sctx->queued.named.dsa->depth_enabled ||
1872 (!sctx->queued.named.blend->cb_target_mask &&
1873 !sctx->queued.named.dsa->depth_write_enabled))))
1874 db_count_control |= S_028004_PERFECT_ZPASS_COUNTS(1);
1875
1876 if (sctx->gfx_level >= GFX10 &&
1877 sctx->occlusion_query_mode != SI_OCCLUSION_QUERY_MODE_CONSERVATIVE_BOOLEAN)
1878 db_count_control |= S_028004_DISABLE_CONSERVATIVE_ZPASS_COUNTS(1);
1879 }
1880
1881 /* This should always be set on GFX11. */
1882 if (sctx->gfx_level >= GFX11)
1883 db_count_control |= S_028004_DISABLE_CONSERVATIVE_ZPASS_COUNTS(1);
1884
1885 db_shader_control |= sctx->ps_db_shader_control;
1886
1887 if (sctx->screen->info.has_export_conflict_bug &&
1888 sctx->queued.named.blend->blend_enable_4bit &&
1889 si_get_num_coverage_samples(sctx) == 1) {
1890 db_shader_control |= S_02880C_OVERRIDE_INTRINSIC_RATE_ENABLE(1) |
1891 S_02880C_OVERRIDE_INTRINSIC_RATE(2);
1892 }
1893
1894 if (sctx->gfx_level >= GFX10_3) {
1895 /* Variable rate shading. */
1896 unsigned mode, log_rate_x, log_rate_y;
1897
1898 if (sctx->allow_flat_shading) {
1899 mode = V_028064_SC_VRS_COMB_MODE_OVERRIDE;
1900 log_rate_x = log_rate_y = 1; /* 2x2 VRS (log2(2) == 1) */
1901 } else {
1902 /* If the shader is using discard, turn off coarse shading because discarding at 2x2 pixel
1903 * granularity degrades quality too much.
1904 *
1905 * The shader writes the VRS rate and we either pass it through or do MIN(shader, 1x1)
1906 * to disable coarse shading.
1907 */
1908 mode = sctx->screen->options.vrs2x2 && G_02880C_KILL_ENABLE(db_shader_control) ?
1909 V_028064_SC_VRS_COMB_MODE_MIN : V_028064_SC_VRS_COMB_MODE_PASSTHRU;
1910 log_rate_x = log_rate_y = 0; /* 1x1 VRS (log2(1) == 0) */
1911 }
1912
1913 if (sctx->gfx_level >= GFX11) {
1914 vrs_override_cntl = S_0283D0_VRS_OVERRIDE_RATE_COMBINER_MODE(mode) |
1915 S_0283D0_VRS_RATE(log_rate_x * 4 + log_rate_y);
1916 } else {
1917 vrs_override_cntl = S_028064_VRS_OVERRIDE_RATE_COMBINER_MODE(mode) |
1918 S_028064_VRS_OVERRIDE_RATE_X(log_rate_x) |
1919 S_028064_VRS_OVERRIDE_RATE_Y(log_rate_y);
1920 }
1921 }
1922
1923 unsigned db_render_override2 =
1924 S_028010_DISABLE_ZMASK_EXPCLEAR_OPTIMIZATION(sctx->db_depth_disable_expclear) |
1925 S_028010_DISABLE_SMEM_EXPCLEAR_OPTIMIZATION(sctx->db_stencil_disable_expclear) |
1926 S_028010_DECOMPRESS_Z_ON_FLUSH(sctx->framebuffer.nr_samples >= 4) |
1927 S_028010_CENTROID_COMPUTATION_MODE(sctx->gfx_level >= GFX10_3 ? 1 : 0);
1928
1929 if (sctx->gfx_level >= GFX12) {
1930 radeon_begin(&sctx->gfx_cs);
1931 gfx12_begin_context_regs();
1932 gfx12_opt_set_context_reg(R_028000_DB_RENDER_CONTROL, SI_TRACKED_DB_RENDER_CONTROL,
1933 db_render_control);
1934 gfx12_opt_set_context_reg(R_028010_DB_RENDER_OVERRIDE2, SI_TRACKED_DB_RENDER_OVERRIDE2,
1935 S_028010_DECOMPRESS_Z_ON_FLUSH(sctx->framebuffer.nr_samples >= 4) |
1936 S_028010_CENTROID_COMPUTATION_MODE(1));
1937 gfx12_opt_set_context_reg(R_028060_DB_COUNT_CONTROL, SI_TRACKED_DB_COUNT_CONTROL,
1938 db_count_control);
1939 gfx12_opt_set_context_reg(R_02806C_DB_SHADER_CONTROL, SI_TRACKED_DB_SHADER_CONTROL,
1940 db_shader_control);
1941 gfx12_opt_set_context_reg(R_0283D0_PA_SC_VRS_OVERRIDE_CNTL,
1942 SI_TRACKED_DB_PA_SC_VRS_OVERRIDE_CNTL, vrs_override_cntl);
1943 gfx12_end_context_regs();
1944 radeon_end(); /* don't track context rolls on GFX12 */
1945 } else if (sctx->screen->info.has_set_context_pairs_packed) {
1946 radeon_begin(&sctx->gfx_cs);
1947 gfx11_begin_packed_context_regs();
1948 gfx11_opt_set_context_reg(R_028000_DB_RENDER_CONTROL, SI_TRACKED_DB_RENDER_CONTROL,
1949 db_render_control);
1950 gfx11_opt_set_context_reg(R_028004_DB_COUNT_CONTROL, SI_TRACKED_DB_COUNT_CONTROL,
1951 db_count_control);
1952 gfx11_opt_set_context_reg(R_028010_DB_RENDER_OVERRIDE2, SI_TRACKED_DB_RENDER_OVERRIDE2,
1953 db_render_override2);
1954 gfx11_opt_set_context_reg(R_02880C_DB_SHADER_CONTROL, SI_TRACKED_DB_SHADER_CONTROL,
1955 db_shader_control);
1956 gfx11_opt_set_context_reg(R_0283D0_PA_SC_VRS_OVERRIDE_CNTL,
1957 SI_TRACKED_DB_PA_SC_VRS_OVERRIDE_CNTL, vrs_override_cntl);
1958 gfx11_end_packed_context_regs();
1959 radeon_end(); /* don't track context rolls on GFX11 */
1960 } else {
1961 radeon_begin(&sctx->gfx_cs);
1962 radeon_opt_set_context_reg2(R_028000_DB_RENDER_CONTROL, SI_TRACKED_DB_RENDER_CONTROL,
1963 db_render_control, db_count_control);
1964 radeon_opt_set_context_reg(R_028010_DB_RENDER_OVERRIDE2,
1965 SI_TRACKED_DB_RENDER_OVERRIDE2, db_render_override2);
1966 radeon_opt_set_context_reg(R_02880C_DB_SHADER_CONTROL, SI_TRACKED_DB_SHADER_CONTROL,
1967 db_shader_control);
1968
1969 if (sctx->gfx_level >= GFX11) {
1970 radeon_opt_set_context_reg(R_0283D0_PA_SC_VRS_OVERRIDE_CNTL,
1971 SI_TRACKED_DB_PA_SC_VRS_OVERRIDE_CNTL, vrs_override_cntl);
1972 } else if (sctx->gfx_level >= GFX10_3) {
1973 radeon_opt_set_context_reg(R_028064_DB_VRS_OVERRIDE_CNTL,
1974 SI_TRACKED_DB_PA_SC_VRS_OVERRIDE_CNTL, vrs_override_cntl);
1975 }
1976 radeon_end_update_context_roll();
1977 }
1978 }
1979
1980 /*
1981 * Texture translation
1982 */
1983
si_translate_texformat(struct pipe_screen * screen,enum pipe_format format,const struct util_format_description * desc,int first_non_void)1984 static uint32_t si_translate_texformat(struct pipe_screen *screen, enum pipe_format format,
1985 const struct util_format_description *desc,
1986 int first_non_void)
1987 {
1988 struct si_screen *sscreen = (struct si_screen *)screen;
1989
1990 assert(sscreen->info.gfx_level <= GFX9);
1991
1992 return ac_translate_tex_dataformat(&sscreen->info, desc, first_non_void);
1993 }
1994
is_wrap_mode_legal(struct si_screen * screen,unsigned wrap)1995 static unsigned is_wrap_mode_legal(struct si_screen *screen, unsigned wrap)
1996 {
1997 if (!screen->info.has_3d_cube_border_color_mipmap) {
1998 switch (wrap) {
1999 case PIPE_TEX_WRAP_CLAMP:
2000 case PIPE_TEX_WRAP_CLAMP_TO_BORDER:
2001 case PIPE_TEX_WRAP_MIRROR_CLAMP:
2002 case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER:
2003 return false;
2004 }
2005 }
2006 return true;
2007 }
2008
si_tex_wrap(unsigned wrap)2009 static unsigned si_tex_wrap(unsigned wrap)
2010 {
2011 switch (wrap) {
2012 default:
2013 case PIPE_TEX_WRAP_REPEAT:
2014 return V_008F30_SQ_TEX_WRAP;
2015 case PIPE_TEX_WRAP_CLAMP:
2016 return V_008F30_SQ_TEX_CLAMP_HALF_BORDER;
2017 case PIPE_TEX_WRAP_CLAMP_TO_EDGE:
2018 return V_008F30_SQ_TEX_CLAMP_LAST_TEXEL;
2019 case PIPE_TEX_WRAP_CLAMP_TO_BORDER:
2020 return V_008F30_SQ_TEX_CLAMP_BORDER;
2021 case PIPE_TEX_WRAP_MIRROR_REPEAT:
2022 return V_008F30_SQ_TEX_MIRROR;
2023 case PIPE_TEX_WRAP_MIRROR_CLAMP:
2024 return V_008F30_SQ_TEX_MIRROR_ONCE_HALF_BORDER;
2025 case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_EDGE:
2026 return V_008F30_SQ_TEX_MIRROR_ONCE_LAST_TEXEL;
2027 case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER:
2028 return V_008F30_SQ_TEX_MIRROR_ONCE_BORDER;
2029 }
2030 }
2031
si_tex_mipfilter(unsigned filter)2032 static unsigned si_tex_mipfilter(unsigned filter)
2033 {
2034 switch (filter) {
2035 case PIPE_TEX_MIPFILTER_NEAREST:
2036 return V_008F38_SQ_TEX_Z_FILTER_POINT;
2037 case PIPE_TEX_MIPFILTER_LINEAR:
2038 return V_008F38_SQ_TEX_Z_FILTER_LINEAR;
2039 default:
2040 case PIPE_TEX_MIPFILTER_NONE:
2041 return V_008F38_SQ_TEX_Z_FILTER_NONE;
2042 }
2043 }
2044
si_tex_compare(unsigned mode,unsigned compare)2045 static unsigned si_tex_compare(unsigned mode, unsigned compare)
2046 {
2047 if (mode == PIPE_TEX_COMPARE_NONE)
2048 return V_008F30_SQ_TEX_DEPTH_COMPARE_NEVER;
2049
2050 switch (compare) {
2051 default:
2052 case PIPE_FUNC_NEVER:
2053 return V_008F30_SQ_TEX_DEPTH_COMPARE_NEVER;
2054 case PIPE_FUNC_LESS:
2055 return V_008F30_SQ_TEX_DEPTH_COMPARE_LESS;
2056 case PIPE_FUNC_EQUAL:
2057 return V_008F30_SQ_TEX_DEPTH_COMPARE_EQUAL;
2058 case PIPE_FUNC_LEQUAL:
2059 return V_008F30_SQ_TEX_DEPTH_COMPARE_LESSEQUAL;
2060 case PIPE_FUNC_GREATER:
2061 return V_008F30_SQ_TEX_DEPTH_COMPARE_GREATER;
2062 case PIPE_FUNC_NOTEQUAL:
2063 return V_008F30_SQ_TEX_DEPTH_COMPARE_NOTEQUAL;
2064 case PIPE_FUNC_GEQUAL:
2065 return V_008F30_SQ_TEX_DEPTH_COMPARE_GREATEREQUAL;
2066 case PIPE_FUNC_ALWAYS:
2067 return V_008F30_SQ_TEX_DEPTH_COMPARE_ALWAYS;
2068 }
2069 }
2070
si_tex_dim(struct si_screen * sscreen,struct si_texture * tex,unsigned view_target,unsigned nr_samples)2071 static unsigned si_tex_dim(struct si_screen *sscreen, struct si_texture *tex, unsigned view_target,
2072 unsigned nr_samples)
2073 {
2074 unsigned res_target = tex->buffer.b.b.target;
2075
2076 if (view_target == PIPE_TEXTURE_CUBE || view_target == PIPE_TEXTURE_CUBE_ARRAY)
2077 res_target = view_target;
2078 /* If interpreting cubemaps as something else, set 2D_ARRAY. */
2079 else if (res_target == PIPE_TEXTURE_CUBE || res_target == PIPE_TEXTURE_CUBE_ARRAY)
2080 res_target = PIPE_TEXTURE_2D_ARRAY;
2081
2082 /* GFX9 allocates 1D textures as 2D. */
2083 if ((res_target == PIPE_TEXTURE_1D || res_target == PIPE_TEXTURE_1D_ARRAY) &&
2084 sscreen->info.gfx_level == GFX9 &&
2085 tex->surface.u.gfx9.resource_type == RADEON_RESOURCE_2D) {
2086 if (res_target == PIPE_TEXTURE_1D)
2087 res_target = PIPE_TEXTURE_2D;
2088 else
2089 res_target = PIPE_TEXTURE_2D_ARRAY;
2090 }
2091
2092 switch (res_target) {
2093 default:
2094 case PIPE_TEXTURE_1D:
2095 return V_008F1C_SQ_RSRC_IMG_1D;
2096 case PIPE_TEXTURE_1D_ARRAY:
2097 return V_008F1C_SQ_RSRC_IMG_1D_ARRAY;
2098 case PIPE_TEXTURE_2D:
2099 case PIPE_TEXTURE_RECT:
2100 return nr_samples > 1 ? V_008F1C_SQ_RSRC_IMG_2D_MSAA : V_008F1C_SQ_RSRC_IMG_2D;
2101 case PIPE_TEXTURE_2D_ARRAY:
2102 return nr_samples > 1 ? V_008F1C_SQ_RSRC_IMG_2D_MSAA_ARRAY : V_008F1C_SQ_RSRC_IMG_2D_ARRAY;
2103 case PIPE_TEXTURE_3D:
2104 return V_008F1C_SQ_RSRC_IMG_3D;
2105 case PIPE_TEXTURE_CUBE:
2106 case PIPE_TEXTURE_CUBE_ARRAY:
2107 return V_008F1C_SQ_RSRC_IMG_CUBE;
2108 }
2109 }
2110
2111 /*
2112 * Format support testing
2113 */
2114
si_is_sampler_format_supported(struct pipe_screen * screen,enum pipe_format format)2115 static bool si_is_sampler_format_supported(struct pipe_screen *screen, enum pipe_format format)
2116 {
2117 struct si_screen *sscreen = (struct si_screen *)screen;
2118 const struct util_format_description *desc = util_format_description(format);
2119
2120 /* Samplers don't support 64 bits per channel. */
2121 if (desc->layout == UTIL_FORMAT_LAYOUT_PLAIN &&
2122 desc->channel[0].size == 64)
2123 return false;
2124
2125 if (sscreen->info.gfx_level >= GFX10) {
2126 const struct gfx10_format *fmt = &ac_get_gfx10_format_table(sscreen->info.gfx_level)[format];
2127 if (!fmt->img_format || fmt->buffers_only)
2128 return false;
2129 return true;
2130 }
2131
2132 const int first_non_void = util_format_get_first_non_void_channel(format);
2133
2134 if (si_translate_texformat(screen, format, desc, first_non_void) == ~0U)
2135 return false;
2136
2137 if (desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB &&
2138 desc->nr_channels != 4 && desc->nr_channels != 1)
2139 return false;
2140
2141 if (desc->layout == UTIL_FORMAT_LAYOUT_ETC && !sscreen->info.has_etc_support)
2142 return false;
2143
2144 if (desc->layout == UTIL_FORMAT_LAYOUT_SUBSAMPLED &&
2145 (desc->format == PIPE_FORMAT_G8B8_G8R8_UNORM ||
2146 desc->format == PIPE_FORMAT_B8G8_R8G8_UNORM))
2147 return false;
2148
2149 /* Other "OTHER" layouts are unsupported. */
2150 if (desc->layout == UTIL_FORMAT_LAYOUT_OTHER &&
2151 desc->format != PIPE_FORMAT_R11G11B10_FLOAT &&
2152 desc->format != PIPE_FORMAT_R9G9B9E5_FLOAT)
2153 return false;
2154
2155 /* This must be before using first_non_void. */
2156 if (desc->layout != UTIL_FORMAT_LAYOUT_PLAIN)
2157 return true;
2158
2159 if (first_non_void < 0 || first_non_void > 3)
2160 return false;
2161
2162 /* Reject SCALED formats because we don't implement them for CB and do the same for texturing. */
2163 if ((desc->channel[first_non_void].type == UTIL_FORMAT_TYPE_UNSIGNED ||
2164 desc->channel[first_non_void].type == UTIL_FORMAT_TYPE_SIGNED) &&
2165 !desc->channel[first_non_void].normalized &&
2166 !desc->channel[first_non_void].pure_integer)
2167 return false;
2168
2169 /* Reject unsupported 32_*NORM and FIXED formats. */
2170 if (desc->channel[first_non_void].size == 32 &&
2171 (desc->channel[first_non_void].normalized ||
2172 desc->channel[first_non_void].type == UTIL_FORMAT_TYPE_FIXED))
2173 return false;
2174
2175 /* Luminace-alpha formats fail tests on Tahiti. */
2176 if (sscreen->info.gfx_level == GFX6 && util_format_is_luminance_alpha(format))
2177 return false;
2178
2179 /* This format fails on Gfx8/Carrizo´. */
2180 if (sscreen->info.family == CHIP_CARRIZO && format == PIPE_FORMAT_A8R8_UNORM)
2181 return false;
2182
2183 /* Reject unsupported 3x 32-bit formats for CB. */
2184 if (desc->nr_channels == 3 && desc->channel[0].size == 32 && desc->channel[1].size == 32 &&
2185 desc->channel[2].size == 32)
2186 return false;
2187
2188 /* Reject all 64-bit formats. */
2189 if (desc->channel[first_non_void].size == 64)
2190 return false;
2191
2192 return true;
2193 }
2194
si_translate_buffer_dataformat(struct pipe_screen * screen,const struct util_format_description * desc,int first_non_void)2195 static uint32_t si_translate_buffer_dataformat(struct pipe_screen *screen,
2196 const struct util_format_description *desc,
2197 int first_non_void)
2198 {
2199 assert(((struct si_screen *)screen)->info.gfx_level <= GFX9);
2200
2201 return ac_translate_buffer_dataformat(desc, first_non_void);
2202 }
2203
si_is_vertex_format_supported(struct pipe_screen * screen,enum pipe_format format,unsigned usage)2204 static unsigned si_is_vertex_format_supported(struct pipe_screen *screen, enum pipe_format format,
2205 unsigned usage)
2206 {
2207 struct si_screen *sscreen = (struct si_screen *)screen;
2208 const struct util_format_description *desc;
2209 int first_non_void;
2210 unsigned data_format;
2211
2212 assert((usage & ~(PIPE_BIND_SHADER_IMAGE | PIPE_BIND_SAMPLER_VIEW | PIPE_BIND_VERTEX_BUFFER)) ==
2213 0);
2214
2215 desc = util_format_description(format);
2216
2217 /* There are no native 8_8_8 or 16_16_16 data formats, and we currently
2218 * select 8_8_8_8 and 16_16_16_16 instead. This works reasonably well
2219 * for read-only access (with caveats surrounding bounds checks), but
2220 * obviously fails for write access which we have to implement for
2221 * shader images. Luckily, OpenGL doesn't expect this to be supported
2222 * anyway, and so the only impact is on PBO uploads / downloads, which
2223 * shouldn't be expected to be fast for GL_RGB anyway.
2224 */
2225 if (desc->block.bits == 3 * 8 || desc->block.bits == 3 * 16) {
2226 if (usage & (PIPE_BIND_SHADER_IMAGE | PIPE_BIND_SAMPLER_VIEW)) {
2227 usage &= ~(PIPE_BIND_SHADER_IMAGE | PIPE_BIND_SAMPLER_VIEW);
2228 if (!usage)
2229 return 0;
2230 }
2231 }
2232
2233 if (sscreen->info.gfx_level >= GFX10) {
2234 const struct gfx10_format *fmt = &ac_get_gfx10_format_table(sscreen->info.gfx_level)[format];
2235 unsigned first_image_only_format = sscreen->info.gfx_level >= GFX11 ? 64 : 128;
2236
2237 if (!fmt->img_format || fmt->img_format >= first_image_only_format)
2238 return 0;
2239 return usage;
2240 }
2241
2242 first_non_void = util_format_get_first_non_void_channel(format);
2243 data_format = si_translate_buffer_dataformat(screen, desc, first_non_void);
2244 if (data_format == V_008F0C_BUF_DATA_FORMAT_INVALID)
2245 return 0;
2246
2247 return usage;
2248 }
2249
si_is_zs_format_supported(enum pipe_format format)2250 static bool si_is_zs_format_supported(enum pipe_format format)
2251 {
2252 if (format == PIPE_FORMAT_Z16_UNORM_S8_UINT)
2253 return false;
2254
2255 return ac_is_zs_format_supported(format);
2256 }
2257
si_is_reduction_mode_supported(struct pipe_screen * screen,enum pipe_format format)2258 static bool si_is_reduction_mode_supported(struct pipe_screen *screen, enum pipe_format format)
2259 {
2260 struct si_screen *sscreen = (struct si_screen *)screen;
2261
2262 return ac_is_reduction_mode_supported(&sscreen->info, format, true);
2263 }
2264
si_is_format_supported(struct pipe_screen * screen,enum pipe_format format,enum pipe_texture_target target,unsigned sample_count,unsigned storage_sample_count,unsigned usage)2265 static bool si_is_format_supported(struct pipe_screen *screen, enum pipe_format format,
2266 enum pipe_texture_target target, unsigned sample_count,
2267 unsigned storage_sample_count, unsigned usage)
2268 {
2269 struct si_screen *sscreen = (struct si_screen *)screen;
2270 unsigned retval = 0;
2271
2272 if (target >= PIPE_MAX_TEXTURE_TYPES) {
2273 PRINT_ERR("radeonsi: unsupported texture type %d\n", target);
2274 return false;
2275 }
2276
2277 /* Require PIPE_BIND_SAMPLER_VIEW support when PIPE_BIND_RENDER_TARGET
2278 * is requested.
2279 */
2280 if (usage & PIPE_BIND_RENDER_TARGET)
2281 usage |= PIPE_BIND_SAMPLER_VIEW;
2282
2283 if ((target == PIPE_TEXTURE_3D || target == PIPE_TEXTURE_CUBE) &&
2284 !sscreen->info.has_3d_cube_border_color_mipmap)
2285 return false;
2286
2287 if (util_format_get_num_planes(format) >= 2)
2288 return false;
2289
2290 if (MAX2(1, sample_count) < MAX2(1, storage_sample_count))
2291 return false;
2292
2293 if (sample_count > 1) {
2294 if (!screen->caps.texture_multisample)
2295 return false;
2296
2297 /* Only power-of-two sample counts are supported. */
2298 if (!util_is_power_of_two_or_zero(sample_count) ||
2299 !util_is_power_of_two_or_zero(storage_sample_count))
2300 return false;
2301
2302 /* Chips with 1 RB don't increment occlusion queries at 16x MSAA sample rate,
2303 * so don't expose 16 samples there.
2304 *
2305 * EQAA also uses max 8 samples because our FMASK fetches only load 32 bits and
2306 * would need to be changed to 64 bits for 16 samples.
2307 */
2308 const unsigned max_samples = 8;
2309
2310 /* MSAA support without framebuffer attachments. */
2311 if (format == PIPE_FORMAT_NONE && sample_count <= max_samples)
2312 return true;
2313
2314 if (!sscreen->info.has_eqaa_surface_allocator || util_format_is_depth_or_stencil(format)) {
2315 /* Color without EQAA or depth/stencil. */
2316 if (sample_count > max_samples || sample_count != storage_sample_count)
2317 return false;
2318 } else {
2319 /* Color with EQAA. */
2320 if (sample_count > max_samples || storage_sample_count > max_samples)
2321 return false;
2322 }
2323 }
2324
2325 if (usage & (PIPE_BIND_SAMPLER_VIEW | PIPE_BIND_SHADER_IMAGE)) {
2326 if (target == PIPE_BUFFER) {
2327 retval |= si_is_vertex_format_supported(
2328 screen, format, usage & (PIPE_BIND_SAMPLER_VIEW | PIPE_BIND_SHADER_IMAGE));
2329 } else {
2330 if (si_is_sampler_format_supported(screen, format))
2331 retval |= usage & (PIPE_BIND_SAMPLER_VIEW | PIPE_BIND_SHADER_IMAGE);
2332 }
2333 }
2334
2335 if ((usage & (PIPE_BIND_RENDER_TARGET | PIPE_BIND_DISPLAY_TARGET | PIPE_BIND_SCANOUT |
2336 PIPE_BIND_SHARED | PIPE_BIND_BLENDABLE)) &&
2337 ac_is_colorbuffer_format_supported(sscreen->info.gfx_level, format)) {
2338 retval |= usage & (PIPE_BIND_RENDER_TARGET | PIPE_BIND_DISPLAY_TARGET | PIPE_BIND_SCANOUT |
2339 PIPE_BIND_SHARED);
2340 if (!util_format_is_pure_integer(format) && !util_format_is_depth_or_stencil(format))
2341 retval |= usage & PIPE_BIND_BLENDABLE;
2342 }
2343
2344 if ((usage & PIPE_BIND_DEPTH_STENCIL) && si_is_zs_format_supported(format)) {
2345 retval |= PIPE_BIND_DEPTH_STENCIL;
2346 }
2347
2348 if (usage & PIPE_BIND_VERTEX_BUFFER) {
2349 retval |= si_is_vertex_format_supported(screen, format, PIPE_BIND_VERTEX_BUFFER);
2350 }
2351
2352 if (usage & PIPE_BIND_INDEX_BUFFER) {
2353 if (format == PIPE_FORMAT_R8_UINT ||
2354 format == PIPE_FORMAT_R16_UINT ||
2355 format == PIPE_FORMAT_R32_UINT)
2356 retval |= PIPE_BIND_INDEX_BUFFER;
2357 }
2358
2359 if ((usage & PIPE_BIND_LINEAR) && !util_format_is_compressed(format) &&
2360 !(usage & PIPE_BIND_DEPTH_STENCIL))
2361 retval |= PIPE_BIND_LINEAR;
2362
2363 if ((usage & PIPE_BIND_SAMPLER_REDUCTION_MINMAX) &&
2364 screen->caps.sampler_reduction_minmax &&
2365 si_is_reduction_mode_supported(screen, format))
2366 retval |= PIPE_BIND_SAMPLER_REDUCTION_MINMAX;
2367
2368 return retval == usage;
2369 }
2370
2371 /*
2372 * framebuffer handling
2373 */
2374
si_choose_spi_color_formats(struct si_surface * surf,unsigned format,unsigned swap,unsigned ntype,bool is_depth)2375 static void si_choose_spi_color_formats(struct si_surface *surf, unsigned format, unsigned swap,
2376 unsigned ntype, bool is_depth)
2377 {
2378 struct ac_spi_color_formats formats = {};
2379
2380 ac_choose_spi_color_formats(format, swap, ntype, is_depth, true, &formats);
2381
2382 surf->spi_shader_col_format = formats.normal;
2383 surf->spi_shader_col_format_alpha = formats.alpha;
2384 surf->spi_shader_col_format_blend = formats.blend;
2385 surf->spi_shader_col_format_blend_alpha = formats.blend_alpha;
2386 }
2387
si_initialize_color_surface(struct si_context * sctx,struct si_surface * surf)2388 static void si_initialize_color_surface(struct si_context *sctx, struct si_surface *surf)
2389 {
2390 struct si_texture *tex = (struct si_texture *)surf->base.texture;
2391 unsigned format, swap, ntype;//, endian;
2392
2393 ntype = ac_get_cb_number_type(surf->base.format);
2394 format = ac_get_cb_format(sctx->gfx_level, surf->base.format);
2395
2396 if (format == V_028C70_COLOR_INVALID) {
2397 PRINT_ERR("Invalid CB format: %d, disabling CB.\n", surf->base.format);
2398 }
2399 assert(format != V_028C70_COLOR_INVALID);
2400 swap = ac_translate_colorswap(sctx->gfx_level, surf->base.format, false);
2401
2402 if (ntype == V_028C70_NUMBER_UINT || ntype == V_028C70_NUMBER_SINT) {
2403 if (format == V_028C70_COLOR_8 || format == V_028C70_COLOR_8_8 ||
2404 format == V_028C70_COLOR_8_8_8_8)
2405 surf->color_is_int8 = true;
2406 else if (format == V_028C70_COLOR_10_10_10_2 || format == V_028C70_COLOR_2_10_10_10)
2407 surf->color_is_int10 = true;
2408 }
2409
2410 const struct ac_cb_state cb_state = {
2411 .surf = &tex->surface,
2412 .format = surf->base.format,
2413 .width = surf->width0,
2414 .height = surf->height0,
2415 .first_layer = surf->base.u.tex.first_layer,
2416 .last_layer = surf->base.u.tex.last_layer,
2417 .num_layers = util_max_layer(&tex->buffer.b.b, 0),
2418 .num_samples = tex->buffer.b.b.nr_samples,
2419 .num_storage_samples = tex->buffer.b.b.nr_storage_samples,
2420 .base_level = surf->base.u.tex.level,
2421 .num_levels = tex->buffer.b.b.last_level + 1,
2422 };
2423
2424 ac_init_cb_surface(&sctx->screen->info, &cb_state, &surf->cb);
2425
2426 /* Determine pixel shader export format */
2427 si_choose_spi_color_formats(surf, format, swap, ntype, tex->is_depth);
2428
2429 surf->color_initialized = true;
2430 }
2431
si_init_depth_surface(struct si_context * sctx,struct si_surface * surf)2432 static void si_init_depth_surface(struct si_context *sctx, struct si_surface *surf)
2433 {
2434 struct si_texture *tex = (struct si_texture *)surf->base.texture;
2435 unsigned level = surf->base.u.tex.level;
2436 unsigned format;
2437
2438 format = ac_translate_dbformat(tex->db_render_format);
2439
2440 assert(format != V_028040_Z_24 || sctx->gfx_level < GFX12);
2441 assert(format != V_028040_Z_INVALID);
2442
2443 if (format == V_028040_Z_INVALID)
2444 PRINT_ERR("Invalid DB format: %d, disabling DB.\n", tex->buffer.b.b.format);
2445
2446 /* Use the original Z format, not db_render_format, so that the polygon offset behaves as
2447 * expected by applications.
2448 */
2449 switch (tex->buffer.b.b.format) {
2450 case PIPE_FORMAT_Z16_UNORM:
2451 surf->db_format_index = 0;
2452 break;
2453 default: /* 24-bit */
2454 surf->db_format_index = 1;
2455 break;
2456 case PIPE_FORMAT_Z32_FLOAT:
2457 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
2458 surf->db_format_index = 2;
2459 break;
2460 }
2461
2462 const struct ac_ds_state ds_state = {
2463 .surf = &tex->surface,
2464 .va = tex->buffer.gpu_address,
2465 .format = tex->db_render_format,
2466 .width = tex->buffer.b.b.width0,
2467 .height = tex->buffer.b.b.height0,
2468 .level = level,
2469 .num_levels = tex->buffer.b.b.last_level + 1,
2470 .num_samples = tex->buffer.b.b.nr_samples,
2471 .first_layer = surf->base.u.tex.first_layer,
2472 .last_layer = surf->base.u.tex.last_layer,
2473 .allow_expclear = true,
2474 .htile_enabled = sctx->gfx_level < GFX12 && si_htile_enabled(tex, level, PIPE_MASK_ZS),
2475 .htile_stencil_disabled = tex->htile_stencil_disabled,
2476 };
2477
2478 ac_init_ds_surface(&sctx->screen->info, &ds_state, &surf->ds);
2479
2480 surf->depth_initialized = true;
2481 }
2482
si_dec_framebuffer_counters(const struct pipe_framebuffer_state * state)2483 static void si_dec_framebuffer_counters(const struct pipe_framebuffer_state *state)
2484 {
2485 for (int i = 0; i < state->nr_cbufs; ++i) {
2486 struct si_surface *surf = NULL;
2487 struct si_texture *tex;
2488
2489 if (!state->cbufs[i])
2490 continue;
2491 surf = (struct si_surface *)state->cbufs[i];
2492 tex = (struct si_texture *)surf->base.texture;
2493
2494 p_atomic_dec(&tex->framebuffers_bound);
2495 }
2496 }
2497
si_mark_display_dcc_dirty(struct si_context * sctx,struct si_texture * tex)2498 void si_mark_display_dcc_dirty(struct si_context *sctx, struct si_texture *tex)
2499 {
2500 assert(sctx->gfx_level < GFX12);
2501
2502 if (!tex->surface.display_dcc_offset || tex->displayable_dcc_dirty)
2503 return;
2504
2505 if (!(tex->buffer.external_usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH)) {
2506 struct hash_entry *entry = _mesa_hash_table_search(sctx->dirty_implicit_resources, tex);
2507 if (!entry) {
2508 struct pipe_resource *dummy = NULL;
2509 pipe_resource_reference(&dummy, &tex->buffer.b.b);
2510 _mesa_hash_table_insert(sctx->dirty_implicit_resources, tex, tex);
2511 }
2512 }
2513 tex->displayable_dcc_dirty = true;
2514 }
2515
si_update_display_dcc_dirty(struct si_context * sctx)2516 static void si_update_display_dcc_dirty(struct si_context *sctx)
2517 {
2518 const struct pipe_framebuffer_state *state = &sctx->framebuffer.state;
2519
2520 for (unsigned i = 0; i < state->nr_cbufs; i++) {
2521 if (state->cbufs[i])
2522 si_mark_display_dcc_dirty(sctx, (struct si_texture *)state->cbufs[i]->texture);
2523 }
2524 }
2525
si_set_framebuffer_state(struct pipe_context * ctx,const struct pipe_framebuffer_state * state)2526 static void si_set_framebuffer_state(struct pipe_context *ctx,
2527 const struct pipe_framebuffer_state *state)
2528 {
2529 struct si_context *sctx = (struct si_context *)ctx;
2530 struct si_surface *surf = NULL;
2531 struct si_texture *tex;
2532 bool old_any_dst_linear = sctx->framebuffer.any_dst_linear;
2533 unsigned old_nr_samples = sctx->framebuffer.nr_samples;
2534 unsigned old_colorbuf_enabled_4bit = sctx->framebuffer.colorbuf_enabled_4bit;
2535 bool old_has_zsbuf = !!sctx->framebuffer.state.zsbuf;
2536 bool old_has_stencil =
2537 old_has_zsbuf &&
2538 ((struct si_texture *)sctx->framebuffer.state.zsbuf->texture)->surface.has_stencil;
2539 uint8_t old_db_format_index =
2540 old_has_zsbuf ?
2541 ((struct si_surface *)sctx->framebuffer.state.zsbuf)->db_format_index : -1;
2542 bool old_has_hiz_his = sctx->framebuffer.has_hiz_his;
2543 int i;
2544
2545 /* Reject zero-sized framebuffers due to a hw bug on GFX6 that occurs
2546 * when PA_SU_HARDWARE_SCREEN_OFFSET != 0 and any_scissor.BR_X/Y <= 0.
2547 * We could implement the full workaround here, but it's a useless case.
2548 */
2549 if ((!state->width || !state->height) && (state->nr_cbufs || state->zsbuf)) {
2550 unreachable("the framebuffer shouldn't have zero area");
2551 return;
2552 }
2553
2554 si_fb_barrier_after_rendering(sctx, SI_FB_BARRIER_SYNC_ALL);
2555
2556 /* Disable DCC if the formats are incompatible. */
2557 if (sctx->gfx_level >= GFX8 && sctx->gfx_level < GFX11) {
2558 for (i = 0; i < state->nr_cbufs; i++) {
2559 if (!state->cbufs[i])
2560 continue;
2561
2562 surf = (struct si_surface *)state->cbufs[i];
2563 tex = (struct si_texture *)surf->base.texture;
2564
2565 if (!surf->dcc_incompatible)
2566 continue;
2567
2568 if (vi_dcc_enabled(tex, surf->base.u.tex.level))
2569 if (!si_texture_disable_dcc(sctx, tex))
2570 si_decompress_dcc(sctx, tex);
2571
2572 surf->dcc_incompatible = false;
2573 }
2574 }
2575
2576 /* Take the maximum of the old and new count. If the new count is lower,
2577 * dirtying is needed to disable the unbound colorbuffers.
2578 */
2579 sctx->framebuffer.dirty_cbufs |=
2580 (1 << MAX2(sctx->framebuffer.state.nr_cbufs, state->nr_cbufs)) - 1;
2581 sctx->framebuffer.dirty_zsbuf |= sctx->framebuffer.state.zsbuf != state->zsbuf;
2582
2583 si_dec_framebuffer_counters(&sctx->framebuffer.state);
2584 util_copy_framebuffer_state(&sctx->framebuffer.state, state);
2585
2586 /* The framebuffer state must be set before the barrier. */
2587 si_fb_barrier_before_rendering(sctx);
2588
2589 /* Recompute layers because frontends and utils might not set it. */
2590 sctx->framebuffer.state.layers = util_framebuffer_get_num_layers(state);
2591
2592 sctx->framebuffer.colorbuf_enabled_4bit = 0;
2593 sctx->framebuffer.spi_shader_col_format = 0;
2594 sctx->framebuffer.spi_shader_col_format_alpha = 0;
2595 sctx->framebuffer.spi_shader_col_format_blend = 0;
2596 sctx->framebuffer.spi_shader_col_format_blend_alpha = 0;
2597 sctx->framebuffer.color_is_int8 = 0;
2598 sctx->framebuffer.color_is_int10 = 0;
2599
2600 sctx->framebuffer.compressed_cb_mask = 0;
2601 sctx->framebuffer.uncompressed_cb_mask = 0;
2602 sctx->framebuffer.nr_samples = util_framebuffer_get_num_samples(state);
2603 sctx->framebuffer.nr_color_samples = sctx->framebuffer.nr_samples;
2604 sctx->framebuffer.log_samples = util_logbase2(sctx->framebuffer.nr_samples);
2605 sctx->framebuffer.any_dst_linear = false;
2606 sctx->framebuffer.CB_has_shader_readable_metadata = false;
2607 sctx->framebuffer.DB_has_shader_readable_metadata = false;
2608 sctx->framebuffer.all_DCC_pipe_aligned = true;
2609 sctx->framebuffer.has_dcc_msaa = false;
2610 sctx->framebuffer.min_bytes_per_pixel = 0;
2611 sctx->framebuffer.disable_vrs_flat_shading = false;
2612 sctx->framebuffer.has_stencil = false;
2613 sctx->framebuffer.has_hiz_his = false;
2614
2615 for (i = 0; i < state->nr_cbufs; i++) {
2616 if (!state->cbufs[i])
2617 continue;
2618
2619 surf = (struct si_surface *)state->cbufs[i];
2620 tex = (struct si_texture *)surf->base.texture;
2621
2622 if (!surf->color_initialized) {
2623 si_initialize_color_surface(sctx, surf);
2624 }
2625
2626 sctx->framebuffer.colorbuf_enabled_4bit |= 0xf << (i * 4);
2627 sctx->framebuffer.spi_shader_col_format |= surf->spi_shader_col_format << (i * 4);
2628 sctx->framebuffer.spi_shader_col_format_alpha |= surf->spi_shader_col_format_alpha << (i * 4);
2629 sctx->framebuffer.spi_shader_col_format_blend |= surf->spi_shader_col_format_blend << (i * 4);
2630 sctx->framebuffer.spi_shader_col_format_blend_alpha |= surf->spi_shader_col_format_blend_alpha
2631 << (i * 4);
2632
2633 if (surf->color_is_int8)
2634 sctx->framebuffer.color_is_int8 |= 1 << i;
2635 if (surf->color_is_int10)
2636 sctx->framebuffer.color_is_int10 |= 1 << i;
2637
2638 if (tex->surface.fmask_offset)
2639 sctx->framebuffer.compressed_cb_mask |= 1 << i;
2640 else
2641 sctx->framebuffer.uncompressed_cb_mask |= 1 << i;
2642
2643 /* Don't update nr_color_samples for non-AA buffers.
2644 * (e.g. destination of MSAA resolve)
2645 */
2646 if (tex->buffer.b.b.nr_samples >= 2 &&
2647 tex->buffer.b.b.nr_storage_samples < tex->buffer.b.b.nr_samples) {
2648 sctx->framebuffer.nr_color_samples =
2649 MIN2(sctx->framebuffer.nr_color_samples, tex->buffer.b.b.nr_storage_samples);
2650 sctx->framebuffer.nr_color_samples = MAX2(1, sctx->framebuffer.nr_color_samples);
2651 }
2652
2653 if (tex->surface.is_linear)
2654 sctx->framebuffer.any_dst_linear = true;
2655
2656 if (vi_dcc_enabled(tex, surf->base.u.tex.level)) {
2657 sctx->framebuffer.CB_has_shader_readable_metadata = true;
2658
2659 if (sctx->gfx_level >= GFX9 && sctx->gfx_level < GFX12 &&
2660 !tex->surface.u.gfx9.color.dcc.pipe_aligned)
2661 sctx->framebuffer.all_DCC_pipe_aligned = false;
2662
2663 if (tex->buffer.b.b.nr_storage_samples >= 2)
2664 sctx->framebuffer.has_dcc_msaa = true;
2665 }
2666
2667 p_atomic_inc(&tex->framebuffers_bound);
2668
2669 /* Update the minimum but don't keep 0. */
2670 if (!sctx->framebuffer.min_bytes_per_pixel ||
2671 tex->surface.bpe < sctx->framebuffer.min_bytes_per_pixel)
2672 sctx->framebuffer.min_bytes_per_pixel = tex->surface.bpe;
2673
2674 /* Disable VRS flat shading where it decreases performance.
2675 * This gives the best results for slow clears for AMD_TEST=blitperf on Navi31.
2676 */
2677 if ((sctx->framebuffer.nr_samples == 8 && tex->surface.bpe != 2) ||
2678 (tex->surface.thick_tiling && tex->surface.bpe == 4 &&
2679 util_format_get_nr_components(surf->base.format) == 4))
2680 sctx->framebuffer.disable_vrs_flat_shading = true;
2681 }
2682
2683 struct si_texture *zstex = NULL;
2684
2685 if (state->zsbuf) {
2686 surf = (struct si_surface *)state->zsbuf;
2687 zstex = (struct si_texture *)surf->base.texture;
2688
2689 if (!surf->depth_initialized) {
2690 si_init_depth_surface(sctx, surf);
2691 }
2692
2693 if (sctx->gfx_level < GFX12 &&
2694 vi_tc_compat_htile_enabled(zstex, surf->base.u.tex.level, PIPE_MASK_ZS))
2695 sctx->framebuffer.DB_has_shader_readable_metadata = true;
2696
2697 /* Update the minimum but don't keep 0. */
2698 if (!sctx->framebuffer.min_bytes_per_pixel ||
2699 zstex->surface.bpe < sctx->framebuffer.min_bytes_per_pixel)
2700 sctx->framebuffer.min_bytes_per_pixel = zstex->surface.bpe;
2701
2702 /* Update polygon offset based on the Z format. */
2703 if (sctx->queued.named.rasterizer->uses_poly_offset &&
2704 surf->db_format_index != old_db_format_index)
2705 sctx->dirty_atoms |= SI_STATE_BIT(rasterizer);
2706
2707 if (util_format_has_stencil(util_format_description(zstex->buffer.b.b.format)))
2708 sctx->framebuffer.has_stencil = true;
2709
2710 if (sctx->gfx_level == GFX12 && !sctx->screen->options.alt_hiz_logic &&
2711 sctx->framebuffer.has_stencil && sctx->queued.named.dsa->stencil_enabled)
2712 zstex->force_disable_hiz_his = true;
2713
2714 if (sctx->gfx_level >= GFX12) {
2715 sctx->framebuffer.has_hiz_his = (zstex->surface.u.gfx9.zs.hiz.offset ||
2716 zstex->surface.u.gfx9.zs.his.offset) &&
2717 !zstex->force_disable_hiz_his;
2718 }
2719 }
2720
2721 si_update_ps_colorbuf0_slot(sctx);
2722 si_mark_atom_dirty(sctx, &sctx->atoms.s.cb_render_state);
2723 si_mark_atom_dirty(sctx, &sctx->atoms.s.framebuffer);
2724
2725 /* NGG cull state uses the sample count. */
2726 if (sctx->screen->use_ngg_culling)
2727 si_mark_atom_dirty(sctx, &sctx->atoms.s.ngg_cull_state);
2728
2729 if (sctx->screen->dpbb_allowed)
2730 si_mark_atom_dirty(sctx, &sctx->atoms.s.dpbb_state);
2731
2732 if (sctx->framebuffer.any_dst_linear != old_any_dst_linear ||
2733 sctx->framebuffer.has_hiz_his != old_has_hiz_his)
2734 si_mark_atom_dirty(sctx, &sctx->atoms.s.msaa_config);
2735
2736 if (sctx->screen->info.has_out_of_order_rast &&
2737 (sctx->framebuffer.colorbuf_enabled_4bit != old_colorbuf_enabled_4bit ||
2738 !!sctx->framebuffer.state.zsbuf != old_has_zsbuf ||
2739 (zstex && zstex->surface.has_stencil != old_has_stencil)))
2740 si_mark_atom_dirty(sctx, &sctx->atoms.s.msaa_config);
2741
2742 if (sctx->framebuffer.nr_samples != old_nr_samples) {
2743 si_mark_atom_dirty(sctx, &sctx->atoms.s.msaa_config);
2744 si_mark_atom_dirty(sctx, &sctx->atoms.s.db_render_state);
2745 si_mark_atom_dirty(sctx, &sctx->atoms.s.sample_locations);
2746 }
2747
2748 si_ps_key_update_framebuffer(sctx);
2749 si_ps_key_update_framebuffer_blend_dsa_rasterizer(sctx);
2750 si_ps_key_update_framebuffer_rasterizer_sample_shading(sctx);
2751 si_ps_key_update_sample_shading(sctx);
2752 si_vs_ps_key_update_rast_prim_smooth_stipple(sctx);
2753 si_update_ps_inputs_read_or_disabled(sctx);
2754 si_update_vrs_flat_shading(sctx);
2755 sctx->do_update_shaders = true;
2756
2757 if (sctx->gfx_level < GFX12 && !sctx->decompression_enabled) {
2758 /* Prevent textures decompression when the framebuffer state
2759 * changes come from the decompression passes themselves.
2760 */
2761 sctx->need_check_render_feedback = true;
2762 }
2763 }
2764
gfx6_emit_framebuffer_state(struct si_context * sctx,unsigned index)2765 static void gfx6_emit_framebuffer_state(struct si_context *sctx, unsigned index)
2766 {
2767 struct radeon_cmdbuf *cs = &sctx->gfx_cs;
2768 struct pipe_framebuffer_state *state = &sctx->framebuffer.state;
2769 unsigned i, nr_cbufs = state->nr_cbufs;
2770 struct si_texture *tex = NULL;
2771 struct si_surface *cb = NULL;
2772 bool is_msaa_resolve = state->nr_cbufs == 2 &&
2773 state->cbufs[0] && state->cbufs[0]->texture->nr_samples > 1 &&
2774 state->cbufs[1] && state->cbufs[1]->texture->nr_samples <= 1;
2775
2776 /* CB can't do MSAA resolve on gfx11. */
2777 assert(!is_msaa_resolve || sctx->gfx_level < GFX11);
2778
2779 radeon_begin(cs);
2780
2781 /* Colorbuffers. */
2782 for (i = 0; i < nr_cbufs; i++) {
2783 if (!(sctx->framebuffer.dirty_cbufs & (1 << i)))
2784 continue;
2785
2786 /* RB+ depth-only rendering. See the comment where we set rbplus_depth_only_opt for more
2787 * information.
2788 */
2789 if (i == 0 &&
2790 sctx->screen->info.rbplus_allowed &&
2791 !sctx->queued.named.blend->cb_target_mask) {
2792 radeon_set_context_reg(R_028C70_CB_COLOR0_INFO + i * 0x3C,
2793 (sctx->gfx_level >= GFX11 ?
2794 S_028C70_FORMAT_GFX11(V_028C70_COLOR_32) :
2795 S_028C70_FORMAT_GFX6(V_028C70_COLOR_32)) |
2796 S_028C70_NUMBER_TYPE(V_028C70_NUMBER_FLOAT));
2797 continue;
2798 }
2799
2800 cb = (struct si_surface *)state->cbufs[i];
2801 if (!cb) {
2802 radeon_set_context_reg(R_028C70_CB_COLOR0_INFO + i * 0x3C,
2803 sctx->gfx_level >= GFX11 ?
2804 S_028C70_FORMAT_GFX11(V_028C70_COLOR_INVALID) :
2805 S_028C70_FORMAT_GFX6(V_028C70_COLOR_INVALID));
2806 continue;
2807 }
2808
2809 tex = (struct si_texture *)cb->base.texture;
2810 radeon_add_to_buffer_list(
2811 sctx, &sctx->gfx_cs, &tex->buffer, RADEON_USAGE_READWRITE | RADEON_USAGE_CB_NEEDS_IMPLICIT_SYNC |
2812 (tex->buffer.b.b.nr_samples > 1 ? RADEON_PRIO_COLOR_BUFFER_MSAA : RADEON_PRIO_COLOR_BUFFER));
2813
2814 if (tex->cmask_buffer && tex->cmask_buffer != &tex->buffer) {
2815 radeon_add_to_buffer_list(sctx, &sctx->gfx_cs, tex->cmask_buffer,
2816 RADEON_USAGE_READWRITE | RADEON_USAGE_CB_NEEDS_IMPLICIT_SYNC |
2817 RADEON_PRIO_SEPARATE_META);
2818 }
2819
2820 /* Compute mutable surface parameters. */
2821 const struct ac_mutable_cb_state mutable_cb_state = {
2822 .surf = &tex->surface,
2823 .cb = &cb->cb,
2824 .va = tex->buffer.gpu_address,
2825 .base_level = cb->base.u.tex.level,
2826 .num_samples = cb->base.texture->nr_samples,
2827 .fmask_enabled = !!tex->surface.fmask_offset,
2828 /* CMASK and fast clears are configured elsewhere. */
2829 .cmask_enabled = false,
2830 .fast_clear_enabled = false,
2831 .dcc_enabled = vi_dcc_enabled(tex, cb->base.u.tex.level) &&
2832 (i != 1 || !is_msaa_resolve),
2833 };
2834 struct ac_cb_surface cb_surf;
2835
2836 ac_set_mutable_cb_surface_fields(&sctx->screen->info, &mutable_cb_state, &cb_surf);
2837
2838 cb_surf.cb_color_info |= tex->cb_color_info;
2839
2840 if (sctx->gfx_level < GFX11) {
2841 if (tex->swap_rgb_to_bgr) {
2842 /* Swap R and B channels. */
2843 static unsigned rgb_to_bgr[4] = {
2844 [V_028C70_SWAP_STD] = V_028C70_SWAP_ALT,
2845 [V_028C70_SWAP_ALT] = V_028C70_SWAP_STD,
2846 [V_028C70_SWAP_STD_REV] = V_028C70_SWAP_ALT_REV,
2847 [V_028C70_SWAP_ALT_REV] = V_028C70_SWAP_STD_REV,
2848 };
2849 unsigned swap = rgb_to_bgr[G_028C70_COMP_SWAP(cb_surf.cb_color_info)];
2850
2851 cb_surf.cb_color_info &= C_028C70_COMP_SWAP;
2852 cb_surf.cb_color_info |= S_028C70_COMP_SWAP(swap);
2853 }
2854
2855 if (cb->base.u.tex.level > 0)
2856 cb_surf.cb_color_info &= C_028C70_FAST_CLEAR;
2857 else
2858 cb_surf.cb_color_cmask = tex->cmask_base_address_reg;
2859 }
2860
2861 if (sctx->gfx_level >= GFX11) {
2862 radeon_set_context_reg(R_028C60_CB_COLOR0_BASE + i * 0x3C, cb_surf.cb_color_base);
2863
2864 radeon_set_context_reg_seq(R_028C6C_CB_COLOR0_VIEW + i * 0x3C, 4);
2865 radeon_emit(cb_surf.cb_color_view); /* CB_COLOR0_VIEW */
2866 radeon_emit(cb_surf.cb_color_info); /* CB_COLOR0_INFO */
2867 radeon_emit(cb_surf.cb_color_attrib); /* CB_COLOR0_ATTRIB */
2868 radeon_emit(cb_surf.cb_dcc_control); /* CB_COLOR0_FDCC_CONTROL */
2869
2870 radeon_set_context_reg(R_028C94_CB_COLOR0_DCC_BASE + i * 0x3C, cb_surf.cb_dcc_base);
2871 radeon_set_context_reg(R_028E40_CB_COLOR0_BASE_EXT + i * 4, cb_surf.cb_color_base >> 32);
2872 radeon_set_context_reg(R_028EA0_CB_COLOR0_DCC_BASE_EXT + i * 4, cb_surf.cb_dcc_base >> 32);
2873 radeon_set_context_reg(R_028EC0_CB_COLOR0_ATTRIB2 + i * 4, cb_surf.cb_color_attrib2);
2874 radeon_set_context_reg(R_028EE0_CB_COLOR0_ATTRIB3 + i * 4, cb_surf.cb_color_attrib3);
2875 } else if (sctx->gfx_level >= GFX10) {
2876 radeon_set_context_reg_seq(R_028C60_CB_COLOR0_BASE + i * 0x3C, 14);
2877 radeon_emit(cb_surf.cb_color_base); /* CB_COLOR0_BASE */
2878 radeon_emit(0); /* hole */
2879 radeon_emit(0); /* hole */
2880 radeon_emit(cb_surf.cb_color_view); /* CB_COLOR0_VIEW */
2881 radeon_emit(cb_surf.cb_color_info); /* CB_COLOR0_INFO */
2882 radeon_emit(cb_surf.cb_color_attrib); /* CB_COLOR0_ATTRIB */
2883 radeon_emit(cb_surf.cb_dcc_control); /* CB_COLOR0_DCC_CONTROL */
2884 radeon_emit(cb_surf.cb_color_cmask); /* CB_COLOR0_CMASK */
2885 radeon_emit(0); /* hole */
2886 radeon_emit(cb_surf.cb_color_fmask); /* CB_COLOR0_FMASK */
2887 radeon_emit(0); /* hole */
2888 radeon_emit(tex->color_clear_value[0]); /* CB_COLOR0_CLEAR_WORD0 */
2889 radeon_emit(tex->color_clear_value[1]); /* CB_COLOR0_CLEAR_WORD1 */
2890 radeon_emit(cb_surf.cb_dcc_base); /* CB_COLOR0_DCC_BASE */
2891
2892 radeon_set_context_reg(R_028E40_CB_COLOR0_BASE_EXT + i * 4, cb_surf.cb_color_base >> 32);
2893 radeon_set_context_reg(R_028E60_CB_COLOR0_CMASK_BASE_EXT + i * 4,
2894 cb_surf.cb_color_cmask >> 32);
2895 radeon_set_context_reg(R_028E80_CB_COLOR0_FMASK_BASE_EXT + i * 4,
2896 cb_surf.cb_color_fmask >> 32);
2897 radeon_set_context_reg(R_028EA0_CB_COLOR0_DCC_BASE_EXT + i * 4, cb_surf.cb_dcc_base >> 32);
2898 radeon_set_context_reg(R_028EC0_CB_COLOR0_ATTRIB2 + i * 4, cb_surf.cb_color_attrib2);
2899 radeon_set_context_reg(R_028EE0_CB_COLOR0_ATTRIB3 + i * 4, cb_surf.cb_color_attrib3);
2900 } else if (sctx->gfx_level == GFX9) {
2901 radeon_set_context_reg_seq(R_028C60_CB_COLOR0_BASE + i * 0x3C, 15);
2902 radeon_emit(cb_surf.cb_color_base); /* CB_COLOR0_BASE */
2903 radeon_emit(S_028C64_BASE_256B(cb_surf.cb_color_base >> 32)); /* CB_COLOR0_BASE_EXT */
2904 radeon_emit(cb_surf.cb_color_attrib2); /* CB_COLOR0_ATTRIB2 */
2905 radeon_emit(cb_surf.cb_color_view); /* CB_COLOR0_VIEW */
2906 radeon_emit(cb_surf.cb_color_info); /* CB_COLOR0_INFO */
2907 radeon_emit(cb_surf.cb_color_attrib); /* CB_COLOR0_ATTRIB */
2908 radeon_emit(cb_surf.cb_dcc_control); /* CB_COLOR0_DCC_CONTROL */
2909 radeon_emit(cb_surf.cb_color_cmask); /* CB_COLOR0_CMASK */
2910 radeon_emit(S_028C80_BASE_256B(cb_surf.cb_color_cmask >> 32)); /* CB_COLOR0_CMASK_BASE_EXT */
2911 radeon_emit(cb_surf.cb_color_fmask); /* CB_COLOR0_FMASK */
2912 radeon_emit(S_028C88_BASE_256B(cb_surf.cb_color_fmask >> 32)); /* CB_COLOR0_FMASK_BASE_EXT */
2913 radeon_emit(tex->color_clear_value[0]); /* CB_COLOR0_CLEAR_WORD0 */
2914 radeon_emit(tex->color_clear_value[1]); /* CB_COLOR0_CLEAR_WORD1 */
2915 radeon_emit(cb_surf.cb_dcc_base); /* CB_COLOR0_DCC_BASE */
2916 radeon_emit(S_028C98_BASE_256B(cb_surf.cb_dcc_base >> 32)); /* CB_COLOR0_DCC_BASE_EXT */
2917
2918 radeon_set_context_reg(R_0287A0_CB_MRT0_EPITCH + i * 4, cb_surf.cb_mrt_epitch);
2919 } else {
2920 /* GFX6-8 */
2921 radeon_set_context_reg_seq(R_028C60_CB_COLOR0_BASE + i * 0x3C,
2922 sctx->gfx_level >= GFX8 ? 14 : 13);
2923 radeon_emit(cb_surf.cb_color_base); /* CB_COLOR0_BASE */
2924 radeon_emit(cb_surf.cb_color_pitch); /* CB_COLOR0_PITCH */
2925 radeon_emit(cb_surf.cb_color_slice); /* CB_COLOR0_SLICE */
2926 radeon_emit(cb_surf.cb_color_view); /* CB_COLOR0_VIEW */
2927 radeon_emit(cb_surf.cb_color_info); /* CB_COLOR0_INFO */
2928 radeon_emit(cb_surf.cb_color_attrib); /* CB_COLOR0_ATTRIB */
2929 radeon_emit(cb_surf.cb_dcc_control); /* CB_COLOR0_DCC_CONTROL */
2930 radeon_emit(cb_surf.cb_color_cmask); /* CB_COLOR0_CMASK */
2931 radeon_emit(tex->surface.u.legacy.color.cmask_slice_tile_max); /* CB_COLOR0_CMASK_SLICE */
2932 radeon_emit(cb_surf.cb_color_fmask); /* CB_COLOR0_FMASK */
2933 radeon_emit(cb_surf.cb_color_fmask_slice); /* CB_COLOR0_FMASK_SLICE */
2934 radeon_emit(tex->color_clear_value[0]); /* CB_COLOR0_CLEAR_WORD0 */
2935 radeon_emit(tex->color_clear_value[1]); /* CB_COLOR0_CLEAR_WORD1 */
2936
2937 if (sctx->gfx_level >= GFX8) /* R_028C94_CB_COLOR0_DCC_BASE */
2938 radeon_emit(cb_surf.cb_dcc_base);
2939 }
2940 }
2941 for (; i < 8; i++)
2942 if (sctx->framebuffer.dirty_cbufs & (1 << i))
2943 radeon_set_context_reg(R_028C70_CB_COLOR0_INFO + i * 0x3C, 0);
2944
2945 /* ZS buffer. */
2946 if (state->zsbuf && sctx->framebuffer.dirty_zsbuf) {
2947 struct si_surface *zb = (struct si_surface *)state->zsbuf;
2948 struct si_texture *tex = (struct si_texture *)zb->base.texture;
2949
2950 radeon_add_to_buffer_list(sctx, &sctx->gfx_cs, &tex->buffer, RADEON_USAGE_READWRITE |
2951 (zb->base.texture->nr_samples > 1 ? RADEON_PRIO_DEPTH_BUFFER_MSAA
2952 : RADEON_PRIO_DEPTH_BUFFER));
2953
2954 const unsigned level = zb->base.u.tex.level;
2955
2956 /* Set mutable fields. */
2957 const struct ac_mutable_ds_state mutable_ds_state = {
2958 .ds = &zb->ds,
2959 .format = tex->db_render_format,
2960 .tc_compat_htile_enabled = vi_tc_compat_htile_enabled(tex, level, PIPE_MASK_ZS),
2961 .zrange_precision = tex->depth_clear_value[level] != 0,
2962 };
2963 struct ac_ds_surface ds;
2964
2965 ac_set_mutable_ds_surface_fields(&sctx->screen->info, &mutable_ds_state, &ds);
2966
2967 if (sctx->gfx_level >= GFX10) {
2968 radeon_set_context_reg(R_028014_DB_HTILE_DATA_BASE, ds.u.gfx6.db_htile_data_base);
2969 radeon_set_context_reg(R_02801C_DB_DEPTH_SIZE_XY, ds.db_depth_size);
2970
2971 if (sctx->gfx_level >= GFX11) {
2972 radeon_set_context_reg_seq(R_028040_DB_Z_INFO, 6);
2973 } else {
2974 radeon_set_context_reg_seq(R_02803C_DB_DEPTH_INFO, 7);
2975 radeon_emit(S_02803C_RESOURCE_LEVEL(1)); /* DB_DEPTH_INFO */
2976 }
2977 radeon_emit(ds.db_z_info); /* DB_Z_INFO */
2978 radeon_emit(ds.db_stencil_info); /* DB_STENCIL_INFO */
2979 radeon_emit(ds.db_depth_base); /* DB_Z_READ_BASE */
2980 radeon_emit(ds.db_stencil_base); /* DB_STENCIL_READ_BASE */
2981 radeon_emit(ds.db_depth_base); /* DB_Z_WRITE_BASE */
2982 radeon_emit(ds.db_stencil_base); /* DB_STENCIL_WRITE_BASE */
2983
2984 radeon_set_context_reg_seq(R_028068_DB_Z_READ_BASE_HI, 5);
2985 radeon_emit(ds.db_depth_base >> 32); /* DB_Z_READ_BASE_HI */
2986 radeon_emit(ds.db_stencil_base >> 32); /* DB_STENCIL_READ_BASE_HI */
2987 radeon_emit(ds.db_depth_base >> 32); /* DB_Z_WRITE_BASE_HI */
2988 radeon_emit(ds.db_stencil_base >> 32); /* DB_STENCIL_WRITE_BASE_HI */
2989 radeon_emit(ds.u.gfx6.db_htile_data_base >> 32); /* DB_HTILE_DATA_BASE_HI */
2990 } else if (sctx->gfx_level == GFX9) {
2991 radeon_set_context_reg_seq(R_028014_DB_HTILE_DATA_BASE, 3);
2992 radeon_emit(ds.u.gfx6.db_htile_data_base); /* DB_HTILE_DATA_BASE */
2993 radeon_emit(S_028018_BASE_HI(ds.u.gfx6.db_htile_data_base >> 32)); /* DB_HTILE_DATA_BASE_HI */
2994 radeon_emit(ds.db_depth_size); /* DB_DEPTH_SIZE */
2995
2996 radeon_set_context_reg_seq(R_028038_DB_Z_INFO, 10);
2997 radeon_emit(ds.db_z_info); /* DB_Z_INFO */
2998 radeon_emit(ds.db_stencil_info); /* DB_STENCIL_INFO */
2999 radeon_emit(ds.db_depth_base); /* DB_Z_READ_BASE */
3000 radeon_emit(S_028044_BASE_HI(ds.db_depth_base >> 32)); /* DB_Z_READ_BASE_HI */
3001 radeon_emit(ds.db_stencil_base); /* DB_STENCIL_READ_BASE */
3002 radeon_emit(S_02804C_BASE_HI(ds.db_stencil_base >> 32)); /* DB_STENCIL_READ_BASE_HI */
3003 radeon_emit(ds.db_depth_base); /* DB_Z_WRITE_BASE */
3004 radeon_emit(S_028054_BASE_HI(ds.db_depth_base >> 32)); /* DB_Z_WRITE_BASE_HI */
3005 radeon_emit(ds.db_stencil_base); /* DB_STENCIL_WRITE_BASE */
3006 radeon_emit(S_02805C_BASE_HI(ds.db_stencil_base >> 32)); /* DB_STENCIL_WRITE_BASE_HI */
3007
3008 radeon_set_context_reg_seq(R_028068_DB_Z_INFO2, 2);
3009 radeon_emit(ds.u.gfx6.db_z_info2); /* DB_Z_INFO2 */
3010 radeon_emit(ds.u.gfx6.db_stencil_info2); /* DB_STENCIL_INFO2 */
3011 } else {
3012 /* GFX6-GFX8 */
3013 radeon_set_context_reg(R_028014_DB_HTILE_DATA_BASE, ds.u.gfx6.db_htile_data_base);
3014
3015 radeon_set_context_reg_seq(R_02803C_DB_DEPTH_INFO, 9);
3016 radeon_emit(ds.u.gfx6.db_depth_info); /* DB_DEPTH_INFO */
3017 radeon_emit(ds.db_z_info); /* DB_Z_INFO */
3018 radeon_emit(ds.db_stencil_info); /* DB_STENCIL_INFO */
3019 radeon_emit(ds.db_depth_base); /* DB_Z_READ_BASE */
3020 radeon_emit(ds.db_stencil_base); /* DB_STENCIL_READ_BASE */
3021 radeon_emit(ds.db_depth_base); /* DB_Z_WRITE_BASE */
3022 radeon_emit(ds.db_stencil_base); /* DB_STENCIL_WRITE_BASE */
3023 radeon_emit(ds.db_depth_size); /* DB_DEPTH_SIZE */
3024 radeon_emit(ds.u.gfx6.db_depth_slice); /* DB_DEPTH_SLICE */
3025 }
3026
3027 radeon_set_context_reg_seq(R_028028_DB_STENCIL_CLEAR, 2);
3028 radeon_emit(tex->stencil_clear_value[level]); /* R_028028_DB_STENCIL_CLEAR */
3029 radeon_emit(fui(tex->depth_clear_value[level])); /* R_02802C_DB_DEPTH_CLEAR */
3030
3031 radeon_set_context_reg(R_028008_DB_DEPTH_VIEW, ds.db_depth_view);
3032 radeon_set_context_reg(R_028ABC_DB_HTILE_SURFACE, ds.u.gfx6.db_htile_surface);
3033 } else if (sctx->framebuffer.dirty_zsbuf) {
3034 if (sctx->gfx_level == GFX9)
3035 radeon_set_context_reg_seq(R_028038_DB_Z_INFO, 2);
3036 else
3037 radeon_set_context_reg_seq(R_028040_DB_Z_INFO, 2);
3038
3039 /* Gfx11+: DB_Z_INFO.NUM_SAMPLES should match the framebuffer samples if no Z/S is bound.
3040 * It determines the sample count for VRS, primitive-ordered pixel shading, and occlusion
3041 * queries.
3042 */
3043 radeon_emit(S_028040_FORMAT(V_028040_Z_INVALID) | /* DB_Z_INFO */
3044 S_028040_NUM_SAMPLES(sctx->gfx_level >= GFX11 ? sctx->framebuffer.log_samples : 0));
3045 radeon_emit(S_028044_FORMAT(V_028044_STENCIL_INVALID)); /* DB_STENCIL_INFO */
3046 }
3047
3048 /* Framebuffer dimensions. */
3049 /* PA_SC_WINDOW_SCISSOR_TL is set to 0,0 in gfx*_init_gfx_preamble_state */
3050 radeon_set_context_reg(R_028208_PA_SC_WINDOW_SCISSOR_BR,
3051 S_028208_BR_X(state->width) | S_028208_BR_Y(state->height));
3052
3053 if (sctx->screen->dpbb_allowed &&
3054 sctx->screen->pbb_context_states_per_bin > 1)
3055 radeon_event_write(V_028A90_BREAK_BATCH);
3056
3057 radeon_end();
3058
3059 si_update_display_dcc_dirty(sctx);
3060
3061 sctx->framebuffer.dirty_cbufs = 0;
3062 sctx->framebuffer.dirty_zsbuf = false;
3063 }
3064
gfx11_dgpu_emit_framebuffer_state(struct si_context * sctx,unsigned index)3065 static void gfx11_dgpu_emit_framebuffer_state(struct si_context *sctx, unsigned index)
3066 {
3067 struct radeon_cmdbuf *cs = &sctx->gfx_cs;
3068 struct pipe_framebuffer_state *state = &sctx->framebuffer.state;
3069 unsigned i, nr_cbufs = state->nr_cbufs;
3070 struct si_texture *tex = NULL;
3071 struct si_surface *cb = NULL;
3072 bool is_msaa_resolve = state->nr_cbufs == 2 &&
3073 state->cbufs[0] && state->cbufs[0]->texture->nr_samples > 1 &&
3074 state->cbufs[1] && state->cbufs[1]->texture->nr_samples <= 1;
3075
3076 /* CB can't do MSAA resolve on gfx11. */
3077 assert(!is_msaa_resolve);
3078
3079 radeon_begin(cs);
3080 gfx11_begin_packed_context_regs();
3081
3082 /* Colorbuffers. */
3083 for (i = 0; i < nr_cbufs; i++) {
3084 if (!(sctx->framebuffer.dirty_cbufs & (1 << i)))
3085 continue;
3086
3087 /* RB+ depth-only rendering. See the comment where we set rbplus_depth_only_opt for more
3088 * information.
3089 */
3090 if (i == 0 &&
3091 sctx->screen->info.rbplus_allowed &&
3092 !sctx->queued.named.blend->cb_target_mask) {
3093 gfx11_set_context_reg(R_028C70_CB_COLOR0_INFO + i * 0x3C,
3094 S_028C70_FORMAT_GFX11(V_028C70_COLOR_32) |
3095 S_028C70_NUMBER_TYPE(V_028C70_NUMBER_FLOAT));
3096 continue;
3097 }
3098
3099 cb = (struct si_surface *)state->cbufs[i];
3100 if (!cb) {
3101 gfx11_set_context_reg(R_028C70_CB_COLOR0_INFO + i * 0x3C,
3102 S_028C70_FORMAT_GFX11(V_028C70_COLOR_INVALID));
3103 continue;
3104 }
3105
3106 tex = (struct si_texture *)cb->base.texture;
3107 radeon_add_to_buffer_list(
3108 sctx, &sctx->gfx_cs, &tex->buffer, RADEON_USAGE_READWRITE | RADEON_USAGE_CB_NEEDS_IMPLICIT_SYNC |
3109 (tex->buffer.b.b.nr_samples > 1 ? RADEON_PRIO_COLOR_BUFFER_MSAA : RADEON_PRIO_COLOR_BUFFER));
3110
3111 if (tex->cmask_buffer && tex->cmask_buffer != &tex->buffer) {
3112 radeon_add_to_buffer_list(sctx, &sctx->gfx_cs, tex->cmask_buffer,
3113 RADEON_USAGE_READWRITE | RADEON_USAGE_CB_NEEDS_IMPLICIT_SYNC |
3114 RADEON_PRIO_SEPARATE_META);
3115 }
3116
3117 /* Compute mutable surface parameters. */
3118 const struct ac_mutable_cb_state mutable_cb_state = {
3119 .surf = &tex->surface,
3120 .cb = &cb->cb,
3121 .va = tex->buffer.gpu_address,
3122 .num_samples = cb->base.texture->nr_samples,
3123 .dcc_enabled = vi_dcc_enabled(tex, cb->base.u.tex.level),
3124 };
3125 struct ac_cb_surface cb_surf;
3126
3127 ac_set_mutable_cb_surface_fields(&sctx->screen->info, &mutable_cb_state, &cb_surf);
3128
3129 cb_surf.cb_color_info |= tex->cb_color_info;
3130
3131 gfx11_set_context_reg(R_028C60_CB_COLOR0_BASE + i * 0x3C, cb_surf.cb_color_base);
3132 gfx11_set_context_reg(R_028C6C_CB_COLOR0_VIEW + i * 0x3C, cb_surf.cb_color_view);
3133 gfx11_set_context_reg(R_028C70_CB_COLOR0_INFO + i * 0x3C, cb_surf.cb_color_info);
3134 gfx11_set_context_reg(R_028C74_CB_COLOR0_ATTRIB + i * 0x3C, cb_surf.cb_color_attrib);
3135 gfx11_set_context_reg(R_028C78_CB_COLOR0_DCC_CONTROL + i * 0x3C, cb_surf.cb_dcc_control);
3136 gfx11_set_context_reg(R_028C94_CB_COLOR0_DCC_BASE + i * 0x3C, cb_surf.cb_dcc_base);
3137 gfx11_set_context_reg(R_028E40_CB_COLOR0_BASE_EXT + i * 4, cb_surf.cb_color_base >> 32);
3138 gfx11_set_context_reg(R_028EA0_CB_COLOR0_DCC_BASE_EXT + i * 4, cb_surf.cb_dcc_base >> 32);
3139 gfx11_set_context_reg(R_028EC0_CB_COLOR0_ATTRIB2 + i * 4, cb_surf.cb_color_attrib2);
3140 gfx11_set_context_reg(R_028EE0_CB_COLOR0_ATTRIB3 + i * 4, cb_surf.cb_color_attrib3);
3141 }
3142 for (; i < 8; i++)
3143 if (sctx->framebuffer.dirty_cbufs & (1 << i))
3144 gfx11_set_context_reg(R_028C70_CB_COLOR0_INFO + i * 0x3C, 0);
3145
3146 /* ZS buffer. */
3147 if (state->zsbuf && sctx->framebuffer.dirty_zsbuf) {
3148 struct si_surface *zb = (struct si_surface *)state->zsbuf;
3149 struct si_texture *tex = (struct si_texture *)zb->base.texture;
3150
3151 radeon_add_to_buffer_list(sctx, &sctx->gfx_cs, &tex->buffer, RADEON_USAGE_READWRITE |
3152 (zb->base.texture->nr_samples > 1 ? RADEON_PRIO_DEPTH_BUFFER_MSAA
3153 : RADEON_PRIO_DEPTH_BUFFER));
3154
3155 const unsigned level = zb->base.u.tex.level;
3156
3157 /* Set mutable fields. */
3158 const struct ac_mutable_ds_state mutable_ds_state = {
3159 .ds = &zb->ds,
3160 .format = tex->db_render_format,
3161 .tc_compat_htile_enabled = vi_tc_compat_htile_enabled(tex, level, PIPE_MASK_ZS),
3162 .zrange_precision = tex->depth_clear_value[level] != 0,
3163 };
3164 struct ac_ds_surface ds;
3165
3166 ac_set_mutable_ds_surface_fields(&sctx->screen->info, &mutable_ds_state, &ds);
3167
3168 gfx11_set_context_reg(R_028014_DB_HTILE_DATA_BASE, ds.u.gfx6.db_htile_data_base);
3169 gfx11_set_context_reg(R_02801C_DB_DEPTH_SIZE_XY, ds.db_depth_size);
3170 gfx11_set_context_reg(R_028040_DB_Z_INFO, ds.db_z_info);
3171 gfx11_set_context_reg(R_028044_DB_STENCIL_INFO, ds.db_stencil_info);
3172 gfx11_set_context_reg(R_028048_DB_Z_READ_BASE, ds.db_depth_base);
3173 gfx11_set_context_reg(R_02804C_DB_STENCIL_READ_BASE, ds.db_stencil_base);
3174 gfx11_set_context_reg(R_028050_DB_Z_WRITE_BASE, ds.db_depth_base);
3175 gfx11_set_context_reg(R_028054_DB_STENCIL_WRITE_BASE, ds.db_stencil_base);
3176 gfx11_set_context_reg(R_028068_DB_Z_READ_BASE_HI, ds.db_depth_base >> 32);
3177 gfx11_set_context_reg(R_02806C_DB_STENCIL_READ_BASE_HI, ds.db_stencil_base >> 32);
3178 gfx11_set_context_reg(R_028070_DB_Z_WRITE_BASE_HI, ds.db_depth_base >> 32);
3179 gfx11_set_context_reg(R_028074_DB_STENCIL_WRITE_BASE_HI, ds.db_stencil_base >> 32);
3180 gfx11_set_context_reg(R_028078_DB_HTILE_DATA_BASE_HI, ds.u.gfx6.db_htile_data_base >> 32);
3181 gfx11_set_context_reg(R_028028_DB_STENCIL_CLEAR, tex->stencil_clear_value[level]);
3182 gfx11_set_context_reg(R_02802C_DB_DEPTH_CLEAR, fui(tex->depth_clear_value[level]));
3183 gfx11_set_context_reg(R_028008_DB_DEPTH_VIEW, ds.db_depth_view);
3184 gfx11_set_context_reg(R_028ABC_DB_HTILE_SURFACE, ds.u.gfx6.db_htile_surface);
3185 } else if (sctx->framebuffer.dirty_zsbuf) {
3186 /* Gfx11+: DB_Z_INFO.NUM_SAMPLES should match the framebuffer samples if no Z/S is bound.
3187 * It determines the sample count for VRS, primitive-ordered pixel shading, and occlusion
3188 * queries.
3189 */
3190 gfx11_set_context_reg(R_028040_DB_Z_INFO,
3191 S_028040_FORMAT(V_028040_Z_INVALID) |
3192 S_028040_NUM_SAMPLES(sctx->framebuffer.log_samples));
3193 gfx11_set_context_reg(R_028044_DB_STENCIL_INFO, S_028044_FORMAT(V_028044_STENCIL_INVALID));
3194 }
3195
3196 /* Framebuffer dimensions. */
3197 /* PA_SC_WINDOW_SCISSOR_TL is set to 0,0 in gfx*_init_gfx_preamble_state */
3198 gfx11_set_context_reg(R_028208_PA_SC_WINDOW_SCISSOR_BR,
3199 S_028208_BR_X(state->width) | S_028208_BR_Y(state->height));
3200 gfx11_end_packed_context_regs();
3201
3202 if (sctx->screen->dpbb_allowed &&
3203 sctx->screen->pbb_context_states_per_bin > 1)
3204 radeon_event_write(V_028A90_BREAK_BATCH);
3205
3206 radeon_end();
3207
3208 si_update_display_dcc_dirty(sctx);
3209
3210 sctx->framebuffer.dirty_cbufs = 0;
3211 sctx->framebuffer.dirty_zsbuf = false;
3212 }
3213
gfx12_emit_framebuffer_state(struct si_context * sctx,unsigned index)3214 static void gfx12_emit_framebuffer_state(struct si_context *sctx, unsigned index)
3215 {
3216 struct radeon_cmdbuf *cs = &sctx->gfx_cs;
3217 struct pipe_framebuffer_state *state = &sctx->framebuffer.state;
3218 unsigned i, nr_cbufs = state->nr_cbufs;
3219 struct si_texture *tex = NULL;
3220 struct si_surface *cb = NULL;
3221 bool is_msaa_resolve = state->nr_cbufs == 2 &&
3222 state->cbufs[0] && state->cbufs[0]->texture->nr_samples > 1 &&
3223 state->cbufs[1] && state->cbufs[1]->texture->nr_samples <= 1;
3224
3225 /* CB can't do MSAA resolve. */
3226 assert(!is_msaa_resolve);
3227
3228 radeon_begin(cs);
3229 gfx12_begin_context_regs();
3230
3231 /* Colorbuffers. */
3232 for (i = 0; i < nr_cbufs; i++) {
3233 if (!(sctx->framebuffer.dirty_cbufs & (1 << i)))
3234 continue;
3235
3236 /* RB+ depth-only rendering. See the comment where we set rbplus_depth_only_opt for more
3237 * information.
3238 */
3239 if (i == 0 &&
3240 sctx->screen->info.rbplus_allowed &&
3241 !sctx->queued.named.blend->cb_target_mask) {
3242 gfx12_set_context_reg(R_028EC0_CB_COLOR0_INFO + i * 4,
3243 S_028EC0_FORMAT(V_028C70_COLOR_32) |
3244 S_028EC0_NUMBER_TYPE(V_028C70_NUMBER_FLOAT));
3245 continue;
3246 }
3247
3248 cb = (struct si_surface *)state->cbufs[i];
3249 if (!cb) {
3250 gfx12_set_context_reg(R_028EC0_CB_COLOR0_INFO + i * 4,
3251 S_028EC0_FORMAT(V_028C70_COLOR_INVALID));
3252 continue;
3253 }
3254
3255 tex = (struct si_texture *)cb->base.texture;
3256 radeon_add_to_buffer_list(
3257 sctx, &sctx->gfx_cs, &tex->buffer, RADEON_USAGE_READWRITE | RADEON_USAGE_CB_NEEDS_IMPLICIT_SYNC |
3258 (tex->buffer.b.b.nr_samples > 1 ? RADEON_PRIO_COLOR_BUFFER_MSAA : RADEON_PRIO_COLOR_BUFFER));
3259
3260 /* Compute mutable surface parameters. */
3261 const struct ac_mutable_cb_state mutable_cb_state = {
3262 .surf = &tex->surface,
3263 .cb = &cb->cb,
3264 .va = tex->buffer.gpu_address,
3265 };
3266 struct ac_cb_surface cb_surf;
3267
3268 ac_set_mutable_cb_surface_fields(&sctx->screen->info, &mutable_cb_state, &cb_surf);
3269
3270 gfx12_set_context_reg(R_028C60_CB_COLOR0_BASE + i * 0x24, cb_surf.cb_color_base);
3271 gfx12_set_context_reg(R_028C64_CB_COLOR0_VIEW + i * 0x24, cb_surf.cb_color_view);
3272 gfx12_set_context_reg(R_028C68_CB_COLOR0_VIEW2 + i * 0x24, cb_surf.cb_color_view2);
3273 gfx12_set_context_reg(R_028C6C_CB_COLOR0_ATTRIB + i * 0x24, cb_surf.cb_color_attrib);
3274 gfx12_set_context_reg(R_028C70_CB_COLOR0_FDCC_CONTROL + i * 0x24, cb_surf.cb_dcc_control);
3275 gfx12_set_context_reg(R_028C78_CB_COLOR0_ATTRIB2 + i * 0x24, cb_surf.cb_color_attrib2);
3276 gfx12_set_context_reg(R_028C7C_CB_COLOR0_ATTRIB3 + i * 0x24, cb_surf.cb_color_attrib3);
3277 gfx12_set_context_reg(R_028E40_CB_COLOR0_BASE_EXT + i * 4, cb_surf.cb_color_base >> 32);
3278 gfx12_set_context_reg(R_028EC0_CB_COLOR0_INFO + i * 4, cb_surf.cb_color_info);
3279 }
3280 /* Set unbound colorbuffers. */
3281 for (; i < 8; i++)
3282 if (sctx->framebuffer.dirty_cbufs & (1 << i))
3283 gfx12_set_context_reg(R_028EC0_CB_COLOR0_INFO + i * 4, 0);
3284
3285 /* ZS buffer. */
3286 if (state->zsbuf && sctx->framebuffer.dirty_zsbuf) {
3287 struct si_surface *zb = (struct si_surface *)state->zsbuf;
3288 struct si_texture *tex = (struct si_texture *)zb->base.texture;
3289
3290 radeon_add_to_buffer_list(sctx, &sctx->gfx_cs, &tex->buffer,
3291 RADEON_USAGE_READWRITE | RADEON_USAGE_DB_NEEDS_IMPLICIT_SYNC |
3292 (zb->base.texture->nr_samples > 1 ? RADEON_PRIO_DEPTH_BUFFER_MSAA
3293 : RADEON_PRIO_DEPTH_BUFFER));
3294 gfx12_set_context_reg(R_028004_DB_DEPTH_VIEW, zb->ds.db_depth_view);
3295 gfx12_set_context_reg(R_028008_DB_DEPTH_VIEW1, zb->ds.u.gfx12.db_depth_view1);
3296 gfx12_set_context_reg(R_028014_DB_DEPTH_SIZE_XY, zb->ds.db_depth_size);
3297 gfx12_set_context_reg(R_028018_DB_Z_INFO, zb->ds.db_z_info);
3298 gfx12_set_context_reg(R_02801C_DB_STENCIL_INFO, zb->ds.db_stencil_info);
3299 gfx12_set_context_reg(R_028020_DB_Z_READ_BASE, zb->ds.db_depth_base);
3300 gfx12_set_context_reg(R_028024_DB_Z_READ_BASE_HI, zb->ds.db_depth_base >> 32);
3301 gfx12_set_context_reg(R_028028_DB_Z_WRITE_BASE, zb->ds.db_depth_base);
3302 gfx12_set_context_reg(R_02802C_DB_Z_WRITE_BASE_HI, zb->ds.db_depth_base >> 32);
3303 gfx12_set_context_reg(R_028030_DB_STENCIL_READ_BASE, zb->ds.db_stencil_base);
3304 gfx12_set_context_reg(R_028034_DB_STENCIL_READ_BASE_HI, zb->ds.db_stencil_base >> 32);
3305 gfx12_set_context_reg(R_028038_DB_STENCIL_WRITE_BASE, zb->ds.db_stencil_base);
3306 gfx12_set_context_reg(R_02803C_DB_STENCIL_WRITE_BASE_HI, zb->ds.db_stencil_base >> 32);
3307
3308 if (tex->force_disable_hiz_his) {
3309 gfx12_set_context_reg(R_028B94_PA_SC_HIZ_INFO, S_028B94_SURFACE_ENABLE(0));
3310 gfx12_set_context_reg(R_028B98_PA_SC_HIS_INFO, S_028B98_SURFACE_ENABLE(0));
3311 } else {
3312 gfx12_set_context_reg(R_028B94_PA_SC_HIZ_INFO, zb->ds.u.gfx12.hiz_info);
3313 gfx12_set_context_reg(R_028B98_PA_SC_HIS_INFO, zb->ds.u.gfx12.his_info);
3314
3315 if (zb->ds.u.gfx12.hiz_info) {
3316 gfx12_set_context_reg(R_028B9C_PA_SC_HIZ_BASE, zb->ds.u.gfx12.hiz_base);
3317 gfx12_set_context_reg(R_028BA0_PA_SC_HIZ_BASE_EXT, zb->ds.u.gfx12.hiz_base >> 32);
3318 gfx12_set_context_reg(R_028BA4_PA_SC_HIZ_SIZE_XY, zb->ds.u.gfx12.hiz_size_xy);
3319 }
3320 if (zb->ds.u.gfx12.his_info) {
3321 gfx12_set_context_reg(R_028BA8_PA_SC_HIS_BASE, zb->ds.u.gfx12.his_base);
3322 gfx12_set_context_reg(R_028BAC_PA_SC_HIS_BASE_EXT, zb->ds.u.gfx12.his_base >> 32);
3323 gfx12_set_context_reg(R_028BB0_PA_SC_HIS_SIZE_XY, zb->ds.u.gfx12.his_size_xy);
3324 }
3325 }
3326 } else if (sctx->framebuffer.dirty_zsbuf) {
3327 gfx12_set_context_reg(R_028018_DB_Z_INFO,
3328 S_028040_FORMAT(V_028040_Z_INVALID) |
3329 S_028040_NUM_SAMPLES(sctx->framebuffer.log_samples));
3330 gfx12_set_context_reg(R_02801C_DB_STENCIL_INFO,
3331 S_028044_FORMAT(V_028044_STENCIL_INVALID)|
3332 S_028044_TILE_STENCIL_DISABLE(1));
3333 gfx12_set_context_reg(R_028B94_PA_SC_HIZ_INFO, S_028B94_SURFACE_ENABLE(0));
3334 gfx12_set_context_reg(R_028B98_PA_SC_HIS_INFO, S_028B98_SURFACE_ENABLE(0));
3335 }
3336
3337 /* Framebuffer dimensions. */
3338 /* PA_SC_WINDOW_SCISSOR_TL is set in gfx12_init_gfx_preamble_state */
3339 gfx12_set_context_reg(R_028208_PA_SC_WINDOW_SCISSOR_BR,
3340 S_028208_BR_X(state->width - 1) | /* inclusive */
3341 S_028208_BR_Y(state->height - 1)); /* inclusive */
3342 gfx12_end_context_regs();
3343
3344 if (sctx->screen->dpbb_allowed &&
3345 sctx->screen->pbb_context_states_per_bin > 1)
3346 radeon_event_write(V_028A90_BREAK_BATCH);
3347
3348 radeon_end();
3349
3350 sctx->framebuffer.dirty_cbufs = 0;
3351 sctx->framebuffer.dirty_zsbuf = false;
3352 }
3353
si_out_of_order_rasterization(struct si_context * sctx)3354 static bool si_out_of_order_rasterization(struct si_context *sctx)
3355 {
3356 struct si_state_blend *blend = sctx->queued.named.blend;
3357 struct si_state_dsa *dsa = sctx->queued.named.dsa;
3358
3359 if (!sctx->screen->info.has_out_of_order_rast)
3360 return false;
3361
3362 unsigned colormask = sctx->framebuffer.colorbuf_enabled_4bit;
3363
3364 colormask &= blend->cb_target_enabled_4bit;
3365
3366 /* Conservative: No logic op. */
3367 if (colormask && blend->logicop_enable)
3368 return false;
3369
3370 struct si_dsa_order_invariance dsa_order_invariant = {.zs = true,
3371 .pass_set = true};
3372
3373 if (sctx->framebuffer.state.zsbuf) {
3374 struct si_texture *zstex = (struct si_texture *)sctx->framebuffer.state.zsbuf->texture;
3375 bool has_stencil = zstex->surface.has_stencil;
3376 dsa_order_invariant = dsa->order_invariance[has_stencil];
3377 if (!dsa_order_invariant.zs)
3378 return false;
3379
3380 /* The set of PS invocations is always order invariant,
3381 * except when early Z/S tests are requested. */
3382 if (sctx->shader.ps.cso && sctx->shader.ps.cso->info.base.writes_memory &&
3383 sctx->shader.ps.cso->info.base.fs.early_fragment_tests &&
3384 !dsa_order_invariant.pass_set)
3385 return false;
3386
3387 if (sctx->occlusion_query_mode == SI_OCCLUSION_QUERY_MODE_PRECISE_INTEGER &&
3388 !dsa_order_invariant.pass_set)
3389 return false;
3390 }
3391
3392 if (!colormask)
3393 return true;
3394
3395 unsigned blendmask = colormask & blend->blend_enable_4bit;
3396
3397 if (blendmask) {
3398 /* Only commutative blending. */
3399 if (blendmask & ~blend->commutative_4bit)
3400 return false;
3401
3402 if (!dsa_order_invariant.pass_set)
3403 return false;
3404 }
3405
3406 if (colormask & ~blendmask)
3407 return false;
3408
3409 return true;
3410 }
3411
si_emit_msaa_config(struct si_context * sctx,unsigned index)3412 static void si_emit_msaa_config(struct si_context *sctx, unsigned index)
3413 {
3414 struct radeon_cmdbuf *cs = &sctx->gfx_cs;
3415 unsigned num_tile_pipes = sctx->screen->info.num_tile_pipes;
3416 /* 33% faster rendering to linear color buffers */
3417 bool dst_is_linear = sctx->framebuffer.any_dst_linear;
3418 bool out_of_order_rast = si_out_of_order_rasterization(sctx);
3419 unsigned sc_mode_cntl_1 =
3420 S_028A4C_WALK_SIZE(dst_is_linear) | S_028A4C_WALK_FENCE_ENABLE(!dst_is_linear) |
3421 S_028A4C_WALK_FENCE_SIZE(num_tile_pipes == 2 ? 2 : 3) |
3422 S_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE(out_of_order_rast) |
3423 S_028A4C_OUT_OF_ORDER_WATER_MARK(sctx->gfx_level >= GFX12 ? 0 : 0x7) |
3424 /* This should also be 0 when the VRS image is enabled. */
3425 S_028A4C_WALK_ALIGN8_PRIM_FITS_ST(!sctx->framebuffer.has_hiz_his) |
3426 /* always 1: */
3427 S_028A4C_SUPERTILE_WALK_ORDER_ENABLE(1) |
3428 S_028A4C_TILE_WALK_ORDER_ENABLE(1) | S_028A4C_MULTI_SHADER_ENGINE_PRIM_DISCARD_ENABLE(1) |
3429 S_028A4C_FORCE_EOV_CNTDWN_ENABLE(1) | S_028A4C_FORCE_EOV_REZ_ENABLE(1);
3430 unsigned db_eqaa = S_028804_HIGH_QUALITY_INTERSECTIONS(1) |
3431 S_028804_INCOHERENT_EQAA_READS(sctx->gfx_level < GFX12) |
3432 S_028804_STATIC_ANCHOR_ASSOCIATIONS(1);
3433 unsigned coverage_samples, z_samples;
3434 struct si_state_rasterizer *rs = sctx->queued.named.rasterizer;
3435
3436 /* S: Coverage samples (up to 16x):
3437 * - Scan conversion samples (PA_SC_AA_CONFIG.MSAA_NUM_SAMPLES)
3438 * - CB FMASK samples (CB_COLORi_ATTRIB.NUM_SAMPLES)
3439 *
3440 * Z: Z/S samples (up to 8x, must be <= coverage samples and >= color samples):
3441 * - Value seen by DB (DB_Z_INFO.NUM_SAMPLES)
3442 * - Value seen by CB, must be correct even if Z/S is unbound (DB_EQAA.MAX_ANCHOR_SAMPLES)
3443 * # Missing samples are derived from Z planes if Z is compressed (up to 16x quality), or
3444 * # from the closest defined sample if Z is uncompressed (same quality as the number of
3445 * # Z samples).
3446 *
3447 * F: Color samples (up to 8x, must be <= coverage samples):
3448 * - CB color samples (CB_COLORi_ATTRIB.NUM_FRAGMENTS)
3449 * - PS iter samples (DB_EQAA.PS_ITER_SAMPLES)
3450 *
3451 * Can be anything between coverage and color samples:
3452 * - SampleMaskIn samples (PA_SC_AA_CONFIG.MSAA_EXPOSED_SAMPLES)
3453 * - SampleMaskOut samples (DB_EQAA.MASK_EXPORT_NUM_SAMPLES)
3454 * - Alpha-to-coverage samples (DB_EQAA.ALPHA_TO_MASK_NUM_SAMPLES)
3455 * - Occlusion query samples (DB_COUNT_CONTROL.SAMPLE_RATE)
3456 * # All are currently set the same as coverage samples.
3457 *
3458 * If color samples < coverage samples, FMASK has a higher bpp to store an "unknown"
3459 * flag for undefined color samples. A shader-based resolve must handle unknowns
3460 * or mask them out with AND. Unknowns can also be guessed from neighbors via
3461 * an edge-detect shader-based resolve, which is required to make "color samples = 1"
3462 * useful. The CB resolve always drops unknowns.
3463 *
3464 * Sensible AA configurations:
3465 * EQAA 16s 8z 8f - might look the same as 16x MSAA if Z is compressed
3466 * EQAA 16s 8z 4f - might look the same as 16x MSAA if Z is compressed
3467 * EQAA 16s 4z 4f - might look the same as 16x MSAA if Z is compressed
3468 * EQAA 8s 8z 8f = 8x MSAA
3469 * EQAA 8s 8z 4f - might look the same as 8x MSAA
3470 * EQAA 8s 8z 2f - might look the same as 8x MSAA with low-density geometry
3471 * EQAA 8s 4z 4f - might look the same as 8x MSAA if Z is compressed
3472 * EQAA 8s 4z 2f - might look the same as 8x MSAA with low-density geometry if Z is compressed
3473 * EQAA 4s 4z 4f = 4x MSAA
3474 * EQAA 4s 4z 2f - might look the same as 4x MSAA with low-density geometry
3475 * EQAA 2s 2z 2f = 2x MSAA
3476 */
3477 coverage_samples = si_get_num_coverage_samples(sctx);
3478
3479 /* DCC_DECOMPRESS and ELIMINATE_FAST_CLEAR require MSAA_NUM_SAMPLES=0. */
3480 if (sctx->gfx_level >= GFX11 && sctx->gfx11_force_msaa_num_samples_zero)
3481 coverage_samples = 1;
3482
3483 /* The DX10 diamond test is not required by GL and decreases line rasterization
3484 * performance, so don't use it.
3485 */
3486 unsigned sc_line_cntl = 0;
3487 unsigned sc_aa_config = 0;
3488
3489 if (coverage_samples > 1 && (rs->multisample_enable ||
3490 sctx->smoothing_enabled)) {
3491 unsigned log_samples = util_logbase2(coverage_samples);
3492
3493 sc_line_cntl |= S_028BDC_EXPAND_LINE_WIDTH(1) |
3494 S_028BDC_PERPENDICULAR_ENDCAP_ENA(rs->perpendicular_end_caps) |
3495 S_028BDC_EXTRA_DX_DY_PRECISION(rs->perpendicular_end_caps &&
3496 (sctx->family == CHIP_VEGA20 ||
3497 sctx->gfx_level >= GFX10));
3498 sc_aa_config = S_028BE0_MSAA_NUM_SAMPLES(log_samples) |
3499 S_028BE0_MSAA_EXPOSED_SAMPLES(log_samples);
3500
3501 if (sctx->gfx_level < GFX12) {
3502 sc_aa_config |= S_028BE0_MAX_SAMPLE_DIST(si_msaa_max_distance[log_samples]) |
3503 S_028BE0_COVERED_CENTROID_IS_CENTER(sctx->gfx_level >= GFX10_3);
3504 }
3505 }
3506
3507 if (sctx->framebuffer.nr_samples > 1 ||
3508 sctx->smoothing_enabled) {
3509 if (sctx->framebuffer.state.zsbuf) {
3510 z_samples = sctx->framebuffer.state.zsbuf->texture->nr_samples;
3511 z_samples = MAX2(1, z_samples);
3512 } else {
3513 z_samples = coverage_samples;
3514 }
3515 unsigned log_samples = util_logbase2(coverage_samples);
3516 unsigned log_z_samples = util_logbase2(z_samples);
3517 unsigned ps_iter_samples = si_get_ps_iter_samples(sctx);
3518 unsigned log_ps_iter_samples = util_logbase2(ps_iter_samples);
3519 if (sctx->framebuffer.nr_samples > 1) {
3520 if (sctx->gfx_level >= GFX12) {
3521 sc_aa_config |= S_028BE0_PS_ITER_SAMPLES(log_ps_iter_samples);
3522 db_eqaa |= S_028078_MASK_EXPORT_NUM_SAMPLES(log_samples) |
3523 S_028078_ALPHA_TO_MASK_NUM_SAMPLES(log_samples);
3524 } else {
3525 db_eqaa |= S_028804_MAX_ANCHOR_SAMPLES(log_z_samples) |
3526 S_028804_PS_ITER_SAMPLES(log_ps_iter_samples) |
3527 S_028804_MASK_EXPORT_NUM_SAMPLES(log_samples) |
3528 S_028804_ALPHA_TO_MASK_NUM_SAMPLES(log_samples);
3529 }
3530 sc_mode_cntl_1 |= S_028A4C_PS_ITER_SAMPLE(ps_iter_samples > 1);
3531 } else if (sctx->smoothing_enabled) {
3532 db_eqaa |= S_028804_OVERRASTERIZATION_AMOUNT(log_samples);
3533 }
3534 }
3535
3536 if (sctx->gfx_level >= GFX12) {
3537 radeon_begin(cs);
3538 gfx12_begin_context_regs();
3539 gfx12_opt_set_context_reg(R_028BDC_PA_SC_LINE_CNTL, SI_TRACKED_PA_SC_LINE_CNTL,
3540 sc_line_cntl);
3541 gfx12_opt_set_context_reg(R_028BE0_PA_SC_AA_CONFIG, SI_TRACKED_PA_SC_AA_CONFIG,
3542 sc_aa_config);
3543 gfx12_opt_set_context_reg(R_028078_DB_EQAA, SI_TRACKED_DB_EQAA, db_eqaa);
3544 gfx12_opt_set_context_reg(R_028A4C_PA_SC_MODE_CNTL_1, SI_TRACKED_PA_SC_MODE_CNTL_1,
3545 sc_mode_cntl_1);
3546 gfx12_end_context_regs();
3547 radeon_end(); /* don't track context rolls on GFX12 */
3548 } else if (sctx->screen->info.has_set_context_pairs_packed) {
3549 radeon_begin(cs);
3550 gfx11_begin_packed_context_regs();
3551 gfx11_opt_set_context_reg(R_028BDC_PA_SC_LINE_CNTL, SI_TRACKED_PA_SC_LINE_CNTL,
3552 sc_line_cntl);
3553 gfx11_opt_set_context_reg(R_028BE0_PA_SC_AA_CONFIG, SI_TRACKED_PA_SC_AA_CONFIG,
3554 sc_aa_config);
3555 gfx11_opt_set_context_reg(R_028804_DB_EQAA, SI_TRACKED_DB_EQAA, db_eqaa);
3556 gfx11_opt_set_context_reg(R_028A4C_PA_SC_MODE_CNTL_1, SI_TRACKED_PA_SC_MODE_CNTL_1,
3557 sc_mode_cntl_1);
3558 gfx11_end_packed_context_regs();
3559 radeon_end(); /* don't track context rolls on GFX11 */
3560 } else {
3561 radeon_begin(cs);
3562 radeon_opt_set_context_reg2(R_028BDC_PA_SC_LINE_CNTL, SI_TRACKED_PA_SC_LINE_CNTL,
3563 sc_line_cntl, sc_aa_config);
3564 radeon_opt_set_context_reg(R_028804_DB_EQAA, SI_TRACKED_DB_EQAA, db_eqaa);
3565 radeon_opt_set_context_reg(R_028A4C_PA_SC_MODE_CNTL_1, SI_TRACKED_PA_SC_MODE_CNTL_1,
3566 sc_mode_cntl_1);
3567 radeon_end_update_context_roll();
3568 }
3569 }
3570
si_update_ps_iter_samples(struct si_context * sctx)3571 void si_update_ps_iter_samples(struct si_context *sctx)
3572 {
3573 if (sctx->ps_iter_samples == sctx->last_ps_iter_samples)
3574 return;
3575
3576 sctx->last_ps_iter_samples = sctx->ps_iter_samples;
3577 si_ps_key_update_sample_shading(sctx);
3578 if (sctx->framebuffer.nr_samples > 1)
3579 si_mark_atom_dirty(sctx, &sctx->atoms.s.msaa_config);
3580 if (sctx->screen->dpbb_allowed)
3581 si_mark_atom_dirty(sctx, &sctx->atoms.s.dpbb_state);
3582 }
3583
si_set_min_samples(struct pipe_context * ctx,unsigned min_samples)3584 static void si_set_min_samples(struct pipe_context *ctx, unsigned min_samples)
3585 {
3586 struct si_context *sctx = (struct si_context *)ctx;
3587
3588 /* The hardware can only do sample shading with 2^n samples. */
3589 min_samples = util_next_power_of_two(min_samples);
3590
3591 if (sctx->ps_iter_samples == min_samples)
3592 return;
3593
3594 sctx->ps_iter_samples = min_samples;
3595
3596 si_ps_key_update_framebuffer_rasterizer_sample_shading(sctx);
3597 sctx->do_update_shaders = true;
3598
3599 si_update_ps_iter_samples(sctx);
3600 }
3601
3602 /*
3603 * Samplers
3604 */
3605
3606 /**
3607 * Build the sampler view descriptor for a buffer texture.
3608 * @param state 256-bit descriptor; only the high 128 bits are filled in
3609 */
si_make_buffer_descriptor(struct si_screen * screen,struct si_resource * buf,enum pipe_format format,unsigned offset,unsigned num_elements,uint32_t * state)3610 void si_make_buffer_descriptor(struct si_screen *screen, struct si_resource *buf,
3611 enum pipe_format format, unsigned offset, unsigned num_elements,
3612 uint32_t *state)
3613 {
3614 const struct util_format_description *desc;
3615 unsigned stride;
3616 unsigned num_records;
3617
3618 desc = util_format_description(format);
3619 stride = desc->block.bits / 8;
3620
3621 num_records = num_elements;
3622 num_records = MIN2(num_records, (buf->b.b.width0 - offset) / stride);
3623
3624 /* The NUM_RECORDS field has a different meaning depending on the chip,
3625 * instruction type, STRIDE, and SWIZZLE_ENABLE.
3626 *
3627 * GFX6-7,10:
3628 * - If STRIDE == 0, it's in byte units.
3629 * - If STRIDE != 0, it's in units of STRIDE, used with inst.IDXEN.
3630 *
3631 * GFX8:
3632 * - For SMEM and STRIDE == 0, it's in byte units.
3633 * - For SMEM and STRIDE != 0, it's in units of STRIDE.
3634 * - For VMEM and STRIDE == 0 or SWIZZLE_ENABLE == 0, it's in byte units.
3635 * - For VMEM and STRIDE != 0 and SWIZZLE_ENABLE == 1, it's in units of STRIDE.
3636 * NOTE: There is incompatibility between VMEM and SMEM opcodes due to SWIZZLE_-
3637 * ENABLE. The workaround is to set STRIDE = 0 if SWIZZLE_ENABLE == 0 when
3638 * using SMEM. This can be done in the shader by clearing STRIDE with s_and.
3639 * That way the same descriptor can be used by both SMEM and VMEM.
3640 *
3641 * GFX9:
3642 * - For SMEM and STRIDE == 0, it's in byte units.
3643 * - For SMEM and STRIDE != 0, it's in units of STRIDE.
3644 * - For VMEM and inst.IDXEN == 0 or STRIDE == 0, it's in byte units.
3645 * - For VMEM and inst.IDXEN == 1 and STRIDE != 0, it's in units of STRIDE.
3646 */
3647 if (screen->info.gfx_level == GFX8)
3648 num_records *= stride;
3649
3650 const struct ac_buffer_state buffer_state = {
3651 .size = num_records,
3652 .format = format,
3653 .swizzle =
3654 {
3655 desc->swizzle[0],
3656 desc->swizzle[1],
3657 desc->swizzle[2],
3658 desc->swizzle[3],
3659 },
3660 .stride = stride,
3661 .gfx10_oob_select = V_008F0C_OOB_SELECT_STRUCTURED_WITH_OFFSET,
3662 };
3663
3664 ac_build_buffer_descriptor(screen->info.gfx_level, &buffer_state, &state[4]);
3665 }
3666
3667 /**
3668 * Translate the parameters to an image descriptor for CDNA image emulation.
3669 * In this function, we choose our own image descriptor format because we emulate image opcodes
3670 * using buffer opcodes.
3671 */
cdna_emu_make_image_descriptor(struct si_screen * screen,struct si_texture * tex,bool sampler,enum pipe_texture_target target,enum pipe_format pipe_format,const unsigned char state_swizzle[4],unsigned first_level,unsigned last_level,unsigned first_layer,unsigned last_layer,unsigned width,unsigned height,unsigned depth,uint32_t * state,uint32_t * fmask_state)3672 static void cdna_emu_make_image_descriptor(struct si_screen *screen, struct si_texture *tex,
3673 bool sampler, enum pipe_texture_target target,
3674 enum pipe_format pipe_format,
3675 const unsigned char state_swizzle[4], unsigned first_level,
3676 unsigned last_level, unsigned first_layer,
3677 unsigned last_layer, unsigned width, unsigned height,
3678 unsigned depth, uint32_t *state, uint32_t *fmask_state)
3679 {
3680 const struct util_format_description *desc = util_format_description(pipe_format);
3681
3682 /* We don't need support these. We only need enough to support VAAPI and OpenMAX. */
3683 if (target == PIPE_TEXTURE_CUBE ||
3684 target == PIPE_TEXTURE_CUBE_ARRAY ||
3685 tex->buffer.b.b.last_level > 0 ||
3686 tex->buffer.b.b.nr_samples >= 2 ||
3687 desc->colorspace != UTIL_FORMAT_COLORSPACE_RGB ||
3688 desc->layout == UTIL_FORMAT_LAYOUT_SUBSAMPLED ||
3689 util_format_is_compressed(pipe_format)) {
3690 assert(!"unexpected texture type");
3691 memset(state, 0, 8 * 4);
3692 return;
3693 }
3694
3695 /* Adjust the image parameters according to the texture type. */
3696 switch (target) {
3697 case PIPE_TEXTURE_1D:
3698 height = 1;
3699 FALLTHROUGH;
3700 case PIPE_TEXTURE_2D:
3701 case PIPE_TEXTURE_RECT:
3702 depth = 1;
3703 break;
3704
3705 case PIPE_TEXTURE_1D_ARRAY:
3706 height = 1;
3707 FALLTHROUGH;
3708 case PIPE_TEXTURE_2D_ARRAY:
3709 first_layer = MIN2(first_layer, tex->buffer.b.b.array_size - 1);
3710 last_layer = MIN2(last_layer, tex->buffer.b.b.array_size - 1);
3711 last_layer = MAX2(last_layer, first_layer);
3712 depth = last_layer - first_layer + 1;
3713 break;
3714
3715 case PIPE_TEXTURE_3D:
3716 first_layer = 0;
3717 break;
3718
3719 default:
3720 unreachable("invalid texture target");
3721 }
3722
3723 unsigned stride = desc->block.bits / 8;
3724 uint64_t num_records = tex->surface.surf_size / stride;
3725 assert(num_records <= UINT32_MAX);
3726
3727 /* Prepare the format fields. */
3728 unsigned char swizzle[4];
3729 util_format_compose_swizzles(desc->swizzle, state_swizzle, swizzle);
3730
3731 /* Buffer descriptor */
3732 const struct ac_buffer_state buffer_state = {
3733 .size = num_records,
3734 .format = pipe_format,
3735 .swizzle =
3736 {
3737 desc->swizzle[0],
3738 desc->swizzle[1],
3739 desc->swizzle[2],
3740 desc->swizzle[3],
3741 },
3742 .stride = stride,
3743 .gfx10_oob_select = V_008F0C_OOB_SELECT_STRUCTURED_WITH_OFFSET,
3744 };
3745
3746 ac_build_buffer_descriptor(screen->info.gfx_level, &buffer_state, &state[0]);
3747
3748 /* Additional fields used by image opcode emulation. */
3749 state[4] = width | (height << 16);
3750 state[5] = depth | (first_layer << 16);
3751 state[6] = tex->surface.u.gfx9.surf_pitch;
3752 state[7] = (uint32_t)tex->surface.u.gfx9.surf_pitch * tex->surface.u.gfx9.surf_height;
3753 }
3754
3755 /**
3756 * Build the sampler view descriptor for a texture.
3757 */
gfx10_make_texture_descriptor(struct si_screen * screen,struct si_texture * tex,bool sampler,enum pipe_texture_target target,enum pipe_format pipe_format,const unsigned char state_swizzle[4],unsigned first_level,unsigned last_level,unsigned first_layer,unsigned last_layer,unsigned width,unsigned height,unsigned depth,bool get_bo_metadata,uint32_t * state,uint32_t * fmask_state)3758 static void gfx10_make_texture_descriptor(
3759 struct si_screen *screen, struct si_texture *tex, bool sampler, enum pipe_texture_target target,
3760 enum pipe_format pipe_format, const unsigned char state_swizzle[4], unsigned first_level,
3761 unsigned last_level, unsigned first_layer, unsigned last_layer, unsigned width, unsigned height,
3762 unsigned depth, bool get_bo_metadata, uint32_t *state, uint32_t *fmask_state)
3763 {
3764 struct pipe_resource *res = &tex->buffer.b.b;
3765 const struct util_format_description *desc;
3766 unsigned char swizzle[4];
3767 unsigned type;
3768
3769 desc = util_format_description(pipe_format);
3770
3771 if (desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS) {
3772 const unsigned char swizzle_xxxx[4] = {0, 0, 0, 0};
3773 const unsigned char swizzle_yyyy[4] = {1, 1, 1, 1};
3774 const unsigned char swizzle_wwww[4] = {3, 3, 3, 3};
3775
3776 switch (pipe_format) {
3777 case PIPE_FORMAT_S8_UINT_Z24_UNORM:
3778 case PIPE_FORMAT_X32_S8X24_UINT:
3779 case PIPE_FORMAT_X8Z24_UNORM:
3780 util_format_compose_swizzles(swizzle_yyyy, state_swizzle, swizzle);
3781 break;
3782 case PIPE_FORMAT_X24S8_UINT:
3783 /*
3784 * X24S8 is implemented as an 8_8_8_8 data format, to
3785 * fix texture gathers. This affects at least
3786 * GL45-CTS.texture_cube_map_array.sampling on GFX8.
3787 */
3788 util_format_compose_swizzles(swizzle_wwww, state_swizzle, swizzle);
3789 break;
3790 default:
3791 util_format_compose_swizzles(swizzle_xxxx, state_swizzle, swizzle);
3792 }
3793 } else {
3794 util_format_compose_swizzles(desc->swizzle, state_swizzle, swizzle);
3795 }
3796
3797 if (!sampler && (res->target == PIPE_TEXTURE_CUBE || res->target == PIPE_TEXTURE_CUBE_ARRAY)) {
3798 /* For the purpose of shader images, treat cube maps as 2D
3799 * arrays.
3800 */
3801 type = V_008F1C_SQ_RSRC_IMG_2D_ARRAY;
3802 } else {
3803 type = si_tex_dim(screen, tex, target, res->nr_samples);
3804 }
3805
3806 if (type == V_008F1C_SQ_RSRC_IMG_1D_ARRAY) {
3807 height = 1;
3808 depth = res->array_size;
3809 } else if (type == V_008F1C_SQ_RSRC_IMG_2D_ARRAY || type == V_008F1C_SQ_RSRC_IMG_2D_MSAA_ARRAY) {
3810 if (sampler || res->target != PIPE_TEXTURE_3D)
3811 depth = res->array_size;
3812 } else if (type == V_008F1C_SQ_RSRC_IMG_CUBE)
3813 depth = res->array_size / 6;
3814
3815 const struct ac_texture_state tex_state = {
3816 .surf = &tex->surface,
3817 .format = pipe_format,
3818 .img_format = res->format,
3819 .width = width,
3820 .height = height,
3821 .depth = (type == V_008F1C_SQ_RSRC_IMG_3D && sampler) ? depth - 1 : last_layer,
3822 .type = type,
3823 .swizzle =
3824 {
3825 swizzle[0],
3826 swizzle[1],
3827 swizzle[2],
3828 swizzle[3],
3829 },
3830 .num_samples = res->nr_samples,
3831 .num_storage_samples = res->nr_storage_samples,
3832 .first_level = first_level,
3833 .last_level = last_level,
3834 .num_levels = res->last_level + 1,
3835 .first_layer = first_layer,
3836 .last_layer = last_layer,
3837 .gfx10 = {
3838 .uav3d = !!(type == V_008F1C_SQ_RSRC_IMG_3D && !sampler),
3839 .upgraded_depth = tex->upgraded_depth,
3840 },
3841 .dcc_enabled = vi_dcc_enabled(tex, first_level),
3842 };
3843
3844 ac_build_texture_descriptor(&screen->info, &tex_state, &state[0]);
3845
3846 /* Initialize the sampler view for FMASK. */
3847 if (tex->surface.fmask_offset) {
3848 const struct ac_fmask_state ac_state = {
3849 .surf = &tex->surface,
3850 .va = tex->buffer.gpu_address,
3851 .width = width,
3852 .height = height,
3853 .depth = depth,
3854 .type = si_tex_dim(screen, tex, target, 0),
3855 .first_layer = first_layer,
3856 .last_layer = last_layer,
3857 .num_samples = res->nr_samples,
3858 .num_storage_samples = res->nr_storage_samples,
3859 };
3860
3861 ac_build_fmask_descriptor(screen->info.gfx_level, &ac_state, &fmask_state[0]);
3862 }
3863 }
3864
3865 /**
3866 * Build the sampler view descriptor for a texture (SI-GFX9).
3867 */
si_make_texture_descriptor(struct si_screen * screen,struct si_texture * tex,bool sampler,enum pipe_texture_target target,enum pipe_format pipe_format,const unsigned char state_swizzle[4],unsigned first_level,unsigned last_level,unsigned first_layer,unsigned last_layer,unsigned width,unsigned height,unsigned depth,bool get_bo_metadata,uint32_t * state,uint32_t * fmask_state)3868 void si_make_texture_descriptor(struct si_screen *screen, struct si_texture *tex,
3869 bool sampler, enum pipe_texture_target target,
3870 enum pipe_format pipe_format,
3871 const unsigned char state_swizzle[4], unsigned first_level,
3872 unsigned last_level, unsigned first_layer,
3873 unsigned last_layer, unsigned width, unsigned height,
3874 unsigned depth, bool get_bo_metadata,
3875 uint32_t *state, uint32_t *fmask_state)
3876 {
3877 if (!screen->info.has_image_opcodes && !get_bo_metadata) {
3878 cdna_emu_make_image_descriptor(screen, tex, sampler, target, pipe_format, state_swizzle,
3879 first_level, last_level, first_layer, last_layer, width,
3880 height, depth, state, fmask_state);
3881 return;
3882 }
3883
3884 if (screen->info.gfx_level >= GFX10) {
3885 gfx10_make_texture_descriptor(screen, tex, sampler, target, pipe_format, state_swizzle,
3886 first_level, last_level, first_layer, last_layer, width,
3887 height, depth, get_bo_metadata, state, fmask_state);
3888 return;
3889 }
3890
3891 struct pipe_resource *res = &tex->buffer.b.b;
3892 const struct util_format_description *desc;
3893 unsigned char swizzle[4];
3894 unsigned type, num_samples;
3895
3896 desc = util_format_description(pipe_format);
3897
3898 num_samples = desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS ? MAX2(1, res->nr_samples)
3899 : MAX2(1, res->nr_storage_samples);
3900
3901 if (desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS) {
3902 const unsigned char swizzle_xxxx[4] = {0, 0, 0, 0};
3903 const unsigned char swizzle_yyyy[4] = {1, 1, 1, 1};
3904 const unsigned char swizzle_wwww[4] = {3, 3, 3, 3};
3905
3906 switch (pipe_format) {
3907 case PIPE_FORMAT_S8_UINT_Z24_UNORM:
3908 case PIPE_FORMAT_X32_S8X24_UINT:
3909 case PIPE_FORMAT_X8Z24_UNORM:
3910 util_format_compose_swizzles(swizzle_yyyy, state_swizzle, swizzle);
3911 break;
3912 case PIPE_FORMAT_X24S8_UINT:
3913 /*
3914 * X24S8 is implemented as an 8_8_8_8 data format, to
3915 * fix texture gathers. This affects at least
3916 * GL45-CTS.texture_cube_map_array.sampling on GFX8.
3917 */
3918 if (screen->info.gfx_level <= GFX8)
3919 util_format_compose_swizzles(swizzle_wwww, state_swizzle, swizzle);
3920 else
3921 util_format_compose_swizzles(swizzle_yyyy, state_swizzle, swizzle);
3922 break;
3923 default:
3924 util_format_compose_swizzles(swizzle_xxxx, state_swizzle, swizzle);
3925 }
3926 } else {
3927 util_format_compose_swizzles(desc->swizzle, state_swizzle, swizzle);
3928 }
3929
3930 if (!sampler && (res->target == PIPE_TEXTURE_CUBE || res->target == PIPE_TEXTURE_CUBE_ARRAY ||
3931 (screen->info.gfx_level <= GFX8 && res->target == PIPE_TEXTURE_3D))) {
3932 /* For the purpose of shader images, treat cube maps and 3D
3933 * textures as 2D arrays. For 3D textures, the address
3934 * calculations for mipmaps are different, so we rely on the
3935 * caller to effectively disable mipmaps.
3936 */
3937 type = V_008F1C_SQ_RSRC_IMG_2D_ARRAY;
3938
3939 assert(res->target != PIPE_TEXTURE_3D || (first_level == 0 && last_level == 0));
3940 } else {
3941 type = si_tex_dim(screen, tex, target, num_samples);
3942 }
3943
3944 if (type == V_008F1C_SQ_RSRC_IMG_1D_ARRAY) {
3945 height = 1;
3946 depth = res->array_size;
3947 } else if (type == V_008F1C_SQ_RSRC_IMG_2D_ARRAY || type == V_008F1C_SQ_RSRC_IMG_2D_MSAA_ARRAY) {
3948 if (sampler || res->target != PIPE_TEXTURE_3D)
3949 depth = res->array_size;
3950 } else if (type == V_008F1C_SQ_RSRC_IMG_CUBE)
3951 depth = res->array_size / 6;
3952
3953 const struct ac_texture_state tex_state = {
3954 .surf = &tex->surface,
3955 .format = pipe_format,
3956 .img_format = res->format,
3957 .width = width,
3958 .height = height,
3959 .depth = depth,
3960 .type = type,
3961 .swizzle =
3962 {
3963 swizzle[0],
3964 swizzle[1],
3965 swizzle[2],
3966 swizzle[3],
3967 },
3968 .num_samples = res->nr_samples,
3969 .num_storage_samples = res->nr_storage_samples,
3970 .first_level = first_level,
3971 .last_level = last_level,
3972 .num_levels = res->last_level + 1,
3973 .first_layer = first_layer,
3974 .last_layer = last_layer,
3975 .dcc_enabled = vi_dcc_enabled(tex, first_level),
3976 .tc_compat_htile_enabled = true,
3977 };
3978
3979 ac_build_texture_descriptor(&screen->info, &tex_state, &state[0]);
3980
3981 /* Initialize the sampler view for FMASK. */
3982 if (tex->surface.fmask_offset) {
3983 const struct ac_fmask_state ac_state = {
3984 .surf = &tex->surface,
3985 .va = tex->buffer.gpu_address,
3986 .width = width,
3987 .height = height,
3988 .depth = depth,
3989 .type = si_tex_dim(screen, tex, target, 0),
3990 .first_layer = first_layer,
3991 .last_layer = last_layer,
3992 .num_samples = res->nr_samples,
3993 .num_storage_samples = res->nr_storage_samples,
3994 };
3995
3996 ac_build_fmask_descriptor(screen->info.gfx_level, &ac_state, &fmask_state[0]);
3997 }
3998 }
3999
4000 /**
4001 * Create a sampler view.
4002 *
4003 * @param ctx context
4004 * @param texture texture
4005 * @param state sampler view template
4006 */
si_create_sampler_view(struct pipe_context * ctx,struct pipe_resource * texture,const struct pipe_sampler_view * state)4007 static struct pipe_sampler_view *si_create_sampler_view(struct pipe_context *ctx,
4008 struct pipe_resource *texture,
4009 const struct pipe_sampler_view *state)
4010 {
4011 struct si_context *sctx = (struct si_context *)ctx;
4012 struct si_sampler_view *view = CALLOC_STRUCT_CL(si_sampler_view);
4013 struct si_texture *tex = (struct si_texture *)texture;
4014 unsigned char state_swizzle[4];
4015 unsigned last_layer = state->u.tex.last_layer;
4016 enum pipe_format pipe_format;
4017 const struct legacy_surf_level *surflevel;
4018
4019 if (!view)
4020 return NULL;
4021
4022 /* initialize base object */
4023 view->base = *state;
4024 view->base.texture = NULL;
4025 view->base.reference.count = 1;
4026 view->base.context = ctx;
4027
4028 assert(texture);
4029 pipe_resource_reference(&view->base.texture, texture);
4030
4031 if (state->format == PIPE_FORMAT_X24S8_UINT || state->format == PIPE_FORMAT_S8X24_UINT ||
4032 state->format == PIPE_FORMAT_X32_S8X24_UINT || state->format == PIPE_FORMAT_S8_UINT)
4033 view->is_stencil_sampler = true;
4034
4035 /* Buffer resource. */
4036 if (texture->target == PIPE_BUFFER) {
4037 uint32_t elements = si_clamp_texture_texel_count(sctx->screen->b.caps.max_texel_buffer_elements,
4038 state->format, state->u.buf.size);
4039
4040 si_make_buffer_descriptor(sctx->screen, si_resource(texture), state->format,
4041 state->u.buf.offset, elements, view->state);
4042 return &view->base;
4043 }
4044
4045 state_swizzle[0] = state->swizzle_r;
4046 state_swizzle[1] = state->swizzle_g;
4047 state_swizzle[2] = state->swizzle_b;
4048 state_swizzle[3] = state->swizzle_a;
4049
4050 /* This is not needed if gallium frontends set last_layer correctly. */
4051 if (state->target == PIPE_TEXTURE_1D || state->target == PIPE_TEXTURE_2D ||
4052 state->target == PIPE_TEXTURE_RECT || state->target == PIPE_TEXTURE_CUBE)
4053 last_layer = state->u.tex.first_layer;
4054
4055 /* Texturing with separate depth and stencil. */
4056 pipe_format = state->format;
4057
4058 /* Depth/stencil texturing sometimes needs separate texture. */
4059 if (tex->is_depth && !si_can_sample_zs(tex, view->is_stencil_sampler)) {
4060 if (!tex->flushed_depth_texture && !si_init_flushed_depth_texture(ctx, texture)) {
4061 pipe_resource_reference(&view->base.texture, NULL);
4062 FREE(view);
4063 return NULL;
4064 }
4065
4066 assert(tex->flushed_depth_texture);
4067
4068 /* Override format for the case where the flushed texture
4069 * contains only Z or only S.
4070 */
4071 if (tex->flushed_depth_texture->buffer.b.b.format != tex->buffer.b.b.format)
4072 pipe_format = tex->flushed_depth_texture->buffer.b.b.format;
4073
4074 tex = tex->flushed_depth_texture;
4075 }
4076
4077 surflevel = tex->surface.u.legacy.level;
4078
4079 if (tex->db_compatible) {
4080 if (!view->is_stencil_sampler)
4081 pipe_format = tex->db_render_format;
4082
4083 switch (pipe_format) {
4084 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
4085 pipe_format = PIPE_FORMAT_Z32_FLOAT;
4086 break;
4087 case PIPE_FORMAT_X8Z24_UNORM:
4088 case PIPE_FORMAT_S8_UINT_Z24_UNORM:
4089 /* Z24 is always stored like this for DB
4090 * compatibility.
4091 */
4092 pipe_format = PIPE_FORMAT_Z24X8_UNORM;
4093 break;
4094 case PIPE_FORMAT_X24S8_UINT:
4095 case PIPE_FORMAT_S8X24_UINT:
4096 case PIPE_FORMAT_X32_S8X24_UINT:
4097 pipe_format = PIPE_FORMAT_S8_UINT;
4098 surflevel = tex->surface.u.legacy.zs.stencil_level;
4099 break;
4100 default:;
4101 }
4102 }
4103
4104 view->dcc_incompatible =
4105 vi_dcc_formats_are_incompatible(texture, state->u.tex.first_level, state->format);
4106
4107 si_make_texture_descriptor(sctx->screen, tex, true, state->target, pipe_format, state_swizzle,
4108 state->u.tex.first_level, state->u.tex.last_level,
4109 state->u.tex.first_layer, last_layer, texture->width0,
4110 texture->height0, texture->depth0, false, view->state,
4111 view->fmask_state);
4112
4113 view->base_level_info = &surflevel[0];
4114 view->block_width = util_format_get_blockwidth(pipe_format);
4115 return &view->base;
4116 }
4117
si_sampler_view_destroy(struct pipe_context * ctx,struct pipe_sampler_view * state)4118 static void si_sampler_view_destroy(struct pipe_context *ctx, struct pipe_sampler_view *state)
4119 {
4120 struct si_sampler_view *view = (struct si_sampler_view *)state;
4121
4122 pipe_resource_reference(&state->texture, NULL);
4123 FREE_CL(view);
4124 }
4125
wrap_mode_uses_border_color(unsigned wrap,bool linear_filter)4126 static bool wrap_mode_uses_border_color(unsigned wrap, bool linear_filter)
4127 {
4128 return wrap == PIPE_TEX_WRAP_CLAMP_TO_BORDER || wrap == PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER ||
4129 (linear_filter && (wrap == PIPE_TEX_WRAP_CLAMP || wrap == PIPE_TEX_WRAP_MIRROR_CLAMP));
4130 }
4131
si_translate_border_color(struct si_context * sctx,const struct pipe_sampler_state * state,const union pipe_color_union * color,bool is_integer,uint32_t * border_color_ptr)4132 static uint32_t si_translate_border_color(struct si_context *sctx,
4133 const struct pipe_sampler_state *state,
4134 const union pipe_color_union *color, bool is_integer,
4135 uint32_t *border_color_ptr)
4136 {
4137 bool linear_filter = state->min_img_filter != PIPE_TEX_FILTER_NEAREST ||
4138 state->mag_img_filter != PIPE_TEX_FILTER_NEAREST;
4139
4140 if (!wrap_mode_uses_border_color(state->wrap_s, linear_filter) &&
4141 !wrap_mode_uses_border_color(state->wrap_t, linear_filter) &&
4142 !wrap_mode_uses_border_color(state->wrap_r, linear_filter))
4143 return V_008F3C_SQ_TEX_BORDER_COLOR_TRANS_BLACK;
4144
4145 #define simple_border_types(elt) \
4146 do { \
4147 if (color->elt[0] == 0 && color->elt[1] == 0 && color->elt[2] == 0 && color->elt[3] == 0) \
4148 return V_008F3C_SQ_TEX_BORDER_COLOR_TRANS_BLACK; \
4149 if (color->elt[0] == 0 && color->elt[1] == 0 && color->elt[2] == 0 && color->elt[3] == 1) \
4150 return V_008F3C_SQ_TEX_BORDER_COLOR_OPAQUE_BLACK; \
4151 if (color->elt[0] == 1 && color->elt[1] == 1 && color->elt[2] == 1 && color->elt[3] == 1) \
4152 return V_008F3C_SQ_TEX_BORDER_COLOR_OPAQUE_WHITE; \
4153 } while (false)
4154
4155 if (is_integer)
4156 simple_border_types(ui);
4157 else
4158 simple_border_types(f);
4159
4160 #undef simple_border_types
4161
4162 int i;
4163
4164 /* Check if the border has been uploaded already. */
4165 for (i = 0; i < sctx->border_color_count; i++)
4166 if (memcmp(&sctx->border_color_table[i], color, sizeof(*color)) == 0)
4167 break;
4168
4169 if (i >= SI_MAX_BORDER_COLORS) {
4170 /* Getting 4096 unique border colors is very unlikely. */
4171 static bool printed;
4172 if (!printed) {
4173 fprintf(stderr, "radeonsi: The border color table is full. "
4174 "Any new border colors will be just black. "
4175 "This is a hardware limitation.\n");
4176 printed = true;
4177 }
4178 return V_008F3C_SQ_TEX_BORDER_COLOR_TRANS_BLACK;
4179 }
4180
4181 if (i == sctx->border_color_count) {
4182 /* Upload a new border color. */
4183 memcpy(&sctx->border_color_table[i], color, sizeof(*color));
4184 util_memcpy_cpu_to_le32(&sctx->border_color_map[i], color, sizeof(*color));
4185 sctx->border_color_count++;
4186 }
4187
4188 *border_color_ptr = i;
4189
4190 return V_008F3C_SQ_TEX_BORDER_COLOR_REGISTER;
4191 }
4192
si_tex_filter(unsigned filter,unsigned max_aniso)4193 static inline unsigned si_tex_filter(unsigned filter, unsigned max_aniso)
4194 {
4195 if (filter == PIPE_TEX_FILTER_LINEAR)
4196 return max_aniso > 1 ? V_008F38_SQ_TEX_XY_FILTER_ANISO_BILINEAR
4197 : V_008F38_SQ_TEX_XY_FILTER_BILINEAR;
4198 else
4199 return max_aniso > 1 ? V_008F38_SQ_TEX_XY_FILTER_ANISO_POINT
4200 : V_008F38_SQ_TEX_XY_FILTER_POINT;
4201 }
4202
si_tex_aniso_filter(unsigned filter)4203 static inline unsigned si_tex_aniso_filter(unsigned filter)
4204 {
4205 if (filter < 2)
4206 return 0;
4207 if (filter < 4)
4208 return 1;
4209 if (filter < 8)
4210 return 2;
4211 if (filter < 16)
4212 return 3;
4213 return 4;
4214 }
4215
si_tex_filter_mode(unsigned mode)4216 static unsigned si_tex_filter_mode(unsigned mode)
4217 {
4218 switch (mode) {
4219 case PIPE_TEX_REDUCTION_WEIGHTED_AVERAGE:
4220 return V_008F30_SQ_IMG_FILTER_MODE_BLEND;
4221 case PIPE_TEX_REDUCTION_MIN:
4222 return V_008F30_SQ_IMG_FILTER_MODE_MIN;
4223 case PIPE_TEX_REDUCTION_MAX:
4224 return V_008F30_SQ_IMG_FILTER_MODE_MAX;
4225 default:
4226 break;
4227 }
4228 return 0;
4229 }
4230
si_create_sampler_state(struct pipe_context * ctx,const struct pipe_sampler_state * state)4231 static void *si_create_sampler_state(struct pipe_context *ctx,
4232 const struct pipe_sampler_state *state)
4233 {
4234 struct si_context *sctx = (struct si_context *)ctx;
4235 struct si_screen *sscreen = sctx->screen;
4236 struct si_sampler_state *rstate = CALLOC_STRUCT(si_sampler_state);
4237 unsigned max_aniso = sscreen->force_aniso >= 0 ? sscreen->force_aniso : state->max_anisotropy;
4238 unsigned max_aniso_ratio = si_tex_aniso_filter(max_aniso);
4239 unsigned filter_mode = si_tex_filter_mode(state->reduction_mode);
4240 bool trunc_coord = (state->min_img_filter == PIPE_TEX_FILTER_NEAREST &&
4241 state->mag_img_filter == PIPE_TEX_FILTER_NEAREST &&
4242 state->compare_mode == PIPE_TEX_COMPARE_NONE) ||
4243 sscreen->info.conformant_trunc_coord;
4244 union pipe_color_union clamped_border_color;
4245
4246 if (!rstate) {
4247 return NULL;
4248 }
4249
4250 /* Validate inputs. */
4251 if (!is_wrap_mode_legal(sscreen, state->wrap_s) ||
4252 !is_wrap_mode_legal(sscreen, state->wrap_t) ||
4253 !is_wrap_mode_legal(sscreen, state->wrap_r) ||
4254 (!sscreen->info.has_3d_cube_border_color_mipmap &&
4255 (state->min_mip_filter != PIPE_TEX_MIPFILTER_NONE ||
4256 state->max_anisotropy > 0))) {
4257 assert(0);
4258 return NULL;
4259 }
4260
4261 #ifndef NDEBUG
4262 rstate->magic = SI_SAMPLER_STATE_MAGIC;
4263 #endif
4264
4265 unsigned border_color_ptr = 0;
4266 unsigned border_color_type =
4267 si_translate_border_color(sctx, state, &state->border_color,
4268 state->border_color_is_integer,
4269 &border_color_ptr);
4270
4271 struct ac_sampler_state ac_state = {
4272 .address_mode_u = si_tex_wrap(state->wrap_s),
4273 .address_mode_v = si_tex_wrap(state->wrap_t),
4274 .address_mode_w = si_tex_wrap(state->wrap_r),
4275 .max_aniso_ratio = max_aniso_ratio,
4276 .depth_compare_func = si_tex_compare(state->compare_mode, state->compare_func),
4277 .unnormalized_coords = state->unnormalized_coords,
4278 .cube_wrap = state->seamless_cube_map,
4279 .trunc_coord = trunc_coord,
4280 .filter_mode = filter_mode,
4281 .mag_filter = si_tex_filter(state->mag_img_filter, max_aniso),
4282 .min_filter = si_tex_filter(state->min_img_filter, max_aniso),
4283 .mip_filter = si_tex_mipfilter(state->min_mip_filter),
4284 .min_lod = state->min_lod,
4285 .max_lod = state->max_lod,
4286 .lod_bias = state->lod_bias,
4287 .border_color_type = border_color_type,
4288 .border_color_ptr = border_color_ptr,
4289 };
4290
4291 ac_build_sampler_descriptor(sscreen->info.gfx_level, &ac_state, rstate->val);
4292
4293 /* Create sampler resource for upgraded depth textures. */
4294 memcpy(rstate->upgraded_depth_val, rstate->val, sizeof(rstate->val));
4295
4296 for (unsigned i = 0; i < 4; ++i) {
4297 /* Use channel 0 on purpose, so that we can use OPAQUE_WHITE
4298 * when the border color is 1.0. */
4299 clamped_border_color.f[i] = CLAMP(state->border_color.f[0], 0, 1);
4300 }
4301
4302 if (memcmp(&state->border_color, &clamped_border_color, sizeof(clamped_border_color)) == 0) {
4303 if (sscreen->info.gfx_level <= GFX9)
4304 rstate->upgraded_depth_val[3] |= S_008F3C_UPGRADED_DEPTH(1);
4305 } else {
4306 border_color_ptr = 0;
4307 border_color_type = si_translate_border_color(sctx, state, &clamped_border_color, false, &border_color_ptr);
4308
4309 rstate->upgraded_depth_val[3] = S_008F3C_BORDER_COLOR_TYPE(border_color_type);
4310
4311 if (sscreen->info.gfx_level >= GFX11) {
4312 rstate->upgraded_depth_val[3] |= S_008F3C_BORDER_COLOR_PTR_GFX11(border_color_ptr);
4313 } else {
4314 rstate->upgraded_depth_val[3] |= S_008F3C_BORDER_COLOR_PTR_GFX6(border_color_ptr);
4315 }
4316 }
4317
4318 return rstate;
4319 }
4320
si_set_sample_mask(struct pipe_context * ctx,unsigned sample_mask)4321 static void si_set_sample_mask(struct pipe_context *ctx, unsigned sample_mask)
4322 {
4323 struct si_context *sctx = (struct si_context *)ctx;
4324
4325 if (sctx->sample_mask == (uint16_t)sample_mask)
4326 return;
4327
4328 sctx->sample_mask = sample_mask;
4329 si_mark_atom_dirty(sctx, &sctx->atoms.s.sample_mask);
4330 }
4331
si_emit_sample_mask(struct si_context * sctx,unsigned index)4332 static void si_emit_sample_mask(struct si_context *sctx, unsigned index)
4333 {
4334 struct radeon_cmdbuf *cs = &sctx->gfx_cs;
4335 unsigned mask = sctx->sample_mask;
4336
4337 /* Needed for line and polygon smoothing as well as for the Polaris
4338 * small primitive filter. We expect the gallium frontend to take care of
4339 * this for us.
4340 */
4341 assert(mask == 0xffff || sctx->framebuffer.nr_samples > 1 ||
4342 (mask & 1 && sctx->blitter_running));
4343
4344 radeon_begin(cs);
4345 radeon_set_context_reg_seq(R_028C38_PA_SC_AA_MASK_X0Y0_X1Y0, 2);
4346 radeon_emit(mask | (mask << 16));
4347 radeon_emit(mask | (mask << 16));
4348 radeon_end();
4349 }
4350
si_delete_sampler_state(struct pipe_context * ctx,void * state)4351 static void si_delete_sampler_state(struct pipe_context *ctx, void *state)
4352 {
4353 #ifndef NDEBUG
4354 struct si_sampler_state *s = state;
4355
4356 assert(s->magic == SI_SAMPLER_STATE_MAGIC);
4357 s->magic = 0;
4358 #endif
4359 free(state);
4360 }
4361
4362 /*
4363 * Vertex elements & buffers
4364 */
4365
4366 struct si_fast_udiv_info32 {
4367 unsigned multiplier; /* the "magic number" multiplier */
4368 unsigned pre_shift; /* shift for the dividend before multiplying */
4369 unsigned post_shift; /* shift for the dividend after multiplying */
4370 int increment; /* 0 or 1; if set then increment the numerator, using one of
4371 the two strategies */
4372 };
4373
si_compute_fast_udiv_info32(uint32_t D,unsigned num_bits)4374 static struct si_fast_udiv_info32 si_compute_fast_udiv_info32(uint32_t D, unsigned num_bits)
4375 {
4376 struct util_fast_udiv_info info = util_compute_fast_udiv_info(D, num_bits, 32);
4377
4378 struct si_fast_udiv_info32 result = {
4379 info.multiplier,
4380 info.pre_shift,
4381 info.post_shift,
4382 info.increment,
4383 };
4384 return result;
4385 }
4386
si_create_vertex_elements(struct pipe_context * ctx,unsigned count,const struct pipe_vertex_element * elements)4387 static void *si_create_vertex_elements(struct pipe_context *ctx, unsigned count,
4388 const struct pipe_vertex_element *elements)
4389 {
4390 struct si_screen *sscreen = (struct si_screen *)ctx->screen;
4391
4392 if (sscreen->debug_flags & DBG(VERTEX_ELEMENTS)) {
4393 for (int i = 0; i < count; ++i) {
4394 const struct pipe_vertex_element *e = elements + i;
4395 fprintf(stderr, "elements[%d]: offset %2d, buffer_index %d, dual_slot %d, format %3d, divisor %u\n",
4396 i, e->src_offset, e->vertex_buffer_index, e->dual_slot, e->src_format, e->instance_divisor);
4397 }
4398 }
4399
4400 struct si_vertex_elements *v = CALLOC_STRUCT(si_vertex_elements);
4401 struct si_fast_udiv_info32 divisor_factors[SI_MAX_ATTRIBS] = {};
4402 STATIC_ASSERT(sizeof(struct si_fast_udiv_info32) == 16);
4403 STATIC_ASSERT(sizeof(divisor_factors[0].multiplier) == 4);
4404 STATIC_ASSERT(sizeof(divisor_factors[0].pre_shift) == 4);
4405 STATIC_ASSERT(sizeof(divisor_factors[0].post_shift) == 4);
4406 STATIC_ASSERT(sizeof(divisor_factors[0].increment) == 4);
4407 int i;
4408
4409 assert(count <= SI_MAX_ATTRIBS);
4410 if (!v)
4411 return NULL;
4412
4413 v->count = count;
4414
4415 unsigned num_vbos_in_user_sgprs = si_num_vbos_in_user_sgprs_inline(sscreen->info.gfx_level);
4416 unsigned alloc_count =
4417 count > num_vbos_in_user_sgprs ? count - num_vbos_in_user_sgprs : 0;
4418 v->vb_desc_list_alloc_size = align(alloc_count * 16, SI_CPDMA_ALIGNMENT);
4419
4420 for (i = 0; i < count; ++i) {
4421 const struct util_format_description *desc;
4422 const struct util_format_channel_description *channel;
4423 int first_non_void;
4424 unsigned vbo_index = elements[i].vertex_buffer_index;
4425
4426 if (vbo_index >= SI_NUM_VERTEX_BUFFERS) {
4427 FREE(v);
4428 return NULL;
4429 }
4430
4431 unsigned instance_divisor = elements[i].instance_divisor;
4432 if (instance_divisor) {
4433 if (instance_divisor == 1) {
4434 v->instance_divisor_is_one |= 1u << i;
4435 } else {
4436 v->instance_divisor_is_fetched |= 1u << i;
4437 divisor_factors[i] = si_compute_fast_udiv_info32(instance_divisor, 32);
4438 }
4439 }
4440
4441 desc = util_format_description(elements[i].src_format);
4442 first_non_void = util_format_get_first_non_void_channel(elements[i].src_format);
4443 channel = first_non_void >= 0 ? &desc->channel[first_non_void] : NULL;
4444
4445 v->elem[i].format_size = desc->block.bits / 8;
4446 v->elem[i].src_offset = elements[i].src_offset;
4447 v->elem[i].stride = elements[i].src_stride;
4448 v->vertex_buffer_index[i] = vbo_index;
4449
4450 bool always_fix = false;
4451 union si_vs_fix_fetch fix_fetch;
4452 unsigned log_hw_load_size; /* the load element size as seen by the hardware */
4453
4454 fix_fetch.bits = 0;
4455 log_hw_load_size = MIN2(2, util_logbase2(desc->block.bits) - 3);
4456
4457 if (channel) {
4458 switch (channel->type) {
4459 case UTIL_FORMAT_TYPE_FLOAT:
4460 fix_fetch.u.format = AC_FETCH_FORMAT_FLOAT;
4461 break;
4462 case UTIL_FORMAT_TYPE_FIXED:
4463 fix_fetch.u.format = AC_FETCH_FORMAT_FIXED;
4464 break;
4465 case UTIL_FORMAT_TYPE_SIGNED: {
4466 if (channel->pure_integer)
4467 fix_fetch.u.format = AC_FETCH_FORMAT_SINT;
4468 else if (channel->normalized)
4469 fix_fetch.u.format = AC_FETCH_FORMAT_SNORM;
4470 else
4471 fix_fetch.u.format = AC_FETCH_FORMAT_SSCALED;
4472 break;
4473 }
4474 case UTIL_FORMAT_TYPE_UNSIGNED: {
4475 if (channel->pure_integer)
4476 fix_fetch.u.format = AC_FETCH_FORMAT_UINT;
4477 else if (channel->normalized)
4478 fix_fetch.u.format = AC_FETCH_FORMAT_UNORM;
4479 else
4480 fix_fetch.u.format = AC_FETCH_FORMAT_USCALED;
4481 break;
4482 }
4483 default:
4484 unreachable("bad format type");
4485 }
4486 } else {
4487 switch (elements[i].src_format) {
4488 case PIPE_FORMAT_R11G11B10_FLOAT:
4489 fix_fetch.u.format = AC_FETCH_FORMAT_FLOAT;
4490 break;
4491 default:
4492 unreachable("bad other format");
4493 }
4494 }
4495
4496 if (desc->channel[0].size == 10) {
4497 fix_fetch.u.log_size = 3; /* special encoding for 2_10_10_10 */
4498 log_hw_load_size = 2;
4499
4500 /* The hardware always treats the 2-bit alpha channel as
4501 * unsigned, so a shader workaround is needed. The affected
4502 * chips are GFX8 and older except Stoney (GFX8.1).
4503 */
4504 always_fix = sscreen->info.gfx_level <= GFX8 && sscreen->info.family != CHIP_STONEY &&
4505 channel->type == UTIL_FORMAT_TYPE_SIGNED;
4506 } else if (elements[i].src_format == PIPE_FORMAT_R11G11B10_FLOAT) {
4507 fix_fetch.u.log_size = 3; /* special encoding */
4508 fix_fetch.u.format = AC_FETCH_FORMAT_FIXED;
4509 log_hw_load_size = 2;
4510 } else {
4511 fix_fetch.u.log_size = util_logbase2(channel->size) - 3;
4512 fix_fetch.u.num_channels_m1 = desc->nr_channels - 1;
4513
4514 /* Always fix up:
4515 * - doubles (multiple loads + truncate to float)
4516 * - 32-bit requiring a conversion
4517 */
4518 always_fix = (fix_fetch.u.log_size == 3) ||
4519 (fix_fetch.u.log_size == 2 && fix_fetch.u.format != AC_FETCH_FORMAT_FLOAT &&
4520 fix_fetch.u.format != AC_FETCH_FORMAT_UINT &&
4521 fix_fetch.u.format != AC_FETCH_FORMAT_SINT);
4522
4523 /* Also fixup 8_8_8 and 16_16_16. */
4524 if (desc->nr_channels == 3 && fix_fetch.u.log_size <= 1) {
4525 always_fix = true;
4526 log_hw_load_size = fix_fetch.u.log_size;
4527 }
4528 }
4529
4530 if (desc->swizzle[0] != PIPE_SWIZZLE_X) {
4531 assert(desc->swizzle[0] == PIPE_SWIZZLE_Z &&
4532 (desc->swizzle[2] == PIPE_SWIZZLE_X || desc->swizzle[2] == PIPE_SWIZZLE_0));
4533 fix_fetch.u.reverse = 1;
4534 }
4535
4536 /* Force the workaround for unaligned access here already if the
4537 * offset relative to the vertex buffer base is unaligned.
4538 *
4539 * There is a theoretical case in which this is too conservative:
4540 * if the vertex buffer's offset is also unaligned in just the
4541 * right way, we end up with an aligned address after all.
4542 * However, this case should be extremely rare in practice (it
4543 * won't happen in well-behaved applications), and taking it
4544 * into account would complicate the fast path (where everything
4545 * is nicely aligned).
4546 */
4547 bool check_alignment =
4548 log_hw_load_size >= 1 &&
4549 (sscreen->info.gfx_level == GFX6 || sscreen->info.gfx_level >= GFX10);
4550 bool opencode = sscreen->options.vs_fetch_always_opencode;
4551
4552 if (check_alignment && ((elements[i].src_offset & ((1 << log_hw_load_size) - 1)) != 0 ||
4553 elements[i].src_stride & 3))
4554 opencode = true;
4555
4556 if (always_fix || check_alignment || opencode)
4557 v->fix_fetch[i] = fix_fetch.bits;
4558
4559 if (opencode)
4560 v->fix_fetch_opencode |= 1 << i;
4561 if (opencode || always_fix)
4562 v->fix_fetch_always |= 1 << i;
4563
4564 if (check_alignment && !opencode) {
4565 assert(log_hw_load_size == 1 || log_hw_load_size == 2);
4566
4567 v->fix_fetch_unaligned |= 1 << i;
4568 v->hw_load_is_dword |= (log_hw_load_size - 1) << i;
4569 v->vb_alignment_check_mask |= 1 << vbo_index;
4570 }
4571
4572 const struct ac_buffer_state buffer_state = {
4573 .format = elements[i].src_format,
4574 .swizzle =
4575 {
4576 desc->swizzle[0],
4577 desc->swizzle[1],
4578 desc->swizzle[2],
4579 desc->swizzle[3],
4580 },
4581 /* OOB_SELECT chooses the out-of-bounds check:
4582 * - 1: index >= NUM_RECORDS (Structured)
4583 * - 3: offset >= NUM_RECORDS (Raw)
4584 */
4585 .gfx10_oob_select = v->elem[i].stride ? V_008F0C_OOB_SELECT_STRUCTURED
4586 : V_008F0C_OOB_SELECT_RAW,
4587 };
4588
4589 ac_set_buf_desc_word3(sscreen->info.gfx_level, &buffer_state, &v->elem[i].rsrc_word3);
4590 }
4591
4592 if (v->instance_divisor_is_fetched) {
4593 unsigned num_divisors = util_last_bit(v->instance_divisor_is_fetched);
4594
4595 v->instance_divisor_factor_buffer = (struct si_resource *)pipe_buffer_create(
4596 &sscreen->b, 0, PIPE_USAGE_DEFAULT, num_divisors * sizeof(divisor_factors[0]));
4597 if (!v->instance_divisor_factor_buffer) {
4598 FREE(v);
4599 return NULL;
4600 }
4601 void *map =
4602 sscreen->ws->buffer_map(sscreen->ws, v->instance_divisor_factor_buffer->buf, NULL, PIPE_MAP_WRITE);
4603 memcpy(map, divisor_factors, num_divisors * sizeof(divisor_factors[0]));
4604 }
4605 return v;
4606 }
4607
si_bind_vertex_elements(struct pipe_context * ctx,void * state)4608 static void si_bind_vertex_elements(struct pipe_context *ctx, void *state)
4609 {
4610 struct si_context *sctx = (struct si_context *)ctx;
4611 struct si_vertex_elements *old = sctx->vertex_elements;
4612 struct si_vertex_elements *v = (struct si_vertex_elements *)state;
4613
4614 if (!v)
4615 v = sctx->no_velems_state;
4616
4617 sctx->vertex_elements = v;
4618 sctx->num_vertex_elements = v->count;
4619 sctx->vertex_buffers_dirty = sctx->num_vertex_elements > 0;
4620
4621 if (old->instance_divisor_is_one != v->instance_divisor_is_one ||
4622 old->instance_divisor_is_fetched != v->instance_divisor_is_fetched ||
4623 (old->vb_alignment_check_mask ^ v->vb_alignment_check_mask) &
4624 sctx->vertex_buffer_unaligned ||
4625 ((v->vb_alignment_check_mask & sctx->vertex_buffer_unaligned) &&
4626 memcmp(old->vertex_buffer_index, v->vertex_buffer_index,
4627 sizeof(v->vertex_buffer_index[0]) * MAX2(old->count, v->count))) ||
4628 /* fix_fetch_{always,opencode,unaligned} and hw_load_is_dword are
4629 * functions of fix_fetch and the src_offset alignment.
4630 * If they change and fix_fetch doesn't, it must be due to different
4631 * src_offset alignment, which is reflected in fix_fetch_opencode. */
4632 old->fix_fetch_opencode != v->fix_fetch_opencode ||
4633 memcmp(old->fix_fetch, v->fix_fetch, sizeof(v->fix_fetch[0]) *
4634 MAX2(old->count, v->count))) {
4635 si_vs_key_update_inputs(sctx);
4636 sctx->do_update_shaders = true;
4637 }
4638
4639 if (v->instance_divisor_is_fetched) {
4640 struct pipe_constant_buffer cb;
4641
4642 cb.buffer = &v->instance_divisor_factor_buffer->b.b;
4643 cb.user_buffer = NULL;
4644 cb.buffer_offset = 0;
4645 cb.buffer_size = 0xffffffff;
4646 si_set_internal_const_buffer(sctx, SI_VS_CONST_INSTANCE_DIVISORS, &cb);
4647 }
4648 }
4649
si_delete_vertex_element(struct pipe_context * ctx,void * state)4650 static void si_delete_vertex_element(struct pipe_context *ctx, void *state)
4651 {
4652 struct si_context *sctx = (struct si_context *)ctx;
4653 struct si_vertex_elements *v = (struct si_vertex_elements *)state;
4654
4655 if (sctx->vertex_elements == state)
4656 si_bind_vertex_elements(ctx, sctx->no_velems_state);
4657
4658 si_resource_reference(&v->instance_divisor_factor_buffer, NULL);
4659 FREE(state);
4660 }
4661
si_set_vertex_buffers(struct pipe_context * ctx,unsigned count,const struct pipe_vertex_buffer * buffers)4662 static void si_set_vertex_buffers(struct pipe_context *ctx, unsigned count,
4663 const struct pipe_vertex_buffer *buffers)
4664 {
4665 struct si_context *sctx = (struct si_context *)ctx;
4666 uint32_t unaligned = 0;
4667 unsigned i;
4668
4669 assert(count <= ARRAY_SIZE(sctx->vertex_buffer));
4670 assert(!count || buffers);
4671
4672 for (i = 0; i < count; i++) {
4673 const struct pipe_vertex_buffer *src = buffers + i;
4674 struct pipe_vertex_buffer *dst = sctx->vertex_buffer + i;
4675 struct pipe_resource *buf = src->buffer.resource;
4676
4677 dst->buffer_offset = src->buffer_offset;
4678
4679 /* Only unreference bound vertex buffers. */
4680 pipe_resource_reference(&dst->buffer.resource, NULL);
4681 dst->buffer.resource = src->buffer.resource;
4682
4683 if (src->buffer_offset & 3)
4684 unaligned |= BITFIELD_BIT(i);
4685
4686 if (buf) {
4687 si_resource(buf)->bind_history |= SI_BIND_VERTEX_BUFFER;
4688 radeon_add_to_buffer_list(sctx, &sctx->gfx_cs, si_resource(buf),
4689 RADEON_USAGE_READ | RADEON_PRIO_VERTEX_BUFFER);
4690 }
4691 }
4692
4693 unsigned last_count = sctx->num_vertex_buffers;
4694 for (; i < last_count; i++)
4695 pipe_resource_reference(&sctx->vertex_buffer[i].buffer.resource, NULL);
4696
4697 sctx->num_vertex_buffers = count;
4698 sctx->vertex_buffers_dirty = sctx->num_vertex_elements > 0;
4699 sctx->vertex_buffer_unaligned = unaligned;
4700
4701 /* Check whether alignment may have changed in a way that requires
4702 * shader changes. This check is conservative: a vertex buffer can only
4703 * trigger a shader change if the misalignment amount changes (e.g.
4704 * from byte-aligned to short-aligned), but we only keep track of
4705 * whether buffers are at least dword-aligned, since that should always
4706 * be the case in well-behaved applications anyway.
4707 */
4708 if (sctx->vertex_elements->vb_alignment_check_mask & unaligned) {
4709 si_vs_key_update_inputs(sctx);
4710 sctx->do_update_shaders = true;
4711 }
4712 }
4713
4714 static struct pipe_vertex_state *
si_create_vertex_state(struct pipe_screen * screen,struct pipe_vertex_buffer * buffer,const struct pipe_vertex_element * elements,unsigned num_elements,struct pipe_resource * indexbuf,uint32_t full_velem_mask)4715 si_create_vertex_state(struct pipe_screen *screen,
4716 struct pipe_vertex_buffer *buffer,
4717 const struct pipe_vertex_element *elements,
4718 unsigned num_elements,
4719 struct pipe_resource *indexbuf,
4720 uint32_t full_velem_mask)
4721 {
4722 struct si_screen *sscreen = (struct si_screen *)screen;
4723 struct si_vertex_state *state = CALLOC_STRUCT(si_vertex_state);
4724
4725 util_init_pipe_vertex_state(screen, buffer, elements, num_elements, indexbuf, full_velem_mask,
4726 &state->b);
4727
4728 /* Initialize the vertex element state in state->element.
4729 * Do it by creating a vertex element state object and copying it there.
4730 */
4731 struct si_context ctx = {};
4732 ctx.b.screen = screen;
4733 struct si_vertex_elements *velems = si_create_vertex_elements(&ctx.b, num_elements, elements);
4734 state->velems = *velems;
4735 si_delete_vertex_element(&ctx.b, velems);
4736
4737 assert(!state->velems.instance_divisor_is_one);
4738 assert(!state->velems.instance_divisor_is_fetched);
4739 assert(!state->velems.fix_fetch_always);
4740 assert(buffer->buffer_offset % 4 == 0);
4741 assert(!buffer->is_user_buffer);
4742 for (unsigned i = 0; i < num_elements; i++) {
4743 assert(elements[i].src_offset % 4 == 0);
4744 assert(!elements[i].dual_slot);
4745 assert(elements[i].src_stride % 4 == 0);
4746 }
4747
4748 for (unsigned i = 0; i < num_elements; i++) {
4749 si_set_vertex_buffer_descriptor(sscreen, &state->velems, &state->b.input.vbuffer, i,
4750 &state->descriptors[i * 4]);
4751 }
4752
4753 return &state->b;
4754 }
4755
si_vertex_state_destroy(struct pipe_screen * screen,struct pipe_vertex_state * state)4756 static void si_vertex_state_destroy(struct pipe_screen *screen,
4757 struct pipe_vertex_state *state)
4758 {
4759 pipe_vertex_buffer_unreference(&state->input.vbuffer);
4760 pipe_resource_reference(&state->input.indexbuf, NULL);
4761 FREE(state);
4762 }
4763
4764 static struct pipe_vertex_state *
si_pipe_create_vertex_state(struct pipe_screen * screen,struct pipe_vertex_buffer * buffer,const struct pipe_vertex_element * elements,unsigned num_elements,struct pipe_resource * indexbuf,uint32_t full_velem_mask)4765 si_pipe_create_vertex_state(struct pipe_screen *screen,
4766 struct pipe_vertex_buffer *buffer,
4767 const struct pipe_vertex_element *elements,
4768 unsigned num_elements,
4769 struct pipe_resource *indexbuf,
4770 uint32_t full_velem_mask)
4771 {
4772 struct si_screen *sscreen = (struct si_screen *)screen;
4773
4774 return util_vertex_state_cache_get(screen, buffer, elements, num_elements, indexbuf,
4775 full_velem_mask, &sscreen->vertex_state_cache);
4776 }
4777
si_pipe_vertex_state_destroy(struct pipe_screen * screen,struct pipe_vertex_state * state)4778 static void si_pipe_vertex_state_destroy(struct pipe_screen *screen,
4779 struct pipe_vertex_state *state)
4780 {
4781 struct si_screen *sscreen = (struct si_screen *)screen;
4782
4783 util_vertex_state_destroy(screen, &sscreen->vertex_state_cache, state);
4784 }
4785
4786 /*
4787 * Misc
4788 */
4789
si_set_tess_state(struct pipe_context * ctx,const float default_outer_level[4],const float default_inner_level[2])4790 static void si_set_tess_state(struct pipe_context *ctx, const float default_outer_level[4],
4791 const float default_inner_level[2])
4792 {
4793 struct si_context *sctx = (struct si_context *)ctx;
4794 struct pipe_constant_buffer cb;
4795 float array[8];
4796
4797 memcpy(array, default_outer_level, sizeof(float) * 4);
4798 memcpy(array + 4, default_inner_level, sizeof(float) * 2);
4799
4800 cb.buffer = NULL;
4801 cb.user_buffer = array;
4802 cb.buffer_offset = 0;
4803 cb.buffer_size = sizeof(array);
4804
4805 si_set_internal_const_buffer(sctx, SI_HS_CONST_DEFAULT_TESS_LEVELS, &cb);
4806 }
4807
si_create_blend_custom(struct si_context * sctx,unsigned mode)4808 static void *si_create_blend_custom(struct si_context *sctx, unsigned mode)
4809 {
4810 struct pipe_blend_state blend;
4811
4812 memset(&blend, 0, sizeof(blend));
4813 blend.independent_blend_enable = true;
4814 blend.rt[0].colormask = 0xf;
4815 return si_create_blend_state_mode(&sctx->b, &blend, mode);
4816 }
4817
si_pm4_emit_sqtt_pipeline(struct si_context * sctx,unsigned index)4818 static void si_pm4_emit_sqtt_pipeline(struct si_context *sctx, unsigned index)
4819 {
4820 struct si_pm4_state *state = sctx->queued.array[index];
4821
4822 si_pm4_emit_state(sctx, index);
4823
4824 radeon_add_to_buffer_list(sctx, &sctx->gfx_cs, ((struct si_sqtt_fake_pipeline*)state)->bo,
4825 RADEON_USAGE_READ | RADEON_PRIO_SHADER_BINARY);
4826 }
4827
si_init_state_compute_functions(struct si_context * sctx)4828 void si_init_state_compute_functions(struct si_context *sctx)
4829 {
4830 sctx->b.create_sampler_state = si_create_sampler_state;
4831 sctx->b.delete_sampler_state = si_delete_sampler_state;
4832 sctx->b.create_sampler_view = si_create_sampler_view;
4833 sctx->b.sampler_view_destroy = si_sampler_view_destroy;
4834 }
4835
si_init_state_functions(struct si_context * sctx)4836 void si_init_state_functions(struct si_context *sctx)
4837 {
4838 sctx->atoms.s.pm4_states[SI_STATE_IDX(blend)].emit = si_pm4_emit_state;
4839 sctx->atoms.s.pm4_states[SI_STATE_IDX(rasterizer)].emit = si_pm4_emit_rasterizer;
4840 sctx->atoms.s.pm4_states[SI_STATE_IDX(dsa)].emit = si_pm4_emit_dsa;
4841 sctx->atoms.s.pm4_states[SI_STATE_IDX(sqtt_pipeline)].emit = si_pm4_emit_sqtt_pipeline;
4842 sctx->atoms.s.pm4_states[SI_STATE_IDX(ls)].emit = si_pm4_emit_shader;
4843 sctx->atoms.s.pm4_states[SI_STATE_IDX(hs)].emit = si_pm4_emit_shader;
4844 sctx->atoms.s.pm4_states[SI_STATE_IDX(es)].emit = si_pm4_emit_shader;
4845 sctx->atoms.s.pm4_states[SI_STATE_IDX(gs)].emit = si_pm4_emit_shader;
4846 sctx->atoms.s.pm4_states[SI_STATE_IDX(vs)].emit = si_pm4_emit_shader;
4847 sctx->atoms.s.pm4_states[SI_STATE_IDX(ps)].emit = si_pm4_emit_shader;
4848
4849 if (sctx->gfx_level >= GFX12)
4850 sctx->atoms.s.framebuffer.emit = gfx12_emit_framebuffer_state;
4851 else if (sctx->screen->info.has_set_context_pairs_packed)
4852 sctx->atoms.s.framebuffer.emit = gfx11_dgpu_emit_framebuffer_state;
4853 else
4854 sctx->atoms.s.framebuffer.emit = gfx6_emit_framebuffer_state;
4855
4856 sctx->atoms.s.db_render_state.emit = si_emit_db_render_state;
4857 sctx->atoms.s.dpbb_state.emit = si_emit_dpbb_state;
4858 sctx->atoms.s.msaa_config.emit = si_emit_msaa_config;
4859 sctx->atoms.s.sample_mask.emit = si_emit_sample_mask;
4860 sctx->atoms.s.cb_render_state.emit = si_emit_cb_render_state;
4861 sctx->atoms.s.blend_color.emit = si_emit_blend_color;
4862 sctx->atoms.s.clip_regs.emit = si_emit_clip_regs;
4863 sctx->atoms.s.clip_state.emit = si_emit_clip_state;
4864 sctx->atoms.s.stencil_ref.emit = si_emit_stencil_ref;
4865
4866 sctx->b.create_blend_state = si_create_blend_state;
4867 sctx->b.bind_blend_state = si_bind_blend_state;
4868 sctx->b.delete_blend_state = si_delete_blend_state;
4869 sctx->b.set_blend_color = si_set_blend_color;
4870
4871 sctx->b.create_rasterizer_state = si_create_rs_state;
4872 sctx->b.bind_rasterizer_state = si_bind_rs_state;
4873 sctx->b.delete_rasterizer_state = si_delete_rs_state;
4874
4875 sctx->b.create_depth_stencil_alpha_state = si_create_dsa_state;
4876 sctx->b.bind_depth_stencil_alpha_state = si_bind_dsa_state;
4877 sctx->b.delete_depth_stencil_alpha_state = si_delete_dsa_state;
4878
4879 sctx->custom_dsa_flush = si_create_db_flush_dsa(sctx);
4880
4881 if (sctx->gfx_level < GFX11) {
4882 sctx->custom_blend_resolve = si_create_blend_custom(sctx, V_028808_CB_RESOLVE);
4883 sctx->custom_blend_fmask_decompress = si_create_blend_custom(sctx, V_028808_CB_FMASK_DECOMPRESS);
4884 sctx->custom_blend_eliminate_fastclear =
4885 si_create_blend_custom(sctx, V_028808_CB_ELIMINATE_FAST_CLEAR);
4886 }
4887
4888 sctx->custom_blend_dcc_decompress =
4889 si_create_blend_custom(sctx,
4890 sctx->gfx_level >= GFX12 ? V_028858_CB_DCC_DECOMPRESS :
4891 sctx->gfx_level >= GFX11 ? V_028808_CB_DCC_DECOMPRESS_GFX11 :
4892 V_028808_CB_DCC_DECOMPRESS_GFX8);
4893
4894 sctx->b.set_clip_state = si_set_clip_state;
4895 sctx->b.set_stencil_ref = si_set_stencil_ref;
4896
4897 sctx->b.set_framebuffer_state = si_set_framebuffer_state;
4898
4899 sctx->b.set_sample_mask = si_set_sample_mask;
4900
4901 sctx->b.create_vertex_elements_state = si_create_vertex_elements;
4902 sctx->b.bind_vertex_elements_state = si_bind_vertex_elements;
4903 sctx->b.delete_vertex_elements_state = si_delete_vertex_element;
4904 sctx->b.set_vertex_buffers = si_set_vertex_buffers;
4905
4906 sctx->b.set_min_samples = si_set_min_samples;
4907 sctx->b.set_tess_state = si_set_tess_state;
4908
4909 sctx->b.set_active_query_state = si_set_active_query_state;
4910 }
4911
si_init_screen_state_functions(struct si_screen * sscreen)4912 void si_init_screen_state_functions(struct si_screen *sscreen)
4913 {
4914 sscreen->b.is_format_supported = si_is_format_supported;
4915 sscreen->b.create_vertex_state = si_pipe_create_vertex_state;
4916 sscreen->b.vertex_state_destroy = si_pipe_vertex_state_destroy;
4917
4918 util_vertex_state_cache_init(&sscreen->vertex_state_cache,
4919 si_create_vertex_state, si_vertex_state_destroy);
4920 }
4921
si_init_compute_preamble_state(struct si_context * sctx,struct si_pm4_state * pm4)4922 static void si_init_compute_preamble_state(struct si_context *sctx,
4923 struct si_pm4_state *pm4)
4924 {
4925 uint64_t border_color_va =
4926 sctx->border_color_buffer ? sctx->border_color_buffer->gpu_address : 0;
4927
4928 const struct ac_preamble_state preamble_state = {
4929 .border_color_va = border_color_va,
4930 .gfx11 = {
4931 .compute_dispatch_interleave = 256,
4932 },
4933 };
4934
4935 ac_init_compute_preamble_state(&preamble_state, &pm4->base);
4936 }
4937
si_init_graphics_preamble_state(struct si_context * sctx,struct si_pm4_state * pm4)4938 static void si_init_graphics_preamble_state(struct si_context *sctx,
4939 struct si_pm4_state *pm4)
4940 {
4941 struct si_screen *sscreen = sctx->screen;
4942 uint64_t border_color_va =
4943 sctx->border_color_buffer ? sctx->border_color_buffer->gpu_address : 0;
4944
4945 const struct ac_preamble_state preamble_state = {
4946 .border_color_va = border_color_va,
4947 .gfx10.cache_rb_gl2 = sctx->gfx_level >= GFX10 && sscreen->options.cache_rb_gl2,
4948 };
4949
4950 ac_init_graphics_preamble_state(&preamble_state, &pm4->base);
4951
4952 if (sctx->gfx_level >= GFX7) {
4953 /* If any sample location uses the -8 coordinate, the EXCLUSION fields should be set to 0. */
4954 ac_pm4_set_reg(&pm4->base, R_02882C_PA_SU_PRIM_FILTER_CNTL,
4955 S_02882C_XMAX_RIGHT_EXCLUSION(1) |
4956 S_02882C_YMAX_BOTTOM_EXCLUSION(1));
4957 }
4958 }
4959
gfx6_init_gfx_preamble_state(struct si_context * sctx)4960 static void gfx6_init_gfx_preamble_state(struct si_context *sctx)
4961 {
4962 struct si_screen *sscreen = sctx->screen;
4963 bool has_clear_state = sscreen->info.has_clear_state;
4964
4965 /* We need more space because the preamble is large. */
4966 struct si_pm4_state *pm4 = si_pm4_create_sized(sscreen, 214, sctx->has_graphics);
4967 if (!pm4)
4968 return;
4969
4970 if (sctx->has_graphics && !sctx->shadowing.registers) {
4971 ac_pm4_cmd_add(&pm4->base, PKT3(PKT3_CONTEXT_CONTROL, 1, 0));
4972 ac_pm4_cmd_add(&pm4->base, CC0_UPDATE_LOAD_ENABLES(1));
4973 ac_pm4_cmd_add(&pm4->base, CC1_UPDATE_SHADOW_ENABLES(1));
4974
4975 if (sscreen->dpbb_allowed) {
4976 ac_pm4_cmd_add(&pm4->base, PKT3(PKT3_EVENT_WRITE, 0, 0));
4977 ac_pm4_cmd_add(&pm4->base, EVENT_TYPE(V_028A90_BREAK_BATCH) | EVENT_INDEX(0));
4978 }
4979
4980 if (has_clear_state) {
4981 ac_pm4_cmd_add(&pm4->base, PKT3(PKT3_CLEAR_STATE, 0, 0));
4982 ac_pm4_cmd_add(&pm4->base, 0);
4983 }
4984 }
4985
4986 si_init_compute_preamble_state(sctx, pm4);
4987
4988 if (!sctx->has_graphics)
4989 goto done;
4990
4991 /* Graphics registers. */
4992 si_init_graphics_preamble_state(sctx, pm4);
4993
4994 if (!has_clear_state) {
4995 ac_pm4_set_reg(&pm4->base, R_02800C_DB_RENDER_OVERRIDE, 0);
4996 ac_pm4_set_reg(&pm4->base, R_0286E0_SPI_BARYC_CNTL, 0);
4997 }
4998
4999 if (sctx->family >= CHIP_POLARIS10 && !sctx->screen->info.has_small_prim_filter_sample_loc_bug) {
5000 /* Polaris10-12 should disable small line culling, but those also have the sample loc bug,
5001 * so they never enter this branch.
5002 */
5003 assert(sctx->family > CHIP_POLARIS12);
5004 ac_pm4_set_reg(&pm4->base, R_028830_PA_SU_SMALL_PRIM_FILTER_CNTL,
5005 S_028830_SMALL_PRIM_FILTER_ENABLE(1));
5006 }
5007
5008 if (sctx->gfx_level <= GFX7 || !has_clear_state) {
5009 ac_pm4_set_reg(&pm4->base, R_028B28_VGT_STRMOUT_DRAW_OPAQUE_OFFSET, 0);
5010 ac_pm4_set_reg(&pm4->base, R_028034_PA_SC_SCREEN_SCISSOR_BR,
5011 S_028034_BR_X(16384) | S_028034_BR_Y(16384));
5012 }
5013
5014 if (sctx->gfx_level == GFX9) {
5015 ac_pm4_set_reg(&pm4->base, R_028C4C_PA_SC_CONSERVATIVE_RASTERIZATION_CNTL,
5016 S_028C4C_NULL_SQUAD_AA_MASK_ENABLE(1));
5017 }
5018
5019 done:
5020 ac_pm4_finalize(&pm4->base);
5021 sctx->cs_preamble_state = pm4;
5022 sctx->cs_preamble_state_tmz = si_pm4_clone(sscreen, pm4); /* Make a copy of the preamble for TMZ. */
5023 }
5024
cdna_init_compute_preamble_state(struct si_context * sctx)5025 static void cdna_init_compute_preamble_state(struct si_context *sctx)
5026 {
5027 struct si_screen *sscreen = sctx->screen;
5028 uint64_t border_color_va =
5029 sctx->border_color_buffer ? sctx->border_color_buffer->gpu_address : 0;
5030 uint32_t compute_cu_en = S_00B858_SH0_CU_EN(sscreen->info.spi_cu_en) |
5031 S_00B858_SH1_CU_EN(sscreen->info.spi_cu_en);
5032
5033 struct si_pm4_state *pm4 = si_pm4_create_sized(sscreen, 48, true);
5034 if (!pm4)
5035 return;
5036
5037 /* Compute registers. */
5038 /* Disable profiling on compute chips. */
5039 ac_pm4_set_reg(&pm4->base, R_00B82C_COMPUTE_PERFCOUNT_ENABLE, 0);
5040 ac_pm4_set_reg(&pm4->base, R_00B834_COMPUTE_PGM_HI, S_00B834_DATA(sctx->screen->info.address32_hi >> 8));
5041 ac_pm4_set_reg(&pm4->base, R_00B858_COMPUTE_STATIC_THREAD_MGMT_SE0, compute_cu_en);
5042 ac_pm4_set_reg(&pm4->base, R_00B85C_COMPUTE_STATIC_THREAD_MGMT_SE1, compute_cu_en);
5043 ac_pm4_set_reg(&pm4->base, R_00B864_COMPUTE_STATIC_THREAD_MGMT_SE2, compute_cu_en);
5044 ac_pm4_set_reg(&pm4->base, R_00B868_COMPUTE_STATIC_THREAD_MGMT_SE3, compute_cu_en);
5045 ac_pm4_set_reg(&pm4->base, R_00B878_COMPUTE_THREAD_TRACE_ENABLE, 0);
5046
5047 if (sscreen->info.family >= CHIP_GFX940) {
5048 ac_pm4_set_reg(&pm4->base, R_00B89C_COMPUTE_TG_CHUNK_SIZE, 0);
5049 ac_pm4_set_reg(&pm4->base, R_00B8B4_COMPUTE_PGM_RSRC3, 0);
5050 } else {
5051 ac_pm4_set_reg(&pm4->base, R_00B894_COMPUTE_STATIC_THREAD_MGMT_SE4, compute_cu_en);
5052 ac_pm4_set_reg(&pm4->base, R_00B898_COMPUTE_STATIC_THREAD_MGMT_SE5, compute_cu_en);
5053 ac_pm4_set_reg(&pm4->base, R_00B89C_COMPUTE_STATIC_THREAD_MGMT_SE6, compute_cu_en);
5054 ac_pm4_set_reg(&pm4->base, R_00B8A0_COMPUTE_STATIC_THREAD_MGMT_SE7, compute_cu_en);
5055 }
5056
5057 ac_pm4_set_reg(&pm4->base, R_0301EC_CP_COHER_START_DELAY, 0);
5058
5059 /* Set the pointer to border colors. Only MI100 supports border colors. */
5060 if (sscreen->info.family == CHIP_MI100) {
5061 ac_pm4_set_reg(&pm4->base, R_030E00_TA_CS_BC_BASE_ADDR, border_color_va >> 8);
5062 ac_pm4_set_reg(&pm4->base, R_030E04_TA_CS_BC_BASE_ADDR_HI,
5063 S_030E04_ADDRESS(border_color_va >> 40));
5064 }
5065
5066 ac_pm4_finalize(&pm4->base);
5067 sctx->cs_preamble_state = pm4;
5068 sctx->cs_preamble_state_tmz = si_pm4_clone(sscreen, pm4); /* Make a copy of the preamble for TMZ. */
5069 }
5070
gfx10_init_gfx_preamble_state(struct si_context * sctx)5071 static void gfx10_init_gfx_preamble_state(struct si_context *sctx)
5072 {
5073 struct si_screen *sscreen = sctx->screen;
5074
5075 /* We need more space because the preamble is large. */
5076 struct si_pm4_state *pm4 = si_pm4_create_sized(sscreen, 214, sctx->has_graphics);
5077 if (!pm4)
5078 return;
5079
5080 if (sctx->has_graphics && !sctx->shadowing.registers) {
5081 ac_pm4_cmd_add(&pm4->base, PKT3(PKT3_CONTEXT_CONTROL, 1, 0));
5082 ac_pm4_cmd_add(&pm4->base, CC0_UPDATE_LOAD_ENABLES(1));
5083 ac_pm4_cmd_add(&pm4->base, CC1_UPDATE_SHADOW_ENABLES(1));
5084
5085 if (sscreen->dpbb_allowed) {
5086 ac_pm4_cmd_add(&pm4->base, PKT3(PKT3_EVENT_WRITE, 0, 0));
5087 ac_pm4_cmd_add(&pm4->base, EVENT_TYPE(V_028A90_BREAK_BATCH) | EVENT_INDEX(0));
5088 }
5089
5090 ac_pm4_cmd_add(&pm4->base, PKT3(PKT3_CLEAR_STATE, 0, 0));
5091 ac_pm4_cmd_add(&pm4->base, 0);
5092 }
5093
5094 si_init_compute_preamble_state(sctx, pm4);
5095
5096 if (!sctx->has_graphics)
5097 goto done;
5098
5099 /* Graphics registers. */
5100 si_init_graphics_preamble_state(sctx, pm4);
5101
5102 ac_pm4_set_reg(&pm4->base, R_028708_SPI_SHADER_IDX_FORMAT,
5103 S_028708_IDX0_EXPORT_FORMAT(V_028708_SPI_SHADER_1COMP));
5104
5105 if (sctx->gfx_level >= GFX10_3) {
5106 /* The rate combiners have no effect if they are disabled like this:
5107 * VERTEX_RATE: BYPASS_VTX_RATE_COMBINER = 1
5108 * PRIMITIVE_RATE: BYPASS_PRIM_RATE_COMBINER = 1
5109 * HTILE_RATE: VRS_HTILE_ENCODING = 0
5110 * SAMPLE_ITER: PS_ITER_SAMPLE = 0
5111 *
5112 * Use OVERRIDE, which will ignore results from previous combiners.
5113 * (e.g. enabled sample shading overrides the vertex rate)
5114 */
5115 ac_pm4_set_reg(&pm4->base, R_028848_PA_CL_VRS_CNTL,
5116 S_028848_VERTEX_RATE_COMBINER_MODE(V_028848_SC_VRS_COMB_MODE_OVERRIDE) |
5117 S_028848_SAMPLE_ITER_COMBINER_MODE(V_028848_SC_VRS_COMB_MODE_OVERRIDE));
5118 }
5119
5120 done:
5121 ac_pm4_finalize(&pm4->base);
5122 sctx->cs_preamble_state = pm4;
5123 sctx->cs_preamble_state_tmz = si_pm4_clone(sscreen, pm4); /* Make a copy of the preamble for TMZ. */
5124 }
5125
gfx12_init_gfx_preamble_state(struct si_context * sctx)5126 static void gfx12_init_gfx_preamble_state(struct si_context *sctx)
5127 {
5128 struct si_screen *sscreen = sctx->screen;
5129
5130 struct si_pm4_state *pm4 = si_pm4_create_sized(sscreen, 300, sctx->has_graphics);
5131 if (!pm4)
5132 return;
5133
5134 if (sctx->has_graphics && !sctx->shadowing.registers) {
5135 ac_pm4_cmd_add(&pm4->base, PKT3(PKT3_CONTEXT_CONTROL, 1, 0));
5136 ac_pm4_cmd_add(&pm4->base, CC0_UPDATE_LOAD_ENABLES(1));
5137 ac_pm4_cmd_add(&pm4->base, CC1_UPDATE_SHADOW_ENABLES(1));
5138 }
5139
5140 if (sctx->has_graphics && sscreen->dpbb_allowed) {
5141 ac_pm4_cmd_add(&pm4->base, PKT3(PKT3_EVENT_WRITE, 0, 0));
5142 ac_pm4_cmd_add(&pm4->base, EVENT_TYPE(V_028A90_BREAK_BATCH) | EVENT_INDEX(0));
5143 }
5144
5145 si_init_compute_preamble_state(sctx, pm4);
5146
5147 if (!sctx->has_graphics)
5148 goto done;
5149
5150 /* Graphics registers. */
5151 si_init_graphics_preamble_state(sctx, pm4);
5152
5153 ac_pm4_set_reg(&pm4->base, R_028648_SPI_SHADER_IDX_FORMAT,
5154 S_028648_IDX0_EXPORT_FORMAT(V_028648_SPI_SHADER_1COMP));
5155 ac_pm4_set_reg(&pm4->base, R_028658_SPI_BARYC_CNTL, 0);
5156
5157 ac_pm4_set_reg(&pm4->base, R_028B28_VGT_STRMOUT_DRAW_OPAQUE_OFFSET, 0);
5158
5159 /* The rate combiners have no effect if they are disabled like this:
5160 * VERTEX_RATE: BYPASS_VTX_RATE_COMBINER = 1
5161 * PRIMITIVE_RATE: BYPASS_PRIM_RATE_COMBINER = 1
5162 * HTILE_RATE: VRS_HTILE_ENCODING = 0
5163 * SAMPLE_ITER: PS_ITER_SAMPLE = 0
5164 *
5165 * Use OVERRIDE, which will ignore results from previous combiners.
5166 * (e.g. enabled sample shading overrides the vertex rate)
5167 */
5168 ac_pm4_set_reg(&pm4->base, R_028848_PA_CL_VRS_CNTL,
5169 S_028848_VERTEX_RATE_COMBINER_MODE(V_028848_SC_VRS_COMB_MODE_OVERRIDE) |
5170 S_028848_SAMPLE_ITER_COMBINER_MODE(V_028848_SC_VRS_COMB_MODE_OVERRIDE));
5171
5172 ac_pm4_set_reg(&pm4->base, R_028C54_PA_SC_CONSERVATIVE_RASTERIZATION_CNTL,
5173 S_028C54_NULL_SQUAD_AA_MASK_ENABLE(1));
5174
5175 done:
5176 sctx->cs_preamble_state = pm4;
5177 sctx->cs_preamble_state_tmz = si_pm4_clone(sscreen, pm4); /* Make a copy of the preamble for TMZ. */
5178 }
5179
si_init_gfx_preamble_state(struct si_context * sctx)5180 void si_init_gfx_preamble_state(struct si_context *sctx)
5181 {
5182 if (!sctx->screen->info.has_graphics)
5183 cdna_init_compute_preamble_state(sctx);
5184 else if (sctx->gfx_level >= GFX12)
5185 gfx12_init_gfx_preamble_state(sctx);
5186 else if (sctx->gfx_level >= GFX10)
5187 gfx10_init_gfx_preamble_state(sctx);
5188 else
5189 gfx6_init_gfx_preamble_state(sctx);
5190 }
5191