1 /*
2 * Copyright 2022 Advanced Micro Devices, Inc.
3 *
4 * SPDX-License-Identifier: MIT
5 */
6
7 /*
8 * This lowering pass converts index based buffer/image/texture access to
9 * explicite descriptor based, which simplify the compiler backend translation.
10 *
11 * For example: load_ubo(1) -> load_ubo(vec4), where the vec4 is the buffer
12 * descriptor with index==1, so compiler backend don't need to do index-to-descriptor
13 * finding which is the most complicated part (move to nir now).
14 */
15
16 #include "nir_builder.h"
17
18 #include "ac_nir.h"
19 #include "si_pipe.h"
20 #include "si_shader_internal.h"
21 #include "sid.h"
22
23 struct lower_resource_state {
24 struct si_shader *shader;
25 struct si_shader_args *args;
26 };
27
load_ubo_desc_fast_path(nir_builder * b,nir_def * addr_lo,struct si_shader_selector * sel)28 static nir_def *load_ubo_desc_fast_path(nir_builder *b, nir_def *addr_lo,
29 struct si_shader_selector *sel)
30 {
31 nir_def *addr_hi =
32 nir_imm_int(b, S_008F04_BASE_ADDRESS_HI(sel->screen->info.address32_hi));
33
34 uint32_t rsrc3 =
35 S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
36 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
37
38 if (sel->screen->info.gfx_level >= GFX11)
39 rsrc3 |= S_008F0C_FORMAT(V_008F0C_GFX11_FORMAT_32_FLOAT) |
40 S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW);
41 else if (sel->screen->info.gfx_level >= GFX10)
42 rsrc3 |= S_008F0C_FORMAT(V_008F0C_GFX10_FORMAT_32_FLOAT) |
43 S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) | S_008F0C_RESOURCE_LEVEL(1);
44 else
45 rsrc3 |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
46 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
47
48 return nir_vec4(b, addr_lo, addr_hi, nir_imm_int(b, sel->info.constbuf0_num_slots * 16),
49 nir_imm_int(b, rsrc3));
50 }
51
clamp_index(nir_builder * b,nir_def * index,unsigned max)52 static nir_def *clamp_index(nir_builder *b, nir_def *index, unsigned max)
53 {
54 if (util_is_power_of_two_or_zero(max))
55 return nir_iand_imm(b, index, max - 1);
56 else {
57 nir_def *clamp = nir_imm_int(b, max - 1);
58 nir_def *cond = nir_uge(b, clamp, index);
59 return nir_bcsel(b, cond, index, clamp);
60 }
61 }
62
load_ubo_desc(nir_builder * b,nir_def * index,struct lower_resource_state * s)63 static nir_def *load_ubo_desc(nir_builder *b, nir_def *index,
64 struct lower_resource_state *s)
65 {
66 struct si_shader_selector *sel = s->shader->selector;
67
68 nir_def *addr = ac_nir_load_arg(b, &s->args->ac, s->args->const_and_shader_buffers);
69
70 if (sel->info.base.num_ubos == 1 && sel->info.base.num_ssbos == 0)
71 return load_ubo_desc_fast_path(b, addr, sel);
72
73 index = clamp_index(b, index, sel->info.base.num_ubos);
74 index = nir_iadd_imm(b, index, SI_NUM_SHADER_BUFFERS);
75
76 nir_def *offset = nir_ishl_imm(b, index, 4);
77 return nir_load_smem_amd(b, 4, addr, offset);
78 }
79
load_ssbo_desc(nir_builder * b,nir_src * index,struct lower_resource_state * s)80 static nir_def *load_ssbo_desc(nir_builder *b, nir_src *index,
81 struct lower_resource_state *s)
82 {
83 struct si_shader_selector *sel = s->shader->selector;
84
85 /* Fast path if the shader buffer is in user SGPRs. */
86 if (nir_src_is_const(*index)) {
87 unsigned slot = nir_src_as_uint(*index);
88 if (slot < sel->cs_num_shaderbufs_in_user_sgprs)
89 return ac_nir_load_arg(b, &s->args->ac, s->args->cs_shaderbuf[slot]);
90 }
91
92 nir_def *addr = ac_nir_load_arg(b, &s->args->ac, s->args->const_and_shader_buffers);
93 nir_def *slot = clamp_index(b, index->ssa, sel->info.base.num_ssbos);
94 slot = nir_isub_imm(b, SI_NUM_SHADER_BUFFERS - 1, slot);
95
96 nir_def *offset = nir_ishl_imm(b, slot, 4);
97 return nir_load_smem_amd(b, 4, addr, offset);
98 }
99
fixup_image_desc(nir_builder * b,nir_def * rsrc,bool uses_store,struct lower_resource_state * s)100 static nir_def *fixup_image_desc(nir_builder *b, nir_def *rsrc, bool uses_store,
101 struct lower_resource_state *s)
102 {
103 struct si_shader_selector *sel = s->shader->selector;
104 struct si_screen *screen = sel->screen;
105
106 /**
107 * Given a 256-bit resource descriptor, force the DCC enable bit to off.
108 *
109 * At least on Tonga, executing image stores on images with DCC enabled and
110 * non-trivial can eventually lead to lockups. This can occur when an
111 * application binds an image as read-only but then uses a shader that writes
112 * to it. The OpenGL spec allows almost arbitrarily bad behavior (including
113 * program termination) in this case, but it doesn't cost much to be a bit
114 * nicer: disabling DCC in the shader still leads to undefined results but
115 * avoids the lockup.
116 */
117 if (uses_store &&
118 screen->info.gfx_level <= GFX9 &&
119 screen->info.gfx_level >= GFX8) {
120 nir_def *tmp = nir_channel(b, rsrc, 6);
121 tmp = nir_iand_imm(b, tmp, C_008F28_COMPRESSION_EN);
122 rsrc = nir_vector_insert_imm(b, rsrc, tmp, 6);
123 }
124
125 if (!uses_store &&
126 screen->info.has_image_load_dcc_bug &&
127 screen->always_allow_dcc_stores) {
128 nir_def *tmp = nir_channel(b, rsrc, 6);
129 tmp = nir_iand_imm(b, tmp, C_00A018_WRITE_COMPRESS_ENABLE);
130 rsrc = nir_vector_insert_imm(b, rsrc, tmp, 6);
131 }
132
133 return rsrc;
134 }
135
136 /* AC_DESC_FMASK is handled exactly like AC_DESC_IMAGE. The caller should
137 * adjust "index" to point to FMASK.
138 */
load_image_desc(nir_builder * b,nir_def * list,nir_def * index,enum ac_descriptor_type desc_type,bool uses_store,struct lower_resource_state * s)139 static nir_def *load_image_desc(nir_builder *b, nir_def *list, nir_def *index,
140 enum ac_descriptor_type desc_type, bool uses_store,
141 struct lower_resource_state *s)
142 {
143 /* index is in uvec8 unit, convert to offset in bytes */
144 nir_def *offset = nir_ishl_imm(b, index, 5);
145
146 unsigned num_channels;
147 if (desc_type == AC_DESC_BUFFER) {
148 offset = nir_iadd_imm(b, offset, 16);
149 num_channels = 4;
150 } else {
151 assert(desc_type == AC_DESC_IMAGE || desc_type == AC_DESC_FMASK);
152 num_channels = 8;
153 }
154
155 nir_def *rsrc = nir_load_smem_amd(b, num_channels, list, offset);
156
157 if (desc_type == AC_DESC_IMAGE)
158 rsrc = fixup_image_desc(b, rsrc, uses_store, s);
159
160 return rsrc;
161 }
162
deref_to_index(nir_builder * b,nir_deref_instr * deref,unsigned max_slots,nir_def ** dynamic_index_ret,unsigned * const_index_ret)163 static nir_def *deref_to_index(nir_builder *b,
164 nir_deref_instr *deref,
165 unsigned max_slots,
166 nir_def **dynamic_index_ret,
167 unsigned *const_index_ret)
168 {
169 unsigned const_index = 0;
170 nir_def *dynamic_index = NULL;
171 while (deref->deref_type != nir_deref_type_var) {
172 assert(deref->deref_type == nir_deref_type_array);
173 unsigned array_size = MAX2(glsl_get_aoa_size(deref->type), 1);
174
175 if (nir_src_is_const(deref->arr.index)) {
176 const_index += array_size * nir_src_as_uint(deref->arr.index);
177 } else {
178 nir_def *tmp = nir_imul_imm(b, deref->arr.index.ssa, array_size);
179 dynamic_index = dynamic_index ? nir_iadd(b, dynamic_index, tmp) : tmp;
180 }
181
182 deref = nir_deref_instr_parent(deref);
183 }
184
185 unsigned base_index = deref->var->data.binding;
186 const_index += base_index;
187
188 /* Redirect invalid resource indices to the first array element. */
189 if (const_index >= max_slots)
190 const_index = base_index;
191
192 nir_def *index = nir_imm_int(b, const_index);
193 if (dynamic_index) {
194 index = nir_iadd(b, dynamic_index, index);
195
196 /* From the GL_ARB_shader_image_load_store extension spec:
197 *
198 * If a shader performs an image load, store, or atomic
199 * operation using an image variable declared as an array,
200 * and if the index used to select an individual element is
201 * negative or greater than or equal to the size of the
202 * array, the results of the operation are undefined but may
203 * not lead to termination.
204 */
205 index = clamp_index(b, index, max_slots);
206 }
207
208 if (dynamic_index_ret)
209 *dynamic_index_ret = dynamic_index;
210 if (const_index_ret)
211 *const_index_ret = const_index;
212
213 return index;
214 }
215
load_deref_image_desc(nir_builder * b,nir_deref_instr * deref,enum ac_descriptor_type desc_type,bool is_load,struct lower_resource_state * s)216 static nir_def *load_deref_image_desc(nir_builder *b, nir_deref_instr *deref,
217 enum ac_descriptor_type desc_type, bool is_load,
218 struct lower_resource_state *s)
219 {
220 unsigned const_index;
221 nir_def *dynamic_index;
222 nir_def *index = deref_to_index(b, deref, s->shader->selector->info.base.num_images,
223 &dynamic_index, &const_index);
224
225 nir_def *desc;
226 if (!dynamic_index && desc_type != AC_DESC_FMASK &&
227 const_index < s->shader->selector->cs_num_images_in_user_sgprs) {
228 /* Fast path if the image is in user SGPRs. */
229 desc = ac_nir_load_arg(b, &s->args->ac, s->args->cs_image[const_index]);
230
231 if (desc_type == AC_DESC_IMAGE)
232 desc = fixup_image_desc(b, desc, !is_load, s);
233 } else {
234 /* FMASKs are separate from images. */
235 if (desc_type == AC_DESC_FMASK)
236 index = nir_iadd_imm(b, index, SI_NUM_IMAGES);
237
238 index = nir_isub_imm(b, SI_NUM_IMAGE_SLOTS - 1, index);
239
240 nir_def *list = ac_nir_load_arg(b, &s->args->ac, s->args->samplers_and_images);
241 desc = load_image_desc(b, list, index, desc_type, !is_load, s);
242 }
243
244 return desc;
245 }
246
load_bindless_image_desc(nir_builder * b,nir_def * index,enum ac_descriptor_type desc_type,bool is_load,struct lower_resource_state * s)247 static nir_def *load_bindless_image_desc(nir_builder *b, nir_def *index,
248 enum ac_descriptor_type desc_type, bool is_load,
249 struct lower_resource_state *s)
250 {
251 /* Bindless image descriptors use 16-dword slots. */
252 index = nir_ishl_imm(b, index, 1);
253
254 /* FMASK is right after the image. */
255 if (desc_type == AC_DESC_FMASK)
256 index = nir_iadd_imm(b, index, 1);
257
258 nir_def *list = ac_nir_load_arg(b, &s->args->ac, s->args->bindless_samplers_and_images);
259 return load_image_desc(b, list, index, desc_type, !is_load, s);
260 }
261
lower_resource_intrinsic(nir_builder * b,nir_intrinsic_instr * intrin,struct lower_resource_state * s)262 static bool lower_resource_intrinsic(nir_builder *b, nir_intrinsic_instr *intrin,
263 struct lower_resource_state *s)
264 {
265 switch (intrin->intrinsic) {
266 case nir_intrinsic_load_ubo: {
267 assert(!(nir_intrinsic_access(intrin) & ACCESS_NON_UNIFORM));
268
269 nir_def *desc = load_ubo_desc(b, intrin->src[0].ssa, s);
270 nir_src_rewrite(&intrin->src[0], desc);
271 break;
272 }
273 case nir_intrinsic_load_ssbo:
274 case nir_intrinsic_ssbo_atomic:
275 case nir_intrinsic_ssbo_atomic_swap: {
276 assert(!(nir_intrinsic_access(intrin) & ACCESS_NON_UNIFORM));
277
278 nir_def *desc = load_ssbo_desc(b, &intrin->src[0], s);
279 nir_src_rewrite(&intrin->src[0], desc);
280 break;
281 }
282 case nir_intrinsic_store_ssbo: {
283 assert(!(nir_intrinsic_access(intrin) & ACCESS_NON_UNIFORM));
284
285 nir_def *desc = load_ssbo_desc(b, &intrin->src[1], s);
286 nir_src_rewrite(&intrin->src[1], desc);
287 break;
288 }
289 case nir_intrinsic_get_ssbo_size: {
290 assert(!(nir_intrinsic_access(intrin) & ACCESS_NON_UNIFORM));
291
292 nir_def *desc = load_ssbo_desc(b, &intrin->src[0], s);
293 nir_def *size = nir_channel(b, desc, 2);
294 nir_def_rewrite_uses(&intrin->def, size);
295 nir_instr_remove(&intrin->instr);
296 break;
297 }
298 case nir_intrinsic_image_deref_load:
299 case nir_intrinsic_image_deref_sparse_load:
300 case nir_intrinsic_image_deref_fragment_mask_load_amd:
301 case nir_intrinsic_image_deref_store:
302 case nir_intrinsic_image_deref_atomic:
303 case nir_intrinsic_image_deref_atomic_swap:
304 case nir_intrinsic_image_deref_descriptor_amd: {
305 assert(!(nir_intrinsic_access(intrin) & ACCESS_NON_UNIFORM));
306
307 nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
308
309 enum ac_descriptor_type desc_type;
310 if (intrin->intrinsic == nir_intrinsic_image_deref_fragment_mask_load_amd) {
311 desc_type = AC_DESC_FMASK;
312 } else {
313 enum glsl_sampler_dim dim = glsl_get_sampler_dim(deref->type);
314 desc_type = dim == GLSL_SAMPLER_DIM_BUF ? AC_DESC_BUFFER : AC_DESC_IMAGE;
315 }
316
317 bool is_load =
318 intrin->intrinsic == nir_intrinsic_image_deref_load ||
319 intrin->intrinsic == nir_intrinsic_image_deref_sparse_load ||
320 intrin->intrinsic == nir_intrinsic_image_deref_fragment_mask_load_amd ||
321 intrin->intrinsic == nir_intrinsic_image_deref_descriptor_amd;
322
323 nir_def *desc = load_deref_image_desc(b, deref, desc_type, is_load, s);
324
325 if (intrin->intrinsic == nir_intrinsic_image_deref_descriptor_amd) {
326 nir_def_rewrite_uses(&intrin->def, desc);
327 nir_instr_remove(&intrin->instr);
328 } else {
329 nir_intrinsic_set_image_dim(intrin, glsl_get_sampler_dim(deref->type));
330 nir_intrinsic_set_image_array(intrin, glsl_sampler_type_is_array(deref->type));
331 nir_rewrite_image_intrinsic(intrin, desc, true);
332 }
333 break;
334 }
335 case nir_intrinsic_bindless_image_load:
336 case nir_intrinsic_bindless_image_sparse_load:
337 case nir_intrinsic_bindless_image_fragment_mask_load_amd:
338 case nir_intrinsic_bindless_image_store:
339 case nir_intrinsic_bindless_image_atomic:
340 case nir_intrinsic_bindless_image_atomic_swap: {
341 assert(!(nir_intrinsic_access(intrin) & ACCESS_NON_UNIFORM));
342
343 enum ac_descriptor_type desc_type;
344 if (intrin->intrinsic == nir_intrinsic_bindless_image_fragment_mask_load_amd) {
345 desc_type = AC_DESC_FMASK;
346 } else {
347 enum glsl_sampler_dim dim = nir_intrinsic_image_dim(intrin);
348 desc_type = dim == GLSL_SAMPLER_DIM_BUF ? AC_DESC_BUFFER : AC_DESC_IMAGE;
349 }
350
351 bool is_load =
352 intrin->intrinsic == nir_intrinsic_bindless_image_load ||
353 intrin->intrinsic == nir_intrinsic_bindless_image_sparse_load ||
354 intrin->intrinsic == nir_intrinsic_bindless_image_fragment_mask_load_amd ||
355 intrin->intrinsic == nir_intrinsic_bindless_image_descriptor_amd;
356
357 nir_def *index = nir_u2u32(b, intrin->src[0].ssa);
358
359 nir_def *desc = load_bindless_image_desc(b, index, desc_type, is_load, s);
360
361 if (intrin->intrinsic == nir_intrinsic_bindless_image_descriptor_amd) {
362 nir_def_rewrite_uses(&intrin->def, desc);
363 nir_instr_remove(&intrin->instr);
364 } else {
365 nir_src_rewrite(&intrin->src[0], desc);
366 }
367 break;
368 }
369 default:
370 return false;
371 }
372
373 return true;
374 }
375
load_sampler_desc(nir_builder * b,nir_def * list,nir_def * index,enum ac_descriptor_type desc_type)376 static nir_def *load_sampler_desc(nir_builder *b, nir_def *list, nir_def *index,
377 enum ac_descriptor_type desc_type)
378 {
379 /* index is in 16 dword unit, convert to offset in bytes */
380 nir_def *offset = nir_ishl_imm(b, index, 6);
381
382 unsigned num_channels = 0;
383 switch (desc_type) {
384 case AC_DESC_IMAGE:
385 /* The image is at [0:7]. */
386 num_channels = 8;
387 break;
388 case AC_DESC_BUFFER:
389 /* The buffer is in [4:7]. */
390 offset = nir_iadd_imm(b, offset, 16);
391 num_channels = 4;
392 break;
393 case AC_DESC_FMASK:
394 /* The FMASK is at [8:15]. */
395 offset = nir_iadd_imm(b, offset, 32);
396 num_channels = 8;
397 break;
398 case AC_DESC_SAMPLER:
399 /* The sampler state is at [12:15]. */
400 offset = nir_iadd_imm(b, offset, 48);
401 num_channels = 4;
402 break;
403 default:
404 unreachable("invalid desc type");
405 break;
406 }
407
408 return nir_load_smem_amd(b, num_channels, list, offset);
409 }
410
load_deref_sampler_desc(nir_builder * b,nir_deref_instr * deref,enum ac_descriptor_type desc_type,struct lower_resource_state * s,bool return_descriptor)411 static nir_def *load_deref_sampler_desc(nir_builder *b, nir_deref_instr *deref,
412 enum ac_descriptor_type desc_type,
413 struct lower_resource_state *s,
414 bool return_descriptor)
415 {
416 unsigned max_slots = BITSET_LAST_BIT(b->shader->info.textures_used);
417 nir_def *index = deref_to_index(b, deref, max_slots, NULL, NULL);
418 index = nir_iadd_imm(b, index, SI_NUM_IMAGE_SLOTS / 2);
419
420 /* return actual desc when required by caller */
421 if (return_descriptor) {
422 nir_def *list = ac_nir_load_arg(b, &s->args->ac, s->args->samplers_and_images);
423 return load_sampler_desc(b, list, index, desc_type);
424 }
425
426 /* Just use index here and let nir-to-llvm backend to translate to actual
427 * descriptor. This is because we need waterfall to handle non-dynamic-uniform
428 * index there.
429 */
430 return index;
431 }
432
load_bindless_sampler_desc(nir_builder * b,nir_def * index,enum ac_descriptor_type desc_type,struct lower_resource_state * s)433 static nir_def *load_bindless_sampler_desc(nir_builder *b, nir_def *index,
434 enum ac_descriptor_type desc_type,
435 struct lower_resource_state *s)
436 {
437 nir_def *list = ac_nir_load_arg(b, &s->args->ac, s->args->bindless_samplers_and_images);
438
439 /* 64 bit to 32 bit */
440 index = nir_u2u32(b, index);
441
442 return load_sampler_desc(b, list, index, desc_type);
443 }
444
fixup_sampler_desc(nir_builder * b,nir_tex_instr * tex,nir_def * sampler,struct lower_resource_state * s)445 static nir_def *fixup_sampler_desc(nir_builder *b,
446 nir_tex_instr *tex,
447 nir_def *sampler,
448 struct lower_resource_state *s)
449 {
450 const struct si_shader_selector *sel = s->shader->selector;
451
452 if (tex->op != nir_texop_tg4 || sel->screen->info.conformant_trunc_coord)
453 return sampler;
454
455 /* Set TRUNC_COORD=0 for textureGather(). */
456 nir_def *dword0 = nir_channel(b, sampler, 0);
457 dword0 = nir_iand_imm(b, dword0, C_008F30_TRUNC_COORD);
458 sampler = nir_vector_insert_imm(b, sampler, dword0, 0);
459 return sampler;
460 }
461
lower_resource_tex(nir_builder * b,nir_tex_instr * tex,struct lower_resource_state * s)462 static bool lower_resource_tex(nir_builder *b, nir_tex_instr *tex,
463 struct lower_resource_state *s)
464 {
465 nir_deref_instr *texture_deref = NULL;
466 nir_deref_instr *sampler_deref = NULL;
467 nir_def *texture_handle = NULL;
468 nir_def *sampler_handle = NULL;
469
470 for (unsigned i = 0; i < tex->num_srcs; i++) {
471 switch (tex->src[i].src_type) {
472 case nir_tex_src_texture_deref:
473 texture_deref = nir_src_as_deref(tex->src[i].src);
474 break;
475 case nir_tex_src_sampler_deref:
476 sampler_deref = nir_src_as_deref(tex->src[i].src);
477 break;
478 case nir_tex_src_texture_handle:
479 texture_handle = tex->src[i].src.ssa;
480 break;
481 case nir_tex_src_sampler_handle:
482 sampler_handle = tex->src[i].src.ssa;
483 break;
484 default:
485 break;
486 }
487 }
488
489 enum ac_descriptor_type desc_type;
490 if (tex->op == nir_texop_fragment_mask_fetch_amd)
491 desc_type = AC_DESC_FMASK;
492 else
493 desc_type = tex->sampler_dim == GLSL_SAMPLER_DIM_BUF ? AC_DESC_BUFFER : AC_DESC_IMAGE;
494
495 if (tex->op == nir_texop_descriptor_amd) {
496 nir_def *image;
497 if (texture_deref)
498 image = load_deref_sampler_desc(b, texture_deref, desc_type, s, true);
499 else
500 image = load_bindless_sampler_desc(b, texture_handle, desc_type, s);
501 nir_def_rewrite_uses(&tex->def, image);
502 nir_instr_remove(&tex->instr);
503 return true;
504 }
505
506 if (tex->op == nir_texop_sampler_descriptor_amd) {
507 nir_def *sampler;
508 if (sampler_deref)
509 sampler = load_deref_sampler_desc(b, sampler_deref, AC_DESC_SAMPLER, s, true);
510 else
511 sampler = load_bindless_sampler_desc(b, sampler_handle, AC_DESC_SAMPLER, s);
512 nir_def_rewrite_uses(&tex->def, sampler);
513 nir_instr_remove(&tex->instr);
514 return true;
515 }
516
517 nir_def *image = texture_deref ?
518 load_deref_sampler_desc(b, texture_deref, desc_type, s, !tex->texture_non_uniform) :
519 load_bindless_sampler_desc(b, texture_handle, desc_type, s);
520
521 nir_def *sampler = NULL;
522 if (sampler_deref)
523 sampler = load_deref_sampler_desc(b, sampler_deref, AC_DESC_SAMPLER, s, !tex->sampler_non_uniform);
524 else if (sampler_handle)
525 sampler = load_bindless_sampler_desc(b, sampler_handle, AC_DESC_SAMPLER, s);
526
527 if (sampler && sampler->num_components > 1)
528 sampler = fixup_sampler_desc(b, tex, sampler, s);
529
530 for (unsigned i = 0; i < tex->num_srcs; i++) {
531 switch (tex->src[i].src_type) {
532 case nir_tex_src_texture_deref:
533 tex->src[i].src_type = nir_tex_src_texture_handle;
534 FALLTHROUGH;
535 case nir_tex_src_texture_handle:
536 nir_src_rewrite(&tex->src[i].src, image);
537 break;
538 case nir_tex_src_sampler_deref:
539 tex->src[i].src_type = nir_tex_src_sampler_handle;
540 FALLTHROUGH;
541 case nir_tex_src_sampler_handle:
542 nir_src_rewrite(&tex->src[i].src, sampler);
543 break;
544 default:
545 break;
546 }
547 }
548
549 return true;
550 }
551
lower_resource_instr(nir_builder * b,nir_instr * instr,void * state)552 static bool lower_resource_instr(nir_builder *b, nir_instr *instr, void *state)
553 {
554 struct lower_resource_state *s = (struct lower_resource_state *)state;
555
556 b->cursor = nir_before_instr(instr);
557
558 switch (instr->type) {
559 case nir_instr_type_intrinsic: {
560 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
561 return lower_resource_intrinsic(b, intrin, s);
562 }
563 case nir_instr_type_tex: {
564 nir_tex_instr *tex = nir_instr_as_tex(instr);
565 return lower_resource_tex(b, tex, s);
566 }
567 default:
568 return false;
569 }
570 }
571
si_nir_lower_resource(nir_shader * nir,struct si_shader * shader,struct si_shader_args * args)572 bool si_nir_lower_resource(nir_shader *nir, struct si_shader *shader,
573 struct si_shader_args *args)
574 {
575 struct lower_resource_state state = {
576 .shader = shader,
577 .args = args,
578 };
579
580 return nir_shader_instructions_pass(nir, lower_resource_instr,
581 nir_metadata_dominance | nir_metadata_block_index,
582 &state);
583 }
584