1 /*
2 * Copyright (C) 2020 Google, Inc.
3 * Copyright (C) 2021 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24
25 #include "nir.h"
26 #include "nir_builder.h"
27
28 /**
29 * Return the intrinsic if it matches the mask in "modes", else return NULL.
30 */
31 static nir_intrinsic_instr *
get_io_intrinsic(nir_instr * instr,nir_variable_mode modes,nir_variable_mode * out_mode)32 get_io_intrinsic(nir_instr *instr, nir_variable_mode modes,
33 nir_variable_mode *out_mode)
34 {
35 if (instr->type != nir_instr_type_intrinsic)
36 return NULL;
37
38 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
39
40 switch (intr->intrinsic) {
41 case nir_intrinsic_load_input:
42 case nir_intrinsic_load_input_vertex:
43 case nir_intrinsic_load_interpolated_input:
44 case nir_intrinsic_load_per_vertex_input:
45 *out_mode = nir_var_shader_in;
46 return modes & nir_var_shader_in ? intr : NULL;
47 case nir_intrinsic_load_output:
48 case nir_intrinsic_load_per_vertex_output:
49 case nir_intrinsic_store_output:
50 case nir_intrinsic_store_per_vertex_output:
51 *out_mode = nir_var_shader_out;
52 return modes & nir_var_shader_out ? intr : NULL;
53 default:
54 return NULL;
55 }
56 }
57
58 /**
59 * Recompute the IO "base" indices from scratch to remove holes or to fix
60 * incorrect base values due to changes in IO locations by using IO locations
61 * to assign new bases. The mapping from locations to bases becomes
62 * monotonically increasing.
63 */
64 bool
nir_recompute_io_bases(nir_function_impl * impl,nir_variable_mode modes)65 nir_recompute_io_bases(nir_function_impl *impl, nir_variable_mode modes)
66 {
67 BITSET_DECLARE(inputs, NUM_TOTAL_VARYING_SLOTS);
68 BITSET_DECLARE(outputs, NUM_TOTAL_VARYING_SLOTS);
69 BITSET_ZERO(inputs);
70 BITSET_ZERO(outputs);
71
72 /* Gather the bitmasks of used locations. */
73 nir_foreach_block_safe (block, impl) {
74 nir_foreach_instr_safe (instr, block) {
75 nir_variable_mode mode;
76 nir_intrinsic_instr *intr = get_io_intrinsic(instr, modes, &mode);
77 if (!intr)
78 continue;
79
80 nir_io_semantics sem = nir_intrinsic_io_semantics(intr);
81 unsigned num_slots = sem.num_slots;
82 if (sem.medium_precision)
83 num_slots = (num_slots + sem.high_16bits + 1) / 2;
84
85 if (mode == nir_var_shader_in) {
86 for (unsigned i = 0; i < num_slots; i++)
87 BITSET_SET(inputs, sem.location + i);
88 } else if (!sem.dual_source_blend_index) {
89 for (unsigned i = 0; i < num_slots; i++)
90 BITSET_SET(outputs, sem.location + i);
91 }
92 }
93 }
94
95 /* Renumber bases. */
96 bool changed = false;
97
98 nir_foreach_block_safe (block, impl) {
99 nir_foreach_instr_safe (instr, block) {
100 nir_variable_mode mode;
101 nir_intrinsic_instr *intr = get_io_intrinsic(instr, modes, &mode);
102 if (!intr)
103 continue;
104
105 nir_io_semantics sem = nir_intrinsic_io_semantics(intr);
106 unsigned num_slots = sem.num_slots;
107 if (sem.medium_precision)
108 num_slots = (num_slots + sem.high_16bits + 1) / 2;
109
110 if (mode == nir_var_shader_in) {
111 nir_intrinsic_set_base(intr,
112 BITSET_PREFIX_SUM(inputs, sem.location));
113 } else if (sem.dual_source_blend_index) {
114 nir_intrinsic_set_base(intr,
115 BITSET_PREFIX_SUM(outputs, NUM_TOTAL_VARYING_SLOTS));
116 } else {
117 nir_intrinsic_set_base(intr,
118 BITSET_PREFIX_SUM(outputs, sem.location));
119 }
120 changed = true;
121 }
122 }
123
124 if (changed) {
125 nir_metadata_preserve(impl, nir_metadata_dominance |
126 nir_metadata_block_index);
127 } else {
128 nir_metadata_preserve(impl, nir_metadata_all);
129 }
130
131 return changed;
132 }
133
134 /**
135 * Lower mediump inputs and/or outputs to 16 bits.
136 *
137 * \param modes Whether to lower inputs, outputs, or both.
138 * \param varying_mask Determines which varyings to skip (VS inputs,
139 * FS outputs, and patch varyings ignore this mask).
140 * \param use_16bit_slots Remap lowered slots to* VARYING_SLOT_VARn_16BIT.
141 */
142 bool
nir_lower_mediump_io(nir_shader * nir,nir_variable_mode modes,uint64_t varying_mask,bool use_16bit_slots)143 nir_lower_mediump_io(nir_shader *nir, nir_variable_mode modes,
144 uint64_t varying_mask, bool use_16bit_slots)
145 {
146 bool changed = false;
147 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
148 assert(impl);
149
150 nir_builder b;
151 nir_builder_init(&b, impl);
152
153 nir_foreach_block_safe (block, impl) {
154 nir_foreach_instr_safe (instr, block) {
155 nir_variable_mode mode;
156 nir_intrinsic_instr *intr = get_io_intrinsic(instr, modes, &mode);
157 if (!intr)
158 continue;
159
160 nir_io_semantics sem = nir_intrinsic_io_semantics(intr);
161 nir_ssa_def *(*convert)(nir_builder *, nir_ssa_def *);
162 bool is_varying = !(nir->info.stage == MESA_SHADER_VERTEX &&
163 mode == nir_var_shader_in) &&
164 !(nir->info.stage == MESA_SHADER_FRAGMENT &&
165 mode == nir_var_shader_out);
166
167 if (!sem.medium_precision ||
168 (is_varying && sem.location <= VARYING_SLOT_VAR31 &&
169 !(varying_mask & BITFIELD64_BIT(sem.location))))
170 continue; /* can't lower */
171
172 if (nir_intrinsic_has_src_type(intr)) {
173 /* Stores. */
174 nir_alu_type type = nir_intrinsic_src_type(intr);
175
176 switch (type) {
177 case nir_type_float32:
178 convert = nir_f2fmp;
179 break;
180 case nir_type_int32:
181 case nir_type_uint32:
182 convert = nir_i2imp;
183 break;
184 default:
185 continue; /* already lowered? */
186 }
187
188 /* Convert the 32-bit store into a 16-bit store. */
189 b.cursor = nir_before_instr(&intr->instr);
190 nir_instr_rewrite_src_ssa(&intr->instr, &intr->src[0],
191 convert(&b, intr->src[0].ssa));
192 nir_intrinsic_set_src_type(intr, (type & ~32) | 16);
193 } else {
194 /* Loads. */
195 nir_alu_type type = nir_intrinsic_dest_type(intr);
196
197 switch (type) {
198 case nir_type_float32:
199 convert = nir_f2f32;
200 break;
201 case nir_type_int32:
202 convert = nir_i2i32;
203 break;
204 case nir_type_uint32:
205 convert = nir_u2u32;
206 break;
207 default:
208 continue; /* already lowered? */
209 }
210
211 /* Convert the 32-bit load into a 16-bit load. */
212 b.cursor = nir_after_instr(&intr->instr);
213 intr->dest.ssa.bit_size = 16;
214 nir_intrinsic_set_dest_type(intr, (type & ~32) | 16);
215 nir_ssa_def *dst = convert(&b, &intr->dest.ssa);
216 nir_ssa_def_rewrite_uses_after(&intr->dest.ssa, dst,
217 dst->parent_instr);
218 }
219
220 if (use_16bit_slots && is_varying &&
221 sem.location >= VARYING_SLOT_VAR0 &&
222 sem.location <= VARYING_SLOT_VAR31) {
223 unsigned index = sem.location - VARYING_SLOT_VAR0;
224
225 sem.location = VARYING_SLOT_VAR0_16BIT + index / 2;
226 sem.high_16bits = index % 2;
227 nir_intrinsic_set_io_semantics(intr, sem);
228 }
229 changed = true;
230 }
231 }
232
233 if (changed && use_16bit_slots)
234 nir_recompute_io_bases(impl, modes);
235
236 if (changed) {
237 nir_metadata_preserve(impl, nir_metadata_dominance |
238 nir_metadata_block_index);
239 } else {
240 nir_metadata_preserve(impl, nir_metadata_all);
241 }
242
243 return changed;
244 }
245
246 /**
247 * Set the mediump precision bit for those shader inputs and outputs that are
248 * set in the "modes" mask. Non-generic varyings (that GLES3 doesn't have)
249 * are ignored. The "types" mask can be (nir_type_float | nir_type_int), etc.
250 */
251 bool
nir_force_mediump_io(nir_shader * nir,nir_variable_mode modes,nir_alu_type types)252 nir_force_mediump_io(nir_shader *nir, nir_variable_mode modes,
253 nir_alu_type types)
254 {
255 bool changed = false;
256 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
257 assert(impl);
258
259 nir_builder b;
260 nir_builder_init(&b, impl);
261
262 nir_foreach_block_safe (block, impl) {
263 nir_foreach_instr_safe (instr, block) {
264 nir_variable_mode mode;
265 nir_intrinsic_instr *intr = get_io_intrinsic(instr, modes, &mode);
266 if (!intr)
267 continue;
268
269 nir_alu_type type;
270 if (nir_intrinsic_has_src_type(intr))
271 type = nir_intrinsic_src_type(intr);
272 else
273 type = nir_intrinsic_dest_type(intr);
274 if (!(type & types))
275 continue;
276
277 nir_io_semantics sem = nir_intrinsic_io_semantics(intr);
278
279 if (nir->info.stage == MESA_SHADER_FRAGMENT &&
280 mode == nir_var_shader_out) {
281 /* Only accept FS outputs. */
282 if (sem.location < FRAG_RESULT_DATA0 &&
283 sem.location != FRAG_RESULT_COLOR)
284 continue;
285 } else if (nir->info.stage == MESA_SHADER_VERTEX &&
286 mode == nir_var_shader_in) {
287 /* Accept all VS inputs. */
288 } else {
289 /* Only accept generic varyings. */
290 if (sem.location < VARYING_SLOT_VAR0 ||
291 sem.location > VARYING_SLOT_VAR31)
292 continue;
293 }
294
295 sem.medium_precision = 1;
296 nir_intrinsic_set_io_semantics(intr, sem);
297 changed = true;
298 }
299 }
300
301 if (changed) {
302 nir_metadata_preserve(impl, nir_metadata_dominance |
303 nir_metadata_block_index);
304 } else {
305 nir_metadata_preserve(impl, nir_metadata_all);
306 }
307
308 return changed;
309 }
310
311 /**
312 * Remap 16-bit varying slots to the original 32-bit varying slots.
313 * This only changes IO semantics and bases.
314 */
315 bool
nir_unpack_16bit_varying_slots(nir_shader * nir,nir_variable_mode modes)316 nir_unpack_16bit_varying_slots(nir_shader *nir, nir_variable_mode modes)
317 {
318 bool changed = false;
319 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
320 assert(impl);
321
322 nir_foreach_block_safe (block, impl) {
323 nir_foreach_instr_safe (instr, block) {
324 nir_variable_mode mode;
325 nir_intrinsic_instr *intr = get_io_intrinsic(instr, modes, &mode);
326 if (!intr)
327 continue;
328
329 nir_io_semantics sem = nir_intrinsic_io_semantics(intr);
330
331 if (sem.location < VARYING_SLOT_VAR0_16BIT ||
332 sem.location > VARYING_SLOT_VAR15_16BIT)
333 continue;
334
335 sem.location = VARYING_SLOT_VAR0 +
336 (sem.location - VARYING_SLOT_VAR0_16BIT) * 2 +
337 sem.high_16bits;
338 sem.high_16bits = 0;
339 nir_intrinsic_set_io_semantics(intr, sem);
340 changed = true;
341 }
342 }
343
344 if (changed)
345 nir_recompute_io_bases(impl, modes);
346
347 if (changed) {
348 nir_metadata_preserve(impl, nir_metadata_dominance |
349 nir_metadata_block_index);
350 } else {
351 nir_metadata_preserve(impl, nir_metadata_all);
352 }
353
354 return changed;
355 }
356
357 static bool
is_n_to_m_conversion(nir_instr * instr,unsigned n,nir_op m)358 is_n_to_m_conversion(nir_instr *instr, unsigned n, nir_op m)
359 {
360 if (instr->type != nir_instr_type_alu)
361 return false;
362
363 nir_alu_instr *alu = nir_instr_as_alu(instr);
364 return alu->op == m && alu->src[0].src.ssa->bit_size == n;
365 }
366
367 static bool
is_f16_to_f32_conversion(nir_instr * instr)368 is_f16_to_f32_conversion(nir_instr *instr)
369 {
370 return is_n_to_m_conversion(instr, 16, nir_op_f2f32);
371 }
372
373 static bool
is_f32_to_f16_conversion(nir_instr * instr)374 is_f32_to_f16_conversion(nir_instr *instr)
375 {
376 return is_n_to_m_conversion(instr, 32, nir_op_f2f16) ||
377 is_n_to_m_conversion(instr, 32, nir_op_f2f16_rtne) ||
378 is_n_to_m_conversion(instr, 32, nir_op_f2fmp);
379 }
380
381 static bool
is_i16_to_i32_conversion(nir_instr * instr)382 is_i16_to_i32_conversion(nir_instr *instr)
383 {
384 return is_n_to_m_conversion(instr, 16, nir_op_i2i32);
385 }
386
387 static bool
is_u16_to_u32_conversion(nir_instr * instr)388 is_u16_to_u32_conversion(nir_instr *instr)
389 {
390 return is_n_to_m_conversion(instr, 16, nir_op_u2u32);
391 }
392
393 static bool
is_i32_to_i16_conversion(nir_instr * instr)394 is_i32_to_i16_conversion(nir_instr *instr)
395 {
396 return is_n_to_m_conversion(instr, 32, nir_op_i2i16);
397 }
398
399 static void
replace_with_mov(nir_builder * b,nir_instr * instr,nir_src * src,nir_alu_instr * alu)400 replace_with_mov(nir_builder *b, nir_instr *instr, nir_src *src,
401 nir_alu_instr *alu)
402 {
403 nir_ssa_def *mov = nir_mov_alu(b, alu->src[0],
404 nir_dest_num_components(alu->dest.dest));
405 assert(!alu->dest.saturate);
406 nir_instr_rewrite_src_ssa(instr, src, mov);
407 }
408
409 /**
410 * If texture source operands use f16->f32 conversions or return values are
411 * followed by f16->f32 or f32->f16, remove those conversions. This benefits
412 * drivers that have texture opcodes that can accept and return 16-bit types.
413 *
414 * "tex_src_types" is a mask of nir_tex_src_* operands that should be handled.
415 * It's always done for the destination.
416 *
417 * This should be run after late algebraic optimizations.
418 * Copy propagation and DCE should be run after this.
419 */
420 bool
nir_fold_16bit_sampler_conversions(nir_shader * nir,unsigned tex_src_types)421 nir_fold_16bit_sampler_conversions(nir_shader *nir,
422 unsigned tex_src_types)
423 {
424 bool changed = false;
425 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
426 assert(impl);
427
428 nir_builder b;
429 nir_builder_init(&b, impl);
430
431 nir_foreach_block_safe (block, impl) {
432 nir_foreach_instr_safe (instr, block) {
433 if (instr->type != nir_instr_type_tex)
434 continue;
435
436 nir_tex_instr *tex = nir_instr_as_tex(instr);
437 nir_instr *src;
438 nir_alu_instr *src_alu;
439
440 /* Skip because AMD doesn't support 16-bit types with these. */
441 if ((tex->op == nir_texop_txs ||
442 tex->op == nir_texop_query_levels) ||
443 tex->sampler_dim == GLSL_SAMPLER_DIM_CUBE)
444 continue;
445
446 /* Optimize source operands. */
447 for (unsigned i = 0; i < tex->num_srcs; i++) {
448 /* Filter out sources that should be ignored. */
449 if (!(BITFIELD_BIT(tex->src[i].src_type) & tex_src_types))
450 continue;
451
452 src = tex->src[i].src.ssa->parent_instr;
453 if (src->type != nir_instr_type_alu)
454 continue;
455
456 src_alu = nir_instr_as_alu(src);
457 b.cursor = nir_before_instr(src);
458
459 if (src_alu->op == nir_op_mov) {
460 assert(!"The IR shouldn't contain any movs to make this pass"
461 " effective.");
462 continue;
463 }
464
465 /* Handle vector sources that are made of scalar instructions. */
466 if (nir_op_is_vec(src_alu->op)) {
467 /* See if the vector is made of f16->f32 opcodes. */
468 unsigned num = nir_dest_num_components(src_alu->dest.dest);
469 bool is_f16_to_f32 = true;
470 bool is_u16_to_u32 = true;
471
472 for (unsigned comp = 0; comp < num; comp++) {
473 nir_instr *instr = src_alu->src[comp].src.ssa->parent_instr;
474 is_f16_to_f32 &= is_f16_to_f32_conversion(instr);
475 /* Zero-extension (u16) and sign-extension (i16) have
476 * the same behavior here - txf returns 0 if bit 15 is set
477 * because it's out of bounds and the higher bits don't
478 * matter.
479 */
480 is_u16_to_u32 &= is_u16_to_u32_conversion(instr) ||
481 is_i16_to_i32_conversion(instr);
482 }
483
484 if (!is_f16_to_f32 && !is_u16_to_u32)
485 continue;
486
487 nir_alu_instr *new_vec = nir_alu_instr_clone(nir, src_alu);
488 nir_instr_insert_after(&src_alu->instr, &new_vec->instr);
489
490 /* Replace conversions with mov. */
491 for (unsigned comp = 0; comp < num; comp++) {
492 nir_instr *instr = new_vec->src[comp].src.ssa->parent_instr;
493 replace_with_mov(&b, &new_vec->instr,
494 &new_vec->src[comp].src,
495 nir_instr_as_alu(instr));
496 }
497
498 new_vec->dest.dest.ssa.bit_size =
499 new_vec->src[0].src.ssa->bit_size;
500 nir_instr_rewrite_src_ssa(&tex->instr, &tex->src[i].src,
501 &new_vec->dest.dest.ssa);
502 changed = true;
503 } else if (is_f16_to_f32_conversion(&src_alu->instr) ||
504 is_u16_to_u32_conversion(&src_alu->instr) ||
505 is_i16_to_i32_conversion(&src_alu->instr)) {
506 /* Handle scalar sources. */
507 replace_with_mov(&b, &tex->instr, &tex->src[i].src, src_alu);
508 changed = true;
509 }
510 }
511
512 /* Optimize the destination. */
513 bool is_f16_to_f32 = true;
514 bool is_f32_to_f16 = true;
515 bool is_i16_to_i32 = true;
516 bool is_i32_to_i16 = true; /* same behavior for int and uint */
517 bool is_u16_to_u32 = true;
518
519 nir_foreach_use(use, &tex->dest.ssa) {
520 is_f16_to_f32 &= is_f16_to_f32_conversion(use->parent_instr);
521 is_f32_to_f16 &= is_f32_to_f16_conversion(use->parent_instr);
522 is_i16_to_i32 &= is_i16_to_i32_conversion(use->parent_instr);
523 is_i32_to_i16 &= is_i32_to_i16_conversion(use->parent_instr);
524 is_u16_to_u32 &= is_u16_to_u32_conversion(use->parent_instr);
525 }
526
527 if (is_f16_to_f32 || is_f32_to_f16 || is_i16_to_i32 ||
528 is_i32_to_i16 || is_u16_to_u32) {
529 /* All uses are the same conversions. Replace them with mov. */
530 nir_foreach_use(use, &tex->dest.ssa) {
531 nir_alu_instr *conv = nir_instr_as_alu(use->parent_instr);
532 conv->op = nir_op_mov;
533 tex->dest.ssa.bit_size = conv->dest.dest.ssa.bit_size;
534 tex->dest_type = (tex->dest_type & (~16 & ~32 & ~64)) |
535 conv->dest.dest.ssa.bit_size;
536 }
537 changed = true;
538 }
539 }
540 }
541
542 if (changed) {
543 nir_metadata_preserve(impl, nir_metadata_dominance |
544 nir_metadata_block_index);
545 } else {
546 nir_metadata_preserve(impl, nir_metadata_all);
547 }
548
549 return changed;
550 }
551
552 /**
553 * Fix types of source operands of texture opcodes according to
554 * the constraints by inserting the appropriate conversion opcodes.
555 *
556 * For example, if the type of derivatives must be equal to texture
557 * coordinates and the type of the texture bias must be 32-bit, there
558 * will be 2 constraints describing that.
559 */
560 bool
nir_legalize_16bit_sampler_srcs(nir_shader * nir,nir_tex_src_type_constraints constraints)561 nir_legalize_16bit_sampler_srcs(nir_shader *nir,
562 nir_tex_src_type_constraints constraints)
563 {
564 bool changed = false;
565 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
566 assert(impl);
567
568 nir_builder b;
569 nir_builder_init(&b, impl);
570
571 nir_foreach_block_safe (block, impl) {
572 nir_foreach_instr_safe (instr, block) {
573 if (instr->type != nir_instr_type_tex)
574 continue;
575
576 nir_tex_instr *tex = nir_instr_as_tex(instr);
577 int8_t map[nir_num_tex_src_types];
578 memset(map, -1, sizeof(map));
579
580 /* Create a mapping from src_type to src[i]. */
581 for (unsigned i = 0; i < tex->num_srcs; i++)
582 map[tex->src[i].src_type] = i;
583
584 /* Legalize src types. */
585 for (unsigned i = 0; i < tex->num_srcs; i++) {
586 nir_tex_src_type_constraint c = constraints[tex->src[i].src_type];
587
588 if (!c.legalize_type)
589 continue;
590
591 /* Determine the required bit size for the src. */
592 unsigned bit_size;
593 if (c.bit_size) {
594 bit_size = c.bit_size;
595 } else {
596 if (map[c.match_src] == -1)
597 continue; /* e.g. txs */
598
599 bit_size = tex->src[map[c.match_src]].src.ssa->bit_size;
600 }
601
602 /* Check if the type is legal. */
603 if (bit_size == tex->src[i].src.ssa->bit_size)
604 continue;
605
606 /* Fix the bit size. */
607 bool is_sint = i == nir_tex_src_offset;
608 bool is_uint = !is_sint &&
609 (tex->op == nir_texop_txf ||
610 tex->op == nir_texop_txf_ms ||
611 tex->op == nir_texop_txs ||
612 tex->op == nir_texop_samples_identical);
613 nir_ssa_def *(*convert)(nir_builder *, nir_ssa_def *);
614
615 switch (bit_size) {
616 case 16:
617 convert = is_sint ? nir_i2i16 :
618 is_uint ? nir_u2u16 : nir_f2f16;
619 break;
620 case 32:
621 convert = is_sint ? nir_i2i32 :
622 is_uint ? nir_u2u32 : nir_f2f32;
623 break;
624 default:
625 assert(!"unexpected bit size");
626 continue;
627 }
628
629 b.cursor = nir_before_instr(&tex->instr);
630 nir_ssa_def *conv =
631 convert(&b, nir_ssa_for_src(&b, tex->src[i].src,
632 tex->src[i].src.ssa->num_components));
633 nir_instr_rewrite_src_ssa(&tex->instr, &tex->src[i].src, conv);
634 changed = true;
635 }
636 }
637 }
638
639 if (changed) {
640 nir_metadata_preserve(impl, nir_metadata_dominance |
641 nir_metadata_block_index);
642 } else {
643 nir_metadata_preserve(impl, nir_metadata_all);
644 }
645
646 return changed;
647 }
648