1 /*
2 * Copyright © 2018 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "nir.h"
25 #include "nir_builder.h"
26
27 static bool
assert_ssa_def_is_not_1bit(nir_ssa_def * def,UNUSED void * unused)28 assert_ssa_def_is_not_1bit(nir_ssa_def *def, UNUSED void *unused)
29 {
30 assert(def->bit_size > 1);
31 return true;
32 }
33
34 static bool
rewrite_1bit_ssa_def_to_32bit(nir_ssa_def * def,void * _progress)35 rewrite_1bit_ssa_def_to_32bit(nir_ssa_def *def, void *_progress)
36 {
37 bool *progress = _progress;
38 if (def->bit_size == 1) {
39 def->bit_size = 32;
40 *progress = true;
41 }
42 return true;
43 }
44
45 static uint32_t
get_bool_convert_opcode(uint32_t dst_bit_size)46 get_bool_convert_opcode(uint32_t dst_bit_size)
47 {
48 switch (dst_bit_size) {
49 case 32: return nir_op_i2i32;
50 case 16: return nir_op_i2i16;
51 case 8: return nir_op_i2i8;
52 default:
53 unreachable("invalid boolean bit-size");
54 }
55 }
56
57 static void
make_sources_canonical(nir_builder * b,nir_alu_instr * alu,uint32_t start_idx)58 make_sources_canonical(nir_builder *b, nir_alu_instr *alu, uint32_t start_idx)
59 {
60 /* TODO: for now we take the bit-size of the first source as the canonical
61 * form but we could try to be smarter.
62 */
63 const nir_op_info *op_info = &nir_op_infos[alu->op];
64 uint32_t bit_size = nir_src_bit_size(alu->src[start_idx].src);
65 for (uint32_t i = start_idx + 1; i < op_info->num_inputs; i++) {
66 if (nir_src_bit_size(alu->src[i].src) != bit_size) {
67 b->cursor = nir_before_instr(&alu->instr);
68 nir_op convert_op = get_bool_convert_opcode(bit_size);
69 nir_ssa_def *new_src =
70 nir_build_alu(b, convert_op, alu->src[i].src.ssa, NULL, NULL, NULL);
71 /* Retain the write mask and swizzle of the original instruction so
72 * that we don’t unnecessarily create a vectorized instruction.
73 */
74 nir_alu_instr *conv_instr =
75 nir_instr_as_alu(nir_builder_last_instr(b));
76 conv_instr->dest.write_mask = alu->dest.write_mask;
77 conv_instr->dest.dest.ssa.num_components =
78 alu->dest.dest.ssa.num_components;
79 memcpy(conv_instr->src[0].swizzle,
80 alu->src[i].swizzle,
81 sizeof(conv_instr->src[0].swizzle));
82 nir_instr_rewrite_src(&alu->instr,
83 &alu->src[i].src, nir_src_for_ssa(new_src));
84 /* The swizzle will have been handled by the conversion instruction
85 * so we can reset it back to the default
86 */
87 for (unsigned j = 0; j < NIR_MAX_VEC_COMPONENTS; j++)
88 alu->src[i].swizzle[j] = j;
89 }
90 }
91 }
92
93 static bool
lower_alu_instr(nir_builder * b,nir_alu_instr * alu)94 lower_alu_instr(nir_builder *b, nir_alu_instr *alu)
95 {
96 const nir_op_info *op_info = &nir_op_infos[alu->op];
97
98 /* For operations that can take multiple boolean sources we need to ensure
99 * that all booleans have the same bit-size
100 */
101 switch (alu->op) {
102 case nir_op_mov:
103 case nir_op_vec2:
104 case nir_op_vec3:
105 case nir_op_vec4:
106 case nir_op_vec5:
107 case nir_op_vec8:
108 case nir_op_vec16:
109 case nir_op_inot:
110 case nir_op_iand:
111 case nir_op_ior:
112 case nir_op_ixor:
113 if (nir_dest_bit_size(alu->dest.dest) > 1)
114 return false; /* Not a boolean instruction */
115 FALLTHROUGH;
116
117 case nir_op_ball_fequal2:
118 case nir_op_ball_fequal3:
119 case nir_op_ball_fequal4:
120 case nir_op_bany_fnequal2:
121 case nir_op_bany_fnequal3:
122 case nir_op_bany_fnequal4:
123 case nir_op_ball_iequal2:
124 case nir_op_ball_iequal3:
125 case nir_op_ball_iequal4:
126 case nir_op_bany_inequal2:
127 case nir_op_bany_inequal3:
128 case nir_op_bany_inequal4:
129 case nir_op_ieq:
130 case nir_op_ine:
131 make_sources_canonical(b, alu, 0);
132 break;
133
134 case nir_op_bcsel:
135 /* bcsel may be choosing between boolean sources too */
136 if (nir_dest_bit_size(alu->dest.dest) == 1)
137 make_sources_canonical(b, alu, 1);
138 break;
139
140 default:
141 break;
142 }
143
144 /* Now that we have a canonical boolean bit-size, go on and rewrite the
145 * instruction to match the canonical bit-size.
146 */
147 uint32_t bit_size = nir_src_bit_size(alu->src[0].src);
148 assert(bit_size > 1);
149
150 nir_op opcode = alu->op;
151 switch (opcode) {
152 case nir_op_mov:
153 case nir_op_vec2:
154 case nir_op_vec3:
155 case nir_op_vec4:
156 case nir_op_vec5:
157 case nir_op_vec8:
158 case nir_op_vec16:
159 case nir_op_inot:
160 case nir_op_iand:
161 case nir_op_ior:
162 case nir_op_ixor:
163 /* Nothing to do here, we do not specialize these opcodes by bit-size */
164 break;
165
166 case nir_op_f2b1:
167 opcode = bit_size == 8 ? nir_op_f2b8 :
168 bit_size == 16 ? nir_op_f2b16 : nir_op_f2b32;
169 break;
170
171 case nir_op_i2b1:
172 opcode = bit_size == 8 ? nir_op_i2b8 :
173 bit_size == 16 ? nir_op_i2b16 : nir_op_i2b32;
174 break;
175
176 case nir_op_b2b1:
177 /* Since the canonical bit size is the size of the src, it's a no-op */
178 opcode = nir_op_mov;
179 break;
180
181 case nir_op_b2b32:
182 /* For up-converting booleans, sign-extend */
183 opcode = nir_op_i2i32;
184 break;
185
186 case nir_op_flt:
187 opcode = bit_size == 8 ? nir_op_flt8 :
188 bit_size == 16 ? nir_op_flt16 : nir_op_flt32;
189 break;
190
191 case nir_op_fge:
192 opcode = bit_size == 8 ? nir_op_fge8 :
193 bit_size == 16 ? nir_op_fge16 : nir_op_fge32;
194 break;
195
196 case nir_op_feq:
197 opcode = bit_size == 8 ? nir_op_feq8 :
198 bit_size == 16 ? nir_op_feq16 : nir_op_feq32;
199 break;
200
201 case nir_op_fneu:
202 opcode = bit_size == 8 ? nir_op_fneu8 :
203 bit_size == 16 ? nir_op_fneu16 : nir_op_fneu32;
204 break;
205
206 case nir_op_ilt:
207 opcode = bit_size == 8 ? nir_op_ilt8 :
208 bit_size == 16 ? nir_op_ilt16 : nir_op_ilt32;
209 break;
210
211 case nir_op_ige:
212 opcode = bit_size == 8 ? nir_op_ige8 :
213 bit_size == 16 ? nir_op_ige16 : nir_op_ige32;
214 break;
215
216 case nir_op_ieq:
217 opcode = bit_size == 8 ? nir_op_ieq8 :
218 bit_size == 16 ? nir_op_ieq16 : nir_op_ieq32;
219 break;
220
221 case nir_op_ine:
222 opcode = bit_size == 8 ? nir_op_ine8 :
223 bit_size == 16 ? nir_op_ine16 : nir_op_ine32;
224 break;
225
226 case nir_op_ult:
227 opcode = bit_size == 8 ? nir_op_ult8 :
228 bit_size == 16 ? nir_op_ult16 : nir_op_ult32;
229 break;
230
231 case nir_op_uge:
232 opcode = bit_size == 8 ? nir_op_uge8 :
233 bit_size == 16 ? nir_op_uge16 : nir_op_uge32;
234 break;
235
236 case nir_op_ball_fequal2:
237 opcode = bit_size == 8 ? nir_op_b8all_fequal2 :
238 bit_size == 16 ? nir_op_b16all_fequal2 :
239 nir_op_b32all_fequal2;
240 break;
241
242 case nir_op_ball_fequal3:
243 opcode = bit_size == 8 ? nir_op_b8all_fequal3 :
244 bit_size == 16 ? nir_op_b16all_fequal3 :
245 nir_op_b32all_fequal3;
246 break;
247
248 case nir_op_ball_fequal4:
249 opcode = bit_size == 8 ? nir_op_b8all_fequal4 :
250 bit_size == 16 ? nir_op_b16all_fequal4 :
251 nir_op_b32all_fequal4;
252 break;
253
254 case nir_op_bany_fnequal2:
255 opcode = bit_size == 8 ? nir_op_b8any_fnequal2 :
256 bit_size == 16 ? nir_op_b16any_fnequal2 :
257 nir_op_b32any_fnequal2;
258 break;
259
260 case nir_op_bany_fnequal3:
261 opcode = bit_size == 8 ? nir_op_b8any_fnequal3 :
262 bit_size == 16 ? nir_op_b16any_fnequal3 :
263 nir_op_b32any_fnequal3;
264 break;
265
266 case nir_op_bany_fnequal4:
267 opcode = bit_size == 8 ? nir_op_b8any_fnequal4 :
268 bit_size == 16 ? nir_op_b16any_fnequal4 :
269 nir_op_b32any_fnequal4;
270 break;
271
272 case nir_op_ball_iequal2:
273 opcode = bit_size == 8 ? nir_op_b8all_iequal2 :
274 bit_size == 16 ? nir_op_b16all_iequal2 :
275 nir_op_b32all_iequal2;
276 break;
277
278 case nir_op_ball_iequal3:
279 opcode = bit_size == 8 ? nir_op_b8all_iequal3 :
280 bit_size == 16 ? nir_op_b16all_iequal3 :
281 nir_op_b32all_iequal3;
282 break;
283
284 case nir_op_ball_iequal4:
285 opcode = bit_size == 8 ? nir_op_b8all_iequal4 :
286 bit_size == 16 ? nir_op_b16all_iequal4 :
287 nir_op_b32all_iequal4;
288 break;
289
290 case nir_op_bany_inequal2:
291 opcode = bit_size == 8 ? nir_op_b8any_inequal2 :
292 bit_size == 16 ? nir_op_b16any_inequal2 :
293 nir_op_b32any_inequal2;
294 break;
295
296 case nir_op_bany_inequal3:
297 opcode = bit_size == 8 ? nir_op_b8any_inequal3 :
298 bit_size == 16 ? nir_op_b16any_inequal3 :
299 nir_op_b32any_inequal3;
300 break;
301
302 case nir_op_bany_inequal4:
303 opcode = bit_size == 8 ? nir_op_b8any_inequal4 :
304 bit_size == 16 ? nir_op_b16any_inequal4 :
305 nir_op_b32any_inequal4;
306 break;
307
308 case nir_op_bcsel:
309 opcode = bit_size == 8 ? nir_op_b8csel :
310 bit_size == 16 ? nir_op_b16csel : nir_op_b32csel;
311
312 /* The destination of the selection may have a different bit-size from
313 * the bcsel condition.
314 */
315 bit_size = nir_src_bit_size(alu->src[1].src);
316 break;
317
318 default:
319 assert(alu->dest.dest.ssa.bit_size > 1);
320 for (unsigned i = 0; i < op_info->num_inputs; i++)
321 assert(alu->src[i].src.ssa->bit_size > 1);
322 return false;
323 }
324
325 alu->op = opcode;
326
327 if (alu->dest.dest.ssa.bit_size == 1)
328 alu->dest.dest.ssa.bit_size = bit_size;
329
330 return true;
331 }
332
333 static bool
lower_load_const_instr(nir_load_const_instr * load)334 lower_load_const_instr(nir_load_const_instr *load)
335 {
336 bool progress = false;
337
338 if (load->def.bit_size > 1)
339 return progress;
340
341 /* TODO: It is not clear if there is any case in which we can ever hit
342 * this path, so for now we just provide a 32-bit default.
343 *
344 * TODO2: after some changed on nir_const_value and other on upstream, we
345 * removed the initialization of a general value like this:
346 * nir_const_value value = load->value
347 *
348 * to initialize per value component. Need to confirm if that is correct,
349 * but look at the TOO before.
350 */
351 for (unsigned i = 0; i < load->def.num_components; i++) {
352 load->value[i].u32 = load->value[i].b ? NIR_TRUE : NIR_FALSE;
353 load->def.bit_size = 32;
354 progress = true;
355 }
356
357 return progress;
358 }
359
360 static bool
lower_phi_instr(nir_builder * b,nir_phi_instr * phi)361 lower_phi_instr(nir_builder *b, nir_phi_instr *phi)
362 {
363 if (nir_dest_bit_size(phi->dest) != 1)
364 return false;
365
366 /* Ensure all phi sources have a canonical bit-size. We choose the
367 * bit-size of the first phi source as the canonical form.
368 *
369 * TODO: maybe we can be smarter about how we choose the canonical form.
370 */
371 uint32_t dst_bit_size = 0;
372 nir_foreach_phi_src(phi_src, phi) {
373 uint32_t src_bit_size = nir_src_bit_size(phi_src->src);
374 if (dst_bit_size == 0) {
375 dst_bit_size = src_bit_size;
376 } else if (src_bit_size != dst_bit_size) {
377 assert(phi_src->src.is_ssa);
378 b->cursor = nir_before_src(&phi_src->src, false);
379 nir_op convert_op = get_bool_convert_opcode(dst_bit_size);
380 nir_ssa_def *new_src =
381 nir_build_alu(b, convert_op, phi_src->src.ssa, NULL, NULL, NULL);
382 nir_instr_rewrite_src(&phi->instr, &phi_src->src,
383 nir_src_for_ssa(new_src));
384 }
385 }
386
387 phi->dest.ssa.bit_size = dst_bit_size;
388
389 return true;
390 }
391
392 static bool
lower_tex_instr(nir_tex_instr * tex)393 lower_tex_instr(nir_tex_instr *tex)
394 {
395 bool progress = false;
396 rewrite_1bit_ssa_def_to_32bit(&tex->dest.ssa, &progress);
397 if (tex->dest_type == nir_type_bool1) {
398 tex->dest_type = nir_type_bool32;
399 progress = true;
400 }
401 return progress;
402 }
403
404 static bool
nir_lower_bool_to_bitsize_impl(nir_builder * b,nir_function_impl * impl)405 nir_lower_bool_to_bitsize_impl(nir_builder *b, nir_function_impl *impl)
406 {
407 bool progress = false;
408
409 nir_foreach_block(block, impl) {
410 nir_foreach_instr_safe(instr, block) {
411 switch (instr->type) {
412 case nir_instr_type_alu:
413 progress |= lower_alu_instr(b, nir_instr_as_alu(instr));
414 break;
415
416 case nir_instr_type_load_const:
417 progress |= lower_load_const_instr(nir_instr_as_load_const(instr));
418 break;
419
420 case nir_instr_type_phi:
421 progress |= lower_phi_instr(b, nir_instr_as_phi(instr));
422 break;
423
424 case nir_instr_type_ssa_undef:
425 case nir_instr_type_intrinsic:
426 nir_foreach_ssa_def(instr, rewrite_1bit_ssa_def_to_32bit,
427 &progress);
428 break;
429
430 case nir_instr_type_tex:
431 progress |= lower_tex_instr(nir_instr_as_tex(instr));
432 break;
433
434 default:
435 nir_foreach_ssa_def(instr, assert_ssa_def_is_not_1bit, NULL);
436 }
437 }
438 }
439
440 if (progress) {
441 nir_metadata_preserve(impl, nir_metadata_block_index |
442 nir_metadata_dominance);
443 }
444
445 return progress;
446 }
447
448 bool
nir_lower_bool_to_bitsize(nir_shader * shader)449 nir_lower_bool_to_bitsize(nir_shader *shader)
450 {
451 nir_builder b;
452 bool progress = false;
453
454 nir_foreach_function(function, shader) {
455 if (function->impl) {
456 nir_builder_init(&b, function->impl);
457 progress = nir_lower_bool_to_bitsize_impl(&b, function->impl) || progress;
458 }
459 }
460
461 return progress;
462 }
463