1 /*
2 * Copyright © 2014-2015 Broadcom
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "nir.h"
25 #include "nir_builder.h"
26
27 struct alu_width_data {
28 nir_vectorize_cb cb;
29 const void *data;
30 };
31
32 /** @file nir_lower_alu_width.c
33 *
34 * Replaces nir_alu_instr operations with more than one channel used in the
35 * arguments with individual per-channel operations.
36 *
37 * Optionally, a callback function which returns the max vectorization width
38 * per instruction can be provided.
39 *
40 * The max vectorization width must be a power of 2.
41 */
42
43 static bool
inst_is_vector_alu(const nir_instr * instr,const void * _state)44 inst_is_vector_alu(const nir_instr *instr, const void *_state)
45 {
46 if (instr->type != nir_instr_type_alu)
47 return false;
48
49 nir_alu_instr *alu = nir_instr_as_alu(instr);
50
51 /* There is no ALU instruction which has a scalar destination, scalar
52 * src[0], and some other vector source.
53 */
54 assert(alu->dest.dest.is_ssa);
55 assert(alu->src[0].src.is_ssa);
56 return alu->dest.dest.ssa.num_components > 1 ||
57 nir_op_infos[alu->op].input_sizes[0] > 1;
58 }
59
60 /* Checks whether all operands of an ALU instruction are swizzled
61 * within the targeted vectorization width.
62 *
63 * The assumption here is that a vecN instruction can only swizzle
64 * within the first N channels of the values it consumes, irrespective
65 * of the capabilities of the instruction which produced those values.
66 * If we assume values are packed consistently (i.e., they always start
67 * at the beginning of a hardware register), we can actually access any
68 * aligned group of N channels so long as we stay within the group.
69 * This means for a vectorization width of 4 that only swizzles from
70 * either [xyzw] or [abcd] etc are allowed. For a width of 2 these are
71 * swizzles from either [xy] or [zw] etc.
72 */
73 static bool
alu_is_swizzled_in_bounds(const nir_alu_instr * alu,unsigned width)74 alu_is_swizzled_in_bounds(const nir_alu_instr *alu, unsigned width)
75 {
76 for (unsigned i = 0; i < nir_op_infos[alu->op].num_inputs; i++) {
77 if (nir_op_infos[alu->op].input_sizes[i] == 1)
78 continue;
79
80 unsigned mask = ~(width - 1);
81 for (unsigned j = 1; j < alu->dest.dest.ssa.num_components; j++) {
82 if ((alu->src[i].swizzle[0] & mask) != (alu->src[i].swizzle[j] & mask))
83 return false;
84 }
85 }
86
87 return true;
88 }
89
90 static void
nir_alu_ssa_dest_init(nir_alu_instr * alu,unsigned num_components,unsigned bit_size)91 nir_alu_ssa_dest_init(nir_alu_instr *alu, unsigned num_components,
92 unsigned bit_size)
93 {
94 nir_ssa_dest_init(&alu->instr, &alu->dest.dest, num_components,
95 bit_size, NULL);
96 alu->dest.write_mask = (1 << num_components) - 1;
97 }
98
99 static nir_ssa_def *
lower_reduction(nir_alu_instr * alu,nir_op chan_op,nir_op merge_op,nir_builder * builder)100 lower_reduction(nir_alu_instr *alu, nir_op chan_op, nir_op merge_op,
101 nir_builder *builder)
102 {
103 unsigned num_components = nir_op_infos[alu->op].input_sizes[0];
104
105 nir_ssa_def *last = NULL;
106 for (int i = num_components - 1; i >= 0; i--) {
107 nir_alu_instr *chan = nir_alu_instr_create(builder->shader, chan_op);
108 nir_alu_ssa_dest_init(chan, 1, alu->dest.dest.ssa.bit_size);
109 nir_alu_src_copy(&chan->src[0], &alu->src[0]);
110 chan->src[0].swizzle[0] = chan->src[0].swizzle[i];
111 if (nir_op_infos[chan_op].num_inputs > 1) {
112 assert(nir_op_infos[chan_op].num_inputs == 2);
113 nir_alu_src_copy(&chan->src[1], &alu->src[1]);
114 chan->src[1].swizzle[0] = chan->src[1].swizzle[i];
115 }
116 chan->exact = alu->exact;
117
118 nir_builder_instr_insert(builder, &chan->instr);
119
120 if (i == num_components - 1) {
121 last = &chan->dest.dest.ssa;
122 } else {
123 last = nir_build_alu(builder, merge_op,
124 last, &chan->dest.dest.ssa, NULL, NULL);
125 }
126 }
127
128 return last;
129 }
130
131 static inline bool
will_lower_ffma(nir_shader * shader,unsigned bit_size)132 will_lower_ffma(nir_shader *shader, unsigned bit_size)
133 {
134 switch (bit_size) {
135 case 16:
136 return shader->options->lower_ffma16;
137 case 32:
138 return shader->options->lower_ffma32;
139 case 64:
140 return shader->options->lower_ffma64;
141 }
142 unreachable("bad bit size");
143 }
144
145 static nir_ssa_def *
lower_fdot(nir_alu_instr * alu,nir_builder * builder)146 lower_fdot(nir_alu_instr *alu, nir_builder *builder)
147 {
148 /* If we don't want to lower ffma, create several ffma instead of fmul+fadd
149 * and fusing later because fusing is not possible for exact fdot instructions.
150 */
151 if (will_lower_ffma(builder->shader, alu->dest.dest.ssa.bit_size))
152 return lower_reduction(alu, nir_op_fmul, nir_op_fadd, builder);
153
154 unsigned num_components = nir_op_infos[alu->op].input_sizes[0];
155
156 nir_ssa_def *prev = NULL;
157 for (int i = num_components - 1; i >= 0; i--) {
158 nir_alu_instr *instr = nir_alu_instr_create(
159 builder->shader, prev ? nir_op_ffma : nir_op_fmul);
160 nir_alu_ssa_dest_init(instr, 1, alu->dest.dest.ssa.bit_size);
161 for (unsigned j = 0; j < 2; j++) {
162 nir_alu_src_copy(&instr->src[j], &alu->src[j]);
163 instr->src[j].swizzle[0] = alu->src[j].swizzle[i];
164 }
165 if (i != num_components - 1)
166 instr->src[2].src = nir_src_for_ssa(prev);
167 instr->exact = builder->exact;
168
169 nir_builder_instr_insert(builder, &instr->instr);
170
171 prev = &instr->dest.dest.ssa;
172 }
173
174 return prev;
175 }
176
177 static nir_ssa_def *
lower_alu_instr_width(nir_builder * b,nir_instr * instr,void * _data)178 lower_alu_instr_width(nir_builder *b, nir_instr *instr, void *_data)
179 {
180 struct alu_width_data *data = _data;
181 nir_alu_instr *alu = nir_instr_as_alu(instr);
182 unsigned num_src = nir_op_infos[alu->op].num_inputs;
183 unsigned i, chan;
184
185 assert(alu->dest.dest.is_ssa);
186 assert(alu->dest.write_mask != 0);
187
188 b->exact = alu->exact;
189
190 unsigned num_components = alu->dest.dest.ssa.num_components;
191 unsigned target_width = 1;
192
193 if (data->cb) {
194 target_width = data->cb(instr, data->data);
195 assert(util_is_power_of_two_or_zero(target_width));
196 if (target_width == 0)
197 return NULL;
198 }
199
200 #define LOWER_REDUCTION(name, chan, merge) \
201 case name##2: \
202 case name##3: \
203 case name##4: \
204 case name##8: \
205 case name##16: \
206 return lower_reduction(alu, chan, merge, b); \
207
208 switch (alu->op) {
209 case nir_op_vec16:
210 case nir_op_vec8:
211 case nir_op_vec5:
212 case nir_op_vec4:
213 case nir_op_vec3:
214 case nir_op_vec2:
215 case nir_op_cube_face_coord_amd:
216 case nir_op_cube_face_index_amd:
217 /* We don't need to scalarize these ops, they're the ones generated to
218 * group up outputs into a value that can be SSAed.
219 */
220 return NULL;
221
222 case nir_op_pack_half_2x16: {
223 if (!b->shader->options->lower_pack_half_2x16)
224 return NULL;
225
226 nir_ssa_def *src_vec2 = nir_ssa_for_alu_src(b, alu, 0);
227 return nir_pack_half_2x16_split(b, nir_channel(b, src_vec2, 0),
228 nir_channel(b, src_vec2, 1));
229 }
230
231 case nir_op_unpack_unorm_4x8:
232 case nir_op_unpack_snorm_4x8:
233 case nir_op_unpack_unorm_2x16:
234 case nir_op_unpack_snorm_2x16:
235 /* There is no scalar version of these ops, unless we were to break it
236 * down to bitshifts and math (which is definitely not intended).
237 */
238 return NULL;
239
240 case nir_op_unpack_half_2x16_flush_to_zero:
241 case nir_op_unpack_half_2x16: {
242 if (!b->shader->options->lower_unpack_half_2x16)
243 return NULL;
244
245 nir_ssa_def *packed = nir_ssa_for_alu_src(b, alu, 0);
246 if (alu->op == nir_op_unpack_half_2x16_flush_to_zero) {
247 return nir_vec2(b,
248 nir_unpack_half_2x16_split_x_flush_to_zero(b,
249 packed),
250 nir_unpack_half_2x16_split_y_flush_to_zero(b,
251 packed));
252 } else {
253 return nir_vec2(b,
254 nir_unpack_half_2x16_split_x(b, packed),
255 nir_unpack_half_2x16_split_y(b, packed));
256 }
257 }
258
259 case nir_op_pack_uvec2_to_uint: {
260 assert(b->shader->options->lower_pack_snorm_2x16 ||
261 b->shader->options->lower_pack_unorm_2x16);
262
263 nir_ssa_def *word = nir_extract_u16(b, nir_ssa_for_alu_src(b, alu, 0),
264 nir_imm_int(b, 0));
265 return nir_ior(b, nir_ishl(b, nir_channel(b, word, 1),
266 nir_imm_int(b, 16)),
267 nir_channel(b, word, 0));
268 }
269
270 case nir_op_pack_uvec4_to_uint: {
271 assert(b->shader->options->lower_pack_snorm_4x8 ||
272 b->shader->options->lower_pack_unorm_4x8);
273
274 nir_ssa_def *byte = nir_extract_u8(b, nir_ssa_for_alu_src(b, alu, 0),
275 nir_imm_int(b, 0));
276 return nir_ior(b, nir_ior(b, nir_ishl(b, nir_channel(b, byte, 3),
277 nir_imm_int(b, 24)),
278 nir_ishl(b, nir_channel(b, byte, 2),
279 nir_imm_int(b, 16))),
280 nir_ior(b, nir_ishl(b, nir_channel(b, byte, 1),
281 nir_imm_int(b, 8)),
282 nir_channel(b, byte, 0)));
283 }
284
285 case nir_op_fdph: {
286 nir_ssa_def *src0_vec = nir_ssa_for_alu_src(b, alu, 0);
287 nir_ssa_def *src1_vec = nir_ssa_for_alu_src(b, alu, 1);
288
289 nir_ssa_def *sum[4];
290 for (unsigned i = 0; i < 3; i++) {
291 sum[i] = nir_fmul(b, nir_channel(b, src0_vec, i),
292 nir_channel(b, src1_vec, i));
293 }
294 sum[3] = nir_channel(b, src1_vec, 3);
295
296 return nir_fadd(b, nir_fadd(b, sum[0], sum[1]),
297 nir_fadd(b, sum[2], sum[3]));
298 }
299
300 case nir_op_pack_64_2x32: {
301 if (!b->shader->options->lower_pack_64_2x32)
302 return NULL;
303
304 nir_ssa_def *src_vec2 = nir_ssa_for_alu_src(b, alu, 0);
305 return nir_pack_64_2x32_split(b, nir_channel(b, src_vec2, 0),
306 nir_channel(b, src_vec2, 1));
307 }
308 case nir_op_pack_64_4x16: {
309 if (!b->shader->options->lower_pack_64_4x16)
310 return NULL;
311
312 nir_ssa_def *src_vec4 = nir_ssa_for_alu_src(b, alu, 0);
313 nir_ssa_def *xy = nir_pack_32_2x16_split(b, nir_channel(b, src_vec4, 0),
314 nir_channel(b, src_vec4, 1));
315 nir_ssa_def *zw = nir_pack_32_2x16_split(b, nir_channel(b, src_vec4, 2),
316 nir_channel(b, src_vec4, 3));
317
318 return nir_pack_64_2x32_split(b, xy, zw);
319 }
320 case nir_op_pack_32_2x16: {
321 if (!b->shader->options->lower_pack_32_2x16)
322 return NULL;
323
324 nir_ssa_def *src_vec2 = nir_ssa_for_alu_src(b, alu, 0);
325 return nir_pack_32_2x16_split(b, nir_channel(b, src_vec2, 0),
326 nir_channel(b, src_vec2, 1));
327 }
328 case nir_op_unpack_64_2x32:
329 case nir_op_unpack_64_4x16:
330 case nir_op_unpack_32_2x16:
331 case nir_op_unpack_double_2x32_dxil:
332 return NULL;
333
334 case nir_op_fdot2:
335 case nir_op_fdot3:
336 case nir_op_fdot4:
337 case nir_op_fdot8:
338 case nir_op_fdot16:
339 return lower_fdot(alu, b);
340
341 LOWER_REDUCTION(nir_op_ball_fequal, nir_op_feq, nir_op_iand);
342 LOWER_REDUCTION(nir_op_ball_iequal, nir_op_ieq, nir_op_iand);
343 LOWER_REDUCTION(nir_op_bany_fnequal, nir_op_fneu, nir_op_ior);
344 LOWER_REDUCTION(nir_op_bany_inequal, nir_op_ine, nir_op_ior);
345 LOWER_REDUCTION(nir_op_b8all_fequal, nir_op_feq8, nir_op_iand);
346 LOWER_REDUCTION(nir_op_b8all_iequal, nir_op_ieq8, nir_op_iand);
347 LOWER_REDUCTION(nir_op_b8any_fnequal, nir_op_fneu8, nir_op_ior);
348 LOWER_REDUCTION(nir_op_b8any_inequal, nir_op_ine8, nir_op_ior);
349 LOWER_REDUCTION(nir_op_b16all_fequal, nir_op_feq16, nir_op_iand);
350 LOWER_REDUCTION(nir_op_b16all_iequal, nir_op_ieq16, nir_op_iand);
351 LOWER_REDUCTION(nir_op_b16any_fnequal, nir_op_fneu16, nir_op_ior);
352 LOWER_REDUCTION(nir_op_b16any_inequal, nir_op_ine16, nir_op_ior);
353 LOWER_REDUCTION(nir_op_b32all_fequal, nir_op_feq32, nir_op_iand);
354 LOWER_REDUCTION(nir_op_b32all_iequal, nir_op_ieq32, nir_op_iand);
355 LOWER_REDUCTION(nir_op_b32any_fnequal, nir_op_fneu32, nir_op_ior);
356 LOWER_REDUCTION(nir_op_b32any_inequal, nir_op_ine32, nir_op_ior);
357 LOWER_REDUCTION(nir_op_fall_equal, nir_op_seq, nir_op_fmin);
358 LOWER_REDUCTION(nir_op_fany_nequal, nir_op_sne, nir_op_fmax);
359
360 default:
361 break;
362 }
363
364 if (num_components == 1)
365 return NULL;
366
367 if (num_components <= target_width) {
368 /* If the ALU instr is swizzled outside the target width,
369 * reduce the target width.
370 */
371 if (alu_is_swizzled_in_bounds(alu, target_width))
372 return NULL;
373 else
374 target_width = DIV_ROUND_UP(num_components, 2);
375 }
376
377 nir_alu_instr *vec = nir_alu_instr_create(b->shader, nir_op_vec(num_components));
378
379 for (chan = 0; chan < num_components; chan += target_width) {
380 unsigned components = MIN2(target_width, num_components - chan);
381 nir_alu_instr *lower = nir_alu_instr_create(b->shader, alu->op);
382
383 for (i = 0; i < num_src; i++) {
384 nir_alu_src_copy(&lower->src[i], &alu->src[i]);
385
386 /* We only handle same-size-as-dest (input_sizes[] == 0) or scalar
387 * args (input_sizes[] == 1).
388 */
389 assert(nir_op_infos[alu->op].input_sizes[i] < 2);
390 for (int j = 0; j < components; j++) {
391 unsigned src_chan = nir_op_infos[alu->op].input_sizes[i] == 1 ? 0 : chan + j;
392 lower->src[i].swizzle[j] = alu->src[i].swizzle[src_chan];
393 }
394 }
395
396 nir_alu_ssa_dest_init(lower, components, alu->dest.dest.ssa.bit_size);
397 lower->dest.saturate = alu->dest.saturate;
398 lower->exact = alu->exact;
399
400 for (i = 0; i < components; i++) {
401 vec->src[chan + i].src = nir_src_for_ssa(&lower->dest.dest.ssa);
402 vec->src[chan + i].swizzle[0] = i;
403 }
404
405 nir_builder_instr_insert(b, &lower->instr);
406 }
407
408 return nir_builder_alu_instr_finish_and_insert(b, vec);
409 }
410
411 bool
nir_lower_alu_width(nir_shader * shader,nir_vectorize_cb cb,const void * _data)412 nir_lower_alu_width(nir_shader *shader, nir_vectorize_cb cb, const void *_data)
413 {
414 struct alu_width_data data = {
415 .cb = cb,
416 .data = _data,
417 };
418
419 return nir_shader_lower_instructions(shader,
420 inst_is_vector_alu,
421 lower_alu_instr_width,
422 &data);
423 }
424
425 struct alu_to_scalar_data {
426 nir_instr_filter_cb cb;
427 const void *data;
428 };
429
430 static uint8_t
scalar_cb(const nir_instr * instr,const void * data)431 scalar_cb(const nir_instr *instr, const void *data)
432 {
433 /* return vectorization-width = 1 for filtered instructions */
434 const struct alu_to_scalar_data *filter = data;
435 return filter->cb(instr, filter->data) ? 1 : 0;
436 }
437
438 bool
nir_lower_alu_to_scalar(nir_shader * shader,nir_instr_filter_cb cb,const void * _data)439 nir_lower_alu_to_scalar(nir_shader *shader, nir_instr_filter_cb cb, const void *_data)
440 {
441 struct alu_to_scalar_data data = {
442 .cb = cb,
443 .data = _data,
444 };
445
446 return nir_lower_alu_width(shader, cb ? scalar_cb : NULL, &data);
447 }
448
449