1 /*
2 * Copyright © 2014-2015 Broadcom
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "nir.h"
25 #include "nir_builder.h"
26
27 struct alu_to_scalar_data {
28 nir_instr_filter_cb cb;
29 const void *data;
30 };
31
32 /** @file nir_lower_alu_to_scalar.c
33 *
34 * Replaces nir_alu_instr operations with more than one channel used in the
35 * arguments with individual per-channel operations.
36 */
37
38 static bool
inst_is_vector_alu(const nir_instr * instr,const void * _state)39 inst_is_vector_alu(const nir_instr *instr, const void *_state)
40 {
41 if (instr->type != nir_instr_type_alu)
42 return false;
43
44 nir_alu_instr *alu = nir_instr_as_alu(instr);
45
46 /* There is no ALU instruction which has a scalar destination, scalar
47 * src[0], and some other vector source.
48 */
49 assert(alu->dest.dest.is_ssa);
50 assert(alu->src[0].src.is_ssa);
51 return alu->dest.dest.ssa.num_components > 1 ||
52 nir_op_infos[alu->op].input_sizes[0] > 1;
53 }
54
55 static void
nir_alu_ssa_dest_init(nir_alu_instr * alu,unsigned num_components,unsigned bit_size)56 nir_alu_ssa_dest_init(nir_alu_instr *alu, unsigned num_components,
57 unsigned bit_size)
58 {
59 nir_ssa_dest_init(&alu->instr, &alu->dest.dest, num_components,
60 bit_size, NULL);
61 alu->dest.write_mask = (1 << num_components) - 1;
62 }
63
64 static nir_ssa_def *
lower_reduction(nir_alu_instr * alu,nir_op chan_op,nir_op merge_op,nir_builder * builder)65 lower_reduction(nir_alu_instr *alu, nir_op chan_op, nir_op merge_op,
66 nir_builder *builder)
67 {
68 unsigned num_components = nir_op_infos[alu->op].input_sizes[0];
69
70 nir_ssa_def *last = NULL;
71 for (int i = num_components - 1; i >= 0; i--) {
72 nir_alu_instr *chan = nir_alu_instr_create(builder->shader, chan_op);
73 nir_alu_ssa_dest_init(chan, 1, alu->dest.dest.ssa.bit_size);
74 nir_alu_src_copy(&chan->src[0], &alu->src[0]);
75 chan->src[0].swizzle[0] = chan->src[0].swizzle[i];
76 if (nir_op_infos[chan_op].num_inputs > 1) {
77 assert(nir_op_infos[chan_op].num_inputs == 2);
78 nir_alu_src_copy(&chan->src[1], &alu->src[1]);
79 chan->src[1].swizzle[0] = chan->src[1].swizzle[i];
80 }
81 chan->exact = alu->exact;
82
83 nir_builder_instr_insert(builder, &chan->instr);
84
85 if (i == num_components - 1) {
86 last = &chan->dest.dest.ssa;
87 } else {
88 last = nir_build_alu(builder, merge_op,
89 last, &chan->dest.dest.ssa, NULL, NULL);
90 }
91 }
92
93 return last;
94 }
95
96 static inline bool
will_lower_ffma(nir_shader * shader,unsigned bit_size)97 will_lower_ffma(nir_shader *shader, unsigned bit_size)
98 {
99 switch (bit_size) {
100 case 16:
101 return shader->options->lower_ffma16;
102 case 32:
103 return shader->options->lower_ffma32;
104 case 64:
105 return shader->options->lower_ffma64;
106 }
107 unreachable("bad bit size");
108 }
109
110 static nir_ssa_def *
lower_fdot(nir_alu_instr * alu,nir_builder * builder)111 lower_fdot(nir_alu_instr *alu, nir_builder *builder)
112 {
113 /* If we don't want to lower ffma, create several ffma instead of fmul+fadd
114 * and fusing later because fusing is not possible for exact fdot instructions.
115 */
116 if (will_lower_ffma(builder->shader, alu->dest.dest.ssa.bit_size))
117 return lower_reduction(alu, nir_op_fmul, nir_op_fadd, builder);
118
119 unsigned num_components = nir_op_infos[alu->op].input_sizes[0];
120
121 nir_ssa_def *prev = NULL;
122 for (int i = num_components - 1; i >= 0; i--) {
123 nir_alu_instr *instr = nir_alu_instr_create(
124 builder->shader, prev ? nir_op_ffma : nir_op_fmul);
125 nir_alu_ssa_dest_init(instr, 1, alu->dest.dest.ssa.bit_size);
126 for (unsigned j = 0; j < 2; j++) {
127 nir_alu_src_copy(&instr->src[j], &alu->src[j]);
128 instr->src[j].swizzle[0] = alu->src[j].swizzle[i];
129 }
130 if (i != num_components - 1)
131 instr->src[2].src = nir_src_for_ssa(prev);
132 instr->exact = builder->exact;
133
134 nir_builder_instr_insert(builder, &instr->instr);
135
136 prev = &instr->dest.dest.ssa;
137 }
138
139 return prev;
140 }
141
142 static nir_ssa_def *
lower_alu_instr_scalar(nir_builder * b,nir_instr * instr,void * _data)143 lower_alu_instr_scalar(nir_builder *b, nir_instr *instr, void *_data)
144 {
145 struct alu_to_scalar_data *data = _data;
146 nir_alu_instr *alu = nir_instr_as_alu(instr);
147 unsigned num_src = nir_op_infos[alu->op].num_inputs;
148 unsigned i, chan;
149
150 assert(alu->dest.dest.is_ssa);
151 assert(alu->dest.write_mask != 0);
152
153 b->cursor = nir_before_instr(&alu->instr);
154 b->exact = alu->exact;
155
156 if (data->cb && !data->cb(instr, data->data))
157 return NULL;
158
159 #define LOWER_REDUCTION(name, chan, merge) \
160 case name##2: \
161 case name##3: \
162 case name##4: \
163 case name##8: \
164 case name##16: \
165 return lower_reduction(alu, chan, merge, b); \
166
167 switch (alu->op) {
168 case nir_op_vec16:
169 case nir_op_vec8:
170 case nir_op_vec5:
171 case nir_op_vec4:
172 case nir_op_vec3:
173 case nir_op_vec2:
174 case nir_op_cube_face_coord_amd:
175 case nir_op_cube_face_index_amd:
176 /* We don't need to scalarize these ops, they're the ones generated to
177 * group up outputs into a value that can be SSAed.
178 */
179 return NULL;
180
181 case nir_op_pack_half_2x16: {
182 if (!b->shader->options->lower_pack_half_2x16)
183 return NULL;
184
185 nir_ssa_def *src_vec2 = nir_ssa_for_alu_src(b, alu, 0);
186 return nir_pack_half_2x16_split(b, nir_channel(b, src_vec2, 0),
187 nir_channel(b, src_vec2, 1));
188 }
189
190 case nir_op_unpack_unorm_4x8:
191 case nir_op_unpack_snorm_4x8:
192 case nir_op_unpack_unorm_2x16:
193 case nir_op_unpack_snorm_2x16:
194 /* There is no scalar version of these ops, unless we were to break it
195 * down to bitshifts and math (which is definitely not intended).
196 */
197 return NULL;
198
199 case nir_op_unpack_half_2x16_flush_to_zero:
200 case nir_op_unpack_half_2x16: {
201 if (!b->shader->options->lower_unpack_half_2x16)
202 return NULL;
203
204 nir_ssa_def *packed = nir_ssa_for_alu_src(b, alu, 0);
205 if (alu->op == nir_op_unpack_half_2x16_flush_to_zero) {
206 return nir_vec2(b,
207 nir_unpack_half_2x16_split_x_flush_to_zero(b,
208 packed),
209 nir_unpack_half_2x16_split_y_flush_to_zero(b,
210 packed));
211 } else {
212 return nir_vec2(b,
213 nir_unpack_half_2x16_split_x(b, packed),
214 nir_unpack_half_2x16_split_y(b, packed));
215 }
216 }
217
218 case nir_op_pack_uvec2_to_uint: {
219 assert(b->shader->options->lower_pack_snorm_2x16 ||
220 b->shader->options->lower_pack_unorm_2x16);
221
222 nir_ssa_def *word = nir_extract_u16(b, nir_ssa_for_alu_src(b, alu, 0),
223 nir_imm_int(b, 0));
224 return nir_ior(b, nir_ishl(b, nir_channel(b, word, 1),
225 nir_imm_int(b, 16)),
226 nir_channel(b, word, 0));
227 }
228
229 case nir_op_pack_uvec4_to_uint: {
230 assert(b->shader->options->lower_pack_snorm_4x8 ||
231 b->shader->options->lower_pack_unorm_4x8);
232
233 nir_ssa_def *byte = nir_extract_u8(b, nir_ssa_for_alu_src(b, alu, 0),
234 nir_imm_int(b, 0));
235 return nir_ior(b, nir_ior(b, nir_ishl(b, nir_channel(b, byte, 3),
236 nir_imm_int(b, 24)),
237 nir_ishl(b, nir_channel(b, byte, 2),
238 nir_imm_int(b, 16))),
239 nir_ior(b, nir_ishl(b, nir_channel(b, byte, 1),
240 nir_imm_int(b, 8)),
241 nir_channel(b, byte, 0)));
242 }
243
244 case nir_op_fdph: {
245 nir_ssa_def *src0_vec = nir_ssa_for_alu_src(b, alu, 0);
246 nir_ssa_def *src1_vec = nir_ssa_for_alu_src(b, alu, 1);
247
248 nir_ssa_def *sum[4];
249 for (unsigned i = 0; i < 3; i++) {
250 sum[i] = nir_fmul(b, nir_channel(b, src0_vec, i),
251 nir_channel(b, src1_vec, i));
252 }
253 sum[3] = nir_channel(b, src1_vec, 3);
254
255 return nir_fadd(b, nir_fadd(b, sum[0], sum[1]),
256 nir_fadd(b, sum[2], sum[3]));
257 }
258
259 case nir_op_pack_64_2x32: {
260 if (!b->shader->options->lower_pack_64_2x32)
261 return NULL;
262
263 nir_ssa_def *src_vec2 = nir_ssa_for_alu_src(b, alu, 0);
264 return nir_pack_64_2x32_split(b, nir_channel(b, src_vec2, 0),
265 nir_channel(b, src_vec2, 1));
266 }
267 case nir_op_pack_64_4x16: {
268 if (!b->shader->options->lower_pack_64_4x16)
269 return NULL;
270
271 nir_ssa_def *src_vec4 = nir_ssa_for_alu_src(b, alu, 0);
272 nir_ssa_def *xy = nir_pack_32_2x16_split(b, nir_channel(b, src_vec4, 0),
273 nir_channel(b, src_vec4, 1));
274 nir_ssa_def *zw = nir_pack_32_2x16_split(b, nir_channel(b, src_vec4, 2),
275 nir_channel(b, src_vec4, 3));
276
277 return nir_pack_64_2x32_split(b, xy, zw);
278 }
279 case nir_op_pack_32_2x16: {
280 if (!b->shader->options->lower_pack_32_2x16)
281 return NULL;
282
283 nir_ssa_def *src_vec2 = nir_ssa_for_alu_src(b, alu, 0);
284 return nir_pack_32_2x16_split(b, nir_channel(b, src_vec2, 0),
285 nir_channel(b, src_vec2, 1));
286 }
287 case nir_op_unpack_64_2x32:
288 case nir_op_unpack_64_4x16:
289 case nir_op_unpack_32_2x16:
290 case nir_op_unpack_double_2x32_dxil:
291 return NULL;
292
293 case nir_op_fdot2:
294 case nir_op_fdot3:
295 case nir_op_fdot4:
296 case nir_op_fdot8:
297 case nir_op_fdot16:
298 return lower_fdot(alu, b);
299
300 LOWER_REDUCTION(nir_op_ball_fequal, nir_op_feq, nir_op_iand);
301 LOWER_REDUCTION(nir_op_ball_iequal, nir_op_ieq, nir_op_iand);
302 LOWER_REDUCTION(nir_op_bany_fnequal, nir_op_fneu, nir_op_ior);
303 LOWER_REDUCTION(nir_op_bany_inequal, nir_op_ine, nir_op_ior);
304 LOWER_REDUCTION(nir_op_b8all_fequal, nir_op_feq8, nir_op_iand);
305 LOWER_REDUCTION(nir_op_b8all_iequal, nir_op_ieq8, nir_op_iand);
306 LOWER_REDUCTION(nir_op_b8any_fnequal, nir_op_fneu8, nir_op_ior);
307 LOWER_REDUCTION(nir_op_b8any_inequal, nir_op_ine8, nir_op_ior);
308 LOWER_REDUCTION(nir_op_b16all_fequal, nir_op_feq16, nir_op_iand);
309 LOWER_REDUCTION(nir_op_b16all_iequal, nir_op_ieq16, nir_op_iand);
310 LOWER_REDUCTION(nir_op_b16any_fnequal, nir_op_fneu16, nir_op_ior);
311 LOWER_REDUCTION(nir_op_b16any_inequal, nir_op_ine16, nir_op_ior);
312 LOWER_REDUCTION(nir_op_b32all_fequal, nir_op_feq32, nir_op_iand);
313 LOWER_REDUCTION(nir_op_b32all_iequal, nir_op_ieq32, nir_op_iand);
314 LOWER_REDUCTION(nir_op_b32any_fnequal, nir_op_fneu32, nir_op_ior);
315 LOWER_REDUCTION(nir_op_b32any_inequal, nir_op_ine32, nir_op_ior);
316 LOWER_REDUCTION(nir_op_fall_equal, nir_op_seq, nir_op_fmin);
317 LOWER_REDUCTION(nir_op_fany_nequal, nir_op_sne, nir_op_fmax);
318
319 default:
320 break;
321 }
322
323 if (alu->dest.dest.ssa.num_components == 1)
324 return NULL;
325
326 unsigned num_components = alu->dest.dest.ssa.num_components;
327 nir_ssa_def *comps[NIR_MAX_VEC_COMPONENTS] = { NULL };
328
329 for (chan = 0; chan < num_components; chan++) {
330 nir_alu_instr *lower = nir_alu_instr_create(b->shader, alu->op);
331 for (i = 0; i < num_src; i++) {
332 /* We only handle same-size-as-dest (input_sizes[] == 0) or scalar
333 * args (input_sizes[] == 1).
334 */
335 assert(nir_op_infos[alu->op].input_sizes[i] < 2);
336 unsigned src_chan = (nir_op_infos[alu->op].input_sizes[i] == 1 ?
337 0 : chan);
338
339 nir_alu_src_copy(&lower->src[i], &alu->src[i]);
340 for (int j = 0; j < NIR_MAX_VEC_COMPONENTS; j++)
341 lower->src[i].swizzle[j] = alu->dest.write_mask & (1 << chan) ?
342 alu->src[i].swizzle[src_chan] : 0;
343 }
344
345 nir_alu_ssa_dest_init(lower, 1, alu->dest.dest.ssa.bit_size);
346 lower->dest.saturate = alu->dest.saturate;
347 comps[chan] = &lower->dest.dest.ssa;
348 lower->exact = alu->exact;
349
350 nir_builder_instr_insert(b, &lower->instr);
351 }
352
353 return nir_vec(b, comps, num_components);
354 }
355
356 bool
nir_lower_alu_to_scalar(nir_shader * shader,nir_instr_filter_cb cb,const void * _data)357 nir_lower_alu_to_scalar(nir_shader *shader, nir_instr_filter_cb cb, const void *_data)
358 {
359 struct alu_to_scalar_data data = {
360 .cb = cb,
361 .data = _data,
362 };
363
364 return nir_shader_lower_instructions(shader,
365 inst_is_vector_alu,
366 lower_alu_instr_scalar,
367 &data);
368 }
369