1 /*
2 * Copyright © 2014-2015 Broadcom
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "nir.h"
25 #include "nir_builder.h"
26
27 struct alu_to_scalar_data {
28 nir_instr_filter_cb cb;
29 const void *data;
30 };
31
32 /** @file nir_lower_alu_to_scalar.c
33 *
34 * Replaces nir_alu_instr operations with more than one channel used in the
35 * arguments with individual per-channel operations.
36 */
37
38 static bool
inst_is_vector_alu(const nir_instr * instr,const void * _state)39 inst_is_vector_alu(const nir_instr *instr, const void *_state)
40 {
41 if (instr->type != nir_instr_type_alu)
42 return false;
43
44 nir_alu_instr *alu = nir_instr_as_alu(instr);
45
46 /* There is no ALU instruction which has a scalar destination, scalar
47 * src[0], and some other vector source.
48 */
49 assert(alu->dest.dest.is_ssa);
50 assert(alu->src[0].src.is_ssa);
51 return alu->dest.dest.ssa.num_components > 1 ||
52 nir_op_infos[alu->op].input_sizes[0] > 1;
53 }
54
55 static void
nir_alu_ssa_dest_init(nir_alu_instr * alu,unsigned num_components,unsigned bit_size)56 nir_alu_ssa_dest_init(nir_alu_instr *alu, unsigned num_components,
57 unsigned bit_size)
58 {
59 nir_ssa_dest_init(&alu->instr, &alu->dest.dest, num_components,
60 bit_size, NULL);
61 alu->dest.write_mask = (1 << num_components) - 1;
62 }
63
64 static nir_ssa_def *
lower_reduction(nir_alu_instr * alu,nir_op chan_op,nir_op merge_op,nir_builder * builder)65 lower_reduction(nir_alu_instr *alu, nir_op chan_op, nir_op merge_op,
66 nir_builder *builder)
67 {
68 unsigned num_components = nir_op_infos[alu->op].input_sizes[0];
69
70 nir_ssa_def *last = NULL;
71 for (int i = num_components - 1; i >= 0; i--) {
72 nir_alu_instr *chan = nir_alu_instr_create(builder->shader, chan_op);
73 nir_alu_ssa_dest_init(chan, 1, alu->dest.dest.ssa.bit_size);
74 nir_alu_src_copy(&chan->src[0], &alu->src[0], chan);
75 chan->src[0].swizzle[0] = chan->src[0].swizzle[i];
76 if (nir_op_infos[chan_op].num_inputs > 1) {
77 assert(nir_op_infos[chan_op].num_inputs == 2);
78 nir_alu_src_copy(&chan->src[1], &alu->src[1], chan);
79 chan->src[1].swizzle[0] = chan->src[1].swizzle[i];
80 }
81 chan->exact = alu->exact;
82
83 nir_builder_instr_insert(builder, &chan->instr);
84
85 if (i == num_components - 1) {
86 last = &chan->dest.dest.ssa;
87 } else {
88 last = nir_build_alu(builder, merge_op,
89 last, &chan->dest.dest.ssa, NULL, NULL);
90 }
91 }
92
93 return last;
94 }
95
96 static nir_ssa_def *
lower_alu_instr_scalar(nir_builder * b,nir_instr * instr,void * _data)97 lower_alu_instr_scalar(nir_builder *b, nir_instr *instr, void *_data)
98 {
99 struct alu_to_scalar_data *data = _data;
100 nir_alu_instr *alu = nir_instr_as_alu(instr);
101 unsigned num_src = nir_op_infos[alu->op].num_inputs;
102 unsigned i, chan;
103
104 assert(alu->dest.dest.is_ssa);
105 assert(alu->dest.write_mask != 0);
106
107 b->cursor = nir_before_instr(&alu->instr);
108 b->exact = alu->exact;
109
110 if (data->cb && !data->cb(instr, data->data))
111 return NULL;
112
113 #define LOWER_REDUCTION(name, chan, merge) \
114 case name##2: \
115 case name##3: \
116 case name##4: \
117 case name##8: \
118 case name##16: \
119 return lower_reduction(alu, chan, merge, b); \
120
121 switch (alu->op) {
122 case nir_op_vec16:
123 case nir_op_vec8:
124 case nir_op_vec4:
125 case nir_op_vec3:
126 case nir_op_vec2:
127 case nir_op_cube_face_coord:
128 case nir_op_cube_face_index:
129 /* We don't need to scalarize these ops, they're the ones generated to
130 * group up outputs into a value that can be SSAed.
131 */
132 return NULL;
133
134 case nir_op_pack_half_2x16: {
135 if (!b->shader->options->lower_pack_half_2x16)
136 return NULL;
137
138 nir_ssa_def *src_vec2 = nir_ssa_for_alu_src(b, alu, 0);
139 return nir_pack_half_2x16_split(b, nir_channel(b, src_vec2, 0),
140 nir_channel(b, src_vec2, 1));
141 }
142
143 case nir_op_unpack_unorm_4x8:
144 case nir_op_unpack_snorm_4x8:
145 case nir_op_unpack_unorm_2x16:
146 case nir_op_unpack_snorm_2x16:
147 /* There is no scalar version of these ops, unless we were to break it
148 * down to bitshifts and math (which is definitely not intended).
149 */
150 return NULL;
151
152 case nir_op_unpack_half_2x16_flush_to_zero:
153 case nir_op_unpack_half_2x16: {
154 if (!b->shader->options->lower_unpack_half_2x16)
155 return NULL;
156
157 nir_ssa_def *packed = nir_ssa_for_alu_src(b, alu, 0);
158 if (alu->op == nir_op_unpack_half_2x16_flush_to_zero) {
159 return nir_vec2(b,
160 nir_unpack_half_2x16_split_x_flush_to_zero(b,
161 packed),
162 nir_unpack_half_2x16_split_y_flush_to_zero(b,
163 packed));
164 } else {
165 return nir_vec2(b,
166 nir_unpack_half_2x16_split_x(b, packed),
167 nir_unpack_half_2x16_split_y(b, packed));
168 }
169 }
170
171 case nir_op_pack_uvec2_to_uint: {
172 assert(b->shader->options->lower_pack_snorm_2x16 ||
173 b->shader->options->lower_pack_unorm_2x16);
174
175 nir_ssa_def *word = nir_extract_u16(b, nir_ssa_for_alu_src(b, alu, 0),
176 nir_imm_int(b, 0));
177 return nir_ior(b, nir_ishl(b, nir_channel(b, word, 1),
178 nir_imm_int(b, 16)),
179 nir_channel(b, word, 0));
180 }
181
182 case nir_op_pack_uvec4_to_uint: {
183 assert(b->shader->options->lower_pack_snorm_4x8 ||
184 b->shader->options->lower_pack_unorm_4x8);
185
186 nir_ssa_def *byte = nir_extract_u8(b, nir_ssa_for_alu_src(b, alu, 0),
187 nir_imm_int(b, 0));
188 return nir_ior(b, nir_ior(b, nir_ishl(b, nir_channel(b, byte, 3),
189 nir_imm_int(b, 24)),
190 nir_ishl(b, nir_channel(b, byte, 2),
191 nir_imm_int(b, 16))),
192 nir_ior(b, nir_ishl(b, nir_channel(b, byte, 1),
193 nir_imm_int(b, 8)),
194 nir_channel(b, byte, 0)));
195 }
196
197 case nir_op_fdph: {
198 nir_ssa_def *src0_vec = nir_ssa_for_alu_src(b, alu, 0);
199 nir_ssa_def *src1_vec = nir_ssa_for_alu_src(b, alu, 1);
200
201 nir_ssa_def *sum[4];
202 for (unsigned i = 0; i < 3; i++) {
203 sum[i] = nir_fmul(b, nir_channel(b, src0_vec, i),
204 nir_channel(b, src1_vec, i));
205 }
206 sum[3] = nir_channel(b, src1_vec, 3);
207
208 return nir_fadd(b, nir_fadd(b, sum[0], sum[1]),
209 nir_fadd(b, sum[2], sum[3]));
210 }
211
212 case nir_op_pack_64_2x32: {
213 if (!b->shader->options->lower_pack_64_2x32)
214 return NULL;
215
216 nir_ssa_def *src_vec2 = nir_ssa_for_alu_src(b, alu, 0);
217 return nir_pack_64_2x32_split(b, nir_channel(b, src_vec2, 0),
218 nir_channel(b, src_vec2, 1));
219 }
220 case nir_op_pack_64_4x16: {
221 if (!b->shader->options->lower_pack_64_4x16)
222 return NULL;
223
224 nir_ssa_def *src_vec4 = nir_ssa_for_alu_src(b, alu, 0);
225 nir_ssa_def *xy = nir_pack_32_2x16_split(b, nir_channel(b, src_vec4, 0),
226 nir_channel(b, src_vec4, 1));
227 nir_ssa_def *zw = nir_pack_32_2x16_split(b, nir_channel(b, src_vec4, 2),
228 nir_channel(b, src_vec4, 3));
229
230 return nir_pack_64_2x32_split(b, xy, zw);
231 }
232 case nir_op_pack_32_2x16: {
233 if (!b->shader->options->lower_pack_32_2x16)
234 return NULL;
235
236 nir_ssa_def *src_vec2 = nir_ssa_for_alu_src(b, alu, 0);
237 return nir_pack_32_2x16_split(b, nir_channel(b, src_vec2, 0),
238 nir_channel(b, src_vec2, 1));
239 }
240 case nir_op_unpack_64_2x32:
241 case nir_op_unpack_64_4x16:
242 case nir_op_unpack_32_2x16:
243 return NULL;
244
245 LOWER_REDUCTION(nir_op_fdot, nir_op_fmul, nir_op_fadd);
246 LOWER_REDUCTION(nir_op_ball_fequal, nir_op_feq, nir_op_iand);
247 LOWER_REDUCTION(nir_op_ball_iequal, nir_op_ieq, nir_op_iand);
248 LOWER_REDUCTION(nir_op_bany_fnequal, nir_op_fneu, nir_op_ior);
249 LOWER_REDUCTION(nir_op_bany_inequal, nir_op_ine, nir_op_ior);
250 LOWER_REDUCTION(nir_op_b8all_fequal, nir_op_feq8, nir_op_iand);
251 LOWER_REDUCTION(nir_op_b8all_iequal, nir_op_ieq8, nir_op_iand);
252 LOWER_REDUCTION(nir_op_b8any_fnequal, nir_op_fneu8, nir_op_ior);
253 LOWER_REDUCTION(nir_op_b8any_inequal, nir_op_ine8, nir_op_ior);
254 LOWER_REDUCTION(nir_op_b16all_fequal, nir_op_feq16, nir_op_iand);
255 LOWER_REDUCTION(nir_op_b16all_iequal, nir_op_ieq16, nir_op_iand);
256 LOWER_REDUCTION(nir_op_b16any_fnequal, nir_op_fneu16, nir_op_ior);
257 LOWER_REDUCTION(nir_op_b16any_inequal, nir_op_ine16, nir_op_ior);
258 LOWER_REDUCTION(nir_op_b32all_fequal, nir_op_feq32, nir_op_iand);
259 LOWER_REDUCTION(nir_op_b32all_iequal, nir_op_ieq32, nir_op_iand);
260 LOWER_REDUCTION(nir_op_b32any_fnequal, nir_op_fneu32, nir_op_ior);
261 LOWER_REDUCTION(nir_op_b32any_inequal, nir_op_ine32, nir_op_ior);
262 LOWER_REDUCTION(nir_op_fall_equal, nir_op_seq, nir_op_fmin);
263 LOWER_REDUCTION(nir_op_fany_nequal, nir_op_sne, nir_op_fmax);
264
265 default:
266 break;
267 }
268
269 if (alu->dest.dest.ssa.num_components == 1)
270 return NULL;
271
272 unsigned num_components = alu->dest.dest.ssa.num_components;
273 nir_ssa_def *comps[NIR_MAX_VEC_COMPONENTS] = { NULL };
274
275 for (chan = 0; chan < NIR_MAX_VEC_COMPONENTS; chan++) {
276 if (!(alu->dest.write_mask & (1 << chan)))
277 continue;
278
279 nir_alu_instr *lower = nir_alu_instr_create(b->shader, alu->op);
280 for (i = 0; i < num_src; i++) {
281 /* We only handle same-size-as-dest (input_sizes[] == 0) or scalar
282 * args (input_sizes[] == 1).
283 */
284 assert(nir_op_infos[alu->op].input_sizes[i] < 2);
285 unsigned src_chan = (nir_op_infos[alu->op].input_sizes[i] == 1 ?
286 0 : chan);
287
288 nir_alu_src_copy(&lower->src[i], &alu->src[i], lower);
289 for (int j = 0; j < NIR_MAX_VEC_COMPONENTS; j++)
290 lower->src[i].swizzle[j] = alu->src[i].swizzle[src_chan];
291 }
292
293 nir_alu_ssa_dest_init(lower, 1, alu->dest.dest.ssa.bit_size);
294 lower->dest.saturate = alu->dest.saturate;
295 comps[chan] = &lower->dest.dest.ssa;
296 lower->exact = alu->exact;
297
298 nir_builder_instr_insert(b, &lower->instr);
299 }
300
301 return nir_vec(b, comps, num_components);
302 }
303
304 bool
nir_lower_alu_to_scalar(nir_shader * shader,nir_instr_filter_cb cb,const void * _data)305 nir_lower_alu_to_scalar(nir_shader *shader, nir_instr_filter_cb cb, const void *_data)
306 {
307 struct alu_to_scalar_data data = {
308 .cb = cb,
309 .data = _data,
310 };
311
312 return nir_shader_lower_instructions(shader,
313 inst_is_vector_alu,
314 lower_alu_instr_scalar,
315 &data);
316 }
317