1 /*
2 * Copyright © 2019 Broadcom
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 /**
25 * Implements lowering for logical operations.
26 *
27 * V3D doesn't have any hardware support for logic ops. Instead, you read the
28 * current contents of the destination from the tile buffer, then do math using
29 * your output color and that destination value, and update the output color
30 * appropriately.
31 */
32
33 #include "util/format/u_format.h"
34 #include "compiler/nir/nir_builder.h"
35 #include "compiler/nir/nir_format_convert.h"
36 #include "v3d_compiler.h"
37
38
39 typedef nir_ssa_def *(*nir_pack_func)(nir_builder *b, nir_ssa_def *c);
40 typedef nir_ssa_def *(*nir_unpack_func)(nir_builder *b, nir_ssa_def *c);
41
42 static bool
logicop_depends_on_dst_color(int logicop_func)43 logicop_depends_on_dst_color(int logicop_func)
44 {
45 switch (logicop_func) {
46 case PIPE_LOGICOP_SET:
47 case PIPE_LOGICOP_CLEAR:
48 case PIPE_LOGICOP_COPY:
49 case PIPE_LOGICOP_COPY_INVERTED:
50 return false;
51 default:
52 return true;
53 }
54 }
55
56 static nir_ssa_def *
v3d_logicop(nir_builder * b,int logicop_func,nir_ssa_def * src,nir_ssa_def * dst)57 v3d_logicop(nir_builder *b, int logicop_func,
58 nir_ssa_def *src, nir_ssa_def *dst)
59 {
60 switch (logicop_func) {
61 case PIPE_LOGICOP_CLEAR:
62 return nir_imm_int(b, 0);
63 case PIPE_LOGICOP_NOR:
64 return nir_inot(b, nir_ior(b, src, dst));
65 case PIPE_LOGICOP_AND_INVERTED:
66 return nir_iand(b, nir_inot(b, src), dst);
67 case PIPE_LOGICOP_COPY_INVERTED:
68 return nir_inot(b, src);
69 case PIPE_LOGICOP_AND_REVERSE:
70 return nir_iand(b, src, nir_inot(b, dst));
71 case PIPE_LOGICOP_INVERT:
72 return nir_inot(b, dst);
73 case PIPE_LOGICOP_XOR:
74 return nir_ixor(b, src, dst);
75 case PIPE_LOGICOP_NAND:
76 return nir_inot(b, nir_iand(b, src, dst));
77 case PIPE_LOGICOP_AND:
78 return nir_iand(b, src, dst);
79 case PIPE_LOGICOP_EQUIV:
80 return nir_inot(b, nir_ixor(b, src, dst));
81 case PIPE_LOGICOP_NOOP:
82 return dst;
83 case PIPE_LOGICOP_OR_INVERTED:
84 return nir_ior(b, nir_inot(b, src), dst);
85 case PIPE_LOGICOP_OR_REVERSE:
86 return nir_ior(b, src, nir_inot(b, dst));
87 case PIPE_LOGICOP_OR:
88 return nir_ior(b, src, dst);
89 case PIPE_LOGICOP_SET:
90 return nir_imm_int(b, ~0);
91 default:
92 fprintf(stderr, "Unknown logic op %d\n", logicop_func);
93 /* FALLTHROUGH */
94 case PIPE_LOGICOP_COPY:
95 return src;
96 }
97 }
98
99 static nir_ssa_def *
v3d_nir_get_swizzled_channel(nir_builder * b,nir_ssa_def ** srcs,int swiz)100 v3d_nir_get_swizzled_channel(nir_builder *b, nir_ssa_def **srcs, int swiz)
101 {
102 switch (swiz) {
103 default:
104 case PIPE_SWIZZLE_NONE:
105 fprintf(stderr, "warning: unknown swizzle\n");
106 /* FALLTHROUGH */
107 case PIPE_SWIZZLE_0:
108 return nir_imm_float(b, 0.0);
109 case PIPE_SWIZZLE_1:
110 return nir_imm_float(b, 1.0);
111 case PIPE_SWIZZLE_X:
112 case PIPE_SWIZZLE_Y:
113 case PIPE_SWIZZLE_Z:
114 case PIPE_SWIZZLE_W:
115 return srcs[swiz];
116 }
117 }
118
119 static nir_ssa_def *
v3d_nir_swizzle_and_pack(nir_builder * b,nir_ssa_def ** chans,const uint8_t * swiz,nir_pack_func pack_func)120 v3d_nir_swizzle_and_pack(nir_builder *b, nir_ssa_def **chans,
121 const uint8_t *swiz, nir_pack_func pack_func)
122 {
123 nir_ssa_def *c[4];
124 for (int i = 0; i < 4; i++)
125 c[i] = v3d_nir_get_swizzled_channel(b, chans, swiz[i]);
126
127 return pack_func(b, nir_vec4(b, c[0], c[1], c[2], c[3]));
128 }
129
130 static nir_ssa_def *
v3d_nir_unpack_and_swizzle(nir_builder * b,nir_ssa_def * packed,const uint8_t * swiz,nir_unpack_func unpack_func)131 v3d_nir_unpack_and_swizzle(nir_builder *b, nir_ssa_def *packed,
132 const uint8_t *swiz, nir_unpack_func unpack_func)
133 {
134 nir_ssa_def *unpacked = unpack_func(b, packed);
135
136 nir_ssa_def *unpacked_chans[4];
137 for (int i = 0; i < 4; i++)
138 unpacked_chans[i] = nir_channel(b, unpacked, i);
139
140 nir_ssa_def *c[4];
141 for (int i = 0; i < 4; i++)
142 c[i] = v3d_nir_get_swizzled_channel(b, unpacked_chans, swiz[i]);
143
144 return nir_vec4(b, c[0], c[1], c[2], c[3]);
145 }
146
147 static nir_ssa_def *
pack_unorm_rgb10a2(nir_builder * b,nir_ssa_def * c)148 pack_unorm_rgb10a2(nir_builder *b, nir_ssa_def *c)
149 {
150 const unsigned bits[4] = { 10, 10, 10, 2 };
151 nir_ssa_def *unorm = nir_format_float_to_unorm(b, c, bits);
152
153 nir_ssa_def *chans[4];
154 for (int i = 0; i < 4; i++)
155 chans[i] = nir_channel(b, unorm, i);
156
157 nir_ssa_def *result = nir_mov(b, chans[0]);
158 int offset = bits[0];
159 for (int i = 1; i < 4; i++) {
160 nir_ssa_def *shifted_chan =
161 nir_ishl(b, chans[i], nir_imm_int(b, offset));
162 result = nir_ior(b, result, shifted_chan);
163 offset += bits[i];
164 }
165 return result;
166 }
167
168 static nir_ssa_def *
unpack_unorm_rgb10a2(nir_builder * b,nir_ssa_def * c)169 unpack_unorm_rgb10a2(nir_builder *b, nir_ssa_def *c)
170 {
171 const unsigned bits[4] = { 10, 10, 10, 2 };
172 const unsigned masks[4] = { BITFIELD_MASK(bits[0]),
173 BITFIELD_MASK(bits[1]),
174 BITFIELD_MASK(bits[2]),
175 BITFIELD_MASK(bits[3]) };
176
177 nir_ssa_def *chans[4];
178 for (int i = 0; i < 4; i++) {
179 nir_ssa_def *unorm = nir_iand(b, c, nir_imm_int(b, masks[i]));
180 chans[i] = nir_format_unorm_to_float(b, unorm, &bits[i]);
181 c = nir_ushr(b, c, nir_imm_int(b, bits[i]));
182 }
183
184 return nir_vec4(b, chans[0], chans[1], chans[2], chans[3]);
185 }
186
187 static const uint8_t *
v3d_get_format_swizzle_for_rt(struct v3d_compile * c,int rt)188 v3d_get_format_swizzle_for_rt(struct v3d_compile *c, int rt)
189 {
190 static const uint8_t ident[4] = { 0, 1, 2, 3 };
191
192 /* We will automatically swap R and B channels for BGRA formats
193 * on tile loads and stores (see 'swap_rb' field in v3d_resource) so
194 * we want to treat these surfaces as if they were regular RGBA formats.
195 */
196 if (c->fs_key->color_fmt[rt].swizzle[0] == 2 &&
197 c->fs_key->color_fmt[rt].format != PIPE_FORMAT_B5G6R5_UNORM) {
198 return ident;
199 } else {
200 return c->fs_key->color_fmt[rt].swizzle;
201 }
202 }
203
204 static nir_ssa_def *
v3d_nir_get_tlb_color(nir_builder * b,int rt,int sample)205 v3d_nir_get_tlb_color(nir_builder *b, int rt, int sample)
206 {
207 nir_ssa_def *color[4];
208 for (int i = 0; i < 4; i++) {
209 nir_intrinsic_instr *load =
210 nir_intrinsic_instr_create(b->shader,
211 nir_intrinsic_load_tlb_color_v3d);
212 load->num_components = 1;
213 nir_intrinsic_set_base(load, sample);
214 nir_intrinsic_set_component(load, i);
215 load->src[0] = nir_src_for_ssa(nir_imm_int(b, rt));
216 nir_ssa_dest_init(&load->instr, &load->dest, 1, 32, NULL);
217 nir_builder_instr_insert(b, &load->instr);
218 color[i] = &load->dest.ssa;
219 }
220
221 return nir_vec4(b, color[0], color[1], color[2], color[3]);
222 }
223
224 static nir_ssa_def *
v3d_emit_logic_op_raw(struct v3d_compile * c,nir_builder * b,nir_ssa_def ** src_chans,nir_ssa_def ** dst_chans,int rt,int sample)225 v3d_emit_logic_op_raw(struct v3d_compile *c, nir_builder *b,
226 nir_ssa_def **src_chans, nir_ssa_def **dst_chans,
227 int rt, int sample)
228 {
229 const uint8_t *fmt_swz = v3d_get_format_swizzle_for_rt(c, rt);
230
231 nir_ssa_def *op_res[4];
232 for (int i = 0; i < 4; i++) {
233 nir_ssa_def *src = src_chans[i];
234 nir_ssa_def *dst =
235 v3d_nir_get_swizzled_channel(b, dst_chans, fmt_swz[i]);
236 op_res[i] = v3d_logicop(b, c->fs_key->logicop_func, src, dst);
237 }
238
239 nir_ssa_def *r[4];
240 for (int i = 0; i < 4; i++)
241 r[i] = v3d_nir_get_swizzled_channel(b, op_res, fmt_swz[i]);
242
243 return nir_vec4(b, r[0], r[1], r[2], r[3]);
244 }
245
246 static nir_ssa_def *
v3d_emit_logic_op_unorm(struct v3d_compile * c,nir_builder * b,nir_ssa_def ** src_chans,nir_ssa_def ** dst_chans,int rt,int sample,nir_pack_func pack_func,nir_unpack_func unpack_func)247 v3d_emit_logic_op_unorm(struct v3d_compile *c, nir_builder *b,
248 nir_ssa_def **src_chans, nir_ssa_def **dst_chans,
249 int rt, int sample,
250 nir_pack_func pack_func, nir_unpack_func unpack_func)
251 {
252 const uint8_t src_swz[4] = { 0, 1, 2, 3 };
253 nir_ssa_def *packed_src =
254 v3d_nir_swizzle_and_pack(b, src_chans, src_swz, pack_func);
255
256 const uint8_t *fmt_swz = v3d_get_format_swizzle_for_rt(c, rt);
257 nir_ssa_def *packed_dst =
258 v3d_nir_swizzle_and_pack(b, dst_chans, fmt_swz, pack_func);
259
260 nir_ssa_def *packed_result =
261 v3d_logicop(b, c->fs_key->logicop_func, packed_src, packed_dst);
262
263 return v3d_nir_unpack_and_swizzle(b, packed_result, fmt_swz, unpack_func);
264 }
265
266 static nir_ssa_def *
v3d_nir_emit_logic_op(struct v3d_compile * c,nir_builder * b,nir_ssa_def * src,int rt,int sample)267 v3d_nir_emit_logic_op(struct v3d_compile *c, nir_builder *b,
268 nir_ssa_def *src, int rt, int sample)
269 {
270 nir_ssa_def *dst = v3d_nir_get_tlb_color(b, rt, sample);
271
272 nir_ssa_def *src_chans[4], *dst_chans[4];
273 for (unsigned i = 0; i < 4; i++) {
274 src_chans[i] = nir_channel(b, src, i);
275 dst_chans[i] = nir_channel(b, dst, i);
276 }
277
278 if (c->fs_key->color_fmt[rt].format == PIPE_FORMAT_R10G10B10A2_UNORM) {
279 return v3d_emit_logic_op_unorm(
280 c, b, src_chans, dst_chans, rt, 0,
281 pack_unorm_rgb10a2, unpack_unorm_rgb10a2);
282 }
283
284 if (util_format_is_unorm(c->fs_key->color_fmt[rt].format)) {
285 return v3d_emit_logic_op_unorm(
286 c, b, src_chans, dst_chans, rt, 0,
287 nir_pack_unorm_4x8, nir_unpack_unorm_4x8);
288 }
289
290 return v3d_emit_logic_op_raw(c, b, src_chans, dst_chans, rt, 0);
291 }
292
293 static void
v3d_emit_ms_output(struct v3d_compile * c,nir_builder * b,nir_ssa_def * color,nir_src * offset,nir_alu_type type,int rt,int sample)294 v3d_emit_ms_output(struct v3d_compile *c, nir_builder *b,
295 nir_ssa_def *color, nir_src *offset,
296 nir_alu_type type, int rt, int sample)
297 {
298
299 nir_intrinsic_instr *store =
300 nir_intrinsic_instr_create(b->shader,
301 nir_intrinsic_store_tlb_sample_color_v3d);
302 store->num_components = 4;
303 nir_intrinsic_set_base(store, sample);
304 nir_intrinsic_set_component(store, 0);
305 nir_intrinsic_set_src_type(store, type);
306 store->src[0] = nir_src_for_ssa(color);
307 store->src[1] = nir_src_for_ssa(nir_imm_int(b, rt));
308 nir_builder_instr_insert(b, &store->instr);
309 }
310
311 static void
v3d_nir_lower_logic_op_instr(struct v3d_compile * c,nir_builder * b,nir_intrinsic_instr * intr,int rt)312 v3d_nir_lower_logic_op_instr(struct v3d_compile *c,
313 nir_builder *b,
314 nir_intrinsic_instr *intr,
315 int rt)
316 {
317 nir_ssa_def *frag_color = intr->src[0].ssa;
318
319
320 const int logic_op = c->fs_key->logicop_func;
321 if (c->fs_key->msaa && logicop_depends_on_dst_color(logic_op)) {
322 c->msaa_per_sample_output = true;
323
324 nir_src *offset = &intr->src[1];
325 nir_alu_type type = nir_intrinsic_src_type(intr);
326 for (int i = 0; i < V3D_MAX_SAMPLES; i++) {
327 nir_ssa_def *sample =
328 v3d_nir_emit_logic_op(c, b, frag_color, rt, i);
329
330 v3d_emit_ms_output(c, b, sample, offset, type, rt, i);
331 }
332
333 nir_instr_remove(&intr->instr);
334 } else {
335 nir_ssa_def *result =
336 v3d_nir_emit_logic_op(c, b, frag_color, rt, 0);
337
338 nir_instr_rewrite_src(&intr->instr, &intr->src[0],
339 nir_src_for_ssa(result));
340 intr->num_components = result->num_components;
341 }
342 }
343
344 static bool
v3d_nir_lower_logic_ops_block(nir_block * block,struct v3d_compile * c)345 v3d_nir_lower_logic_ops_block(nir_block *block, struct v3d_compile *c)
346 {
347 nir_foreach_instr_safe(instr, block) {
348 if (instr->type != nir_instr_type_intrinsic)
349 continue;
350
351 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
352 if (intr->intrinsic != nir_intrinsic_store_output)
353 continue;
354
355 nir_foreach_shader_out_variable(var, c->s) {
356 const int driver_loc = var->data.driver_location;
357 if (driver_loc != nir_intrinsic_base(intr))
358 continue;
359
360 const int loc = var->data.location;
361 if (loc != FRAG_RESULT_COLOR &&
362 (loc < FRAG_RESULT_DATA0 ||
363 loc >= FRAG_RESULT_DATA0 + V3D_MAX_DRAW_BUFFERS)) {
364 continue;
365 }
366
367 /* Logic operations do not apply on floating point or
368 * sRGB enabled render targets.
369 */
370 const int rt = driver_loc;
371 assert(rt < V3D_MAX_DRAW_BUFFERS);
372
373 const enum pipe_format format =
374 c->fs_key->color_fmt[rt].format;
375 if (util_format_is_float(format) ||
376 util_format_is_srgb(format)) {
377 continue;
378 }
379
380 nir_function_impl *impl =
381 nir_cf_node_get_function(&block->cf_node);
382 nir_builder b;
383 nir_builder_init(&b, impl);
384 b.cursor = nir_before_instr(&intr->instr);
385 v3d_nir_lower_logic_op_instr(c, &b, intr, rt);
386 }
387 }
388
389 return true;
390 }
391
392 void
v3d_nir_lower_logic_ops(nir_shader * s,struct v3d_compile * c)393 v3d_nir_lower_logic_ops(nir_shader *s, struct v3d_compile *c)
394 {
395 /* Nothing to do if logic op is 'copy src to dst' or if logic ops are
396 * disabled (we set the logic op to copy in that case).
397 */
398 if (c->fs_key->logicop_func == PIPE_LOGICOP_COPY)
399 return;
400
401 nir_foreach_function(function, s) {
402 if (function->impl) {
403 nir_foreach_block(block, function->impl)
404 v3d_nir_lower_logic_ops_block(block, c);
405
406 nir_metadata_preserve(function->impl,
407 nir_metadata_block_index |
408 nir_metadata_dominance);
409 }
410 }
411 }
412