1 /*
2 * Copyright © 2010 Intel Corporation
3 * Copyright © 2018 Broadcom
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25 #include "nir.h"
26 #include "nir_builder.h"
27
28 /** nir_lower_alu.c
29 *
30 * NIR's home for miscellaneous ALU operation lowering implementations.
31 *
32 * Most NIR ALU lowering occurs in nir_opt_algebraic.py, since it's generally
33 * easy to write them there. However, if terms appear multiple times in the
34 * lowered code, it can get very verbose and cause a lot of work for CSE, so
35 * it may end up being easier to write out in C code.
36 *
37 * The shader must be in SSA for this pass.
38 */
39
40 #define LOWER_MUL_HIGH (1 << 0)
41
42 static bool
lower_alu_instr(nir_builder * b,nir_instr * instr_,UNUSED void * cb_data)43 lower_alu_instr(nir_builder *b, nir_instr *instr_, UNUSED void *cb_data)
44 {
45 if (instr_->type != nir_instr_type_alu)
46 return false;
47
48 nir_alu_instr *instr = nir_instr_as_alu(instr_);
49
50 nir_def *lowered = NULL;
51
52 b->cursor = nir_before_instr(&instr->instr);
53 b->exact = instr->exact;
54
55 switch (instr->op) {
56 case nir_op_bitfield_reverse:
57 if (b->shader->options->lower_bitfield_reverse) {
58 /* For more details, see:
59 *
60 * http://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel
61 */
62 nir_def *c1 = nir_imm_int(b, 1);
63 nir_def *c2 = nir_imm_int(b, 2);
64 nir_def *c4 = nir_imm_int(b, 4);
65 nir_def *c8 = nir_imm_int(b, 8);
66 nir_def *c16 = nir_imm_int(b, 16);
67 nir_def *c33333333 = nir_imm_int(b, 0x33333333);
68 nir_def *c55555555 = nir_imm_int(b, 0x55555555);
69 nir_def *c0f0f0f0f = nir_imm_int(b, 0x0f0f0f0f);
70 nir_def *c00ff00ff = nir_imm_int(b, 0x00ff00ff);
71
72 lowered = nir_ssa_for_alu_src(b, instr, 0);
73
74 /* Swap odd and even bits. */
75 lowered = nir_ior(b,
76 nir_iand(b, nir_ushr(b, lowered, c1), c55555555),
77 nir_ishl(b, nir_iand(b, lowered, c55555555), c1));
78
79 /* Swap consecutive pairs. */
80 lowered = nir_ior(b,
81 nir_iand(b, nir_ushr(b, lowered, c2), c33333333),
82 nir_ishl(b, nir_iand(b, lowered, c33333333), c2));
83
84 /* Swap nibbles. */
85 lowered = nir_ior(b,
86 nir_iand(b, nir_ushr(b, lowered, c4), c0f0f0f0f),
87 nir_ishl(b, nir_iand(b, lowered, c0f0f0f0f), c4));
88
89 /* Swap bytes. */
90 lowered = nir_ior(b,
91 nir_iand(b, nir_ushr(b, lowered, c8), c00ff00ff),
92 nir_ishl(b, nir_iand(b, lowered, c00ff00ff), c8));
93
94 lowered = nir_ior(b,
95 nir_ushr(b, lowered, c16),
96 nir_ishl(b, lowered, c16));
97 }
98 break;
99
100 case nir_op_bit_count:
101 if (b->shader->options->lower_bit_count) {
102 /* For more details, see:
103 *
104 * http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
105 */
106 nir_def *c1 = nir_imm_int(b, 1);
107 nir_def *c2 = nir_imm_int(b, 2);
108 nir_def *c4 = nir_imm_int(b, 4);
109 nir_def *c24 = nir_imm_int(b, 24);
110 nir_def *c33333333 = nir_imm_int(b, 0x33333333);
111 nir_def *c55555555 = nir_imm_int(b, 0x55555555);
112 nir_def *c0f0f0f0f = nir_imm_int(b, 0x0f0f0f0f);
113 nir_def *c01010101 = nir_imm_int(b, 0x01010101);
114
115 lowered = nir_ssa_for_alu_src(b, instr, 0);
116
117 lowered = nir_isub(b, lowered,
118 nir_iand(b, nir_ushr(b, lowered, c1), c55555555));
119
120 lowered = nir_iadd(b,
121 nir_iand(b, lowered, c33333333),
122 nir_iand(b, nir_ushr(b, lowered, c2), c33333333));
123
124 lowered = nir_ushr(b,
125 nir_imul(b,
126 nir_iand(b,
127 nir_iadd(b,
128 lowered,
129 nir_ushr(b, lowered, c4)),
130 c0f0f0f0f),
131 c01010101),
132 c24);
133 }
134 break;
135
136 case nir_op_imul_high:
137 case nir_op_umul_high:
138 if (b->shader->options->lower_mul_high) {
139 nir_def *src0 = nir_ssa_for_alu_src(b, instr, 0);
140 nir_def *src1 = nir_ssa_for_alu_src(b, instr, 1);
141 if (src0->bit_size < 32) {
142 /* Just do the math in 32-bit space and shift the result */
143 nir_alu_type base_type = nir_op_infos[instr->op].output_type;
144
145 nir_def *src0_32 = nir_type_convert(b, src0, base_type, base_type | 32, nir_rounding_mode_undef);
146 nir_def *src1_32 = nir_type_convert(b, src1, base_type, base_type | 32, nir_rounding_mode_undef);
147 nir_def *dest_32 = nir_imul(b, src0_32, src1_32);
148 nir_def *dest_shifted = nir_ishr_imm(b, dest_32, src0->bit_size);
149 lowered = nir_type_convert(b, dest_shifted, base_type, base_type | src0->bit_size, nir_rounding_mode_undef);
150 } else {
151 nir_def *cshift = nir_imm_int(b, src0->bit_size / 2);
152 nir_def *cmask = nir_imm_intN_t(b, (1ull << (src0->bit_size / 2)) - 1, src0->bit_size);
153 nir_def *different_signs = NULL;
154 if (instr->op == nir_op_imul_high) {
155 nir_def *c0 = nir_imm_intN_t(b, 0, src0->bit_size);
156 different_signs = nir_ixor(b,
157 nir_ilt(b, src0, c0),
158 nir_ilt(b, src1, c0));
159 src0 = nir_iabs(b, src0);
160 src1 = nir_iabs(b, src1);
161 }
162
163 /* ABCD
164 * * EFGH
165 * ======
166 * (GH * CD) + (GH * AB) << 16 + (EF * CD) << 16 + (EF * AB) << 32
167 *
168 * Start by splitting into the 4 multiplies.
169 */
170 nir_def *src0l = nir_iand(b, src0, cmask);
171 nir_def *src1l = nir_iand(b, src1, cmask);
172 nir_def *src0h = nir_ushr(b, src0, cshift);
173 nir_def *src1h = nir_ushr(b, src1, cshift);
174
175 nir_def *lo = nir_imul(b, src0l, src1l);
176 nir_def *m1 = nir_imul(b, src0l, src1h);
177 nir_def *m2 = nir_imul(b, src0h, src1l);
178 nir_def *hi = nir_imul(b, src0h, src1h);
179
180 nir_def *tmp;
181
182 tmp = nir_ishl(b, m1, cshift);
183 hi = nir_iadd(b, hi, nir_uadd_carry(b, lo, tmp));
184 lo = nir_iadd(b, lo, tmp);
185 hi = nir_iadd(b, hi, nir_ushr(b, m1, cshift));
186
187 tmp = nir_ishl(b, m2, cshift);
188 hi = nir_iadd(b, hi, nir_uadd_carry(b, lo, tmp));
189 lo = nir_iadd(b, lo, tmp);
190 hi = nir_iadd(b, hi, nir_ushr(b, m2, cshift));
191
192 if (instr->op == nir_op_imul_high) {
193 /* For channels where different_signs is set we have to perform a
194 * 64-bit negation. This is *not* the same as just negating the
195 * high 32-bits. Consider -3 * 2. The high 32-bits is 0, but the
196 * desired result is -1, not -0! Recall -x == ~x + 1.
197 */
198 nir_def *c1 = nir_imm_intN_t(b, 1, src0->bit_size);
199 hi = nir_bcsel(b, different_signs,
200 nir_iadd(b,
201 nir_inot(b, hi),
202 nir_uadd_carry(b, nir_inot(b, lo), c1)),
203 hi);
204 }
205
206 lowered = hi;
207 }
208 }
209 break;
210
211 default:
212 break;
213 }
214
215 if (lowered) {
216 nir_def_rewrite_uses(&instr->def, lowered);
217 nir_instr_remove(&instr->instr);
218 return true;
219 } else {
220 return false;
221 }
222 }
223
224 bool
nir_lower_alu(nir_shader * shader)225 nir_lower_alu(nir_shader *shader)
226 {
227 if (!shader->options->lower_bitfield_reverse &&
228 !shader->options->lower_bit_count &&
229 !shader->options->lower_mul_high)
230 return false;
231
232 return nir_shader_instructions_pass(shader, lower_alu_instr,
233 nir_metadata_block_index |
234 nir_metadata_dominance,
235 NULL);
236 }
237