1 #include "ir3_nir.h"
2
3 #include "nir.h"
4 #include "nir_builder.h"
5 #include "nir_search.h"
6 #include "nir_search_helpers.h"
7
8 /* What follows is NIR algebraic transform code for the following 2
9 * transforms:
10 * ('imul', 'a@32', 'b@32') => ('imadsh_mix16', 'b', 'a', ('imadsh_mix16', 'a', 'b', ('umul_low', 'a', 'b')))
11 * ('iadd', ('imul24', 'a', 'b'), 'c') => ('imad24_ir3', 'a', 'b', 'c')
12 */
13
14
15 static const nir_search_variable search0_0 = {
16 { nir_search_value_variable, 32 },
17 0, /* a */
18 false,
19 nir_type_invalid,
20 NULL,
21 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
22 };
23
24 static const nir_search_variable search0_1 = {
25 { nir_search_value_variable, 32 },
26 1, /* b */
27 false,
28 nir_type_invalid,
29 NULL,
30 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
31 };
32 static const nir_search_expression search0 = {
33 { nir_search_value_expression, 32 },
34 false, false,
35 0, 1,
36 nir_op_imul,
37 { &search0_0.value, &search0_1.value },
38 NULL,
39 };
40
41 /* replace0_0 -> search0_1 in the cache */
42
43 /* replace0_1 -> search0_0 in the cache */
44
45 /* replace0_2_0 -> search0_0 in the cache */
46
47 /* replace0_2_1 -> search0_1 in the cache */
48
49 /* replace0_2_2_0 -> search0_0 in the cache */
50
51 /* replace0_2_2_1 -> search0_1 in the cache */
52 static const nir_search_expression replace0_2_2 = {
53 { nir_search_value_expression, 32 },
54 false, false,
55 0, 1,
56 nir_op_umul_low,
57 { &search0_0.value, &search0_1.value },
58 NULL,
59 };
60 static const nir_search_expression replace0_2 = {
61 { nir_search_value_expression, 32 },
62 false, false,
63 -1, 1,
64 nir_op_imadsh_mix16,
65 { &search0_0.value, &search0_1.value, &replace0_2_2.value },
66 NULL,
67 };
68 static const nir_search_expression replace0 = {
69 { nir_search_value_expression, 32 },
70 false, false,
71 -1, 1,
72 nir_op_imadsh_mix16,
73 { &search0_1.value, &search0_0.value, &replace0_2.value },
74 NULL,
75 };
76
77 /* search1_0_0 -> search0_0 in the cache */
78
79 /* search1_0_1 -> search0_1 in the cache */
80 static const nir_search_expression search1_0 = {
81 { nir_search_value_expression, 32 },
82 false, false,
83 1, 1,
84 nir_op_imul24,
85 { &search0_0.value, &search0_1.value },
86 NULL,
87 };
88
89 static const nir_search_variable search1_1 = {
90 { nir_search_value_variable, 32 },
91 2, /* c */
92 false,
93 nir_type_invalid,
94 NULL,
95 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
96 };
97 static const nir_search_expression search1 = {
98 { nir_search_value_expression, 32 },
99 false, false,
100 0, 2,
101 nir_op_iadd,
102 { &search1_0.value, &search1_1.value },
103 NULL,
104 };
105
106 /* replace1_0 -> search0_0 in the cache */
107
108 /* replace1_1 -> search0_1 in the cache */
109
110 /* replace1_2 -> search1_1 in the cache */
111 static const nir_search_expression replace1 = {
112 { nir_search_value_expression, 32 },
113 false, false,
114 0, 1,
115 nir_op_imad24_ir3,
116 { &search0_0.value, &search0_1.value, &search1_1.value },
117 NULL,
118 };
119
120
121 static const struct transform ir3_nir_lower_imul_state2_xforms[] = {
122 { &search0, &replace0.value, 0 },
123 };
124 static const struct transform ir3_nir_lower_imul_state4_xforms[] = {
125 { &search1, &replace1.value, 0 },
126 };
127
128 static const struct per_op_table ir3_nir_lower_imul_table[nir_num_search_ops] = {
129 [nir_op_imul] = {
130 .filter = (uint16_t []) {
131 0,
132 0,
133 0,
134 0,
135 0,
136 },
137
138 .num_filtered_states = 1,
139 .table = (uint16_t []) {
140
141 2,
142 },
143 },
144 [nir_op_iadd] = {
145 .filter = (uint16_t []) {
146 0,
147 0,
148 0,
149 1,
150 0,
151 },
152
153 .num_filtered_states = 2,
154 .table = (uint16_t []) {
155
156 0,
157 4,
158 4,
159 4,
160 },
161 },
162 [nir_op_imul24] = {
163 .filter = (uint16_t []) {
164 0,
165 0,
166 0,
167 0,
168 0,
169 },
170
171 .num_filtered_states = 1,
172 .table = (uint16_t []) {
173
174 3,
175 },
176 },
177 };
178
179 const struct transform *ir3_nir_lower_imul_transforms[] = {
180 NULL,
181 NULL,
182 ir3_nir_lower_imul_state2_xforms,
183 NULL,
184 ir3_nir_lower_imul_state4_xforms,
185 };
186
187 const uint16_t ir3_nir_lower_imul_transform_counts[] = {
188 0,
189 0,
190 (uint16_t)ARRAY_SIZE(ir3_nir_lower_imul_state2_xforms),
191 0,
192 (uint16_t)ARRAY_SIZE(ir3_nir_lower_imul_state4_xforms),
193 };
194
195 bool
ir3_nir_lower_imul(nir_shader * shader)196 ir3_nir_lower_imul(nir_shader *shader)
197 {
198 bool progress = false;
199 bool condition_flags[1];
200 const nir_shader_compiler_options *options = shader->options;
201 const shader_info *info = &shader->info;
202 (void) options;
203 (void) info;
204
205 condition_flags[0] = true;
206
207 nir_foreach_function(function, shader) {
208 if (function->impl) {
209 progress |= nir_algebraic_impl(function->impl, condition_flags,
210 ir3_nir_lower_imul_transforms,
211 ir3_nir_lower_imul_transform_counts,
212 ir3_nir_lower_imul_table);
213 }
214 }
215
216 return progress;
217 }
218
219