1 /*
2 * Copyright © 2019 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "nir.h"
25 #include "nir/nir_builder.h"
26 #include "nir_constant_expressions.h"
27 #include "nir_control_flow.h"
28 #include "nir_loop_analyze.h"
29
30 static bool
is_two_src_comparison(const nir_alu_instr * instr)31 is_two_src_comparison(const nir_alu_instr *instr)
32 {
33 switch (instr->op) {
34 case nir_op_flt:
35 case nir_op_flt32:
36 case nir_op_fge:
37 case nir_op_fge32:
38 case nir_op_feq:
39 case nir_op_feq32:
40 case nir_op_fneu:
41 case nir_op_fneu32:
42 case nir_op_ilt:
43 case nir_op_ilt32:
44 case nir_op_ult:
45 case nir_op_ult32:
46 case nir_op_ige:
47 case nir_op_ige32:
48 case nir_op_uge:
49 case nir_op_uge32:
50 case nir_op_ieq:
51 case nir_op_ieq32:
52 case nir_op_ine:
53 case nir_op_ine32:
54 return true;
55 default:
56 return false;
57 }
58 }
59
60 static bool
all_srcs_are_ssa(const nir_alu_instr * instr)61 all_srcs_are_ssa(const nir_alu_instr *instr)
62 {
63 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
64 if (!instr->src[i].src.is_ssa)
65 return false;
66 }
67
68 return true;
69 }
70
71
72 static bool
all_uses_are_bcsel(const nir_alu_instr * instr)73 all_uses_are_bcsel(const nir_alu_instr *instr)
74 {
75 if (!instr->dest.dest.is_ssa)
76 return false;
77
78 nir_foreach_use(use, &instr->dest.dest.ssa) {
79 if (use->parent_instr->type != nir_instr_type_alu)
80 return false;
81
82 nir_alu_instr *const alu = nir_instr_as_alu(use->parent_instr);
83 if (alu->op != nir_op_bcsel &&
84 alu->op != nir_op_b32csel)
85 return false;
86
87 /* Not only must the result be used by a bcsel, but it must be used as
88 * the first source (the condition).
89 */
90 if (alu->src[0].src.ssa != &instr->dest.dest.ssa)
91 return false;
92 }
93
94 return true;
95 }
96
97 static bool
nir_opt_rematerialize_compares_impl(nir_shader * shader,nir_function_impl * impl)98 nir_opt_rematerialize_compares_impl(nir_shader *shader, nir_function_impl *impl)
99 {
100 bool progress = false;
101
102 nir_foreach_block(block, impl) {
103 nir_foreach_instr(instr, block) {
104 if (instr->type != nir_instr_type_alu)
105 continue;
106
107 nir_alu_instr *const alu = nir_instr_as_alu(instr);
108 if (!is_two_src_comparison(alu))
109 continue;
110
111 if (!all_srcs_are_ssa(alu))
112 continue;
113
114 if (!all_uses_are_bcsel(alu))
115 continue;
116
117 /* At this point it is known that alu is a comparison instruction
118 * that is only used by nir_op_bcsel and possibly by if-statements
119 * (though the latter has not been explicitly checked).
120 *
121 * Iterate through each use of the comparison. For every use (or use
122 * by an if-statement) that is in a different block, emit a copy of
123 * the comparison. Care must be taken here. The original
124 * instruction must be duplicated only once in each block because CSE
125 * cannot be run after this pass.
126 */
127 nir_foreach_use_safe(use, &alu->dest.dest.ssa) {
128 nir_instr *const use_instr = use->parent_instr;
129
130 /* If the use is in the same block as the def, don't
131 * rematerialize.
132 */
133 if (use_instr->block == alu->instr.block)
134 continue;
135
136 nir_alu_instr *clone = nir_alu_instr_clone(shader, alu);
137
138 nir_instr_insert_before(use_instr, &clone->instr);
139
140 nir_alu_instr *const use_alu = nir_instr_as_alu(use_instr);
141 for (unsigned i = 0; i < nir_op_infos[use_alu->op].num_inputs; i++) {
142 if (use_alu->src[i].src.ssa == &alu->dest.dest.ssa) {
143 nir_instr_rewrite_src(&use_alu->instr,
144 &use_alu->src[i].src,
145 nir_src_for_ssa(&clone->dest.dest.ssa));
146 progress = true;
147 }
148 }
149 }
150
151 nir_foreach_if_use_safe(use, &alu->dest.dest.ssa) {
152 nir_if *const if_stmt = use->parent_if;
153
154 nir_block *const prev_block =
155 nir_cf_node_as_block(nir_cf_node_prev(&if_stmt->cf_node));
156
157 /* If the compare is from the previous block, don't
158 * rematerialize.
159 */
160 if (prev_block == alu->instr.block)
161 continue;
162
163 nir_alu_instr *clone = nir_alu_instr_clone(shader, alu);
164
165 nir_instr_insert_after_block(prev_block, &clone->instr);
166
167 nir_if_rewrite_condition(if_stmt,
168 nir_src_for_ssa(&clone->dest.dest.ssa));
169 progress = true;
170 }
171 }
172 }
173
174 if (progress) {
175 nir_metadata_preserve(impl, nir_metadata_block_index |
176 nir_metadata_dominance);
177 } else {
178 nir_metadata_preserve(impl, nir_metadata_all);
179 }
180
181 return progress;
182 }
183
184 bool
nir_opt_rematerialize_compares(nir_shader * shader)185 nir_opt_rematerialize_compares(nir_shader *shader)
186 {
187 bool progress = false;
188
189 nir_foreach_function(function, shader) {
190 if (function->impl == NULL)
191 continue;
192
193 progress = nir_opt_rematerialize_compares_impl(shader, function->impl)
194 || progress;
195 }
196
197 return progress;
198 }
199