1 /*
2 * Copyright (C) 2018-2020 Collabora, Ltd.
3 * Copyright (C) 2019-2020 Icecream95
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24
25 #include "pan_ir.h"
26 #include "compiler/nir/nir_builder.h"
27
28 /* Midgard can write all of color, depth and stencil in a single writeout
29 * operation, so we merge depth/stencil stores with color stores.
30 * If there are no color stores, we add a write to the "depth RT".
31 *
32 * For Bifrost, we want these combined so we can properly order
33 * +ZS_EMIT with respect to +ATEST and +BLEND, as well as combining
34 * depth/stencil stores into a single +ZS_EMIT op.
35 */
36 bool
pan_nir_lower_zs_store(nir_shader * nir)37 pan_nir_lower_zs_store(nir_shader *nir)
38 {
39 if (nir->info.stage != MESA_SHADER_FRAGMENT)
40 return false;
41
42 nir_variable *z_var = NULL, *s_var = NULL;
43
44 nir_foreach_shader_out_variable(var, nir) {
45 if (var->data.location == FRAG_RESULT_DEPTH)
46 z_var = var;
47 else if (var->data.location == FRAG_RESULT_STENCIL)
48 s_var = var;
49 }
50
51 if (!z_var && !s_var)
52 return false;
53
54 bool progress = false;
55
56 nir_foreach_function(function, nir) {
57 if (!function->impl) continue;
58
59 nir_intrinsic_instr *z_store = NULL, *s_store = NULL;
60
61 nir_foreach_block(block, function->impl) {
62 nir_foreach_instr_safe(instr, block) {
63 if (instr->type != nir_instr_type_intrinsic)
64 continue;
65
66 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
67 if (intr->intrinsic != nir_intrinsic_store_output)
68 continue;
69
70 if (z_var && nir_intrinsic_base(intr) == z_var->data.driver_location) {
71 assert(!z_store);
72 z_store = intr;
73 }
74
75 if (s_var && nir_intrinsic_base(intr) == s_var->data.driver_location) {
76 assert(!s_store);
77 s_store = intr;
78 }
79 }
80 }
81
82 if (!z_store && !s_store) continue;
83
84 bool replaced = false;
85
86 nir_foreach_block(block, function->impl) {
87 nir_foreach_instr_safe(instr, block) {
88 if (instr->type != nir_instr_type_intrinsic)
89 continue;
90
91 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
92 if (intr->intrinsic != nir_intrinsic_store_output)
93 continue;
94
95 const nir_variable *var = nir_find_variable_with_driver_location(nir, nir_var_shader_out, nir_intrinsic_base(intr));
96 assert(var);
97
98 if (var->data.location != FRAG_RESULT_COLOR &&
99 var->data.location < FRAG_RESULT_DATA0)
100 continue;
101
102 if (var->data.index)
103 continue;
104
105 assert(nir_src_is_const(intr->src[1]) && "no indirect outputs");
106
107 nir_builder b;
108 nir_builder_init(&b, function->impl);
109
110 assert(!z_store || z_store->instr.block == instr->block);
111 assert(!s_store || s_store->instr.block == instr->block);
112 b.cursor = nir_after_block_before_jump(instr->block);
113
114 nir_intrinsic_instr *combined_store;
115 combined_store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_store_combined_output_pan);
116
117 combined_store->num_components = intr->src[0].ssa->num_components;
118
119 nir_intrinsic_set_base(combined_store, nir_intrinsic_base(intr));
120 nir_intrinsic_set_src_type(combined_store, nir_intrinsic_src_type(intr));
121
122 unsigned writeout = PAN_WRITEOUT_C;
123 if (z_store)
124 writeout |= PAN_WRITEOUT_Z;
125 if (s_store)
126 writeout |= PAN_WRITEOUT_S;
127
128 nir_intrinsic_set_component(combined_store, writeout);
129
130 struct nir_ssa_def *zero = nir_imm_int(&b, 0);
131
132 struct nir_ssa_def *src[4] = {
133 intr->src[0].ssa,
134 intr->src[1].ssa,
135 z_store ? z_store->src[0].ssa : zero,
136 s_store ? s_store->src[0].ssa : zero,
137 };
138
139 for (int i = 0; i < 4; ++i)
140 combined_store->src[i] = nir_src_for_ssa(src[i]);
141
142 nir_builder_instr_insert(&b, &combined_store->instr);
143
144 nir_instr_remove(instr);
145
146 replaced = true;
147 }
148 }
149
150 /* Insert a store to the depth RT (0xff) if needed */
151 if (!replaced) {
152 nir_builder b;
153 nir_builder_init(&b, function->impl);
154
155 nir_block *block = NULL;
156 if (z_store && s_store)
157 assert(z_store->instr.block == s_store->instr.block);
158
159 if (z_store)
160 block = z_store->instr.block;
161 else
162 block = s_store->instr.block;
163
164 b.cursor = nir_after_block_before_jump(block);
165
166 nir_intrinsic_instr *combined_store;
167 combined_store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_store_combined_output_pan);
168
169 combined_store->num_components = 4;
170
171 unsigned base;
172 if (z_store)
173 base = nir_intrinsic_base(z_store);
174 else
175 base = nir_intrinsic_base(s_store);
176 nir_intrinsic_set_base(combined_store, base);
177 nir_intrinsic_set_src_type(combined_store, nir_type_float32);
178
179 unsigned writeout = 0;
180 if (z_store)
181 writeout |= PAN_WRITEOUT_Z;
182 if (s_store)
183 writeout |= PAN_WRITEOUT_S;
184
185 nir_intrinsic_set_component(combined_store, writeout);
186
187 struct nir_ssa_def *zero = nir_imm_int(&b, 0);
188
189 struct nir_ssa_def *src[4] = {
190 nir_imm_vec4(&b, 0, 0, 0, 0),
191 zero,
192 z_store ? z_store->src[0].ssa : zero,
193 s_store ? s_store->src[0].ssa : zero,
194 };
195
196 for (int i = 0; i < 4; ++i)
197 combined_store->src[i] = nir_src_for_ssa(src[i]);
198
199 nir_builder_instr_insert(&b, &combined_store->instr);
200 }
201
202 if (z_store)
203 nir_instr_remove(&z_store->instr);
204
205 if (s_store)
206 nir_instr_remove(&s_store->instr);
207
208 nir_metadata_preserve(function->impl, nir_metadata_block_index | nir_metadata_dominance);
209 progress = true;
210 }
211
212 return progress;
213 }
214
215 /* Real writeout stores, which break execution, need to be moved to after
216 * dual-source stores, which are just standard register writes. */
217 bool
pan_nir_reorder_writeout(nir_shader * nir)218 pan_nir_reorder_writeout(nir_shader *nir)
219 {
220 bool progress = false;
221
222 nir_foreach_function(function, nir) {
223 if (!function->impl) continue;
224
225 nir_foreach_block(block, function->impl) {
226 nir_instr *last_writeout = NULL;
227
228 nir_foreach_instr_reverse_safe(instr, block) {
229 if (instr->type != nir_instr_type_intrinsic)
230 continue;
231
232 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
233 if (intr->intrinsic != nir_intrinsic_store_output)
234 continue;
235
236 const nir_variable *var = nir_find_variable_with_driver_location(nir, nir_var_shader_out, nir_intrinsic_base(intr));
237
238 if (var->data.index) {
239 if (!last_writeout)
240 last_writeout = instr;
241 continue;
242 }
243
244 if (!last_writeout)
245 continue;
246
247 /* This is a real store, so move it to after dual-source stores */
248 exec_node_remove(&instr->node);
249 exec_node_insert_after(&last_writeout->node, &instr->node);
250
251 progress = true;
252 }
253 }
254 }
255
256 return progress;
257 }
258