1 /*
2 * Copyright © 2019 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "nir.h"
25
26 /* This pass optimizes GL access qualifiers. So far it does three things:
27 *
28 * - Infer readonly when it's missing.
29 * - Infer writeonly when it's missing.
30 * - Infer ACCESS_CAN_REORDER when the following are true:
31 * - Either there are no writes, or ACCESS_NON_WRITEABLE is set. In either
32 * case there are no writes to the underlying memory.
33 * - ACCESS_VOLATILE is not set.
34 *
35 * If these conditions are true, then image and buffer reads may be treated as
36 * if they were uniform buffer reads, i.e. they may be arbitrarily moved,
37 * combined, rematerialized etc.
38 */
39
40 struct access_state {
41 nir_shader *shader;
42 bool infer_non_readable;
43
44 struct set *vars_written;
45 struct set *vars_read;
46 bool images_written;
47 bool buffers_written;
48 bool images_read;
49 bool buffers_read;
50 };
51
52 static void
gather_buffer_access(struct access_state * state,nir_ssa_def * def,bool read,bool write)53 gather_buffer_access(struct access_state *state, nir_ssa_def *def, bool read, bool write)
54 {
55 state->buffers_read |= read;
56 state->buffers_written |= write;
57
58 if (!def)
59 return;
60
61 const nir_variable *var = nir_get_binding_variable(
62 state->shader, nir_chase_binding(nir_src_for_ssa(def)));
63 if (var) {
64 if (read)
65 _mesa_set_add(state->vars_read, var);
66 if (write)
67 _mesa_set_add(state->vars_written, var);
68 } else {
69 nir_foreach_variable_with_modes(possible_var, state->shader, nir_var_mem_ssbo) {
70 if (read)
71 _mesa_set_add(state->vars_read, possible_var);
72 if (write)
73 _mesa_set_add(state->vars_written, possible_var);
74 }
75 }
76 }
77
78 static void
gather_intrinsic(struct access_state * state,nir_intrinsic_instr * instr)79 gather_intrinsic(struct access_state *state, nir_intrinsic_instr *instr)
80 {
81 const nir_variable *var;
82 bool read, write;
83 switch (instr->intrinsic) {
84 case nir_intrinsic_image_deref_load:
85 case nir_intrinsic_image_deref_store:
86 case nir_intrinsic_image_deref_sparse_load:
87 case nir_intrinsic_image_deref_atomic_add:
88 case nir_intrinsic_image_deref_atomic_imin:
89 case nir_intrinsic_image_deref_atomic_umin:
90 case nir_intrinsic_image_deref_atomic_imax:
91 case nir_intrinsic_image_deref_atomic_umax:
92 case nir_intrinsic_image_deref_atomic_and:
93 case nir_intrinsic_image_deref_atomic_or:
94 case nir_intrinsic_image_deref_atomic_xor:
95 case nir_intrinsic_image_deref_atomic_exchange:
96 case nir_intrinsic_image_deref_atomic_comp_swap:
97 case nir_intrinsic_image_deref_atomic_fadd:
98 case nir_intrinsic_image_deref_atomic_fmin:
99 case nir_intrinsic_image_deref_atomic_fmax:
100 var = nir_intrinsic_get_var(instr, 0);
101 read = instr->intrinsic != nir_intrinsic_image_deref_store;
102 write = instr->intrinsic != nir_intrinsic_image_deref_load &&
103 instr->intrinsic != nir_intrinsic_image_deref_sparse_load;
104
105 /* In OpenGL, buffer images use normal buffer objects, whereas other
106 * image types use textures which cannot alias with buffer objects.
107 * Therefore we have to group buffer samplers together with SSBO's.
108 */
109 if (glsl_get_sampler_dim(glsl_without_array(var->type)) ==
110 GLSL_SAMPLER_DIM_BUF) {
111 state->buffers_read |= read;
112 state->buffers_written |= write;
113 } else {
114 state->images_read |= read;
115 state->images_written |= write;
116 }
117
118 if ((var->data.mode == nir_var_uniform ||
119 var->data.mode == nir_var_image) && read)
120 _mesa_set_add(state->vars_read, var);
121 if ((var->data.mode == nir_var_uniform ||
122 var->data.mode == nir_var_image) && write)
123 _mesa_set_add(state->vars_written, var);
124 break;
125
126 case nir_intrinsic_bindless_image_load:
127 case nir_intrinsic_bindless_image_store:
128 case nir_intrinsic_bindless_image_sparse_load:
129 case nir_intrinsic_bindless_image_atomic_add:
130 case nir_intrinsic_bindless_image_atomic_imin:
131 case nir_intrinsic_bindless_image_atomic_umin:
132 case nir_intrinsic_bindless_image_atomic_imax:
133 case nir_intrinsic_bindless_image_atomic_umax:
134 case nir_intrinsic_bindless_image_atomic_and:
135 case nir_intrinsic_bindless_image_atomic_or:
136 case nir_intrinsic_bindless_image_atomic_xor:
137 case nir_intrinsic_bindless_image_atomic_exchange:
138 case nir_intrinsic_bindless_image_atomic_comp_swap:
139 case nir_intrinsic_bindless_image_atomic_fadd:
140 case nir_intrinsic_bindless_image_atomic_fmin:
141 case nir_intrinsic_bindless_image_atomic_fmax:
142 read = instr->intrinsic != nir_intrinsic_bindless_image_store;
143 write = instr->intrinsic != nir_intrinsic_bindless_image_load &&
144 instr->intrinsic != nir_intrinsic_bindless_image_sparse_load;
145
146 if (nir_intrinsic_image_dim(instr) == GLSL_SAMPLER_DIM_BUF) {
147 state->buffers_read |= read;
148 state->buffers_written |= write;
149 } else {
150 state->images_read |= read;
151 state->images_written |= write;
152 }
153 break;
154
155 case nir_intrinsic_load_deref:
156 case nir_intrinsic_store_deref:
157 case nir_intrinsic_deref_atomic_add:
158 case nir_intrinsic_deref_atomic_imin:
159 case nir_intrinsic_deref_atomic_umin:
160 case nir_intrinsic_deref_atomic_imax:
161 case nir_intrinsic_deref_atomic_umax:
162 case nir_intrinsic_deref_atomic_and:
163 case nir_intrinsic_deref_atomic_or:
164 case nir_intrinsic_deref_atomic_xor:
165 case nir_intrinsic_deref_atomic_exchange:
166 case nir_intrinsic_deref_atomic_comp_swap:
167 case nir_intrinsic_deref_atomic_fadd:
168 case nir_intrinsic_deref_atomic_fmin:
169 case nir_intrinsic_deref_atomic_fmax:
170 case nir_intrinsic_deref_atomic_fcomp_swap: {
171 nir_deref_instr *deref = nir_src_as_deref(instr->src[0]);
172 if (!nir_deref_mode_may_be(deref, nir_var_mem_ssbo | nir_var_mem_global))
173 break;
174
175 bool ssbo = nir_deref_mode_is(deref, nir_var_mem_ssbo);
176 gather_buffer_access(state, ssbo ? instr->src[0].ssa : NULL,
177 instr->intrinsic != nir_intrinsic_store_deref,
178 instr->intrinsic != nir_intrinsic_load_deref);
179 break;
180 }
181
182 default:
183 break;
184 }
185 }
186
187 static bool
process_variable(struct access_state * state,nir_variable * var)188 process_variable(struct access_state *state, nir_variable *var)
189 {
190 const struct glsl_type *type = glsl_without_array(var->type);
191 if (var->data.mode != nir_var_mem_ssbo &&
192 !(var->data.mode == nir_var_uniform && glsl_type_is_image(type)) &&
193 var->data.mode != nir_var_image)
194 return false;
195
196 /* Ignore variables we've already marked */
197 if (var->data.access & ACCESS_CAN_REORDER)
198 return false;
199
200 unsigned access = var->data.access;
201 bool is_buffer = var->data.mode == nir_var_mem_ssbo ||
202 glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF;
203
204 if (!(access & ACCESS_NON_WRITEABLE)) {
205 if (is_buffer ? !state->buffers_written : !state->images_written)
206 access |= ACCESS_NON_WRITEABLE;
207 else if ((access & ACCESS_RESTRICT) && !_mesa_set_search(state->vars_written, var))
208 access |= ACCESS_NON_WRITEABLE;
209 }
210
211 if (state->infer_non_readable && !(access & ACCESS_NON_READABLE)) {
212 if (is_buffer ? !state->buffers_read : !state->images_read)
213 access |= ACCESS_NON_READABLE;
214 else if ((access & ACCESS_RESTRICT) && !_mesa_set_search(state->vars_read, var))
215 access |= ACCESS_NON_READABLE;
216 }
217
218 bool changed = var->data.access != access;
219 var->data.access = access;
220 return changed;
221 }
222
223 static bool
update_access(struct access_state * state,nir_intrinsic_instr * instr,bool is_buffer,bool is_global)224 update_access(struct access_state *state, nir_intrinsic_instr *instr, bool is_buffer, bool is_global)
225 {
226 enum gl_access_qualifier access = nir_intrinsic_access(instr);
227
228 bool is_memory_readonly = access & ACCESS_NON_WRITEABLE;
229 bool is_memory_writeonly = access & ACCESS_NON_READABLE;
230
231 if (instr->intrinsic != nir_intrinsic_bindless_image_load &&
232 instr->intrinsic != nir_intrinsic_bindless_image_store &&
233 instr->intrinsic != nir_intrinsic_bindless_image_sparse_load &&
234 !is_global) {
235 const nir_variable *var = nir_get_binding_variable(
236 state->shader, nir_chase_binding(instr->src[0]));
237 is_memory_readonly |= var && (var->data.access & ACCESS_NON_WRITEABLE);
238 is_memory_writeonly |= var && (var->data.access & ACCESS_NON_READABLE);
239 }
240
241 if (is_global) {
242 is_memory_readonly |= !state->buffers_written && !state->images_written;
243 is_memory_writeonly |= !state->buffers_read && !state->images_read;
244 } else {
245 is_memory_readonly |= is_buffer ? !state->buffers_written : !state->images_written;
246 is_memory_writeonly |= is_buffer ? !state->buffers_read : !state->images_read;
247 }
248
249 if (is_memory_readonly)
250 access |= ACCESS_NON_WRITEABLE;
251 if (state->infer_non_readable && is_memory_writeonly)
252 access |= ACCESS_NON_READABLE;
253 if (!(access & ACCESS_VOLATILE) && is_memory_readonly)
254 access |= ACCESS_CAN_REORDER;
255
256 bool progress = nir_intrinsic_access(instr) != access;
257 nir_intrinsic_set_access(instr, access);
258 return progress;
259 }
260
261 static bool
process_intrinsic(struct access_state * state,nir_intrinsic_instr * instr)262 process_intrinsic(struct access_state *state, nir_intrinsic_instr *instr)
263 {
264 switch (instr->intrinsic) {
265 case nir_intrinsic_bindless_image_load:
266 case nir_intrinsic_bindless_image_store:
267 case nir_intrinsic_bindless_image_sparse_load:
268 return update_access(state, instr, nir_intrinsic_image_dim(instr) == GLSL_SAMPLER_DIM_BUF,
269 false);
270
271 case nir_intrinsic_load_deref:
272 case nir_intrinsic_store_deref: {
273 if (nir_deref_mode_is(nir_src_as_deref(instr->src[0]), nir_var_mem_global))
274 return update_access(state, instr, false, true);
275 else if (nir_deref_mode_is(nir_src_as_deref(instr->src[0]), nir_var_mem_ssbo))
276 return update_access(state, instr, true, false);
277 else
278 return false;
279 }
280
281 case nir_intrinsic_image_deref_load:
282 case nir_intrinsic_image_deref_store:
283 case nir_intrinsic_image_deref_sparse_load: {
284 nir_variable *var = nir_intrinsic_get_var(instr, 0);
285
286 bool is_buffer =
287 glsl_get_sampler_dim(glsl_without_array(var->type)) == GLSL_SAMPLER_DIM_BUF;
288
289 return update_access(state, instr, is_buffer, false);
290 }
291
292 default:
293 return false;
294 }
295 }
296
297 static bool
opt_access_impl(struct access_state * state,nir_function_impl * impl)298 opt_access_impl(struct access_state *state,
299 nir_function_impl *impl)
300 {
301 bool progress = false;
302
303 nir_foreach_block(block, impl) {
304 nir_foreach_instr(instr, block) {
305 if (instr->type == nir_instr_type_intrinsic)
306 progress |= process_intrinsic(state,
307 nir_instr_as_intrinsic(instr));
308 }
309 }
310
311 if (progress) {
312 nir_metadata_preserve(impl,
313 nir_metadata_block_index |
314 nir_metadata_dominance |
315 nir_metadata_live_ssa_defs |
316 nir_metadata_loop_analysis);
317 }
318
319
320 return progress;
321 }
322
323 bool
nir_opt_access(nir_shader * shader,const nir_opt_access_options * options)324 nir_opt_access(nir_shader *shader, const nir_opt_access_options *options)
325 {
326 struct access_state state = {
327 .shader = shader,
328 .infer_non_readable = options->infer_non_readable,
329 .vars_written = _mesa_pointer_set_create(NULL),
330 .vars_read = _mesa_pointer_set_create(NULL),
331 };
332
333 bool var_progress = false;
334 bool progress = false;
335
336 nir_foreach_function(func, shader) {
337 if (func->impl) {
338 nir_foreach_block(block, func->impl) {
339 nir_foreach_instr(instr, block) {
340 if (instr->type == nir_instr_type_intrinsic)
341 gather_intrinsic(&state, nir_instr_as_intrinsic(instr));
342 }
343 }
344 }
345 }
346
347 /* In Vulkan, buffers and images can alias. */
348 if (options->is_vulkan) {
349 state.buffers_written |= state.images_written;
350 state.images_written |= state.buffers_written;
351 state.buffers_read |= state.images_read;
352 state.images_read |= state.buffers_read;
353 }
354
355 nir_foreach_variable_with_modes(var, shader, nir_var_uniform |
356 nir_var_mem_ubo |
357 nir_var_mem_ssbo |
358 nir_var_image)
359 var_progress |= process_variable(&state, var);
360
361 nir_foreach_function(func, shader) {
362 if (func->impl) {
363 progress |= opt_access_impl(&state, func->impl);
364
365 /* If we make a change to the uniforms, update all the impls. */
366 if (var_progress) {
367 nir_metadata_preserve(func->impl,
368 nir_metadata_block_index |
369 nir_metadata_dominance |
370 nir_metadata_live_ssa_defs |
371 nir_metadata_loop_analysis);
372 }
373 }
374 }
375
376 progress |= var_progress;
377
378 _mesa_set_destroy(state.vars_read, NULL);
379 _mesa_set_destroy(state.vars_written, NULL);
380 return progress;
381 }
382