1 /*
2 * Copyright © 2014 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Connor Abbott (cwabbott0@gmail.com)
25 *
26 */
27
28 #include "compiler/nir/nir.h"
29 #include "compiler/nir/nir_builder.h"
30 #include "gl_nir.h"
31 #include "ir_uniform.h"
32 #include "main/config.h"
33 #include "main/shader_types.h"
34 #include <assert.h>
35
36 /*
37 * replace atomic counter intrinsics that use a variable with intrinsics
38 * that directly store the buffer index and byte offset
39 */
40
41 static bool
lower_deref_instr(nir_builder * b,nir_intrinsic_instr * instr,const struct gl_shader_program * shader_program,nir_shader * shader,bool use_binding_as_idx)42 lower_deref_instr(nir_builder *b, nir_intrinsic_instr *instr,
43 const struct gl_shader_program *shader_program,
44 nir_shader *shader, bool use_binding_as_idx)
45 {
46 nir_intrinsic_op op;
47 switch (instr->intrinsic) {
48 case nir_intrinsic_atomic_counter_read_deref:
49 op = nir_intrinsic_atomic_counter_read;
50 break;
51
52 case nir_intrinsic_atomic_counter_inc_deref:
53 op = nir_intrinsic_atomic_counter_inc;
54 break;
55
56 case nir_intrinsic_atomic_counter_pre_dec_deref:
57 op = nir_intrinsic_atomic_counter_pre_dec;
58 break;
59
60 case nir_intrinsic_atomic_counter_post_dec_deref:
61 op = nir_intrinsic_atomic_counter_post_dec;
62 break;
63
64 case nir_intrinsic_atomic_counter_add_deref:
65 op = nir_intrinsic_atomic_counter_add;
66 break;
67
68 case nir_intrinsic_atomic_counter_min_deref:
69 op = nir_intrinsic_atomic_counter_min;
70 break;
71
72 case nir_intrinsic_atomic_counter_max_deref:
73 op = nir_intrinsic_atomic_counter_max;
74 break;
75
76 case nir_intrinsic_atomic_counter_and_deref:
77 op = nir_intrinsic_atomic_counter_and;
78 break;
79
80 case nir_intrinsic_atomic_counter_or_deref:
81 op = nir_intrinsic_atomic_counter_or;
82 break;
83
84 case nir_intrinsic_atomic_counter_xor_deref:
85 op = nir_intrinsic_atomic_counter_xor;
86 break;
87
88 case nir_intrinsic_atomic_counter_exchange_deref:
89 op = nir_intrinsic_atomic_counter_exchange;
90 break;
91
92 case nir_intrinsic_atomic_counter_comp_swap_deref:
93 op = nir_intrinsic_atomic_counter_comp_swap;
94 break;
95
96 default:
97 return false;
98 }
99
100 nir_deref_instr *deref = nir_src_as_deref(instr->src[0]);
101 nir_variable *var = nir_deref_instr_get_variable(deref);
102
103 if (var->data.mode != nir_var_uniform &&
104 var->data.mode != nir_var_mem_ssbo &&
105 var->data.mode != nir_var_mem_shared)
106 return false; /* atomics passed as function arguments can't be lowered */
107
108 const unsigned uniform_loc = var->data.location;
109 const unsigned idx = use_binding_as_idx ? var->data.binding :
110 shader_program->data->UniformStorage[uniform_loc].opaque[shader->info.stage].index;
111
112 b->cursor = nir_before_instr(&instr->instr);
113
114 nir_ssa_def *offset = nir_imm_int(b, var->data.offset);
115 for (nir_deref_instr *d = deref; d->deref_type != nir_deref_type_var;
116 d = nir_deref_instr_parent(d)) {
117 assert(d->deref_type == nir_deref_type_array);
118 assert(d->arr.index.is_ssa);
119
120 unsigned array_stride = ATOMIC_COUNTER_SIZE;
121 if (glsl_type_is_array(d->type))
122 array_stride *= glsl_get_aoa_size(d->type);
123
124 offset = nir_iadd(b, offset, nir_imul(b, d->arr.index.ssa,
125 nir_imm_int(b, array_stride)));
126 }
127
128 /* Since the first source is a deref and the first source in the lowered
129 * instruction is the offset, we can just swap it out and change the
130 * opcode.
131 */
132 instr->intrinsic = op;
133 nir_instr_rewrite_src(&instr->instr, &instr->src[0],
134 nir_src_for_ssa(offset));
135 nir_intrinsic_set_base(instr, idx);
136
137 nir_deref_instr_remove_if_unused(deref);
138
139 return true;
140 }
141
142 bool
gl_nir_lower_atomics(nir_shader * shader,const struct gl_shader_program * shader_program,bool use_binding_as_idx)143 gl_nir_lower_atomics(nir_shader *shader,
144 const struct gl_shader_program *shader_program,
145 bool use_binding_as_idx)
146 {
147 bool progress = false;
148
149 nir_foreach_function(function, shader) {
150 if (!function->impl)
151 continue;
152
153 bool impl_progress = false;
154
155 nir_builder build;
156 nir_builder_init(&build, function->impl);
157
158 nir_foreach_block(block, function->impl) {
159 nir_foreach_instr_safe(instr, block) {
160 if (instr->type != nir_instr_type_intrinsic)
161 continue;
162
163 impl_progress |= lower_deref_instr(&build,
164 nir_instr_as_intrinsic(instr),
165 shader_program, shader,
166 use_binding_as_idx);
167 }
168 }
169
170 if (impl_progress) {
171 nir_metadata_preserve(function->impl, nir_metadata_block_index |
172 nir_metadata_dominance);
173 progress = true;
174 }
175 }
176
177 return progress;
178 }
179