1 /*
2 * Copyright © 2014 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Connor Abbott (cwabbott0@gmail.com)
25 *
26 */
27
28 #include "compiler/nir/nir.h"
29 #include "compiler/nir/nir_builder.h"
30 #include "gl_nir.h"
31 #include "main/config.h"
32 #include "main/shader_types.h"
33 #include <assert.h>
34
35 /*
36 * replace atomic counter intrinsics that use a variable with intrinsics
37 * that directly store the buffer index and byte offset
38 */
39
40 static bool
lower_deref_instr(nir_builder * b,nir_intrinsic_instr * instr,const struct gl_shader_program * shader_program,nir_shader * shader,bool use_binding_as_idx)41 lower_deref_instr(nir_builder *b, nir_intrinsic_instr *instr,
42 const struct gl_shader_program *shader_program,
43 nir_shader *shader, bool use_binding_as_idx)
44 {
45 nir_intrinsic_op op;
46 switch (instr->intrinsic) {
47 case nir_intrinsic_atomic_counter_read_deref:
48 op = nir_intrinsic_atomic_counter_read;
49 break;
50
51 case nir_intrinsic_atomic_counter_inc_deref:
52 op = nir_intrinsic_atomic_counter_inc;
53 break;
54
55 case nir_intrinsic_atomic_counter_pre_dec_deref:
56 op = nir_intrinsic_atomic_counter_pre_dec;
57 break;
58
59 case nir_intrinsic_atomic_counter_post_dec_deref:
60 op = nir_intrinsic_atomic_counter_post_dec;
61 break;
62
63 case nir_intrinsic_atomic_counter_add_deref:
64 op = nir_intrinsic_atomic_counter_add;
65 break;
66
67 case nir_intrinsic_atomic_counter_min_deref:
68 op = nir_intrinsic_atomic_counter_min;
69 break;
70
71 case nir_intrinsic_atomic_counter_max_deref:
72 op = nir_intrinsic_atomic_counter_max;
73 break;
74
75 case nir_intrinsic_atomic_counter_and_deref:
76 op = nir_intrinsic_atomic_counter_and;
77 break;
78
79 case nir_intrinsic_atomic_counter_or_deref:
80 op = nir_intrinsic_atomic_counter_or;
81 break;
82
83 case nir_intrinsic_atomic_counter_xor_deref:
84 op = nir_intrinsic_atomic_counter_xor;
85 break;
86
87 case nir_intrinsic_atomic_counter_exchange_deref:
88 op = nir_intrinsic_atomic_counter_exchange;
89 break;
90
91 case nir_intrinsic_atomic_counter_comp_swap_deref:
92 op = nir_intrinsic_atomic_counter_comp_swap;
93 break;
94
95 default:
96 return false;
97 }
98
99 nir_deref_instr *deref = nir_src_as_deref(instr->src[0]);
100 nir_variable *var = nir_deref_instr_get_variable(deref);
101
102 if (var->data.mode != nir_var_uniform &&
103 var->data.mode != nir_var_mem_ssbo &&
104 var->data.mode != nir_var_mem_shared)
105 return false; /* atomics passed as function arguments can't be lowered */
106
107 const unsigned uniform_loc = var->data.location;
108 const unsigned idx = use_binding_as_idx ? var->data.binding :
109 shader_program->data->UniformStorage[uniform_loc].opaque[shader->info.stage].index;
110
111 b->cursor = nir_before_instr(&instr->instr);
112
113 int offset_value = 0;
114 int range_base = 0;
115 if (!b->shader->options->lower_atomic_offset_to_range_base)
116 offset_value = var->data.offset;
117 else
118 range_base = var->data.offset;
119
120 nir_def *offset = nir_imm_int(b, offset_value);
121 for (nir_deref_instr *d = deref; d->deref_type != nir_deref_type_var;
122 d = nir_deref_instr_parent(d)) {
123 assert(d->deref_type == nir_deref_type_array);
124
125 unsigned array_stride = ATOMIC_COUNTER_SIZE;
126 if (glsl_type_is_array(d->type))
127 array_stride *= glsl_get_aoa_size(d->type);
128
129 offset = nir_iadd(b, offset, nir_imul(b, d->arr.index.ssa,
130 nir_imm_int(b, array_stride)));
131 }
132
133 /* Since the first source is a deref and the first source in the lowered
134 * instruction is the offset, we can just swap it out and change the
135 * opcode.
136 */
137 instr->intrinsic = op;
138 nir_intrinsic_set_range_base(instr, range_base);
139
140 nir_src_rewrite(&instr->src[0], offset);
141 nir_intrinsic_set_base(instr, idx);
142
143 nir_deref_instr_remove_if_unused(deref);
144
145 return true;
146 }
147
148 struct lower_atomics_data {
149 bool use_binding_as_idx;
150 nir_shader *shader;
151 const struct gl_shader_program *shader_program;
152 };
153
154 static bool
gl_nir_lower_atomics_instr(nir_builder * b,nir_instr * instr,void * cb_data)155 gl_nir_lower_atomics_instr(nir_builder *b, nir_instr *instr, void *cb_data)
156 {
157 if (instr->type != nir_instr_type_intrinsic)
158 return false;
159
160 struct lower_atomics_data *data = cb_data;
161
162 return lower_deref_instr(b,
163 nir_instr_as_intrinsic(instr),
164 data->shader_program,
165 data->shader,
166 data->use_binding_as_idx);
167 }
168
169 bool
gl_nir_lower_atomics(nir_shader * shader,const struct gl_shader_program * shader_program,bool use_binding_as_idx)170 gl_nir_lower_atomics(nir_shader *shader,
171 const struct gl_shader_program *shader_program,
172 bool use_binding_as_idx)
173 {
174 struct lower_atomics_data data = {
175 .use_binding_as_idx = use_binding_as_idx,
176 .shader = shader,
177 .shader_program = shader_program,
178 };
179
180 return nir_shader_instructions_pass(shader, gl_nir_lower_atomics_instr,
181 nir_metadata_control_flow,
182 &data);
183 }
184