1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "nir.h"
25 #include "nir_builder.h"
26
27 static void
build_constant_load(nir_builder * b,nir_deref_instr * deref,nir_constant * c)28 build_constant_load(nir_builder *b, nir_deref_instr *deref, nir_constant *c)
29 {
30 if (glsl_type_is_vector_or_scalar(deref->type)) {
31 nir_load_const_instr *load =
32 nir_load_const_instr_create(b->shader,
33 glsl_get_vector_elements(deref->type),
34 glsl_get_bit_size(deref->type));
35 memcpy(load->value, c->values, sizeof(*load->value) * load->def.num_components);
36 nir_builder_instr_insert(b, &load->instr);
37 nir_store_deref(b, deref, &load->def, ~0);
38 } else if (glsl_type_is_struct_or_ifc(deref->type)) {
39 unsigned len = glsl_get_length(deref->type);
40 for (unsigned i = 0; i < len; i++) {
41 build_constant_load(b, nir_build_deref_struct(b, deref, i),
42 c->elements[i]);
43 }
44 } else {
45 assert(glsl_type_is_array(deref->type) ||
46 glsl_type_is_matrix(deref->type));
47 unsigned len = glsl_get_length(deref->type);
48 for (unsigned i = 0; i < len; i++) {
49 build_constant_load(b,
50 nir_build_deref_array_imm(b, deref, i),
51 c->elements[i]);
52 }
53 }
54 }
55
56 static bool
lower_const_initializer(struct nir_builder * b,struct exec_list * var_list,nir_variable_mode modes)57 lower_const_initializer(struct nir_builder *b, struct exec_list *var_list,
58 nir_variable_mode modes)
59 {
60 bool progress = false;
61
62 b->cursor = nir_before_impl(b->impl);
63
64 nir_foreach_variable_in_list(var, var_list) {
65 if (!(var->data.mode & modes))
66 continue;
67
68 if (var->constant_initializer) {
69 build_constant_load(b, nir_build_deref_var(b, var),
70 var->constant_initializer);
71
72 progress = true;
73 var->constant_initializer = NULL;
74 } else if (var->pointer_initializer) {
75 nir_deref_instr *src_deref = nir_build_deref_var(b, var->pointer_initializer);
76 nir_deref_instr *dst_deref = nir_build_deref_var(b, var);
77
78 /* Note that this stores a pointer to src into dst */
79 nir_store_deref(b, dst_deref, &src_deref->def, ~0);
80
81 progress = true;
82 var->pointer_initializer = NULL;
83 }
84 }
85
86 return progress;
87 }
88
89 bool
nir_lower_variable_initializers(nir_shader * shader,nir_variable_mode modes)90 nir_lower_variable_initializers(nir_shader *shader, nir_variable_mode modes)
91 {
92 bool progress = false;
93
94 /* Only some variables have initializers that we want to lower. Others
95 * such as uniforms have initializers which are useful later during linking
96 * so we want to skip over those. Restrict to only variable types where
97 * initializers make sense so that callers can use nir_var_all.
98 */
99 modes &= nir_var_shader_out |
100 nir_var_shader_temp |
101 nir_var_function_temp |
102 nir_var_system_value;
103
104 nir_foreach_function_with_impl(func, impl, shader) {
105 bool impl_progress = false;
106 nir_builder builder = nir_builder_create(impl);
107
108 if ((modes & ~nir_var_function_temp) && func->is_entrypoint) {
109 impl_progress |= lower_const_initializer(&builder,
110 &shader->variables,
111 modes);
112 }
113
114 if (modes & nir_var_function_temp) {
115 impl_progress |= lower_const_initializer(&builder,
116 &impl->locals,
117 nir_var_function_temp);
118 }
119
120 if (impl_progress) {
121 progress = true;
122 nir_metadata_preserve(impl, nir_metadata_block_index |
123 nir_metadata_dominance |
124 nir_metadata_live_defs);
125 } else {
126 nir_metadata_preserve(impl, nir_metadata_all);
127 }
128 }
129
130 return progress;
131 }
132
133 /* Zero initialize shared_size bytes of shared memory by splitting work writes
134 * of chunk_size bytes among the invocations.
135 *
136 * Used for implementing VK_KHR_zero_initialize_workgroup_memory.
137 */
138 bool
nir_zero_initialize_shared_memory(nir_shader * shader,const unsigned shared_size,const unsigned chunk_size)139 nir_zero_initialize_shared_memory(nir_shader *shader,
140 const unsigned shared_size,
141 const unsigned chunk_size)
142 {
143 assert(shared_size > 0);
144 assert(chunk_size > 0);
145 assert(chunk_size % 4 == 0);
146
147 nir_function_impl *impl = nir_shader_get_entrypoint(shader);
148 nir_builder b = nir_builder_at(nir_before_impl(impl));
149
150 assert(!shader->info.workgroup_size_variable);
151 const unsigned local_count = shader->info.workgroup_size[0] *
152 shader->info.workgroup_size[1] *
153 shader->info.workgroup_size[2];
154
155 /* The initialization logic is simplified if we can always split the memory
156 * in full chunk_size units.
157 */
158 assert(shared_size % chunk_size == 0);
159
160 const unsigned chunk_comps = chunk_size / 4;
161
162 nir_variable *it = nir_local_variable_create(b.impl, glsl_uint_type(),
163 "zero_init_iterator");
164 nir_def *local_index = nir_load_local_invocation_index(&b);
165 nir_def *first_offset = nir_imul_imm(&b, local_index, chunk_size);
166 nir_store_var(&b, it, first_offset, 0x1);
167
168 nir_loop *loop = nir_push_loop(&b);
169 {
170 nir_def *offset = nir_load_var(&b, it);
171
172 nir_push_if(&b, nir_uge_imm(&b, offset, shared_size));
173 {
174 nir_jump(&b, nir_jump_break);
175 }
176 nir_pop_if(&b, NULL);
177
178 nir_store_shared(&b, nir_imm_zero(&b, chunk_comps, 32), offset,
179 .align_mul = chunk_size,
180 .write_mask = ((1 << chunk_comps) - 1));
181
182 nir_def *new_offset = nir_iadd_imm(&b, offset, chunk_size * local_count);
183 nir_store_var(&b, it, new_offset, 0x1);
184 }
185 nir_pop_loop(&b, loop);
186
187 nir_barrier(&b, SCOPE_WORKGROUP, SCOPE_WORKGROUP, NIR_MEMORY_ACQ_REL,
188 nir_var_mem_shared);
189
190 nir_metadata_preserve(nir_shader_get_entrypoint(shader), nir_metadata_none);
191
192 return true;
193 }
194
195
196 /** Clears all shared memory to zero at the end of the shader
197 *
198 * To easily get to the end of the shader it relies on all exits
199 * being lowered. Designed to be called late in the lowering process,
200 * e.g. doesn't need to lower vars to ssa.
201 */
202 bool
nir_clear_shared_memory(nir_shader * shader,const unsigned shared_size,const unsigned chunk_size)203 nir_clear_shared_memory(nir_shader *shader,
204 const unsigned shared_size,
205 const unsigned chunk_size)
206 {
207 assert(chunk_size > 0);
208 assert(chunk_size % 4 == 0);
209
210 if (shared_size == 0)
211 return false;
212
213 nir_function_impl *impl = nir_shader_get_entrypoint(shader);
214 nir_builder b = nir_builder_at(nir_after_impl(impl));
215
216 /* The initialization logic is simplified if we can always split the memory
217 * in full chunk_size units.
218 */
219 assert(shared_size % chunk_size == 0);
220
221 const unsigned chunk_comps = chunk_size / 4;
222
223 nir_barrier(&b, SCOPE_WORKGROUP, SCOPE_WORKGROUP, NIR_MEMORY_ACQ_REL,
224 nir_var_mem_shared);
225
226 nir_def *local_index = nir_load_local_invocation_index(&b);
227 nir_def *first_offset = nir_imul_imm(&b, local_index, chunk_size);
228
229 unsigned iterations = UINT_MAX;
230 unsigned size_per_iteration = 0;
231 if (!shader->info.workgroup_size_variable) {
232 size_per_iteration = nir_static_workgroup_size(shader) * chunk_size;
233 iterations = DIV_ROUND_UP(shared_size, size_per_iteration);
234 }
235
236 if (iterations <= shader->options->max_unroll_iterations) {
237 /* Doing a manual inline here because (a) we may not optimize after and
238 * (b) the loop unroll pass doesn't deal well with the potential partial
239 * last iteration.*/
240 for (unsigned i = 0; i < iterations; ++i) {
241 const unsigned base = size_per_iteration * i;
242 bool use_check = i >= shared_size / size_per_iteration;
243 if (use_check)
244 nir_push_if(&b, nir_ult_imm(&b, first_offset, shared_size - base));
245
246 nir_store_shared(&b, nir_imm_zero(&b, chunk_comps, 32),
247 nir_iadd_imm(&b, first_offset, base),
248 .align_mul = chunk_size,
249 .write_mask = ((1 << chunk_comps) - 1));
250 if (use_check)
251 nir_pop_if(&b, NULL);
252 }
253 } else {
254 nir_phi_instr *offset_phi = nir_phi_instr_create(shader);
255 nir_def_init(&offset_phi->instr, &offset_phi->def, 1, 32);
256 nir_phi_instr_add_src(offset_phi, nir_cursor_current_block(b.cursor), first_offset);
257
258 nir_def *size_per_iteration_def = shader->info.workgroup_size_variable ?
259 nir_imul_imm(&b, nir_load_workgroup_size(&b), chunk_size) :
260 nir_imm_int(&b, size_per_iteration);
261 nir_def *value = nir_imm_zero(&b, chunk_comps, 32);
262
263 nir_loop *loop = nir_push_loop(&b);
264 nir_block *loop_block = nir_cursor_current_block(b.cursor);
265 {
266 nir_def *offset = &offset_phi->def;
267
268 nir_push_if(&b, nir_uge_imm(&b, offset, shared_size));
269 {
270 nir_jump(&b, nir_jump_break);
271 }
272 nir_pop_if(&b, NULL);
273 nir_store_shared(&b, value, offset,
274 .align_mul = chunk_size,
275 .write_mask = ((1 << chunk_comps) - 1));
276
277 nir_def *new_offset = nir_iadd(&b, offset, size_per_iteration_def);
278 nir_phi_instr_add_src(offset_phi, nir_cursor_current_block(b.cursor), new_offset);
279 }
280 nir_pop_loop(&b, loop);
281
282 b.cursor = nir_before_block(loop_block);
283 nir_builder_instr_insert(&b, &offset_phi->instr);
284 }
285
286 nir_metadata_preserve(nir_shader_get_entrypoint(shader), nir_metadata_none);
287
288 return true;
289 }
290