1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "anv_nir.h"
25 #include "program/prog_parameter.h"
26 #include "nir/nir_builder.h"
27
28 struct apply_pipeline_layout_state {
29 nir_shader *shader;
30 nir_builder builder;
31
32 struct anv_pipeline_layout *layout;
33 bool add_bounds_checks;
34
35 struct {
36 BITSET_WORD *used;
37 uint8_t *surface_offsets;
38 uint8_t *sampler_offsets;
39 uint8_t *image_offsets;
40 } set[MAX_SETS];
41 };
42
43 static void
add_binding(struct apply_pipeline_layout_state * state,uint32_t set,uint32_t binding)44 add_binding(struct apply_pipeline_layout_state *state,
45 uint32_t set, uint32_t binding)
46 {
47 BITSET_SET(state->set[set].used, binding);
48 }
49
50 static void
add_var_binding(struct apply_pipeline_layout_state * state,nir_variable * var)51 add_var_binding(struct apply_pipeline_layout_state *state, nir_variable *var)
52 {
53 add_binding(state, var->data.descriptor_set, var->data.binding);
54 }
55
56 static void
get_used_bindings_block(nir_block * block,struct apply_pipeline_layout_state * state)57 get_used_bindings_block(nir_block *block,
58 struct apply_pipeline_layout_state *state)
59 {
60 nir_foreach_instr_safe(instr, block) {
61 switch (instr->type) {
62 case nir_instr_type_intrinsic: {
63 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
64 switch (intrin->intrinsic) {
65 case nir_intrinsic_vulkan_resource_index:
66 add_binding(state, nir_intrinsic_desc_set(intrin),
67 nir_intrinsic_binding(intrin));
68 break;
69
70 case nir_intrinsic_image_load:
71 case nir_intrinsic_image_store:
72 case nir_intrinsic_image_atomic_add:
73 case nir_intrinsic_image_atomic_min:
74 case nir_intrinsic_image_atomic_max:
75 case nir_intrinsic_image_atomic_and:
76 case nir_intrinsic_image_atomic_or:
77 case nir_intrinsic_image_atomic_xor:
78 case nir_intrinsic_image_atomic_exchange:
79 case nir_intrinsic_image_atomic_comp_swap:
80 case nir_intrinsic_image_size:
81 case nir_intrinsic_image_samples:
82 add_var_binding(state, intrin->variables[0]->var);
83 break;
84
85 default:
86 break;
87 }
88 break;
89 }
90 case nir_instr_type_tex: {
91 nir_tex_instr *tex = nir_instr_as_tex(instr);
92 assert(tex->texture);
93 add_var_binding(state, tex->texture->var);
94 if (tex->sampler)
95 add_var_binding(state, tex->sampler->var);
96 break;
97 }
98 default:
99 continue;
100 }
101 }
102 }
103
104 static void
lower_res_index_intrinsic(nir_intrinsic_instr * intrin,struct apply_pipeline_layout_state * state)105 lower_res_index_intrinsic(nir_intrinsic_instr *intrin,
106 struct apply_pipeline_layout_state *state)
107 {
108 nir_builder *b = &state->builder;
109
110 b->cursor = nir_before_instr(&intrin->instr);
111
112 uint32_t set = nir_intrinsic_desc_set(intrin);
113 uint32_t binding = nir_intrinsic_binding(intrin);
114
115 uint32_t surface_index = state->set[set].surface_offsets[binding];
116 uint32_t array_size =
117 state->layout->set[set].layout->binding[binding].array_size;
118
119 nir_ssa_def *block_index = nir_ssa_for_src(b, intrin->src[0], 1);
120
121 if (state->add_bounds_checks)
122 block_index = nir_umin(b, block_index, nir_imm_int(b, array_size - 1));
123
124 block_index = nir_iadd(b, nir_imm_int(b, surface_index), block_index);
125
126 assert(intrin->dest.is_ssa);
127 nir_ssa_def_rewrite_uses(&intrin->dest.ssa, nir_src_for_ssa(block_index));
128 nir_instr_remove(&intrin->instr);
129 }
130
131 static void
lower_tex_deref(nir_tex_instr * tex,nir_deref_var * deref,unsigned * const_index,unsigned array_size,nir_tex_src_type src_type,struct apply_pipeline_layout_state * state)132 lower_tex_deref(nir_tex_instr *tex, nir_deref_var *deref,
133 unsigned *const_index, unsigned array_size,
134 nir_tex_src_type src_type,
135 struct apply_pipeline_layout_state *state)
136 {
137 nir_builder *b = &state->builder;
138
139 if (deref->deref.child) {
140 assert(deref->deref.child->deref_type == nir_deref_type_array);
141 nir_deref_array *deref_array = nir_deref_as_array(deref->deref.child);
142
143 if (deref_array->deref_array_type == nir_deref_array_type_indirect) {
144 nir_ssa_def *index =
145 nir_iadd(b, nir_imm_int(b, deref_array->base_offset),
146 nir_ssa_for_src(b, deref_array->indirect, 1));
147
148 if (state->add_bounds_checks)
149 index = nir_umin(b, index, nir_imm_int(b, array_size - 1));
150
151 nir_tex_src *new_srcs = rzalloc_array(tex, nir_tex_src,
152 tex->num_srcs + 1);
153
154 for (unsigned i = 0; i < tex->num_srcs; i++) {
155 new_srcs[i].src_type = tex->src[i].src_type;
156 nir_instr_move_src(&tex->instr, &new_srcs[i].src, &tex->src[i].src);
157 }
158
159 ralloc_free(tex->src);
160 tex->src = new_srcs;
161
162 /* Now we can go ahead and move the source over to being a
163 * first-class texture source.
164 */
165 tex->src[tex->num_srcs].src_type = src_type;
166 nir_instr_rewrite_src(&tex->instr, &tex->src[tex->num_srcs].src,
167 nir_src_for_ssa(index));
168 tex->num_srcs++;
169 } else {
170 *const_index += MIN2(deref_array->base_offset, array_size - 1);
171 }
172 }
173 }
174
175 static void
cleanup_tex_deref(nir_tex_instr * tex,nir_deref_var * deref)176 cleanup_tex_deref(nir_tex_instr *tex, nir_deref_var *deref)
177 {
178 if (deref->deref.child == NULL)
179 return;
180
181 nir_deref_array *deref_array = nir_deref_as_array(deref->deref.child);
182
183 if (deref_array->deref_array_type != nir_deref_array_type_indirect)
184 return;
185
186 nir_instr_rewrite_src(&tex->instr, &deref_array->indirect, NIR_SRC_INIT);
187 }
188
189 static void
lower_tex(nir_tex_instr * tex,struct apply_pipeline_layout_state * state)190 lower_tex(nir_tex_instr *tex, struct apply_pipeline_layout_state *state)
191 {
192 /* No one should have come by and lowered it already */
193 assert(tex->texture);
194
195 state->builder.cursor = nir_before_instr(&tex->instr);
196
197 unsigned set = tex->texture->var->data.descriptor_set;
198 unsigned binding = tex->texture->var->data.binding;
199 unsigned array_size =
200 state->layout->set[set].layout->binding[binding].array_size;
201 tex->texture_index = state->set[set].surface_offsets[binding];
202 lower_tex_deref(tex, tex->texture, &tex->texture_index, array_size,
203 nir_tex_src_texture_offset, state);
204
205 if (tex->sampler) {
206 unsigned set = tex->sampler->var->data.descriptor_set;
207 unsigned binding = tex->sampler->var->data.binding;
208 unsigned array_size =
209 state->layout->set[set].layout->binding[binding].array_size;
210 tex->sampler_index = state->set[set].sampler_offsets[binding];
211 lower_tex_deref(tex, tex->sampler, &tex->sampler_index, array_size,
212 nir_tex_src_sampler_offset, state);
213 }
214
215 /* The backend only ever uses this to mark used surfaces. We don't care
216 * about that little optimization so it just needs to be non-zero.
217 */
218 tex->texture_array_size = 1;
219
220 cleanup_tex_deref(tex, tex->texture);
221 if (tex->sampler)
222 cleanup_tex_deref(tex, tex->sampler);
223 tex->texture = NULL;
224 tex->sampler = NULL;
225 }
226
227 static void
apply_pipeline_layout_block(nir_block * block,struct apply_pipeline_layout_state * state)228 apply_pipeline_layout_block(nir_block *block,
229 struct apply_pipeline_layout_state *state)
230 {
231 nir_foreach_instr_safe(instr, block) {
232 switch (instr->type) {
233 case nir_instr_type_intrinsic: {
234 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
235 if (intrin->intrinsic == nir_intrinsic_vulkan_resource_index) {
236 lower_res_index_intrinsic(intrin, state);
237 }
238 break;
239 }
240 case nir_instr_type_tex:
241 lower_tex(nir_instr_as_tex(instr), state);
242 break;
243 default:
244 continue;
245 }
246 }
247 }
248
249 static void
setup_vec4_uniform_value(const union gl_constant_value ** params,const union gl_constant_value * values,unsigned n)250 setup_vec4_uniform_value(const union gl_constant_value **params,
251 const union gl_constant_value *values,
252 unsigned n)
253 {
254 static const gl_constant_value zero = { 0 };
255
256 for (unsigned i = 0; i < n; ++i)
257 params[i] = &values[i];
258
259 for (unsigned i = n; i < 4; ++i)
260 params[i] = &zero;
261 }
262
263 void
anv_nir_apply_pipeline_layout(struct anv_pipeline * pipeline,nir_shader * shader,struct brw_stage_prog_data * prog_data,struct anv_pipeline_bind_map * map)264 anv_nir_apply_pipeline_layout(struct anv_pipeline *pipeline,
265 nir_shader *shader,
266 struct brw_stage_prog_data *prog_data,
267 struct anv_pipeline_bind_map *map)
268 {
269 struct anv_pipeline_layout *layout = pipeline->layout;
270
271 struct apply_pipeline_layout_state state = {
272 .shader = shader,
273 .layout = layout,
274 .add_bounds_checks = pipeline->device->robust_buffer_access,
275 };
276
277 void *mem_ctx = ralloc_context(NULL);
278
279 for (unsigned s = 0; s < layout->num_sets; s++) {
280 const unsigned count = layout->set[s].layout->binding_count;
281 const unsigned words = BITSET_WORDS(count);
282 state.set[s].used = rzalloc_array(mem_ctx, BITSET_WORD, words);
283 state.set[s].surface_offsets = rzalloc_array(mem_ctx, uint8_t, count);
284 state.set[s].sampler_offsets = rzalloc_array(mem_ctx, uint8_t, count);
285 state.set[s].image_offsets = rzalloc_array(mem_ctx, uint8_t, count);
286 }
287
288 nir_foreach_function(function, shader) {
289 if (!function->impl)
290 continue;
291
292 nir_foreach_block(block, function->impl)
293 get_used_bindings_block(block, &state);
294 }
295
296 for (uint32_t set = 0; set < layout->num_sets; set++) {
297 struct anv_descriptor_set_layout *set_layout = layout->set[set].layout;
298
299 BITSET_WORD b, _tmp;
300 BITSET_FOREACH_SET(b, _tmp, state.set[set].used,
301 set_layout->binding_count) {
302 if (set_layout->binding[b].stage[shader->stage].surface_index >= 0)
303 map->surface_count += set_layout->binding[b].array_size;
304 if (set_layout->binding[b].stage[shader->stage].sampler_index >= 0)
305 map->sampler_count += set_layout->binding[b].array_size;
306 if (set_layout->binding[b].stage[shader->stage].image_index >= 0)
307 map->image_count += set_layout->binding[b].array_size;
308 }
309 }
310
311 unsigned surface = 0;
312 unsigned sampler = 0;
313 unsigned image = 0;
314 for (uint32_t set = 0; set < layout->num_sets; set++) {
315 struct anv_descriptor_set_layout *set_layout = layout->set[set].layout;
316
317 BITSET_WORD b, _tmp;
318 BITSET_FOREACH_SET(b, _tmp, state.set[set].used,
319 set_layout->binding_count) {
320 unsigned array_size = set_layout->binding[b].array_size;
321
322 if (set_layout->binding[b].stage[shader->stage].surface_index >= 0) {
323 state.set[set].surface_offsets[b] = surface;
324 for (unsigned i = 0; i < array_size; i++) {
325 map->surface_to_descriptor[surface + i].set = set;
326 map->surface_to_descriptor[surface + i].binding = b;
327 map->surface_to_descriptor[surface + i].index = i;
328 }
329 surface += array_size;
330 }
331
332 if (set_layout->binding[b].stage[shader->stage].sampler_index >= 0) {
333 state.set[set].sampler_offsets[b] = sampler;
334 for (unsigned i = 0; i < array_size; i++) {
335 map->sampler_to_descriptor[sampler + i].set = set;
336 map->sampler_to_descriptor[sampler + i].binding = b;
337 map->sampler_to_descriptor[sampler + i].index = i;
338 }
339 sampler += array_size;
340 }
341
342 if (set_layout->binding[b].stage[shader->stage].image_index >= 0) {
343 state.set[set].image_offsets[b] = image;
344 image += array_size;
345 }
346 }
347 }
348
349 nir_foreach_variable(var, &shader->uniforms) {
350 if (!glsl_type_is_image(var->interface_type))
351 continue;
352
353 enum glsl_sampler_dim dim = glsl_get_sampler_dim(var->interface_type);
354 if (dim != GLSL_SAMPLER_DIM_SUBPASS &&
355 dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
356 continue;
357
358 const uint32_t set = var->data.descriptor_set;
359 const uint32_t binding = var->data.binding;
360 const uint32_t array_size =
361 layout->set[set].layout->binding[binding].array_size;
362
363 if (!BITSET_TEST(state.set[set].used, binding))
364 continue;
365
366 struct anv_pipeline_binding *pipe_binding =
367 &map->surface_to_descriptor[state.set[set].surface_offsets[binding]];
368 for (unsigned i = 0; i < array_size; i++) {
369 assert(pipe_binding[i].set == set);
370 assert(pipe_binding[i].binding == binding);
371 assert(pipe_binding[i].index == i);
372 pipe_binding[i].input_attachment_index = var->data.index + i;
373 }
374 }
375
376 nir_foreach_function(function, shader) {
377 if (!function->impl)
378 continue;
379
380 nir_builder_init(&state.builder, function->impl);
381 nir_foreach_block(block, function->impl)
382 apply_pipeline_layout_block(block, &state);
383 nir_metadata_preserve(function->impl, nir_metadata_block_index |
384 nir_metadata_dominance);
385 }
386
387 if (map->image_count > 0) {
388 assert(map->image_count <= MAX_IMAGES);
389 nir_foreach_variable(var, &shader->uniforms) {
390 if (glsl_type_is_image(var->type) ||
391 (glsl_type_is_array(var->type) &&
392 glsl_type_is_image(glsl_get_array_element(var->type)))) {
393 /* Images are represented as uniform push constants and the actual
394 * information required for reading/writing to/from the image is
395 * storred in the uniform.
396 */
397 unsigned set = var->data.descriptor_set;
398 unsigned binding = var->data.binding;
399 unsigned image_index = state.set[set].image_offsets[binding];
400
401 var->data.driver_location = shader->num_uniforms +
402 image_index * BRW_IMAGE_PARAM_SIZE * 4;
403 }
404 }
405
406 struct anv_push_constants *null_data = NULL;
407 const gl_constant_value **param =
408 prog_data->param + (shader->num_uniforms / 4);
409 const struct brw_image_param *image_param = null_data->images;
410 for (uint32_t i = 0; i < map->image_count; i++) {
411 setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_SURFACE_IDX_OFFSET,
412 (const union gl_constant_value *)&image_param->surface_idx, 1);
413 setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_OFFSET_OFFSET,
414 (const union gl_constant_value *)image_param->offset, 2);
415 setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_SIZE_OFFSET,
416 (const union gl_constant_value *)image_param->size, 3);
417 setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_STRIDE_OFFSET,
418 (const union gl_constant_value *)image_param->stride, 4);
419 setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_TILING_OFFSET,
420 (const union gl_constant_value *)image_param->tiling, 3);
421 setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_SWIZZLING_OFFSET,
422 (const union gl_constant_value *)image_param->swizzling, 2);
423
424 param += BRW_IMAGE_PARAM_SIZE;
425 image_param ++;
426 }
427
428 shader->num_uniforms += map->image_count * BRW_IMAGE_PARAM_SIZE * 4;
429 }
430
431 ralloc_free(mem_ctx);
432 }
433