1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 * IN THE SOFTWARE.
23 */
24 #include "radv_meta.h"
25 #include "nir/nir_builder.h"
26
27 /*
28 * GFX queue: Compute shader implementation of image->buffer copy
29 * Compute queue: implementation also of buffer->image, image->image, and image clear.
30 */
31
32 /* GFX9 needs to use a 3D sampler to access 3D resources, so the shader has the options
33 * for that.
34 */
35 static nir_shader *
build_nir_itob_compute_shader(struct radv_device * dev,bool is_3d)36 build_nir_itob_compute_shader(struct radv_device *dev, bool is_3d)
37 {
38 nir_builder b;
39 enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D;
40 const struct glsl_type *sampler_type = glsl_sampler_type(dim,
41 false,
42 false,
43 GLSL_TYPE_FLOAT);
44 const struct glsl_type *img_type = glsl_sampler_type(GLSL_SAMPLER_DIM_BUF,
45 false,
46 false,
47 GLSL_TYPE_FLOAT);
48 nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
49 b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_itob_cs_3d" : "meta_itob_cs");
50 b.shader->info.cs.local_size[0] = 16;
51 b.shader->info.cs.local_size[1] = 16;
52 b.shader->info.cs.local_size[2] = 1;
53 nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
54 sampler_type, "s_tex");
55 input_img->data.descriptor_set = 0;
56 input_img->data.binding = 0;
57
58 nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
59 img_type, "out_img");
60 output_img->data.descriptor_set = 0;
61 output_img->data.binding = 1;
62
63 nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
64 nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
65 nir_ssa_def *block_size = nir_imm_ivec4(&b,
66 b.shader->info.cs.local_size[0],
67 b.shader->info.cs.local_size[1],
68 b.shader->info.cs.local_size[2], 0);
69
70 nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
71
72
73
74 nir_intrinsic_instr *offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
75 nir_intrinsic_set_base(offset, 0);
76 nir_intrinsic_set_range(offset, 16);
77 offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
78 offset->num_components = is_3d ? 3 : 2;
79 nir_ssa_dest_init(&offset->instr, &offset->dest, is_3d ? 3 : 2, 32, "offset");
80 nir_builder_instr_insert(&b, &offset->instr);
81
82 nir_intrinsic_instr *stride = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
83 nir_intrinsic_set_base(stride, 0);
84 nir_intrinsic_set_range(stride, 16);
85 stride->src[0] = nir_src_for_ssa(nir_imm_int(&b, 12));
86 stride->num_components = 1;
87 nir_ssa_dest_init(&stride->instr, &stride->dest, 1, 32, "stride");
88 nir_builder_instr_insert(&b, &stride->instr);
89
90 nir_ssa_def *img_coord = nir_iadd(&b, global_id, &offset->dest.ssa);
91 nir_tex_instr *tex = nir_tex_instr_create(b.shader, 2);
92 tex->sampler_dim = dim;
93 tex->op = nir_texop_txf;
94 tex->src[0].src_type = nir_tex_src_coord;
95 tex->src[0].src = nir_src_for_ssa(nir_channels(&b, img_coord, is_3d ? 0x7 : 0x3));
96 tex->src[1].src_type = nir_tex_src_lod;
97 tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
98 tex->dest_type = nir_type_float;
99 tex->is_array = false;
100 tex->coord_components = is_3d ? 3 : 2;
101 tex->texture = nir_deref_var_create(tex, input_img);
102 tex->sampler = NULL;
103
104 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
105 nir_builder_instr_insert(&b, &tex->instr);
106
107 nir_ssa_def *pos_x = nir_channel(&b, global_id, 0);
108 nir_ssa_def *pos_y = nir_channel(&b, global_id, 1);
109
110 nir_ssa_def *tmp = nir_imul(&b, pos_y, &stride->dest.ssa);
111 tmp = nir_iadd(&b, tmp, pos_x);
112
113 nir_ssa_def *coord = nir_vec4(&b, tmp, tmp, tmp, tmp);
114
115 nir_ssa_def *outval = &tex->dest.ssa;
116 nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
117 store->src[0] = nir_src_for_ssa(coord);
118 store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
119 store->src[2] = nir_src_for_ssa(outval);
120 store->variables[0] = nir_deref_var_create(store, output_img);
121
122 nir_builder_instr_insert(&b, &store->instr);
123 return b.shader;
124 }
125
126 /* Image to buffer - don't write use image accessors */
127 static VkResult
radv_device_init_meta_itob_state(struct radv_device * device)128 radv_device_init_meta_itob_state(struct radv_device *device)
129 {
130 VkResult result;
131 struct radv_shader_module cs = { .nir = NULL };
132 struct radv_shader_module cs_3d = { .nir = NULL };
133
134 cs.nir = build_nir_itob_compute_shader(device, false);
135 if (device->physical_device->rad_info.chip_class >= GFX9)
136 cs_3d.nir = build_nir_itob_compute_shader(device, true);
137
138 /*
139 * two descriptors one for the image being sampled
140 * one for the buffer being written.
141 */
142 VkDescriptorSetLayoutCreateInfo ds_create_info = {
143 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
144 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
145 .bindingCount = 2,
146 .pBindings = (VkDescriptorSetLayoutBinding[]) {
147 {
148 .binding = 0,
149 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
150 .descriptorCount = 1,
151 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
152 .pImmutableSamplers = NULL
153 },
154 {
155 .binding = 1,
156 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
157 .descriptorCount = 1,
158 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
159 .pImmutableSamplers = NULL
160 },
161 }
162 };
163
164 result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
165 &ds_create_info,
166 &device->meta_state.alloc,
167 &device->meta_state.itob.img_ds_layout);
168 if (result != VK_SUCCESS)
169 goto fail;
170
171
172 VkPipelineLayoutCreateInfo pl_create_info = {
173 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
174 .setLayoutCount = 1,
175 .pSetLayouts = &device->meta_state.itob.img_ds_layout,
176 .pushConstantRangeCount = 1,
177 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 16},
178 };
179
180 result = radv_CreatePipelineLayout(radv_device_to_handle(device),
181 &pl_create_info,
182 &device->meta_state.alloc,
183 &device->meta_state.itob.img_p_layout);
184 if (result != VK_SUCCESS)
185 goto fail;
186
187 /* compute shader */
188
189 VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
190 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
191 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
192 .module = radv_shader_module_to_handle(&cs),
193 .pName = "main",
194 .pSpecializationInfo = NULL,
195 };
196
197 VkComputePipelineCreateInfo vk_pipeline_info = {
198 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
199 .stage = pipeline_shader_stage,
200 .flags = 0,
201 .layout = device->meta_state.itob.img_p_layout,
202 };
203
204 result = radv_CreateComputePipelines(radv_device_to_handle(device),
205 radv_pipeline_cache_to_handle(&device->meta_state.cache),
206 1, &vk_pipeline_info, NULL,
207 &device->meta_state.itob.pipeline);
208 if (result != VK_SUCCESS)
209 goto fail;
210
211 if (device->physical_device->rad_info.chip_class >= GFX9) {
212 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = {
213 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
214 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
215 .module = radv_shader_module_to_handle(&cs_3d),
216 .pName = "main",
217 .pSpecializationInfo = NULL,
218 };
219
220 VkComputePipelineCreateInfo vk_pipeline_info_3d = {
221 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
222 .stage = pipeline_shader_stage_3d,
223 .flags = 0,
224 .layout = device->meta_state.itob.img_p_layout,
225 };
226
227 result = radv_CreateComputePipelines(radv_device_to_handle(device),
228 radv_pipeline_cache_to_handle(&device->meta_state.cache),
229 1, &vk_pipeline_info_3d, NULL,
230 &device->meta_state.itob.pipeline_3d);
231 if (result != VK_SUCCESS)
232 goto fail;
233 ralloc_free(cs_3d.nir);
234 }
235 ralloc_free(cs.nir);
236
237 return VK_SUCCESS;
238 fail:
239 ralloc_free(cs.nir);
240 ralloc_free(cs_3d.nir);
241 return result;
242 }
243
244 static void
radv_device_finish_meta_itob_state(struct radv_device * device)245 radv_device_finish_meta_itob_state(struct radv_device *device)
246 {
247 struct radv_meta_state *state = &device->meta_state;
248
249 radv_DestroyPipelineLayout(radv_device_to_handle(device),
250 state->itob.img_p_layout, &state->alloc);
251 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
252 state->itob.img_ds_layout,
253 &state->alloc);
254 radv_DestroyPipeline(radv_device_to_handle(device),
255 state->itob.pipeline, &state->alloc);
256 if (device->physical_device->rad_info.chip_class >= GFX9)
257 radv_DestroyPipeline(radv_device_to_handle(device),
258 state->itob.pipeline_3d, &state->alloc);
259 }
260
261 static nir_shader *
build_nir_btoi_compute_shader(struct radv_device * dev,bool is_3d)262 build_nir_btoi_compute_shader(struct radv_device *dev, bool is_3d)
263 {
264 nir_builder b;
265 enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D;
266 const struct glsl_type *buf_type = glsl_sampler_type(GLSL_SAMPLER_DIM_BUF,
267 false,
268 false,
269 GLSL_TYPE_FLOAT);
270 const struct glsl_type *img_type = glsl_sampler_type(dim,
271 false,
272 false,
273 GLSL_TYPE_FLOAT);
274 nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
275 b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_btoi_cs_3d" : "meta_btoi_cs");
276 b.shader->info.cs.local_size[0] = 16;
277 b.shader->info.cs.local_size[1] = 16;
278 b.shader->info.cs.local_size[2] = 1;
279 nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
280 buf_type, "s_tex");
281 input_img->data.descriptor_set = 0;
282 input_img->data.binding = 0;
283
284 nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
285 img_type, "out_img");
286 output_img->data.descriptor_set = 0;
287 output_img->data.binding = 1;
288
289 nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
290 nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
291 nir_ssa_def *block_size = nir_imm_ivec4(&b,
292 b.shader->info.cs.local_size[0],
293 b.shader->info.cs.local_size[1],
294 b.shader->info.cs.local_size[2], 0);
295
296 nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
297
298 nir_intrinsic_instr *offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
299 nir_intrinsic_set_base(offset, 0);
300 nir_intrinsic_set_range(offset, 16);
301 offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
302 offset->num_components = is_3d ? 3 : 2;
303 nir_ssa_dest_init(&offset->instr, &offset->dest, is_3d ? 3 : 2, 32, "offset");
304 nir_builder_instr_insert(&b, &offset->instr);
305
306 nir_intrinsic_instr *stride = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
307 nir_intrinsic_set_base(stride, 0);
308 nir_intrinsic_set_range(stride, 16);
309 stride->src[0] = nir_src_for_ssa(nir_imm_int(&b, 12));
310 stride->num_components = 1;
311 nir_ssa_dest_init(&stride->instr, &stride->dest, 1, 32, "stride");
312 nir_builder_instr_insert(&b, &stride->instr);
313
314 nir_ssa_def *pos_x = nir_channel(&b, global_id, 0);
315 nir_ssa_def *pos_y = nir_channel(&b, global_id, 1);
316
317 nir_ssa_def *tmp = nir_imul(&b, pos_y, &stride->dest.ssa);
318 tmp = nir_iadd(&b, tmp, pos_x);
319
320 nir_ssa_def *buf_coord = nir_vec4(&b, tmp, tmp, tmp, tmp);
321
322 nir_ssa_def *img_coord = nir_iadd(&b, global_id, &offset->dest.ssa);
323
324 nir_tex_instr *tex = nir_tex_instr_create(b.shader, 2);
325 tex->sampler_dim = GLSL_SAMPLER_DIM_BUF;
326 tex->op = nir_texop_txf;
327 tex->src[0].src_type = nir_tex_src_coord;
328 tex->src[0].src = nir_src_for_ssa(nir_channels(&b, buf_coord, 1));
329 tex->src[1].src_type = nir_tex_src_lod;
330 tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
331 tex->dest_type = nir_type_float;
332 tex->is_array = false;
333 tex->coord_components = 1;
334 tex->texture = nir_deref_var_create(tex, input_img);
335 tex->sampler = NULL;
336
337 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
338 nir_builder_instr_insert(&b, &tex->instr);
339
340 nir_ssa_def *outval = &tex->dest.ssa;
341 nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
342 store->src[0] = nir_src_for_ssa(img_coord);
343 store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
344 store->src[2] = nir_src_for_ssa(outval);
345 store->variables[0] = nir_deref_var_create(store, output_img);
346
347 nir_builder_instr_insert(&b, &store->instr);
348 return b.shader;
349 }
350
351 /* Buffer to image - don't write use image accessors */
352 static VkResult
radv_device_init_meta_btoi_state(struct radv_device * device)353 radv_device_init_meta_btoi_state(struct radv_device *device)
354 {
355 VkResult result;
356 struct radv_shader_module cs = { .nir = NULL };
357 struct radv_shader_module cs_3d = { .nir = NULL };
358 cs.nir = build_nir_btoi_compute_shader(device, false);
359 if (device->physical_device->rad_info.chip_class >= GFX9)
360 cs_3d.nir = build_nir_btoi_compute_shader(device, true);
361 /*
362 * two descriptors one for the image being sampled
363 * one for the buffer being written.
364 */
365 VkDescriptorSetLayoutCreateInfo ds_create_info = {
366 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
367 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
368 .bindingCount = 2,
369 .pBindings = (VkDescriptorSetLayoutBinding[]) {
370 {
371 .binding = 0,
372 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
373 .descriptorCount = 1,
374 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
375 .pImmutableSamplers = NULL
376 },
377 {
378 .binding = 1,
379 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
380 .descriptorCount = 1,
381 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
382 .pImmutableSamplers = NULL
383 },
384 }
385 };
386
387 result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
388 &ds_create_info,
389 &device->meta_state.alloc,
390 &device->meta_state.btoi.img_ds_layout);
391 if (result != VK_SUCCESS)
392 goto fail;
393
394
395 VkPipelineLayoutCreateInfo pl_create_info = {
396 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
397 .setLayoutCount = 1,
398 .pSetLayouts = &device->meta_state.btoi.img_ds_layout,
399 .pushConstantRangeCount = 1,
400 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 16},
401 };
402
403 result = radv_CreatePipelineLayout(radv_device_to_handle(device),
404 &pl_create_info,
405 &device->meta_state.alloc,
406 &device->meta_state.btoi.img_p_layout);
407 if (result != VK_SUCCESS)
408 goto fail;
409
410 /* compute shader */
411
412 VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
413 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
414 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
415 .module = radv_shader_module_to_handle(&cs),
416 .pName = "main",
417 .pSpecializationInfo = NULL,
418 };
419
420 VkComputePipelineCreateInfo vk_pipeline_info = {
421 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
422 .stage = pipeline_shader_stage,
423 .flags = 0,
424 .layout = device->meta_state.btoi.img_p_layout,
425 };
426
427 result = radv_CreateComputePipelines(radv_device_to_handle(device),
428 radv_pipeline_cache_to_handle(&device->meta_state.cache),
429 1, &vk_pipeline_info, NULL,
430 &device->meta_state.btoi.pipeline);
431 if (result != VK_SUCCESS)
432 goto fail;
433
434 if (device->physical_device->rad_info.chip_class >= GFX9) {
435 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = {
436 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
437 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
438 .module = radv_shader_module_to_handle(&cs_3d),
439 .pName = "main",
440 .pSpecializationInfo = NULL,
441 };
442
443 VkComputePipelineCreateInfo vk_pipeline_info_3d = {
444 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
445 .stage = pipeline_shader_stage_3d,
446 .flags = 0,
447 .layout = device->meta_state.btoi.img_p_layout,
448 };
449
450 result = radv_CreateComputePipelines(radv_device_to_handle(device),
451 radv_pipeline_cache_to_handle(&device->meta_state.cache),
452 1, &vk_pipeline_info_3d, NULL,
453 &device->meta_state.btoi.pipeline_3d);
454 ralloc_free(cs_3d.nir);
455 }
456 ralloc_free(cs.nir);
457
458 return VK_SUCCESS;
459 fail:
460 ralloc_free(cs_3d.nir);
461 ralloc_free(cs.nir);
462 return result;
463 }
464
465 static void
radv_device_finish_meta_btoi_state(struct radv_device * device)466 radv_device_finish_meta_btoi_state(struct radv_device *device)
467 {
468 struct radv_meta_state *state = &device->meta_state;
469
470 radv_DestroyPipelineLayout(radv_device_to_handle(device),
471 state->btoi.img_p_layout, &state->alloc);
472 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
473 state->btoi.img_ds_layout,
474 &state->alloc);
475 radv_DestroyPipeline(radv_device_to_handle(device),
476 state->btoi.pipeline, &state->alloc);
477 radv_DestroyPipeline(radv_device_to_handle(device),
478 state->btoi.pipeline_3d, &state->alloc);
479 }
480
481 static nir_shader *
build_nir_itoi_compute_shader(struct radv_device * dev,bool is_3d)482 build_nir_itoi_compute_shader(struct radv_device *dev, bool is_3d)
483 {
484 nir_builder b;
485 enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D;
486 const struct glsl_type *buf_type = glsl_sampler_type(dim,
487 false,
488 false,
489 GLSL_TYPE_FLOAT);
490 const struct glsl_type *img_type = glsl_sampler_type(dim,
491 false,
492 false,
493 GLSL_TYPE_FLOAT);
494 nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
495 b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_itoi_cs_3d" : "meta_itoi_cs");
496 b.shader->info.cs.local_size[0] = 16;
497 b.shader->info.cs.local_size[1] = 16;
498 b.shader->info.cs.local_size[2] = 1;
499 nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
500 buf_type, "s_tex");
501 input_img->data.descriptor_set = 0;
502 input_img->data.binding = 0;
503
504 nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
505 img_type, "out_img");
506 output_img->data.descriptor_set = 0;
507 output_img->data.binding = 1;
508
509 nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
510 nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
511 nir_ssa_def *block_size = nir_imm_ivec4(&b,
512 b.shader->info.cs.local_size[0],
513 b.shader->info.cs.local_size[1],
514 b.shader->info.cs.local_size[2], 0);
515
516 nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
517
518 nir_intrinsic_instr *src_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
519 nir_intrinsic_set_base(src_offset, 0);
520 nir_intrinsic_set_range(src_offset, 24);
521 src_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
522 src_offset->num_components = is_3d ? 3 : 2;
523 nir_ssa_dest_init(&src_offset->instr, &src_offset->dest, is_3d ? 3 : 2, 32, "src_offset");
524 nir_builder_instr_insert(&b, &src_offset->instr);
525
526 nir_intrinsic_instr *dst_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
527 nir_intrinsic_set_base(dst_offset, 0);
528 nir_intrinsic_set_range(dst_offset, 24);
529 dst_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 12));
530 dst_offset->num_components = is_3d ? 3 : 2;
531 nir_ssa_dest_init(&dst_offset->instr, &dst_offset->dest, is_3d ? 3 : 2, 32, "dst_offset");
532 nir_builder_instr_insert(&b, &dst_offset->instr);
533
534 nir_ssa_def *src_coord = nir_iadd(&b, global_id, &src_offset->dest.ssa);
535
536 nir_ssa_def *dst_coord = nir_iadd(&b, global_id, &dst_offset->dest.ssa);
537
538 nir_tex_instr *tex = nir_tex_instr_create(b.shader, 2);
539 tex->sampler_dim = dim;
540 tex->op = nir_texop_txf;
541 tex->src[0].src_type = nir_tex_src_coord;
542 tex->src[0].src = nir_src_for_ssa(nir_channels(&b, src_coord, is_3d ? 0x7 : 0x3));
543 tex->src[1].src_type = nir_tex_src_lod;
544 tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
545 tex->dest_type = nir_type_float;
546 tex->is_array = false;
547 tex->coord_components = is_3d ? 3 : 2;
548 tex->texture = nir_deref_var_create(tex, input_img);
549 tex->sampler = NULL;
550
551 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
552 nir_builder_instr_insert(&b, &tex->instr);
553
554 nir_ssa_def *outval = &tex->dest.ssa;
555 nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
556 store->src[0] = nir_src_for_ssa(dst_coord);
557 store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
558 store->src[2] = nir_src_for_ssa(outval);
559 store->variables[0] = nir_deref_var_create(store, output_img);
560
561 nir_builder_instr_insert(&b, &store->instr);
562 return b.shader;
563 }
564
565 /* image to image - don't write use image accessors */
566 static VkResult
radv_device_init_meta_itoi_state(struct radv_device * device)567 radv_device_init_meta_itoi_state(struct radv_device *device)
568 {
569 VkResult result;
570 struct radv_shader_module cs = { .nir = NULL };
571 struct radv_shader_module cs_3d = { .nir = NULL };
572 cs.nir = build_nir_itoi_compute_shader(device, false);
573 if (device->physical_device->rad_info.chip_class >= GFX9)
574 cs_3d.nir = build_nir_itoi_compute_shader(device, true);
575 /*
576 * two descriptors one for the image being sampled
577 * one for the buffer being written.
578 */
579 VkDescriptorSetLayoutCreateInfo ds_create_info = {
580 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
581 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
582 .bindingCount = 2,
583 .pBindings = (VkDescriptorSetLayoutBinding[]) {
584 {
585 .binding = 0,
586 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
587 .descriptorCount = 1,
588 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
589 .pImmutableSamplers = NULL
590 },
591 {
592 .binding = 1,
593 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
594 .descriptorCount = 1,
595 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
596 .pImmutableSamplers = NULL
597 },
598 }
599 };
600
601 result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
602 &ds_create_info,
603 &device->meta_state.alloc,
604 &device->meta_state.itoi.img_ds_layout);
605 if (result != VK_SUCCESS)
606 goto fail;
607
608
609 VkPipelineLayoutCreateInfo pl_create_info = {
610 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
611 .setLayoutCount = 1,
612 .pSetLayouts = &device->meta_state.itoi.img_ds_layout,
613 .pushConstantRangeCount = 1,
614 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 24},
615 };
616
617 result = radv_CreatePipelineLayout(radv_device_to_handle(device),
618 &pl_create_info,
619 &device->meta_state.alloc,
620 &device->meta_state.itoi.img_p_layout);
621 if (result != VK_SUCCESS)
622 goto fail;
623
624 /* compute shader */
625
626 VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
627 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
628 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
629 .module = radv_shader_module_to_handle(&cs),
630 .pName = "main",
631 .pSpecializationInfo = NULL,
632 };
633
634 VkComputePipelineCreateInfo vk_pipeline_info = {
635 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
636 .stage = pipeline_shader_stage,
637 .flags = 0,
638 .layout = device->meta_state.itoi.img_p_layout,
639 };
640
641 result = radv_CreateComputePipelines(radv_device_to_handle(device),
642 radv_pipeline_cache_to_handle(&device->meta_state.cache),
643 1, &vk_pipeline_info, NULL,
644 &device->meta_state.itoi.pipeline);
645 if (result != VK_SUCCESS)
646 goto fail;
647
648 if (device->physical_device->rad_info.chip_class >= GFX9) {
649 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = {
650 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
651 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
652 .module = radv_shader_module_to_handle(&cs_3d),
653 .pName = "main",
654 .pSpecializationInfo = NULL,
655 };
656
657 VkComputePipelineCreateInfo vk_pipeline_info_3d = {
658 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
659 .stage = pipeline_shader_stage_3d,
660 .flags = 0,
661 .layout = device->meta_state.itoi.img_p_layout,
662 };
663
664 result = radv_CreateComputePipelines(radv_device_to_handle(device),
665 radv_pipeline_cache_to_handle(&device->meta_state.cache),
666 1, &vk_pipeline_info_3d, NULL,
667 &device->meta_state.itoi.pipeline_3d);
668
669 ralloc_free(cs_3d.nir);
670 }
671 ralloc_free(cs.nir);
672
673 return VK_SUCCESS;
674 fail:
675 ralloc_free(cs.nir);
676 ralloc_free(cs_3d.nir);
677 return result;
678 }
679
680 static void
radv_device_finish_meta_itoi_state(struct radv_device * device)681 radv_device_finish_meta_itoi_state(struct radv_device *device)
682 {
683 struct radv_meta_state *state = &device->meta_state;
684
685 radv_DestroyPipelineLayout(radv_device_to_handle(device),
686 state->itoi.img_p_layout, &state->alloc);
687 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
688 state->itoi.img_ds_layout,
689 &state->alloc);
690 radv_DestroyPipeline(radv_device_to_handle(device),
691 state->itoi.pipeline, &state->alloc);
692 if (device->physical_device->rad_info.chip_class >= GFX9)
693 radv_DestroyPipeline(radv_device_to_handle(device),
694 state->itoi.pipeline_3d, &state->alloc);
695 }
696
697 static nir_shader *
build_nir_cleari_compute_shader(struct radv_device * dev,bool is_3d)698 build_nir_cleari_compute_shader(struct radv_device *dev, bool is_3d)
699 {
700 nir_builder b;
701 enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D;
702 const struct glsl_type *img_type = glsl_sampler_type(dim,
703 false,
704 false,
705 GLSL_TYPE_FLOAT);
706 nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
707 b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_cleari_cs_3d" : "meta_cleari_cs");
708 b.shader->info.cs.local_size[0] = 16;
709 b.shader->info.cs.local_size[1] = 16;
710 b.shader->info.cs.local_size[2] = 1;
711
712 nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
713 img_type, "out_img");
714 output_img->data.descriptor_set = 0;
715 output_img->data.binding = 0;
716
717 nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
718 nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
719 nir_ssa_def *block_size = nir_imm_ivec4(&b,
720 b.shader->info.cs.local_size[0],
721 b.shader->info.cs.local_size[1],
722 b.shader->info.cs.local_size[2], 0);
723
724 nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
725
726 nir_intrinsic_instr *clear_val = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
727 nir_intrinsic_set_base(clear_val, 0);
728 nir_intrinsic_set_range(clear_val, 20);
729 clear_val->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
730 clear_val->num_components = 4;
731 nir_ssa_dest_init(&clear_val->instr, &clear_val->dest, 4, 32, "clear_value");
732 nir_builder_instr_insert(&b, &clear_val->instr);
733
734 nir_intrinsic_instr *layer = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
735 nir_intrinsic_set_base(layer, 0);
736 nir_intrinsic_set_range(layer, 20);
737 layer->src[0] = nir_src_for_ssa(nir_imm_int(&b, 16));
738 layer->num_components = 1;
739 nir_ssa_dest_init(&layer->instr, &layer->dest, 1, 32, "layer");
740 nir_builder_instr_insert(&b, &layer->instr);
741
742 nir_ssa_def *global_z = nir_iadd(&b, nir_channel(&b, global_id, 2), &layer->dest.ssa);
743
744 nir_ssa_def *comps[4];
745 comps[0] = nir_channel(&b, global_id, 0);
746 comps[1] = nir_channel(&b, global_id, 1);
747 comps[2] = global_z;
748 comps[3] = nir_imm_int(&b, 0);
749 global_id = nir_vec(&b, comps, 4);
750
751 nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
752 store->src[0] = nir_src_for_ssa(global_id);
753 store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
754 store->src[2] = nir_src_for_ssa(&clear_val->dest.ssa);
755 store->variables[0] = nir_deref_var_create(store, output_img);
756
757 nir_builder_instr_insert(&b, &store->instr);
758 return b.shader;
759 }
760
761 static VkResult
radv_device_init_meta_cleari_state(struct radv_device * device)762 radv_device_init_meta_cleari_state(struct radv_device *device)
763 {
764 VkResult result;
765 struct radv_shader_module cs = { .nir = NULL };
766 struct radv_shader_module cs_3d = { .nir = NULL };
767 cs.nir = build_nir_cleari_compute_shader(device, false);
768 if (device->physical_device->rad_info.chip_class >= GFX9)
769 cs_3d.nir = build_nir_cleari_compute_shader(device, true);
770
771 /*
772 * two descriptors one for the image being sampled
773 * one for the buffer being written.
774 */
775 VkDescriptorSetLayoutCreateInfo ds_create_info = {
776 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
777 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
778 .bindingCount = 1,
779 .pBindings = (VkDescriptorSetLayoutBinding[]) {
780 {
781 .binding = 0,
782 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
783 .descriptorCount = 1,
784 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
785 .pImmutableSamplers = NULL
786 },
787 }
788 };
789
790 result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
791 &ds_create_info,
792 &device->meta_state.alloc,
793 &device->meta_state.cleari.img_ds_layout);
794 if (result != VK_SUCCESS)
795 goto fail;
796
797
798 VkPipelineLayoutCreateInfo pl_create_info = {
799 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
800 .setLayoutCount = 1,
801 .pSetLayouts = &device->meta_state.cleari.img_ds_layout,
802 .pushConstantRangeCount = 1,
803 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 20},
804 };
805
806 result = radv_CreatePipelineLayout(radv_device_to_handle(device),
807 &pl_create_info,
808 &device->meta_state.alloc,
809 &device->meta_state.cleari.img_p_layout);
810 if (result != VK_SUCCESS)
811 goto fail;
812
813 /* compute shader */
814
815 VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
816 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
817 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
818 .module = radv_shader_module_to_handle(&cs),
819 .pName = "main",
820 .pSpecializationInfo = NULL,
821 };
822
823 VkComputePipelineCreateInfo vk_pipeline_info = {
824 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
825 .stage = pipeline_shader_stage,
826 .flags = 0,
827 .layout = device->meta_state.cleari.img_p_layout,
828 };
829
830 result = radv_CreateComputePipelines(radv_device_to_handle(device),
831 radv_pipeline_cache_to_handle(&device->meta_state.cache),
832 1, &vk_pipeline_info, NULL,
833 &device->meta_state.cleari.pipeline);
834 if (result != VK_SUCCESS)
835 goto fail;
836
837
838 if (device->physical_device->rad_info.chip_class >= GFX9) {
839 /* compute shader */
840 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = {
841 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
842 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
843 .module = radv_shader_module_to_handle(&cs_3d),
844 .pName = "main",
845 .pSpecializationInfo = NULL,
846 };
847
848 VkComputePipelineCreateInfo vk_pipeline_info_3d = {
849 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
850 .stage = pipeline_shader_stage_3d,
851 .flags = 0,
852 .layout = device->meta_state.cleari.img_p_layout,
853 };
854
855 result = radv_CreateComputePipelines(radv_device_to_handle(device),
856 radv_pipeline_cache_to_handle(&device->meta_state.cache),
857 1, &vk_pipeline_info_3d, NULL,
858 &device->meta_state.cleari.pipeline_3d);
859 if (result != VK_SUCCESS)
860 goto fail;
861
862 ralloc_free(cs_3d.nir);
863 }
864 ralloc_free(cs.nir);
865 return VK_SUCCESS;
866 fail:
867 ralloc_free(cs.nir);
868 ralloc_free(cs_3d.nir);
869 return result;
870 }
871
872 static void
radv_device_finish_meta_cleari_state(struct radv_device * device)873 radv_device_finish_meta_cleari_state(struct radv_device *device)
874 {
875 struct radv_meta_state *state = &device->meta_state;
876
877 radv_DestroyPipelineLayout(radv_device_to_handle(device),
878 state->cleari.img_p_layout, &state->alloc);
879 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
880 state->cleari.img_ds_layout,
881 &state->alloc);
882 radv_DestroyPipeline(radv_device_to_handle(device),
883 state->cleari.pipeline, &state->alloc);
884 radv_DestroyPipeline(radv_device_to_handle(device),
885 state->cleari.pipeline_3d, &state->alloc);
886 }
887
888 void
radv_device_finish_meta_bufimage_state(struct radv_device * device)889 radv_device_finish_meta_bufimage_state(struct radv_device *device)
890 {
891 radv_device_finish_meta_itob_state(device);
892 radv_device_finish_meta_btoi_state(device);
893 radv_device_finish_meta_itoi_state(device);
894 radv_device_finish_meta_cleari_state(device);
895 }
896
897 VkResult
radv_device_init_meta_bufimage_state(struct radv_device * device)898 radv_device_init_meta_bufimage_state(struct radv_device *device)
899 {
900 VkResult result;
901
902 result = radv_device_init_meta_itob_state(device);
903 if (result != VK_SUCCESS)
904 goto fail_itob;
905
906 result = radv_device_init_meta_btoi_state(device);
907 if (result != VK_SUCCESS)
908 goto fail_btoi;
909
910 result = radv_device_init_meta_itoi_state(device);
911 if (result != VK_SUCCESS)
912 goto fail_itoi;
913
914 result = radv_device_init_meta_cleari_state(device);
915 if (result != VK_SUCCESS)
916 goto fail_cleari;
917
918 return VK_SUCCESS;
919 fail_cleari:
920 radv_device_finish_meta_cleari_state(device);
921 fail_itoi:
922 radv_device_finish_meta_itoi_state(device);
923 fail_btoi:
924 radv_device_finish_meta_btoi_state(device);
925 fail_itob:
926 radv_device_finish_meta_itob_state(device);
927 return result;
928 }
929
930 static void
create_iview(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * surf,struct radv_image_view * iview)931 create_iview(struct radv_cmd_buffer *cmd_buffer,
932 struct radv_meta_blit2d_surf *surf,
933 struct radv_image_view *iview)
934 {
935 VkImageViewType view_type = cmd_buffer->device->physical_device->rad_info.chip_class < GFX9 ? VK_IMAGE_VIEW_TYPE_2D :
936 radv_meta_get_view_type(surf->image);
937 radv_image_view_init(iview, cmd_buffer->device,
938 &(VkImageViewCreateInfo) {
939 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
940 .image = radv_image_to_handle(surf->image),
941 .viewType = view_type,
942 .format = surf->format,
943 .subresourceRange = {
944 .aspectMask = surf->aspect_mask,
945 .baseMipLevel = surf->level,
946 .levelCount = 1,
947 .baseArrayLayer = surf->layer,
948 .layerCount = 1
949 },
950 });
951 }
952
953 static void
create_bview(struct radv_cmd_buffer * cmd_buffer,struct radv_buffer * buffer,unsigned offset,VkFormat format,struct radv_buffer_view * bview)954 create_bview(struct radv_cmd_buffer *cmd_buffer,
955 struct radv_buffer *buffer,
956 unsigned offset,
957 VkFormat format,
958 struct radv_buffer_view *bview)
959 {
960 radv_buffer_view_init(bview, cmd_buffer->device,
961 &(VkBufferViewCreateInfo) {
962 .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO,
963 .flags = 0,
964 .buffer = radv_buffer_to_handle(buffer),
965 .format = format,
966 .offset = offset,
967 .range = VK_WHOLE_SIZE,
968 });
969
970 }
971
972 static void
itob_bind_descriptors(struct radv_cmd_buffer * cmd_buffer,struct radv_image_view * src,struct radv_buffer_view * dst)973 itob_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
974 struct radv_image_view *src,
975 struct radv_buffer_view *dst)
976 {
977 struct radv_device *device = cmd_buffer->device;
978
979 radv_meta_push_descriptor_set(cmd_buffer,
980 VK_PIPELINE_BIND_POINT_COMPUTE,
981 device->meta_state.itob.img_p_layout,
982 0, /* set */
983 2, /* descriptorWriteCount */
984 (VkWriteDescriptorSet[]) {
985 {
986 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
987 .dstBinding = 0,
988 .dstArrayElement = 0,
989 .descriptorCount = 1,
990 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
991 .pImageInfo = (VkDescriptorImageInfo[]) {
992 {
993 .sampler = VK_NULL_HANDLE,
994 .imageView = radv_image_view_to_handle(src),
995 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
996 },
997 }
998 },
999 {
1000 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1001 .dstBinding = 1,
1002 .dstArrayElement = 0,
1003 .descriptorCount = 1,
1004 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
1005 .pTexelBufferView = (VkBufferView[]) { radv_buffer_view_to_handle(dst) },
1006 }
1007 });
1008 }
1009
1010 void
radv_meta_image_to_buffer(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * src,struct radv_meta_blit2d_buffer * dst,unsigned num_rects,struct radv_meta_blit2d_rect * rects)1011 radv_meta_image_to_buffer(struct radv_cmd_buffer *cmd_buffer,
1012 struct radv_meta_blit2d_surf *src,
1013 struct radv_meta_blit2d_buffer *dst,
1014 unsigned num_rects,
1015 struct radv_meta_blit2d_rect *rects)
1016 {
1017 VkPipeline pipeline = cmd_buffer->device->meta_state.itob.pipeline;
1018 struct radv_device *device = cmd_buffer->device;
1019 struct radv_image_view src_view;
1020 struct radv_buffer_view dst_view;
1021
1022 create_iview(cmd_buffer, src, &src_view);
1023 create_bview(cmd_buffer, dst->buffer, dst->offset, dst->format, &dst_view);
1024 itob_bind_descriptors(cmd_buffer, &src_view, &dst_view);
1025
1026 if (device->physical_device->rad_info.chip_class >= GFX9 &&
1027 src->image->type == VK_IMAGE_TYPE_3D)
1028 pipeline = cmd_buffer->device->meta_state.itob.pipeline_3d;
1029
1030 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1031 VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1032
1033 for (unsigned r = 0; r < num_rects; ++r) {
1034 unsigned push_constants[4] = {
1035 rects[r].src_x,
1036 rects[r].src_y,
1037 src->layer,
1038 dst->pitch
1039 };
1040 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1041 device->meta_state.itob.img_p_layout,
1042 VK_SHADER_STAGE_COMPUTE_BIT, 0, 16,
1043 push_constants);
1044
1045 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
1046 }
1047 }
1048
1049 static void
btoi_bind_descriptors(struct radv_cmd_buffer * cmd_buffer,struct radv_buffer_view * src,struct radv_image_view * dst)1050 btoi_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
1051 struct radv_buffer_view *src,
1052 struct radv_image_view *dst)
1053 {
1054 struct radv_device *device = cmd_buffer->device;
1055
1056 radv_meta_push_descriptor_set(cmd_buffer,
1057 VK_PIPELINE_BIND_POINT_COMPUTE,
1058 device->meta_state.btoi.img_p_layout,
1059 0, /* set */
1060 2, /* descriptorWriteCount */
1061 (VkWriteDescriptorSet[]) {
1062 {
1063 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1064 .dstBinding = 0,
1065 .dstArrayElement = 0,
1066 .descriptorCount = 1,
1067 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
1068 .pTexelBufferView = (VkBufferView[]) { radv_buffer_view_to_handle(src) },
1069 },
1070 {
1071 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1072 .dstBinding = 1,
1073 .dstArrayElement = 0,
1074 .descriptorCount = 1,
1075 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
1076 .pImageInfo = (VkDescriptorImageInfo[]) {
1077 {
1078 .sampler = VK_NULL_HANDLE,
1079 .imageView = radv_image_view_to_handle(dst),
1080 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1081 },
1082 }
1083 }
1084 });
1085 }
1086
1087 void
radv_meta_buffer_to_image_cs(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_buffer * src,struct radv_meta_blit2d_surf * dst,unsigned num_rects,struct radv_meta_blit2d_rect * rects)1088 radv_meta_buffer_to_image_cs(struct radv_cmd_buffer *cmd_buffer,
1089 struct radv_meta_blit2d_buffer *src,
1090 struct radv_meta_blit2d_surf *dst,
1091 unsigned num_rects,
1092 struct radv_meta_blit2d_rect *rects)
1093 {
1094 VkPipeline pipeline = cmd_buffer->device->meta_state.btoi.pipeline;
1095 struct radv_device *device = cmd_buffer->device;
1096 struct radv_buffer_view src_view;
1097 struct radv_image_view dst_view;
1098
1099 create_bview(cmd_buffer, src->buffer, src->offset, src->format, &src_view);
1100 create_iview(cmd_buffer, dst, &dst_view);
1101 btoi_bind_descriptors(cmd_buffer, &src_view, &dst_view);
1102
1103 if (device->physical_device->rad_info.chip_class >= GFX9 &&
1104 dst->image->type == VK_IMAGE_TYPE_3D)
1105 pipeline = cmd_buffer->device->meta_state.btoi.pipeline_3d;
1106 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1107 VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1108
1109 for (unsigned r = 0; r < num_rects; ++r) {
1110 unsigned push_constants[4] = {
1111 rects[r].dst_x,
1112 rects[r].dst_y,
1113 dst->layer,
1114 src->pitch,
1115 };
1116 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1117 device->meta_state.btoi.img_p_layout,
1118 VK_SHADER_STAGE_COMPUTE_BIT, 0, 16,
1119 push_constants);
1120
1121 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
1122 }
1123 }
1124
1125 static void
itoi_bind_descriptors(struct radv_cmd_buffer * cmd_buffer,struct radv_image_view * src,struct radv_image_view * dst)1126 itoi_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
1127 struct radv_image_view *src,
1128 struct radv_image_view *dst)
1129 {
1130 struct radv_device *device = cmd_buffer->device;
1131
1132 radv_meta_push_descriptor_set(cmd_buffer,
1133 VK_PIPELINE_BIND_POINT_COMPUTE,
1134 device->meta_state.itoi.img_p_layout,
1135 0, /* set */
1136 2, /* descriptorWriteCount */
1137 (VkWriteDescriptorSet[]) {
1138 {
1139 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1140 .dstBinding = 0,
1141 .dstArrayElement = 0,
1142 .descriptorCount = 1,
1143 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
1144 .pImageInfo = (VkDescriptorImageInfo[]) {
1145 {
1146 .sampler = VK_NULL_HANDLE,
1147 .imageView = radv_image_view_to_handle(src),
1148 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1149 },
1150 }
1151 },
1152 {
1153 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1154 .dstBinding = 1,
1155 .dstArrayElement = 0,
1156 .descriptorCount = 1,
1157 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
1158 .pImageInfo = (VkDescriptorImageInfo[]) {
1159 {
1160 .sampler = VK_NULL_HANDLE,
1161 .imageView = radv_image_view_to_handle(dst),
1162 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1163 },
1164 }
1165 }
1166 });
1167 }
1168
1169 void
radv_meta_image_to_image_cs(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * src,struct radv_meta_blit2d_surf * dst,unsigned num_rects,struct radv_meta_blit2d_rect * rects)1170 radv_meta_image_to_image_cs(struct radv_cmd_buffer *cmd_buffer,
1171 struct radv_meta_blit2d_surf *src,
1172 struct radv_meta_blit2d_surf *dst,
1173 unsigned num_rects,
1174 struct radv_meta_blit2d_rect *rects)
1175 {
1176 VkPipeline pipeline = cmd_buffer->device->meta_state.itoi.pipeline;
1177 struct radv_device *device = cmd_buffer->device;
1178 struct radv_image_view src_view, dst_view;
1179
1180 create_iview(cmd_buffer, src, &src_view);
1181 create_iview(cmd_buffer, dst, &dst_view);
1182
1183 itoi_bind_descriptors(cmd_buffer, &src_view, &dst_view);
1184
1185 if (device->physical_device->rad_info.chip_class >= GFX9 &&
1186 src->image->type == VK_IMAGE_TYPE_3D)
1187 pipeline = cmd_buffer->device->meta_state.itoi.pipeline_3d;
1188 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1189 VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1190
1191 for (unsigned r = 0; r < num_rects; ++r) {
1192 unsigned push_constants[6] = {
1193 rects[r].src_x,
1194 rects[r].src_y,
1195 src->layer,
1196 rects[r].dst_x,
1197 rects[r].dst_y,
1198 dst->layer,
1199 };
1200 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1201 device->meta_state.itoi.img_p_layout,
1202 VK_SHADER_STAGE_COMPUTE_BIT, 0, 24,
1203 push_constants);
1204
1205 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
1206 }
1207 }
1208
1209 static void
cleari_bind_descriptors(struct radv_cmd_buffer * cmd_buffer,struct radv_image_view * dst_iview)1210 cleari_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
1211 struct radv_image_view *dst_iview)
1212 {
1213 struct radv_device *device = cmd_buffer->device;
1214
1215 radv_meta_push_descriptor_set(cmd_buffer,
1216 VK_PIPELINE_BIND_POINT_COMPUTE,
1217 device->meta_state.cleari.img_p_layout,
1218 0, /* set */
1219 1, /* descriptorWriteCount */
1220 (VkWriteDescriptorSet[]) {
1221 {
1222 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1223 .dstBinding = 0,
1224 .dstArrayElement = 0,
1225 .descriptorCount = 1,
1226 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
1227 .pImageInfo = (VkDescriptorImageInfo[]) {
1228 {
1229 .sampler = VK_NULL_HANDLE,
1230 .imageView = radv_image_view_to_handle(dst_iview),
1231 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1232 },
1233 }
1234 },
1235 });
1236 }
1237
1238 void
radv_meta_clear_image_cs(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * dst,const VkClearColorValue * clear_color)1239 radv_meta_clear_image_cs(struct radv_cmd_buffer *cmd_buffer,
1240 struct radv_meta_blit2d_surf *dst,
1241 const VkClearColorValue *clear_color)
1242 {
1243 VkPipeline pipeline = cmd_buffer->device->meta_state.cleari.pipeline;
1244 struct radv_device *device = cmd_buffer->device;
1245 struct radv_image_view dst_iview;
1246
1247 create_iview(cmd_buffer, dst, &dst_iview);
1248 cleari_bind_descriptors(cmd_buffer, &dst_iview);
1249
1250 if (device->physical_device->rad_info.chip_class >= GFX9 &&
1251 dst->image->type == VK_IMAGE_TYPE_3D)
1252 pipeline = cmd_buffer->device->meta_state.cleari.pipeline_3d;
1253
1254 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1255 VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1256
1257 unsigned push_constants[5] = {
1258 clear_color->uint32[0],
1259 clear_color->uint32[1],
1260 clear_color->uint32[2],
1261 clear_color->uint32[3],
1262 dst->layer,
1263 };
1264
1265 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1266 device->meta_state.cleari.img_p_layout,
1267 VK_SHADER_STAGE_COMPUTE_BIT, 0, 20,
1268 push_constants);
1269
1270 radv_unaligned_dispatch(cmd_buffer, dst->image->info.width, dst->image->info.height, 1);
1271 }
1272