1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "nir/nir_builder.h"
25 #include "radv_meta.h"
26 #include "vk_common_entrypoints.h"
27
28 struct blit_region {
29 VkOffset3D src_offset;
30 VkExtent3D src_extent;
31 VkOffset3D dst_offset;
32 VkExtent3D dst_extent;
33 };
34
35 static VkResult build_pipeline(struct radv_device *device, VkImageAspectFlagBits aspect, enum glsl_sampler_dim tex_dim,
36 VkFormat format, VkPipeline *pipeline);
37
38 static nir_shader *
build_nir_vertex_shader(struct radv_device * dev)39 build_nir_vertex_shader(struct radv_device *dev)
40 {
41 const struct glsl_type *vec4 = glsl_vec4_type();
42 nir_builder b = radv_meta_init_shader(dev, MESA_SHADER_VERTEX, "meta_blit_vs");
43
44 nir_variable *pos_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "gl_Position");
45 pos_out->data.location = VARYING_SLOT_POS;
46
47 nir_variable *tex_pos_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "v_tex_pos");
48 tex_pos_out->data.location = VARYING_SLOT_VAR0;
49 tex_pos_out->data.interpolation = INTERP_MODE_SMOOTH;
50
51 nir_def *outvec = nir_gen_rect_vertices(&b, NULL, NULL);
52
53 nir_store_var(&b, pos_out, outvec, 0xf);
54
55 nir_def *src_box = nir_load_push_constant(&b, 4, 32, nir_imm_int(&b, 0), .range = 16);
56 nir_def *src0_z = nir_load_push_constant(&b, 1, 32, nir_imm_int(&b, 0), .base = 16, .range = 4);
57
58 nir_def *vertex_id = nir_load_vertex_id_zero_base(&b);
59
60 /* vertex 0 - src0_x, src0_y, src0_z */
61 /* vertex 1 - src0_x, src1_y, src0_z*/
62 /* vertex 2 - src1_x, src0_y, src0_z */
63 /* so channel 0 is vertex_id != 2 ? src_x : src_x + w
64 channel 1 is vertex id != 1 ? src_y : src_y + w */
65
66 nir_def *c0cmp = nir_ine_imm(&b, vertex_id, 2);
67 nir_def *c1cmp = nir_ine_imm(&b, vertex_id, 1);
68
69 nir_def *comp[4];
70 comp[0] = nir_bcsel(&b, c0cmp, nir_channel(&b, src_box, 0), nir_channel(&b, src_box, 2));
71
72 comp[1] = nir_bcsel(&b, c1cmp, nir_channel(&b, src_box, 1), nir_channel(&b, src_box, 3));
73 comp[2] = src0_z;
74 comp[3] = nir_imm_float(&b, 1.0);
75 nir_def *out_tex_vec = nir_vec(&b, comp, 4);
76 nir_store_var(&b, tex_pos_out, out_tex_vec, 0xf);
77 return b.shader;
78 }
79
80 static nir_shader *
build_nir_copy_fragment_shader(struct radv_device * dev,enum glsl_sampler_dim tex_dim)81 build_nir_copy_fragment_shader(struct radv_device *dev, enum glsl_sampler_dim tex_dim)
82 {
83 const struct glsl_type *vec4 = glsl_vec4_type();
84 nir_builder b = radv_meta_init_shader(dev, MESA_SHADER_FRAGMENT, "meta_blit_fs.%d", tex_dim);
85
86 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec4, "v_tex_pos");
87 tex_pos_in->data.location = VARYING_SLOT_VAR0;
88
89 /* Swizzle the array index which comes in as Z coordinate into the right
90 * position.
91 */
92 unsigned swz[] = {0, (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 1), 2};
93 nir_def *const tex_pos =
94 nir_swizzle(&b, nir_load_var(&b, tex_pos_in), swz, (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 3));
95
96 const struct glsl_type *sampler_type =
97 glsl_sampler_type(tex_dim, false, tex_dim != GLSL_SAMPLER_DIM_3D, glsl_get_base_type(vec4));
98 nir_variable *sampler = nir_variable_create(b.shader, nir_var_uniform, sampler_type, "s_tex");
99 sampler->data.descriptor_set = 0;
100 sampler->data.binding = 0;
101
102 nir_deref_instr *tex_deref = nir_build_deref_var(&b, sampler);
103 nir_def *color = nir_tex_deref(&b, tex_deref, tex_deref, tex_pos);
104
105 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
106 color_out->data.location = FRAG_RESULT_DATA0;
107 nir_store_var(&b, color_out, color, 0xf);
108
109 return b.shader;
110 }
111
112 static nir_shader *
build_nir_copy_fragment_shader_depth(struct radv_device * dev,enum glsl_sampler_dim tex_dim)113 build_nir_copy_fragment_shader_depth(struct radv_device *dev, enum glsl_sampler_dim tex_dim)
114 {
115 const struct glsl_type *vec4 = glsl_vec4_type();
116 nir_builder b = radv_meta_init_shader(dev, MESA_SHADER_FRAGMENT, "meta_blit_depth_fs.%d", tex_dim);
117
118 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec4, "v_tex_pos");
119 tex_pos_in->data.location = VARYING_SLOT_VAR0;
120
121 /* Swizzle the array index which comes in as Z coordinate into the right
122 * position.
123 */
124 unsigned swz[] = {0, (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 1), 2};
125 nir_def *const tex_pos =
126 nir_swizzle(&b, nir_load_var(&b, tex_pos_in), swz, (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 3));
127
128 const struct glsl_type *sampler_type =
129 glsl_sampler_type(tex_dim, false, tex_dim != GLSL_SAMPLER_DIM_3D, glsl_get_base_type(vec4));
130 nir_variable *sampler = nir_variable_create(b.shader, nir_var_uniform, sampler_type, "s_tex");
131 sampler->data.descriptor_set = 0;
132 sampler->data.binding = 0;
133
134 nir_deref_instr *tex_deref = nir_build_deref_var(&b, sampler);
135 nir_def *color = nir_tex_deref(&b, tex_deref, tex_deref, tex_pos);
136
137 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
138 color_out->data.location = FRAG_RESULT_DEPTH;
139 nir_store_var(&b, color_out, color, 0x1);
140
141 return b.shader;
142 }
143
144 static nir_shader *
build_nir_copy_fragment_shader_stencil(struct radv_device * dev,enum glsl_sampler_dim tex_dim)145 build_nir_copy_fragment_shader_stencil(struct radv_device *dev, enum glsl_sampler_dim tex_dim)
146 {
147 const struct glsl_type *vec4 = glsl_vec4_type();
148 nir_builder b = radv_meta_init_shader(dev, MESA_SHADER_FRAGMENT, "meta_blit_stencil_fs.%d", tex_dim);
149
150 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec4, "v_tex_pos");
151 tex_pos_in->data.location = VARYING_SLOT_VAR0;
152
153 /* Swizzle the array index which comes in as Z coordinate into the right
154 * position.
155 */
156 unsigned swz[] = {0, (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 1), 2};
157 nir_def *const tex_pos =
158 nir_swizzle(&b, nir_load_var(&b, tex_pos_in), swz, (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 3));
159
160 const struct glsl_type *sampler_type =
161 glsl_sampler_type(tex_dim, false, tex_dim != GLSL_SAMPLER_DIM_3D, glsl_get_base_type(vec4));
162 nir_variable *sampler = nir_variable_create(b.shader, nir_var_uniform, sampler_type, "s_tex");
163 sampler->data.descriptor_set = 0;
164 sampler->data.binding = 0;
165
166 nir_deref_instr *tex_deref = nir_build_deref_var(&b, sampler);
167 nir_def *color = nir_tex_deref(&b, tex_deref, tex_deref, tex_pos);
168
169 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
170 color_out->data.location = FRAG_RESULT_STENCIL;
171 nir_store_var(&b, color_out, color, 0x1);
172
173 return b.shader;
174 }
175
176 static enum glsl_sampler_dim
translate_sampler_dim(VkImageType type)177 translate_sampler_dim(VkImageType type)
178 {
179 switch (type) {
180 case VK_IMAGE_TYPE_1D:
181 return GLSL_SAMPLER_DIM_1D;
182 case VK_IMAGE_TYPE_2D:
183 return GLSL_SAMPLER_DIM_2D;
184 case VK_IMAGE_TYPE_3D:
185 return GLSL_SAMPLER_DIM_3D;
186 default:
187 unreachable("Unhandled image type");
188 }
189 }
190
191 static void
meta_emit_blit(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,struct radv_image_view * src_iview,VkImageLayout src_image_layout,float src_offset_0[3],float src_offset_1[3],struct radv_image * dst_image,struct radv_image_view * dst_iview,VkImageLayout dst_image_layout,VkRect2D dst_box,VkSampler sampler)192 meta_emit_blit(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, struct radv_image_view *src_iview,
193 VkImageLayout src_image_layout, float src_offset_0[3], float src_offset_1[3],
194 struct radv_image *dst_image, struct radv_image_view *dst_iview, VkImageLayout dst_image_layout,
195 VkRect2D dst_box, VkSampler sampler)
196 {
197 struct radv_device *device = cmd_buffer->device;
198 uint32_t src_width = radv_minify(src_iview->image->vk.extent.width, src_iview->vk.base_mip_level);
199 uint32_t src_height = radv_minify(src_iview->image->vk.extent.height, src_iview->vk.base_mip_level);
200 uint32_t src_depth = radv_minify(src_iview->image->vk.extent.depth, src_iview->vk.base_mip_level);
201 uint32_t dst_width = radv_minify(dst_iview->image->vk.extent.width, dst_iview->vk.base_mip_level);
202 uint32_t dst_height = radv_minify(dst_iview->image->vk.extent.height, dst_iview->vk.base_mip_level);
203
204 assert(src_image->vk.samples == dst_image->vk.samples);
205
206 float vertex_push_constants[5] = {
207 src_offset_0[0] / (float)src_width, src_offset_0[1] / (float)src_height, src_offset_1[0] / (float)src_width,
208 src_offset_1[1] / (float)src_height, src_offset_0[2] / (float)src_depth,
209 };
210
211 vk_common_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), device->meta_state.blit.pipeline_layout,
212 VK_SHADER_STAGE_VERTEX_BIT, 0, 20, vertex_push_constants);
213
214 VkPipeline *pipeline = NULL;
215 unsigned fs_key = 0;
216 VkFormat format = VK_FORMAT_UNDEFINED;
217
218 switch (src_iview->vk.aspects) {
219 case VK_IMAGE_ASPECT_COLOR_BIT: {
220 fs_key = radv_format_meta_fs_key(device, dst_image->vk.format);
221 format = radv_fs_key_format_exemplars[fs_key];
222
223 switch (src_image->vk.image_type) {
224 case VK_IMAGE_TYPE_1D:
225 pipeline = &device->meta_state.blit.pipeline_1d_src[fs_key];
226 break;
227 case VK_IMAGE_TYPE_2D:
228 pipeline = &device->meta_state.blit.pipeline_2d_src[fs_key];
229 break;
230 case VK_IMAGE_TYPE_3D:
231 pipeline = &device->meta_state.blit.pipeline_3d_src[fs_key];
232 break;
233 default:
234 unreachable("bad VkImageType");
235 }
236 break;
237 }
238 case VK_IMAGE_ASPECT_DEPTH_BIT: {
239 format = VK_FORMAT_D32_SFLOAT;
240
241 switch (src_image->vk.image_type) {
242 case VK_IMAGE_TYPE_1D:
243 pipeline = &device->meta_state.blit.depth_only_1d_pipeline;
244 break;
245 case VK_IMAGE_TYPE_2D:
246 pipeline = &device->meta_state.blit.depth_only_2d_pipeline;
247 break;
248 case VK_IMAGE_TYPE_3D:
249 pipeline = &device->meta_state.blit.depth_only_3d_pipeline;
250 break;
251 default:
252 unreachable("bad VkImageType");
253 }
254 break;
255 }
256 case VK_IMAGE_ASPECT_STENCIL_BIT: {
257 format = VK_FORMAT_S8_UINT;
258
259 switch (src_image->vk.image_type) {
260 case VK_IMAGE_TYPE_1D:
261 pipeline = &device->meta_state.blit.stencil_only_1d_pipeline;
262 break;
263 case VK_IMAGE_TYPE_2D:
264 pipeline = &device->meta_state.blit.stencil_only_2d_pipeline;
265 break;
266 case VK_IMAGE_TYPE_3D:
267 pipeline = &device->meta_state.blit.stencil_only_3d_pipeline;
268 break;
269 default:
270 unreachable("bad VkImageType");
271 }
272 break;
273 }
274 default:
275 unreachable("bad VkImageType");
276 }
277
278 if (!*pipeline) {
279 VkResult ret = build_pipeline(device, src_iview->vk.aspects, translate_sampler_dim(src_image->vk.image_type),
280 format, pipeline);
281 if (ret != VK_SUCCESS) {
282 vk_command_buffer_set_error(&cmd_buffer->vk, ret);
283 return;
284 }
285 }
286
287 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS, *pipeline);
288
289 radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, device->meta_state.blit.pipeline_layout,
290 0, /* set */
291 1, /* descriptorWriteCount */
292 (VkWriteDescriptorSet[]){{.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
293 .dstBinding = 0,
294 .dstArrayElement = 0,
295 .descriptorCount = 1,
296 .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
297 .pImageInfo = (VkDescriptorImageInfo[]){
298 {
299 .sampler = sampler,
300 .imageView = radv_image_view_to_handle(src_iview),
301 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
302 },
303 }}});
304
305 VkRenderingInfo rendering_info = {
306 .sType = VK_STRUCTURE_TYPE_RENDERING_INFO,
307 .renderArea =
308 {
309 .offset = {0, 0},
310 .extent = {dst_width, dst_height},
311 },
312 .layerCount = 1,
313 };
314
315 VkRenderingAttachmentInfo color_att;
316 if (src_iview->image->vk.aspects == VK_IMAGE_ASPECT_COLOR_BIT) {
317 unsigned dst_layout = radv_meta_dst_layout_from_layout(dst_image_layout);
318 VkImageLayout layout = radv_meta_dst_layout_to_layout(dst_layout);
319
320 color_att = (VkRenderingAttachmentInfo){
321 .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
322 .imageView = radv_image_view_to_handle(dst_iview),
323 .imageLayout = layout,
324 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
325 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
326 };
327 rendering_info.colorAttachmentCount = 1;
328 rendering_info.pColorAttachments = &color_att;
329 }
330
331 VkRenderingAttachmentInfo depth_att;
332 if (src_iview->image->vk.aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
333 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst_image_layout);
334 VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
335
336 depth_att = (VkRenderingAttachmentInfo){
337 .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
338 .imageView = radv_image_view_to_handle(dst_iview),
339 .imageLayout = layout,
340 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
341 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
342 };
343 rendering_info.pDepthAttachment = &depth_att;
344 }
345
346 VkRenderingAttachmentInfo stencil_att;
347 if (src_iview->image->vk.aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
348 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst_image_layout);
349 VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
350
351 stencil_att = (VkRenderingAttachmentInfo){
352 .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
353 .imageView = radv_image_view_to_handle(dst_iview),
354 .imageLayout = layout,
355 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
356 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
357 };
358 rendering_info.pStencilAttachment = &stencil_att;
359 }
360
361 radv_CmdBeginRendering(radv_cmd_buffer_to_handle(cmd_buffer), &rendering_info);
362
363 radv_CmdDraw(radv_cmd_buffer_to_handle(cmd_buffer), 3, 1, 0, 0);
364
365 radv_CmdEndRendering(radv_cmd_buffer_to_handle(cmd_buffer));
366 }
367
368 static bool
flip_coords(unsigned * src0,unsigned * src1,unsigned * dst0,unsigned * dst1)369 flip_coords(unsigned *src0, unsigned *src1, unsigned *dst0, unsigned *dst1)
370 {
371 bool flip = false;
372 if (*src0 > *src1) {
373 unsigned tmp = *src0;
374 *src0 = *src1;
375 *src1 = tmp;
376 flip = !flip;
377 }
378
379 if (*dst0 > *dst1) {
380 unsigned tmp = *dst0;
381 *dst0 = *dst1;
382 *dst1 = tmp;
383 flip = !flip;
384 }
385 return flip;
386 }
387
388 static void
blit_image(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,VkImageLayout src_image_layout,struct radv_image * dst_image,VkImageLayout dst_image_layout,const VkImageBlit2 * region,VkFilter filter)389 blit_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, VkImageLayout src_image_layout,
390 struct radv_image *dst_image, VkImageLayout dst_image_layout, const VkImageBlit2 *region, VkFilter filter)
391 {
392 const VkImageSubresourceLayers *src_res = ®ion->srcSubresource;
393 const VkImageSubresourceLayers *dst_res = ®ion->dstSubresource;
394 struct radv_device *device = cmd_buffer->device;
395 struct radv_meta_saved_state saved_state;
396 VkSampler sampler;
397
398 /* From the Vulkan 1.0 spec:
399 *
400 * vkCmdBlitImage must not be used for multisampled source or
401 * destination images. Use vkCmdResolveImage for this purpose.
402 */
403 assert(src_image->vk.samples == 1);
404 assert(dst_image->vk.samples == 1);
405
406 radv_CreateSampler(radv_device_to_handle(device),
407 &(VkSamplerCreateInfo){
408 .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO,
409 .magFilter = filter,
410 .minFilter = filter,
411 .addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
412 .addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
413 .addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
414 },
415 &cmd_buffer->vk.pool->alloc, &sampler);
416
417 /* VK_EXT_conditional_rendering says that blit commands should not be
418 * affected by conditional rendering.
419 */
420 radv_meta_save(&saved_state, cmd_buffer,
421 RADV_META_SAVE_GRAPHICS_PIPELINE | RADV_META_SAVE_CONSTANTS | RADV_META_SAVE_DESCRIPTORS |
422 RADV_META_SUSPEND_PREDICATING);
423
424 unsigned dst_start, dst_end;
425 if (dst_image->vk.image_type == VK_IMAGE_TYPE_3D) {
426 assert(dst_res->baseArrayLayer == 0);
427 dst_start = region->dstOffsets[0].z;
428 dst_end = region->dstOffsets[1].z;
429 } else {
430 dst_start = dst_res->baseArrayLayer;
431 dst_end = dst_start + vk_image_subresource_layer_count(&dst_image->vk, dst_res);
432 }
433
434 unsigned src_start, src_end;
435 if (src_image->vk.image_type == VK_IMAGE_TYPE_3D) {
436 assert(src_res->baseArrayLayer == 0);
437 src_start = region->srcOffsets[0].z;
438 src_end = region->srcOffsets[1].z;
439 } else {
440 src_start = src_res->baseArrayLayer;
441 src_end = src_start + vk_image_subresource_layer_count(&src_image->vk, src_res);
442 }
443
444 bool flip_z = flip_coords(&src_start, &src_end, &dst_start, &dst_end);
445 float src_z_step = (float)(src_end - src_start) / (float)(dst_end - dst_start);
446
447 /* There is no interpolation to the pixel center during
448 * rendering, so add the 0.5 offset ourselves here. */
449 float depth_center_offset = 0;
450 if (src_image->vk.image_type == VK_IMAGE_TYPE_3D)
451 depth_center_offset = 0.5 / (dst_end - dst_start) * (src_end - src_start);
452
453 if (flip_z) {
454 src_start = src_end;
455 src_z_step *= -1;
456 depth_center_offset *= -1;
457 }
458
459 unsigned src_x0 = region->srcOffsets[0].x;
460 unsigned src_x1 = region->srcOffsets[1].x;
461 unsigned dst_x0 = region->dstOffsets[0].x;
462 unsigned dst_x1 = region->dstOffsets[1].x;
463
464 unsigned src_y0 = region->srcOffsets[0].y;
465 unsigned src_y1 = region->srcOffsets[1].y;
466 unsigned dst_y0 = region->dstOffsets[0].y;
467 unsigned dst_y1 = region->dstOffsets[1].y;
468
469 VkRect2D dst_box;
470 dst_box.offset.x = MIN2(dst_x0, dst_x1);
471 dst_box.offset.y = MIN2(dst_y0, dst_y1);
472 dst_box.extent.width = dst_x1 - dst_x0;
473 dst_box.extent.height = dst_y1 - dst_y0;
474
475 const VkOffset2D dst_offset_0 = {
476 .x = dst_x0,
477 .y = dst_y0,
478 };
479 const VkOffset2D dst_offset_1 = {
480 .x = dst_x1,
481 .y = dst_y1,
482 };
483
484 radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
485 &(VkViewport){.x = dst_offset_0.x,
486 .y = dst_offset_0.y,
487 .width = dst_offset_1.x - dst_offset_0.x,
488 .height = dst_offset_1.y - dst_offset_0.y,
489 .minDepth = 0.0f,
490 .maxDepth = 1.0f});
491
492 radv_CmdSetScissor(
493 radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
494 &(VkRect2D){
495 .offset = (VkOffset2D){MIN2(dst_offset_0.x, dst_offset_1.x), MIN2(dst_offset_0.y, dst_offset_1.y)},
496 .extent = (VkExtent2D){abs(dst_offset_1.x - dst_offset_0.x), abs(dst_offset_1.y - dst_offset_0.y)},
497 });
498
499 const unsigned num_layers = dst_end - dst_start;
500 for (unsigned i = 0; i < num_layers; i++) {
501 struct radv_image_view dst_iview, src_iview;
502
503 float src_offset_0[3] = {
504 src_x0,
505 src_y0,
506 src_start + i * src_z_step + depth_center_offset,
507 };
508 float src_offset_1[3] = {
509 src_x1,
510 src_y1,
511 src_start + i * src_z_step + depth_center_offset,
512 };
513 const uint32_t dst_array_slice = dst_start + i;
514
515 /* 3D images have just 1 layer */
516 const uint32_t src_array_slice = src_image->vk.image_type == VK_IMAGE_TYPE_3D ? 0 : src_start + i;
517
518 radv_image_view_init(&dst_iview, cmd_buffer->device,
519 &(VkImageViewCreateInfo){
520 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
521 .image = radv_image_to_handle(dst_image),
522 .viewType = radv_meta_get_view_type(dst_image),
523 .format = dst_image->vk.format,
524 .subresourceRange = {.aspectMask = dst_res->aspectMask,
525 .baseMipLevel = dst_res->mipLevel,
526 .levelCount = 1,
527 .baseArrayLayer = dst_array_slice,
528 .layerCount = 1},
529 },
530 0, NULL);
531 radv_image_view_init(&src_iview, cmd_buffer->device,
532 &(VkImageViewCreateInfo){
533 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
534 .image = radv_image_to_handle(src_image),
535 .viewType = radv_meta_get_view_type(src_image),
536 .format = src_image->vk.format,
537 .subresourceRange = {.aspectMask = src_res->aspectMask,
538 .baseMipLevel = src_res->mipLevel,
539 .levelCount = 1,
540 .baseArrayLayer = src_array_slice,
541 .layerCount = 1},
542 },
543 0, NULL);
544 meta_emit_blit(cmd_buffer, src_image, &src_iview, src_image_layout, src_offset_0, src_offset_1, dst_image,
545 &dst_iview, dst_image_layout, dst_box, sampler);
546
547 radv_image_view_finish(&dst_iview);
548 radv_image_view_finish(&src_iview);
549 }
550
551 radv_meta_restore(&saved_state, cmd_buffer);
552
553 radv_DestroySampler(radv_device_to_handle(device), sampler, &cmd_buffer->vk.pool->alloc);
554 }
555
556 VKAPI_ATTR void VKAPI_CALL
radv_CmdBlitImage2(VkCommandBuffer commandBuffer,const VkBlitImageInfo2 * pBlitImageInfo)557 radv_CmdBlitImage2(VkCommandBuffer commandBuffer, const VkBlitImageInfo2 *pBlitImageInfo)
558 {
559 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
560 RADV_FROM_HANDLE(radv_image, src_image, pBlitImageInfo->srcImage);
561 RADV_FROM_HANDLE(radv_image, dst_image, pBlitImageInfo->dstImage);
562
563 for (unsigned r = 0; r < pBlitImageInfo->regionCount; r++) {
564 blit_image(cmd_buffer, src_image, pBlitImageInfo->srcImageLayout, dst_image, pBlitImageInfo->dstImageLayout,
565 &pBlitImageInfo->pRegions[r], pBlitImageInfo->filter);
566 }
567 }
568
569 void
radv_device_finish_meta_blit_state(struct radv_device * device)570 radv_device_finish_meta_blit_state(struct radv_device *device)
571 {
572 struct radv_meta_state *state = &device->meta_state;
573
574 for (unsigned i = 0; i < NUM_META_FS_KEYS; ++i) {
575 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.pipeline_1d_src[i], &state->alloc);
576 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.pipeline_2d_src[i], &state->alloc);
577 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.pipeline_3d_src[i], &state->alloc);
578 }
579
580 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.depth_only_1d_pipeline, &state->alloc);
581 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.depth_only_2d_pipeline, &state->alloc);
582 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.depth_only_3d_pipeline, &state->alloc);
583
584 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.stencil_only_1d_pipeline, &state->alloc);
585 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.stencil_only_2d_pipeline, &state->alloc);
586 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.stencil_only_3d_pipeline, &state->alloc);
587
588 radv_DestroyPipelineLayout(radv_device_to_handle(device), state->blit.pipeline_layout, &state->alloc);
589 device->vk.dispatch_table.DestroyDescriptorSetLayout(radv_device_to_handle(device), state->blit.ds_layout,
590 &state->alloc);
591 }
592
593 static VkResult
build_pipeline(struct radv_device * device,VkImageAspectFlagBits aspect,enum glsl_sampler_dim tex_dim,VkFormat format,VkPipeline * pipeline)594 build_pipeline(struct radv_device *device, VkImageAspectFlagBits aspect, enum glsl_sampler_dim tex_dim, VkFormat format,
595 VkPipeline *pipeline)
596 {
597 VkResult result = VK_SUCCESS;
598
599 mtx_lock(&device->meta_state.mtx);
600
601 if (*pipeline) {
602 mtx_unlock(&device->meta_state.mtx);
603 return VK_SUCCESS;
604 }
605
606 nir_shader *fs;
607 nir_shader *vs = build_nir_vertex_shader(device);
608
609 VkPipelineRenderingCreateInfo rendering_create_info = {
610 .sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO,
611 };
612
613 switch (aspect) {
614 case VK_IMAGE_ASPECT_COLOR_BIT:
615 fs = build_nir_copy_fragment_shader(device, tex_dim);
616 rendering_create_info.colorAttachmentCount = 1;
617 rendering_create_info.pColorAttachmentFormats = &format;
618 break;
619 case VK_IMAGE_ASPECT_DEPTH_BIT:
620 fs = build_nir_copy_fragment_shader_depth(device, tex_dim);
621 rendering_create_info.depthAttachmentFormat = format;
622 break;
623 case VK_IMAGE_ASPECT_STENCIL_BIT:
624 fs = build_nir_copy_fragment_shader_stencil(device, tex_dim);
625 rendering_create_info.stencilAttachmentFormat = format;
626 break;
627 default:
628 unreachable("Unhandled aspect");
629 }
630 VkPipelineVertexInputStateCreateInfo vi_create_info = {
631 .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
632 .vertexBindingDescriptionCount = 0,
633 .vertexAttributeDescriptionCount = 0,
634 };
635
636 VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
637 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
638 .stage = VK_SHADER_STAGE_VERTEX_BIT,
639 .module = vk_shader_module_handle_from_nir(vs),
640 .pName = "main",
641 .pSpecializationInfo = NULL},
642 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
643 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
644 .module = vk_shader_module_handle_from_nir(fs),
645 .pName = "main",
646 .pSpecializationInfo = NULL},
647 };
648
649 VkGraphicsPipelineCreateInfo vk_pipeline_info = {
650 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
651 .pNext = &rendering_create_info,
652 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
653 .pStages = pipeline_shader_stages,
654 .pVertexInputState = &vi_create_info,
655 .pInputAssemblyState =
656 &(VkPipelineInputAssemblyStateCreateInfo){
657 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
658 .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
659 .primitiveRestartEnable = false,
660 },
661 .pViewportState =
662 &(VkPipelineViewportStateCreateInfo){
663 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
664 .viewportCount = 1,
665 .scissorCount = 1,
666 },
667 .pRasterizationState =
668 &(VkPipelineRasterizationStateCreateInfo){.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
669 .rasterizerDiscardEnable = false,
670 .polygonMode = VK_POLYGON_MODE_FILL,
671 .cullMode = VK_CULL_MODE_NONE,
672 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE,
673 .lineWidth = 1.0f},
674 .pMultisampleState =
675 &(VkPipelineMultisampleStateCreateInfo){
676 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
677 .rasterizationSamples = 1,
678 .sampleShadingEnable = false,
679 .pSampleMask = (VkSampleMask[]){UINT32_MAX},
680 },
681 .pDynamicState =
682 &(VkPipelineDynamicStateCreateInfo){
683 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
684 .dynamicStateCount = 2,
685 .pDynamicStates =
686 (VkDynamicState[]){
687 VK_DYNAMIC_STATE_VIEWPORT,
688 VK_DYNAMIC_STATE_SCISSOR,
689 },
690 },
691 .flags = 0,
692 .layout = device->meta_state.blit.pipeline_layout,
693 .renderPass = VK_NULL_HANDLE,
694 .subpass = 0,
695 };
696
697 VkPipelineColorBlendStateCreateInfo color_blend_info = {
698 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
699 .attachmentCount = 1,
700 .pAttachments =
701 (VkPipelineColorBlendAttachmentState[]){
702 {.colorWriteMask = VK_COLOR_COMPONENT_A_BIT | VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT |
703 VK_COLOR_COMPONENT_B_BIT},
704 },
705 .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f}};
706
707 VkPipelineDepthStencilStateCreateInfo depth_info = {
708 .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
709 .depthTestEnable = true,
710 .depthWriteEnable = true,
711 .depthCompareOp = VK_COMPARE_OP_ALWAYS,
712 };
713
714 VkPipelineDepthStencilStateCreateInfo stencil_info = {
715 .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
716 .depthTestEnable = false,
717 .depthWriteEnable = false,
718 .stencilTestEnable = true,
719 .front = {.failOp = VK_STENCIL_OP_REPLACE,
720 .passOp = VK_STENCIL_OP_REPLACE,
721 .depthFailOp = VK_STENCIL_OP_REPLACE,
722 .compareOp = VK_COMPARE_OP_ALWAYS,
723 .compareMask = 0xff,
724 .writeMask = 0xff,
725 .reference = 0},
726 .back = {.failOp = VK_STENCIL_OP_REPLACE,
727 .passOp = VK_STENCIL_OP_REPLACE,
728 .depthFailOp = VK_STENCIL_OP_REPLACE,
729 .compareOp = VK_COMPARE_OP_ALWAYS,
730 .compareMask = 0xff,
731 .writeMask = 0xff,
732 .reference = 0},
733 .depthCompareOp = VK_COMPARE_OP_ALWAYS,
734 };
735
736 switch (aspect) {
737 case VK_IMAGE_ASPECT_COLOR_BIT:
738 vk_pipeline_info.pColorBlendState = &color_blend_info;
739 break;
740 case VK_IMAGE_ASPECT_DEPTH_BIT:
741 vk_pipeline_info.pDepthStencilState = &depth_info;
742 break;
743 case VK_IMAGE_ASPECT_STENCIL_BIT:
744 vk_pipeline_info.pDepthStencilState = &stencil_info;
745 break;
746 default:
747 unreachable("Unhandled aspect");
748 }
749
750 const struct radv_graphics_pipeline_create_info radv_pipeline_info = {.use_rectlist = true};
751
752 result = radv_graphics_pipeline_create(radv_device_to_handle(device), device->meta_state.cache, &vk_pipeline_info,
753 &radv_pipeline_info, &device->meta_state.alloc, pipeline);
754 ralloc_free(vs);
755 ralloc_free(fs);
756 mtx_unlock(&device->meta_state.mtx);
757 return result;
758 }
759
760 static VkResult
radv_device_init_meta_blit_color(struct radv_device * device,bool on_demand)761 radv_device_init_meta_blit_color(struct radv_device *device, bool on_demand)
762 {
763 VkResult result;
764
765 for (unsigned i = 0; i < NUM_META_FS_KEYS; ++i) {
766 VkFormat format = radv_fs_key_format_exemplars[i];
767 unsigned key = radv_format_meta_fs_key(device, format);
768
769 if (on_demand)
770 continue;
771
772 result = build_pipeline(device, VK_IMAGE_ASPECT_COLOR_BIT, GLSL_SAMPLER_DIM_1D, format,
773 &device->meta_state.blit.pipeline_1d_src[key]);
774 if (result != VK_SUCCESS)
775 goto fail;
776
777 result = build_pipeline(device, VK_IMAGE_ASPECT_COLOR_BIT, GLSL_SAMPLER_DIM_2D, format,
778 &device->meta_state.blit.pipeline_2d_src[key]);
779 if (result != VK_SUCCESS)
780 goto fail;
781
782 result = build_pipeline(device, VK_IMAGE_ASPECT_COLOR_BIT, GLSL_SAMPLER_DIM_3D, format,
783 &device->meta_state.blit.pipeline_3d_src[key]);
784 if (result != VK_SUCCESS)
785 goto fail;
786 }
787
788 result = VK_SUCCESS;
789 fail:
790 return result;
791 }
792
793 static VkResult
radv_device_init_meta_blit_depth(struct radv_device * device,bool on_demand)794 radv_device_init_meta_blit_depth(struct radv_device *device, bool on_demand)
795 {
796 VkResult result;
797
798 if (on_demand)
799 return VK_SUCCESS;
800
801 result = build_pipeline(device, VK_IMAGE_ASPECT_DEPTH_BIT, GLSL_SAMPLER_DIM_1D, VK_FORMAT_D32_SFLOAT,
802 &device->meta_state.blit.depth_only_1d_pipeline);
803 if (result != VK_SUCCESS)
804 goto fail;
805
806 result = build_pipeline(device, VK_IMAGE_ASPECT_DEPTH_BIT, GLSL_SAMPLER_DIM_2D, VK_FORMAT_D32_SFLOAT,
807 &device->meta_state.blit.depth_only_2d_pipeline);
808 if (result != VK_SUCCESS)
809 goto fail;
810
811 result = build_pipeline(device, VK_IMAGE_ASPECT_DEPTH_BIT, GLSL_SAMPLER_DIM_3D, VK_FORMAT_D32_SFLOAT,
812 &device->meta_state.blit.depth_only_3d_pipeline);
813 if (result != VK_SUCCESS)
814 goto fail;
815
816 fail:
817 return result;
818 }
819
820 static VkResult
radv_device_init_meta_blit_stencil(struct radv_device * device,bool on_demand)821 radv_device_init_meta_blit_stencil(struct radv_device *device, bool on_demand)
822 {
823 VkResult result;
824
825 if (on_demand)
826 return VK_SUCCESS;
827
828 result = build_pipeline(device, VK_IMAGE_ASPECT_STENCIL_BIT, GLSL_SAMPLER_DIM_1D, VK_FORMAT_S8_UINT,
829 &device->meta_state.blit.stencil_only_1d_pipeline);
830 if (result != VK_SUCCESS)
831 goto fail;
832
833 result = build_pipeline(device, VK_IMAGE_ASPECT_STENCIL_BIT, GLSL_SAMPLER_DIM_2D, VK_FORMAT_S8_UINT,
834 &device->meta_state.blit.stencil_only_2d_pipeline);
835 if (result != VK_SUCCESS)
836 goto fail;
837
838 result = build_pipeline(device, VK_IMAGE_ASPECT_STENCIL_BIT, GLSL_SAMPLER_DIM_3D, VK_FORMAT_S8_UINT,
839 &device->meta_state.blit.stencil_only_3d_pipeline);
840 if (result != VK_SUCCESS)
841 goto fail;
842
843 fail:
844 return result;
845 }
846
847 VkResult
radv_device_init_meta_blit_state(struct radv_device * device,bool on_demand)848 radv_device_init_meta_blit_state(struct radv_device *device, bool on_demand)
849 {
850 VkResult result;
851
852 VkDescriptorSetLayoutCreateInfo ds_layout_info = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
853 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
854 .bindingCount = 1,
855 .pBindings = (VkDescriptorSetLayoutBinding[]){
856 {.binding = 0,
857 .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
858 .descriptorCount = 1,
859 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
860 .pImmutableSamplers = NULL},
861 }};
862 result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device), &ds_layout_info, &device->meta_state.alloc,
863 &device->meta_state.blit.ds_layout);
864 if (result != VK_SUCCESS)
865 return result;
866
867 const VkPushConstantRange push_constant_range = {VK_SHADER_STAGE_VERTEX_BIT, 0, 20};
868
869 result = radv_CreatePipelineLayout(radv_device_to_handle(device),
870 &(VkPipelineLayoutCreateInfo){
871 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
872 .setLayoutCount = 1,
873 .pSetLayouts = &device->meta_state.blit.ds_layout,
874 .pushConstantRangeCount = 1,
875 .pPushConstantRanges = &push_constant_range,
876 },
877 &device->meta_state.alloc, &device->meta_state.blit.pipeline_layout);
878 if (result != VK_SUCCESS)
879 return result;
880
881 result = radv_device_init_meta_blit_color(device, on_demand);
882 if (result != VK_SUCCESS)
883 return result;
884
885 result = radv_device_init_meta_blit_depth(device, on_demand);
886 if (result != VK_SUCCESS)
887 return result;
888
889 return radv_device_init_meta_blit_stencil(device, on_demand);
890 }
891