1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "nir/nir_builder.h"
25 #include "radv_meta.h"
26
27 struct blit_region {
28 VkOffset3D src_offset;
29 VkExtent3D src_extent;
30 VkOffset3D dest_offset;
31 VkExtent3D dest_extent;
32 };
33
34 static VkResult build_pipeline(struct radv_device *device, VkImageAspectFlagBits aspect,
35 enum glsl_sampler_dim tex_dim, VkFormat format,
36 VkPipeline *pipeline);
37
38 static nir_shader *
build_nir_vertex_shader(struct radv_device * dev)39 build_nir_vertex_shader(struct radv_device *dev)
40 {
41 const struct glsl_type *vec4 = glsl_vec4_type();
42 nir_builder b = radv_meta_init_shader(dev, MESA_SHADER_VERTEX, "meta_blit_vs");
43
44 nir_variable *pos_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "gl_Position");
45 pos_out->data.location = VARYING_SLOT_POS;
46
47 nir_variable *tex_pos_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "v_tex_pos");
48 tex_pos_out->data.location = VARYING_SLOT_VAR0;
49 tex_pos_out->data.interpolation = INTERP_MODE_SMOOTH;
50
51 nir_ssa_def *outvec = nir_gen_rect_vertices(&b, NULL, NULL);
52
53 nir_store_var(&b, pos_out, outvec, 0xf);
54
55 nir_ssa_def *src_box = nir_load_push_constant(&b, 4, 32, nir_imm_int(&b, 0), .range = 16);
56 nir_ssa_def *src0_z =
57 nir_load_push_constant(&b, 1, 32, nir_imm_int(&b, 0), .base = 16, .range = 4);
58
59 nir_ssa_def *vertex_id = nir_load_vertex_id_zero_base(&b);
60
61 /* vertex 0 - src0_x, src0_y, src0_z */
62 /* vertex 1 - src0_x, src1_y, src0_z*/
63 /* vertex 2 - src1_x, src0_y, src0_z */
64 /* so channel 0 is vertex_id != 2 ? src_x : src_x + w
65 channel 1 is vertex id != 1 ? src_y : src_y + w */
66
67 nir_ssa_def *c0cmp = nir_ine_imm(&b, vertex_id, 2);
68 nir_ssa_def *c1cmp = nir_ine_imm(&b, vertex_id, 1);
69
70 nir_ssa_def *comp[4];
71 comp[0] = nir_bcsel(&b, c0cmp, nir_channel(&b, src_box, 0), nir_channel(&b, src_box, 2));
72
73 comp[1] = nir_bcsel(&b, c1cmp, nir_channel(&b, src_box, 1), nir_channel(&b, src_box, 3));
74 comp[2] = src0_z;
75 comp[3] = nir_imm_float(&b, 1.0);
76 nir_ssa_def *out_tex_vec = nir_vec(&b, comp, 4);
77 nir_store_var(&b, tex_pos_out, out_tex_vec, 0xf);
78 return b.shader;
79 }
80
81 static nir_shader *
build_nir_copy_fragment_shader(struct radv_device * dev,enum glsl_sampler_dim tex_dim)82 build_nir_copy_fragment_shader(struct radv_device *dev, enum glsl_sampler_dim tex_dim)
83 {
84 const struct glsl_type *vec4 = glsl_vec4_type();
85 nir_builder b = radv_meta_init_shader(dev, MESA_SHADER_FRAGMENT, "meta_blit_fs.%d", tex_dim);
86
87 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec4, "v_tex_pos");
88 tex_pos_in->data.location = VARYING_SLOT_VAR0;
89
90 /* Swizzle the array index which comes in as Z coordinate into the right
91 * position.
92 */
93 unsigned swz[] = {0, (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 1), 2};
94 nir_ssa_def *const tex_pos =
95 nir_swizzle(&b, nir_load_var(&b, tex_pos_in), swz, (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 3));
96
97 const struct glsl_type *sampler_type =
98 glsl_sampler_type(tex_dim, false, tex_dim != GLSL_SAMPLER_DIM_3D, glsl_get_base_type(vec4));
99 nir_variable *sampler = nir_variable_create(b.shader, nir_var_uniform, sampler_type, "s_tex");
100 sampler->data.descriptor_set = 0;
101 sampler->data.binding = 0;
102
103 nir_ssa_def *tex_deref = &nir_build_deref_var(&b, sampler)->dest.ssa;
104
105 nir_tex_instr *tex = nir_tex_instr_create(b.shader, 3);
106 tex->sampler_dim = tex_dim;
107 tex->op = nir_texop_tex;
108 tex->src[0].src_type = nir_tex_src_coord;
109 tex->src[0].src = nir_src_for_ssa(tex_pos);
110 tex->src[1].src_type = nir_tex_src_texture_deref;
111 tex->src[1].src = nir_src_for_ssa(tex_deref);
112 tex->src[2].src_type = nir_tex_src_sampler_deref;
113 tex->src[2].src = nir_src_for_ssa(tex_deref);
114 tex->dest_type = nir_type_float32; /* TODO */
115 tex->is_array = glsl_sampler_type_is_array(sampler_type);
116 tex->coord_components = tex_pos->num_components;
117
118 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
119 nir_builder_instr_insert(&b, &tex->instr);
120
121 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
122 color_out->data.location = FRAG_RESULT_DATA0;
123 nir_store_var(&b, color_out, &tex->dest.ssa, 0xf);
124
125 return b.shader;
126 }
127
128 static nir_shader *
build_nir_copy_fragment_shader_depth(struct radv_device * dev,enum glsl_sampler_dim tex_dim)129 build_nir_copy_fragment_shader_depth(struct radv_device *dev, enum glsl_sampler_dim tex_dim)
130 {
131 const struct glsl_type *vec4 = glsl_vec4_type();
132 nir_builder b =
133 radv_meta_init_shader(dev, MESA_SHADER_FRAGMENT, "meta_blit_depth_fs.%d", tex_dim);
134
135 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec4, "v_tex_pos");
136 tex_pos_in->data.location = VARYING_SLOT_VAR0;
137
138 /* Swizzle the array index which comes in as Z coordinate into the right
139 * position.
140 */
141 unsigned swz[] = {0, (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 1), 2};
142 nir_ssa_def *const tex_pos =
143 nir_swizzle(&b, nir_load_var(&b, tex_pos_in), swz, (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 3));
144
145 const struct glsl_type *sampler_type =
146 glsl_sampler_type(tex_dim, false, tex_dim != GLSL_SAMPLER_DIM_3D, glsl_get_base_type(vec4));
147 nir_variable *sampler = nir_variable_create(b.shader, nir_var_uniform, sampler_type, "s_tex");
148 sampler->data.descriptor_set = 0;
149 sampler->data.binding = 0;
150
151 nir_ssa_def *tex_deref = &nir_build_deref_var(&b, sampler)->dest.ssa;
152
153 nir_tex_instr *tex = nir_tex_instr_create(b.shader, 3);
154 tex->sampler_dim = tex_dim;
155 tex->op = nir_texop_tex;
156 tex->src[0].src_type = nir_tex_src_coord;
157 tex->src[0].src = nir_src_for_ssa(tex_pos);
158 tex->src[1].src_type = nir_tex_src_texture_deref;
159 tex->src[1].src = nir_src_for_ssa(tex_deref);
160 tex->src[2].src_type = nir_tex_src_sampler_deref;
161 tex->src[2].src = nir_src_for_ssa(tex_deref);
162 tex->dest_type = nir_type_float32; /* TODO */
163 tex->is_array = glsl_sampler_type_is_array(sampler_type);
164 tex->coord_components = tex_pos->num_components;
165
166 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
167 nir_builder_instr_insert(&b, &tex->instr);
168
169 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
170 color_out->data.location = FRAG_RESULT_DEPTH;
171 nir_store_var(&b, color_out, &tex->dest.ssa, 0x1);
172
173 return b.shader;
174 }
175
176 static nir_shader *
build_nir_copy_fragment_shader_stencil(struct radv_device * dev,enum glsl_sampler_dim tex_dim)177 build_nir_copy_fragment_shader_stencil(struct radv_device *dev, enum glsl_sampler_dim tex_dim)
178 {
179 const struct glsl_type *vec4 = glsl_vec4_type();
180 nir_builder b =
181 radv_meta_init_shader(dev, MESA_SHADER_FRAGMENT, "meta_blit_stencil_fs.%d", tex_dim);
182
183 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec4, "v_tex_pos");
184 tex_pos_in->data.location = VARYING_SLOT_VAR0;
185
186 /* Swizzle the array index which comes in as Z coordinate into the right
187 * position.
188 */
189 unsigned swz[] = {0, (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 1), 2};
190 nir_ssa_def *const tex_pos =
191 nir_swizzle(&b, nir_load_var(&b, tex_pos_in), swz, (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 3));
192
193 const struct glsl_type *sampler_type =
194 glsl_sampler_type(tex_dim, false, tex_dim != GLSL_SAMPLER_DIM_3D, glsl_get_base_type(vec4));
195 nir_variable *sampler = nir_variable_create(b.shader, nir_var_uniform, sampler_type, "s_tex");
196 sampler->data.descriptor_set = 0;
197 sampler->data.binding = 0;
198
199 nir_ssa_def *tex_deref = &nir_build_deref_var(&b, sampler)->dest.ssa;
200
201 nir_tex_instr *tex = nir_tex_instr_create(b.shader, 3);
202 tex->sampler_dim = tex_dim;
203 tex->op = nir_texop_tex;
204 tex->src[0].src_type = nir_tex_src_coord;
205 tex->src[0].src = nir_src_for_ssa(tex_pos);
206 tex->src[1].src_type = nir_tex_src_texture_deref;
207 tex->src[1].src = nir_src_for_ssa(tex_deref);
208 tex->src[2].src_type = nir_tex_src_sampler_deref;
209 tex->src[2].src = nir_src_for_ssa(tex_deref);
210 tex->dest_type = nir_type_float32; /* TODO */
211 tex->is_array = glsl_sampler_type_is_array(sampler_type);
212 tex->coord_components = tex_pos->num_components;
213
214 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
215 nir_builder_instr_insert(&b, &tex->instr);
216
217 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
218 color_out->data.location = FRAG_RESULT_STENCIL;
219 nir_store_var(&b, color_out, &tex->dest.ssa, 0x1);
220
221 return b.shader;
222 }
223
224 static enum glsl_sampler_dim
translate_sampler_dim(VkImageType type)225 translate_sampler_dim(VkImageType type)
226 {
227 switch (type) {
228 case VK_IMAGE_TYPE_1D:
229 return GLSL_SAMPLER_DIM_1D;
230 case VK_IMAGE_TYPE_2D:
231 return GLSL_SAMPLER_DIM_2D;
232 case VK_IMAGE_TYPE_3D:
233 return GLSL_SAMPLER_DIM_3D;
234 default:
235 unreachable("Unhandled image type");
236 }
237 }
238
239 static void
meta_emit_blit(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,struct radv_image_view * src_iview,VkImageLayout src_image_layout,float src_offset_0[3],float src_offset_1[3],struct radv_image * dest_image,struct radv_image_view * dest_iview,VkImageLayout dest_image_layout,VkOffset2D dest_offset_0,VkOffset2D dest_offset_1,VkRect2D dest_box,VkSampler sampler)240 meta_emit_blit(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
241 struct radv_image_view *src_iview, VkImageLayout src_image_layout,
242 float src_offset_0[3], float src_offset_1[3], struct radv_image *dest_image,
243 struct radv_image_view *dest_iview, VkImageLayout dest_image_layout,
244 VkOffset2D dest_offset_0, VkOffset2D dest_offset_1, VkRect2D dest_box,
245 VkSampler sampler)
246 {
247 struct radv_device *device = cmd_buffer->device;
248 uint32_t src_width = radv_minify(src_iview->image->info.width, src_iview->vk.base_mip_level);
249 uint32_t src_height = radv_minify(src_iview->image->info.height, src_iview->vk.base_mip_level);
250 uint32_t src_depth = radv_minify(src_iview->image->info.depth, src_iview->vk.base_mip_level);
251 uint32_t dst_width = radv_minify(dest_iview->image->info.width, dest_iview->vk.base_mip_level);
252 uint32_t dst_height = radv_minify(dest_iview->image->info.height, dest_iview->vk.base_mip_level);
253
254 assert(src_image->info.samples == dest_image->info.samples);
255
256 float vertex_push_constants[5] = {
257 src_offset_0[0] / (float)src_width, src_offset_0[1] / (float)src_height,
258 src_offset_1[0] / (float)src_width, src_offset_1[1] / (float)src_height,
259 src_offset_0[2] / (float)src_depth,
260 };
261
262 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
263 device->meta_state.blit.pipeline_layout, VK_SHADER_STAGE_VERTEX_BIT, 0, 20,
264 vertex_push_constants);
265
266 VkPipeline *pipeline = NULL;
267 unsigned fs_key = 0;
268 VkFormat format = VK_FORMAT_UNDEFINED;
269
270 switch (src_iview->vk.aspects) {
271 case VK_IMAGE_ASPECT_COLOR_BIT: {
272 fs_key = radv_format_meta_fs_key(device, dest_image->vk.format);
273 format = radv_fs_key_format_exemplars[fs_key];
274
275 switch (src_image->vk.image_type) {
276 case VK_IMAGE_TYPE_1D:
277 pipeline = &device->meta_state.blit.pipeline_1d_src[fs_key];
278 break;
279 case VK_IMAGE_TYPE_2D:
280 pipeline = &device->meta_state.blit.pipeline_2d_src[fs_key];
281 break;
282 case VK_IMAGE_TYPE_3D:
283 pipeline = &device->meta_state.blit.pipeline_3d_src[fs_key];
284 break;
285 default:
286 unreachable("bad VkImageType");
287 }
288 break;
289 }
290 case VK_IMAGE_ASPECT_DEPTH_BIT: {
291 format = VK_FORMAT_D32_SFLOAT;
292
293 switch (src_image->vk.image_type) {
294 case VK_IMAGE_TYPE_1D:
295 pipeline = &device->meta_state.blit.depth_only_1d_pipeline;
296 break;
297 case VK_IMAGE_TYPE_2D:
298 pipeline = &device->meta_state.blit.depth_only_2d_pipeline;
299 break;
300 case VK_IMAGE_TYPE_3D:
301 pipeline = &device->meta_state.blit.depth_only_3d_pipeline;
302 break;
303 default:
304 unreachable("bad VkImageType");
305 }
306 break;
307 }
308 case VK_IMAGE_ASPECT_STENCIL_BIT: {
309 format = VK_FORMAT_S8_UINT;
310
311 switch (src_image->vk.image_type) {
312 case VK_IMAGE_TYPE_1D:
313 pipeline = &device->meta_state.blit.stencil_only_1d_pipeline;
314 break;
315 case VK_IMAGE_TYPE_2D:
316 pipeline = &device->meta_state.blit.stencil_only_2d_pipeline;
317 break;
318 case VK_IMAGE_TYPE_3D:
319 pipeline = &device->meta_state.blit.stencil_only_3d_pipeline;
320 break;
321 default:
322 unreachable("bad VkImageType");
323 }
324 break;
325 }
326 default:
327 unreachable("bad VkImageType");
328 }
329
330 if (!*pipeline) {
331 VkResult ret = build_pipeline(device, src_iview->vk.aspects,
332 translate_sampler_dim(src_image->vk.image_type),
333 format, pipeline);
334 if (ret != VK_SUCCESS) {
335 cmd_buffer->record_result = ret;
336 return;
337 }
338 }
339
340 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS,
341 *pipeline);
342
343 radv_meta_push_descriptor_set(
344 cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, device->meta_state.blit.pipeline_layout,
345 0, /* set */
346 1, /* descriptorWriteCount */
347 (VkWriteDescriptorSet[]){{.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
348 .dstBinding = 0,
349 .dstArrayElement = 0,
350 .descriptorCount = 1,
351 .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
352 .pImageInfo = (VkDescriptorImageInfo[]){
353 {
354 .sampler = sampler,
355 .imageView = radv_image_view_to_handle(src_iview),
356 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
357 },
358 }}});
359
360 radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
361 &(VkViewport){.x = dest_offset_0.x,
362 .y = dest_offset_0.y,
363 .width = dest_offset_1.x - dest_offset_0.x,
364 .height = dest_offset_1.y - dest_offset_0.y,
365 .minDepth = 0.0f,
366 .maxDepth = 1.0f});
367
368 radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
369 &(VkRect2D){
370 .offset = (VkOffset2D){MIN2(dest_offset_0.x, dest_offset_1.x),
371 MIN2(dest_offset_0.y, dest_offset_1.y)},
372 .extent = (VkExtent2D){abs(dest_offset_1.x - dest_offset_0.x),
373 abs(dest_offset_1.y - dest_offset_0.y)},
374 });
375
376 VkRenderingInfo rendering_info = {
377 .sType = VK_STRUCTURE_TYPE_RENDERING_INFO,
378 .renderArea = {
379 .offset = { 0, 0 },
380 .extent = { dst_width, dst_height },
381 },
382 .layerCount = 1,
383 };
384
385 VkRenderingAttachmentInfo color_att;
386 if (src_iview->image->vk.aspects == VK_IMAGE_ASPECT_COLOR_BIT) {
387 unsigned dst_layout = radv_meta_dst_layout_from_layout(dest_image_layout);
388 VkImageLayout layout = radv_meta_dst_layout_to_layout(dst_layout);
389
390 color_att = (VkRenderingAttachmentInfo) {
391 .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
392 .imageView = radv_image_view_to_handle(dest_iview),
393 .imageLayout = layout,
394 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
395 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
396 };
397 rendering_info.colorAttachmentCount = 1;
398 rendering_info.pColorAttachments = &color_att;
399 }
400
401 VkRenderingAttachmentInfo depth_att;
402 if (src_iview->image->vk.aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
403 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dest_image_layout);
404 VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
405
406 depth_att = (VkRenderingAttachmentInfo) {
407 .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
408 .imageView = radv_image_view_to_handle(dest_iview),
409 .imageLayout = layout,
410 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
411 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
412 };
413 rendering_info.pDepthAttachment = &depth_att;
414 }
415
416 VkRenderingAttachmentInfo stencil_att;
417 if (src_iview->image->vk.aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
418 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dest_image_layout);
419 VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
420
421 stencil_att = (VkRenderingAttachmentInfo) {
422 .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
423 .imageView = radv_image_view_to_handle(dest_iview),
424 .imageLayout = layout,
425 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
426 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
427 };
428 rendering_info.pStencilAttachment = &stencil_att;
429 }
430
431 radv_CmdBeginRendering(radv_cmd_buffer_to_handle(cmd_buffer), &rendering_info);
432
433 radv_CmdDraw(radv_cmd_buffer_to_handle(cmd_buffer), 3, 1, 0, 0);
434
435 radv_CmdEndRendering(radv_cmd_buffer_to_handle(cmd_buffer));
436 }
437
438 static bool
flip_coords(unsigned * src0,unsigned * src1,unsigned * dst0,unsigned * dst1)439 flip_coords(unsigned *src0, unsigned *src1, unsigned *dst0, unsigned *dst1)
440 {
441 bool flip = false;
442 if (*src0 > *src1) {
443 unsigned tmp = *src0;
444 *src0 = *src1;
445 *src1 = tmp;
446 flip = !flip;
447 }
448
449 if (*dst0 > *dst1) {
450 unsigned tmp = *dst0;
451 *dst0 = *dst1;
452 *dst1 = tmp;
453 flip = !flip;
454 }
455 return flip;
456 }
457
458 static void
blit_image(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,VkImageLayout src_image_layout,struct radv_image * dst_image,VkImageLayout dst_image_layout,const VkImageBlit2 * region,VkFilter filter)459 blit_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
460 VkImageLayout src_image_layout, struct radv_image *dst_image,
461 VkImageLayout dst_image_layout, const VkImageBlit2 *region, VkFilter filter)
462 {
463 const VkImageSubresourceLayers *src_res = ®ion->srcSubresource;
464 const VkImageSubresourceLayers *dst_res = ®ion->dstSubresource;
465 struct radv_device *device = cmd_buffer->device;
466 struct radv_meta_saved_state saved_state;
467 VkSampler sampler;
468
469 /* From the Vulkan 1.0 spec:
470 *
471 * vkCmdBlitImage must not be used for multisampled source or
472 * destination images. Use vkCmdResolveImage for this purpose.
473 */
474 assert(src_image->info.samples == 1);
475 assert(dst_image->info.samples == 1);
476
477 radv_CreateSampler(radv_device_to_handle(device),
478 &(VkSamplerCreateInfo){
479 .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO,
480 .magFilter = filter,
481 .minFilter = filter,
482 .addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
483 .addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
484 .addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
485 },
486 &cmd_buffer->pool->vk.alloc, &sampler);
487
488 /* VK_EXT_conditional_rendering says that blit commands should not be
489 * affected by conditional rendering.
490 */
491 radv_meta_save(&saved_state, cmd_buffer,
492 RADV_META_SAVE_GRAPHICS_PIPELINE | RADV_META_SAVE_CONSTANTS |
493 RADV_META_SAVE_DESCRIPTORS | RADV_META_SUSPEND_PREDICATING);
494
495 unsigned dst_start, dst_end;
496 if (dst_image->vk.image_type == VK_IMAGE_TYPE_3D) {
497 assert(dst_res->baseArrayLayer == 0);
498 dst_start = region->dstOffsets[0].z;
499 dst_end = region->dstOffsets[1].z;
500 } else {
501 dst_start = dst_res->baseArrayLayer;
502 dst_end = dst_start + dst_res->layerCount;
503 }
504
505 unsigned src_start, src_end;
506 if (src_image->vk.image_type == VK_IMAGE_TYPE_3D) {
507 assert(src_res->baseArrayLayer == 0);
508 src_start = region->srcOffsets[0].z;
509 src_end = region->srcOffsets[1].z;
510 } else {
511 src_start = src_res->baseArrayLayer;
512 src_end = src_start + src_res->layerCount;
513 }
514
515 bool flip_z = flip_coords(&src_start, &src_end, &dst_start, &dst_end);
516 float src_z_step = (float)(src_end - src_start) / (float)(dst_end - dst_start);
517
518 /* There is no interpolation to the pixel center during
519 * rendering, so add the 0.5 offset ourselves here. */
520 float depth_center_offset = 0;
521 if (src_image->vk.image_type == VK_IMAGE_TYPE_3D)
522 depth_center_offset = 0.5 / (dst_end - dst_start) * (src_end - src_start);
523
524 if (flip_z) {
525 src_start = src_end;
526 src_z_step *= -1;
527 depth_center_offset *= -1;
528 }
529
530 unsigned src_x0 = region->srcOffsets[0].x;
531 unsigned src_x1 = region->srcOffsets[1].x;
532 unsigned dst_x0 = region->dstOffsets[0].x;
533 unsigned dst_x1 = region->dstOffsets[1].x;
534
535 unsigned src_y0 = region->srcOffsets[0].y;
536 unsigned src_y1 = region->srcOffsets[1].y;
537 unsigned dst_y0 = region->dstOffsets[0].y;
538 unsigned dst_y1 = region->dstOffsets[1].y;
539
540 VkRect2D dst_box;
541 dst_box.offset.x = MIN2(dst_x0, dst_x1);
542 dst_box.offset.y = MIN2(dst_y0, dst_y1);
543 dst_box.extent.width = dst_x1 - dst_x0;
544 dst_box.extent.height = dst_y1 - dst_y0;
545
546 const unsigned num_layers = dst_end - dst_start;
547 for (unsigned i = 0; i < num_layers; i++) {
548 struct radv_image_view dst_iview, src_iview;
549
550 const VkOffset2D dst_offset_0 = {
551 .x = dst_x0,
552 .y = dst_y0,
553 };
554 const VkOffset2D dst_offset_1 = {
555 .x = dst_x1,
556 .y = dst_y1,
557 };
558
559 float src_offset_0[3] = {
560 src_x0,
561 src_y0,
562 src_start + i * src_z_step + depth_center_offset,
563 };
564 float src_offset_1[3] = {
565 src_x1,
566 src_y1,
567 src_start + i * src_z_step + depth_center_offset,
568 };
569 const uint32_t dst_array_slice = dst_start + i;
570
571 /* 3D images have just 1 layer */
572 const uint32_t src_array_slice = src_image->vk.image_type == VK_IMAGE_TYPE_3D ? 0 : src_start + i;
573
574 radv_image_view_init(&dst_iview, cmd_buffer->device,
575 &(VkImageViewCreateInfo){
576 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
577 .image = radv_image_to_handle(dst_image),
578 .viewType = radv_meta_get_view_type(dst_image),
579 .format = dst_image->vk.format,
580 .subresourceRange = {.aspectMask = dst_res->aspectMask,
581 .baseMipLevel = dst_res->mipLevel,
582 .levelCount = 1,
583 .baseArrayLayer = dst_array_slice,
584 .layerCount = 1},
585 },
586 0, NULL);
587 radv_image_view_init(&src_iview, cmd_buffer->device,
588 &(VkImageViewCreateInfo){
589 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
590 .image = radv_image_to_handle(src_image),
591 .viewType = radv_meta_get_view_type(src_image),
592 .format = src_image->vk.format,
593 .subresourceRange = {.aspectMask = src_res->aspectMask,
594 .baseMipLevel = src_res->mipLevel,
595 .levelCount = 1,
596 .baseArrayLayer = src_array_slice,
597 .layerCount = 1},
598 },
599 0, NULL);
600 meta_emit_blit(cmd_buffer, src_image, &src_iview, src_image_layout, src_offset_0,
601 src_offset_1, dst_image, &dst_iview, dst_image_layout, dst_offset_0,
602 dst_offset_1, dst_box, sampler);
603
604 radv_image_view_finish(&dst_iview);
605 radv_image_view_finish(&src_iview);
606 }
607
608 radv_meta_restore(&saved_state, cmd_buffer);
609
610 radv_DestroySampler(radv_device_to_handle(device), sampler, &cmd_buffer->pool->vk.alloc);
611 }
612
613 VKAPI_ATTR void VKAPI_CALL
radv_CmdBlitImage2(VkCommandBuffer commandBuffer,const VkBlitImageInfo2 * pBlitImageInfo)614 radv_CmdBlitImage2(VkCommandBuffer commandBuffer, const VkBlitImageInfo2 *pBlitImageInfo)
615 {
616 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
617 RADV_FROM_HANDLE(radv_image, src_image, pBlitImageInfo->srcImage);
618 RADV_FROM_HANDLE(radv_image, dst_image, pBlitImageInfo->dstImage);
619
620 for (unsigned r = 0; r < pBlitImageInfo->regionCount; r++) {
621 blit_image(cmd_buffer, src_image, pBlitImageInfo->srcImageLayout, dst_image,
622 pBlitImageInfo->dstImageLayout, &pBlitImageInfo->pRegions[r],
623 pBlitImageInfo->filter);
624 }
625 }
626
627 void
radv_device_finish_meta_blit_state(struct radv_device * device)628 radv_device_finish_meta_blit_state(struct radv_device *device)
629 {
630 struct radv_meta_state *state = &device->meta_state;
631
632 for (unsigned i = 0; i < NUM_META_FS_KEYS; ++i) {
633 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.pipeline_1d_src[i],
634 &state->alloc);
635 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.pipeline_2d_src[i],
636 &state->alloc);
637 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.pipeline_3d_src[i],
638 &state->alloc);
639 }
640
641 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.depth_only_1d_pipeline,
642 &state->alloc);
643 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.depth_only_2d_pipeline,
644 &state->alloc);
645 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.depth_only_3d_pipeline,
646 &state->alloc);
647
648 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.stencil_only_1d_pipeline,
649 &state->alloc);
650 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.stencil_only_2d_pipeline,
651 &state->alloc);
652 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.stencil_only_3d_pipeline,
653 &state->alloc);
654
655 radv_DestroyPipelineLayout(radv_device_to_handle(device), state->blit.pipeline_layout,
656 &state->alloc);
657 device->vk.dispatch_table.DestroyDescriptorSetLayout(radv_device_to_handle(device),
658 state->blit.ds_layout, &state->alloc);
659 }
660
661 static VkResult
build_pipeline(struct radv_device * device,VkImageAspectFlagBits aspect,enum glsl_sampler_dim tex_dim,VkFormat format,VkPipeline * pipeline)662 build_pipeline(struct radv_device *device, VkImageAspectFlagBits aspect,
663 enum glsl_sampler_dim tex_dim, VkFormat format, VkPipeline *pipeline)
664 {
665 VkResult result = VK_SUCCESS;
666
667 mtx_lock(&device->meta_state.mtx);
668
669 if (*pipeline) {
670 mtx_unlock(&device->meta_state.mtx);
671 return VK_SUCCESS;
672 }
673
674 nir_shader *fs;
675 nir_shader *vs = build_nir_vertex_shader(device);
676
677 VkPipelineRenderingCreateInfo rendering_create_info = {
678 .sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO,
679 };
680
681 switch (aspect) {
682 case VK_IMAGE_ASPECT_COLOR_BIT:
683 fs = build_nir_copy_fragment_shader(device, tex_dim);
684 rendering_create_info.colorAttachmentCount = 1;
685 rendering_create_info.pColorAttachmentFormats = &format;
686 break;
687 case VK_IMAGE_ASPECT_DEPTH_BIT:
688 fs = build_nir_copy_fragment_shader_depth(device, tex_dim);
689 rendering_create_info.depthAttachmentFormat = format;
690 break;
691 case VK_IMAGE_ASPECT_STENCIL_BIT:
692 fs = build_nir_copy_fragment_shader_stencil(device, tex_dim);
693 rendering_create_info.stencilAttachmentFormat = format;
694 break;
695 default:
696 unreachable("Unhandled aspect");
697 }
698 VkPipelineVertexInputStateCreateInfo vi_create_info = {
699 .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
700 .vertexBindingDescriptionCount = 0,
701 .vertexAttributeDescriptionCount = 0,
702 };
703
704 VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
705 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
706 .stage = VK_SHADER_STAGE_VERTEX_BIT,
707 .module = vk_shader_module_handle_from_nir(vs),
708 .pName = "main",
709 .pSpecializationInfo = NULL},
710 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
711 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
712 .module = vk_shader_module_handle_from_nir(fs),
713 .pName = "main",
714 .pSpecializationInfo = NULL},
715 };
716
717 VkGraphicsPipelineCreateInfo vk_pipeline_info = {
718 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
719 .pNext = &rendering_create_info,
720 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
721 .pStages = pipeline_shader_stages,
722 .pVertexInputState = &vi_create_info,
723 .pInputAssemblyState =
724 &(VkPipelineInputAssemblyStateCreateInfo){
725 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
726 .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
727 .primitiveRestartEnable = false,
728 },
729 .pViewportState =
730 &(VkPipelineViewportStateCreateInfo){
731 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
732 .viewportCount = 1,
733 .scissorCount = 1,
734 },
735 .pRasterizationState =
736 &(VkPipelineRasterizationStateCreateInfo){
737 .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
738 .rasterizerDiscardEnable = false,
739 .polygonMode = VK_POLYGON_MODE_FILL,
740 .cullMode = VK_CULL_MODE_NONE,
741 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE,
742 .lineWidth = 1.0f},
743 .pMultisampleState =
744 &(VkPipelineMultisampleStateCreateInfo){
745 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
746 .rasterizationSamples = 1,
747 .sampleShadingEnable = false,
748 .pSampleMask = (VkSampleMask[]){UINT32_MAX},
749 },
750 .pDynamicState =
751 &(VkPipelineDynamicStateCreateInfo){
752 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
753 .dynamicStateCount = 2,
754 .pDynamicStates =
755 (VkDynamicState[]){
756 VK_DYNAMIC_STATE_VIEWPORT,
757 VK_DYNAMIC_STATE_SCISSOR,
758 },
759 },
760 .flags = 0,
761 .layout = device->meta_state.blit.pipeline_layout,
762 .renderPass = VK_NULL_HANDLE,
763 .subpass = 0,
764 };
765
766 VkPipelineColorBlendStateCreateInfo color_blend_info = {
767 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
768 .attachmentCount = 1,
769 .pAttachments = (VkPipelineColorBlendAttachmentState[]){
770 {.colorWriteMask = VK_COLOR_COMPONENT_A_BIT | VK_COLOR_COMPONENT_R_BIT |
771 VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT},
772 },
773 .blendConstants = { 0.0f, 0.0f, 0.0f, 0.0f }};
774
775 VkPipelineDepthStencilStateCreateInfo depth_info = {
776 .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
777 .depthTestEnable = true,
778 .depthWriteEnable = true,
779 .depthCompareOp = VK_COMPARE_OP_ALWAYS,
780 };
781
782 VkPipelineDepthStencilStateCreateInfo stencil_info = {
783 .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
784 .depthTestEnable = false,
785 .depthWriteEnable = false,
786 .stencilTestEnable = true,
787 .front = {.failOp = VK_STENCIL_OP_REPLACE,
788 .passOp = VK_STENCIL_OP_REPLACE,
789 .depthFailOp = VK_STENCIL_OP_REPLACE,
790 .compareOp = VK_COMPARE_OP_ALWAYS,
791 .compareMask = 0xff,
792 .writeMask = 0xff,
793 .reference = 0},
794 .back = {.failOp = VK_STENCIL_OP_REPLACE,
795 .passOp = VK_STENCIL_OP_REPLACE,
796 .depthFailOp = VK_STENCIL_OP_REPLACE,
797 .compareOp = VK_COMPARE_OP_ALWAYS,
798 .compareMask = 0xff,
799 .writeMask = 0xff,
800 .reference = 0},
801 .depthCompareOp = VK_COMPARE_OP_ALWAYS,
802 };
803
804 switch (aspect) {
805 case VK_IMAGE_ASPECT_COLOR_BIT:
806 vk_pipeline_info.pColorBlendState = &color_blend_info;
807 break;
808 case VK_IMAGE_ASPECT_DEPTH_BIT:
809 vk_pipeline_info.pDepthStencilState = &depth_info;
810 break;
811 case VK_IMAGE_ASPECT_STENCIL_BIT:
812 vk_pipeline_info.pDepthStencilState = &stencil_info;
813 break;
814 default:
815 unreachable("Unhandled aspect");
816 }
817
818 const struct radv_graphics_pipeline_create_info radv_pipeline_info = {.use_rectlist = true};
819
820 result = radv_graphics_pipeline_create(
821 radv_device_to_handle(device), radv_pipeline_cache_to_handle(&device->meta_state.cache),
822 &vk_pipeline_info, &radv_pipeline_info, &device->meta_state.alloc, pipeline);
823 ralloc_free(vs);
824 ralloc_free(fs);
825 mtx_unlock(&device->meta_state.mtx);
826 return result;
827 }
828
829 static VkResult
radv_device_init_meta_blit_color(struct radv_device * device,bool on_demand)830 radv_device_init_meta_blit_color(struct radv_device *device, bool on_demand)
831 {
832 VkResult result;
833
834 for (unsigned i = 0; i < NUM_META_FS_KEYS; ++i) {
835 VkFormat format = radv_fs_key_format_exemplars[i];
836 unsigned key = radv_format_meta_fs_key(device, format);
837
838 if (on_demand)
839 continue;
840
841 result = build_pipeline(device, VK_IMAGE_ASPECT_COLOR_BIT, GLSL_SAMPLER_DIM_1D, format,
842 &device->meta_state.blit.pipeline_1d_src[key]);
843 if (result != VK_SUCCESS)
844 goto fail;
845
846 result = build_pipeline(device, VK_IMAGE_ASPECT_COLOR_BIT, GLSL_SAMPLER_DIM_2D, format,
847 &device->meta_state.blit.pipeline_2d_src[key]);
848 if (result != VK_SUCCESS)
849 goto fail;
850
851 result = build_pipeline(device, VK_IMAGE_ASPECT_COLOR_BIT, GLSL_SAMPLER_DIM_3D, format,
852 &device->meta_state.blit.pipeline_3d_src[key]);
853 if (result != VK_SUCCESS)
854 goto fail;
855 }
856
857 result = VK_SUCCESS;
858 fail:
859 return result;
860 }
861
862 static VkResult
radv_device_init_meta_blit_depth(struct radv_device * device,bool on_demand)863 radv_device_init_meta_blit_depth(struct radv_device *device, bool on_demand)
864 {
865 VkResult result;
866
867 if (on_demand)
868 return VK_SUCCESS;
869
870 result = build_pipeline(device, VK_IMAGE_ASPECT_DEPTH_BIT, GLSL_SAMPLER_DIM_1D,
871 VK_FORMAT_D32_SFLOAT,
872 &device->meta_state.blit.depth_only_1d_pipeline);
873 if (result != VK_SUCCESS)
874 goto fail;
875
876 result = build_pipeline(device, VK_IMAGE_ASPECT_DEPTH_BIT, GLSL_SAMPLER_DIM_2D,
877 VK_FORMAT_D32_SFLOAT,
878 &device->meta_state.blit.depth_only_2d_pipeline);
879 if (result != VK_SUCCESS)
880 goto fail;
881
882 result = build_pipeline(device, VK_IMAGE_ASPECT_DEPTH_BIT, GLSL_SAMPLER_DIM_3D,
883 VK_FORMAT_D32_SFLOAT,
884 &device->meta_state.blit.depth_only_3d_pipeline);
885 if (result != VK_SUCCESS)
886 goto fail;
887
888 fail:
889 return result;
890 }
891
892 static VkResult
radv_device_init_meta_blit_stencil(struct radv_device * device,bool on_demand)893 radv_device_init_meta_blit_stencil(struct radv_device *device, bool on_demand)
894 {
895 VkResult result;
896
897 if (on_demand)
898 return VK_SUCCESS;
899
900 result = build_pipeline(device, VK_IMAGE_ASPECT_STENCIL_BIT, GLSL_SAMPLER_DIM_1D,
901 VK_FORMAT_S8_UINT,
902 &device->meta_state.blit.stencil_only_1d_pipeline);
903 if (result != VK_SUCCESS)
904 goto fail;
905
906 result = build_pipeline(device, VK_IMAGE_ASPECT_STENCIL_BIT, GLSL_SAMPLER_DIM_2D,
907 VK_FORMAT_S8_UINT,
908 &device->meta_state.blit.stencil_only_2d_pipeline);
909 if (result != VK_SUCCESS)
910 goto fail;
911
912 result = build_pipeline(device, VK_IMAGE_ASPECT_STENCIL_BIT, GLSL_SAMPLER_DIM_3D,
913 VK_FORMAT_S8_UINT,
914 &device->meta_state.blit.stencil_only_3d_pipeline);
915 if (result != VK_SUCCESS)
916 goto fail;
917
918 fail:
919 return result;
920 }
921
922 VkResult
radv_device_init_meta_blit_state(struct radv_device * device,bool on_demand)923 radv_device_init_meta_blit_state(struct radv_device *device, bool on_demand)
924 {
925 VkResult result;
926
927 VkDescriptorSetLayoutCreateInfo ds_layout_info = {
928 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
929 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
930 .bindingCount = 1,
931 .pBindings = (VkDescriptorSetLayoutBinding[]){
932 {.binding = 0,
933 .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
934 .descriptorCount = 1,
935 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
936 .pImmutableSamplers = NULL},
937 }};
938 result =
939 radv_CreateDescriptorSetLayout(radv_device_to_handle(device), &ds_layout_info,
940 &device->meta_state.alloc, &device->meta_state.blit.ds_layout);
941 if (result != VK_SUCCESS)
942 return result;
943
944 const VkPushConstantRange push_constant_range = {VK_SHADER_STAGE_VERTEX_BIT, 0, 20};
945
946 result = radv_CreatePipelineLayout(radv_device_to_handle(device),
947 &(VkPipelineLayoutCreateInfo){
948 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
949 .setLayoutCount = 1,
950 .pSetLayouts = &device->meta_state.blit.ds_layout,
951 .pushConstantRangeCount = 1,
952 .pPushConstantRanges = &push_constant_range,
953 },
954 &device->meta_state.alloc,
955 &device->meta_state.blit.pipeline_layout);
956 if (result != VK_SUCCESS)
957 return result;
958
959 result = radv_device_init_meta_blit_color(device, on_demand);
960 if (result != VK_SUCCESS)
961 return result;
962
963 result = radv_device_init_meta_blit_depth(device, on_demand);
964 if (result != VK_SUCCESS)
965 return result;
966
967 return radv_device_init_meta_blit_stencil(device, on_demand);
968 }
969