1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * SPDX-License-Identifier: MIT
5 */
6
7 #include <assert.h>
8 #include <stdbool.h>
9
10 #include "nir/nir_builder.h"
11 #include "radv_entrypoints.h"
12 #include "radv_meta.h"
13 #include "sid.h"
14 #include "vk_format.h"
15
16 static nir_shader *
build_nir_fs(struct radv_device * dev)17 build_nir_fs(struct radv_device *dev)
18 {
19 const struct glsl_type *vec4 = glsl_vec4_type();
20 nir_variable *f_color;
21
22 nir_builder b = radv_meta_init_shader(dev, MESA_SHADER_FRAGMENT, "meta_resolve_fs");
23
24 f_color = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
25 f_color->data.location = FRAG_RESULT_DATA0;
26 nir_store_var(&b, f_color, nir_imm_vec4(&b, 0.0, 0.0, 0.0, 1.0), 0xf);
27
28 return b.shader;
29 }
30
31 static VkResult
get_pipeline(struct radv_device * device,unsigned fs_key,VkPipeline * pipeline_out,VkPipelineLayout * layout_out)32 get_pipeline(struct radv_device *device, unsigned fs_key, VkPipeline *pipeline_out, VkPipelineLayout *layout_out)
33 {
34 const VkFormat format = radv_fs_key_format_exemplars[fs_key];
35 char key_data[64];
36 VkResult result;
37
38 snprintf(key_data, sizeof(key_data), "radv-resolve-hw-%d", fs_key);
39
40 result = radv_meta_get_noop_pipeline_layout(device, layout_out);
41 if (result != VK_SUCCESS)
42 return result;
43
44 nir_shader *vs_module = radv_meta_build_nir_vs_generate_vertices(device);
45 nir_shader *fs_module = build_nir_fs(device);
46
47 const VkGraphicsPipelineCreateInfoRADV radv_info = {
48 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO_RADV,
49 .custom_blend_mode = V_028808_CB_RESOLVE,
50 };
51
52 const VkGraphicsPipelineCreateInfo pipeline_create_info = {
53 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
54 .pNext = &radv_info,
55 .stageCount = 2,
56 .pStages =
57 (VkPipelineShaderStageCreateInfo[]){
58 {
59 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
60 .stage = VK_SHADER_STAGE_VERTEX_BIT,
61 .module = vk_shader_module_handle_from_nir(vs_module),
62 .pName = "main",
63 },
64 {
65 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
66 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
67 .module = vk_shader_module_handle_from_nir(fs_module),
68 .pName = "main",
69 },
70 },
71 .pVertexInputState =
72 &(VkPipelineVertexInputStateCreateInfo){
73 .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
74 .vertexBindingDescriptionCount = 0,
75 .vertexAttributeDescriptionCount = 0,
76 },
77 .pInputAssemblyState =
78 &(VkPipelineInputAssemblyStateCreateInfo){
79 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
80 .topology = VK_PRIMITIVE_TOPOLOGY_META_RECT_LIST_MESA,
81 .primitiveRestartEnable = false,
82 },
83 .pViewportState =
84 &(VkPipelineViewportStateCreateInfo){
85 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
86 .viewportCount = 1,
87 .scissorCount = 1,
88 },
89 .pRasterizationState =
90 &(VkPipelineRasterizationStateCreateInfo){
91 .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
92 .depthClampEnable = false,
93 .rasterizerDiscardEnable = false,
94 .polygonMode = VK_POLYGON_MODE_FILL,
95 .cullMode = VK_CULL_MODE_NONE,
96 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE,
97 },
98 .pMultisampleState =
99 &(VkPipelineMultisampleStateCreateInfo){
100 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
101 .rasterizationSamples = 1,
102 .sampleShadingEnable = false,
103 .pSampleMask = NULL,
104 .alphaToCoverageEnable = false,
105 .alphaToOneEnable = false,
106 },
107 .pColorBlendState =
108 &(VkPipelineColorBlendStateCreateInfo){
109 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
110 .logicOpEnable = false,
111 .attachmentCount = 2,
112 .pAttachments =
113 (VkPipelineColorBlendAttachmentState[]){
114 {
115 .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT |
116 VK_COLOR_COMPONENT_A_BIT,
117 },
118 {
119 .colorWriteMask = 0,
120
121 }},
122 },
123 .pDynamicState =
124 &(VkPipelineDynamicStateCreateInfo){
125 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
126 .dynamicStateCount = 2,
127 .pDynamicStates =
128 (VkDynamicState[]){
129 VK_DYNAMIC_STATE_VIEWPORT,
130 VK_DYNAMIC_STATE_SCISSOR,
131 },
132 },
133 .layout = *layout_out,
134 };
135
136 struct vk_meta_rendering_info render = {
137 .color_attachment_count = 2,
138 .color_attachment_formats = {format, format},
139 };
140
141 result = vk_meta_create_graphics_pipeline(&device->vk, &device->meta_state.device, &pipeline_create_info, &render,
142 key_data, strlen(key_data), pipeline_out);
143
144 ralloc_free(vs_module);
145 ralloc_free(fs_module);
146 return result;
147 }
148
149 static void
emit_resolve(struct radv_cmd_buffer * cmd_buffer,const struct radv_image * src_image,const struct radv_image * dst_image,VkFormat vk_format)150 emit_resolve(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *src_image, const struct radv_image *dst_image,
151 VkFormat vk_format)
152 {
153 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
154 VkCommandBuffer cmd_buffer_h = radv_cmd_buffer_to_handle(cmd_buffer);
155 unsigned fs_key = radv_format_meta_fs_key(device, vk_format);
156 VkPipelineLayout layout;
157 VkPipeline pipeline;
158 VkResult result;
159
160 result = get_pipeline(device, fs_key, &pipeline, &layout);
161 if (result != VK_SUCCESS) {
162 vk_command_buffer_set_error(&cmd_buffer->vk, result);
163 return;
164 }
165
166 cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
167 VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, src_image, NULL) |
168 radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
169 VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT, src_image, NULL);
170
171 radv_CmdBindPipeline(cmd_buffer_h, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
172
173 radv_CmdDraw(cmd_buffer_h, 3, 1, 0, 0);
174 cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
175 VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, dst_image, NULL);
176 }
177
178 enum radv_resolve_method {
179 RESOLVE_HW,
180 RESOLVE_COMPUTE,
181 RESOLVE_FRAGMENT,
182 };
183
184 static bool
image_hw_resolve_compat(const struct radv_device * device,struct radv_image * src_image,struct radv_image * dst_image)185 image_hw_resolve_compat(const struct radv_device *device, struct radv_image *src_image, struct radv_image *dst_image)
186 {
187 const struct radv_physical_device *pdev = radv_device_physical(device);
188 if (pdev->info.gfx_level >= GFX9) {
189 return dst_image->planes[0].surface.u.gfx9.swizzle_mode == src_image->planes[0].surface.u.gfx9.swizzle_mode;
190 } else {
191 return dst_image->planes[0].surface.micro_tile_mode == src_image->planes[0].surface.micro_tile_mode;
192 }
193 }
194
195 static void
radv_pick_resolve_method_images(struct radv_device * device,struct radv_image * src_image,VkFormat src_format,struct radv_image * dst_image,unsigned dst_level,VkImageLayout dst_image_layout,struct radv_cmd_buffer * cmd_buffer,enum radv_resolve_method * method)196 radv_pick_resolve_method_images(struct radv_device *device, struct radv_image *src_image, VkFormat src_format,
197 struct radv_image *dst_image, unsigned dst_level, VkImageLayout dst_image_layout,
198 struct radv_cmd_buffer *cmd_buffer, enum radv_resolve_method *method)
199
200 {
201 uint32_t queue_mask = radv_image_queue_family_mask(dst_image, cmd_buffer->qf, cmd_buffer->qf);
202
203 if (vk_format_is_color(src_format)) {
204 /* Using the fragment resolve path is currently a hint to
205 * avoid decompressing DCC for partial resolves and
206 * re-initialize it after resolving using compute.
207 * TODO: Add support for layered and int to the fragment path.
208 */
209 if (radv_layout_dcc_compressed(device, dst_image, dst_level, dst_image_layout, queue_mask)) {
210 *method = RESOLVE_FRAGMENT;
211 } else if (!image_hw_resolve_compat(device, src_image, dst_image)) {
212 /* The micro tile mode only needs to match for the HW
213 * resolve path which is the default path for non-DCC
214 * resolves.
215 */
216 *method = RESOLVE_COMPUTE;
217 }
218
219 if (src_format == VK_FORMAT_R16G16_UNORM || src_format == VK_FORMAT_R16G16_SNORM)
220 *method = RESOLVE_COMPUTE;
221 else if (vk_format_is_int(src_format))
222 *method = RESOLVE_COMPUTE;
223 else if (src_image->vk.array_layers > 1 || dst_image->vk.array_layers > 1)
224 *method = RESOLVE_COMPUTE;
225 } else {
226 assert(dst_image_layout == VK_IMAGE_LAYOUT_UNDEFINED);
227 if (src_image->vk.array_layers > 1 || dst_image->vk.array_layers > 1 ||
228 (dst_image->planes[0].surface.flags & RADEON_SURF_NO_RENDER_TARGET))
229 *method = RESOLVE_COMPUTE;
230 else
231 *method = RESOLVE_FRAGMENT;
232 }
233 }
234
235 static void
radv_meta_resolve_hardware_image(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,VkImageLayout src_image_layout,struct radv_image * dst_image,VkImageLayout dst_image_layout,const VkImageResolve2 * region)236 radv_meta_resolve_hardware_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
237 VkImageLayout src_image_layout, struct radv_image *dst_image,
238 VkImageLayout dst_image_layout, const VkImageResolve2 *region)
239 {
240 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
241 struct radv_meta_saved_state saved_state;
242
243 radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_GRAPHICS_PIPELINE);
244
245 assert(src_image->vk.samples > 1);
246 assert(dst_image->vk.samples == 1);
247
248 /* From the Vulkan 1.0 spec:
249 *
250 * - The aspectMask member of srcSubresource and dstSubresource must
251 * only contain VK_IMAGE_ASPECT_COLOR_BIT
252 */
253 assert(region->srcSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
254 assert(region->dstSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
255 /* Multi-layer resolves are handled by compute */
256 assert(vk_image_subresource_layer_count(&src_image->vk, ®ion->srcSubresource) == 1 &&
257 vk_image_subresource_layer_count(&dst_image->vk, ®ion->dstSubresource) == 1);
258 /**
259 * From Vulkan 1.0.6 spec: 18.6 Resolving Multisample Images
260 *
261 * extent is the size in texels of the source image to resolve in width,
262 * height and depth. 1D images use only x and width. 2D images use x, y,
263 * width and height. 3D images use x, y, z, width, height and depth.
264 *
265 * srcOffset and dstOffset select the initial x, y, and z offsets in
266 * texels of the sub-regions of the source and destination image data.
267 * extent is the size in texels of the source image to resolve in width,
268 * height and depth. 1D images use only x and width. 2D images use x, y,
269 * width and height. 3D images use x, y, z, width, height and depth.
270 */
271 const struct VkExtent3D extent = vk_image_sanitize_extent(&src_image->vk, region->extent);
272 const struct VkOffset3D dstOffset = vk_image_sanitize_offset(&dst_image->vk, region->dstOffset);
273
274 uint32_t queue_mask = radv_image_queue_family_mask(dst_image, cmd_buffer->qf, cmd_buffer->qf);
275
276 if (radv_layout_dcc_compressed(device, dst_image, region->dstSubresource.mipLevel, dst_image_layout, queue_mask)) {
277 VkImageSubresourceRange range = {
278 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
279 .baseMipLevel = region->dstSubresource.mipLevel,
280 .levelCount = 1,
281 .baseArrayLayer = 0,
282 .layerCount = 1,
283 };
284
285 cmd_buffer->state.flush_bits |= radv_init_dcc(cmd_buffer, dst_image, &range, 0xffffffff);
286 }
287
288 VkRect2D resolve_area = {
289 .offset = {dstOffset.x, dstOffset.y},
290 .extent = {extent.width, extent.height},
291 };
292
293 radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
294 &(VkViewport){.x = resolve_area.offset.x,
295 .y = resolve_area.offset.y,
296 .width = resolve_area.extent.width,
297 .height = resolve_area.extent.height,
298 .minDepth = 0.0f,
299 .maxDepth = 1.0f});
300
301 radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &resolve_area);
302
303 struct radv_image_view src_iview;
304 radv_image_view_init(&src_iview, device,
305 &(VkImageViewCreateInfo){
306 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
307 .image = radv_image_to_handle(src_image),
308 .viewType = VK_IMAGE_VIEW_TYPE_2D,
309 .format = src_image->vk.format,
310 .subresourceRange =
311 {
312 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
313 .baseMipLevel = 0,
314 .levelCount = 1,
315 .baseArrayLayer = 0,
316 .layerCount = 1,
317 },
318 },
319 NULL);
320
321 struct radv_image_view dst_iview;
322 radv_image_view_init(&dst_iview, device,
323 &(VkImageViewCreateInfo){
324 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
325 .image = radv_image_to_handle(dst_image),
326 .viewType = radv_meta_get_view_type(dst_image),
327 .format = dst_image->vk.format,
328 .subresourceRange =
329 {
330 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
331 .baseMipLevel = region->dstSubresource.mipLevel,
332 .levelCount = 1,
333 .baseArrayLayer = 0,
334 .layerCount = 1,
335 },
336 },
337 NULL);
338
339 const VkRenderingAttachmentInfo color_atts[2] = {
340 {
341 .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
342 .imageView = radv_image_view_to_handle(&src_iview),
343 .imageLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
344 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
345 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
346 },
347 {
348 .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
349 .imageView = radv_image_view_to_handle(&dst_iview),
350 .imageLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
351 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
352 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
353 },
354 };
355
356 const VkRenderingInfo rendering_info = {
357 .sType = VK_STRUCTURE_TYPE_RENDERING_INFO,
358 .flags = VK_RENDERING_INPUT_ATTACHMENT_NO_CONCURRENT_WRITES_BIT_MESA,
359 .renderArea = resolve_area,
360 .layerCount = 1,
361 .colorAttachmentCount = 2,
362 .pColorAttachments = color_atts,
363 };
364
365 radv_CmdBeginRendering(radv_cmd_buffer_to_handle(cmd_buffer), &rendering_info);
366
367 emit_resolve(cmd_buffer, src_image, dst_image, dst_iview.vk.format);
368
369 radv_CmdEndRendering(radv_cmd_buffer_to_handle(cmd_buffer));
370
371 radv_image_view_finish(&src_iview);
372 radv_image_view_finish(&dst_iview);
373
374 radv_meta_restore(&saved_state, cmd_buffer);
375 }
376
377 static void
resolve_image(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,VkImageLayout src_image_layout,struct radv_image * dst_image,VkImageLayout dst_image_layout,const VkImageResolve2 * region,enum radv_resolve_method resolve_method)378 resolve_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, VkImageLayout src_image_layout,
379 struct radv_image *dst_image, VkImageLayout dst_image_layout, const VkImageResolve2 *region,
380 enum radv_resolve_method resolve_method)
381 {
382 switch (resolve_method) {
383 case RESOLVE_HW:
384 radv_meta_resolve_hardware_image(cmd_buffer, src_image, src_image_layout, dst_image, dst_image_layout, region);
385 break;
386 case RESOLVE_FRAGMENT:
387 radv_decompress_resolve_src(cmd_buffer, src_image, src_image_layout, region);
388
389 radv_meta_resolve_fragment_image(cmd_buffer, src_image, src_image_layout, dst_image, dst_image_layout, region);
390 break;
391 case RESOLVE_COMPUTE:
392 radv_decompress_resolve_src(cmd_buffer, src_image, src_image_layout, region);
393
394 radv_meta_resolve_compute_image(cmd_buffer, src_image, src_image->vk.format, src_image_layout, dst_image,
395 dst_image->vk.format, dst_image_layout, region);
396 break;
397 default:
398 assert(!"Invalid resolve method selected");
399 }
400 }
401
402 VKAPI_ATTR void VKAPI_CALL
radv_CmdResolveImage2(VkCommandBuffer commandBuffer,const VkResolveImageInfo2 * pResolveImageInfo)403 radv_CmdResolveImage2(VkCommandBuffer commandBuffer, const VkResolveImageInfo2 *pResolveImageInfo)
404 {
405 VK_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
406 VK_FROM_HANDLE(radv_image, src_image, pResolveImageInfo->srcImage);
407 VK_FROM_HANDLE(radv_image, dst_image, pResolveImageInfo->dstImage);
408 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
409 const struct radv_physical_device *pdev = radv_device_physical(device);
410 VkImageLayout src_image_layout = pResolveImageInfo->srcImageLayout;
411 VkImageLayout dst_image_layout = pResolveImageInfo->dstImageLayout;
412 enum radv_resolve_method resolve_method = pdev->info.gfx_level >= GFX11 ? RESOLVE_FRAGMENT : RESOLVE_HW;
413
414 /* we can use the hw resolve only for single full resolves */
415 if (pResolveImageInfo->regionCount == 1) {
416 if (pResolveImageInfo->pRegions[0].srcOffset.x || pResolveImageInfo->pRegions[0].srcOffset.y ||
417 pResolveImageInfo->pRegions[0].srcOffset.z)
418 resolve_method = RESOLVE_COMPUTE;
419 if (pResolveImageInfo->pRegions[0].dstOffset.x || pResolveImageInfo->pRegions[0].dstOffset.y ||
420 pResolveImageInfo->pRegions[0].dstOffset.z)
421 resolve_method = RESOLVE_COMPUTE;
422
423 if (pResolveImageInfo->pRegions[0].extent.width != src_image->vk.extent.width ||
424 pResolveImageInfo->pRegions[0].extent.height != src_image->vk.extent.height ||
425 pResolveImageInfo->pRegions[0].extent.depth != src_image->vk.extent.depth)
426 resolve_method = RESOLVE_COMPUTE;
427 } else
428 resolve_method = RESOLVE_COMPUTE;
429
430 for (uint32_t r = 0; r < pResolveImageInfo->regionCount; r++) {
431 const VkImageResolve2 *region = &pResolveImageInfo->pRegions[r];
432
433 radv_pick_resolve_method_images(device, src_image, src_image->vk.format, dst_image,
434 region->dstSubresource.mipLevel, dst_image_layout, cmd_buffer, &resolve_method);
435
436 resolve_image(cmd_buffer, src_image, src_image_layout, dst_image, dst_image_layout, region, resolve_method);
437 }
438 }
439
440 static void
radv_cmd_buffer_resolve_rendering_hw(struct radv_cmd_buffer * cmd_buffer,struct radv_image_view * src_iview,VkImageLayout src_layout,struct radv_image_view * dst_iview,VkImageLayout dst_layout)441 radv_cmd_buffer_resolve_rendering_hw(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview,
442 VkImageLayout src_layout, struct radv_image_view *dst_iview,
443 VkImageLayout dst_layout)
444 {
445 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
446 struct radv_meta_saved_state saved_state;
447
448 radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_GRAPHICS_PIPELINE | RADV_META_SAVE_RENDER);
449
450 VkRect2D *resolve_area = &saved_state.render.area;
451
452 radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
453 &(VkViewport){.x = resolve_area->offset.x,
454 .y = resolve_area->offset.y,
455 .width = resolve_area->extent.width,
456 .height = resolve_area->extent.height,
457 .minDepth = 0.0f,
458 .maxDepth = 1.0f});
459
460 radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, resolve_area);
461
462 struct radv_image *src_img = src_iview->image;
463 struct radv_image *dst_img = dst_iview->image;
464 uint32_t queue_mask = radv_image_queue_family_mask(dst_img, cmd_buffer->qf, cmd_buffer->qf);
465
466 if (radv_layout_dcc_compressed(device, dst_img, dst_iview->vk.base_mip_level, dst_layout, queue_mask)) {
467 VkImageSubresourceRange range = {
468 .aspectMask = dst_iview->vk.aspects,
469 .baseMipLevel = dst_iview->vk.base_mip_level,
470 .levelCount = 1,
471 .baseArrayLayer = 0,
472 .layerCount = 1,
473 };
474
475 cmd_buffer->state.flush_bits |= radv_init_dcc(cmd_buffer, dst_img, &range, 0xffffffff);
476 }
477
478 const VkRenderingAttachmentInfo color_atts[2] = {
479 {
480 .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
481 .imageView = radv_image_view_to_handle(src_iview),
482 .imageLayout = src_layout,
483 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
484 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
485 },
486 {
487 .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
488 .imageView = radv_image_view_to_handle(dst_iview),
489 .imageLayout = dst_layout,
490 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
491 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
492 },
493 };
494
495 const VkRenderingInfo rendering_info = {
496 .sType = VK_STRUCTURE_TYPE_RENDERING_INFO,
497 .flags = VK_RENDERING_INPUT_ATTACHMENT_NO_CONCURRENT_WRITES_BIT_MESA,
498 .renderArea = saved_state.render.area,
499 .layerCount = 1,
500 .viewMask = saved_state.render.view_mask,
501 .colorAttachmentCount = 2,
502 .pColorAttachments = color_atts,
503 };
504
505 radv_CmdBeginRendering(radv_cmd_buffer_to_handle(cmd_buffer), &rendering_info);
506
507 emit_resolve(cmd_buffer, src_img, dst_img, dst_iview->vk.format);
508
509 radv_CmdEndRendering(radv_cmd_buffer_to_handle(cmd_buffer));
510
511 radv_meta_restore(&saved_state, cmd_buffer);
512 }
513
514 /**
515 * Emit any needed resolves for the current subpass.
516 */
517 void
radv_cmd_buffer_resolve_rendering(struct radv_cmd_buffer * cmd_buffer)518 radv_cmd_buffer_resolve_rendering(struct radv_cmd_buffer *cmd_buffer)
519 {
520 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
521 const struct radv_physical_device *pdev = radv_device_physical(device);
522 const struct radv_rendering_state *render = &cmd_buffer->state.render;
523 enum radv_resolve_method resolve_method = pdev->info.gfx_level >= GFX11 ? RESOLVE_FRAGMENT : RESOLVE_HW;
524
525 bool has_color_resolve = false;
526 for (uint32_t i = 0; i < render->color_att_count; ++i) {
527 if (render->color_att[i].resolve_iview != NULL)
528 has_color_resolve = true;
529 }
530 bool has_ds_resolve = render->ds_att.resolve_iview != NULL;
531
532 if (!has_color_resolve && !has_ds_resolve)
533 return;
534
535 radv_describe_begin_render_pass_resolve(cmd_buffer);
536
537 if (render->ds_att.resolve_iview != NULL) {
538 struct radv_image_view *src_iview = render->ds_att.iview;
539 struct radv_image_view *dst_iview = render->ds_att.resolve_iview;
540
541 radv_pick_resolve_method_images(device, src_iview->image, src_iview->vk.format, dst_iview->image,
542 dst_iview->vk.base_mip_level, VK_IMAGE_LAYOUT_UNDEFINED, cmd_buffer,
543 &resolve_method);
544
545 if ((src_iview->vk.aspects & VK_IMAGE_ASPECT_DEPTH_BIT) && render->ds_att.resolve_mode != VK_RESOLVE_MODE_NONE) {
546 if (resolve_method == RESOLVE_FRAGMENT) {
547 radv_depth_stencil_resolve_rendering_fs(cmd_buffer, VK_IMAGE_ASPECT_DEPTH_BIT, render->ds_att.resolve_mode);
548 } else {
549 assert(resolve_method == RESOLVE_COMPUTE);
550 radv_depth_stencil_resolve_rendering_cs(cmd_buffer, VK_IMAGE_ASPECT_DEPTH_BIT, render->ds_att.resolve_mode);
551 }
552 }
553
554 if ((src_iview->vk.aspects & VK_IMAGE_ASPECT_STENCIL_BIT) &&
555 render->ds_att.stencil_resolve_mode != VK_RESOLVE_MODE_NONE) {
556 if (resolve_method == RESOLVE_FRAGMENT) {
557 radv_depth_stencil_resolve_rendering_fs(cmd_buffer, VK_IMAGE_ASPECT_STENCIL_BIT,
558 render->ds_att.stencil_resolve_mode);
559 } else {
560 assert(resolve_method == RESOLVE_COMPUTE);
561 radv_depth_stencil_resolve_rendering_cs(cmd_buffer, VK_IMAGE_ASPECT_STENCIL_BIT,
562 render->ds_att.stencil_resolve_mode);
563 }
564 }
565
566 /* From the Vulkan spec 1.2.165:
567 *
568 * "VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT specifies
569 * write access to a color, resolve, or depth/stencil
570 * resolve attachment during a render pass or via
571 * certain subpass load and store operations."
572 *
573 * Yes, it's counterintuitive but it makes sense because ds
574 * resolve operations happen late at the end of the subpass.
575 *
576 * That said, RADV is wrong because it executes the subpass
577 * end barrier *before* any subpass resolves instead of after.
578 *
579 * TODO: Fix this properly by executing subpass end barriers
580 * after subpass resolves.
581 */
582 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB;
583 if (radv_image_has_htile(dst_iview->image))
584 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB_META;
585 }
586
587 if (has_color_resolve) {
588 uint32_t layer_count = render->layer_count;
589 VkRect2D resolve_area = render->area;
590 struct radv_resolve_barrier barrier;
591
592 if (render->view_mask)
593 layer_count = util_last_bit(render->view_mask);
594
595 /* Resolves happen before the end-of-subpass barriers get executed, so we have to make the
596 * attachment shader-readable.
597 */
598 barrier.src_stage_mask = VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT;
599 barrier.dst_stage_mask = VK_PIPELINE_STAGE_2_RESOLVE_BIT;
600 barrier.src_access_mask = VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT;
601 barrier.dst_access_mask = VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT;
602 radv_emit_resolve_barrier(cmd_buffer, &barrier);
603
604 for (uint32_t i = 0; i < render->color_att_count; ++i) {
605 if (render->color_att[i].resolve_iview == NULL)
606 continue;
607
608 struct radv_image_view *src_iview = render->color_att[i].iview;
609 VkImageLayout src_layout = render->color_att[i].layout;
610 struct radv_image *src_img = src_iview->image;
611 struct radv_image_view *dst_iview = render->color_att[i].resolve_iview;
612 VkImageLayout dst_layout = render->color_att[i].resolve_layout;
613 struct radv_image *dst_img = dst_iview->image;
614
615 radv_pick_resolve_method_images(device, src_img, src_iview->vk.format, dst_img, dst_iview->vk.base_mip_level,
616 dst_layout, cmd_buffer, &resolve_method);
617 VkImageResolve2 region = {
618 .sType = VK_STRUCTURE_TYPE_IMAGE_RESOLVE_2,
619 .extent =
620 {
621 .width = resolve_area.extent.width,
622 .height = resolve_area.extent.height,
623 .depth = 1,
624 },
625 .srcSubresource =
626 (VkImageSubresourceLayers){
627 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
628 .mipLevel = src_iview->vk.base_mip_level,
629 .baseArrayLayer = src_iview->vk.base_array_layer,
630 .layerCount = layer_count,
631 },
632 .dstSubresource =
633 (VkImageSubresourceLayers){
634 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
635 .mipLevel = dst_iview->vk.base_mip_level,
636 .baseArrayLayer = dst_iview->vk.base_array_layer,
637 .layerCount = layer_count,
638 },
639 .srcOffset = {resolve_area.offset.x, resolve_area.offset.y, 0},
640 .dstOffset = {resolve_area.offset.x, resolve_area.offset.y, 0},
641 };
642
643 switch (resolve_method) {
644 case RESOLVE_HW:
645 radv_cmd_buffer_resolve_rendering_hw(cmd_buffer, src_iview, src_layout, dst_iview, dst_layout);
646 break;
647 case RESOLVE_COMPUTE:
648 radv_decompress_resolve_src(cmd_buffer, src_iview->image, src_layout, ®ion);
649
650 radv_cmd_buffer_resolve_rendering_cs(cmd_buffer, src_iview, src_layout, dst_iview, dst_layout, ®ion);
651 break;
652 case RESOLVE_FRAGMENT:
653 radv_decompress_resolve_src(cmd_buffer, src_iview->image, src_layout, ®ion);
654
655 radv_cmd_buffer_resolve_rendering_fs(cmd_buffer, src_iview, src_layout, dst_iview, dst_layout);
656 break;
657 default:
658 unreachable("Invalid resolve method");
659 }
660 }
661 }
662
663 radv_describe_end_render_pass_resolve(cmd_buffer);
664 }
665
666 /**
667 * Decompress CMask/FMask before resolving a multisampled source image inside a
668 * subpass.
669 */
670 void
radv_decompress_resolve_rendering_src(struct radv_cmd_buffer * cmd_buffer)671 radv_decompress_resolve_rendering_src(struct radv_cmd_buffer *cmd_buffer)
672 {
673 const struct radv_rendering_state *render = &cmd_buffer->state.render;
674
675 uint32_t layer_count = render->layer_count;
676 if (render->view_mask)
677 layer_count = util_last_bit(render->view_mask);
678
679 for (uint32_t i = 0; i < render->color_att_count; ++i) {
680 if (render->color_att[i].resolve_iview == NULL)
681 continue;
682
683 struct radv_image_view *src_iview = render->color_att[i].iview;
684 VkImageLayout src_layout = render->color_att[i].layout;
685 struct radv_image *src_image = src_iview->image;
686
687 VkImageResolve2 region = {0};
688 region.sType = VK_STRUCTURE_TYPE_IMAGE_RESOLVE_2;
689 region.srcSubresource.aspectMask = src_iview->vk.aspects;
690 region.srcSubresource.mipLevel = 0;
691 region.srcSubresource.baseArrayLayer = src_iview->vk.base_array_layer;
692 region.srcSubresource.layerCount = layer_count;
693
694 radv_decompress_resolve_src(cmd_buffer, src_image, src_layout, ®ion);
695 }
696 }
697
698 /**
699 * Decompress CMask/FMask before resolving a multisampled source image.
700 */
701 void
radv_decompress_resolve_src(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,VkImageLayout src_image_layout,const VkImageResolve2 * region)702 radv_decompress_resolve_src(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
703 VkImageLayout src_image_layout, const VkImageResolve2 *region)
704 {
705 VkImageMemoryBarrier2 barrier = {
706 .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
707 .srcStageMask = VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT,
708 .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT,
709 .dstStageMask = VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT,
710 .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT,
711 .oldLayout = src_image_layout,
712 .newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
713 .image = radv_image_to_handle(src_image),
714 .subresourceRange = (VkImageSubresourceRange){
715 .aspectMask = region->srcSubresource.aspectMask,
716 .baseMipLevel = 0,
717 .levelCount = 1,
718 .baseArrayLayer = region->srcSubresource.baseArrayLayer,
719 .layerCount = vk_image_subresource_layer_count(&src_image->vk, ®ion->srcSubresource),
720 }};
721
722 VkSampleLocationsInfoEXT sample_loc_info;
723 if (src_image->vk.create_flags & VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT) {
724 /* If the depth/stencil image uses different sample
725 * locations, we need them during HTILE decompressions.
726 */
727 struct radv_sample_locations_state *sample_locs = &cmd_buffer->state.render.sample_locations;
728
729 sample_loc_info = (VkSampleLocationsInfoEXT){
730 .sType = VK_STRUCTURE_TYPE_SAMPLE_LOCATIONS_INFO_EXT,
731 .sampleLocationsPerPixel = sample_locs->per_pixel,
732 .sampleLocationGridSize = sample_locs->grid_size,
733 .sampleLocationsCount = sample_locs->count,
734 .pSampleLocations = sample_locs->locations,
735 };
736 barrier.pNext = &sample_loc_info;
737 }
738
739 VkDependencyInfo dep_info = {
740 .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
741 .imageMemoryBarrierCount = 1,
742 .pImageMemoryBarriers = &barrier,
743 };
744
745 radv_CmdPipelineBarrier2(radv_cmd_buffer_to_handle(cmd_buffer), &dep_info);
746 }
747