1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include <assert.h>
25 #include <stdbool.h>
26
27 #include "radv_meta.h"
28 #include "radv_private.h"
29 #include "vk_format.h"
30 #include "nir/nir_builder.h"
31 #include "sid.h"
32
33 /* emit 0, 0, 0, 1 */
34 static nir_shader *
build_nir_fs(void)35 build_nir_fs(void)
36 {
37 const struct glsl_type *vec4 = glsl_vec4_type();
38 nir_builder b;
39 nir_variable *f_color; /* vec4, fragment output color */
40
41 nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
42 b.shader->info.name = ralloc_asprintf(b.shader,
43 "meta_resolve_fs");
44
45 f_color = nir_variable_create(b.shader, nir_var_shader_out, vec4,
46 "f_color");
47 f_color->data.location = FRAG_RESULT_DATA0;
48 nir_store_var(&b, f_color, nir_imm_vec4(&b, 0.0, 0.0, 0.0, 1.0), 0xf);
49
50 return b.shader;
51 }
52
53 static VkResult
create_pass(struct radv_device * device,VkFormat vk_format,VkRenderPass * pass)54 create_pass(struct radv_device *device, VkFormat vk_format, VkRenderPass *pass)
55 {
56 VkResult result;
57 VkDevice device_h = radv_device_to_handle(device);
58 const VkAllocationCallbacks *alloc = &device->meta_state.alloc;
59 VkAttachmentDescription attachments[2];
60 int i;
61
62 for (i = 0; i < 2; i++) {
63 attachments[i].format = vk_format;
64 attachments[i].samples = 1;
65 attachments[i].loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
66 attachments[i].storeOp = VK_ATTACHMENT_STORE_OP_STORE;
67 }
68 attachments[0].initialLayout = VK_IMAGE_LAYOUT_GENERAL;
69 attachments[0].finalLayout = VK_IMAGE_LAYOUT_GENERAL;
70 attachments[1].initialLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
71 attachments[1].finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
72
73 result = radv_CreateRenderPass(device_h,
74 &(VkRenderPassCreateInfo) {
75 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
76 .attachmentCount = 2,
77 .pAttachments = attachments,
78 .subpassCount = 1,
79 .pSubpasses = &(VkSubpassDescription) {
80 .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
81 .inputAttachmentCount = 0,
82 .colorAttachmentCount = 2,
83 .pColorAttachments = (VkAttachmentReference[]) {
84 {
85 .attachment = 0,
86 .layout = VK_IMAGE_LAYOUT_GENERAL,
87 },
88 {
89 .attachment = 1,
90 .layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
91 },
92 },
93 .pResolveAttachments = NULL,
94 .pDepthStencilAttachment = &(VkAttachmentReference) {
95 .attachment = VK_ATTACHMENT_UNUSED,
96 },
97 .preserveAttachmentCount = 0,
98 .pPreserveAttachments = NULL,
99 },
100 .dependencyCount = 2,
101 .pDependencies = (VkSubpassDependency[]) {
102 {
103 .srcSubpass = VK_SUBPASS_EXTERNAL,
104 .dstSubpass = 0,
105 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
106 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
107 .srcAccessMask = 0,
108 .dstAccessMask = 0,
109 .dependencyFlags = 0
110 },
111 {
112 .srcSubpass = 0,
113 .dstSubpass = VK_SUBPASS_EXTERNAL,
114 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
115 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
116 .srcAccessMask = 0,
117 .dstAccessMask = 0,
118 .dependencyFlags = 0
119 }
120 },
121 },
122 alloc,
123 pass);
124
125 return result;
126 }
127
128 static VkResult
create_pipeline(struct radv_device * device,VkShaderModule vs_module_h,VkPipeline * pipeline,VkRenderPass pass)129 create_pipeline(struct radv_device *device,
130 VkShaderModule vs_module_h,
131 VkPipeline *pipeline,
132 VkRenderPass pass)
133 {
134 VkResult result;
135 VkDevice device_h = radv_device_to_handle(device);
136
137 struct radv_shader_module fs_module = {
138 .nir = build_nir_fs(),
139 };
140
141 if (!fs_module.nir) {
142 /* XXX: Need more accurate error */
143 result = VK_ERROR_OUT_OF_HOST_MEMORY;
144 goto cleanup;
145 }
146
147 VkPipelineLayoutCreateInfo pl_create_info = {
148 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
149 .setLayoutCount = 0,
150 .pSetLayouts = NULL,
151 .pushConstantRangeCount = 0,
152 .pPushConstantRanges = NULL,
153 };
154
155 if (!device->meta_state.resolve.p_layout) {
156 result = radv_CreatePipelineLayout(radv_device_to_handle(device),
157 &pl_create_info,
158 &device->meta_state.alloc,
159 &device->meta_state.resolve.p_layout);
160 if (result != VK_SUCCESS)
161 goto cleanup;
162 }
163
164 result = radv_graphics_pipeline_create(device_h,
165 radv_pipeline_cache_to_handle(&device->meta_state.cache),
166 &(VkGraphicsPipelineCreateInfo) {
167 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
168 .stageCount = 2,
169 .pStages = (VkPipelineShaderStageCreateInfo[]) {
170 {
171 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
172 .stage = VK_SHADER_STAGE_VERTEX_BIT,
173 .module = vs_module_h,
174 .pName = "main",
175 },
176 {
177 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
178 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
179 .module = radv_shader_module_to_handle(&fs_module),
180 .pName = "main",
181 },
182 },
183 .pVertexInputState = &(VkPipelineVertexInputStateCreateInfo) {
184 .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
185 .vertexBindingDescriptionCount = 0,
186 .vertexAttributeDescriptionCount = 0,
187 },
188 .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
189 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
190 .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
191 .primitiveRestartEnable = false,
192 },
193 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
194 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
195 .viewportCount = 1,
196 .scissorCount = 1,
197 },
198 .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
199 .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
200 .depthClampEnable = false,
201 .rasterizerDiscardEnable = false,
202 .polygonMode = VK_POLYGON_MODE_FILL,
203 .cullMode = VK_CULL_MODE_NONE,
204 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE,
205 },
206 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
207 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
208 .rasterizationSamples = 1,
209 .sampleShadingEnable = false,
210 .pSampleMask = NULL,
211 .alphaToCoverageEnable = false,
212 .alphaToOneEnable = false,
213 },
214 .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
215 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
216 .logicOpEnable = false,
217 .attachmentCount = 2,
218 .pAttachments = (VkPipelineColorBlendAttachmentState []) {
219 {
220 .colorWriteMask = VK_COLOR_COMPONENT_R_BIT |
221 VK_COLOR_COMPONENT_G_BIT |
222 VK_COLOR_COMPONENT_B_BIT |
223 VK_COLOR_COMPONENT_A_BIT,
224 },
225 {
226 .colorWriteMask = 0,
227
228 }
229 },
230 },
231 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
232 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
233 .dynamicStateCount = 2,
234 .pDynamicStates = (VkDynamicState[]) {
235 VK_DYNAMIC_STATE_VIEWPORT,
236 VK_DYNAMIC_STATE_SCISSOR,
237 },
238 },
239 .layout = device->meta_state.resolve.p_layout,
240 .renderPass = pass,
241 .subpass = 0,
242 },
243 &(struct radv_graphics_pipeline_create_info) {
244 .use_rectlist = true,
245 .custom_blend_mode = V_028808_CB_RESOLVE,
246 },
247 &device->meta_state.alloc, pipeline);
248 if (result != VK_SUCCESS)
249 goto cleanup;
250
251 goto cleanup;
252
253 cleanup:
254 ralloc_free(fs_module.nir);
255 return result;
256 }
257
258 void
radv_device_finish_meta_resolve_state(struct radv_device * device)259 radv_device_finish_meta_resolve_state(struct radv_device *device)
260 {
261 struct radv_meta_state *state = &device->meta_state;
262
263 for (uint32_t j = 0; j < NUM_META_FS_KEYS; j++) {
264 radv_DestroyRenderPass(radv_device_to_handle(device),
265 state->resolve.pass[j], &state->alloc);
266 radv_DestroyPipeline(radv_device_to_handle(device),
267 state->resolve.pipeline[j], &state->alloc);
268 }
269 radv_DestroyPipelineLayout(radv_device_to_handle(device),
270 state->resolve.p_layout, &state->alloc);
271
272 }
273
274 VkResult
radv_device_init_meta_resolve_state(struct radv_device * device,bool on_demand)275 radv_device_init_meta_resolve_state(struct radv_device *device, bool on_demand)
276 {
277 if (on_demand)
278 return VK_SUCCESS;
279
280 VkResult res = VK_SUCCESS;
281 struct radv_meta_state *state = &device->meta_state;
282 struct radv_shader_module vs_module = { .nir = radv_meta_build_nir_vs_generate_vertices() };
283 if (!vs_module.nir) {
284 /* XXX: Need more accurate error */
285 res = VK_ERROR_OUT_OF_HOST_MEMORY;
286 goto fail;
287 }
288
289 for (uint32_t i = 0; i < NUM_META_FS_KEYS; ++i) {
290 VkFormat format = radv_fs_key_format_exemplars[i];
291 unsigned fs_key = radv_format_meta_fs_key(format);
292 res = create_pass(device, format, &state->resolve.pass[fs_key]);
293 if (res != VK_SUCCESS)
294 goto fail;
295
296 VkShaderModule vs_module_h = radv_shader_module_to_handle(&vs_module);
297 res = create_pipeline(device, vs_module_h,
298 &state->resolve.pipeline[fs_key], state->resolve.pass[fs_key]);
299 if (res != VK_SUCCESS)
300 goto fail;
301 }
302
303 goto cleanup;
304
305 fail:
306 radv_device_finish_meta_resolve_state(device);
307
308 cleanup:
309 ralloc_free(vs_module.nir);
310
311 return res;
312 }
313
314 static void
emit_resolve(struct radv_cmd_buffer * cmd_buffer,VkFormat vk_format,const VkOffset2D * dest_offset,const VkExtent2D * resolve_extent)315 emit_resolve(struct radv_cmd_buffer *cmd_buffer,
316 VkFormat vk_format,
317 const VkOffset2D *dest_offset,
318 const VkExtent2D *resolve_extent)
319 {
320 struct radv_device *device = cmd_buffer->device;
321 VkCommandBuffer cmd_buffer_h = radv_cmd_buffer_to_handle(cmd_buffer);
322 unsigned fs_key = radv_format_meta_fs_key(vk_format);
323
324 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB;
325
326 radv_CmdBindPipeline(cmd_buffer_h, VK_PIPELINE_BIND_POINT_GRAPHICS,
327 device->meta_state.resolve.pipeline[fs_key]);
328
329 radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkViewport) {
330 .x = dest_offset->x,
331 .y = dest_offset->y,
332 .width = resolve_extent->width,
333 .height = resolve_extent->height,
334 .minDepth = 0.0f,
335 .maxDepth = 1.0f
336 });
337
338 radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkRect2D) {
339 .offset = *dest_offset,
340 .extent = *resolve_extent,
341 });
342
343 radv_CmdDraw(cmd_buffer_h, 3, 1, 0, 0);
344 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB;
345 }
346
347 enum radv_resolve_method {
348 RESOLVE_HW,
349 RESOLVE_COMPUTE,
350 RESOLVE_FRAGMENT,
351 };
352
radv_pick_resolve_method_images(struct radv_device * device,struct radv_image * src_image,VkFormat src_format,struct radv_image * dest_image,VkImageLayout dest_image_layout,bool dest_render_loop,struct radv_cmd_buffer * cmd_buffer,enum radv_resolve_method * method)353 static void radv_pick_resolve_method_images(struct radv_device *device,
354 struct radv_image *src_image,
355 VkFormat src_format,
356 struct radv_image *dest_image,
357 VkImageLayout dest_image_layout,
358 bool dest_render_loop,
359 struct radv_cmd_buffer *cmd_buffer,
360 enum radv_resolve_method *method)
361
362 {
363 uint32_t queue_mask = radv_image_queue_family_mask(dest_image,
364 cmd_buffer->queue_family_index,
365 cmd_buffer->queue_family_index);
366
367 if (vk_format_is_color(src_format)) {
368 if (src_format == VK_FORMAT_R16G16_UNORM ||
369 src_format == VK_FORMAT_R16G16_SNORM)
370 *method = RESOLVE_COMPUTE;
371 else if (vk_format_is_int(src_format))
372 *method = RESOLVE_COMPUTE;
373 else if (src_image->info.array_size > 1 ||
374 dest_image->info.array_size > 1)
375 *method = RESOLVE_COMPUTE;
376
377 if (radv_layout_dcc_compressed(device, dest_image, dest_image_layout,
378 dest_render_loop, queue_mask)) {
379 *method = RESOLVE_FRAGMENT;
380 } else if (dest_image->planes[0].surface.micro_tile_mode !=
381 src_image->planes[0].surface.micro_tile_mode) {
382 *method = RESOLVE_COMPUTE;
383 }
384 } else {
385 if (src_image->info.array_size > 1 ||
386 dest_image->info.array_size > 1)
387 *method = RESOLVE_COMPUTE;
388 else
389 *method = RESOLVE_FRAGMENT;
390 }
391 }
392
393 static VkResult
build_resolve_pipeline(struct radv_device * device,unsigned fs_key)394 build_resolve_pipeline(struct radv_device *device,
395 unsigned fs_key)
396 {
397 VkResult result = VK_SUCCESS;
398
399 if (device->meta_state.resolve.pipeline[fs_key])
400 return result;
401
402 mtx_lock(&device->meta_state.mtx);
403 if (device->meta_state.resolve.pipeline[fs_key]) {
404 mtx_unlock(&device->meta_state.mtx);
405 return result;
406 }
407
408 struct radv_shader_module vs_module = { .nir = radv_meta_build_nir_vs_generate_vertices() };
409
410 result = create_pass(device, radv_fs_key_format_exemplars[fs_key], &device->meta_state.resolve.pass[fs_key]);
411 if (result != VK_SUCCESS)
412 goto fail;
413
414 VkShaderModule vs_module_h = radv_shader_module_to_handle(&vs_module);
415 result = create_pipeline(device, vs_module_h, &device->meta_state.resolve.pipeline[fs_key], device->meta_state.resolve.pass[fs_key]);
416
417 fail:
418 ralloc_free(vs_module.nir);
419 mtx_unlock(&device->meta_state.mtx);
420 return result;
421 }
422
423 static void
radv_meta_resolve_hardware_image(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,VkImageLayout src_image_layout,struct radv_image * dst_image,VkImageLayout dst_image_layout,const VkImageResolve2KHR * region)424 radv_meta_resolve_hardware_image(struct radv_cmd_buffer *cmd_buffer,
425 struct radv_image *src_image,
426 VkImageLayout src_image_layout,
427 struct radv_image *dst_image,
428 VkImageLayout dst_image_layout,
429 const VkImageResolve2KHR *region)
430 {
431 struct radv_device *device = cmd_buffer->device;
432 struct radv_meta_saved_state saved_state;
433
434 radv_meta_save(&saved_state, cmd_buffer,
435 RADV_META_SAVE_GRAPHICS_PIPELINE);
436
437 assert(src_image->info.samples > 1);
438 if (src_image->info.samples <= 1) {
439 /* this causes GPU hangs if we get past here */
440 fprintf(stderr, "radv: Illegal resolve operation (src not multisampled), will hang GPU.");
441 return;
442 }
443 assert(dst_image->info.samples == 1);
444
445 if (src_image->info.array_size > 1)
446 radv_finishme("vkCmdResolveImage: multisample array images");
447
448 unsigned fs_key = radv_format_meta_fs_key(dst_image->vk_format);
449
450 /* From the Vulkan 1.0 spec:
451 *
452 * - The aspectMask member of srcSubresource and dstSubresource must
453 * only contain VK_IMAGE_ASPECT_COLOR_BIT
454 *
455 * - The layerCount member of srcSubresource and dstSubresource must
456 * match
457 */
458 assert(region->srcSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
459 assert(region->dstSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
460 assert(region->srcSubresource.layerCount ==
461 region->dstSubresource.layerCount);
462
463 const uint32_t src_base_layer =
464 radv_meta_get_iview_layer(src_image, ®ion->srcSubresource,
465 ®ion->srcOffset);
466
467 const uint32_t dst_base_layer =
468 radv_meta_get_iview_layer(dst_image, ®ion->dstSubresource,
469 ®ion->dstOffset);
470
471 /**
472 * From Vulkan 1.0.6 spec: 18.6 Resolving Multisample Images
473 *
474 * extent is the size in texels of the source image to resolve in width,
475 * height and depth. 1D images use only x and width. 2D images use x, y,
476 * width and height. 3D images use x, y, z, width, height and depth.
477 *
478 * srcOffset and dstOffset select the initial x, y, and z offsets in
479 * texels of the sub-regions of the source and destination image data.
480 * extent is the size in texels of the source image to resolve in width,
481 * height and depth. 1D images use only x and width. 2D images use x, y,
482 * width and height. 3D images use x, y, z, width, height and depth.
483 */
484 const struct VkExtent3D extent =
485 radv_sanitize_image_extent(src_image->type, region->extent);
486 const struct VkOffset3D dstOffset =
487 radv_sanitize_image_offset(dst_image->type, region->dstOffset);
488
489 if (radv_dcc_enabled(dst_image, region->dstSubresource.mipLevel)) {
490 VkImageSubresourceRange range = {
491 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
492 .baseMipLevel = region->dstSubresource.mipLevel,
493 .levelCount = 1,
494 .baseArrayLayer = dst_base_layer,
495 .layerCount = region->dstSubresource.layerCount,
496 };
497
498 radv_initialize_dcc(cmd_buffer, dst_image, &range, 0xffffffff);
499 }
500
501 for (uint32_t layer = 0; layer < region->srcSubresource.layerCount;
502 ++layer) {
503
504 VkResult ret = build_resolve_pipeline(device, fs_key);
505 if (ret != VK_SUCCESS) {
506 cmd_buffer->record_result = ret;
507 break;
508 }
509
510 struct radv_image_view src_iview;
511 radv_image_view_init(&src_iview, cmd_buffer->device,
512 &(VkImageViewCreateInfo) {
513 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
514 .image = radv_image_to_handle(src_image),
515 .viewType = radv_meta_get_view_type(src_image),
516 .format = src_image->vk_format,
517 .subresourceRange = {
518 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
519 .baseMipLevel = region->srcSubresource.mipLevel,
520 .levelCount = 1,
521 .baseArrayLayer = src_base_layer + layer,
522 .layerCount = 1,
523 },
524 }, NULL);
525
526 struct radv_image_view dst_iview;
527 radv_image_view_init(&dst_iview, cmd_buffer->device,
528 &(VkImageViewCreateInfo) {
529 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
530 .image = radv_image_to_handle(dst_image),
531 .viewType = radv_meta_get_view_type(dst_image),
532 .format = dst_image->vk_format,
533 .subresourceRange = {
534 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
535 .baseMipLevel = region->dstSubresource.mipLevel,
536 .levelCount = 1,
537 .baseArrayLayer = dst_base_layer + layer,
538 .layerCount = 1,
539 },
540 }, NULL);
541
542 VkFramebuffer fb_h;
543 radv_CreateFramebuffer(radv_device_to_handle(device),
544 &(VkFramebufferCreateInfo) {
545 .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
546 .attachmentCount = 2,
547 .pAttachments = (VkImageView[]) {
548 radv_image_view_to_handle(&src_iview),
549 radv_image_view_to_handle(&dst_iview),
550 },
551 .width = radv_minify(dst_image->info.width,
552 region->dstSubresource.mipLevel),
553 .height = radv_minify(dst_image->info.height,
554 region->dstSubresource.mipLevel),
555 .layers = 1
556 },
557 &cmd_buffer->pool->alloc,
558 &fb_h);
559
560 radv_cmd_buffer_begin_render_pass(cmd_buffer,
561 &(VkRenderPassBeginInfo) {
562 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
563 .renderPass = device->meta_state.resolve.pass[fs_key],
564 .framebuffer = fb_h,
565 .renderArea = {
566 .offset = {
567 dstOffset.x,
568 dstOffset.y,
569 },
570 .extent = {
571 extent.width,
572 extent.height,
573 }
574 },
575 .clearValueCount = 0,
576 .pClearValues = NULL,
577 });
578
579 radv_cmd_buffer_set_subpass(cmd_buffer,
580 &cmd_buffer->state.pass->subpasses[0]);
581
582 emit_resolve(cmd_buffer,
583 dst_iview.vk_format,
584 &(VkOffset2D) {
585 .x = dstOffset.x,
586 .y = dstOffset.y,
587 },
588 &(VkExtent2D) {
589 .width = extent.width,
590 .height = extent.height,
591 });
592
593 radv_cmd_buffer_end_render_pass(cmd_buffer);
594
595 radv_DestroyFramebuffer(radv_device_to_handle(device),
596 fb_h, &cmd_buffer->pool->alloc);
597 }
598
599 radv_meta_restore(&saved_state, cmd_buffer);
600 }
601
602 static void
resolve_image(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,VkImageLayout src_image_layout,struct radv_image * dst_image,VkImageLayout dst_image_layout,const VkImageResolve2KHR * region,enum radv_resolve_method resolve_method)603 resolve_image(struct radv_cmd_buffer *cmd_buffer,
604 struct radv_image *src_image,
605 VkImageLayout src_image_layout,
606 struct radv_image *dst_image,
607 VkImageLayout dst_image_layout,
608 const VkImageResolve2KHR *region,
609 enum radv_resolve_method resolve_method)
610 {
611 switch (resolve_method) {
612 case RESOLVE_HW:
613 radv_meta_resolve_hardware_image(cmd_buffer,
614 src_image,
615 src_image_layout,
616 dst_image,
617 dst_image_layout,
618 region);
619 break;
620 case RESOLVE_FRAGMENT:
621 radv_meta_resolve_fragment_image(cmd_buffer,
622 src_image,
623 src_image_layout,
624 dst_image,
625 dst_image_layout,
626 region);
627 break;
628 case RESOLVE_COMPUTE:
629 radv_meta_resolve_compute_image(cmd_buffer,
630 src_image,
631 src_image->vk_format,
632 src_image_layout,
633 dst_image,
634 dst_image->vk_format,
635 dst_image_layout,
636 region);
637 break;
638 default:
639 assert(!"Invalid resolve method selected");
640 }
641 }
642
radv_CmdResolveImage(VkCommandBuffer cmd_buffer_h,VkImage src_image_h,VkImageLayout src_image_layout,VkImage dest_image_h,VkImageLayout dest_image_layout,uint32_t region_count,const VkImageResolve * regions)643 void radv_CmdResolveImage(
644 VkCommandBuffer cmd_buffer_h,
645 VkImage src_image_h,
646 VkImageLayout src_image_layout,
647 VkImage dest_image_h,
648 VkImageLayout dest_image_layout,
649 uint32_t region_count,
650 const VkImageResolve* regions)
651 {
652 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, cmd_buffer_h);
653 RADV_FROM_HANDLE(radv_image, src_image, src_image_h);
654 RADV_FROM_HANDLE(radv_image, dest_image, dest_image_h);
655 enum radv_resolve_method resolve_method = RESOLVE_HW;
656 /* we can use the hw resolve only for single full resolves */
657 if (region_count == 1) {
658 if (regions[0].srcOffset.x ||
659 regions[0].srcOffset.y ||
660 regions[0].srcOffset.z)
661 resolve_method = RESOLVE_COMPUTE;
662 if (regions[0].dstOffset.x ||
663 regions[0].dstOffset.y ||
664 regions[0].dstOffset.z)
665 resolve_method = RESOLVE_COMPUTE;
666
667 if (regions[0].extent.width != src_image->info.width ||
668 regions[0].extent.height != src_image->info.height ||
669 regions[0].extent.depth != src_image->info.depth)
670 resolve_method = RESOLVE_COMPUTE;
671 } else
672 resolve_method = RESOLVE_COMPUTE;
673
674 radv_pick_resolve_method_images(cmd_buffer->device, src_image,
675 src_image->vk_format, dest_image,
676 dest_image_layout, false, cmd_buffer,
677 &resolve_method);
678
679 for (uint32_t r = 0; r < region_count; r++) {
680 VkImageResolve2KHR region = {
681 .sType = VK_STRUCTURE_TYPE_IMAGE_RESOLVE_2_KHR,
682 .srcSubresource = regions[r].srcSubresource,
683 .srcOffset = regions[r].srcOffset,
684 .dstSubresource = regions[r].dstSubresource,
685 .dstOffset = regions[r].dstOffset,
686 .extent = regions[r].extent,
687 };
688
689 resolve_image(cmd_buffer, src_image, src_image_layout,
690 dest_image, dest_image_layout,
691 ®ion, resolve_method);
692 }
693 }
694
radv_CmdResolveImage2KHR(VkCommandBuffer commandBuffer,const VkResolveImageInfo2KHR * pResolveImageInfo)695 void radv_CmdResolveImage2KHR(
696 VkCommandBuffer commandBuffer,
697 const VkResolveImageInfo2KHR* pResolveImageInfo)
698 {
699 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
700 RADV_FROM_HANDLE(radv_image, src_image, pResolveImageInfo->srcImage);
701 RADV_FROM_HANDLE(radv_image, dst_image, pResolveImageInfo->dstImage);
702 VkImageLayout src_image_layout = pResolveImageInfo->srcImageLayout;
703 VkImageLayout dst_image_layout = pResolveImageInfo->dstImageLayout;
704 enum radv_resolve_method resolve_method = RESOLVE_HW;
705 /* we can use the hw resolve only for single full resolves */
706 if (pResolveImageInfo->regionCount == 1) {
707 if (pResolveImageInfo->pRegions[0].srcOffset.x ||
708 pResolveImageInfo->pRegions[0].srcOffset.y ||
709 pResolveImageInfo->pRegions[0].srcOffset.z)
710 resolve_method = RESOLVE_COMPUTE;
711 if (pResolveImageInfo->pRegions[0].dstOffset.x ||
712 pResolveImageInfo->pRegions[0].dstOffset.y ||
713 pResolveImageInfo->pRegions[0].dstOffset.z)
714 resolve_method = RESOLVE_COMPUTE;
715
716 if (pResolveImageInfo->pRegions[0].extent.width != src_image->info.width ||
717 pResolveImageInfo->pRegions[0].extent.height != src_image->info.height ||
718 pResolveImageInfo->pRegions[0].extent.depth != src_image->info.depth)
719 resolve_method = RESOLVE_COMPUTE;
720 } else
721 resolve_method = RESOLVE_COMPUTE;
722
723 radv_pick_resolve_method_images(cmd_buffer->device, src_image,
724 src_image->vk_format, dst_image,
725 dst_image_layout, false, cmd_buffer,
726 &resolve_method);
727
728 for (uint32_t r = 0; r < pResolveImageInfo->regionCount; r++) {
729 resolve_image(cmd_buffer, src_image, src_image_layout,
730 dst_image, dst_image_layout,
731 &pResolveImageInfo->pRegions[r], resolve_method);
732 }
733 }
734
735 /**
736 * Emit any needed resolves for the current subpass.
737 */
738 void
radv_cmd_buffer_resolve_subpass(struct radv_cmd_buffer * cmd_buffer)739 radv_cmd_buffer_resolve_subpass(struct radv_cmd_buffer *cmd_buffer)
740 {
741 struct radv_framebuffer *fb = cmd_buffer->state.framebuffer;
742 const struct radv_subpass *subpass = cmd_buffer->state.subpass;
743 struct radv_meta_saved_state saved_state;
744 enum radv_resolve_method resolve_method = RESOLVE_HW;
745
746 if (subpass->ds_resolve_attachment) {
747 struct radv_subpass_attachment src_att = *subpass->depth_stencil_attachment;
748 struct radv_subpass_attachment dst_att = *subpass->ds_resolve_attachment;
749 struct radv_image_view *src_iview =
750 cmd_buffer->state.attachments[src_att.attachment].iview;
751 struct radv_image_view *dst_iview =
752 cmd_buffer->state.attachments[dst_att.attachment].iview;
753
754 /* Make sure to not clear the depth/stencil attachment after resolves. */
755 cmd_buffer->state.attachments[dst_att.attachment].pending_clear_aspects = 0;
756
757 radv_pick_resolve_method_images(cmd_buffer->device,
758 src_iview->image,
759 src_iview->vk_format,
760 dst_iview->image,
761 dst_att.layout,
762 dst_att.in_render_loop,
763 cmd_buffer,
764 &resolve_method);
765
766 if ((src_iview->aspect_mask & VK_IMAGE_ASPECT_DEPTH_BIT) &&
767 subpass->depth_resolve_mode != VK_RESOLVE_MODE_NONE_KHR) {
768 if (resolve_method == RESOLVE_FRAGMENT) {
769 radv_depth_stencil_resolve_subpass_fs(cmd_buffer,
770 VK_IMAGE_ASPECT_DEPTH_BIT,
771 subpass->depth_resolve_mode);
772 } else {
773 assert(resolve_method == RESOLVE_COMPUTE);
774 radv_depth_stencil_resolve_subpass_cs(cmd_buffer,
775 VK_IMAGE_ASPECT_DEPTH_BIT,
776 subpass->depth_resolve_mode);
777 }
778 }
779
780 if ((src_iview->aspect_mask & VK_IMAGE_ASPECT_STENCIL_BIT) &&
781 subpass->stencil_resolve_mode != VK_RESOLVE_MODE_NONE_KHR) {
782 if (resolve_method == RESOLVE_FRAGMENT) {
783 radv_depth_stencil_resolve_subpass_fs(cmd_buffer,
784 VK_IMAGE_ASPECT_STENCIL_BIT,
785 subpass->stencil_resolve_mode);
786 } else {
787 assert(resolve_method == RESOLVE_COMPUTE);
788 radv_depth_stencil_resolve_subpass_cs(cmd_buffer,
789 VK_IMAGE_ASPECT_STENCIL_BIT,
790 subpass->stencil_resolve_mode);
791 }
792 }
793
794 /* From the Vulkan spec 1.2.165:
795 *
796 * "VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT specifies
797 * write access to a color, resolve, or depth/stencil
798 * resolve attachment during a render pass or via
799 * certain subpass load and store operations."
800 *
801 * Yes, it's counterintuitive but it makes sense because ds
802 * resolve operations happen late at the end of the subpass.
803 *
804 * That said, RADV is wrong because it executes the subpass
805 * end barrier *before* any subpass resolves instead of after.
806 *
807 * TODO: Fix this properly by executing subpass end barriers
808 * after subpass resolves.
809 */
810 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB;
811 if (radv_image_has_htile(dst_iview->image))
812 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB_META;
813 }
814
815 if (!subpass->has_color_resolve)
816 return;
817
818 for (uint32_t i = 0; i < subpass->color_count; ++i) {
819 struct radv_subpass_attachment src_att = subpass->color_attachments[i];
820 struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i];
821
822 if (dest_att.attachment == VK_ATTACHMENT_UNUSED)
823 continue;
824
825 /* Make sure to not clear color attachments after resolves. */
826 cmd_buffer->state.attachments[dest_att.attachment].pending_clear_aspects = 0;
827
828 struct radv_image *dst_img = cmd_buffer->state.attachments[dest_att.attachment].iview->image;
829 struct radv_image_view *src_iview= cmd_buffer->state.attachments[src_att.attachment].iview;
830 struct radv_image *src_img = src_iview->image;
831
832 radv_pick_resolve_method_images(cmd_buffer->device, src_img,
833 src_iview->vk_format, dst_img,
834 dest_att.layout,
835 dest_att.in_render_loop,
836 cmd_buffer, &resolve_method);
837
838 if (resolve_method == RESOLVE_FRAGMENT) {
839 break;
840 }
841 }
842
843 if (resolve_method == RESOLVE_COMPUTE) {
844 radv_cmd_buffer_resolve_subpass_cs(cmd_buffer);
845 return;
846 } else if (resolve_method == RESOLVE_FRAGMENT) {
847 radv_cmd_buffer_resolve_subpass_fs(cmd_buffer);
848 return;
849 }
850
851 radv_meta_save(&saved_state, cmd_buffer,
852 RADV_META_SAVE_GRAPHICS_PIPELINE);
853
854 for (uint32_t i = 0; i < subpass->color_count; ++i) {
855 struct radv_subpass_attachment src_att = subpass->color_attachments[i];
856 struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i];
857
858 if (dest_att.attachment == VK_ATTACHMENT_UNUSED)
859 continue;
860
861 struct radv_image_view *dest_iview = cmd_buffer->state.attachments[dest_att.attachment].iview;
862 struct radv_image *dst_img = dest_iview->image;
863
864 if (radv_dcc_enabled(dst_img, dest_iview->base_mip)) {
865 VkImageSubresourceRange range = {
866 .aspectMask = dest_iview->aspect_mask,
867 .baseMipLevel = dest_iview->base_mip,
868 .levelCount = dest_iview->level_count,
869 .baseArrayLayer = dest_iview->base_layer,
870 .layerCount = dest_iview->layer_count,
871 };
872
873 radv_initialize_dcc(cmd_buffer, dst_img, &range, 0xffffffff);
874 cmd_buffer->state.attachments[dest_att.attachment].current_layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
875 }
876
877 struct radv_subpass resolve_subpass = {
878 .color_count = 2,
879 .color_attachments = (struct radv_subpass_attachment[]) { src_att, dest_att },
880 .depth_stencil_attachment = NULL,
881 };
882
883 radv_cmd_buffer_set_subpass(cmd_buffer, &resolve_subpass);
884
885 VkResult ret = build_resolve_pipeline(cmd_buffer->device, radv_format_meta_fs_key(dest_iview->vk_format));
886 if (ret != VK_SUCCESS) {
887 cmd_buffer->record_result = ret;
888 continue;
889 }
890
891 emit_resolve(cmd_buffer,
892 dest_iview->vk_format,
893 &(VkOffset2D) { 0, 0 },
894 &(VkExtent2D) { fb->width, fb->height });
895 }
896
897 radv_cmd_buffer_set_subpass(cmd_buffer, subpass);
898
899 radv_meta_restore(&saved_state, cmd_buffer);
900 }
901
902 /**
903 * Decompress CMask/FMask before resolving a multisampled source image inside a
904 * subpass.
905 */
906 void
radv_decompress_resolve_subpass_src(struct radv_cmd_buffer * cmd_buffer)907 radv_decompress_resolve_subpass_src(struct radv_cmd_buffer *cmd_buffer)
908 {
909 const struct radv_subpass *subpass = cmd_buffer->state.subpass;
910 struct radv_framebuffer *fb = cmd_buffer->state.framebuffer;
911 uint32_t layer_count = fb->layers;
912
913 if (subpass->view_mask)
914 layer_count = util_last_bit(subpass->view_mask);
915
916 for (uint32_t i = 0; i < subpass->color_count; ++i) {
917 struct radv_subpass_attachment src_att = subpass->color_attachments[i];
918 struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i];
919
920 if (dest_att.attachment == VK_ATTACHMENT_UNUSED)
921 continue;
922
923 struct radv_image_view *src_iview = cmd_buffer->state.attachments[src_att.attachment].iview;
924 struct radv_image *src_image = src_iview->image;
925
926 VkImageResolve2KHR region = {0};
927 region.sType = VK_STRUCTURE_TYPE_IMAGE_RESOLVE_2_KHR;
928 region.srcSubresource.aspectMask = src_iview->aspect_mask;
929 region.srcSubresource.mipLevel = 0;
930 region.srcSubresource.baseArrayLayer = src_iview->base_layer;
931 region.srcSubresource.layerCount = layer_count;
932
933 radv_decompress_resolve_src(cmd_buffer, src_image,
934 src_att.layout, ®ion);
935 }
936
937 if (subpass->ds_resolve_attachment) {
938 struct radv_subpass_attachment src_att = *subpass->depth_stencil_attachment;
939 struct radv_image_view *src_iview = fb->attachments[src_att.attachment];
940 struct radv_image *src_image = src_iview->image;
941
942 VkImageResolve2KHR region = {0};
943 region.sType = VK_STRUCTURE_TYPE_IMAGE_RESOLVE_2_KHR;
944 region.srcSubresource.aspectMask = src_iview->aspect_mask;
945 region.srcSubresource.mipLevel = 0;
946 region.srcSubresource.baseArrayLayer = src_iview->base_layer;
947 region.srcSubresource.layerCount = layer_count;
948
949 radv_decompress_resolve_src(cmd_buffer, src_image,
950 src_att.layout, ®ion);
951 }
952 }
953
954 static struct radv_sample_locations_state *
radv_get_resolve_sample_locations(struct radv_cmd_buffer * cmd_buffer)955 radv_get_resolve_sample_locations(struct radv_cmd_buffer *cmd_buffer)
956 {
957 struct radv_cmd_state *state = &cmd_buffer->state;
958 uint32_t subpass_id = radv_get_subpass_id(cmd_buffer);
959
960 for (uint32_t i = 0; i < state->num_subpass_sample_locs; i++) {
961 if (state->subpass_sample_locs[i].subpass_idx == subpass_id)
962 return &state->subpass_sample_locs[i].sample_location;
963 }
964
965 return NULL;
966 }
967
968 /**
969 * Decompress CMask/FMask before resolving a multisampled source image.
970 */
971 void
radv_decompress_resolve_src(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,VkImageLayout src_image_layout,const VkImageResolve2KHR * region)972 radv_decompress_resolve_src(struct radv_cmd_buffer *cmd_buffer,
973 struct radv_image *src_image,
974 VkImageLayout src_image_layout,
975 const VkImageResolve2KHR *region)
976 {
977 const uint32_t src_base_layer =
978 radv_meta_get_iview_layer(src_image, ®ion->srcSubresource,
979 ®ion->srcOffset);
980
981 VkImageMemoryBarrier barrier = {0};
982 barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
983 barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
984 barrier.oldLayout = src_image_layout;
985 barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
986 barrier.image = radv_image_to_handle(src_image);
987 barrier.subresourceRange = (VkImageSubresourceRange) {
988 .aspectMask = region->srcSubresource.aspectMask,
989 .baseMipLevel = region->srcSubresource.mipLevel,
990 .levelCount = 1,
991 .baseArrayLayer = src_base_layer,
992 .layerCount = region->srcSubresource.layerCount,
993 };
994
995 if (src_image->flags & VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT) {
996 /* If the depth/stencil image uses different sample
997 * locations, we need them during HTILE decompressions.
998 */
999 struct radv_sample_locations_state *sample_locs =
1000 radv_get_resolve_sample_locations(cmd_buffer);
1001
1002 barrier.pNext = &(VkSampleLocationsInfoEXT) {
1003 .sType = VK_STRUCTURE_TYPE_SAMPLE_LOCATIONS_INFO_EXT,
1004 .sampleLocationsPerPixel = sample_locs->per_pixel,
1005 .sampleLocationGridSize = sample_locs->grid_size,
1006 .sampleLocationsCount = sample_locs->count,
1007 .pSampleLocations = sample_locs->locations,
1008 };
1009 }
1010
1011 radv_CmdPipelineBarrier(radv_cmd_buffer_to_handle(cmd_buffer),
1012 VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
1013 VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
1014 false, 0, NULL, 0, NULL, 1, &barrier);
1015 }
1016