1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include <assert.h>
25 #include <stdbool.h>
26
27 #include "radv_meta.h"
28 #include "radv_private.h"
29 #include "sid.h"
30
31
32 static nir_shader *
build_dcc_decompress_compute_shader(struct radv_device * dev)33 build_dcc_decompress_compute_shader(struct radv_device *dev)
34 {
35 nir_builder b;
36 const struct glsl_type *buf_type = glsl_sampler_type(GLSL_SAMPLER_DIM_2D,
37 false,
38 false,
39 GLSL_TYPE_FLOAT);
40 const struct glsl_type *img_type = glsl_image_type(GLSL_SAMPLER_DIM_2D,
41 false,
42 GLSL_TYPE_FLOAT);
43 nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
44 b.shader->info.name = ralloc_strdup(b.shader, "dcc_decompress_compute");
45
46 /* We need at least 16/16/1 to cover an entire DCC block in a single workgroup. */
47 b.shader->info.cs.local_size[0] = 16;
48 b.shader->info.cs.local_size[1] = 16;
49 b.shader->info.cs.local_size[2] = 1;
50 nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
51 buf_type, "s_tex");
52 input_img->data.descriptor_set = 0;
53 input_img->data.binding = 0;
54
55 nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
56 img_type, "out_img");
57 output_img->data.descriptor_set = 0;
58 output_img->data.binding = 1;
59
60 nir_ssa_def *invoc_id = nir_load_local_invocation_id(&b);
61 nir_ssa_def *wg_id = nir_load_work_group_id(&b, 32);
62 nir_ssa_def *block_size = nir_imm_ivec4(&b,
63 b.shader->info.cs.local_size[0],
64 b.shader->info.cs.local_size[1],
65 b.shader->info.cs.local_size[2], 0);
66
67 nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
68 nir_ssa_def *input_img_deref = &nir_build_deref_var(&b, input_img)->dest.ssa;
69
70 nir_tex_instr *tex = nir_tex_instr_create(b.shader, 3);
71 tex->sampler_dim = GLSL_SAMPLER_DIM_2D;
72 tex->op = nir_texop_txf;
73 tex->src[0].src_type = nir_tex_src_coord;
74 tex->src[0].src = nir_src_for_ssa(nir_channels(&b, global_id, 3));
75 tex->src[1].src_type = nir_tex_src_lod;
76 tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
77 tex->src[2].src_type = nir_tex_src_texture_deref;
78 tex->src[2].src = nir_src_for_ssa(input_img_deref);
79 tex->dest_type = nir_type_float;
80 tex->is_array = false;
81 tex->coord_components = 2;
82
83 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
84 nir_builder_instr_insert(&b, &tex->instr);
85
86 nir_scoped_barrier(&b, NIR_SCOPE_WORKGROUP, NIR_SCOPE_WORKGROUP,
87 NIR_MEMORY_ACQ_REL, nir_var_mem_ssbo);
88
89 nir_ssa_def *outval = &tex->dest.ssa;
90 nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_deref_store);
91 store->num_components = 4;
92 store->src[0] = nir_src_for_ssa(&nir_build_deref_var(&b, output_img)->dest.ssa);
93 store->src[1] = nir_src_for_ssa(global_id);
94 store->src[2] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
95 store->src[3] = nir_src_for_ssa(outval);
96 store->src[4] = nir_src_for_ssa(nir_imm_int(&b, 0));
97
98 nir_builder_instr_insert(&b, &store->instr);
99 return b.shader;
100 }
101
102 static VkResult
create_dcc_compress_compute(struct radv_device * device)103 create_dcc_compress_compute(struct radv_device *device)
104 {
105 VkResult result = VK_SUCCESS;
106 struct radv_shader_module cs = { .nir = NULL };
107
108 cs.nir = build_dcc_decompress_compute_shader(device);
109
110 VkDescriptorSetLayoutCreateInfo ds_create_info = {
111 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
112 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
113 .bindingCount = 2,
114 .pBindings = (VkDescriptorSetLayoutBinding[]) {
115 {
116 .binding = 0,
117 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
118 .descriptorCount = 1,
119 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
120 .pImmutableSamplers = NULL
121 },
122 {
123 .binding = 1,
124 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
125 .descriptorCount = 1,
126 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
127 .pImmutableSamplers = NULL
128 },
129 }
130 };
131
132 result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
133 &ds_create_info,
134 &device->meta_state.alloc,
135 &device->meta_state.fast_clear_flush.dcc_decompress_compute_ds_layout);
136 if (result != VK_SUCCESS)
137 goto cleanup;
138
139
140 VkPipelineLayoutCreateInfo pl_create_info = {
141 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
142 .setLayoutCount = 1,
143 .pSetLayouts = &device->meta_state.fast_clear_flush.dcc_decompress_compute_ds_layout,
144 .pushConstantRangeCount = 1,
145 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 8},
146 };
147
148 result = radv_CreatePipelineLayout(radv_device_to_handle(device),
149 &pl_create_info,
150 &device->meta_state.alloc,
151 &device->meta_state.fast_clear_flush.dcc_decompress_compute_p_layout);
152 if (result != VK_SUCCESS)
153 goto cleanup;
154
155 /* compute shader */
156
157 VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
158 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
159 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
160 .module = radv_shader_module_to_handle(&cs),
161 .pName = "main",
162 .pSpecializationInfo = NULL,
163 };
164
165 VkComputePipelineCreateInfo vk_pipeline_info = {
166 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
167 .stage = pipeline_shader_stage,
168 .flags = 0,
169 .layout = device->meta_state.fast_clear_flush.dcc_decompress_compute_p_layout,
170 };
171
172 result = radv_CreateComputePipelines(radv_device_to_handle(device),
173 radv_pipeline_cache_to_handle(&device->meta_state.cache),
174 1, &vk_pipeline_info, NULL,
175 &device->meta_state.fast_clear_flush.dcc_decompress_compute_pipeline);
176 if (result != VK_SUCCESS)
177 goto cleanup;
178
179 cleanup:
180 ralloc_free(cs.nir);
181 return result;
182 }
183
184 static VkResult
create_pass(struct radv_device * device)185 create_pass(struct radv_device *device)
186 {
187 VkResult result;
188 VkDevice device_h = radv_device_to_handle(device);
189 const VkAllocationCallbacks *alloc = &device->meta_state.alloc;
190 VkAttachmentDescription attachment;
191
192 attachment.format = VK_FORMAT_UNDEFINED;
193 attachment.samples = 1;
194 attachment.loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
195 attachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
196 attachment.initialLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
197 attachment.finalLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
198
199 result = radv_CreateRenderPass(device_h,
200 &(VkRenderPassCreateInfo) {
201 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
202 .attachmentCount = 1,
203 .pAttachments = &attachment,
204 .subpassCount = 1,
205 .pSubpasses = &(VkSubpassDescription) {
206 .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
207 .inputAttachmentCount = 0,
208 .colorAttachmentCount = 1,
209 .pColorAttachments = (VkAttachmentReference[]) {
210 {
211 .attachment = 0,
212 .layout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
213 },
214 },
215 .pResolveAttachments = NULL,
216 .pDepthStencilAttachment = &(VkAttachmentReference) {
217 .attachment = VK_ATTACHMENT_UNUSED,
218 },
219 .preserveAttachmentCount = 0,
220 .pPreserveAttachments = NULL,
221 },
222 .dependencyCount = 2,
223 .pDependencies = (VkSubpassDependency[]) {
224 {
225 .srcSubpass = VK_SUBPASS_EXTERNAL,
226 .dstSubpass = 0,
227 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
228 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
229 .srcAccessMask = 0,
230 .dstAccessMask = 0,
231 .dependencyFlags = 0
232 },
233 {
234 .srcSubpass = 0,
235 .dstSubpass = VK_SUBPASS_EXTERNAL,
236 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
237 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
238 .srcAccessMask = 0,
239 .dstAccessMask = 0,
240 .dependencyFlags = 0
241 }
242 },
243 },
244 alloc,
245 &device->meta_state.fast_clear_flush.pass);
246
247 return result;
248 }
249
250 static VkResult
create_pipeline_layout(struct radv_device * device,VkPipelineLayout * layout)251 create_pipeline_layout(struct radv_device *device, VkPipelineLayout *layout)
252 {
253 VkPipelineLayoutCreateInfo pl_create_info = {
254 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
255 .setLayoutCount = 0,
256 .pSetLayouts = NULL,
257 .pushConstantRangeCount = 0,
258 .pPushConstantRanges = NULL,
259 };
260
261 return radv_CreatePipelineLayout(radv_device_to_handle(device),
262 &pl_create_info,
263 &device->meta_state.alloc,
264 layout);
265 }
266
267 static VkResult
create_pipeline(struct radv_device * device,VkShaderModule vs_module_h,VkPipelineLayout layout)268 create_pipeline(struct radv_device *device,
269 VkShaderModule vs_module_h,
270 VkPipelineLayout layout)
271 {
272 VkResult result;
273 VkDevice device_h = radv_device_to_handle(device);
274
275 struct radv_shader_module fs_module = {
276 .nir = radv_meta_build_nir_fs_noop(),
277 };
278
279 if (!fs_module.nir) {
280 /* XXX: Need more accurate error */
281 result = VK_ERROR_OUT_OF_HOST_MEMORY;
282 goto cleanup;
283 }
284
285 const VkPipelineShaderStageCreateInfo stages[2] = {
286 {
287 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
288 .stage = VK_SHADER_STAGE_VERTEX_BIT,
289 .module = vs_module_h,
290 .pName = "main",
291 },
292 {
293 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
294 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
295 .module = radv_shader_module_to_handle(&fs_module),
296 .pName = "main",
297 },
298 };
299
300 const VkPipelineVertexInputStateCreateInfo vi_state = {
301 .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
302 .vertexBindingDescriptionCount = 0,
303 .vertexAttributeDescriptionCount = 0,
304 };
305
306 const VkPipelineInputAssemblyStateCreateInfo ia_state = {
307 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
308 .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
309 .primitiveRestartEnable = false,
310 };
311
312 const VkPipelineColorBlendStateCreateInfo blend_state = {
313 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
314 .logicOpEnable = false,
315 .attachmentCount = 1,
316 .pAttachments = (VkPipelineColorBlendAttachmentState []) {
317 {
318 .colorWriteMask = VK_COLOR_COMPONENT_R_BIT |
319 VK_COLOR_COMPONENT_G_BIT |
320 VK_COLOR_COMPONENT_B_BIT |
321 VK_COLOR_COMPONENT_A_BIT,
322 },
323 }
324 };
325 const VkPipelineRasterizationStateCreateInfo rs_state = {
326 .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
327 .depthClampEnable = false,
328 .rasterizerDiscardEnable = false,
329 .polygonMode = VK_POLYGON_MODE_FILL,
330 .cullMode = VK_CULL_MODE_NONE,
331 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE,
332 };
333
334 result = radv_graphics_pipeline_create(device_h,
335 radv_pipeline_cache_to_handle(&device->meta_state.cache),
336 &(VkGraphicsPipelineCreateInfo) {
337 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
338 .stageCount = 2,
339 .pStages = stages,
340
341 .pVertexInputState = &vi_state,
342 .pInputAssemblyState = &ia_state,
343
344 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
345 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
346 .viewportCount = 1,
347 .scissorCount = 1,
348 },
349 .pRasterizationState = &rs_state,
350 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
351 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
352 .rasterizationSamples = 1,
353 .sampleShadingEnable = false,
354 .pSampleMask = NULL,
355 .alphaToCoverageEnable = false,
356 .alphaToOneEnable = false,
357 },
358 .pColorBlendState = &blend_state,
359 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
360 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
361 .dynamicStateCount = 2,
362 .pDynamicStates = (VkDynamicState[]) {
363 VK_DYNAMIC_STATE_VIEWPORT,
364 VK_DYNAMIC_STATE_SCISSOR,
365 },
366 },
367 .layout = layout,
368 .renderPass = device->meta_state.fast_clear_flush.pass,
369 .subpass = 0,
370 },
371 &(struct radv_graphics_pipeline_create_info) {
372 .use_rectlist = true,
373 .custom_blend_mode = V_028808_CB_ELIMINATE_FAST_CLEAR,
374 },
375 &device->meta_state.alloc,
376 &device->meta_state.fast_clear_flush.cmask_eliminate_pipeline);
377 if (result != VK_SUCCESS)
378 goto cleanup;
379
380 result = radv_graphics_pipeline_create(device_h,
381 radv_pipeline_cache_to_handle(&device->meta_state.cache),
382 &(VkGraphicsPipelineCreateInfo) {
383 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
384 .stageCount = 2,
385 .pStages = stages,
386
387 .pVertexInputState = &vi_state,
388 .pInputAssemblyState = &ia_state,
389
390 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
391 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
392 .viewportCount = 1,
393 .scissorCount = 1,
394 },
395 .pRasterizationState = &rs_state,
396 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
397 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
398 .rasterizationSamples = 1,
399 .sampleShadingEnable = false,
400 .pSampleMask = NULL,
401 .alphaToCoverageEnable = false,
402 .alphaToOneEnable = false,
403 },
404 .pColorBlendState = &blend_state,
405 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
406 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
407 .dynamicStateCount = 2,
408 .pDynamicStates = (VkDynamicState[]) {
409 VK_DYNAMIC_STATE_VIEWPORT,
410 VK_DYNAMIC_STATE_SCISSOR,
411 },
412 },
413 .layout = layout,
414 .renderPass = device->meta_state.fast_clear_flush.pass,
415 .subpass = 0,
416 },
417 &(struct radv_graphics_pipeline_create_info) {
418 .use_rectlist = true,
419 .custom_blend_mode = V_028808_CB_FMASK_DECOMPRESS,
420 },
421 &device->meta_state.alloc,
422 &device->meta_state.fast_clear_flush.fmask_decompress_pipeline);
423 if (result != VK_SUCCESS)
424 goto cleanup;
425
426 result = radv_graphics_pipeline_create(device_h,
427 radv_pipeline_cache_to_handle(&device->meta_state.cache),
428 &(VkGraphicsPipelineCreateInfo) {
429 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
430 .stageCount = 2,
431 .pStages = stages,
432
433 .pVertexInputState = &vi_state,
434 .pInputAssemblyState = &ia_state,
435
436 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
437 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
438 .viewportCount = 1,
439 .scissorCount = 1,
440 },
441 .pRasterizationState = &rs_state,
442 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
443 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
444 .rasterizationSamples = 1,
445 .sampleShadingEnable = false,
446 .pSampleMask = NULL,
447 .alphaToCoverageEnable = false,
448 .alphaToOneEnable = false,
449 },
450 .pColorBlendState = &blend_state,
451 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
452 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
453 .dynamicStateCount = 2,
454 .pDynamicStates = (VkDynamicState[]) {
455 VK_DYNAMIC_STATE_VIEWPORT,
456 VK_DYNAMIC_STATE_SCISSOR,
457 },
458 },
459 .layout = layout,
460 .renderPass = device->meta_state.fast_clear_flush.pass,
461 .subpass = 0,
462 },
463 &(struct radv_graphics_pipeline_create_info) {
464 .use_rectlist = true,
465 .custom_blend_mode = V_028808_CB_DCC_DECOMPRESS,
466 },
467 &device->meta_state.alloc,
468 &device->meta_state.fast_clear_flush.dcc_decompress_pipeline);
469 if (result != VK_SUCCESS)
470 goto cleanup;
471
472 goto cleanup;
473
474 cleanup:
475 ralloc_free(fs_module.nir);
476 return result;
477 }
478
479 void
radv_device_finish_meta_fast_clear_flush_state(struct radv_device * device)480 radv_device_finish_meta_fast_clear_flush_state(struct radv_device *device)
481 {
482 struct radv_meta_state *state = &device->meta_state;
483
484 radv_DestroyPipeline(radv_device_to_handle(device),
485 state->fast_clear_flush.dcc_decompress_pipeline,
486 &state->alloc);
487 radv_DestroyPipeline(radv_device_to_handle(device),
488 state->fast_clear_flush.fmask_decompress_pipeline,
489 &state->alloc);
490 radv_DestroyPipeline(radv_device_to_handle(device),
491 state->fast_clear_flush.cmask_eliminate_pipeline,
492 &state->alloc);
493 radv_DestroyRenderPass(radv_device_to_handle(device),
494 state->fast_clear_flush.pass, &state->alloc);
495 radv_DestroyPipelineLayout(radv_device_to_handle(device),
496 state->fast_clear_flush.p_layout,
497 &state->alloc);
498
499 radv_DestroyPipeline(radv_device_to_handle(device),
500 state->fast_clear_flush.dcc_decompress_compute_pipeline,
501 &state->alloc);
502 radv_DestroyPipelineLayout(radv_device_to_handle(device),
503 state->fast_clear_flush.dcc_decompress_compute_p_layout,
504 &state->alloc);
505 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
506 state->fast_clear_flush.dcc_decompress_compute_ds_layout,
507 &state->alloc);
508 }
509
510 static VkResult
radv_device_init_meta_fast_clear_flush_state_internal(struct radv_device * device)511 radv_device_init_meta_fast_clear_flush_state_internal(struct radv_device *device)
512 {
513 VkResult res = VK_SUCCESS;
514
515 mtx_lock(&device->meta_state.mtx);
516 if (device->meta_state.fast_clear_flush.cmask_eliminate_pipeline) {
517 mtx_unlock(&device->meta_state.mtx);
518 return VK_SUCCESS;
519 }
520
521 struct radv_shader_module vs_module = { .nir = radv_meta_build_nir_vs_generate_vertices() };
522 if (!vs_module.nir) {
523 /* XXX: Need more accurate error */
524 res = VK_ERROR_OUT_OF_HOST_MEMORY;
525 goto fail;
526 }
527
528 res = create_pass(device);
529 if (res != VK_SUCCESS)
530 goto fail;
531
532 res = create_pipeline_layout(device,
533 &device->meta_state.fast_clear_flush.p_layout);
534 if (res != VK_SUCCESS)
535 goto fail;
536
537 VkShaderModule vs_module_h = radv_shader_module_to_handle(&vs_module);
538 res = create_pipeline(device, vs_module_h,
539 device->meta_state.fast_clear_flush.p_layout);
540 if (res != VK_SUCCESS)
541 goto fail;
542
543 res = create_dcc_compress_compute(device);
544 if (res != VK_SUCCESS)
545 goto fail;
546
547 goto cleanup;
548
549 fail:
550 radv_device_finish_meta_fast_clear_flush_state(device);
551
552 cleanup:
553 ralloc_free(vs_module.nir);
554 mtx_unlock(&device->meta_state.mtx);
555
556 return res;
557 }
558
559
560 VkResult
radv_device_init_meta_fast_clear_flush_state(struct radv_device * device,bool on_demand)561 radv_device_init_meta_fast_clear_flush_state(struct radv_device *device, bool on_demand)
562 {
563 if (on_demand)
564 return VK_SUCCESS;
565
566 return radv_device_init_meta_fast_clear_flush_state_internal(device);
567 }
568
569 static void
radv_emit_set_predication_state_from_image(struct radv_cmd_buffer * cmd_buffer,struct radv_image * image,uint64_t pred_offset,bool value)570 radv_emit_set_predication_state_from_image(struct radv_cmd_buffer *cmd_buffer,
571 struct radv_image *image,
572 uint64_t pred_offset, bool value)
573 {
574 uint64_t va = 0;
575
576 if (value) {
577 va = radv_buffer_get_va(image->bo) + image->offset;
578 va += pred_offset;
579 }
580
581 si_emit_set_predication_state(cmd_buffer, true, va);
582 }
583
584 static void
radv_process_color_image_layer(struct radv_cmd_buffer * cmd_buffer,struct radv_image * image,const VkImageSubresourceRange * range,int level,int layer,bool flush_cb)585 radv_process_color_image_layer(struct radv_cmd_buffer *cmd_buffer,
586 struct radv_image *image,
587 const VkImageSubresourceRange *range,
588 int level, int layer, bool flush_cb)
589 {
590 struct radv_device *device = cmd_buffer->device;
591 struct radv_image_view iview;
592 uint32_t width, height;
593
594 width = radv_minify(image->info.width, range->baseMipLevel + level);
595 height = radv_minify(image->info.height, range->baseMipLevel + level);
596
597 radv_image_view_init(&iview, device,
598 &(VkImageViewCreateInfo) {
599 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
600 .image = radv_image_to_handle(image),
601 .viewType = radv_meta_get_view_type(image),
602 .format = image->vk_format,
603 .subresourceRange = {
604 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
605 .baseMipLevel = range->baseMipLevel + level,
606 .levelCount = 1,
607 .baseArrayLayer = range->baseArrayLayer + layer,
608 .layerCount = 1,
609 },
610 }, NULL);
611
612 VkFramebuffer fb_h;
613 radv_CreateFramebuffer(radv_device_to_handle(device),
614 &(VkFramebufferCreateInfo) {
615 .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
616 .attachmentCount = 1,
617 .pAttachments = (VkImageView[]) {
618 radv_image_view_to_handle(&iview)
619 },
620 .width = width,
621 .height = height,
622 .layers = 1
623 }, &cmd_buffer->pool->alloc, &fb_h);
624
625 radv_cmd_buffer_begin_render_pass(cmd_buffer,
626 &(VkRenderPassBeginInfo) {
627 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
628 .renderPass = device->meta_state.fast_clear_flush.pass,
629 .framebuffer = fb_h,
630 .renderArea = {
631 .offset = { 0, 0, },
632 .extent = { width, height, }
633 },
634 .clearValueCount = 0,
635 .pClearValues = NULL,
636 });
637
638 radv_cmd_buffer_set_subpass(cmd_buffer,
639 &cmd_buffer->state.pass->subpasses[0]);
640
641 if (flush_cb)
642 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB |
643 RADV_CMD_FLAG_FLUSH_AND_INV_CB_META;
644
645 radv_CmdDraw(radv_cmd_buffer_to_handle(cmd_buffer), 3, 1, 0, 0);
646
647 if (flush_cb)
648 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB |
649 RADV_CMD_FLAG_FLUSH_AND_INV_CB_META;
650
651 radv_cmd_buffer_end_render_pass(cmd_buffer);
652
653 radv_DestroyFramebuffer(radv_device_to_handle(device), fb_h,
654 &cmd_buffer->pool->alloc);
655 }
656
657 static void
radv_process_color_image(struct radv_cmd_buffer * cmd_buffer,struct radv_image * image,const VkImageSubresourceRange * subresourceRange,bool decompress_dcc)658 radv_process_color_image(struct radv_cmd_buffer *cmd_buffer,
659 struct radv_image *image,
660 const VkImageSubresourceRange *subresourceRange,
661 bool decompress_dcc)
662 {
663 struct radv_device *device = cmd_buffer->device;
664 struct radv_meta_saved_state saved_state;
665 bool flush_cb = false;
666 VkPipeline *pipeline;
667
668 if (decompress_dcc && radv_dcc_enabled(image, subresourceRange->baseMipLevel)) {
669 pipeline = &device->meta_state.fast_clear_flush.dcc_decompress_pipeline;
670 } else if (radv_image_has_fmask(image) && !image->tc_compatible_cmask) {
671 pipeline = &device->meta_state.fast_clear_flush.fmask_decompress_pipeline;
672 } else {
673 pipeline = &device->meta_state.fast_clear_flush.cmask_eliminate_pipeline;
674 }
675
676 if (!*pipeline) {
677 VkResult ret;
678
679 ret = radv_device_init_meta_fast_clear_flush_state_internal(device);
680 if (ret != VK_SUCCESS) {
681 cmd_buffer->record_result = ret;
682 return;
683 }
684 }
685
686 if (pipeline == &device->meta_state.fast_clear_flush.dcc_decompress_pipeline ||
687 pipeline == &device->meta_state.fast_clear_flush.fmask_decompress_pipeline) {
688 /* Flushing CB is required before and after DCC_DECOMPRESS or
689 * FMASK_DECOMPRESS.
690 */
691 flush_cb = true;
692 }
693
694 radv_meta_save(&saved_state, cmd_buffer,
695 RADV_META_SAVE_GRAPHICS_PIPELINE |
696 RADV_META_SAVE_PASS);
697
698 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
699 VK_PIPELINE_BIND_POINT_GRAPHICS, *pipeline);
700
701 for (uint32_t l = 0; l < radv_get_levelCount(image, subresourceRange); ++l) {
702 uint32_t width, height;
703
704 /* Do not decompress levels without DCC. */
705 if (decompress_dcc &&
706 !radv_dcc_enabled(image, subresourceRange->baseMipLevel + l))
707 continue;
708
709 width = radv_minify(image->info.width,
710 subresourceRange->baseMipLevel + l);
711 height = radv_minify(image->info.height,
712 subresourceRange->baseMipLevel + l);
713
714 radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
715 &(VkViewport) {
716 .x = 0,
717 .y = 0,
718 .width = width,
719 .height = height,
720 .minDepth = 0.0f,
721 .maxDepth = 1.0f
722 });
723
724 radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
725 &(VkRect2D) {
726 .offset = { 0, 0 },
727 .extent = { width, height },
728 });
729
730 for (uint32_t s = 0; s < radv_get_layerCount(image, subresourceRange); s++) {
731 radv_process_color_image_layer(cmd_buffer, image,
732 subresourceRange, l, s,
733 flush_cb);
734 }
735 }
736
737 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB |
738 RADV_CMD_FLAG_FLUSH_AND_INV_CB_META;
739
740 radv_meta_restore(&saved_state, cmd_buffer);
741 }
742
743 static void
radv_emit_color_decompress(struct radv_cmd_buffer * cmd_buffer,struct radv_image * image,const VkImageSubresourceRange * subresourceRange,bool decompress_dcc)744 radv_emit_color_decompress(struct radv_cmd_buffer *cmd_buffer,
745 struct radv_image *image,
746 const VkImageSubresourceRange *subresourceRange,
747 bool decompress_dcc)
748 {
749 bool old_predicating = false;
750
751 assert(cmd_buffer->queue_family_index == RADV_QUEUE_GENERAL);
752
753 if (radv_dcc_enabled(image, subresourceRange->baseMipLevel)) {
754 uint64_t pred_offset = decompress_dcc ? image->dcc_pred_offset :
755 image->fce_pred_offset;
756 pred_offset += 8 * subresourceRange->baseMipLevel;
757
758 old_predicating = cmd_buffer->state.predicating;
759
760 radv_emit_set_predication_state_from_image(cmd_buffer, image, pred_offset, true);
761 cmd_buffer->state.predicating = true;
762 }
763
764 radv_process_color_image(cmd_buffer, image, subresourceRange,
765 decompress_dcc);
766
767 if (radv_dcc_enabled(image, subresourceRange->baseMipLevel)) {
768 uint64_t pred_offset = decompress_dcc ? image->dcc_pred_offset :
769 image->fce_pred_offset;
770 pred_offset += 8 * subresourceRange->baseMipLevel;
771
772 cmd_buffer->state.predicating = old_predicating;
773
774 radv_emit_set_predication_state_from_image(cmd_buffer, image, pred_offset, false);
775
776 if (cmd_buffer->state.predication_type != -1) {
777 /* Restore previous conditional rendering user state. */
778 si_emit_set_predication_state(cmd_buffer,
779 cmd_buffer->state.predication_type,
780 cmd_buffer->state.predication_va);
781 }
782 }
783
784 if (radv_dcc_enabled(image, subresourceRange->baseMipLevel)) {
785 /* Clear the image's fast-clear eliminate predicate because
786 * FMASK and DCC also imply a fast-clear eliminate.
787 */
788 radv_update_fce_metadata(cmd_buffer, image, subresourceRange, false);
789
790 /* Mark the image as being decompressed. */
791 if (decompress_dcc)
792 radv_update_dcc_metadata(cmd_buffer, image, subresourceRange, false);
793 }
794 }
795
796 void
radv_fast_clear_flush_image_inplace(struct radv_cmd_buffer * cmd_buffer,struct radv_image * image,const VkImageSubresourceRange * subresourceRange)797 radv_fast_clear_flush_image_inplace(struct radv_cmd_buffer *cmd_buffer,
798 struct radv_image *image,
799 const VkImageSubresourceRange *subresourceRange)
800 {
801 struct radv_barrier_data barrier = {0};
802
803 if (radv_image_has_fmask(image)) {
804 barrier.layout_transitions.fmask_decompress = 1;
805 } else {
806 barrier.layout_transitions.fast_clear_eliminate = 1;
807 }
808 radv_describe_layout_transition(cmd_buffer, &barrier);
809
810 radv_emit_color_decompress(cmd_buffer, image, subresourceRange, false);
811 }
812
813 static void
radv_decompress_dcc_gfx(struct radv_cmd_buffer * cmd_buffer,struct radv_image * image,const VkImageSubresourceRange * subresourceRange)814 radv_decompress_dcc_gfx(struct radv_cmd_buffer *cmd_buffer,
815 struct radv_image *image,
816 const VkImageSubresourceRange *subresourceRange)
817 {
818 radv_emit_color_decompress(cmd_buffer, image, subresourceRange, true);
819 }
820
821 static void
radv_decompress_dcc_compute(struct radv_cmd_buffer * cmd_buffer,struct radv_image * image,const VkImageSubresourceRange * subresourceRange)822 radv_decompress_dcc_compute(struct radv_cmd_buffer *cmd_buffer,
823 struct radv_image *image,
824 const VkImageSubresourceRange *subresourceRange)
825 {
826 struct radv_meta_saved_state saved_state;
827 struct radv_image_view load_iview = {0};
828 struct radv_image_view store_iview = {0};
829 struct radv_device *device = cmd_buffer->device;
830
831 /* This assumes the image is 2d with 1 layer */
832 struct radv_cmd_state *state = &cmd_buffer->state;
833
834 state->flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB |
835 RADV_CMD_FLAG_FLUSH_AND_INV_CB_META;
836
837 if (!cmd_buffer->device->meta_state.fast_clear_flush.cmask_eliminate_pipeline) {
838 VkResult ret = radv_device_init_meta_fast_clear_flush_state_internal(cmd_buffer->device);
839 if (ret != VK_SUCCESS) {
840 cmd_buffer->record_result = ret;
841 return;
842 }
843 }
844
845 radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_DESCRIPTORS |
846 RADV_META_SAVE_COMPUTE_PIPELINE);
847
848 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
849 VK_PIPELINE_BIND_POINT_COMPUTE,
850 device->meta_state.fast_clear_flush.dcc_decompress_compute_pipeline);
851
852 for (uint32_t l = 0; l < radv_get_levelCount(image, subresourceRange); l++) {
853 uint32_t width, height;
854
855 /* Do not decompress levels without DCC. */
856 if (!radv_dcc_enabled(image, subresourceRange->baseMipLevel + l))
857 continue;
858
859 width = radv_minify(image->info.width,
860 subresourceRange->baseMipLevel + l);
861 height = radv_minify(image->info.height,
862 subresourceRange->baseMipLevel + l);
863
864 for (uint32_t s = 0; s < radv_get_layerCount(image, subresourceRange); s++) {
865 radv_image_view_init(&load_iview, cmd_buffer->device,
866 &(VkImageViewCreateInfo) {
867 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
868 .image = radv_image_to_handle(image),
869 .viewType = VK_IMAGE_VIEW_TYPE_2D,
870 .format = image->vk_format,
871 .subresourceRange = {
872 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
873 .baseMipLevel = subresourceRange->baseMipLevel + l,
874 .levelCount = 1,
875 .baseArrayLayer = subresourceRange->baseArrayLayer + s,
876 .layerCount = 1
877 },
878 }, NULL);
879 radv_image_view_init(&store_iview, cmd_buffer->device,
880 &(VkImageViewCreateInfo) {
881 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
882 .image = radv_image_to_handle(image),
883 .viewType = VK_IMAGE_VIEW_TYPE_2D,
884 .format = image->vk_format,
885 .subresourceRange = {
886 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
887 .baseMipLevel = subresourceRange->baseMipLevel + l,
888 .levelCount = 1,
889 .baseArrayLayer = subresourceRange->baseArrayLayer + s,
890 .layerCount = 1
891 },
892 }, &(struct radv_image_view_extra_create_info) {
893 .disable_compression = true
894 });
895
896 radv_meta_push_descriptor_set(cmd_buffer,
897 VK_PIPELINE_BIND_POINT_COMPUTE,
898 device->meta_state.fast_clear_flush.dcc_decompress_compute_p_layout,
899 0, /* set */
900 2, /* descriptorWriteCount */
901 (VkWriteDescriptorSet[]) {
902 {
903 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
904 .dstBinding = 0,
905 .dstArrayElement = 0,
906 .descriptorCount = 1,
907 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
908 .pImageInfo = (VkDescriptorImageInfo[]) {
909 {
910 .sampler = VK_NULL_HANDLE,
911 .imageView = radv_image_view_to_handle(&load_iview),
912 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
913 },
914 }
915 },
916 {
917 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
918 .dstBinding = 1,
919 .dstArrayElement = 0,
920 .descriptorCount = 1,
921 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
922 .pImageInfo = (VkDescriptorImageInfo[]) {
923 {
924 .sampler = VK_NULL_HANDLE,
925 .imageView = radv_image_view_to_handle(&store_iview),
926 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
927 },
928 }
929 }
930 });
931
932 radv_unaligned_dispatch(cmd_buffer, width, height, 1);
933 }
934 }
935
936 /* Mark this image as actually being decompressed. */
937 radv_update_dcc_metadata(cmd_buffer, image, subresourceRange, false);
938
939 /* The fill buffer below does its own saving */
940 radv_meta_restore(&saved_state, cmd_buffer);
941
942 state->flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH |
943 RADV_CMD_FLAG_INV_VCACHE;
944
945
946 /* Initialize the DCC metadata as "fully expanded". */
947 radv_initialize_dcc(cmd_buffer, image, subresourceRange, 0xffffffff);
948 }
949
950 void
radv_decompress_dcc(struct radv_cmd_buffer * cmd_buffer,struct radv_image * image,const VkImageSubresourceRange * subresourceRange)951 radv_decompress_dcc(struct radv_cmd_buffer *cmd_buffer,
952 struct radv_image *image,
953 const VkImageSubresourceRange *subresourceRange)
954 {
955 struct radv_barrier_data barrier = {0};
956
957 barrier.layout_transitions.dcc_decompress = 1;
958 radv_describe_layout_transition(cmd_buffer, &barrier);
959
960 if (cmd_buffer->queue_family_index == RADV_QUEUE_GENERAL)
961 radv_decompress_dcc_gfx(cmd_buffer, image, subresourceRange);
962 else
963 radv_decompress_dcc_compute(cmd_buffer, image, subresourceRange);
964 }
965