• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright © 2016 Dave Airlie
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  */
23 
24 
25 #include <assert.h>
26 #include <stdbool.h>
27 
28 #include "radv_meta.h"
29 #include "radv_private.h"
30 #include "nir/nir_builder.h"
31 #include "sid.h"
32 #include "vk_format.h"
33 
34 static nir_shader *
build_nir_vertex_shader(void)35 build_nir_vertex_shader(void)
36 {
37 	const struct glsl_type *vec4 = glsl_vec4_type();
38 	nir_builder b;
39 
40 	nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_VERTEX, NULL);
41 	b.shader->info.name = ralloc_strdup(b.shader, "meta_resolve_vs");
42 
43 	nir_variable *pos_out = nir_variable_create(b.shader, nir_var_shader_out,
44 						    vec4, "gl_Position");
45 	pos_out->data.location = VARYING_SLOT_POS;
46 
47 	nir_ssa_def *outvec = radv_meta_gen_rect_vertices(&b);
48 
49 	nir_store_var(&b, pos_out, outvec, 0xf);
50 	return b.shader;
51 }
52 
53 static nir_shader *
build_resolve_fragment_shader(struct radv_device * dev,bool is_integer,int samples)54 build_resolve_fragment_shader(struct radv_device *dev, bool is_integer, int samples)
55 {
56 	nir_builder b;
57 	char name[64];
58 	const struct glsl_type *vec4 = glsl_vec4_type();
59 	const struct glsl_type *sampler_type = glsl_sampler_type(GLSL_SAMPLER_DIM_MS,
60 								 false,
61 								 false,
62 								 GLSL_TYPE_FLOAT);
63 
64 	snprintf(name, 64, "meta_resolve_fs-%d-%s", samples, is_integer ? "int" : "float");
65 	nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
66 	b.shader->info.name = ralloc_strdup(b.shader, name);
67 
68 	nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
69 						      sampler_type, "s_tex");
70 	input_img->data.descriptor_set = 0;
71 	input_img->data.binding = 0;
72 
73 	nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out,
74 						      vec4, "f_color");
75 	color_out->data.location = FRAG_RESULT_DATA0;
76 
77 	nir_ssa_def *pos_in = nir_channels(&b, nir_load_frag_coord(&b), 0x3);
78 	nir_intrinsic_instr *src_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
79 	nir_intrinsic_set_base(src_offset, 0);
80 	nir_intrinsic_set_range(src_offset, 8);
81 	src_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
82 	src_offset->num_components = 2;
83 	nir_ssa_dest_init(&src_offset->instr, &src_offset->dest, 2, 32, "src_offset");
84 	nir_builder_instr_insert(&b, &src_offset->instr);
85 
86 	nir_ssa_def *pos_int = nir_f2i32(&b, pos_in);
87 
88 	nir_ssa_def *img_coord = nir_channels(&b, nir_iadd(&b, pos_int, &src_offset->dest.ssa), 0x3);
89 	nir_variable *color = nir_local_variable_create(b.impl, glsl_vec4_type(), "color");
90 
91 	radv_meta_build_resolve_shader_core(&b, is_integer, samples, input_img,
92 	                                    color, img_coord);
93 
94 	nir_ssa_def *outval = nir_load_var(&b, color);
95 	nir_store_var(&b, color_out, outval, 0xf);
96 	return b.shader;
97 }
98 
99 
100 static VkResult
create_layout(struct radv_device * device)101 create_layout(struct radv_device *device)
102 {
103 	VkResult result;
104 	/*
105 	 * one descriptors for the image being sampled
106 	 */
107 	VkDescriptorSetLayoutCreateInfo ds_create_info = {
108 		.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
109 		.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
110 		.bindingCount = 1,
111 		.pBindings = (VkDescriptorSetLayoutBinding[]) {
112 			{
113 				.binding = 0,
114 				.descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
115 				.descriptorCount = 1,
116 				.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
117 				.pImmutableSamplers = NULL
118 			},
119 		}
120 	};
121 
122 	result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
123 						&ds_create_info,
124 						&device->meta_state.alloc,
125 						&device->meta_state.resolve_fragment.ds_layout);
126 	if (result != VK_SUCCESS)
127 		goto fail;
128 
129 
130 	VkPipelineLayoutCreateInfo pl_create_info = {
131 		.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
132 		.setLayoutCount = 1,
133 		.pSetLayouts = &device->meta_state.resolve_fragment.ds_layout,
134 		.pushConstantRangeCount = 1,
135 		.pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_FRAGMENT_BIT, 0, 8},
136 	};
137 
138 	result = radv_CreatePipelineLayout(radv_device_to_handle(device),
139 					  &pl_create_info,
140 					  &device->meta_state.alloc,
141 					  &device->meta_state.resolve_fragment.p_layout);
142 	if (result != VK_SUCCESS)
143 		goto fail;
144 	return VK_SUCCESS;
145 fail:
146 	return result;
147 }
148 
149 static const VkPipelineVertexInputStateCreateInfo normal_vi_create_info = {
150 	.sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
151 	.vertexBindingDescriptionCount = 0,
152 	.vertexAttributeDescriptionCount = 0,
153 };
154 
155 static VkResult
create_resolve_pipeline(struct radv_device * device,int samples_log2,VkFormat format)156 create_resolve_pipeline(struct radv_device *device,
157 			int samples_log2,
158 			VkFormat format)
159 {
160 	mtx_lock(&device->meta_state.mtx);
161 
162 	unsigned fs_key = radv_format_meta_fs_key(format);
163 	VkPipeline *pipeline = &device->meta_state.resolve_fragment.rc[samples_log2].pipeline[fs_key];
164 	if (*pipeline) {
165 		mtx_unlock(&device->meta_state.mtx);
166 		return VK_SUCCESS;
167 	}
168 
169 	VkResult result;
170 	bool is_integer = false;
171 	uint32_t samples = 1 << samples_log2;
172 	const VkPipelineVertexInputStateCreateInfo *vi_create_info;
173 	vi_create_info = &normal_vi_create_info;
174 	if (vk_format_is_int(format))
175 		is_integer = true;
176 
177 	struct radv_shader_module fs = { .nir = NULL };
178 	fs.nir = build_resolve_fragment_shader(device, is_integer, samples);
179 	struct radv_shader_module vs = {
180 		.nir = build_nir_vertex_shader(),
181 	};
182 
183 	VkRenderPass *rp = &device->meta_state.resolve_fragment.rc[samples_log2].render_pass[fs_key][0];
184 
185 	assert(!*rp);
186 
187 	VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
188 		{
189 			.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
190 			.stage = VK_SHADER_STAGE_VERTEX_BIT,
191 			.module = radv_shader_module_to_handle(&vs),
192 			.pName = "main",
193 			.pSpecializationInfo = NULL
194 		}, {
195 			.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
196 			.stage = VK_SHADER_STAGE_FRAGMENT_BIT,
197 			.module = radv_shader_module_to_handle(&fs),
198 			.pName = "main",
199 			.pSpecializationInfo = NULL
200 		},
201 	};
202 
203 
204 	for (unsigned dst_layout = 0; dst_layout < RADV_META_DST_LAYOUT_COUNT; ++dst_layout) {
205 		VkImageLayout layout = radv_meta_dst_layout_to_layout(dst_layout);
206 		result = radv_CreateRenderPass(radv_device_to_handle(device),
207 					&(VkRenderPassCreateInfo) {
208 						.sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
209 						.attachmentCount = 1,
210 						.pAttachments = &(VkAttachmentDescription) {
211 							.format = format,
212 							.loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
213 							.storeOp = VK_ATTACHMENT_STORE_OP_STORE,
214 							.initialLayout = layout,
215 							.finalLayout = layout,
216 						},
217 						.subpassCount = 1,
218 						.pSubpasses = &(VkSubpassDescription) {
219 							.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
220 							.inputAttachmentCount = 0,
221 							.colorAttachmentCount = 1,
222 							.pColorAttachments = &(VkAttachmentReference) {
223 								.attachment = 0,
224 								.layout = layout,
225 							},
226 						.pResolveAttachments = NULL,
227 						.pDepthStencilAttachment = &(VkAttachmentReference) {
228 							.attachment = VK_ATTACHMENT_UNUSED,
229 							.layout = VK_IMAGE_LAYOUT_GENERAL,
230 						},
231 						.preserveAttachmentCount = 0,
232 						.pPreserveAttachments = NULL,
233 					},
234 					.dependencyCount = 2,
235 					.pDependencies = (VkSubpassDependency[]) {
236 						{
237 							.srcSubpass = VK_SUBPASS_EXTERNAL,
238 							.dstSubpass = 0,
239 							.srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
240 							.dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
241 							.srcAccessMask = 0,
242 							.dstAccessMask = 0,
243 							.dependencyFlags = 0
244 						},
245 						{
246 							.srcSubpass = 0,
247 							.dstSubpass = VK_SUBPASS_EXTERNAL,
248 							.srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
249 							.dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
250 							.srcAccessMask = 0,
251 							.dstAccessMask = 0,
252 							.dependencyFlags = 0
253 						}
254 					},
255 				}, &device->meta_state.alloc, rp + dst_layout);
256 	}
257 
258 
259 	const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
260 		.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
261 		.stageCount = ARRAY_SIZE(pipeline_shader_stages),
262 		.pStages = pipeline_shader_stages,
263 		.pVertexInputState = vi_create_info,
264 		.pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
265 			.sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
266 			.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
267 			.primitiveRestartEnable = false,
268 		},
269 		.pViewportState = &(VkPipelineViewportStateCreateInfo) {
270 			.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
271 			.viewportCount = 1,
272 			.scissorCount = 1,
273 		},
274 		.pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
275 			.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
276 			.rasterizerDiscardEnable = false,
277 			.polygonMode = VK_POLYGON_MODE_FILL,
278 			.cullMode = VK_CULL_MODE_NONE,
279 			.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
280 		},
281 		.pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
282 			.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
283 			.rasterizationSamples = 1,
284 			.sampleShadingEnable = false,
285 			.pSampleMask = (VkSampleMask[]) { UINT32_MAX },
286 		},
287 		.pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
288 			.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
289 			.attachmentCount = 1,
290 			.pAttachments = (VkPipelineColorBlendAttachmentState []) {
291 				{ .colorWriteMask =
292 				  VK_COLOR_COMPONENT_A_BIT |
293 				  VK_COLOR_COMPONENT_R_BIT |
294 				  VK_COLOR_COMPONENT_G_BIT |
295 				  VK_COLOR_COMPONENT_B_BIT },
296 			}
297 		},
298 		.pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
299 			.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
300 			.dynamicStateCount = 9,
301 			.pDynamicStates = (VkDynamicState[]) {
302 				VK_DYNAMIC_STATE_VIEWPORT,
303 				VK_DYNAMIC_STATE_SCISSOR,
304 				VK_DYNAMIC_STATE_LINE_WIDTH,
305 				VK_DYNAMIC_STATE_DEPTH_BIAS,
306 				VK_DYNAMIC_STATE_BLEND_CONSTANTS,
307 				VK_DYNAMIC_STATE_DEPTH_BOUNDS,
308 				VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
309 				VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
310 				VK_DYNAMIC_STATE_STENCIL_REFERENCE,
311 			},
312 		},
313 		.flags = 0,
314 		.layout = device->meta_state.resolve_fragment.p_layout,
315 		.renderPass = *rp,
316 		.subpass = 0,
317 	};
318 
319 	const struct radv_graphics_pipeline_create_info radv_pipeline_info = {
320 		.use_rectlist = true
321 	};
322 
323 	result = radv_graphics_pipeline_create(radv_device_to_handle(device),
324 					       radv_pipeline_cache_to_handle(&device->meta_state.cache),
325 					       &vk_pipeline_info, &radv_pipeline_info,
326 					       &device->meta_state.alloc,
327 					       pipeline);
328 	ralloc_free(vs.nir);
329 	ralloc_free(fs.nir);
330 
331 	mtx_unlock(&device->meta_state.mtx);
332 	return result;
333 }
334 
335 enum {
336 	DEPTH_RESOLVE,
337 	STENCIL_RESOLVE
338 };
339 
340 static const char *
get_resolve_mode_str(VkResolveModeFlagBits resolve_mode)341 get_resolve_mode_str(VkResolveModeFlagBits resolve_mode)
342 {
343 	switch (resolve_mode) {
344 	case VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR:
345 		return "zero";
346 	case VK_RESOLVE_MODE_AVERAGE_BIT_KHR:
347 		return "average";
348 	case VK_RESOLVE_MODE_MIN_BIT_KHR:
349 		return "min";
350 	case VK_RESOLVE_MODE_MAX_BIT_KHR:
351 		return "max";
352 	default:
353 		unreachable("invalid resolve mode");
354 	}
355 }
356 
357 static nir_shader *
build_depth_stencil_resolve_fragment_shader(struct radv_device * dev,int samples,int index,VkResolveModeFlagBits resolve_mode)358 build_depth_stencil_resolve_fragment_shader(struct radv_device *dev, int samples,
359 					    int index,
360 					    VkResolveModeFlagBits resolve_mode)
361 {
362 	nir_builder b;
363 	char name[64];
364 	const struct glsl_type *vec4 = glsl_vec4_type();
365 	const struct glsl_type *sampler_type = glsl_sampler_type(GLSL_SAMPLER_DIM_2D,
366 								 false,
367 								 false,
368 								 GLSL_TYPE_FLOAT);
369 
370 	snprintf(name, 64, "meta_resolve_fs_%s-%s-%d",
371 		 index == DEPTH_RESOLVE ? "depth" : "stencil",
372 		 get_resolve_mode_str(resolve_mode), samples);
373 
374 	nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
375 	b.shader->info.name = ralloc_strdup(b.shader, name);
376 
377 	nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
378 						      sampler_type, "s_tex");
379 	input_img->data.descriptor_set = 0;
380 	input_img->data.binding = 0;
381 
382 	nir_variable *fs_out = nir_variable_create(b.shader,
383 						   nir_var_shader_out, vec4,
384 						   "f_out");
385 	fs_out->data.location =
386 		index == DEPTH_RESOLVE ? FRAG_RESULT_DEPTH : FRAG_RESULT_STENCIL;
387 
388 	nir_ssa_def *pos_in = nir_channels(&b, nir_load_frag_coord(&b), 0x3);
389 
390 	nir_intrinsic_instr *src_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
391 	nir_intrinsic_set_base(src_offset, 0);
392 	nir_intrinsic_set_range(src_offset, 8);
393 	src_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
394 	src_offset->num_components = 2;
395 	nir_ssa_dest_init(&src_offset->instr, &src_offset->dest, 2, 32, "src_offset");
396 	nir_builder_instr_insert(&b, &src_offset->instr);
397 
398 	nir_ssa_def *pos_int = nir_f2i32(&b, pos_in);
399 
400 	nir_ssa_def *img_coord = nir_channels(&b, nir_iadd(&b, pos_int, &src_offset->dest.ssa), 0x3);
401 
402 	nir_ssa_def *input_img_deref = &nir_build_deref_var(&b, input_img)->dest.ssa;
403 
404 	nir_alu_type type = index == DEPTH_RESOLVE ? nir_type_float : nir_type_uint;
405 
406 	nir_tex_instr *tex = nir_tex_instr_create(b.shader, 3);
407 	tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
408 	tex->op = nir_texop_txf_ms;
409 	tex->src[0].src_type = nir_tex_src_coord;
410 	tex->src[0].src = nir_src_for_ssa(img_coord);
411 	tex->src[1].src_type = nir_tex_src_ms_index;
412 	tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
413 	tex->src[2].src_type = nir_tex_src_texture_deref;
414 	tex->src[2].src = nir_src_for_ssa(input_img_deref);
415 	tex->dest_type = type;
416 	tex->is_array = false;
417 	tex->coord_components = 2;
418 
419 	nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
420 	nir_builder_instr_insert(&b, &tex->instr);
421 
422 	nir_ssa_def *outval = &tex->dest.ssa;
423 
424 	if (resolve_mode != VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR) {
425 		for (int i = 1; i < samples; i++) {
426 			nir_tex_instr *tex_add = nir_tex_instr_create(b.shader, 3);
427 			tex_add->sampler_dim = GLSL_SAMPLER_DIM_MS;
428 			tex_add->op = nir_texop_txf_ms;
429 			tex_add->src[0].src_type = nir_tex_src_coord;
430 			tex_add->src[0].src = nir_src_for_ssa(img_coord);
431 			tex_add->src[1].src_type = nir_tex_src_ms_index;
432 			tex_add->src[1].src = nir_src_for_ssa(nir_imm_int(&b, i));
433 			tex_add->src[2].src_type = nir_tex_src_texture_deref;
434 			tex_add->src[2].src = nir_src_for_ssa(input_img_deref);
435 			tex_add->dest_type = type;
436 			tex_add->is_array = false;
437 			tex_add->coord_components = 2;
438 
439 			nir_ssa_dest_init(&tex_add->instr, &tex_add->dest, 4, 32, "tex");
440 			nir_builder_instr_insert(&b, &tex_add->instr);
441 
442 			switch (resolve_mode) {
443 			case VK_RESOLVE_MODE_AVERAGE_BIT_KHR:
444 				assert(index == DEPTH_RESOLVE);
445 				outval = nir_fadd(&b, outval, &tex_add->dest.ssa);
446 				break;
447 			case VK_RESOLVE_MODE_MIN_BIT_KHR:
448 				if (index == DEPTH_RESOLVE)
449 					outval = nir_fmin(&b, outval, &tex_add->dest.ssa);
450 				else
451 					outval = nir_umin(&b, outval, &tex_add->dest.ssa);
452 				break;
453 			case VK_RESOLVE_MODE_MAX_BIT_KHR:
454 				if (index == DEPTH_RESOLVE)
455 					outval = nir_fmax(&b, outval, &tex_add->dest.ssa);
456 				else
457 					outval = nir_umax(&b, outval, &tex_add->dest.ssa);
458 				break;
459 			default:
460 				unreachable("invalid resolve mode");
461 			}
462 		}
463 
464 		if (resolve_mode == VK_RESOLVE_MODE_AVERAGE_BIT_KHR)
465 			outval = nir_fdiv(&b, outval, nir_imm_float(&b, samples));
466 	}
467 
468 	nir_store_var(&b, fs_out, outval, 0x1);
469 
470 	return b.shader;
471 }
472 
473 static VkResult
create_depth_stencil_resolve_pipeline(struct radv_device * device,int samples_log2,int index,VkResolveModeFlagBits resolve_mode)474 create_depth_stencil_resolve_pipeline(struct radv_device *device,
475 				      int samples_log2,
476 				      int index,
477 				      VkResolveModeFlagBits resolve_mode)
478 {
479 	VkRenderPass *render_pass;
480 	VkPipeline *pipeline;
481 	VkFormat src_format;
482 	VkResult result;
483 
484 	mtx_lock(&device->meta_state.mtx);
485 
486 	switch (resolve_mode) {
487 	case VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR:
488 		if (index == DEPTH_RESOLVE)
489 			pipeline = &device->meta_state.resolve_fragment.depth_zero_pipeline;
490 		else
491 			pipeline = &device->meta_state.resolve_fragment.stencil_zero_pipeline;
492 		break;
493 	case VK_RESOLVE_MODE_AVERAGE_BIT_KHR:
494 		assert(index == DEPTH_RESOLVE);
495 		pipeline = &device->meta_state.resolve_fragment.depth[samples_log2].average_pipeline;
496 		break;
497 	case VK_RESOLVE_MODE_MIN_BIT_KHR:
498 		if (index == DEPTH_RESOLVE)
499 			pipeline = &device->meta_state.resolve_fragment.depth[samples_log2].min_pipeline;
500 		else
501 			pipeline = &device->meta_state.resolve_fragment.stencil[samples_log2].min_pipeline;
502 		break;
503 	case VK_RESOLVE_MODE_MAX_BIT_KHR:
504 		if (index == DEPTH_RESOLVE)
505 			pipeline = &device->meta_state.resolve_fragment.depth[samples_log2].max_pipeline;
506 		else
507 			pipeline = &device->meta_state.resolve_fragment.stencil[samples_log2].max_pipeline;
508 		break;
509 	default:
510 		unreachable("invalid resolve mode");
511 	}
512 
513 	if (*pipeline) {
514 		mtx_unlock(&device->meta_state.mtx);
515 		return VK_SUCCESS;
516 	}
517 
518 	struct radv_shader_module fs = { .nir = NULL };
519 	struct radv_shader_module vs = { .nir = NULL };
520 	uint32_t samples = 1 << samples_log2;
521 
522 	vs.nir = build_nir_vertex_shader();
523 	fs.nir = build_depth_stencil_resolve_fragment_shader(device, samples,
524 							     index, resolve_mode);
525 
526 	VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
527 		{
528 			.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
529 			.stage = VK_SHADER_STAGE_VERTEX_BIT,
530 			.module = radv_shader_module_to_handle(&vs),
531 			.pName = "main",
532 			.pSpecializationInfo = NULL
533 		}, {
534 			.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
535 			.stage = VK_SHADER_STAGE_FRAGMENT_BIT,
536 			.module = radv_shader_module_to_handle(&fs),
537 			.pName = "main",
538 			.pSpecializationInfo = NULL
539 		},
540 	};
541 
542 	if (index == DEPTH_RESOLVE) {
543 		src_format = VK_FORMAT_D32_SFLOAT;
544 		render_pass = &device->meta_state.resolve_fragment.depth_render_pass;
545 	} else {
546 		render_pass = &device->meta_state.resolve_fragment.stencil_render_pass;
547 		src_format = VK_FORMAT_S8_UINT;
548 	}
549 
550 	if (!*render_pass) {
551 		result = radv_CreateRenderPass(radv_device_to_handle(device),
552 					       &(VkRenderPassCreateInfo) {
553 							.sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
554 							.attachmentCount = 1,
555 							.pAttachments = &(VkAttachmentDescription) {
556 								.format = src_format,
557 								.loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE,
558 								.storeOp = VK_ATTACHMENT_STORE_OP_DONT_CARE,
559 								.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
560 								.stencilStoreOp = VK_ATTACHMENT_STORE_OP_STORE,
561 								.initialLayout = VK_IMAGE_LAYOUT_GENERAL,
562 								.finalLayout = VK_IMAGE_LAYOUT_GENERAL,
563 							},
564 							.subpassCount = 1,
565 							.pSubpasses = &(VkSubpassDescription) {
566 								.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
567 								.inputAttachmentCount = 0,
568 								.colorAttachmentCount = 0,
569 								.pColorAttachments = NULL,
570 							.pResolveAttachments = NULL,
571 							.pDepthStencilAttachment = &(VkAttachmentReference) {
572 								.attachment = 0,
573 								.layout = VK_IMAGE_LAYOUT_GENERAL,
574 							},
575 							.preserveAttachmentCount = 0,
576 							.pPreserveAttachments = NULL,
577 						},
578 						.dependencyCount = 2,
579 						.pDependencies = (VkSubpassDependency[]) {
580 							{
581 								.srcSubpass = VK_SUBPASS_EXTERNAL,
582 								.dstSubpass = 0,
583 								.srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
584 								.dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
585 								.srcAccessMask = 0,
586 								.dstAccessMask = 0,
587 								.dependencyFlags = 0
588 							},
589 							{
590 								.srcSubpass = 0,
591 								.dstSubpass = VK_SUBPASS_EXTERNAL,
592 								.srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
593 								.dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
594 								.srcAccessMask = 0,
595 								.dstAccessMask = 0,
596 								.dependencyFlags = 0
597 							}
598 						},
599 					}, &device->meta_state.alloc, render_pass);
600 	}
601 
602 	VkStencilOp stencil_op =
603 		index == DEPTH_RESOLVE ? VK_STENCIL_OP_KEEP : VK_STENCIL_OP_REPLACE;
604 
605 	VkPipelineDepthStencilStateCreateInfo depth_stencil_state = {
606 		.sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
607 		.depthTestEnable = true,
608 		.depthWriteEnable = index == DEPTH_RESOLVE,
609 		.stencilTestEnable = index == STENCIL_RESOLVE,
610 		.depthCompareOp = VK_COMPARE_OP_ALWAYS,
611 		.front = {
612 			.failOp = stencil_op,
613 			.passOp = stencil_op,
614 			.depthFailOp = stencil_op,
615 			.compareOp = VK_COMPARE_OP_ALWAYS,
616 		},
617 		.back = {
618 			.failOp = stencil_op,
619 			.passOp = stencil_op,
620 			.depthFailOp = stencil_op,
621 			.compareOp = VK_COMPARE_OP_ALWAYS,
622 		}
623 	};
624 
625 	const VkPipelineVertexInputStateCreateInfo *vi_create_info;
626 	vi_create_info = &normal_vi_create_info;
627 
628 	const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
629 		.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
630 		.stageCount = ARRAY_SIZE(pipeline_shader_stages),
631 		.pStages = pipeline_shader_stages,
632 		.pVertexInputState = vi_create_info,
633 		.pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
634 			.sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
635 			.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
636 			.primitiveRestartEnable = false,
637 		},
638 		.pViewportState = &(VkPipelineViewportStateCreateInfo) {
639 			.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
640 			.viewportCount = 1,
641 			.scissorCount = 1,
642 		},
643 		.pDepthStencilState = &depth_stencil_state,
644 		.pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
645 			.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
646 			.rasterizerDiscardEnable = false,
647 			.polygonMode = VK_POLYGON_MODE_FILL,
648 			.cullMode = VK_CULL_MODE_NONE,
649 			.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
650 		},
651 		.pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
652 			.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
653 			.rasterizationSamples = 1,
654 			.sampleShadingEnable = false,
655 			.pSampleMask = (VkSampleMask[]) { UINT32_MAX },
656 		},
657 		.pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
658 			.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
659 			.attachmentCount = 0,
660 			.pAttachments = (VkPipelineColorBlendAttachmentState []) {
661 				{ .colorWriteMask =
662 				  VK_COLOR_COMPONENT_A_BIT |
663 				  VK_COLOR_COMPONENT_R_BIT |
664 				  VK_COLOR_COMPONENT_G_BIT |
665 				  VK_COLOR_COMPONENT_B_BIT },
666 			}
667 		},
668 		.pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
669 			.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
670 			.dynamicStateCount = 9,
671 			.pDynamicStates = (VkDynamicState[]) {
672 				VK_DYNAMIC_STATE_VIEWPORT,
673 				VK_DYNAMIC_STATE_SCISSOR,
674 				VK_DYNAMIC_STATE_LINE_WIDTH,
675 				VK_DYNAMIC_STATE_DEPTH_BIAS,
676 				VK_DYNAMIC_STATE_BLEND_CONSTANTS,
677 				VK_DYNAMIC_STATE_DEPTH_BOUNDS,
678 				VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
679 				VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
680 				VK_DYNAMIC_STATE_STENCIL_REFERENCE,
681 			},
682 		},
683 		.flags = 0,
684 		.layout = device->meta_state.resolve_fragment.p_layout,
685 		.renderPass = *render_pass,
686 		.subpass = 0,
687 	};
688 
689 	const struct radv_graphics_pipeline_create_info radv_pipeline_info = {
690 		.use_rectlist = true
691 	};
692 
693 	result = radv_graphics_pipeline_create(radv_device_to_handle(device),
694 					       radv_pipeline_cache_to_handle(&device->meta_state.cache),
695 					       &vk_pipeline_info, &radv_pipeline_info,
696 					       &device->meta_state.alloc,
697 					       pipeline);
698 
699 	ralloc_free(vs.nir);
700 	ralloc_free(fs.nir);
701 
702 	mtx_unlock(&device->meta_state.mtx);
703 	return result;
704 }
705 
706 VkResult
radv_device_init_meta_resolve_fragment_state(struct radv_device * device,bool on_demand)707 radv_device_init_meta_resolve_fragment_state(struct radv_device *device, bool on_demand)
708 {
709 	VkResult res;
710 
711 	res = create_layout(device);
712 	if (res != VK_SUCCESS)
713 		goto fail;
714 
715 	if (on_demand)
716 		return VK_SUCCESS;
717 
718 	for (uint32_t i = 0; i < MAX_SAMPLES_LOG2; ++i) {
719 		for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
720 			res = create_resolve_pipeline(device, i, radv_fs_key_format_exemplars[j]);
721 			if (res != VK_SUCCESS)
722 				goto fail;
723 		}
724 
725 		res = create_depth_stencil_resolve_pipeline(device, i, DEPTH_RESOLVE,
726 							    VK_RESOLVE_MODE_AVERAGE_BIT_KHR);
727 		if (res != VK_SUCCESS)
728 			goto fail;
729 
730 		res = create_depth_stencil_resolve_pipeline(device, i, DEPTH_RESOLVE,
731 							    VK_RESOLVE_MODE_MIN_BIT_KHR);
732 		if (res != VK_SUCCESS)
733 			goto fail;
734 
735 		res = create_depth_stencil_resolve_pipeline(device, i, DEPTH_RESOLVE,
736 							    VK_RESOLVE_MODE_MAX_BIT_KHR);
737 		if (res != VK_SUCCESS)
738 			goto fail;
739 
740 		res = create_depth_stencil_resolve_pipeline(device, i, STENCIL_RESOLVE,
741 							    VK_RESOLVE_MODE_MIN_BIT_KHR);
742 		if (res != VK_SUCCESS)
743 			goto fail;
744 
745 		res = create_depth_stencil_resolve_pipeline(device, i, STENCIL_RESOLVE,
746 							    VK_RESOLVE_MODE_MAX_BIT_KHR);
747 		if (res != VK_SUCCESS)
748 			goto fail;
749 	}
750 
751 	res = create_depth_stencil_resolve_pipeline(device, 0, DEPTH_RESOLVE,
752 						    VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR);
753 	if (res != VK_SUCCESS)
754 		goto fail;
755 
756 	res = create_depth_stencil_resolve_pipeline(device, 0, STENCIL_RESOLVE,
757 						    VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR);
758 	if (res != VK_SUCCESS)
759 		goto fail;
760 
761 	return VK_SUCCESS;
762 fail:
763 	radv_device_finish_meta_resolve_fragment_state(device);
764 	return res;
765 }
766 
767 void
radv_device_finish_meta_resolve_fragment_state(struct radv_device * device)768 radv_device_finish_meta_resolve_fragment_state(struct radv_device *device)
769 {
770 	struct radv_meta_state *state = &device->meta_state;
771 	for (uint32_t i = 0; i < MAX_SAMPLES_LOG2; ++i) {
772 		for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
773 			for(unsigned k =0; k < RADV_META_DST_LAYOUT_COUNT; ++k) {
774 				radv_DestroyRenderPass(radv_device_to_handle(device),
775 				                       state->resolve_fragment.rc[i].render_pass[j][k],
776 				                       &state->alloc);
777 			}
778 			radv_DestroyPipeline(radv_device_to_handle(device),
779 					     state->resolve_fragment.rc[i].pipeline[j],
780 					     &state->alloc);
781 		}
782 
783 		radv_DestroyPipeline(radv_device_to_handle(device),
784 				     state->resolve_fragment.depth[i].average_pipeline,
785 				     &state->alloc);
786 
787 		radv_DestroyPipeline(radv_device_to_handle(device),
788 				     state->resolve_fragment.depth[i].max_pipeline,
789 				     &state->alloc);
790 
791 		radv_DestroyPipeline(radv_device_to_handle(device),
792 				     state->resolve_fragment.depth[i].min_pipeline,
793 				     &state->alloc);
794 
795 		radv_DestroyPipeline(radv_device_to_handle(device),
796 				     state->resolve_fragment.stencil[i].max_pipeline,
797 				     &state->alloc);
798 
799 		radv_DestroyPipeline(radv_device_to_handle(device),
800 				     state->resolve_fragment.stencil[i].min_pipeline,
801 				     &state->alloc);
802 	}
803 
804 	radv_DestroyRenderPass(radv_device_to_handle(device),
805 			       state->resolve_fragment.depth_render_pass,
806 			       &state->alloc);
807 	radv_DestroyRenderPass(radv_device_to_handle(device),
808 			       state->resolve_fragment.stencil_render_pass,
809 			       &state->alloc);
810 
811 	radv_DestroyPipeline(radv_device_to_handle(device),
812 			     state->resolve_fragment.depth_zero_pipeline,
813 			     &state->alloc);
814 	radv_DestroyPipeline(radv_device_to_handle(device),
815 			     state->resolve_fragment.stencil_zero_pipeline,
816 			     &state->alloc);
817 
818 	radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
819 					state->resolve_fragment.ds_layout,
820 					&state->alloc);
821 	radv_DestroyPipelineLayout(radv_device_to_handle(device),
822 				   state->resolve_fragment.p_layout,
823 				   &state->alloc);
824 }
825 
826 static VkPipeline *
radv_get_resolve_pipeline(struct radv_cmd_buffer * cmd_buffer,struct radv_image_view * src_iview,struct radv_image_view * dst_iview)827 radv_get_resolve_pipeline(struct radv_cmd_buffer *cmd_buffer,
828 			  struct radv_image_view *src_iview,
829 			  struct radv_image_view *dst_iview)
830 {
831 	struct radv_device *device = cmd_buffer->device;
832 	unsigned fs_key = radv_format_meta_fs_key(dst_iview->vk_format);
833 	const uint32_t samples = src_iview->image->info.samples;
834 	const uint32_t samples_log2 = ffs(samples) - 1;
835 	VkPipeline *pipeline;
836 
837 	pipeline = &device->meta_state.resolve_fragment.rc[samples_log2].pipeline[fs_key];
838 	if (!*pipeline ) {
839 		VkResult ret;
840 
841 		ret = create_resolve_pipeline(device, samples_log2,
842 					      radv_fs_key_format_exemplars[fs_key]);
843 		if (ret != VK_SUCCESS) {
844 			cmd_buffer->record_result = ret;
845 			return NULL;
846 		}
847 	}
848 
849 	return pipeline;
850 }
851 
852 static void
emit_resolve(struct radv_cmd_buffer * cmd_buffer,struct radv_image_view * src_iview,struct radv_image_view * dest_iview,const VkOffset2D * src_offset,const VkOffset2D * dest_offset,const VkExtent2D * resolve_extent)853 emit_resolve(struct radv_cmd_buffer *cmd_buffer,
854 	     struct radv_image_view *src_iview,
855 	     struct radv_image_view *dest_iview,
856 	     const VkOffset2D *src_offset,
857              const VkOffset2D *dest_offset,
858              const VkExtent2D *resolve_extent)
859 {
860 	struct radv_device *device = cmd_buffer->device;
861 	VkCommandBuffer cmd_buffer_h = radv_cmd_buffer_to_handle(cmd_buffer);
862 	VkPipeline *pipeline;
863 
864 	radv_meta_push_descriptor_set(cmd_buffer,
865 				      VK_PIPELINE_BIND_POINT_GRAPHICS,
866 				      cmd_buffer->device->meta_state.resolve_fragment.p_layout,
867 				      0, /* set */
868 				      1, /* descriptorWriteCount */
869 				      (VkWriteDescriptorSet[]) {
870 					      {
871 						      .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
872 					              .dstBinding = 0,
873 					              .dstArrayElement = 0,
874 					              .descriptorCount = 1,
875 					              .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
876 						      .pImageInfo = (VkDescriptorImageInfo[]) {
877 						      {
878 						      .sampler = VK_NULL_HANDLE,
879 						      .imageView = radv_image_view_to_handle(src_iview),
880 						      .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
881 						      },
882 						      }
883 					      },
884 				      });
885 
886 	cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB;
887 
888 	unsigned push_constants[2] = {
889 		src_offset->x - dest_offset->x,
890 		src_offset->y - dest_offset->y,
891 	};
892 	radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
893 			      device->meta_state.resolve_fragment.p_layout,
894 			      VK_SHADER_STAGE_FRAGMENT_BIT, 0, 8,
895 			      push_constants);
896 
897 	pipeline = radv_get_resolve_pipeline(cmd_buffer, src_iview, dest_iview);
898 
899 	radv_CmdBindPipeline(cmd_buffer_h, VK_PIPELINE_BIND_POINT_GRAPHICS,
900 			     *pipeline);
901 
902 	radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkViewport) {
903 		.x = dest_offset->x,
904 		.y = dest_offset->y,
905 		.width = resolve_extent->width,
906 		.height = resolve_extent->height,
907 		.minDepth = 0.0f,
908 		.maxDepth = 1.0f
909 	});
910 
911 	radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkRect2D) {
912 		.offset = *dest_offset,
913 		.extent = *resolve_extent,
914 	});
915 
916 	radv_CmdDraw(cmd_buffer_h, 3, 1, 0, 0);
917 	cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB;
918 }
919 
920 static void
emit_depth_stencil_resolve(struct radv_cmd_buffer * cmd_buffer,struct radv_image_view * src_iview,struct radv_image_view * dst_iview,const VkOffset2D * src_offset,const VkOffset2D * dst_offset,const VkExtent2D * resolve_extent,VkImageAspectFlags aspects,VkResolveModeFlagBits resolve_mode)921 emit_depth_stencil_resolve(struct radv_cmd_buffer *cmd_buffer,
922 			   struct radv_image_view *src_iview,
923 			   struct radv_image_view *dst_iview,
924 			   const VkOffset2D *src_offset,
925 			   const VkOffset2D *dst_offset,
926 			   const VkExtent2D *resolve_extent,
927 			   VkImageAspectFlags aspects,
928 			   VkResolveModeFlagBits resolve_mode)
929 {
930 	struct radv_device *device = cmd_buffer->device;
931 	const uint32_t samples = src_iview->image->info.samples;
932 	const uint32_t samples_log2 = ffs(samples) - 1;
933 	VkPipeline *pipeline;
934 
935 	radv_meta_push_descriptor_set(cmd_buffer,
936 				      VK_PIPELINE_BIND_POINT_GRAPHICS,
937 				      cmd_buffer->device->meta_state.resolve_fragment.p_layout,
938 				      0, /* set */
939 				      1, /* descriptorWriteCount */
940 				      (VkWriteDescriptorSet[]) {
941 					      {
942 						      .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
943 					              .dstBinding = 0,
944 					              .dstArrayElement = 0,
945 					              .descriptorCount = 1,
946 					              .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
947 						      .pImageInfo = (VkDescriptorImageInfo[]) {
948 						      {
949 						      .sampler = VK_NULL_HANDLE,
950 						      .imageView = radv_image_view_to_handle(src_iview),
951 						      .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
952 						      },
953 						      }
954 					      },
955 				      });
956 
957 	unsigned push_constants[2] = {
958 		src_offset->x - dst_offset->x,
959 		src_offset->y - dst_offset->y,
960 	};
961 	radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
962 			      device->meta_state.resolve_fragment.p_layout,
963 			      VK_SHADER_STAGE_FRAGMENT_BIT, 0, 8,
964 			      push_constants);
965 
966 	switch (resolve_mode) {
967 	case VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR:
968 		if (aspects == VK_IMAGE_ASPECT_DEPTH_BIT)
969 			pipeline = &device->meta_state.resolve_fragment.depth_zero_pipeline;
970 		else
971 			pipeline = &device->meta_state.resolve_fragment.stencil_zero_pipeline;
972 		break;
973 	case VK_RESOLVE_MODE_AVERAGE_BIT_KHR:
974 		assert(aspects == VK_IMAGE_ASPECT_DEPTH_BIT);
975 		pipeline = &device->meta_state.resolve_fragment.depth[samples_log2].average_pipeline;
976 		break;
977 	case VK_RESOLVE_MODE_MIN_BIT_KHR:
978 		if (aspects == VK_IMAGE_ASPECT_DEPTH_BIT)
979 			pipeline = &device->meta_state.resolve_fragment.depth[samples_log2].min_pipeline;
980 		else
981 			pipeline = &device->meta_state.resolve_fragment.stencil[samples_log2].min_pipeline;
982 		break;
983 	case VK_RESOLVE_MODE_MAX_BIT_KHR:
984 		if (aspects == VK_IMAGE_ASPECT_DEPTH_BIT)
985 			pipeline = &device->meta_state.resolve_fragment.depth[samples_log2].max_pipeline;
986 		else
987 			pipeline = &device->meta_state.resolve_fragment.stencil[samples_log2].max_pipeline;
988 		break;
989 	default:
990 		unreachable("invalid resolve mode");
991 	}
992 
993 	if (!*pipeline) {
994 		int index = aspects == VK_IMAGE_ASPECT_DEPTH_BIT ? DEPTH_RESOLVE : STENCIL_RESOLVE;
995 		VkResult ret;
996 
997 		ret = create_depth_stencil_resolve_pipeline(device, samples_log2,
998 							    index, resolve_mode);
999 		if (ret != VK_SUCCESS) {
1000 			cmd_buffer->record_result = ret;
1001 			return;
1002 		}
1003 	}
1004 
1005 	radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1006 			     VK_PIPELINE_BIND_POINT_GRAPHICS, *pipeline);
1007 
1008 	radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkViewport) {
1009 		.x = dst_offset->x,
1010 		.y = dst_offset->y,
1011 		.width = resolve_extent->width,
1012 		.height = resolve_extent->height,
1013 		.minDepth = 0.0f,
1014 		.maxDepth = 1.0f
1015 	});
1016 
1017 	radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkRect2D) {
1018 		.offset = *dst_offset,
1019 		.extent = *resolve_extent,
1020 	});
1021 
1022 	radv_CmdDraw(radv_cmd_buffer_to_handle(cmd_buffer), 3, 1, 0, 0);
1023 }
1024 
radv_meta_resolve_fragment_image(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,VkImageLayout src_image_layout,struct radv_image * dest_image,VkImageLayout dest_image_layout,const VkImageResolve2KHR * region)1025 void radv_meta_resolve_fragment_image(struct radv_cmd_buffer *cmd_buffer,
1026 				      struct radv_image *src_image,
1027 				      VkImageLayout src_image_layout,
1028 				      struct radv_image *dest_image,
1029 				      VkImageLayout dest_image_layout,
1030 				      const VkImageResolve2KHR *region)
1031 {
1032 	struct radv_device *device = cmd_buffer->device;
1033 	struct radv_meta_saved_state saved_state;
1034 	const uint32_t samples = src_image->info.samples;
1035 	const uint32_t samples_log2 = ffs(samples) - 1;
1036 	unsigned fs_key = radv_format_meta_fs_key(dest_image->vk_format);
1037 	unsigned dst_layout = radv_meta_dst_layout_from_layout(dest_image_layout);
1038 	VkRenderPass rp;
1039 
1040 	radv_decompress_resolve_src(cmd_buffer, src_image, src_image_layout,
1041 				    region);
1042 
1043 	if (!device->meta_state.resolve_fragment.rc[samples_log2].render_pass[fs_key][dst_layout]) {
1044 		VkResult ret = create_resolve_pipeline(device, samples_log2, radv_fs_key_format_exemplars[fs_key]);
1045 		if (ret != VK_SUCCESS) {
1046 			cmd_buffer->record_result = ret;
1047 			return;
1048 		}
1049 	}
1050 
1051 	rp = device->meta_state.resolve_fragment.rc[samples_log2].render_pass[fs_key][dst_layout];
1052 
1053 	radv_meta_save(&saved_state, cmd_buffer,
1054 		       RADV_META_SAVE_GRAPHICS_PIPELINE |
1055 		       RADV_META_SAVE_CONSTANTS |
1056 		       RADV_META_SAVE_DESCRIPTORS);
1057 
1058 	assert(region->srcSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
1059 	assert(region->dstSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
1060 	assert(region->srcSubresource.layerCount == region->dstSubresource.layerCount);
1061 
1062 	const uint32_t src_base_layer =
1063 		radv_meta_get_iview_layer(src_image, &region->srcSubresource,
1064 					  &region->srcOffset);
1065 
1066 	const uint32_t dest_base_layer =
1067 		radv_meta_get_iview_layer(dest_image, &region->dstSubresource,
1068 					  &region->dstOffset);
1069 
1070 	const struct VkExtent3D extent =
1071 		radv_sanitize_image_extent(src_image->type, region->extent);
1072 	const struct VkOffset3D srcOffset =
1073 		radv_sanitize_image_offset(src_image->type, region->srcOffset);
1074 	const struct VkOffset3D dstOffset =
1075 		radv_sanitize_image_offset(dest_image->type, region->dstOffset);
1076 
1077 	for (uint32_t layer = 0; layer < region->srcSubresource.layerCount;
1078 	     ++layer) {
1079 
1080 		struct radv_image_view src_iview;
1081 		radv_image_view_init(&src_iview, cmd_buffer->device,
1082 				     &(VkImageViewCreateInfo) {
1083 					     .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
1084 						     .image = radv_image_to_handle(src_image),
1085 						     .viewType = radv_meta_get_view_type(src_image),
1086 						     .format = src_image->vk_format,
1087 						     .subresourceRange = {
1088 						     .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
1089 						     .baseMipLevel = region->srcSubresource.mipLevel,
1090 						     .levelCount = 1,
1091 						     .baseArrayLayer = src_base_layer + layer,
1092 						     .layerCount = 1,
1093 					     },
1094 				     }, NULL);
1095 
1096 		struct radv_image_view dest_iview;
1097 		radv_image_view_init(&dest_iview, cmd_buffer->device,
1098 				     &(VkImageViewCreateInfo) {
1099 					     .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
1100 						     .image = radv_image_to_handle(dest_image),
1101 						     .viewType = radv_meta_get_view_type(dest_image),
1102 						     .format = dest_image->vk_format,
1103 						     .subresourceRange = {
1104 						     .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
1105 						     .baseMipLevel = region->dstSubresource.mipLevel,
1106 						     .levelCount = 1,
1107 						     .baseArrayLayer = dest_base_layer + layer,
1108 						     .layerCount = 1,
1109 					     },
1110 				     }, NULL);
1111 
1112 
1113 		VkFramebuffer fb;
1114 		radv_CreateFramebuffer(radv_device_to_handle(cmd_buffer->device),
1115 		       &(VkFramebufferCreateInfo) {
1116 			       .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
1117 				       .attachmentCount = 1,
1118 				       .pAttachments = (VkImageView[]) {
1119 				       radv_image_view_to_handle(&dest_iview),
1120 			       },
1121 			       .width = extent.width + dstOffset.x,
1122 			       .height = extent.height + dstOffset.y,
1123 			       .layers = 1
1124 			}, &cmd_buffer->pool->alloc, &fb);
1125 
1126 		radv_cmd_buffer_begin_render_pass(cmd_buffer,
1127 						  &(VkRenderPassBeginInfo) {
1128 							.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
1129 								.renderPass = rp,
1130 								.framebuffer = fb,
1131 								.renderArea = {
1132 									.offset = { dstOffset.x, dstOffset.y, },
1133 									.extent = { extent.width, extent.height },
1134 								},
1135 							.clearValueCount = 0,
1136 							.pClearValues = NULL,
1137 					});
1138 
1139 		radv_cmd_buffer_set_subpass(cmd_buffer,
1140 					    &cmd_buffer->state.pass->subpasses[0]);
1141 
1142 		emit_resolve(cmd_buffer,
1143 			     &src_iview,
1144 			     &dest_iview,
1145 			     &(VkOffset2D) { srcOffset.x, srcOffset.y },
1146 			     &(VkOffset2D) { dstOffset.x, dstOffset.y },
1147 			     &(VkExtent2D) { extent.width, extent.height });
1148 
1149 		radv_cmd_buffer_end_render_pass(cmd_buffer);
1150 
1151 		radv_DestroyFramebuffer(radv_device_to_handle(cmd_buffer->device), fb, &cmd_buffer->pool->alloc);
1152 	}
1153 
1154 	radv_meta_restore(&saved_state, cmd_buffer);
1155 }
1156 
1157 
1158 /**
1159  * Emit any needed resolves for the current subpass.
1160  */
1161 void
radv_cmd_buffer_resolve_subpass_fs(struct radv_cmd_buffer * cmd_buffer)1162 radv_cmd_buffer_resolve_subpass_fs(struct radv_cmd_buffer *cmd_buffer)
1163 {
1164 	struct radv_framebuffer *fb = cmd_buffer->state.framebuffer;
1165 	const struct radv_subpass *subpass = cmd_buffer->state.subpass;
1166 	struct radv_meta_saved_state saved_state;
1167 	struct radv_subpass_barrier barrier;
1168 
1169 	/* Resolves happen before the end-of-subpass barriers get executed,
1170 	 * so we have to make the attachment shader-readable */
1171 	barrier.src_stage_mask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
1172 	barrier.src_access_mask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
1173 	barrier.dst_access_mask = VK_ACCESS_INPUT_ATTACHMENT_READ_BIT;
1174 	radv_subpass_barrier(cmd_buffer, &barrier);
1175 
1176 	radv_decompress_resolve_subpass_src(cmd_buffer);
1177 
1178 	radv_meta_save(&saved_state, cmd_buffer,
1179 		       RADV_META_SAVE_GRAPHICS_PIPELINE |
1180 		       RADV_META_SAVE_CONSTANTS |
1181 		       RADV_META_SAVE_DESCRIPTORS);
1182 
1183 	for (uint32_t i = 0; i < subpass->color_count; ++i) {
1184 		struct radv_subpass_attachment src_att = subpass->color_attachments[i];
1185 		struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i];
1186 
1187 		if (dest_att.attachment == VK_ATTACHMENT_UNUSED)
1188 			continue;
1189 
1190 		struct radv_image_view *dest_iview = cmd_buffer->state.attachments[dest_att.attachment].iview;
1191 		struct radv_image_view *src_iview = cmd_buffer->state.attachments[src_att.attachment].iview;
1192 
1193 		struct radv_subpass resolve_subpass = {
1194 			.color_count = 1,
1195 			.color_attachments = (struct radv_subpass_attachment[]) { dest_att },
1196 			.depth_stencil_attachment = NULL,
1197 		};
1198 
1199 		radv_cmd_buffer_set_subpass(cmd_buffer, &resolve_subpass);
1200 
1201 		emit_resolve(cmd_buffer,
1202 			     src_iview,
1203 			     dest_iview,
1204 			     &(VkOffset2D) { 0, 0 },
1205 			     &(VkOffset2D) { 0, 0 },
1206 			     &(VkExtent2D) { fb->width, fb->height });
1207 	}
1208 
1209 	radv_cmd_buffer_set_subpass(cmd_buffer, subpass);
1210 
1211 	radv_meta_restore(&saved_state, cmd_buffer);
1212 }
1213 
1214 /**
1215  * Depth/stencil resolves for the current subpass.
1216  */
1217 void
radv_depth_stencil_resolve_subpass_fs(struct radv_cmd_buffer * cmd_buffer,VkImageAspectFlags aspects,VkResolveModeFlagBits resolve_mode)1218 radv_depth_stencil_resolve_subpass_fs(struct radv_cmd_buffer *cmd_buffer,
1219 				      VkImageAspectFlags aspects,
1220 				      VkResolveModeFlagBits resolve_mode)
1221 {
1222 	struct radv_framebuffer *fb = cmd_buffer->state.framebuffer;
1223 	const struct radv_subpass *subpass = cmd_buffer->state.subpass;
1224 	struct radv_meta_saved_state saved_state;
1225 	struct radv_subpass_barrier barrier;
1226 
1227 	/* Resolves happen before the end-of-subpass barriers get executed,
1228 	 * so we have to make the attachment shader-readable */
1229 	barrier.src_stage_mask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
1230 	barrier.src_access_mask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
1231 	barrier.dst_access_mask = VK_ACCESS_INPUT_ATTACHMENT_READ_BIT;
1232 	radv_subpass_barrier(cmd_buffer, &barrier);
1233 
1234 	radv_decompress_resolve_subpass_src(cmd_buffer);
1235 
1236 	radv_meta_save(&saved_state, cmd_buffer,
1237 		       RADV_META_SAVE_GRAPHICS_PIPELINE |
1238 		       RADV_META_SAVE_CONSTANTS |
1239 		       RADV_META_SAVE_DESCRIPTORS);
1240 
1241 	struct radv_subpass_attachment src_att = *subpass->depth_stencil_attachment;
1242 	struct radv_subpass_attachment dst_att = *subpass->ds_resolve_attachment;
1243 
1244 	struct radv_image_view *src_iview =
1245 		cmd_buffer->state.attachments[src_att.attachment].iview;
1246 	struct radv_image *src_image = src_iview->image;
1247 	struct radv_image_view *dst_iview =
1248 		cmd_buffer->state.attachments[dst_att.attachment].iview;
1249 
1250 	struct radv_subpass resolve_subpass = {
1251 		.color_count = 0,
1252 		.color_attachments = NULL,
1253 		.depth_stencil_attachment = (struct radv_subpass_attachment *) { &dst_att },
1254 	};
1255 
1256 	radv_cmd_buffer_set_subpass(cmd_buffer, &resolve_subpass);
1257 
1258 	struct radv_image_view tsrc_iview;
1259 	radv_image_view_init(&tsrc_iview, cmd_buffer->device,
1260 			     &(VkImageViewCreateInfo) {
1261 				.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
1262 				.image = radv_image_to_handle(src_image),
1263 				.viewType = radv_meta_get_view_type(src_image),
1264 				.format = src_iview->vk_format,
1265 				.subresourceRange = {
1266 					.aspectMask = aspects,
1267 					.baseMipLevel = 0,
1268 					.levelCount = 1,
1269 					.baseArrayLayer = 0,
1270 					.layerCount = 1,
1271 				},
1272 			      }, NULL);
1273 
1274 	emit_depth_stencil_resolve(cmd_buffer, &tsrc_iview, dst_iview,
1275 				   &(VkOffset2D) { 0, 0 },
1276 				   &(VkOffset2D) { 0, 0 },
1277 				   &(VkExtent2D) { fb->width, fb->height },
1278 				   aspects,
1279 				   resolve_mode);
1280 
1281 	radv_cmd_buffer_set_subpass(cmd_buffer, subpass);
1282 
1283 	radv_meta_restore(&saved_state, cmd_buffer);
1284 }
1285