1 /*
2 * Copyright © 2016 Red Hat
3 *
4 * based on anv driver:
5 * Copyright © 2016 Intel Corporation
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the next
15 * paragraph) shall be included in all copies or substantial portions of the
16 * Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * IN THE SOFTWARE.
25 */
26
27 #include "radv_meta.h"
28 #include "nir/nir_builder.h"
29 #include "vk_format.h"
30
31 enum blit2d_src_type {
32 BLIT2D_SRC_TYPE_IMAGE,
33 BLIT2D_SRC_TYPE_IMAGE_3D,
34 BLIT2D_SRC_TYPE_BUFFER,
35 BLIT2D_NUM_SRC_TYPES,
36 };
37
38 static VkResult
39 blit2d_init_color_pipeline(struct radv_device *device,
40 enum blit2d_src_type src_type,
41 VkFormat format,
42 uint32_t log2_samples);
43
44 static VkResult
45 blit2d_init_depth_only_pipeline(struct radv_device *device,
46 enum blit2d_src_type src_type,
47 uint32_t log2_samples);
48
49 static VkResult
50 blit2d_init_stencil_only_pipeline(struct radv_device *device,
51 enum blit2d_src_type src_type,
52 uint32_t log2_samples);
53
54 static void
create_iview(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * surf,struct radv_image_view * iview,VkFormat depth_format,VkImageAspectFlagBits aspects)55 create_iview(struct radv_cmd_buffer *cmd_buffer,
56 struct radv_meta_blit2d_surf *surf,
57 struct radv_image_view *iview, VkFormat depth_format,
58 VkImageAspectFlagBits aspects)
59 {
60 VkFormat format;
61 VkImageViewType view_type = cmd_buffer->device->physical_device->rad_info.chip_class < GFX9 ? VK_IMAGE_VIEW_TYPE_2D :
62 radv_meta_get_view_type(surf->image);
63
64 if (depth_format)
65 format = depth_format;
66 else
67 format = surf->format;
68
69 radv_image_view_init(iview, cmd_buffer->device,
70 &(VkImageViewCreateInfo) {
71 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
72 .image = radv_image_to_handle(surf->image),
73 .viewType = view_type,
74 .format = format,
75 .subresourceRange = {
76 .aspectMask = aspects,
77 .baseMipLevel = surf->level,
78 .levelCount = 1,
79 .baseArrayLayer = surf->layer,
80 .layerCount = 1
81 },
82 }, NULL);
83 }
84
85 static void
create_bview(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_buffer * src,struct radv_buffer_view * bview,VkFormat depth_format)86 create_bview(struct radv_cmd_buffer *cmd_buffer,
87 struct radv_meta_blit2d_buffer *src,
88 struct radv_buffer_view *bview, VkFormat depth_format)
89 {
90 VkFormat format;
91
92 if (depth_format)
93 format = depth_format;
94 else
95 format = src->format;
96 radv_buffer_view_init(bview, cmd_buffer->device,
97 &(VkBufferViewCreateInfo) {
98 .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO,
99 .flags = 0,
100 .buffer = radv_buffer_to_handle(src->buffer),
101 .format = format,
102 .offset = src->offset,
103 .range = VK_WHOLE_SIZE,
104 });
105
106 }
107
108 struct blit2d_src_temps {
109 struct radv_image_view iview;
110 struct radv_buffer_view bview;
111 };
112
113 static void
blit2d_bind_src(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * src_img,struct radv_meta_blit2d_buffer * src_buf,struct blit2d_src_temps * tmp,enum blit2d_src_type src_type,VkFormat depth_format,VkImageAspectFlagBits aspects,uint32_t log2_samples)114 blit2d_bind_src(struct radv_cmd_buffer *cmd_buffer,
115 struct radv_meta_blit2d_surf *src_img,
116 struct radv_meta_blit2d_buffer *src_buf,
117 struct blit2d_src_temps *tmp,
118 enum blit2d_src_type src_type, VkFormat depth_format,
119 VkImageAspectFlagBits aspects,
120 uint32_t log2_samples)
121 {
122 struct radv_device *device = cmd_buffer->device;
123
124 if (src_type == BLIT2D_SRC_TYPE_BUFFER) {
125 create_bview(cmd_buffer, src_buf, &tmp->bview, depth_format);
126
127 radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
128 device->meta_state.blit2d[log2_samples].p_layouts[src_type],
129 0, /* set */
130 1, /* descriptorWriteCount */
131 (VkWriteDescriptorSet[]) {
132 {
133 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
134 .dstBinding = 0,
135 .dstArrayElement = 0,
136 .descriptorCount = 1,
137 .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER,
138 .pTexelBufferView = (VkBufferView[]) { radv_buffer_view_to_handle(&tmp->bview) }
139 }
140 });
141
142 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
143 device->meta_state.blit2d[log2_samples].p_layouts[src_type],
144 VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4,
145 &src_buf->pitch);
146 } else {
147 create_iview(cmd_buffer, src_img, &tmp->iview, depth_format, aspects);
148
149 if (src_type == BLIT2D_SRC_TYPE_IMAGE_3D)
150 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
151 device->meta_state.blit2d[log2_samples].p_layouts[src_type],
152 VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4,
153 &src_img->layer);
154
155 radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
156 device->meta_state.blit2d[log2_samples].p_layouts[src_type],
157 0, /* set */
158 1, /* descriptorWriteCount */
159 (VkWriteDescriptorSet[]) {
160 {
161 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
162 .dstBinding = 0,
163 .dstArrayElement = 0,
164 .descriptorCount = 1,
165 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
166 .pImageInfo = (VkDescriptorImageInfo[]) {
167 {
168 .sampler = VK_NULL_HANDLE,
169 .imageView = radv_image_view_to_handle(&tmp->iview),
170 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
171 },
172 }
173 }
174 });
175 }
176 }
177
178 struct blit2d_dst_temps {
179 VkImage image;
180 struct radv_image_view iview;
181 VkFramebuffer fb;
182 };
183
184 static void
blit2d_bind_dst(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * dst,uint32_t width,uint32_t height,VkFormat depth_format,struct blit2d_dst_temps * tmp,VkImageAspectFlagBits aspects)185 blit2d_bind_dst(struct radv_cmd_buffer *cmd_buffer,
186 struct radv_meta_blit2d_surf *dst,
187 uint32_t width,
188 uint32_t height,
189 VkFormat depth_format,
190 struct blit2d_dst_temps *tmp,
191 VkImageAspectFlagBits aspects)
192 {
193 create_iview(cmd_buffer, dst, &tmp->iview, depth_format, aspects);
194
195 radv_CreateFramebuffer(radv_device_to_handle(cmd_buffer->device),
196 &(VkFramebufferCreateInfo) {
197 .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
198 .attachmentCount = 1,
199 .pAttachments = (VkImageView[]) {
200 radv_image_view_to_handle(&tmp->iview),
201 },
202 .width = width,
203 .height = height,
204 .layers = 1
205 }, &cmd_buffer->pool->alloc, &tmp->fb);
206 }
207
208 static void
bind_pipeline(struct radv_cmd_buffer * cmd_buffer,enum blit2d_src_type src_type,unsigned fs_key,uint32_t log2_samples)209 bind_pipeline(struct radv_cmd_buffer *cmd_buffer,
210 enum blit2d_src_type src_type, unsigned fs_key,
211 uint32_t log2_samples)
212 {
213 VkPipeline pipeline =
214 cmd_buffer->device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key];
215
216 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
217 VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
218 }
219
220 static void
bind_depth_pipeline(struct radv_cmd_buffer * cmd_buffer,enum blit2d_src_type src_type,uint32_t log2_samples)221 bind_depth_pipeline(struct radv_cmd_buffer *cmd_buffer,
222 enum blit2d_src_type src_type,
223 uint32_t log2_samples)
224 {
225 VkPipeline pipeline =
226 cmd_buffer->device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type];
227
228 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
229 VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
230 }
231
232 static void
bind_stencil_pipeline(struct radv_cmd_buffer * cmd_buffer,enum blit2d_src_type src_type,uint32_t log2_samples)233 bind_stencil_pipeline(struct radv_cmd_buffer *cmd_buffer,
234 enum blit2d_src_type src_type,
235 uint32_t log2_samples)
236 {
237 VkPipeline pipeline =
238 cmd_buffer->device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type];
239
240 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
241 VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
242 }
243
244 static void
radv_meta_blit2d_normal_dst(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * src_img,struct radv_meta_blit2d_buffer * src_buf,struct radv_meta_blit2d_surf * dst,unsigned num_rects,struct radv_meta_blit2d_rect * rects,enum blit2d_src_type src_type,uint32_t log2_samples)245 radv_meta_blit2d_normal_dst(struct radv_cmd_buffer *cmd_buffer,
246 struct radv_meta_blit2d_surf *src_img,
247 struct radv_meta_blit2d_buffer *src_buf,
248 struct radv_meta_blit2d_surf *dst,
249 unsigned num_rects,
250 struct radv_meta_blit2d_rect *rects, enum blit2d_src_type src_type,
251 uint32_t log2_samples)
252 {
253 struct radv_device *device = cmd_buffer->device;
254
255 for (unsigned r = 0; r < num_rects; ++r) {
256 unsigned i;
257 for_each_bit(i, dst->aspect_mask) {
258 unsigned aspect_mask = 1u << i;
259 unsigned src_aspect_mask = aspect_mask;
260 VkFormat depth_format = 0;
261 if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT)
262 depth_format = vk_format_stencil_only(dst->image->vk_format);
263 else if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT)
264 depth_format = vk_format_depth_only(dst->image->vk_format);
265 else if (src_img)
266 src_aspect_mask = src_img->aspect_mask;
267
268 struct blit2d_src_temps src_temps;
269 blit2d_bind_src(cmd_buffer, src_img, src_buf, &src_temps, src_type, depth_format, src_aspect_mask, log2_samples);
270
271 struct blit2d_dst_temps dst_temps;
272 blit2d_bind_dst(cmd_buffer, dst, rects[r].dst_x + rects[r].width,
273 rects[r].dst_y + rects[r].height, depth_format, &dst_temps, aspect_mask);
274
275 float vertex_push_constants[4] = {
276 rects[r].src_x,
277 rects[r].src_y,
278 rects[r].src_x + rects[r].width,
279 rects[r].src_y + rects[r].height,
280 };
281
282 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
283 device->meta_state.blit2d[log2_samples].p_layouts[src_type],
284 VK_SHADER_STAGE_VERTEX_BIT, 0, 16,
285 vertex_push_constants);
286
287 if (aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT ||
288 aspect_mask == VK_IMAGE_ASPECT_PLANE_0_BIT ||
289 aspect_mask == VK_IMAGE_ASPECT_PLANE_1_BIT ||
290 aspect_mask == VK_IMAGE_ASPECT_PLANE_2_BIT) {
291 unsigned fs_key = radv_format_meta_fs_key(dst_temps.iview.vk_format);
292 unsigned dst_layout = radv_meta_dst_layout_from_layout(dst->current_layout);
293
294 if (device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key] == VK_NULL_HANDLE) {
295 VkResult ret = blit2d_init_color_pipeline(device, src_type, radv_fs_key_format_exemplars[fs_key], log2_samples);
296 if (ret != VK_SUCCESS) {
297 cmd_buffer->record_result = ret;
298 goto fail_pipeline;
299 }
300 }
301
302 radv_cmd_buffer_begin_render_pass(cmd_buffer,
303 &(VkRenderPassBeginInfo) {
304 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
305 .renderPass = device->meta_state.blit2d_render_passes[fs_key][dst_layout],
306 .framebuffer = dst_temps.fb,
307 .renderArea = {
308 .offset = { rects[r].dst_x, rects[r].dst_y, },
309 .extent = { rects[r].width, rects[r].height },
310 },
311 .clearValueCount = 0,
312 .pClearValues = NULL,
313 });
314
315 radv_cmd_buffer_set_subpass(cmd_buffer,
316 &cmd_buffer->state.pass->subpasses[0]);
317
318 bind_pipeline(cmd_buffer, src_type, fs_key, log2_samples);
319 } else if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT) {
320 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst->current_layout);
321
322 if (device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type] == VK_NULL_HANDLE) {
323 VkResult ret = blit2d_init_depth_only_pipeline(device, src_type, log2_samples);
324 if (ret != VK_SUCCESS) {
325 cmd_buffer->record_result = ret;
326 goto fail_pipeline;
327 }
328 }
329
330 radv_cmd_buffer_begin_render_pass(cmd_buffer,
331 &(VkRenderPassBeginInfo) {
332 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
333 .renderPass = device->meta_state.blit2d_depth_only_rp[ds_layout],
334 .framebuffer = dst_temps.fb,
335 .renderArea = {
336 .offset = { rects[r].dst_x, rects[r].dst_y, },
337 .extent = { rects[r].width, rects[r].height },
338 },
339 .clearValueCount = 0,
340 .pClearValues = NULL,
341 });
342
343 radv_cmd_buffer_set_subpass(cmd_buffer,
344 &cmd_buffer->state.pass->subpasses[0]);
345
346 bind_depth_pipeline(cmd_buffer, src_type, log2_samples);
347
348 } else if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT) {
349 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst->current_layout);
350
351 if (device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type] == VK_NULL_HANDLE) {
352 VkResult ret = blit2d_init_stencil_only_pipeline(device, src_type, log2_samples);
353 if (ret != VK_SUCCESS) {
354 cmd_buffer->record_result = ret;
355 goto fail_pipeline;
356 }
357 }
358
359 radv_cmd_buffer_begin_render_pass(cmd_buffer,
360 &(VkRenderPassBeginInfo) {
361 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
362 .renderPass = device->meta_state.blit2d_stencil_only_rp[ds_layout],
363 .framebuffer = dst_temps.fb,
364 .renderArea = {
365 .offset = { rects[r].dst_x, rects[r].dst_y, },
366 .extent = { rects[r].width, rects[r].height },
367 },
368 .clearValueCount = 0,
369 .pClearValues = NULL,
370 });
371
372 radv_cmd_buffer_set_subpass(cmd_buffer,
373 &cmd_buffer->state.pass->subpasses[0]);
374
375 bind_stencil_pipeline(cmd_buffer, src_type, log2_samples);
376 } else
377 unreachable("Processing blit2d with multiple aspects.");
378
379 radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkViewport) {
380 .x = rects[r].dst_x,
381 .y = rects[r].dst_y,
382 .width = rects[r].width,
383 .height = rects[r].height,
384 .minDepth = 0.0f,
385 .maxDepth = 1.0f
386 });
387
388 radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkRect2D) {
389 .offset = (VkOffset2D) { rects[r].dst_x, rects[r].dst_y },
390 .extent = (VkExtent2D) { rects[r].width, rects[r].height },
391 });
392
393
394
395 radv_CmdDraw(radv_cmd_buffer_to_handle(cmd_buffer), 3, 1, 0, 0);
396 radv_cmd_buffer_end_render_pass(cmd_buffer);
397
398 fail_pipeline:
399 /* At the point where we emit the draw call, all data from the
400 * descriptor sets, etc. has been used. We are free to delete it.
401 */
402 radv_DestroyFramebuffer(radv_device_to_handle(device),
403 dst_temps.fb,
404 &cmd_buffer->pool->alloc);
405 }
406 }
407 }
408
409 void
radv_meta_blit2d(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * src_img,struct radv_meta_blit2d_buffer * src_buf,struct radv_meta_blit2d_surf * dst,unsigned num_rects,struct radv_meta_blit2d_rect * rects)410 radv_meta_blit2d(struct radv_cmd_buffer *cmd_buffer,
411 struct radv_meta_blit2d_surf *src_img,
412 struct radv_meta_blit2d_buffer *src_buf,
413 struct radv_meta_blit2d_surf *dst,
414 unsigned num_rects,
415 struct radv_meta_blit2d_rect *rects)
416 {
417 bool use_3d = cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9 &&
418 (src_img && src_img->image->type == VK_IMAGE_TYPE_3D);
419 enum blit2d_src_type src_type = src_buf ? BLIT2D_SRC_TYPE_BUFFER :
420 use_3d ? BLIT2D_SRC_TYPE_IMAGE_3D : BLIT2D_SRC_TYPE_IMAGE;
421 radv_meta_blit2d_normal_dst(cmd_buffer, src_img, src_buf, dst,
422 num_rects, rects, src_type,
423 src_img ? util_logbase2(src_img->image->info.samples) : 0);
424 }
425
426 static nir_shader *
build_nir_vertex_shader(void)427 build_nir_vertex_shader(void)
428 {
429 const struct glsl_type *vec4 = glsl_vec4_type();
430 const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
431 nir_builder b;
432
433 nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_VERTEX, NULL);
434 b.shader->info.name = ralloc_strdup(b.shader, "meta_blit2d_vs");
435
436 nir_variable *pos_out = nir_variable_create(b.shader, nir_var_shader_out,
437 vec4, "gl_Position");
438 pos_out->data.location = VARYING_SLOT_POS;
439
440 nir_variable *tex_pos_out = nir_variable_create(b.shader, nir_var_shader_out,
441 vec2, "v_tex_pos");
442 tex_pos_out->data.location = VARYING_SLOT_VAR0;
443 tex_pos_out->data.interpolation = INTERP_MODE_SMOOTH;
444
445 nir_ssa_def *outvec = radv_meta_gen_rect_vertices(&b);
446 nir_store_var(&b, pos_out, outvec, 0xf);
447
448 nir_intrinsic_instr *src_box = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
449 src_box->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
450 nir_intrinsic_set_base(src_box, 0);
451 nir_intrinsic_set_range(src_box, 16);
452 src_box->num_components = 4;
453 nir_ssa_dest_init(&src_box->instr, &src_box->dest, 4, 32, "src_box");
454 nir_builder_instr_insert(&b, &src_box->instr);
455
456 nir_intrinsic_instr *vertex_id = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_vertex_id_zero_base);
457 nir_ssa_dest_init(&vertex_id->instr, &vertex_id->dest, 1, 32, "vertexid");
458 nir_builder_instr_insert(&b, &vertex_id->instr);
459
460 /* vertex 0 - src_x, src_y */
461 /* vertex 1 - src_x, src_y+h */
462 /* vertex 2 - src_x+w, src_y */
463 /* so channel 0 is vertex_id != 2 ? src_x : src_x + w
464 channel 1 is vertex id != 1 ? src_y : src_y + w */
465
466 nir_ssa_def *c0cmp = nir_ine(&b, &vertex_id->dest.ssa,
467 nir_imm_int(&b, 2));
468 nir_ssa_def *c1cmp = nir_ine(&b, &vertex_id->dest.ssa,
469 nir_imm_int(&b, 1));
470
471 nir_ssa_def *comp[2];
472 comp[0] = nir_bcsel(&b, c0cmp,
473 nir_channel(&b, &src_box->dest.ssa, 0),
474 nir_channel(&b, &src_box->dest.ssa, 2));
475
476 comp[1] = nir_bcsel(&b, c1cmp,
477 nir_channel(&b, &src_box->dest.ssa, 1),
478 nir_channel(&b, &src_box->dest.ssa, 3));
479 nir_ssa_def *out_tex_vec = nir_vec(&b, comp, 2);
480 nir_store_var(&b, tex_pos_out, out_tex_vec, 0x3);
481 return b.shader;
482 }
483
484 typedef nir_ssa_def* (*texel_fetch_build_func)(struct nir_builder *,
485 struct radv_device *,
486 nir_ssa_def *, bool, bool);
487
488 static nir_ssa_def *
build_nir_texel_fetch(struct nir_builder * b,struct radv_device * device,nir_ssa_def * tex_pos,bool is_3d,bool is_multisampled)489 build_nir_texel_fetch(struct nir_builder *b, struct radv_device *device,
490 nir_ssa_def *tex_pos, bool is_3d, bool is_multisampled)
491 {
492 enum glsl_sampler_dim dim =
493 is_3d ? GLSL_SAMPLER_DIM_3D : is_multisampled ? GLSL_SAMPLER_DIM_MS : GLSL_SAMPLER_DIM_2D;
494 const struct glsl_type *sampler_type =
495 glsl_sampler_type(dim, false, false, GLSL_TYPE_UINT);
496 nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform,
497 sampler_type, "s_tex");
498 sampler->data.descriptor_set = 0;
499 sampler->data.binding = 0;
500
501 nir_ssa_def *tex_pos_3d = NULL;
502 nir_intrinsic_instr *sample_idx = NULL;
503 if (is_3d) {
504 nir_intrinsic_instr *layer = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_push_constant);
505 nir_intrinsic_set_base(layer, 16);
506 nir_intrinsic_set_range(layer, 4);
507 layer->src[0] = nir_src_for_ssa(nir_imm_int(b, 0));
508 layer->num_components = 1;
509 nir_ssa_dest_init(&layer->instr, &layer->dest, 1, 32, "layer");
510 nir_builder_instr_insert(b, &layer->instr);
511
512 nir_ssa_def *chans[3];
513 chans[0] = nir_channel(b, tex_pos, 0);
514 chans[1] = nir_channel(b, tex_pos, 1);
515 chans[2] = &layer->dest.ssa;
516 tex_pos_3d = nir_vec(b, chans, 3);
517 }
518 if (is_multisampled) {
519 sample_idx = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_sample_id);
520 nir_ssa_dest_init(&sample_idx->instr, &sample_idx->dest, 1, 32, "sample_idx");
521 nir_builder_instr_insert(b, &sample_idx->instr);
522 }
523
524 nir_ssa_def *tex_deref = &nir_build_deref_var(b, sampler)->dest.ssa;
525
526 nir_tex_instr *tex = nir_tex_instr_create(b->shader, is_multisampled ? 4 : 3);
527 tex->sampler_dim = dim;
528 tex->op = is_multisampled ? nir_texop_txf_ms : nir_texop_txf;
529 tex->src[0].src_type = nir_tex_src_coord;
530 tex->src[0].src = nir_src_for_ssa(is_3d ? tex_pos_3d : tex_pos);
531 tex->src[1].src_type = is_multisampled ? nir_tex_src_ms_index : nir_tex_src_lod;
532 tex->src[1].src = nir_src_for_ssa(is_multisampled ? &sample_idx->dest.ssa : nir_imm_int(b, 0));
533 tex->src[2].src_type = nir_tex_src_texture_deref;
534 tex->src[2].src = nir_src_for_ssa(tex_deref);
535 if (is_multisampled) {
536 tex->src[3].src_type = nir_tex_src_lod;
537 tex->src[3].src = nir_src_for_ssa(nir_imm_int(b, 0));
538 }
539 tex->dest_type = nir_type_uint;
540 tex->is_array = false;
541 tex->coord_components = is_3d ? 3 : 2;
542
543 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
544 nir_builder_instr_insert(b, &tex->instr);
545
546 return &tex->dest.ssa;
547 }
548
549
550 static nir_ssa_def *
build_nir_buffer_fetch(struct nir_builder * b,struct radv_device * device,nir_ssa_def * tex_pos,bool is_3d,bool is_multisampled)551 build_nir_buffer_fetch(struct nir_builder *b, struct radv_device *device,
552 nir_ssa_def *tex_pos, bool is_3d, bool is_multisampled)
553 {
554 const struct glsl_type *sampler_type =
555 glsl_sampler_type(GLSL_SAMPLER_DIM_BUF, false, false, GLSL_TYPE_UINT);
556 nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform,
557 sampler_type, "s_tex");
558 sampler->data.descriptor_set = 0;
559 sampler->data.binding = 0;
560
561 nir_intrinsic_instr *width = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_push_constant);
562 nir_intrinsic_set_base(width, 16);
563 nir_intrinsic_set_range(width, 4);
564 width->src[0] = nir_src_for_ssa(nir_imm_int(b, 0));
565 width->num_components = 1;
566 nir_ssa_dest_init(&width->instr, &width->dest, 1, 32, "width");
567 nir_builder_instr_insert(b, &width->instr);
568
569 nir_ssa_def *pos_x = nir_channel(b, tex_pos, 0);
570 nir_ssa_def *pos_y = nir_channel(b, tex_pos, 1);
571 pos_y = nir_imul(b, pos_y, &width->dest.ssa);
572 pos_x = nir_iadd(b, pos_x, pos_y);
573
574 nir_ssa_def *tex_deref = &nir_build_deref_var(b, sampler)->dest.ssa;
575
576 nir_tex_instr *tex = nir_tex_instr_create(b->shader, 2);
577 tex->sampler_dim = GLSL_SAMPLER_DIM_BUF;
578 tex->op = nir_texop_txf;
579 tex->src[0].src_type = nir_tex_src_coord;
580 tex->src[0].src = nir_src_for_ssa(pos_x);
581 tex->src[1].src_type = nir_tex_src_texture_deref;
582 tex->src[1].src = nir_src_for_ssa(tex_deref);
583 tex->dest_type = nir_type_uint;
584 tex->is_array = false;
585 tex->coord_components = 1;
586
587 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
588 nir_builder_instr_insert(b, &tex->instr);
589
590 return &tex->dest.ssa;
591 }
592
593 static const VkPipelineVertexInputStateCreateInfo normal_vi_create_info = {
594 .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
595 .vertexBindingDescriptionCount = 0,
596 .vertexAttributeDescriptionCount = 0,
597 };
598
599 static nir_shader *
build_nir_copy_fragment_shader(struct radv_device * device,texel_fetch_build_func txf_func,const char * name,bool is_3d,bool is_multisampled)600 build_nir_copy_fragment_shader(struct radv_device *device,
601 texel_fetch_build_func txf_func, const char* name, bool is_3d,
602 bool is_multisampled)
603 {
604 const struct glsl_type *vec4 = glsl_vec4_type();
605 const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
606 nir_builder b;
607
608 nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
609 b.shader->info.name = ralloc_strdup(b.shader, name);
610
611 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in,
612 vec2, "v_tex_pos");
613 tex_pos_in->data.location = VARYING_SLOT_VAR0;
614
615 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out,
616 vec4, "f_color");
617 color_out->data.location = FRAG_RESULT_DATA0;
618
619 nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
620 nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
621
622 nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
623 nir_store_var(&b, color_out, color, 0xf);
624
625 return b.shader;
626 }
627
628 static nir_shader *
build_nir_copy_fragment_shader_depth(struct radv_device * device,texel_fetch_build_func txf_func,const char * name,bool is_3d,bool is_multisampled)629 build_nir_copy_fragment_shader_depth(struct radv_device *device,
630 texel_fetch_build_func txf_func, const char* name, bool is_3d,
631 bool is_multisampled)
632 {
633 const struct glsl_type *vec4 = glsl_vec4_type();
634 const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
635 nir_builder b;
636
637 nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
638 b.shader->info.name = ralloc_strdup(b.shader, name);
639
640 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in,
641 vec2, "v_tex_pos");
642 tex_pos_in->data.location = VARYING_SLOT_VAR0;
643
644 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out,
645 vec4, "f_color");
646 color_out->data.location = FRAG_RESULT_DEPTH;
647
648 nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
649 nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
650
651 nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
652 nir_store_var(&b, color_out, color, 0x1);
653
654 return b.shader;
655 }
656
657 static nir_shader *
build_nir_copy_fragment_shader_stencil(struct radv_device * device,texel_fetch_build_func txf_func,const char * name,bool is_3d,bool is_multisampled)658 build_nir_copy_fragment_shader_stencil(struct radv_device *device,
659 texel_fetch_build_func txf_func, const char* name, bool is_3d,
660 bool is_multisampled)
661 {
662 const struct glsl_type *vec4 = glsl_vec4_type();
663 const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
664 nir_builder b;
665
666 nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
667 b.shader->info.name = ralloc_strdup(b.shader, name);
668
669 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in,
670 vec2, "v_tex_pos");
671 tex_pos_in->data.location = VARYING_SLOT_VAR0;
672
673 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out,
674 vec4, "f_color");
675 color_out->data.location = FRAG_RESULT_STENCIL;
676
677 nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
678 nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
679
680 nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
681 nir_store_var(&b, color_out, color, 0x1);
682
683 return b.shader;
684 }
685
686 void
radv_device_finish_meta_blit2d_state(struct radv_device * device)687 radv_device_finish_meta_blit2d_state(struct radv_device *device)
688 {
689 struct radv_meta_state *state = &device->meta_state;
690
691 for(unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
692 for (unsigned k = 0; k < RADV_META_DST_LAYOUT_COUNT; ++k) {
693 radv_DestroyRenderPass(radv_device_to_handle(device),
694 state->blit2d_render_passes[j][k],
695 &state->alloc);
696 }
697 }
698
699 for (enum radv_blit_ds_layout j = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; j < RADV_BLIT_DS_LAYOUT_COUNT; j++) {
700 radv_DestroyRenderPass(radv_device_to_handle(device),
701 state->blit2d_depth_only_rp[j], &state->alloc);
702 radv_DestroyRenderPass(radv_device_to_handle(device),
703 state->blit2d_stencil_only_rp[j], &state->alloc);
704 }
705
706 for (unsigned log2_samples = 0; log2_samples < MAX_SAMPLES_LOG2; ++log2_samples) {
707 for (unsigned src = 0; src < BLIT2D_NUM_SRC_TYPES; src++) {
708 radv_DestroyPipelineLayout(radv_device_to_handle(device),
709 state->blit2d[log2_samples].p_layouts[src],
710 &state->alloc);
711 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
712 state->blit2d[log2_samples].ds_layouts[src],
713 &state->alloc);
714
715 for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
716 radv_DestroyPipeline(radv_device_to_handle(device),
717 state->blit2d[log2_samples].pipelines[src][j],
718 &state->alloc);
719 }
720
721 radv_DestroyPipeline(radv_device_to_handle(device),
722 state->blit2d[log2_samples].depth_only_pipeline[src],
723 &state->alloc);
724 radv_DestroyPipeline(radv_device_to_handle(device),
725 state->blit2d[log2_samples].stencil_only_pipeline[src],
726 &state->alloc);
727 }
728 }
729 }
730
731 static VkResult
blit2d_init_color_pipeline(struct radv_device * device,enum blit2d_src_type src_type,VkFormat format,uint32_t log2_samples)732 blit2d_init_color_pipeline(struct radv_device *device,
733 enum blit2d_src_type src_type,
734 VkFormat format,
735 uint32_t log2_samples)
736 {
737 VkResult result;
738 unsigned fs_key = radv_format_meta_fs_key(format);
739 const char *name;
740
741 mtx_lock(&device->meta_state.mtx);
742 if (device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key]) {
743 mtx_unlock(&device->meta_state.mtx);
744 return VK_SUCCESS;
745 }
746
747 texel_fetch_build_func src_func;
748 switch(src_type) {
749 case BLIT2D_SRC_TYPE_IMAGE:
750 src_func = build_nir_texel_fetch;
751 name = "meta_blit2d_image_fs";
752 break;
753 case BLIT2D_SRC_TYPE_IMAGE_3D:
754 src_func = build_nir_texel_fetch;
755 name = "meta_blit3d_image_fs";
756 break;
757 case BLIT2D_SRC_TYPE_BUFFER:
758 src_func = build_nir_buffer_fetch;
759 name = "meta_blit2d_buffer_fs";
760 break;
761 default:
762 unreachable("unknown blit src type\n");
763 break;
764 }
765
766 const VkPipelineVertexInputStateCreateInfo *vi_create_info;
767 struct radv_shader_module fs = { .nir = NULL };
768
769
770 fs.nir = build_nir_copy_fragment_shader(device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
771 vi_create_info = &normal_vi_create_info;
772
773 struct radv_shader_module vs = {
774 .nir = build_nir_vertex_shader(),
775 };
776
777 VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
778 {
779 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
780 .stage = VK_SHADER_STAGE_VERTEX_BIT,
781 .module = radv_shader_module_to_handle(&vs),
782 .pName = "main",
783 .pSpecializationInfo = NULL
784 }, {
785 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
786 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
787 .module = radv_shader_module_to_handle(&fs),
788 .pName = "main",
789 .pSpecializationInfo = NULL
790 },
791 };
792
793 for (unsigned dst_layout = 0; dst_layout < RADV_META_DST_LAYOUT_COUNT; ++dst_layout) {
794 if (!device->meta_state.blit2d_render_passes[fs_key][dst_layout]) {
795 VkImageLayout layout = radv_meta_dst_layout_to_layout(dst_layout);
796
797 result = radv_CreateRenderPass(radv_device_to_handle(device),
798 &(VkRenderPassCreateInfo) {
799 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
800 .attachmentCount = 1,
801 .pAttachments = &(VkAttachmentDescription) {
802 .format = format,
803 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
804 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
805 .initialLayout = layout,
806 .finalLayout = layout,
807 },
808 .subpassCount = 1,
809 .pSubpasses = &(VkSubpassDescription) {
810 .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
811 .inputAttachmentCount = 0,
812 .colorAttachmentCount = 1,
813 .pColorAttachments = &(VkAttachmentReference) {
814 .attachment = 0,
815 .layout = layout,
816 },
817 .pResolveAttachments = NULL,
818 .pDepthStencilAttachment = &(VkAttachmentReference) {
819 .attachment = VK_ATTACHMENT_UNUSED,
820 .layout = layout,
821 },
822 .preserveAttachmentCount = 0,
823 .pPreserveAttachments = NULL,
824 },
825 .dependencyCount = 2,
826 .pDependencies = (VkSubpassDependency[]) {
827 {
828 .srcSubpass = VK_SUBPASS_EXTERNAL,
829 .dstSubpass = 0,
830 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
831 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
832 .srcAccessMask = 0,
833 .dstAccessMask = 0,
834 .dependencyFlags = 0
835 },
836 {
837 .srcSubpass = 0,
838 .dstSubpass = VK_SUBPASS_EXTERNAL,
839 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
840 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
841 .srcAccessMask = 0,
842 .dstAccessMask = 0,
843 .dependencyFlags = 0
844 }
845 },
846 }, &device->meta_state.alloc, &device->meta_state.blit2d_render_passes[fs_key][dst_layout]);
847 }
848 }
849
850 const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
851 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
852 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
853 .pStages = pipeline_shader_stages,
854 .pVertexInputState = vi_create_info,
855 .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
856 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
857 .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
858 .primitiveRestartEnable = false,
859 },
860 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
861 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
862 .viewportCount = 1,
863 .scissorCount = 1,
864 },
865 .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
866 .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
867 .rasterizerDiscardEnable = false,
868 .polygonMode = VK_POLYGON_MODE_FILL,
869 .cullMode = VK_CULL_MODE_NONE,
870 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
871 },
872 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
873 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
874 .rasterizationSamples = 1 << log2_samples,
875 .sampleShadingEnable = log2_samples > 1,
876 .minSampleShading = 1.0,
877 .pSampleMask = (VkSampleMask[]) { UINT32_MAX },
878 },
879 .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
880 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
881 .attachmentCount = 1,
882 .pAttachments = (VkPipelineColorBlendAttachmentState []) {
883 { .colorWriteMask =
884 VK_COLOR_COMPONENT_A_BIT |
885 VK_COLOR_COMPONENT_R_BIT |
886 VK_COLOR_COMPONENT_G_BIT |
887 VK_COLOR_COMPONENT_B_BIT },
888 }
889 },
890 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
891 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
892 .dynamicStateCount = 9,
893 .pDynamicStates = (VkDynamicState[]) {
894 VK_DYNAMIC_STATE_VIEWPORT,
895 VK_DYNAMIC_STATE_SCISSOR,
896 VK_DYNAMIC_STATE_LINE_WIDTH,
897 VK_DYNAMIC_STATE_DEPTH_BIAS,
898 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
899 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
900 VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
901 VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
902 VK_DYNAMIC_STATE_STENCIL_REFERENCE,
903 },
904 },
905 .flags = 0,
906 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
907 .renderPass = device->meta_state.blit2d_render_passes[fs_key][0],
908 .subpass = 0,
909 };
910
911 const struct radv_graphics_pipeline_create_info radv_pipeline_info = {
912 .use_rectlist = true
913 };
914
915 result = radv_graphics_pipeline_create(radv_device_to_handle(device),
916 radv_pipeline_cache_to_handle(&device->meta_state.cache),
917 &vk_pipeline_info, &radv_pipeline_info,
918 &device->meta_state.alloc,
919 &device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key]);
920
921
922 ralloc_free(vs.nir);
923 ralloc_free(fs.nir);
924
925 mtx_unlock(&device->meta_state.mtx);
926 return result;
927 }
928
929 static VkResult
blit2d_init_depth_only_pipeline(struct radv_device * device,enum blit2d_src_type src_type,uint32_t log2_samples)930 blit2d_init_depth_only_pipeline(struct radv_device *device,
931 enum blit2d_src_type src_type,
932 uint32_t log2_samples)
933 {
934 VkResult result;
935 const char *name;
936
937 mtx_lock(&device->meta_state.mtx);
938 if (device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type]) {
939 mtx_unlock(&device->meta_state.mtx);
940 return VK_SUCCESS;
941 }
942
943 texel_fetch_build_func src_func;
944 switch(src_type) {
945 case BLIT2D_SRC_TYPE_IMAGE:
946 src_func = build_nir_texel_fetch;
947 name = "meta_blit2d_depth_image_fs";
948 break;
949 case BLIT2D_SRC_TYPE_IMAGE_3D:
950 src_func = build_nir_texel_fetch;
951 name = "meta_blit3d_depth_image_fs";
952 break;
953 case BLIT2D_SRC_TYPE_BUFFER:
954 src_func = build_nir_buffer_fetch;
955 name = "meta_blit2d_depth_buffer_fs";
956 break;
957 default:
958 unreachable("unknown blit src type\n");
959 break;
960 }
961
962 const VkPipelineVertexInputStateCreateInfo *vi_create_info;
963 struct radv_shader_module fs = { .nir = NULL };
964
965 fs.nir = build_nir_copy_fragment_shader_depth(device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
966 vi_create_info = &normal_vi_create_info;
967
968 struct radv_shader_module vs = {
969 .nir = build_nir_vertex_shader(),
970 };
971
972 VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
973 {
974 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
975 .stage = VK_SHADER_STAGE_VERTEX_BIT,
976 .module = radv_shader_module_to_handle(&vs),
977 .pName = "main",
978 .pSpecializationInfo = NULL
979 }, {
980 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
981 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
982 .module = radv_shader_module_to_handle(&fs),
983 .pName = "main",
984 .pSpecializationInfo = NULL
985 },
986 };
987
988 for (enum radv_blit_ds_layout ds_layout = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; ds_layout < RADV_BLIT_DS_LAYOUT_COUNT; ds_layout++) {
989 if (!device->meta_state.blit2d_depth_only_rp[ds_layout]) {
990 VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
991 result = radv_CreateRenderPass(radv_device_to_handle(device),
992 &(VkRenderPassCreateInfo) {
993 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
994 .attachmentCount = 1,
995 .pAttachments = &(VkAttachmentDescription) {
996 .format = VK_FORMAT_D32_SFLOAT,
997 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
998 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
999 .initialLayout = layout,
1000 .finalLayout = layout,
1001 },
1002 .subpassCount = 1,
1003 .pSubpasses = &(VkSubpassDescription) {
1004 .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
1005 .inputAttachmentCount = 0,
1006 .colorAttachmentCount = 0,
1007 .pColorAttachments = NULL,
1008 .pResolveAttachments = NULL,
1009 .pDepthStencilAttachment = &(VkAttachmentReference) {
1010 .attachment = 0,
1011 .layout = layout,
1012 },
1013 .preserveAttachmentCount = 0,
1014 .pPreserveAttachments = NULL,
1015 },
1016 .dependencyCount = 2,
1017 .pDependencies = (VkSubpassDependency[]) {
1018 {
1019 .srcSubpass = VK_SUBPASS_EXTERNAL,
1020 .dstSubpass = 0,
1021 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
1022 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
1023 .srcAccessMask = 0,
1024 .dstAccessMask = 0,
1025 .dependencyFlags = 0
1026 },
1027 {
1028 .srcSubpass = 0,
1029 .dstSubpass = VK_SUBPASS_EXTERNAL,
1030 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
1031 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
1032 .srcAccessMask = 0,
1033 .dstAccessMask = 0,
1034 .dependencyFlags = 0
1035 }
1036 },
1037 }, &device->meta_state.alloc, &device->meta_state.blit2d_depth_only_rp[ds_layout]);
1038 }
1039 }
1040
1041 const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
1042 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
1043 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
1044 .pStages = pipeline_shader_stages,
1045 .pVertexInputState = vi_create_info,
1046 .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
1047 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
1048 .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
1049 .primitiveRestartEnable = false,
1050 },
1051 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
1052 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
1053 .viewportCount = 1,
1054 .scissorCount = 1,
1055 },
1056 .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
1057 .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
1058 .rasterizerDiscardEnable = false,
1059 .polygonMode = VK_POLYGON_MODE_FILL,
1060 .cullMode = VK_CULL_MODE_NONE,
1061 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
1062 },
1063 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
1064 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
1065 .rasterizationSamples = 1 << log2_samples,
1066 .sampleShadingEnable = false,
1067 .pSampleMask = (VkSampleMask[]) { UINT32_MAX },
1068 },
1069 .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
1070 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
1071 .attachmentCount = 0,
1072 .pAttachments = NULL,
1073 },
1074 .pDepthStencilState = &(VkPipelineDepthStencilStateCreateInfo) {
1075 .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
1076 .depthTestEnable = true,
1077 .depthWriteEnable = true,
1078 .depthCompareOp = VK_COMPARE_OP_ALWAYS,
1079 },
1080 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
1081 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
1082 .dynamicStateCount = 9,
1083 .pDynamicStates = (VkDynamicState[]) {
1084 VK_DYNAMIC_STATE_VIEWPORT,
1085 VK_DYNAMIC_STATE_SCISSOR,
1086 VK_DYNAMIC_STATE_LINE_WIDTH,
1087 VK_DYNAMIC_STATE_DEPTH_BIAS,
1088 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
1089 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
1090 VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
1091 VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
1092 VK_DYNAMIC_STATE_STENCIL_REFERENCE,
1093 },
1094 },
1095 .flags = 0,
1096 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
1097 .renderPass = device->meta_state.blit2d_depth_only_rp[0],
1098 .subpass = 0,
1099 };
1100
1101 const struct radv_graphics_pipeline_create_info radv_pipeline_info = {
1102 .use_rectlist = true
1103 };
1104
1105 result = radv_graphics_pipeline_create(radv_device_to_handle(device),
1106 radv_pipeline_cache_to_handle(&device->meta_state.cache),
1107 &vk_pipeline_info, &radv_pipeline_info,
1108 &device->meta_state.alloc,
1109 &device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type]);
1110
1111
1112 ralloc_free(vs.nir);
1113 ralloc_free(fs.nir);
1114
1115 mtx_unlock(&device->meta_state.mtx);
1116 return result;
1117 }
1118
1119 static VkResult
blit2d_init_stencil_only_pipeline(struct radv_device * device,enum blit2d_src_type src_type,uint32_t log2_samples)1120 blit2d_init_stencil_only_pipeline(struct radv_device *device,
1121 enum blit2d_src_type src_type,
1122 uint32_t log2_samples)
1123 {
1124 VkResult result;
1125 const char *name;
1126
1127 mtx_lock(&device->meta_state.mtx);
1128 if (device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type]) {
1129 mtx_unlock(&device->meta_state.mtx);
1130 return VK_SUCCESS;
1131 }
1132
1133 texel_fetch_build_func src_func;
1134 switch(src_type) {
1135 case BLIT2D_SRC_TYPE_IMAGE:
1136 src_func = build_nir_texel_fetch;
1137 name = "meta_blit2d_stencil_image_fs";
1138 break;
1139 case BLIT2D_SRC_TYPE_IMAGE_3D:
1140 src_func = build_nir_texel_fetch;
1141 name = "meta_blit3d_stencil_image_fs";
1142 break;
1143 case BLIT2D_SRC_TYPE_BUFFER:
1144 src_func = build_nir_buffer_fetch;
1145 name = "meta_blit2d_stencil_buffer_fs";
1146 break;
1147 default:
1148 unreachable("unknown blit src type\n");
1149 break;
1150 }
1151
1152 const VkPipelineVertexInputStateCreateInfo *vi_create_info;
1153 struct radv_shader_module fs = { .nir = NULL };
1154
1155 fs.nir = build_nir_copy_fragment_shader_stencil(device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
1156 vi_create_info = &normal_vi_create_info;
1157
1158 struct radv_shader_module vs = {
1159 .nir = build_nir_vertex_shader(),
1160 };
1161
1162 VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
1163 {
1164 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
1165 .stage = VK_SHADER_STAGE_VERTEX_BIT,
1166 .module = radv_shader_module_to_handle(&vs),
1167 .pName = "main",
1168 .pSpecializationInfo = NULL
1169 }, {
1170 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
1171 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
1172 .module = radv_shader_module_to_handle(&fs),
1173 .pName = "main",
1174 .pSpecializationInfo = NULL
1175 },
1176 };
1177
1178 for (enum radv_blit_ds_layout ds_layout = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; ds_layout < RADV_BLIT_DS_LAYOUT_COUNT; ds_layout++) {
1179 if (!device->meta_state.blit2d_stencil_only_rp[ds_layout]) {
1180 VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
1181 result = radv_CreateRenderPass(radv_device_to_handle(device),
1182 &(VkRenderPassCreateInfo) {
1183 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
1184 .attachmentCount = 1,
1185 .pAttachments = &(VkAttachmentDescription) {
1186 .format = VK_FORMAT_S8_UINT,
1187 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
1188 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
1189 .initialLayout = layout,
1190 .finalLayout = layout,
1191 },
1192 .subpassCount = 1,
1193 .pSubpasses = &(VkSubpassDescription) {
1194 .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
1195 .inputAttachmentCount = 0,
1196 .colorAttachmentCount = 0,
1197 .pColorAttachments = NULL,
1198 .pResolveAttachments = NULL,
1199 .pDepthStencilAttachment = &(VkAttachmentReference) {
1200 .attachment = 0,
1201 .layout = layout,
1202 },
1203 .preserveAttachmentCount = 0,
1204 .pPreserveAttachments = NULL,
1205 },
1206 .dependencyCount = 2,
1207 .pDependencies = (VkSubpassDependency[]) {
1208 {
1209 .srcSubpass = VK_SUBPASS_EXTERNAL,
1210 .dstSubpass = 0,
1211 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
1212 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
1213 .srcAccessMask = 0,
1214 .dstAccessMask = 0,
1215 .dependencyFlags = 0
1216 },
1217 {
1218 .srcSubpass = 0,
1219 .dstSubpass = VK_SUBPASS_EXTERNAL,
1220 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
1221 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
1222 .srcAccessMask = 0,
1223 .dstAccessMask = 0,
1224 .dependencyFlags = 0
1225 }
1226 },
1227 }, &device->meta_state.alloc, &device->meta_state.blit2d_stencil_only_rp[ds_layout]);
1228 }
1229 }
1230
1231 const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
1232 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
1233 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
1234 .pStages = pipeline_shader_stages,
1235 .pVertexInputState = vi_create_info,
1236 .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
1237 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
1238 .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
1239 .primitiveRestartEnable = false,
1240 },
1241 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
1242 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
1243 .viewportCount = 1,
1244 .scissorCount = 1,
1245 },
1246 .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
1247 .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
1248 .rasterizerDiscardEnable = false,
1249 .polygonMode = VK_POLYGON_MODE_FILL,
1250 .cullMode = VK_CULL_MODE_NONE,
1251 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
1252 },
1253 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
1254 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
1255 .rasterizationSamples = 1 << log2_samples,
1256 .sampleShadingEnable = false,
1257 .pSampleMask = (VkSampleMask[]) { UINT32_MAX },
1258 },
1259 .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
1260 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
1261 .attachmentCount = 0,
1262 .pAttachments = NULL,
1263 },
1264 .pDepthStencilState = &(VkPipelineDepthStencilStateCreateInfo) {
1265 .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
1266 .depthTestEnable = false,
1267 .depthWriteEnable = false,
1268 .stencilTestEnable = true,
1269 .front = {
1270 .failOp = VK_STENCIL_OP_REPLACE,
1271 .passOp = VK_STENCIL_OP_REPLACE,
1272 .depthFailOp = VK_STENCIL_OP_REPLACE,
1273 .compareOp = VK_COMPARE_OP_ALWAYS,
1274 .compareMask = 0xff,
1275 .writeMask = 0xff,
1276 .reference = 0
1277 },
1278 .back = {
1279 .failOp = VK_STENCIL_OP_REPLACE,
1280 .passOp = VK_STENCIL_OP_REPLACE,
1281 .depthFailOp = VK_STENCIL_OP_REPLACE,
1282 .compareOp = VK_COMPARE_OP_ALWAYS,
1283 .compareMask = 0xff,
1284 .writeMask = 0xff,
1285 .reference = 0
1286 },
1287 .depthCompareOp = VK_COMPARE_OP_ALWAYS,
1288 },
1289 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
1290 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
1291 .dynamicStateCount = 6,
1292 .pDynamicStates = (VkDynamicState[]) {
1293 VK_DYNAMIC_STATE_VIEWPORT,
1294 VK_DYNAMIC_STATE_SCISSOR,
1295 VK_DYNAMIC_STATE_LINE_WIDTH,
1296 VK_DYNAMIC_STATE_DEPTH_BIAS,
1297 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
1298 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
1299 },
1300 },
1301 .flags = 0,
1302 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
1303 .renderPass = device->meta_state.blit2d_stencil_only_rp[0],
1304 .subpass = 0,
1305 };
1306
1307 const struct radv_graphics_pipeline_create_info radv_pipeline_info = {
1308 .use_rectlist = true
1309 };
1310
1311 result = radv_graphics_pipeline_create(radv_device_to_handle(device),
1312 radv_pipeline_cache_to_handle(&device->meta_state.cache),
1313 &vk_pipeline_info, &radv_pipeline_info,
1314 &device->meta_state.alloc,
1315 &device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type]);
1316
1317
1318 ralloc_free(vs.nir);
1319 ralloc_free(fs.nir);
1320
1321 mtx_unlock(&device->meta_state.mtx);
1322 return result;
1323 }
1324
1325 static VkResult
meta_blit2d_create_pipe_layout(struct radv_device * device,int idx,uint32_t log2_samples)1326 meta_blit2d_create_pipe_layout(struct radv_device *device,
1327 int idx,
1328 uint32_t log2_samples)
1329 {
1330 VkResult result;
1331 VkDescriptorType desc_type = (idx == BLIT2D_SRC_TYPE_BUFFER) ? VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER : VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE;
1332 const VkPushConstantRange push_constant_ranges[] = {
1333 {VK_SHADER_STAGE_VERTEX_BIT, 0, 16},
1334 {VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4},
1335 };
1336 int num_push_constant_range = (idx != BLIT2D_SRC_TYPE_IMAGE || log2_samples > 0) ? 2 : 1;
1337
1338 result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
1339 &(VkDescriptorSetLayoutCreateInfo) {
1340 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
1341 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
1342 .bindingCount = 1,
1343 .pBindings = (VkDescriptorSetLayoutBinding[]) {
1344 {
1345 .binding = 0,
1346 .descriptorType = desc_type,
1347 .descriptorCount = 1,
1348 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
1349 .pImmutableSamplers = NULL
1350 },
1351 }
1352 }, &device->meta_state.alloc, &device->meta_state.blit2d[log2_samples].ds_layouts[idx]);
1353 if (result != VK_SUCCESS)
1354 goto fail;
1355
1356 result = radv_CreatePipelineLayout(radv_device_to_handle(device),
1357 &(VkPipelineLayoutCreateInfo) {
1358 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
1359 .setLayoutCount = 1,
1360 .pSetLayouts = &device->meta_state.blit2d[log2_samples].ds_layouts[idx],
1361 .pushConstantRangeCount = num_push_constant_range,
1362 .pPushConstantRanges = push_constant_ranges,
1363 },
1364 &device->meta_state.alloc, &device->meta_state.blit2d[log2_samples].p_layouts[idx]);
1365 if (result != VK_SUCCESS)
1366 goto fail;
1367 return VK_SUCCESS;
1368 fail:
1369 return result;
1370 }
1371
1372 VkResult
radv_device_init_meta_blit2d_state(struct radv_device * device,bool on_demand)1373 radv_device_init_meta_blit2d_state(struct radv_device *device, bool on_demand)
1374 {
1375 VkResult result;
1376 bool create_3d = device->physical_device->rad_info.chip_class >= GFX9;
1377
1378 for (unsigned log2_samples = 0; log2_samples < MAX_SAMPLES_LOG2; log2_samples++) {
1379 for (unsigned src = 0; src < BLIT2D_NUM_SRC_TYPES; src++) {
1380 if (src == BLIT2D_SRC_TYPE_IMAGE_3D && !create_3d)
1381 continue;
1382
1383 /* Don't need to handle copies between buffers and multisample images. */
1384 if (src == BLIT2D_SRC_TYPE_BUFFER && log2_samples > 0)
1385 continue;
1386
1387 result = meta_blit2d_create_pipe_layout(device, src, log2_samples);
1388 if (result != VK_SUCCESS)
1389 goto fail;
1390
1391 if (on_demand)
1392 continue;
1393
1394 for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
1395 result = blit2d_init_color_pipeline(device, src, radv_fs_key_format_exemplars[j], log2_samples);
1396 if (result != VK_SUCCESS)
1397 goto fail;
1398 }
1399
1400 result = blit2d_init_depth_only_pipeline(device, src, log2_samples);
1401 if (result != VK_SUCCESS)
1402 goto fail;
1403
1404 result = blit2d_init_stencil_only_pipeline(device, src, log2_samples);
1405 if (result != VK_SUCCESS)
1406 goto fail;
1407 }
1408 }
1409
1410 return VK_SUCCESS;
1411
1412 fail:
1413 radv_device_finish_meta_blit2d_state(device);
1414 return result;
1415 }
1416