• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright © 2016 Red Hat
3  *
4  * based on anv driver:
5  * Copyright © 2016 Intel Corporation
6  *
7  * Permission is hereby granted, free of charge, to any person obtaining a
8  * copy of this software and associated documentation files (the "Software"),
9  * to deal in the Software without restriction, including without limitation
10  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11  * and/or sell copies of the Software, and to permit persons to whom the
12  * Software is furnished to do so, subject to the following conditions:
13  *
14  * The above copyright notice and this permission notice (including the next
15  * paragraph) shall be included in all copies or substantial portions of the
16  * Software.
17  *
18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
21  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24  * IN THE SOFTWARE.
25  */
26 
27 #include "nir/nir_builder.h"
28 #include "radv_meta.h"
29 #include "vk_format.h"
30 
31 enum blit2d_src_type {
32    BLIT2D_SRC_TYPE_IMAGE,
33    BLIT2D_SRC_TYPE_IMAGE_3D,
34    BLIT2D_SRC_TYPE_BUFFER,
35    BLIT2D_NUM_SRC_TYPES,
36 };
37 
38 static VkResult blit2d_init_color_pipeline(struct radv_device *device,
39                                            enum blit2d_src_type src_type, VkFormat format,
40                                            uint32_t log2_samples);
41 
42 static VkResult blit2d_init_depth_only_pipeline(struct radv_device *device,
43                                                 enum blit2d_src_type src_type,
44                                                 uint32_t log2_samples);
45 
46 static VkResult blit2d_init_stencil_only_pipeline(struct radv_device *device,
47                                                   enum blit2d_src_type src_type,
48                                                   uint32_t log2_samples);
49 
50 static void
create_iview(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * surf,struct radv_image_view * iview,VkFormat depth_format,VkImageAspectFlagBits aspects)51 create_iview(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *surf,
52              struct radv_image_view *iview, VkFormat depth_format, VkImageAspectFlagBits aspects)
53 {
54    VkFormat format;
55    VkImageViewType view_type = cmd_buffer->device->physical_device->rad_info.chip_class < GFX9
56                                   ? VK_IMAGE_VIEW_TYPE_2D
57                                   : radv_meta_get_view_type(surf->image);
58 
59    if (depth_format)
60       format = depth_format;
61    else
62       format = surf->format;
63 
64    radv_image_view_init(iview, cmd_buffer->device,
65                         &(VkImageViewCreateInfo){
66                            .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
67                            .image = radv_image_to_handle(surf->image),
68                            .viewType = view_type,
69                            .format = format,
70                            .subresourceRange = {.aspectMask = aspects,
71                                                 .baseMipLevel = surf->level,
72                                                 .levelCount = 1,
73                                                 .baseArrayLayer = surf->layer,
74                                                 .layerCount = 1},
75                         },
76                         NULL);
77 }
78 
79 static void
create_bview(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_buffer * src,struct radv_buffer_view * bview,VkFormat depth_format)80 create_bview(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_buffer *src,
81              struct radv_buffer_view *bview, VkFormat depth_format)
82 {
83    VkFormat format;
84 
85    if (depth_format)
86       format = depth_format;
87    else
88       format = src->format;
89    radv_buffer_view_init(bview, cmd_buffer->device,
90                          &(VkBufferViewCreateInfo){
91                             .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO,
92                             .flags = 0,
93                             .buffer = radv_buffer_to_handle(src->buffer),
94                             .format = format,
95                             .offset = src->offset,
96                             .range = VK_WHOLE_SIZE,
97                          });
98 }
99 
100 struct blit2d_src_temps {
101    struct radv_image_view iview;
102    struct radv_buffer_view bview;
103 };
104 
105 static void
blit2d_bind_src(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * src_img,struct radv_meta_blit2d_buffer * src_buf,struct blit2d_src_temps * tmp,enum blit2d_src_type src_type,VkFormat depth_format,VkImageAspectFlagBits aspects,uint32_t log2_samples)106 blit2d_bind_src(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src_img,
107                 struct radv_meta_blit2d_buffer *src_buf, struct blit2d_src_temps *tmp,
108                 enum blit2d_src_type src_type, VkFormat depth_format, VkImageAspectFlagBits aspects,
109                 uint32_t log2_samples)
110 {
111    struct radv_device *device = cmd_buffer->device;
112 
113    if (src_type == BLIT2D_SRC_TYPE_BUFFER) {
114       create_bview(cmd_buffer, src_buf, &tmp->bview, depth_format);
115 
116       radv_meta_push_descriptor_set(
117          cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
118          device->meta_state.blit2d[log2_samples].p_layouts[src_type], 0, /* set */
119          1,                                                              /* descriptorWriteCount */
120          (VkWriteDescriptorSet[]){
121             {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
122              .dstBinding = 0,
123              .dstArrayElement = 0,
124              .descriptorCount = 1,
125              .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER,
126              .pTexelBufferView = (VkBufferView[]){radv_buffer_view_to_handle(&tmp->bview)}}});
127 
128       radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
129                             device->meta_state.blit2d[log2_samples].p_layouts[src_type],
130                             VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4, &src_buf->pitch);
131    } else {
132       create_iview(cmd_buffer, src_img, &tmp->iview, depth_format, aspects);
133 
134       if (src_type == BLIT2D_SRC_TYPE_IMAGE_3D)
135          radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
136                                device->meta_state.blit2d[log2_samples].p_layouts[src_type],
137                                VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4, &src_img->layer);
138 
139       radv_meta_push_descriptor_set(
140          cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
141          device->meta_state.blit2d[log2_samples].p_layouts[src_type], 0, /* set */
142          1,                                                              /* descriptorWriteCount */
143          (VkWriteDescriptorSet[]){{.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
144                                    .dstBinding = 0,
145                                    .dstArrayElement = 0,
146                                    .descriptorCount = 1,
147                                    .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
148                                    .pImageInfo = (VkDescriptorImageInfo[]){
149                                       {
150                                          .sampler = VK_NULL_HANDLE,
151                                          .imageView = radv_image_view_to_handle(&tmp->iview),
152                                          .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
153                                       },
154                                    }}});
155    }
156 }
157 
158 struct blit2d_dst_temps {
159    VkImage image;
160    struct radv_image_view iview;
161    VkFramebuffer fb;
162 };
163 
164 static void
blit2d_bind_dst(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * dst,uint32_t width,uint32_t height,VkFormat depth_format,struct blit2d_dst_temps * tmp,VkImageAspectFlagBits aspects)165 blit2d_bind_dst(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *dst,
166                 uint32_t width, uint32_t height, VkFormat depth_format,
167                 struct blit2d_dst_temps *tmp, VkImageAspectFlagBits aspects)
168 {
169    create_iview(cmd_buffer, dst, &tmp->iview, depth_format, aspects);
170 
171    radv_CreateFramebuffer(
172       radv_device_to_handle(cmd_buffer->device),
173       &(VkFramebufferCreateInfo){.sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
174                                  .attachmentCount = 1,
175                                  .pAttachments =
176                                     (VkImageView[]){
177                                        radv_image_view_to_handle(&tmp->iview),
178                                     },
179                                  .width = width,
180                                  .height = height,
181                                  .layers = 1},
182       &cmd_buffer->pool->alloc, &tmp->fb);
183 }
184 
185 static void
bind_pipeline(struct radv_cmd_buffer * cmd_buffer,enum blit2d_src_type src_type,unsigned fs_key,uint32_t log2_samples)186 bind_pipeline(struct radv_cmd_buffer *cmd_buffer, enum blit2d_src_type src_type, unsigned fs_key,
187               uint32_t log2_samples)
188 {
189    VkPipeline pipeline =
190       cmd_buffer->device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key];
191 
192    radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS,
193                         pipeline);
194 }
195 
196 static void
bind_depth_pipeline(struct radv_cmd_buffer * cmd_buffer,enum blit2d_src_type src_type,uint32_t log2_samples)197 bind_depth_pipeline(struct radv_cmd_buffer *cmd_buffer, enum blit2d_src_type src_type,
198                     uint32_t log2_samples)
199 {
200    VkPipeline pipeline =
201       cmd_buffer->device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type];
202 
203    radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS,
204                         pipeline);
205 }
206 
207 static void
bind_stencil_pipeline(struct radv_cmd_buffer * cmd_buffer,enum blit2d_src_type src_type,uint32_t log2_samples)208 bind_stencil_pipeline(struct radv_cmd_buffer *cmd_buffer, enum blit2d_src_type src_type,
209                       uint32_t log2_samples)
210 {
211    VkPipeline pipeline =
212       cmd_buffer->device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type];
213 
214    radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS,
215                         pipeline);
216 }
217 
218 static void
radv_meta_blit2d_normal_dst(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * src_img,struct radv_meta_blit2d_buffer * src_buf,struct radv_meta_blit2d_surf * dst,unsigned num_rects,struct radv_meta_blit2d_rect * rects,enum blit2d_src_type src_type,uint32_t log2_samples)219 radv_meta_blit2d_normal_dst(struct radv_cmd_buffer *cmd_buffer,
220                             struct radv_meta_blit2d_surf *src_img,
221                             struct radv_meta_blit2d_buffer *src_buf,
222                             struct radv_meta_blit2d_surf *dst, unsigned num_rects,
223                             struct radv_meta_blit2d_rect *rects, enum blit2d_src_type src_type,
224                             uint32_t log2_samples)
225 {
226    struct radv_device *device = cmd_buffer->device;
227 
228    for (unsigned r = 0; r < num_rects; ++r) {
229       u_foreach_bit(i, dst->aspect_mask)
230       {
231          unsigned aspect_mask = 1u << i;
232          unsigned src_aspect_mask = aspect_mask;
233          VkFormat depth_format = 0;
234          if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT)
235             depth_format = vk_format_stencil_only(dst->image->vk_format);
236          else if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT)
237             depth_format = vk_format_depth_only(dst->image->vk_format);
238          else if (src_img)
239             src_aspect_mask = src_img->aspect_mask;
240 
241          struct blit2d_src_temps src_temps;
242          blit2d_bind_src(cmd_buffer, src_img, src_buf, &src_temps, src_type, depth_format,
243                          src_aspect_mask, log2_samples);
244 
245          struct blit2d_dst_temps dst_temps;
246          blit2d_bind_dst(cmd_buffer, dst, rects[r].dst_x + rects[r].width,
247                          rects[r].dst_y + rects[r].height, depth_format, &dst_temps, aspect_mask);
248 
249          float vertex_push_constants[4] = {
250             rects[r].src_x,
251             rects[r].src_y,
252             rects[r].src_x + rects[r].width,
253             rects[r].src_y + rects[r].height,
254          };
255 
256          radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
257                                device->meta_state.blit2d[log2_samples].p_layouts[src_type],
258                                VK_SHADER_STAGE_VERTEX_BIT, 0, 16, vertex_push_constants);
259 
260          if (aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT ||
261              aspect_mask == VK_IMAGE_ASPECT_PLANE_0_BIT ||
262              aspect_mask == VK_IMAGE_ASPECT_PLANE_1_BIT ||
263              aspect_mask == VK_IMAGE_ASPECT_PLANE_2_BIT) {
264             unsigned fs_key = radv_format_meta_fs_key(device, dst_temps.iview.vk_format);
265             unsigned dst_layout = radv_meta_dst_layout_from_layout(dst->current_layout);
266 
267             if (device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key] ==
268                 VK_NULL_HANDLE) {
269                VkResult ret = blit2d_init_color_pipeline(
270                   device, src_type, radv_fs_key_format_exemplars[fs_key], log2_samples);
271                if (ret != VK_SUCCESS) {
272                   cmd_buffer->record_result = ret;
273                   goto fail_pipeline;
274                }
275             }
276 
277             radv_cmd_buffer_begin_render_pass(
278                cmd_buffer,
279                &(VkRenderPassBeginInfo){
280                   .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
281                   .renderPass = device->meta_state.blit2d_render_passes[fs_key][dst_layout],
282                   .framebuffer = dst_temps.fb,
283                   .renderArea =
284                      {
285                         .offset =
286                            {
287                               rects[r].dst_x,
288                               rects[r].dst_y,
289                            },
290                         .extent = {rects[r].width, rects[r].height},
291                      },
292                   .clearValueCount = 0,
293                   .pClearValues = NULL,
294                },
295                &(struct radv_extra_render_pass_begin_info){.disable_dcc =
296                                                               dst->disable_compression});
297 
298             radv_cmd_buffer_set_subpass(cmd_buffer, &cmd_buffer->state.pass->subpasses[0]);
299 
300             bind_pipeline(cmd_buffer, src_type, fs_key, log2_samples);
301          } else if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT) {
302             enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst->current_layout);
303 
304             if (device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type] ==
305                 VK_NULL_HANDLE) {
306                VkResult ret = blit2d_init_depth_only_pipeline(device, src_type, log2_samples);
307                if (ret != VK_SUCCESS) {
308                   cmd_buffer->record_result = ret;
309                   goto fail_pipeline;
310                }
311             }
312 
313             radv_cmd_buffer_begin_render_pass(
314                cmd_buffer,
315                &(VkRenderPassBeginInfo){
316                   .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
317                   .renderPass = device->meta_state.blit2d_depth_only_rp[ds_layout],
318                   .framebuffer = dst_temps.fb,
319                   .renderArea =
320                      {
321                         .offset =
322                            {
323                               rects[r].dst_x,
324                               rects[r].dst_y,
325                            },
326                         .extent = {rects[r].width, rects[r].height},
327                      },
328                   .clearValueCount = 0,
329                   .pClearValues = NULL,
330                },
331                NULL);
332 
333             radv_cmd_buffer_set_subpass(cmd_buffer, &cmd_buffer->state.pass->subpasses[0]);
334 
335             bind_depth_pipeline(cmd_buffer, src_type, log2_samples);
336 
337          } else if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT) {
338             enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst->current_layout);
339 
340             if (device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type] ==
341                 VK_NULL_HANDLE) {
342                VkResult ret = blit2d_init_stencil_only_pipeline(device, src_type, log2_samples);
343                if (ret != VK_SUCCESS) {
344                   cmd_buffer->record_result = ret;
345                   goto fail_pipeline;
346                }
347             }
348 
349             radv_cmd_buffer_begin_render_pass(
350                cmd_buffer,
351                &(VkRenderPassBeginInfo){
352                   .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
353                   .renderPass = device->meta_state.blit2d_stencil_only_rp[ds_layout],
354                   .framebuffer = dst_temps.fb,
355                   .renderArea =
356                      {
357                         .offset =
358                            {
359                               rects[r].dst_x,
360                               rects[r].dst_y,
361                            },
362                         .extent = {rects[r].width, rects[r].height},
363                      },
364                   .clearValueCount = 0,
365                   .pClearValues = NULL,
366                },
367                NULL);
368 
369             radv_cmd_buffer_set_subpass(cmd_buffer, &cmd_buffer->state.pass->subpasses[0]);
370 
371             bind_stencil_pipeline(cmd_buffer, src_type, log2_samples);
372          } else
373             unreachable("Processing blit2d with multiple aspects.");
374 
375          radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
376                              &(VkViewport){.x = rects[r].dst_x,
377                                            .y = rects[r].dst_y,
378                                            .width = rects[r].width,
379                                            .height = rects[r].height,
380                                            .minDepth = 0.0f,
381                                            .maxDepth = 1.0f});
382 
383          radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
384                             &(VkRect2D){
385                                .offset = (VkOffset2D){rects[r].dst_x, rects[r].dst_y},
386                                .extent = (VkExtent2D){rects[r].width, rects[r].height},
387                             });
388 
389          radv_CmdDraw(radv_cmd_buffer_to_handle(cmd_buffer), 3, 1, 0, 0);
390          radv_cmd_buffer_end_render_pass(cmd_buffer);
391 
392       fail_pipeline:
393          /* At the point where we emit the draw call, all data from the
394           * descriptor sets, etc. has been used.  We are free to delete it.
395           */
396          radv_DestroyFramebuffer(radv_device_to_handle(device), dst_temps.fb,
397                                  &cmd_buffer->pool->alloc);
398 
399          if (src_type == BLIT2D_SRC_TYPE_BUFFER)
400             radv_buffer_view_finish(&src_temps.bview);
401          else
402             radv_image_view_finish(&dst_temps.iview);
403 
404          radv_image_view_finish(&dst_temps.iview);
405       }
406    }
407 }
408 
409 void
radv_meta_blit2d(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * src_img,struct radv_meta_blit2d_buffer * src_buf,struct radv_meta_blit2d_surf * dst,unsigned num_rects,struct radv_meta_blit2d_rect * rects)410 radv_meta_blit2d(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src_img,
411                  struct radv_meta_blit2d_buffer *src_buf, struct radv_meta_blit2d_surf *dst,
412                  unsigned num_rects, struct radv_meta_blit2d_rect *rects)
413 {
414    bool use_3d = cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9 &&
415                  (src_img && src_img->image->type == VK_IMAGE_TYPE_3D);
416    enum blit2d_src_type src_type = src_buf  ? BLIT2D_SRC_TYPE_BUFFER
417                                    : use_3d ? BLIT2D_SRC_TYPE_IMAGE_3D
418                                             : BLIT2D_SRC_TYPE_IMAGE;
419    radv_meta_blit2d_normal_dst(cmd_buffer, src_img, src_buf, dst, num_rects, rects, src_type,
420                                src_img ? util_logbase2(src_img->image->info.samples) : 0);
421 }
422 
423 static nir_shader *
build_nir_vertex_shader(void)424 build_nir_vertex_shader(void)
425 {
426    const struct glsl_type *vec4 = glsl_vec4_type();
427    const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
428    nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_VERTEX, NULL, "meta_blit2d_vs");
429 
430    nir_variable *pos_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "gl_Position");
431    pos_out->data.location = VARYING_SLOT_POS;
432 
433    nir_variable *tex_pos_out = nir_variable_create(b.shader, nir_var_shader_out, vec2, "v_tex_pos");
434    tex_pos_out->data.location = VARYING_SLOT_VAR0;
435    tex_pos_out->data.interpolation = INTERP_MODE_SMOOTH;
436 
437    nir_ssa_def *outvec = radv_meta_gen_rect_vertices(&b);
438    nir_store_var(&b, pos_out, outvec, 0xf);
439 
440    nir_ssa_def *src_box = nir_load_push_constant(&b, 4, 32, nir_imm_int(&b, 0), .range = 16);
441    nir_ssa_def *vertex_id = nir_load_vertex_id_zero_base(&b);
442 
443    /* vertex 0 - src_x, src_y */
444    /* vertex 1 - src_x, src_y+h */
445    /* vertex 2 - src_x+w, src_y */
446    /* so channel 0 is vertex_id != 2 ? src_x : src_x + w
447       channel 1 is vertex id != 1 ? src_y : src_y + w */
448 
449    nir_ssa_def *c0cmp = nir_ine(&b, vertex_id, nir_imm_int(&b, 2));
450    nir_ssa_def *c1cmp = nir_ine(&b, vertex_id, nir_imm_int(&b, 1));
451 
452    nir_ssa_def *comp[2];
453    comp[0] = nir_bcsel(&b, c0cmp, nir_channel(&b, src_box, 0), nir_channel(&b, src_box, 2));
454 
455    comp[1] = nir_bcsel(&b, c1cmp, nir_channel(&b, src_box, 1), nir_channel(&b, src_box, 3));
456    nir_ssa_def *out_tex_vec = nir_vec(&b, comp, 2);
457    nir_store_var(&b, tex_pos_out, out_tex_vec, 0x3);
458    return b.shader;
459 }
460 
461 typedef nir_ssa_def *(*texel_fetch_build_func)(struct nir_builder *, struct radv_device *,
462                                                nir_ssa_def *, bool, bool);
463 
464 static nir_ssa_def *
build_nir_texel_fetch(struct nir_builder * b,struct radv_device * device,nir_ssa_def * tex_pos,bool is_3d,bool is_multisampled)465 build_nir_texel_fetch(struct nir_builder *b, struct radv_device *device, nir_ssa_def *tex_pos,
466                       bool is_3d, bool is_multisampled)
467 {
468    enum glsl_sampler_dim dim = is_3d             ? GLSL_SAMPLER_DIM_3D
469                                : is_multisampled ? GLSL_SAMPLER_DIM_MS
470                                                  : GLSL_SAMPLER_DIM_2D;
471    const struct glsl_type *sampler_type = glsl_sampler_type(dim, false, false, GLSL_TYPE_UINT);
472    nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform, sampler_type, "s_tex");
473    sampler->data.descriptor_set = 0;
474    sampler->data.binding = 0;
475 
476    nir_ssa_def *tex_pos_3d = NULL;
477    nir_ssa_def *sample_idx = NULL;
478    if (is_3d) {
479       nir_ssa_def *layer =
480          nir_load_push_constant(b, 1, 32, nir_imm_int(b, 0), .base = 16, .range = 4);
481 
482       nir_ssa_def *chans[3];
483       chans[0] = nir_channel(b, tex_pos, 0);
484       chans[1] = nir_channel(b, tex_pos, 1);
485       chans[2] = layer;
486       tex_pos_3d = nir_vec(b, chans, 3);
487    }
488    if (is_multisampled) {
489       sample_idx = nir_load_sample_id(b);
490    }
491 
492    nir_ssa_def *tex_deref = &nir_build_deref_var(b, sampler)->dest.ssa;
493 
494    nir_tex_instr *tex = nir_tex_instr_create(b->shader, is_multisampled ? 4 : 3);
495    tex->sampler_dim = dim;
496    tex->op = is_multisampled ? nir_texop_txf_ms : nir_texop_txf;
497    tex->src[0].src_type = nir_tex_src_coord;
498    tex->src[0].src = nir_src_for_ssa(is_3d ? tex_pos_3d : tex_pos);
499    tex->src[1].src_type = is_multisampled ? nir_tex_src_ms_index : nir_tex_src_lod;
500    tex->src[1].src = nir_src_for_ssa(is_multisampled ? sample_idx : nir_imm_int(b, 0));
501    tex->src[2].src_type = nir_tex_src_texture_deref;
502    tex->src[2].src = nir_src_for_ssa(tex_deref);
503    if (is_multisampled) {
504       tex->src[3].src_type = nir_tex_src_lod;
505       tex->src[3].src = nir_src_for_ssa(nir_imm_int(b, 0));
506    }
507    tex->dest_type = nir_type_uint32;
508    tex->is_array = false;
509    tex->coord_components = is_3d ? 3 : 2;
510 
511    nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
512    nir_builder_instr_insert(b, &tex->instr);
513 
514    return &tex->dest.ssa;
515 }
516 
517 static nir_ssa_def *
build_nir_buffer_fetch(struct nir_builder * b,struct radv_device * device,nir_ssa_def * tex_pos,bool is_3d,bool is_multisampled)518 build_nir_buffer_fetch(struct nir_builder *b, struct radv_device *device, nir_ssa_def *tex_pos,
519                        bool is_3d, bool is_multisampled)
520 {
521    const struct glsl_type *sampler_type =
522       glsl_sampler_type(GLSL_SAMPLER_DIM_BUF, false, false, GLSL_TYPE_UINT);
523    nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform, sampler_type, "s_tex");
524    sampler->data.descriptor_set = 0;
525    sampler->data.binding = 0;
526 
527    nir_ssa_def *width = nir_load_push_constant(b, 1, 32, nir_imm_int(b, 0), .base = 16, .range = 4);
528 
529    nir_ssa_def *pos_x = nir_channel(b, tex_pos, 0);
530    nir_ssa_def *pos_y = nir_channel(b, tex_pos, 1);
531    pos_y = nir_imul(b, pos_y, width);
532    pos_x = nir_iadd(b, pos_x, pos_y);
533 
534    nir_ssa_def *tex_deref = &nir_build_deref_var(b, sampler)->dest.ssa;
535 
536    nir_tex_instr *tex = nir_tex_instr_create(b->shader, 2);
537    tex->sampler_dim = GLSL_SAMPLER_DIM_BUF;
538    tex->op = nir_texop_txf;
539    tex->src[0].src_type = nir_tex_src_coord;
540    tex->src[0].src = nir_src_for_ssa(pos_x);
541    tex->src[1].src_type = nir_tex_src_texture_deref;
542    tex->src[1].src = nir_src_for_ssa(tex_deref);
543    tex->dest_type = nir_type_uint32;
544    tex->is_array = false;
545    tex->coord_components = 1;
546 
547    nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
548    nir_builder_instr_insert(b, &tex->instr);
549 
550    return &tex->dest.ssa;
551 }
552 
553 static const VkPipelineVertexInputStateCreateInfo normal_vi_create_info = {
554    .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
555    .vertexBindingDescriptionCount = 0,
556    .vertexAttributeDescriptionCount = 0,
557 };
558 
559 static nir_shader *
build_nir_copy_fragment_shader(struct radv_device * device,texel_fetch_build_func txf_func,const char * name,bool is_3d,bool is_multisampled)560 build_nir_copy_fragment_shader(struct radv_device *device, texel_fetch_build_func txf_func,
561                                const char *name, bool is_3d, bool is_multisampled)
562 {
563    const struct glsl_type *vec4 = glsl_vec4_type();
564    const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
565    nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, NULL, "%s", name);
566 
567    nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec2, "v_tex_pos");
568    tex_pos_in->data.location = VARYING_SLOT_VAR0;
569 
570    nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
571    color_out->data.location = FRAG_RESULT_DATA0;
572 
573    nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
574    nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
575 
576    nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
577    nir_store_var(&b, color_out, color, 0xf);
578 
579    return b.shader;
580 }
581 
582 static nir_shader *
build_nir_copy_fragment_shader_depth(struct radv_device * device,texel_fetch_build_func txf_func,const char * name,bool is_3d,bool is_multisampled)583 build_nir_copy_fragment_shader_depth(struct radv_device *device, texel_fetch_build_func txf_func,
584                                      const char *name, bool is_3d, bool is_multisampled)
585 {
586    const struct glsl_type *vec4 = glsl_vec4_type();
587    const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
588    nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, NULL, "%s", name);
589 
590    nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec2, "v_tex_pos");
591    tex_pos_in->data.location = VARYING_SLOT_VAR0;
592 
593    nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
594    color_out->data.location = FRAG_RESULT_DEPTH;
595 
596    nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
597    nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
598 
599    nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
600    nir_store_var(&b, color_out, color, 0x1);
601 
602    return b.shader;
603 }
604 
605 static nir_shader *
build_nir_copy_fragment_shader_stencil(struct radv_device * device,texel_fetch_build_func txf_func,const char * name,bool is_3d,bool is_multisampled)606 build_nir_copy_fragment_shader_stencil(struct radv_device *device, texel_fetch_build_func txf_func,
607                                        const char *name, bool is_3d, bool is_multisampled)
608 {
609    const struct glsl_type *vec4 = glsl_vec4_type();
610    const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
611    nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, NULL, "%s", name);
612 
613    nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec2, "v_tex_pos");
614    tex_pos_in->data.location = VARYING_SLOT_VAR0;
615 
616    nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
617    color_out->data.location = FRAG_RESULT_STENCIL;
618 
619    nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
620    nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
621 
622    nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
623    nir_store_var(&b, color_out, color, 0x1);
624 
625    return b.shader;
626 }
627 
628 void
radv_device_finish_meta_blit2d_state(struct radv_device * device)629 radv_device_finish_meta_blit2d_state(struct radv_device *device)
630 {
631    struct radv_meta_state *state = &device->meta_state;
632 
633    for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
634       for (unsigned k = 0; k < RADV_META_DST_LAYOUT_COUNT; ++k) {
635          radv_DestroyRenderPass(radv_device_to_handle(device), state->blit2d_render_passes[j][k],
636                                 &state->alloc);
637       }
638    }
639 
640    for (enum radv_blit_ds_layout j = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; j < RADV_BLIT_DS_LAYOUT_COUNT;
641         j++) {
642       radv_DestroyRenderPass(radv_device_to_handle(device), state->blit2d_depth_only_rp[j],
643                              &state->alloc);
644       radv_DestroyRenderPass(radv_device_to_handle(device), state->blit2d_stencil_only_rp[j],
645                              &state->alloc);
646    }
647 
648    for (unsigned log2_samples = 0; log2_samples < MAX_SAMPLES_LOG2; ++log2_samples) {
649       for (unsigned src = 0; src < BLIT2D_NUM_SRC_TYPES; src++) {
650          radv_DestroyPipelineLayout(radv_device_to_handle(device),
651                                     state->blit2d[log2_samples].p_layouts[src], &state->alloc);
652          radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
653                                          state->blit2d[log2_samples].ds_layouts[src],
654                                          &state->alloc);
655 
656          for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
657             radv_DestroyPipeline(radv_device_to_handle(device),
658                                  state->blit2d[log2_samples].pipelines[src][j], &state->alloc);
659          }
660 
661          radv_DestroyPipeline(radv_device_to_handle(device),
662                               state->blit2d[log2_samples].depth_only_pipeline[src], &state->alloc);
663          radv_DestroyPipeline(radv_device_to_handle(device),
664                               state->blit2d[log2_samples].stencil_only_pipeline[src],
665                               &state->alloc);
666       }
667    }
668 }
669 
670 static VkResult
blit2d_init_color_pipeline(struct radv_device * device,enum blit2d_src_type src_type,VkFormat format,uint32_t log2_samples)671 blit2d_init_color_pipeline(struct radv_device *device, enum blit2d_src_type src_type,
672                            VkFormat format, uint32_t log2_samples)
673 {
674    VkResult result;
675    unsigned fs_key = radv_format_meta_fs_key(device, format);
676    const char *name;
677 
678    mtx_lock(&device->meta_state.mtx);
679    if (device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key]) {
680       mtx_unlock(&device->meta_state.mtx);
681       return VK_SUCCESS;
682    }
683 
684    texel_fetch_build_func src_func;
685    switch (src_type) {
686    case BLIT2D_SRC_TYPE_IMAGE:
687       src_func = build_nir_texel_fetch;
688       name = "meta_blit2d_image_fs";
689       break;
690    case BLIT2D_SRC_TYPE_IMAGE_3D:
691       src_func = build_nir_texel_fetch;
692       name = "meta_blit3d_image_fs";
693       break;
694    case BLIT2D_SRC_TYPE_BUFFER:
695       src_func = build_nir_buffer_fetch;
696       name = "meta_blit2d_buffer_fs";
697       break;
698    default:
699       unreachable("unknown blit src type\n");
700       break;
701    }
702 
703    const VkPipelineVertexInputStateCreateInfo *vi_create_info;
704    nir_shader *fs = build_nir_copy_fragment_shader(
705       device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
706    nir_shader *vs = build_nir_vertex_shader();
707 
708    vi_create_info = &normal_vi_create_info;
709 
710    VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
711       {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
712        .stage = VK_SHADER_STAGE_VERTEX_BIT,
713        .module = vk_shader_module_handle_from_nir(vs),
714        .pName = "main",
715        .pSpecializationInfo = NULL},
716       {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
717        .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
718        .module = vk_shader_module_handle_from_nir(fs),
719        .pName = "main",
720        .pSpecializationInfo = NULL},
721    };
722 
723    for (unsigned dst_layout = 0; dst_layout < RADV_META_DST_LAYOUT_COUNT; ++dst_layout) {
724       if (!device->meta_state.blit2d_render_passes[fs_key][dst_layout]) {
725          VkImageLayout layout = radv_meta_dst_layout_to_layout(dst_layout);
726 
727          result = radv_CreateRenderPass2(
728             radv_device_to_handle(device),
729             &(VkRenderPassCreateInfo2){
730                .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO_2,
731                .attachmentCount = 1,
732                .pAttachments =
733                   &(VkAttachmentDescription2){
734                      .sType = VK_STRUCTURE_TYPE_ATTACHMENT_DESCRIPTION_2,
735                      .format = format,
736                      .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
737                      .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
738                      .initialLayout = layout,
739                      .finalLayout = layout,
740                   },
741                .subpassCount = 1,
742                .pSubpasses =
743                   &(VkSubpassDescription2){
744                      .sType = VK_STRUCTURE_TYPE_SUBPASS_DESCRIPTION_2,
745                      .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
746                      .inputAttachmentCount = 0,
747                      .colorAttachmentCount = 1,
748                      .pColorAttachments =
749                         &(VkAttachmentReference2){
750                            .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2,
751                            .attachment = 0,
752                            .layout = layout,
753                         },
754                      .pResolveAttachments = NULL,
755                      .pDepthStencilAttachment =
756                         &(VkAttachmentReference2){
757                            .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2,
758                            .attachment = VK_ATTACHMENT_UNUSED,
759                            .layout = layout,
760                         },
761                      .preserveAttachmentCount = 0,
762                      .pPreserveAttachments = NULL,
763                   },
764                .dependencyCount = 2,
765                .pDependencies =
766                   (VkSubpassDependency2[]){{.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2,
767                                             .srcSubpass = VK_SUBPASS_EXTERNAL,
768                                             .dstSubpass = 0,
769                                             .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
770                                             .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
771                                             .srcAccessMask = 0,
772                                             .dstAccessMask = 0,
773                                             .dependencyFlags = 0},
774                                            {.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2,
775                                             .srcSubpass = 0,
776                                             .dstSubpass = VK_SUBPASS_EXTERNAL,
777                                             .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
778                                             .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
779                                             .srcAccessMask = 0,
780                                             .dstAccessMask = 0,
781                                             .dependencyFlags = 0}},
782             },
783             &device->meta_state.alloc,
784             &device->meta_state.blit2d_render_passes[fs_key][dst_layout]);
785       }
786    }
787 
788    const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
789       .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
790       .stageCount = ARRAY_SIZE(pipeline_shader_stages),
791       .pStages = pipeline_shader_stages,
792       .pVertexInputState = vi_create_info,
793       .pInputAssemblyState =
794          &(VkPipelineInputAssemblyStateCreateInfo){
795             .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
796             .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
797             .primitiveRestartEnable = false,
798          },
799       .pViewportState =
800          &(VkPipelineViewportStateCreateInfo){
801             .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
802             .viewportCount = 1,
803             .scissorCount = 1,
804          },
805       .pRasterizationState =
806          &(VkPipelineRasterizationStateCreateInfo){
807             .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
808             .rasterizerDiscardEnable = false,
809             .polygonMode = VK_POLYGON_MODE_FILL,
810             .cullMode = VK_CULL_MODE_NONE,
811             .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE},
812       .pMultisampleState =
813          &(VkPipelineMultisampleStateCreateInfo){
814             .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
815             .rasterizationSamples = 1 << log2_samples,
816             .sampleShadingEnable = log2_samples > 1,
817             .minSampleShading = 1.0,
818             .pSampleMask = (VkSampleMask[]){UINT32_MAX},
819          },
820       .pColorBlendState =
821          &(VkPipelineColorBlendStateCreateInfo){
822             .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
823             .attachmentCount = 1,
824             .pAttachments =
825                (VkPipelineColorBlendAttachmentState[]){
826                   {.colorWriteMask = VK_COLOR_COMPONENT_A_BIT | VK_COLOR_COMPONENT_R_BIT |
827                                      VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT},
828                }},
829       .pDynamicState =
830          &(VkPipelineDynamicStateCreateInfo){
831             .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
832             .dynamicStateCount = 9,
833             .pDynamicStates =
834                (VkDynamicState[]){
835                   VK_DYNAMIC_STATE_VIEWPORT,
836                   VK_DYNAMIC_STATE_SCISSOR,
837                   VK_DYNAMIC_STATE_LINE_WIDTH,
838                   VK_DYNAMIC_STATE_DEPTH_BIAS,
839                   VK_DYNAMIC_STATE_BLEND_CONSTANTS,
840                   VK_DYNAMIC_STATE_DEPTH_BOUNDS,
841                   VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
842                   VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
843                   VK_DYNAMIC_STATE_STENCIL_REFERENCE,
844                },
845          },
846       .flags = 0,
847       .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
848       .renderPass = device->meta_state.blit2d_render_passes[fs_key][0],
849       .subpass = 0,
850    };
851 
852    const struct radv_graphics_pipeline_create_info radv_pipeline_info = {.use_rectlist = true};
853 
854    result = radv_graphics_pipeline_create(
855       radv_device_to_handle(device), radv_pipeline_cache_to_handle(&device->meta_state.cache),
856       &vk_pipeline_info, &radv_pipeline_info, &device->meta_state.alloc,
857       &device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key]);
858 
859    ralloc_free(vs);
860    ralloc_free(fs);
861 
862    mtx_unlock(&device->meta_state.mtx);
863    return result;
864 }
865 
866 static VkResult
blit2d_init_depth_only_pipeline(struct radv_device * device,enum blit2d_src_type src_type,uint32_t log2_samples)867 blit2d_init_depth_only_pipeline(struct radv_device *device, enum blit2d_src_type src_type,
868                                 uint32_t log2_samples)
869 {
870    VkResult result;
871    const char *name;
872 
873    mtx_lock(&device->meta_state.mtx);
874    if (device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type]) {
875       mtx_unlock(&device->meta_state.mtx);
876       return VK_SUCCESS;
877    }
878 
879    texel_fetch_build_func src_func;
880    switch (src_type) {
881    case BLIT2D_SRC_TYPE_IMAGE:
882       src_func = build_nir_texel_fetch;
883       name = "meta_blit2d_depth_image_fs";
884       break;
885    case BLIT2D_SRC_TYPE_IMAGE_3D:
886       src_func = build_nir_texel_fetch;
887       name = "meta_blit3d_depth_image_fs";
888       break;
889    case BLIT2D_SRC_TYPE_BUFFER:
890       src_func = build_nir_buffer_fetch;
891       name = "meta_blit2d_depth_buffer_fs";
892       break;
893    default:
894       unreachable("unknown blit src type\n");
895       break;
896    }
897 
898    const VkPipelineVertexInputStateCreateInfo *vi_create_info;
899    nir_shader *fs = build_nir_copy_fragment_shader_depth(
900       device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
901    nir_shader *vs = build_nir_vertex_shader();
902 
903    vi_create_info = &normal_vi_create_info;
904 
905    VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
906       {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
907        .stage = VK_SHADER_STAGE_VERTEX_BIT,
908        .module = vk_shader_module_handle_from_nir(vs),
909        .pName = "main",
910        .pSpecializationInfo = NULL},
911       {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
912        .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
913        .module = vk_shader_module_handle_from_nir(fs),
914        .pName = "main",
915        .pSpecializationInfo = NULL},
916    };
917 
918    for (enum radv_blit_ds_layout ds_layout = RADV_BLIT_DS_LAYOUT_TILE_ENABLE;
919         ds_layout < RADV_BLIT_DS_LAYOUT_COUNT; ds_layout++) {
920       if (!device->meta_state.blit2d_depth_only_rp[ds_layout]) {
921          VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
922          result = radv_CreateRenderPass2(
923             radv_device_to_handle(device),
924             &(VkRenderPassCreateInfo2){
925                .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO_2,
926                .attachmentCount = 1,
927                .pAttachments =
928                   &(VkAttachmentDescription2){
929                      .sType = VK_STRUCTURE_TYPE_ATTACHMENT_DESCRIPTION_2,
930                      .format = VK_FORMAT_D32_SFLOAT,
931                      .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
932                      .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
933                      .initialLayout = layout,
934                      .finalLayout = layout,
935                   },
936                .subpassCount = 1,
937                .pSubpasses =
938                   &(VkSubpassDescription2){
939                      .sType = VK_STRUCTURE_TYPE_SUBPASS_DESCRIPTION_2,
940                      .inputAttachmentCount = 0,
941                      .colorAttachmentCount = 0,
942                      .pColorAttachments = NULL,
943                      .pResolveAttachments = NULL,
944                      .pDepthStencilAttachment =
945                         &(VkAttachmentReference2){
946                            .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2,
947                            .attachment = 0,
948                            .layout = layout,
949                         },
950                      .preserveAttachmentCount = 0,
951                      .pPreserveAttachments = NULL,
952                   },
953                .dependencyCount = 2,
954                .pDependencies =
955                   (VkSubpassDependency2[]){{.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2,
956                                             .srcSubpass = VK_SUBPASS_EXTERNAL,
957                                             .dstSubpass = 0,
958                                             .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
959                                             .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
960                                             .srcAccessMask = 0,
961                                             .dstAccessMask = 0,
962                                             .dependencyFlags = 0},
963                                            {.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2,
964                                             .srcSubpass = 0,
965                                             .dstSubpass = VK_SUBPASS_EXTERNAL,
966                                             .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
967                                             .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
968                                             .srcAccessMask = 0,
969                                             .dstAccessMask = 0,
970                                             .dependencyFlags = 0}},
971             },
972             &device->meta_state.alloc, &device->meta_state.blit2d_depth_only_rp[ds_layout]);
973       }
974    }
975 
976    const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
977       .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
978       .stageCount = ARRAY_SIZE(pipeline_shader_stages),
979       .pStages = pipeline_shader_stages,
980       .pVertexInputState = vi_create_info,
981       .pInputAssemblyState =
982          &(VkPipelineInputAssemblyStateCreateInfo){
983             .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
984             .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
985             .primitiveRestartEnable = false,
986          },
987       .pViewportState =
988          &(VkPipelineViewportStateCreateInfo){
989             .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
990             .viewportCount = 1,
991             .scissorCount = 1,
992          },
993       .pRasterizationState =
994          &(VkPipelineRasterizationStateCreateInfo){
995             .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
996             .rasterizerDiscardEnable = false,
997             .polygonMode = VK_POLYGON_MODE_FILL,
998             .cullMode = VK_CULL_MODE_NONE,
999             .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE},
1000       .pMultisampleState =
1001          &(VkPipelineMultisampleStateCreateInfo){
1002             .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
1003             .rasterizationSamples = 1 << log2_samples,
1004             .sampleShadingEnable = false,
1005             .pSampleMask = (VkSampleMask[]){UINT32_MAX},
1006          },
1007       .pColorBlendState =
1008          &(VkPipelineColorBlendStateCreateInfo){
1009             .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
1010             .attachmentCount = 0,
1011             .pAttachments = NULL,
1012          },
1013       .pDepthStencilState =
1014          &(VkPipelineDepthStencilStateCreateInfo){
1015             .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
1016             .depthTestEnable = true,
1017             .depthWriteEnable = true,
1018             .depthCompareOp = VK_COMPARE_OP_ALWAYS,
1019          },
1020       .pDynamicState =
1021          &(VkPipelineDynamicStateCreateInfo){
1022             .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
1023             .dynamicStateCount = 9,
1024             .pDynamicStates =
1025                (VkDynamicState[]){
1026                   VK_DYNAMIC_STATE_VIEWPORT,
1027                   VK_DYNAMIC_STATE_SCISSOR,
1028                   VK_DYNAMIC_STATE_LINE_WIDTH,
1029                   VK_DYNAMIC_STATE_DEPTH_BIAS,
1030                   VK_DYNAMIC_STATE_BLEND_CONSTANTS,
1031                   VK_DYNAMIC_STATE_DEPTH_BOUNDS,
1032                   VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
1033                   VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
1034                   VK_DYNAMIC_STATE_STENCIL_REFERENCE,
1035                },
1036          },
1037       .flags = 0,
1038       .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
1039       .renderPass = device->meta_state.blit2d_depth_only_rp[0],
1040       .subpass = 0,
1041    };
1042 
1043    const struct radv_graphics_pipeline_create_info radv_pipeline_info = {.use_rectlist = true};
1044 
1045    result = radv_graphics_pipeline_create(
1046       radv_device_to_handle(device), radv_pipeline_cache_to_handle(&device->meta_state.cache),
1047       &vk_pipeline_info, &radv_pipeline_info, &device->meta_state.alloc,
1048       &device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type]);
1049 
1050    ralloc_free(vs);
1051    ralloc_free(fs);
1052 
1053    mtx_unlock(&device->meta_state.mtx);
1054    return result;
1055 }
1056 
1057 static VkResult
blit2d_init_stencil_only_pipeline(struct radv_device * device,enum blit2d_src_type src_type,uint32_t log2_samples)1058 blit2d_init_stencil_only_pipeline(struct radv_device *device, enum blit2d_src_type src_type,
1059                                   uint32_t log2_samples)
1060 {
1061    VkResult result;
1062    const char *name;
1063 
1064    mtx_lock(&device->meta_state.mtx);
1065    if (device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type]) {
1066       mtx_unlock(&device->meta_state.mtx);
1067       return VK_SUCCESS;
1068    }
1069 
1070    texel_fetch_build_func src_func;
1071    switch (src_type) {
1072    case BLIT2D_SRC_TYPE_IMAGE:
1073       src_func = build_nir_texel_fetch;
1074       name = "meta_blit2d_stencil_image_fs";
1075       break;
1076    case BLIT2D_SRC_TYPE_IMAGE_3D:
1077       src_func = build_nir_texel_fetch;
1078       name = "meta_blit3d_stencil_image_fs";
1079       break;
1080    case BLIT2D_SRC_TYPE_BUFFER:
1081       src_func = build_nir_buffer_fetch;
1082       name = "meta_blit2d_stencil_buffer_fs";
1083       break;
1084    default:
1085       unreachable("unknown blit src type\n");
1086       break;
1087    }
1088 
1089    const VkPipelineVertexInputStateCreateInfo *vi_create_info;
1090    nir_shader *fs = build_nir_copy_fragment_shader_stencil(
1091       device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
1092    nir_shader *vs = build_nir_vertex_shader();
1093 
1094    vi_create_info = &normal_vi_create_info;
1095 
1096    VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
1097       {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
1098        .stage = VK_SHADER_STAGE_VERTEX_BIT,
1099        .module = vk_shader_module_handle_from_nir(vs),
1100        .pName = "main",
1101        .pSpecializationInfo = NULL},
1102       {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
1103        .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
1104        .module = vk_shader_module_handle_from_nir(fs),
1105        .pName = "main",
1106        .pSpecializationInfo = NULL},
1107    };
1108 
1109    for (enum radv_blit_ds_layout ds_layout = RADV_BLIT_DS_LAYOUT_TILE_ENABLE;
1110         ds_layout < RADV_BLIT_DS_LAYOUT_COUNT; ds_layout++) {
1111       if (!device->meta_state.blit2d_stencil_only_rp[ds_layout]) {
1112          VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
1113          result = radv_CreateRenderPass2(
1114             radv_device_to_handle(device),
1115             &(VkRenderPassCreateInfo2){
1116                .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO_2,
1117                .attachmentCount = 1,
1118                .pAttachments =
1119                   &(VkAttachmentDescription2){
1120                      .sType = VK_STRUCTURE_TYPE_ATTACHMENT_DESCRIPTION_2,
1121                      .format = VK_FORMAT_S8_UINT,
1122                      .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
1123                      .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
1124                      .initialLayout = layout,
1125                      .finalLayout = layout,
1126                   },
1127                .subpassCount = 1,
1128                .pSubpasses =
1129                   &(VkSubpassDescription2){
1130                      .sType = VK_STRUCTURE_TYPE_SUBPASS_DESCRIPTION_2,
1131                      .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
1132                      .inputAttachmentCount = 0,
1133                      .colorAttachmentCount = 0,
1134                      .pColorAttachments = NULL,
1135                      .pResolveAttachments = NULL,
1136                      .pDepthStencilAttachment =
1137                         &(VkAttachmentReference2){
1138                            .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2,
1139                            .attachment = 0,
1140                            .layout = layout,
1141                         },
1142                      .preserveAttachmentCount = 0,
1143                      .pPreserveAttachments = NULL,
1144                   },
1145                .dependencyCount = 2,
1146                .pDependencies =
1147                   (VkSubpassDependency2[]){{.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2,
1148                                             .srcSubpass = VK_SUBPASS_EXTERNAL,
1149                                             .dstSubpass = 0,
1150                                             .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
1151                                             .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
1152                                             .srcAccessMask = 0,
1153                                             .dstAccessMask = 0,
1154                                             .dependencyFlags = 0},
1155                                            {.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2,
1156                                             .srcSubpass = 0,
1157                                             .dstSubpass = VK_SUBPASS_EXTERNAL,
1158                                             .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
1159                                             .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
1160                                             .srcAccessMask = 0,
1161                                             .dstAccessMask = 0,
1162                                             .dependencyFlags = 0}},
1163             },
1164             &device->meta_state.alloc, &device->meta_state.blit2d_stencil_only_rp[ds_layout]);
1165       }
1166    }
1167 
1168    const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
1169       .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
1170       .stageCount = ARRAY_SIZE(pipeline_shader_stages),
1171       .pStages = pipeline_shader_stages,
1172       .pVertexInputState = vi_create_info,
1173       .pInputAssemblyState =
1174          &(VkPipelineInputAssemblyStateCreateInfo){
1175             .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
1176             .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
1177             .primitiveRestartEnable = false,
1178          },
1179       .pViewportState =
1180          &(VkPipelineViewportStateCreateInfo){
1181             .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
1182             .viewportCount = 1,
1183             .scissorCount = 1,
1184          },
1185       .pRasterizationState =
1186          &(VkPipelineRasterizationStateCreateInfo){
1187             .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
1188             .rasterizerDiscardEnable = false,
1189             .polygonMode = VK_POLYGON_MODE_FILL,
1190             .cullMode = VK_CULL_MODE_NONE,
1191             .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE},
1192       .pMultisampleState =
1193          &(VkPipelineMultisampleStateCreateInfo){
1194             .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
1195             .rasterizationSamples = 1 << log2_samples,
1196             .sampleShadingEnable = false,
1197             .pSampleMask = (VkSampleMask[]){UINT32_MAX},
1198          },
1199       .pColorBlendState =
1200          &(VkPipelineColorBlendStateCreateInfo){
1201             .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
1202             .attachmentCount = 0,
1203             .pAttachments = NULL,
1204          },
1205       .pDepthStencilState =
1206          &(VkPipelineDepthStencilStateCreateInfo){
1207             .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
1208             .depthTestEnable = false,
1209             .depthWriteEnable = false,
1210             .stencilTestEnable = true,
1211             .front = {.failOp = VK_STENCIL_OP_REPLACE,
1212                       .passOp = VK_STENCIL_OP_REPLACE,
1213                       .depthFailOp = VK_STENCIL_OP_REPLACE,
1214                       .compareOp = VK_COMPARE_OP_ALWAYS,
1215                       .compareMask = 0xff,
1216                       .writeMask = 0xff,
1217                       .reference = 0},
1218             .back = {.failOp = VK_STENCIL_OP_REPLACE,
1219                      .passOp = VK_STENCIL_OP_REPLACE,
1220                      .depthFailOp = VK_STENCIL_OP_REPLACE,
1221                      .compareOp = VK_COMPARE_OP_ALWAYS,
1222                      .compareMask = 0xff,
1223                      .writeMask = 0xff,
1224                      .reference = 0},
1225             .depthCompareOp = VK_COMPARE_OP_ALWAYS,
1226          },
1227       .pDynamicState =
1228          &(VkPipelineDynamicStateCreateInfo){
1229             .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
1230             .dynamicStateCount = 6,
1231             .pDynamicStates =
1232                (VkDynamicState[]){
1233                   VK_DYNAMIC_STATE_VIEWPORT,
1234                   VK_DYNAMIC_STATE_SCISSOR,
1235                   VK_DYNAMIC_STATE_LINE_WIDTH,
1236                   VK_DYNAMIC_STATE_DEPTH_BIAS,
1237                   VK_DYNAMIC_STATE_BLEND_CONSTANTS,
1238                   VK_DYNAMIC_STATE_DEPTH_BOUNDS,
1239                },
1240          },
1241       .flags = 0,
1242       .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
1243       .renderPass = device->meta_state.blit2d_stencil_only_rp[0],
1244       .subpass = 0,
1245    };
1246 
1247    const struct radv_graphics_pipeline_create_info radv_pipeline_info = {.use_rectlist = true};
1248 
1249    result = radv_graphics_pipeline_create(
1250       radv_device_to_handle(device), radv_pipeline_cache_to_handle(&device->meta_state.cache),
1251       &vk_pipeline_info, &radv_pipeline_info, &device->meta_state.alloc,
1252       &device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type]);
1253 
1254    ralloc_free(vs);
1255    ralloc_free(fs);
1256 
1257    mtx_unlock(&device->meta_state.mtx);
1258    return result;
1259 }
1260 
1261 static VkResult
meta_blit2d_create_pipe_layout(struct radv_device * device,int idx,uint32_t log2_samples)1262 meta_blit2d_create_pipe_layout(struct radv_device *device, int idx, uint32_t log2_samples)
1263 {
1264    VkResult result;
1265    VkDescriptorType desc_type = (idx == BLIT2D_SRC_TYPE_BUFFER)
1266                                    ? VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER
1267                                    : VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE;
1268    const VkPushConstantRange push_constant_ranges[] = {
1269       {VK_SHADER_STAGE_VERTEX_BIT, 0, 16},
1270       {VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4},
1271    };
1272    int num_push_constant_range = (idx != BLIT2D_SRC_TYPE_IMAGE || log2_samples > 0) ? 2 : 1;
1273 
1274    result = radv_CreateDescriptorSetLayout(
1275       radv_device_to_handle(device),
1276       &(VkDescriptorSetLayoutCreateInfo){
1277          .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
1278          .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
1279          .bindingCount = 1,
1280          .pBindings =
1281             (VkDescriptorSetLayoutBinding[]){
1282                {.binding = 0,
1283                 .descriptorType = desc_type,
1284                 .descriptorCount = 1,
1285                 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
1286                 .pImmutableSamplers = NULL},
1287             }},
1288       &device->meta_state.alloc, &device->meta_state.blit2d[log2_samples].ds_layouts[idx]);
1289    if (result != VK_SUCCESS)
1290       goto fail;
1291 
1292    result = radv_CreatePipelineLayout(
1293       radv_device_to_handle(device),
1294       &(VkPipelineLayoutCreateInfo){
1295          .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
1296          .setLayoutCount = 1,
1297          .pSetLayouts = &device->meta_state.blit2d[log2_samples].ds_layouts[idx],
1298          .pushConstantRangeCount = num_push_constant_range,
1299          .pPushConstantRanges = push_constant_ranges,
1300       },
1301       &device->meta_state.alloc, &device->meta_state.blit2d[log2_samples].p_layouts[idx]);
1302    if (result != VK_SUCCESS)
1303       goto fail;
1304    return VK_SUCCESS;
1305 fail:
1306    return result;
1307 }
1308 
1309 VkResult
radv_device_init_meta_blit2d_state(struct radv_device * device,bool on_demand)1310 radv_device_init_meta_blit2d_state(struct radv_device *device, bool on_demand)
1311 {
1312    VkResult result;
1313    bool create_3d = device->physical_device->rad_info.chip_class >= GFX9;
1314 
1315    for (unsigned log2_samples = 0; log2_samples < MAX_SAMPLES_LOG2; log2_samples++) {
1316       for (unsigned src = 0; src < BLIT2D_NUM_SRC_TYPES; src++) {
1317          if (src == BLIT2D_SRC_TYPE_IMAGE_3D && !create_3d)
1318             continue;
1319 
1320          /* Don't need to handle copies between buffers and multisample images. */
1321          if (src == BLIT2D_SRC_TYPE_BUFFER && log2_samples > 0)
1322             continue;
1323 
1324          /* There are no multisampled 3D images. */
1325          if (src == BLIT2D_SRC_TYPE_IMAGE_3D && log2_samples > 0)
1326             continue;
1327 
1328          result = meta_blit2d_create_pipe_layout(device, src, log2_samples);
1329          if (result != VK_SUCCESS)
1330             goto fail;
1331 
1332          if (on_demand)
1333             continue;
1334 
1335          for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
1336             result = blit2d_init_color_pipeline(device, src, radv_fs_key_format_exemplars[j],
1337                                                 log2_samples);
1338             if (result != VK_SUCCESS)
1339                goto fail;
1340          }
1341 
1342          result = blit2d_init_depth_only_pipeline(device, src, log2_samples);
1343          if (result != VK_SUCCESS)
1344             goto fail;
1345 
1346          result = blit2d_init_stencil_only_pipeline(device, src, log2_samples);
1347          if (result != VK_SUCCESS)
1348             goto fail;
1349       }
1350    }
1351 
1352    return VK_SUCCESS;
1353 
1354 fail:
1355    radv_device_finish_meta_blit2d_state(device);
1356    return result;
1357 }
1358