1 /*
2 * Copyright © 2016 Red Hat
3 * based on intel anv code:
4 * Copyright © 2015 Intel Corporation
5 *
6 * SPDX-License-Identifier: MIT
7 */
8
9 #ifndef RADV_META_H
10 #define RADV_META_H
11
12 #include "radv_buffer.h"
13 #include "radv_buffer_view.h"
14 #include "radv_cmd_buffer.h"
15 #include "radv_device.h"
16 #include "radv_device_memory.h"
17 #include "radv_entrypoints.h"
18 #include "radv_image.h"
19 #include "radv_image_view.h"
20 #include "radv_physical_device.h"
21 #include "radv_pipeline.h"
22 #include "radv_pipeline_compute.h"
23 #include "radv_pipeline_graphics.h"
24 #include "radv_queue.h"
25 #include "radv_shader.h"
26 #include "radv_shader_object.h"
27 #include "radv_sqtt.h"
28
29 #include "vk_render_pass.h"
30 #include "vk_shader_module.h"
31
32 #ifdef __cplusplus
33 extern "C" {
34 #endif
35
36 enum radv_meta_save_flags {
37 RADV_META_SAVE_RENDER = (1 << 0),
38 RADV_META_SAVE_CONSTANTS = (1 << 1),
39 RADV_META_SAVE_DESCRIPTORS = (1 << 2),
40 RADV_META_SAVE_GRAPHICS_PIPELINE = (1 << 3),
41 RADV_META_SAVE_COMPUTE_PIPELINE = (1 << 4),
42 RADV_META_SUSPEND_PREDICATING = (1 << 5),
43 };
44
45 struct radv_meta_saved_state {
46 uint32_t flags;
47
48 struct radv_descriptor_set *old_descriptor_set0;
49 struct radv_graphics_pipeline *old_graphics_pipeline;
50 struct radv_compute_pipeline *old_compute_pipeline;
51 struct radv_dynamic_state dynamic;
52
53 struct radv_shader_object *old_shader_objs[MESA_VULKAN_SHADER_STAGES];
54
55 char push_constants[MAX_PUSH_CONSTANTS_SIZE];
56
57 struct radv_rendering_state render;
58
59 unsigned active_emulated_pipeline_queries;
60 unsigned active_emulated_prims_gen_queries;
61 unsigned active_emulated_prims_xfb_queries;
62 unsigned active_occlusion_queries;
63
64 bool predicating;
65 };
66
67 enum radv_blit_ds_layout {
68 RADV_BLIT_DS_LAYOUT_TILE_ENABLE,
69 RADV_BLIT_DS_LAYOUT_TILE_DISABLE,
70 RADV_BLIT_DS_LAYOUT_COUNT,
71 };
72
73 static inline enum radv_blit_ds_layout
radv_meta_blit_ds_to_type(VkImageLayout layout)74 radv_meta_blit_ds_to_type(VkImageLayout layout)
75 {
76 return (layout == VK_IMAGE_LAYOUT_GENERAL) ? RADV_BLIT_DS_LAYOUT_TILE_DISABLE : RADV_BLIT_DS_LAYOUT_TILE_ENABLE;
77 }
78
79 static inline VkImageLayout
radv_meta_blit_ds_to_layout(enum radv_blit_ds_layout ds_layout)80 radv_meta_blit_ds_to_layout(enum radv_blit_ds_layout ds_layout)
81 {
82 return ds_layout == RADV_BLIT_DS_LAYOUT_TILE_ENABLE ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL : VK_IMAGE_LAYOUT_GENERAL;
83 }
84
85 enum radv_meta_dst_layout {
86 RADV_META_DST_LAYOUT_GENERAL,
87 RADV_META_DST_LAYOUT_OPTIMAL,
88 RADV_META_DST_LAYOUT_COUNT,
89 };
90
91 static inline enum radv_meta_dst_layout
radv_meta_dst_layout_from_layout(VkImageLayout layout)92 radv_meta_dst_layout_from_layout(VkImageLayout layout)
93 {
94 return (layout == VK_IMAGE_LAYOUT_GENERAL) ? RADV_META_DST_LAYOUT_GENERAL : RADV_META_DST_LAYOUT_OPTIMAL;
95 }
96
97 static inline VkImageLayout
radv_meta_dst_layout_to_layout(enum radv_meta_dst_layout layout)98 radv_meta_dst_layout_to_layout(enum radv_meta_dst_layout layout)
99 {
100 return layout == RADV_META_DST_LAYOUT_OPTIMAL ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL : VK_IMAGE_LAYOUT_GENERAL;
101 }
102
103 extern const VkFormat radv_fs_key_format_exemplars[NUM_META_FS_KEYS];
104
105 enum radv_meta_object_key_type {
106 RADV_META_OBJECT_KEY_NOOP = VK_META_OBJECT_KEY_DRIVER_OFFSET,
107 RADV_META_OBJECT_KEY_BLIT,
108 RADV_META_OBJECT_KEY_BLIT2D,
109 RADV_META_OBJECT_KEY_BLIT2D_COLOR,
110 RADV_META_OBJECT_KEY_BLIT2D_DEPTH,
111 RADV_META_OBJECT_KEY_BLIT2D_STENCIL,
112 RADV_META_OBJECT_KEY_FILL_BUFFER,
113 RADV_META_OBJECT_KEY_COPY_BUFFER,
114 RADV_META_OBJECT_KEY_COPY_IMAGE_TO_BUFFER,
115 RADV_META_OBJECT_KEY_COPY_BUFFER_TO_IMAGE,
116 RADV_META_OBJECT_KEY_COPY_BUFFER_TO_IMAGE_R32G32B32,
117 RADV_META_OBJECT_KEY_COPY_IMAGE,
118 RADV_META_OBJECT_KEY_COPY_IMAGE_R32G32B32,
119 RADV_META_OBJECT_KEY_COPY_VRS_HTILE,
120 RADV_META_OBJECT_KEY_CLEAR_CS,
121 RADV_META_OBJECT_KEY_CLEAR_CS_R32G32B32,
122 RADV_META_OBJECT_KEY_CLEAR_COLOR,
123 RADV_META_OBJECT_KEY_CLEAR_DS,
124 RADV_META_OBJECT_KEY_CLEAR_HTILE,
125 RADV_META_OBJECT_KEY_CLEAR_DCC_COMP_TO_SINGLE,
126 RADV_META_OBJECT_KEY_FAST_CLEAR_ELIMINATE,
127 RADV_META_OBJECT_KEY_DCC_DECOMPRESS,
128 RADV_META_OBJECT_KEY_DCC_RETILE,
129 RADV_META_OBJECT_KEY_HTILE_EXPAND_GFX,
130 RADV_META_OBJECT_KEY_HTILE_EXPAND_CS,
131 RADV_META_OBJECT_KEY_FMASK_COPY,
132 RADV_META_OBJECT_KEY_FMASK_EXPAND,
133 RADV_META_OBJECT_KEY_FMASK_DECOMPRESS,
134 RADV_META_OBJECT_KEY_RESOLVE_HW,
135 RADV_META_OBJECT_KEY_RESOLVE_CS,
136 RADV_META_OBJECT_KEY_RESOLVE_COLOR_CS,
137 RADV_META_OBJECT_KEY_RESOLVE_DS_CS,
138 RADV_META_OBJECT_KEY_RESOLVE_FS,
139 RADV_META_OBJECT_KEY_RESOLVE_COLOR_FS,
140 RADV_META_OBJECT_KEY_RESOLVE_DS_FS,
141 RADV_META_OBJECT_KEY_DGC,
142 RADV_META_OBJECT_KEY_QUERY,
143 RADV_META_OBJECT_KEY_QUERY_OCCLUSION,
144 RADV_META_OBJECT_KEY_QUERY_PIPELINE_STATS,
145 RADV_META_OBJECT_KEY_QUERY_TFB,
146 RADV_META_OBJECT_KEY_QUERY_TIMESTAMP,
147 RADV_META_OBJECT_KEY_QUERY_PRIMS_GEN,
148 RADV_META_OBJECT_KEY_QUERY_MESH_PRIMS_GEN,
149 };
150
151 VkResult radv_device_init_meta(struct radv_device *device);
152 void radv_device_finish_meta(struct radv_device *device);
153
154 VkResult radv_device_init_null_accel_struct(struct radv_device *device);
155 VkResult radv_device_init_accel_struct_build_state(struct radv_device *device);
156 void radv_device_finish_accel_struct_build_state(struct radv_device *device);
157
158 void radv_meta_save(struct radv_meta_saved_state *saved_state, struct radv_cmd_buffer *cmd_buffer, uint32_t flags);
159
160 void radv_meta_restore(const struct radv_meta_saved_state *state, struct radv_cmd_buffer *cmd_buffer);
161
162 VkImageViewType radv_meta_get_view_type(const struct radv_image *image);
163
164 uint32_t radv_meta_get_iview_layer(const struct radv_image *dst_image, const VkImageSubresourceLayers *dst_subresource,
165 const VkOffset3D *dst_offset);
166
167 struct radv_meta_blit2d_surf {
168 /** The size of an element in bytes. */
169 uint8_t bs;
170 VkFormat format;
171
172 struct radv_image *image;
173 unsigned level;
174 unsigned layer;
175 VkImageAspectFlags aspect_mask;
176 VkImageLayout current_layout;
177 bool disable_compression;
178 };
179
180 struct radv_meta_blit2d_buffer {
181 struct radv_buffer *buffer;
182 uint32_t offset;
183 uint32_t pitch;
184 uint8_t bs;
185 VkFormat format;
186 };
187
188 struct radv_meta_blit2d_rect {
189 uint32_t src_x, src_y;
190 uint32_t dst_x, dst_y;
191 uint32_t width, height;
192 };
193
194 void radv_meta_begin_blit2d(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_saved_state *save);
195
196 void radv_meta_blit2d(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src_img,
197 struct radv_meta_blit2d_buffer *src_buf, struct radv_meta_blit2d_surf *dst,
198 struct radv_meta_blit2d_rect *rect);
199
200 void radv_meta_end_blit2d(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_saved_state *save);
201
202 void radv_meta_image_to_buffer(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src,
203 struct radv_meta_blit2d_buffer *dst, struct radv_meta_blit2d_rect *rect);
204
205 void radv_meta_buffer_to_image_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_buffer *src,
206 struct radv_meta_blit2d_surf *dst, struct radv_meta_blit2d_rect *rect);
207 void radv_meta_image_to_image_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src,
208 struct radv_meta_blit2d_surf *dst, struct radv_meta_blit2d_rect *rect);
209 void radv_meta_clear_image_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *dst,
210 const VkClearColorValue *clear_color);
211
212 void radv_expand_depth_stencil(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
213 const VkImageSubresourceRange *subresourceRange,
214 struct radv_sample_locations_state *sample_locs);
215 void radv_fast_clear_flush_image_inplace(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
216 const VkImageSubresourceRange *subresourceRange);
217 void radv_decompress_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
218 const VkImageSubresourceRange *subresourceRange);
219 void radv_retile_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image);
220 void radv_expand_fmask_image_inplace(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
221 const VkImageSubresourceRange *subresourceRange);
222 void radv_copy_vrs_htile(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *vrs_iview, const VkRect2D *rect,
223 struct radv_image *dst_image, struct radv_buffer *htile_buffer, bool read_htile_value);
224
225 bool radv_can_use_fmask_copy(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *src_image,
226 const struct radv_image *dst_image, const struct radv_meta_blit2d_rect *rect);
227 void radv_fmask_copy(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src,
228 struct radv_meta_blit2d_surf *dst);
229
230 void radv_meta_resolve_compute_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
231 VkFormat src_format, VkImageLayout src_image_layout, struct radv_image *dst_image,
232 VkFormat dst_format, VkImageLayout dst_image_layout,
233 const VkImageResolve2 *region);
234
235 void radv_meta_resolve_fragment_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
236 VkImageLayout src_image_layout, struct radv_image *dst_image,
237 VkImageLayout dst_image_layout, const VkImageResolve2 *region);
238
239 void radv_decompress_resolve_rendering_src(struct radv_cmd_buffer *cmd_buffer);
240
241 void radv_decompress_resolve_src(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
242 VkImageLayout src_image_layout, const VkImageResolve2 *region);
243
244 uint32_t radv_clear_cmask(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
245 const VkImageSubresourceRange *range, uint32_t value);
246 uint32_t radv_clear_fmask(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
247 const VkImageSubresourceRange *range, uint32_t value);
248 uint32_t radv_clear_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
249 const VkImageSubresourceRange *range, uint32_t value);
250 uint32_t radv_clear_htile(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *image,
251 const VkImageSubresourceRange *range, uint32_t value, bool is_clear);
252
253 void radv_update_buffer_cp(struct radv_cmd_buffer *cmd_buffer, uint64_t va, const void *data, uint64_t size);
254
255 void radv_meta_decode_etc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, VkImageLayout layout,
256 const VkImageSubresourceLayers *subresource, VkOffset3D offset, VkExtent3D extent);
257 void radv_meta_decode_astc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, VkImageLayout layout,
258 const VkImageSubresourceLayers *subresource, VkOffset3D offset, VkExtent3D extent);
259
260 /* common nir builder helpers */
261 #include "nir/nir_builder.h"
262
263 nir_builder PRINTFLIKE(3, 4)
264 radv_meta_init_shader(struct radv_device *dev, gl_shader_stage stage, const char *name, ...);
265
266 nir_shader *radv_meta_build_nir_vs_generate_vertices(struct radv_device *dev);
267 nir_shader *radv_meta_build_nir_fs_noop(struct radv_device *dev);
268
269 void radv_meta_build_resolve_shader_core(struct radv_device *device, nir_builder *b, bool is_integer, int samples,
270 nir_variable *input_img, nir_variable *color, nir_def *img_coord);
271
272 nir_def *radv_meta_load_descriptor(nir_builder *b, unsigned desc_set, unsigned binding);
273
274 nir_def *get_global_ids(nir_builder *b, unsigned num_components);
275
276 void radv_break_on_count(nir_builder *b, nir_variable *var, nir_def *count);
277
278 uint32_t radv_fill_buffer(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *image,
279 struct radeon_winsys_bo *bo, uint64_t va, uint64_t size, uint32_t value);
280
281 void radv_copy_buffer(struct radv_cmd_buffer *cmd_buffer, struct radeon_winsys_bo *src_bo,
282 struct radeon_winsys_bo *dst_bo, uint64_t src_offset, uint64_t dst_offset, uint64_t size);
283
284 void radv_cmd_buffer_clear_attachment(struct radv_cmd_buffer *cmd_buffer, const VkClearAttachment *attachment);
285
286 void radv_cmd_buffer_clear_rendering(struct radv_cmd_buffer *cmd_buffer, const VkRenderingInfo *render_info);
287
288 void radv_cmd_buffer_resolve_rendering(struct radv_cmd_buffer *cmd_buffer);
289
290 void radv_cmd_buffer_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview,
291 VkImageLayout src_layout, struct radv_image_view *dst_iview,
292 VkImageLayout dst_layout, const VkImageResolve2 *region);
293
294 void radv_depth_stencil_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, VkImageAspectFlags aspects,
295 VkResolveModeFlagBits resolve_mode);
296
297 void radv_cmd_buffer_resolve_rendering_fs(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview,
298 VkImageLayout src_layout, struct radv_image_view *dst_iview,
299 VkImageLayout dst_layout);
300
301 void radv_depth_stencil_resolve_rendering_fs(struct radv_cmd_buffer *cmd_buffer, VkImageAspectFlags aspects,
302 VkResolveModeFlagBits resolve_mode);
303
304 VkResult radv_meta_get_noop_pipeline_layout(struct radv_device *device, VkPipelineLayout *layout_out);
305
306 #ifdef __cplusplus
307 }
308 #endif
309
310 #endif /* RADV_META_H */
311