1 /*
2 * Copyright © 2016 Red Hat
3 * based on intel anv code:
4 * Copyright © 2015 Intel Corporation
5 *
6 * SPDX-License-Identifier: MIT
7 */
8
9 #ifndef RADV_META_H
10 #define RADV_META_H
11
12 #include "radv_buffer.h"
13 #include "radv_buffer_view.h"
14 #include "radv_cmd_buffer.h"
15 #include "radv_device.h"
16 #include "radv_device_memory.h"
17 #include "radv_entrypoints.h"
18 #include "radv_image.h"
19 #include "radv_image_view.h"
20 #include "radv_physical_device.h"
21 #include "radv_pipeline.h"
22 #include "radv_pipeline_compute.h"
23 #include "radv_pipeline_graphics.h"
24 #include "radv_queue.h"
25 #include "radv_shader.h"
26 #include "radv_shader_object.h"
27 #include "radv_sqtt.h"
28
29 #include "vk_render_pass.h"
30 #include "vk_shader_module.h"
31
32 #ifdef __cplusplus
33 extern "C" {
34 #endif
35
36 enum radv_meta_save_flags {
37 RADV_META_SAVE_RENDER = (1 << 0),
38 RADV_META_SAVE_CONSTANTS = (1 << 1),
39 RADV_META_SAVE_DESCRIPTORS = (1 << 2),
40 RADV_META_SAVE_GRAPHICS_PIPELINE = (1 << 3),
41 RADV_META_SAVE_COMPUTE_PIPELINE = (1 << 4),
42 RADV_META_SUSPEND_PREDICATING = (1 << 5),
43 };
44
45 struct radv_meta_saved_state {
46 uint32_t flags;
47
48 struct radv_descriptor_set *old_descriptor_set0;
49 struct radv_graphics_pipeline *old_graphics_pipeline;
50 struct radv_compute_pipeline *old_compute_pipeline;
51 struct radv_dynamic_state dynamic;
52
53 struct radv_shader_object *old_shader_objs[MESA_VULKAN_SHADER_STAGES];
54
55 char push_constants[MAX_PUSH_CONSTANTS_SIZE];
56
57 struct radv_rendering_state render;
58
59 unsigned active_emulated_pipeline_queries;
60 unsigned active_emulated_prims_gen_queries;
61 unsigned active_emulated_prims_xfb_queries;
62 unsigned active_occlusion_queries;
63
64 bool predicating;
65 };
66
67 enum radv_blit_ds_layout {
68 RADV_BLIT_DS_LAYOUT_TILE_ENABLE,
69 RADV_BLIT_DS_LAYOUT_TILE_DISABLE,
70 RADV_BLIT_DS_LAYOUT_COUNT,
71 };
72
73 static inline enum radv_blit_ds_layout
radv_meta_blit_ds_to_type(VkImageLayout layout)74 radv_meta_blit_ds_to_type(VkImageLayout layout)
75 {
76 return (layout == VK_IMAGE_LAYOUT_GENERAL) ? RADV_BLIT_DS_LAYOUT_TILE_DISABLE : RADV_BLIT_DS_LAYOUT_TILE_ENABLE;
77 }
78
79 static inline VkImageLayout
radv_meta_blit_ds_to_layout(enum radv_blit_ds_layout ds_layout)80 radv_meta_blit_ds_to_layout(enum radv_blit_ds_layout ds_layout)
81 {
82 return ds_layout == RADV_BLIT_DS_LAYOUT_TILE_ENABLE ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL : VK_IMAGE_LAYOUT_GENERAL;
83 }
84
85 enum radv_meta_dst_layout {
86 RADV_META_DST_LAYOUT_GENERAL,
87 RADV_META_DST_LAYOUT_OPTIMAL,
88 RADV_META_DST_LAYOUT_COUNT,
89 };
90
91 static inline enum radv_meta_dst_layout
radv_meta_dst_layout_from_layout(VkImageLayout layout)92 radv_meta_dst_layout_from_layout(VkImageLayout layout)
93 {
94 return (layout == VK_IMAGE_LAYOUT_GENERAL) ? RADV_META_DST_LAYOUT_GENERAL : RADV_META_DST_LAYOUT_OPTIMAL;
95 }
96
97 static inline VkImageLayout
radv_meta_dst_layout_to_layout(enum radv_meta_dst_layout layout)98 radv_meta_dst_layout_to_layout(enum radv_meta_dst_layout layout)
99 {
100 return layout == RADV_META_DST_LAYOUT_OPTIMAL ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL : VK_IMAGE_LAYOUT_GENERAL;
101 }
102
103 extern const VkFormat radv_fs_key_format_exemplars[NUM_META_FS_KEYS];
104
105 VkResult radv_device_init_meta(struct radv_device *device);
106 void radv_device_finish_meta(struct radv_device *device);
107
108 VkResult radv_device_init_null_accel_struct(struct radv_device *device);
109 VkResult radv_device_init_accel_struct_build_state(struct radv_device *device);
110 void radv_device_finish_accel_struct_build_state(struct radv_device *device);
111
112 void radv_meta_save(struct radv_meta_saved_state *saved_state, struct radv_cmd_buffer *cmd_buffer, uint32_t flags);
113
114 void radv_meta_restore(const struct radv_meta_saved_state *state, struct radv_cmd_buffer *cmd_buffer);
115
116 VkImageViewType radv_meta_get_view_type(const struct radv_image *image);
117
118 uint32_t radv_meta_get_iview_layer(const struct radv_image *dst_image, const VkImageSubresourceLayers *dst_subresource,
119 const VkOffset3D *dst_offset);
120
121 struct radv_meta_blit2d_surf {
122 /** The size of an element in bytes. */
123 uint8_t bs;
124 VkFormat format;
125
126 struct radv_image *image;
127 unsigned level;
128 unsigned layer;
129 VkImageAspectFlags aspect_mask;
130 VkImageLayout current_layout;
131 bool disable_compression;
132 };
133
134 struct radv_meta_blit2d_buffer {
135 struct radv_buffer *buffer;
136 uint32_t offset;
137 uint32_t pitch;
138 uint8_t bs;
139 VkFormat format;
140 };
141
142 struct radv_meta_blit2d_rect {
143 uint32_t src_x, src_y;
144 uint32_t dst_x, dst_y;
145 uint32_t width, height;
146 };
147
148 void radv_meta_begin_blit2d(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_saved_state *save);
149
150 void radv_meta_blit2d(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src_img,
151 struct radv_meta_blit2d_buffer *src_buf, struct radv_meta_blit2d_surf *dst,
152 struct radv_meta_blit2d_rect *rect);
153
154 void radv_meta_end_blit2d(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_saved_state *save);
155
156 void radv_meta_image_to_buffer(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src,
157 struct radv_meta_blit2d_buffer *dst, struct radv_meta_blit2d_rect *rect);
158
159 void radv_meta_buffer_to_image_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_buffer *src,
160 struct radv_meta_blit2d_surf *dst, struct radv_meta_blit2d_rect *rect);
161 void radv_meta_image_to_image_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src,
162 struct radv_meta_blit2d_surf *dst, struct radv_meta_blit2d_rect *rect);
163 void radv_meta_clear_image_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *dst,
164 const VkClearColorValue *clear_color);
165
166 void radv_expand_depth_stencil(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
167 const VkImageSubresourceRange *subresourceRange,
168 struct radv_sample_locations_state *sample_locs);
169 void radv_fast_clear_flush_image_inplace(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
170 const VkImageSubresourceRange *subresourceRange);
171 void radv_decompress_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
172 const VkImageSubresourceRange *subresourceRange);
173 void radv_retile_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image);
174 void radv_expand_fmask_image_inplace(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
175 const VkImageSubresourceRange *subresourceRange);
176 void radv_copy_vrs_htile(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *vrs_iview, const VkRect2D *rect,
177 struct radv_image *dst_image, struct radv_buffer *htile_buffer, bool read_htile_value);
178
179 bool radv_can_use_fmask_copy(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *src_image,
180 const struct radv_image *dst_image, const struct radv_meta_blit2d_rect *rect);
181 void radv_fmask_copy(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src,
182 struct radv_meta_blit2d_surf *dst);
183
184 void radv_meta_resolve_compute_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
185 VkFormat src_format, VkImageLayout src_image_layout, struct radv_image *dst_image,
186 VkFormat dst_format, VkImageLayout dst_image_layout,
187 const VkImageResolve2 *region);
188
189 void radv_meta_resolve_fragment_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
190 VkImageLayout src_image_layout, struct radv_image *dst_image,
191 VkImageLayout dst_image_layout, const VkImageResolve2 *region);
192
193 void radv_decompress_resolve_rendering_src(struct radv_cmd_buffer *cmd_buffer);
194
195 void radv_decompress_resolve_src(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
196 VkImageLayout src_image_layout, const VkImageResolve2 *region);
197
198 uint32_t radv_clear_cmask(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
199 const VkImageSubresourceRange *range, uint32_t value);
200 uint32_t radv_clear_fmask(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
201 const VkImageSubresourceRange *range, uint32_t value);
202 uint32_t radv_clear_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
203 const VkImageSubresourceRange *range, uint32_t value);
204 uint32_t radv_clear_htile(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *image,
205 const VkImageSubresourceRange *range, uint32_t value, bool is_clear);
206
207 void radv_update_buffer_cp(struct radv_cmd_buffer *cmd_buffer, uint64_t va, const void *data, uint64_t size);
208
209 void radv_meta_decode_etc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, VkImageLayout layout,
210 const VkImageSubresourceLayers *subresource, VkOffset3D offset, VkExtent3D extent);
211 void radv_meta_decode_astc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, VkImageLayout layout,
212 const VkImageSubresourceLayers *subresource, VkOffset3D offset, VkExtent3D extent);
213
214 /* common nir builder helpers */
215 #include "nir/nir_builder.h"
216
217 nir_builder PRINTFLIKE(3, 4)
218 radv_meta_init_shader(struct radv_device *dev, gl_shader_stage stage, const char *name, ...);
219
220 nir_shader *radv_meta_build_nir_vs_generate_vertices(struct radv_device *dev);
221 nir_shader *radv_meta_build_nir_fs_noop(struct radv_device *dev);
222
223 void radv_meta_build_resolve_shader_core(struct radv_device *device, nir_builder *b, bool is_integer, int samples,
224 nir_variable *input_img, nir_variable *color, nir_def *img_coord);
225
226 nir_def *radv_meta_load_descriptor(nir_builder *b, unsigned desc_set, unsigned binding);
227
228 nir_def *get_global_ids(nir_builder *b, unsigned num_components);
229
230 void radv_break_on_count(nir_builder *b, nir_variable *var, nir_def *count);
231
232 uint32_t radv_fill_buffer(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *image,
233 struct radeon_winsys_bo *bo, uint64_t va, uint64_t size, uint32_t value);
234
235 void radv_copy_buffer(struct radv_cmd_buffer *cmd_buffer, struct radeon_winsys_bo *src_bo,
236 struct radeon_winsys_bo *dst_bo, uint64_t src_offset, uint64_t dst_offset, uint64_t size);
237
238 void radv_cmd_buffer_clear_attachment(struct radv_cmd_buffer *cmd_buffer, const VkClearAttachment *attachment);
239
240 void radv_cmd_buffer_clear_rendering(struct radv_cmd_buffer *cmd_buffer, const VkRenderingInfo *render_info);
241
242 void radv_cmd_buffer_resolve_rendering(struct radv_cmd_buffer *cmd_buffer);
243
244 void radv_cmd_buffer_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview,
245 VkImageLayout src_layout, struct radv_image_view *dst_iview,
246 VkImageLayout dst_layout, const VkImageResolve2 *region);
247
248 void radv_depth_stencil_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, VkImageAspectFlags aspects,
249 VkResolveModeFlagBits resolve_mode);
250
251 void radv_cmd_buffer_resolve_rendering_fs(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview,
252 VkImageLayout src_layout, struct radv_image_view *dst_iview,
253 VkImageLayout dst_layout);
254
255 void radv_depth_stencil_resolve_rendering_fs(struct radv_cmd_buffer *cmd_buffer, VkImageAspectFlags aspects,
256 VkResolveModeFlagBits resolve_mode);
257
258 VkResult radv_meta_get_noop_pipeline_layout(struct radv_device *device, VkPipelineLayout *layout_out);
259
260 #ifdef __cplusplus
261 }
262 #endif
263
264 #endif /* RADV_META_H */
265