1 /*
2 * Copyright © 2016 Red Hat
3 * based on intel anv code:
4 * Copyright © 2015 Intel Corporation
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
15 * Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
23 * IN THE SOFTWARE.
24 */
25
26 #ifndef RADV_META_H
27 #define RADV_META_H
28
29 #include "radv_private.h"
30 #include "radv_shader.h"
31
32 #ifdef __cplusplus
33 extern "C" {
34 #endif
35
36 enum radv_meta_save_flags {
37 RADV_META_SAVE_RENDER = (1 << 0),
38 RADV_META_SAVE_CONSTANTS = (1 << 1),
39 RADV_META_SAVE_DESCRIPTORS = (1 << 2),
40 RADV_META_SAVE_GRAPHICS_PIPELINE = (1 << 3),
41 RADV_META_SAVE_COMPUTE_PIPELINE = (1 << 4),
42 RADV_META_SUSPEND_PREDICATING = (1 << 5),
43 };
44
45 struct radv_meta_saved_state {
46 uint32_t flags;
47
48 struct radv_descriptor_set *old_descriptor_set0;
49 struct radv_graphics_pipeline *old_graphics_pipeline;
50 struct radv_compute_pipeline *old_compute_pipeline;
51 struct radv_dynamic_state dynamic;
52
53 struct radv_shader_object *old_shader_objs[MESA_VULKAN_SHADER_STAGES];
54
55 char push_constants[MAX_PUSH_CONSTANTS_SIZE];
56
57 struct radv_rendering_state render;
58
59 unsigned active_pipeline_gds_queries;
60 unsigned active_prims_gen_gds_queries;
61 unsigned active_prims_xfb_gds_queries;
62 unsigned active_occlusion_queries;
63
64 bool predicating;
65 };
66
67 enum radv_blit_ds_layout {
68 RADV_BLIT_DS_LAYOUT_TILE_ENABLE,
69 RADV_BLIT_DS_LAYOUT_TILE_DISABLE,
70 RADV_BLIT_DS_LAYOUT_COUNT,
71 };
72
73 static inline enum radv_blit_ds_layout
radv_meta_blit_ds_to_type(VkImageLayout layout)74 radv_meta_blit_ds_to_type(VkImageLayout layout)
75 {
76 return (layout == VK_IMAGE_LAYOUT_GENERAL) ? RADV_BLIT_DS_LAYOUT_TILE_DISABLE : RADV_BLIT_DS_LAYOUT_TILE_ENABLE;
77 }
78
79 static inline VkImageLayout
radv_meta_blit_ds_to_layout(enum radv_blit_ds_layout ds_layout)80 radv_meta_blit_ds_to_layout(enum radv_blit_ds_layout ds_layout)
81 {
82 return ds_layout == RADV_BLIT_DS_LAYOUT_TILE_ENABLE ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL : VK_IMAGE_LAYOUT_GENERAL;
83 }
84
85 enum radv_meta_dst_layout {
86 RADV_META_DST_LAYOUT_GENERAL,
87 RADV_META_DST_LAYOUT_OPTIMAL,
88 RADV_META_DST_LAYOUT_COUNT,
89 };
90
91 static inline enum radv_meta_dst_layout
radv_meta_dst_layout_from_layout(VkImageLayout layout)92 radv_meta_dst_layout_from_layout(VkImageLayout layout)
93 {
94 return (layout == VK_IMAGE_LAYOUT_GENERAL) ? RADV_META_DST_LAYOUT_GENERAL : RADV_META_DST_LAYOUT_OPTIMAL;
95 }
96
97 static inline VkImageLayout
radv_meta_dst_layout_to_layout(enum radv_meta_dst_layout layout)98 radv_meta_dst_layout_to_layout(enum radv_meta_dst_layout layout)
99 {
100 return layout == RADV_META_DST_LAYOUT_OPTIMAL ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL : VK_IMAGE_LAYOUT_GENERAL;
101 }
102
103 extern const VkFormat radv_fs_key_format_exemplars[NUM_META_FS_KEYS];
104 unsigned radv_format_meta_fs_key(struct radv_device *device, VkFormat format);
105
106 VkResult radv_device_init_meta(struct radv_device *device);
107 void radv_device_finish_meta(struct radv_device *device);
108
109 VkResult radv_device_init_meta_clear_state(struct radv_device *device, bool on_demand);
110 void radv_device_finish_meta_clear_state(struct radv_device *device);
111
112 VkResult radv_device_init_meta_resolve_state(struct radv_device *device, bool on_demand);
113 void radv_device_finish_meta_resolve_state(struct radv_device *device);
114
115 VkResult radv_device_init_meta_depth_decomp_state(struct radv_device *device, bool on_demand);
116 void radv_device_finish_meta_depth_decomp_state(struct radv_device *device);
117
118 VkResult radv_device_init_meta_fast_clear_flush_state(struct radv_device *device, bool on_demand);
119 void radv_device_finish_meta_fast_clear_flush_state(struct radv_device *device);
120
121 VkResult radv_device_init_meta_blit_state(struct radv_device *device, bool on_demand);
122 void radv_device_finish_meta_blit_state(struct radv_device *device);
123
124 VkResult radv_device_init_meta_blit2d_state(struct radv_device *device, bool on_demand);
125 void radv_device_finish_meta_blit2d_state(struct radv_device *device);
126
127 VkResult radv_device_init_meta_buffer_state(struct radv_device *device);
128 void radv_device_finish_meta_buffer_state(struct radv_device *device);
129
130 VkResult radv_device_init_meta_query_state(struct radv_device *device, bool on_demand);
131 void radv_device_finish_meta_query_state(struct radv_device *device);
132
133 VkResult radv_device_init_meta_resolve_compute_state(struct radv_device *device, bool on_demand);
134 void radv_device_finish_meta_resolve_compute_state(struct radv_device *device);
135
136 VkResult radv_device_init_meta_resolve_fragment_state(struct radv_device *device, bool on_demand);
137 void radv_device_finish_meta_resolve_fragment_state(struct radv_device *device);
138
139 VkResult radv_device_init_meta_fmask_copy_state(struct radv_device *device, bool on_demand);
140 void radv_device_finish_meta_fmask_copy_state(struct radv_device *device);
141
142 VkResult radv_device_init_meta_fmask_expand_state(struct radv_device *device, bool on_demand);
143 void radv_device_finish_meta_fmask_expand_state(struct radv_device *device);
144
145 void radv_device_finish_meta_dcc_retile_state(struct radv_device *device);
146
147 void radv_device_finish_meta_copy_vrs_htile_state(struct radv_device *device);
148
149 VkResult radv_device_init_null_accel_struct(struct radv_device *device);
150 VkResult radv_device_init_accel_struct_build_state(struct radv_device *device);
151 void radv_device_finish_accel_struct_build_state(struct radv_device *device);
152
153 VkResult radv_device_init_meta_etc_decode_state(struct radv_device *device, bool on_demand);
154 void radv_device_finish_meta_etc_decode_state(struct radv_device *device);
155
156 VkResult radv_device_init_meta_astc_decode_state(struct radv_device *device, bool on_demand);
157 void radv_device_finish_meta_astc_decode_state(struct radv_device *device);
158
159 VkResult radv_device_init_dgc_prepare_state(struct radv_device *device);
160 void radv_device_finish_dgc_prepare_state(struct radv_device *device);
161
162 void radv_meta_save(struct radv_meta_saved_state *saved_state, struct radv_cmd_buffer *cmd_buffer, uint32_t flags);
163
164 void radv_meta_restore(const struct radv_meta_saved_state *state, struct radv_cmd_buffer *cmd_buffer);
165
166 VkImageViewType radv_meta_get_view_type(const struct radv_image *image);
167
168 uint32_t radv_meta_get_iview_layer(const struct radv_image *dst_image, const VkImageSubresourceLayers *dst_subresource,
169 const VkOffset3D *dst_offset);
170
171 struct radv_meta_blit2d_surf {
172 /** The size of an element in bytes. */
173 uint8_t bs;
174 VkFormat format;
175
176 struct radv_image *image;
177 unsigned level;
178 unsigned layer;
179 VkImageAspectFlags aspect_mask;
180 VkImageLayout current_layout;
181 bool disable_compression;
182 };
183
184 struct radv_meta_blit2d_buffer {
185 struct radv_buffer *buffer;
186 uint32_t offset;
187 uint32_t pitch;
188 uint8_t bs;
189 VkFormat format;
190 };
191
192 struct radv_meta_blit2d_rect {
193 uint32_t src_x, src_y;
194 uint32_t dst_x, dst_y;
195 uint32_t width, height;
196 };
197
198 void radv_meta_begin_blit2d(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_saved_state *save);
199
200 void radv_meta_blit2d(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src_img,
201 struct radv_meta_blit2d_buffer *src_buf, struct radv_meta_blit2d_surf *dst, unsigned num_rects,
202 struct radv_meta_blit2d_rect *rects);
203
204 void radv_meta_end_blit2d(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_saved_state *save);
205
206 VkResult radv_device_init_meta_bufimage_state(struct radv_device *device);
207 void radv_device_finish_meta_bufimage_state(struct radv_device *device);
208 void radv_meta_image_to_buffer(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src,
209 struct radv_meta_blit2d_buffer *dst, unsigned num_rects,
210 struct radv_meta_blit2d_rect *rects);
211
212 void radv_meta_buffer_to_image_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_buffer *src,
213 struct radv_meta_blit2d_surf *dst, unsigned num_rects,
214 struct radv_meta_blit2d_rect *rects);
215 void radv_meta_image_to_image_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src,
216 struct radv_meta_blit2d_surf *dst, unsigned num_rects,
217 struct radv_meta_blit2d_rect *rects);
218 void radv_meta_clear_image_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *dst,
219 const VkClearColorValue *clear_color);
220
221 void radv_expand_depth_stencil(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
222 const VkImageSubresourceRange *subresourceRange,
223 struct radv_sample_locations_state *sample_locs);
224 void radv_resummarize_depth_stencil(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
225 const VkImageSubresourceRange *subresourceRange,
226 struct radv_sample_locations_state *sample_locs);
227 void radv_fast_clear_flush_image_inplace(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
228 const VkImageSubresourceRange *subresourceRange);
229 void radv_decompress_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
230 const VkImageSubresourceRange *subresourceRange);
231 void radv_retile_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image);
232 void radv_expand_fmask_image_inplace(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
233 const VkImageSubresourceRange *subresourceRange);
234 void radv_copy_vrs_htile(struct radv_cmd_buffer *cmd_buffer, struct radv_image *vrs_image, const VkRect2D *rect,
235 struct radv_image *dst_image, struct radv_buffer *htile_buffer, bool read_htile_value);
236
237 bool radv_can_use_fmask_copy(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *src_image,
238 const struct radv_image *dst_image, unsigned num_rects,
239 const struct radv_meta_blit2d_rect *rects);
240 void radv_fmask_copy(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src,
241 struct radv_meta_blit2d_surf *dst);
242
243 void radv_meta_resolve_compute_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
244 VkFormat src_format, VkImageLayout src_image_layout, struct radv_image *dst_image,
245 VkFormat dst_format, VkImageLayout dst_image_layout,
246 const VkImageResolve2 *region);
247
248 void radv_meta_resolve_fragment_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
249 VkImageLayout src_image_layout, struct radv_image *dst_image,
250 VkImageLayout dst_image_layout, const VkImageResolve2 *region);
251
252 void radv_decompress_resolve_rendering_src(struct radv_cmd_buffer *cmd_buffer);
253
254 void radv_decompress_resolve_src(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
255 VkImageLayout src_image_layout, const VkImageResolve2 *region);
256
257 uint32_t radv_clear_cmask(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
258 const VkImageSubresourceRange *range, uint32_t value);
259 uint32_t radv_clear_fmask(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
260 const VkImageSubresourceRange *range, uint32_t value);
261 uint32_t radv_clear_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
262 const VkImageSubresourceRange *range, uint32_t value);
263 uint32_t radv_clear_htile(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *image,
264 const VkImageSubresourceRange *range, uint32_t value);
265
266 void radv_update_buffer_cp(struct radv_cmd_buffer *cmd_buffer, uint64_t va, const void *data, uint64_t size);
267
268 void radv_meta_decode_etc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, VkImageLayout layout,
269 const VkImageSubresourceLayers *subresource, VkOffset3D offset, VkExtent3D extent);
270 void radv_meta_decode_astc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, VkImageLayout layout,
271 const VkImageSubresourceLayers *subresource, VkOffset3D offset, VkExtent3D extent);
272
273 /* common nir builder helpers */
274 #include "nir/nir_builder.h"
275
276 nir_builder PRINTFLIKE(3, 4)
277 radv_meta_init_shader(struct radv_device *dev, gl_shader_stage stage, const char *name, ...);
278
279 nir_shader *radv_meta_build_nir_vs_generate_vertices(struct radv_device *dev);
280 nir_shader *radv_meta_build_nir_fs_noop(struct radv_device *dev);
281
282 void radv_meta_build_resolve_shader_core(struct radv_device *device, nir_builder *b, bool is_integer, int samples,
283 nir_variable *input_img, nir_variable *color, nir_def *img_coord);
284
285 nir_def *radv_meta_load_descriptor(nir_builder *b, unsigned desc_set, unsigned binding);
286
287 nir_def *get_global_ids(nir_builder *b, unsigned num_components);
288
289 void radv_break_on_count(nir_builder *b, nir_variable *var, nir_def *count);
290
291 uint32_t radv_fill_buffer(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *image,
292 struct radeon_winsys_bo *bo, uint64_t va, uint64_t size, uint32_t value);
293
294 void radv_copy_buffer(struct radv_cmd_buffer *cmd_buffer, struct radeon_winsys_bo *src_bo,
295 struct radeon_winsys_bo *dst_bo, uint64_t src_offset, uint64_t dst_offset, uint64_t size);
296
297 void radv_cmd_buffer_clear_attachment(struct radv_cmd_buffer *cmd_buffer, const VkClearAttachment *attachment);
298
299 void radv_cmd_buffer_clear_rendering(struct radv_cmd_buffer *cmd_buffer, const VkRenderingInfo *render_info);
300
301 void radv_cmd_buffer_resolve_rendering(struct radv_cmd_buffer *cmd_buffer);
302
303 void radv_cmd_buffer_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview,
304 VkImageLayout src_layout, struct radv_image_view *dst_iview,
305 VkImageLayout dst_layout, const VkImageResolve2 *region);
306
307 void radv_depth_stencil_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, VkImageAspectFlags aspects,
308 VkResolveModeFlagBits resolve_mode);
309
310 void radv_cmd_buffer_resolve_rendering_fs(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview,
311 VkImageLayout src_layout, struct radv_image_view *dst_iview,
312 VkImageLayout dst_layout);
313
314 void radv_depth_stencil_resolve_rendering_fs(struct radv_cmd_buffer *cmd_buffer, VkImageAspectFlags aspects,
315 VkResolveModeFlagBits resolve_mode);
316
317 #ifdef __cplusplus
318 }
319 #endif
320
321 #endif /* RADV_META_H */
322