1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * based in part on anv driver which is:
6 * Copyright © 2015 Intel Corporation
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the "Software"),
10 * to deal in the Software without restriction, including without limitation
11 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 * and/or sell copies of the Software, and to permit persons to whom the
13 * Software is furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the next
16 * paragraph) shall be included in all copies or substantial portions of the
17 * Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 * IN THE SOFTWARE.
26 */
27
28 #ifndef RADV_PRIVATE_H
29 #define RADV_PRIVATE_H
30
31 #include <assert.h>
32 #include <stdbool.h>
33 #include <stdint.h>
34 #include <stdio.h>
35 #include <stdlib.h>
36 #include <string.h>
37 #ifdef HAVE_VALGRIND
38 #include <memcheck.h>
39 #include <valgrind.h>
40 #define VG(x) x
41 #else
42 #define VG(x) ((void)0)
43 #endif
44
45 #include "c11/threads.h"
46 #ifndef _WIN32
47 #include <amdgpu.h>
48 #include <xf86drm.h>
49 #endif
50 #include "compiler/shader_enums.h"
51 #include "util/bitscan.h"
52 #include "util/cnd_monotonic.h"
53 #include "util/list.h"
54 #include "util/macros.h"
55 #include "util/rwlock.h"
56 #include "util/xmlconfig.h"
57 #include "vk_alloc.h"
58 #include "vk_debug_report.h"
59 #include "vk_device.h"
60 #include "vk_format.h"
61 #include "vk_instance.h"
62 #include "vk_log.h"
63 #include "vk_physical_device.h"
64 #include "vk_shader_module.h"
65 #include "vk_command_buffer.h"
66 #include "vk_queue.h"
67 #include "vk_util.h"
68
69 #include "ac_binary.h"
70 #include "ac_gpu_info.h"
71 #include "ac_shader_util.h"
72 #include "ac_sqtt.h"
73 #include "ac_surface.h"
74 #include "radv_constants.h"
75 #include "radv_descriptor_set.h"
76 #include "radv_radeon_winsys.h"
77 #include "radv_shader.h"
78 #include "sid.h"
79
80 /* Pre-declarations needed for WSI entrypoints */
81 struct wl_surface;
82 struct wl_display;
83 typedef struct xcb_connection_t xcb_connection_t;
84 typedef uint32_t xcb_visualid_t;
85 typedef uint32_t xcb_window_t;
86
87 #include <vulkan/vk_android_native_buffer.h>
88 #include <vulkan/vk_icd.h>
89 #include <vulkan/vulkan.h>
90 #include <vulkan/vulkan_android.h>
91
92 #include "radv_entrypoints.h"
93
94 #include "wsi_common.h"
95
96 #ifdef __cplusplus
97 extern "C"
98 {
99 #endif
100
101 /* Helper to determine if we should compile
102 * any of the Android AHB support.
103 *
104 * To actually enable the ext we also need
105 * the necessary kernel support.
106 */
107 #if defined(ANDROID) && ANDROID_API_LEVEL >= 26
108 #define RADV_SUPPORT_ANDROID_HARDWARE_BUFFER 1
109 #include <vndk/hardware_buffer.h>
110 #else
111 #define RADV_SUPPORT_ANDROID_HARDWARE_BUFFER 0
112 #endif
113
114 #ifdef _WIN32
115 #define RADV_SUPPORT_CALIBRATED_TIMESTAMPS 0
116 #else
117 #define RADV_SUPPORT_CALIBRATED_TIMESTAMPS 1
118 #endif
119
120 #ifdef _WIN32
121 #define radv_printflike(a, b)
122 #else
123 #define radv_printflike(a, b) __attribute__((__format__(__printf__, a, b)))
124 #endif
125
126 static inline uint32_t
align_u32(uint32_t v,uint32_t a)127 align_u32(uint32_t v, uint32_t a)
128 {
129 assert(a != 0 && a == (a & -a));
130 return (v + a - 1) & ~(a - 1);
131 }
132
133 static inline uint32_t
align_u32_npot(uint32_t v,uint32_t a)134 align_u32_npot(uint32_t v, uint32_t a)
135 {
136 return (v + a - 1) / a * a;
137 }
138
139 static inline uint64_t
align_u64(uint64_t v,uint64_t a)140 align_u64(uint64_t v, uint64_t a)
141 {
142 assert(a != 0 && a == (a & -a));
143 return (v + a - 1) & ~(a - 1);
144 }
145
146 static inline int32_t
align_i32(int32_t v,int32_t a)147 align_i32(int32_t v, int32_t a)
148 {
149 assert(a != 0 && a == (a & -a));
150 return (v + a - 1) & ~(a - 1);
151 }
152
153 /** Alignment must be a power of 2. */
154 static inline bool
radv_is_aligned(uintmax_t n,uintmax_t a)155 radv_is_aligned(uintmax_t n, uintmax_t a)
156 {
157 assert(a == (a & -a));
158 return (n & (a - 1)) == 0;
159 }
160
161 static inline uint32_t
round_up_u32(uint32_t v,uint32_t a)162 round_up_u32(uint32_t v, uint32_t a)
163 {
164 return (v + a - 1) / a;
165 }
166
167 static inline uint64_t
round_up_u64(uint64_t v,uint64_t a)168 round_up_u64(uint64_t v, uint64_t a)
169 {
170 return (v + a - 1) / a;
171 }
172
173 static inline uint32_t
radv_minify(uint32_t n,uint32_t levels)174 radv_minify(uint32_t n, uint32_t levels)
175 {
176 if (unlikely(n == 0))
177 return 0;
178 else
179 return MAX2(n >> levels, 1);
180 }
181 static inline float
radv_clamp_f(float f,float min,float max)182 radv_clamp_f(float f, float min, float max)
183 {
184 assert(min < max);
185
186 if (f > max)
187 return max;
188 else if (f < min)
189 return min;
190 else
191 return f;
192 }
193
194 static inline bool
radv_clear_mask(uint32_t * inout_mask,uint32_t clear_mask)195 radv_clear_mask(uint32_t *inout_mask, uint32_t clear_mask)
196 {
197 if (*inout_mask & clear_mask) {
198 *inout_mask &= ~clear_mask;
199 return true;
200 } else {
201 return false;
202 }
203 }
204
205 /* Whenever we generate an error, pass it through this function. Useful for
206 * debugging, where we can break on it. Only call at error site, not when
207 * propagating errors. Might be useful to plug in a stack trace here.
208 */
209
210 struct radv_image_view;
211 struct radv_instance;
212
213 void radv_loge(const char *format, ...) radv_printflike(1, 2);
214 void radv_loge_v(const char *format, va_list va);
215 void radv_logi(const char *format, ...) radv_printflike(1, 2);
216 void radv_logi_v(const char *format, va_list va);
217
218 /* A non-fatal assert. Useful for debugging. */
219 #ifdef NDEBUG
220 #define radv_assert(x) \
221 do { \
222 } while (0)
223 #else
224 #define radv_assert(x) \
225 do { \
226 if (unlikely(!(x))) \
227 fprintf(stderr, "%s:%d ASSERT: %s\n", __FILE__, __LINE__, #x); \
228 } while (0)
229 #endif
230
231 int radv_get_instance_entrypoint_index(const char *name);
232 int radv_get_device_entrypoint_index(const char *name);
233 int radv_get_physical_device_entrypoint_index(const char *name);
234
235 const char *radv_get_instance_entry_name(int index);
236 const char *radv_get_physical_device_entry_name(int index);
237 const char *radv_get_device_entry_name(int index);
238
239 struct radv_physical_device {
240 struct vk_physical_device vk;
241
242 /* Link in radv_instance::physical_devices */
243 struct list_head link;
244
245 struct radv_instance *instance;
246
247 struct radeon_winsys *ws;
248 struct radeon_info rad_info;
249 char name[VK_MAX_PHYSICAL_DEVICE_NAME_SIZE];
250 uint8_t driver_uuid[VK_UUID_SIZE];
251 uint8_t device_uuid[VK_UUID_SIZE];
252 uint8_t cache_uuid[VK_UUID_SIZE];
253
254 int local_fd;
255 int master_fd;
256 struct wsi_device wsi_device;
257
258 bool out_of_order_rast_allowed;
259
260 /* Whether DCC should be enabled for MSAA textures. */
261 bool dcc_msaa_allowed;
262
263 /* Whether to enable NGG. */
264 bool use_ngg;
265
266 /* Whether to enable NGG culling. */
267 bool use_ngg_culling;
268
269 /* Whether to enable NGG streamout. */
270 bool use_ngg_streamout;
271
272 /* Number of threads per wave. */
273 uint8_t ps_wave_size;
274 uint8_t cs_wave_size;
275 uint8_t ge_wave_size;
276
277 /* Whether to use the LLVM compiler backend */
278 bool use_llvm;
279
280 /* This is the drivers on-disk cache used as a fallback as opposed to
281 * the pipeline cache defined by apps.
282 */
283 struct disk_cache *disk_cache;
284
285 VkPhysicalDeviceMemoryProperties memory_properties;
286 enum radeon_bo_domain memory_domains[VK_MAX_MEMORY_TYPES];
287 enum radeon_bo_flag memory_flags[VK_MAX_MEMORY_TYPES];
288 unsigned heaps;
289
290 #ifndef _WIN32
291 int available_nodes;
292 drmPciBusInfo bus_info;
293
294 dev_t primary_devid;
295 dev_t render_devid;
296 #endif
297
298 nir_shader_compiler_options nir_options;
299 };
300
301 struct radv_instance {
302 struct vk_instance vk;
303
304 VkAllocationCallbacks alloc;
305
306 uint64_t debug_flags;
307 uint64_t perftest_flags;
308
309 bool physical_devices_enumerated;
310 struct list_head physical_devices;
311
312 struct driOptionCache dri_options;
313 struct driOptionCache available_dri_options;
314
315 /**
316 * Workarounds for game bugs.
317 */
318 bool enable_mrt_output_nan_fixup;
319 bool disable_tc_compat_htile_in_general;
320 bool disable_shrink_image_store;
321 bool absolute_depth_bias;
322 bool report_apu_as_dgpu;
323 };
324
325 VkResult radv_init_wsi(struct radv_physical_device *physical_device);
326 void radv_finish_wsi(struct radv_physical_device *physical_device);
327
328 struct cache_entry;
329
330 struct radv_pipeline_cache {
331 struct vk_object_base base;
332 struct radv_device *device;
333 mtx_t mutex;
334 VkPipelineCacheCreateFlags flags;
335
336 uint32_t total_size;
337 uint32_t table_size;
338 uint32_t kernel_count;
339 struct cache_entry **hash_table;
340 bool modified;
341
342 VkAllocationCallbacks alloc;
343 };
344
345 struct radv_shader_binary;
346 struct radv_shader_variant;
347 struct radv_pipeline_shader_stack_size;
348
349 void radv_pipeline_cache_init(struct radv_pipeline_cache *cache, struct radv_device *device);
350 void radv_pipeline_cache_finish(struct radv_pipeline_cache *cache);
351 bool radv_pipeline_cache_load(struct radv_pipeline_cache *cache, const void *data, size_t size);
352
353 bool radv_create_shader_variants_from_pipeline_cache(
354 struct radv_device *device, struct radv_pipeline_cache *cache, const unsigned char *sha1,
355 struct radv_shader_variant **variants, struct radv_pipeline_shader_stack_size **stack_sizes,
356 uint32_t *num_stack_sizes, bool *found_in_application_cache);
357
358 void radv_pipeline_cache_insert_shaders(
359 struct radv_device *device, struct radv_pipeline_cache *cache, const unsigned char *sha1,
360 struct radv_shader_variant **variants, struct radv_shader_binary *const *binaries,
361 const struct radv_pipeline_shader_stack_size *stack_sizes, uint32_t num_stack_sizes);
362
363 enum radv_blit_ds_layout {
364 RADV_BLIT_DS_LAYOUT_TILE_ENABLE,
365 RADV_BLIT_DS_LAYOUT_TILE_DISABLE,
366 RADV_BLIT_DS_LAYOUT_COUNT,
367 };
368
369 static inline enum radv_blit_ds_layout
radv_meta_blit_ds_to_type(VkImageLayout layout)370 radv_meta_blit_ds_to_type(VkImageLayout layout)
371 {
372 return (layout == VK_IMAGE_LAYOUT_GENERAL) ? RADV_BLIT_DS_LAYOUT_TILE_DISABLE
373 : RADV_BLIT_DS_LAYOUT_TILE_ENABLE;
374 }
375
376 static inline VkImageLayout
radv_meta_blit_ds_to_layout(enum radv_blit_ds_layout ds_layout)377 radv_meta_blit_ds_to_layout(enum radv_blit_ds_layout ds_layout)
378 {
379 return ds_layout == RADV_BLIT_DS_LAYOUT_TILE_ENABLE ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL
380 : VK_IMAGE_LAYOUT_GENERAL;
381 }
382
383 enum radv_meta_dst_layout {
384 RADV_META_DST_LAYOUT_GENERAL,
385 RADV_META_DST_LAYOUT_OPTIMAL,
386 RADV_META_DST_LAYOUT_COUNT,
387 };
388
389 static inline enum radv_meta_dst_layout
radv_meta_dst_layout_from_layout(VkImageLayout layout)390 radv_meta_dst_layout_from_layout(VkImageLayout layout)
391 {
392 return (layout == VK_IMAGE_LAYOUT_GENERAL) ? RADV_META_DST_LAYOUT_GENERAL
393 : RADV_META_DST_LAYOUT_OPTIMAL;
394 }
395
396 static inline VkImageLayout
radv_meta_dst_layout_to_layout(enum radv_meta_dst_layout layout)397 radv_meta_dst_layout_to_layout(enum radv_meta_dst_layout layout)
398 {
399 return layout == RADV_META_DST_LAYOUT_OPTIMAL ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL
400 : VK_IMAGE_LAYOUT_GENERAL;
401 }
402
403 struct radv_meta_state {
404 VkAllocationCallbacks alloc;
405
406 struct radv_pipeline_cache cache;
407
408 /*
409 * For on-demand pipeline creation, makes sure that
410 * only one thread tries to build a pipeline at the same time.
411 */
412 mtx_t mtx;
413
414 /**
415 * Use array element `i` for images with `2^i` samples.
416 */
417 struct {
418 VkRenderPass render_pass[NUM_META_FS_KEYS];
419 VkPipeline color_pipelines[NUM_META_FS_KEYS];
420
421 VkRenderPass depthstencil_rp;
422 VkPipeline depth_only_pipeline[NUM_DEPTH_CLEAR_PIPELINES];
423 VkPipeline stencil_only_pipeline[NUM_DEPTH_CLEAR_PIPELINES];
424 VkPipeline depthstencil_pipeline[NUM_DEPTH_CLEAR_PIPELINES];
425
426 VkPipeline depth_only_unrestricted_pipeline[NUM_DEPTH_CLEAR_PIPELINES];
427 VkPipeline stencil_only_unrestricted_pipeline[NUM_DEPTH_CLEAR_PIPELINES];
428 VkPipeline depthstencil_unrestricted_pipeline[NUM_DEPTH_CLEAR_PIPELINES];
429 } clear[MAX_SAMPLES_LOG2];
430
431 VkPipelineLayout clear_color_p_layout;
432 VkPipelineLayout clear_depth_p_layout;
433 VkPipelineLayout clear_depth_unrestricted_p_layout;
434
435 /* Optimized compute fast HTILE clear for stencil or depth only. */
436 VkPipeline clear_htile_mask_pipeline;
437 VkPipelineLayout clear_htile_mask_p_layout;
438 VkDescriptorSetLayout clear_htile_mask_ds_layout;
439
440 /* Copy VRS into HTILE. */
441 VkPipeline copy_vrs_htile_pipeline;
442 VkPipelineLayout copy_vrs_htile_p_layout;
443 VkDescriptorSetLayout copy_vrs_htile_ds_layout;
444
445 /* Clear DCC with comp-to-single. */
446 VkPipeline clear_dcc_comp_to_single_pipeline[2]; /* 0: 1x, 1: 2x/4x/8x */
447 VkPipelineLayout clear_dcc_comp_to_single_p_layout;
448 VkDescriptorSetLayout clear_dcc_comp_to_single_ds_layout;
449
450 struct {
451 VkRenderPass render_pass[NUM_META_FS_KEYS][RADV_META_DST_LAYOUT_COUNT];
452
453 /** Pipeline that blits from a 1D image. */
454 VkPipeline pipeline_1d_src[NUM_META_FS_KEYS];
455
456 /** Pipeline that blits from a 2D image. */
457 VkPipeline pipeline_2d_src[NUM_META_FS_KEYS];
458
459 /** Pipeline that blits from a 3D image. */
460 VkPipeline pipeline_3d_src[NUM_META_FS_KEYS];
461
462 VkRenderPass depth_only_rp[RADV_BLIT_DS_LAYOUT_COUNT];
463 VkPipeline depth_only_1d_pipeline;
464 VkPipeline depth_only_2d_pipeline;
465 VkPipeline depth_only_3d_pipeline;
466
467 VkRenderPass stencil_only_rp[RADV_BLIT_DS_LAYOUT_COUNT];
468 VkPipeline stencil_only_1d_pipeline;
469 VkPipeline stencil_only_2d_pipeline;
470 VkPipeline stencil_only_3d_pipeline;
471 VkPipelineLayout pipeline_layout;
472 VkDescriptorSetLayout ds_layout;
473 } blit;
474
475 struct {
476 VkPipelineLayout p_layouts[5];
477 VkDescriptorSetLayout ds_layouts[5];
478 VkPipeline pipelines[5][NUM_META_FS_KEYS];
479
480 VkPipeline depth_only_pipeline[5];
481
482 VkPipeline stencil_only_pipeline[5];
483 } blit2d[MAX_SAMPLES_LOG2];
484
485 VkRenderPass blit2d_render_passes[NUM_META_FS_KEYS][RADV_META_DST_LAYOUT_COUNT];
486 VkRenderPass blit2d_depth_only_rp[RADV_BLIT_DS_LAYOUT_COUNT];
487 VkRenderPass blit2d_stencil_only_rp[RADV_BLIT_DS_LAYOUT_COUNT];
488
489 struct {
490 VkPipelineLayout img_p_layout;
491 VkDescriptorSetLayout img_ds_layout;
492 VkPipeline pipeline;
493 VkPipeline pipeline_3d;
494 } itob;
495 struct {
496 VkPipelineLayout img_p_layout;
497 VkDescriptorSetLayout img_ds_layout;
498 VkPipeline pipeline;
499 VkPipeline pipeline_3d;
500 } btoi;
501 struct {
502 VkPipelineLayout img_p_layout;
503 VkDescriptorSetLayout img_ds_layout;
504 VkPipeline pipeline;
505 } btoi_r32g32b32;
506 struct {
507 VkPipelineLayout img_p_layout;
508 VkDescriptorSetLayout img_ds_layout;
509 VkPipeline pipeline[MAX_SAMPLES_LOG2];
510 VkPipeline pipeline_3d;
511 } itoi;
512 struct {
513 VkPipelineLayout img_p_layout;
514 VkDescriptorSetLayout img_ds_layout;
515 VkPipeline pipeline;
516 } itoi_r32g32b32;
517 struct {
518 VkPipelineLayout img_p_layout;
519 VkDescriptorSetLayout img_ds_layout;
520 VkPipeline pipeline[MAX_SAMPLES_LOG2];
521 VkPipeline pipeline_3d;
522 } cleari;
523 struct {
524 VkPipelineLayout img_p_layout;
525 VkDescriptorSetLayout img_ds_layout;
526 VkPipeline pipeline;
527 } cleari_r32g32b32;
528
529 struct {
530 VkPipelineLayout p_layout;
531 VkPipeline pipeline[NUM_META_FS_KEYS];
532 VkRenderPass pass[NUM_META_FS_KEYS];
533 } resolve;
534
535 struct {
536 VkDescriptorSetLayout ds_layout;
537 VkPipelineLayout p_layout;
538 struct {
539 VkPipeline pipeline;
540 VkPipeline i_pipeline;
541 VkPipeline srgb_pipeline;
542 } rc[MAX_SAMPLES_LOG2];
543
544 VkPipeline depth_zero_pipeline;
545 struct {
546 VkPipeline average_pipeline;
547 VkPipeline max_pipeline;
548 VkPipeline min_pipeline;
549 } depth[MAX_SAMPLES_LOG2];
550
551 VkPipeline stencil_zero_pipeline;
552 struct {
553 VkPipeline max_pipeline;
554 VkPipeline min_pipeline;
555 } stencil[MAX_SAMPLES_LOG2];
556 } resolve_compute;
557
558 struct {
559 VkDescriptorSetLayout ds_layout;
560 VkPipelineLayout p_layout;
561
562 struct {
563 VkRenderPass render_pass[NUM_META_FS_KEYS][RADV_META_DST_LAYOUT_COUNT];
564 VkPipeline pipeline[NUM_META_FS_KEYS];
565 } rc[MAX_SAMPLES_LOG2];
566
567 VkRenderPass depth_render_pass;
568 VkPipeline depth_zero_pipeline;
569 struct {
570 VkPipeline average_pipeline;
571 VkPipeline max_pipeline;
572 VkPipeline min_pipeline;
573 } depth[MAX_SAMPLES_LOG2];
574
575 VkRenderPass stencil_render_pass;
576 VkPipeline stencil_zero_pipeline;
577 struct {
578 VkPipeline max_pipeline;
579 VkPipeline min_pipeline;
580 } stencil[MAX_SAMPLES_LOG2];
581 } resolve_fragment;
582
583 struct {
584 VkPipelineLayout p_layout;
585 VkPipeline decompress_pipeline;
586 VkPipeline resummarize_pipeline;
587 VkRenderPass pass;
588 } depth_decomp[MAX_SAMPLES_LOG2];
589
590 VkDescriptorSetLayout expand_depth_stencil_compute_ds_layout;
591 VkPipelineLayout expand_depth_stencil_compute_p_layout;
592 VkPipeline expand_depth_stencil_compute_pipeline;
593
594 struct {
595 VkPipelineLayout p_layout;
596 VkPipeline cmask_eliminate_pipeline;
597 VkPipeline fmask_decompress_pipeline;
598 VkPipeline dcc_decompress_pipeline;
599 VkRenderPass pass;
600
601 VkDescriptorSetLayout dcc_decompress_compute_ds_layout;
602 VkPipelineLayout dcc_decompress_compute_p_layout;
603 VkPipeline dcc_decompress_compute_pipeline;
604 } fast_clear_flush;
605
606 struct {
607 VkPipelineLayout fill_p_layout;
608 VkPipelineLayout copy_p_layout;
609 VkDescriptorSetLayout fill_ds_layout;
610 VkDescriptorSetLayout copy_ds_layout;
611 VkPipeline fill_pipeline;
612 VkPipeline copy_pipeline;
613 } buffer;
614
615 struct {
616 VkDescriptorSetLayout ds_layout;
617 VkPipelineLayout p_layout;
618 VkPipeline occlusion_query_pipeline;
619 VkPipeline pipeline_statistics_query_pipeline;
620 VkPipeline tfb_query_pipeline;
621 VkPipeline timestamp_query_pipeline;
622 } query;
623
624 struct {
625 VkDescriptorSetLayout ds_layout;
626 VkPipelineLayout p_layout;
627 VkPipeline pipeline[MAX_SAMPLES_LOG2];
628 } fmask_expand;
629
630 struct {
631 VkDescriptorSetLayout ds_layout;
632 VkPipelineLayout p_layout;
633 VkPipeline pipeline[32];
634 } dcc_retile;
635
636 struct {
637 VkPipelineLayout leaf_p_layout;
638 VkPipeline leaf_pipeline;
639 VkPipelineLayout internal_p_layout;
640 VkPipeline internal_pipeline;
641 VkPipelineLayout copy_p_layout;
642 VkPipeline copy_pipeline;
643 } accel_struct_build;
644 };
645
646 /* queue types */
647 #define RADV_QUEUE_GENERAL 0
648 #define RADV_QUEUE_COMPUTE 1
649 #define RADV_QUEUE_TRANSFER 2
650
651 /* Not a real queue family */
652 #define RADV_QUEUE_FOREIGN 3
653
654 #define RADV_MAX_QUEUE_FAMILIES 3
655
656 #define RADV_NUM_HW_CTX (RADEON_CTX_PRIORITY_REALTIME + 1)
657
658 struct radv_deferred_queue_submission;
659
660 enum ring_type radv_queue_family_to_ring(int f);
661
662 struct radv_queue {
663 struct vk_queue vk;
664 struct radv_device *device;
665 struct radeon_winsys_ctx *hw_ctx;
666 enum radeon_ctx_priority priority;
667
668 uint32_t scratch_size_per_wave;
669 uint32_t scratch_waves;
670 uint32_t compute_scratch_size_per_wave;
671 uint32_t compute_scratch_waves;
672 uint32_t esgs_ring_size;
673 uint32_t gsvs_ring_size;
674 bool has_tess_rings;
675 bool has_gds;
676 bool has_gds_oa;
677 bool has_sample_positions;
678
679 struct radeon_winsys_bo *scratch_bo;
680 struct radeon_winsys_bo *descriptor_bo;
681 struct radeon_winsys_bo *compute_scratch_bo;
682 struct radeon_winsys_bo *esgs_ring_bo;
683 struct radeon_winsys_bo *gsvs_ring_bo;
684 struct radeon_winsys_bo *tess_rings_bo;
685 struct radeon_winsys_bo *gds_bo;
686 struct radeon_winsys_bo *gds_oa_bo;
687 struct radeon_cmdbuf *initial_preamble_cs;
688 struct radeon_cmdbuf *initial_full_flush_preamble_cs;
689 struct radeon_cmdbuf *continue_preamble_cs;
690
691 struct list_head pending_submissions;
692 mtx_t pending_mutex;
693
694 mtx_t thread_mutex;
695 struct u_cnd_monotonic thread_cond;
696 struct radv_deferred_queue_submission *thread_submission;
697 thrd_t submission_thread;
698 bool thread_exit;
699 bool thread_running;
700 bool cond_created;
701 };
702
703 #define RADV_BORDER_COLOR_COUNT 4096
704 #define RADV_BORDER_COLOR_BUFFER_SIZE (sizeof(VkClearColorValue) * RADV_BORDER_COLOR_COUNT)
705
706 struct radv_device_border_color_data {
707 bool used[RADV_BORDER_COLOR_COUNT];
708
709 struct radeon_winsys_bo *bo;
710 VkClearColorValue *colors_gpu_ptr;
711
712 /* Mutex is required to guarantee vkCreateSampler thread safety
713 * given that we are writing to a buffer and checking color occupation */
714 mtx_t mutex;
715 };
716
717 enum radv_force_vrs {
718 RADV_FORCE_VRS_NONE = 0,
719 RADV_FORCE_VRS_2x2,
720 RADV_FORCE_VRS_2x1,
721 RADV_FORCE_VRS_1x2,
722 };
723
724 struct radv_device {
725 struct vk_device vk;
726
727 struct radv_instance *instance;
728 struct radeon_winsys *ws;
729
730 struct radeon_winsys_ctx *hw_ctx[RADV_NUM_HW_CTX];
731 struct radv_meta_state meta_state;
732
733 struct radv_queue *queues[RADV_MAX_QUEUE_FAMILIES];
734 int queue_count[RADV_MAX_QUEUE_FAMILIES];
735 struct radeon_cmdbuf *empty_cs[RADV_MAX_QUEUE_FAMILIES];
736
737 bool pbb_allowed;
738 uint32_t tess_offchip_block_dw_size;
739 uint32_t scratch_waves;
740 uint32_t dispatch_initiator;
741
742 uint32_t gs_table_depth;
743
744 /* MSAA sample locations.
745 * The first index is the sample index.
746 * The second index is the coordinate: X, Y. */
747 float sample_locations_1x[1][2];
748 float sample_locations_2x[2][2];
749 float sample_locations_4x[4][2];
750 float sample_locations_8x[8][2];
751
752 /* GFX7 and later */
753 uint32_t gfx_init_size_dw;
754 struct radeon_winsys_bo *gfx_init;
755
756 struct radeon_winsys_bo *trace_bo;
757 uint32_t *trace_id_ptr;
758
759 /* Whether to keep shader debug info, for tracing or VK_AMD_shader_info */
760 bool keep_shader_info;
761
762 struct radv_physical_device *physical_device;
763
764 /* Backup in-memory cache to be used if the app doesn't provide one */
765 struct radv_pipeline_cache *mem_cache;
766
767 /*
768 * use different counters so MSAA MRTs get consecutive surface indices,
769 * even if MASK is allocated in between.
770 */
771 uint32_t image_mrt_offset_counter;
772 uint32_t fmask_mrt_offset_counter;
773
774 struct list_head shader_arenas;
775 uint8_t shader_free_list_mask;
776 struct list_head shader_free_lists[RADV_SHADER_ALLOC_NUM_FREE_LISTS];
777 struct list_head shader_block_obj_pool;
778 mtx_t shader_arena_mutex;
779
780 /* For detecting VM faults reported by dmesg. */
781 uint64_t dmesg_timestamp;
782
783 /* Whether the app has enabled the robustBufferAccess/robustBufferAccess2 features. */
784 bool robust_buffer_access;
785 bool robust_buffer_access2;
786
787 /* Whether gl_FragCoord.z should be adjusted for VRS due to a hw bug
788 * on some GFX10.3 chips.
789 */
790 bool adjust_frag_coord_z;
791
792 /* Whether the driver uses a global BO list. */
793 bool use_global_bo_list;
794
795 /* Whether attachment VRS is enabled. */
796 bool attachment_vrs_enabled;
797
798 /* Whether shader image 32-bit float atomics are enabled. */
799 bool image_float32_atomics;
800
801 /* Whether anisotropy is forced with RADV_TEX_ANISO (-1 is disabled). */
802 int force_aniso;
803
804 struct radv_device_border_color_data border_color_data;
805
806 /* Condition variable for legacy timelines, to notify waiters when a
807 * new point gets submitted. */
808 struct u_cnd_monotonic timeline_cond;
809
810 /* Thread trace. */
811 struct ac_thread_trace_data thread_trace;
812
813 /* Trap handler. */
814 struct radv_shader_variant *trap_handler_shader;
815 struct radeon_winsys_bo *tma_bo; /* Trap Memory Address */
816 uint32_t *tma_ptr;
817
818 /* Overallocation. */
819 bool overallocation_disallowed;
820 uint64_t allocated_memory_size[VK_MAX_MEMORY_HEAPS];
821 mtx_t overallocation_mutex;
822
823 /* Track the number of device loss occurs. */
824 int lost;
825
826 /* Whether the user forced VRS rates on GFX10.3+. */
827 enum radv_force_vrs force_vrs;
828
829 /* Depth image for VRS when not bound by the app. */
830 struct {
831 struct radv_image *image;
832 struct radv_buffer *buffer; /* HTILE */
833 struct radv_device_memory *mem;
834 } vrs;
835
836 struct u_rwlock vs_prologs_lock;
837 struct hash_table *vs_prologs;
838
839 struct radv_shader_prolog *simple_vs_prologs[MAX_VERTEX_ATTRIBS];
840 struct radv_shader_prolog *instance_rate_vs_prologs[816];
841 };
842
843 VkResult _radv_device_set_lost(struct radv_device *device, const char *file, int line,
844 const char *msg, ...) radv_printflike(4, 5);
845
846 #define radv_device_set_lost(dev, ...) _radv_device_set_lost(dev, __FILE__, __LINE__, __VA_ARGS__)
847
848 static inline bool
radv_device_is_lost(const struct radv_device * device)849 radv_device_is_lost(const struct radv_device *device)
850 {
851 return unlikely(p_atomic_read(&device->lost));
852 }
853
854 struct radv_device_memory {
855 struct vk_object_base base;
856 struct radeon_winsys_bo *bo;
857 /* for dedicated allocations */
858 struct radv_image *image;
859 struct radv_buffer *buffer;
860 uint32_t heap_index;
861 uint64_t alloc_size;
862 void *map;
863 void *user_ptr;
864
865 #if RADV_SUPPORT_ANDROID_HARDWARE_BUFFER
866 struct AHardwareBuffer *android_hardware_buffer;
867 #endif
868 };
869
870 void radv_device_memory_init(struct radv_device_memory *mem, struct radv_device *device,
871 struct radeon_winsys_bo *bo);
872 void radv_device_memory_finish(struct radv_device_memory *mem);
873
874 struct radv_descriptor_range {
875 uint64_t va;
876 uint32_t size;
877 };
878
879 struct radv_descriptor_set_header {
880 struct vk_object_base base;
881 const struct radv_descriptor_set_layout *layout;
882 uint32_t size;
883 uint32_t buffer_count;
884
885 struct radeon_winsys_bo *bo;
886 uint64_t va;
887 uint32_t *mapped_ptr;
888 struct radv_descriptor_range *dynamic_descriptors;
889 };
890
891 struct radv_descriptor_set {
892 struct radv_descriptor_set_header header;
893
894 struct radeon_winsys_bo *descriptors[];
895 };
896
897 struct radv_push_descriptor_set {
898 struct radv_descriptor_set_header set;
899 uint32_t capacity;
900 };
901
902 struct radv_descriptor_pool_entry {
903 uint32_t offset;
904 uint32_t size;
905 struct radv_descriptor_set *set;
906 };
907
908 struct radv_descriptor_pool {
909 struct vk_object_base base;
910 struct radeon_winsys_bo *bo;
911 uint8_t *host_bo;
912 uint8_t *mapped_ptr;
913 uint64_t current_offset;
914 uint64_t size;
915
916 uint8_t *host_memory_base;
917 uint8_t *host_memory_ptr;
918 uint8_t *host_memory_end;
919
920 uint32_t entry_count;
921 uint32_t max_entry_count;
922 struct radv_descriptor_pool_entry entries[0];
923 };
924
925 struct radv_descriptor_update_template_entry {
926 VkDescriptorType descriptor_type;
927
928 /* The number of descriptors to update */
929 uint32_t descriptor_count;
930
931 /* Into mapped_ptr or dynamic_descriptors, in units of the respective array */
932 uint32_t dst_offset;
933
934 /* In dwords. Not valid/used for dynamic descriptors */
935 uint32_t dst_stride;
936
937 uint32_t buffer_offset;
938
939 /* Only valid for combined image samplers and samplers */
940 uint8_t has_sampler;
941 uint8_t sampler_offset;
942
943 /* In bytes */
944 size_t src_offset;
945 size_t src_stride;
946
947 /* For push descriptors */
948 const uint32_t *immutable_samplers;
949 };
950
951 struct radv_descriptor_update_template {
952 struct vk_object_base base;
953 uint32_t entry_count;
954 VkPipelineBindPoint bind_point;
955 struct radv_descriptor_update_template_entry entry[0];
956 };
957
958 struct radv_buffer {
959 struct vk_object_base base;
960 VkDeviceSize size;
961
962 VkBufferUsageFlags usage;
963 VkBufferCreateFlags flags;
964
965 /* Set when bound */
966 struct radeon_winsys_bo *bo;
967 VkDeviceSize offset;
968
969 bool shareable;
970 };
971
972 void radv_buffer_init(struct radv_buffer *buffer, struct radv_device *device,
973 struct radeon_winsys_bo *bo, uint64_t size, uint64_t offset);
974 void radv_buffer_finish(struct radv_buffer *buffer);
975
976 enum radv_dynamic_state_bits {
977 RADV_DYNAMIC_VIEWPORT = 1ull << 0,
978 RADV_DYNAMIC_SCISSOR = 1ull << 1,
979 RADV_DYNAMIC_LINE_WIDTH = 1ull << 2,
980 RADV_DYNAMIC_DEPTH_BIAS = 1ull << 3,
981 RADV_DYNAMIC_BLEND_CONSTANTS = 1ull << 4,
982 RADV_DYNAMIC_DEPTH_BOUNDS = 1ull << 5,
983 RADV_DYNAMIC_STENCIL_COMPARE_MASK = 1ull << 6,
984 RADV_DYNAMIC_STENCIL_WRITE_MASK = 1ull << 7,
985 RADV_DYNAMIC_STENCIL_REFERENCE = 1ull << 8,
986 RADV_DYNAMIC_DISCARD_RECTANGLE = 1ull << 9,
987 RADV_DYNAMIC_SAMPLE_LOCATIONS = 1ull << 10,
988 RADV_DYNAMIC_LINE_STIPPLE = 1ull << 11,
989 RADV_DYNAMIC_CULL_MODE = 1ull << 12,
990 RADV_DYNAMIC_FRONT_FACE = 1ull << 13,
991 RADV_DYNAMIC_PRIMITIVE_TOPOLOGY = 1ull << 14,
992 RADV_DYNAMIC_DEPTH_TEST_ENABLE = 1ull << 15,
993 RADV_DYNAMIC_DEPTH_WRITE_ENABLE = 1ull << 16,
994 RADV_DYNAMIC_DEPTH_COMPARE_OP = 1ull << 17,
995 RADV_DYNAMIC_DEPTH_BOUNDS_TEST_ENABLE = 1ull << 18,
996 RADV_DYNAMIC_STENCIL_TEST_ENABLE = 1ull << 19,
997 RADV_DYNAMIC_STENCIL_OP = 1ull << 20,
998 RADV_DYNAMIC_VERTEX_INPUT_BINDING_STRIDE = 1ull << 21,
999 RADV_DYNAMIC_FRAGMENT_SHADING_RATE = 1ull << 22,
1000 RADV_DYNAMIC_PATCH_CONTROL_POINTS = 1ull << 23,
1001 RADV_DYNAMIC_RASTERIZER_DISCARD_ENABLE = 1ull << 24,
1002 RADV_DYNAMIC_DEPTH_BIAS_ENABLE = 1ull << 25,
1003 RADV_DYNAMIC_LOGIC_OP = 1ull << 26,
1004 RADV_DYNAMIC_PRIMITIVE_RESTART_ENABLE = 1ull << 27,
1005 RADV_DYNAMIC_COLOR_WRITE_ENABLE = 1ull << 28,
1006 RADV_DYNAMIC_VERTEX_INPUT = 1ull << 29,
1007 RADV_DYNAMIC_ALL = (1ull << 30) - 1,
1008 };
1009
1010 enum radv_cmd_dirty_bits {
1011 /* Keep the dynamic state dirty bits in sync with
1012 * enum radv_dynamic_state_bits */
1013 RADV_CMD_DIRTY_DYNAMIC_VIEWPORT = 1ull << 0,
1014 RADV_CMD_DIRTY_DYNAMIC_SCISSOR = 1ull << 1,
1015 RADV_CMD_DIRTY_DYNAMIC_LINE_WIDTH = 1ull << 2,
1016 RADV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS = 1ull << 3,
1017 RADV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS = 1ull << 4,
1018 RADV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS = 1ull << 5,
1019 RADV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK = 1ull << 6,
1020 RADV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK = 1ull << 7,
1021 RADV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE = 1ull << 8,
1022 RADV_CMD_DIRTY_DYNAMIC_DISCARD_RECTANGLE = 1ull << 9,
1023 RADV_CMD_DIRTY_DYNAMIC_SAMPLE_LOCATIONS = 1ull << 10,
1024 RADV_CMD_DIRTY_DYNAMIC_LINE_STIPPLE = 1ull << 11,
1025 RADV_CMD_DIRTY_DYNAMIC_CULL_MODE = 1ull << 12,
1026 RADV_CMD_DIRTY_DYNAMIC_FRONT_FACE = 1ull << 13,
1027 RADV_CMD_DIRTY_DYNAMIC_PRIMITIVE_TOPOLOGY = 1ull << 14,
1028 RADV_CMD_DIRTY_DYNAMIC_DEPTH_TEST_ENABLE = 1ull << 15,
1029 RADV_CMD_DIRTY_DYNAMIC_DEPTH_WRITE_ENABLE = 1ull << 16,
1030 RADV_CMD_DIRTY_DYNAMIC_DEPTH_COMPARE_OP = 1ull << 17,
1031 RADV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS_TEST_ENABLE = 1ull << 18,
1032 RADV_CMD_DIRTY_DYNAMIC_STENCIL_TEST_ENABLE = 1ull << 19,
1033 RADV_CMD_DIRTY_DYNAMIC_STENCIL_OP = 1ull << 20,
1034 RADV_CMD_DIRTY_DYNAMIC_VERTEX_INPUT_BINDING_STRIDE = 1ull << 21,
1035 RADV_CMD_DIRTY_DYNAMIC_FRAGMENT_SHADING_RATE = 1ull << 22,
1036 RADV_CMD_DIRTY_DYNAMIC_PATCH_CONTROL_POINTS = 1ull << 23,
1037 RADV_CMD_DIRTY_DYNAMIC_RASTERIZER_DISCARD_ENABLE = 1ull << 24,
1038 RADV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS_ENABLE = 1ull << 25,
1039 RADV_CMD_DIRTY_DYNAMIC_LOGIC_OP = 1ull << 26,
1040 RADV_CMD_DIRTY_DYNAMIC_PRIMITIVE_RESTART_ENABLE = 1ull << 27,
1041 RADV_CMD_DIRTY_DYNAMIC_COLOR_WRITE_ENABLE = 1ull << 28,
1042 RADV_CMD_DIRTY_DYNAMIC_VERTEX_INPUT = 1ull << 29,
1043 RADV_CMD_DIRTY_DYNAMIC_ALL = (1ull << 30) - 1,
1044 RADV_CMD_DIRTY_PIPELINE = 1ull << 30,
1045 RADV_CMD_DIRTY_INDEX_BUFFER = 1ull << 31,
1046 RADV_CMD_DIRTY_FRAMEBUFFER = 1ull << 32,
1047 RADV_CMD_DIRTY_VERTEX_BUFFER = 1ull << 33,
1048 RADV_CMD_DIRTY_STREAMOUT_BUFFER = 1ull << 34,
1049 };
1050
1051 enum radv_cmd_flush_bits {
1052 /* Instruction cache. */
1053 RADV_CMD_FLAG_INV_ICACHE = 1 << 0,
1054 /* Scalar L1 cache. */
1055 RADV_CMD_FLAG_INV_SCACHE = 1 << 1,
1056 /* Vector L1 cache. */
1057 RADV_CMD_FLAG_INV_VCACHE = 1 << 2,
1058 /* L2 cache + L2 metadata cache writeback & invalidate.
1059 * GFX6-8: Used by shaders only. GFX9-10: Used by everything. */
1060 RADV_CMD_FLAG_INV_L2 = 1 << 3,
1061 /* L2 writeback (write dirty L2 lines to memory for non-L2 clients).
1062 * Only used for coherency with non-L2 clients like CB, DB, CP on GFX6-8.
1063 * GFX6-7 will do complete invalidation, because the writeback is unsupported. */
1064 RADV_CMD_FLAG_WB_L2 = 1 << 4,
1065 /* Invalidate the metadata cache. To be used when the DCC/HTILE metadata
1066 * changed and we want to read an image from shaders. */
1067 RADV_CMD_FLAG_INV_L2_METADATA = 1 << 5,
1068 /* Framebuffer caches */
1069 RADV_CMD_FLAG_FLUSH_AND_INV_CB_META = 1 << 6,
1070 RADV_CMD_FLAG_FLUSH_AND_INV_DB_META = 1 << 7,
1071 RADV_CMD_FLAG_FLUSH_AND_INV_DB = 1 << 8,
1072 RADV_CMD_FLAG_FLUSH_AND_INV_CB = 1 << 9,
1073 /* Engine synchronization. */
1074 RADV_CMD_FLAG_VS_PARTIAL_FLUSH = 1 << 10,
1075 RADV_CMD_FLAG_PS_PARTIAL_FLUSH = 1 << 11,
1076 RADV_CMD_FLAG_CS_PARTIAL_FLUSH = 1 << 12,
1077 RADV_CMD_FLAG_VGT_FLUSH = 1 << 13,
1078 /* Pipeline query controls. */
1079 RADV_CMD_FLAG_START_PIPELINE_STATS = 1 << 14,
1080 RADV_CMD_FLAG_STOP_PIPELINE_STATS = 1 << 15,
1081 RADV_CMD_FLAG_VGT_STREAMOUT_SYNC = 1 << 16,
1082
1083 RADV_CMD_FLUSH_AND_INV_FRAMEBUFFER =
1084 (RADV_CMD_FLAG_FLUSH_AND_INV_CB | RADV_CMD_FLAG_FLUSH_AND_INV_CB_META |
1085 RADV_CMD_FLAG_FLUSH_AND_INV_DB | RADV_CMD_FLAG_FLUSH_AND_INV_DB_META)
1086 };
1087
1088 struct radv_vertex_binding {
1089 struct radv_buffer *buffer;
1090 VkDeviceSize offset;
1091 VkDeviceSize size;
1092 VkDeviceSize stride;
1093 };
1094
1095 struct radv_streamout_binding {
1096 struct radv_buffer *buffer;
1097 VkDeviceSize offset;
1098 VkDeviceSize size;
1099 };
1100
1101 struct radv_streamout_state {
1102 /* Mask of bound streamout buffers. */
1103 uint8_t enabled_mask;
1104
1105 /* External state that comes from the last vertex stage, it must be
1106 * set explicitely when binding a new graphics pipeline.
1107 */
1108 uint16_t stride_in_dw[MAX_SO_BUFFERS];
1109 uint32_t enabled_stream_buffers_mask; /* stream0 buffers0-3 in 4 LSB */
1110
1111 /* State of VGT_STRMOUT_BUFFER_(CONFIG|END) */
1112 uint32_t hw_enabled_mask;
1113
1114 /* State of VGT_STRMOUT_(CONFIG|EN) */
1115 bool streamout_enabled;
1116 };
1117
1118 struct radv_viewport_state {
1119 uint32_t count;
1120 VkViewport viewports[MAX_VIEWPORTS];
1121 struct {
1122 float scale[3];
1123 float translate[3];
1124 } xform[MAX_VIEWPORTS];
1125 };
1126
1127 struct radv_scissor_state {
1128 uint32_t count;
1129 VkRect2D scissors[MAX_SCISSORS];
1130 };
1131
1132 struct radv_discard_rectangle_state {
1133 uint32_t count;
1134 VkRect2D rectangles[MAX_DISCARD_RECTANGLES];
1135 };
1136
1137 struct radv_sample_locations_state {
1138 VkSampleCountFlagBits per_pixel;
1139 VkExtent2D grid_size;
1140 uint32_t count;
1141 VkSampleLocationEXT locations[MAX_SAMPLE_LOCATIONS];
1142 };
1143
1144 struct radv_dynamic_state {
1145 /**
1146 * Bitmask of (1ull << VK_DYNAMIC_STATE_*).
1147 * Defines the set of saved dynamic state.
1148 */
1149 uint64_t mask;
1150
1151 struct radv_viewport_state viewport;
1152
1153 struct radv_scissor_state scissor;
1154
1155 float line_width;
1156
1157 struct {
1158 float bias;
1159 float clamp;
1160 float slope;
1161 } depth_bias;
1162
1163 float blend_constants[4];
1164
1165 struct {
1166 float min;
1167 float max;
1168 } depth_bounds;
1169
1170 struct {
1171 uint32_t front;
1172 uint32_t back;
1173 } stencil_compare_mask;
1174
1175 struct {
1176 uint32_t front;
1177 uint32_t back;
1178 } stencil_write_mask;
1179
1180 struct {
1181 struct {
1182 VkStencilOp fail_op;
1183 VkStencilOp pass_op;
1184 VkStencilOp depth_fail_op;
1185 VkCompareOp compare_op;
1186 } front;
1187
1188 struct {
1189 VkStencilOp fail_op;
1190 VkStencilOp pass_op;
1191 VkStencilOp depth_fail_op;
1192 VkCompareOp compare_op;
1193 } back;
1194 } stencil_op;
1195
1196 struct {
1197 uint32_t front;
1198 uint32_t back;
1199 } stencil_reference;
1200
1201 struct radv_discard_rectangle_state discard_rectangle;
1202
1203 struct radv_sample_locations_state sample_location;
1204
1205 struct {
1206 uint32_t factor;
1207 uint16_t pattern;
1208 } line_stipple;
1209
1210 VkCullModeFlags cull_mode;
1211 VkFrontFace front_face;
1212 unsigned primitive_topology;
1213
1214 bool depth_test_enable;
1215 bool depth_write_enable;
1216 VkCompareOp depth_compare_op;
1217 bool depth_bounds_test_enable;
1218 bool stencil_test_enable;
1219
1220 struct {
1221 VkExtent2D size;
1222 VkFragmentShadingRateCombinerOpKHR combiner_ops[2];
1223 } fragment_shading_rate;
1224
1225 bool depth_bias_enable;
1226 bool primitive_restart_enable;
1227 bool rasterizer_discard_enable;
1228
1229 unsigned logic_op;
1230
1231 uint32_t color_write_enable;
1232 };
1233
1234 extern const struct radv_dynamic_state default_dynamic_state;
1235
1236 const char *radv_get_debug_option_name(int id);
1237
1238 const char *radv_get_perftest_option_name(int id);
1239
1240 int radv_get_int_debug_option(const char *name, int default_value);
1241
1242 struct radv_color_buffer_info {
1243 uint64_t cb_color_base;
1244 uint64_t cb_color_cmask;
1245 uint64_t cb_color_fmask;
1246 uint64_t cb_dcc_base;
1247 uint32_t cb_color_slice;
1248 uint32_t cb_color_view;
1249 uint32_t cb_color_info;
1250 uint32_t cb_color_attrib;
1251 uint32_t cb_color_attrib2; /* GFX9 and later */
1252 uint32_t cb_color_attrib3; /* GFX10 and later */
1253 uint32_t cb_dcc_control;
1254 uint32_t cb_color_cmask_slice;
1255 uint32_t cb_color_fmask_slice;
1256 union {
1257 uint32_t cb_color_pitch; // GFX6-GFX8
1258 uint32_t cb_mrt_epitch; // GFX9+
1259 };
1260 };
1261
1262 struct radv_ds_buffer_info {
1263 uint64_t db_z_read_base;
1264 uint64_t db_stencil_read_base;
1265 uint64_t db_z_write_base;
1266 uint64_t db_stencil_write_base;
1267 uint64_t db_htile_data_base;
1268 uint32_t db_depth_info;
1269 uint32_t db_z_info;
1270 uint32_t db_stencil_info;
1271 uint32_t db_depth_view;
1272 uint32_t db_depth_size;
1273 uint32_t db_depth_slice;
1274 uint32_t db_htile_surface;
1275 uint32_t pa_su_poly_offset_db_fmt_cntl;
1276 uint32_t db_z_info2; /* GFX9 only */
1277 uint32_t db_stencil_info2; /* GFX9 only */
1278 };
1279
1280 void radv_initialise_color_surface(struct radv_device *device, struct radv_color_buffer_info *cb,
1281 struct radv_image_view *iview);
1282 void radv_initialise_ds_surface(struct radv_device *device, struct radv_ds_buffer_info *ds,
1283 struct radv_image_view *iview);
1284 void radv_initialise_vrs_surface(struct radv_image *image, struct radv_buffer *htile_buffer,
1285 struct radv_ds_buffer_info *ds);
1286
1287 /**
1288 * Attachment state when recording a renderpass instance.
1289 *
1290 * The clear value is valid only if there exists a pending clear.
1291 */
1292 struct radv_attachment_state {
1293 VkImageAspectFlags pending_clear_aspects;
1294 uint32_t cleared_views;
1295 VkClearValue clear_value;
1296 VkImageLayout current_layout;
1297 VkImageLayout current_stencil_layout;
1298 bool current_in_render_loop;
1299 bool disable_dcc;
1300 struct radv_sample_locations_state sample_location;
1301
1302 union {
1303 struct radv_color_buffer_info cb;
1304 struct radv_ds_buffer_info ds;
1305 };
1306 struct radv_image_view *iview;
1307 };
1308
1309 struct radv_descriptor_state {
1310 struct radv_descriptor_set *sets[MAX_SETS];
1311 uint32_t dirty;
1312 uint32_t valid;
1313 struct radv_push_descriptor_set push_set;
1314 bool push_dirty;
1315 uint32_t dynamic_buffers[4 * MAX_DYNAMIC_BUFFERS];
1316 };
1317
1318 struct radv_subpass_sample_locs_state {
1319 uint32_t subpass_idx;
1320 struct radv_sample_locations_state sample_location;
1321 };
1322
1323 enum rgp_flush_bits {
1324 RGP_FLUSH_WAIT_ON_EOP_TS = 0x1,
1325 RGP_FLUSH_VS_PARTIAL_FLUSH = 0x2,
1326 RGP_FLUSH_PS_PARTIAL_FLUSH = 0x4,
1327 RGP_FLUSH_CS_PARTIAL_FLUSH = 0x8,
1328 RGP_FLUSH_PFP_SYNC_ME = 0x10,
1329 RGP_FLUSH_SYNC_CP_DMA = 0x20,
1330 RGP_FLUSH_INVAL_VMEM_L0 = 0x40,
1331 RGP_FLUSH_INVAL_ICACHE = 0x80,
1332 RGP_FLUSH_INVAL_SMEM_L0 = 0x100,
1333 RGP_FLUSH_FLUSH_L2 = 0x200,
1334 RGP_FLUSH_INVAL_L2 = 0x400,
1335 RGP_FLUSH_FLUSH_CB = 0x800,
1336 RGP_FLUSH_INVAL_CB = 0x1000,
1337 RGP_FLUSH_FLUSH_DB = 0x2000,
1338 RGP_FLUSH_INVAL_DB = 0x4000,
1339 RGP_FLUSH_INVAL_L1 = 0x8000,
1340 };
1341
1342 struct radv_cmd_state {
1343 /* Vertex descriptors */
1344 uint64_t vb_va;
1345
1346 bool predicating;
1347 uint64_t dirty;
1348
1349 uint32_t prefetch_L2_mask;
1350
1351 struct radv_pipeline *pipeline;
1352 struct radv_pipeline *emitted_pipeline;
1353 struct radv_pipeline *compute_pipeline;
1354 struct radv_pipeline *emitted_compute_pipeline;
1355 struct radv_pipeline *rt_pipeline; /* emitted = emitted_compute_pipeline */
1356 struct radv_framebuffer *framebuffer;
1357 struct radv_render_pass *pass;
1358 const struct radv_subpass *subpass;
1359 struct radv_dynamic_state dynamic;
1360 struct radv_vs_input_state dynamic_vs_input;
1361 struct radv_attachment_state *attachments;
1362 struct radv_streamout_state streamout;
1363 VkRect2D render_area;
1364
1365 uint32_t num_subpass_sample_locs;
1366 struct radv_subpass_sample_locs_state *subpass_sample_locs;
1367
1368 /* Index buffer */
1369 struct radv_buffer *index_buffer;
1370 uint64_t index_offset;
1371 uint32_t index_type;
1372 uint32_t max_index_count;
1373 uint64_t index_va;
1374 int32_t last_index_type;
1375
1376 int32_t last_primitive_reset_en;
1377 uint32_t last_primitive_reset_index;
1378 enum radv_cmd_flush_bits flush_bits;
1379 unsigned active_occlusion_queries;
1380 bool perfect_occlusion_queries_enabled;
1381 unsigned active_pipeline_queries;
1382 unsigned active_pipeline_gds_queries;
1383 uint32_t trace_id;
1384 uint32_t last_ia_multi_vgt_param;
1385
1386 uint32_t last_num_instances;
1387 uint32_t last_first_instance;
1388 uint32_t last_vertex_offset;
1389 uint32_t last_drawid;
1390
1391 uint32_t last_sx_ps_downconvert;
1392 uint32_t last_sx_blend_opt_epsilon;
1393 uint32_t last_sx_blend_opt_control;
1394
1395 /* Whether CP DMA is busy/idle. */
1396 bool dma_is_busy;
1397
1398 /* Whether any images that are not L2 coherent are dirty from the CB. */
1399 bool rb_noncoherent_dirty;
1400
1401 /* Conditional rendering info. */
1402 uint8_t predication_op; /* 32-bit or 64-bit predicate value */
1403 int predication_type; /* -1: disabled, 0: normal, 1: inverted */
1404 uint64_t predication_va;
1405
1406 /* Inheritance info. */
1407 VkQueryPipelineStatisticFlags inherited_pipeline_statistics;
1408
1409 bool context_roll_without_scissor_emitted;
1410
1411 /* SQTT related state. */
1412 uint32_t current_event_type;
1413 uint32_t num_events;
1414 uint32_t num_layout_transitions;
1415 bool pending_sqtt_barrier_end;
1416 enum rgp_flush_bits sqtt_flush_bits;
1417
1418 /* NGG culling state. */
1419 uint32_t last_nggc_settings;
1420 int8_t last_nggc_settings_sgpr_idx;
1421 bool last_nggc_skip;
1422
1423 uint8_t cb_mip[MAX_RTS];
1424
1425 /* Whether DRAW_{INDEX}_INDIRECT_MULTI is emitted. */
1426 bool uses_draw_indirect_multi;
1427
1428 uint32_t rt_stack_size;
1429
1430 struct radv_shader_prolog *emitted_vs_prolog;
1431 uint32_t *emitted_vs_prolog_key;
1432 uint32_t emitted_vs_prolog_key_hash;
1433 uint32_t vbo_misaligned_mask;
1434 uint32_t vbo_bound_mask;
1435 };
1436
1437 struct radv_cmd_pool {
1438 struct vk_object_base base;
1439 VkAllocationCallbacks alloc;
1440 struct list_head cmd_buffers;
1441 struct list_head free_cmd_buffers;
1442 uint32_t queue_family_index;
1443 };
1444
1445 struct radv_cmd_buffer_upload {
1446 uint8_t *map;
1447 unsigned offset;
1448 uint64_t size;
1449 struct radeon_winsys_bo *upload_bo;
1450 struct list_head list;
1451 };
1452
1453 enum radv_cmd_buffer_status {
1454 RADV_CMD_BUFFER_STATUS_INVALID,
1455 RADV_CMD_BUFFER_STATUS_INITIAL,
1456 RADV_CMD_BUFFER_STATUS_RECORDING,
1457 RADV_CMD_BUFFER_STATUS_EXECUTABLE,
1458 RADV_CMD_BUFFER_STATUS_PENDING,
1459 };
1460
1461 struct radv_cmd_buffer {
1462 struct vk_command_buffer vk;
1463
1464 struct radv_device *device;
1465
1466 struct radv_cmd_pool *pool;
1467 struct list_head pool_link;
1468
1469 VkCommandBufferUsageFlags usage_flags;
1470 VkCommandBufferLevel level;
1471 enum radv_cmd_buffer_status status;
1472 struct radeon_cmdbuf *cs;
1473 struct radv_cmd_state state;
1474 struct radv_vertex_binding vertex_bindings[MAX_VBS];
1475 struct radv_streamout_binding streamout_bindings[MAX_SO_BUFFERS];
1476 uint32_t queue_family_index;
1477
1478 uint8_t push_constants[MAX_PUSH_CONSTANTS_SIZE];
1479 VkShaderStageFlags push_constant_stages;
1480 struct radv_descriptor_set_header meta_push_descriptors;
1481
1482 struct radv_descriptor_state descriptors[MAX_BIND_POINTS];
1483
1484 struct radv_cmd_buffer_upload upload;
1485
1486 uint32_t scratch_size_per_wave_needed;
1487 uint32_t scratch_waves_wanted;
1488 uint32_t compute_scratch_size_per_wave_needed;
1489 uint32_t compute_scratch_waves_wanted;
1490 uint32_t esgs_ring_size_needed;
1491 uint32_t gsvs_ring_size_needed;
1492 bool tess_rings_needed;
1493 bool gds_needed; /* for GFX10 streamout and NGG GS queries */
1494 bool gds_oa_needed; /* for GFX10 streamout */
1495 bool sample_positions_needed;
1496
1497 VkResult record_result;
1498
1499 uint64_t gfx9_fence_va;
1500 uint32_t gfx9_fence_idx;
1501 uint64_t gfx9_eop_bug_va;
1502
1503 /**
1504 * Whether a query pool has been resetted and we have to flush caches.
1505 */
1506 bool pending_reset_query;
1507
1508 /**
1509 * Bitmask of pending active query flushes.
1510 */
1511 enum radv_cmd_flush_bits active_query_flush_bits;
1512 };
1513
1514 struct radv_image;
1515 struct radv_image_view;
1516
1517 bool radv_cmd_buffer_uses_mec(struct radv_cmd_buffer *cmd_buffer);
1518
1519 void si_emit_graphics(struct radv_device *device, struct radeon_cmdbuf *cs);
1520 void si_emit_compute(struct radv_device *device, struct radeon_cmdbuf *cs);
1521
1522 void cik_create_gfx_config(struct radv_device *device);
1523
1524 void si_write_scissors(struct radeon_cmdbuf *cs, int first, int count, const VkRect2D *scissors,
1525 const VkViewport *viewports, bool can_use_guardband);
1526 uint32_t si_get_ia_multi_vgt_param(struct radv_cmd_buffer *cmd_buffer, bool instanced_draw,
1527 bool indirect_draw, bool count_from_stream_output,
1528 uint32_t draw_vertex_count, unsigned topology,
1529 bool prim_restart_enable);
1530 void si_cs_emit_write_event_eop(struct radeon_cmdbuf *cs, enum chip_class chip_class, bool is_mec,
1531 unsigned event, unsigned event_flags, unsigned dst_sel,
1532 unsigned data_sel, uint64_t va, uint32_t new_fence,
1533 uint64_t gfx9_eop_bug_va);
1534
1535 void radv_cp_wait_mem(struct radeon_cmdbuf *cs, uint32_t op, uint64_t va, uint32_t ref,
1536 uint32_t mask);
1537 void si_cs_emit_cache_flush(struct radeon_cmdbuf *cs, enum chip_class chip_class,
1538 uint32_t *fence_ptr, uint64_t va, bool is_mec,
1539 enum radv_cmd_flush_bits flush_bits,
1540 enum rgp_flush_bits *sqtt_flush_bits, uint64_t gfx9_eop_bug_va);
1541 void si_emit_cache_flush(struct radv_cmd_buffer *cmd_buffer);
1542 void si_emit_set_predication_state(struct radv_cmd_buffer *cmd_buffer, bool draw_visible,
1543 unsigned pred_op, uint64_t va);
1544 void si_cp_dma_buffer_copy(struct radv_cmd_buffer *cmd_buffer, uint64_t src_va, uint64_t dest_va,
1545 uint64_t size);
1546 void si_cp_dma_prefetch(struct radv_cmd_buffer *cmd_buffer, uint64_t va, unsigned size);
1547 void si_cp_dma_clear_buffer(struct radv_cmd_buffer *cmd_buffer, uint64_t va, uint64_t size,
1548 unsigned value);
1549 void si_cp_dma_wait_for_idle(struct radv_cmd_buffer *cmd_buffer);
1550
1551 void radv_set_db_count_control(struct radv_cmd_buffer *cmd_buffer);
1552
1553 unsigned radv_instance_rate_prolog_index(unsigned num_attributes, uint32_t instance_rate_inputs);
1554 uint32_t radv_hash_vs_prolog(const void *key_);
1555 bool radv_cmp_vs_prolog(const void *a_, const void *b_);
1556
1557 bool radv_cmd_buffer_upload_alloc(struct radv_cmd_buffer *cmd_buffer, unsigned size,
1558 unsigned *out_offset, void **ptr);
1559 void radv_cmd_buffer_set_subpass(struct radv_cmd_buffer *cmd_buffer,
1560 const struct radv_subpass *subpass);
1561 void radv_cmd_buffer_restore_subpass(struct radv_cmd_buffer *cmd_buffer,
1562 const struct radv_subpass *subpass);
1563 bool radv_cmd_buffer_upload_data(struct radv_cmd_buffer *cmd_buffer, unsigned size,
1564 const void *data, unsigned *out_offset);
1565
1566 void radv_cmd_buffer_clear_subpass(struct radv_cmd_buffer *cmd_buffer);
1567 void radv_cmd_buffer_resolve_subpass(struct radv_cmd_buffer *cmd_buffer);
1568 void radv_cmd_buffer_resolve_subpass_cs(struct radv_cmd_buffer *cmd_buffer);
1569 void radv_depth_stencil_resolve_subpass_cs(struct radv_cmd_buffer *cmd_buffer,
1570 VkImageAspectFlags aspects,
1571 VkResolveModeFlagBits resolve_mode);
1572 void radv_cmd_buffer_resolve_subpass_fs(struct radv_cmd_buffer *cmd_buffer);
1573 void radv_depth_stencil_resolve_subpass_fs(struct radv_cmd_buffer *cmd_buffer,
1574 VkImageAspectFlags aspects,
1575 VkResolveModeFlagBits resolve_mode);
1576 void radv_emit_default_sample_locations(struct radeon_cmdbuf *cs, int nr_samples);
1577 unsigned radv_get_default_max_sample_dist(int log_samples);
1578 void radv_device_init_msaa(struct radv_device *device);
1579 VkResult radv_device_init_vrs_state(struct radv_device *device);
1580
1581 void radv_update_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer,
1582 const struct radv_image_view *iview,
1583 VkClearDepthStencilValue ds_clear_value,
1584 VkImageAspectFlags aspects);
1585
1586 void radv_update_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer,
1587 const struct radv_image_view *iview, int cb_idx,
1588 uint32_t color_values[2]);
1589
1590 bool radv_image_use_dcc_image_stores(const struct radv_device *device,
1591 const struct radv_image *image);
1592 bool radv_image_use_dcc_predication(const struct radv_device *device,
1593 const struct radv_image *image);
1594
1595 void radv_update_fce_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
1596 const VkImageSubresourceRange *range, bool value);
1597
1598 void radv_update_dcc_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
1599 const VkImageSubresourceRange *range, bool value);
1600 enum radv_cmd_flush_bits radv_src_access_flush(struct radv_cmd_buffer *cmd_buffer,
1601 VkAccessFlags src_flags,
1602 const struct radv_image *image);
1603 enum radv_cmd_flush_bits radv_dst_access_flush(struct radv_cmd_buffer *cmd_buffer,
1604 VkAccessFlags dst_flags,
1605 const struct radv_image *image);
1606 uint32_t radv_fill_buffer(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *image,
1607 struct radeon_winsys_bo *bo, uint64_t offset, uint64_t size,
1608 uint32_t value);
1609 void radv_cmd_buffer_trace_emit(struct radv_cmd_buffer *cmd_buffer);
1610 bool radv_get_memory_fd(struct radv_device *device, struct radv_device_memory *memory, int *pFD);
1611 void radv_free_memory(struct radv_device *device, const VkAllocationCallbacks *pAllocator,
1612 struct radv_device_memory *mem);
1613
1614 static inline void
radv_emit_shader_pointer_head(struct radeon_cmdbuf * cs,unsigned sh_offset,unsigned pointer_count,bool use_32bit_pointers)1615 radv_emit_shader_pointer_head(struct radeon_cmdbuf *cs, unsigned sh_offset, unsigned pointer_count,
1616 bool use_32bit_pointers)
1617 {
1618 radeon_emit(cs, PKT3(PKT3_SET_SH_REG, pointer_count * (use_32bit_pointers ? 1 : 2), 0));
1619 radeon_emit(cs, (sh_offset - SI_SH_REG_OFFSET) >> 2);
1620 }
1621
1622 static inline void
radv_emit_shader_pointer_body(struct radv_device * device,struct radeon_cmdbuf * cs,uint64_t va,bool use_32bit_pointers)1623 radv_emit_shader_pointer_body(struct radv_device *device, struct radeon_cmdbuf *cs, uint64_t va,
1624 bool use_32bit_pointers)
1625 {
1626 radeon_emit(cs, va);
1627
1628 if (use_32bit_pointers) {
1629 assert(va == 0 || (va >> 32) == device->physical_device->rad_info.address32_hi);
1630 } else {
1631 radeon_emit(cs, va >> 32);
1632 }
1633 }
1634
1635 static inline void
radv_emit_shader_pointer(struct radv_device * device,struct radeon_cmdbuf * cs,uint32_t sh_offset,uint64_t va,bool global)1636 radv_emit_shader_pointer(struct radv_device *device, struct radeon_cmdbuf *cs, uint32_t sh_offset,
1637 uint64_t va, bool global)
1638 {
1639 bool use_32bit_pointers = !global;
1640
1641 radv_emit_shader_pointer_head(cs, sh_offset, 1, use_32bit_pointers);
1642 radv_emit_shader_pointer_body(device, cs, va, use_32bit_pointers);
1643 }
1644
1645 static inline struct radv_descriptor_state *
radv_get_descriptors_state(struct radv_cmd_buffer * cmd_buffer,VkPipelineBindPoint bind_point)1646 radv_get_descriptors_state(struct radv_cmd_buffer *cmd_buffer, VkPipelineBindPoint bind_point)
1647 {
1648 switch (bind_point) {
1649 case VK_PIPELINE_BIND_POINT_GRAPHICS:
1650 case VK_PIPELINE_BIND_POINT_COMPUTE:
1651 return &cmd_buffer->descriptors[bind_point];
1652 case VK_PIPELINE_BIND_POINT_RAY_TRACING_KHR:
1653 return &cmd_buffer->descriptors[2];
1654 default:
1655 unreachable("Unhandled bind point");
1656 }
1657 }
1658
1659 void
1660 radv_get_viewport_xform(const VkViewport *viewport, float scale[3], float translate[3]);
1661
1662 /*
1663 * Takes x,y,z as exact numbers of invocations, instead of blocks.
1664 *
1665 * Limitations: Can't call normal dispatch functions without binding or rebinding
1666 * the compute pipeline.
1667 */
1668 void radv_unaligned_dispatch(struct radv_cmd_buffer *cmd_buffer, uint32_t x, uint32_t y,
1669 uint32_t z);
1670
1671 void radv_indirect_dispatch(struct radv_cmd_buffer *cmd_buffer, struct radeon_winsys_bo *bo,
1672 uint64_t va);
1673
1674 struct radv_event {
1675 struct vk_object_base base;
1676 struct radeon_winsys_bo *bo;
1677 uint64_t *map;
1678 };
1679
1680 #define RADV_HASH_SHADER_CS_WAVE32 (1 << 1)
1681 #define RADV_HASH_SHADER_PS_WAVE32 (1 << 2)
1682 #define RADV_HASH_SHADER_GE_WAVE32 (1 << 3)
1683 #define RADV_HASH_SHADER_LLVM (1 << 4)
1684 #define RADV_HASH_SHADER_KEEP_STATISTICS (1 << 8)
1685 #define RADV_HASH_SHADER_USE_NGG_CULLING (1 << 13)
1686 #define RADV_HASH_SHADER_ROBUST_BUFFER_ACCESS (1 << 14)
1687 #define RADV_HASH_SHADER_ROBUST_BUFFER_ACCESS2 (1 << 15)
1688 #define RADV_HASH_SHADER_FORCE_EMULATE_RT (1 << 16)
1689
1690 struct radv_pipeline_key;
1691
1692 void radv_hash_shaders(unsigned char *hash, const VkPipelineShaderStageCreateInfo **stages,
1693 const struct radv_pipeline_layout *layout,
1694 const struct radv_pipeline_key *key, uint32_t flags);
1695
1696 void radv_hash_rt_shaders(unsigned char *hash, const VkRayTracingPipelineCreateInfoKHR *pCreateInfo,
1697 uint32_t flags);
1698
1699 uint32_t radv_get_hash_flags(const struct radv_device *device, bool stats);
1700
1701 bool radv_rt_pipeline_has_dynamic_stack_size(const VkRayTracingPipelineCreateInfoKHR *pCreateInfo);
1702
1703 #define RADV_STAGE_MASK ((1 << MESA_SHADER_STAGES) - 1)
1704
1705 #define radv_foreach_stage(stage, stage_bits) \
1706 for (gl_shader_stage stage, __tmp = (gl_shader_stage)((stage_bits)&RADV_STAGE_MASK); \
1707 stage = ffs(__tmp) - 1, __tmp; __tmp &= ~(1 << (stage)))
1708
1709 extern const VkFormat radv_fs_key_format_exemplars[NUM_META_FS_KEYS];
1710 unsigned radv_format_meta_fs_key(struct radv_device *device, VkFormat format);
1711
1712 struct radv_multisample_state {
1713 uint32_t db_eqaa;
1714 uint32_t pa_sc_mode_cntl_0;
1715 uint32_t pa_sc_mode_cntl_1;
1716 uint32_t pa_sc_aa_config;
1717 uint32_t pa_sc_aa_mask[2];
1718 unsigned num_samples;
1719 };
1720
1721 struct radv_vrs_state {
1722 uint32_t pa_cl_vrs_cntl;
1723 };
1724
1725 struct radv_prim_vertex_count {
1726 uint8_t min;
1727 uint8_t incr;
1728 };
1729
1730 struct radv_ia_multi_vgt_param_helpers {
1731 uint32_t base;
1732 bool partial_es_wave;
1733 uint8_t primgroup_size;
1734 bool ia_switch_on_eoi;
1735 bool partial_vs_wave;
1736 };
1737
1738 struct radv_binning_state {
1739 uint32_t pa_sc_binner_cntl_0;
1740 };
1741
1742 #define SI_GS_PER_ES 128
1743
1744 enum radv_pipeline_type {
1745 RADV_PIPELINE_GRAPHICS,
1746 /* Compute pipeline (incl raytracing pipeline) */
1747 RADV_PIPELINE_COMPUTE,
1748 /* Pipeline library. This can't actually run and merely is a partial pipeline. */
1749 RADV_PIPELINE_LIBRARY
1750 };
1751
1752 struct radv_pipeline_group_handle {
1753 uint32_t handles[2];
1754 };
1755
1756 struct radv_pipeline_shader_stack_size {
1757 uint32_t recursive_size;
1758 /* anyhit + intersection */
1759 uint32_t non_recursive_size;
1760 };
1761
1762 struct radv_pipeline {
1763 struct vk_object_base base;
1764 enum radv_pipeline_type type;
1765
1766 struct radv_device *device;
1767 struct radv_dynamic_state dynamic_state;
1768
1769 bool need_indirect_descriptor_sets;
1770 struct radv_shader_variant *shaders[MESA_SHADER_STAGES];
1771 struct radv_shader_variant *gs_copy_shader;
1772 VkShaderStageFlags active_stages;
1773
1774 struct radeon_cmdbuf cs;
1775 uint32_t ctx_cs_hash;
1776 struct radeon_cmdbuf ctx_cs;
1777
1778 uint32_t binding_stride[MAX_VBS];
1779
1780 uint8_t attrib_bindings[MAX_VERTEX_ATTRIBS];
1781 uint32_t attrib_ends[MAX_VERTEX_ATTRIBS];
1782 uint32_t attrib_index_offset[MAX_VERTEX_ATTRIBS];
1783
1784 bool use_per_attribute_vb_descs;
1785 bool can_use_simple_input;
1786 uint8_t last_vertex_attrib_bit;
1787 uint8_t next_vertex_stage : 8;
1788 uint32_t vb_desc_usage_mask;
1789 uint32_t vb_desc_alloc_size;
1790
1791 uint32_t user_data_0[MESA_SHADER_STAGES];
1792 union {
1793 struct {
1794 struct radv_multisample_state ms;
1795 struct radv_binning_state binning;
1796 struct radv_vrs_state vrs;
1797 uint32_t spi_baryc_cntl;
1798 unsigned esgs_ring_size;
1799 unsigned gsvs_ring_size;
1800 uint32_t vtx_base_sgpr;
1801 struct radv_ia_multi_vgt_param_helpers ia_multi_vgt_param;
1802 uint8_t vtx_emit_num;
1803 bool uses_drawid;
1804 bool uses_baseinstance;
1805 bool can_use_guardband;
1806 uint64_t needed_dynamic_state;
1807 bool disable_out_of_order_rast_for_occlusion;
1808 unsigned tess_patch_control_points;
1809 unsigned pa_su_sc_mode_cntl;
1810 unsigned db_depth_control;
1811 unsigned pa_cl_clip_cntl;
1812 unsigned cb_color_control;
1813 bool uses_dynamic_stride;
1814 bool uses_conservative_overestimate;
1815
1816 /* Used for rbplus */
1817 uint32_t col_format;
1818 uint32_t cb_target_mask;
1819
1820 /* Whether the pipeline uses NGG (GFX10+). */
1821 bool is_ngg;
1822 bool has_ngg_culling;
1823
1824 /* Last pre-PS API stage */
1825 gl_shader_stage last_vgt_api_stage;
1826 } graphics;
1827 struct {
1828 struct radv_pipeline_group_handle *rt_group_handles;
1829 struct radv_pipeline_shader_stack_size *rt_stack_sizes;
1830 bool dynamic_stack_size;
1831 uint32_t group_count;
1832 } compute;
1833 struct {
1834 unsigned stage_count;
1835 VkPipelineShaderStageCreateInfo *stages;
1836 unsigned group_count;
1837 VkRayTracingShaderGroupCreateInfoKHR *groups;
1838 } library;
1839 };
1840
1841 unsigned max_waves;
1842 unsigned scratch_bytes_per_wave;
1843
1844 /* Not NULL if graphics pipeline uses streamout. */
1845 struct radv_shader_variant *streamout_shader;
1846
1847 /* Unique pipeline hash identifier. */
1848 uint64_t pipeline_hash;
1849
1850 /* Pipeline layout info. */
1851 uint32_t push_constant_size;
1852 uint32_t dynamic_offset_count;
1853 };
1854
1855 static inline bool
radv_pipeline_has_gs(const struct radv_pipeline * pipeline)1856 radv_pipeline_has_gs(const struct radv_pipeline *pipeline)
1857 {
1858 return pipeline->shaders[MESA_SHADER_GEOMETRY] ? true : false;
1859 }
1860
1861 static inline bool
radv_pipeline_has_tess(const struct radv_pipeline * pipeline)1862 radv_pipeline_has_tess(const struct radv_pipeline *pipeline)
1863 {
1864 return pipeline->shaders[MESA_SHADER_TESS_CTRL] ? true : false;
1865 }
1866
1867 bool radv_pipeline_has_ngg_passthrough(const struct radv_pipeline *pipeline);
1868
1869 bool radv_pipeline_has_gs_copy_shader(const struct radv_pipeline *pipeline);
1870
1871 struct radv_userdata_info *radv_lookup_user_sgpr(struct radv_pipeline *pipeline,
1872 gl_shader_stage stage, int idx);
1873
1874 struct radv_shader_variant *radv_get_shader(const struct radv_pipeline *pipeline,
1875 gl_shader_stage stage);
1876
1877 struct radv_graphics_pipeline_create_info {
1878 bool use_rectlist;
1879 bool db_depth_clear;
1880 bool db_stencil_clear;
1881 bool depth_compress_disable;
1882 bool stencil_compress_disable;
1883 bool resummarize_enable;
1884 uint32_t custom_blend_mode;
1885 };
1886
1887 VkResult radv_graphics_pipeline_create(VkDevice device, VkPipelineCache cache,
1888 const VkGraphicsPipelineCreateInfo *pCreateInfo,
1889 const struct radv_graphics_pipeline_create_info *extra,
1890 const VkAllocationCallbacks *alloc, VkPipeline *pPipeline);
1891
1892 VkResult radv_compute_pipeline_create(VkDevice _device, VkPipelineCache _cache,
1893 const VkComputePipelineCreateInfo *pCreateInfo,
1894 const VkAllocationCallbacks *pAllocator,
1895 const uint8_t *custom_hash,
1896 struct radv_pipeline_shader_stack_size *rt_stack_sizes,
1897 uint32_t rt_group_count, VkPipeline *pPipeline);
1898
1899 void radv_pipeline_destroy(struct radv_device *device, struct radv_pipeline *pipeline,
1900 const VkAllocationCallbacks *allocator);
1901
1902 struct radv_binning_settings {
1903 unsigned context_states_per_bin; /* allowed range: [1, 6] */
1904 unsigned persistent_states_per_bin; /* allowed range: [1, 32] */
1905 unsigned fpovs_per_batch; /* allowed range: [0, 255], 0 = unlimited */
1906 };
1907
1908 struct radv_binning_settings radv_get_binning_settings(const struct radv_physical_device *pdev);
1909
1910 struct vk_format_description;
1911 uint32_t radv_translate_buffer_dataformat(const struct util_format_description *desc,
1912 int first_non_void);
1913 uint32_t radv_translate_buffer_numformat(const struct util_format_description *desc,
1914 int first_non_void);
1915 bool radv_is_buffer_format_supported(VkFormat format, bool *scaled);
1916 void radv_translate_vertex_format(const struct radv_physical_device *pdevice, VkFormat format,
1917 const struct util_format_description *desc, unsigned *dfmt,
1918 unsigned *nfmt, bool *post_shuffle,
1919 enum radv_vs_input_alpha_adjust *alpha_adjust);
1920 uint32_t radv_translate_colorformat(VkFormat format);
1921 uint32_t radv_translate_color_numformat(VkFormat format, const struct util_format_description *desc,
1922 int first_non_void);
1923 uint32_t radv_colorformat_endian_swap(uint32_t colorformat);
1924 unsigned radv_translate_colorswap(VkFormat format, bool do_endian_swap);
1925 uint32_t radv_translate_dbformat(VkFormat format);
1926 uint32_t radv_translate_tex_dataformat(VkFormat format, const struct util_format_description *desc,
1927 int first_non_void);
1928 uint32_t radv_translate_tex_numformat(VkFormat format, const struct util_format_description *desc,
1929 int first_non_void);
1930 bool radv_format_pack_clear_color(VkFormat format, uint32_t clear_vals[2],
1931 VkClearColorValue *value);
1932 bool radv_is_storage_image_format_supported(struct radv_physical_device *physical_device,
1933 VkFormat format);
1934 bool radv_is_colorbuffer_format_supported(const struct radv_physical_device *pdevice,
1935 VkFormat format, bool *blendable);
1936 bool radv_dcc_formats_compatible(VkFormat format1, VkFormat format2, bool *sign_reinterpret);
1937 bool radv_is_atomic_format_supported(VkFormat format);
1938 bool radv_device_supports_etc(struct radv_physical_device *physical_device);
1939
1940 struct radv_image_plane {
1941 VkFormat format;
1942 struct radeon_surf surface;
1943 };
1944
1945 struct radv_image {
1946 struct vk_object_base base;
1947 VkImageType type;
1948 /* The original VkFormat provided by the client. This may not match any
1949 * of the actual surface formats.
1950 */
1951 VkFormat vk_format;
1952 VkImageUsageFlags usage; /**< Superset of VkImageCreateInfo::usage. */
1953 struct ac_surf_info info;
1954 VkImageTiling tiling; /** VkImageCreateInfo::tiling */
1955 VkImageCreateFlags flags; /** VkImageCreateInfo::flags */
1956
1957 VkDeviceSize size;
1958 uint32_t alignment;
1959
1960 unsigned queue_family_mask;
1961 bool exclusive;
1962 bool shareable;
1963 bool l2_coherent;
1964 bool dcc_sign_reinterpret;
1965 bool support_comp_to_single;
1966
1967 /* Set when bound */
1968 struct radeon_winsys_bo *bo;
1969 VkDeviceSize offset;
1970 bool tc_compatible_cmask;
1971
1972 uint64_t clear_value_offset;
1973 uint64_t fce_pred_offset;
1974 uint64_t dcc_pred_offset;
1975
1976 /*
1977 * Metadata for the TC-compat zrange workaround. If the 32-bit value
1978 * stored at this offset is UINT_MAX, the driver will emit
1979 * DB_Z_INFO.ZRANGE_PRECISION=0, otherwise it will skip the
1980 * SET_CONTEXT_REG packet.
1981 */
1982 uint64_t tc_compat_zrange_offset;
1983
1984 /* For VK_ANDROID_native_buffer, the WSI image owns the memory, */
1985 VkDeviceMemory owned_memory;
1986
1987 unsigned plane_count;
1988 struct radv_image_plane planes[0];
1989 };
1990
1991 /* Whether the image has a htile that is known consistent with the contents of
1992 * the image and is allowed to be in compressed form.
1993 *
1994 * If this is false reads that don't use the htile should be able to return
1995 * correct results.
1996 */
1997 bool radv_layout_is_htile_compressed(const struct radv_device *device,
1998 const struct radv_image *image, VkImageLayout layout,
1999 bool in_render_loop, unsigned queue_mask);
2000
2001 bool radv_layout_can_fast_clear(const struct radv_device *device, const struct radv_image *image,
2002 unsigned level, VkImageLayout layout, bool in_render_loop,
2003 unsigned queue_mask);
2004
2005 bool radv_layout_dcc_compressed(const struct radv_device *device, const struct radv_image *image,
2006 unsigned level, VkImageLayout layout, bool in_render_loop,
2007 unsigned queue_mask);
2008
2009 bool radv_layout_fmask_compressed(const struct radv_device *device, const struct radv_image *image,
2010 VkImageLayout layout, unsigned queue_mask);
2011
2012 /**
2013 * Return whether the image has CMASK metadata for color surfaces.
2014 */
2015 static inline bool
radv_image_has_cmask(const struct radv_image * image)2016 radv_image_has_cmask(const struct radv_image *image)
2017 {
2018 return image->planes[0].surface.cmask_offset;
2019 }
2020
2021 /**
2022 * Return whether the image has FMASK metadata for color surfaces.
2023 */
2024 static inline bool
radv_image_has_fmask(const struct radv_image * image)2025 radv_image_has_fmask(const struct radv_image *image)
2026 {
2027 return image->planes[0].surface.fmask_offset;
2028 }
2029
2030 /**
2031 * Return whether the image has DCC metadata for color surfaces.
2032 */
2033 static inline bool
radv_image_has_dcc(const struct radv_image * image)2034 radv_image_has_dcc(const struct radv_image *image)
2035 {
2036 return !(image->planes[0].surface.flags & RADEON_SURF_Z_OR_SBUFFER) &&
2037 image->planes[0].surface.meta_offset;
2038 }
2039
2040 /**
2041 * Return whether the image is TC-compatible CMASK.
2042 */
2043 static inline bool
radv_image_is_tc_compat_cmask(const struct radv_image * image)2044 radv_image_is_tc_compat_cmask(const struct radv_image *image)
2045 {
2046 return radv_image_has_fmask(image) && image->tc_compatible_cmask;
2047 }
2048
2049 /**
2050 * Return whether DCC metadata is enabled for a level.
2051 */
2052 static inline bool
radv_dcc_enabled(const struct radv_image * image,unsigned level)2053 radv_dcc_enabled(const struct radv_image *image, unsigned level)
2054 {
2055 return radv_image_has_dcc(image) && level < image->planes[0].surface.num_meta_levels;
2056 }
2057
2058 /**
2059 * Return whether the image has CB metadata.
2060 */
2061 static inline bool
radv_image_has_CB_metadata(const struct radv_image * image)2062 radv_image_has_CB_metadata(const struct radv_image *image)
2063 {
2064 return radv_image_has_cmask(image) || radv_image_has_fmask(image) || radv_image_has_dcc(image);
2065 }
2066
2067 /**
2068 * Return whether the image has HTILE metadata for depth surfaces.
2069 */
2070 static inline bool
radv_image_has_htile(const struct radv_image * image)2071 radv_image_has_htile(const struct radv_image *image)
2072 {
2073 return image->planes[0].surface.flags & RADEON_SURF_Z_OR_SBUFFER &&
2074 image->planes[0].surface.meta_size;
2075 }
2076
2077 /**
2078 * Return whether the image has VRS HTILE metadata for depth surfaces
2079 */
2080 static inline bool
radv_image_has_vrs_htile(const struct radv_device * device,const struct radv_image * image)2081 radv_image_has_vrs_htile(const struct radv_device *device, const struct radv_image *image)
2082 {
2083 /* Any depth buffer can potentially use VRS. */
2084 return device->attachment_vrs_enabled && radv_image_has_htile(image) &&
2085 (image->usage & VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT);
2086 }
2087
2088 /**
2089 * Return whether HTILE metadata is enabled for a level.
2090 */
2091 static inline bool
radv_htile_enabled(const struct radv_image * image,unsigned level)2092 radv_htile_enabled(const struct radv_image *image, unsigned level)
2093 {
2094 return radv_image_has_htile(image) && level < image->planes[0].surface.num_meta_levels;
2095 }
2096
2097 /**
2098 * Return whether the image is TC-compatible HTILE.
2099 */
2100 static inline bool
radv_image_is_tc_compat_htile(const struct radv_image * image)2101 radv_image_is_tc_compat_htile(const struct radv_image *image)
2102 {
2103 return radv_image_has_htile(image) &&
2104 (image->planes[0].surface.flags & RADEON_SURF_TC_COMPATIBLE_HTILE);
2105 }
2106
2107 /**
2108 * Return whether the entire HTILE buffer can be used for depth in order to
2109 * improve HiZ Z-Range precision.
2110 */
2111 static inline bool
radv_image_tile_stencil_disabled(const struct radv_device * device,const struct radv_image * image)2112 radv_image_tile_stencil_disabled(const struct radv_device *device, const struct radv_image *image)
2113 {
2114 if (device->physical_device->rad_info.chip_class >= GFX9) {
2115 return !vk_format_has_stencil(image->vk_format) && !radv_image_has_vrs_htile(device, image);
2116 } else {
2117 /* Due to a hw bug, TILE_STENCIL_DISABLE must be set to 0 for
2118 * the TC-compat ZRANGE issue even if no stencil is used.
2119 */
2120 return !vk_format_has_stencil(image->vk_format) && !radv_image_is_tc_compat_htile(image);
2121 }
2122 }
2123
2124 static inline bool
radv_image_has_clear_value(const struct radv_image * image)2125 radv_image_has_clear_value(const struct radv_image *image)
2126 {
2127 return image->clear_value_offset != 0;
2128 }
2129
2130 static inline uint64_t
radv_image_get_fast_clear_va(const struct radv_image * image,uint32_t base_level)2131 radv_image_get_fast_clear_va(const struct radv_image *image, uint32_t base_level)
2132 {
2133 assert(radv_image_has_clear_value(image));
2134
2135 uint64_t va = radv_buffer_get_va(image->bo);
2136 va += image->offset + image->clear_value_offset + base_level * 8;
2137 return va;
2138 }
2139
2140 static inline uint64_t
radv_image_get_fce_pred_va(const struct radv_image * image,uint32_t base_level)2141 radv_image_get_fce_pred_va(const struct radv_image *image, uint32_t base_level)
2142 {
2143 assert(image->fce_pred_offset != 0);
2144
2145 uint64_t va = radv_buffer_get_va(image->bo);
2146 va += image->offset + image->fce_pred_offset + base_level * 8;
2147 return va;
2148 }
2149
2150 static inline uint64_t
radv_image_get_dcc_pred_va(const struct radv_image * image,uint32_t base_level)2151 radv_image_get_dcc_pred_va(const struct radv_image *image, uint32_t base_level)
2152 {
2153 assert(image->dcc_pred_offset != 0);
2154
2155 uint64_t va = radv_buffer_get_va(image->bo);
2156 va += image->offset + image->dcc_pred_offset + base_level * 8;
2157 return va;
2158 }
2159
2160 static inline uint64_t
radv_get_tc_compat_zrange_va(const struct radv_image * image,uint32_t base_level)2161 radv_get_tc_compat_zrange_va(const struct radv_image *image, uint32_t base_level)
2162 {
2163 assert(image->tc_compat_zrange_offset != 0);
2164
2165 uint64_t va = radv_buffer_get_va(image->bo);
2166 va += image->offset + image->tc_compat_zrange_offset + base_level * 4;
2167 return va;
2168 }
2169
2170 static inline uint64_t
radv_get_ds_clear_value_va(const struct radv_image * image,uint32_t base_level)2171 radv_get_ds_clear_value_va(const struct radv_image *image, uint32_t base_level)
2172 {
2173 assert(radv_image_has_clear_value(image));
2174
2175 uint64_t va = radv_buffer_get_va(image->bo);
2176 va += image->offset + image->clear_value_offset + base_level * 8;
2177 return va;
2178 }
2179
2180 static inline uint32_t
radv_get_htile_initial_value(const struct radv_device * device,const struct radv_image * image)2181 radv_get_htile_initial_value(const struct radv_device *device, const struct radv_image *image)
2182 {
2183 uint32_t initial_value;
2184
2185 if (radv_image_tile_stencil_disabled(device, image)) {
2186 /* Z only (no stencil):
2187 *
2188 * |31 18|17 4|3 0|
2189 * +---------+---------+-------+
2190 * | Max Z | Min Z | ZMask |
2191 */
2192 initial_value = 0xfffc000f;
2193 } else {
2194 /* Z and stencil:
2195 *
2196 * |31 12|11 10|9 8|7 6|5 4|3 0|
2197 * +-----------+-----+------+-----+-----+-------+
2198 * | Z Range | | SMem | SR1 | SR0 | ZMask |
2199 *
2200 * SR0/SR1 contains the stencil test results. Initializing
2201 * SR0/SR1 to 0x3 means the stencil test result is unknown.
2202 *
2203 * Z, stencil and 4 bit VRS encoding:
2204 * |31 12|11 10|9 8|7 6|5 4|3 0|
2205 * +-----------+------------+------+------------+-----+-------+
2206 * | Z Range | VRS y-rate | SMem | VRS x-rate | SR0 | ZMask |
2207 */
2208 if (radv_image_has_vrs_htile(device, image)) {
2209 /* Initialize the VRS x-rate value at 0, so the hw interprets it as 1 sample. */
2210 initial_value = 0xfffff33f;
2211 } else {
2212 initial_value = 0xfffff3ff;
2213 }
2214 }
2215
2216 return initial_value;
2217 }
2218
2219 static inline bool
radv_image_get_iterate256(struct radv_device * device,struct radv_image * image)2220 radv_image_get_iterate256(struct radv_device *device, struct radv_image *image)
2221 {
2222 /* ITERATE_256 is required for depth or stencil MSAA images that are TC-compatible HTILE. */
2223 return device->physical_device->rad_info.chip_class >= GFX10 &&
2224 (image->usage & (VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT |
2225 VK_IMAGE_USAGE_TRANSFER_DST_BIT)) &&
2226 radv_image_is_tc_compat_htile(image) &&
2227 image->info.samples > 1;
2228 }
2229
2230 unsigned radv_image_queue_family_mask(const struct radv_image *image, uint32_t family,
2231 uint32_t queue_family);
2232
2233 static inline uint32_t
radv_get_layerCount(const struct radv_image * image,const VkImageSubresourceRange * range)2234 radv_get_layerCount(const struct radv_image *image, const VkImageSubresourceRange *range)
2235 {
2236 return range->layerCount == VK_REMAINING_ARRAY_LAYERS
2237 ? image->info.array_size - range->baseArrayLayer
2238 : range->layerCount;
2239 }
2240
2241 static inline uint32_t
radv_get_levelCount(const struct radv_image * image,const VkImageSubresourceRange * range)2242 radv_get_levelCount(const struct radv_image *image, const VkImageSubresourceRange *range)
2243 {
2244 return range->levelCount == VK_REMAINING_MIP_LEVELS ? image->info.levels - range->baseMipLevel
2245 : range->levelCount;
2246 }
2247
2248 bool radv_image_is_renderable(struct radv_device *device, struct radv_image *image);
2249
2250 struct radeon_bo_metadata;
2251 void radv_init_metadata(struct radv_device *device, struct radv_image *image,
2252 struct radeon_bo_metadata *metadata);
2253
2254 void radv_image_override_offset_stride(struct radv_device *device, struct radv_image *image,
2255 uint64_t offset, uint32_t stride);
2256
2257 union radv_descriptor {
2258 struct {
2259 uint32_t plane0_descriptor[8];
2260 uint32_t fmask_descriptor[8];
2261 };
2262 struct {
2263 uint32_t plane_descriptors[3][8];
2264 };
2265 };
2266
2267 struct radv_image_view {
2268 struct vk_object_base base;
2269 struct radv_image *image; /**< VkImageViewCreateInfo::image */
2270
2271 VkImageViewType type;
2272 VkImageAspectFlags aspect_mask;
2273 VkFormat vk_format;
2274 unsigned plane_id;
2275 uint32_t base_layer;
2276 uint32_t layer_count;
2277 uint32_t base_mip;
2278 uint32_t level_count;
2279 VkExtent3D extent; /**< Extent of VkImageViewCreateInfo::baseMipLevel. */
2280
2281 /* Whether the image iview supports fast clear. */
2282 bool support_fast_clear;
2283
2284 union radv_descriptor descriptor;
2285
2286 /* Descriptor for use as a storage image as opposed to a sampled image.
2287 * This has a few differences for cube maps (e.g. type).
2288 */
2289 union radv_descriptor storage_descriptor;
2290 };
2291
2292 struct radv_image_create_info {
2293 const VkImageCreateInfo *vk_info;
2294 bool scanout;
2295 bool no_metadata_planes;
2296 const struct radeon_bo_metadata *bo_metadata;
2297 };
2298
2299 VkResult
2300 radv_image_create_layout(struct radv_device *device, struct radv_image_create_info create_info,
2301 const struct VkImageDrmFormatModifierExplicitCreateInfoEXT *mod_info,
2302 struct radv_image *image);
2303
2304 VkResult radv_image_create(VkDevice _device, const struct radv_image_create_info *info,
2305 const VkAllocationCallbacks *alloc, VkImage *pImage);
2306
2307 bool radv_are_formats_dcc_compatible(const struct radv_physical_device *pdev, const void *pNext,
2308 VkFormat format, VkImageCreateFlags flags,
2309 bool *sign_reinterpret);
2310
2311 bool vi_alpha_is_on_msb(struct radv_device *device, VkFormat format);
2312
2313 VkResult radv_image_from_gralloc(VkDevice device_h, const VkImageCreateInfo *base_info,
2314 const VkNativeBufferANDROID *gralloc_info,
2315 const VkAllocationCallbacks *alloc, VkImage *out_image_h);
2316 uint64_t radv_ahb_usage_from_vk_usage(const VkImageCreateFlags vk_create,
2317 const VkImageUsageFlags vk_usage);
2318 VkResult radv_import_ahb_memory(struct radv_device *device, struct radv_device_memory *mem,
2319 unsigned priority,
2320 const VkImportAndroidHardwareBufferInfoANDROID *info);
2321 VkResult radv_create_ahb_memory(struct radv_device *device, struct radv_device_memory *mem,
2322 unsigned priority, const VkMemoryAllocateInfo *pAllocateInfo);
2323
2324 VkFormat radv_select_android_external_format(const void *next, VkFormat default_format);
2325
2326 bool radv_android_gralloc_supports_format(VkFormat format, VkImageUsageFlagBits usage);
2327
2328 struct radv_image_view_extra_create_info {
2329 bool disable_compression;
2330 bool enable_compression;
2331 };
2332
2333 void radv_image_view_init(struct radv_image_view *view, struct radv_device *device,
2334 const VkImageViewCreateInfo *pCreateInfo,
2335 const struct radv_image_view_extra_create_info *extra_create_info);
2336 void radv_image_view_finish(struct radv_image_view *iview);
2337
2338 VkFormat radv_get_aspect_format(struct radv_image *image, VkImageAspectFlags mask);
2339
2340 struct radv_sampler_ycbcr_conversion {
2341 struct vk_object_base base;
2342 VkFormat format;
2343 VkSamplerYcbcrModelConversion ycbcr_model;
2344 VkSamplerYcbcrRange ycbcr_range;
2345 VkComponentMapping components;
2346 VkChromaLocation chroma_offsets[2];
2347 VkFilter chroma_filter;
2348 };
2349
2350 struct radv_buffer_view {
2351 struct vk_object_base base;
2352 struct radeon_winsys_bo *bo;
2353 VkFormat vk_format;
2354 uint64_t range; /**< VkBufferViewCreateInfo::range */
2355 uint32_t state[4];
2356 };
2357 void radv_buffer_view_init(struct radv_buffer_view *view, struct radv_device *device,
2358 const VkBufferViewCreateInfo *pCreateInfo);
2359 void radv_buffer_view_finish(struct radv_buffer_view *view);
2360
2361 static inline struct VkExtent3D
radv_sanitize_image_extent(const VkImageType imageType,const struct VkExtent3D imageExtent)2362 radv_sanitize_image_extent(const VkImageType imageType, const struct VkExtent3D imageExtent)
2363 {
2364 switch (imageType) {
2365 case VK_IMAGE_TYPE_1D:
2366 return (VkExtent3D){imageExtent.width, 1, 1};
2367 case VK_IMAGE_TYPE_2D:
2368 return (VkExtent3D){imageExtent.width, imageExtent.height, 1};
2369 case VK_IMAGE_TYPE_3D:
2370 return imageExtent;
2371 default:
2372 unreachable("invalid image type");
2373 }
2374 }
2375
2376 static inline struct VkOffset3D
radv_sanitize_image_offset(const VkImageType imageType,const struct VkOffset3D imageOffset)2377 radv_sanitize_image_offset(const VkImageType imageType, const struct VkOffset3D imageOffset)
2378 {
2379 switch (imageType) {
2380 case VK_IMAGE_TYPE_1D:
2381 return (VkOffset3D){imageOffset.x, 0, 0};
2382 case VK_IMAGE_TYPE_2D:
2383 return (VkOffset3D){imageOffset.x, imageOffset.y, 0};
2384 case VK_IMAGE_TYPE_3D:
2385 return imageOffset;
2386 default:
2387 unreachable("invalid image type");
2388 }
2389 }
2390
2391 static inline bool
radv_image_extent_compare(const struct radv_image * image,const VkExtent3D * extent)2392 radv_image_extent_compare(const struct radv_image *image, const VkExtent3D *extent)
2393 {
2394 if (extent->width != image->info.width || extent->height != image->info.height ||
2395 extent->depth != image->info.depth)
2396 return false;
2397 return true;
2398 }
2399
2400 struct radv_sampler {
2401 struct vk_object_base base;
2402 uint32_t state[4];
2403 struct radv_sampler_ycbcr_conversion *ycbcr_sampler;
2404 uint32_t border_color_slot;
2405 };
2406
2407 struct radv_framebuffer {
2408 struct vk_object_base base;
2409 uint32_t width;
2410 uint32_t height;
2411 uint32_t layers;
2412
2413 bool imageless;
2414
2415 uint32_t attachment_count;
2416 struct radv_image_view *attachments[0];
2417 };
2418
2419 struct radv_subpass_barrier {
2420 VkPipelineStageFlags src_stage_mask;
2421 VkAccessFlags src_access_mask;
2422 VkAccessFlags dst_access_mask;
2423 };
2424
2425 void radv_emit_subpass_barrier(struct radv_cmd_buffer *cmd_buffer,
2426 const struct radv_subpass_barrier *barrier);
2427
2428 struct radv_subpass_attachment {
2429 uint32_t attachment;
2430 VkImageLayout layout;
2431 VkImageLayout stencil_layout;
2432 bool in_render_loop;
2433 };
2434
2435 struct radv_subpass {
2436 uint32_t attachment_count;
2437 struct radv_subpass_attachment *attachments;
2438
2439 uint32_t input_count;
2440 uint32_t color_count;
2441 struct radv_subpass_attachment *input_attachments;
2442 struct radv_subpass_attachment *color_attachments;
2443 struct radv_subpass_attachment *resolve_attachments;
2444 struct radv_subpass_attachment *depth_stencil_attachment;
2445 struct radv_subpass_attachment *ds_resolve_attachment;
2446 struct radv_subpass_attachment *vrs_attachment;
2447 VkResolveModeFlagBits depth_resolve_mode;
2448 VkResolveModeFlagBits stencil_resolve_mode;
2449
2450 /** Subpass has at least one color resolve attachment */
2451 bool has_color_resolve;
2452
2453 /** Subpass has at least one color attachment */
2454 bool has_color_att;
2455
2456 struct radv_subpass_barrier start_barrier;
2457
2458 uint32_t view_mask;
2459
2460 VkSampleCountFlagBits color_sample_count;
2461 VkSampleCountFlagBits depth_sample_count;
2462 VkSampleCountFlagBits max_sample_count;
2463
2464 /* Whether the subpass has ingoing/outgoing external dependencies. */
2465 bool has_ingoing_dep;
2466 bool has_outgoing_dep;
2467 };
2468
2469 uint32_t radv_get_subpass_id(struct radv_cmd_buffer *cmd_buffer);
2470
2471 struct radv_render_pass_attachment {
2472 VkFormat format;
2473 uint32_t samples;
2474 VkAttachmentLoadOp load_op;
2475 VkAttachmentLoadOp stencil_load_op;
2476 VkImageLayout initial_layout;
2477 VkImageLayout final_layout;
2478 VkImageLayout stencil_initial_layout;
2479 VkImageLayout stencil_final_layout;
2480
2481 /* The subpass id in which the attachment will be used first/last. */
2482 uint32_t first_subpass_idx;
2483 uint32_t last_subpass_idx;
2484 };
2485
2486 struct radv_render_pass {
2487 struct vk_object_base base;
2488 uint32_t attachment_count;
2489 uint32_t subpass_count;
2490 struct radv_subpass_attachment *subpass_attachments;
2491 struct radv_render_pass_attachment *attachments;
2492 struct radv_subpass_barrier end_barrier;
2493 struct radv_subpass subpasses[0];
2494 };
2495
2496 VkResult radv_device_init_meta(struct radv_device *device);
2497 void radv_device_finish_meta(struct radv_device *device);
2498
2499 struct radv_query_pool {
2500 struct vk_object_base base;
2501 struct radeon_winsys_bo *bo;
2502 uint32_t stride;
2503 uint32_t availability_offset;
2504 uint64_t size;
2505 char *ptr;
2506 VkQueryType type;
2507 uint32_t pipeline_stats_mask;
2508 };
2509
2510 typedef enum {
2511 RADV_SEMAPHORE_NONE,
2512 RADV_SEMAPHORE_SYNCOBJ,
2513 RADV_SEMAPHORE_TIMELINE_SYNCOBJ,
2514 RADV_SEMAPHORE_TIMELINE,
2515 } radv_semaphore_kind;
2516
2517 struct radv_deferred_queue_submission;
2518
2519 struct radv_timeline_waiter {
2520 struct list_head list;
2521 struct radv_deferred_queue_submission *submission;
2522 uint64_t value;
2523 };
2524
2525 struct radv_timeline_point {
2526 struct list_head list;
2527
2528 uint64_t value;
2529 uint32_t syncobj;
2530
2531 /* Separate from the list to accomodate CPU wait being async, as well
2532 * as prevent point deletion during submission. */
2533 unsigned wait_count;
2534 };
2535
2536 struct radv_timeline {
2537 mtx_t mutex;
2538
2539 uint64_t highest_signaled;
2540 uint64_t highest_submitted;
2541
2542 struct list_head points;
2543
2544 /* Keep free points on hand so we do not have to recreate syncobjs all
2545 * the time. */
2546 struct list_head free_points;
2547
2548 /* Submissions that are deferred waiting for a specific value to be
2549 * submitted. */
2550 struct list_head waiters;
2551 };
2552
2553 struct radv_timeline_syncobj {
2554 /* Keep syncobj first, so common-code can just handle this as
2555 * non-timeline syncobj. */
2556 uint32_t syncobj;
2557 uint64_t max_point; /* max submitted point. */
2558 };
2559
2560 struct radv_semaphore_part {
2561 radv_semaphore_kind kind;
2562 union {
2563 uint32_t syncobj;
2564 struct radv_timeline timeline;
2565 struct radv_timeline_syncobj timeline_syncobj;
2566 };
2567 };
2568
2569 struct radv_semaphore {
2570 struct vk_object_base base;
2571 struct radv_semaphore_part permanent;
2572 struct radv_semaphore_part temporary;
2573 };
2574
2575 bool radv_queue_internal_submit(struct radv_queue *queue, struct radeon_cmdbuf *cs);
2576
2577 void radv_set_descriptor_set(struct radv_cmd_buffer *cmd_buffer, VkPipelineBindPoint bind_point,
2578 struct radv_descriptor_set *set, unsigned idx);
2579
2580 void radv_update_descriptor_sets(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer,
2581 VkDescriptorSet overrideSet, uint32_t descriptorWriteCount,
2582 const VkWriteDescriptorSet *pDescriptorWrites,
2583 uint32_t descriptorCopyCount,
2584 const VkCopyDescriptorSet *pDescriptorCopies);
2585
2586 void radv_update_descriptor_set_with_template(struct radv_device *device,
2587 struct radv_cmd_buffer *cmd_buffer,
2588 struct radv_descriptor_set *set,
2589 VkDescriptorUpdateTemplate descriptorUpdateTemplate,
2590 const void *pData);
2591
2592 void radv_meta_push_descriptor_set(struct radv_cmd_buffer *cmd_buffer,
2593 VkPipelineBindPoint pipelineBindPoint, VkPipelineLayout _layout,
2594 uint32_t set, uint32_t descriptorWriteCount,
2595 const VkWriteDescriptorSet *pDescriptorWrites);
2596
2597 uint32_t radv_init_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
2598 const VkImageSubresourceRange *range, uint32_t value);
2599
2600 uint32_t radv_init_fmask(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
2601 const VkImageSubresourceRange *range);
2602
2603 typedef enum {
2604 RADV_FENCE_NONE,
2605 RADV_FENCE_SYNCOBJ,
2606 } radv_fence_kind;
2607
2608 struct radv_fence_part {
2609 radv_fence_kind kind;
2610
2611 /* DRM syncobj handle for syncobj-based fences. */
2612 uint32_t syncobj;
2613 };
2614
2615 struct radv_fence {
2616 struct vk_object_base base;
2617 struct radv_fence_part permanent;
2618 struct radv_fence_part temporary;
2619 };
2620
2621 /* radv_nir_to_llvm.c */
2622 struct radv_shader_args;
2623
2624 void llvm_compile_shader(struct radv_device *device, unsigned shader_count,
2625 struct nir_shader *const *shaders, struct radv_shader_binary **binary,
2626 struct radv_shader_args *args);
2627
2628 /* radv_shader_info.h */
2629 struct radv_shader_info;
2630
2631 void radv_nir_shader_info_pass(struct radv_device *device, const struct nir_shader *nir,
2632 const struct radv_pipeline_layout *layout,
2633 const struct radv_pipeline_key *pipeline_key,
2634 struct radv_shader_info *info);
2635
2636 void radv_nir_shader_info_init(struct radv_shader_info *info);
2637
2638 bool radv_thread_trace_init(struct radv_device *device);
2639 void radv_thread_trace_finish(struct radv_device *device);
2640 bool radv_begin_thread_trace(struct radv_queue *queue);
2641 bool radv_end_thread_trace(struct radv_queue *queue);
2642 bool radv_get_thread_trace(struct radv_queue *queue, struct ac_thread_trace *thread_trace);
2643 void radv_emit_thread_trace_userdata(const struct radv_device *device, struct radeon_cmdbuf *cs,
2644 const void *data, uint32_t num_dwords);
2645 bool radv_is_instruction_timing_enabled(void);
2646
2647 /* radv_sqtt_layer_.c */
2648 struct radv_barrier_data {
2649 union {
2650 struct {
2651 uint16_t depth_stencil_expand : 1;
2652 uint16_t htile_hiz_range_expand : 1;
2653 uint16_t depth_stencil_resummarize : 1;
2654 uint16_t dcc_decompress : 1;
2655 uint16_t fmask_decompress : 1;
2656 uint16_t fast_clear_eliminate : 1;
2657 uint16_t fmask_color_expand : 1;
2658 uint16_t init_mask_ram : 1;
2659 uint16_t reserved : 8;
2660 };
2661 uint16_t all;
2662 } layout_transitions;
2663 };
2664
2665 /**
2666 * Value for the reason field of an RGP barrier start marker originating from
2667 * the Vulkan client (does not include PAL-defined values). (Table 15)
2668 */
2669 enum rgp_barrier_reason {
2670 RGP_BARRIER_UNKNOWN_REASON = 0xFFFFFFFF,
2671
2672 /* External app-generated barrier reasons, i.e. API synchronization
2673 * commands Range of valid values: [0x00000001 ... 0x7FFFFFFF].
2674 */
2675 RGP_BARRIER_EXTERNAL_CMD_PIPELINE_BARRIER = 0x00000001,
2676 RGP_BARRIER_EXTERNAL_RENDER_PASS_SYNC = 0x00000002,
2677 RGP_BARRIER_EXTERNAL_CMD_WAIT_EVENTS = 0x00000003,
2678
2679 /* Internal barrier reasons, i.e. implicit synchronization inserted by
2680 * the Vulkan driver Range of valid values: [0xC0000000 ... 0xFFFFFFFE].
2681 */
2682 RGP_BARRIER_INTERNAL_BASE = 0xC0000000,
2683 RGP_BARRIER_INTERNAL_PRE_RESET_QUERY_POOL_SYNC = RGP_BARRIER_INTERNAL_BASE + 0,
2684 RGP_BARRIER_INTERNAL_POST_RESET_QUERY_POOL_SYNC = RGP_BARRIER_INTERNAL_BASE + 1,
2685 RGP_BARRIER_INTERNAL_GPU_EVENT_RECYCLE_STALL = RGP_BARRIER_INTERNAL_BASE + 2,
2686 RGP_BARRIER_INTERNAL_PRE_COPY_QUERY_POOL_RESULTS_SYNC = RGP_BARRIER_INTERNAL_BASE + 3
2687 };
2688
2689 void radv_describe_begin_cmd_buffer(struct radv_cmd_buffer *cmd_buffer);
2690 void radv_describe_end_cmd_buffer(struct radv_cmd_buffer *cmd_buffer);
2691 void radv_describe_draw(struct radv_cmd_buffer *cmd_buffer);
2692 void radv_describe_dispatch(struct radv_cmd_buffer *cmd_buffer, int x, int y, int z);
2693 void radv_describe_begin_render_pass_clear(struct radv_cmd_buffer *cmd_buffer,
2694 VkImageAspectFlagBits aspects);
2695 void radv_describe_end_render_pass_clear(struct radv_cmd_buffer *cmd_buffer);
2696 void radv_describe_begin_render_pass_resolve(struct radv_cmd_buffer *cmd_buffer);
2697 void radv_describe_end_render_pass_resolve(struct radv_cmd_buffer *cmd_buffer);
2698 void radv_describe_barrier_start(struct radv_cmd_buffer *cmd_buffer,
2699 enum rgp_barrier_reason reason);
2700 void radv_describe_barrier_end(struct radv_cmd_buffer *cmd_buffer);
2701 void radv_describe_barrier_end_delayed(struct radv_cmd_buffer *cmd_buffer);
2702 void radv_describe_layout_transition(struct radv_cmd_buffer *cmd_buffer,
2703 const struct radv_barrier_data *barrier);
2704
2705 uint64_t radv_get_current_time(void);
2706
2707 static inline uint32_t
si_conv_gl_prim_to_vertices(unsigned gl_prim)2708 si_conv_gl_prim_to_vertices(unsigned gl_prim)
2709 {
2710 switch (gl_prim) {
2711 case 0: /* GL_POINTS */
2712 return 1;
2713 case 1: /* GL_LINES */
2714 case 3: /* GL_LINE_STRIP */
2715 return 2;
2716 case 4: /* GL_TRIANGLES */
2717 case 5: /* GL_TRIANGLE_STRIP */
2718 return 3;
2719 case 0xA: /* GL_LINE_STRIP_ADJACENCY_ARB */
2720 return 4;
2721 case 0xc: /* GL_TRIANGLES_ADJACENCY_ARB */
2722 return 6;
2723 case 7: /* GL_QUADS */
2724 return V_028A6C_TRISTRIP;
2725 default:
2726 assert(0);
2727 return 0;
2728 }
2729 }
2730
2731 static inline uint32_t
si_conv_prim_to_gs_out(enum VkPrimitiveTopology topology)2732 si_conv_prim_to_gs_out(enum VkPrimitiveTopology topology)
2733 {
2734 switch (topology) {
2735 case VK_PRIMITIVE_TOPOLOGY_POINT_LIST:
2736 case VK_PRIMITIVE_TOPOLOGY_PATCH_LIST:
2737 return V_028A6C_POINTLIST;
2738 case VK_PRIMITIVE_TOPOLOGY_LINE_LIST:
2739 case VK_PRIMITIVE_TOPOLOGY_LINE_STRIP:
2740 case VK_PRIMITIVE_TOPOLOGY_LINE_LIST_WITH_ADJACENCY:
2741 case VK_PRIMITIVE_TOPOLOGY_LINE_STRIP_WITH_ADJACENCY:
2742 return V_028A6C_LINESTRIP;
2743 case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST:
2744 case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP:
2745 case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN:
2746 case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST_WITH_ADJACENCY:
2747 case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP_WITH_ADJACENCY:
2748 return V_028A6C_TRISTRIP;
2749 default:
2750 assert(0);
2751 return 0;
2752 }
2753 }
2754
2755 struct radv_extra_render_pass_begin_info {
2756 bool disable_dcc;
2757 };
2758
2759 void radv_cmd_buffer_begin_render_pass(struct radv_cmd_buffer *cmd_buffer,
2760 const VkRenderPassBeginInfo *pRenderPassBegin,
2761 const struct radv_extra_render_pass_begin_info *extra_info);
2762 void radv_cmd_buffer_end_render_pass(struct radv_cmd_buffer *cmd_buffer);
2763
2764 static inline uint32_t
si_translate_prim(unsigned topology)2765 si_translate_prim(unsigned topology)
2766 {
2767 switch (topology) {
2768 case VK_PRIMITIVE_TOPOLOGY_POINT_LIST:
2769 return V_008958_DI_PT_POINTLIST;
2770 case VK_PRIMITIVE_TOPOLOGY_LINE_LIST:
2771 return V_008958_DI_PT_LINELIST;
2772 case VK_PRIMITIVE_TOPOLOGY_LINE_STRIP:
2773 return V_008958_DI_PT_LINESTRIP;
2774 case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST:
2775 return V_008958_DI_PT_TRILIST;
2776 case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP:
2777 return V_008958_DI_PT_TRISTRIP;
2778 case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN:
2779 return V_008958_DI_PT_TRIFAN;
2780 case VK_PRIMITIVE_TOPOLOGY_LINE_LIST_WITH_ADJACENCY:
2781 return V_008958_DI_PT_LINELIST_ADJ;
2782 case VK_PRIMITIVE_TOPOLOGY_LINE_STRIP_WITH_ADJACENCY:
2783 return V_008958_DI_PT_LINESTRIP_ADJ;
2784 case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST_WITH_ADJACENCY:
2785 return V_008958_DI_PT_TRILIST_ADJ;
2786 case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP_WITH_ADJACENCY:
2787 return V_008958_DI_PT_TRISTRIP_ADJ;
2788 case VK_PRIMITIVE_TOPOLOGY_PATCH_LIST:
2789 return V_008958_DI_PT_PATCH;
2790 default:
2791 assert(0);
2792 return 0;
2793 }
2794 }
2795
2796 static inline uint32_t
si_translate_stencil_op(enum VkStencilOp op)2797 si_translate_stencil_op(enum VkStencilOp op)
2798 {
2799 switch (op) {
2800 case VK_STENCIL_OP_KEEP:
2801 return V_02842C_STENCIL_KEEP;
2802 case VK_STENCIL_OP_ZERO:
2803 return V_02842C_STENCIL_ZERO;
2804 case VK_STENCIL_OP_REPLACE:
2805 return V_02842C_STENCIL_REPLACE_TEST;
2806 case VK_STENCIL_OP_INCREMENT_AND_CLAMP:
2807 return V_02842C_STENCIL_ADD_CLAMP;
2808 case VK_STENCIL_OP_DECREMENT_AND_CLAMP:
2809 return V_02842C_STENCIL_SUB_CLAMP;
2810 case VK_STENCIL_OP_INVERT:
2811 return V_02842C_STENCIL_INVERT;
2812 case VK_STENCIL_OP_INCREMENT_AND_WRAP:
2813 return V_02842C_STENCIL_ADD_WRAP;
2814 case VK_STENCIL_OP_DECREMENT_AND_WRAP:
2815 return V_02842C_STENCIL_SUB_WRAP;
2816 default:
2817 return 0;
2818 }
2819 }
2820
2821 static inline uint32_t
si_translate_blend_logic_op(VkLogicOp op)2822 si_translate_blend_logic_op(VkLogicOp op)
2823 {
2824 switch (op) {
2825 case VK_LOGIC_OP_CLEAR:
2826 return V_028808_ROP3_CLEAR;
2827 case VK_LOGIC_OP_AND:
2828 return V_028808_ROP3_AND;
2829 case VK_LOGIC_OP_AND_REVERSE:
2830 return V_028808_ROP3_AND_REVERSE;
2831 case VK_LOGIC_OP_COPY:
2832 return V_028808_ROP3_COPY;
2833 case VK_LOGIC_OP_AND_INVERTED:
2834 return V_028808_ROP3_AND_INVERTED;
2835 case VK_LOGIC_OP_NO_OP:
2836 return V_028808_ROP3_NO_OP;
2837 case VK_LOGIC_OP_XOR:
2838 return V_028808_ROP3_XOR;
2839 case VK_LOGIC_OP_OR:
2840 return V_028808_ROP3_OR;
2841 case VK_LOGIC_OP_NOR:
2842 return V_028808_ROP3_NOR;
2843 case VK_LOGIC_OP_EQUIVALENT:
2844 return V_028808_ROP3_EQUIVALENT;
2845 case VK_LOGIC_OP_INVERT:
2846 return V_028808_ROP3_INVERT;
2847 case VK_LOGIC_OP_OR_REVERSE:
2848 return V_028808_ROP3_OR_REVERSE;
2849 case VK_LOGIC_OP_COPY_INVERTED:
2850 return V_028808_ROP3_COPY_INVERTED;
2851 case VK_LOGIC_OP_OR_INVERTED:
2852 return V_028808_ROP3_OR_INVERTED;
2853 case VK_LOGIC_OP_NAND:
2854 return V_028808_ROP3_NAND;
2855 case VK_LOGIC_OP_SET:
2856 return V_028808_ROP3_SET;
2857 default:
2858 unreachable("Unhandled logic op");
2859 }
2860 }
2861
2862 /**
2863 * Helper used for debugging compiler issues by enabling/disabling LLVM for a
2864 * specific shader stage (developers only).
2865 */
2866 static inline bool
radv_use_llvm_for_stage(struct radv_device * device,UNUSED gl_shader_stage stage)2867 radv_use_llvm_for_stage(struct radv_device *device, UNUSED gl_shader_stage stage)
2868 {
2869 return device->physical_device->use_llvm;
2870 }
2871
2872 struct radv_acceleration_structure {
2873 struct vk_object_base base;
2874
2875 struct radeon_winsys_bo *bo;
2876 uint64_t mem_offset;
2877 uint64_t size;
2878 };
2879
2880 static inline uint64_t
radv_accel_struct_get_va(const struct radv_acceleration_structure * accel)2881 radv_accel_struct_get_va(const struct radv_acceleration_structure *accel)
2882 {
2883 return radv_buffer_get_va(accel->bo) + accel->mem_offset;
2884 }
2885
2886 #define RADV_FROM_HANDLE(__radv_type, __name, __handle) \
2887 VK_FROM_HANDLE(__radv_type, __name, __handle)
2888
2889 VK_DEFINE_HANDLE_CASTS(radv_cmd_buffer, vk.base, VkCommandBuffer,
2890 VK_OBJECT_TYPE_COMMAND_BUFFER)
2891 VK_DEFINE_HANDLE_CASTS(radv_device, vk.base, VkDevice, VK_OBJECT_TYPE_DEVICE)
2892 VK_DEFINE_HANDLE_CASTS(radv_instance, vk.base, VkInstance, VK_OBJECT_TYPE_INSTANCE)
2893 VK_DEFINE_HANDLE_CASTS(radv_physical_device, vk.base, VkPhysicalDevice,
2894 VK_OBJECT_TYPE_PHYSICAL_DEVICE)
2895 VK_DEFINE_HANDLE_CASTS(radv_queue, vk.base, VkQueue, VK_OBJECT_TYPE_QUEUE)
2896 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_acceleration_structure, base,
2897 VkAccelerationStructureKHR,
2898 VK_OBJECT_TYPE_ACCELERATION_STRUCTURE_KHR)
2899 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_cmd_pool, base, VkCommandPool,
2900 VK_OBJECT_TYPE_COMMAND_POOL)
2901 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_buffer, base, VkBuffer, VK_OBJECT_TYPE_BUFFER)
2902 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_buffer_view, base, VkBufferView,
2903 VK_OBJECT_TYPE_BUFFER_VIEW)
2904 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_descriptor_pool, base, VkDescriptorPool,
2905 VK_OBJECT_TYPE_DESCRIPTOR_POOL)
2906 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_descriptor_set, header.base, VkDescriptorSet,
2907 VK_OBJECT_TYPE_DESCRIPTOR_SET)
2908 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_descriptor_set_layout, base,
2909 VkDescriptorSetLayout,
2910 VK_OBJECT_TYPE_DESCRIPTOR_SET_LAYOUT)
2911 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_descriptor_update_template, base,
2912 VkDescriptorUpdateTemplate,
2913 VK_OBJECT_TYPE_DESCRIPTOR_UPDATE_TEMPLATE)
2914 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_device_memory, base, VkDeviceMemory,
2915 VK_OBJECT_TYPE_DEVICE_MEMORY)
2916 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_fence, base, VkFence, VK_OBJECT_TYPE_FENCE)
2917 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_event, base, VkEvent, VK_OBJECT_TYPE_EVENT)
2918 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_framebuffer, base, VkFramebuffer,
2919 VK_OBJECT_TYPE_FRAMEBUFFER)
2920 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_image, base, VkImage, VK_OBJECT_TYPE_IMAGE)
2921 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_image_view, base, VkImageView,
2922 VK_OBJECT_TYPE_IMAGE_VIEW);
2923 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_pipeline_cache, base, VkPipelineCache,
2924 VK_OBJECT_TYPE_PIPELINE_CACHE)
2925 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_pipeline, base, VkPipeline,
2926 VK_OBJECT_TYPE_PIPELINE)
2927 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_pipeline_layout, base, VkPipelineLayout,
2928 VK_OBJECT_TYPE_PIPELINE_LAYOUT)
2929 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_query_pool, base, VkQueryPool,
2930 VK_OBJECT_TYPE_QUERY_POOL)
2931 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_render_pass, base, VkRenderPass,
2932 VK_OBJECT_TYPE_RENDER_PASS)
2933 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_sampler, base, VkSampler,
2934 VK_OBJECT_TYPE_SAMPLER)
2935 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_sampler_ycbcr_conversion, base,
2936 VkSamplerYcbcrConversion,
2937 VK_OBJECT_TYPE_SAMPLER_YCBCR_CONVERSION)
2938 VK_DEFINE_NONDISP_HANDLE_CASTS(radv_semaphore, base, VkSemaphore,
2939 VK_OBJECT_TYPE_SEMAPHORE)
2940
2941 #ifdef __cplusplus
2942 }
2943 #endif
2944
2945 #endif /* RADV_PRIVATE_H */
2946