1 /*
2 * Copyright © 2019 Red Hat.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #pragma once
25 #include <stdlib.h>
26 #include <stdio.h>
27 #include <stdbool.h>
28 #include <string.h>
29 #include <assert.h>
30 #include <stdint.h>
31
32 #include <llvm/Config/llvm-config.h>
33
34 #include "util/macros.h"
35 #include "util/list.h"
36 #include "util/u_dynarray.h"
37 #include "util/simple_mtx.h"
38 #include "util/u_queue.h"
39 #include "util/u_upload_mgr.h"
40
41 #include "compiler/shader_enums.h"
42 #include "pipe/p_screen.h"
43 #include "pipe/p_state.h"
44 #include "cso_cache/cso_context.h"
45 #include "nir.h"
46
47 #ifdef HAVE_LIBDRM
48 #include <drm-uapi/drm.h>
49 #include "drm-uapi/drm_fourcc.h"
50 #endif
51
52 #if DETECT_OS_ANDROID
53 #include <vndk/hardware_buffer.h>
54 #endif
55
56 /* Pre-declarations needed for WSI entrypoints */
57 struct wl_surface;
58 struct wl_display;
59 typedef struct xcb_connection_t xcb_connection_t;
60 typedef uint32_t xcb_visualid_t;
61 typedef uint32_t xcb_window_t;
62
63 #define VK_PROTOTYPES
64 #include <vulkan/vulkan.h>
65 #include <vulkan/vk_icd.h>
66
67 #include "lvp_entrypoints.h"
68 #include "vk_acceleration_structure.h"
69 #include "vk_buffer.h"
70 #include "vk_buffer_view.h"
71 #include "vk_device.h"
72 #include "vk_device_generated_commands.h"
73 #include "vk_instance.h"
74 #include "vk_image.h"
75 #include "vk_log.h"
76 #include "vk_physical_device.h"
77 #include "vk_shader_module.h"
78 #include "vk_util.h"
79 #include "vk_format.h"
80 #include "vk_cmd_queue.h"
81 #include "vk_command_buffer.h"
82 #include "vk_command_pool.h"
83 #include "vk_descriptor_set_layout.h"
84 #include "vk_graphics_state.h"
85 #include "vk_pipeline_layout.h"
86 #include "vk_queue.h"
87 #include "vk_sampler.h"
88 #include "vk_sync.h"
89 #include "vk_sync_timeline.h"
90 #include "vk_ycbcr_conversion.h"
91 #include "lp_jit.h"
92
93 #include "wsi_common.h"
94
95 #include <assert.h>
96 #ifdef __cplusplus
97 extern "C" {
98 #endif
99
100 #define MAX_SETS 8
101 #define MAX_DESCRIPTORS 1000000 /* Required by vkd3d-proton */
102 #define MAX_PUSH_CONSTANTS_SIZE 256
103 #define MAX_PUSH_DESCRIPTORS 32
104 #define MAX_DESCRIPTOR_UNIFORM_BLOCK_SIZE 4096
105 #define MAX_PER_STAGE_DESCRIPTOR_UNIFORM_BLOCKS 8
106 #define MAX_DGC_STREAMS 16
107 #define MAX_DGC_TOKENS 16
108 /* Currently lavapipe does not support more than 1 image plane */
109 #define LVP_MAX_PLANE_COUNT 1
110
111 #ifdef _WIN32
112 #define lvp_printflike(a, b)
113 #else
114 #define lvp_printflike(a, b) __attribute__((__format__(__printf__, a, b)))
115 #endif
116
117 #define LVP_DEBUG_ALL_ENTRYPOINTS (1 << 0)
118
119 void __lvp_finishme(const char *file, int line, const char *format, ...)
120 lvp_printflike(3, 4);
121
122 #define lvp_finishme(format, ...) \
123 __lvp_finishme(__FILE__, __LINE__, format, ##__VA_ARGS__);
124
125 #define stub_return(v) \
126 do { \
127 lvp_finishme("stub %s", __func__); \
128 return (v); \
129 } while (0)
130
131 #define stub() \
132 do { \
133 lvp_finishme("stub %s", __func__); \
134 return; \
135 } while (0)
136
137 #define LVP_SHADER_STAGES (MESA_SHADER_CALLABLE + 1)
138 #define LVP_STAGE_MASK BITFIELD_MASK(LVP_SHADER_STAGES)
139 #define LVP_STAGE_MASK_GFX (BITFIELD_MASK(PIPE_SHADER_MESH_TYPES) & ~BITFIELD_BIT(MESA_SHADER_COMPUTE))
140
141 #define lvp_foreach_stage(stage, stage_bits) \
142 for (gl_shader_stage stage, \
143 __tmp = (gl_shader_stage)((stage_bits) & LVP_STAGE_MASK); \
144 stage = ffs(__tmp) - 1, __tmp; \
145 __tmp &= ~(1 << (stage)))
146
147 #define lvp_forall_stage(stage) \
148 for (gl_shader_stage stage = MESA_SHADER_VERTEX; stage < LVP_SHADER_STAGES; stage++)
149
150 #define lvp_forall_gfx_stage(stage) \
151 for (gl_shader_stage stage, \
152 __tmp = (gl_shader_stage)(LVP_STAGE_MASK_GFX); \
153 stage = ffs(__tmp) - 1, __tmp; \
154 __tmp &= ~(1 << (stage)))
155
156 struct lvp_physical_device {
157 struct vk_physical_device vk;
158
159 struct pipe_loader_device *pld;
160 struct pipe_screen *pscreen;
161 const nir_shader_compiler_options *drv_options[LVP_SHADER_STAGES];
162 uint32_t max_images;
163 bool snorm_blend;
164
165 struct vk_sync_timeline_type sync_timeline_type;
166 const struct vk_sync_type *sync_types[3];
167
168 struct wsi_device wsi_device;
169 };
170
171 struct lvp_instance {
172 struct vk_instance vk;
173
174 uint32_t apiVersion;
175
176 uint64_t debug_flags;
177
178 struct pipe_loader_device *devs;
179 int num_devices;
180 };
181
182 VkResult lvp_init_wsi(struct lvp_physical_device *physical_device);
183 void lvp_finish_wsi(struct lvp_physical_device *physical_device);
184
185 bool lvp_physical_device_extension_supported(struct lvp_physical_device *dev,
186 const char *name);
187
188 struct lvp_queue {
189 struct vk_queue vk;
190 struct lvp_device * device;
191 struct pipe_context *ctx;
192 struct cso_context *cso;
193 struct u_upload_mgr *uploader;
194 struct pipe_fence_handle *last_fence;
195 void *state;
196 struct util_dynarray pipeline_destroys;
197 simple_mtx_t lock;
198 };
199
200 struct lvp_pipeline_cache {
201 struct vk_object_base base;
202 struct lvp_device * device;
203 VkAllocationCallbacks alloc;
204 };
205
206 struct lvp_device {
207 struct vk_device vk;
208
209 struct lvp_queue queue;
210 struct lvp_instance * instance;
211 struct lvp_physical_device *physical_device;
212 struct pipe_screen *pscreen;
213 void *noop_fs;
214 simple_mtx_t bda_lock;
215 struct hash_table bda;
216 struct pipe_resource *zero_buffer; /* for zeroed bda */
217 bool poison_mem;
218 bool print_cmds;
219
220 struct lp_texture_handle *null_texture_handle;
221 struct lp_texture_handle *null_image_handle;
222 struct util_dynarray bda_texture_handles;
223 struct util_dynarray bda_image_handles;
224
225 uint32_t group_handle_alloc;
226 };
227
228 void lvp_device_get_cache_uuid(void *uuid);
229
230 enum lvp_device_memory_type {
231 LVP_DEVICE_MEMORY_TYPE_DEFAULT,
232 LVP_DEVICE_MEMORY_TYPE_USER_PTR,
233 LVP_DEVICE_MEMORY_TYPE_OPAQUE_FD,
234 LVP_DEVICE_MEMORY_TYPE_DMA_BUF,
235 };
236
237 struct lvp_device_memory {
238 struct vk_object_base base;
239 struct pipe_memory_allocation *pmem;
240 struct llvmpipe_memory_allocation mem_alloc;
241 uint32_t type_index;
242 VkDeviceSize map_size;
243 VkDeviceSize size;
244 void * map;
245 enum lvp_device_memory_type memory_type;
246 int backed_fd;
247 #if DETECT_OS_ANDROID
248 struct AHardwareBuffer *android_hardware_buffer;
249 #endif
250 };
251
252 struct lvp_pipe_sync {
253 struct vk_sync base;
254
255 mtx_t lock;
256 cnd_t changed;
257
258 bool signaled;
259 struct pipe_fence_handle *fence;
260 };
261
262 extern const struct vk_sync_type lvp_pipe_sync_type;
263
264 void lvp_pipe_sync_signal_with_fence(struct lvp_device *device,
265 struct lvp_pipe_sync *sync,
266 struct pipe_fence_handle *fence);
267
268 static inline struct lvp_pipe_sync *
vk_sync_as_lvp_pipe_sync(struct vk_sync * sync)269 vk_sync_as_lvp_pipe_sync(struct vk_sync *sync)
270 {
271 assert(sync->type == &lvp_pipe_sync_type);
272 return container_of(sync, struct lvp_pipe_sync, base);
273 }
274
275 struct lvp_image_plane {
276 struct pipe_resource *bo;
277 struct pipe_memory_allocation *pmem;
278 VkDeviceSize plane_offset;
279 VkDeviceSize memory_offset;
280 VkDeviceSize size;
281 };
282
283 struct lvp_image {
284 struct vk_image vk;
285 VkDeviceSize offset;
286 VkDeviceSize size;
287 uint32_t alignment;
288 bool disjoint;
289 uint8_t plane_count;
290 struct lvp_image_plane planes[3];
291 };
292
293 struct lvp_image_view {
294 struct vk_image_view vk;
295 const struct lvp_image *image; /**< VkImageViewCreateInfo::image */
296
297 enum pipe_format pformat;
298
299 struct pipe_surface *surface; /* have we created a pipe surface for this? */
300 struct lvp_image_view *multisample; //VK_EXT_multisampled_render_to_single_sampled
301
302 uint8_t plane_count;
303 struct {
304 unsigned image_plane;
305 struct pipe_sampler_view *sv;
306 struct pipe_image_view iv;
307 struct lp_texture_handle *texture_handle;
308 struct lp_texture_handle *image_handle;
309 } planes[3];
310 };
311
312 struct lvp_sampler {
313 struct vk_sampler vk;
314 struct lp_descriptor desc;
315 };
316
317 struct lvp_descriptor_set_binding_layout {
318 uint32_t descriptor_index;
319 /* Number of array elements in this binding */
320 VkDescriptorType type;
321 uint32_t stride; /* used for planar samplers */
322 uint32_t array_size;
323 bool valid;
324
325 uint32_t dynamic_index;
326
327 uint32_t uniform_block_offset;
328 uint32_t uniform_block_size;
329
330 /* Immutable samplers (or NULL if no immutable samplers) */
331 struct lvp_sampler **immutable_samplers;
332 };
333
334 struct lvp_descriptor_set_layout {
335 struct vk_descriptor_set_layout vk;
336
337 /* add new members after this */
338
339 uint32_t immutable_sampler_count;
340
341 /* Number of bindings in this descriptor set */
342 uint32_t binding_count;
343
344 /* Total size of the descriptor set with room for all array entries */
345 uint32_t size;
346
347 /* Shader stages affected by this descriptor set */
348 uint32_t shader_stages;
349
350 /* Number of dynamic offsets used by this descriptor set */
351 uint32_t dynamic_offset_count;
352
353 /* if this layout is comprised solely of immutable samplers, this will be a bindable set */
354 struct lvp_descriptor_set *immutable_set;
355
356 /* Bindings in this descriptor set */
357 struct lvp_descriptor_set_binding_layout binding[0];
358 };
359
360 static inline const struct lvp_descriptor_set_layout *
vk_to_lvp_descriptor_set_layout(const struct vk_descriptor_set_layout * layout)361 vk_to_lvp_descriptor_set_layout(const struct vk_descriptor_set_layout *layout)
362 {
363 return container_of(layout, const struct lvp_descriptor_set_layout, vk);
364 }
365
366 struct lvp_descriptor_set {
367 struct vk_object_base base;
368 struct lvp_descriptor_set_layout *layout;
369 struct list_head link;
370
371 /* Buffer holding the descriptors. */
372 struct pipe_memory_allocation *pmem;
373 struct pipe_resource *bo;
374 void *map;
375 };
376
377 struct lvp_descriptor_pool {
378 struct vk_object_base base;
379 VkDescriptorPoolCreateFlags flags;
380 uint32_t max_sets;
381
382 struct list_head sets;
383 };
384
385 uint32_t lvp_descriptor_update_template_entry_size(VkDescriptorType type);
386
387 VkResult
388 lvp_descriptor_set_create(struct lvp_device *device,
389 struct lvp_descriptor_set_layout *layout,
390 struct lvp_descriptor_set **out_set);
391
392 void
393 lvp_descriptor_set_destroy(struct lvp_device *device,
394 struct lvp_descriptor_set *set);
395
396 void
397 lvp_descriptor_set_update_with_template(VkDevice _device, VkDescriptorSet descriptorSet,
398 VkDescriptorUpdateTemplate descriptorUpdateTemplate,
399 const void *pData);
400
401 struct lvp_pipeline_layout {
402 struct vk_pipeline_layout vk;
403
404 uint32_t push_constant_size;
405 VkShaderStageFlags push_constant_stages;
406 };
407
408
409 struct lvp_pipeline_layout *
410 lvp_pipeline_layout_create(struct lvp_device *device,
411 const VkPipelineLayoutCreateInfo* pCreateInfo,
412 const VkAllocationCallbacks* pAllocator);
413
414 struct lvp_pipeline_nir {
415 int ref_cnt;
416 nir_shader *nir;
417 };
418
419 struct lvp_pipeline_nir *
420 lvp_create_pipeline_nir(nir_shader *nir);
421
422 static inline void
lvp_pipeline_nir_ref(struct lvp_pipeline_nir ** dst,struct lvp_pipeline_nir * src)423 lvp_pipeline_nir_ref(struct lvp_pipeline_nir **dst, struct lvp_pipeline_nir *src)
424 {
425 struct lvp_pipeline_nir *old_dst = *dst;
426 if (old_dst == src || (old_dst && src && old_dst->nir == src->nir))
427 return;
428
429 if (old_dst && p_atomic_dec_zero(&old_dst->ref_cnt)) {
430 ralloc_free(old_dst->nir);
431 ralloc_free(old_dst);
432 }
433 if (src)
434 p_atomic_inc(&src->ref_cnt);
435 *dst = src;
436 }
437
438 struct lvp_inline_variant {
439 uint32_t mask;
440 uint32_t vals[PIPE_MAX_CONSTANT_BUFFERS][MAX_INLINABLE_UNIFORMS];
441 void *cso;
442 };
443
444 struct lvp_shader {
445 struct vk_object_base base;
446 struct lvp_pipeline_layout *layout;
447 struct lvp_pipeline_nir *pipeline_nir;
448 struct lvp_pipeline_nir *tess_ccw;
449 void *shader_cso;
450 void *tess_ccw_cso;
451 struct {
452 uint32_t uniform_offsets[PIPE_MAX_CONSTANT_BUFFERS][MAX_INLINABLE_UNIFORMS];
453 uint8_t count[PIPE_MAX_CONSTANT_BUFFERS];
454 bool must_inline;
455 uint32_t can_inline; //bitmask
456 struct set variants;
457 } inlines;
458 struct pipe_stream_output_info stream_output;
459 struct blob blob; //preserved for GetShaderBinaryDataEXT
460 uint32_t push_constant_size;
461 };
462
463 enum lvp_pipeline_type {
464 LVP_PIPELINE_GRAPHICS,
465 LVP_PIPELINE_COMPUTE,
466 LVP_PIPELINE_RAY_TRACING,
467 LVP_PIPELINE_EXEC_GRAPH,
468 LVP_PIPELINE_TYPE_COUNT,
469 };
470
471 static inline enum lvp_pipeline_type
lvp_pipeline_type_from_bind_point(VkPipelineBindPoint bind_point)472 lvp_pipeline_type_from_bind_point(VkPipelineBindPoint bind_point)
473 {
474 switch (bind_point) {
475 case VK_PIPELINE_BIND_POINT_GRAPHICS: return LVP_PIPELINE_GRAPHICS;
476 case VK_PIPELINE_BIND_POINT_COMPUTE: return LVP_PIPELINE_COMPUTE;
477 case VK_PIPELINE_BIND_POINT_RAY_TRACING_KHR: return LVP_PIPELINE_RAY_TRACING;
478 #ifdef VK_ENABLE_BETA_EXTENSIONS
479 case VK_PIPELINE_BIND_POINT_EXECUTION_GRAPH_AMDX: return LVP_PIPELINE_EXEC_GRAPH;
480 #endif
481 default: unreachable("Unsupported VkPipelineBindPoint");
482 }
483 }
484
485 #define LVP_RAY_TRACING_STAGES (VK_SHADER_STAGE_RAYGEN_BIT_KHR | VK_SHADER_STAGE_ANY_HIT_BIT_KHR | \
486 VK_SHADER_STAGE_CLOSEST_HIT_BIT_KHR | VK_SHADER_STAGE_MISS_BIT_KHR | \
487 VK_SHADER_STAGE_INTERSECTION_BIT_KHR | VK_SHADER_STAGE_CALLABLE_BIT_KHR)
488
489 static inline uint32_t
lvp_pipeline_types_from_shader_stages(VkShaderStageFlags stageFlags)490 lvp_pipeline_types_from_shader_stages(VkShaderStageFlags stageFlags)
491 {
492 uint32_t types = 0;
493 #ifdef VK_ENABLE_BETA_EXTENSIONS
494 if (stageFlags & MESA_VK_SHADER_STAGE_WORKGRAPH_HACK_BIT_FIXME)
495 types |= BITFIELD_BIT(LVP_PIPELINE_EXEC_GRAPH);
496 #endif
497 if (stageFlags & LVP_RAY_TRACING_STAGES)
498 types |= BITFIELD_BIT(LVP_PIPELINE_RAY_TRACING);
499 if (stageFlags & VK_SHADER_STAGE_COMPUTE_BIT)
500 types |= BITFIELD_BIT(LVP_PIPELINE_COMPUTE);
501 if (stageFlags & (VK_SHADER_STAGE_ALL_GRAPHICS | VK_SHADER_STAGE_MESH_BIT_EXT | VK_SHADER_STAGE_TASK_BIT_EXT))
502 types |= BITFIELD_BIT(LVP_PIPELINE_GRAPHICS);
503 return types;
504 }
505
506 #define LVP_RAY_TRACING_GROUP_HANDLE_SIZE 32
507 #define LVP_RAY_HIT_ATTRIBS_SIZE 32
508
509 struct lvp_ray_tracing_group_handle {
510 uint32_t index;
511 };
512
513 struct lvp_ray_tracing_group {
514 struct lvp_ray_tracing_group_handle handle;
515 uint32_t recursive_index;
516 uint32_t ahit_index;
517 uint32_t isec_index;
518 };
519
520 struct lvp_pipeline {
521 struct vk_object_base base;
522 struct lvp_device * device;
523 struct lvp_pipeline_layout * layout;
524
525 enum lvp_pipeline_type type;
526 VkPipelineCreateFlags2KHR flags;
527
528 void *state_data;
529 bool force_min_sample;
530 struct lvp_shader shaders[LVP_SHADER_STAGES];
531 gl_shader_stage last_vertex;
532 struct vk_graphics_pipeline_state graphics_state;
533 VkGraphicsPipelineLibraryFlagsEXT stages;
534 bool line_smooth;
535 bool disable_multisample;
536 bool line_rectangular;
537 bool library;
538 bool compiled;
539 bool used;
540
541 struct {
542 const char *name;
543 const char *next_name;
544 uint32_t index;
545 uint32_t scratch_size;
546 } exec_graph;
547
548 struct {
549 struct lvp_pipeline_nir **stages;
550 struct lvp_ray_tracing_group *groups;
551 uint32_t stage_count;
552 uint32_t group_count;
553 } rt;
554
555 unsigned num_groups;
556 unsigned num_groups_total;
557 VkPipeline groups[0];
558 };
559
560 /* Minimum requirement by the spec. */
561 #define LVP_MAX_EXEC_GRAPH_PAYLOADS 256
562
563 struct lvp_exec_graph_shader_output {
564 uint32_t payload_count;
565 uint32_t node_index;
566 };
567
568 struct lvp_exec_graph_internal_data {
569 /* inputs */
570 void *payload_in;
571 void *payloads;
572 /* outputs */
573 struct lvp_exec_graph_shader_output outputs[LVP_MAX_EXEC_GRAPH_PAYLOADS];
574 };
575
576 bool
577 lvp_lower_exec_graph(struct lvp_pipeline *pipeline, nir_shader *nir);
578
579 void
580 lvp_pipeline_shaders_compile(struct lvp_pipeline *pipeline, bool locked);
581
582 struct lvp_event {
583 struct vk_object_base base;
584 volatile uint64_t event_storage;
585 };
586
587 struct lvp_buffer {
588 struct vk_buffer vk;
589
590 struct lvp_device_memory *mem;
591 struct pipe_resource *bo;
592 uint64_t total_size;
593 uint64_t offset;
594 void *map;
595 struct pipe_transfer *transfer;
596 };
597
598 struct lvp_buffer_view {
599 struct vk_buffer_view vk;
600 enum pipe_format pformat;
601 struct pipe_sampler_view *sv;
602 struct pipe_image_view iv;
603
604 struct lp_texture_handle *texture_handle;
605 struct lp_texture_handle *image_handle;
606 };
607
608 #define LVP_QUERY_ACCELERATION_STRUCTURE_COMPACTED_SIZE (PIPE_QUERY_TYPES)
609 #define LVP_QUERY_ACCELERATION_STRUCTURE_SERIALIZATION_SIZE (PIPE_QUERY_TYPES + 1)
610 #define LVP_QUERY_ACCELERATION_STRUCTURE_SIZE (PIPE_QUERY_TYPES + 2)
611 #define LVP_QUERY_ACCELERATION_STRUCTURE_INSTANCE_COUNT (PIPE_QUERY_TYPES + 3)
612
613 struct lvp_query_pool {
614 struct vk_object_base base;
615 VkQueryType type;
616 uint32_t count;
617 VkQueryPipelineStatisticFlags pipeline_stats;
618 enum pipe_query_type base_type;
619 void *data; /* Used by queries that are not implemented by pipe_query */
620 struct pipe_query *queries[0];
621 };
622
623 struct lvp_cmd_buffer {
624 struct vk_command_buffer vk;
625
626 struct lvp_device * device;
627
628 uint8_t push_constants[MAX_PUSH_CONSTANTS_SIZE];
629 };
630
631 struct lvp_indirect_command_layout_nv {
632 struct vk_object_base base;
633 uint8_t stream_count;
634 uint8_t token_count;
635 uint16_t stream_strides[MAX_DGC_STREAMS];
636 VkPipelineBindPoint bind_point;
637 VkIndirectCommandsLayoutUsageFlagsNV flags;
638 VkIndirectCommandsLayoutTokenNV tokens[0];
639 };
640
641 struct lvp_indirect_execution_set {
642 struct vk_object_base base;
643 bool is_shaders;
644 #if VK_USE_64_BIT_PTR_DEFINES
645 void *array[0];
646 #else
647 uint64_t array[0];
648 #endif
649 };
650
651 enum lvp_indirect_layout_type {
652 LVP_INDIRECT_COMMAND_LAYOUT_DRAW,
653 LVP_INDIRECT_COMMAND_LAYOUT_DRAW_COUNT,
654 LVP_INDIRECT_COMMAND_LAYOUT_DISPATCH,
655 LVP_INDIRECT_COMMAND_LAYOUT_RAYS,
656 };
657
658 struct lvp_indirect_command_layout_ext {
659 struct vk_indirect_command_layout vk;
660 enum lvp_indirect_layout_type type;
661 VkIndirectCommandsLayoutTokenEXT tokens[0];
662 };
663
664 extern const struct vk_command_buffer_ops lvp_cmd_buffer_ops;
665
666 static inline const struct lvp_descriptor_set_layout *
get_set_layout(const struct lvp_pipeline_layout * layout,uint32_t set)667 get_set_layout(const struct lvp_pipeline_layout *layout, uint32_t set)
668 {
669 return container_of(layout->vk.set_layouts[set],
670 const struct lvp_descriptor_set_layout, vk);
671 }
672
673 static inline const struct lvp_descriptor_set_binding_layout *
get_binding_layout(const struct lvp_pipeline_layout * layout,uint32_t set,uint32_t binding)674 get_binding_layout(const struct lvp_pipeline_layout *layout,
675 uint32_t set, uint32_t binding)
676 {
677 return &get_set_layout(layout, set)->binding[binding];
678 }
679
680 #define LVP_FROM_HANDLE(__lvp_type, __name, __handle) \
681 struct __lvp_type *__name = __lvp_type ## _from_handle(__handle)
682
683 VK_DEFINE_HANDLE_CASTS(lvp_cmd_buffer, vk.base, VkCommandBuffer,
684 VK_OBJECT_TYPE_COMMAND_BUFFER)
685 VK_DEFINE_HANDLE_CASTS(lvp_device, vk.base, VkDevice, VK_OBJECT_TYPE_DEVICE)
686 VK_DEFINE_HANDLE_CASTS(lvp_instance, vk.base, VkInstance, VK_OBJECT_TYPE_INSTANCE)
687 VK_DEFINE_HANDLE_CASTS(lvp_physical_device, vk.base, VkPhysicalDevice,
688 VK_OBJECT_TYPE_PHYSICAL_DEVICE)
689 VK_DEFINE_HANDLE_CASTS(lvp_queue, vk.base, VkQueue, VK_OBJECT_TYPE_QUEUE)
690
691 VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_buffer, vk.base, VkBuffer,
692 VK_OBJECT_TYPE_BUFFER)
693 VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_buffer_view, vk.base, VkBufferView,
694 VK_OBJECT_TYPE_BUFFER_VIEW)
695 VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_descriptor_pool, base, VkDescriptorPool,
696 VK_OBJECT_TYPE_DESCRIPTOR_POOL)
697 VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_descriptor_set, base, VkDescriptorSet,
698 VK_OBJECT_TYPE_DESCRIPTOR_SET)
699 VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_descriptor_set_layout, vk.base, VkDescriptorSetLayout,
700 VK_OBJECT_TYPE_DESCRIPTOR_SET_LAYOUT)
701 VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_device_memory, base, VkDeviceMemory,
702 VK_OBJECT_TYPE_DEVICE_MEMORY)
703 VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_event, base, VkEvent, VK_OBJECT_TYPE_EVENT)
704 VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_image, vk.base, VkImage, VK_OBJECT_TYPE_IMAGE)
705 VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_image_view, vk.base, VkImageView,
706 VK_OBJECT_TYPE_IMAGE_VIEW);
707 VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_pipeline_cache, base, VkPipelineCache,
708 VK_OBJECT_TYPE_PIPELINE_CACHE)
709 VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_pipeline, base, VkPipeline,
710 VK_OBJECT_TYPE_PIPELINE)
711 VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_shader, base, VkShaderEXT,
712 VK_OBJECT_TYPE_SHADER_EXT)
713 VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_pipeline_layout, vk.base, VkPipelineLayout,
714 VK_OBJECT_TYPE_PIPELINE_LAYOUT)
715 VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_query_pool, base, VkQueryPool,
716 VK_OBJECT_TYPE_QUERY_POOL)
717 VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_sampler, vk.base, VkSampler,
718 VK_OBJECT_TYPE_SAMPLER)
719 VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_indirect_command_layout_nv, base, VkIndirectCommandsLayoutNV,
720 VK_OBJECT_TYPE_INDIRECT_COMMANDS_LAYOUT_NV)
721 VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_indirect_command_layout_ext, vk.base, VkIndirectCommandsLayoutEXT,
722 VK_OBJECT_TYPE_INDIRECT_COMMANDS_LAYOUT_EXT)
723 VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_indirect_execution_set, base, VkIndirectExecutionSetEXT,
724 VK_OBJECT_TYPE_INDIRECT_EXECUTION_SET_EXT)
725
726 void lvp_add_enqueue_cmd_entrypoints(struct vk_device_dispatch_table *disp);
727
728 VkResult lvp_buffer_bind_sparse(struct lvp_device *device,
729 struct lvp_queue *queue,
730 VkSparseBufferMemoryBindInfo *bind);
731 VkResult lvp_image_bind_opaque_sparse(struct lvp_device *device,
732 struct lvp_queue *queue,
733 VkSparseImageOpaqueMemoryBindInfo *bind);
734 VkResult lvp_image_bind_sparse(struct lvp_device *device,
735 struct lvp_queue *queue,
736 VkSparseImageMemoryBindInfo *bind);
737
738 VkResult lvp_execute_cmds(struct lvp_device *device,
739 struct lvp_queue *queue,
740 struct lvp_cmd_buffer *cmd_buffer);
741 size_t
742 lvp_get_rendering_state_size(void);
743 struct lvp_image *lvp_swapchain_get_image(VkSwapchainKHR swapchain,
744 uint32_t index);
745
746 static inline enum pipe_format
lvp_vk_format_to_pipe_format(VkFormat format)747 lvp_vk_format_to_pipe_format(VkFormat format)
748 {
749 /* Some formats cause problems with CTS right now.*/
750 switch (format) {
751 case VK_FORMAT_R4G4B4A4_UNORM_PACK16:
752 case VK_FORMAT_R8_SRGB:
753 case VK_FORMAT_R8G8_SRGB:
754 case VK_FORMAT_R64G64B64A64_SFLOAT:
755 case VK_FORMAT_R64_SFLOAT:
756 case VK_FORMAT_R64G64_SFLOAT:
757 case VK_FORMAT_R64G64B64_SFLOAT:
758 case VK_FORMAT_A2R10G10B10_SINT_PACK32:
759 case VK_FORMAT_A2B10G10R10_SINT_PACK32:
760 case VK_FORMAT_D16_UNORM_S8_UINT:
761 return PIPE_FORMAT_NONE;
762 case VK_FORMAT_R10X6_UNORM_PACK16:
763 case VK_FORMAT_R12X4_UNORM_PACK16:
764 return PIPE_FORMAT_R16_UNORM;
765 case VK_FORMAT_R10X6G10X6_UNORM_2PACK16:
766 case VK_FORMAT_R12X4G12X4_UNORM_2PACK16:
767 return PIPE_FORMAT_R16G16_UNORM;
768 default:
769 return vk_format_to_pipe_format(format);
770 }
771 }
772
773 void
774 lvp_sampler_init(struct lvp_device *device, struct lp_descriptor *desc, const VkSamplerCreateInfo *pCreateInfo, const struct vk_sampler *sampler);
775
776 static inline uint8_t
lvp_image_aspects_to_plane(ASSERTED const struct lvp_image * image,VkImageAspectFlags aspectMask)777 lvp_image_aspects_to_plane(ASSERTED const struct lvp_image *image,
778 VkImageAspectFlags aspectMask)
779 {
780 /* If we are requesting the first plane of image with only one plane, return that */
781 if (image->vk.aspects == VK_IMAGE_ASPECT_COLOR_BIT && aspectMask == VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT)
782 return 0;
783
784 /* Verify that the aspects are actually in the image */
785 assert(!(aspectMask & ~image->vk.aspects));
786
787 /* Must only be one aspect unless it's depth/stencil */
788 assert(aspectMask == (VK_IMAGE_ASPECT_DEPTH_BIT |
789 VK_IMAGE_ASPECT_STENCIL_BIT) ||
790 util_bitcount(aspectMask) == 1);
791
792 switch(aspectMask) {
793 case VK_IMAGE_ASPECT_PLANE_1_BIT: return 1;
794 case VK_IMAGE_ASPECT_PLANE_2_BIT: return 2;
795 default: return 0;
796 }
797 }
798
799 void
800 lvp_pipeline_destroy(struct lvp_device *device, struct lvp_pipeline *pipeline, bool locked);
801
802 void
803 queue_thread_noop(void *data, void *gdata, int thread_index);
804
805 VkResult
806 lvp_spirv_to_nir(struct lvp_pipeline *pipeline, const VkPipelineShaderStageCreateInfo *sinfo,
807 nir_shader **out_nir);
808
809 void
810 lvp_shader_init(struct lvp_shader *shader, nir_shader *nir);
811
812 void
813 lvp_shader_optimize(nir_shader *nir);
814 bool
815 lvp_find_inlinable_uniforms(struct lvp_shader *shader, nir_shader *nir);
816 void
817 lvp_inline_uniforms(nir_shader *nir, const struct lvp_shader *shader, const uint32_t *uniform_values, uint32_t ubo);
818 void *
819 lvp_shader_compile(struct lvp_device *device, struct lvp_shader *shader, nir_shader *nir, bool locked);
820 bool
821 lvp_nir_lower_ray_queries(struct nir_shader *shader);
822 bool
823 lvp_nir_lower_sparse_residency(struct nir_shader *shader);
824 enum vk_cmd_type
825 lvp_nv_dgc_token_to_cmd_type(const VkIndirectCommandsLayoutTokenNV *token);
826
827 #if DETECT_OS_ANDROID
828 VkResult
829 lvp_import_ahb_memory(struct lvp_device *device, struct lvp_device_memory *mem,
830 const VkImportAndroidHardwareBufferInfoANDROID *info);
831 VkResult
832 lvp_create_ahb_memory(struct lvp_device *device, struct lvp_device_memory *mem,
833 const VkMemoryAllocateInfo *pAllocateInfo);
834 #endif
835
836 enum vk_cmd_type
837 lvp_ext_dgc_token_to_cmd_type(const struct lvp_indirect_command_layout_ext *elayout, const VkIndirectCommandsLayoutTokenEXT *token);
838 size_t
839 lvp_ext_dgc_token_size(const struct lvp_indirect_command_layout_ext *elayout, const VkIndirectCommandsLayoutTokenEXT *token);
840 #ifdef __cplusplus
841 }
842 #endif
843