1 /*
2 * Copyright © 2022 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Mike Blumenkrantz <michael.blumenkrantz@gmail.com>
25 */
26
27 #ifndef ZINK_TYPES_H
28 #define ZINK_TYPES_H
29
30 #include <vulkan/vulkan_core.h>
31
32 #include "compiler/nir/nir.h"
33
34 #include "pipe/p_context.h"
35 #include "pipe/p_defines.h"
36 #include "pipe/p_state.h"
37
38 #include "pipebuffer/pb_cache.h"
39 #include "pipebuffer/pb_slab.h"
40
41 #include "util/disk_cache.h"
42 #include "util/hash_table.h"
43 #include "util/list.h"
44 #include "util/log.h"
45 #include "util/rwlock.h"
46 #include "util/set.h"
47 #include "util/simple_mtx.h"
48 #include "util/slab.h"
49 #include "util/u_dynarray.h"
50 #include "util/u_idalloc.h"
51 #include "util/u_live_shader_cache.h"
52 #include "util/u_queue.h"
53 #include "util/u_range.h"
54 #include "util/u_threaded_context.h"
55 #include "util/u_transfer.h"
56 #include "util/u_vertex_state_cache.h"
57
58 #include "vulkan/util/vk_util.h"
59
60 #include "zink_device_info.h"
61 #include "zink_instance.h"
62 #include "zink_shader_keys.h"
63 #include "vk_dispatch_table.h"
64
65 #ifdef HAVE_RENDERDOC_APP_H
66 #include "renderdoc_app.h"
67 #endif
68
69 /* the descriptor binding id for fbfetch/input attachment */
70 #define ZINK_FBFETCH_BINDING 5
71 #define ZINK_GFX_SHADER_COUNT 5
72
73 /* number of descriptors to allocate in a pool */
74 #define MAX_LAZY_DESCRIPTORS 500
75 /* explicit clamping because descriptor caching used to exist */
76 #define ZINK_MAX_SHADER_IMAGES 32
77 /* total number of bindless ids that can be allocated */
78 #define ZINK_MAX_BINDLESS_HANDLES 1024
79
80 /* enum zink_descriptor_type */
81 #define ZINK_MAX_DESCRIPTOR_SETS 6
82 #define ZINK_MAX_DESCRIPTORS_PER_TYPE (32 * ZINK_GFX_SHADER_COUNT)
83 /* Descriptor size reported by lavapipe. */
84 #define ZINK_FBFETCH_DESCRIPTOR_SIZE 280
85
86 /* suballocator defines */
87 #define NUM_SLAB_ALLOCATORS 3
88 #define MIN_SLAB_ORDER 8
89
90
91 /* this is the spec minimum */
92 #define ZINK_SPARSE_BUFFER_PAGE_SIZE (64 * 1024)
93
94 /* flag to create screen->copy_context */
95 #define ZINK_CONTEXT_COPY_ONLY (1<<30)
96
97 /* convenience macros for accessing dispatch table functions */
98 #define VKCTX(fn) zink_screen(ctx->base.screen)->vk.fn
99 #define VKSCR(fn) screen->vk.fn
100
101 #ifdef __cplusplus
102 extern "C" {
103 #endif
104
105 extern uint32_t zink_debug;
106 extern bool zink_tracing;
107
108 #ifdef __cplusplus
109 }
110 #endif
111
112
113 /** enums */
114
115 /* features for draw/program templates */
116 typedef enum {
117 ZINK_NO_MULTIDRAW,
118 ZINK_MULTIDRAW,
119 } zink_multidraw;
120
121 typedef enum {
122 ZINK_NO_DYNAMIC_STATE,
123 ZINK_DYNAMIC_STATE,
124 ZINK_DYNAMIC_STATE2,
125 ZINK_DYNAMIC_VERTEX_INPUT2,
126 ZINK_DYNAMIC_STATE3,
127 ZINK_DYNAMIC_VERTEX_INPUT,
128 } zink_dynamic_state;
129
130 typedef enum {
131 ZINK_PIPELINE_NO_DYNAMIC_STATE,
132 ZINK_PIPELINE_DYNAMIC_STATE,
133 ZINK_PIPELINE_DYNAMIC_STATE2,
134 ZINK_PIPELINE_DYNAMIC_STATE2_PCP,
135 ZINK_PIPELINE_DYNAMIC_VERTEX_INPUT2,
136 ZINK_PIPELINE_DYNAMIC_VERTEX_INPUT2_PCP,
137 ZINK_PIPELINE_DYNAMIC_STATE3,
138 ZINK_PIPELINE_DYNAMIC_STATE3_PCP,
139 ZINK_PIPELINE_DYNAMIC_VERTEX_INPUT,
140 ZINK_PIPELINE_DYNAMIC_VERTEX_INPUT_PCP,
141 } zink_pipeline_dynamic_state;
142
143 enum zink_blit_flags {
144 ZINK_BLIT_NORMAL = 1 << 0,
145 ZINK_BLIT_SAVE_FS = 1 << 1,
146 ZINK_BLIT_SAVE_FB = 1 << 2,
147 ZINK_BLIT_SAVE_TEXTURES = 1 << 3,
148 ZINK_BLIT_NO_COND_RENDER = 1 << 4,
149 ZINK_BLIT_SAVE_FS_CONST_BUF = 1 << 5,
150 };
151
152 /* descriptor types; also the ordering of the sets
153 * ...except that ZINK_DESCRIPTOR_BASE_TYPES is actually ZINK_DESCRIPTOR_TYPE_UNIFORMS,
154 * and all base type values are thus +1 to get the set id (using screen->desc_set_id[idx])
155 */
156 enum zink_descriptor_type {
157 ZINK_DESCRIPTOR_TYPE_UBO,
158 ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW,
159 ZINK_DESCRIPTOR_TYPE_SSBO,
160 ZINK_DESCRIPTOR_TYPE_IMAGE,
161 ZINK_DESCRIPTOR_BASE_TYPES, /**< the count/iterator for basic descriptor types */
162 ZINK_DESCRIPTOR_BINDLESS,
163 ZINK_DESCRIPTOR_ALL_TYPES,
164 ZINK_DESCRIPTOR_TYPE_UNIFORMS = ZINK_DESCRIPTOR_BASE_TYPES, /**< this is aliased for convenience */
165 ZINK_DESCRIPTOR_NON_BINDLESS_TYPES = ZINK_DESCRIPTOR_BASE_TYPES + 1, /**< for struct sizing */
166 };
167
168 enum zink_descriptor_mode {
169 ZINK_DESCRIPTOR_MODE_AUTO,
170 ZINK_DESCRIPTOR_MODE_LAZY,
171 ZINK_DESCRIPTOR_MODE_DB,
172 };
173
174 /* the current mode */
175 extern enum zink_descriptor_mode zink_descriptor_mode;
176
177 /* indexing for descriptor template management */
178 enum zink_descriptor_size_index {
179 ZDS_INDEX_UBO,
180 ZDS_INDEX_COMBINED_SAMPLER,
181 ZDS_INDEX_UNIFORM_TEXELS,
182 ZDS_INDEX_SAMPLER,
183 ZDS_INDEX_STORAGE_BUFFER,
184 ZDS_INDEX_STORAGE_IMAGE,
185 ZDS_INDEX_STORAGE_TEXELS,
186 ZDS_INDEX_MAX,
187 };
188
189 /* indexing for descriptor template management in COMPACT mode */
190 enum zink_descriptor_size_index_compact {
191 ZDS_INDEX_COMP_UBO,
192 ZDS_INDEX_COMP_STORAGE_BUFFER,
193 ZDS_INDEX_COMP_COMBINED_SAMPLER,
194 ZDS_INDEX_COMP_UNIFORM_TEXELS,
195 ZDS_INDEX_COMP_SAMPLER,
196 ZDS_INDEX_COMP_STORAGE_IMAGE,
197 ZDS_INDEX_COMP_STORAGE_TEXELS,
198 };
199
200 enum zink_resource_access {
201 ZINK_RESOURCE_ACCESS_READ = 1,
202 ZINK_RESOURCE_ACCESS_WRITE = 32,
203 ZINK_RESOURCE_ACCESS_RW = ZINK_RESOURCE_ACCESS_READ | ZINK_RESOURCE_ACCESS_WRITE,
204 };
205
206
207 /* zink heaps are based off of vulkan memory types, but are not a 1-to-1 mapping to vulkan memory type indices and have no direct relation to vulkan memory heaps*/
208 enum zink_heap {
209 ZINK_HEAP_DEVICE_LOCAL,
210 ZINK_HEAP_DEVICE_LOCAL_SPARSE,
211 ZINK_HEAP_DEVICE_LOCAL_LAZY,
212 ZINK_HEAP_DEVICE_LOCAL_VISIBLE,
213 ZINK_HEAP_HOST_VISIBLE_COHERENT,
214 ZINK_HEAP_HOST_VISIBLE_COHERENT_CACHED,
215 ZINK_HEAP_MAX,
216 };
217
218 enum zink_alloc_flag {
219 ZINK_ALLOC_SPARSE = 1<<0,
220 ZINK_ALLOC_NO_SUBALLOC = 1<<1,
221 };
222
223 enum zink_debug {
224 ZINK_DEBUG_NIR = (1<<0),
225 ZINK_DEBUG_SPIRV = (1<<1),
226 ZINK_DEBUG_TGSI = (1<<2),
227 ZINK_DEBUG_VALIDATION = (1<<3),
228 ZINK_DEBUG_SYNC = (1<<4),
229 ZINK_DEBUG_COMPACT = (1<<5),
230 ZINK_DEBUG_NOREORDER = (1<<6),
231 ZINK_DEBUG_GPL = (1<<7),
232 ZINK_DEBUG_SHADERDB = (1<<8),
233 ZINK_DEBUG_RP = (1<<9),
234 ZINK_DEBUG_NORP = (1<<10),
235 ZINK_DEBUG_MAP = (1<<11),
236 ZINK_DEBUG_FLUSHSYNC = (1<<12),
237 ZINK_DEBUG_NOSHOBJ = (1<<13),
238 ZINK_DEBUG_OPTIMAL_KEYS = (1<<14),
239 ZINK_DEBUG_NOOPT = (1<<15),
240 ZINK_DEBUG_NOBGC = (1<<16),
241 ZINK_DEBUG_DGC = (1<<17),
242 ZINK_DEBUG_MEM = (1<<18),
243 ZINK_DEBUG_QUIET = (1<<19),
244 };
245
246 enum zink_pv_emulation_primitive {
247 ZINK_PVE_PRIMITIVE_NONE = 0,
248 ZINK_PVE_PRIMITIVE_SIMPLE = 1,
249 /* when triangle or quad strips are used and the gs outputs triangles */
250 ZINK_PVE_PRIMITIVE_TRISTRIP = 2,
251 ZINK_PVE_PRIMITIVE_FAN = 3,
252 };
253
254 enum zink_dgc_buffer {
255 ZINK_DGC_VBO,
256 ZINK_DGC_IB,
257 ZINK_DGC_PSO,
258 ZINK_DGC_PUSH,
259 ZINK_DGC_DRAW,
260 ZINK_DGC_MAX,
261 };
262
263 /** fence types */
264 struct tc_unflushed_batch_token;
265
266 /* an async fence created for tc */
267 struct zink_tc_fence {
268 struct pipe_reference reference;
269 /* enables distinction between tc fence submission and vk queue submission */
270 uint32_t submit_count;
271 /* when the tc fence is signaled for use */
272 struct util_queue_fence ready;
273 struct tc_unflushed_batch_token *tc_token;
274 /* for deferred flushes */
275 struct pipe_context *deferred_ctx;
276 /* multiple tc fences may point to a real fence */
277 struct zink_fence *fence;
278 /* for use with semaphore/imported fences */
279 VkSemaphore sem;
280 };
281
282 /* a fence is actually a zink_batch_state, but these are split out for logical consistency */
283 struct zink_fence {
284 uint64_t batch_id;
285 bool submitted;
286 bool completed;
287 struct util_dynarray mfences;
288 };
289
290
291 /** state types */
292
293 struct zink_vertex_elements_hw_state {
294 uint32_t hash;
295 uint32_t num_bindings, num_attribs;
296 /* VK_EXT_vertex_input_dynamic_state uses different types */
297 union {
298 VkVertexInputAttributeDescription attribs[PIPE_MAX_ATTRIBS];
299 VkVertexInputAttributeDescription2EXT dynattribs[PIPE_MAX_ATTRIBS];
300 };
301 union {
302 struct {
303 VkVertexInputBindingDivisorDescriptionEXT divisors[PIPE_MAX_ATTRIBS];
304 VkVertexInputBindingDescription bindings[PIPE_MAX_ATTRIBS]; // combination of element_state and stride
305 VkDeviceSize strides[PIPE_MAX_ATTRIBS];
306 uint8_t divisors_present;
307 } b;
308 VkVertexInputBindingDescription2EXT dynbindings[PIPE_MAX_ATTRIBS];
309 };
310 uint8_t binding_map[PIPE_MAX_ATTRIBS];
311 };
312
313 struct zink_vertex_elements_state {
314 /* decomposed attributes read only a single component for format compatibility */
315 bool has_decomposed_attrs;
316 struct {
317 uint32_t binding;
318 VkVertexInputRate inputRate;
319 } bindings[PIPE_MAX_ATTRIBS];
320 uint32_t divisor[PIPE_MAX_ATTRIBS];
321 uint32_t min_stride[PIPE_MAX_ATTRIBS]; //for dynamic_state1
322 uint32_t decomposed_attrs;
323 unsigned decomposed_attrs_size;
324 uint32_t decomposed_attrs_without_w;
325 unsigned decomposed_attrs_without_w_size;
326 struct zink_vertex_elements_hw_state hw_state;
327 };
328
329 /* for vertex state draws */
330 struct zink_vertex_state {
331 struct pipe_vertex_state b;
332 struct zink_vertex_elements_state velems;
333 };
334
335 struct zink_rasterizer_hw_state {
336 unsigned polygon_mode : 2; //VkPolygonMode
337 unsigned line_mode : 2; //VkLineRasterizationModeEXT
338 unsigned depth_clip:1;
339 unsigned depth_clamp:1;
340 unsigned pv_last:1;
341 unsigned line_stipple_enable:1;
342 unsigned clip_halfz:1;
343 };
344
345 struct zink_rasterizer_state {
346 struct pipe_rasterizer_state base;
347 bool offset_fill;
348 float offset_units, offset_clamp, offset_scale;
349 float line_width;
350 VkFrontFace front_face;
351 VkCullModeFlags cull_mode;
352 VkLineRasterizationModeEXT dynamic_line_mode;
353 struct zink_rasterizer_hw_state hw_state;
354 };
355
356 struct zink_blend_state {
357 uint32_t hash;
358 unsigned num_rts;
359 VkPipelineColorBlendAttachmentState attachments[PIPE_MAX_COLOR_BUFS];
360
361 struct {
362 VkBool32 enables[PIPE_MAX_COLOR_BUFS];
363 VkColorBlendEquationEXT eq[PIPE_MAX_COLOR_BUFS];
364 VkColorComponentFlags wrmask[PIPE_MAX_COLOR_BUFS];
365 } ds3;
366
367 VkBool32 logicop_enable;
368 VkLogicOp logicop_func;
369
370 VkBool32 alpha_to_coverage;
371 VkBool32 alpha_to_one;
372
373 uint32_t wrmask;
374 uint8_t enables;
375
376 bool dual_src_blend;
377 };
378
379 struct zink_depth_stencil_alpha_hw_state {
380 VkBool32 depth_test;
381 VkCompareOp depth_compare_op;
382
383 VkBool32 depth_bounds_test;
384 float min_depth_bounds, max_depth_bounds;
385
386 VkBool32 stencil_test;
387 VkStencilOpState stencil_front;
388 VkStencilOpState stencil_back;
389
390 VkBool32 depth_write;
391 };
392
393 struct zink_depth_stencil_alpha_state {
394 struct pipe_depth_stencil_alpha_state base;
395 struct zink_depth_stencil_alpha_hw_state hw_state;
396 };
397
398
399 /** descriptor types */
400
401 /* zink_descriptor_layout objects are cached: this is the key for one */
402 struct zink_descriptor_layout_key {
403 unsigned num_bindings;
404 VkDescriptorSetLayoutBinding *bindings;
405 };
406
407 struct zink_descriptor_layout {
408 VkDescriptorSetLayout layout;
409 };
410
411 /* descriptor pools are cached: zink_descriptor_pool_key::id is the id for a type of pool */
412 struct zink_descriptor_pool_key {
413 unsigned use_count;
414 unsigned num_type_sizes;
415 unsigned id;
416 VkDescriptorPoolSize sizes[4];
417 struct zink_descriptor_layout_key *layout;
418 };
419
420 /* a template used for updating descriptor buffers */
421 struct zink_descriptor_template {
422 uint16_t stride; //the stride between mem pointers
423 uint16_t db_size; //the size of the entry in the buffer
424 unsigned count; //the number of descriptors
425 size_t offset; //the offset of the base host pointer to update from
426 };
427
428 /* ctx->dd; created at context creation */
429 struct zink_descriptor_data {
430 bool bindless_bound;
431 bool bindless_init;
432 bool has_fbfetch;
433 bool push_state_changed[2]; //gfx, compute
434 uint8_t state_changed[2]; //gfx, compute
435 struct zink_descriptor_layout_key *push_layout_keys[2]; //gfx, compute
436 struct zink_descriptor_layout *push_dsl[2]; //gfx, compute
437 VkDescriptorUpdateTemplate push_template[2]; //gfx, compute
438
439 struct zink_descriptor_layout *dummy_dsl;
440
441 union {
442 struct {
443 VkDescriptorPool bindless_pool;
444 VkDescriptorSet bindless_set;
445 } t;
446 struct {
447 struct zink_resource *bindless_db;
448 uint8_t *bindless_db_map;
449 struct pipe_transfer *bindless_db_xfer;
450 uint32_t bindless_db_offsets[4];
451 unsigned max_db_size;
452 } db;
453 };
454
455 struct zink_program *pg[2]; //gfx, compute
456
457 VkDescriptorUpdateTemplateEntry push_entries[MESA_SHADER_STAGES]; //gfx+fbfetch
458 VkDescriptorUpdateTemplateEntry compute_push_entry;
459
460 /* push descriptor layout size and binding offsets */
461 uint32_t db_size[2]; //gfx, compute
462 uint32_t db_offset[ZINK_GFX_SHADER_COUNT + 1]; //gfx + fbfetch
463 /* compute offset is always 0 */
464 };
465
466 /* pg->dd; created at program creation */
467 struct zink_program_descriptor_data {
468 bool bindless;
469 bool fbfetch;
470 /* bitmask of ubo0 usage for stages */
471 uint8_t push_usage;
472 /* bitmask of which sets are used by the program */
473 uint8_t binding_usage;
474 /* all the pool keys for the program */
475 struct zink_descriptor_pool_key *pool_key[ZINK_DESCRIPTOR_BASE_TYPES]; //push set doesn't need one
476 /* all the layouts for the program */
477 struct zink_descriptor_layout *layouts[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES];
478 /* all the templates for the program */
479 union {
480 VkDescriptorUpdateTemplate templates[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES];
481 struct zink_descriptor_template *db_template[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES];
482 };
483 uint32_t db_size[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //the total size of the layout
484 uint32_t *db_offset[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //the offset of each binding in the layout
485 };
486
487 struct zink_descriptor_pool {
488 /* the current index of 'sets' */
489 unsigned set_idx;
490 /* number of sets allocated */
491 unsigned sets_alloc;
492 VkDescriptorPool pool;
493 /* sets are lazily allocated */
494 VkDescriptorSet sets[MAX_LAZY_DESCRIPTORS];
495 };
496
497 /* a zink_descriptor_pool_key matches up to this struct */
498 struct zink_descriptor_pool_multi {
499 /* for flagging when overflowed pools must be destroyed instead of reused */
500 bool reinit_overflow;
501 /* this flips to split usable overflow from in-use overflow */
502 unsigned overflow_idx;
503 /* zink_descriptor_pool objects that have exceeded MAX_LAZY_DESCRIPTORS sets */
504 struct util_dynarray overflowed_pools[2];
505 /* the current pool; may be null */
506 struct zink_descriptor_pool *pool;
507 /* pool key for convenience */
508 const struct zink_descriptor_pool_key *pool_key;
509 };
510
511 /* bs->dd; created on batch state creation */
512 struct zink_batch_descriptor_data {
513 /* pools have fbfetch initialized */
514 bool has_fbfetch;
515 /* are descriptor buffers bound */
516 bool db_bound;
517 /* real size of 'pools' */
518 unsigned pool_size[ZINK_DESCRIPTOR_BASE_TYPES];
519 /* this array is sized based on the max zink_descriptor_pool_key::id used by the batch; members may be NULL */
520 struct util_dynarray pools[ZINK_DESCRIPTOR_BASE_TYPES];
521 struct zink_descriptor_pool_multi push_pool[2]; //gfx, compute
522 /* the current program (for descriptor updating) */
523 struct zink_program *pg[2]; //gfx, compute
524 /* the current pipeline compatibility id (for pipeline compatibility rules) */
525 uint32_t compat_id[2]; //gfx, compute
526 /* the current set layout */
527 VkDescriptorSetLayout dsl[2][ZINK_DESCRIPTOR_BASE_TYPES]; //gfx, compute
528 union {
529 /* the current set for a given type; used for rebinding if pipeline compat id changes and current set must be rebound */
530 VkDescriptorSet sets[2][ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //gfx, compute
531 uint64_t cur_db_offset[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //gfx, compute; the current offset of a descriptor buffer for rebinds
532 };
533 /* mask of push descriptor usage */
534 unsigned push_usage[2]; //gfx, compute
535
536 struct zink_resource *db; //the descriptor buffer for a given type
537 uint8_t *db_map; //the host map for the buffer
538 struct pipe_transfer *db_xfer; //the transfer map for the buffer
539 uint64_t db_offset; //the "next" offset that will be used when the buffer is updated
540 };
541
542 /** batch types */
543 /* zink_batch_usage concepts:
544 * - batch "usage" is an indicator of when and how a BO was accessed
545 * - batch "tracking" is the batch state(s) containing an extra ref for a BO
546 *
547 * - usage prevents a BO from being mapped while it has pending+conflicting access
548 * - usage affects pipeline barrier generation for synchronizing reads and writes
549 * - usage MUST be removed before context destruction to avoid crashing during BO
550 * reclaiming in suballocator
551 *
552 * - tracking prevents a BO from being destroyed early
553 * - tracking enables usage to be pruned
554 *
555 *
556 * tracking is added:
557 * - any time a BO is used in a "one-off" operation (e.g., blit, index buffer, indirect buffer)
558 * - any time a descriptor is unbound
559 * - when a buffer is replaced (IFF: resource is bound as a descriptor or usage previously existed)
560 *
561 * tracking is removed:
562 * - in zink_reset_batch_state()
563 *
564 * usage is added:
565 * - any time a BO is used in a "one-off" operation (e.g., blit, index buffer, indirect buffer)
566 * - any time a descriptor is bound
567 * - any time a descriptor is unbound (IFF: usage previously existed)
568 * - for all bound descriptors on the first draw/dispatch after a flush (zink_update_descriptor_refs)
569 *
570 * usage is removed:
571 * - when tracking is removed (IFF: BO usage == tracking, i.e., this is the last batch that a BO was active on)
572 */
573 struct zink_batch_usage {
574 uint32_t usage;
575 /* this is a monotonic int used to disambiguate internal fences from their tc fence references */
576 uint32_t submit_count;
577 cnd_t flush;
578 mtx_t mtx;
579 bool unflushed;
580 };
581
582 struct zink_bo_usage {
583 uint32_t submit_count;
584 struct zink_batch_usage *u;
585 };
586
587 struct zink_batch_obj_list {
588 unsigned max_buffers;
589 unsigned num_buffers;
590 struct zink_resource_object **objs;
591 };
592
593 struct zink_batch_state {
594 struct zink_fence fence;
595 struct zink_batch_state *next;
596
597 struct zink_batch_usage usage;
598 struct zink_context *ctx;
599 VkCommandPool cmdpool;
600 VkCommandBuffer cmdbuf;
601 VkCommandBuffer reordered_cmdbuf;
602 VkCommandPool unsynchronized_cmdpool;
603 VkCommandBuffer unsynchronized_cmdbuf;
604 VkSemaphore signal_semaphore; //external signal semaphore
605 struct util_dynarray signal_semaphores; //external signal semaphores
606 struct util_dynarray wait_semaphores; //external wait semaphores
607 struct util_dynarray wait_semaphore_stages; //external wait semaphores
608 struct util_dynarray fd_wait_semaphores; //dmabuf wait semaphores
609 struct util_dynarray fd_wait_semaphore_stages; //dmabuf wait semaphores
610 struct util_dynarray fences; //zink_tc_fence refs
611
612 VkSemaphore present;
613 struct zink_resource *swapchain;
614 struct util_dynarray acquires;
615 struct util_dynarray acquire_flags;
616
617 struct {
618 struct util_dynarray pipelines;
619 struct util_dynarray layouts;
620 } dgc;
621
622 VkAccessFlags unordered_write_access;
623 VkPipelineStageFlags unordered_write_stages;
624
625 simple_mtx_t exportable_lock;
626
627 struct util_queue_fence flush_completed;
628
629 struct set programs;
630 struct set dmabuf_exports;
631
632 #define BUFFER_HASHLIST_SIZE 32768
633 /* buffer_indices_hashlist[hash(bo)] returns -1 if the bo
634 * isn't part of any buffer lists or the index where the bo could be found.
635 * Since 1) hash collisions of 2 different bo can happen and 2) we use a
636 * single hashlist for the 3 buffer list, this is only a hint.
637 * batch_find_resource uses this hint to speed up buffers look up.
638 */
639 int16_t buffer_indices_hashlist[BUFFER_HASHLIST_SIZE];
640 struct zink_batch_obj_list real_objs;
641 struct zink_batch_obj_list slab_objs;
642 struct zink_batch_obj_list sparse_objs;
643 struct zink_resource_object *last_added_obj;
644 struct util_dynarray swapchain_obj; //this doesn't have a zink_bo and must be handled differently
645
646 struct util_dynarray unref_resources;
647 struct util_dynarray bindless_releases[2];
648
649 struct util_dynarray zombie_samplers;
650
651 struct set active_queries; /* zink_query objects which were active at some point in this batch */
652 struct util_dynarray dead_querypools;
653
654 struct util_dynarray freed_sparse_backing_bos;
655
656 struct zink_batch_descriptor_data dd;
657
658 VkDeviceSize resource_size;
659
660 bool is_device_lost;
661 bool has_barriers;
662 bool has_unsync;
663 };
664
665 static inline struct zink_batch_state *
zink_batch_state(struct zink_fence * fence)666 zink_batch_state(struct zink_fence *fence)
667 {
668 return (struct zink_batch_state *)fence;
669 }
670
671 struct zink_batch {
672 struct zink_batch_state *state;
673
674 struct zink_batch_usage *last_batch_usage;
675 struct zink_resource *swapchain;
676
677 unsigned work_count;
678
679 simple_mtx_t ref_lock;
680
681 bool has_work;
682 bool last_was_compute;
683 bool in_rp; //renderpass is currently active
684 };
685
686
687 /** bo types */
688 struct bo_export {
689 /** File descriptor associated with a handle export. */
690 int drm_fd;
691
692 /** GEM handle in drm_fd */
693 uint32_t gem_handle;
694
695 struct list_head link;
696 };
697
698 struct zink_bo {
699 struct pb_buffer base;
700
701 union {
702 struct {
703 void *cpu_ptr; /* for user_ptr and permanent maps */
704 int map_count;
705 struct list_head exports;
706 simple_mtx_t export_lock;
707
708 bool is_user_ptr;
709 bool use_reusable_pool;
710
711 /* Whether buffer_get_handle or buffer_from_handle has been called,
712 * it can only transition from false to true. Protected by lock.
713 */
714 bool is_shared;
715 } real;
716 struct {
717 struct pb_slab_entry entry;
718 struct zink_bo *real;
719 } slab;
720 struct {
721 uint32_t num_va_pages;
722 uint32_t num_backing_pages;
723
724 struct list_head backing;
725
726 /* Commitment information for each page of the virtual memory area. */
727 struct zink_sparse_commitment *commitments;
728 } sparse;
729 } u;
730
731 VkDeviceMemory mem;
732 uint64_t offset;
733
734 uint32_t unique_id;
735 const char *name;
736
737 simple_mtx_t lock;
738
739 struct zink_bo_usage reads;
740 struct zink_bo_usage writes;
741
742 struct pb_cache_entry cache_entry[];
743 };
744
745 static inline struct zink_bo *
zink_bo(struct pb_buffer * pbuf)746 zink_bo(struct pb_buffer *pbuf)
747 {
748 return (struct zink_bo*)pbuf;
749 }
750
751 /** clear types */
752 struct zink_framebuffer_clear_data {
753 union {
754 union pipe_color_union color;
755 struct {
756 float depth;
757 unsigned stencil;
758 uint8_t bits : 2; // PIPE_CLEAR_DEPTH, PIPE_CLEAR_STENCIL
759 } zs;
760 };
761 struct pipe_scissor_state scissor;
762 bool has_scissor;
763 bool conditional;
764 };
765
766 struct zink_framebuffer_clear {
767 struct util_dynarray clears;
768 };
769
770
771 /** compiler types */
772 struct zink_shader_info {
773 uint16_t stride[PIPE_MAX_SO_BUFFERS];
774 uint32_t sampler_mask;
775 bool have_sparse;
776 bool have_vulkan_memory_model;
777 bool have_workgroup_memory_explicit_layout;
778 struct {
779 uint8_t flush_denorms:3; // 16, 32, 64
780 uint8_t preserve_denorms:3; // 16, 32, 64
781 bool denorms_32_bit_independence:1;
782 bool denorms_all_independence:1;
783 } float_controls;
784 unsigned bindless_set_idx;
785 };
786
787 enum zink_rast_prim {
788 ZINK_PRIM_POINTS,
789 ZINK_PRIM_LINES,
790 ZINK_PRIM_TRIANGLES,
791 ZINK_PRIM_MAX,
792 };
793
794 struct zink_shader_object {
795 union {
796 VkShaderEXT obj;
797 VkShaderModule mod;
798 };
799 struct spirv_shader *spirv;
800 };
801
802 struct zink_shader {
803 struct util_live_shader base;
804 uint32_t hash;
805 struct blob blob;
806 struct shader_info info;
807
808 struct zink_shader_info sinfo;
809
810 struct {
811 int index;
812 int binding;
813 VkDescriptorType type;
814 unsigned char size;
815 } bindings[ZINK_DESCRIPTOR_BASE_TYPES][ZINK_MAX_DESCRIPTORS_PER_TYPE];
816 size_t num_bindings[ZINK_DESCRIPTOR_BASE_TYPES];
817 unsigned num_texel_buffers;
818 uint32_t ubos_used; // bitfield of which ubo indices are used
819 uint32_t ssbos_used; // bitfield of which ssbo indices are used
820 uint64_t flat_flags;
821 bool bindless;
822 bool can_inline;
823 bool has_uniforms;
824 bool has_edgeflags;
825 bool needs_inlining;
826 struct spirv_shader *spirv;
827
828 struct {
829 struct util_queue_fence fence;
830 struct zink_shader_object obj;
831 VkDescriptorSetLayout dsl;
832 VkPipelineLayout layout;
833 VkPipeline gpl;
834 VkDescriptorSetLayoutBinding *bindings;
835 unsigned num_bindings;
836 struct zink_descriptor_template *db_template;
837 unsigned db_size;
838 unsigned *db_offset;
839 } precompile;
840
841 simple_mtx_t lock;
842 struct set *programs;
843 struct util_dynarray pipeline_libs;
844
845 union {
846 struct {
847 struct zink_shader *generated_tcs; // a generated shader that this shader "owns"; only valid in the tes stage
848 struct zink_shader *generated_gs[MESA_PRIM_COUNT][ZINK_PRIM_MAX]; // generated shaders that this shader "owns"
849 struct zink_shader *parent; // for a generated gs this points to the shader that "owns" it
850
851 bool is_generated; // if this is a driver-created shader (e.g., tcs)
852 } non_fs;
853
854 struct {
855 /* Bitmask of textures that have shadow sampling result components
856 * other than RED accessed. This is a subset of !is_new_style_shadow
857 * (GLSL <1.30, ARB_fp) shadow sampling usage.
858 */
859 uint32_t legacy_shadow_mask;
860 nir_variable *fbfetch; //for fs output
861 } fs;
862 };
863 };
864
865
866 /** pipeline types */
867 struct zink_pipeline_dynamic_state1 {
868 uint8_t front_face; //VkFrontFace:1
869 uint8_t cull_mode; //VkCullModeFlags:2
870 uint16_t num_viewports;
871 struct zink_depth_stencil_alpha_hw_state *depth_stencil_alpha_state; //must be last
872 };
873
874 struct zink_pipeline_dynamic_state2 {
875 bool primitive_restart;
876 bool rasterizer_discard;
877 uint16_t vertices_per_patch; //5 bits
878 };
879
880 #define zink_pipeline_dynamic_state3 zink_rasterizer_hw_state
881
882 struct zink_gfx_pipeline_state {
883 /* order matches zink_gfx_output_key */
884 unsigned force_persample_interp:1;
885 uint32_t rast_samples:6;
886 uint32_t min_samples:6;
887 uint32_t feedback_loop : 1;
888 uint32_t feedback_loop_zs : 1;
889 uint32_t rast_attachment_order : 1;
890 uint32_t rp_state : 16;
891 VkSampleMask sample_mask;
892 uint32_t blend_id;
893
894 /* Pre-hashed value for table lookup, invalid when zero.
895 * Members after this point are not included in pipeline state hash key */
896 uint32_t hash;
897 bool dirty;
898
899 struct zink_pipeline_dynamic_state1 dyn_state1;
900
901 struct zink_pipeline_dynamic_state2 dyn_state2;
902 struct zink_pipeline_dynamic_state3 dyn_state3;
903
904 union {
905 VkShaderModule modules[MESA_SHADER_STAGES - 1];
906 uint32_t optimal_key;
907 };
908 bool modules_changed;
909
910 uint32_t vertex_hash;
911
912 uint32_t final_hash;
913
914 uint32_t _pad2;
915 /* order matches zink_gfx_input_key */
916 union {
917 struct {
918 unsigned idx:8;
919 bool uses_dynamic_stride;
920 };
921 uint32_t input;
922 };
923 uint32_t vertex_buffers_enabled_mask;
924 uint32_t vertex_strides[PIPE_MAX_ATTRIBS];
925 struct zink_vertex_elements_hw_state *element_state;
926 struct zink_zs_swizzle_key *shadow;
927 bool sample_locations_enabled;
928 enum mesa_prim shader_rast_prim, rast_prim; /* reduced type or max for unknown */
929 union {
930 struct {
931 struct zink_shader_key key[5];
932 struct zink_shader_key last_vertex;
933 } shader_keys;
934 struct {
935 union zink_shader_key_optimal key;
936 } shader_keys_optimal;
937 };
938 struct zink_blend_state *blend_state;
939 struct zink_render_pass *render_pass;
940 struct zink_render_pass *next_render_pass; //will be used next time rp is begun
941 VkFormat rendering_formats[PIPE_MAX_COLOR_BUFS];
942 VkPipelineRenderingCreateInfo rendering_info;
943 VkPipeline pipeline;
944 enum mesa_prim gfx_prim_mode; //pending mode
945 };
946
947 struct zink_compute_pipeline_state {
948 /* Pre-hashed value for table lookup, invalid when zero.
949 * Members after this point are not included in pipeline state hash key */
950 uint32_t hash;
951 uint32_t final_hash;
952 bool dirty;
953 uint32_t local_size[3];
954 uint32_t variable_shared_mem;
955
956 uint32_t module_hash;
957 VkShaderModule module;
958 bool module_changed;
959
960 struct zink_shader_key key;
961
962 VkPipeline pipeline;
963 };
964
965
966 /** program types */
967
968 /* create_gfx_pushconst must be kept in sync with this struct */
969 struct zink_gfx_push_constant {
970 unsigned draw_mode_is_indexed;
971 unsigned draw_id;
972 unsigned framebuffer_is_layered;
973 float default_inner_level[2];
974 float default_outer_level[4];
975 uint32_t line_stipple_pattern;
976 float viewport_scale[2];
977 float line_width;
978 };
979
980 /* The order of the enums MUST match the order of the zink_gfx_push_constant
981 * members.
982 */
983 enum zink_gfx_push_constant_member {
984 ZINK_GFX_PUSHCONST_DRAW_MODE_IS_INDEXED,
985 ZINK_GFX_PUSHCONST_DRAW_ID,
986 ZINK_GFX_PUSHCONST_FRAMEBUFFER_IS_LAYERED,
987 ZINK_GFX_PUSHCONST_DEFAULT_INNER_LEVEL,
988 ZINK_GFX_PUSHCONST_DEFAULT_OUTER_LEVEL,
989 ZINK_GFX_PUSHCONST_LINE_STIPPLE_PATTERN,
990 ZINK_GFX_PUSHCONST_VIEWPORT_SCALE,
991 ZINK_GFX_PUSHCONST_LINE_WIDTH,
992 ZINK_GFX_PUSHCONST_MAX
993 };
994
995 /* a shader module is used for directly reusing a shader module between programs,
996 * e.g., in the case where we're swapping out only one shader,
997 * allowing us to skip going through shader keys
998 */
999 struct zink_shader_module {
1000 struct zink_shader_object obj;
1001 uint32_t hash;
1002 bool shobj;
1003 bool default_variant;
1004 bool has_nonseamless;
1005 bool needs_zs_shader_swizzle;
1006 uint8_t num_uniforms;
1007 uint8_t key_size;
1008 uint8_t key[0]; /* | key | uniforms | zs shader swizzle | */
1009 };
1010
1011 struct zink_program {
1012 struct pipe_reference reference;
1013 struct zink_context *ctx;
1014 unsigned char sha1[20];
1015 struct util_queue_fence cache_fence;
1016 struct u_rwlock pipeline_cache_lock;
1017 VkPipelineCache pipeline_cache;
1018 size_t pipeline_cache_size;
1019 struct zink_batch_usage *batch_uses;
1020 bool is_compute;
1021 bool can_precompile;
1022 bool uses_shobj; //whether shader objects are used; programs CANNOT mix shader objects and shader modules
1023
1024 struct zink_program_descriptor_data dd;
1025
1026 uint32_t compat_id;
1027 VkPipelineLayout layout;
1028 VkDescriptorSetLayout dsl[ZINK_DESCRIPTOR_ALL_TYPES]; // one for each type + push + bindless
1029 unsigned num_dsl;
1030
1031 bool removed;
1032 };
1033
1034 #define STAGE_MASK_OPTIMAL (1<<16)
1035 #define STAGE_MASK_OPTIMAL_SHADOW (1<<17)
1036 typedef bool (*equals_gfx_pipeline_state_func)(const void *a, const void *b);
1037
1038 struct zink_gfx_library_key {
1039 uint32_t optimal_key; //equals_pipeline_lib_optimal
1040 VkShaderModule modules[ZINK_GFX_SHADER_COUNT];
1041 VkPipeline pipeline;
1042 };
1043
1044 struct zink_gfx_input_key {
1045 union {
1046 struct {
1047 unsigned idx:8;
1048 bool uses_dynamic_stride;
1049 };
1050 uint32_t input;
1051 };
1052 uint32_t vertex_buffers_enabled_mask;
1053 uint32_t vertex_strides[PIPE_MAX_ATTRIBS];
1054 struct zink_vertex_elements_hw_state *element_state;
1055 VkPipeline pipeline;
1056 };
1057
1058 struct zink_gfx_output_key {
1059 /* order matches zink_gfx_output_key */
1060 union {
1061 struct {
1062 unsigned force_persample_interp:1;
1063 uint32_t rast_samples:6;
1064 uint32_t min_samples:6;
1065 uint32_t feedback_loop : 1;
1066 uint32_t feedback_loop_zs : 1;
1067 uint32_t rast_attachment_order : 1;
1068 uint32_t rp_state : 16;
1069 };
1070 uint32_t key;
1071 };
1072
1073 /* TODO: compress these */
1074 VkSampleMask sample_mask;
1075 uint32_t blend_id;
1076 VkPipeline pipeline;
1077 };
1078
1079 struct zink_gfx_pipeline_cache_entry {
1080 struct zink_gfx_pipeline_state state;
1081 VkPipeline pipeline;
1082 struct zink_gfx_program *prog;
1083 /* GPL only */
1084 struct util_queue_fence fence;
1085 union {
1086 struct {
1087 struct zink_gfx_input_key *ikey;
1088 struct zink_gfx_library_key *gkey;
1089 struct zink_gfx_output_key *okey;
1090 VkPipeline unoptimized_pipeline;
1091 } gpl;
1092 struct zink_shader_object shobjs[ZINK_GFX_SHADER_COUNT];
1093 };
1094 };
1095
1096 struct zink_gfx_lib_cache {
1097 /* for hashing */
1098 struct zink_shader *shaders[ZINK_GFX_SHADER_COUNT];
1099 unsigned refcount;
1100 bool removed; //once removed from cache
1101 uint8_t stages_present;
1102
1103 simple_mtx_t lock;
1104 struct set libs; //zink_gfx_library_key -> VkPipeline
1105 };
1106
1107 struct zink_gfx_program {
1108 struct zink_program base;
1109
1110 bool is_separable; //not a full program
1111 struct zink_context *ctx; //the owner context
1112
1113 uint32_t stages_present; //mask of stages present in this program
1114 uint32_t stages_remaining; //mask of zink_shader remaining in this program
1115 uint32_t gfx_hash; //from ctx->gfx_hash
1116
1117 struct zink_shader *shaders[ZINK_GFX_SHADER_COUNT];
1118 struct zink_shader *last_vertex_stage;
1119 struct zink_shader_object objs[ZINK_GFX_SHADER_COUNT];
1120
1121 /* full */
1122 VkShaderEXT objects[ZINK_GFX_SHADER_COUNT];
1123 uint32_t module_hash[ZINK_GFX_SHADER_COUNT];
1124 struct blob blobs[ZINK_GFX_SHADER_COUNT];
1125 struct util_dynarray shader_cache[ZINK_GFX_SHADER_COUNT][2][2]; //normal, nonseamless cubes, inline uniforms
1126 unsigned inlined_variant_count[ZINK_GFX_SHADER_COUNT];
1127 uint32_t default_variant_hash;
1128 uint8_t inline_variants; //which stages are using inlined uniforms
1129 bool needs_inlining; // whether this program requires some uniforms to be inlined
1130 bool has_edgeflags;
1131 bool optimal_keys;
1132
1133 /* separable */
1134 struct zink_gfx_program *full_prog;
1135
1136 struct hash_table pipelines[2][11]; // [dynamic, renderpass][number of draw modes we support]
1137 uint32_t last_variant_hash;
1138
1139 uint32_t last_finalized_hash[2][4]; //[dynamic, renderpass][primtype idx]
1140 VkPipeline last_pipeline[2][4]; //[dynamic, renderpass][primtype idx]
1141
1142 struct zink_gfx_lib_cache *libs;
1143 };
1144
1145 struct zink_compute_program {
1146 struct zink_program base;
1147
1148 bool use_local_size;
1149 bool has_variable_shared_mem;
1150
1151 unsigned scratch_size;
1152
1153 unsigned num_inlinable_uniforms;
1154 nir_shader *nir; //only until precompile finishes
1155
1156 struct zink_shader_module *curr;
1157
1158 struct zink_shader_module *module; //base
1159 struct util_dynarray shader_cache[2]; //nonseamless cubes, inline uniforms
1160 unsigned inlined_variant_count;
1161
1162 struct zink_shader *shader;
1163 struct hash_table pipelines;
1164
1165 simple_mtx_t cache_lock; //extra lock because threads are insane and sand was not meant to think
1166
1167 VkPipeline base_pipeline;
1168 };
1169
1170
1171 /** renderpass types */
1172
1173 struct zink_rt_attrib {
1174 VkFormat format;
1175 VkSampleCountFlagBits samples;
1176 bool clear_color;
1177 union {
1178 bool clear_stencil;
1179 bool fbfetch;
1180 };
1181 bool invalid;
1182 bool needs_write;
1183 bool resolve;
1184 bool feedback_loop;
1185 };
1186
1187 struct zink_render_pass_state {
1188 union {
1189 struct {
1190 uint8_t num_cbufs : 5; /* PIPE_MAX_COLOR_BUFS = 8 */
1191 uint8_t have_zsbuf : 1;
1192 uint8_t samples:1; //for fs samplemask
1193 uint32_t num_zsresolves : 1;
1194 uint32_t num_cresolves : 24; /* PIPE_MAX_COLOR_BUFS, but this is a struct hole */
1195 };
1196 uint32_t val; //for comparison
1197 };
1198 struct zink_rt_attrib rts[PIPE_MAX_COLOR_BUFS + 1];
1199 unsigned num_rts;
1200 uint32_t clears; //for extra verification and update flagging
1201 uint16_t msaa_expand_mask;
1202 uint16_t msaa_samples; //used with VK_EXT_multisampled_render_to_single_sampled
1203 };
1204
1205 struct zink_pipeline_rt {
1206 VkFormat format;
1207 VkSampleCountFlagBits samples;
1208 };
1209
1210 struct zink_render_pass_pipeline_state {
1211 uint32_t num_attachments:14;
1212 uint32_t msaa_samples : 8;
1213 uint32_t fbfetch:1;
1214 uint32_t color_read:1;
1215 uint32_t depth_read:1;
1216 uint32_t depth_write:1;
1217 uint32_t num_cresolves:4;
1218 uint32_t num_zsresolves:1;
1219 bool samples:1; //for fs samplemask
1220 struct zink_pipeline_rt attachments[PIPE_MAX_COLOR_BUFS + 1];
1221 unsigned id;
1222 };
1223
1224 struct zink_render_pass {
1225 VkRenderPass render_pass;
1226 struct zink_render_pass_state state;
1227 unsigned pipeline_state;
1228 };
1229
1230
1231 /** resource types */
1232 struct zink_resource_object {
1233 struct pipe_reference reference;
1234
1235 VkPipelineStageFlags access_stage;
1236 VkAccessFlags access;
1237 VkPipelineStageFlags unordered_access_stage;
1238 VkAccessFlags unordered_access;
1239 VkAccessFlags last_write;
1240
1241 /* 'access' is propagated from unordered_access to handle ops occurring
1242 * in the ordered cmdbuf which can promote barriers to unordered
1243 */
1244 bool ordered_access_is_copied;
1245 bool unordered_read;
1246 bool unordered_write;
1247 bool unsync_access;
1248 bool copies_valid;
1249 bool copies_need_reset; //for use with batch state resets
1250
1251 struct u_rwlock copy_lock;
1252 struct util_dynarray copies[16]; //regions being copied to; for barrier omission
1253
1254 VkBuffer storage_buffer;
1255 simple_mtx_t view_lock;
1256 uint32_t view_prune_count; //how many views to prune
1257 uint32_t view_prune_timeline; //when to prune
1258 struct util_dynarray views;
1259
1260 union {
1261 VkBuffer buffer;
1262 VkImage image;
1263 };
1264 VkDeviceAddress bda;
1265
1266 VkSampleLocationsInfoEXT zs_evaluate;
1267 bool needs_zs_evaluate;
1268
1269 bool storage_init; //layout was set for image
1270 bool transfer_dst;
1271 bool render_target;
1272 bool is_buffer;
1273 bool exportable;
1274
1275 /* TODO: this should be a union */
1276 int handle;
1277 struct zink_bo *bo;
1278 // struct {
1279 struct kopper_displaytarget *dt;
1280 uint32_t dt_idx;
1281 uint32_t last_dt_idx;
1282 VkSemaphore present;
1283 bool new_dt;
1284 bool indefinite_acquire;
1285 // }
1286
1287
1288 VkDeviceSize offset, size, alignment;
1289 uint64_t vkflags;
1290 uint64_t vkusage;
1291 VkFormatFeatureFlags vkfeats;
1292 uint64_t modifier;
1293 VkImageAspectFlags modifier_aspect;
1294 VkSamplerYcbcrConversion sampler_conversion;
1295 unsigned plane_offsets[3];
1296 unsigned plane_strides[3];
1297 unsigned plane_count;
1298
1299 bool host_visible;
1300 bool coherent;
1301 bool is_aux;
1302 };
1303
1304 struct zink_resource {
1305 struct threaded_resource base;
1306
1307 enum pipe_format internal_format:16;
1308
1309 struct zink_resource_object *obj;
1310 uint32_t queue;
1311 union {
1312 struct {
1313 struct util_range valid_buffer_range;
1314 uint32_t vbo_bind_mask : PIPE_MAX_ATTRIBS;
1315 uint8_t ubo_bind_count[2];
1316 uint8_t ssbo_bind_count[2];
1317 uint8_t vbo_bind_count;
1318 uint8_t so_bind_count; //not counted in all_binds
1319 bool so_valid;
1320 uint32_t ubo_bind_mask[MESA_SHADER_STAGES];
1321 uint32_t ssbo_bind_mask[MESA_SHADER_STAGES];
1322 };
1323 struct {
1324 bool linear;
1325 bool need_2D;
1326 bool valid;
1327 uint8_t fb_bind_count; //not counted in all_binds
1328 uint16_t fb_binds; /* mask of attachment idx; zs is PIPE_MAX_COLOR_BUFS */
1329 VkSparseImageMemoryRequirements sparse;
1330 VkFormat format;
1331 VkImageLayout layout;
1332 VkImageAspectFlags aspect;
1333 };
1334 };
1335 uint32_t sampler_binds[MESA_SHADER_STAGES];
1336 uint32_t image_binds[MESA_SHADER_STAGES];
1337 uint16_t sampler_bind_count[2]; //gfx, compute
1338 uint16_t image_bind_count[2]; //gfx, compute
1339 uint16_t write_bind_count[2]; //gfx, compute
1340 union {
1341 uint16_t bindless[2]; //tex, img
1342 uint32_t all_bindless;
1343 };
1344 union {
1345 uint16_t bind_count[2]; //gfx, compute
1346 uint32_t all_binds;
1347 };
1348
1349 VkPipelineStageFlagBits gfx_barrier;
1350 VkAccessFlagBits barrier_access[2]; //gfx, compute
1351
1352 union {
1353 struct {
1354 struct hash_table bufferview_cache;
1355 simple_mtx_t bufferview_mtx;
1356 };
1357 struct {
1358 struct hash_table surface_cache;
1359 simple_mtx_t surface_mtx;
1360 };
1361 };
1362
1363 bool copies_warned;
1364 bool swapchain;
1365 bool dmabuf;
1366 unsigned dt_stride;
1367
1368 uint8_t modifiers_count;
1369 uint64_t *modifiers;
1370 };
1371
1372 static inline struct zink_resource *
zink_resource(struct pipe_resource * r)1373 zink_resource(struct pipe_resource *r)
1374 {
1375 return (struct zink_resource *)r;
1376 }
1377
1378
1379 struct zink_transfer {
1380 struct threaded_transfer base;
1381 struct pipe_resource *staging_res;
1382 unsigned offset;
1383 unsigned depthPitch;
1384 };
1385
1386
1387 /** screen types */
1388 struct zink_modifier_prop {
1389 uint32_t drmFormatModifierCount;
1390 VkDrmFormatModifierPropertiesEXT* pDrmFormatModifierProperties;
1391 };
1392
1393 struct zink_format_props {
1394 VkFormatFeatureFlags2 linearTilingFeatures;
1395 VkFormatFeatureFlags2 optimalTilingFeatures;
1396 VkFormatFeatureFlags2 bufferFeatures;
1397 };
1398
1399 struct zink_screen {
1400 struct pipe_screen base;
1401
1402 struct util_dl_library *loader_lib;
1403 PFN_vkGetInstanceProcAddr vk_GetInstanceProcAddr;
1404 PFN_vkGetDeviceProcAddr vk_GetDeviceProcAddr;
1405
1406 bool threaded;
1407 bool threaded_submit;
1408 bool is_cpu;
1409 bool abort_on_hang;
1410 bool frame_marker_emitted;
1411 uint64_t curr_batch; //the current batch id
1412 uint32_t last_finished;
1413 VkSemaphore sem;
1414 VkFence fence;
1415 struct util_queue flush_queue;
1416 simple_mtx_t copy_context_lock;
1417 struct zink_context *copy_context;
1418
1419 struct zink_batch_state *free_batch_states; //unused batch states
1420 struct zink_batch_state *last_free_batch_state; //for appending
1421 simple_mtx_t free_batch_states_lock;
1422
1423 simple_mtx_t semaphores_lock;
1424 struct util_dynarray semaphores;
1425 struct util_dynarray fd_semaphores;
1426
1427 unsigned buffer_rebind_counter;
1428 unsigned image_rebind_counter;
1429 unsigned robust_ctx_count;
1430
1431 struct hash_table dts;
1432 simple_mtx_t dt_lock;
1433
1434 bool device_lost;
1435 int drm_fd;
1436
1437 struct slab_parent_pool transfer_pool;
1438 struct disk_cache *disk_cache;
1439 struct util_queue cache_put_thread;
1440 struct util_queue cache_get_thread;
1441
1442 /* there are 5 gfx stages, but VS and FS are assumed to be always present,
1443 * thus only 3 stages need to be considered, giving 2^3 = 8 program caches.
1444 */
1445 struct set pipeline_libs[8];
1446 simple_mtx_t pipeline_libs_lock[8];
1447
1448 simple_mtx_t desc_set_layouts_lock;
1449 struct hash_table desc_set_layouts[ZINK_DESCRIPTOR_BASE_TYPES];
1450 simple_mtx_t desc_pool_keys_lock;
1451 struct set desc_pool_keys[ZINK_DESCRIPTOR_BASE_TYPES];
1452 struct util_live_shader_cache shaders;
1453
1454 uint64_t db_size[ZINK_DESCRIPTOR_ALL_TYPES];
1455 unsigned base_descriptor_size;
1456 VkDescriptorSetLayout bindless_layout;
1457
1458 struct {
1459 struct pb_cache bo_cache;
1460 struct pb_slabs bo_slabs[NUM_SLAB_ALLOCATORS];
1461 unsigned min_alloc_size;
1462 uint32_t next_bo_unique_id;
1463 } pb;
1464 uint8_t heap_map[ZINK_HEAP_MAX][VK_MAX_MEMORY_TYPES]; // mapping from zink heaps to memory type indices
1465 uint8_t heap_count[ZINK_HEAP_MAX]; // number of memory types per zink heap
1466 bool resizable_bar;
1467
1468 uint64_t total_video_mem;
1469 uint64_t clamp_video_mem;
1470 uint64_t total_mem;
1471 uint64_t mapped_vram;
1472
1473 VkInstance instance;
1474 struct zink_instance_info instance_info;
1475
1476 struct hash_table *debug_mem_sizes;
1477 simple_mtx_t debug_mem_lock;
1478
1479 VkPhysicalDevice pdev;
1480 uint32_t vk_version, spirv_version;
1481 struct util_idalloc_mt buffer_ids;
1482 struct util_vertex_state_cache vertex_state_cache;
1483
1484 struct zink_device_info info;
1485 struct nir_shader_compiler_options nir_options;
1486
1487 bool optimal_keys;
1488 bool have_full_ds3;
1489 bool have_X8_D24_UNORM_PACK32;
1490 bool have_D24_UNORM_S8_UINT;
1491 bool have_D32_SFLOAT_S8_UINT;
1492 bool have_triangle_fans;
1493 bool need_decompose_attrs;
1494 bool need_2D_zs;
1495 bool need_2D_sparse;
1496 bool faked_e5sparse; //drivers may not expose R9G9B9E5 but cts requires it
1497 bool can_hic_shader_read;
1498
1499 uint32_t gfx_queue;
1500 uint32_t sparse_queue;
1501 uint32_t max_queues;
1502 uint32_t timestamp_valid_bits;
1503 VkDevice dev;
1504 VkQueue queue; //gfx+compute
1505 VkQueue queue_sparse;
1506 simple_mtx_t queue_lock;
1507 VkDebugUtilsMessengerEXT debugUtilsCallbackHandle;
1508
1509 uint32_t cur_custom_border_color_samplers;
1510
1511 unsigned screen_id;
1512
1513 #ifdef HAVE_RENDERDOC_APP_H
1514 RENDERDOC_API_1_0_0 *renderdoc_api;
1515 unsigned renderdoc_capture_start;
1516 unsigned renderdoc_capture_end;
1517 unsigned renderdoc_frame;
1518 bool renderdoc_capturing;
1519 bool renderdoc_capture_all;
1520 #endif
1521
1522 struct vk_dispatch_table vk;
1523
1524 void (*buffer_barrier)(struct zink_context *ctx, struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline);
1525 void (*image_barrier)(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline);
1526 void (*image_barrier_unsync)(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline);
1527
1528 bool compact_descriptors; /**< toggled if descriptor set ids are compacted */
1529 uint8_t desc_set_id[ZINK_MAX_DESCRIPTOR_SETS]; /**< converts enum zink_descriptor_type -> the actual set id */
1530
1531 struct {
1532 bool dual_color_blend_by_location;
1533 bool inline_uniforms;
1534 bool emulate_point_smooth;
1535 bool zink_shader_object_enable;
1536 } driconf;
1537
1538 struct zink_format_props format_props[PIPE_FORMAT_COUNT];
1539 struct zink_modifier_prop modifier_props[PIPE_FORMAT_COUNT];
1540
1541 VkExtent2D maxSampleLocationGridSize[5];
1542 VkPipelineLayout gfx_push_constant_layout;
1543
1544 struct {
1545 bool broken_l4a4;
1546 /* https://gitlab.khronos.org/vulkan/vulkan/-/issues/3306
1547 * HI TURNIP
1548 */
1549 bool broken_cache_semantics;
1550 bool missing_a8_unorm;
1551 bool implicit_sync;
1552 bool disable_optimized_compile;
1553 bool always_feedback_loop;
1554 bool always_feedback_loop_zs;
1555 bool needs_sanitised_layer;
1556 bool track_renderpasses;
1557 bool no_linestipple;
1558 bool no_linesmooth;
1559 bool no_hw_gl_point;
1560 bool lower_robustImageAccess2;
1561 bool needs_zs_shader_swizzle;
1562 bool can_do_invalid_linear_modifier;
1563 unsigned z16_unscaled_bias;
1564 unsigned z24_unscaled_bias;
1565 } driver_workarounds;
1566 };
1567
1568 static inline struct zink_screen *
zink_screen(struct pipe_screen * pipe)1569 zink_screen(struct pipe_screen *pipe)
1570 {
1571 return (struct zink_screen *)pipe;
1572 }
1573
1574 /** surface types */
1575
1576 /* info for validating/creating imageless framebuffers */
1577 struct zink_surface_info {
1578 VkImageCreateFlags flags;
1579 VkImageUsageFlags usage;
1580 uint32_t width;
1581 uint32_t height;
1582 uint32_t layerCount;
1583 VkFormat format[2]; //base format, srgb format (for srgb framebuffer)
1584 };
1585
1586 /* an imageview for a zink_resource:
1587 - may be a fb attachment, samplerview, or shader image
1588 - cached on the parent zink_resource_object
1589 - also handles swapchains
1590 */
1591 struct zink_surface {
1592 struct pipe_surface base;
1593 /* all the info for creating a new imageview */
1594 VkImageViewCreateInfo ivci;
1595 VkImageViewUsageCreateInfo usage_info;
1596 /* for framebuffer use */
1597 struct zink_surface_info info;
1598 bool is_swapchain;
1599 /* the current imageview */
1600 VkImageView image_view;
1601 /* array of imageviews for swapchains, one for each image */
1602 VkImageView *swapchain;
1603 unsigned swapchain_size;
1604 void *obj; //backing resource object; used to determine rebinds
1605 void *dt_swapchain; //current swapchain object; used to determine swapchain rebinds
1606 uint32_t hash; //for surface caching
1607 };
1608
1609 /* wrapper object that preserves the gallium expectation of having
1610 * pipe_surface::context match the context used to create the surface
1611 */
1612 struct zink_ctx_surface {
1613 struct pipe_surface base;
1614 struct zink_surface *surf; //the actual surface
1615 struct zink_ctx_surface *transient; //for use with EXT_multisample_render_to_texture
1616 bool transient_init; //whether the transient surface has data
1617 bool needs_mutable;
1618 };
1619
1620 /* use this cast for framebuffer surfaces */
1621 static inline struct zink_surface *
zink_csurface(struct pipe_surface * psurface)1622 zink_csurface(struct pipe_surface *psurface)
1623 {
1624 return psurface ? ((struct zink_ctx_surface *)psurface)->surf : NULL;
1625 }
1626
1627 /* use this cast for checking transient framebuffer surfaces */
1628 static inline struct zink_surface *
zink_transient_surface(struct pipe_surface * psurface)1629 zink_transient_surface(struct pipe_surface *psurface)
1630 {
1631 return psurface ? ((struct zink_ctx_surface *)psurface)->transient ? ((struct zink_ctx_surface *)psurface)->transient->surf : NULL : NULL;
1632 }
1633
1634 /* use this cast for internal surfaces */
1635 static inline struct zink_surface *
zink_surface(struct pipe_surface * psurface)1636 zink_surface(struct pipe_surface *psurface)
1637 {
1638 return (struct zink_surface *)psurface;
1639 }
1640
1641
1642 /** framebuffer types */
1643 struct zink_framebuffer_state {
1644 uint32_t width;
1645 uint16_t height;
1646 uint32_t layers:6;
1647 uint32_t samples:6;
1648 uint32_t num_attachments:4;
1649 struct zink_surface_info infos[PIPE_MAX_COLOR_BUFS + 1];
1650 };
1651
1652 struct zink_framebuffer {
1653 struct pipe_reference reference;
1654
1655 /* current objects */
1656 VkFramebuffer fb;
1657 struct zink_render_pass *rp;
1658
1659 struct zink_framebuffer_state state;
1660 VkFramebufferAttachmentImageInfo infos[PIPE_MAX_COLOR_BUFS + 1];
1661 struct hash_table objects;
1662 };
1663
1664
1665 /** context types */
1666 struct zink_sampler_state {
1667 VkSampler sampler;
1668 VkSampler sampler_clamped;
1669 bool custom_border_color;
1670 bool emulate_nonseamless;
1671 };
1672
1673 struct zink_buffer_view {
1674 struct pipe_reference reference;
1675 struct pipe_resource *pres;
1676 VkBufferViewCreateInfo bvci;
1677 VkBufferView buffer_view;
1678 uint32_t hash;
1679 };
1680
1681 struct zink_sampler_view {
1682 struct pipe_sampler_view base;
1683 union {
1684 struct zink_surface *image_view;
1685 struct zink_buffer_view *buffer_view;
1686 unsigned tbo_size;
1687 };
1688 struct zink_surface *cube_array;
1689 /* Optional sampler view returning red (depth) in all channels, for shader rewrites. */
1690 struct zink_surface *zs_view;
1691 struct zink_zs_swizzle swizzle;
1692 };
1693
1694 struct zink_image_view {
1695 struct pipe_image_view base;
1696 union {
1697 struct zink_surface *surface;
1698 struct zink_buffer_view *buffer_view;
1699 };
1700 };
1701
1702 static inline struct zink_sampler_view *
zink_sampler_view(struct pipe_sampler_view * pview)1703 zink_sampler_view(struct pipe_sampler_view *pview)
1704 {
1705 return (struct zink_sampler_view *)pview;
1706 }
1707
1708 struct zink_so_target {
1709 struct pipe_stream_output_target base;
1710 struct pipe_resource *counter_buffer;
1711 VkDeviceSize counter_buffer_offset;
1712 uint32_t stride;
1713 bool counter_buffer_valid;
1714 };
1715
1716 static inline struct zink_so_target *
zink_so_target(struct pipe_stream_output_target * so_target)1717 zink_so_target(struct pipe_stream_output_target *so_target)
1718 {
1719 return (struct zink_so_target *)so_target;
1720 }
1721
1722 struct zink_viewport_state {
1723 struct pipe_viewport_state viewport_states[PIPE_MAX_VIEWPORTS];
1724 struct pipe_scissor_state scissor_states[PIPE_MAX_VIEWPORTS];
1725 uint8_t num_viewports;
1726 };
1727
1728 struct zink_descriptor_db_info {
1729 unsigned offset;
1730 unsigned size;
1731 enum pipe_format format;
1732 struct pipe_resource *pres;
1733 };
1734
1735 struct zink_descriptor_surface {
1736 union {
1737 struct zink_surface *surface;
1738 struct zink_buffer_view *bufferview;
1739 struct zink_descriptor_db_info db;
1740 };
1741 bool is_buffer;
1742 };
1743
1744 struct zink_bindless_descriptor {
1745 struct zink_descriptor_surface ds;
1746 struct zink_sampler_state *sampler;
1747 uint32_t handle;
1748 uint32_t access; //PIPE_ACCESS_...
1749 };
1750
1751 struct zink_rendering_info {
1752 VkPipelineRenderingCreateInfo info;
1753 unsigned id;
1754 };
1755
1756
1757 typedef void (*pipe_draw_vertex_state_func)(struct pipe_context *ctx,
1758 struct pipe_vertex_state *vstate,
1759 uint32_t partial_velem_mask,
1760 struct pipe_draw_vertex_state_info info,
1761 const struct pipe_draw_start_count_bias *draws,
1762 unsigned num_draws);
1763 typedef void (*pipe_launch_grid_func)(struct pipe_context *pipe, const struct pipe_grid_info *info);
1764
1765
1766 enum zink_ds3_state {
1767 ZINK_DS3_RAST_STIPPLE,
1768 ZINK_DS3_RAST_CLIP,
1769 ZINK_DS3_RAST_CLAMP,
1770 ZINK_DS3_RAST_POLYGON,
1771 ZINK_DS3_RAST_HALFZ,
1772 ZINK_DS3_RAST_PV,
1773 ZINK_DS3_RAST_LINE,
1774 ZINK_DS3_RAST_STIPPLE_ON,
1775 ZINK_DS3_BLEND_A2C,
1776 ZINK_DS3_BLEND_A21,
1777 ZINK_DS3_BLEND_ON,
1778 ZINK_DS3_BLEND_WRITE,
1779 ZINK_DS3_BLEND_EQ,
1780 ZINK_DS3_BLEND_LOGIC_ON,
1781 ZINK_DS3_BLEND_LOGIC,
1782 };
1783
1784 struct zink_context {
1785 struct pipe_context base;
1786 struct threaded_context *tc;
1787 struct slab_child_pool transfer_pool;
1788 struct slab_child_pool transfer_pool_unsync;
1789 struct blitter_context *blitter;
1790 struct util_debug_callback dbg;
1791
1792 unsigned flags;
1793
1794 pipe_draw_func draw_vbo[2]; //batch changed
1795 pipe_draw_vertex_state_func draw_state[2]; //batch changed
1796 pipe_launch_grid_func launch_grid[2]; //batch changed
1797
1798 struct pipe_device_reset_callback reset;
1799
1800 struct util_queue_fence unsync_fence; //unsigned during unsync recording (blocks flush ops)
1801 struct util_queue_fence flush_fence; //unsigned during flush (blocks unsync ops)
1802
1803 struct zink_fence *deferred_fence;
1804 struct zink_fence *last_fence; //the last command buffer submitted
1805 struct zink_batch_state *batch_states; //list of submitted batch states: ordered by increasing timeline id
1806 unsigned batch_states_count; //number of states in `batch_states`
1807 struct zink_batch_state *free_batch_states; //unused batch states
1808 struct zink_batch_state *last_free_batch_state; //for appending
1809 bool oom_flush;
1810 bool oom_stall;
1811 bool track_renderpasses;
1812 struct zink_batch batch;
1813
1814 unsigned shader_has_inlinable_uniforms_mask;
1815 unsigned inlinable_uniforms_valid_mask;
1816
1817 struct pipe_constant_buffer ubos[MESA_SHADER_STAGES][PIPE_MAX_CONSTANT_BUFFERS];
1818 struct pipe_shader_buffer ssbos[MESA_SHADER_STAGES][PIPE_MAX_SHADER_BUFFERS];
1819 uint32_t writable_ssbos[MESA_SHADER_STAGES];
1820 struct zink_image_view image_views[MESA_SHADER_STAGES][ZINK_MAX_SHADER_IMAGES];
1821
1822 uint32_t transient_attachments;
1823 struct pipe_framebuffer_state fb_state;
1824 struct hash_table framebuffer_cache;
1825
1826 struct zink_vertex_elements_state *element_state;
1827 struct zink_rasterizer_state *rast_state;
1828 struct zink_depth_stencil_alpha_state *dsa_state;
1829
1830 bool pipeline_changed[2]; //gfx, compute
1831
1832 struct zink_shader *gfx_stages[ZINK_GFX_SHADER_COUNT];
1833 struct zink_shader *last_vertex_stage;
1834 bool shader_reads_drawid;
1835 bool shader_reads_basevertex;
1836 struct zink_gfx_pipeline_state gfx_pipeline_state;
1837 /* there are 5 gfx stages, but VS and FS are assumed to be always present,
1838 * thus only 3 stages need to be considered, giving 2^3 = 8 program caches.
1839 */
1840 struct hash_table program_cache[8];
1841 simple_mtx_t program_lock[8];
1842 uint32_t gfx_hash;
1843 struct zink_gfx_program *curr_program;
1844 struct set gfx_inputs;
1845 struct set gfx_outputs;
1846
1847 struct zink_descriptor_data dd;
1848
1849 struct zink_compute_pipeline_state compute_pipeline_state;
1850 struct zink_compute_program *curr_compute;
1851
1852 unsigned shader_stages : ZINK_GFX_SHADER_COUNT; /* mask of bound gfx shader stages */
1853 uint8_t dirty_gfx_stages; /* mask of changed gfx shader stages */
1854 bool last_vertex_stage_dirty;
1855 bool compute_dirty;
1856 bool is_generated_gs_bound;
1857
1858 struct {
1859 VkRenderingAttachmentInfo attachments[PIPE_MAX_COLOR_BUFS + 2]; //+depth, +stencil
1860 VkRenderingInfo info;
1861 struct tc_renderpass_info tc_info;
1862 } dynamic_fb;
1863 uint32_t fb_layer_mismatch; //bitmask
1864 unsigned depth_bias_scale_factor;
1865 struct set rendering_state_cache[6]; //[util_logbase2_ceil(msrtss samplecount)]
1866 struct set render_pass_state_cache;
1867 struct hash_table *render_pass_cache;
1868 VkExtent2D swapchain_size;
1869 bool fb_changed;
1870 bool rp_changed; //force renderpass restart
1871 bool rp_layout_changed; //renderpass changed, maybe restart
1872 bool rp_loadop_changed; //renderpass changed, don't restart
1873 bool zsbuf_unused;
1874 bool zsbuf_readonly;
1875
1876 struct zink_framebuffer *framebuffer;
1877 struct zink_framebuffer_clear fb_clears[PIPE_MAX_COLOR_BUFS + 1];
1878 uint16_t clears_enabled;
1879 uint16_t rp_clears_enabled;
1880 uint16_t void_clears;
1881 uint16_t fbfetch_outputs;
1882 uint16_t feedback_loops;
1883 struct zink_resource *needs_present;
1884
1885 struct pipe_vertex_buffer vertex_buffers[PIPE_MAX_ATTRIBS];
1886 bool vertex_buffers_dirty;
1887
1888 struct zink_sampler_state *sampler_states[MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS];
1889 struct pipe_sampler_view *sampler_views[MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS];
1890
1891 struct zink_viewport_state vp_state;
1892 bool vp_state_changed;
1893 bool scissor_changed;
1894
1895 float blend_constants[4];
1896
1897 bool sample_locations_changed;
1898 VkSampleLocationEXT vk_sample_locations[PIPE_MAX_SAMPLE_LOCATION_GRID_SIZE * PIPE_MAX_SAMPLE_LOCATION_GRID_SIZE];
1899 uint8_t sample_locations[2 * 4 * 8 * 16];
1900
1901 struct pipe_stencil_ref stencil_ref;
1902
1903 union {
1904 struct {
1905 float default_inner_level[2];
1906 float default_outer_level[4];
1907 };
1908 float tess_levels[6];
1909 };
1910
1911 struct zink_vk_query *curr_xfb_queries[PIPE_MAX_VERTEX_STREAMS];
1912 struct zink_shader *null_fs;
1913 struct zink_shader *saved_fs;
1914
1915 struct list_head query_pools;
1916 struct list_head suspended_queries;
1917 struct list_head primitives_generated_queries;
1918 struct zink_query *vertices_query;
1919 bool disable_fs;
1920 bool disable_color_writes;
1921 bool was_line_loop;
1922 bool fs_query_active;
1923 bool occlusion_query_active;
1924 bool primitives_generated_active;
1925 bool primitives_generated_suspended;
1926 bool queries_disabled, render_condition_active;
1927 bool queries_in_rp;
1928 struct {
1929 struct zink_query *query;
1930 bool inverted;
1931 bool active; //this is the internal vk state
1932 } render_condition;
1933 struct {
1934 uint64_t render_passes;
1935 } hud;
1936
1937 struct {
1938 bool valid;
1939 struct u_upload_mgr *upload[ZINK_DGC_MAX];
1940 struct zink_resource *buffers[ZINK_DGC_MAX];
1941 struct zink_gfx_program *last_prog;
1942 uint8_t *maps[ZINK_DGC_MAX];
1943 size_t bind_offsets[ZINK_DGC_MAX];
1944 size_t cur_offsets[ZINK_DGC_MAX];
1945 size_t max_size[ZINK_DGC_MAX];
1946 struct util_dynarray pipelines;
1947 struct util_dynarray tokens;
1948 } dgc;
1949
1950 struct pipe_resource *dummy_vertex_buffer;
1951 struct pipe_resource *dummy_xfb_buffer;
1952 struct pipe_surface *dummy_surface[7];
1953 struct zink_buffer_view *dummy_bufferview;
1954
1955 unsigned buffer_rebind_counter;
1956 unsigned image_rebind_counter;
1957
1958 struct {
1959 /* descriptor info */
1960 uint32_t push_valid;
1961 uint8_t num_ubos[MESA_SHADER_STAGES];
1962
1963 uint8_t num_ssbos[MESA_SHADER_STAGES];
1964 struct util_dynarray global_bindings;
1965
1966 VkDescriptorImageInfo textures[MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS];
1967 uint32_t emulate_nonseamless[MESA_SHADER_STAGES];
1968 uint32_t cubes[MESA_SHADER_STAGES];
1969 uint8_t num_samplers[MESA_SHADER_STAGES];
1970 uint8_t num_sampler_views[MESA_SHADER_STAGES];
1971
1972 VkDescriptorImageInfo images[MESA_SHADER_STAGES][ZINK_MAX_SHADER_IMAGES];
1973 uint8_t num_images[MESA_SHADER_STAGES];
1974
1975 union {
1976 struct {
1977 VkDescriptorBufferInfo ubos[MESA_SHADER_STAGES][PIPE_MAX_CONSTANT_BUFFERS];
1978 VkDescriptorBufferInfo ssbos[MESA_SHADER_STAGES][PIPE_MAX_SHADER_BUFFERS];
1979 VkBufferView tbos[MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS];
1980 VkBufferView texel_images[MESA_SHADER_STAGES][ZINK_MAX_SHADER_IMAGES];
1981 } t;
1982 struct {
1983 VkDescriptorAddressInfoEXT ubos[MESA_SHADER_STAGES][PIPE_MAX_CONSTANT_BUFFERS];
1984 VkDescriptorAddressInfoEXT ssbos[MESA_SHADER_STAGES][PIPE_MAX_SHADER_BUFFERS];
1985 VkDescriptorAddressInfoEXT tbos[MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS];
1986 VkDescriptorAddressInfoEXT texel_images[MESA_SHADER_STAGES][ZINK_MAX_SHADER_IMAGES];
1987 } db;
1988 };
1989
1990 VkDescriptorImageInfo fbfetch;
1991 uint8_t fbfetch_db[ZINK_FBFETCH_DESCRIPTOR_SIZE];
1992
1993 /* the current state of the zs swizzle data */
1994 struct zink_zs_swizzle_key zs_swizzle[MESA_SHADER_STAGES];
1995
1996 struct zink_resource *descriptor_res[ZINK_DESCRIPTOR_BASE_TYPES][MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS];
1997
1998 struct {
1999 struct util_idalloc tex_slots; //img, buffer
2000 struct util_idalloc img_slots; //img, buffer
2001 struct hash_table tex_handles; //img, buffer
2002 struct hash_table img_handles; //img, buffer
2003 union {
2004 struct {
2005 VkBufferView *buffer_infos; //tex, img
2006 } t;
2007 struct {
2008 VkDescriptorAddressInfoEXT *buffer_infos;
2009 } db;
2010 };
2011 VkDescriptorImageInfo *img_infos; //tex, img
2012 struct util_dynarray updates; //texture, img
2013 struct util_dynarray resident; //texture, img
2014 } bindless[2];
2015 union {
2016 bool bindless_dirty[2]; //tex, img
2017 uint16_t any_bindless_dirty;
2018 };
2019 bool bindless_refs_dirty;
2020 bool null_fbfetch_init;
2021 } di;
2022 void (*invalidate_descriptor_state)(struct zink_context *ctx, gl_shader_stage shader, enum zink_descriptor_type type, unsigned, unsigned);
2023 struct set *need_barriers[2]; //gfx, compute
2024 struct set update_barriers[2][2]; //[gfx, compute][current, next]
2025 uint8_t barrier_set_idx[2];
2026 unsigned memory_barrier;
2027
2028 uint32_t ds3_states;
2029
2030 uint32_t num_so_targets;
2031 struct pipe_stream_output_target *so_targets[PIPE_MAX_SO_BUFFERS];
2032 bool dirty_so_targets;
2033
2034 bool gfx_dirty;
2035
2036 bool shobj_draw : 1; //using shader objects for draw
2037 bool is_device_lost;
2038 bool primitive_restart;
2039 bool blitting : 1;
2040 bool unordered_blitting : 1;
2041 bool vertex_state_changed : 1;
2042 bool blend_state_changed : 1;
2043 bool blend_color_changed : 1;
2044 bool sample_mask_changed : 1;
2045 bool rast_state_changed : 1;
2046 bool line_width_changed : 1;
2047 bool dsa_state_changed : 1;
2048 bool stencil_ref_changed : 1;
2049 bool rasterizer_discard_changed : 1;
2050 bool rp_tc_info_updated : 1;
2051 };
2052
2053 static inline struct zink_context *
zink_context(struct pipe_context * context)2054 zink_context(struct pipe_context *context)
2055 {
2056 return (struct zink_context *)context;
2057 }
2058
2059 #endif
2060