1 /*
2 * Copyright © 2022 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Mike Blumenkrantz <michael.blumenkrantz@gmail.com>
25 */
26
27 #ifndef ZINK_TYPES_H
28 #define ZINK_TYPES_H
29
30 #include <vulkan/vulkan_core.h>
31
32 #include "compiler/nir/nir.h"
33
34 #include "pipe/p_context.h"
35 #include "pipe/p_defines.h"
36 #include "pipe/p_state.h"
37
38 #include "pipebuffer/pb_cache.h"
39 #include "pipebuffer/pb_slab.h"
40
41 #include "util/disk_cache.h"
42 #include "util/hash_table.h"
43 #include "util/list.h"
44 #include "util/log.h"
45 #include "util/rwlock.h"
46 #include "util/set.h"
47 #include "util/simple_mtx.h"
48 #include "util/slab.h"
49 #include "util/u_blitter.h"
50 #include "util/u_dynarray.h"
51 #include "util/u_idalloc.h"
52 #include "util/u_live_shader_cache.h"
53 #include "util/u_queue.h"
54 #include "util/u_range.h"
55 #include "util/u_threaded_context.h"
56 #include "util/u_transfer.h"
57 #include "util/u_vertex_state_cache.h"
58
59 #include "vk_util.h"
60
61 #include "zink_device_info.h"
62 #include "zink_instance.h"
63 #include "zink_shader_keys.h"
64 #include "vk_dispatch_table.h"
65
66 #include "renderdoc_app.h"
67
68 /* the descriptor binding id for fbfetch/input attachment */
69 #define ZINK_FBFETCH_BINDING 5
70 #define ZINK_GFX_SHADER_COUNT 5
71
72 /* number of descriptors to allocate in a pool */
73 #define MAX_LAZY_DESCRIPTORS 500
74 /* explicit clamping because descriptor caching used to exist */
75 #define ZINK_MAX_SHADER_IMAGES 32
76 /* total number of bindless ids that can be allocated */
77 #define ZINK_MAX_BINDLESS_HANDLES 1024
78
79 /* enum zink_descriptor_type */
80 #define ZINK_MAX_DESCRIPTOR_SETS 6
81 #define ZINK_MAX_DESCRIPTORS_PER_TYPE (32 * ZINK_GFX_SHADER_COUNT)
82 /* Descriptor size reported by lavapipe. */
83 #define ZINK_FBFETCH_DESCRIPTOR_SIZE 280
84
85 /* suballocator defines */
86 #define NUM_SLAB_ALLOCATORS 3
87 #define MIN_SLAB_ORDER 8
88
89
90 /* this is the spec minimum */
91 #define ZINK_SPARSE_BUFFER_PAGE_SIZE (64 * 1024)
92
93 /* flag to create screen->copy_context */
94 #define ZINK_CONTEXT_COPY_ONLY (1<<30)
95
96 /* convenience macros for accessing dispatch table functions */
97 #define VKCTX(fn) zink_screen(ctx->base.screen)->vk.fn
98 #define VKSCR(fn) screen->vk.fn
99
100 #ifdef __cplusplus
101 extern "C" {
102 #endif
103
104 extern uint32_t zink_debug;
105 extern bool zink_tracing;
106
107 #ifdef __cplusplus
108 }
109 #endif
110
111
112 /** enums */
113
114 /* features for draw/program templates */
115 typedef enum {
116 ZINK_NO_MULTIDRAW,
117 ZINK_MULTIDRAW,
118 } zink_multidraw;
119
120 typedef enum {
121 ZINK_NO_DYNAMIC_STATE,
122 ZINK_DYNAMIC_STATE,
123 ZINK_DYNAMIC_STATE2,
124 ZINK_DYNAMIC_VERTEX_INPUT2,
125 ZINK_DYNAMIC_STATE3,
126 ZINK_DYNAMIC_VERTEX_INPUT,
127 } zink_dynamic_state;
128
129 typedef enum {
130 ZINK_PIPELINE_NO_DYNAMIC_STATE,
131 ZINK_PIPELINE_DYNAMIC_STATE,
132 ZINK_PIPELINE_DYNAMIC_STATE2,
133 ZINK_PIPELINE_DYNAMIC_STATE2_PCP,
134 ZINK_PIPELINE_DYNAMIC_VERTEX_INPUT2,
135 ZINK_PIPELINE_DYNAMIC_VERTEX_INPUT2_PCP,
136 ZINK_PIPELINE_DYNAMIC_STATE3,
137 ZINK_PIPELINE_DYNAMIC_STATE3_PCP,
138 ZINK_PIPELINE_DYNAMIC_VERTEX_INPUT,
139 ZINK_PIPELINE_DYNAMIC_VERTEX_INPUT_PCP,
140 } zink_pipeline_dynamic_state;
141
142 enum zink_blit_flags {
143 ZINK_BLIT_NORMAL = 1 << 0,
144 ZINK_BLIT_SAVE_FS = 1 << 1,
145 ZINK_BLIT_SAVE_FB = 1 << 2,
146 ZINK_BLIT_SAVE_TEXTURES = 1 << 3,
147 ZINK_BLIT_NO_COND_RENDER = 1 << 4,
148 ZINK_BLIT_SAVE_FS_CONST_BUF = 1 << 5,
149 };
150
151 /* descriptor types; also the ordering of the sets
152 * ...except that ZINK_DESCRIPTOR_BASE_TYPES is actually ZINK_DESCRIPTOR_TYPE_UNIFORMS,
153 * and all base type values are thus +1 to get the set id (using screen->desc_set_id[idx])
154 */
155 enum zink_descriptor_type {
156 ZINK_DESCRIPTOR_TYPE_UBO,
157 ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW,
158 ZINK_DESCRIPTOR_TYPE_SSBO,
159 ZINK_DESCRIPTOR_TYPE_IMAGE,
160 ZINK_DESCRIPTOR_BASE_TYPES, /**< the count/iterator for basic descriptor types */
161 ZINK_DESCRIPTOR_BINDLESS,
162 ZINK_DESCRIPTOR_ALL_TYPES,
163 ZINK_DESCRIPTOR_TYPE_UNIFORMS = ZINK_DESCRIPTOR_BASE_TYPES, /**< this is aliased for convenience */
164 ZINK_DESCRIPTOR_NON_BINDLESS_TYPES = ZINK_DESCRIPTOR_BASE_TYPES + 1, /**< for struct sizing */
165 };
166
167 enum zink_descriptor_mode {
168 ZINK_DESCRIPTOR_MODE_AUTO,
169 ZINK_DESCRIPTOR_MODE_LAZY,
170 ZINK_DESCRIPTOR_MODE_DB,
171 };
172
173 /* the current mode */
174 extern enum zink_descriptor_mode zink_descriptor_mode;
175
176 /* indexing for descriptor template management */
177 enum zink_descriptor_size_index {
178 ZDS_INDEX_UBO,
179 ZDS_INDEX_COMBINED_SAMPLER,
180 ZDS_INDEX_UNIFORM_TEXELS,
181 ZDS_INDEX_SAMPLER,
182 ZDS_INDEX_STORAGE_BUFFER,
183 ZDS_INDEX_STORAGE_IMAGE,
184 ZDS_INDEX_STORAGE_TEXELS,
185 ZDS_INDEX_MAX,
186 };
187
188 /* indexing for descriptor template management in COMPACT mode */
189 enum zink_descriptor_size_index_compact {
190 ZDS_INDEX_COMP_UBO,
191 ZDS_INDEX_COMP_STORAGE_BUFFER,
192 ZDS_INDEX_COMP_COMBINED_SAMPLER,
193 ZDS_INDEX_COMP_UNIFORM_TEXELS,
194 ZDS_INDEX_COMP_SAMPLER,
195 ZDS_INDEX_COMP_STORAGE_IMAGE,
196 ZDS_INDEX_COMP_STORAGE_TEXELS,
197 };
198
199 enum zink_resource_access {
200 ZINK_RESOURCE_ACCESS_READ = 1,
201 ZINK_RESOURCE_ACCESS_WRITE = 32,
202 ZINK_RESOURCE_ACCESS_RW = ZINK_RESOURCE_ACCESS_READ | ZINK_RESOURCE_ACCESS_WRITE,
203 };
204
205
206 /* zink heaps are based off of vulkan memory types, but are not a 1-to-1 mapping to vulkan memory type indices and have no direct relation to vulkan memory heaps*/
207 enum zink_heap {
208 ZINK_HEAP_DEVICE_LOCAL,
209 ZINK_HEAP_DEVICE_LOCAL_SPARSE,
210 ZINK_HEAP_DEVICE_LOCAL_LAZY,
211 ZINK_HEAP_DEVICE_LOCAL_VISIBLE,
212 ZINK_HEAP_HOST_VISIBLE_COHERENT,
213 ZINK_HEAP_HOST_VISIBLE_COHERENT_CACHED,
214 ZINK_HEAP_MAX,
215 };
216
217 enum zink_alloc_flag {
218 ZINK_ALLOC_SPARSE = 1<<0,
219 ZINK_ALLOC_NO_SUBALLOC = 1<<1,
220 };
221
222 enum zink_debug {
223 ZINK_DEBUG_NIR = (1<<0),
224 ZINK_DEBUG_SPIRV = (1<<1),
225 ZINK_DEBUG_TGSI = (1<<2),
226 ZINK_DEBUG_VALIDATION = (1<<3),
227 ZINK_DEBUG_SYNC = (1<<4),
228 ZINK_DEBUG_COMPACT = (1<<5),
229 ZINK_DEBUG_NOREORDER = (1<<6),
230 ZINK_DEBUG_GPL = (1<<7),
231 ZINK_DEBUG_SHADERDB = (1<<8),
232 ZINK_DEBUG_RP = (1<<9),
233 ZINK_DEBUG_NORP = (1<<10),
234 ZINK_DEBUG_MAP = (1<<11),
235 ZINK_DEBUG_FLUSHSYNC = (1<<12),
236 ZINK_DEBUG_NOSHOBJ = (1<<13),
237 ZINK_DEBUG_OPTIMAL_KEYS = (1<<14),
238 ZINK_DEBUG_NOOPT = (1<<15),
239 ZINK_DEBUG_NOBGC = (1<<16),
240 ZINK_DEBUG_MEM = (1<<17),
241 ZINK_DEBUG_QUIET = (1<<18),
242 ZINK_DEBUG_IOOPT = (1<<19),
243 ZINK_DEBUG_NOPC = (1<<20),
244 };
245
246 enum zink_pv_emulation_primitive {
247 ZINK_PVE_PRIMITIVE_NONE = 0,
248 ZINK_PVE_PRIMITIVE_SIMPLE = 1,
249 /* when triangle or quad strips are used and the gs outputs triangles */
250 ZINK_PVE_PRIMITIVE_TRISTRIP = 2,
251 ZINK_PVE_PRIMITIVE_FAN = 3,
252 };
253
254 /** fence types */
255 struct tc_unflushed_batch_token;
256
257 /* an async fence created for tc */
258 struct zink_tc_fence {
259 struct pipe_reference reference;
260 /* enables distinction between tc fence submission and vk queue submission */
261 uint32_t submit_count;
262 /* when the tc fence is signaled for use */
263 struct util_queue_fence ready;
264 struct tc_unflushed_batch_token *tc_token;
265 /* for deferred flushes */
266 struct pipe_context *deferred_ctx;
267 /* multiple tc fences may point to a real fence */
268 struct zink_fence *fence;
269 /* for use with semaphore/imported fences */
270 VkSemaphore sem;
271 };
272
273 /* a fence is actually a zink_batch_state, but these are split out for logical consistency */
274 struct zink_fence {
275 uint64_t batch_id;
276 bool submitted;
277 bool completed;
278 struct util_dynarray mfences;
279 };
280
281
282 /** state types */
283
284 struct zink_vertex_elements_hw_state {
285 uint32_t hash;
286 uint32_t num_bindings, num_attribs;
287 /* VK_EXT_vertex_input_dynamic_state uses different types */
288 union {
289 VkVertexInputAttributeDescription attribs[PIPE_MAX_ATTRIBS];
290 VkVertexInputAttributeDescription2EXT dynattribs[PIPE_MAX_ATTRIBS];
291 };
292 union {
293 struct {
294 VkVertexInputBindingDivisorDescriptionEXT divisors[PIPE_MAX_ATTRIBS];
295 VkVertexInputBindingDescription bindings[PIPE_MAX_ATTRIBS]; // combination of element_state and stride
296 VkDeviceSize strides[PIPE_MAX_ATTRIBS];
297 uint8_t divisors_present;
298 } b;
299 VkVertexInputBindingDescription2EXT dynbindings[PIPE_MAX_ATTRIBS];
300 };
301 uint8_t binding_map[PIPE_MAX_ATTRIBS];
302 };
303
304 struct zink_vertex_elements_state {
305 /* decomposed attributes read only a single component for format compatibility */
306 bool has_decomposed_attrs;
307 struct {
308 uint32_t binding;
309 VkVertexInputRate inputRate;
310 } bindings[PIPE_MAX_ATTRIBS];
311 uint32_t divisor[PIPE_MAX_ATTRIBS];
312 uint32_t min_stride[PIPE_MAX_ATTRIBS]; //for dynamic_state1
313 uint32_t decomposed_attrs;
314 unsigned decomposed_attrs_size;
315 uint32_t decomposed_attrs_without_w;
316 unsigned decomposed_attrs_without_w_size;
317 struct zink_vertex_elements_hw_state hw_state;
318 };
319
320 /* for vertex state draws */
321 struct zink_vertex_state {
322 struct pipe_vertex_state b;
323 struct zink_vertex_elements_state velems;
324 };
325
326 struct zink_rasterizer_hw_state {
327 unsigned polygon_mode : 2; //VkPolygonMode
328 unsigned line_mode : 2; //VkLineRasterizationModeEXT
329 unsigned depth_clip:1;
330 unsigned depth_clamp:1;
331 unsigned pv_last:1;
332 unsigned line_stipple_enable:1;
333 unsigned clip_halfz:1;
334 };
335
336 struct zink_rasterizer_state {
337 struct pipe_rasterizer_state base;
338 bool offset_fill;
339 float offset_units, offset_clamp, offset_scale;
340 float line_width;
341 VkFrontFace front_face;
342 VkCullModeFlags cull_mode;
343 VkLineRasterizationModeEXT dynamic_line_mode;
344 struct zink_rasterizer_hw_state hw_state;
345 };
346
347 struct zink_blend_state {
348 uint32_t hash;
349 unsigned num_rts;
350 VkPipelineColorBlendAttachmentState attachments[PIPE_MAX_COLOR_BUFS];
351
352 struct {
353 VkBool32 enables[PIPE_MAX_COLOR_BUFS];
354 VkColorBlendEquationEXT eq[PIPE_MAX_COLOR_BUFS];
355 VkColorComponentFlags wrmask[PIPE_MAX_COLOR_BUFS];
356 } ds3;
357
358 VkBool32 logicop_enable;
359 VkLogicOp logicop_func;
360
361 VkBool32 alpha_to_coverage;
362 VkBool32 alpha_to_one;
363
364 uint32_t wrmask;
365 uint8_t enables;
366
367 bool dual_src_blend;
368 };
369
370 struct zink_depth_stencil_alpha_hw_state {
371 VkBool32 depth_test;
372 VkCompareOp depth_compare_op;
373
374 VkBool32 depth_bounds_test;
375 float min_depth_bounds, max_depth_bounds;
376
377 VkBool32 stencil_test;
378 VkStencilOpState stencil_front;
379 VkStencilOpState stencil_back;
380
381 VkBool32 depth_write;
382 };
383
384 struct zink_depth_stencil_alpha_state {
385 struct pipe_depth_stencil_alpha_state base;
386 struct zink_depth_stencil_alpha_hw_state hw_state;
387 };
388
389
390 /** descriptor types */
391
392 /* zink_descriptor_layout objects are cached: this is the key for one */
393 struct zink_descriptor_layout_key {
394 unsigned num_bindings;
395 VkDescriptorSetLayoutBinding *bindings;
396 };
397
398 struct zink_descriptor_layout {
399 VkDescriptorSetLayout layout;
400 };
401
402 /* descriptor pools are cached: zink_descriptor_pool_key::id is the id for a type of pool */
403 struct zink_descriptor_pool_key {
404 unsigned use_count;
405 unsigned num_type_sizes;
406 unsigned id;
407 VkDescriptorPoolSize sizes[4];
408 struct zink_descriptor_layout_key *layout;
409 };
410
411 /* a template used for updating descriptor buffers */
412 struct zink_descriptor_template {
413 uint16_t stride; //the stride between mem pointers
414 uint16_t db_size; //the size of the entry in the buffer
415 unsigned count; //the number of descriptors
416 size_t offset; //the offset of the base host pointer to update from
417 };
418
419 /* ctx->dd; created at context creation */
420 struct zink_descriptor_data {
421 bool bindless_bound;
422 bool bindless_init;
423 bool has_fbfetch;
424 bool push_state_changed[2]; //gfx, compute
425 uint8_t state_changed[2]; //gfx, compute
426 struct zink_descriptor_layout_key *push_layout_keys[2]; //gfx, compute
427 struct zink_descriptor_layout *push_dsl[2]; //gfx, compute
428 VkDescriptorUpdateTemplate push_template[2]; //gfx, compute
429
430 struct zink_descriptor_layout *dummy_dsl;
431
432 union {
433 struct {
434 VkDescriptorPool bindless_pool;
435 VkDescriptorSet bindless_set;
436 } t;
437 struct {
438 struct zink_resource *bindless_db;
439 uint8_t *bindless_db_map;
440 struct pipe_transfer *bindless_db_xfer;
441 uint32_t bindless_db_offsets[4];
442 unsigned max_db_size;
443 unsigned size_enlarge_scale;
444 } db;
445 };
446
447 struct zink_program *pg[2]; //gfx, compute
448
449 VkDescriptorUpdateTemplateEntry push_entries[MESA_SHADER_STAGES]; //gfx+fbfetch
450 VkDescriptorUpdateTemplateEntry compute_push_entry;
451
452 /* push descriptor layout size and binding offsets */
453 uint32_t db_size[2]; //gfx, compute
454 uint32_t db_offset[ZINK_GFX_SHADER_COUNT + 1]; //gfx + fbfetch
455 /* compute offset is always 0 */
456 };
457
458 /* pg->dd; created at program creation */
459 struct zink_program_descriptor_data {
460 bool bindless;
461 bool fbfetch;
462 /* bitmask of ubo0 usage for stages */
463 uint8_t push_usage;
464 /* bitmask of which sets are used by the program */
465 uint8_t binding_usage;
466 /* all the pool keys for the program */
467 struct zink_descriptor_pool_key *pool_key[ZINK_DESCRIPTOR_BASE_TYPES]; //push set doesn't need one
468 /* all the layouts for the program */
469 struct zink_descriptor_layout *layouts[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES];
470 /* all the templates for the program */
471 union {
472 VkDescriptorUpdateTemplate templates[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES];
473 struct zink_descriptor_template *db_template[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES];
474 };
475 uint32_t db_size[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //the total size of the layout
476 uint32_t *db_offset[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //the offset of each binding in the layout
477 };
478
479 struct zink_descriptor_pool {
480 /* the current index of 'sets' */
481 unsigned set_idx;
482 /* number of sets allocated */
483 unsigned sets_alloc;
484 VkDescriptorPool pool;
485 /* sets are lazily allocated */
486 VkDescriptorSet sets[MAX_LAZY_DESCRIPTORS];
487 };
488
489 /* a zink_descriptor_pool_key matches up to this struct */
490 struct zink_descriptor_pool_multi {
491 /* for flagging when overflowed pools must be destroyed instead of reused */
492 bool reinit_overflow;
493 /* this flips to split usable overflow from in-use overflow */
494 unsigned overflow_idx;
495 /* zink_descriptor_pool objects that have exceeded MAX_LAZY_DESCRIPTORS sets */
496 struct util_dynarray overflowed_pools[2];
497 /* the current pool; may be null */
498 struct zink_descriptor_pool *pool;
499 /* pool key for convenience */
500 const struct zink_descriptor_pool_key *pool_key;
501 };
502
503 /* bs->dd; created on batch state creation */
504 struct zink_batch_descriptor_data {
505 /* pools have fbfetch initialized */
506 bool has_fbfetch;
507 /* are descriptor buffers bound */
508 bool db_bound;
509 /* real size of 'pools' */
510 unsigned pool_size[ZINK_DESCRIPTOR_BASE_TYPES];
511 /* this array is sized based on the max zink_descriptor_pool_key::id used by the batch; members may be NULL */
512 struct util_dynarray pools[ZINK_DESCRIPTOR_BASE_TYPES];
513 struct zink_descriptor_pool_multi push_pool[2]; //gfx, compute
514 /* the current program (for descriptor updating) */
515 struct zink_program *pg[2]; //gfx, compute
516 /* the current pipeline compatibility id (for pipeline compatibility rules) */
517 uint32_t compat_id[2]; //gfx, compute
518 /* the current set layout */
519 VkDescriptorSetLayout dsl[2][ZINK_DESCRIPTOR_BASE_TYPES]; //gfx, compute
520 union {
521 /* the current set for a given type; used for rebinding if pipeline compat id changes and current set must be rebound */
522 VkDescriptorSet sets[2][ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //gfx, compute
523 uint64_t cur_db_offset[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //gfx, compute; the current offset of a descriptor buffer for rebinds
524 };
525 /* mask of push descriptor usage */
526 unsigned push_usage[2]; //gfx, compute
527
528 struct zink_resource *db; //the descriptor buffer for a given type
529 uint8_t *db_map; //the host map for the buffer
530 struct pipe_transfer *db_xfer; //the transfer map for the buffer
531 uint64_t db_offset; //the "next" offset that will be used when the buffer is updated
532 };
533
534 /** batch types */
535 /* zink_batch_usage concepts:
536 * - batch "usage" is an indicator of when and how a BO was accessed
537 * - batch "tracking" is the batch state(s) containing an extra ref for a BO
538 *
539 * - usage prevents a BO from being mapped while it has pending+conflicting access
540 * - usage affects pipeline barrier generation for synchronizing reads and writes
541 * - usage MUST be removed before context destruction to avoid crashing during BO
542 * reclaiming in suballocator
543 *
544 * - tracking prevents a BO from being destroyed early
545 * - tracking enables usage to be pruned
546 *
547 *
548 * tracking is added:
549 * - any time a BO is used in a "one-off" operation (e.g., blit, index buffer, indirect buffer)
550 * - any time a descriptor is unbound
551 * - when a buffer is replaced (IFF: resource is bound as a descriptor or usage previously existed)
552 *
553 * tracking is removed:
554 * - in zink_reset_batch_state()
555 *
556 * usage is added:
557 * - any time a BO is used in a "one-off" operation (e.g., blit, index buffer, indirect buffer)
558 * - any time a descriptor is bound
559 * - any time a descriptor is unbound (IFF: usage previously existed)
560 * - for all bound descriptors on the first draw/dispatch after a flush (zink_update_descriptor_refs)
561 *
562 * usage is removed:
563 * - when tracking is removed (IFF: BO usage == tracking, i.e., this is the last batch that a BO was active on)
564 */
565 struct zink_batch_usage {
566 uint32_t usage;
567 /* this is a monotonic int used to disambiguate internal fences from their tc fence references */
568 uint32_t submit_count;
569 cnd_t flush;
570 mtx_t mtx;
571 bool unflushed;
572 };
573
574 struct zink_bo_usage {
575 uint32_t submit_count;
576 struct zink_batch_usage *u;
577 };
578
579 struct zink_batch_obj_list {
580 unsigned max_buffers;
581 unsigned num_buffers;
582 struct zink_resource_object **objs;
583 };
584
585 struct zink_batch_state {
586 struct zink_fence fence;
587 struct zink_batch_state *next;
588
589 struct zink_batch_usage usage;
590 struct zink_context *ctx;
591 VkCommandPool cmdpool;
592 VkCommandBuffer cmdbuf;
593 VkCommandBuffer reordered_cmdbuf;
594 VkCommandPool unsynchronized_cmdpool;
595 VkCommandBuffer unsynchronized_cmdbuf;
596 VkSemaphore signal_semaphore; //external signal semaphore
597 struct util_dynarray signal_semaphores; //external signal semaphores
598 struct util_dynarray wait_semaphores; //external wait semaphores
599 struct util_dynarray wait_semaphore_stages; //external wait semaphores
600 struct util_dynarray fd_wait_semaphores; //dmabuf wait semaphores
601 struct util_dynarray fd_wait_semaphore_stages; //dmabuf wait semaphores
602 struct util_dynarray tracked_semaphores; //semaphores which are just tracked
603 VkSemaphore sparse_semaphore; //current sparse wait semaphore
604 struct util_dynarray fences; //zink_tc_fence refs
605 simple_mtx_t ref_lock;
606
607 VkSemaphore present;
608 struct zink_resource *swapchain;
609 struct util_dynarray acquires;
610 struct util_dynarray acquire_flags;
611
612 VkAccessFlags unordered_write_access;
613 VkPipelineStageFlags unordered_write_stages;
614
615 simple_mtx_t exportable_lock;
616
617 struct util_queue_fence flush_completed;
618
619 struct set programs;
620 struct set dmabuf_exports;
621
622 #define BUFFER_HASHLIST_SIZE 32768
623 /* buffer_indices_hashlist[hash(bo)] returns -1 if the bo
624 * isn't part of any buffer lists or the index where the bo could be found.
625 * Since 1) hash collisions of 2 different bo can happen and 2) we use a
626 * single hashlist for the 3 buffer list, this is only a hint.
627 * batch_find_resource uses this hint to speed up buffers look up.
628 */
629 int16_t buffer_indices_hashlist[BUFFER_HASHLIST_SIZE];
630 uint16_t hashlist_min;
631 uint16_t hashlist_max;
632 struct zink_batch_obj_list real_objs;
633 struct zink_batch_obj_list slab_objs;
634 struct zink_batch_obj_list sparse_objs;
635 struct zink_resource_object *last_added_obj;
636 struct util_dynarray swapchain_obj; //this doesn't have a zink_bo and must be handled differently
637
638 struct util_dynarray unref_resources;
639 struct util_dynarray bindless_releases[2];
640
641 struct util_dynarray zombie_samplers;
642
643 struct set active_queries; /* zink_query objects which were active at some point in this batch */
644 struct util_dynarray dead_querypools;
645
646 struct util_dynarray freed_sparse_backing_bos;
647
648 struct zink_batch_descriptor_data dd;
649
650 VkDeviceSize resource_size;
651
652 bool is_device_lost;
653 /* these flags correspond to the matching cmdbufs */
654 bool has_work;
655 bool has_reordered_work;
656 bool has_unsync;
657 };
658
659 static inline struct zink_batch_state *
zink_batch_state(struct zink_fence * fence)660 zink_batch_state(struct zink_fence *fence)
661 {
662 return (struct zink_batch_state *)fence;
663 }
664
665 /** bo types */
666 struct bo_export {
667 /** File descriptor associated with a handle export. */
668 int drm_fd;
669
670 /** GEM handle in drm_fd */
671 uint32_t gem_handle;
672
673 struct list_head link;
674 };
675
676 struct zink_bo {
677 struct pb_buffer base;
678
679 union {
680 struct {
681 void *cpu_ptr; /* for user_ptr and permanent maps */
682 int map_count;
683 struct list_head exports;
684 simple_mtx_t export_lock;
685
686 bool is_user_ptr;
687 bool use_reusable_pool;
688
689 /* Whether buffer_get_handle or buffer_from_handle has been called,
690 * it can only transition from false to true. Protected by lock.
691 */
692 bool is_shared;
693 } real;
694 struct {
695 struct pb_slab_entry entry;
696 struct zink_bo *real;
697 } slab;
698 struct {
699 uint32_t num_va_pages;
700 uint32_t num_backing_pages;
701
702 struct list_head backing;
703
704 /* Commitment information for each page of the virtual memory area. */
705 struct zink_sparse_commitment *commitments;
706 } sparse;
707 } u;
708
709 VkDeviceMemory mem;
710 uint64_t offset;
711
712 uint32_t unique_id;
713 const char *name;
714
715 simple_mtx_t lock;
716
717 struct zink_bo_usage reads;
718 struct zink_bo_usage writes;
719
720 struct pb_cache_entry cache_entry[];
721 };
722
723 static inline struct zink_bo *
zink_bo(struct pb_buffer * pbuf)724 zink_bo(struct pb_buffer *pbuf)
725 {
726 return (struct zink_bo*)pbuf;
727 }
728
729 /** clear types */
730 struct zink_framebuffer_clear_data {
731 union {
732 union pipe_color_union color;
733 struct {
734 float depth;
735 unsigned stencil;
736 uint8_t bits : 2; // PIPE_CLEAR_DEPTH, PIPE_CLEAR_STENCIL
737 } zs;
738 };
739 struct pipe_scissor_state scissor;
740 bool has_scissor;
741 bool conditional;
742 };
743
744 struct zink_framebuffer_clear {
745 struct util_dynarray clears;
746 };
747
748
749 /** compiler types */
750 struct zink_shader_info {
751 uint16_t stride[PIPE_MAX_SO_BUFFERS];
752 uint32_t sampler_mask;
753 bool have_sparse;
754 bool have_vulkan_memory_model;
755 bool have_workgroup_memory_explicit_layout;
756 bool broken_arbitary_type_const;
757 struct {
758 uint8_t flush_denorms:3; // 16, 32, 64
759 uint8_t preserve_denorms:3; // 16, 32, 64
760 bool denorms_32_bit_independence:1;
761 bool denorms_all_independence:1;
762 } float_controls;
763 unsigned bindless_set_idx;
764 };
765
766 enum zink_rast_prim {
767 ZINK_PRIM_POINTS,
768 ZINK_PRIM_LINES,
769 ZINK_PRIM_TRIANGLES,
770 ZINK_PRIM_MAX,
771 };
772
773 struct zink_shader_object {
774 union {
775 VkShaderEXT obj;
776 VkShaderModule mod;
777 };
778 struct spirv_shader *spirv;
779 };
780
781 struct zink_shader {
782 struct util_live_shader base;
783 uint32_t hash;
784 struct blob blob;
785 struct shader_info info;
786 /* this is deleted in zink_shader_init */
787 nir_shader *nir;
788
789 struct zink_shader_info sinfo;
790
791 struct {
792 int index;
793 int binding;
794 VkDescriptorType type;
795 unsigned char size;
796 } bindings[ZINK_DESCRIPTOR_BASE_TYPES][ZINK_MAX_DESCRIPTORS_PER_TYPE];
797 size_t num_bindings[ZINK_DESCRIPTOR_BASE_TYPES];
798 uint32_t ubos_used; // bitfield of which ubo indices are used
799 uint32_t ssbos_used; // bitfield of which ssbo indices are used
800 uint64_t arrayed_inputs; //mask of locations using arrayed io
801 uint64_t arrayed_outputs; //mask of locations using arrayed io
802 uint64_t flat_flags;
803 bool bindless;
804 bool can_inline;
805 bool has_uniforms;
806 bool has_edgeflags;
807 bool needs_inlining;
808 struct spirv_shader *spirv;
809
810 struct {
811 struct util_queue_fence fence;
812 struct zink_shader_object obj;
813 VkDescriptorSetLayout dsl;
814 VkPipelineLayout layout;
815 VkPipeline gpl;
816 VkDescriptorSetLayoutBinding *bindings;
817 unsigned num_bindings;
818 struct zink_descriptor_template *db_template;
819 unsigned db_size;
820 unsigned *db_offset;
821 } precompile;
822
823 simple_mtx_t lock;
824 struct set *programs;
825 struct util_dynarray pipeline_libs;
826
827 union {
828 struct {
829 struct zink_shader *generated_tcs; // a generated shader that this shader "owns"; only valid in the tes stage
830 struct zink_shader *generated_gs[MESA_PRIM_COUNT][ZINK_PRIM_MAX]; // generated shaders that this shader "owns"
831 struct zink_shader *parent; // for a generated gs this points to the shader that "owns" it
832
833 bool is_generated; // if this is a driver-created shader (e.g., tcs)
834 } non_fs;
835
836 struct {
837 /* Bitmask of textures that have shadow sampling result components
838 * other than RED accessed. This is a subset of !is_new_style_shadow
839 * (GLSL <1.30, ARB_fp) shadow sampling usage.
840 */
841 uint32_t legacy_shadow_mask;
842 nir_variable *fbfetch; //for fs output
843 } fs;
844 };
845 };
846
847
848 /** pipeline types */
849 struct zink_pipeline_dynamic_state1 {
850 uint8_t front_face; //VkFrontFace:1
851 uint8_t cull_mode; //VkCullModeFlags:2
852 uint16_t num_viewports;
853 struct zink_depth_stencil_alpha_hw_state *depth_stencil_alpha_state; //must be last
854 };
855
856 struct zink_pipeline_dynamic_state2 {
857 bool primitive_restart;
858 bool rasterizer_discard;
859 uint16_t vertices_per_patch; //5 bits
860 };
861
862 #define zink_pipeline_dynamic_state3 zink_rasterizer_hw_state
863
864 struct zink_gfx_pipeline_state {
865 /* order matches zink_gfx_output_key */
866 unsigned force_persample_interp:1;
867 uint32_t rast_samples:6;
868 uint32_t min_samples:6;
869 uint32_t feedback_loop : 1;
870 uint32_t feedback_loop_zs : 1;
871 uint32_t rast_attachment_order : 1;
872 uint32_t rp_state : 16;
873 VkSampleMask sample_mask;
874 uint32_t blend_id;
875
876 /* Pre-hashed value for table lookup, invalid when zero.
877 * Members after this point are not included in pipeline state hash key */
878 uint32_t hash;
879 bool dirty;
880
881 struct zink_pipeline_dynamic_state1 dyn_state1;
882
883 struct zink_pipeline_dynamic_state2 dyn_state2;
884 struct zink_pipeline_dynamic_state3 dyn_state3;
885
886 union {
887 VkShaderModule modules[MESA_SHADER_STAGES - 1];
888 uint32_t optimal_key;
889 };
890 bool modules_changed;
891
892 uint32_t vertex_hash;
893
894 uint32_t final_hash;
895
896 uint32_t _pad2;
897 /* order matches zink_gfx_input_key */
898 union {
899 struct {
900 unsigned idx:8;
901 bool uses_dynamic_stride;
902 };
903 uint32_t input;
904 };
905 uint32_t vertex_buffers_enabled_mask;
906 uint32_t vertex_strides[PIPE_MAX_ATTRIBS];
907 struct zink_vertex_elements_hw_state *element_state;
908 struct zink_zs_swizzle_key *shadow;
909 bool sample_locations_enabled;
910 enum mesa_prim shader_rast_prim, rast_prim; /* reduced type or max for unknown */
911 union {
912 struct {
913 struct zink_shader_key key[5];
914 struct zink_shader_key last_vertex;
915 } shader_keys;
916 struct {
917 union zink_shader_key_optimal key;
918 } shader_keys_optimal;
919 };
920 struct zink_blend_state *blend_state;
921 struct zink_render_pass *render_pass;
922 struct zink_render_pass *next_render_pass; //will be used next time rp is begun
923 VkFormat rendering_formats[PIPE_MAX_COLOR_BUFS];
924 VkPipelineRenderingCreateInfo rendering_info;
925 VkPipeline pipeline;
926 enum mesa_prim gfx_prim_mode; //pending mode
927 };
928
929 struct zink_compute_pipeline_state {
930 /* Pre-hashed value for table lookup, invalid when zero.
931 * Members after this point are not included in pipeline state hash key */
932 uint32_t hash;
933 uint32_t final_hash;
934 bool dirty;
935 uint32_t local_size[3];
936 uint32_t variable_shared_mem;
937
938 uint32_t module_hash;
939 VkShaderModule module;
940 bool module_changed;
941
942 struct zink_shader_key key;
943
944 VkPipeline pipeline;
945 };
946
947
948 /** program types */
949
950 /* create_gfx_pushconst must be kept in sync with this struct */
951 struct zink_gfx_push_constant {
952 unsigned draw_mode_is_indexed;
953 unsigned draw_id;
954 unsigned framebuffer_is_layered;
955 float default_inner_level[2];
956 float default_outer_level[4];
957 uint32_t line_stipple_pattern;
958 float viewport_scale[2];
959 float line_width;
960 };
961
962 /* The order of the enums MUST match the order of the zink_gfx_push_constant
963 * members.
964 */
965 enum zink_gfx_push_constant_member {
966 ZINK_GFX_PUSHCONST_DRAW_MODE_IS_INDEXED,
967 ZINK_GFX_PUSHCONST_DRAW_ID,
968 ZINK_GFX_PUSHCONST_FRAMEBUFFER_IS_LAYERED,
969 ZINK_GFX_PUSHCONST_DEFAULT_INNER_LEVEL,
970 ZINK_GFX_PUSHCONST_DEFAULT_OUTER_LEVEL,
971 ZINK_GFX_PUSHCONST_LINE_STIPPLE_PATTERN,
972 ZINK_GFX_PUSHCONST_VIEWPORT_SCALE,
973 ZINK_GFX_PUSHCONST_LINE_WIDTH,
974 ZINK_GFX_PUSHCONST_MAX
975 };
976
977 /* a shader module is used for directly reusing a shader module between programs,
978 * e.g., in the case where we're swapping out only one shader,
979 * allowing us to skip going through shader keys
980 */
981 struct zink_shader_module {
982 struct zink_shader_object obj;
983 uint32_t hash;
984 bool shobj;
985 bool default_variant;
986 bool has_nonseamless;
987 bool needs_zs_shader_swizzle;
988 uint8_t num_uniforms;
989 uint8_t key_size;
990 uint8_t key[0]; /* | key | uniforms | zs shader swizzle | */
991 };
992
993 struct zink_program {
994 struct pipe_reference reference;
995 struct zink_context *ctx;
996 blake3_hash blake3;
997 struct util_queue_fence cache_fence;
998 struct u_rwlock pipeline_cache_lock;
999 VkPipelineCache pipeline_cache;
1000 size_t pipeline_cache_size;
1001 struct zink_batch_usage *batch_uses;
1002 bool is_compute;
1003 bool can_precompile;
1004 bool uses_shobj; //whether shader objects are used; programs CANNOT mix shader objects and shader modules
1005
1006 struct zink_program_descriptor_data dd;
1007
1008 uint32_t compat_id;
1009 VkPipelineLayout layout;
1010 VkDescriptorSetLayout dsl[ZINK_DESCRIPTOR_ALL_TYPES]; // one for each type + push + bindless
1011 unsigned num_dsl;
1012
1013 bool removed;
1014 };
1015
1016 #define STAGE_MASK_OPTIMAL (1<<16)
1017 #define STAGE_MASK_OPTIMAL_SHADOW (1<<17)
1018 typedef bool (*equals_gfx_pipeline_state_func)(const void *a, const void *b);
1019
1020 struct zink_gfx_library_key {
1021 uint32_t optimal_key; //equals_pipeline_lib_optimal
1022 VkShaderModule modules[ZINK_GFX_SHADER_COUNT];
1023 VkPipeline pipeline;
1024 };
1025
1026 struct zink_gfx_input_key {
1027 union {
1028 struct {
1029 unsigned idx:8;
1030 bool uses_dynamic_stride;
1031 };
1032 uint32_t input;
1033 };
1034 uint32_t vertex_buffers_enabled_mask;
1035 uint32_t vertex_strides[PIPE_MAX_ATTRIBS];
1036 struct zink_vertex_elements_hw_state *element_state;
1037 VkPipeline pipeline;
1038 };
1039
1040 struct zink_gfx_output_key {
1041 /* order matches zink_gfx_output_key */
1042 union {
1043 struct {
1044 unsigned force_persample_interp:1;
1045 uint32_t rast_samples:6;
1046 uint32_t min_samples:6;
1047 uint32_t feedback_loop : 1;
1048 uint32_t feedback_loop_zs : 1;
1049 uint32_t rast_attachment_order : 1;
1050 uint32_t rp_state : 16;
1051 };
1052 uint32_t key;
1053 };
1054
1055 /* TODO: compress these */
1056 VkSampleMask sample_mask;
1057 uint32_t blend_id;
1058 VkPipeline pipeline;
1059 };
1060
1061 struct zink_gfx_pipeline_cache_entry {
1062 struct zink_gfx_pipeline_state state;
1063 VkPipeline pipeline;
1064 struct zink_gfx_program *prog;
1065 /* GPL only */
1066 struct util_queue_fence fence;
1067 union {
1068 struct {
1069 struct zink_gfx_input_key *ikey;
1070 struct zink_gfx_library_key *gkey;
1071 struct zink_gfx_output_key *okey;
1072 VkPipeline unoptimized_pipeline;
1073 } gpl;
1074 struct zink_shader_object shobjs[ZINK_GFX_SHADER_COUNT];
1075 };
1076 };
1077
1078 struct zink_gfx_lib_cache {
1079 /* for hashing */
1080 struct zink_shader *shaders[ZINK_GFX_SHADER_COUNT];
1081 unsigned refcount;
1082 bool removed; //once removed from cache
1083 uint8_t stages_present;
1084
1085 simple_mtx_t lock;
1086 struct set libs; //zink_gfx_library_key -> VkPipeline
1087 };
1088
1089 struct zink_gfx_program {
1090 struct zink_program base;
1091
1092 bool is_separable; //not a full program
1093
1094 uint32_t stages_present; //mask of stages present in this program
1095 uint32_t stages_remaining; //mask of zink_shader remaining in this program
1096 uint32_t gfx_hash; //from ctx->gfx_hash
1097
1098 struct zink_shader *shaders[ZINK_GFX_SHADER_COUNT];
1099 struct zink_shader *last_vertex_stage;
1100 struct zink_shader_object objs[ZINK_GFX_SHADER_COUNT];
1101
1102 /* full */
1103 VkShaderEXT objects[ZINK_GFX_SHADER_COUNT];
1104 uint32_t module_hash[ZINK_GFX_SHADER_COUNT];
1105 struct blob blobs[ZINK_GFX_SHADER_COUNT];
1106 struct util_dynarray shader_cache[ZINK_GFX_SHADER_COUNT][2][2]; //normal, nonseamless cubes, inline uniforms
1107 unsigned inlined_variant_count[ZINK_GFX_SHADER_COUNT];
1108 uint32_t default_variant_hash;
1109 uint8_t inline_variants; //which stages are using inlined uniforms
1110 bool needs_inlining; // whether this program requires some uniforms to be inlined
1111 bool has_edgeflags;
1112 bool optimal_keys;
1113
1114 /* separable */
1115 struct zink_gfx_program *full_prog;
1116
1117 struct hash_table pipelines[2][11]; // [dynamic, renderpass][number of draw modes we support]
1118 uint32_t last_variant_hash;
1119
1120 uint32_t last_finalized_hash[2][4]; //[dynamic, renderpass][primtype idx]
1121 struct zink_gfx_pipeline_cache_entry *last_pipeline[2][4]; //[dynamic, renderpass][primtype idx]
1122
1123 struct zink_gfx_lib_cache *libs;
1124 };
1125
1126 struct zink_compute_program {
1127 struct zink_program base;
1128
1129 bool use_local_size;
1130 bool has_variable_shared_mem;
1131
1132 unsigned scratch_size;
1133
1134 unsigned num_inlinable_uniforms;
1135 nir_shader *nir; //only until precompile finishes
1136
1137 struct zink_shader_module *curr;
1138
1139 struct zink_shader_module *module; //base
1140 struct util_dynarray shader_cache[2]; //nonseamless cubes, inline uniforms
1141 unsigned inlined_variant_count;
1142
1143 struct zink_shader *shader;
1144 struct hash_table pipelines;
1145
1146 simple_mtx_t cache_lock; //extra lock because threads are insane and sand was not meant to think
1147
1148 VkPipeline base_pipeline;
1149 };
1150
1151
1152 /** renderpass types */
1153
1154 struct zink_rt_attrib {
1155 VkFormat format;
1156 VkSampleCountFlagBits samples;
1157 bool clear_color;
1158 union {
1159 bool clear_stencil;
1160 bool fbfetch;
1161 };
1162 bool invalid;
1163 bool needs_write;
1164 bool resolve;
1165 bool feedback_loop;
1166 };
1167
1168 struct zink_render_pass_state {
1169 union {
1170 struct {
1171 uint8_t num_cbufs : 5; /* PIPE_MAX_COLOR_BUFS = 8 */
1172 uint8_t have_zsbuf : 1;
1173 uint8_t samples:1; //for fs samplemask
1174 uint32_t num_zsresolves : 1;
1175 uint32_t num_cresolves : 24; /* PIPE_MAX_COLOR_BUFS, but this is a struct hole */
1176 };
1177 uint32_t val; //for comparison
1178 };
1179 struct zink_rt_attrib rts[PIPE_MAX_COLOR_BUFS + 1];
1180 unsigned num_rts;
1181 uint32_t clears; //for extra verification and update flagging
1182 uint16_t msaa_expand_mask;
1183 uint16_t msaa_samples; //used with VK_EXT_multisampled_render_to_single_sampled
1184 };
1185
1186 struct zink_pipeline_rt {
1187 VkFormat format;
1188 VkSampleCountFlagBits samples;
1189 };
1190
1191 struct zink_render_pass_pipeline_state {
1192 uint32_t num_attachments:14;
1193 uint32_t msaa_samples : 8;
1194 uint32_t fbfetch:1;
1195 uint32_t color_read:1;
1196 uint32_t depth_read:1;
1197 uint32_t depth_write:1;
1198 uint32_t num_cresolves:4;
1199 uint32_t num_zsresolves:1;
1200 bool samples:1; //for fs samplemask
1201 struct zink_pipeline_rt attachments[PIPE_MAX_COLOR_BUFS + 1];
1202 unsigned id;
1203 };
1204
1205 struct zink_render_pass {
1206 VkRenderPass render_pass;
1207 struct zink_render_pass_state state;
1208 unsigned pipeline_state;
1209 };
1210
1211
1212 /** resource types */
1213 struct zink_resource_object {
1214 struct pipe_reference reference;
1215
1216 VkPipelineStageFlags access_stage;
1217 VkAccessFlags access;
1218 VkPipelineStageFlags unordered_access_stage;
1219 VkAccessFlags unordered_access;
1220 VkAccessFlags last_write;
1221
1222 /* 'access' is propagated from unordered_access to handle ops occurring
1223 * in the ordered cmdbuf which can promote barriers to unordered
1224 */
1225 bool ordered_access_is_copied;
1226 bool unordered_read;
1227 bool unordered_write;
1228 bool unsync_access;
1229 bool copies_valid;
1230 bool copies_need_reset; //for use with batch state resets
1231
1232 struct u_rwlock copy_lock;
1233 struct util_dynarray copies[16]; //regions being copied to; for barrier omission
1234
1235 VkBuffer storage_buffer;
1236 simple_mtx_t view_lock;
1237 uint32_t view_prune_count; //how many views to prune
1238 uint32_t view_prune_timeline; //when to prune
1239 struct util_dynarray views;
1240
1241 union {
1242 VkBuffer buffer;
1243 VkImage image;
1244 };
1245 VkDeviceAddress bda;
1246
1247 VkSampleLocationsInfoEXT zs_evaluate;
1248 bool needs_zs_evaluate;
1249
1250 bool storage_init; //layout was set for image
1251 bool transfer_dst;
1252 bool render_target;
1253 bool is_buffer;
1254 bool exportable;
1255
1256 /* TODO: this should be a union */
1257 int handle;
1258 struct zink_bo *bo;
1259 // struct {
1260 struct kopper_displaytarget *dt;
1261 uint32_t dt_idx;
1262 uint32_t last_dt_idx;
1263 VkSemaphore present;
1264 bool new_dt;
1265 bool indefinite_acquire;
1266 // }
1267
1268
1269 unsigned miptail_commits;
1270 VkDeviceSize offset, size, alignment;
1271 uint64_t vkflags;
1272 uint64_t vkusage;
1273 VkFormatFeatureFlags vkfeats;
1274 uint64_t modifier;
1275 VkImageAspectFlags modifier_aspect;
1276 unsigned plane_offsets[3];
1277 unsigned plane_strides[3];
1278 unsigned plane_count;
1279
1280 bool host_visible;
1281 bool coherent;
1282 bool is_aux;
1283 };
1284
1285 struct zink_resource {
1286 struct threaded_resource base;
1287
1288 enum pipe_format internal_format:16;
1289
1290 struct zink_resource_object *obj;
1291 struct pipe_surface *surface; //for swapchain images
1292 uint32_t queue;
1293 union {
1294 struct {
1295 struct util_range valid_buffer_range;
1296 struct util_range *real_buffer_range; //only set on tc replace_buffer src
1297 uint32_t vbo_bind_mask : PIPE_MAX_ATTRIBS;
1298 uint8_t ubo_bind_count[2];
1299 uint8_t ssbo_bind_count[2];
1300 uint8_t vbo_bind_count;
1301 uint8_t so_bind_count; //not counted in all_binds
1302 bool so_valid;
1303 uint32_t ubo_bind_mask[MESA_SHADER_STAGES];
1304 uint32_t ssbo_bind_mask[MESA_SHADER_STAGES];
1305 };
1306 struct {
1307 bool linear;
1308 bool need_2D;
1309 bool valid;
1310 uint8_t fb_bind_count; //not counted in all_binds
1311 uint16_t fb_binds; /* mask of attachment idx; zs is PIPE_MAX_COLOR_BUFS */
1312 VkSparseImageMemoryRequirements sparse;
1313 VkFormat format;
1314 VkImageLayout layout;
1315 VkImageAspectFlags aspect;
1316 };
1317 };
1318 uint32_t sampler_binds[MESA_SHADER_STAGES];
1319 uint32_t image_binds[MESA_SHADER_STAGES];
1320 uint16_t sampler_bind_count[2]; //gfx, compute
1321 uint16_t image_bind_count[2]; //gfx, compute
1322 uint16_t write_bind_count[2]; //gfx, compute
1323 union {
1324 uint16_t bindless[2]; //tex, img
1325 uint32_t all_bindless;
1326 };
1327 union {
1328 uint16_t bind_count[2]; //gfx, compute
1329 uint32_t all_binds;
1330 };
1331
1332 VkPipelineStageFlagBits gfx_barrier;
1333 VkAccessFlagBits barrier_access[2]; //gfx, compute
1334
1335 union {
1336 struct {
1337 struct hash_table bufferview_cache;
1338 simple_mtx_t bufferview_mtx;
1339 };
1340 struct {
1341 struct hash_table surface_cache;
1342 simple_mtx_t surface_mtx;
1343 };
1344 };
1345
1346 VkRect2D damage;
1347 bool use_damage;
1348
1349 bool copies_warned;
1350 bool swapchain;
1351 bool dmabuf;
1352 unsigned dt_stride;
1353
1354 uint8_t modifiers_count;
1355 uint64_t *modifiers;
1356 };
1357
1358 static inline struct zink_resource *
zink_resource(struct pipe_resource * r)1359 zink_resource(struct pipe_resource *r)
1360 {
1361 return (struct zink_resource *)r;
1362 }
1363
1364
1365 struct zink_transfer {
1366 struct threaded_transfer base;
1367 struct pipe_resource *staging_res;
1368 unsigned offset;
1369 unsigned depthPitch;
1370 };
1371
1372
1373 /** screen types */
1374 struct zink_modifier_props {
1375 uint32_t drmFormatModifierCount;
1376 VkDrmFormatModifierPropertiesEXT* pDrmFormatModifierProperties;
1377 };
1378
1379 struct zink_format_props {
1380 VkFormatFeatureFlags2 linearTilingFeatures;
1381 VkFormatFeatureFlags2 optimalTilingFeatures;
1382 VkFormatFeatureFlags2 bufferFeatures;
1383 };
1384
1385 struct zink_screen {
1386 struct pipe_screen base;
1387
1388 const char *vendor_name;
1389 const char *device_name;
1390
1391 struct util_dl_library *loader_lib;
1392 PFN_vkGetInstanceProcAddr vk_GetInstanceProcAddr;
1393 PFN_vkGetDeviceProcAddr vk_GetDeviceProcAddr;
1394
1395 bool threaded;
1396 bool threaded_submit;
1397 bool is_cpu;
1398 bool abort_on_hang;
1399 bool frame_marker_emitted;
1400 bool driver_name_is_inferred;
1401 uint64_t curr_batch; //the current batch id
1402 uint32_t last_finished;
1403 VkSemaphore sem;
1404 VkFence fence;
1405 struct util_queue flush_queue;
1406 simple_mtx_t copy_context_lock;
1407 struct zink_context *copy_context;
1408
1409 struct zink_batch_state *free_batch_states; //unused batch states
1410 struct zink_batch_state *last_free_batch_state; //for appending
1411 simple_mtx_t free_batch_states_lock;
1412
1413 simple_mtx_t semaphores_lock;
1414 struct util_dynarray semaphores;
1415 struct util_dynarray fd_semaphores;
1416
1417 unsigned buffer_rebind_counter;
1418 unsigned image_rebind_counter;
1419 unsigned robust_ctx_count;
1420
1421 struct hash_table dts;
1422 simple_mtx_t dt_lock;
1423
1424 bool device_lost;
1425 int drm_fd;
1426
1427 struct slab_parent_pool transfer_pool;
1428 struct disk_cache *disk_cache;
1429 struct util_queue cache_put_thread;
1430 struct util_queue cache_get_thread;
1431
1432 /* there are 5 gfx stages, but VS and FS are assumed to be always present,
1433 * thus only 3 stages need to be considered, giving 2^3 = 8 program caches.
1434 */
1435 struct set pipeline_libs[8];
1436 simple_mtx_t pipeline_libs_lock[8];
1437
1438 simple_mtx_t desc_set_layouts_lock;
1439 struct hash_table desc_set_layouts[ZINK_DESCRIPTOR_BASE_TYPES];
1440 simple_mtx_t desc_pool_keys_lock;
1441 struct set desc_pool_keys[ZINK_DESCRIPTOR_BASE_TYPES];
1442 struct util_live_shader_cache shaders;
1443
1444 uint64_t db_size[ZINK_DESCRIPTOR_ALL_TYPES];
1445 unsigned base_descriptor_size;
1446 VkDescriptorSetLayout bindless_layout;
1447
1448 struct {
1449 struct pb_cache bo_cache;
1450 struct pb_slabs bo_slabs[NUM_SLAB_ALLOCATORS];
1451 unsigned min_alloc_size;
1452 uint32_t next_bo_unique_id;
1453 } pb;
1454 uint8_t heap_map[ZINK_HEAP_MAX][VK_MAX_MEMORY_TYPES]; // mapping from zink heaps to memory type indices
1455 uint8_t heap_count[ZINK_HEAP_MAX]; // number of memory types per zink heap
1456 bool resizable_bar;
1457
1458 uint64_t total_video_mem;
1459 uint64_t clamp_video_mem;
1460 uint64_t total_mem;
1461 uint64_t mapped_vram;
1462
1463 VkInstance instance;
1464 struct zink_instance_info instance_info;
1465
1466 struct hash_table *debug_mem_sizes;
1467 simple_mtx_t debug_mem_lock;
1468
1469 VkPhysicalDevice pdev;
1470 uint32_t vk_version, spirv_version;
1471 struct util_idalloc_mt buffer_ids;
1472 struct util_vertex_state_cache vertex_state_cache;
1473
1474 struct zink_device_info info;
1475 struct nir_shader_compiler_options nir_options;
1476
1477 bool optimal_keys;
1478 bool have_full_ds3;
1479 bool have_X8_D24_UNORM_PACK32;
1480 bool have_D24_UNORM_S8_UINT;
1481 bool have_D32_SFLOAT_S8_UINT;
1482 bool have_triangle_fans;
1483 bool have_dynamic_state_vertex_input_binding_stride;
1484 bool need_decompose_attrs;
1485 bool need_2D_zs;
1486 bool need_2D_sparse;
1487 bool can_hic_shader_read;
1488
1489 uint32_t gfx_queue;
1490 uint32_t sparse_queue;
1491 uint32_t max_queues;
1492 uint32_t timestamp_valid_bits;
1493 VkDevice dev;
1494 VkQueue queue; //gfx+compute
1495 VkQueue queue_sparse;
1496 simple_mtx_t queue_lock;
1497 VkDebugUtilsMessengerEXT debugUtilsCallbackHandle;
1498
1499 uint32_t cur_custom_border_color_samplers;
1500
1501 unsigned screen_id;
1502
1503 RENDERDOC_API_1_0_0 *renderdoc_api;
1504 unsigned renderdoc_capture_start;
1505 unsigned renderdoc_capture_end;
1506 unsigned renderdoc_frame;
1507 bool renderdoc_capturing;
1508 bool renderdoc_capture_all;
1509
1510 struct vk_uncompacted_dispatch_table vk;
1511
1512 void (*buffer_barrier)(struct zink_context *ctx, struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline);
1513 void (*image_barrier)(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline);
1514 void (*image_barrier_unsync)(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline);
1515
1516 bool compact_descriptors; /**< toggled if descriptor set ids are compacted */
1517 uint8_t desc_set_id[ZINK_MAX_DESCRIPTOR_SETS]; /**< converts enum zink_descriptor_type -> the actual set id */
1518
1519 struct {
1520 bool dual_color_blend_by_location;
1521 bool inline_uniforms;
1522 bool emulate_point_smooth;
1523 bool zink_shader_object_enable;
1524 } driconf;
1525
1526 struct zink_format_props format_props[PIPE_FORMAT_COUNT];
1527 struct zink_modifier_props modifier_props[PIPE_FORMAT_COUNT];
1528 bool format_props_init[PIPE_FORMAT_COUNT];
1529
1530 VkExtent2D maxSampleLocationGridSize[5];
1531 VkPipelineLayout gfx_push_constant_layout;
1532
1533 struct {
1534 /* these affect shader cache */
1535 bool lower_robustImageAccess2;
1536 bool needs_zs_shader_swizzle;
1537 bool needs_sanitised_layer;
1538 bool io_opt;
1539 bool broken_const;
1540 } driver_compiler_workarounds;
1541 struct {
1542 bool broken_l4a4;
1543 /* https://gitlab.khronos.org/vulkan/vulkan/-/issues/3306
1544 * HI TURNIP
1545 */
1546 bool broken_cache_semantics;
1547 bool missing_a8_unorm;
1548 bool implicit_sync;
1549 bool disable_optimized_compile;
1550 bool always_feedback_loop;
1551 bool always_feedback_loop_zs;
1552 bool track_renderpasses;
1553 bool no_linestipple;
1554 bool no_linesmooth;
1555 bool no_hw_gl_point;
1556 bool can_do_invalid_linear_modifier;
1557 bool inconsistent_interpolation;
1558 bool can_2d_view_sparse;
1559 bool general_depth_layout;
1560 unsigned z16_unscaled_bias;
1561 unsigned z24_unscaled_bias;
1562 } driver_workarounds;
1563 };
1564
1565 static inline struct zink_screen *
zink_screen(struct pipe_screen * pipe)1566 zink_screen(struct pipe_screen *pipe)
1567 {
1568 return (struct zink_screen *)pipe;
1569 }
1570
1571 /** surface types */
1572
1573 /* info for validating/creating imageless framebuffers */
1574 struct zink_surface_info {
1575 VkImageCreateFlags flags;
1576 VkImageUsageFlags usage;
1577 uint32_t width;
1578 uint32_t height;
1579 uint32_t layerCount;
1580 VkFormat format[2]; //base format, srgb format (for srgb framebuffer)
1581 };
1582
1583 /* an imageview for a zink_resource:
1584 - may be a fb attachment, samplerview, or shader image
1585 - cached on the parent zink_resource_object
1586 - also handles swapchains
1587 */
1588 struct zink_surface {
1589 struct pipe_surface base;
1590 /* all the info for creating a new imageview */
1591 VkImageViewCreateInfo ivci;
1592 VkImageViewUsageCreateInfo usage_info;
1593 /* for framebuffer use */
1594 struct zink_surface_info info;
1595 bool is_swapchain;
1596 /* the current imageview */
1597 VkImageView image_view;
1598 /* array of imageviews for swapchains, one for each image */
1599 VkImageView *swapchain;
1600 unsigned swapchain_size;
1601 void *obj; //backing resource object; used to determine rebinds
1602 void *dt_swapchain; //current swapchain object; used to determine swapchain rebinds
1603 uint32_t hash; //for surface caching
1604 };
1605
1606 /* wrapper object that preserves the gallium expectation of having
1607 * pipe_surface::context match the context used to create the surface
1608 */
1609 struct zink_ctx_surface {
1610 struct pipe_surface base;
1611 struct zink_surface *surf; //the actual surface
1612 struct zink_ctx_surface *transient; //for use with EXT_multisample_render_to_texture
1613 bool transient_init; //whether the transient surface has data
1614 bool needs_mutable;
1615 };
1616
1617 /* use this cast for framebuffer surfaces */
1618 static inline struct zink_surface *
zink_csurface(struct pipe_surface * psurface)1619 zink_csurface(struct pipe_surface *psurface)
1620 {
1621 return psurface ? ((struct zink_ctx_surface *)psurface)->surf : NULL;
1622 }
1623
1624 /* use this cast for checking transient framebuffer surfaces */
1625 static inline struct zink_surface *
zink_transient_surface(struct pipe_surface * psurface)1626 zink_transient_surface(struct pipe_surface *psurface)
1627 {
1628 return psurface ? ((struct zink_ctx_surface *)psurface)->transient ? ((struct zink_ctx_surface *)psurface)->transient->surf : NULL : NULL;
1629 }
1630
1631 /* use this cast for internal surfaces */
1632 static inline struct zink_surface *
zink_surface(struct pipe_surface * psurface)1633 zink_surface(struct pipe_surface *psurface)
1634 {
1635 return (struct zink_surface *)psurface;
1636 }
1637
1638
1639 /** framebuffer types */
1640 struct zink_framebuffer_state {
1641 uint32_t width;
1642 uint16_t height;
1643 uint32_t layers:6;
1644 uint32_t samples:6;
1645 uint32_t num_attachments:4;
1646 struct zink_surface_info infos[PIPE_MAX_COLOR_BUFS + 1];
1647 };
1648
1649 struct zink_framebuffer {
1650 struct pipe_reference reference;
1651
1652 /* current objects */
1653 VkFramebuffer fb;
1654 struct zink_render_pass *rp;
1655
1656 struct zink_framebuffer_state state;
1657 VkFramebufferAttachmentImageInfo infos[PIPE_MAX_COLOR_BUFS + 1];
1658 struct hash_table objects;
1659 };
1660
1661
1662 /** context types */
1663 struct zink_sampler_state {
1664 VkSampler sampler;
1665 VkSampler sampler_clamped;
1666 bool custom_border_color;
1667 bool emulate_nonseamless;
1668 };
1669
1670 struct zink_buffer_view {
1671 struct pipe_reference reference;
1672 struct pipe_resource *pres;
1673 VkBufferViewCreateInfo bvci;
1674 VkBufferView buffer_view;
1675 uint32_t hash;
1676 };
1677
1678 struct zink_sampler_view {
1679 struct pipe_sampler_view base;
1680 union {
1681 struct zink_surface *image_view;
1682 struct zink_buffer_view *buffer_view;
1683 unsigned tbo_size;
1684 };
1685 struct zink_surface *cube_array;
1686 /* Optional sampler view returning red (depth) in all channels, for shader rewrites. */
1687 struct zink_surface *zs_view;
1688 struct zink_zs_swizzle swizzle;
1689 };
1690
1691 struct zink_image_view {
1692 struct pipe_image_view base;
1693 union {
1694 struct zink_surface *surface;
1695 struct zink_buffer_view *buffer_view;
1696 };
1697 };
1698
1699 static inline struct zink_sampler_view *
zink_sampler_view(struct pipe_sampler_view * pview)1700 zink_sampler_view(struct pipe_sampler_view *pview)
1701 {
1702 return (struct zink_sampler_view *)pview;
1703 }
1704
1705 struct zink_so_target {
1706 struct pipe_stream_output_target base;
1707 struct pipe_resource *counter_buffer;
1708 VkDeviceSize counter_buffer_offset;
1709 uint32_t stride;
1710 bool counter_buffer_valid;
1711 };
1712
1713 static inline struct zink_so_target *
zink_so_target(struct pipe_stream_output_target * so_target)1714 zink_so_target(struct pipe_stream_output_target *so_target)
1715 {
1716 return (struct zink_so_target *)so_target;
1717 }
1718
1719 struct zink_viewport_state {
1720 struct pipe_viewport_state viewport_states[PIPE_MAX_VIEWPORTS];
1721 struct pipe_scissor_state scissor_states[PIPE_MAX_VIEWPORTS];
1722 uint8_t num_viewports;
1723 };
1724
1725 struct zink_descriptor_db_info {
1726 unsigned offset;
1727 unsigned size;
1728 enum pipe_format format;
1729 struct pipe_resource *pres;
1730 };
1731
1732 struct zink_descriptor_surface {
1733 union {
1734 struct zink_surface *surface;
1735 struct zink_buffer_view *bufferview;
1736 struct zink_descriptor_db_info db;
1737 };
1738 bool is_buffer;
1739 };
1740
1741 struct zink_bindless_descriptor {
1742 struct zink_descriptor_surface ds;
1743 struct zink_sampler_state *sampler;
1744 uint32_t handle;
1745 uint32_t access; //PIPE_ACCESS_...
1746 };
1747
1748 struct zink_rendering_info {
1749 VkPipelineRenderingCreateInfo info;
1750 unsigned id;
1751 };
1752
1753
1754 typedef void (*pipe_draw_vertex_state_func)(struct pipe_context *ctx,
1755 struct pipe_vertex_state *vstate,
1756 uint32_t partial_velem_mask,
1757 struct pipe_draw_vertex_state_info info,
1758 const struct pipe_draw_start_count_bias *draws,
1759 unsigned num_draws);
1760 typedef void (*pipe_launch_grid_func)(struct pipe_context *pipe, const struct pipe_grid_info *info);
1761
1762
1763 enum zink_ds3_state {
1764 ZINK_DS3_RAST_STIPPLE,
1765 ZINK_DS3_RAST_CLIP,
1766 ZINK_DS3_RAST_CLAMP,
1767 ZINK_DS3_RAST_POLYGON,
1768 ZINK_DS3_RAST_HALFZ,
1769 ZINK_DS3_RAST_PV,
1770 ZINK_DS3_RAST_LINE,
1771 ZINK_DS3_RAST_STIPPLE_ON,
1772 ZINK_DS3_BLEND_A2C,
1773 ZINK_DS3_BLEND_A21,
1774 ZINK_DS3_BLEND_ON,
1775 ZINK_DS3_BLEND_WRITE,
1776 ZINK_DS3_BLEND_EQ,
1777 ZINK_DS3_BLEND_LOGIC_ON,
1778 ZINK_DS3_BLEND_LOGIC,
1779 };
1780
1781 struct zink_context {
1782 struct pipe_context base;
1783 struct threaded_context *tc;
1784 struct slab_child_pool transfer_pool;
1785 struct slab_child_pool transfer_pool_unsync;
1786 struct blitter_context *blitter;
1787 struct util_debug_callback dbg;
1788
1789 unsigned flags;
1790
1791 pipe_draw_func draw_vbo[2]; //batch changed
1792 pipe_draw_vertex_state_func draw_state[2]; //batch changed
1793 pipe_launch_grid_func launch_grid[2]; //batch changed
1794
1795 struct pipe_device_reset_callback reset;
1796
1797 struct util_queue_fence unsync_fence; //unsigned during unsync recording (blocks flush ops)
1798 struct util_queue_fence flush_fence; //unsigned during flush (blocks unsync ops)
1799
1800 struct zink_fence *deferred_fence;
1801 struct zink_batch_state *last_batch_state; //the last command buffer submitted
1802 struct zink_batch_state *batch_states; //list of submitted batch states: ordered by increasing timeline id
1803 unsigned batch_states_count; //number of states in `batch_states`
1804 struct zink_batch_state *free_batch_states; //unused batch states
1805 struct zink_batch_state *last_free_batch_state; //for appending
1806 bool oom_flush;
1807 bool oom_stall;
1808 bool track_renderpasses;
1809 bool no_reorder;
1810 struct zink_batch_state *bs;
1811
1812 unsigned shader_has_inlinable_uniforms_mask;
1813 unsigned inlinable_uniforms_valid_mask;
1814
1815 struct pipe_constant_buffer ubos[MESA_SHADER_STAGES][PIPE_MAX_CONSTANT_BUFFERS];
1816 struct pipe_shader_buffer ssbos[MESA_SHADER_STAGES][PIPE_MAX_SHADER_BUFFERS];
1817 uint32_t writable_ssbos[MESA_SHADER_STAGES];
1818 struct zink_image_view image_views[MESA_SHADER_STAGES][ZINK_MAX_SHADER_IMAGES];
1819
1820 uint32_t transient_attachments;
1821 struct pipe_framebuffer_state fb_state;
1822 struct hash_table framebuffer_cache;
1823
1824 struct zink_vertex_elements_state *element_state;
1825 struct zink_rasterizer_state *rast_state;
1826 struct zink_depth_stencil_alpha_state *dsa_state;
1827
1828 bool pipeline_changed[2]; //gfx, compute
1829
1830 struct zink_shader *gfx_stages[ZINK_GFX_SHADER_COUNT];
1831 struct zink_shader *last_vertex_stage;
1832 bool shader_reads_drawid;
1833 bool shader_reads_basevertex;
1834 struct zink_gfx_pipeline_state gfx_pipeline_state;
1835 /* there are 5 gfx stages, but VS and FS are assumed to be always present,
1836 * thus only 3 stages need to be considered, giving 2^3 = 8 program caches.
1837 */
1838 struct hash_table program_cache[8];
1839 simple_mtx_t program_lock[8];
1840 uint32_t gfx_hash;
1841 struct zink_gfx_program *curr_program;
1842 struct set gfx_inputs;
1843 struct set gfx_outputs;
1844
1845 struct zink_descriptor_data dd;
1846
1847 struct zink_compute_pipeline_state compute_pipeline_state;
1848 struct zink_compute_program *curr_compute;
1849
1850 unsigned shader_stages : ZINK_GFX_SHADER_COUNT; /* mask of bound gfx shader stages */
1851 uint8_t dirty_gfx_stages; /* mask of changed gfx shader stages */
1852 bool last_vertex_stage_dirty;
1853 bool compute_dirty;
1854 bool is_generated_gs_bound;
1855
1856 struct {
1857 VkRenderingAttachmentInfo attachments[PIPE_MAX_COLOR_BUFS + 2]; //+depth, +stencil
1858 VkRenderingInfo info;
1859 struct tc_renderpass_info tc_info;
1860 } dynamic_fb;
1861 uint32_t fb_layer_mismatch; //bitmask
1862 unsigned depth_bias_scale_factor;
1863 struct set rendering_state_cache[6]; //[util_logbase2_ceil(msrtss samplecount)]
1864 struct set render_pass_state_cache;
1865 struct hash_table *render_pass_cache;
1866 struct zink_resource *swapchain;
1867 VkExtent2D swapchain_size;
1868 bool fb_changed;
1869 bool in_rp; //renderpass is currently active
1870 bool rp_changed; //force renderpass restart
1871 bool rp_layout_changed; //renderpass changed, maybe restart
1872 bool rp_loadop_changed; //renderpass changed, don't restart
1873 bool zsbuf_unused;
1874 bool zsbuf_readonly;
1875
1876 struct zink_framebuffer *framebuffer;
1877 struct zink_framebuffer_clear fb_clears[PIPE_MAX_COLOR_BUFS + 1];
1878 uint16_t clears_enabled;
1879 uint16_t rp_clears_enabled;
1880 uint16_t void_clears;
1881 uint16_t fbfetch_outputs;
1882 uint16_t feedback_loops;
1883 struct zink_resource *needs_present;
1884
1885 struct pipe_vertex_buffer vertex_buffers[PIPE_MAX_ATTRIBS];
1886 bool vertex_buffers_dirty;
1887
1888 struct zink_sampler_state *sampler_states[MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS];
1889 struct pipe_sampler_view *sampler_views[MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS];
1890
1891 struct zink_viewport_state vp_state;
1892 bool vp_state_changed;
1893 bool scissor_changed;
1894
1895 float blend_constants[4];
1896
1897 bool sample_locations_changed;
1898 VkSampleLocationEXT vk_sample_locations[PIPE_MAX_SAMPLE_LOCATION_GRID_SIZE * PIPE_MAX_SAMPLE_LOCATION_GRID_SIZE];
1899 uint8_t sample_locations[2 * 4 * 8 * 16];
1900
1901 struct pipe_stencil_ref stencil_ref;
1902
1903 union {
1904 struct {
1905 float default_inner_level[2];
1906 float default_outer_level[4];
1907 };
1908 float tess_levels[6];
1909 };
1910
1911 struct zink_vk_query *curr_xfb_queries[PIPE_MAX_VERTEX_STREAMS];
1912 struct zink_shader *null_fs;
1913 struct zink_shader *saved_fs;
1914
1915 struct list_head query_pools;
1916 struct list_head suspended_queries;
1917 struct list_head primitives_generated_queries;
1918 struct zink_query *vertices_query;
1919 bool disable_fs;
1920 bool disable_color_writes;
1921 bool was_line_loop;
1922 bool fs_query_active;
1923 bool occlusion_query_active;
1924 bool primitives_generated_active;
1925 bool primitives_generated_suspended;
1926 bool queries_disabled, render_condition_active;
1927 bool queries_in_rp;
1928 struct {
1929 struct zink_query *query;
1930 bool inverted;
1931 bool active; //this is the internal vk state
1932 } render_condition;
1933 struct {
1934 uint64_t render_passes;
1935 } hud;
1936
1937 struct pipe_resource *dummy_vertex_buffer;
1938 struct pipe_resource *dummy_xfb_buffer;
1939 struct pipe_surface *dummy_surface[7];
1940 struct zink_buffer_view *dummy_bufferview;
1941
1942 unsigned buffer_rebind_counter;
1943 unsigned image_rebind_counter;
1944
1945 struct {
1946 /* descriptor info */
1947 uint8_t num_ubos[MESA_SHADER_STAGES];
1948
1949 uint8_t num_ssbos[MESA_SHADER_STAGES];
1950 struct util_dynarray global_bindings;
1951
1952 VkDescriptorImageInfo textures[MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS];
1953 uint32_t emulate_nonseamless[MESA_SHADER_STAGES];
1954 uint32_t cubes[MESA_SHADER_STAGES];
1955 uint8_t num_samplers[MESA_SHADER_STAGES];
1956 uint8_t num_sampler_views[MESA_SHADER_STAGES];
1957
1958 VkDescriptorImageInfo images[MESA_SHADER_STAGES][ZINK_MAX_SHADER_IMAGES];
1959 uint8_t num_images[MESA_SHADER_STAGES];
1960
1961 union {
1962 struct {
1963 VkDescriptorBufferInfo ubos[MESA_SHADER_STAGES][PIPE_MAX_CONSTANT_BUFFERS];
1964 VkDescriptorBufferInfo ssbos[MESA_SHADER_STAGES][PIPE_MAX_SHADER_BUFFERS];
1965 VkBufferView tbos[MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS];
1966 VkBufferView texel_images[MESA_SHADER_STAGES][ZINK_MAX_SHADER_IMAGES];
1967 } t;
1968 struct {
1969 VkDescriptorAddressInfoEXT ubos[MESA_SHADER_STAGES][PIPE_MAX_CONSTANT_BUFFERS];
1970 VkDescriptorAddressInfoEXT ssbos[MESA_SHADER_STAGES][PIPE_MAX_SHADER_BUFFERS];
1971 VkDescriptorAddressInfoEXT tbos[MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS];
1972 VkDescriptorAddressInfoEXT texel_images[MESA_SHADER_STAGES][ZINK_MAX_SHADER_IMAGES];
1973 } db;
1974 };
1975
1976 VkDescriptorImageInfo fbfetch;
1977 uint8_t fbfetch_db[ZINK_FBFETCH_DESCRIPTOR_SIZE];
1978
1979 /* the current state of the zs swizzle data */
1980 struct zink_zs_swizzle_key zs_swizzle[MESA_SHADER_STAGES];
1981
1982 struct zink_resource *descriptor_res[ZINK_DESCRIPTOR_BASE_TYPES][MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS];
1983
1984 struct {
1985 struct util_idalloc tex_slots; //img, buffer
1986 struct util_idalloc img_slots; //img, buffer
1987 struct hash_table tex_handles; //img, buffer
1988 struct hash_table img_handles; //img, buffer
1989 union {
1990 struct {
1991 VkBufferView *buffer_infos; //tex, img
1992 } t;
1993 struct {
1994 VkDescriptorAddressInfoEXT *buffer_infos;
1995 } db;
1996 };
1997 VkDescriptorImageInfo *img_infos; //tex, img
1998 struct util_dynarray updates; //texture, img
1999 struct util_dynarray resident; //texture, img
2000 } bindless[2];
2001 union {
2002 bool bindless_dirty[2]; //tex, img
2003 uint16_t any_bindless_dirty;
2004 };
2005 bool bindless_refs_dirty;
2006 bool null_fbfetch_init;
2007 } di;
2008 void (*invalidate_descriptor_state)(struct zink_context *ctx, gl_shader_stage shader, enum zink_descriptor_type type, unsigned, unsigned);
2009 struct set *need_barriers[2]; //gfx, compute
2010 struct set update_barriers[2][2]; //[gfx, compute][current, next]
2011 uint8_t barrier_set_idx[2];
2012 unsigned memory_barrier;
2013
2014 uint32_t ds3_states;
2015 unsigned work_count;
2016
2017 uint32_t num_so_targets;
2018 struct pipe_stream_output_target *so_targets[PIPE_MAX_SO_BUFFERS];
2019 bool dirty_so_targets;
2020
2021 bool gfx_dirty;
2022
2023 bool shobj_draw : 1; //using shader objects for draw
2024 bool is_device_lost;
2025 bool primitive_restart;
2026 bool blitting : 1;
2027 bool blit_scissor : 1;
2028 bool blit_nearest : 1;
2029 bool unordered_blitting : 1;
2030 bool vertex_state_changed : 1;
2031 bool blend_state_changed : 1;
2032 bool blend_color_changed : 1;
2033 bool sample_mask_changed : 1;
2034 bool rast_state_changed : 1;
2035 bool line_width_changed : 1;
2036 bool dsa_state_changed : 1;
2037 bool stencil_ref_changed : 1;
2038 bool rasterizer_discard_changed : 1;
2039 bool rp_tc_info_updated : 1;
2040 bool last_work_was_compute : 1;
2041 };
2042
2043 static inline struct zink_context *
zink_context(struct pipe_context * context)2044 zink_context(struct pipe_context *context)
2045 {
2046 return (struct zink_context *)context;
2047 }
2048
2049 #endif
2050