1 /*
2 * Copyright © 2021 Collabora Ltd.
3 *
4 * derived from tu_private.h driver which is:
5 * Copyright © 2016 Red Hat.
6 * Copyright © 2016 Bas Nieuwenhuizen
7 * Copyright © 2015 Intel Corporation
8 *
9 * Permission is hereby granted, free of charge, to any person obtaining a
10 * copy of this software and associated documentation files (the "Software"),
11 * to deal in the Software without restriction, including without limitation
12 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
13 * and/or sell copies of the Software, and to permit persons to whom the
14 * Software is furnished to do so, subject to the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the next
17 * paragraph) shall be included in all copies or substantial portions of the
18 * Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
27 */
28
29 #ifndef PANVK_PRIVATE_H
30 #define PANVK_PRIVATE_H
31
32 #include <assert.h>
33 #include <pthread.h>
34 #include <stdbool.h>
35 #include <stdint.h>
36 #include <stdio.h>
37 #include <stdlib.h>
38 #include <string.h>
39 #ifdef HAVE_VALGRIND
40 #include <memcheck.h>
41 #include <valgrind.h>
42 #define VG(x) x
43 #else
44 #define VG(x)
45 #endif
46
47 #include "c11/threads.h"
48 #include "compiler/shader_enums.h"
49 #include "util/list.h"
50 #include "util/macros.h"
51 #include "vk_alloc.h"
52 #include "vk_buffer.h"
53 #include "vk_command_buffer.h"
54 #include "vk_command_pool.h"
55 #include "vk_descriptor_set_layout.h"
56 #include "vk_device.h"
57 #include "vk_image.h"
58 #include "vk_instance.h"
59 #include "vk_log.h"
60 #include "vk_object.h"
61 #include "vk_physical_device.h"
62 #include "vk_pipeline_layout.h"
63 #include "vk_queue.h"
64 #include "vk_sync.h"
65 #include "wsi_common.h"
66
67 #include "drm-uapi/panfrost_drm.h"
68
69 #include "pan_blend.h"
70 #include "pan_blitter.h"
71 #include "pan_desc.h"
72 #include "pan_jc.h"
73 #include "pan_texture.h"
74 #include "panvk_mempool.h"
75 #include "panvk_varyings.h"
76 #include "vk_extensions.h"
77
78 #include "kmod/pan_kmod.h"
79
80 /* Pre-declarations needed for WSI entrypoints */
81 struct wl_surface;
82 struct wl_display;
83 typedef struct xcb_connection_t xcb_connection_t;
84 typedef uint32_t xcb_visualid_t;
85 typedef uint32_t xcb_window_t;
86
87 #include <vulkan/vk_android_native_buffer.h>
88 #include <vulkan/vk_icd.h>
89 #include <vulkan/vulkan.h>
90
91 #include "panvk_entrypoints.h"
92
93 #define MAX_BIND_POINTS 2 /* compute + graphics */
94 #define MAX_VBS 16
95 #define MAX_VERTEX_ATTRIBS 16
96 #define MAX_RTS 8
97 #define MAX_VSC_PIPES 32
98 #define MAX_VIEWPORTS 1
99 #define MAX_SCISSORS 16
100 #define MAX_DISCARD_RECTANGLES 4
101 #define MAX_PUSH_CONSTANTS_SIZE 128
102 #define MAX_PUSH_DESCRIPTORS 32
103 #define MAX_DYNAMIC_UNIFORM_BUFFERS 16
104 #define MAX_DYNAMIC_STORAGE_BUFFERS 8
105 #define MAX_DYNAMIC_BUFFERS \
106 (MAX_DYNAMIC_UNIFORM_BUFFERS + MAX_DYNAMIC_STORAGE_BUFFERS)
107 #define MAX_SAMPLES_LOG2 4
108 #define NUM_META_FS_KEYS 13
109 #define MAX_VIEWS 8
110
111 #define NUM_DEPTH_CLEAR_PIPELINES 3
112
113 #define PANVK_SYSVAL_UBO_INDEX 0
114 #define PANVK_PUSH_CONST_UBO_INDEX 1
115 #define PANVK_NUM_BUILTIN_UBOS 2
116
117 #define panvk_stub() assert(!"stub")
118
119 struct panvk_device;
120
121 /* Used for internal object allocation. */
122 struct panvk_priv_bo {
123 struct panvk_device *dev;
124 struct pan_kmod_bo *bo;
125 struct {
126 mali_ptr dev;
127 void *host;
128 } addr;
129 };
130
131 struct panvk_priv_bo *panvk_priv_bo_create(struct panvk_device *dev,
132 size_t size, uint32_t flags,
133 const VkAllocationCallbacks *alloc,
134 VkSystemAllocationScope scope);
135
136 void panvk_priv_bo_destroy(struct panvk_priv_bo *bo,
137 const VkAllocationCallbacks *alloc);
138
139 #define PANVK_META_COPY_BUF2IMG_NUM_FORMATS 12
140 #define PANVK_META_COPY_IMG2BUF_NUM_FORMATS 12
141 #define PANVK_META_COPY_IMG2IMG_NUM_FORMATS 14
142 #define PANVK_META_COPY_NUM_TEX_TYPES 5
143 #define PANVK_META_COPY_BUF2BUF_NUM_BLKSIZES 5
144
145 static inline unsigned
panvk_meta_copy_tex_type(unsigned dim,bool isarray)146 panvk_meta_copy_tex_type(unsigned dim, bool isarray)
147 {
148 assert(dim > 0 && dim <= 3);
149 assert(dim < 3 || !isarray);
150 return (((dim - 1) << 1) | (isarray ? 1 : 0));
151 }
152
153 struct panvk_meta {
154
155 struct panvk_pool bin_pool;
156 struct panvk_pool desc_pool;
157
158 /* Access to the blitter pools are protected by the blitter
159 * shader/rsd locks. They can't be merged with other binary/desc
160 * pools unless we patch pan_blitter.c to external pool locks.
161 */
162 struct {
163 struct panvk_pool bin_pool;
164 struct panvk_pool desc_pool;
165 struct pan_blitter_cache cache;
166 } blitter;
167
168 struct pan_blend_shader_cache blend_shader_cache;
169
170 struct {
171 struct {
172 mali_ptr shader;
173 struct pan_shader_info shader_info;
174 } color[3]; /* 3 base types */
175 } clear_attachment;
176
177 struct {
178 struct {
179 mali_ptr rsd;
180 } buf2img[PANVK_META_COPY_BUF2IMG_NUM_FORMATS];
181 struct {
182 mali_ptr rsd;
183 } img2buf[PANVK_META_COPY_NUM_TEX_TYPES]
184 [PANVK_META_COPY_IMG2BUF_NUM_FORMATS];
185 struct {
186 mali_ptr rsd;
187 } img2img[2][PANVK_META_COPY_NUM_TEX_TYPES]
188 [PANVK_META_COPY_IMG2IMG_NUM_FORMATS];
189 struct {
190 mali_ptr rsd;
191 } buf2buf[PANVK_META_COPY_BUF2BUF_NUM_BLKSIZES];
192 struct {
193 mali_ptr rsd;
194 } fillbuf;
195 } copy;
196 };
197
198 struct panvk_physical_device {
199 struct vk_physical_device vk;
200
201 struct {
202 struct pan_kmod_dev *dev;
203 struct pan_kmod_dev_props props;
204 } kmod;
205
206 const struct panfrost_model *model;
207 struct {
208 const struct pan_blendable_format *blendable;
209 const struct panfrost_format *all;
210 } formats;
211
212 struct panvk_instance *instance;
213
214 char name[VK_MAX_PHYSICAL_DEVICE_NAME_SIZE];
215 uint8_t driver_uuid[VK_UUID_SIZE];
216 uint8_t device_uuid[VK_UUID_SIZE];
217 uint8_t cache_uuid[VK_UUID_SIZE];
218
219 struct vk_sync_type drm_syncobj_type;
220 const struct vk_sync_type *sync_types[2];
221
222 struct wsi_device wsi_device;
223
224 int master_fd;
225 };
226
227 enum panvk_debug_flags {
228 PANVK_DEBUG_STARTUP = 1 << 0,
229 PANVK_DEBUG_NIR = 1 << 1,
230 PANVK_DEBUG_TRACE = 1 << 2,
231 PANVK_DEBUG_SYNC = 1 << 3,
232 PANVK_DEBUG_AFBC = 1 << 4,
233 PANVK_DEBUG_LINEAR = 1 << 5,
234 PANVK_DEBUG_DUMP = 1 << 6,
235 PANVK_DEBUG_NO_KNOWN_WARN = 1 << 7,
236 };
237
238 struct panvk_instance {
239 struct vk_instance vk;
240
241 uint32_t api_version;
242
243 enum panvk_debug_flags debug_flags;
244
245 struct {
246 struct pan_kmod_allocator allocator;
247 } kmod;
248 };
249
250 VkResult panvk_wsi_init(struct panvk_physical_device *physical_device);
251 void panvk_wsi_finish(struct panvk_physical_device *physical_device);
252
253 bool panvk_instance_extension_supported(const char *name);
254 uint32_t panvk_physical_device_api_version(struct panvk_physical_device *dev);
255 bool
256 panvk_physical_device_extension_supported(struct panvk_physical_device *dev,
257 const char *name);
258
259 struct panvk_pipeline_cache {
260 struct vk_object_base base;
261 VkAllocationCallbacks alloc;
262 };
263
264 #define PANVK_MAX_QUEUE_FAMILIES 1
265
266 struct panvk_queue {
267 struct vk_queue vk;
268 struct panvk_device *device;
269 uint32_t sync;
270 };
271
272 struct panvk_device {
273 struct vk_device vk;
274
275 struct {
276 struct pan_kmod_vm *vm;
277 struct pan_kmod_dev *dev;
278 struct pan_kmod_allocator allocator;
279 } kmod;
280
281 struct panvk_priv_bo *tiler_heap;
282 struct panvk_priv_bo *sample_positions;
283
284 struct panvk_meta meta;
285
286 struct vk_device_dispatch_table cmd_dispatch;
287
288 struct panvk_instance *instance;
289
290 struct panvk_queue *queues[PANVK_MAX_QUEUE_FAMILIES];
291 int queue_count[PANVK_MAX_QUEUE_FAMILIES];
292
293 struct panvk_physical_device *physical_device;
294
295 struct {
296 struct pandecode_context *decode_ctx;
297 } debug;
298
299 int _lost;
300 };
301
302 VkResult _panvk_device_set_lost(struct panvk_device *device, const char *file,
303 int line, const char *msg, ...)
304 PRINTFLIKE(4, 5);
305 #define panvk_device_set_lost(dev, ...) \
306 _panvk_device_set_lost(dev, __FILE__, __LINE__, __VA_ARGS__)
307
308 static inline bool
panvk_device_is_lost(struct panvk_device * device)309 panvk_device_is_lost(struct panvk_device *device)
310 {
311 return unlikely(p_atomic_read(&device->_lost));
312 }
313
314 #define TILER_DESC_WORDS 56
315
316 struct panvk_batch {
317 struct list_head node;
318 struct util_dynarray jobs;
319 struct util_dynarray event_ops;
320 struct pan_jc jc;
321 struct {
322 const struct panvk_framebuffer *info;
323 struct panfrost_ptr desc;
324 } fb;
325 struct {
326 struct pan_kmod_bo *src, *dst;
327 } blit;
328 struct panfrost_ptr tls;
329 mali_ptr fragment_job;
330 struct {
331 struct pan_tiler_context ctx;
332 struct panfrost_ptr descs;
333 uint32_t templ[TILER_DESC_WORDS];
334 } tiler;
335 struct pan_tls_info tlsinfo;
336 unsigned wls_total_size;
337 bool issued;
338 };
339
340 enum panvk_event_op_type {
341 PANVK_EVENT_OP_SET,
342 PANVK_EVENT_OP_RESET,
343 PANVK_EVENT_OP_WAIT,
344 };
345
346 struct panvk_event_op {
347 enum panvk_event_op_type type;
348 struct panvk_event *event;
349 };
350
351 struct panvk_device_memory {
352 struct vk_object_base base;
353 struct pan_kmod_bo *bo;
354 struct {
355 mali_ptr dev;
356 void *host;
357 } addr;
358 };
359
360 struct panvk_buffer_desc {
361 struct panvk_buffer *buffer;
362 VkDeviceSize offset;
363 VkDeviceSize size;
364 };
365
366 struct panvk_descriptor_set {
367 struct vk_object_base base;
368 struct panvk_descriptor_pool *pool;
369 const struct panvk_descriptor_set_layout *layout;
370 struct panvk_buffer_desc *dyn_ssbos;
371 void *ubos;
372 struct panvk_buffer_desc *dyn_ubos;
373 void *samplers;
374 void *textures;
375 void *img_attrib_bufs;
376 uint32_t *img_fmts;
377
378 struct panvk_priv_bo *desc_bo;
379 };
380
381 #define MAX_SETS 4
382
383 struct panvk_descriptor_set_binding_layout {
384 VkDescriptorType type;
385
386 /* Number of array elements in this binding */
387 unsigned array_size;
388
389 /* Indices in the desc arrays */
390 union {
391 struct {
392 union {
393 unsigned sampler_idx;
394 unsigned img_idx;
395 };
396 unsigned tex_idx;
397 };
398 unsigned dyn_ssbo_idx;
399 unsigned ubo_idx;
400 unsigned dyn_ubo_idx;
401 };
402
403 /* Offset into the descriptor UBO where this binding starts */
404 uint32_t desc_ubo_offset;
405
406 /* Stride between descriptors in this binding in the UBO */
407 uint16_t desc_ubo_stride;
408
409 /* Shader stages affected by this set+binding */
410 uint16_t shader_stages;
411
412 struct panvk_sampler **immutable_samplers;
413 };
414
415 struct panvk_descriptor_set_layout {
416 struct vk_descriptor_set_layout vk;
417
418 /* Shader stages affected by this descriptor set */
419 uint16_t shader_stages;
420
421 unsigned num_samplers;
422 unsigned num_textures;
423 unsigned num_ubos;
424 unsigned num_dyn_ubos;
425 unsigned num_dyn_ssbos;
426 unsigned num_imgs;
427
428 /* Size of the descriptor UBO */
429 uint32_t desc_ubo_size;
430
431 /* Index of the descriptor UBO */
432 unsigned desc_ubo_index;
433
434 /* Number of bindings in this descriptor set */
435 uint32_t binding_count;
436
437 /* Bindings in this descriptor set */
438 struct panvk_descriptor_set_binding_layout bindings[0];
439 };
440
441 static inline const struct panvk_descriptor_set_layout *
vk_to_panvk_descriptor_set_layout(const struct vk_descriptor_set_layout * layout)442 vk_to_panvk_descriptor_set_layout(const struct vk_descriptor_set_layout *layout)
443 {
444 return container_of(layout, const struct panvk_descriptor_set_layout, vk);
445 }
446
447 struct panvk_pipeline_layout {
448 struct vk_pipeline_layout vk;
449
450 unsigned char sha1[20];
451
452 unsigned num_samplers;
453 unsigned num_textures;
454 unsigned num_ubos;
455 unsigned num_dyn_ubos;
456 unsigned num_dyn_ssbos;
457 uint32_t num_imgs;
458
459 struct {
460 uint32_t size;
461 } push_constants;
462
463 struct {
464 unsigned sampler_offset;
465 unsigned tex_offset;
466 unsigned ubo_offset;
467 unsigned dyn_ubo_offset;
468 unsigned dyn_ssbo_offset;
469 unsigned img_offset;
470 } sets[MAX_SETS];
471 };
472
473 static unsigned
panvk_pipeline_layout_ubo_start(const struct panvk_pipeline_layout * layout,unsigned set,bool is_dynamic)474 panvk_pipeline_layout_ubo_start(const struct panvk_pipeline_layout *layout,
475 unsigned set, bool is_dynamic)
476 {
477 const struct panvk_descriptor_set_layout *set_layout =
478 vk_to_panvk_descriptor_set_layout(layout->vk.set_layouts[set]);
479
480 unsigned offset = PANVK_NUM_BUILTIN_UBOS + layout->sets[set].ubo_offset +
481 layout->sets[set].dyn_ubo_offset;
482
483 if (is_dynamic)
484 offset += set_layout->num_ubos;
485
486 return offset;
487 }
488
489 static unsigned
panvk_pipeline_layout_ubo_index(const struct panvk_pipeline_layout * layout,unsigned set,unsigned binding,unsigned array_index)490 panvk_pipeline_layout_ubo_index(const struct panvk_pipeline_layout *layout,
491 unsigned set, unsigned binding,
492 unsigned array_index)
493 {
494 const struct panvk_descriptor_set_layout *set_layout =
495 vk_to_panvk_descriptor_set_layout(layout->vk.set_layouts[set]);
496 const struct panvk_descriptor_set_binding_layout *binding_layout =
497 &set_layout->bindings[binding];
498
499 const bool is_dynamic =
500 binding_layout->type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC;
501 const uint32_t ubo_idx =
502 is_dynamic ? binding_layout->dyn_ubo_idx : binding_layout->ubo_idx;
503
504 return panvk_pipeline_layout_ubo_start(layout, set, is_dynamic) + ubo_idx +
505 array_index;
506 }
507
508 struct panvk_desc_pool_counters {
509 unsigned samplers;
510 unsigned combined_image_samplers;
511 unsigned sampled_images;
512 unsigned storage_images;
513 unsigned uniform_texel_bufs;
514 unsigned storage_texel_bufs;
515 unsigned input_attachments;
516 unsigned uniform_bufs;
517 unsigned storage_bufs;
518 unsigned uniform_dyn_bufs;
519 unsigned storage_dyn_bufs;
520 unsigned sets;
521 };
522
523 struct panvk_descriptor_pool {
524 struct vk_object_base base;
525 struct panvk_desc_pool_counters max;
526 struct panvk_desc_pool_counters cur;
527 struct panvk_descriptor_set *sets;
528 };
529
530 struct panvk_buffer {
531 struct vk_buffer vk;
532
533 mali_ptr dev_addr;
534
535 /* TODO: See if we can rework the synchronization logic so we don't need to
536 * pass BOs around.
537 */
538 struct pan_kmod_bo *bo;
539
540 /* FIXME: Only used for index buffers to do the min/max index retrieval on
541 * the CPU side. This is all broken anyway and the min/max search should be
542 * done with a compute shader that also patches the job descriptor
543 * accordingly (basically an indirect draw).
544 *
545 * Make sure this field goes away as soon as we fixed indirect draws.
546 */
547 void *host_ptr;
548 };
549
550 static inline mali_ptr
panvk_buffer_gpu_ptr(const struct panvk_buffer * buffer,uint64_t offset)551 panvk_buffer_gpu_ptr(const struct panvk_buffer *buffer, uint64_t offset)
552 {
553 if (buffer->bo == NULL)
554 return 0;
555
556 return buffer->dev_addr + offset;
557 }
558
559 static inline uint64_t
panvk_buffer_range(const struct panvk_buffer * buffer,uint64_t offset,uint64_t range)560 panvk_buffer_range(const struct panvk_buffer *buffer, uint64_t offset,
561 uint64_t range)
562 {
563 if (buffer->bo == NULL)
564 return 0;
565
566 return vk_buffer_range(&buffer->vk, offset, range);
567 }
568
569 enum panvk_dynamic_state_bits {
570 PANVK_DYNAMIC_VIEWPORT = 1 << 0,
571 PANVK_DYNAMIC_SCISSOR = 1 << 1,
572 PANVK_DYNAMIC_LINE_WIDTH = 1 << 2,
573 PANVK_DYNAMIC_DEPTH_BIAS = 1 << 3,
574 PANVK_DYNAMIC_BLEND_CONSTANTS = 1 << 4,
575 PANVK_DYNAMIC_DEPTH_BOUNDS = 1 << 5,
576 PANVK_DYNAMIC_STENCIL_COMPARE_MASK = 1 << 6,
577 PANVK_DYNAMIC_STENCIL_WRITE_MASK = 1 << 7,
578 PANVK_DYNAMIC_STENCIL_REFERENCE = 1 << 8,
579 PANVK_DYNAMIC_DISCARD_RECTANGLE = 1 << 9,
580 PANVK_DYNAMIC_SSBO = 1 << 10,
581 PANVK_DYNAMIC_VERTEX_INSTANCE_OFFSETS = 1 << 11,
582 PANVK_DYNAMIC_ALL = (1 << 12) - 1,
583 };
584
585 /* This has to match nir_address_format_64bit_bounded_global */
586 struct panvk_ssbo_addr {
587 uint64_t base_addr;
588 uint32_t size;
589 uint32_t zero; /* Must be zero! */
590 };
591
592 union panvk_sysval_vec4 {
593 float f32[4];
594 uint32_t u32[4];
595 };
596
597 struct panvk_sysvals {
598 union {
599 struct {
600 /* Only for graphics */
601 union panvk_sysval_vec4 viewport_scale;
602 union panvk_sysval_vec4 viewport_offset;
603 union panvk_sysval_vec4 blend_constants;
604
605 uint32_t first_vertex;
606 uint32_t base_vertex;
607 uint32_t base_instance;
608 };
609
610 struct {
611 /* Only for compute */
612 union panvk_sysval_vec4 num_work_groups;
613 union panvk_sysval_vec4 local_group_size;
614 };
615 };
616
617 /* The back-end compiler doesn't know about any sysvals after this point */
618
619 struct panvk_ssbo_addr dyn_ssbos[MAX_DYNAMIC_STORAGE_BUFFERS];
620 };
621
622 struct panvk_descriptor_state {
623 uint32_t dirty;
624 const struct panvk_descriptor_set *sets[MAX_SETS];
625 struct panvk_sysvals sysvals;
626 struct {
627 struct panvk_buffer_desc ubos[MAX_DYNAMIC_UNIFORM_BUFFERS];
628 struct panvk_buffer_desc ssbos[MAX_DYNAMIC_STORAGE_BUFFERS];
629 } dyn;
630 mali_ptr sysvals_ptr;
631 mali_ptr ubos;
632 mali_ptr textures;
633 mali_ptr samplers;
634 mali_ptr push_constants;
635 mali_ptr vs_attribs;
636 mali_ptr vs_attrib_bufs;
637 mali_ptr non_vs_attribs;
638 mali_ptr non_vs_attrib_bufs;
639 };
640
641 #define INVOCATION_DESC_WORDS 2
642
643 struct panvk_draw_info {
644 unsigned first_index;
645 unsigned index_count;
646 unsigned index_size;
647 unsigned first_vertex;
648 unsigned vertex_count;
649 unsigned vertex_range;
650 unsigned padded_vertex_count;
651 unsigned first_instance;
652 unsigned instance_count;
653 int vertex_offset;
654 unsigned offset_start;
655 uint32_t invocation[INVOCATION_DESC_WORDS];
656 struct {
657 mali_ptr varyings;
658 mali_ptr attributes;
659 mali_ptr attribute_bufs;
660 mali_ptr push_constants;
661 } stages[MESA_SHADER_STAGES];
662 mali_ptr varying_bufs;
663 mali_ptr textures;
664 mali_ptr samplers;
665 mali_ptr ubos;
666 mali_ptr position;
667 mali_ptr indices;
668 union {
669 mali_ptr psiz;
670 float line_width;
671 };
672 mali_ptr tls;
673 mali_ptr fb;
674 const struct pan_tiler_context *tiler_ctx;
675 mali_ptr fs_rsd;
676 mali_ptr viewport;
677 struct {
678 struct panfrost_ptr vertex;
679 struct panfrost_ptr tiler;
680 } jobs;
681 };
682
683 struct panvk_dispatch_info {
684 struct pan_compute_dim wg_count;
685 mali_ptr attributes;
686 mali_ptr attribute_bufs;
687 mali_ptr tsd;
688 mali_ptr ubos;
689 mali_ptr push_uniforms;
690 mali_ptr textures;
691 mali_ptr samplers;
692 };
693
694 struct panvk_attrib_info {
695 unsigned buf;
696 unsigned offset;
697 enum pipe_format format;
698 };
699
700 struct panvk_attrib_buf_info {
701 bool special;
702 union {
703 struct {
704 unsigned stride;
705 bool per_instance;
706 uint32_t instance_divisor;
707 };
708 unsigned special_id;
709 };
710 };
711
712 struct panvk_attribs_info {
713 struct panvk_attrib_info attrib[PAN_MAX_ATTRIBUTE];
714 unsigned attrib_count;
715 struct panvk_attrib_buf_info buf[PAN_MAX_ATTRIBUTE];
716 unsigned buf_count;
717 };
718
719 struct panvk_attrib_buf {
720 mali_ptr address;
721 unsigned size;
722 };
723
724 struct panvk_cmd_state {
725 uint32_t dirty;
726
727 struct panvk_varyings_info varyings;
728 mali_ptr fs_rsd;
729
730 struct {
731 float constants[4];
732 } blend;
733
734 struct {
735 struct {
736 float constant_factor;
737 float clamp;
738 float slope_factor;
739 } depth_bias;
740 float line_width;
741 } rast;
742
743 struct {
744 struct panvk_attrib_buf bufs[MAX_VBS];
745 unsigned count;
746 } vb;
747
748 /* Index buffer */
749 struct {
750 struct panvk_buffer *buffer;
751 uint64_t offset;
752 uint8_t index_size;
753 uint32_t first_vertex, base_vertex, base_instance;
754 } ib;
755
756 struct {
757 struct {
758 uint8_t compare_mask;
759 uint8_t write_mask;
760 uint8_t ref;
761 } s_front, s_back;
762 } zs;
763
764 struct {
765 struct pan_fb_info info;
766 bool crc_valid[MAX_RTS];
767 } fb;
768
769 const struct panvk_render_pass *pass;
770 const struct panvk_subpass *subpass;
771 const struct panvk_framebuffer *framebuffer;
772 VkRect2D render_area;
773
774 struct panvk_clear_value *clear;
775
776 mali_ptr vpd;
777 VkViewport viewport;
778 VkRect2D scissor;
779
780 struct panvk_batch *batch;
781 };
782
783 struct panvk_cmd_pool {
784 struct vk_command_pool vk;
785 struct panvk_bo_pool desc_bo_pool;
786 struct panvk_bo_pool varying_bo_pool;
787 struct panvk_bo_pool tls_bo_pool;
788 };
789
790 struct panvk_cmd_bind_point_state {
791 struct panvk_descriptor_state desc_state;
792 const struct panvk_pipeline *pipeline;
793 };
794
795 struct panvk_cmd_buffer {
796 struct vk_command_buffer vk;
797
798 struct panvk_device *device;
799
800 struct panvk_pool desc_pool;
801 struct panvk_pool varying_pool;
802 struct panvk_pool tls_pool;
803 struct list_head batches;
804
805 VkCommandBufferUsageFlags usage_flags;
806
807 struct panvk_cmd_state state;
808
809 uint8_t push_constants[MAX_PUSH_CONSTANTS_SIZE];
810 VkShaderStageFlags push_constant_stages;
811 struct panvk_descriptor_set meta_push_descriptors;
812
813 struct panvk_cmd_bind_point_state bind_points[MAX_BIND_POINTS];
814 };
815
816 #define panvk_cmd_get_bind_point_state(cmdbuf, bindpoint) \
817 &(cmdbuf)->bind_points[VK_PIPELINE_BIND_POINT_##bindpoint]
818
819 #define panvk_cmd_get_pipeline(cmdbuf, bindpoint) \
820 (cmdbuf)->bind_points[VK_PIPELINE_BIND_POINT_##bindpoint].pipeline
821
822 #define panvk_cmd_get_desc_state(cmdbuf, bindpoint) \
823 &(cmdbuf)->bind_points[VK_PIPELINE_BIND_POINT_##bindpoint].desc_state
824
825 struct panvk_batch *panvk_cmd_open_batch(struct panvk_cmd_buffer *cmdbuf);
826
827 void panvk_cmd_fb_info_set_subpass(struct panvk_cmd_buffer *cmdbuf);
828
829 void panvk_cmd_fb_info_init(struct panvk_cmd_buffer *cmdbuf);
830
831 void panvk_cmd_preload_fb_after_batch_split(struct panvk_cmd_buffer *cmdbuf);
832
833 void panvk_pack_color(struct panvk_clear_value *out,
834 const VkClearColorValue *in, enum pipe_format format);
835
836 struct panvk_event {
837 struct vk_object_base base;
838 uint32_t syncobj;
839 };
840
841 struct panvk_shader {
842 struct pan_shader_info info;
843 struct util_dynarray binary;
844 unsigned sysval_ubo;
845 struct pan_compute_dim local_size;
846 bool has_img_access;
847 };
848
849 struct panvk_shader *
850 panvk_shader_create(struct panvk_device *dev, gl_shader_stage stage,
851 const VkPipelineShaderStageCreateInfo *stage_info,
852 const struct panvk_pipeline_layout *layout,
853 unsigned sysval_ubo, struct pan_blend_state *blend_state,
854 bool static_blend_constants,
855 const VkAllocationCallbacks *alloc);
856
857 void panvk_shader_destroy(struct panvk_device *dev, struct panvk_shader *shader,
858 const VkAllocationCallbacks *alloc);
859
860 #define RSD_WORDS 16
861 #define BLEND_DESC_WORDS 4
862
863 struct panvk_pipeline {
864 struct vk_object_base base;
865
866 struct panvk_varyings_info varyings;
867 struct panvk_attribs_info attribs;
868
869 const struct panvk_pipeline_layout *layout;
870
871 unsigned active_stages;
872
873 uint32_t dynamic_state_mask;
874
875 struct panvk_priv_bo *binary_bo;
876 struct panvk_priv_bo *state_bo;
877
878 mali_ptr vpd;
879 mali_ptr rsds[MESA_SHADER_STAGES];
880
881 /* shader stage bit is set of the stage accesses storage images */
882 uint32_t img_access_mask;
883
884 unsigned num_ubos;
885
886 struct {
887 unsigned ubo_idx;
888 } sysvals[MESA_SHADER_STAGES];
889
890 unsigned tls_size;
891 unsigned wls_size;
892
893 struct {
894 mali_ptr address;
895 struct pan_shader_info info;
896 uint32_t rsd_template[RSD_WORDS];
897 bool required;
898 bool dynamic_rsd;
899 uint8_t rt_mask;
900 } fs;
901
902 struct {
903 struct pan_compute_dim local_size;
904 } cs;
905
906 struct {
907 unsigned topology;
908 bool writes_point_size;
909 bool primitive_restart;
910 } ia;
911
912 struct {
913 bool clamp_depth;
914 float line_width;
915 struct {
916 bool enable;
917 float constant_factor;
918 float clamp;
919 float slope_factor;
920 } depth_bias;
921 bool front_ccw;
922 bool cull_front_face;
923 bool cull_back_face;
924 bool enable;
925 } rast;
926
927 struct {
928 bool z_test;
929 bool z_write;
930 unsigned z_compare_func;
931 bool s_test;
932 struct {
933 unsigned fail_op;
934 unsigned pass_op;
935 unsigned z_fail_op;
936 unsigned compare_func;
937 uint8_t compare_mask;
938 uint8_t write_mask;
939 uint8_t ref;
940 } s_front, s_back;
941 } zs;
942
943 struct {
944 uint8_t rast_samples;
945 uint8_t min_samples;
946 uint16_t sample_mask;
947 bool alpha_to_coverage;
948 bool alpha_to_one;
949 } ms;
950
951 struct {
952 struct pan_blend_state state;
953 uint32_t bd_template[8][BLEND_DESC_WORDS];
954 struct {
955 uint8_t index;
956 uint16_t bifrost_factor;
957 } constant[8];
958 bool reads_dest;
959 } blend;
960
961 VkViewport viewport;
962 VkRect2D scissor;
963 };
964
965 #define PANVK_MAX_PLANES 1
966
967 struct panvk_image {
968 struct vk_image vk;
969
970 /* TODO: See if we can rework the synchronization logic so we don't need to
971 * pass BOs around.
972 */
973 struct pan_kmod_bo *bo;
974
975 struct pan_image pimage;
976 };
977
978 unsigned panvk_image_get_plane_size(const struct panvk_image *image,
979 unsigned plane);
980
981 unsigned panvk_image_get_total_size(const struct panvk_image *image);
982
983 #define TEXTURE_DESC_WORDS 8
984 #define ATTRIB_BUF_DESC_WORDS 4
985
986 struct panvk_image_view {
987 struct vk_image_view vk;
988
989 struct pan_image_view pview;
990
991 struct panvk_priv_bo *bo;
992 struct {
993 uint32_t tex[TEXTURE_DESC_WORDS];
994 uint32_t img_attrib_buf[ATTRIB_BUF_DESC_WORDS * 2];
995 } descs;
996 };
997
998 #define SAMPLER_DESC_WORDS 8
999
1000 struct panvk_sampler {
1001 struct vk_object_base base;
1002 uint32_t desc[SAMPLER_DESC_WORDS];
1003 };
1004
1005 struct panvk_buffer_view {
1006 struct vk_object_base base;
1007 struct panvk_priv_bo *bo;
1008 struct {
1009 uint32_t tex[TEXTURE_DESC_WORDS];
1010 uint32_t img_attrib_buf[ATTRIB_BUF_DESC_WORDS * 2];
1011 } descs;
1012 enum pipe_format fmt;
1013 uint32_t elems;
1014 };
1015
1016 struct panvk_attachment_info {
1017 struct panvk_image_view *iview;
1018 };
1019
1020 struct panvk_framebuffer {
1021 struct vk_object_base base;
1022
1023 uint32_t width;
1024 uint32_t height;
1025 uint32_t layers;
1026
1027 uint32_t attachment_count;
1028 struct panvk_attachment_info attachments[0];
1029 };
1030
1031 struct panvk_clear_value {
1032 union {
1033 uint32_t color[4];
1034 struct {
1035 float depth;
1036 uint8_t stencil;
1037 };
1038 };
1039 };
1040
1041 struct panvk_subpass_attachment {
1042 uint32_t idx;
1043 VkImageLayout layout;
1044 bool clear;
1045 bool preload;
1046 };
1047
1048 struct panvk_subpass {
1049 uint32_t input_count;
1050 uint32_t color_count;
1051 struct panvk_subpass_attachment *input_attachments;
1052 uint8_t active_color_attachments;
1053 struct panvk_subpass_attachment *color_attachments;
1054 struct panvk_subpass_attachment *resolve_attachments;
1055 struct panvk_subpass_attachment zs_attachment;
1056
1057 uint32_t view_mask;
1058 };
1059
1060 struct panvk_render_pass_attachment {
1061 VkAttachmentDescriptionFlags flags;
1062 enum pipe_format format;
1063 unsigned samples;
1064 VkAttachmentLoadOp load_op;
1065 VkAttachmentStoreOp store_op;
1066 VkAttachmentLoadOp stencil_load_op;
1067 VkAttachmentStoreOp stencil_store_op;
1068 VkImageLayout initial_layout;
1069 VkImageLayout final_layout;
1070 unsigned view_mask;
1071 unsigned first_used_in_subpass;
1072 };
1073
1074 struct panvk_render_pass {
1075 struct vk_object_base base;
1076
1077 uint32_t attachment_count;
1078 uint32_t subpass_count;
1079 struct panvk_subpass_attachment *subpass_attachments;
1080 struct panvk_render_pass_attachment *attachments;
1081 struct panvk_subpass subpasses[0];
1082 };
1083
1084 VK_DEFINE_HANDLE_CASTS(panvk_cmd_buffer, vk.base, VkCommandBuffer,
1085 VK_OBJECT_TYPE_COMMAND_BUFFER)
1086 VK_DEFINE_HANDLE_CASTS(panvk_device, vk.base, VkDevice, VK_OBJECT_TYPE_DEVICE)
1087 VK_DEFINE_HANDLE_CASTS(panvk_instance, vk.base, VkInstance,
1088 VK_OBJECT_TYPE_INSTANCE)
1089 VK_DEFINE_HANDLE_CASTS(panvk_physical_device, vk.base, VkPhysicalDevice,
1090 VK_OBJECT_TYPE_PHYSICAL_DEVICE)
1091 VK_DEFINE_HANDLE_CASTS(panvk_queue, vk.base, VkQueue, VK_OBJECT_TYPE_QUEUE)
1092
1093 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_cmd_pool, vk.base, VkCommandPool,
1094 VK_OBJECT_TYPE_COMMAND_POOL)
1095 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_buffer, vk.base, VkBuffer,
1096 VK_OBJECT_TYPE_BUFFER)
1097 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_buffer_view, base, VkBufferView,
1098 VK_OBJECT_TYPE_BUFFER_VIEW)
1099 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_descriptor_pool, base, VkDescriptorPool,
1100 VK_OBJECT_TYPE_DESCRIPTOR_POOL)
1101 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_descriptor_set, base, VkDescriptorSet,
1102 VK_OBJECT_TYPE_DESCRIPTOR_SET)
1103 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_descriptor_set_layout, vk.base,
1104 VkDescriptorSetLayout,
1105 VK_OBJECT_TYPE_DESCRIPTOR_SET_LAYOUT)
1106 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_device_memory, base, VkDeviceMemory,
1107 VK_OBJECT_TYPE_DEVICE_MEMORY)
1108 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_event, base, VkEvent, VK_OBJECT_TYPE_EVENT)
1109 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_framebuffer, base, VkFramebuffer,
1110 VK_OBJECT_TYPE_FRAMEBUFFER)
1111 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_image, vk.base, VkImage,
1112 VK_OBJECT_TYPE_IMAGE)
1113 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_image_view, vk.base, VkImageView,
1114 VK_OBJECT_TYPE_IMAGE_VIEW);
1115 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_pipeline_cache, base, VkPipelineCache,
1116 VK_OBJECT_TYPE_PIPELINE_CACHE)
1117 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_pipeline, base, VkPipeline,
1118 VK_OBJECT_TYPE_PIPELINE)
1119 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_pipeline_layout, vk.base, VkPipelineLayout,
1120 VK_OBJECT_TYPE_PIPELINE_LAYOUT)
1121 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_render_pass, base, VkRenderPass,
1122 VK_OBJECT_TYPE_RENDER_PASS)
1123 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_sampler, base, VkSampler,
1124 VK_OBJECT_TYPE_SAMPLER)
1125
1126 #define panvk_arch_name(name, version) panvk_##version##_##name
1127
1128 #define panvk_arch_dispatch(arch, name, ...) \
1129 do { \
1130 switch (arch) { \
1131 case 6: \
1132 panvk_arch_name(name, v6)(__VA_ARGS__); \
1133 break; \
1134 case 7: \
1135 panvk_arch_name(name, v7)(__VA_ARGS__); \
1136 break; \
1137 default: \
1138 unreachable("Unsupported architecture"); \
1139 } \
1140 } while (0)
1141
1142 #ifdef PAN_ARCH
1143 #if PAN_ARCH == 6
1144 #define panvk_per_arch(name) panvk_arch_name(name, v6)
1145 #elif PAN_ARCH == 7
1146 #define panvk_per_arch(name) panvk_arch_name(name, v7)
1147 #endif
1148 #include "panvk_vX_cmd_buffer.h"
1149 #include "panvk_vX_cs.h"
1150 #include "panvk_vX_device.h"
1151 #include "panvk_vX_meta.h"
1152 #else
1153 #define PAN_ARCH 6
1154 #define panvk_per_arch(name) panvk_arch_name(name, v6)
1155 #include "panvk_vX_cmd_buffer.h"
1156 #include "panvk_vX_cs.h"
1157 #include "panvk_vX_device.h"
1158 #include "panvk_vX_meta.h"
1159 #undef PAN_ARCH
1160 #undef panvk_per_arch
1161 #define PAN_ARCH 7
1162 #define panvk_per_arch(name) panvk_arch_name(name, v7)
1163 #include "panvk_vX_cmd_buffer.h"
1164 #include "panvk_vX_cs.h"
1165 #include "panvk_vX_device.h"
1166 #include "panvk_vX_meta.h"
1167 #undef PAN_ARCH
1168 #undef panvk_per_arch
1169 #endif
1170
1171 #ifdef PAN_ARCH
1172 bool panvk_per_arch(blend_needs_lowering)(const struct panvk_device *dev,
1173 const struct pan_blend_state *state,
1174 unsigned rt);
1175
1176 struct panvk_shader *panvk_per_arch(shader_create)(
1177 struct panvk_device *dev, gl_shader_stage stage,
1178 const VkPipelineShaderStageCreateInfo *stage_info,
1179 const struct panvk_pipeline_layout *layout, unsigned sysval_ubo,
1180 struct pan_blend_state *blend_state, bool static_blend_constants,
1181 const VkAllocationCallbacks *alloc);
1182 struct nir_shader;
1183
1184 bool panvk_per_arch(nir_lower_descriptors)(
1185 struct nir_shader *nir, struct panvk_device *dev,
1186 const struct panvk_pipeline_layout *layout, bool *has_img_access_out);
1187 #endif
1188
1189 #endif /* PANVK_PRIVATE_H */
1190