1 /*
2 * Copyright © 2021 Collabora Ltd.
3 *
4 * derived from tu_private.h driver which is:
5 * Copyright © 2016 Red Hat.
6 * Copyright © 2016 Bas Nieuwenhuizen
7 * Copyright © 2015 Intel Corporation
8 *
9 * Permission is hereby granted, free of charge, to any person obtaining a
10 * copy of this software and associated documentation files (the "Software"),
11 * to deal in the Software without restriction, including without limitation
12 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
13 * and/or sell copies of the Software, and to permit persons to whom the
14 * Software is furnished to do so, subject to the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the next
17 * paragraph) shall be included in all copies or substantial portions of the
18 * Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
27 */
28
29 #ifndef PANVK_PRIVATE_H
30 #define PANVK_PRIVATE_H
31
32 #include <assert.h>
33 #include <pthread.h>
34 #include <stdbool.h>
35 #include <stdint.h>
36 #include <stdio.h>
37 #include <stdlib.h>
38 #include <string.h>
39 #ifdef HAVE_VALGRIND
40 #include <memcheck.h>
41 #include <valgrind.h>
42 #define VG(x) x
43 #else
44 #define VG(x)
45 #endif
46
47 #include "c11/threads.h"
48 #include "compiler/shader_enums.h"
49 #include "util/list.h"
50 #include "util/macros.h"
51 #include "vk_alloc.h"
52 #include "vk_buffer.h"
53 #include "vk_command_buffer.h"
54 #include "vk_command_pool.h"
55 #include "vk_descriptor_set_layout.h"
56 #include "vk_device.h"
57 #include "vk_image.h"
58 #include "vk_instance.h"
59 #include "vk_log.h"
60 #include "vk_object.h"
61 #include "vk_physical_device.h"
62 #include "vk_pipeline_layout.h"
63 #include "vk_queue.h"
64 #include "vk_sync.h"
65 #include "wsi_common.h"
66
67 #include "drm-uapi/panfrost_drm.h"
68
69 #include "pan_blend.h"
70 #include "pan_cs.h"
71 #include "pan_device.h"
72 #include "panvk_mempool.h"
73 #include "pan_texture.h"
74 #include "pan_scoreboard.h"
75 #include "vk_extensions.h"
76 #include "panvk_varyings.h"
77
78 /* Pre-declarations needed for WSI entrypoints */
79 struct wl_surface;
80 struct wl_display;
81 typedef struct xcb_connection_t xcb_connection_t;
82 typedef uint32_t xcb_visualid_t;
83 typedef uint32_t xcb_window_t;
84
85 #include <vulkan/vk_android_native_buffer.h>
86 #include <vulkan/vk_icd.h>
87 #include <vulkan/vulkan.h>
88
89 #include "panvk_entrypoints.h"
90
91 #define MAX_BIND_POINTS 2 /* compute + graphics */
92 #define MAX_VBS 16
93 #define MAX_VERTEX_ATTRIBS 16
94 #define MAX_RTS 8
95 #define MAX_VSC_PIPES 32
96 #define MAX_VIEWPORTS 1
97 #define MAX_SCISSORS 16
98 #define MAX_DISCARD_RECTANGLES 4
99 #define MAX_PUSH_CONSTANTS_SIZE 128
100 #define MAX_PUSH_DESCRIPTORS 32
101 #define MAX_DYNAMIC_UNIFORM_BUFFERS 16
102 #define MAX_DYNAMIC_STORAGE_BUFFERS 8
103 #define MAX_DYNAMIC_BUFFERS \
104 (MAX_DYNAMIC_UNIFORM_BUFFERS + MAX_DYNAMIC_STORAGE_BUFFERS)
105 #define MAX_SAMPLES_LOG2 4
106 #define NUM_META_FS_KEYS 13
107 #define PANVK_MAX_DRM_DEVICES 1
108 #define MAX_VIEWS 8
109
110 #define NUM_DEPTH_CLEAR_PIPELINES 3
111
112 #define PANVK_SYSVAL_UBO_INDEX 0
113 #define PANVK_PUSH_CONST_UBO_INDEX 1
114 #define PANVK_NUM_BUILTIN_UBOS 2
115
116 #define panvk_printflike(a, b) __attribute__((__format__(__printf__, a, b)))
117
118 void
119 panvk_logi(const char *format, ...) panvk_printflike(1, 2);
120 void
121 panvk_logi_v(const char *format, va_list va);
122
123 #define panvk_stub() assert(!"stub")
124
125 #define PANVK_META_COPY_BUF2IMG_NUM_FORMATS 12
126 #define PANVK_META_COPY_IMG2BUF_NUM_FORMATS 12
127 #define PANVK_META_COPY_IMG2IMG_NUM_FORMATS 14
128 #define PANVK_META_COPY_NUM_TEX_TYPES 5
129 #define PANVK_META_COPY_BUF2BUF_NUM_BLKSIZES 5
130
131 static inline unsigned
panvk_meta_copy_tex_type(unsigned dim,bool isarray)132 panvk_meta_copy_tex_type(unsigned dim, bool isarray)
133 {
134 assert(dim > 0 && dim <= 3);
135 assert(dim < 3 || !isarray);
136 return (((dim - 1) << 1) | (isarray ? 1 : 0));
137 }
138
139 struct panvk_meta {
140 struct panvk_pool bin_pool;
141 struct panvk_pool desc_pool;
142
143 /* Access to the blitter pools are protected by the blitter
144 * shader/rsd locks. They can't be merged with other binary/desc
145 * pools unless we patch pan_blitter.c to external pool locks.
146 */
147 struct {
148 struct panvk_pool bin_pool;
149 struct panvk_pool desc_pool;
150 } blitter;
151
152 struct {
153 struct {
154 mali_ptr shader;
155 struct pan_shader_info shader_info;
156 } color[3]; /* 3 base types */
157 } clear_attachment;
158
159 struct {
160 struct {
161 mali_ptr rsd;
162 } buf2img[PANVK_META_COPY_BUF2IMG_NUM_FORMATS];
163 struct {
164 mali_ptr rsd;
165 } img2buf[PANVK_META_COPY_NUM_TEX_TYPES][PANVK_META_COPY_IMG2BUF_NUM_FORMATS];
166 struct {
167 mali_ptr rsd;
168 } img2img[2][PANVK_META_COPY_NUM_TEX_TYPES][PANVK_META_COPY_IMG2IMG_NUM_FORMATS];
169 struct {
170 mali_ptr rsd;
171 } buf2buf[PANVK_META_COPY_BUF2BUF_NUM_BLKSIZES];
172 struct {
173 mali_ptr rsd;
174 } fillbuf;
175 } copy;
176 };
177
178 struct panvk_physical_device {
179 struct vk_physical_device vk;
180
181 /* The API agnostic device object. */
182 struct panfrost_device pdev;
183
184 struct panvk_instance *instance;
185
186 char path[20];
187 char name[VK_MAX_PHYSICAL_DEVICE_NAME_SIZE];
188 uint8_t driver_uuid[VK_UUID_SIZE];
189 uint8_t device_uuid[VK_UUID_SIZE];
190 uint8_t cache_uuid[VK_UUID_SIZE];
191
192 struct vk_sync_type drm_syncobj_type;
193 const struct vk_sync_type *sync_types[2];
194
195 struct wsi_device wsi_device;
196 struct panvk_meta meta;
197
198 int master_fd;
199 };
200
201 enum panvk_debug_flags {
202 PANVK_DEBUG_STARTUP = 1 << 0,
203 PANVK_DEBUG_NIR = 1 << 1,
204 PANVK_DEBUG_TRACE = 1 << 2,
205 PANVK_DEBUG_SYNC = 1 << 3,
206 PANVK_DEBUG_AFBC = 1 << 4,
207 PANVK_DEBUG_LINEAR = 1 << 5,
208 PANVK_DEBUG_DUMP = 1 << 6,
209 };
210
211 struct panvk_instance {
212 struct vk_instance vk;
213
214 uint32_t api_version;
215 int physical_device_count;
216 struct panvk_physical_device physical_devices[PANVK_MAX_DRM_DEVICES];
217
218 enum panvk_debug_flags debug_flags;
219 };
220
221 VkResult
222 panvk_wsi_init(struct panvk_physical_device *physical_device);
223 void
224 panvk_wsi_finish(struct panvk_physical_device *physical_device);
225
226 bool
227 panvk_instance_extension_supported(const char *name);
228 uint32_t
229 panvk_physical_device_api_version(struct panvk_physical_device *dev);
230 bool
231 panvk_physical_device_extension_supported(struct panvk_physical_device *dev,
232 const char *name);
233
234 struct panvk_pipeline_cache {
235 struct vk_object_base base;
236 VkAllocationCallbacks alloc;
237 };
238
239 /* queue types */
240 #define PANVK_QUEUE_GENERAL 0
241
242 #define PANVK_MAX_QUEUE_FAMILIES 1
243
244 struct panvk_queue {
245 struct vk_queue vk;
246 struct panvk_device *device;
247 uint32_t sync;
248 };
249
250 struct panvk_device {
251 struct vk_device vk;
252
253 struct vk_device_dispatch_table cmd_dispatch;
254
255 struct panvk_instance *instance;
256
257 struct panvk_queue *queues[PANVK_MAX_QUEUE_FAMILIES];
258 int queue_count[PANVK_MAX_QUEUE_FAMILIES];
259
260 struct panvk_physical_device *physical_device;
261 int _lost;
262 };
263
264 VkResult _panvk_device_set_lost(struct panvk_device *device,
265 const char *file, int line,
266 const char *msg, ...) PRINTFLIKE(4, 5);
267 #define panvk_device_set_lost(dev, ...) \
268 _panvk_device_set_lost(dev, __FILE__, __LINE__, __VA_ARGS__)
269
270 static inline bool
panvk_device_is_lost(struct panvk_device * device)271 panvk_device_is_lost(struct panvk_device *device)
272 {
273 return unlikely(p_atomic_read(&device->_lost));
274 }
275
276 #define TILER_DESC_WORDS 56
277
278 struct panvk_batch {
279 struct list_head node;
280 struct util_dynarray jobs;
281 struct util_dynarray event_ops;
282 struct pan_scoreboard scoreboard;
283 struct {
284 const struct panvk_framebuffer *info;
285 struct panfrost_ptr desc;
286 } fb;
287 struct {
288 struct panfrost_bo *src, *dst;
289 } blit;
290 struct panfrost_ptr tls;
291 mali_ptr fragment_job;
292 struct {
293 struct pan_tiler_context ctx;
294 struct panfrost_ptr descs;
295 uint32_t templ[TILER_DESC_WORDS];
296 } tiler;
297 struct pan_tls_info tlsinfo;
298 unsigned wls_total_size;
299 bool issued;
300 };
301
302 enum panvk_event_op_type {
303 PANVK_EVENT_OP_SET,
304 PANVK_EVENT_OP_RESET,
305 PANVK_EVENT_OP_WAIT,
306 };
307
308 struct panvk_event_op {
309 enum panvk_event_op_type type;
310 struct panvk_event *event;
311 };
312
313 struct panvk_device_memory {
314 struct vk_object_base base;
315 struct panfrost_bo *bo;
316 };
317
318 struct panvk_buffer_desc {
319 struct panvk_buffer *buffer;
320 VkDeviceSize offset;
321 VkDeviceSize size;
322 };
323
324 struct panvk_descriptor_set {
325 struct vk_object_base base;
326 struct panvk_descriptor_pool *pool;
327 const struct panvk_descriptor_set_layout *layout;
328 struct panvk_buffer_desc *dyn_ssbos;
329 void *ubos;
330 struct panvk_buffer_desc *dyn_ubos;
331 void *samplers;
332 void *textures;
333 void *img_attrib_bufs;
334 uint32_t *img_fmts;
335
336 struct panfrost_bo *desc_bo;
337 };
338
339 #define MAX_SETS 4
340
341 struct panvk_descriptor_set_binding_layout {
342 VkDescriptorType type;
343
344 /* Number of array elements in this binding */
345 unsigned array_size;
346
347 /* Indices in the desc arrays */
348 union {
349 struct {
350 union {
351 unsigned sampler_idx;
352 unsigned img_idx;
353 };
354 unsigned tex_idx;
355 };
356 unsigned dyn_ssbo_idx;
357 unsigned ubo_idx;
358 unsigned dyn_ubo_idx;
359 };
360
361 /* Offset into the descriptor UBO where this binding starts */
362 uint32_t desc_ubo_offset;
363
364 /* Stride between descriptors in this binding in the UBO */
365 uint16_t desc_ubo_stride;
366
367 /* Shader stages affected by this set+binding */
368 uint16_t shader_stages;
369
370 struct panvk_sampler **immutable_samplers;
371 };
372
373 struct panvk_descriptor_set_layout {
374 struct vk_descriptor_set_layout vk;
375
376 /* Shader stages affected by this descriptor set */
377 uint16_t shader_stages;
378
379 unsigned num_samplers;
380 unsigned num_textures;
381 unsigned num_ubos;
382 unsigned num_dyn_ubos;
383 unsigned num_dyn_ssbos;
384 unsigned num_imgs;
385
386 /* Size of the descriptor UBO */
387 uint32_t desc_ubo_size;
388
389 /* Index of the descriptor UBO */
390 unsigned desc_ubo_index;
391
392 /* Number of bindings in this descriptor set */
393 uint32_t binding_count;
394
395 /* Bindings in this descriptor set */
396 struct panvk_descriptor_set_binding_layout bindings[0];
397 };
398
399 static inline const struct panvk_descriptor_set_layout *
vk_to_panvk_descriptor_set_layout(const struct vk_descriptor_set_layout * layout)400 vk_to_panvk_descriptor_set_layout(const struct vk_descriptor_set_layout *layout)
401 {
402 return container_of(layout, const struct panvk_descriptor_set_layout, vk);
403 }
404
405 struct panvk_pipeline_layout {
406 struct vk_pipeline_layout vk;
407
408 unsigned char sha1[20];
409
410 unsigned num_samplers;
411 unsigned num_textures;
412 unsigned num_ubos;
413 unsigned num_dyn_ubos;
414 unsigned num_dyn_ssbos;
415 uint32_t num_imgs;
416 uint32_t num_sets;
417
418 struct {
419 uint32_t size;
420 } push_constants;
421
422 struct {
423 unsigned sampler_offset;
424 unsigned tex_offset;
425 unsigned ubo_offset;
426 unsigned dyn_ubo_offset;
427 unsigned dyn_ssbo_offset;
428 unsigned img_offset;
429 } sets[MAX_SETS];
430 };
431
432 static unsigned
panvk_pipeline_layout_ubo_start(const struct panvk_pipeline_layout * layout,unsigned set,bool is_dynamic)433 panvk_pipeline_layout_ubo_start(const struct panvk_pipeline_layout *layout,
434 unsigned set, bool is_dynamic)
435 {
436 const struct panvk_descriptor_set_layout *set_layout =
437 vk_to_panvk_descriptor_set_layout(layout->vk.set_layouts[set]);
438
439 unsigned offset = PANVK_NUM_BUILTIN_UBOS +
440 layout->sets[set].ubo_offset +
441 layout->sets[set].dyn_ubo_offset;
442
443 if (is_dynamic)
444 offset += set_layout->num_ubos;
445
446 return offset;
447 }
448
449 static unsigned
panvk_pipeline_layout_ubo_index(const struct panvk_pipeline_layout * layout,unsigned set,unsigned binding,unsigned array_index)450 panvk_pipeline_layout_ubo_index(const struct panvk_pipeline_layout *layout,
451 unsigned set, unsigned binding,
452 unsigned array_index)
453 {
454 const struct panvk_descriptor_set_layout *set_layout =
455 vk_to_panvk_descriptor_set_layout(layout->vk.set_layouts[set]);
456 const struct panvk_descriptor_set_binding_layout *binding_layout =
457 &set_layout->bindings[binding];
458
459 const bool is_dynamic =
460 binding_layout->type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC;
461 const uint32_t ubo_idx = is_dynamic ? binding_layout->dyn_ubo_idx :
462 binding_layout->ubo_idx;
463
464 return panvk_pipeline_layout_ubo_start(layout, set, is_dynamic) +
465 ubo_idx + array_index;
466 }
467
468 struct panvk_desc_pool_counters {
469 unsigned samplers;
470 unsigned combined_image_samplers;
471 unsigned sampled_images;
472 unsigned storage_images;
473 unsigned uniform_texel_bufs;
474 unsigned storage_texel_bufs;
475 unsigned input_attachments;
476 unsigned uniform_bufs;
477 unsigned storage_bufs;
478 unsigned uniform_dyn_bufs;
479 unsigned storage_dyn_bufs;
480 unsigned sets;
481 };
482
483 struct panvk_descriptor_pool {
484 struct vk_object_base base;
485 struct panvk_desc_pool_counters max;
486 struct panvk_desc_pool_counters cur;
487 struct panvk_descriptor_set *sets;
488 };
489
490 struct panvk_buffer {
491 struct vk_buffer vk;
492
493 struct panfrost_bo *bo;
494 VkDeviceSize bo_offset;
495 };
496
497 static inline mali_ptr
panvk_buffer_gpu_ptr(const struct panvk_buffer * buffer,uint64_t offset)498 panvk_buffer_gpu_ptr(const struct panvk_buffer *buffer, uint64_t offset)
499 {
500 if (buffer->bo == NULL)
501 return 0;
502
503 return buffer->bo->ptr.gpu + buffer->bo_offset + offset;
504 }
505
506 static inline uint64_t
panvk_buffer_range(const struct panvk_buffer * buffer,uint64_t offset,uint64_t range)507 panvk_buffer_range(const struct panvk_buffer *buffer,
508 uint64_t offset, uint64_t range)
509 {
510 if (buffer->bo == NULL)
511 return 0;
512
513 return vk_buffer_range(&buffer->vk, offset, range);
514 }
515
516 enum panvk_dynamic_state_bits {
517 PANVK_DYNAMIC_VIEWPORT = 1 << 0,
518 PANVK_DYNAMIC_SCISSOR = 1 << 1,
519 PANVK_DYNAMIC_LINE_WIDTH = 1 << 2,
520 PANVK_DYNAMIC_DEPTH_BIAS = 1 << 3,
521 PANVK_DYNAMIC_BLEND_CONSTANTS = 1 << 4,
522 PANVK_DYNAMIC_DEPTH_BOUNDS = 1 << 5,
523 PANVK_DYNAMIC_STENCIL_COMPARE_MASK = 1 << 6,
524 PANVK_DYNAMIC_STENCIL_WRITE_MASK = 1 << 7,
525 PANVK_DYNAMIC_STENCIL_REFERENCE = 1 << 8,
526 PANVK_DYNAMIC_DISCARD_RECTANGLE = 1 << 9,
527 PANVK_DYNAMIC_SSBO = 1 << 10,
528 PANVK_DYNAMIC_VERTEX_INSTANCE_OFFSETS = 1 << 11,
529 PANVK_DYNAMIC_ALL = (1 << 12) - 1,
530 };
531
532 /* This has to match nir_address_format_64bit_bounded_global */
533 struct panvk_ssbo_addr {
534 uint64_t base_addr;
535 uint32_t size;
536 uint32_t zero; /* Must be zero! */
537 };
538
539 union panvk_sysval_vec4 {
540 float f32[4];
541 uint32_t u32[4];
542 };
543
544 struct panvk_sysvals {
545 union {
546 struct {
547 /* Only for graphics */
548 union panvk_sysval_vec4 viewport_scale;
549 union panvk_sysval_vec4 viewport_offset;
550 union {
551 union panvk_sysval_vec4 vertex_instance_offsets;
552 struct {
553 uint32_t first_vertex;
554 uint32_t base_vertex;
555 uint32_t base_instance;
556 };
557 };
558 union panvk_sysval_vec4 blend_constants;
559 };
560
561 struct {
562 /* Only for compute */
563 union panvk_sysval_vec4 num_work_groups;
564 union panvk_sysval_vec4 local_group_size;
565 };
566 };
567
568 /* The back-end compiler doesn't know about any sysvals after this point */
569
570 struct panvk_ssbo_addr dyn_ssbos[MAX_DYNAMIC_STORAGE_BUFFERS];
571 };
572
573 struct panvk_descriptor_state {
574 uint32_t dirty;
575 const struct panvk_descriptor_set *sets[MAX_SETS];
576 struct panvk_sysvals sysvals;
577 struct {
578 struct panvk_buffer_desc ubos[MAX_DYNAMIC_UNIFORM_BUFFERS];
579 struct panvk_buffer_desc ssbos[MAX_DYNAMIC_STORAGE_BUFFERS];
580 } dyn;
581 mali_ptr sysvals_ptr;
582 mali_ptr ubos;
583 mali_ptr textures;
584 mali_ptr samplers;
585 mali_ptr push_constants;
586 mali_ptr vs_attribs;
587 mali_ptr vs_attrib_bufs;
588 mali_ptr non_vs_attribs;
589 mali_ptr non_vs_attrib_bufs;
590 };
591
592 #define INVOCATION_DESC_WORDS 2
593
594 struct panvk_draw_info {
595 unsigned first_index;
596 unsigned index_count;
597 unsigned index_size;
598 unsigned first_vertex;
599 unsigned vertex_count;
600 unsigned vertex_range;
601 unsigned padded_vertex_count;
602 unsigned first_instance;
603 unsigned instance_count;
604 int vertex_offset;
605 unsigned offset_start;
606 uint32_t invocation[INVOCATION_DESC_WORDS];
607 struct {
608 mali_ptr varyings;
609 mali_ptr attributes;
610 mali_ptr attribute_bufs;
611 mali_ptr push_constants;
612 } stages[MESA_SHADER_STAGES];
613 mali_ptr varying_bufs;
614 mali_ptr textures;
615 mali_ptr samplers;
616 mali_ptr ubos;
617 mali_ptr position;
618 mali_ptr indices;
619 union {
620 mali_ptr psiz;
621 float line_width;
622 };
623 mali_ptr tls;
624 mali_ptr fb;
625 const struct pan_tiler_context *tiler_ctx;
626 mali_ptr fs_rsd;
627 mali_ptr viewport;
628 struct {
629 struct panfrost_ptr vertex;
630 struct panfrost_ptr tiler;
631 } jobs;
632 };
633
634 struct panvk_dispatch_info {
635 struct pan_compute_dim wg_count;
636 mali_ptr attributes;
637 mali_ptr attribute_bufs;
638 mali_ptr tsd;
639 mali_ptr ubos;
640 mali_ptr push_uniforms;
641 mali_ptr textures;
642 mali_ptr samplers;
643 };
644
645 struct panvk_attrib_info {
646 unsigned buf;
647 unsigned offset;
648 enum pipe_format format;
649 };
650
651 struct panvk_attrib_buf_info {
652 bool special;
653 union {
654 struct {
655 unsigned stride;
656 bool per_instance;
657 uint32_t instance_divisor;
658 };
659 unsigned special_id;
660 };
661 };
662
663 struct panvk_attribs_info {
664 struct panvk_attrib_info attrib[PAN_MAX_ATTRIBUTE];
665 unsigned attrib_count;
666 struct panvk_attrib_buf_info buf[PAN_MAX_ATTRIBUTE];
667 unsigned buf_count;
668 };
669
670 struct panvk_attrib_buf {
671 mali_ptr address;
672 unsigned size;
673 };
674
675 struct panvk_cmd_state {
676 uint32_t dirty;
677
678 struct panvk_varyings_info varyings;
679 mali_ptr fs_rsd;
680
681 struct {
682 float constants[4];
683 } blend;
684
685 struct {
686 struct {
687 float constant_factor;
688 float clamp;
689 float slope_factor;
690 } depth_bias;
691 float line_width;
692 } rast;
693
694 struct {
695 struct panvk_attrib_buf bufs[MAX_VBS];
696 unsigned count;
697 } vb;
698
699 /* Index buffer */
700 struct {
701 struct panvk_buffer *buffer;
702 uint64_t offset;
703 uint8_t index_size;
704 uint32_t first_vertex, base_vertex, base_instance;
705 } ib;
706
707 struct {
708 struct {
709 uint8_t compare_mask;
710 uint8_t write_mask;
711 uint8_t ref;
712 } s_front, s_back;
713 } zs;
714
715 struct {
716 struct pan_fb_info info;
717 bool crc_valid[MAX_RTS];
718 } fb;
719
720 const struct panvk_render_pass *pass;
721 const struct panvk_subpass *subpass;
722 const struct panvk_framebuffer *framebuffer;
723 VkRect2D render_area;
724
725 struct panvk_clear_value *clear;
726
727 mali_ptr vpd;
728 VkViewport viewport;
729 VkRect2D scissor;
730
731 struct panvk_batch *batch;
732 };
733
734 struct panvk_cmd_pool {
735 struct vk_command_pool vk;
736 struct list_head active_cmd_buffers;
737 struct list_head free_cmd_buffers;
738 struct panvk_bo_pool desc_bo_pool;
739 struct panvk_bo_pool varying_bo_pool;
740 struct panvk_bo_pool tls_bo_pool;
741 };
742
743 enum panvk_cmd_buffer_status {
744 PANVK_CMD_BUFFER_STATUS_INVALID,
745 PANVK_CMD_BUFFER_STATUS_INITIAL,
746 PANVK_CMD_BUFFER_STATUS_RECORDING,
747 PANVK_CMD_BUFFER_STATUS_EXECUTABLE,
748 PANVK_CMD_BUFFER_STATUS_PENDING,
749 };
750
751 struct panvk_cmd_bind_point_state {
752 struct panvk_descriptor_state desc_state;
753 const struct panvk_pipeline *pipeline;
754 };
755
756 struct panvk_cmd_buffer {
757 struct vk_command_buffer vk;
758
759 struct panvk_device *device;
760
761 struct panvk_cmd_pool *pool;
762 struct list_head pool_link;
763 struct panvk_pool desc_pool;
764 struct panvk_pool varying_pool;
765 struct panvk_pool tls_pool;
766 struct list_head batches;
767
768 VkCommandBufferUsageFlags usage_flags;
769 enum panvk_cmd_buffer_status status;
770
771 struct panvk_cmd_state state;
772 uint32_t queue_family_index;
773
774 uint8_t push_constants[MAX_PUSH_CONSTANTS_SIZE];
775 VkShaderStageFlags push_constant_stages;
776 struct panvk_descriptor_set meta_push_descriptors;
777
778 struct panvk_cmd_bind_point_state bind_points[MAX_BIND_POINTS];
779
780 VkResult record_result;
781 };
782
783 #define panvk_cmd_get_bind_point_state(cmdbuf, bindpoint) \
784 &(cmdbuf)->bind_points[VK_PIPELINE_BIND_POINT_ ## bindpoint]
785
786 #define panvk_cmd_get_pipeline(cmdbuf, bindpoint) \
787 (cmdbuf)->bind_points[VK_PIPELINE_BIND_POINT_ ## bindpoint].pipeline
788
789 #define panvk_cmd_get_desc_state(cmdbuf, bindpoint) \
790 &(cmdbuf)->bind_points[VK_PIPELINE_BIND_POINT_ ## bindpoint].desc_state
791
792 struct panvk_batch *
793 panvk_cmd_open_batch(struct panvk_cmd_buffer *cmdbuf);
794
795 void
796 panvk_cmd_fb_info_set_subpass(struct panvk_cmd_buffer *cmdbuf);
797
798 void
799 panvk_cmd_fb_info_init(struct panvk_cmd_buffer *cmdbuf);
800
801 void
802 panvk_cmd_preload_fb_after_batch_split(struct panvk_cmd_buffer *cmdbuf);
803
804 void
805 panvk_pack_color(struct panvk_clear_value *out,
806 const VkClearColorValue *in,
807 enum pipe_format format);
808
809 struct panvk_event {
810 struct vk_object_base base;
811 uint32_t syncobj;
812 };
813
814 struct panvk_shader {
815 struct pan_shader_info info;
816 struct util_dynarray binary;
817 unsigned sysval_ubo;
818 struct pan_compute_dim local_size;
819 bool has_img_access;
820 };
821
822 struct panvk_shader *
823 panvk_shader_create(struct panvk_device *dev,
824 gl_shader_stage stage,
825 const VkPipelineShaderStageCreateInfo *stage_info,
826 const struct panvk_pipeline_layout *layout,
827 unsigned sysval_ubo,
828 struct pan_blend_state *blend_state,
829 bool static_blend_constants,
830 const VkAllocationCallbacks *alloc);
831
832 void
833 panvk_shader_destroy(struct panvk_device *dev,
834 struct panvk_shader *shader,
835 const VkAllocationCallbacks *alloc);
836
837 #define RSD_WORDS 16
838 #define BLEND_DESC_WORDS 4
839
840 struct panvk_pipeline {
841 struct vk_object_base base;
842
843 struct panvk_varyings_info varyings;
844 struct panvk_attribs_info attribs;
845
846 const struct panvk_pipeline_layout *layout;
847
848 unsigned active_stages;
849
850 uint32_t dynamic_state_mask;
851
852 struct panfrost_bo *binary_bo;
853 struct panfrost_bo *state_bo;
854
855 mali_ptr vpd;
856 mali_ptr rsds[MESA_SHADER_STAGES];
857
858 /* shader stage bit is set of the stage accesses storage images */
859 uint32_t img_access_mask;
860
861 unsigned num_ubos;
862
863 struct {
864 unsigned ubo_idx;
865 struct panfrost_sysvals ids;
866 } sysvals[MESA_SHADER_STAGES];
867
868 unsigned tls_size;
869 unsigned wls_size;
870
871 struct {
872 mali_ptr address;
873 struct pan_shader_info info;
874 uint32_t rsd_template[RSD_WORDS];
875 bool required;
876 bool dynamic_rsd;
877 uint8_t rt_mask;
878 } fs;
879
880 struct {
881 struct pan_compute_dim local_size;
882 } cs;
883
884 struct {
885 unsigned topology;
886 bool writes_point_size;
887 bool primitive_restart;
888 } ia;
889
890 struct {
891 bool clamp_depth;
892 float line_width;
893 struct {
894 bool enable;
895 float constant_factor;
896 float clamp;
897 float slope_factor;
898 } depth_bias;
899 bool front_ccw;
900 bool cull_front_face;
901 bool cull_back_face;
902 bool enable;
903 } rast;
904
905 struct {
906 bool z_test;
907 bool z_write;
908 unsigned z_compare_func;
909 bool s_test;
910 struct {
911 unsigned fail_op;
912 unsigned pass_op;
913 unsigned z_fail_op;
914 unsigned compare_func;
915 uint8_t compare_mask;
916 uint8_t write_mask;
917 uint8_t ref;
918 } s_front, s_back;
919 } zs;
920
921 struct {
922 uint8_t rast_samples;
923 uint8_t min_samples;
924 uint16_t sample_mask;
925 bool alpha_to_coverage;
926 bool alpha_to_one;
927 } ms;
928
929 struct {
930 struct pan_blend_state state;
931 uint32_t bd_template[8][BLEND_DESC_WORDS];
932 struct {
933 uint8_t index;
934 uint16_t bifrost_factor;
935 } constant[8];
936 bool reads_dest;
937 } blend;
938
939 VkViewport viewport;
940 VkRect2D scissor;
941 };
942
943 #define PANVK_MAX_PLANES 1
944
945 struct panvk_image {
946 struct vk_image vk;
947
948 struct pan_image pimage;
949 };
950
951 unsigned
952 panvk_image_get_plane_size(const struct panvk_image *image, unsigned plane);
953
954 unsigned
955 panvk_image_get_total_size(const struct panvk_image *image);
956
957 #define TEXTURE_DESC_WORDS 8
958 #define ATTRIB_BUF_DESC_WORDS 4
959
960 struct panvk_image_view {
961 struct vk_image_view vk;
962
963 struct pan_image_view pview;
964
965 struct panfrost_bo *bo;
966 struct {
967 uint32_t tex[TEXTURE_DESC_WORDS];
968 uint32_t img_attrib_buf[ATTRIB_BUF_DESC_WORDS * 2];
969 } descs;
970 };
971
972 #define SAMPLER_DESC_WORDS 8
973
974 struct panvk_sampler {
975 struct vk_object_base base;
976 uint32_t desc[SAMPLER_DESC_WORDS];
977 };
978
979 struct panvk_buffer_view {
980 struct vk_object_base base;
981 struct panfrost_bo *bo;
982 struct {
983 uint32_t tex[TEXTURE_DESC_WORDS];
984 uint32_t img_attrib_buf[ATTRIB_BUF_DESC_WORDS * 2];
985 } descs;
986 enum pipe_format fmt;
987 uint32_t elems;
988 };
989
990 struct panvk_attachment_info {
991 struct panvk_image_view *iview;
992 };
993
994 struct panvk_framebuffer {
995 struct vk_object_base base;
996
997 uint32_t width;
998 uint32_t height;
999 uint32_t layers;
1000
1001 uint32_t attachment_count;
1002 struct panvk_attachment_info attachments[0];
1003 };
1004
1005 struct panvk_clear_value {
1006 union {
1007 uint32_t color[4];
1008 struct {
1009 float depth;
1010 uint8_t stencil;
1011 };
1012 };
1013 };
1014
1015 struct panvk_subpass_attachment {
1016 uint32_t idx;
1017 VkImageLayout layout;
1018 bool clear;
1019 bool preload;
1020 };
1021
1022 struct panvk_subpass {
1023 uint32_t input_count;
1024 uint32_t color_count;
1025 struct panvk_subpass_attachment *input_attachments;
1026 uint8_t active_color_attachments;
1027 struct panvk_subpass_attachment *color_attachments;
1028 struct panvk_subpass_attachment *resolve_attachments;
1029 struct panvk_subpass_attachment zs_attachment;
1030
1031 uint32_t view_mask;
1032 };
1033
1034 struct panvk_render_pass_attachment {
1035 VkAttachmentDescriptionFlags flags;
1036 enum pipe_format format;
1037 unsigned samples;
1038 VkAttachmentLoadOp load_op;
1039 VkAttachmentStoreOp store_op;
1040 VkAttachmentLoadOp stencil_load_op;
1041 VkAttachmentStoreOp stencil_store_op;
1042 VkImageLayout initial_layout;
1043 VkImageLayout final_layout;
1044 unsigned view_mask;
1045 unsigned first_used_in_subpass;
1046 };
1047
1048 struct panvk_render_pass {
1049 struct vk_object_base base;
1050
1051 uint32_t attachment_count;
1052 uint32_t subpass_count;
1053 struct panvk_subpass_attachment *subpass_attachments;
1054 struct panvk_render_pass_attachment *attachments;
1055 struct panvk_subpass subpasses[0];
1056 };
1057
1058 VK_DEFINE_HANDLE_CASTS(panvk_cmd_buffer, vk.base, VkCommandBuffer, VK_OBJECT_TYPE_COMMAND_BUFFER)
1059 VK_DEFINE_HANDLE_CASTS(panvk_device, vk.base, VkDevice, VK_OBJECT_TYPE_DEVICE)
1060 VK_DEFINE_HANDLE_CASTS(panvk_instance, vk.base, VkInstance, VK_OBJECT_TYPE_INSTANCE)
1061 VK_DEFINE_HANDLE_CASTS(panvk_physical_device, vk.base, VkPhysicalDevice, VK_OBJECT_TYPE_PHYSICAL_DEVICE)
1062 VK_DEFINE_HANDLE_CASTS(panvk_queue, vk.base, VkQueue, VK_OBJECT_TYPE_QUEUE)
1063
1064 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_cmd_pool, vk.base, VkCommandPool, VK_OBJECT_TYPE_COMMAND_POOL)
1065 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_buffer, vk.base, VkBuffer, VK_OBJECT_TYPE_BUFFER)
1066 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_buffer_view, base, VkBufferView, VK_OBJECT_TYPE_BUFFER_VIEW)
1067 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_descriptor_pool, base, VkDescriptorPool, VK_OBJECT_TYPE_DESCRIPTOR_POOL)
1068 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_descriptor_set, base, VkDescriptorSet, VK_OBJECT_TYPE_DESCRIPTOR_SET)
1069 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_descriptor_set_layout, vk.base,
1070 VkDescriptorSetLayout, VK_OBJECT_TYPE_DESCRIPTOR_SET_LAYOUT)
1071 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_device_memory, base, VkDeviceMemory, VK_OBJECT_TYPE_DEVICE_MEMORY)
1072 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_event, base, VkEvent, VK_OBJECT_TYPE_EVENT)
1073 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_framebuffer, base, VkFramebuffer, VK_OBJECT_TYPE_FRAMEBUFFER)
1074 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_image, vk.base, VkImage, VK_OBJECT_TYPE_IMAGE)
1075 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_image_view, vk.base, VkImageView, VK_OBJECT_TYPE_IMAGE_VIEW);
1076 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_pipeline_cache, base, VkPipelineCache, VK_OBJECT_TYPE_PIPELINE_CACHE)
1077 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_pipeline, base, VkPipeline, VK_OBJECT_TYPE_PIPELINE)
1078 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_pipeline_layout, vk.base, VkPipelineLayout, VK_OBJECT_TYPE_PIPELINE_LAYOUT)
1079 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_render_pass, base, VkRenderPass, VK_OBJECT_TYPE_RENDER_PASS)
1080 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_sampler, base, VkSampler, VK_OBJECT_TYPE_SAMPLER)
1081
1082 #define panvk_arch_name(name, version) panvk_## version ## _ ## name
1083
1084 #define panvk_arch_dispatch(arch, name, ...) \
1085 do { \
1086 switch (arch) { \
1087 case 6: panvk_arch_name(name, v6)(__VA_ARGS__); break; \
1088 case 7: panvk_arch_name(name, v7)(__VA_ARGS__); break; \
1089 default: unreachable("Invalid arch"); \
1090 } \
1091 } while (0)
1092
1093 #ifdef PAN_ARCH
1094 #if PAN_ARCH == 6
1095 #define panvk_per_arch(name) panvk_arch_name(name, v6)
1096 #elif PAN_ARCH == 7
1097 #define panvk_per_arch(name) panvk_arch_name(name, v7)
1098 #endif
1099 #include "panvk_vX_cmd_buffer.h"
1100 #include "panvk_vX_cs.h"
1101 #include "panvk_vX_device.h"
1102 #include "panvk_vX_meta.h"
1103 #else
1104 #define PAN_ARCH 6
1105 #define panvk_per_arch(name) panvk_arch_name(name, v6)
1106 #include "panvk_vX_cmd_buffer.h"
1107 #include "panvk_vX_cs.h"
1108 #include "panvk_vX_device.h"
1109 #include "panvk_vX_meta.h"
1110 #undef PAN_ARCH
1111 #undef panvk_per_arch
1112 #define PAN_ARCH 7
1113 #define panvk_per_arch(name) panvk_arch_name(name, v7)
1114 #include "panvk_vX_cmd_buffer.h"
1115 #include "panvk_vX_cs.h"
1116 #include "panvk_vX_device.h"
1117 #include "panvk_vX_meta.h"
1118 #undef PAN_ARCH
1119 #undef panvk_per_arch
1120 #endif
1121
1122 #ifdef PAN_ARCH
1123 bool
1124 panvk_per_arch(blend_needs_lowering)(const struct panfrost_device *dev,
1125 const struct pan_blend_state *state,
1126 unsigned rt);
1127
1128 struct panvk_shader *
1129 panvk_per_arch(shader_create)(struct panvk_device *dev,
1130 gl_shader_stage stage,
1131 const VkPipelineShaderStageCreateInfo *stage_info,
1132 const struct panvk_pipeline_layout *layout,
1133 unsigned sysval_ubo,
1134 struct pan_blend_state *blend_state,
1135 bool static_blend_constants,
1136 const VkAllocationCallbacks *alloc);
1137 struct nir_shader;
1138
1139 bool
1140 panvk_per_arch(nir_lower_descriptors)(struct nir_shader *nir,
1141 struct panvk_device *dev,
1142 const struct panvk_pipeline_layout *layout,
1143 bool *has_img_access_out);
1144 #endif
1145
1146 #endif /* PANVK_PRIVATE_H */
1147