1 /*
2 * Copyright 2024 Valve Corporation
3 * Copyright 2024 Alyssa Rosenzweig
4 * Copyright 2022-2023 Collabora Ltd. and Red Hat Inc.
5 * SPDX-License-Identifier: MIT
6 */
7
8 #pragma once
9
10 #include "asahi/lib/agx_device.h"
11 #include "util/rwlock.h"
12 #include "util/simple_mtx.h"
13 #include "util/u_dynarray.h"
14 #include "agx_bg_eot.h"
15 #include "agx_pack.h"
16 #include "agx_scratch.h"
17 #include "decode.h"
18 #include "vk_cmd_queue.h"
19 #include "vk_dispatch_table.h"
20
21 #include "hk_private.h"
22
23 #include "hk_descriptor_table.h"
24 #include "hk_queue.h"
25 #include "vk_device.h"
26 #include "vk_meta.h"
27 #include "vk_queue.h"
28
29 struct hk_physical_device;
30 struct vk_pipeline_cache;
31
32 /* Fixed offsets for reserved null image descriptors */
33 #define HK_NULL_TEX_OFFSET (0)
34 #define HK_NULL_PBE_OFFSET (24)
35
36 typedef void (*hk_internal_builder_t)(struct nir_builder *b, const void *key);
37
38 struct hk_internal_key {
39 hk_internal_builder_t builder;
40 size_t key_size;
41 uint8_t key[];
42 };
43
44 struct hk_internal_shaders {
45 simple_mtx_t lock;
46 struct hash_table *ht;
47 };
48
49 struct hk_rc_sampler {
50 struct agx_sampler_packed key;
51
52 /* Reference count for this hardware sampler, protected by the heap mutex */
53 uint16_t refcount;
54
55 /* Index of this hardware sampler in the hardware sampler heap */
56 uint16_t index;
57 };
58
59 struct hk_sampler_heap {
60 simple_mtx_t lock;
61
62 struct hk_descriptor_table table;
63
64 /* Map of agx_sampler_packed to hk_rc_sampler */
65 struct hash_table *ht;
66 };
67
68 struct hk_device {
69 struct vk_device vk;
70 struct agx_device dev;
71 struct agxdecode_ctx *decode_ctx;
72
73 struct hk_descriptor_table images;
74 struct hk_descriptor_table occlusion_queries;
75 struct hk_sampler_heap samplers;
76
77 struct hk_queue queue;
78
79 struct vk_pipeline_cache *mem_cache;
80
81 struct vk_meta_device meta;
82 struct agx_bg_eot_cache bg_eot;
83
84 struct {
85 struct agx_bo *bo;
86 struct agx_usc_uniform_packed image_heap;
87 uint64_t null_sink, zero_sink;
88 uint64_t geometry_state;
89 } rodata;
90
91 struct hk_internal_shaders prolog_epilog;
92 struct hk_internal_shaders kernels;
93 struct hk_api_shader *write_shader;
94
95 /* Indirected for common secondary emulation */
96 struct vk_device_dispatch_table cmd_dispatch;
97
98 /* Heap used for GPU-side memory allocation for geometry/tessellation.
99 *
100 * Control streams accessing the heap must be serialized. This is not
101 * expected to be a legitimate problem. If it is, we can rework later.
102 */
103 struct agx_bo *heap;
104
105 struct {
106 struct agx_scratch vs, fs, cs;
107 simple_mtx_t lock;
108 } scratch;
109
110 uint32_t perftest;
111
112 struct {
113 struct u_rwlock lock;
114 struct util_dynarray list;
115 struct util_dynarray counts;
116 } external_bos;
117 };
118
119 VK_DEFINE_HANDLE_CASTS(hk_device, vk.base, VkDevice, VK_OBJECT_TYPE_DEVICE)
120
121 enum hk_perftest {
122 HK_PERF_NOTESS = BITFIELD_BIT(0),
123 HK_PERF_NOBORDER = BITFIELD_BIT(1),
124 HK_PERF_NOBARRIER = BITFIELD_BIT(2),
125 HK_PERF_BATCH = BITFIELD_BIT(3),
126 HK_PERF_NOROBUST = BITFIELD_BIT(4),
127 };
128
129 #define HK_PERF(dev, flag) unlikely((dev)->perftest &HK_PERF_##flag)
130
131 static inline struct hk_physical_device *
hk_device_physical(struct hk_device * dev)132 hk_device_physical(struct hk_device *dev)
133 {
134 return (struct hk_physical_device *)dev->vk.physical;
135 }
136
137 VkResult hk_device_init_meta(struct hk_device *dev);
138 void hk_device_finish_meta(struct hk_device *dev);
139
140 VkResult hk_sampler_heap_add(struct hk_device *dev,
141 struct agx_sampler_packed desc,
142 struct hk_rc_sampler **out);
143
144 void hk_sampler_heap_remove(struct hk_device *dev, struct hk_rc_sampler *rc);
145
146 static inline struct agx_scratch *
hk_device_scratch_locked(struct hk_device * dev,enum pipe_shader_type stage)147 hk_device_scratch_locked(struct hk_device *dev, enum pipe_shader_type stage)
148 {
149 simple_mtx_assert_locked(&dev->scratch.lock);
150
151 switch (stage) {
152 case PIPE_SHADER_FRAGMENT:
153 return &dev->scratch.fs;
154 case PIPE_SHADER_VERTEX:
155 return &dev->scratch.vs;
156 default:
157 return &dev->scratch.cs;
158 }
159 }
160
161 static inline void
hk_device_alloc_scratch(struct hk_device * dev,enum pipe_shader_type stage,unsigned size)162 hk_device_alloc_scratch(struct hk_device *dev, enum pipe_shader_type stage,
163 unsigned size)
164 {
165 simple_mtx_lock(&dev->scratch.lock);
166 agx_scratch_alloc(hk_device_scratch_locked(dev, stage), size, 0);
167 simple_mtx_unlock(&dev->scratch.lock);
168 }
169