1 /*
2 * Copyright 2021 Alyssa Rosenzweig
3 * SPDX-License-Identifier: MIT
4 */
5
6 #pragma once
7
8 #include <stdint.h>
9 #include <xf86drm.h>
10 #include "util/ralloc.h"
11 #include "util/simple_mtx.h"
12 #include "util/sparse_array.h"
13 #include "util/timespec.h"
14 #include "util/u_printf.h"
15 #include "util/vma.h"
16 #include "agx_bo.h"
17 #include "agx_pack.h"
18 #include "decode.h"
19 #include "layout.h"
20 #include "libagx_dgc.h"
21 #include "unstable_asahi_drm.h"
22
23 #include "vdrm.h"
24 #include "virglrenderer_hw.h"
25
26 #include "asahi_proto.h"
27
28 // TODO: this is a lie right now
29 static const uint64_t AGX_SUPPORTED_INCOMPAT_FEATURES =
30 DRM_ASAHI_FEAT_MANDATORY_ZS_COMPRESSION;
31
32 enum agx_dbg {
33 AGX_DBG_TRACE = BITFIELD_BIT(0),
34 AGX_DBG_BODUMP = BITFIELD_BIT(1),
35 AGX_DBG_NO16 = BITFIELD_BIT(2),
36 AGX_DBG_DIRTY = BITFIELD_BIT(3),
37 AGX_DBG_PRECOMPILE = BITFIELD_BIT(4),
38 AGX_DBG_PERF = BITFIELD_BIT(5),
39 AGX_DBG_NOCOMPRESS = BITFIELD_BIT(6),
40 AGX_DBG_NOCLUSTER = BITFIELD_BIT(7),
41 AGX_DBG_SYNC = BITFIELD_BIT(8),
42 AGX_DBG_STATS = BITFIELD_BIT(9),
43 AGX_DBG_RESOURCE = BITFIELD_BIT(10),
44 AGX_DBG_BATCH = BITFIELD_BIT(11),
45 AGX_DBG_NOWC = BITFIELD_BIT(12),
46 AGX_DBG_SYNCTVB = BITFIELD_BIT(13),
47 AGX_DBG_SMALLTILE = BITFIELD_BIT(14),
48 AGX_DBG_NOMSAA = BITFIELD_BIT(15),
49 AGX_DBG_NOSHADOW = BITFIELD_BIT(16),
50 AGX_DBG_BODUMPVERBOSE = BITFIELD_BIT(17),
51 AGX_DBG_SCRATCH = BITFIELD_BIT(18),
52 AGX_DBG_NOSOFT = BITFIELD_BIT(19),
53 AGX_DBG_FEEDBACK = BITFIELD_BIT(20),
54 AGX_DBG_1QUEUE = BITFIELD_BIT(21),
55 };
56
57 /* How many power-of-two levels in the BO cache do we want? 2^14 minimum chosen
58 * as it is the page size that all allocations are rounded to
59 */
60 #define MIN_BO_CACHE_BUCKET (14) /* 2^14 = 16KB */
61 #define MAX_BO_CACHE_BUCKET (22) /* 2^22 = 4MB */
62
63 /* Fencepost problem, hence the off-by-one */
64 #define NR_BO_CACHE_BUCKETS (MAX_BO_CACHE_BUCKET - MIN_BO_CACHE_BUCKET + 1)
65
66 /* Forward decl only, do not pull in all of NIR */
67 struct nir_shader;
68
69 #define BARRIER_RENDER (1 << DRM_ASAHI_SUBQUEUE_RENDER)
70 #define BARRIER_COMPUTE (1 << DRM_ASAHI_SUBQUEUE_COMPUTE)
71
72 struct agx_submit_virt {
73 uint32_t vbo_res_id;
74 uint32_t extres_count;
75 struct asahi_ccmd_submit_res *extres;
76 };
77
78 typedef struct {
79 struct agx_bo *(*bo_alloc)(struct agx_device *dev, size_t size, size_t align,
80 enum agx_bo_flags flags);
81 int (*bo_bind)(struct agx_device *dev, struct agx_bo *bo, uint64_t addr,
82 size_t size_B, uint64_t offset_B, uint32_t flags,
83 bool unbind);
84 void (*bo_mmap)(struct agx_device *dev, struct agx_bo *bo);
85 ssize_t (*get_params)(struct agx_device *dev, void *buf, size_t size);
86 int (*submit)(struct agx_device *dev, struct drm_asahi_submit *submit,
87 struct agx_submit_virt *virt);
88 int (*bo_bind_object)(struct agx_device *dev, struct agx_bo *bo,
89 uint32_t *object_handle, size_t size_B,
90 uint64_t offset_B, uint32_t flags);
91 int (*bo_unbind_object)(struct agx_device *dev, uint32_t object_handle,
92 uint32_t flags);
93
94 } agx_device_ops_t;
95
96 struct agx_device {
97 uint32_t debug;
98
99 /* NIR library of AGX helpers/shaders. Immutable once created. */
100 const struct nir_shader *libagx;
101
102 /* Precompiled libagx binary table */
103 const uint32_t **libagx_programs;
104
105 char name[64];
106 struct drm_asahi_params_global params;
107 uint64_t next_global_id, last_global_id;
108 bool is_virtio;
109 agx_device_ops_t ops;
110
111 /* vdrm device */
112 struct vdrm_device *vdrm;
113 uint32_t next_blob_id;
114
115 /* Device handle */
116 int fd;
117
118 /* VM handle */
119 uint32_t vm_id;
120
121 /* Global queue handle */
122 uint32_t queue_id;
123
124 /* VMA heaps */
125 simple_mtx_t vma_lock;
126 uint64_t shader_base;
127 struct util_vma_heap main_heap;
128 struct util_vma_heap usc_heap;
129 uint64_t guard_size;
130
131 struct renderonly *ro;
132
133 pthread_mutex_t bo_map_lock;
134 struct util_sparse_array bo_map;
135 uint32_t max_handle;
136
137 struct {
138 simple_mtx_t lock;
139
140 /* List containing all cached BOs sorted in LRU (Least Recently Used)
141 * order so we can quickly evict BOs that are more than 1 second old.
142 */
143 struct list_head lru;
144
145 /* The BO cache is a set of buckets with power-of-two sizes. Each bucket
146 * is a linked list of free panfrost_bo objects.
147 */
148 struct list_head buckets[NR_BO_CACHE_BUCKETS];
149
150 /* Current size of the BO cache in bytes (sum of sizes of cached BOs) */
151 size_t size;
152
153 /* Number of hits/misses for the BO cache */
154 uint64_t hits, misses;
155 } bo_cache;
156
157 struct agxdecode_ctx *agxdecode;
158
159 /* Prepacked USC Sampler word to bind the txf sampler, used for
160 * precompiled shaders on both drivers.
161 */
162 struct agx_usc_sampler_packed txf_sampler;
163
164 /* Simplified device selection */
165 enum agx_chip chip;
166
167 struct {
168 uint64_t num;
169 uint64_t den;
170 } timestamp_to_ns;
171
172 struct {
173 uint64_t num;
174 uint64_t den;
175 } user_timestamp_to_ns;
176
177 struct u_printf_ctx printf;
178 };
179
180 static inline void *
agx_bo_map(struct agx_bo * bo)181 agx_bo_map(struct agx_bo *bo)
182 {
183 if (!bo->_map)
184 bo->dev->ops.bo_mmap(bo->dev, bo);
185
186 return bo->_map;
187 }
188
189 static inline bool
agx_has_soft_fault(struct agx_device * dev)190 agx_has_soft_fault(struct agx_device *dev)
191 {
192 return (dev->params.feat_compat & DRM_ASAHI_FEAT_SOFT_FAULTS) &&
193 !(dev->debug & AGX_DBG_NOSOFT);
194 }
195
196 static uint32_t
agx_usc_addr(struct agx_device * dev,uint64_t addr)197 agx_usc_addr(struct agx_device *dev, uint64_t addr)
198 {
199 assert(addr >= dev->shader_base);
200 assert((addr - dev->shader_base) <= UINT32_MAX);
201
202 return addr - dev->shader_base;
203 }
204
205 bool agx_open_device(void *memctx, struct agx_device *dev);
206
207 void agx_close_device(struct agx_device *dev);
208
209 static inline struct agx_bo *
agx_lookup_bo(struct agx_device * dev,uint32_t handle)210 agx_lookup_bo(struct agx_device *dev, uint32_t handle)
211 {
212 return util_sparse_array_get(&dev->bo_map, handle);
213 }
214
215 uint64_t agx_get_global_id(struct agx_device *dev);
216
217 uint32_t agx_create_command_queue(struct agx_device *dev, uint32_t caps,
218 uint32_t priority);
219 int agx_destroy_command_queue(struct agx_device *dev, uint32_t queue_id);
220
221 int agx_import_sync_file(struct agx_device *dev, struct agx_bo *bo, int fd);
222 int agx_export_sync_file(struct agx_device *dev, struct agx_bo *bo);
223
224 void agx_debug_fault(struct agx_device *dev, uint64_t addr);
225
226 uint64_t agx_get_gpu_timestamp(struct agx_device *dev);
227
228 static inline uint64_t
agx_gpu_time_to_ns(struct agx_device * dev,uint64_t gpu_time)229 agx_gpu_time_to_ns(struct agx_device *dev, uint64_t gpu_time)
230 {
231 return (gpu_time * dev->timestamp_to_ns.num) / dev->timestamp_to_ns.den;
232 }
233
234 static inline uint64_t
agx_gpu_timestamp_to_ns(struct agx_device * dev,uint64_t gpu_timestamp)235 agx_gpu_timestamp_to_ns(struct agx_device *dev, uint64_t gpu_timestamp)
236 {
237 return (gpu_timestamp * dev->user_timestamp_to_ns.num) /
238 dev->user_timestamp_to_ns.den;
239 }
240
241 void agx_get_device_uuid(const struct agx_device *dev, void *uuid);
242 void agx_get_driver_uuid(void *uuid);
243 unsigned agx_get_num_cores(const struct agx_device *dev);
244
245 struct agx_device_key agx_gather_device_key(struct agx_device *dev);
246
247 struct agx_va *agx_va_alloc(struct agx_device *dev, uint64_t size_B,
248 uint64_t align_B, enum agx_va_flags flags,
249 uint64_t fixed_va);
250 void agx_va_free(struct agx_device *dev, struct agx_va *va);
251
252 static inline bool
agx_supports_timestamps(const struct agx_device * dev)253 agx_supports_timestamps(const struct agx_device *dev)
254 {
255 return (dev->params.feat_compat & DRM_ASAHI_FEAT_USER_TIMESTAMPS);
256 }
257