1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * Based on radeon_winsys.h which is:
6 * Copyright 2008 Corbin Simpson <MostAwesomeDude@gmail.com>
7 * Copyright 2010 Marek Olšák <maraeo@gmail.com>
8 *
9 * SPDX-License-Identifier: MIT
10 */
11
12 #ifndef RADV_RADEON_WINSYS_H
13 #define RADV_RADEON_WINSYS_H
14
15 #include <stdbool.h>
16 #include <stdint.h>
17 #include <stdio.h>
18 #include <stdlib.h>
19 #include <string.h>
20 #include "util/u_math.h"
21 #include "util/u_memory.h"
22 #include <vulkan/vulkan.h>
23 #include "amd_family.h"
24
25 struct radeon_info;
26 struct vk_sync_type;
27 struct vk_sync_wait;
28 struct vk_sync_signal;
29
30 enum radeon_bo_domain { /* bitfield */
31 RADEON_DOMAIN_GTT = 2,
32 RADEON_DOMAIN_VRAM = 4,
33 RADEON_DOMAIN_VRAM_GTT = RADEON_DOMAIN_VRAM | RADEON_DOMAIN_GTT,
34 RADEON_DOMAIN_GDS = 8,
35 RADEON_DOMAIN_OA = 16,
36 };
37
38 enum radeon_bo_flag { /* bitfield */
39 RADEON_FLAG_GTT_WC = (1 << 0),
40 RADEON_FLAG_CPU_ACCESS = (1 << 1),
41 RADEON_FLAG_NO_CPU_ACCESS = (1 << 2),
42 RADEON_FLAG_VIRTUAL = (1 << 3),
43 RADEON_FLAG_VA_UNCACHED = (1 << 4),
44 RADEON_FLAG_IMPLICIT_SYNC = (1 << 5),
45 RADEON_FLAG_NO_INTERPROCESS_SHARING = (1 << 6),
46 RADEON_FLAG_READ_ONLY = (1 << 7),
47 RADEON_FLAG_32BIT = (1 << 8),
48 RADEON_FLAG_PREFER_LOCAL_BO = (1 << 9),
49 RADEON_FLAG_ZERO_VRAM = (1 << 10),
50 RADEON_FLAG_REPLAYABLE = (1 << 11),
51 RADEON_FLAG_DISCARDABLE = (1 << 12),
52 RADEON_FLAG_GFX12_ALLOW_DCC = (1 << 13),
53 };
54
55 enum radeon_ctx_priority {
56 RADEON_CTX_PRIORITY_INVALID = -1,
57 RADEON_CTX_PRIORITY_LOW = 0,
58 RADEON_CTX_PRIORITY_MEDIUM,
59 RADEON_CTX_PRIORITY_HIGH,
60 RADEON_CTX_PRIORITY_REALTIME,
61 };
62
63 enum radeon_ctx_pstate {
64 RADEON_CTX_PSTATE_NONE = 0,
65 RADEON_CTX_PSTATE_STANDARD,
66 RADEON_CTX_PSTATE_MIN_SCLK,
67 RADEON_CTX_PSTATE_MIN_MCLK,
68 RADEON_CTX_PSTATE_PEAK,
69 };
70
71 enum radeon_value_id {
72 RADEON_ALLOCATED_VRAM,
73 RADEON_ALLOCATED_VRAM_VIS,
74 RADEON_ALLOCATED_GTT,
75 RADEON_TIMESTAMP,
76 RADEON_NUM_BYTES_MOVED,
77 RADEON_NUM_EVICTIONS,
78 RADEON_NUM_VRAM_CPU_PAGE_FAULTS,
79 RADEON_VRAM_USAGE,
80 RADEON_VRAM_VIS_USAGE,
81 RADEON_GTT_USAGE,
82 RADEON_GPU_TEMPERATURE,
83 RADEON_CURRENT_SCLK,
84 RADEON_CURRENT_MCLK,
85 };
86
87 struct radeon_cmdbuf {
88 /* These are uint64_t to tell the compiler that buf can't alias them.
89 * If they're uint32_t the generated code needs to redundantly
90 * store and reload them between buf writes. */
91 uint64_t cdw; /* Number of used dwords. */
92 uint64_t max_dw; /* Maximum number of dwords. */
93 uint64_t reserved_dw; /* Number of dwords reserved through radeon_check_space() */
94 uint32_t *buf; /* The base pointer of the chunk. */
95 };
96
97 #define RADEON_SURF_TYPE_MASK 0xFF
98 #define RADEON_SURF_TYPE_SHIFT 0
99 #define RADEON_SURF_TYPE_1D 0
100 #define RADEON_SURF_TYPE_2D 1
101 #define RADEON_SURF_TYPE_3D 2
102 #define RADEON_SURF_TYPE_CUBEMAP 3
103 #define RADEON_SURF_TYPE_1D_ARRAY 4
104 #define RADEON_SURF_TYPE_2D_ARRAY 5
105 #define RADEON_SURF_MODE_MASK 0xFF
106 #define RADEON_SURF_MODE_SHIFT 8
107
108 #define RADEON_SURF_GET(v, field) (((v) >> RADEON_SURF_##field##_SHIFT) & RADEON_SURF_##field##_MASK)
109 #define RADEON_SURF_SET(v, field) (((v)&RADEON_SURF_##field##_MASK) << RADEON_SURF_##field##_SHIFT)
110 #define RADEON_SURF_CLR(v, field) ((v) & ~(RADEON_SURF_##field##_MASK << RADEON_SURF_##field##_SHIFT))
111
112 enum radeon_bo_layout {
113 RADEON_LAYOUT_LINEAR = 0,
114 RADEON_LAYOUT_TILED,
115 RADEON_LAYOUT_SQUARETILED,
116
117 RADEON_LAYOUT_UNKNOWN
118 };
119
120 /* Tiling info for display code, DRI sharing, and other data. */
121 struct radeon_bo_metadata {
122 /* Tiling flags describing the texture layout for display code
123 * and DRI sharing.
124 */
125 union {
126 struct {
127 enum radeon_bo_layout microtile;
128 enum radeon_bo_layout macrotile;
129 unsigned pipe_config;
130 unsigned bankw;
131 unsigned bankh;
132 unsigned tile_split;
133 unsigned mtilea;
134 unsigned num_banks;
135 unsigned stride;
136 bool scanout;
137 } legacy;
138
139 struct {
140 /* surface flags */
141 unsigned swizzle_mode : 5;
142 bool scanout;
143 uint32_t dcc_offset_256b;
144 uint32_t dcc_pitch_max;
145 bool dcc_independent_64b_blocks;
146 bool dcc_independent_128b_blocks;
147 unsigned dcc_max_compressed_block_size;
148 } gfx9;
149
150 struct {
151 unsigned swizzle_mode : 3;
152 unsigned dcc_max_compressed_block : 3;
153 unsigned dcc_data_format : 6;
154 unsigned dcc_number_type : 3;
155 bool dcc_write_compress_disable;
156 bool scanout;
157 } gfx12;
158 } u;
159
160 /* Additional metadata associated with the buffer, in bytes.
161 * The maximum size is 64 * 4. This is opaque for the winsys & kernel.
162 * Supported by amdgpu only.
163 */
164 uint32_t size_metadata;
165 uint32_t metadata[64];
166 };
167
168 struct radeon_winsys_ctx;
169
170 struct radeon_winsys_bo {
171 uint64_t va;
172 uint64_t size;
173 /* buffer is created with AMDGPU_GEM_CREATE_VM_ALWAYS_VALID */
174 bool is_local;
175 bool vram_no_cpu_access;
176 /* buffer is added to the BO list of all submissions */
177 bool use_global_list;
178 bool gfx12_allow_dcc;
179 enum radeon_bo_domain initial_domain;
180 };
181
182 struct radv_winsys_submit_info {
183 enum amd_ip_type ip_type;
184 int queue_index;
185 unsigned cs_count;
186 unsigned initial_preamble_count;
187 unsigned continue_preamble_count;
188 unsigned postamble_count;
189 struct radeon_cmdbuf **cs_array;
190 struct radeon_cmdbuf **initial_preamble_cs;
191 struct radeon_cmdbuf **continue_preamble_cs;
192 struct radeon_cmdbuf **postamble_cs;
193 bool uses_shadow_regs;
194 };
195
196 /* Kernel effectively allows 0-31. This sets some priorities for fixed
197 * functionality buffers */
198 enum {
199 RADV_BO_PRIORITY_APPLICATION_MAX = 28,
200
201 /* virtual buffers have 0 priority since the priority is not used. */
202 RADV_BO_PRIORITY_VIRTUAL = 0,
203
204 RADV_BO_PRIORITY_METADATA = 10,
205 /* This should be considerably lower than most of the stuff below,
206 * but how much lower is hard to say since we don't know application
207 * assignments. Put it pretty high since it is GTT anyway. */
208 RADV_BO_PRIORITY_QUERY_POOL = 29,
209
210 RADV_BO_PRIORITY_DESCRIPTOR = 30,
211 RADV_BO_PRIORITY_UPLOAD_BUFFER = 30,
212 RADV_BO_PRIORITY_FENCE = 30,
213 RADV_BO_PRIORITY_SHADER = 31,
214 RADV_BO_PRIORITY_SCRATCH = 31,
215 RADV_BO_PRIORITY_CS = 31,
216 };
217
218 struct radv_winsys_gpuvm_fault_info {
219 uint64_t addr;
220 uint32_t status;
221 uint32_t vmhub;
222 };
223
224 enum radv_cs_dump_type {
225 RADV_CS_DUMP_TYPE_IBS,
226 RADV_CS_DUMP_TYPE_CTX_ROLLS,
227 };
228
229 struct radeon_winsys {
230 void (*destroy)(struct radeon_winsys *ws);
231
232 void (*query_info)(struct radeon_winsys *ws, struct radeon_info *gpu_info);
233
234 uint64_t (*query_value)(struct radeon_winsys *ws, enum radeon_value_id value);
235
236 bool (*read_registers)(struct radeon_winsys *ws, unsigned reg_offset, unsigned num_registers, uint32_t *out);
237
238 const char *(*get_chip_name)(struct radeon_winsys *ws);
239
240 bool (*query_gpuvm_fault)(struct radeon_winsys *ws, struct radv_winsys_gpuvm_fault_info *fault_info);
241
242 VkResult (*buffer_create)(struct radeon_winsys *ws, uint64_t size, unsigned alignment, enum radeon_bo_domain domain,
243 enum radeon_bo_flag flags, unsigned priority, uint64_t address,
244 struct radeon_winsys_bo **out_bo);
245
246 void (*buffer_destroy)(struct radeon_winsys *ws, struct radeon_winsys_bo *bo);
247 void *(*buffer_map)(struct radeon_winsys *ws, struct radeon_winsys_bo *bo, bool use_fixed_addr, void *fixed_addr);
248
249 VkResult (*buffer_from_ptr)(struct radeon_winsys *ws, void *pointer, uint64_t size, unsigned priority,
250 struct radeon_winsys_bo **out_bo);
251
252 VkResult (*buffer_from_fd)(struct radeon_winsys *ws, int fd, unsigned priority, struct radeon_winsys_bo **out_bo,
253 uint64_t *alloc_size);
254
255 bool (*buffer_get_fd)(struct radeon_winsys *ws, struct radeon_winsys_bo *bo, int *fd);
256
257 bool (*buffer_get_flags_from_fd)(struct radeon_winsys *ws, int fd, enum radeon_bo_domain *domains,
258 enum radeon_bo_flag *flags);
259
260 void (*buffer_unmap)(struct radeon_winsys *ws, struct radeon_winsys_bo *bo, bool replace);
261
262 void (*buffer_set_metadata)(struct radeon_winsys *ws, struct radeon_winsys_bo *bo, struct radeon_bo_metadata *md);
263 void (*buffer_get_metadata)(struct radeon_winsys *ws, struct radeon_winsys_bo *bo, struct radeon_bo_metadata *md);
264
265 VkResult (*buffer_virtual_bind)(struct radeon_winsys *ws, struct radeon_winsys_bo *parent, uint64_t offset,
266 uint64_t size, struct radeon_winsys_bo *bo, uint64_t bo_offset);
267
268 VkResult (*buffer_make_resident)(struct radeon_winsys *ws, struct radeon_winsys_bo *bo, bool resident);
269
270 VkResult (*ctx_create)(struct radeon_winsys *ws, enum radeon_ctx_priority priority, struct radeon_winsys_ctx **ctx);
271 void (*ctx_destroy)(struct radeon_winsys_ctx *ctx);
272
273 bool (*ctx_wait_idle)(struct radeon_winsys_ctx *ctx, enum amd_ip_type amd_ip_type, int ring_index);
274
275 int (*ctx_set_pstate)(struct radeon_winsys_ctx *ctx, uint32_t pstate);
276
277 enum radeon_bo_domain (*cs_domain)(const struct radeon_winsys *ws);
278
279 struct radeon_cmdbuf *(*cs_create)(struct radeon_winsys *ws, enum amd_ip_type amd_ip_type, bool is_secondary);
280
281 void (*cs_destroy)(struct radeon_cmdbuf *cs);
282
283 void (*cs_reset)(struct radeon_cmdbuf *cs);
284
285 bool (*cs_chain)(struct radeon_cmdbuf *cs, struct radeon_cmdbuf *next_cs, bool pre_en);
286
287 void (*cs_unchain)(struct radeon_cmdbuf *cs);
288
289 VkResult (*cs_finalize)(struct radeon_cmdbuf *cs);
290
291 void (*cs_grow)(struct radeon_cmdbuf *cs, size_t min_size);
292
293 VkResult (*cs_submit)(struct radeon_winsys_ctx *ctx, const struct radv_winsys_submit_info *submit,
294 uint32_t wait_count, const struct vk_sync_wait *waits, uint32_t signal_count,
295 const struct vk_sync_signal *signals);
296
297 void (*cs_add_buffer)(struct radeon_cmdbuf *cs, struct radeon_winsys_bo *bo);
298
299 void (*cs_execute_secondary)(struct radeon_cmdbuf *parent, struct radeon_cmdbuf *child, bool allow_ib2);
300
301 void (*cs_execute_ib)(struct radeon_cmdbuf *cs, struct radeon_winsys_bo *bo, const uint64_t va, const uint32_t cdw,
302 const bool predicate);
303
304 void (*cs_chain_dgc_ib)(struct radeon_cmdbuf *cs, uint64_t va, uint32_t cdw, uint64_t trailer_va,
305 const bool predicate);
306
307 void (*cs_dump)(struct radeon_cmdbuf *cs, FILE *file, const int *trace_ids, int trace_id_count,
308 enum radv_cs_dump_type type);
309
310 void (*cs_annotate)(struct radeon_cmdbuf *cs, const char *marker);
311
312 void (*cs_pad)(struct radeon_cmdbuf *cs, unsigned leave_dw_space);
313
314 void (*dump_bo_ranges)(struct radeon_winsys *ws, FILE *file);
315
316 void (*dump_bo_log)(struct radeon_winsys *ws, FILE *file);
317
318 int (*get_fd)(struct radeon_winsys *ws);
319
320 const struct vk_sync_type *const *(*get_sync_types)(struct radeon_winsys *ws);
321 };
322
323 static inline void
radeon_emit(struct radeon_cmdbuf * cs,uint32_t value)324 radeon_emit(struct radeon_cmdbuf *cs, uint32_t value)
325 {
326 assert(cs->cdw < cs->reserved_dw);
327 cs->buf[cs->cdw++] = value;
328 }
329
330 static inline void
radeon_emit_direct(struct radeon_cmdbuf * cs,uint32_t offset,uint32_t value)331 radeon_emit_direct(struct radeon_cmdbuf *cs, uint32_t offset, uint32_t value)
332 {
333 assert(offset < cs->reserved_dw);
334 cs->buf[offset] = value;
335 }
336
337 static inline void
radeon_emit_array(struct radeon_cmdbuf * cs,const uint32_t * values,unsigned count)338 radeon_emit_array(struct radeon_cmdbuf *cs, const uint32_t *values, unsigned count)
339 {
340 assert(cs->cdw + count <= cs->reserved_dw);
341 memcpy(cs->buf + cs->cdw, values, count * 4);
342 cs->cdw += count;
343 }
344
345 static inline uint64_t
radv_buffer_get_va(const struct radeon_winsys_bo * bo)346 radv_buffer_get_va(const struct radeon_winsys_bo *bo)
347 {
348 return bo->va;
349 }
350
351 static inline bool
radv_buffer_is_resident(const struct radeon_winsys_bo * bo)352 radv_buffer_is_resident(const struct radeon_winsys_bo *bo)
353 {
354 return bo->use_global_list || bo->is_local;
355 }
356
357 static inline void
radv_cs_add_buffer(struct radeon_winsys * ws,struct radeon_cmdbuf * cs,struct radeon_winsys_bo * bo)358 radv_cs_add_buffer(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, struct radeon_winsys_bo *bo)
359 {
360 if (radv_buffer_is_resident(bo))
361 return;
362
363 ws->cs_add_buffer(cs, bo);
364 }
365
366 static inline void *
radv_buffer_map(struct radeon_winsys * ws,struct radeon_winsys_bo * bo)367 radv_buffer_map(struct radeon_winsys *ws, struct radeon_winsys_bo *bo)
368 {
369 return ws->buffer_map(ws, bo, false, NULL);
370 }
371
372 #endif /* RADV_RADEON_WINSYS_H */
373