1 /*
2 * Copyright © 2011 Marek Olšák <maraeo@gmail.com>
3 * Copyright © 2015 Advanced Micro Devices, Inc.
4 *
5 * SPDX-License-Identifier: MIT
6 */
7
8 #ifndef AMDGPU_CS_H
9 #define AMDGPU_CS_H
10
11 #include "amdgpu_bo.h"
12 #include "util/u_memory.h"
13 #include "ac_linux_drm.h"
14 #include "drm-uapi/amdgpu_drm.h"
15
16 #ifdef __cplusplus
17 extern "C" {
18 #endif
19
20 /* Smaller submits means the GPU gets busy sooner and there is less
21 * waiting for buffers and fences. Proof:
22 * http://www.phoronix.com/scan.php?page=article&item=mesa-111-si&num=1
23 */
24 #define IB_MAX_SUBMIT_BYTES (80 * 1024)
25
26 struct amdgpu_ctx {
27 struct pipe_reference reference;
28 uint32_t ctx_handle;
29 struct amdgpu_winsys *aws;
30 ac_drm_bo user_fence_bo;
31 uint32_t user_fence_bo_kms_handle;
32 uint64_t *user_fence_cpu_address_base;
33
34 /* If true, report lost contexts and skip command submission.
35 * If false, terminate the process.
36 */
37 bool allow_context_lost;
38
39 /* Lost context status due to ioctl and allocation failures. */
40 enum pipe_reset_status sw_status;
41 };
42
43 struct amdgpu_cs_buffer {
44 struct amdgpu_winsys_bo *bo;
45 unsigned usage;
46 };
47
48 enum ib_type {
49 IB_PREAMBLE,
50 IB_MAIN,
51 IB_NUM,
52 };
53
54 struct amdgpu_ib {
55 /* A buffer out of which new IBs are allocated. */
56 struct pb_buffer_lean *big_buffer;
57 uint8_t *big_buffer_cpu_ptr;
58 uint64_t gpu_address;
59 unsigned used_ib_space;
60
61 /* The maximum seen size from cs_check_space. If the driver does
62 * cs_check_space and flush, the newly allocated IB should have at least
63 * this size.
64 */
65 unsigned max_check_space_size;
66
67 unsigned max_ib_bytes;
68 /* ptr_ib_size initially points to cs->csc->chunk_ib->ib_bytes.
69 * If in amdgpu_cs_check_space() ib chaining is required, then ptr_ib_size will point
70 * to indirect buffer packet size field.
71 */
72 uint32_t *ptr_ib_size;
73 bool is_chained_ib;
74 };
75
76 struct amdgpu_fence_list {
77 struct pipe_fence_handle **list;
78 unsigned num;
79 unsigned max;
80 };
81
82 struct amdgpu_buffer_list {
83 unsigned max_buffers;
84 unsigned num_buffers;
85 struct amdgpu_cs_buffer *buffers;
86 };
87
88 struct amdgpu_cs_context {
89 struct drm_amdgpu_cs_chunk_ib chunk_ib[IB_NUM];
90 uint32_t *ib_main_addr; /* the beginning of IB before chaining */
91
92 struct amdgpu_winsys *aws;
93
94 /* Buffers. */
95 struct amdgpu_buffer_list buffer_lists[NUM_BO_LIST_TYPES];
96 int16_t *buffer_indices_hashlist;
97
98 struct amdgpu_winsys_bo *last_added_bo;
99 unsigned last_added_bo_usage;
100
101 struct amdgpu_seq_no_fences seq_no_dependencies;
102
103 struct amdgpu_fence_list syncobj_dependencies;
104 struct amdgpu_fence_list syncobj_to_signal;
105
106 struct pipe_fence_handle *fence;
107
108 /* the error returned from cs_flush for non-async submissions */
109 int error_code;
110
111 /* TMZ: will this command be submitted using the TMZ flag */
112 bool secure;
113 };
114
115 /* This high limit is needed for viewperf2020/catia. */
116 #define BUFFER_HASHLIST_SIZE 32768
117
118 struct amdgpu_cs {
119 struct amdgpu_ib main_ib; /* must be first because this is inherited */
120 struct amdgpu_winsys *aws;
121 struct amdgpu_ctx *ctx;
122
123 /*
124 * Ensure a 64-bit alignment for drm_amdgpu_cs_chunk_fence.
125 */
126 struct drm_amdgpu_cs_chunk_fence fence_chunk;
127 enum amd_ip_type ip_type;
128 unsigned queue_index;
129
130 /* Whether this queue uses amdgpu_winsys_bo::alt_fence instead of generating its own
131 * sequence numbers for synchronization.
132 */
133 bool uses_alt_fence;
134
135 /* We flip between these two CS. While one is being consumed
136 * by the kernel in another thread, the other one is being filled
137 * by the pipe driver. */
138 struct amdgpu_cs_context csc1;
139 struct amdgpu_cs_context csc2;
140 /* The currently-used CS. */
141 struct amdgpu_cs_context *csc;
142 /* The CS being currently-owned by the other thread. */
143 struct amdgpu_cs_context *cst;
144 /* buffer_indices_hashlist[hash(bo)] returns -1 if the bo
145 * isn't part of any buffer lists or the index where the bo could be found.
146 * Since 1) hash collisions of 2 different bo can happen and 2) we use a
147 * single hashlist for the 3 buffer list, this is only a hint.
148 * amdgpu_lookup_buffer uses this hint to speed up buffers look up.
149 */
150 int16_t buffer_indices_hashlist[BUFFER_HASHLIST_SIZE];
151
152 /* Flush CS. */
153 void (*flush_cs)(void *ctx, unsigned flags, struct pipe_fence_handle **fence);
154 void *flush_data;
155 bool noop;
156 bool has_chaining;
157
158 struct util_queue_fence flush_completed;
159 struct pipe_fence_handle *next_fence;
160 struct pb_buffer_lean *preamble_ib_bo;
161
162 struct drm_amdgpu_cs_chunk_cp_gfx_shadow mcbp_fw_shadow_chunk;
163 };
164
165 struct amdgpu_fence {
166 struct pipe_reference reference;
167 uint32_t syncobj;
168
169 struct amdgpu_winsys *aws;
170
171 /* The following field aren't set for imported fences. */
172 struct amdgpu_ctx *ctx; /* submission context */
173 uint32_t ip_type;
174 uint64_t *user_fence_cpu_address;
175 uint64_t seq_no;
176
177 /* If the fence has been submitted. This is unsignalled for deferred fences
178 * (cs->next_fence) and while an IB is still being submitted in the submit
179 * thread. */
180 struct util_queue_fence submitted;
181
182 volatile int signalled; /* bool (int for atomicity) */
183 bool imported;
184 uint8_t queue_index; /* for non-imported fences */
185 uint_seq_no queue_seq_no; /* winsys-generated sequence number */
186 };
187
188 void amdgpu_fence_destroy(struct amdgpu_fence *fence);
189
amdgpu_ctx_reference(struct amdgpu_ctx ** dst,struct amdgpu_ctx * src)190 static inline void amdgpu_ctx_reference(struct amdgpu_ctx **dst, struct amdgpu_ctx *src)
191 {
192 struct amdgpu_ctx *old_dst = *dst;
193
194 if (pipe_reference(old_dst ? &old_dst->reference : NULL,
195 src ? &src->reference : NULL)) {
196 ac_drm_device *dev = old_dst->aws->dev;
197 ac_drm_bo_cpu_unmap(dev, old_dst->user_fence_bo);
198 ac_drm_bo_free(dev, old_dst->user_fence_bo);
199 ac_drm_cs_ctx_free(dev, old_dst->ctx_handle);
200 FREE(old_dst);
201 }
202 *dst = src;
203 }
204
amdgpu_fence_reference(struct pipe_fence_handle ** dst,struct pipe_fence_handle * src)205 static inline void amdgpu_fence_reference(struct pipe_fence_handle **dst,
206 struct pipe_fence_handle *src)
207 {
208 struct amdgpu_fence **adst = (struct amdgpu_fence **)dst;
209 struct amdgpu_fence *asrc = (struct amdgpu_fence *)src;
210
211 if (pipe_reference(&(*adst)->reference, &asrc->reference))
212 amdgpu_fence_destroy(*adst);
213
214 *adst = asrc;
215 }
216
217 /* Same as amdgpu_fence_reference, but ignore the value in *dst. */
amdgpu_fence_set_reference(struct pipe_fence_handle ** dst,struct pipe_fence_handle * src)218 static inline void amdgpu_fence_set_reference(struct pipe_fence_handle **dst,
219 struct pipe_fence_handle *src)
220 {
221 *dst = src;
222 pipe_reference(NULL, &((struct amdgpu_fence *)src)->reference); /* only increment refcount */
223 }
224
225 /* Unreference dst, but don't assign anything. */
amdgpu_fence_drop_reference(struct pipe_fence_handle * dst)226 static inline void amdgpu_fence_drop_reference(struct pipe_fence_handle *dst)
227 {
228 struct amdgpu_fence *adst = (struct amdgpu_fence *)dst;
229
230 if (pipe_reference(&adst->reference, NULL)) /* only decrement refcount */
231 amdgpu_fence_destroy(adst);
232 }
233
234 struct amdgpu_cs_buffer *
235 amdgpu_lookup_buffer_any_type(struct amdgpu_cs_context *cs, struct amdgpu_winsys_bo *bo);
236
237 static inline struct amdgpu_cs *
amdgpu_cs(struct radeon_cmdbuf * rcs)238 amdgpu_cs(struct radeon_cmdbuf *rcs)
239 {
240 struct amdgpu_cs *cs = (struct amdgpu_cs*)rcs->priv;
241 assert(cs);
242 return cs;
243 }
244
245 #define get_container(member_ptr, container_type, container_member) \
246 (container_type *)((char *)(member_ptr) - offsetof(container_type, container_member))
247
248 static inline bool
amdgpu_bo_is_referenced_by_cs(struct amdgpu_cs * cs,struct amdgpu_winsys_bo * bo)249 amdgpu_bo_is_referenced_by_cs(struct amdgpu_cs *cs,
250 struct amdgpu_winsys_bo *bo)
251 {
252 return amdgpu_lookup_buffer_any_type(cs->csc, bo) != NULL;
253 }
254
get_buf_list_idx(struct amdgpu_winsys_bo * bo)255 static inline unsigned get_buf_list_idx(struct amdgpu_winsys_bo *bo)
256 {
257 /* AMDGPU_BO_REAL_REUSABLE* maps to AMDGPU_BO_REAL. */
258 static_assert(ARRAY_SIZE(((struct amdgpu_cs_context*)NULL)->buffer_lists) == NUM_BO_LIST_TYPES, "");
259 return MIN2(bo->type, AMDGPU_BO_REAL);
260 }
261
262 static inline bool
amdgpu_bo_is_referenced_by_cs_with_usage(struct amdgpu_cs * cs,struct amdgpu_winsys_bo * bo,unsigned usage)263 amdgpu_bo_is_referenced_by_cs_with_usage(struct amdgpu_cs *cs,
264 struct amdgpu_winsys_bo *bo,
265 unsigned usage)
266 {
267 struct amdgpu_cs_buffer *buffer = amdgpu_lookup_buffer_any_type(cs->csc, bo);
268
269 return buffer && (buffer->usage & usage) != 0;
270 }
271
272 bool amdgpu_fence_wait(struct pipe_fence_handle *fence, uint64_t timeout,
273 bool absolute);
274 void amdgpu_cs_sync_flush(struct radeon_cmdbuf *rcs);
275 void amdgpu_cs_init_functions(struct amdgpu_screen_winsys *sws);
276
277 #ifdef __cplusplus
278 }
279 #endif
280
281 #endif
282