1 /*
2 * Copyright © 2022 Imagination Technologies Ltd.
3 *
4 * based in part on anv driver which is:
5 * Copyright © 2015 Intel Corporation
6 *
7 * based in part on v3dv_cl.h which is:
8 * Copyright © 2019 Raspberry Pi
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice (including the next
18 * paragraph) shall be included in all copies or substantial portions of the
19 * Software.
20 *
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27 * SOFTWARE.
28 */
29
30 #ifndef PVR_CSB_H
31 #define PVR_CSB_H
32
33 #include <assert.h>
34 #include <stdbool.h>
35 #include <stdint.h>
36 #include <vulkan/vulkan.h>
37
38 #include "pvr_bo.h"
39 #include "pvr_types.h"
40 #include "pvr_winsys.h"
41 #include "util/list.h"
42 #include "util/macros.h"
43 #include "util/u_dynarray.h"
44
45 #define __pvr_address_type pvr_dev_addr_t
46 #define __pvr_get_address(pvr_dev_addr) (pvr_dev_addr).addr
47 /* clang-format off */
48 #define __pvr_make_address(addr_u64) PVR_DEV_ADDR(addr_u64)
49 /* clang-format on */
50
51 #include "csbgen/rogue_hwdefs.h"
52
53 /**
54 * \brief Size of the individual csb buffer object.
55 */
56 #define PVR_CMD_BUFFER_CSB_BO_SIZE 4096
57
58 struct pvr_device;
59
60 enum pvr_cmd_stream_type {
61 PVR_CMD_STREAM_TYPE_INVALID = 0, /* explicitly treat 0 as invalid */
62 PVR_CMD_STREAM_TYPE_GRAPHICS,
63 PVR_CMD_STREAM_TYPE_GRAPHICS_DEFERRED,
64 PVR_CMD_STREAM_TYPE_COMPUTE,
65 };
66
67 struct pvr_csb {
68 struct pvr_device *device;
69
70 /* Pointer to current csb buffer object */
71 struct pvr_bo *pvr_bo;
72
73 /* pointers to current bo memory */
74 void *start;
75 void *end;
76 void *next;
77
78 /* When extending the control stream we can't break state updates across bos.
79 * This indicates where the current state update starts, so that it can be
80 * be relocated into the new bo without breaking the update.
81 */
82 void *relocation_mark;
83 #if MESA_DEBUG
84 /* Used to track the state of the `relocation_mark` and to catch cases where
85 * the driver might have emitted to the cs without using the
86 * `relocation_mark`. Doing so is mostly harmless but will waste memory in
87 * case the cs is extended while an untracked state update is emitted, as
88 * we'll have to relocate the cs contents from the last tracked state update
89 * instead of just the one currently being emitted.
90 */
91 enum pvr_csb_relocation_mark_status {
92 PVR_CSB_RELOCATION_MARK_UNINITIALIZED,
93 PVR_CSB_RELOCATION_MARK_SET,
94 PVR_CSB_RELOCATION_MARK_SET_AND_CONSUMED,
95 PVR_CSB_RELOCATION_MARK_CLEARED,
96 } relocation_mark_status;
97 #endif
98
99 /* List of csb buffer objects */
100 struct list_head pvr_bo_list;
101
102 struct util_dynarray deferred_cs_mem;
103
104 enum pvr_cmd_stream_type stream_type;
105
106 /* Current error status of the command buffer. Used to track inconsistent
107 * or incomplete command buffer states that are the consequence of run-time
108 * errors such as out of memory scenarios. We want to track this in the
109 * csb because the command buffer object is not visible to some parts
110 * of the driver.
111 */
112 VkResult status;
113 };
114
115 /**
116 * \brief Gets the status of the csb.
117 *
118 * \param[in] csb Control Stream Builder object.
119 * \return VK_SUCCESS if the csb hasn't encountered any error or error code
120 * otherwise.
121 */
pvr_csb_get_status(const struct pvr_csb * csb)122 static inline VkResult pvr_csb_get_status(const struct pvr_csb *csb)
123 {
124 return csb->status;
125 }
126
127 /**
128 * \brief Checks if the control stream is empty or not.
129 *
130 * \param[in] csb Control Stream Builder object.
131 * \return true if csb is empty false otherwise.
132 */
pvr_csb_is_empty(const struct pvr_csb * csb)133 static inline bool pvr_csb_is_empty(const struct pvr_csb *csb)
134 {
135 return list_is_empty(&csb->pvr_bo_list);
136 }
137
138 static inline pvr_dev_addr_t
pvr_csb_get_start_address(const struct pvr_csb * csb)139 pvr_csb_get_start_address(const struct pvr_csb *csb)
140 {
141 if (!pvr_csb_is_empty(csb)) {
142 struct pvr_bo *pvr_bo =
143 list_first_entry(&csb->pvr_bo_list, struct pvr_bo, link);
144
145 return pvr_bo->vma->dev_addr;
146 }
147
148 return PVR_DEV_ADDR_INVALID;
149 }
150
151 /** \defgroup CSB relocation marking.
152 * Functions and macros related to relocation marking for control stream words.
153 *
154 * When there is no more space left in the current bo, csb needs has to extend
155 * the control stream by allocating a new bo and emitting a link to it. State
156 * updates have to be contiguous so cannot be broken by a link. Thus csb copies
157 * the current, in construction, state update into the new bo and emits a link
158 * in its place in the old bo. To do so however, it needs a hint from the driver
159 * to determine where the current state update started from, so a relocation
160 * mark is used.
161 *
162 * List of words demarking the beginning of state updates (i.e. state update
163 * headers):
164 * - ROGUE_VDMCTRL_PPP_STATE0
165 * - ROGUE_VDMCTRL_PDS_STATE0
166 * - ROGUE_VDMCTRL_VDM_STATE0
167 * - ROGUE_VDMCTRL_INDEX_LIST0
168 * - ROGUE_VDMCTRL_STREAM_LINK0
169 * - ROGUE_VDMCTRL_STREAM_RETURN
170 * - ROGUE_VDMCTRL_STREAM_TERMINATE
171 *
172 * - ROGUE_CDMCTRL_KERNEL0
173 * - ROGUE_CDMCTRL_STREAM_LINK0
174 * - ROGUE_CDMCTRL_STREAM_TERMINATE
175 *
176 * The driver should set the relocation mark whenever a new state update is
177 * started. And clear it when the state update is fully formed.
178 *
179 * PVR_CSB_RELOCATION_MARK state machine:
180 *
181 * UNINITIALIZED
182 * ↓
183 * ┌─── → SET ─────────┐
184 * │ ↓ │
185 * │ SET_AND_CONSUMED │
186 * │ ↓ │
187 * │ CLEARED ← ─────┘
188 * └───────┘
189 *
190 * @{
191 */
192 /* TODO: Add in the IPF transfer control stream state updates to the list once
193 * csb gets used for it
194 */
195
196 /**
197 * \brief Set the relocation mark.
198 *
199 * Indicates to csb that on cs extension it should relocate all words, starting
200 * from now, into the new bo.
201 */
pvr_csb_set_relocation_mark(struct pvr_csb * csb)202 static inline void pvr_csb_set_relocation_mark(struct pvr_csb *csb)
203 {
204 #if MESA_DEBUG
205 assert(csb->relocation_mark_status ==
206 PVR_CSB_RELOCATION_MARK_UNINITIALIZED ||
207 csb->relocation_mark_status == PVR_CSB_RELOCATION_MARK_CLEARED);
208
209 csb->relocation_mark_status = PVR_CSB_RELOCATION_MARK_SET;
210 #endif
211
212 csb->relocation_mark = csb->next;
213 }
214
215 /**
216 * \brief Clear the relocation mark.
217 *
218 * Indicate to csb that the state update is fully formed so it doesn't need to
219 * relocate it in case of cs extension.
220 */
pvr_csb_clear_relocation_mark(UNUSED struct pvr_csb * csb)221 static inline void pvr_csb_clear_relocation_mark(UNUSED struct pvr_csb *csb)
222 {
223 #if MESA_DEBUG
224 assert(csb->relocation_mark_status == PVR_CSB_RELOCATION_MARK_SET ||
225 csb->relocation_mark_status ==
226 PVR_CSB_RELOCATION_MARK_SET_AND_CONSUMED);
227
228 csb->relocation_mark_status = PVR_CSB_RELOCATION_MARK_CLEARED;
229 #endif
230 }
231
232 /** @} */
233 /* End of \defgroup CSB relocation marking. */
234
235 void pvr_csb_init(struct pvr_device *device,
236 enum pvr_cmd_stream_type stream_type,
237 struct pvr_csb *csb);
238 void pvr_csb_finish(struct pvr_csb *csb);
239 VkResult pvr_csb_bake(struct pvr_csb *csb, struct list_head *bo_list_out);
240 void *pvr_csb_alloc_dwords(struct pvr_csb *csb, uint32_t num_dwords);
241 VkResult pvr_csb_copy(struct pvr_csb *csb_dst, struct pvr_csb *csb_src);
242 void pvr_csb_emit_link(struct pvr_csb *csb, pvr_dev_addr_t addr, bool ret);
243 VkResult pvr_csb_emit_return(struct pvr_csb *csb);
244 VkResult pvr_csb_emit_terminate(struct pvr_csb *csb);
245
246 void pvr_csb_dump(const struct pvr_csb *csb,
247 uint32_t frame_num,
248 uint32_t job_num);
249
250 #define pvr_cmd_length(x) ROGUE_##x##_length
251 #define pvr_cmd_header(x) ROGUE_##x##_header
252 #define pvr_cmd_pack(x) ROGUE_##x##_pack
253 #define pvr_cmd_unpack(x) ROGUE_##x##_unpack
254 #define pvr_cmd_enum_to_str(x) ROGUE_##x##_to_str
255
256 /**
257 * \brief Merges dwords0 and dwords1 arrays and stores the result into the
258 * control stream pointed by the csb object.
259 *
260 * \param[in] csb Control Stream Builder object.
261 * \param[in] dwords0 Dwords0 array.
262 * \param[in] dwords1 Dwords1 array.
263 */
264 #define pvr_csb_emit_merge(csb, dwords0, dwords1) \
265 do { \
266 uint32_t *dw; \
267 STATIC_ASSERT(ARRAY_SIZE(dwords0) == ARRAY_SIZE(dwords1)); \
268 dw = pvr_csb_alloc_dwords(csb, ARRAY_SIZE(dwords0)); \
269 if (!dw) \
270 break; \
271 for (uint32_t i = 0; i < ARRAY_SIZE(dwords0); i++) \
272 dw[i] = (dwords0)[i] | (dwords1)[i]; \
273 } while (0)
274
275 /**
276 * \brief Packs a command/state into one or more dwords and stores them into
277 * the control stream pointed by the csb object.
278 *
279 * \param[in] csb Control Stream Builder object.
280 * \param[in] cmd Command/state type.
281 * \param[in,out] name Name to give to the command/state structure variable,
282 * which contains the information to be packed. This can be
283 * used by the caller to modify the command or state
284 * information before it's packed.
285 */
286 #define pvr_csb_emit(csb, cmd, name) \
287 for (struct ROGUE_##cmd \
288 name = { pvr_cmd_header(cmd) }, \
289 *_dst = pvr_csb_alloc_dwords(csb, pvr_cmd_length(cmd)); \
290 __builtin_expect(_dst != NULL, 1); \
291 ({ \
292 pvr_cmd_pack(cmd)(_dst, &name); \
293 _dst = NULL; \
294 }))
295
296 /**
297 * \brief Stores dword into the control stream pointed by the csb object.
298 *
299 * \param[in] csb Control Stream Builder object.
300 * \param[in] dword Dword to store into control stream.
301 */
302 #define pvr_csb_emit_dword(csb, dword) \
303 do { \
304 uint32_t *dw; \
305 STATIC_ASSERT(sizeof(dword) == sizeof(uint32_t)); \
306 dw = pvr_csb_alloc_dwords(csb, 1U); \
307 if (!dw) \
308 break; \
309 *dw = dword; \
310 } while (0)
311
312 /**
313 * \name Raw command/state buffer helpers.
314 * These provide functionality to read or write control/state words from/to a
315 * raw buffer, accessed through a pointer, with some extra checks.
316 *
317 * The raw buffer doesn't have to be related to a control stream builder object
318 * so these can be used with any cpu accessible buffer.
319 */
320 /**@{*/
321
322 /**
323 * \brief Packs a command/state into one or more dwords and stores them in the
324 * memory pointed to by _dst.
325 *
326 * \param[out] _dst Pointer to store the packed command/state.
327 * \param[in] cmd Command/state type.
328 * \param[in,out] name Name to give to the command/state structure variable,
329 * which contains the information to be packed and emitted.
330 * This can be used by the caller to modify the command or
331 * state information before it's packed.
332 */
333 #define pvr_csb_pack(_dst, cmd, name) \
334 for (struct ROGUE_##cmd name = { pvr_cmd_header(cmd) }, \
335 *_loop_terminate = &name; \
336 __builtin_expect(_loop_terminate != NULL, 1); \
337 ({ \
338 STATIC_ASSERT(sizeof(*(_dst)) == \
339 PVR_DW_TO_BYTES(pvr_cmd_length(cmd))); \
340 pvr_cmd_pack(cmd)((_dst), &name); \
341 _loop_terminate = NULL; \
342 }))
343
344 /**
345 * \brief Unpacks one or more dwords into a command/state struct.
346 *
347 * Unlike pvr_csb_pack, this returns the stack-allocated struct directly
348 * since it is not needed afterwards.
349 *
350 * \param[in] _src Pointer to read the packed command/state from.
351 * \param[in] cmd Command/state type.
352 */
353 #define pvr_csb_unpack(_src, cmd) \
354 ({ \
355 struct ROGUE_##cmd _name; \
356 STATIC_ASSERT(sizeof(*(_src)) == PVR_DW_TO_BYTES(pvr_cmd_length(cmd))); \
357 pvr_cmd_unpack(cmd)((_src), &_name); \
358 _name; \
359 })
360
361 /**
362 * \brief Writes a command/state word value into a raw buffer and advance.
363 *
364 * The buffer pointer is incremented appropriately based on the control stream
365 * word length.
366 *
367 * \param[in,out] dst Raw buffer pointer for writing.
368 * \param[in] cmd Command/state type.
369 * \param[in] val Pre-packed value to write.
370 */
371 #define pvr_csb_write_value(dst, cmd, val) \
372 do { \
373 static_assert(sizeof(*(dst)) == PVR_DW_TO_BYTES(pvr_cmd_length(cmd)), \
374 "Size mismatch"); \
375 static_assert(sizeof(*(dst)) == sizeof(val), "Size mismatch"); \
376 *(dst) = (val); \
377 (dst)++; \
378 } while (0)
379
380 /**
381 * \brief Packs a command/state word struct and writes the value into a raw
382 * buffer and advance.
383 *
384 * The buffer pointer is incremented appropriately based on the control stream
385 * word length.
386 *
387 * \param[in,out] dst Raw buffer pointer for writing.
388 * \param[in] cmd Command/state type.
389 * \param[in] val Command/state struct to pack and write.
390 */
391 #define pvr_csb_write_struct(dst, cmd, val) \
392 do { \
393 static_assert(sizeof(*(dst)) == PVR_DW_TO_BYTES(pvr_cmd_length(cmd)), \
394 "Size mismatch"); \
395 pvr_cmd_pack(cmd)((dst), (val)); \
396 (dst)++; \
397 } while (0)
398
399 /**@}*/
400 /* End of \name Raw command/state buffer helpers. */
401
402 #endif /* PVR_CSB_H */
403