1 /*
2 * Copyright © 2022 Collabora Ltd. and Red Hat Inc.
3 * SPDX-License-Identifier: MIT
4 */
5 #ifndef NVK_CMD_BUFFER_H
6 #define NVK_CMD_BUFFER_H 1
7
8 #include "nvk_private.h"
9
10 #include "nv_push.h"
11 #include "nvk_cmd_pool.h"
12 #include "nvk_descriptor_set.h"
13 #include "nvk_image.h"
14 #include "nvk_shader.h"
15
16 #include "util/u_dynarray.h"
17
18 #include "vk_command_buffer.h"
19
20 #include <stdio.h>
21
22 struct nvk_buffer;
23 struct nvk_cbuf;
24 struct nvk_cmd_mem;
25 struct nvk_cmd_buffer;
26 struct nvk_cmd_pool;
27 struct nvk_image_view;
28 struct nvk_push_descriptor_set;
29 struct nvk_shader;
30 struct vk_shader;
31
32 /** Root descriptor table. This gets pushed to the GPU directly */
33 struct nvk_root_descriptor_table {
34 union {
35 struct {
36 uint32_t base_vertex;
37 uint32_t base_instance;
38 uint32_t draw_index;
39 uint32_t view_index;
40 struct nak_sample_location sample_locations[NVK_MAX_SAMPLES];
41 struct nak_sample_mask sample_masks[NVK_MAX_SAMPLES];
42 } draw;
43 struct {
44 uint32_t base_group[3];
45 uint32_t group_count[3];
46 } cs;
47 };
48
49 /* Client push constants */
50 uint8_t push[NVK_MAX_PUSH_SIZE];
51
52 /* Descriptor set addresses */
53 struct nvk_buffer_address sets[NVK_MAX_SETS];
54
55 /* For each descriptor set, the index in dynamic_buffers where that set's
56 * dynamic buffers start. This is maintained for every set, regardless
57 * of whether or not anything is bound there.
58 */
59 uint8_t set_dynamic_buffer_start[NVK_MAX_SETS];
60
61 /* Dynamic buffer bindings */
62 union nvk_buffer_descriptor dynamic_buffers[NVK_MAX_DYNAMIC_BUFFERS];
63
64 /* enfore alignment to 0x100 as needed pre pascal */
65 uint8_t __padding[0xb8];
66 };
67
68 /* helper macro for computing root descriptor byte offsets */
69 #define nvk_root_descriptor_offset(member)\
70 offsetof(struct nvk_root_descriptor_table, member)
71
72 enum ENUM_PACKED nvk_descriptor_set_type {
73 NVK_DESCRIPTOR_SET_TYPE_NONE,
74 NVK_DESCRIPTOR_SET_TYPE_SET,
75 NVK_DESCRIPTOR_SET_TYPE_PUSH,
76 NVK_DESCRIPTOR_SET_TYPE_BUFFER,
77 };
78
79 struct nvk_descriptor_set_binding {
80 enum nvk_descriptor_set_type type;
81 struct nvk_descriptor_set *set;
82 struct nvk_push_descriptor_set *push;
83 };
84
85 struct nvk_descriptor_state {
86 alignas(16) char root[sizeof(struct nvk_root_descriptor_table)];
87 void (*flush_root)(struct nvk_cmd_buffer *cmd,
88 struct nvk_descriptor_state *desc,
89 size_t offset, size_t size);
90
91 struct nvk_descriptor_set_binding sets[NVK_MAX_SETS];
92 uint32_t push_dirty;
93 };
94
95 #define nvk_descriptor_state_get_root(desc, member, dst) do { \
96 const struct nvk_root_descriptor_table *root = \
97 (const struct nvk_root_descriptor_table *)(desc)->root; \
98 *dst = root->member; \
99 } while (0)
100
101 #define nvk_descriptor_state_get_root_array(desc, member, \
102 start, count, dst) do { \
103 const struct nvk_root_descriptor_table *root = \
104 (const struct nvk_root_descriptor_table *)(desc)->root; \
105 unsigned _start = start; \
106 unsigned _count = count; \
107 assert(_start + _count <= ARRAY_SIZE(root->member)); \
108 for (unsigned i = 0; i < _count; i++) \
109 (dst)[i] = root->member[i + _start]; \
110 } while (0)
111
112 #define nvk_descriptor_state_set_root(cmd, desc, member, src) do { \
113 struct nvk_descriptor_state *_desc = (desc); \
114 struct nvk_root_descriptor_table *root = \
115 (struct nvk_root_descriptor_table *)_desc->root; \
116 root->member = (src); \
117 if (_desc->flush_root != NULL) { \
118 size_t offset = (char *)&root->member - (char *)root; \
119 _desc->flush_root((cmd), _desc, offset, sizeof(root->member)); \
120 } \
121 } while (0)
122
123 #define nvk_descriptor_state_set_root_array(cmd, desc, member, \
124 start, count, src) do { \
125 struct nvk_descriptor_state *_desc = (desc); \
126 struct nvk_root_descriptor_table *root = \
127 (struct nvk_root_descriptor_table *)_desc->root; \
128 unsigned _start = start; \
129 unsigned _count = count; \
130 assert(_start + _count <= ARRAY_SIZE(root->member)); \
131 for (unsigned i = 0; i < _count; i++) \
132 root->member[i + _start] = (src)[i]; \
133 if (_desc->flush_root != NULL) { \
134 size_t offset = (char *)&root->member[_start] - (char *)root; \
135 _desc->flush_root((cmd), _desc, offset, \
136 _count * sizeof(root->member[0])); \
137 } \
138 } while (0)
139
140 struct nvk_attachment {
141 VkFormat vk_format;
142 struct nvk_image_view *iview;
143
144 VkResolveModeFlagBits resolve_mode;
145 struct nvk_image_view *resolve_iview;
146
147 /* Needed to track the value of storeOp in case we need to copy images for
148 * the DRM_FORMAT_MOD_LINEAR case */
149 VkAttachmentStoreOp store_op;
150 };
151
152 struct nvk_rendering_state {
153 VkRenderingFlagBits flags;
154
155 VkRect2D area;
156 uint32_t layer_count;
157 uint32_t view_mask;
158 uint32_t samples;
159
160 uint32_t color_att_count;
161 struct nvk_attachment color_att[NVK_MAX_RTS];
162 struct nvk_attachment depth_att;
163 struct nvk_attachment stencil_att;
164 struct nvk_attachment fsr_att;
165
166 bool all_linear;
167 };
168
169 struct nvk_graphics_state {
170 struct nvk_rendering_state render;
171 struct nvk_descriptor_state descriptors;
172
173 VkShaderStageFlags shaders_dirty;
174 struct nvk_shader *shaders[MESA_SHADER_MESH + 1];
175
176 struct nvk_cbuf_group {
177 uint16_t dirty;
178 struct nvk_cbuf cbufs[16];
179 } cbuf_groups[5];
180
181 /* Used for meta save/restore */
182 struct nvk_addr_range vb0;
183
184 /* Needed by vk_command_buffer::dynamic_graphics_state */
185 struct vk_vertex_input_state _dynamic_vi;
186 struct vk_sample_locations_state _dynamic_sl;
187 };
188
189 struct nvk_compute_state {
190 struct nvk_descriptor_state descriptors;
191 struct nvk_shader *shader;
192 };
193
194 struct nvk_cmd_push {
195 void *map;
196 uint64_t addr;
197 uint32_t range;
198 bool no_prefetch;
199 };
200
201 struct nvk_cmd_buffer {
202 struct vk_command_buffer vk;
203
204 struct nvk_cmd_state {
205 uint64_t descriptor_buffers[NVK_MAX_SETS];
206 struct nvk_graphics_state gfx;
207 struct nvk_compute_state cs;
208 } state;
209
210 /** List of nvk_cmd_mem
211 *
212 * This list exists entirely for ownership tracking. Everything in here
213 * must also be in pushes or bo_refs if it is to be referenced by this
214 * command buffer.
215 */
216 struct list_head owned_mem;
217 struct list_head owned_gart_mem;
218
219 struct nvk_cmd_mem *upload_mem;
220 uint32_t upload_offset;
221
222 struct nvk_cmd_mem *cond_render_gart_mem;
223 uint32_t cond_render_gart_offset;
224
225 struct nvk_cmd_mem *push_mem;
226 uint32_t *push_mem_limit;
227 struct nv_push push;
228
229 /** Array of struct nvk_cmd_push
230 *
231 * This acts both as a BO reference as well as provides a range in the
232 * buffer to use as a pushbuf.
233 */
234 struct util_dynarray pushes;
235
236 uint64_t tls_space_needed;
237 };
238
239 VK_DEFINE_HANDLE_CASTS(nvk_cmd_buffer, vk.base, VkCommandBuffer,
240 VK_OBJECT_TYPE_COMMAND_BUFFER)
241
242 extern const struct vk_command_buffer_ops nvk_cmd_buffer_ops;
243
244 static inline struct nvk_device *
nvk_cmd_buffer_device(struct nvk_cmd_buffer * cmd)245 nvk_cmd_buffer_device(struct nvk_cmd_buffer *cmd)
246 {
247 return (struct nvk_device *)cmd->vk.base.device;
248 }
249
250 static inline struct nvk_cmd_pool *
nvk_cmd_buffer_pool(struct nvk_cmd_buffer * cmd)251 nvk_cmd_buffer_pool(struct nvk_cmd_buffer *cmd)
252 {
253 return (struct nvk_cmd_pool *)cmd->vk.pool;
254 }
255
256 void nvk_cmd_buffer_new_push(struct nvk_cmd_buffer *cmd);
257
258 #define NVK_CMD_BUFFER_MAX_PUSH 512
259
260 static inline struct nv_push *
nvk_cmd_buffer_push(struct nvk_cmd_buffer * cmd,uint32_t dw_count)261 nvk_cmd_buffer_push(struct nvk_cmd_buffer *cmd, uint32_t dw_count)
262 {
263 assert(dw_count <= NVK_CMD_BUFFER_MAX_PUSH);
264
265 /* Compare to the actual limit on our push bo */
266 if (unlikely(cmd->push.end + dw_count > cmd->push_mem_limit))
267 nvk_cmd_buffer_new_push(cmd);
268
269 cmd->push.limit = cmd->push.end + dw_count;
270
271 return &cmd->push;
272 }
273
274 void
275 nvk_cmd_buffer_push_indirect(struct nvk_cmd_buffer *cmd,
276 uint64_t addr, uint32_t dw_count);
277
278 void nvk_cmd_buffer_begin_graphics(struct nvk_cmd_buffer *cmd,
279 const VkCommandBufferBeginInfo *pBeginInfo);
280 void nvk_cmd_buffer_begin_compute(struct nvk_cmd_buffer *cmd,
281 const VkCommandBufferBeginInfo *pBeginInfo);
282
283 void nvk_cmd_invalidate_graphics_state(struct nvk_cmd_buffer *cmd);
284 void nvk_cmd_invalidate_compute_state(struct nvk_cmd_buffer *cmd);
285
286 void nvk_cmd_bind_shaders(struct vk_command_buffer *vk_cmd,
287 uint32_t stage_count,
288 const gl_shader_stage *stages,
289 struct vk_shader ** const shaders);
290
291 void nvk_cmd_bind_graphics_shader(struct nvk_cmd_buffer *cmd,
292 const gl_shader_stage stage,
293 struct nvk_shader *shader);
294
295 void nvk_cmd_bind_compute_shader(struct nvk_cmd_buffer *cmd,
296 struct nvk_shader *shader);
297
298 void nvk_cmd_dirty_cbufs_for_descriptors(struct nvk_cmd_buffer *cmd,
299 VkShaderStageFlags stages,
300 uint32_t sets_start, uint32_t sets_end);
301 void nvk_cmd_bind_vertex_buffer(struct nvk_cmd_buffer *cmd, uint32_t vb_idx,
302 struct nvk_addr_range addr_range);
303
304 static inline struct nvk_descriptor_state *
nvk_get_descriptors_state(struct nvk_cmd_buffer * cmd,VkPipelineBindPoint bind_point)305 nvk_get_descriptors_state(struct nvk_cmd_buffer *cmd,
306 VkPipelineBindPoint bind_point)
307 {
308 switch (bind_point) {
309 case VK_PIPELINE_BIND_POINT_GRAPHICS:
310 return &cmd->state.gfx.descriptors;
311 case VK_PIPELINE_BIND_POINT_COMPUTE:
312 return &cmd->state.cs.descriptors;
313 default:
314 unreachable("Unhandled bind point");
315 }
316 }
317
318 static inline struct nvk_descriptor_state *
nvk_get_descriptor_state_for_stages(struct nvk_cmd_buffer * cmd,VkShaderStageFlags stages)319 nvk_get_descriptor_state_for_stages(struct nvk_cmd_buffer *cmd,
320 VkShaderStageFlags stages)
321 {
322 if (stages & VK_SHADER_STAGE_COMPUTE_BIT) {
323 assert(stages == VK_SHADER_STAGE_COMPUTE_BIT);
324 return &cmd->state.cs.descriptors;
325 } else if (stages & NVK_SHADER_STAGE_GRAPHICS_BITS) {
326 assert(!(stages & ~NVK_SHADER_STAGE_GRAPHICS_BITS));
327 return &cmd->state.gfx.descriptors;
328 } else {
329 unreachable("Unknown shader stage");
330 }
331 }
332
333 VkResult nvk_cmd_buffer_upload_alloc(struct nvk_cmd_buffer *cmd,
334 uint32_t size, uint32_t alignment,
335 uint64_t *addr, void **ptr);
336
337 VkResult nvk_cmd_buffer_upload_data(struct nvk_cmd_buffer *cmd,
338 const void *data, uint32_t size,
339 uint32_t alignment, uint64_t *addr);
340
341 VkResult nvk_cmd_buffer_cond_render_alloc(struct nvk_cmd_buffer *cmd,
342 uint64_t *addr);
343
344 void nvk_cmd_flush_wait_dep(struct nvk_cmd_buffer *cmd,
345 const VkDependencyInfo *dep,
346 bool wait);
347
348 void nvk_cmd_invalidate_deps(struct nvk_cmd_buffer *cmd,
349 uint32_t dep_count,
350 const VkDependencyInfo *deps);
351
352 void
353 nvk_cmd_buffer_flush_push_descriptors(struct nvk_cmd_buffer *cmd,
354 struct nvk_descriptor_state *desc);
355
356 bool
357 nvk_cmd_buffer_get_cbuf_addr(struct nvk_cmd_buffer *cmd,
358 const struct nvk_descriptor_state *desc,
359 const struct nvk_shader *shader,
360 const struct nvk_cbuf *cbuf,
361 struct nvk_buffer_address *addr_out);
362 uint64_t
363 nvk_cmd_buffer_get_cbuf_descriptor_addr(struct nvk_cmd_buffer *cmd,
364 const struct nvk_descriptor_state *desc,
365 const struct nvk_cbuf *cbuf);
366
367 VkResult nvk_cmd_flush_cs_qmd(struct nvk_cmd_buffer *cmd,
368 uint32_t global_size[3],
369 uint64_t *qmd_addr_out,
370 uint64_t *root_desc_addr_out);
371
372 void nvk_cmd_flush_gfx_dynamic_state(struct nvk_cmd_buffer *cmd);
373 void nvk_cmd_flush_gfx_shaders(struct nvk_cmd_buffer *cmd);
374 void nvk_cmd_flush_gfx_cbufs(struct nvk_cmd_buffer *cmd);
375
376 void nvk_cmd_dispatch_shader(struct nvk_cmd_buffer *cmd,
377 struct nvk_shader *shader,
378 const void *push_data, size_t push_size,
379 uint32_t groupCountX,
380 uint32_t groupCountY,
381 uint32_t groupCountZ);
382
383 void nvk_meta_resolve_rendering(struct nvk_cmd_buffer *cmd,
384 const VkRenderingInfo *pRenderingInfo);
385
386 void nvk_cmd_buffer_dump(struct nvk_cmd_buffer *cmd, FILE *fp);
387
388 void nvk_linear_render_copy(struct nvk_cmd_buffer *cmd,
389 const struct nvk_image_view *iview,
390 VkRect2D copy_rect,
391 bool copy_to_tiled_shadow);
392
393 #endif
394