1 /*
2 * Copyright 2018 Collabora Ltd.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #ifndef ZINK_CONTEXT_H
25 #define ZINK_CONTEXT_H
26
27 #define ZINK_FBFETCH_BINDING 6 //COMPUTE+1
28 #define ZINK_SHADER_COUNT (PIPE_SHADER_TYPES - 1)
29
30 #define ZINK_DEFAULT_MAX_DESCS 5000
31 #define ZINK_DEFAULT_DESC_CLAMP (ZINK_DEFAULT_MAX_DESCS * 0.9)
32
33 #define ZINK_MAX_BINDLESS_HANDLES 1024
34
35 #include "zink_clear.h"
36 #include "zink_pipeline.h"
37 #include "zink_batch.h"
38 #include "zink_compiler.h"
39 #include "zink_descriptors.h"
40 #include "zink_surface.h"
41
42 #include "pipe/p_context.h"
43 #include "pipe/p_state.h"
44 #include "util/u_rect.h"
45 #include "util/u_threaded_context.h"
46 #include "util/u_idalloc.h"
47 #include "util/slab.h"
48 #include "util/list.h"
49 #include "util/u_dynarray.h"
50
51 #include <vulkan/vulkan.h>
52
53 #ifdef __cplusplus
54 extern "C" {
55 #endif
56
57 struct blitter_context;
58 struct list_head;
59
60 struct zink_blend_state;
61 struct zink_depth_stencil_alpha_state;
62 struct zink_gfx_program;
63 struct zink_rasterizer_state;
64 struct zink_resource;
65 struct zink_vertex_elements_state;
66
67 enum zink_blit_flags {
68 ZINK_BLIT_NORMAL = 1 << 0,
69 ZINK_BLIT_SAVE_FS = 1 << 1,
70 ZINK_BLIT_SAVE_FB = 1 << 2,
71 ZINK_BLIT_SAVE_TEXTURES = 1 << 3,
72 ZINK_BLIT_NO_COND_RENDER = 1 << 4,
73 };
74
75 struct zink_sampler_state {
76 VkSampler sampler;
77 uint32_t hash;
78 struct zink_descriptor_refs desc_set_refs;
79 struct zink_batch_usage *batch_uses;
80 bool custom_border_color;
81 };
82
83 struct zink_buffer_view {
84 struct pipe_reference reference;
85 struct pipe_resource *pres;
86 VkBufferViewCreateInfo bvci;
87 VkBufferView buffer_view;
88 uint32_t hash;
89 struct zink_batch_usage *batch_uses;
90 struct zink_descriptor_refs desc_set_refs;
91 };
92
93 struct zink_sampler_view {
94 struct pipe_sampler_view base;
95 union {
96 struct zink_surface *image_view;
97 struct zink_buffer_view *buffer_view;
98 };
99 };
100
101 struct zink_image_view {
102 struct pipe_image_view base;
103 union {
104 struct zink_surface *surface;
105 struct zink_buffer_view *buffer_view;
106 };
107 };
108
109 static inline struct zink_sampler_view *
zink_sampler_view(struct pipe_sampler_view * pview)110 zink_sampler_view(struct pipe_sampler_view *pview)
111 {
112 return (struct zink_sampler_view *)pview;
113 }
114
115 struct zink_so_target {
116 struct pipe_stream_output_target base;
117 struct pipe_resource *counter_buffer;
118 VkDeviceSize counter_buffer_offset;
119 uint32_t stride;
120 bool counter_buffer_valid;
121 };
122
123 static inline struct zink_so_target *
zink_so_target(struct pipe_stream_output_target * so_target)124 zink_so_target(struct pipe_stream_output_target *so_target)
125 {
126 return (struct zink_so_target *)so_target;
127 }
128
129 struct zink_viewport_state {
130 struct pipe_viewport_state viewport_states[PIPE_MAX_VIEWPORTS];
131 struct pipe_scissor_state scissor_states[PIPE_MAX_VIEWPORTS];
132 uint8_t num_viewports;
133 };
134
135
136 struct zink_descriptor_surface {
137 union {
138 struct zink_surface *surface;
139 struct zink_buffer_view *bufferview;
140 };
141 bool is_buffer;
142 };
143
144 struct zink_bindless_descriptor {
145 struct zink_descriptor_surface ds;
146 struct zink_sampler_state *sampler;
147 uint32_t handle;
148 uint32_t access; //PIPE_ACCESS_...
149 };
150
151 static inline struct zink_resource *
zink_descriptor_surface_resource(struct zink_descriptor_surface * ds)152 zink_descriptor_surface_resource(struct zink_descriptor_surface *ds)
153 {
154 return ds->is_buffer ? (struct zink_resource*)ds->bufferview->pres : (struct zink_resource*)ds->surface->base.texture;
155 }
156
157 typedef void (*pipe_draw_vbo_func)(struct pipe_context *pipe,
158 const struct pipe_draw_info *info,
159 unsigned drawid_offset,
160 const struct pipe_draw_indirect_info *indirect,
161 const struct pipe_draw_start_count_bias *draws,
162 unsigned num_draws);
163
164 typedef void (*pipe_launch_grid_func)(struct pipe_context *pipe, const struct pipe_grid_info *info);
165
166 typedef enum {
167 ZINK_NO_MULTIDRAW,
168 ZINK_MULTIDRAW,
169 } zink_multidraw;
170
171 typedef enum {
172 ZINK_NO_DYNAMIC_STATE,
173 ZINK_DYNAMIC_STATE,
174 } zink_dynamic_state;
175
176 typedef enum {
177 ZINK_NO_DYNAMIC_STATE2,
178 ZINK_DYNAMIC_STATE2,
179 } zink_dynamic_state2;
180
181 typedef enum {
182 ZINK_NO_DYNAMIC_VERTEX_INPUT,
183 ZINK_DYNAMIC_VERTEX_INPUT,
184 } zink_dynamic_vertex_input;
185
186 struct zink_context {
187 struct pipe_context base;
188 struct threaded_context *tc;
189 struct slab_child_pool transfer_pool;
190 struct slab_child_pool transfer_pool_unsync;
191 struct blitter_context *blitter;
192
193 pipe_draw_vbo_func draw_vbo[2]; //batch changed
194 pipe_launch_grid_func launch_grid[2]; //batch changed
195
196 struct pipe_device_reset_callback reset;
197
198 simple_mtx_t batch_mtx;
199 struct zink_fence *deferred_fence;
200 struct zink_fence *last_fence; //the last command buffer submitted
201 struct zink_batch_state *batch_states; //list of submitted batch states: ordered by increasing timeline id
202 unsigned batch_states_count; //number of states in `batch_states`
203 struct util_dynarray free_batch_states; //unused batch states
204 bool oom_flush;
205 bool oom_stall;
206 struct zink_batch batch;
207
208 unsigned shader_has_inlinable_uniforms_mask;
209 unsigned inlinable_uniforms_valid_mask;
210 uint32_t compute_inlinable_uniforms[MAX_INLINABLE_UNIFORMS];
211
212 struct pipe_constant_buffer ubos[PIPE_SHADER_TYPES][PIPE_MAX_CONSTANT_BUFFERS];
213 struct pipe_shader_buffer ssbos[PIPE_SHADER_TYPES][PIPE_MAX_SHADER_BUFFERS];
214 uint32_t writable_ssbos[PIPE_SHADER_TYPES];
215 struct zink_image_view image_views[PIPE_SHADER_TYPES][PIPE_MAX_SHADER_IMAGES];
216
217 struct pipe_framebuffer_state fb_state;
218 struct zink_framebuffer *(*get_framebuffer)(struct zink_context*);
219 void (*init_framebuffer)(struct zink_screen *screen, struct zink_framebuffer *fb, struct zink_render_pass *rp);
220 struct hash_table framebuffer_cache;
221
222 struct zink_vertex_elements_state *element_state;
223 struct zink_rasterizer_state *rast_state;
224 struct zink_depth_stencil_alpha_state *dsa_state;
225
226 struct hash_table desc_set_layouts[ZINK_DESCRIPTOR_TYPES];
227 bool pipeline_changed[2]; //gfx, compute
228
229 struct zink_shader *gfx_stages[ZINK_SHADER_COUNT];
230 struct zink_shader *last_vertex_stage;
231 bool shader_reads_drawid;
232 bool shader_reads_basevertex;
233 struct zink_gfx_pipeline_state gfx_pipeline_state;
234 /* there are 5 gfx stages, but VS and FS are assumed to be always present,
235 * thus only 3 stages need to be considered, giving 2^3 = 8 program caches.
236 */
237 struct hash_table program_cache[8];
238 uint32_t gfx_hash;
239 struct zink_gfx_program *curr_program;
240
241 struct zink_descriptor_data *dd;
242
243 struct zink_shader *compute_stage;
244 struct zink_compute_pipeline_state compute_pipeline_state;
245 struct hash_table compute_program_cache;
246 struct zink_compute_program *curr_compute;
247
248 unsigned shader_stages : ZINK_SHADER_COUNT; /* mask of bound gfx shader stages */
249 unsigned dirty_shader_stages : 6; /* mask of changed shader stages */
250 bool last_vertex_stage_dirty;
251
252 struct set render_pass_state_cache;
253 struct hash_table *render_pass_cache;
254 bool new_swapchain;
255 bool fb_changed;
256 bool rp_changed;
257
258 struct zink_framebuffer *framebuffer;
259 struct zink_framebuffer_clear fb_clears[PIPE_MAX_COLOR_BUFS + 1];
260 uint16_t clears_enabled;
261 uint16_t rp_clears_enabled;
262 uint16_t fbfetch_outputs;
263
264 VkBuffer vbufs[PIPE_MAX_ATTRIBS];
265 unsigned vbuf_offsets[PIPE_MAX_ATTRIBS];
266 struct pipe_vertex_buffer vertex_buffers[PIPE_MAX_ATTRIBS];
267 bool vertex_buffers_dirty;
268
269 void *sampler_states[PIPE_SHADER_TYPES][PIPE_MAX_SAMPLERS];
270 struct pipe_sampler_view *sampler_views[PIPE_SHADER_TYPES][PIPE_MAX_SAMPLERS];
271
272 struct zink_viewport_state vp_state;
273 bool vp_state_changed;
274 bool scissor_changed;
275
276 float blend_constants[4];
277
278 bool sample_locations_changed;
279 VkSampleLocationEXT vk_sample_locations[PIPE_MAX_SAMPLE_LOCATION_GRID_SIZE * PIPE_MAX_SAMPLE_LOCATION_GRID_SIZE];
280 uint8_t sample_locations[2 * 4 * 8 * 16];
281
282 struct pipe_stencil_ref stencil_ref;
283
284 union {
285 struct {
286 float default_inner_level[2];
287 float default_outer_level[4];
288 };
289 float tess_levels[6];
290 };
291
292 struct list_head suspended_queries;
293 struct list_head primitives_generated_queries;
294 bool queries_disabled, render_condition_active;
295 struct {
296 struct zink_query *query;
297 bool inverted;
298 } render_condition;
299
300 struct pipe_resource *dummy_vertex_buffer;
301 struct pipe_resource *dummy_xfb_buffer;
302 struct pipe_surface *dummy_surface[7];
303 struct zink_buffer_view *dummy_bufferview;
304
305 unsigned buffer_rebind_counter;
306
307 struct {
308 /* descriptor info */
309 VkDescriptorBufferInfo ubos[PIPE_SHADER_TYPES][PIPE_MAX_CONSTANT_BUFFERS];
310 uint32_t push_valid;
311 uint8_t num_ubos[PIPE_SHADER_TYPES];
312
313 VkDescriptorBufferInfo ssbos[PIPE_SHADER_TYPES][PIPE_MAX_SHADER_BUFFERS];
314 uint8_t num_ssbos[PIPE_SHADER_TYPES];
315
316 VkDescriptorImageInfo textures[PIPE_SHADER_TYPES][PIPE_MAX_SAMPLERS];
317 VkBufferView tbos[PIPE_SHADER_TYPES][PIPE_MAX_SAMPLERS];
318 uint8_t num_samplers[PIPE_SHADER_TYPES];
319 uint8_t num_sampler_views[PIPE_SHADER_TYPES];
320
321 VkDescriptorImageInfo images[PIPE_SHADER_TYPES][PIPE_MAX_SHADER_IMAGES];
322 VkBufferView texel_images[PIPE_SHADER_TYPES][PIPE_MAX_SHADER_IMAGES];
323 uint8_t num_images[PIPE_SHADER_TYPES];
324
325 VkDescriptorImageInfo fbfetch;
326
327 struct zink_resource *descriptor_res[ZINK_DESCRIPTOR_TYPES][PIPE_SHADER_TYPES][PIPE_MAX_SAMPLERS];
328 struct zink_descriptor_surface sampler_surfaces[PIPE_SHADER_TYPES][PIPE_MAX_SAMPLERS];
329 struct zink_descriptor_surface image_surfaces[PIPE_SHADER_TYPES][PIPE_MAX_SHADER_IMAGES];
330
331 struct {
332 struct util_idalloc tex_slots;
333 struct util_idalloc img_slots;
334 struct hash_table tex_handles;
335 struct hash_table img_handles;
336 VkBufferView *buffer_infos; //tex, img
337 VkDescriptorImageInfo *img_infos; //tex, img
338 struct util_dynarray updates;
339 struct util_dynarray resident;
340 } bindless[2]; //img, buffer
341 union {
342 bool bindless_dirty[2]; //tex, img
343 uint16_t any_bindless_dirty;
344 };
345 bool bindless_refs_dirty;
346 } di;
347 struct set *need_barriers[2]; //gfx, compute
348 struct set update_barriers[2][2]; //[gfx, compute][current, next]
349 uint8_t barrier_set_idx[2];
350 unsigned memory_barrier;
351
352 uint32_t num_so_targets;
353 struct pipe_stream_output_target *so_targets[PIPE_MAX_SO_OUTPUTS];
354 bool dirty_so_targets;
355 bool xfb_barrier;
356 bool first_frame_done;
357 bool have_timelines;
358
359 bool gfx_dirty;
360
361 bool is_device_lost;
362 bool primitive_restart;
363 bool vertex_state_changed : 1;
364 bool blend_state_changed : 1;
365 bool rast_state_changed : 1;
366 bool dsa_state_changed : 1;
367 bool stencil_ref_changed : 1;
368 };
369
370 static inline struct zink_context *
zink_context(struct pipe_context * context)371 zink_context(struct pipe_context *context)
372 {
373 return (struct zink_context *)context;
374 }
375
376 static inline bool
zink_fb_clear_enabled(const struct zink_context * ctx,unsigned idx)377 zink_fb_clear_enabled(const struct zink_context *ctx, unsigned idx)
378 {
379 if (idx == PIPE_MAX_COLOR_BUFS)
380 return ctx->clears_enabled & PIPE_CLEAR_DEPTHSTENCIL;
381 return ctx->clears_enabled & (PIPE_CLEAR_COLOR0 << idx);
382 }
383
384 void
385 zink_fence_wait(struct pipe_context *ctx);
386
387 void
388 zink_wait_on_batch(struct zink_context *ctx, uint32_t batch_id);
389
390 bool
391 zink_check_batch_completion(struct zink_context *ctx, uint32_t batch_id, bool have_lock);
392
393 void
394 zink_flush_queue(struct zink_context *ctx);
395 void
396 zink_update_fbfetch(struct zink_context *ctx);
397 bool
398 zink_resource_access_is_write(VkAccessFlags flags);
399
400 void
401 zink_resource_buffer_barrier(struct zink_context *ctx, struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline);
402 void
403 zink_fake_buffer_barrier(struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline);
404 bool
405 zink_resource_image_needs_barrier(struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline);
406 bool
407 zink_resource_image_barrier_init(VkImageMemoryBarrier *imb, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline);
408 void
409 zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res,
410 VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline);
411
412 bool
413 zink_resource_needs_barrier(struct zink_resource *res, VkImageLayout layout, VkAccessFlags flags, VkPipelineStageFlags pipeline);
414 void
415 zink_update_descriptor_refs(struct zink_context *ctx, bool compute);
416 void
417 zink_init_vk_sample_locations(struct zink_context *ctx, VkSampleLocationsInfoEXT *loc);
418
419 void
420 zink_begin_render_pass(struct zink_context *ctx);
421 void
422 zink_end_render_pass(struct zink_context *ctx);
423
424 static inline void
zink_batch_rp(struct zink_context * ctx)425 zink_batch_rp(struct zink_context *ctx)
426 {
427 if (!ctx->batch.in_rp)
428 zink_begin_render_pass(ctx);
429 }
430
431 static inline void
zink_batch_no_rp(struct zink_context * ctx)432 zink_batch_no_rp(struct zink_context *ctx)
433 {
434 zink_end_render_pass(ctx);
435 assert(!ctx->batch.in_rp);
436 }
437
438 static inline VkPipelineStageFlags
zink_pipeline_flags_from_pipe_stage(enum pipe_shader_type pstage)439 zink_pipeline_flags_from_pipe_stage(enum pipe_shader_type pstage)
440 {
441 switch (pstage) {
442 case PIPE_SHADER_VERTEX:
443 return VK_PIPELINE_STAGE_VERTEX_SHADER_BIT;
444 case PIPE_SHADER_FRAGMENT:
445 return VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
446 case PIPE_SHADER_GEOMETRY:
447 return VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT;
448 case PIPE_SHADER_TESS_CTRL:
449 return VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT;
450 case PIPE_SHADER_TESS_EVAL:
451 return VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT;
452 case PIPE_SHADER_COMPUTE:
453 return VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
454 default:
455 unreachable("unknown shader stage");
456 }
457 }
458
459 void
460 zink_rebind_all_buffers(struct zink_context *ctx);
461
462 void
463 zink_flush_memory_barrier(struct zink_context *ctx, bool is_compute);
464 void
465 zink_init_draw_functions(struct zink_context *ctx, struct zink_screen *screen);
466 void
467 zink_init_grid_functions(struct zink_context *ctx);
468
469 #ifdef __cplusplus
470 }
471 #endif
472
473 #ifndef __cplusplus
474 VkPipelineStageFlags
475 zink_pipeline_flags_from_stage(VkShaderStageFlagBits stage);
476
477 VkShaderStageFlagBits
478 zink_shader_stage(enum pipe_shader_type type);
479
480 struct pipe_context *
481 zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags);
482
483 void
484 zink_context_query_init(struct pipe_context *ctx);
485
486 void
487 zink_blit_begin(struct zink_context *ctx, enum zink_blit_flags flags);
488
489 void
490 zink_blit(struct pipe_context *pctx,
491 const struct pipe_blit_info *info);
492
493 bool
494 zink_blit_region_fills(struct u_rect region, unsigned width, unsigned height);
495
496 bool
497 zink_blit_region_covers(struct u_rect region, struct u_rect covers);
498
499 static inline struct u_rect
zink_rect_from_box(const struct pipe_box * box)500 zink_rect_from_box(const struct pipe_box *box)
501 {
502 return (struct u_rect){box->x, box->x + box->width, box->y, box->y + box->height};
503 }
504
505 static inline VkComponentSwizzle
zink_component_mapping(enum pipe_swizzle swizzle)506 zink_component_mapping(enum pipe_swizzle swizzle)
507 {
508 switch (swizzle) {
509 case PIPE_SWIZZLE_X: return VK_COMPONENT_SWIZZLE_R;
510 case PIPE_SWIZZLE_Y: return VK_COMPONENT_SWIZZLE_G;
511 case PIPE_SWIZZLE_Z: return VK_COMPONENT_SWIZZLE_B;
512 case PIPE_SWIZZLE_W: return VK_COMPONENT_SWIZZLE_A;
513 case PIPE_SWIZZLE_0: return VK_COMPONENT_SWIZZLE_ZERO;
514 case PIPE_SWIZZLE_1: return VK_COMPONENT_SWIZZLE_ONE;
515 case PIPE_SWIZZLE_NONE: return VK_COMPONENT_SWIZZLE_IDENTITY; // ???
516 default:
517 unreachable("unexpected swizzle");
518 }
519 }
520
521 enum pipe_swizzle
522 zink_clamp_void_swizzle(const struct util_format_description *desc, enum pipe_swizzle swizzle);
523
524 bool
525 zink_resource_rebind(struct zink_context *ctx, struct zink_resource *res);
526
527 void
528 zink_rebind_framebuffer(struct zink_context *ctx, struct zink_resource *res);
529
530 void
531 zink_copy_buffer(struct zink_context *ctx, struct zink_resource *dst, struct zink_resource *src,
532 unsigned dst_offset, unsigned src_offset, unsigned size);
533
534 void
535 zink_copy_image_buffer(struct zink_context *ctx, struct zink_resource *dst, struct zink_resource *src,
536 unsigned dst_level, unsigned dstx, unsigned dsty, unsigned dstz,
537 unsigned src_level, const struct pipe_box *src_box, enum pipe_map_flags map_flags);
538
539 void
540 zink_destroy_buffer_view(struct zink_screen *screen, struct zink_buffer_view *buffer_view);
541
542 void
543 debug_describe_zink_buffer_view(char *buf, const struct zink_buffer_view *ptr);
544
545 static inline void
zink_buffer_view_reference(struct zink_screen * screen,struct zink_buffer_view ** dst,struct zink_buffer_view * src)546 zink_buffer_view_reference(struct zink_screen *screen,
547 struct zink_buffer_view **dst,
548 struct zink_buffer_view *src)
549 {
550 struct zink_buffer_view *old_dst = dst ? *dst : NULL;
551
552 if (pipe_reference_described(old_dst ? &old_dst->reference : NULL, &src->reference,
553 (debug_reference_descriptor)debug_describe_zink_buffer_view))
554 zink_destroy_buffer_view(screen, old_dst);
555 if (dst) *dst = src;
556 }
557 #endif
558
559 #endif
560