1 /*
2 * Copyright 2018 Collabora Ltd.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "zink_clear.h"
25 #include "zink_context.h"
26 #include "zink_descriptors.h"
27 #include "zink_fence.h"
28 #include "zink_format.h"
29 #include "zink_framebuffer.h"
30 #include "zink_helpers.h"
31 #include "zink_inlines.h"
32 #include "zink_kopper.h"
33 #include "zink_program.h"
34 #include "zink_query.h"
35 #include "zink_render_pass.h"
36 #include "zink_resource.h"
37 #include "zink_screen.h"
38 #include "zink_state.h"
39 #include "zink_surface.h"
40
41
42 #include "nir/pipe_nir.h"
43 #include "util/u_blitter.h"
44 #include "util/u_debug.h"
45 #include "util/format_srgb.h"
46 #include "util/format/u_format.h"
47 #include "util/u_helpers.h"
48 #include "util/u_inlines.h"
49 #include "util/u_sample_positions.h"
50 #include "util/u_string.h"
51 #include "util/u_thread.h"
52 #include "util/perf/u_trace.h"
53 #include "util/u_cpu_detect.h"
54 #include "util/thread_sched.h"
55 #include "util/strndup.h"
56 #include "nir.h"
57 #include "nir_builder.h"
58
59 #include "vk_format.h"
60
61 #include "driver_trace/tr_context.h"
62
63 #include "util/u_memory.h"
64 #include "util/u_upload_mgr.h"
65
66 #define XXH_INLINE_ALL
67 #include "util/xxhash.h"
68
69 static void
update_tc_info(struct zink_context * ctx)70 update_tc_info(struct zink_context *ctx)
71 {
72 if (ctx->track_renderpasses) {
73 const struct tc_renderpass_info *info = threaded_context_get_renderpass_info(ctx->tc);
74 ctx->rp_changed |= ctx->dynamic_fb.tc_info.data != info->data;
75 ctx->dynamic_fb.tc_info.data = info->data;
76 } else {
77 struct tc_renderpass_info info = ctx->dynamic_fb.tc_info;
78 bool zsbuf_used = !ctx->zsbuf_unused;
79 bool zsbuf_write = zink_is_zsbuf_write(ctx);
80 ctx->dynamic_fb.tc_info.data32[0] = 0;
81 if (ctx->clears_enabled & PIPE_CLEAR_DEPTHSTENCIL)
82 ctx->dynamic_fb.tc_info.zsbuf_clear_partial = true;
83 if (ctx->rp_clears_enabled & PIPE_CLEAR_DEPTHSTENCIL)
84 ctx->dynamic_fb.tc_info.zsbuf_clear = true;
85 if (ctx->dynamic_fb.tc_info.zsbuf_clear != info.zsbuf_clear)
86 ctx->rp_loadop_changed = true;
87 if (zink_is_zsbuf_write(ctx) != zsbuf_write)
88 ctx->rp_layout_changed = true;
89 ctx->rp_changed |= zink_is_zsbuf_used(ctx) != zsbuf_used;
90 }
91 }
92
93 void
debug_describe_zink_buffer_view(char * buf,const struct zink_buffer_view * ptr)94 debug_describe_zink_buffer_view(char *buf, const struct zink_buffer_view *ptr)
95 {
96 sprintf(buf, "zink_buffer_view");
97 }
98
99 ALWAYS_INLINE static void
check_resource_for_batch_ref(struct zink_context * ctx,struct zink_resource * res)100 check_resource_for_batch_ref(struct zink_context *ctx, struct zink_resource *res)
101 {
102 if (!zink_resource_has_binds(res)) {
103 /* avoid desync between usage and tracking:
104 * - if usage exists, it must be removed before the context is destroyed
105 * - having usage does not imply having tracking
106 * - if tracking will be added here, also reapply usage to avoid dangling usage once tracking is removed
107 * TODO: somehow fix this for perf because it's an extra hash lookup
108 */
109 if (!res->obj->dt && zink_resource_has_usage(res))
110 zink_batch_reference_resource_rw(ctx, res, !!res->obj->bo->writes.u);
111 else
112 zink_batch_reference_resource(ctx, res);
113 }
114 }
115
116 static void
zink_context_destroy(struct pipe_context * pctx)117 zink_context_destroy(struct pipe_context *pctx)
118 {
119 struct zink_context *ctx = zink_context(pctx);
120 struct zink_screen *screen = zink_screen(pctx->screen);
121
122 struct pipe_framebuffer_state fb = {0};
123 pctx->set_framebuffer_state(pctx, &fb);
124
125 if (util_queue_is_initialized(&screen->flush_queue))
126 util_queue_finish(&screen->flush_queue);
127 if (ctx->bs && !screen->device_lost) {
128 simple_mtx_lock(&screen->queue_lock);
129 VkResult result = VKSCR(QueueWaitIdle)(screen->queue);
130 simple_mtx_unlock(&screen->queue_lock);
131
132 if (result != VK_SUCCESS)
133 mesa_loge("ZINK: vkQueueWaitIdle failed (%s)", vk_Result_to_str(result));
134 }
135
136 for (unsigned i = 0; i < ARRAY_SIZE(ctx->program_cache); i++) {
137 simple_mtx_lock((&ctx->program_lock[i]));
138 hash_table_foreach(&ctx->program_cache[i], entry) {
139 struct zink_program *pg = entry->data;
140 zink_program_finish(ctx, pg);
141 pg->removed = true;
142 }
143 simple_mtx_unlock((&ctx->program_lock[i]));
144 }
145
146 if (ctx->blitter)
147 util_blitter_destroy(ctx->blitter);
148 for (unsigned i = 0; i < ctx->fb_state.nr_cbufs; i++)
149 pipe_surface_release(&ctx->base, &ctx->fb_state.cbufs[i]);
150 pipe_surface_release(&ctx->base, &ctx->fb_state.zsbuf);
151
152 pipe_resource_reference(&ctx->dummy_vertex_buffer, NULL);
153 pipe_resource_reference(&ctx->dummy_xfb_buffer, NULL);
154
155 for (unsigned i = 0; i < ARRAY_SIZE(ctx->dummy_surface); i++)
156 pipe_surface_release(&ctx->base, &ctx->dummy_surface[i]);
157 zink_buffer_view_reference(screen, &ctx->dummy_bufferview, NULL);
158
159 zink_descriptors_deinit_bindless(ctx);
160
161 struct zink_batch_state *bs = ctx->batch_states;
162 while (bs) {
163 struct zink_batch_state *bs_next = bs->next;
164 zink_clear_batch_state(ctx, bs);
165 /* restore link as we insert them into the screens free_batch_states
166 * list below
167 */
168 bs->next = bs_next;
169 bs = bs_next;
170 }
171 bs = ctx->free_batch_states;
172 while (bs) {
173 struct zink_batch_state *bs_next = bs->next;
174 zink_clear_batch_state(ctx, bs);
175 bs->ctx = NULL;
176 /* restore link as we insert them into the screens free_batch_states
177 * list below
178 */
179 bs->next = bs_next;
180 bs = bs_next;
181 }
182 simple_mtx_lock(&screen->free_batch_states_lock);
183 if (ctx->batch_states) {
184 if (screen->free_batch_states)
185 screen->last_free_batch_state->next = ctx->batch_states;
186 else {
187 screen->free_batch_states = ctx->batch_states;
188 screen->last_free_batch_state = screen->free_batch_states;
189 }
190 }
191 while (screen->last_free_batch_state && screen->last_free_batch_state->next)
192 screen->last_free_batch_state = screen->last_free_batch_state->next;
193 if (ctx->free_batch_states) {
194 if (screen->free_batch_states)
195 screen->last_free_batch_state->next = ctx->free_batch_states;
196 else {
197 screen->free_batch_states = ctx->free_batch_states;
198 screen->last_free_batch_state = ctx->last_free_batch_state;
199 }
200 }
201 while (screen->last_free_batch_state && screen->last_free_batch_state->next)
202 screen->last_free_batch_state = screen->last_free_batch_state->next;
203 if (ctx->bs) {
204 zink_clear_batch_state(ctx, ctx->bs);
205 if (screen->free_batch_states)
206 screen->last_free_batch_state->next = ctx->bs;
207 else {
208 screen->free_batch_states = ctx->bs;
209 screen->last_free_batch_state = screen->free_batch_states;
210 }
211 }
212 while (screen->last_free_batch_state && screen->last_free_batch_state->next)
213 screen->last_free_batch_state = screen->last_free_batch_state->next;
214 simple_mtx_unlock(&screen->free_batch_states_lock);
215
216 for (unsigned i = 0; i < 2; i++) {
217 util_idalloc_fini(&ctx->di.bindless[i].tex_slots);
218 util_idalloc_fini(&ctx->di.bindless[i].img_slots);
219 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB)
220 free(ctx->di.bindless[i].db.buffer_infos);
221 else
222 free(ctx->di.bindless[i].t.buffer_infos);
223 free(ctx->di.bindless[i].img_infos);
224 util_dynarray_fini(&ctx->di.bindless[i].updates);
225 util_dynarray_fini(&ctx->di.bindless[i].resident);
226 }
227
228 if (ctx->null_fs)
229 pctx->delete_fs_state(pctx, ctx->null_fs);
230
231 hash_table_foreach(&ctx->framebuffer_cache, he)
232 zink_destroy_framebuffer(screen, he->data);
233
234 hash_table_foreach(ctx->render_pass_cache, he)
235 zink_destroy_render_pass(screen, he->data);
236
237 zink_context_destroy_query_pools(ctx);
238 set_foreach(&ctx->gfx_inputs, he) {
239 struct zink_gfx_input_key *ikey = (void*)he->key;
240 VKSCR(DestroyPipeline)(screen->dev, ikey->pipeline, NULL);
241 }
242 set_foreach(&ctx->gfx_outputs, he) {
243 struct zink_gfx_output_key *okey = (void*)he->key;
244 VKSCR(DestroyPipeline)(screen->dev, okey->pipeline, NULL);
245 }
246 u_upload_destroy(pctx->stream_uploader);
247 u_upload_destroy(pctx->const_uploader);
248 slab_destroy_child(&ctx->transfer_pool);
249 for (unsigned i = 0; i < ARRAY_SIZE(ctx->program_cache); i++)
250 _mesa_hash_table_clear(&ctx->program_cache[i], NULL);
251 for (unsigned i = 0; i < ARRAY_SIZE(ctx->program_lock); i++)
252 simple_mtx_destroy(&ctx->program_lock[i]);
253 _mesa_hash_table_destroy(ctx->render_pass_cache, NULL);
254 slab_destroy_child(&ctx->transfer_pool_unsync);
255
256 zink_descriptors_deinit(ctx);
257
258 if (!(ctx->flags & ZINK_CONTEXT_COPY_ONLY))
259 p_atomic_dec(&screen->base.num_contexts);
260
261 util_dynarray_foreach(&ctx->di.global_bindings, struct pipe_resource *, res) {
262 pipe_resource_reference(res, NULL);
263 }
264 util_dynarray_fini(&ctx->di.global_bindings);
265
266 ralloc_free(ctx);
267 }
268
269 static void
check_device_lost(struct zink_context * ctx)270 check_device_lost(struct zink_context *ctx)
271 {
272 if (!zink_screen(ctx->base.screen)->device_lost || ctx->is_device_lost)
273 return;
274 debug_printf("ZINK: device lost detected!\n");
275 if (ctx->reset.reset)
276 ctx->reset.reset(ctx->reset.data, PIPE_GUILTY_CONTEXT_RESET);
277 ctx->is_device_lost = true;
278 }
279
280 static enum pipe_reset_status
zink_get_device_reset_status(struct pipe_context * pctx)281 zink_get_device_reset_status(struct pipe_context *pctx)
282 {
283 struct zink_context *ctx = zink_context(pctx);
284
285 enum pipe_reset_status status = PIPE_NO_RESET;
286
287 if (ctx->is_device_lost) {
288 // Since we don't know what really happened to the hardware, just
289 // assume that we are in the wrong
290 status = PIPE_GUILTY_CONTEXT_RESET;
291
292 debug_printf("ZINK: device lost detected!\n");
293
294 if (ctx->reset.reset)
295 ctx->reset.reset(ctx->reset.data, status);
296 }
297
298 return status;
299 }
300
301 static void
zink_set_device_reset_callback(struct pipe_context * pctx,const struct pipe_device_reset_callback * cb)302 zink_set_device_reset_callback(struct pipe_context *pctx,
303 const struct pipe_device_reset_callback *cb)
304 {
305 struct zink_context *ctx = zink_context(pctx);
306 bool had_reset = !!ctx->reset.reset;
307
308 if (cb)
309 ctx->reset = *cb;
310 else
311 memset(&ctx->reset, 0, sizeof(ctx->reset));
312
313 bool have_reset = !!ctx->reset.reset;
314 if (had_reset != have_reset) {
315 if (have_reset)
316 p_atomic_inc(&zink_screen(pctx->screen)->robust_ctx_count);
317 else
318 p_atomic_dec(&zink_screen(pctx->screen)->robust_ctx_count);
319 }
320 }
321
322 static void
zink_set_context_param(struct pipe_context * pctx,enum pipe_context_param param,unsigned value)323 zink_set_context_param(struct pipe_context *pctx, enum pipe_context_param param,
324 unsigned value)
325 {
326 struct zink_context *ctx = zink_context(pctx);
327 struct zink_screen *screen = zink_screen(ctx->base.screen);
328
329 switch (param) {
330 case PIPE_CONTEXT_PARAM_UPDATE_THREAD_SCHEDULING:
331 if (screen->threaded_submit)
332 util_thread_sched_apply_policy(screen->flush_queue.threads[0],
333 UTIL_THREAD_DRIVER_SUBMIT, value, NULL);
334 break;
335 default:
336 break;
337 }
338 }
339
340 static void
zink_set_debug_callback(struct pipe_context * pctx,const struct util_debug_callback * cb)341 zink_set_debug_callback(struct pipe_context *pctx, const struct util_debug_callback *cb)
342 {
343 struct zink_context *ctx = zink_context(pctx);
344
345 if (cb)
346 ctx->dbg = *cb;
347 else
348 memset(&ctx->dbg, 0, sizeof(ctx->dbg));
349 }
350
351 static VkSamplerMipmapMode
sampler_mipmap_mode(enum pipe_tex_mipfilter filter)352 sampler_mipmap_mode(enum pipe_tex_mipfilter filter)
353 {
354 switch (filter) {
355 case PIPE_TEX_MIPFILTER_NEAREST: return VK_SAMPLER_MIPMAP_MODE_NEAREST;
356 case PIPE_TEX_MIPFILTER_LINEAR: return VK_SAMPLER_MIPMAP_MODE_LINEAR;
357 case PIPE_TEX_MIPFILTER_NONE:
358 unreachable("PIPE_TEX_MIPFILTER_NONE should be dealt with earlier");
359 }
360 unreachable("unexpected filter");
361 }
362
363 static VkSamplerAddressMode
sampler_address_mode(enum pipe_tex_wrap filter)364 sampler_address_mode(enum pipe_tex_wrap filter)
365 {
366 switch (filter) {
367 case PIPE_TEX_WRAP_REPEAT: return VK_SAMPLER_ADDRESS_MODE_REPEAT;
368 case PIPE_TEX_WRAP_CLAMP_TO_EDGE: return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
369 case PIPE_TEX_WRAP_CLAMP_TO_BORDER: return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
370 case PIPE_TEX_WRAP_MIRROR_REPEAT: return VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT;
371 case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_EDGE: return VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE;
372 case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER: return VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE; /* not technically correct, but kinda works */
373 default: break;
374 }
375 unreachable("unexpected wrap");
376 }
377
378 /* unnormalizedCoordinates only support CLAMP_TO_EDGE or CLAMP_TO_BORDER */
379 static VkSamplerAddressMode
sampler_address_mode_unnormalized(enum pipe_tex_wrap filter)380 sampler_address_mode_unnormalized(enum pipe_tex_wrap filter)
381 {
382 switch (filter) {
383 case PIPE_TEX_WRAP_CLAMP_TO_EDGE: return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
384 case PIPE_TEX_WRAP_CLAMP_TO_BORDER: return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
385 default: break;
386 }
387 unreachable("unexpected wrap");
388 }
389
390 static VkCompareOp
compare_op(enum pipe_compare_func op)391 compare_op(enum pipe_compare_func op)
392 {
393 switch (op) {
394 case PIPE_FUNC_NEVER: return VK_COMPARE_OP_NEVER;
395 case PIPE_FUNC_LESS: return VK_COMPARE_OP_LESS;
396 case PIPE_FUNC_EQUAL: return VK_COMPARE_OP_EQUAL;
397 case PIPE_FUNC_LEQUAL: return VK_COMPARE_OP_LESS_OR_EQUAL;
398 case PIPE_FUNC_GREATER: return VK_COMPARE_OP_GREATER;
399 case PIPE_FUNC_NOTEQUAL: return VK_COMPARE_OP_NOT_EQUAL;
400 case PIPE_FUNC_GEQUAL: return VK_COMPARE_OP_GREATER_OR_EQUAL;
401 case PIPE_FUNC_ALWAYS: return VK_COMPARE_OP_ALWAYS;
402 }
403 unreachable("unexpected compare");
404 }
405
406 static inline bool
wrap_needs_border_color(unsigned wrap)407 wrap_needs_border_color(unsigned wrap)
408 {
409 return wrap == PIPE_TEX_WRAP_CLAMP || wrap == PIPE_TEX_WRAP_CLAMP_TO_BORDER ||
410 wrap == PIPE_TEX_WRAP_MIRROR_CLAMP || wrap == PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER;
411 }
412
413 static VkBorderColor
get_border_color(const union pipe_color_union * color,bool is_integer,bool need_custom)414 get_border_color(const union pipe_color_union *color, bool is_integer, bool need_custom)
415 {
416 if (is_integer) {
417 if (color->ui[0] == 0 && color->ui[1] == 0 && color->ui[2] == 0 && color->ui[3] == 0)
418 return VK_BORDER_COLOR_INT_TRANSPARENT_BLACK;
419 if (color->ui[0] == 0 && color->ui[1] == 0 && color->ui[2] == 0 && color->ui[3] == 1)
420 return VK_BORDER_COLOR_INT_OPAQUE_BLACK;
421 if (color->ui[0] == 1 && color->ui[1] == 1 && color->ui[2] == 1 && color->ui[3] == 1)
422 return VK_BORDER_COLOR_INT_OPAQUE_WHITE;
423 return need_custom ? VK_BORDER_COLOR_INT_CUSTOM_EXT : VK_BORDER_COLOR_INT_TRANSPARENT_BLACK;
424 }
425
426 if (color->f[0] == 0 && color->f[1] == 0 && color->f[2] == 0 && color->f[3] == 0)
427 return VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
428 if (color->f[0] == 0 && color->f[1] == 0 && color->f[2] == 0 && color->f[3] == 1)
429 return VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK;
430 if (color->f[0] == 1 && color->f[1] == 1 && color->f[2] == 1 && color->f[3] == 1)
431 return VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE;
432 return need_custom ? VK_BORDER_COLOR_FLOAT_CUSTOM_EXT : VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
433 }
434
435 static void *
zink_create_sampler_state(struct pipe_context * pctx,const struct pipe_sampler_state * state)436 zink_create_sampler_state(struct pipe_context *pctx,
437 const struct pipe_sampler_state *state)
438 {
439 struct zink_screen *screen = zink_screen(pctx->screen);
440 ASSERTED struct zink_context *zink = zink_context(pctx);
441 bool need_custom = false;
442 bool need_clamped_border_color = false;
443 VkSamplerCreateInfo sci = {0};
444 VkSamplerCustomBorderColorCreateInfoEXT cbci = {0};
445 VkSamplerCustomBorderColorCreateInfoEXT cbci_clamped = {0};
446 sci.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO;
447 if (screen->info.have_EXT_non_seamless_cube_map && !state->seamless_cube_map)
448 sci.flags |= VK_SAMPLER_CREATE_NON_SEAMLESS_CUBE_MAP_BIT_EXT;
449 if (state->unnormalized_coords) {
450 assert(zink->flags & PIPE_CONTEXT_COMPUTE_ONLY);
451 sci.unnormalizedCoordinates = state->unnormalized_coords;
452 }
453 sci.magFilter = zink_filter(state->mag_img_filter);
454 if (sci.unnormalizedCoordinates)
455 sci.minFilter = sci.magFilter;
456 else
457 sci.minFilter = zink_filter(state->min_img_filter);
458
459 VkSamplerReductionModeCreateInfo rci;
460 rci.sType = VK_STRUCTURE_TYPE_SAMPLER_REDUCTION_MODE_CREATE_INFO;
461 rci.pNext = NULL;
462 switch (state->reduction_mode) {
463 case PIPE_TEX_REDUCTION_MIN:
464 rci.reductionMode = VK_SAMPLER_REDUCTION_MODE_MIN;
465 break;
466 case PIPE_TEX_REDUCTION_MAX:
467 rci.reductionMode = VK_SAMPLER_REDUCTION_MODE_MAX;
468 break;
469 default:
470 rci.reductionMode = VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE;
471 break;
472 }
473 if (state->reduction_mode)
474 sci.pNext = &rci;
475
476 if (sci.unnormalizedCoordinates) {
477 sci.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST;
478 } else if (state->min_mip_filter != PIPE_TEX_MIPFILTER_NONE) {
479 sci.mipmapMode = sampler_mipmap_mode(state->min_mip_filter);
480 sci.minLod = state->min_lod;
481 sci.maxLod = MAX2(state->max_lod, state->min_lod);
482 } else {
483 sci.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST;
484 sci.minLod = CLAMP(state->min_lod, 0.0f, 0.25f);
485 sci.maxLod = CLAMP(state->max_lod, 0.0f, 0.25f);
486 }
487
488 if (!sci.unnormalizedCoordinates) {
489 sci.addressModeU = sampler_address_mode(state->wrap_s);
490 sci.addressModeV = sampler_address_mode(state->wrap_t);
491 sci.addressModeW = sampler_address_mode(state->wrap_r);
492 } else {
493 sci.addressModeU = sampler_address_mode_unnormalized(state->wrap_s);
494 sci.addressModeV = sampler_address_mode_unnormalized(state->wrap_t);
495 sci.addressModeW = sampler_address_mode_unnormalized(state->wrap_r);
496 }
497
498 sci.mipLodBias = CLAMP(state->lod_bias,
499 -screen->info.props.limits.maxSamplerLodBias,
500 screen->info.props.limits.maxSamplerLodBias);
501
502 need_custom |= wrap_needs_border_color(state->wrap_s);
503 need_custom |= wrap_needs_border_color(state->wrap_t);
504 need_custom |= wrap_needs_border_color(state->wrap_r);
505
506 if (state->compare_mode == PIPE_TEX_COMPARE_NONE)
507 sci.compareOp = VK_COMPARE_OP_NEVER;
508 else {
509 sci.compareOp = compare_op(state->compare_func);
510 sci.compareEnable = VK_TRUE;
511 }
512
513 bool is_integer = state->border_color_is_integer;
514
515 sci.borderColor = get_border_color(&state->border_color, is_integer, need_custom);
516 if (sci.borderColor > VK_BORDER_COLOR_INT_OPAQUE_WHITE && need_custom) {
517 if (!screen->info.border_color_feats.customBorderColorWithoutFormat &&
518 zink_driverid(screen) != VK_DRIVER_ID_MESA_TURNIP) {
519 static bool warned = false;
520 warn_missing_feature(warned, "customBorderColorWithoutFormat");
521 }
522 if (screen->info.have_EXT_custom_border_color &&
523 (screen->info.border_color_feats.customBorderColorWithoutFormat || state->border_color_format)) {
524 if (!screen->info.have_EXT_border_color_swizzle) {
525 static bool warned = false;
526 warn_missing_feature(warned, "VK_EXT_border_color_swizzle");
527 }
528
529 if (!is_integer && !screen->have_D24_UNORM_S8_UINT) {
530 union pipe_color_union clamped_border_color;
531 for (unsigned i = 0; i < 4; ++i) {
532 /* Use channel 0 on purpose, so that we can use OPAQUE_WHITE
533 * when the border color is 1.0. */
534 clamped_border_color.f[i] = CLAMP(state->border_color.f[0], 0, 1);
535 }
536 if (memcmp(&state->border_color, &clamped_border_color, sizeof(clamped_border_color)) != 0) {
537 need_clamped_border_color = true;
538 cbci_clamped.sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT;
539 cbci_clamped.format = VK_FORMAT_UNDEFINED;
540 /* these are identical unions */
541 memcpy(&cbci_clamped.customBorderColor, &clamped_border_color, sizeof(union pipe_color_union));
542 }
543 }
544 cbci.sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT;
545 if (screen->info.border_color_feats.customBorderColorWithoutFormat) {
546 cbci.format = VK_FORMAT_UNDEFINED;
547 /* these are identical unions */
548 memcpy(&cbci.customBorderColor, &state->border_color, sizeof(union pipe_color_union));
549 } else {
550 if (util_format_is_depth_or_stencil(state->border_color_format)) {
551 if (is_integer) {
552 cbci.format = VK_FORMAT_S8_UINT;
553 for (unsigned i = 0; i < 4; i++)
554 cbci.customBorderColor.uint32[i] = CLAMP(state->border_color.ui[i], 0, 255);
555 } else {
556 cbci.format = zink_get_format(screen, util_format_get_depth_only(state->border_color_format));
557 /* these are identical unions */
558 memcpy(&cbci.customBorderColor, &state->border_color, sizeof(union pipe_color_union));
559 }
560 } else {
561 cbci.format = zink_get_format(screen, state->border_color_format);
562 union pipe_color_union color;
563 for (unsigned i = 0; i < 4; i++) {
564 zink_format_clamp_channel_srgb(util_format_description(state->border_color_format), &color, &state->border_color, i);
565 }
566 zink_convert_color(screen, state->border_color_format, (void*)&cbci.customBorderColor, &color);
567 }
568 }
569 cbci.pNext = sci.pNext;
570 sci.pNext = &cbci;
571 UNUSED uint32_t check = p_atomic_inc_return(&screen->cur_custom_border_color_samplers);
572 assert(check <= screen->info.border_color_props.maxCustomBorderColorSamplers);
573 } else
574 sci.borderColor = VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK; // TODO with custom shader if we're super interested?
575 }
576
577 if (state->max_anisotropy > 1) {
578 sci.maxAnisotropy = state->max_anisotropy;
579 sci.anisotropyEnable = VK_TRUE;
580 }
581
582 struct zink_sampler_state *sampler = CALLOC_STRUCT(zink_sampler_state);
583 if (!sampler)
584 return NULL;
585
586 VkResult result = VKSCR(CreateSampler)(screen->dev, &sci, NULL, &sampler->sampler);
587 if (result != VK_SUCCESS) {
588 mesa_loge("ZINK: vkCreateSampler failed (%s)", vk_Result_to_str(result));
589 FREE(sampler);
590 return NULL;
591 }
592 if (need_clamped_border_color) {
593 sci.pNext = &cbci_clamped;
594 result = VKSCR(CreateSampler)(screen->dev, &sci, NULL, &sampler->sampler_clamped);
595 if (result != VK_SUCCESS) {
596 mesa_loge("ZINK: vkCreateSampler failed (%s)", vk_Result_to_str(result));
597 VKSCR(DestroySampler)(screen->dev, sampler->sampler, NULL);
598 FREE(sampler);
599 return NULL;
600 }
601 }
602 sampler->custom_border_color = need_custom;
603 if (!screen->info.have_EXT_non_seamless_cube_map)
604 sampler->emulate_nonseamless = !state->seamless_cube_map;
605
606 return sampler;
607 }
608
609 ALWAYS_INLINE static VkImageLayout
get_layout_for_binding(const struct zink_context * ctx,struct zink_resource * res,enum zink_descriptor_type type,bool is_compute)610 get_layout_for_binding(const struct zink_context *ctx, struct zink_resource *res, enum zink_descriptor_type type, bool is_compute)
611 {
612 if (res->obj->is_buffer)
613 return 0;
614 switch (type) {
615 case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW:
616 return zink_descriptor_util_image_layout_eval(ctx, res, is_compute);
617 case ZINK_DESCRIPTOR_TYPE_IMAGE:
618 return VK_IMAGE_LAYOUT_GENERAL;
619 default:
620 break;
621 }
622 return 0;
623 }
624
625 ALWAYS_INLINE static struct zink_surface *
get_imageview_for_binding(struct zink_context * ctx,gl_shader_stage stage,enum zink_descriptor_type type,unsigned idx)626 get_imageview_for_binding(struct zink_context *ctx, gl_shader_stage stage, enum zink_descriptor_type type, unsigned idx)
627 {
628 switch (type) {
629 case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW: {
630 struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[stage][idx]);
631 if (!sampler_view || !sampler_view->base.texture)
632 return NULL;
633 /* if this is a non-seamless cube sampler, return the cube array view */
634 if (ctx->di.emulate_nonseamless[stage] & ctx->di.cubes[stage] & BITFIELD_BIT(idx))
635 return sampler_view->cube_array;
636 bool needs_zs_shader_swizzle = (ctx->di.zs_swizzle[stage].mask & BITFIELD_BIT(idx)) &&
637 zink_screen(ctx->base.screen)->driver_compiler_workarounds.needs_zs_shader_swizzle;
638 bool needs_shadow_shader_swizzle = (stage == MESA_SHADER_FRAGMENT) && ctx->gfx_stages[MESA_SHADER_FRAGMENT] &&
639 (ctx->di.zs_swizzle[MESA_SHADER_FRAGMENT].mask & ctx->gfx_stages[MESA_SHADER_FRAGMENT]->fs.legacy_shadow_mask & BITFIELD_BIT(idx));
640 if (sampler_view->zs_view && (needs_zs_shader_swizzle || needs_shadow_shader_swizzle))
641 return sampler_view->zs_view;
642 return sampler_view->image_view;
643 }
644 case ZINK_DESCRIPTOR_TYPE_IMAGE: {
645 struct zink_image_view *image_view = &ctx->image_views[stage][idx];
646 return image_view->base.resource ? image_view->surface : NULL;
647 }
648 default:
649 break;
650 }
651 unreachable("ACK");
652 return VK_NULL_HANDLE;
653 }
654
655 ALWAYS_INLINE static struct zink_buffer_view *
get_bufferview_for_binding(struct zink_context * ctx,gl_shader_stage stage,enum zink_descriptor_type type,unsigned idx)656 get_bufferview_for_binding(struct zink_context *ctx, gl_shader_stage stage, enum zink_descriptor_type type, unsigned idx)
657 {
658 switch (type) {
659 case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW: {
660 struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[stage][idx]);
661 return sampler_view->base.texture ? sampler_view->buffer_view : NULL;
662 }
663 case ZINK_DESCRIPTOR_TYPE_IMAGE: {
664 struct zink_image_view *image_view = &ctx->image_views[stage][idx];
665 return image_view->base.resource ? image_view->buffer_view : NULL;
666 }
667 default:
668 break;
669 }
670 unreachable("ACK");
671 return VK_NULL_HANDLE;
672 }
673
674 ALWAYS_INLINE static struct zink_resource *
update_descriptor_state_ubo_db(struct zink_context * ctx,gl_shader_stage shader,unsigned slot,struct zink_resource * res)675 update_descriptor_state_ubo_db(struct zink_context *ctx, gl_shader_stage shader, unsigned slot, struct zink_resource *res)
676 {
677 struct zink_screen *screen = zink_screen(ctx->base.screen);
678 ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_UBO][shader][slot] = res;
679 if (res) {
680 ctx->di.db.ubos[shader][slot].address = res->obj->bda + ctx->ubos[shader][slot].buffer_offset;
681 ctx->di.db.ubos[shader][slot].range = ctx->ubos[shader][slot].buffer_size;
682 assert(ctx->di.db.ubos[shader][slot].range == VK_WHOLE_SIZE ||
683 ctx->di.db.ubos[shader][slot].range <= screen->info.props.limits.maxUniformBufferRange);
684 } else {
685 ctx->di.db.ubos[shader][slot].address = 0;
686 ctx->di.db.ubos[shader][slot].range = VK_WHOLE_SIZE;
687 }
688 return res;
689 }
690
691 ALWAYS_INLINE static struct zink_resource *
update_descriptor_state_ubo_lazy(struct zink_context * ctx,gl_shader_stage shader,unsigned slot,struct zink_resource * res)692 update_descriptor_state_ubo_lazy(struct zink_context *ctx, gl_shader_stage shader, unsigned slot, struct zink_resource *res)
693 {
694 struct zink_screen *screen = zink_screen(ctx->base.screen);
695 ctx->di.t.ubos[shader][slot].offset = ctx->ubos[shader][slot].buffer_offset;
696 ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_UBO][shader][slot] = res;
697 if (res) {
698 ctx->di.t.ubos[shader][slot].buffer = res->obj->buffer;
699 ctx->di.t.ubos[shader][slot].range = ctx->ubos[shader][slot].buffer_size;
700 assert(ctx->di.t.ubos[shader][slot].range <= screen->info.props.limits.maxUniformBufferRange);
701 }
702 else {
703 bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor;
704 VkBuffer null_buffer = zink_resource(ctx->dummy_vertex_buffer)->obj->buffer;
705 ctx->di.t.ubos[shader][slot].buffer = have_null_descriptors ? VK_NULL_HANDLE : null_buffer;
706 ctx->di.t.ubos[shader][slot].range = VK_WHOLE_SIZE;
707 }
708 return res;
709 }
710
711 ALWAYS_INLINE static struct zink_resource *
update_descriptor_state_ssbo_db(struct zink_context * ctx,gl_shader_stage shader,unsigned slot,struct zink_resource * res)712 update_descriptor_state_ssbo_db(struct zink_context *ctx, gl_shader_stage shader, unsigned slot, struct zink_resource *res)
713 {
714 ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_SSBO][shader][slot] = res;
715 if (res) {
716 ctx->di.db.ssbos[shader][slot].address = res->obj->bda + ctx->ssbos[shader][slot].buffer_offset;
717 ctx->di.db.ssbos[shader][slot].range = ctx->ssbos[shader][slot].buffer_size;
718 } else {
719 ctx->di.db.ssbos[shader][slot].address = 0;
720 ctx->di.db.ssbos[shader][slot].range = VK_WHOLE_SIZE;
721 }
722 return res;
723 }
724
725 ALWAYS_INLINE static struct zink_resource *
update_descriptor_state_ssbo_lazy(struct zink_context * ctx,gl_shader_stage shader,unsigned slot,struct zink_resource * res)726 update_descriptor_state_ssbo_lazy(struct zink_context *ctx, gl_shader_stage shader, unsigned slot, struct zink_resource *res)
727 {
728 struct zink_screen *screen = zink_screen(ctx->base.screen);
729 ctx->di.t.ssbos[shader][slot].offset = ctx->ssbos[shader][slot].buffer_offset;
730 ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_SSBO][shader][slot] = res;
731 if (res) {
732 ctx->di.t.ssbos[shader][slot].buffer = res->obj->buffer;
733 ctx->di.t.ssbos[shader][slot].range = ctx->ssbos[shader][slot].buffer_size;
734 } else {
735 bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor;
736 VkBuffer null_buffer = zink_resource(ctx->dummy_vertex_buffer)->obj->buffer;
737 ctx->di.t.ssbos[shader][slot].buffer = have_null_descriptors ? VK_NULL_HANDLE : null_buffer;
738 ctx->di.t.ssbos[shader][slot].range = VK_WHOLE_SIZE;
739 }
740 return res;
741 }
742
743 ALWAYS_INLINE static struct zink_resource *
update_descriptor_state_sampler(struct zink_context * ctx,gl_shader_stage shader,unsigned slot,struct zink_resource * res)744 update_descriptor_state_sampler(struct zink_context *ctx, gl_shader_stage shader, unsigned slot, struct zink_resource *res)
745 {
746 struct zink_screen *screen = zink_screen(ctx->base.screen);
747 bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor;
748 const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW;
749 ctx->di.descriptor_res[type][shader][slot] = res;
750 if (res) {
751 if (res->obj->is_buffer) {
752 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
753 ctx->di.db.tbos[shader][slot].address = res->obj->bda + ctx->sampler_views[shader][slot]->u.buf.offset;
754 ctx->di.db.tbos[shader][slot].range = zink_sampler_view(ctx->sampler_views[shader][slot])->tbo_size;
755 ctx->di.db.tbos[shader][slot].format = zink_get_format(screen, ctx->sampler_views[shader][slot]->format);
756 } else {
757 struct zink_buffer_view *bv = get_bufferview_for_binding(ctx, shader, type, slot);
758 ctx->di.t.tbos[shader][slot] = bv->buffer_view;
759 }
760 } else {
761 struct zink_surface *surface = get_imageview_for_binding(ctx, shader, type, slot);
762 ctx->di.textures[shader][slot].imageLayout = ctx->blitting ? res->layout : get_layout_for_binding(ctx, res, type, shader == MESA_SHADER_COMPUTE);
763 ctx->di.textures[shader][slot].imageView = surface->image_view;
764 if (!screen->have_D24_UNORM_S8_UINT &&
765 ctx->sampler_states[shader][slot] && ctx->sampler_states[shader][slot]->sampler_clamped) {
766 struct zink_sampler_state *state = ctx->sampler_states[shader][slot];
767 VkSampler sampler = (surface->base.format == PIPE_FORMAT_Z24X8_UNORM && surface->ivci.format == VK_FORMAT_D32_SFLOAT) ||
768 (surface->base.format == PIPE_FORMAT_Z24_UNORM_S8_UINT && surface->ivci.format == VK_FORMAT_D32_SFLOAT_S8_UINT) ?
769 state->sampler_clamped :
770 state->sampler;
771 if (ctx->di.textures[shader][slot].sampler != sampler) {
772 ctx->invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1);
773 ctx->di.textures[shader][slot].sampler = sampler;
774 }
775 }
776 }
777 } else {
778 if (likely(have_null_descriptors)) {
779 ctx->di.textures[shader][slot].imageView = VK_NULL_HANDLE;
780 ctx->di.textures[shader][slot].imageLayout = VK_IMAGE_LAYOUT_UNDEFINED;
781 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
782 ctx->di.db.tbos[shader][slot].address = 0;
783 ctx->di.db.tbos[shader][slot].range = VK_WHOLE_SIZE;
784 } else {
785 ctx->di.t.tbos[shader][slot] = VK_NULL_HANDLE;
786 }
787 } else {
788 assert(zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB);
789 struct zink_surface *null_surface = zink_get_dummy_surface(ctx, 0);
790 struct zink_buffer_view *null_bufferview = ctx->dummy_bufferview;
791 ctx->di.textures[shader][slot].imageView = null_surface->image_view;
792 ctx->di.textures[shader][slot].imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
793 ctx->di.t.tbos[shader][slot] = null_bufferview->buffer_view;
794 }
795 }
796 return res;
797 }
798
799 void
zink_update_shadow_samplerviews(struct zink_context * ctx,unsigned mask)800 zink_update_shadow_samplerviews(struct zink_context *ctx, unsigned mask)
801 {
802 u_foreach_bit(slot, mask)
803 update_descriptor_state_sampler(ctx, MESA_SHADER_FRAGMENT, slot, ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW][MESA_SHADER_FRAGMENT][slot]);
804 }
805
806 ALWAYS_INLINE static struct zink_resource *
update_descriptor_state_image(struct zink_context * ctx,gl_shader_stage shader,unsigned slot,struct zink_resource * res)807 update_descriptor_state_image(struct zink_context *ctx, gl_shader_stage shader, unsigned slot, struct zink_resource *res)
808 {
809 struct zink_screen *screen = zink_screen(ctx->base.screen);
810 bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor;
811 const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_IMAGE;
812 ctx->di.descriptor_res[type][shader][slot] = res;
813 if (res) {
814 if (res->obj->is_buffer) {
815 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
816 ctx->di.db.texel_images[shader][slot].address = res->obj->bda + ctx->image_views[shader][slot].base.u.buf.offset;
817 ctx->di.db.texel_images[shader][slot].range = ctx->image_views[shader][slot].base.u.buf.size;
818 ctx->di.db.texel_images[shader][slot].format = zink_get_format(screen, ctx->image_views[shader][slot].base.format);
819 } else {
820 struct zink_buffer_view *bv = get_bufferview_for_binding(ctx, shader, type, slot);
821 ctx->di.t.texel_images[shader][slot] = bv->buffer_view;
822 }
823 } else {
824 struct zink_surface *surface = get_imageview_for_binding(ctx, shader, type, slot);
825 ctx->di.images[shader][slot].imageLayout = VK_IMAGE_LAYOUT_GENERAL;
826 ctx->di.images[shader][slot].imageView = surface->image_view;
827 }
828 } else {
829 if (likely(have_null_descriptors)) {
830 memset(&ctx->di.images[shader][slot], 0, sizeof(ctx->di.images[shader][slot]));
831 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
832 ctx->di.db.texel_images[shader][slot].address = 0;
833 ctx->di.db.texel_images[shader][slot].range = VK_WHOLE_SIZE;
834 } else {
835 ctx->di.t.texel_images[shader][slot] = VK_NULL_HANDLE;
836 }
837 } else {
838 assert(zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB);
839 struct zink_surface *null_surface = zink_get_dummy_surface(ctx, 0);
840 struct zink_buffer_view *null_bufferview = ctx->dummy_bufferview;
841 ctx->di.images[shader][slot].imageView = null_surface->image_view;
842 ctx->di.images[shader][slot].imageLayout = VK_IMAGE_LAYOUT_GENERAL;
843 ctx->di.t.texel_images[shader][slot] = null_bufferview->buffer_view;
844 }
845 }
846 return res;
847 }
848
849 static void
update_nonseamless_shader_key(struct zink_context * ctx,gl_shader_stage pstage)850 update_nonseamless_shader_key(struct zink_context *ctx, gl_shader_stage pstage)
851 {
852 const uint32_t new_mask = ctx->di.emulate_nonseamless[pstage] & ctx->di.cubes[pstage];
853 if (pstage == MESA_SHADER_COMPUTE) {
854 if (ctx->compute_pipeline_state.key.base.nonseamless_cube_mask != new_mask)
855 ctx->compute_dirty = true;
856 ctx->compute_pipeline_state.key.base.nonseamless_cube_mask = new_mask;
857 } else {
858 if (zink_get_shader_key_base(ctx, pstage)->nonseamless_cube_mask != new_mask)
859 zink_set_shader_key_base(ctx, pstage)->nonseamless_cube_mask = new_mask;
860 }
861 }
862
863 static void
zink_bind_sampler_states(struct pipe_context * pctx,gl_shader_stage shader,unsigned start_slot,unsigned num_samplers,void ** samplers)864 zink_bind_sampler_states(struct pipe_context *pctx,
865 gl_shader_stage shader,
866 unsigned start_slot,
867 unsigned num_samplers,
868 void **samplers)
869 {
870 struct zink_context *ctx = zink_context(pctx);
871 struct zink_screen *screen = zink_screen(pctx->screen);
872 for (unsigned i = 0; i < num_samplers; ++i) {
873 struct zink_sampler_state *state = samplers[i];
874 if (samplers[i] == ctx->sampler_states[shader][start_slot + i])
875 continue;
876 ctx->invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot, 1);
877 ctx->sampler_states[shader][start_slot + i] = state;
878 if (state) {
879 ctx->di.textures[shader][start_slot + i].sampler = state->sampler;
880 if (state->sampler_clamped && !screen->have_D24_UNORM_S8_UINT) {
881 struct zink_surface *surface = get_imageview_for_binding(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot + i);
882 if (surface &&
883 ((surface->base.format == PIPE_FORMAT_Z24X8_UNORM && surface->ivci.format == VK_FORMAT_D32_SFLOAT) ||
884 (surface->base.format == PIPE_FORMAT_Z24_UNORM_S8_UINT && surface->ivci.format == VK_FORMAT_D32_SFLOAT_S8_UINT)))
885 ctx->di.textures[shader][start_slot + i].sampler = state->sampler_clamped;
886 }
887 } else {
888 ctx->di.textures[shader][start_slot + i].sampler = VK_NULL_HANDLE;
889 }
890 }
891 ctx->di.num_samplers[shader] = start_slot + num_samplers;
892 }
893
894 static void
zink_bind_sampler_states_nonseamless(struct pipe_context * pctx,gl_shader_stage shader,unsigned start_slot,unsigned num_samplers,void ** samplers)895 zink_bind_sampler_states_nonseamless(struct pipe_context *pctx,
896 gl_shader_stage shader,
897 unsigned start_slot,
898 unsigned num_samplers,
899 void **samplers)
900 {
901 struct zink_context *ctx = zink_context(pctx);
902 uint32_t old_mask = ctx->di.emulate_nonseamless[shader];
903 uint32_t mask = BITFIELD_RANGE(start_slot, num_samplers);
904 ctx->di.emulate_nonseamless[shader] &= ~mask;
905 for (unsigned i = 0; i < num_samplers; ++i) {
906 struct zink_sampler_state *state = samplers[i];
907 const uint32_t bit = BITFIELD_BIT(start_slot + i);
908 if (!state)
909 continue;
910 if (state->emulate_nonseamless)
911 ctx->di.emulate_nonseamless[shader] |= bit;
912 if (state->emulate_nonseamless != (old_mask & bit) && (ctx->di.cubes[shader] & bit)) {
913 struct zink_surface *surface = get_imageview_for_binding(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot + i);
914 if (surface && ctx->di.images[shader][start_slot + i].imageView != surface->image_view) {
915 ctx->di.images[shader][start_slot + i].imageView = surface->image_view;
916 update_descriptor_state_sampler(ctx, shader, start_slot + i, zink_resource(surface->base.texture));
917 ctx->invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot + i, 1);
918 }
919 }
920 }
921 zink_bind_sampler_states(pctx, shader, start_slot, num_samplers, samplers);
922 update_nonseamless_shader_key(ctx, shader);
923 }
924
925 static void
zink_delete_sampler_state(struct pipe_context * pctx,void * sampler_state)926 zink_delete_sampler_state(struct pipe_context *pctx,
927 void *sampler_state)
928 {
929 struct zink_sampler_state *sampler = sampler_state;
930 struct zink_batch_state *bs = zink_context(pctx)->bs;
931 /* may be called if context_create fails */
932 if (bs) {
933 util_dynarray_append(&bs->zombie_samplers, VkSampler,
934 sampler->sampler);
935 if (sampler->sampler_clamped)
936 util_dynarray_append(&bs->zombie_samplers, VkSampler,
937 sampler->sampler_clamped);
938 }
939 if (sampler->custom_border_color)
940 p_atomic_dec(&zink_screen(pctx->screen)->cur_custom_border_color_samplers);
941 FREE(sampler);
942 }
943
944 static VkImageAspectFlags
sampler_aspect_from_format(enum pipe_format fmt)945 sampler_aspect_from_format(enum pipe_format fmt)
946 {
947 if (util_format_is_depth_or_stencil(fmt)) {
948 const struct util_format_description *desc = util_format_description(fmt);
949 if (util_format_has_depth(desc))
950 return VK_IMAGE_ASPECT_DEPTH_BIT;
951 assert(util_format_has_stencil(desc));
952 return VK_IMAGE_ASPECT_STENCIL_BIT;
953 } else
954 return VK_IMAGE_ASPECT_COLOR_BIT;
955 }
956
957 static uint32_t
hash_bufferview(void * bvci)958 hash_bufferview(void *bvci)
959 {
960 size_t offset = offsetof(VkBufferViewCreateInfo, flags);
961 return _mesa_hash_data((char*)bvci + offset, sizeof(VkBufferViewCreateInfo) - offset);
962 }
963
964 static VkBufferViewCreateInfo
create_bvci(struct zink_context * ctx,struct zink_resource * res,enum pipe_format format,uint32_t offset,uint32_t range)965 create_bvci(struct zink_context *ctx, struct zink_resource *res, enum pipe_format format, uint32_t offset, uint32_t range)
966 {
967 struct zink_screen *screen = zink_screen(ctx->base.screen);
968 VkBufferViewCreateInfo bvci;
969 // Zero whole struct (including alignment holes), so hash_bufferview
970 // does not access potentially uninitialized data.
971 memset(&bvci, 0, sizeof(bvci));
972 bvci.sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO;
973 bvci.pNext = NULL;
974 if (zink_get_format_props(screen, format)->bufferFeatures & VK_FORMAT_FEATURE_STORAGE_TEXEL_BUFFER_BIT)
975 bvci.buffer = res->obj->storage_buffer ? res->obj->storage_buffer : res->obj->buffer;
976 else
977 bvci.buffer = res->obj->buffer;
978 bvci.format = zink_get_format(screen, format);
979 assert(bvci.format);
980 bvci.offset = offset;
981 bvci.range = !offset && range == res->base.b.width0 ? VK_WHOLE_SIZE : range;
982 unsigned blocksize = util_format_get_blocksize(format);
983 if (bvci.range != VK_WHOLE_SIZE) {
984 /* clamp out partial texels */
985 bvci.range -= bvci.range % blocksize;
986 if (bvci.offset + bvci.range >= res->base.b.width0)
987 bvci.range = VK_WHOLE_SIZE;
988 }
989 uint64_t clamp = (uint64_t)blocksize * (uint64_t)screen->info.props.limits.maxTexelBufferElements;
990 if (bvci.range == VK_WHOLE_SIZE && res->base.b.width0 > clamp)
991 bvci.range = clamp;
992 bvci.flags = 0;
993 return bvci;
994 }
995
996 static struct zink_buffer_view *
get_buffer_view(struct zink_context * ctx,struct zink_resource * res,VkBufferViewCreateInfo * bvci)997 get_buffer_view(struct zink_context *ctx, struct zink_resource *res, VkBufferViewCreateInfo *bvci)
998 {
999 struct zink_screen *screen = zink_screen(ctx->base.screen);
1000 struct zink_buffer_view *buffer_view = NULL;
1001
1002 uint32_t hash = hash_bufferview(bvci);
1003 simple_mtx_lock(&res->bufferview_mtx);
1004 struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&res->bufferview_cache, hash, bvci);
1005 if (he) {
1006 buffer_view = he->data;
1007 p_atomic_inc(&buffer_view->reference.count);
1008 } else {
1009 VkBufferView view;
1010 VkResult result = VKSCR(CreateBufferView)(screen->dev, bvci, NULL, &view);
1011 if (result != VK_SUCCESS) {
1012 mesa_loge("ZINK: vkCreateBufferView failed (%s)", vk_Result_to_str(result));
1013 goto out;
1014 }
1015 buffer_view = CALLOC_STRUCT(zink_buffer_view);
1016 if (!buffer_view) {
1017 VKSCR(DestroyBufferView)(screen->dev, view, NULL);
1018 goto out;
1019 }
1020 pipe_reference_init(&buffer_view->reference, 1);
1021 pipe_resource_reference(&buffer_view->pres, &res->base.b);
1022 buffer_view->bvci = *bvci;
1023 buffer_view->buffer_view = view;
1024 buffer_view->hash = hash;
1025 _mesa_hash_table_insert_pre_hashed(&res->bufferview_cache, hash, &buffer_view->bvci, buffer_view);
1026 }
1027 out:
1028 simple_mtx_unlock(&res->bufferview_mtx);
1029 return buffer_view;
1030 }
1031
1032 enum pipe_swizzle
zink_clamp_void_swizzle(const struct util_format_description * desc,enum pipe_swizzle swizzle)1033 zink_clamp_void_swizzle(const struct util_format_description *desc, enum pipe_swizzle swizzle)
1034 {
1035 switch (swizzle) {
1036 case PIPE_SWIZZLE_X:
1037 case PIPE_SWIZZLE_Y:
1038 case PIPE_SWIZZLE_Z:
1039 case PIPE_SWIZZLE_W:
1040 return desc->channel[swizzle].type == UTIL_FORMAT_TYPE_VOID ? PIPE_SWIZZLE_1 : swizzle;
1041 default:
1042 break;
1043 }
1044 return swizzle;
1045 }
1046
1047 ALWAYS_INLINE static enum pipe_swizzle
clamp_zs_swizzle(enum pipe_swizzle swizzle)1048 clamp_zs_swizzle(enum pipe_swizzle swizzle)
1049 {
1050 switch (swizzle) {
1051 case PIPE_SWIZZLE_X:
1052 case PIPE_SWIZZLE_Y:
1053 case PIPE_SWIZZLE_Z:
1054 case PIPE_SWIZZLE_W:
1055 return PIPE_SWIZZLE_X;
1056 default:
1057 break;
1058 }
1059 return swizzle;
1060 }
1061
1062 ALWAYS_INLINE static enum pipe_swizzle
clamp_alpha_swizzle(enum pipe_swizzle swizzle)1063 clamp_alpha_swizzle(enum pipe_swizzle swizzle)
1064 {
1065 if (swizzle == PIPE_SWIZZLE_W)
1066 return PIPE_SWIZZLE_X;
1067 if (swizzle < PIPE_SWIZZLE_W)
1068 return PIPE_SWIZZLE_0;
1069 return swizzle;
1070 }
1071
1072 ALWAYS_INLINE static enum pipe_swizzle
clamp_luminance_swizzle(enum pipe_swizzle swizzle)1073 clamp_luminance_swizzle(enum pipe_swizzle swizzle)
1074 {
1075 if (swizzle == PIPE_SWIZZLE_W)
1076 return PIPE_SWIZZLE_1;
1077 if (swizzle < PIPE_SWIZZLE_W)
1078 return PIPE_SWIZZLE_X;
1079 return swizzle;
1080 }
1081
1082 ALWAYS_INLINE static enum pipe_swizzle
clamp_luminance_alpha_swizzle(enum pipe_swizzle swizzle)1083 clamp_luminance_alpha_swizzle(enum pipe_swizzle swizzle)
1084 {
1085 if (swizzle == PIPE_SWIZZLE_W)
1086 return PIPE_SWIZZLE_Y;
1087 if (swizzle < PIPE_SWIZZLE_W)
1088 return PIPE_SWIZZLE_X;
1089 return swizzle;
1090 }
1091
1092 ALWAYS_INLINE static bool
viewtype_is_cube(const VkImageViewCreateInfo * ivci)1093 viewtype_is_cube(const VkImageViewCreateInfo *ivci)
1094 {
1095 return ivci->viewType == VK_IMAGE_VIEW_TYPE_CUBE ||
1096 ivci->viewType == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY;
1097 }
1098
1099 static struct pipe_sampler_view *
zink_create_sampler_view(struct pipe_context * pctx,struct pipe_resource * pres,const struct pipe_sampler_view * state)1100 zink_create_sampler_view(struct pipe_context *pctx, struct pipe_resource *pres,
1101 const struct pipe_sampler_view *state)
1102 {
1103 struct zink_screen *screen = zink_screen(pctx->screen);
1104 struct zink_resource *res = zink_resource(pres);
1105 struct zink_context *ctx = zink_context(pctx);
1106 struct zink_sampler_view *sampler_view = CALLOC_STRUCT_CL(zink_sampler_view);
1107 bool err;
1108
1109 if (!sampler_view) {
1110 mesa_loge("ZINK: failed to allocate sampler_view!");
1111 return NULL;
1112 }
1113
1114 sampler_view->base = *state;
1115 sampler_view->base.texture = NULL;
1116 pipe_resource_reference(&sampler_view->base.texture, pres);
1117 sampler_view->base.reference.count = 1;
1118 sampler_view->base.context = pctx;
1119
1120 if (state->target != PIPE_BUFFER) {
1121 VkImageViewCreateInfo ivci;
1122
1123 struct pipe_surface templ = {0};
1124 templ.u.tex.level = state->u.tex.first_level;
1125 templ.format = state->format;
1126 /* avoid needing mutable for depth/stencil sampling */
1127 if (util_format_is_depth_and_stencil(pres->format))
1128 templ.format = pres->format;
1129 if (state->target != PIPE_TEXTURE_3D) {
1130 templ.u.tex.first_layer = state->u.tex.first_layer;
1131 templ.u.tex.last_layer = state->u.tex.last_layer;
1132 }
1133
1134 if (zink_is_swapchain(res)) {
1135 if (!zink_kopper_acquire(ctx, res, UINT64_MAX)) {
1136 FREE_CL(sampler_view);
1137 return NULL;
1138 }
1139 }
1140
1141 ivci = create_ivci(screen, res, &templ, state->target);
1142 ivci.subresourceRange.levelCount = state->u.tex.last_level - state->u.tex.first_level + 1;
1143 ivci.subresourceRange.aspectMask = util_format_is_depth_or_stencil(state->format) ? sampler_aspect_from_format(state->format) : res->aspect;
1144 bool red_depth_sampler_view = false;
1145 /* samplers for stencil aspects of packed formats need to always use stencil swizzle */
1146 if (ivci.subresourceRange.aspectMask & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) {
1147 ivci.components.r = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_r));
1148 ivci.components.g = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_g));
1149 ivci.components.b = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_b));
1150 ivci.components.a = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_a));
1151
1152 /* If we're sampling depth and we might need to do shader rewrites for
1153 * legacy shadow sampling, then set up an extra image view that just
1154 * returns the red (depth) component, so you can always have the shadow
1155 * result available in the red component for the in-shader swizzling.
1156 * (Or if we have PVR's needs_zs_shader_swizzle and are sampling ONE
1157 * value for stencil, which also uses that view).
1158 */
1159 if (ivci.subresourceRange.aspectMask == VK_IMAGE_ASPECT_DEPTH_BIT ||
1160 zink_screen(ctx->base.screen)->driver_compiler_workarounds.needs_zs_shader_swizzle) {
1161 VkComponentSwizzle *swizzle = (VkComponentSwizzle*)&ivci.components;
1162 for (unsigned i = 0; i < 4; i++) {
1163 if (swizzle[i] == VK_COMPONENT_SWIZZLE_ONE ||
1164 (swizzle[i] == VK_COMPONENT_SWIZZLE_ZERO && ivci.subresourceRange.aspectMask == VK_IMAGE_ASPECT_DEPTH_BIT))
1165 red_depth_sampler_view = true;
1166 }
1167 /* this is the data that will be used in shader rewrites */
1168 sampler_view->swizzle.s[0] = clamp_zs_swizzle(sampler_view->base.swizzle_r);
1169 sampler_view->swizzle.s[1] = clamp_zs_swizzle(sampler_view->base.swizzle_g);
1170 sampler_view->swizzle.s[2] = clamp_zs_swizzle(sampler_view->base.swizzle_b);
1171 sampler_view->swizzle.s[3] = clamp_zs_swizzle(sampler_view->base.swizzle_a);
1172 }
1173 } else {
1174 enum pipe_swizzle swizzle[4] = {
1175 sampler_view->base.swizzle_r,
1176 sampler_view->base.swizzle_g,
1177 sampler_view->base.swizzle_b,
1178 sampler_view->base.swizzle_a
1179 };
1180 /* if we have e.g., R8G8B8X8, then we have to ignore alpha since we're just emulating
1181 * these formats
1182 */
1183 if (zink_format_is_voidable_rgba_variant(state->format)) {
1184 const struct util_format_description *view_desc = util_format_description(state->format);
1185 for (int i = 0; i < 4; ++i)
1186 swizzle[i] = zink_clamp_void_swizzle(view_desc, swizzle[i]);
1187 } else if (util_format_is_alpha(state->format) && res->format != VK_FORMAT_A8_UNORM_KHR) {
1188 for (int i = 0; i < 4; ++i)
1189 swizzle[i] = clamp_alpha_swizzle(swizzle[i]);
1190 } else if (util_format_is_luminance(pres->format) ||
1191 util_format_is_luminance_alpha(pres->format)) {
1192 if (util_format_is_luminance(pres->format)) {
1193 for (int i = 0; i < 4; ++i)
1194 swizzle[i] = clamp_luminance_swizzle(swizzle[i]);
1195 } else {
1196 for (int i = 0; i < 4; ++i)
1197 swizzle[i] = clamp_luminance_alpha_swizzle(swizzle[i]);
1198 }
1199 if (state->format != pres->format) {
1200 /* luminance / luminance-alpha formats can be reinterpreted
1201 * as red / red-alpha formats by the state-tracker, and we
1202 * need to whack the green/blue channels here to the
1203 * correct values for that to work.
1204 */
1205 enum pipe_format linear = util_format_linear(pres->format);
1206 if (state->format == util_format_luminance_to_red(linear)) {
1207 assert(swizzle[1] == PIPE_SWIZZLE_X ||
1208 swizzle[1] == PIPE_SWIZZLE_0);
1209 assert(swizzle[2] == PIPE_SWIZZLE_X ||
1210 swizzle[2] == PIPE_SWIZZLE_0);
1211 swizzle[1] = swizzle[2] = PIPE_SWIZZLE_0;
1212 } else
1213 assert(state->format == linear);
1214 }
1215 } else if (util_format_is_red_alpha(pres->format)) {
1216 /* RA formats are mapped to RG with adjusted swizzle */
1217 assert(util_format_is_red_green(vk_format_to_pipe_format(ivci.format)));
1218 swizzle[3] = PIPE_SWIZZLE_Y;
1219 }
1220
1221 ivci.components.r = zink_component_mapping(swizzle[0]);
1222 ivci.components.g = zink_component_mapping(swizzle[1]);
1223 ivci.components.b = zink_component_mapping(swizzle[2]);
1224 ivci.components.a = zink_component_mapping(swizzle[3]);
1225 }
1226 assert(ivci.format);
1227
1228 sampler_view->image_view = zink_get_surface(ctx, pres, &templ, &ivci);
1229 if (!screen->info.have_EXT_non_seamless_cube_map && viewtype_is_cube(&sampler_view->image_view->ivci)) {
1230 ivci.viewType = VK_IMAGE_VIEW_TYPE_2D_ARRAY;
1231 sampler_view->cube_array = zink_get_surface(ctx, pres, &templ, &ivci);
1232 } else if (red_depth_sampler_view) {
1233 /* there is only one component, and real swizzling can't be done here,
1234 * so ensure the shader gets the sampled data
1235 */
1236 ivci.components.r = VK_COMPONENT_SWIZZLE_R;
1237 ivci.components.g = VK_COMPONENT_SWIZZLE_R;
1238 ivci.components.b = VK_COMPONENT_SWIZZLE_R;
1239 ivci.components.a = VK_COMPONENT_SWIZZLE_R;
1240 sampler_view->zs_view = zink_get_surface(ctx, pres, &templ, &ivci);
1241 }
1242 err = !sampler_view->image_view;
1243 } else {
1244 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
1245 /* always enforce limit clamping */
1246 unsigned blocksize = util_format_get_blocksize(state->format);
1247 sampler_view->tbo_size = MIN2(state->u.buf.size / blocksize, screen->info.props.limits.maxTexelBufferElements) * blocksize;
1248 return &sampler_view->base;
1249 }
1250 VkBufferViewCreateInfo bvci = create_bvci(ctx, res, state->format, state->u.buf.offset, state->u.buf.size);
1251 sampler_view->buffer_view = get_buffer_view(ctx, res, &bvci);
1252 err = !sampler_view->buffer_view;
1253 }
1254 if (err) {
1255 FREE_CL(sampler_view);
1256 return NULL;
1257 }
1258 return &sampler_view->base;
1259 }
1260
1261 void
zink_destroy_buffer_view(struct zink_screen * screen,struct zink_buffer_view * buffer_view)1262 zink_destroy_buffer_view(struct zink_screen *screen, struct zink_buffer_view *buffer_view)
1263 {
1264 struct zink_resource *res = zink_resource(buffer_view->pres);
1265 simple_mtx_lock(&res->bufferview_mtx);
1266 if (buffer_view->reference.count) {
1267 /* got a cache hit during deletion */
1268 simple_mtx_unlock(&res->bufferview_mtx);
1269 return;
1270 }
1271 struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&res->bufferview_cache, buffer_view->hash, &buffer_view->bvci);
1272 assert(he);
1273 _mesa_hash_table_remove(&res->bufferview_cache, he);
1274 simple_mtx_unlock(&res->bufferview_mtx);
1275 simple_mtx_lock(&res->obj->view_lock);
1276 util_dynarray_append(&res->obj->views, VkBufferView, buffer_view->buffer_view);
1277 simple_mtx_unlock(&res->obj->view_lock);
1278 pipe_resource_reference(&buffer_view->pres, NULL);
1279 FREE(buffer_view);
1280 }
1281
1282 static void
zink_sampler_view_destroy(struct pipe_context * pctx,struct pipe_sampler_view * pview)1283 zink_sampler_view_destroy(struct pipe_context *pctx,
1284 struct pipe_sampler_view *pview)
1285 {
1286 struct zink_sampler_view *view = zink_sampler_view(pview);
1287 if (pview->texture->target == PIPE_BUFFER) {
1288 if (zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB)
1289 zink_buffer_view_reference(zink_screen(pctx->screen), &view->buffer_view, NULL);
1290 } else {
1291 zink_surface_reference(zink_screen(pctx->screen), &view->image_view, NULL);
1292 zink_surface_reference(zink_screen(pctx->screen), &view->cube_array, NULL);
1293 zink_surface_reference(zink_screen(pctx->screen), &view->zs_view, NULL);
1294 }
1295 pipe_resource_reference(&pview->texture, NULL);
1296 FREE_CL(view);
1297 }
1298
1299 static void
zink_get_sample_position(struct pipe_context * ctx,unsigned sample_count,unsigned sample_index,float * out_value)1300 zink_get_sample_position(struct pipe_context *ctx,
1301 unsigned sample_count,
1302 unsigned sample_index,
1303 float *out_value)
1304 {
1305 /* TODO: handle this I guess */
1306 assert(zink_screen(ctx->screen)->info.props.limits.standardSampleLocations);
1307 u_default_get_sample_position(ctx, sample_count, sample_index, out_value);
1308 }
1309
1310 static void
zink_set_polygon_stipple(struct pipe_context * pctx,const struct pipe_poly_stipple * ps)1311 zink_set_polygon_stipple(struct pipe_context *pctx,
1312 const struct pipe_poly_stipple *ps)
1313 {
1314 }
1315
1316 ALWAYS_INLINE static void
update_res_bind_count(struct zink_context * ctx,struct zink_resource * res,bool is_compute,bool decrement)1317 update_res_bind_count(struct zink_context *ctx, struct zink_resource *res, bool is_compute, bool decrement)
1318 {
1319 if (decrement) {
1320 assert(res->bind_count[is_compute]);
1321 if (!--res->bind_count[is_compute])
1322 _mesa_set_remove_key(ctx->need_barriers[is_compute], res);
1323 check_resource_for_batch_ref(ctx, res);
1324 } else
1325 res->bind_count[is_compute]++;
1326 }
1327
1328 ALWAYS_INLINE static void
update_existing_vbo(struct zink_context * ctx,unsigned slot)1329 update_existing_vbo(struct zink_context *ctx, unsigned slot)
1330 {
1331 if (!ctx->vertex_buffers[slot].buffer.resource)
1332 return;
1333 struct zink_resource *res = zink_resource(ctx->vertex_buffers[slot].buffer.resource);
1334 res->vbo_bind_count--;
1335 res->vbo_bind_mask &= ~BITFIELD_BIT(slot);
1336 if (!res->vbo_bind_count) {
1337 res->gfx_barrier &= ~VK_PIPELINE_STAGE_VERTEX_INPUT_BIT;
1338 res->barrier_access[0] &= ~VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT;
1339 }
1340 update_res_bind_count(ctx, res, false, true);
1341 }
1342
1343 static void
zink_set_vertex_buffers(struct pipe_context * pctx,unsigned num_buffers,const struct pipe_vertex_buffer * buffers)1344 zink_set_vertex_buffers(struct pipe_context *pctx,
1345 unsigned num_buffers,
1346 const struct pipe_vertex_buffer *buffers)
1347 {
1348 struct zink_context *ctx = zink_context(pctx);
1349 const bool have_input_state = zink_screen(pctx->screen)->info.have_EXT_vertex_input_dynamic_state;
1350 const bool need_state_change = !zink_screen(pctx->screen)->info.have_EXT_extended_dynamic_state &&
1351 !have_input_state;
1352 unsigned last_count = util_last_bit(ctx->gfx_pipeline_state.vertex_buffers_enabled_mask);
1353 uint32_t enabled_buffers = BITFIELD_MASK(num_buffers);
1354
1355 assert(!num_buffers || buffers);
1356
1357 for (unsigned i = 0; i < num_buffers; ++i) {
1358 const struct pipe_vertex_buffer *vb = buffers + i;
1359 struct pipe_vertex_buffer *ctx_vb = &ctx->vertex_buffers[i];
1360 update_existing_vbo(ctx, i);
1361 pipe_resource_reference(&ctx_vb->buffer.resource, NULL);
1362 ctx_vb->buffer.resource = vb->buffer.resource;
1363
1364 if (vb->buffer.resource) {
1365 struct zink_resource *res = zink_resource(vb->buffer.resource);
1366 res->vbo_bind_mask |= BITFIELD_BIT(i);
1367 res->vbo_bind_count++;
1368 res->gfx_barrier |= VK_PIPELINE_STAGE_VERTEX_INPUT_BIT;
1369 res->barrier_access[0] |= VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT;
1370 update_res_bind_count(ctx, res, false, false);
1371 ctx_vb->buffer_offset = vb->buffer_offset;
1372 /* always barrier before possible rebind */
1373 zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT,
1374 VK_PIPELINE_STAGE_VERTEX_INPUT_BIT);
1375 zink_batch_resource_usage_set(ctx->bs, res, false, true);
1376 res->obj->unordered_read = false;
1377 } else {
1378 enabled_buffers &= ~BITFIELD_BIT(i);
1379 }
1380 }
1381 for (unsigned i = num_buffers; i < last_count; i++) {
1382 update_existing_vbo(ctx, i);
1383 pipe_resource_reference(&ctx->vertex_buffers[i].buffer.resource, NULL);
1384 }
1385 if (need_state_change)
1386 ctx->vertex_state_changed = true;
1387 else if (!have_input_state && ctx->gfx_pipeline_state.vertex_buffers_enabled_mask != enabled_buffers)
1388 ctx->vertex_state_changed = true;
1389 ctx->gfx_pipeline_state.vertex_buffers_enabled_mask = enabled_buffers;
1390 ctx->vertex_buffers_dirty = num_buffers > 0;
1391 #ifndef NDEBUG
1392 u_foreach_bit(b, enabled_buffers)
1393 assert(ctx->vertex_buffers[b].buffer.resource);
1394 #endif
1395 }
1396
1397 static void
zink_set_viewport_states(struct pipe_context * pctx,unsigned start_slot,unsigned num_viewports,const struct pipe_viewport_state * state)1398 zink_set_viewport_states(struct pipe_context *pctx,
1399 unsigned start_slot,
1400 unsigned num_viewports,
1401 const struct pipe_viewport_state *state)
1402 {
1403 struct zink_context *ctx = zink_context(pctx);
1404
1405 for (unsigned i = 0; i < num_viewports; ++i)
1406 ctx->vp_state.viewport_states[start_slot + i] = state[i];
1407
1408 ctx->vp_state_changed = true;
1409 }
1410
1411 static void
zink_set_scissor_states(struct pipe_context * pctx,unsigned start_slot,unsigned num_scissors,const struct pipe_scissor_state * states)1412 zink_set_scissor_states(struct pipe_context *pctx,
1413 unsigned start_slot, unsigned num_scissors,
1414 const struct pipe_scissor_state *states)
1415 {
1416 struct zink_context *ctx = zink_context(pctx);
1417
1418 for (unsigned i = 0; i < num_scissors; i++)
1419 ctx->vp_state.scissor_states[start_slot + i] = states[i];
1420 ctx->scissor_changed = true;
1421 }
1422
1423 static void
zink_set_inlinable_constants(struct pipe_context * pctx,gl_shader_stage shader,uint num_values,uint32_t * values)1424 zink_set_inlinable_constants(struct pipe_context *pctx,
1425 gl_shader_stage shader,
1426 uint num_values, uint32_t *values)
1427 {
1428 struct zink_context *ctx = (struct zink_context *)pctx;
1429 const uint32_t bit = BITFIELD_BIT(shader);
1430 uint32_t *inlinable_uniforms;
1431 struct zink_shader_key *key = NULL;
1432
1433 if (shader == MESA_SHADER_COMPUTE) {
1434 key = &ctx->compute_pipeline_state.key;
1435 } else {
1436 assert(!zink_screen(pctx->screen)->optimal_keys ||
1437 (shader == MESA_SHADER_GEOMETRY &&
1438 ctx->gfx_stages[MESA_SHADER_GEOMETRY] &&
1439 ctx->gfx_stages[MESA_SHADER_GEOMETRY]->non_fs.is_generated));
1440 key = &ctx->gfx_pipeline_state.shader_keys.key[shader];
1441 }
1442 inlinable_uniforms = key->base.inlined_uniform_values;
1443 if (!(ctx->inlinable_uniforms_valid_mask & bit) ||
1444 memcmp(inlinable_uniforms, values, num_values * 4)) {
1445 memcpy(inlinable_uniforms, values, num_values * 4);
1446 if (shader == MESA_SHADER_COMPUTE)
1447 ctx->compute_dirty = true;
1448 else
1449 ctx->dirty_gfx_stages |= bit;
1450 ctx->inlinable_uniforms_valid_mask |= bit;
1451 key->inline_uniforms = true;
1452 }
1453 }
1454
1455 ALWAYS_INLINE static void
unbind_descriptor_stage(struct zink_resource * res,gl_shader_stage pstage)1456 unbind_descriptor_stage(struct zink_resource *res, gl_shader_stage pstage)
1457 {
1458 if (!res->sampler_binds[pstage] && !res->image_binds[pstage] && !res->all_bindless)
1459 res->gfx_barrier &= ~zink_pipeline_flags_from_pipe_stage(pstage);
1460 }
1461
1462 ALWAYS_INLINE static void
unbind_buffer_descriptor_stage(struct zink_resource * res,gl_shader_stage pstage)1463 unbind_buffer_descriptor_stage(struct zink_resource *res, gl_shader_stage pstage)
1464 {
1465 if (!res->ubo_bind_mask[pstage] && !res->ssbo_bind_mask[pstage])
1466 unbind_descriptor_stage(res, pstage);
1467 }
1468
1469 ALWAYS_INLINE static void
unbind_ubo(struct zink_context * ctx,struct zink_resource * res,gl_shader_stage pstage,unsigned slot)1470 unbind_ubo(struct zink_context *ctx, struct zink_resource *res, gl_shader_stage pstage, unsigned slot)
1471 {
1472 if (!res)
1473 return;
1474 res->ubo_bind_mask[pstage] &= ~BITFIELD_BIT(slot);
1475 res->ubo_bind_count[pstage == MESA_SHADER_COMPUTE]--;
1476 unbind_buffer_descriptor_stage(res, pstage);
1477 if (!res->ubo_bind_count[pstage == MESA_SHADER_COMPUTE])
1478 res->barrier_access[pstage == MESA_SHADER_COMPUTE] &= ~VK_ACCESS_UNIFORM_READ_BIT;
1479 update_res_bind_count(ctx, res, pstage == MESA_SHADER_COMPUTE, true);
1480 }
1481
1482 static void
invalidate_inlined_uniforms(struct zink_context * ctx,gl_shader_stage pstage)1483 invalidate_inlined_uniforms(struct zink_context *ctx, gl_shader_stage pstage)
1484 {
1485 unsigned bit = BITFIELD_BIT(pstage);
1486 if (!(ctx->inlinable_uniforms_valid_mask & bit))
1487 return;
1488 ctx->inlinable_uniforms_valid_mask &= ~bit;
1489 if (pstage == MESA_SHADER_COMPUTE) {
1490 ctx->compute_dirty = true;
1491 return;
1492 }
1493 assert(!zink_screen(ctx->base.screen)->optimal_keys || (pstage == MESA_SHADER_GEOMETRY && ctx->is_generated_gs_bound));
1494 ctx->dirty_gfx_stages |= bit;
1495 struct zink_shader_key *key = &ctx->gfx_pipeline_state.shader_keys.key[pstage];
1496 key->inline_uniforms = false;
1497 }
1498
1499 ALWAYS_INLINE static void
zink_set_constant_buffer_internal(struct pipe_context * pctx,gl_shader_stage shader,uint index,bool take_ownership,const struct pipe_constant_buffer * cb,bool use_db)1500 zink_set_constant_buffer_internal(struct pipe_context *pctx,
1501 gl_shader_stage shader, uint index,
1502 bool take_ownership,
1503 const struct pipe_constant_buffer *cb,
1504 bool use_db)
1505 {
1506 struct zink_context *ctx = zink_context(pctx);
1507 bool update = false;
1508
1509 struct zink_resource *res = zink_resource(ctx->ubos[shader][index].buffer);
1510 if (cb) {
1511 struct pipe_resource *buffer = cb->buffer;
1512 unsigned offset = cb->buffer_offset;
1513 struct zink_screen *screen = zink_screen(pctx->screen);
1514 if (cb->user_buffer) {
1515 u_upload_data(ctx->base.const_uploader, 0, cb->buffer_size,
1516 screen->info.props.limits.minUniformBufferOffsetAlignment,
1517 cb->user_buffer, &offset, &buffer);
1518 }
1519 struct zink_resource *new_res = zink_resource(buffer);
1520 if (new_res) {
1521 if (new_res != res) {
1522 unbind_ubo(ctx, res, shader, index);
1523 new_res->ubo_bind_count[shader == MESA_SHADER_COMPUTE]++;
1524 new_res->ubo_bind_mask[shader] |= BITFIELD_BIT(index);
1525 new_res->gfx_barrier |= zink_pipeline_flags_from_pipe_stage(shader);
1526 new_res->barrier_access[shader == MESA_SHADER_COMPUTE] |= VK_ACCESS_UNIFORM_READ_BIT;
1527 update_res_bind_count(ctx, new_res, shader == MESA_SHADER_COMPUTE, false);
1528 }
1529 zink_screen(ctx->base.screen)->buffer_barrier(ctx, new_res, VK_ACCESS_UNIFORM_READ_BIT,
1530 new_res->gfx_barrier);
1531 zink_batch_resource_usage_set(ctx->bs, new_res, false, true);
1532 if (!ctx->unordered_blitting)
1533 new_res->obj->unordered_read = false;
1534 }
1535 update |= ctx->ubos[shader][index].buffer_offset != offset ||
1536 !!res != !!buffer || (res && res->obj->buffer != new_res->obj->buffer) ||
1537 ctx->ubos[shader][index].buffer_size != cb->buffer_size;
1538
1539 if (take_ownership) {
1540 pipe_resource_reference(&ctx->ubos[shader][index].buffer, NULL);
1541 ctx->ubos[shader][index].buffer = buffer;
1542 } else {
1543 pipe_resource_reference(&ctx->ubos[shader][index].buffer, buffer);
1544 }
1545 ctx->ubos[shader][index].buffer_offset = offset;
1546 ctx->ubos[shader][index].buffer_size = cb->buffer_size;
1547 ctx->ubos[shader][index].user_buffer = NULL;
1548
1549 if (cb->user_buffer)
1550 pipe_resource_reference(&buffer, NULL);
1551
1552 if (index + 1 >= ctx->di.num_ubos[shader])
1553 ctx->di.num_ubos[shader] = index + 1;
1554
1555 if (use_db) {
1556 update_descriptor_state_ubo_db(ctx, shader, index, new_res);
1557 } else {
1558 update_descriptor_state_ubo_lazy(ctx, shader, index, new_res);
1559 }
1560 } else {
1561 ctx->ubos[shader][index].buffer_offset = 0;
1562 ctx->ubos[shader][index].buffer_size = 0;
1563 ctx->ubos[shader][index].user_buffer = NULL;
1564 if (res) {
1565 unbind_ubo(ctx, res, shader, index);
1566 if (use_db) {
1567 update_descriptor_state_ubo_db(ctx, shader, index, NULL);
1568 } else {
1569 update_descriptor_state_ubo_lazy(ctx, shader, index, NULL);
1570 }
1571 }
1572 update = !!ctx->ubos[shader][index].buffer;
1573
1574 pipe_resource_reference(&ctx->ubos[shader][index].buffer, NULL);
1575 if (ctx->di.num_ubos[shader] == index + 1)
1576 ctx->di.num_ubos[shader]--;
1577 }
1578 if (index == 0) {
1579 /* Invalidate current inlinable uniforms. */
1580 invalidate_inlined_uniforms(ctx, shader);
1581 }
1582
1583 if (update)
1584 ctx->invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_UBO, index, 1);
1585 }
1586
1587 static void
zink_set_constant_buffer_db(struct pipe_context * pctx,gl_shader_stage shader,uint index,bool take_ownership,const struct pipe_constant_buffer * cb)1588 zink_set_constant_buffer_db(struct pipe_context *pctx,
1589 gl_shader_stage shader, uint index,
1590 bool take_ownership,
1591 const struct pipe_constant_buffer *cb)
1592 {
1593 zink_set_constant_buffer_internal(pctx, shader, index, take_ownership, cb, true);
1594 }
1595
1596 static void
zink_set_constant_buffer_lazy(struct pipe_context * pctx,gl_shader_stage shader,uint index,bool take_ownership,const struct pipe_constant_buffer * cb)1597 zink_set_constant_buffer_lazy(struct pipe_context *pctx,
1598 gl_shader_stage shader, uint index,
1599 bool take_ownership,
1600 const struct pipe_constant_buffer *cb)
1601 {
1602 zink_set_constant_buffer_internal(pctx, shader, index, take_ownership, cb, false);
1603 }
1604
1605 ALWAYS_INLINE static void
unbind_descriptor_reads(struct zink_resource * res,bool is_compute)1606 unbind_descriptor_reads(struct zink_resource *res, bool is_compute)
1607 {
1608 if (!res->sampler_bind_count[is_compute] && !res->image_bind_count[is_compute] && !res->all_bindless)
1609 res->barrier_access[is_compute] &= ~VK_ACCESS_SHADER_READ_BIT;
1610 }
1611
1612 ALWAYS_INLINE static void
unbind_buffer_descriptor_reads(struct zink_resource * res,bool is_compute)1613 unbind_buffer_descriptor_reads(struct zink_resource *res, bool is_compute)
1614 {
1615 if (!res->ssbo_bind_count[is_compute] && !res->all_bindless)
1616 unbind_descriptor_reads(res, is_compute);
1617 }
1618
1619 ALWAYS_INLINE static void
unbind_ssbo(struct zink_context * ctx,struct zink_resource * res,gl_shader_stage pstage,unsigned slot,bool writable)1620 unbind_ssbo(struct zink_context *ctx, struct zink_resource *res, gl_shader_stage pstage, unsigned slot, bool writable)
1621 {
1622 if (!res)
1623 return;
1624 res->ssbo_bind_mask[pstage] &= ~BITFIELD_BIT(slot);
1625 res->ssbo_bind_count[pstage == MESA_SHADER_COMPUTE]--;
1626 unbind_buffer_descriptor_stage(res, pstage);
1627 unbind_buffer_descriptor_reads(res, pstage == MESA_SHADER_COMPUTE);
1628 update_res_bind_count(ctx, res, pstage == MESA_SHADER_COMPUTE, true);
1629 if (writable)
1630 res->write_bind_count[pstage == MESA_SHADER_COMPUTE]--;
1631 if (!res->write_bind_count[pstage == MESA_SHADER_COMPUTE])
1632 res->barrier_access[pstage == MESA_SHADER_COMPUTE] &= ~VK_ACCESS_SHADER_WRITE_BIT;
1633 }
1634
1635 ALWAYS_INLINE static void
zink_set_shader_buffers_internal(struct pipe_context * pctx,gl_shader_stage p_stage,unsigned start_slot,unsigned count,const struct pipe_shader_buffer * buffers,unsigned writable_bitmask,bool use_db)1636 zink_set_shader_buffers_internal(struct pipe_context *pctx,
1637 gl_shader_stage p_stage,
1638 unsigned start_slot, unsigned count,
1639 const struct pipe_shader_buffer *buffers,
1640 unsigned writable_bitmask,
1641 bool use_db)
1642 {
1643 struct zink_context *ctx = zink_context(pctx);
1644 bool update = false;
1645 unsigned max_slot = 0;
1646
1647 unsigned modified_bits = u_bit_consecutive(start_slot, count);
1648 unsigned old_writable_mask = ctx->writable_ssbos[p_stage];
1649 assert(!ctx->unordered_blitting);
1650 ctx->writable_ssbos[p_stage] &= ~modified_bits;
1651 ctx->writable_ssbos[p_stage] |= writable_bitmask << start_slot;
1652
1653 for (unsigned i = 0; i < count; i++) {
1654 unsigned slot = start_slot + i;
1655 struct pipe_shader_buffer *ssbo = &ctx->ssbos[p_stage][slot];
1656 struct zink_resource *res = ssbo->buffer ? zink_resource(ssbo->buffer) : NULL;
1657 bool was_writable = old_writable_mask & BITFIELD64_BIT(slot);
1658 if (buffers && buffers[i].buffer) {
1659 struct zink_resource *new_res = zink_resource(buffers[i].buffer);
1660 if (new_res != res) {
1661 unbind_ssbo(ctx, res, p_stage, slot, was_writable);
1662 new_res->ssbo_bind_mask[p_stage] |= BITFIELD_BIT(slot);
1663 new_res->ssbo_bind_count[p_stage == MESA_SHADER_COMPUTE]++;
1664 new_res->gfx_barrier |= zink_pipeline_flags_from_pipe_stage(p_stage);
1665 update_res_bind_count(ctx, new_res, p_stage == MESA_SHADER_COMPUTE, false);
1666 }
1667 VkAccessFlags access = VK_ACCESS_SHADER_READ_BIT;
1668 if (ctx->writable_ssbos[p_stage] & BITFIELD64_BIT(slot)) {
1669 new_res->write_bind_count[p_stage == MESA_SHADER_COMPUTE]++;
1670 access |= VK_ACCESS_SHADER_WRITE_BIT;
1671 }
1672 pipe_resource_reference(&ssbo->buffer, &new_res->base.b);
1673 new_res->barrier_access[p_stage == MESA_SHADER_COMPUTE] |= access;
1674 ssbo->buffer_offset = buffers[i].buffer_offset;
1675 ssbo->buffer_size = MIN2(buffers[i].buffer_size, new_res->base.b.width0 - ssbo->buffer_offset);
1676 util_range_add(&new_res->base.b, &new_res->valid_buffer_range, ssbo->buffer_offset,
1677 ssbo->buffer_offset + ssbo->buffer_size);
1678 zink_screen(ctx->base.screen)->buffer_barrier(ctx, new_res, access,
1679 new_res->gfx_barrier);
1680 zink_batch_resource_usage_set(ctx->bs, new_res, access & VK_ACCESS_SHADER_WRITE_BIT, true);
1681 update = true;
1682 max_slot = MAX2(max_slot, slot);
1683 if (use_db) {
1684 update_descriptor_state_ssbo_db(ctx, p_stage, slot, new_res);
1685 } else {
1686 update_descriptor_state_ssbo_lazy(ctx, p_stage, slot, new_res);
1687 }
1688 if (zink_resource_access_is_write(access))
1689 new_res->obj->unordered_write = false;
1690 new_res->obj->unordered_read = false;
1691 } else {
1692 if (res)
1693 update = true;
1694 ssbo->buffer_offset = 0;
1695 ssbo->buffer_size = 0;
1696 if (res) {
1697 unbind_ssbo(ctx, res, p_stage, slot, was_writable);
1698 if (use_db) {
1699 update_descriptor_state_ssbo_db(ctx, p_stage, slot, NULL);
1700 } else {
1701 update_descriptor_state_ssbo_lazy(ctx, p_stage, slot, NULL);
1702 }
1703
1704 }
1705 pipe_resource_reference(&ssbo->buffer, NULL);
1706 }
1707 }
1708 if (start_slot + count >= ctx->di.num_ssbos[p_stage])
1709 ctx->di.num_ssbos[p_stage] = max_slot + 1;
1710 if (update)
1711 ctx->invalidate_descriptor_state(ctx, p_stage, ZINK_DESCRIPTOR_TYPE_SSBO, start_slot, count);
1712 }
1713
1714 static void
zink_set_shader_buffers_db(struct pipe_context * pctx,gl_shader_stage p_stage,unsigned start_slot,unsigned count,const struct pipe_shader_buffer * buffers,unsigned writable_bitmask)1715 zink_set_shader_buffers_db(struct pipe_context *pctx,
1716 gl_shader_stage p_stage,
1717 unsigned start_slot, unsigned count,
1718 const struct pipe_shader_buffer *buffers,
1719 unsigned writable_bitmask)
1720 {
1721 zink_set_shader_buffers_internal(pctx, p_stage, start_slot, count, buffers, writable_bitmask, true);
1722 }
1723
1724 static void
zink_set_shader_buffers_lazy(struct pipe_context * pctx,gl_shader_stage p_stage,unsigned start_slot,unsigned count,const struct pipe_shader_buffer * buffers,unsigned writable_bitmask)1725 zink_set_shader_buffers_lazy(struct pipe_context *pctx,
1726 gl_shader_stage p_stage,
1727 unsigned start_slot, unsigned count,
1728 const struct pipe_shader_buffer *buffers,
1729 unsigned writable_bitmask)
1730 {
1731 zink_set_shader_buffers_internal(pctx, p_stage, start_slot, count, buffers, writable_bitmask, false);
1732 }
1733
1734 static void
update_binds_for_samplerviews(struct zink_context * ctx,struct zink_resource * res,bool is_compute)1735 update_binds_for_samplerviews(struct zink_context *ctx, struct zink_resource *res, bool is_compute)
1736 {
1737 VkImageLayout layout = get_layout_for_binding(ctx, res, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, is_compute);
1738 if (is_compute) {
1739 u_foreach_bit(slot, res->sampler_binds[MESA_SHADER_COMPUTE]) {
1740 if (ctx->di.textures[MESA_SHADER_COMPUTE][slot].imageLayout != layout) {
1741 update_descriptor_state_sampler(ctx, MESA_SHADER_COMPUTE, slot, res);
1742 ctx->invalidate_descriptor_state(ctx, MESA_SHADER_COMPUTE, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1);
1743 }
1744 }
1745 } else {
1746 for (unsigned i = 0; i < ZINK_GFX_SHADER_COUNT; i++) {
1747 u_foreach_bit(slot, res->sampler_binds[i]) {
1748 if (ctx->di.textures[i][slot].imageLayout != layout) {
1749 update_descriptor_state_sampler(ctx, i, slot, res);
1750 ctx->invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1);
1751 }
1752 }
1753 }
1754 }
1755 }
1756
1757 static void
flush_pending_clears(struct zink_context * ctx,struct zink_resource * res)1758 flush_pending_clears(struct zink_context *ctx, struct zink_resource *res)
1759 {
1760 if (res->fb_bind_count && ctx->clears_enabled)
1761 zink_fb_clears_apply(ctx, &res->base.b);
1762 }
1763
1764 static inline void
unbind_shader_image_counts(struct zink_context * ctx,struct zink_resource * res,bool is_compute,bool writable)1765 unbind_shader_image_counts(struct zink_context *ctx, struct zink_resource *res, bool is_compute, bool writable)
1766 {
1767 update_res_bind_count(ctx, res, is_compute, true);
1768 if (writable)
1769 res->write_bind_count[is_compute]--;
1770 res->image_bind_count[is_compute]--;
1771 /* if this was the last image bind, the sampler bind layouts must be updated */
1772 if (!res->obj->is_buffer && !res->image_bind_count[is_compute] && res->bind_count[is_compute])
1773 update_binds_for_samplerviews(ctx, res, is_compute);
1774 }
1775
1776 ALWAYS_INLINE static bool
check_for_layout_update(struct zink_context * ctx,struct zink_resource * res,bool is_compute)1777 check_for_layout_update(struct zink_context *ctx, struct zink_resource *res, bool is_compute)
1778 {
1779 VkImageLayout layout = res->bind_count[is_compute] ? zink_descriptor_util_image_layout_eval(ctx, res, is_compute) : VK_IMAGE_LAYOUT_UNDEFINED;
1780 VkImageLayout other_layout = res->bind_count[!is_compute] ? zink_descriptor_util_image_layout_eval(ctx, res, !is_compute) : VK_IMAGE_LAYOUT_UNDEFINED;
1781 bool ret = false;
1782 if (!is_compute && res->fb_binds && !(ctx->feedback_loops & res->fb_binds)) {
1783 /* always double check feedback loops */
1784 ret = !!_mesa_set_add(ctx->need_barriers[0], res);
1785 } else {
1786 if (res->bind_count[is_compute] && layout && res->layout != layout)
1787 ret = !!_mesa_set_add(ctx->need_barriers[is_compute], res);
1788 if (res->bind_count[!is_compute] && other_layout && (layout != other_layout || res->layout != other_layout))
1789 ret = !!_mesa_set_add(ctx->need_barriers[!is_compute], res);
1790 }
1791 return ret;
1792 }
1793
1794 static void
unbind_shader_image(struct zink_context * ctx,gl_shader_stage stage,unsigned slot)1795 unbind_shader_image(struct zink_context *ctx, gl_shader_stage stage, unsigned slot)
1796 {
1797 struct zink_image_view *image_view = &ctx->image_views[stage][slot];
1798 bool is_compute = stage == MESA_SHADER_COMPUTE;
1799 if (!image_view->base.resource)
1800 return;
1801
1802 struct zink_resource *res = zink_resource(image_view->base.resource);
1803 res->image_binds[stage] &= ~BITFIELD_BIT(slot);
1804 unbind_shader_image_counts(ctx, res, is_compute, image_view->base.access & PIPE_IMAGE_ACCESS_WRITE);
1805 if (!res->write_bind_count[is_compute])
1806 res->barrier_access[stage == MESA_SHADER_COMPUTE] &= ~VK_ACCESS_SHADER_WRITE_BIT;
1807
1808 if (image_view->base.resource->target == PIPE_BUFFER) {
1809 unbind_buffer_descriptor_stage(res, stage);
1810 unbind_buffer_descriptor_reads(res, stage == MESA_SHADER_COMPUTE);
1811 zink_buffer_view_reference(zink_screen(ctx->base.screen), &image_view->buffer_view, NULL);
1812 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB)
1813 pipe_resource_reference(&image_view->base.resource, NULL);
1814 } else {
1815 unbind_descriptor_stage(res, stage);
1816 unbind_descriptor_reads(res, stage == MESA_SHADER_COMPUTE);
1817 if (!res->image_bind_count[is_compute])
1818 check_for_layout_update(ctx, res, is_compute);
1819 zink_surface_reference(zink_screen(ctx->base.screen), &image_view->surface, NULL);
1820 }
1821 image_view->base.resource = NULL;
1822 image_view->surface = NULL;
1823 }
1824
1825 static struct zink_buffer_view *
create_image_bufferview(struct zink_context * ctx,const struct pipe_image_view * view)1826 create_image_bufferview(struct zink_context *ctx, const struct pipe_image_view *view)
1827 {
1828 struct zink_resource *res = zink_resource(view->resource);
1829 VkBufferViewCreateInfo bvci = create_bvci(ctx, res, view->format, view->u.buf.offset, view->u.buf.size);
1830 struct zink_buffer_view *buffer_view = get_buffer_view(ctx, res, &bvci);
1831 if (!buffer_view)
1832 return NULL;
1833 util_range_add(&res->base.b, &res->valid_buffer_range, view->u.buf.offset,
1834 view->u.buf.offset + view->u.buf.size);
1835 return buffer_view;
1836 }
1837
1838 static void
finalize_image_bind(struct zink_context * ctx,struct zink_resource * res,bool is_compute)1839 finalize_image_bind(struct zink_context *ctx, struct zink_resource *res, bool is_compute)
1840 {
1841 /* if this is the first image bind and there are sampler binds, the image's sampler layout
1842 * must be updated to GENERAL
1843 */
1844 if (res->image_bind_count[is_compute] == 1 &&
1845 res->bind_count[is_compute] > 1)
1846 update_binds_for_samplerviews(ctx, res, is_compute);
1847 if (!check_for_layout_update(ctx, res, is_compute)) {
1848 /* no deferred barrier: unset unordered usage immediately */
1849 // TODO: figure out a way to link up layouts between unordered and main cmdbuf
1850 // if (zink_resource_access_is_write(res->barrier_access[is_compute]))
1851 res->obj->unordered_write = false;
1852 res->obj->unordered_read = false;
1853 }
1854 }
1855
1856 static struct zink_surface *
create_image_surface(struct zink_context * ctx,const struct pipe_image_view * view,bool is_compute)1857 create_image_surface(struct zink_context *ctx, const struct pipe_image_view *view, bool is_compute)
1858 {
1859 struct zink_screen *screen = zink_screen(ctx->base.screen);
1860 struct zink_resource *res = zink_resource(view->resource);
1861 struct pipe_surface tmpl = {0};
1862 enum pipe_texture_target target = res->base.b.target;
1863 tmpl.format = view->format;
1864 tmpl.u.tex.level = view->u.tex.level;
1865 tmpl.u.tex.first_layer = view->u.tex.first_layer;
1866 tmpl.u.tex.last_layer = view->u.tex.last_layer;
1867 unsigned depth = 1 + tmpl.u.tex.last_layer - tmpl.u.tex.first_layer;
1868 switch (target) {
1869 case PIPE_TEXTURE_3D:
1870 if (depth < u_minify(res->base.b.depth0, view->u.tex.level)) {
1871 assert(depth == 1);
1872 target = PIPE_TEXTURE_2D;
1873 if (!screen->info.have_EXT_image_2d_view_of_3d ||
1874 !screen->info.view2d_feats.image2DViewOf3D) {
1875 static bool warned = false;
1876 warn_missing_feature(warned, "image2DViewOf3D");
1877 }
1878 } else {
1879 assert(tmpl.u.tex.first_layer == 0);
1880 tmpl.u.tex.last_layer = 0;
1881 }
1882 break;
1883 case PIPE_TEXTURE_2D_ARRAY:
1884 case PIPE_TEXTURE_1D_ARRAY:
1885 if (depth < res->base.b.array_size && depth == 1)
1886 target = target == PIPE_TEXTURE_2D_ARRAY ? PIPE_TEXTURE_2D : PIPE_TEXTURE_1D;
1887 break;
1888 default: break;
1889 }
1890 if (zink_format_needs_mutable(view->resource->format, view->format))
1891 /* mutable not set by default */
1892 zink_resource_object_init_mutable(ctx, res);
1893 VkImageViewCreateInfo ivci = create_ivci(screen, res, &tmpl, target);
1894 struct zink_surface *surface = zink_get_surface(ctx, view->resource, &tmpl, &ivci);
1895 if (!surface)
1896 return NULL;
1897 if (is_compute)
1898 flush_pending_clears(ctx, res);
1899 return surface;
1900 }
1901
1902 static void
zink_set_shader_images(struct pipe_context * pctx,gl_shader_stage shader_type,unsigned start_slot,unsigned count,unsigned unbind_num_trailing_slots,const struct pipe_image_view * images)1903 zink_set_shader_images(struct pipe_context *pctx,
1904 gl_shader_stage shader_type,
1905 unsigned start_slot, unsigned count,
1906 unsigned unbind_num_trailing_slots,
1907 const struct pipe_image_view *images)
1908 {
1909 struct zink_context *ctx = zink_context(pctx);
1910 struct zink_screen *screen = zink_screen(pctx->screen);
1911 bool update = false;
1912 bool is_compute = shader_type == MESA_SHADER_COMPUTE;
1913 assert(!ctx->unordered_blitting);
1914 for (unsigned i = 0; i < count; i++) {
1915 struct zink_image_view *a = &ctx->image_views[shader_type][start_slot + i];
1916 const struct pipe_image_view *b = images ? &images[i] : NULL;
1917 struct zink_resource *res = b ? zink_resource(b->resource) : NULL;
1918 if (b && b->resource) {
1919 if (!zink_resource_object_init_storage(ctx, res)) {
1920 debug_printf("couldn't create storage image!");
1921 continue;
1922 }
1923
1924 VkAccessFlags access = 0;
1925 if (b->access & PIPE_IMAGE_ACCESS_WRITE) {
1926 access |= VK_ACCESS_SHADER_WRITE_BIT;
1927 }
1928 if (b->access & PIPE_IMAGE_ACCESS_READ) {
1929 access |= VK_ACCESS_SHADER_READ_BIT;
1930 }
1931
1932 bool changed = false;
1933 if (!a->base.resource || a->base.resource != b->resource) {
1934 /* this needs a full unbind+bind */
1935 changed = true;
1936 unbind_shader_image(ctx, shader_type, start_slot + i);
1937 update_res_bind_count(ctx, res, is_compute, false);
1938 res->image_bind_count[is_compute]++;
1939 /* always increment write_bind_count on new bind */
1940 if (b->access & PIPE_IMAGE_ACCESS_WRITE)
1941 res->write_bind_count[is_compute]++;
1942 /* db mode refcounts these */
1943 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB && b->resource->target == PIPE_BUFFER)
1944 pipe_resource_reference(&a->base.resource, b->resource);
1945 } else {
1946 /* resource matches: check for write flag change and partial rebind */
1947
1948 /* previous bind didn't have write: increment */
1949 if ((b->access & PIPE_IMAGE_ACCESS_WRITE) && !(a->base.access & PIPE_IMAGE_ACCESS_WRITE))
1950 res->write_bind_count[is_compute]++;
1951 /* previous bind had write: decrement */
1952 else if (!(b->access & PIPE_IMAGE_ACCESS_WRITE) && (a->base.access & PIPE_IMAGE_ACCESS_WRITE)) {
1953 res->write_bind_count[is_compute]--;
1954 if (!res->write_bind_count[is_compute])
1955 res->barrier_access[is_compute] &= ~VK_ACCESS_SHADER_WRITE_BIT;
1956 }
1957
1958 /* this may need a partial rebind */
1959 changed = a->base.format != b->format || zink_resource(a->base.resource)->obj != res->obj;
1960 if (!changed) {
1961 if (b->resource->target == PIPE_BUFFER) {
1962 /* db mode has no partial rebind */
1963 if (zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB)
1964 changed = !!memcmp(&a->base.u.buf, &b->u.buf, sizeof(b->u.buf));
1965 } else {
1966 /* no memcmp, these are bitfields */
1967 changed = a->base.u.tex.first_layer != b->u.tex.first_layer ||
1968 a->base.u.tex.last_layer != b->u.tex.last_layer ||
1969 a->base.u.tex.level != b->u.tex.level;
1970 }
1971 }
1972 }
1973
1974 if (changed) {
1975 /* this is a partial rebind */
1976 if (b->resource->target == PIPE_BUFFER) {
1977 /* db has no partial rebind */
1978 if (zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB) {
1979 /* bufferview rebind: get updated bufferview and unref old one */
1980 struct zink_buffer_view *bv = create_image_bufferview(ctx, b);
1981 /* identical rebind was already checked above */
1982 assert(bv && bv != a->buffer_view);
1983 zink_buffer_view_reference(screen, &a->buffer_view, NULL);
1984 /* ref already added by create */
1985 a->buffer_view = bv;
1986 }
1987 } else {
1988 /* image rebind: get updated surface and unref old one */
1989 struct zink_surface *surface = create_image_surface(ctx, b, is_compute);
1990 /* identical rebind was already checked above */
1991 assert(surface && surface != a->surface);
1992 zink_surface_reference(screen, &a->surface, NULL);
1993 /* ref already added by create */
1994 a->surface = surface;
1995 }
1996 }
1997
1998 /* these operations occur regardless of binding/rebinding */
1999 res->gfx_barrier |= zink_pipeline_flags_from_pipe_stage(shader_type);
2000 res->barrier_access[is_compute] |= access;
2001 if (b->resource->target == PIPE_BUFFER) {
2002 screen->buffer_barrier(ctx, res, access,
2003 res->gfx_barrier);
2004 zink_batch_resource_usage_set(ctx->bs, res,
2005 zink_resource_access_is_write(access), true);
2006 if (zink_resource_access_is_write(access))
2007 res->obj->unordered_write = false;
2008 res->obj->unordered_read = false;
2009 } else {
2010 finalize_image_bind(ctx, res, is_compute);
2011 zink_batch_resource_usage_set(ctx->bs, res,
2012 zink_resource_access_is_write(access), false);
2013 }
2014 memcpy(&a->base, images + i, sizeof(struct pipe_image_view));
2015 if (b->resource->target == PIPE_BUFFER) {
2016 /* always enforce limit clamping */
2017 unsigned blocksize = util_format_get_blocksize(a->base.format);
2018 a->base.u.buf.size = MIN2(a->base.u.buf.size / blocksize, screen->info.props.limits.maxTexelBufferElements) * blocksize;
2019 }
2020 update = true;
2021 res->image_binds[shader_type] |= BITFIELD_BIT(start_slot + i);
2022 } else if (a->base.resource) {
2023 update = true;
2024 unbind_shader_image(ctx, shader_type, start_slot + i);
2025 }
2026 update_descriptor_state_image(ctx, shader_type, start_slot + i, res);
2027 }
2028 for (unsigned i = 0; i < unbind_num_trailing_slots; i++) {
2029 update |= !!ctx->image_views[shader_type][start_slot + count + i].base.resource;
2030 unbind_shader_image(ctx, shader_type, start_slot + count + i);
2031 update_descriptor_state_image(ctx, shader_type, start_slot + count + i, NULL);
2032 }
2033 ctx->di.num_images[shader_type] = start_slot + count;
2034 if (update)
2035 ctx->invalidate_descriptor_state(ctx, shader_type, ZINK_DESCRIPTOR_TYPE_IMAGE, start_slot, count);
2036 }
2037
2038 static void
update_feedback_loop_dynamic_state(struct zink_context * ctx)2039 update_feedback_loop_dynamic_state(struct zink_context *ctx)
2040 {
2041 if (!zink_screen(ctx->base.screen)->info.have_EXT_attachment_feedback_loop_dynamic_state)
2042 return;
2043 VkImageAspectFlags aspects = 0;
2044 if (ctx->feedback_loops & BITFIELD_MASK(PIPE_MAX_COLOR_BUFS))
2045 aspects |= VK_IMAGE_ASPECT_COLOR_BIT;
2046 if (ctx->feedback_loops & BITFIELD_BIT(PIPE_MAX_COLOR_BUFS))
2047 aspects |= VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT;
2048 VKCTX(CmdSetAttachmentFeedbackLoopEnableEXT)(ctx->bs->cmdbuf, aspects);
2049 }
2050
2051 static void
update_feedback_loop_state(struct zink_context * ctx,unsigned idx,unsigned feedback_loops)2052 update_feedback_loop_state(struct zink_context *ctx, unsigned idx, unsigned feedback_loops)
2053 {
2054 if (feedback_loops != ctx->feedback_loops) {
2055 if (idx == PIPE_MAX_COLOR_BUFS && !zink_screen(ctx->base.screen)->driver_workarounds.always_feedback_loop_zs) {
2056 if (ctx->gfx_pipeline_state.feedback_loop_zs)
2057 ctx->gfx_pipeline_state.dirty = true;
2058 ctx->gfx_pipeline_state.feedback_loop_zs = false;
2059 } else if (idx < PIPE_MAX_COLOR_BUFS && !zink_screen(ctx->base.screen)->driver_workarounds.always_feedback_loop) {
2060 if (ctx->gfx_pipeline_state.feedback_loop)
2061 ctx->gfx_pipeline_state.dirty = true;
2062 ctx->gfx_pipeline_state.feedback_loop = false;
2063 }
2064 update_feedback_loop_dynamic_state(ctx);
2065 }
2066 ctx->feedback_loops = feedback_loops;
2067 }
2068
2069 ALWAYS_INLINE static void
unbind_samplerview(struct zink_context * ctx,gl_shader_stage stage,unsigned slot)2070 unbind_samplerview(struct zink_context *ctx, gl_shader_stage stage, unsigned slot)
2071 {
2072 struct zink_sampler_view *sv = zink_sampler_view(ctx->sampler_views[stage][slot]);
2073 if (!sv || !sv->base.texture)
2074 return;
2075 struct zink_resource *res = zink_resource(sv->base.texture);
2076 res->sampler_bind_count[stage == MESA_SHADER_COMPUTE]--;
2077 if (stage != MESA_SHADER_COMPUTE && !res->sampler_bind_count[0] && res->fb_bind_count) {
2078 u_foreach_bit(idx, res->fb_binds) {
2079 if (ctx->feedback_loops & BITFIELD_BIT(idx)) {
2080 ctx->dynamic_fb.attachments[idx].imageLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
2081 ctx->rp_layout_changed = true;
2082 }
2083 update_feedback_loop_state(ctx, idx, ctx->feedback_loops & ~BITFIELD_BIT(idx));
2084 }
2085 }
2086 update_res_bind_count(ctx, res, stage == MESA_SHADER_COMPUTE, true);
2087 res->sampler_binds[stage] &= ~BITFIELD_BIT(slot);
2088 if (res->obj->is_buffer) {
2089 unbind_buffer_descriptor_stage(res, stage);
2090 unbind_buffer_descriptor_reads(res, stage == MESA_SHADER_COMPUTE);
2091 } else {
2092 unbind_descriptor_stage(res, stage);
2093 unbind_descriptor_reads(res, stage == MESA_SHADER_COMPUTE);
2094 if (!res->sampler_bind_count[stage == MESA_SHADER_COMPUTE])
2095 check_for_layout_update(ctx, res, stage == MESA_SHADER_COMPUTE);
2096 }
2097 assert(slot < 32);
2098 ctx->di.zs_swizzle[stage].mask &= ~BITFIELD_BIT(slot);
2099 }
2100
2101 static void
zink_set_sampler_views(struct pipe_context * pctx,gl_shader_stage shader_type,unsigned start_slot,unsigned num_views,unsigned unbind_num_trailing_slots,bool take_ownership,struct pipe_sampler_view ** views)2102 zink_set_sampler_views(struct pipe_context *pctx,
2103 gl_shader_stage shader_type,
2104 unsigned start_slot,
2105 unsigned num_views,
2106 unsigned unbind_num_trailing_slots,
2107 bool take_ownership,
2108 struct pipe_sampler_view **views)
2109 {
2110 struct zink_context *ctx = zink_context(pctx);
2111
2112 const uint32_t mask = BITFIELD_RANGE(start_slot, num_views);
2113 uint32_t shadow_mask = ctx->di.zs_swizzle[shader_type].mask;
2114 ctx->di.cubes[shader_type] &= ~mask;
2115
2116 bool update = false;
2117 bool shadow_update = false;
2118 if (views) {
2119 for (unsigned i = 0; i < num_views; ++i) {
2120 struct pipe_sampler_view *pview = views[i];
2121 struct zink_sampler_view *a = zink_sampler_view(ctx->sampler_views[shader_type][start_slot + i]);
2122 struct zink_sampler_view *b = zink_sampler_view(pview);
2123
2124 if (a == b) {
2125 if (take_ownership) {
2126 struct pipe_sampler_view *view = views[i];
2127 pipe_sampler_view_reference(&view, NULL);
2128 }
2129 continue;
2130 }
2131
2132 struct zink_resource *res = b ? zink_resource(b->base.texture) : NULL;
2133 if (b && b->base.texture) {
2134 if (!a || zink_resource(a->base.texture) != res) {
2135 if (a)
2136 unbind_samplerview(ctx, shader_type, start_slot + i);
2137 update_res_bind_count(ctx, res, shader_type == MESA_SHADER_COMPUTE, false);
2138 res->sampler_bind_count[shader_type == MESA_SHADER_COMPUTE]++;
2139 res->gfx_barrier |= zink_pipeline_flags_from_pipe_stage(shader_type);
2140 res->barrier_access[shader_type == MESA_SHADER_COMPUTE] |= VK_ACCESS_SHADER_READ_BIT;
2141 }
2142 if (res->base.b.target == PIPE_BUFFER) {
2143 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
2144 if (!a || a->base.texture != b->base.texture || zink_resource(a->base.texture)->obj != res->obj ||
2145 memcmp(&a->base.u.buf, &b->base.u.buf, sizeof(b->base.u.buf)))
2146 update = true;
2147 } else if (b->buffer_view->bvci.buffer != res->obj->buffer) {
2148 /* if this resource has been rebound while it wasn't set here,
2149 * its backing resource will have changed and thus we need to update
2150 * the bufferview
2151 */
2152 VkBufferViewCreateInfo bvci = b->buffer_view->bvci;
2153 bvci.buffer = res->obj->buffer;
2154 struct zink_buffer_view *buffer_view = get_buffer_view(ctx, res, &bvci);
2155 assert(buffer_view != b->buffer_view);
2156 zink_buffer_view_reference(zink_screen(ctx->base.screen), &b->buffer_view, NULL);
2157 b->buffer_view = buffer_view;
2158 update = true;
2159 } else if (!a || a->buffer_view->buffer_view != b->buffer_view->buffer_view)
2160 update = true;
2161 zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, VK_ACCESS_SHADER_READ_BIT,
2162 res->gfx_barrier);
2163 zink_batch_resource_usage_set(ctx->bs, res, false, true);
2164 if (!ctx->unordered_blitting)
2165 res->obj->unordered_read = false;
2166 } else {
2167 if (zink_format_needs_mutable(res->base.b.format, b->image_view->base.format))
2168 /* mutable not set by default */
2169 zink_resource_object_init_mutable(ctx, res);
2170 if (res->obj != b->image_view->obj) {
2171 struct pipe_surface *psurf = &b->image_view->base;
2172 VkImageView iv = b->image_view->image_view;
2173 zink_rebind_surface(ctx, &psurf);
2174 b->image_view = zink_surface(psurf);
2175 update |= iv != b->image_view->image_view;
2176 } else if (a != b)
2177 update = true;
2178 if (shader_type == MESA_SHADER_COMPUTE)
2179 flush_pending_clears(ctx, res);
2180 if (b->cube_array) {
2181 ctx->di.cubes[shader_type] |= BITFIELD_BIT(start_slot + i);
2182 }
2183 if (!check_for_layout_update(ctx, res, shader_type == MESA_SHADER_COMPUTE) && !ctx->unordered_blitting) {
2184 /* no deferred barrier: unset unordered usage immediately */
2185 res->obj->unordered_read = false;
2186 // TODO: figure out a way to link up layouts between unordered and main cmdbuf
2187 res->obj->unordered_write = false;
2188 }
2189 if (!a)
2190 update = true;
2191 zink_batch_resource_usage_set(ctx->bs, res, false, false);
2192 if (b->zs_view) {
2193 assert(start_slot + i < 32); //bitfield size
2194 ctx->di.zs_swizzle[shader_type].mask |= BITFIELD_BIT(start_slot + i);
2195 /* this is already gonna be slow, so don't bother trying to micro-optimize */
2196 shadow_update |= memcmp(&ctx->di.zs_swizzle[shader_type].swizzle[start_slot + i],
2197 &b->swizzle, sizeof(struct zink_zs_swizzle));
2198 memcpy(&ctx->di.zs_swizzle[shader_type].swizzle[start_slot + i], &b->swizzle, sizeof(struct zink_zs_swizzle));
2199 } else {
2200 assert(start_slot + i < 32); //bitfield size
2201 ctx->di.zs_swizzle[shader_type].mask &= ~BITFIELD_BIT(start_slot + i);
2202 }
2203 }
2204 res->sampler_binds[shader_type] |= BITFIELD_BIT(start_slot + i);
2205 } else if (a) {
2206 unbind_samplerview(ctx, shader_type, start_slot + i);
2207 update = true;
2208 }
2209 if (take_ownership) {
2210 pipe_sampler_view_reference(&ctx->sampler_views[shader_type][start_slot + i], NULL);
2211 ctx->sampler_views[shader_type][start_slot + i] = pview;
2212 } else {
2213 pipe_sampler_view_reference(&ctx->sampler_views[shader_type][start_slot + i], pview);
2214 }
2215 update_descriptor_state_sampler(ctx, shader_type, start_slot + i, res);
2216 }
2217 } else {
2218 unbind_num_trailing_slots += num_views;
2219 num_views = 0;
2220 }
2221 for (unsigned i = 0; i < unbind_num_trailing_slots; ++i) {
2222 unsigned slot = start_slot + num_views + i;
2223 update |= !!ctx->sampler_views[shader_type][slot];
2224 unbind_samplerview(ctx, shader_type, slot);
2225 pipe_sampler_view_reference(
2226 &ctx->sampler_views[shader_type][slot],
2227 NULL);
2228 update_descriptor_state_sampler(ctx, shader_type, slot, NULL);
2229 }
2230 ctx->di.num_sampler_views[shader_type] = start_slot + num_views;
2231 if (update) {
2232 struct zink_screen *screen = zink_screen(pctx->screen);
2233 ctx->invalidate_descriptor_state(ctx, shader_type, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot, num_views);
2234 if (!screen->info.have_EXT_non_seamless_cube_map)
2235 update_nonseamless_shader_key(ctx, shader_type);
2236 shadow_update |= shadow_mask != ctx->di.zs_swizzle[shader_type].mask;
2237 zink_set_zs_needs_shader_swizzle_key(ctx, shader_type, shadow_update);
2238 }
2239 }
2240
2241 static uint64_t
zink_create_texture_handle(struct pipe_context * pctx,struct pipe_sampler_view * view,const struct pipe_sampler_state * state)2242 zink_create_texture_handle(struct pipe_context *pctx, struct pipe_sampler_view *view, const struct pipe_sampler_state *state)
2243 {
2244 struct zink_context *ctx = zink_context(pctx);
2245 struct zink_resource *res = zink_resource(view->texture);
2246 struct zink_sampler_view *sv = zink_sampler_view(view);
2247 struct zink_bindless_descriptor *bd;
2248 bd = calloc(1, sizeof(struct zink_bindless_descriptor));
2249 if (!bd)
2250 return 0;
2251
2252 bd->sampler = pctx->create_sampler_state(pctx, state);
2253 if (!bd->sampler) {
2254 free(bd);
2255 return 0;
2256 }
2257
2258 bd->ds.is_buffer = res->base.b.target == PIPE_BUFFER;
2259 if (res->base.b.target == PIPE_BUFFER) {
2260 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
2261 pipe_resource_reference(&bd->ds.db.pres, view->texture);
2262 bd->ds.db.format = view->format;
2263 bd->ds.db.offset = view->u.buf.offset;
2264 bd->ds.db.size = view->u.buf.size;
2265 } else {
2266 zink_buffer_view_reference(zink_screen(pctx->screen), &bd->ds.bufferview, sv->buffer_view);
2267 }
2268 } else {
2269 zink_surface_reference(zink_screen(pctx->screen), &bd->ds.surface, sv->image_view);
2270 }
2271 uint64_t handle = util_idalloc_alloc(&ctx->di.bindless[bd->ds.is_buffer].tex_slots);
2272 if (bd->ds.is_buffer)
2273 handle += ZINK_MAX_BINDLESS_HANDLES;
2274 bd->handle = handle;
2275 _mesa_hash_table_insert(&ctx->di.bindless[bd->ds.is_buffer].tex_handles, (void*)(uintptr_t)handle, bd);
2276 return handle;
2277 }
2278
2279 static void
zink_delete_texture_handle(struct pipe_context * pctx,uint64_t handle)2280 zink_delete_texture_handle(struct pipe_context *pctx, uint64_t handle)
2281 {
2282 struct zink_context *ctx = zink_context(pctx);
2283 bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle);
2284 struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].tex_handles, (void*)(uintptr_t)handle);
2285 assert(he);
2286 struct zink_bindless_descriptor *bd = he->data;
2287 struct zink_descriptor_surface *ds = &bd->ds;
2288 _mesa_hash_table_remove(&ctx->di.bindless[is_buffer].tex_handles, he);
2289 uint32_t h = handle;
2290 util_dynarray_append(&ctx->bs->bindless_releases[0], uint32_t, h);
2291
2292 if (ds->is_buffer) {
2293 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
2294 pipe_resource_reference(&ds->db.pres, NULL);
2295 } else {
2296 zink_buffer_view_reference(zink_screen(pctx->screen), &ds->bufferview, NULL);
2297 }
2298 } else {
2299 zink_surface_reference(zink_screen(pctx->screen), &ds->surface, NULL);
2300 pctx->delete_sampler_state(pctx, bd->sampler);
2301 }
2302 free(ds);
2303 }
2304
2305 static void
rebind_bindless_bufferview(struct zink_context * ctx,struct zink_resource * res,struct zink_descriptor_surface * ds)2306 rebind_bindless_bufferview(struct zink_context *ctx, struct zink_resource *res, struct zink_descriptor_surface *ds)
2307 {
2308 /* descriptor buffer is unaffected by this */
2309 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB)
2310 return;
2311 /* if this resource has been rebound while it wasn't set here,
2312 * its backing resource will have changed and thus we need to update
2313 * the bufferview
2314 */
2315 VkBufferViewCreateInfo bvci = ds->bufferview->bvci;
2316 bvci.buffer = res->obj->buffer;
2317 struct zink_buffer_view *buffer_view = get_buffer_view(ctx, res, &bvci);
2318 assert(buffer_view != ds->bufferview);
2319 zink_buffer_view_reference(zink_screen(ctx->base.screen), &ds->bufferview, NULL);
2320 ds->bufferview = buffer_view;
2321 }
2322
2323 static void
zero_bindless_descriptor(struct zink_context * ctx,uint32_t handle,bool is_buffer,bool is_image)2324 zero_bindless_descriptor(struct zink_context *ctx, uint32_t handle, bool is_buffer, bool is_image)
2325 {
2326 if (likely(zink_screen(ctx->base.screen)->info.rb2_feats.nullDescriptor)) {
2327 if (is_buffer) {
2328 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
2329 ctx->di.bindless[is_image].db.buffer_infos[handle].address = 0;
2330 ctx->di.bindless[is_image].db.buffer_infos[handle].range = 0;
2331 } else {
2332 VkBufferView *bv = &ctx->di.bindless[is_image].t.buffer_infos[handle];
2333 *bv = VK_NULL_HANDLE;
2334 }
2335 } else {
2336 VkDescriptorImageInfo *ii = &ctx->di.bindless[is_image].img_infos[handle];
2337 memset(ii, 0, sizeof(*ii));
2338 }
2339 } else {
2340 if (is_buffer) {
2341 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
2342 ctx->di.bindless[is_image].db.buffer_infos[handle].address = zink_resource(ctx->dummy_bufferview->pres)->obj->bda;
2343 ctx->di.bindless[is_image].db.buffer_infos[handle].range = 1;
2344 } else {
2345 VkBufferView *bv = &ctx->di.bindless[is_image].t.buffer_infos[handle];
2346 struct zink_buffer_view *null_bufferview = ctx->dummy_bufferview;
2347 *bv = null_bufferview->buffer_view;
2348 }
2349 } else {
2350 struct zink_surface *null_surface = zink_get_dummy_surface(ctx, 0);
2351 VkDescriptorImageInfo *ii = &ctx->di.bindless[is_image].img_infos[handle];
2352 ii->sampler = VK_NULL_HANDLE;
2353 ii->imageView = null_surface->image_view;
2354 ii->imageLayout = VK_IMAGE_LAYOUT_GENERAL;
2355 }
2356 }
2357 }
2358
2359 static void
unbind_bindless_descriptor(struct zink_context * ctx,struct zink_resource * res)2360 unbind_bindless_descriptor(struct zink_context *ctx, struct zink_resource *res)
2361 {
2362 if (!res->bindless[1]) {
2363 /* check to remove write access */
2364 for (unsigned i = 0; i < 2; i++) {
2365 if (!res->write_bind_count[i])
2366 res->barrier_access[i] &= ~VK_ACCESS_SHADER_WRITE_BIT;
2367 }
2368 }
2369 bool is_buffer = res->base.b.target == PIPE_BUFFER;
2370 if (!res->all_bindless) {
2371 /* check to remove read access */
2372 if (is_buffer) {
2373 for (unsigned i = 0; i < 2; i++)
2374 unbind_buffer_descriptor_reads(res, i);
2375 } else {
2376 for (unsigned i = 0; i < 2; i++)
2377 unbind_descriptor_reads(res, i);
2378 }
2379 }
2380 for (unsigned i = 0; i < 2; i++) {
2381 if (!res->image_bind_count[i])
2382 check_for_layout_update(ctx, res, i);
2383 }
2384 }
2385
2386 static void
zink_make_texture_handle_resident(struct pipe_context * pctx,uint64_t handle,bool resident)2387 zink_make_texture_handle_resident(struct pipe_context *pctx, uint64_t handle, bool resident)
2388 {
2389 struct zink_context *ctx = zink_context(pctx);
2390 bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle);
2391 struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].tex_handles, (void*)(uintptr_t)handle);
2392 assert(he);
2393 struct zink_bindless_descriptor *bd = he->data;
2394 struct zink_descriptor_surface *ds = &bd->ds;
2395 struct zink_resource *res = zink_descriptor_surface_resource(ds);
2396 if (is_buffer)
2397 handle -= ZINK_MAX_BINDLESS_HANDLES;
2398 if (resident) {
2399 update_res_bind_count(ctx, res, false, false);
2400 update_res_bind_count(ctx, res, true, false);
2401 res->bindless[0]++;
2402 if (is_buffer) {
2403 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
2404 ctx->di.bindless[0].db.buffer_infos[handle].address = res->obj->bda + ds->db.offset;
2405 ctx->di.bindless[0].db.buffer_infos[handle].range = ds->db.size;
2406 ctx->di.bindless[0].db.buffer_infos[handle].format = zink_get_format(zink_screen(ctx->base.screen), ds->db.format);
2407 } else {
2408 if (ds->bufferview->bvci.buffer != res->obj->buffer)
2409 rebind_bindless_bufferview(ctx, res, ds);
2410 VkBufferView *bv = &ctx->di.bindless[0].t.buffer_infos[handle];
2411 *bv = ds->bufferview->buffer_view;
2412 }
2413 zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, VK_ACCESS_SHADER_READ_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
2414 zink_batch_resource_usage_set(ctx->bs, res, false, true);
2415 res->obj->unordered_read = false;
2416 } else {
2417 VkDescriptorImageInfo *ii = &ctx->di.bindless[0].img_infos[handle];
2418 ii->sampler = bd->sampler->sampler;
2419 ii->imageView = ds->surface->image_view;
2420 ii->imageLayout = zink_descriptor_util_image_layout_eval(ctx, res, false);
2421 flush_pending_clears(ctx, res);
2422 if (!check_for_layout_update(ctx, res, false)) {
2423 res->obj->unordered_read = false;
2424 // TODO: figure out a way to link up layouts between unordered and main cmdbuf
2425 res->obj->unordered_write = false;
2426 }
2427 if (!check_for_layout_update(ctx, res, true)) {
2428 res->obj->unordered_read = false;
2429 // TODO: figure out a way to link up layouts between unordered and main cmdbuf
2430 res->obj->unordered_write = false;
2431 }
2432 zink_batch_resource_usage_set(ctx->bs, res, false, false);
2433 res->obj->unordered_write = false;
2434 }
2435 res->gfx_barrier |= VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
2436 res->barrier_access[0] |= VK_ACCESS_SHADER_READ_BIT;
2437 res->barrier_access[1] |= VK_ACCESS_SHADER_READ_BIT;
2438 util_dynarray_append(&ctx->di.bindless[0].resident, struct zink_bindless_descriptor *, bd);
2439 uint32_t h = is_buffer ? handle + ZINK_MAX_BINDLESS_HANDLES : handle;
2440 util_dynarray_append(&ctx->di.bindless[0].updates, uint32_t, h);
2441 } else {
2442 zero_bindless_descriptor(ctx, handle, is_buffer, false);
2443 util_dynarray_delete_unordered(&ctx->di.bindless[0].resident, struct zink_bindless_descriptor *, bd);
2444 update_res_bind_count(ctx, res, false, true);
2445 update_res_bind_count(ctx, res, true, true);
2446 res->bindless[0]--;
2447 unbind_bindless_descriptor(ctx, res);
2448 }
2449 ctx->di.bindless_dirty[0] = true;
2450 }
2451
2452 static uint64_t
zink_create_image_handle(struct pipe_context * pctx,const struct pipe_image_view * view)2453 zink_create_image_handle(struct pipe_context *pctx, const struct pipe_image_view *view)
2454 {
2455 struct zink_context *ctx = zink_context(pctx);
2456 struct zink_resource *res = zink_resource(view->resource);
2457 struct zink_bindless_descriptor *bd;
2458 if (!zink_resource_object_init_storage(ctx, res)) {
2459 debug_printf("couldn't create storage image!");
2460 return 0;
2461 }
2462 bd = calloc(1, sizeof(struct zink_bindless_descriptor));
2463 if (!bd)
2464 return 0;
2465 bd->sampler = NULL;
2466
2467 bd->ds.is_buffer = res->base.b.target == PIPE_BUFFER;
2468 if (res->base.b.target == PIPE_BUFFER)
2469 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
2470 pipe_resource_reference(&bd->ds.db.pres, view->resource);
2471 bd->ds.db.format = view->format;
2472 bd->ds.db.offset = view->u.buf.offset;
2473 bd->ds.db.size = view->u.buf.size;
2474 } else {
2475 bd->ds.bufferview = create_image_bufferview(ctx, view);
2476 }
2477 else
2478 bd->ds.surface = create_image_surface(ctx, view, false);
2479 uint64_t handle = util_idalloc_alloc(&ctx->di.bindless[bd->ds.is_buffer].img_slots);
2480 if (bd->ds.is_buffer)
2481 handle += ZINK_MAX_BINDLESS_HANDLES;
2482 bd->handle = handle;
2483 _mesa_hash_table_insert(&ctx->di.bindless[bd->ds.is_buffer].img_handles, (void*)(uintptr_t)handle, bd);
2484 return handle;
2485 }
2486
2487 static void
zink_delete_image_handle(struct pipe_context * pctx,uint64_t handle)2488 zink_delete_image_handle(struct pipe_context *pctx, uint64_t handle)
2489 {
2490 struct zink_context *ctx = zink_context(pctx);
2491 bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle);
2492 struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].img_handles, (void*)(uintptr_t)handle);
2493 assert(he);
2494 struct zink_descriptor_surface *ds = he->data;
2495 _mesa_hash_table_remove(&ctx->di.bindless[is_buffer].img_handles, he);
2496 uint32_t h = handle;
2497 util_dynarray_append(&ctx->bs->bindless_releases[1], uint32_t, h);
2498
2499 if (ds->is_buffer) {
2500 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
2501 pipe_resource_reference(&ds->db.pres, NULL);
2502 } else {
2503 zink_buffer_view_reference(zink_screen(pctx->screen), &ds->bufferview, NULL);
2504 }
2505 } else {
2506 zink_surface_reference(zink_screen(pctx->screen), &ds->surface, NULL);
2507 }
2508 free(ds);
2509 }
2510
2511 static void
zink_make_image_handle_resident(struct pipe_context * pctx,uint64_t handle,unsigned paccess,bool resident)2512 zink_make_image_handle_resident(struct pipe_context *pctx, uint64_t handle, unsigned paccess, bool resident)
2513 {
2514 struct zink_context *ctx = zink_context(pctx);
2515 bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle);
2516 struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].img_handles, (void*)(uintptr_t)handle);
2517 assert(he);
2518 struct zink_bindless_descriptor *bd = he->data;
2519 struct zink_descriptor_surface *ds = &bd->ds;
2520 bd->access = paccess;
2521 struct zink_resource *res = zink_descriptor_surface_resource(ds);
2522 VkAccessFlags access = 0;
2523 if (paccess & PIPE_IMAGE_ACCESS_WRITE) {
2524 if (resident) {
2525 res->write_bind_count[0]++;
2526 res->write_bind_count[1]++;
2527 } else {
2528 res->write_bind_count[0]--;
2529 res->write_bind_count[1]--;
2530 }
2531 access |= VK_ACCESS_SHADER_WRITE_BIT;
2532 }
2533 if (paccess & PIPE_IMAGE_ACCESS_READ) {
2534 access |= VK_ACCESS_SHADER_READ_BIT;
2535 }
2536 if (is_buffer)
2537 handle -= ZINK_MAX_BINDLESS_HANDLES;
2538 if (resident) {
2539 update_res_bind_count(ctx, res, false, false);
2540 update_res_bind_count(ctx, res, true, false);
2541 res->image_bind_count[0]++;
2542 res->image_bind_count[1]++;
2543 res->bindless[1]++;
2544 if (is_buffer) {
2545 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
2546 ctx->di.bindless[0].db.buffer_infos[handle].address = res->obj->bda + ds->db.offset;
2547 ctx->di.bindless[0].db.buffer_infos[handle].range = ds->db.size;
2548 ctx->di.bindless[0].db.buffer_infos[handle].format = zink_get_format(zink_screen(ctx->base.screen), ds->db.format);
2549 } else {
2550 if (ds->bufferview->bvci.buffer != res->obj->buffer)
2551 rebind_bindless_bufferview(ctx, res, ds);
2552 VkBufferView *bv = &ctx->di.bindless[1].t.buffer_infos[handle];
2553 *bv = ds->bufferview->buffer_view;
2554 }
2555 zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, access, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
2556 zink_batch_resource_usage_set(ctx->bs, res, zink_resource_access_is_write(access), true);
2557 if (zink_resource_access_is_write(access))
2558 res->obj->unordered_write = false;
2559 res->obj->unordered_read = false;
2560 } else {
2561 VkDescriptorImageInfo *ii = &ctx->di.bindless[1].img_infos[handle];
2562 ii->sampler = VK_NULL_HANDLE;
2563 ii->imageView = ds->surface->image_view;
2564 ii->imageLayout = VK_IMAGE_LAYOUT_GENERAL;
2565 finalize_image_bind(ctx, res, false);
2566 finalize_image_bind(ctx, res, true);
2567 zink_batch_resource_usage_set(ctx->bs, res, zink_resource_access_is_write(access), false);
2568 res->obj->unordered_write = false;
2569 }
2570 res->gfx_barrier |= VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
2571 res->barrier_access[0] |= access;
2572 res->barrier_access[1] |= access;
2573 util_dynarray_append(&ctx->di.bindless[1].resident, struct zink_bindless_descriptor *, bd);
2574 uint32_t h = is_buffer ? handle + ZINK_MAX_BINDLESS_HANDLES : handle;
2575 util_dynarray_append(&ctx->di.bindless[1].updates, uint32_t, h);
2576 } else {
2577 zero_bindless_descriptor(ctx, handle, is_buffer, true);
2578 util_dynarray_delete_unordered(&ctx->di.bindless[1].resident, struct zink_bindless_descriptor *, bd);
2579 unbind_shader_image_counts(ctx, res, false, false);
2580 unbind_shader_image_counts(ctx, res, true, false);
2581 res->bindless[1]--;
2582 unbind_bindless_descriptor(ctx, res);
2583 }
2584 ctx->di.bindless_dirty[1] = true;
2585 }
2586
2587 static void
zink_set_global_binding(struct pipe_context * pctx,unsigned first,unsigned count,struct pipe_resource ** resources,uint32_t ** handles)2588 zink_set_global_binding(struct pipe_context *pctx,
2589 unsigned first, unsigned count,
2590 struct pipe_resource **resources,
2591 uint32_t **handles)
2592 {
2593 struct zink_context *ctx = zink_context(pctx);
2594
2595 size_t size = ctx->di.global_bindings.capacity;
2596 if (!util_dynarray_resize(&ctx->di.global_bindings, struct pipe_resource*, first + count + 8))
2597 unreachable("zink: out of memory somehow");
2598 if (size != ctx->di.global_bindings.capacity) {
2599 uint8_t *data = ctx->di.global_bindings.data;
2600 memset(data + size, 0, ctx->di.global_bindings.capacity - size);
2601 }
2602
2603 struct pipe_resource **globals = ctx->di.global_bindings.data;
2604 for (unsigned i = 0; i < count; i++) {
2605 if (resources && resources[i]) {
2606 struct zink_resource *res = zink_resource(resources[i]);
2607
2608 util_range_add(&res->base.b, &res->valid_buffer_range, 0, res->base.b.width0);
2609 pipe_resource_reference(&globals[first + i], resources[i]);
2610
2611 uint64_t addr = 0;
2612 memcpy(&addr, handles[i], sizeof(addr));
2613 addr += zink_resource_get_address(zink_screen(pctx->screen), res);
2614 memcpy(handles[i], &addr, sizeof(addr));
2615 zink_resource_usage_set(res, ctx->bs, true);
2616 res->obj->unordered_read = res->obj->unordered_write = false;
2617 zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
2618 } else if (globals[i]) {
2619 zink_batch_reference_resource(ctx, zink_resource(globals[first + i]));
2620 pipe_resource_reference(&globals[first + i], NULL);
2621 }
2622 }
2623 }
2624
2625 static void
zink_set_stencil_ref(struct pipe_context * pctx,const struct pipe_stencil_ref ref)2626 zink_set_stencil_ref(struct pipe_context *pctx,
2627 const struct pipe_stencil_ref ref)
2628 {
2629 struct zink_context *ctx = zink_context(pctx);
2630 ctx->stencil_ref = ref;
2631 ctx->stencil_ref_changed = true;
2632 }
2633
2634 static void
zink_set_clip_state(struct pipe_context * pctx,const struct pipe_clip_state * pcs)2635 zink_set_clip_state(struct pipe_context *pctx,
2636 const struct pipe_clip_state *pcs)
2637 {
2638 }
2639
2640 static void
zink_set_tess_state(struct pipe_context * pctx,const float default_outer_level[4],const float default_inner_level[2])2641 zink_set_tess_state(struct pipe_context *pctx,
2642 const float default_outer_level[4],
2643 const float default_inner_level[2])
2644 {
2645 struct zink_context *ctx = zink_context(pctx);
2646 memcpy(&ctx->default_inner_level, default_inner_level, sizeof(ctx->default_inner_level));
2647 memcpy(&ctx->default_outer_level, default_outer_level, sizeof(ctx->default_outer_level));
2648 }
2649
2650 static void
zink_set_patch_vertices(struct pipe_context * pctx,uint8_t patch_vertices)2651 zink_set_patch_vertices(struct pipe_context *pctx, uint8_t patch_vertices)
2652 {
2653 struct zink_context *ctx = zink_context(pctx);
2654 if (zink_set_tcs_key_patches(ctx, patch_vertices)) {
2655 ctx->gfx_pipeline_state.dyn_state2.vertices_per_patch = patch_vertices;
2656 if (zink_screen(ctx->base.screen)->info.dynamic_state2_feats.extendedDynamicState2PatchControlPoints)
2657 VKCTX(CmdSetPatchControlPointsEXT)(ctx->bs->cmdbuf, patch_vertices);
2658 else
2659 ctx->gfx_pipeline_state.dirty = true;
2660 }
2661 }
2662
2663 static void
init_null_fbfetch(struct zink_context * ctx)2664 init_null_fbfetch(struct zink_context *ctx)
2665 {
2666 struct zink_screen *screen = zink_screen(ctx->base.screen);
2667 ctx->di.null_fbfetch_init = true;
2668 if (zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB)
2669 return;
2670 VkDescriptorGetInfoEXT info;
2671 info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_GET_INFO_EXT;
2672 info.pNext = NULL;
2673 info.type = VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT;
2674 info.data.pInputAttachmentImage = &ctx->di.fbfetch;
2675 if (screen->info.db_props.inputAttachmentDescriptorSize)
2676 VKSCR(GetDescriptorEXT)(screen->dev, &info, screen->info.db_props.inputAttachmentDescriptorSize, ctx->di.fbfetch_db);
2677 }
2678
2679 bool
zink_update_fbfetch(struct zink_context * ctx)2680 zink_update_fbfetch(struct zink_context *ctx)
2681 {
2682 const bool had_fbfetch = ctx->di.fbfetch.imageLayout == VK_IMAGE_LAYOUT_GENERAL;
2683 if (!ctx->gfx_stages[MESA_SHADER_FRAGMENT] ||
2684 !ctx->gfx_stages[MESA_SHADER_FRAGMENT]->info.fs.uses_fbfetch_output) {
2685 if (!had_fbfetch)
2686 return false;
2687 zink_batch_no_rp(ctx);
2688 ctx->di.fbfetch.imageLayout = VK_IMAGE_LAYOUT_UNDEFINED;
2689 ctx->di.fbfetch.imageView = zink_screen(ctx->base.screen)->info.rb2_feats.nullDescriptor ?
2690 VK_NULL_HANDLE :
2691 zink_get_dummy_surface(ctx, 0)->image_view;
2692 ctx->invalidate_descriptor_state(ctx, MESA_SHADER_FRAGMENT, ZINK_DESCRIPTOR_TYPE_UBO, 0, 1);
2693 return true;
2694 }
2695
2696 bool changed = !had_fbfetch;
2697 if (ctx->fb_state.cbufs[0]) {
2698 VkImageView fbfetch = zink_csurface(ctx->fb_state.cbufs[0])->image_view;
2699 if (!fbfetch)
2700 /* swapchain image: retry later */
2701 return false;
2702 changed |= fbfetch != ctx->di.fbfetch.imageView;
2703 ctx->di.fbfetch.imageView = zink_csurface(ctx->fb_state.cbufs[0])->image_view;
2704
2705 bool fbfetch_ms = ctx->fb_state.cbufs[0]->texture->nr_samples > 1;
2706 if (zink_get_fs_base_key(ctx)->fbfetch_ms != fbfetch_ms)
2707 zink_set_fs_base_key(ctx)->fbfetch_ms = fbfetch_ms;
2708 } else {
2709 ctx->di.fbfetch.imageView = zink_screen(ctx->base.screen)->info.rb2_feats.nullDescriptor ?
2710 VK_NULL_HANDLE :
2711 zink_get_dummy_surface(ctx, 0)->image_view;
2712 }
2713 bool ret = false;
2714 ctx->di.fbfetch.imageLayout = VK_IMAGE_LAYOUT_GENERAL;
2715 if (changed) {
2716 ctx->invalidate_descriptor_state(ctx, MESA_SHADER_FRAGMENT, ZINK_DESCRIPTOR_TYPE_UBO, 0, 1);
2717 if (!had_fbfetch) {
2718 ret = true;
2719 zink_batch_no_rp(ctx);
2720 }
2721 }
2722 return ret;
2723 }
2724
2725 void
zink_update_vk_sample_locations(struct zink_context * ctx)2726 zink_update_vk_sample_locations(struct zink_context *ctx)
2727 {
2728 if (ctx->gfx_pipeline_state.sample_locations_enabled && ctx->sample_locations_changed) {
2729 unsigned samples = ctx->gfx_pipeline_state.rast_samples + 1;
2730 unsigned idx = util_logbase2_ceil(MAX2(samples, 1));
2731 VkExtent2D grid_size = zink_screen(ctx->base.screen)->maxSampleLocationGridSize[idx];
2732
2733 for (unsigned pixel = 0; pixel < grid_size.width * grid_size.height; pixel++) {
2734 for (unsigned sample = 0; sample < samples; sample++) {
2735 unsigned pixel_x = pixel % grid_size.width;
2736 unsigned pixel_y = pixel / grid_size.width;
2737 unsigned wi = pixel * samples + sample;
2738 unsigned ri = (pixel_y * grid_size.width + pixel_x % grid_size.width);
2739 ri = ri * samples + sample;
2740 ctx->vk_sample_locations[wi].x = (ctx->sample_locations[ri] & 0xf) / 16.0f;
2741 ctx->vk_sample_locations[wi].y = (16 - (ctx->sample_locations[ri] >> 4)) / 16.0f;
2742 }
2743 }
2744 }
2745 }
2746
2747 static unsigned
find_rp_state(struct zink_context * ctx)2748 find_rp_state(struct zink_context *ctx)
2749 {
2750 bool found = false;
2751 /* calc the state idx using the samples to account for msrtss */
2752 unsigned idx = zink_screen(ctx->base.screen)->info.have_EXT_multisampled_render_to_single_sampled && ctx->transient_attachments ?
2753 util_logbase2_ceil(ctx->gfx_pipeline_state.rast_samples + 1) : 0;
2754 struct set_entry *he = _mesa_set_search_or_add(&ctx->rendering_state_cache[idx], &ctx->gfx_pipeline_state.rendering_info, &found);
2755 struct zink_rendering_info *info;
2756 if (found) {
2757 info = (void*)he->key;
2758 return info->id;
2759 }
2760 info = ralloc(ctx, struct zink_rendering_info);
2761 memcpy(info, &ctx->gfx_pipeline_state.rendering_info, sizeof(VkPipelineRenderingCreateInfo));
2762 info->id = ctx->rendering_state_cache[idx].entries;
2763 he->key = info;
2764 return info->id;
2765 }
2766
2767 unsigned
zink_update_rendering_info(struct zink_context * ctx)2768 zink_update_rendering_info(struct zink_context *ctx)
2769 {
2770 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2771 struct zink_surface *surf = zink_csurface(ctx->fb_state.cbufs[i]);
2772 ctx->gfx_pipeline_state.rendering_formats[i] = surf ? surf->info.format[0] : VK_FORMAT_UNDEFINED;
2773 }
2774 ctx->gfx_pipeline_state.rendering_info.viewMask = ctx->fb_state.viewmask;
2775 ctx->gfx_pipeline_state.rendering_info.depthAttachmentFormat = VK_FORMAT_UNDEFINED;
2776 ctx->gfx_pipeline_state.rendering_info.stencilAttachmentFormat = VK_FORMAT_UNDEFINED;
2777 if (ctx->fb_state.zsbuf && zink_is_zsbuf_used(ctx)) {
2778 struct zink_surface *surf = zink_csurface(ctx->fb_state.zsbuf);
2779 bool has_depth = util_format_has_depth(util_format_description(ctx->fb_state.zsbuf->format));
2780 bool has_stencil = util_format_has_stencil(util_format_description(ctx->fb_state.zsbuf->format));
2781
2782 if (has_depth)
2783 ctx->gfx_pipeline_state.rendering_info.depthAttachmentFormat = surf->info.format[0];
2784 if (has_stencil)
2785 ctx->gfx_pipeline_state.rendering_info.stencilAttachmentFormat = surf->info.format[0];
2786 }
2787 return find_rp_state(ctx);
2788 }
2789
2790 static unsigned
calc_max_dummy_fbo_size(struct zink_context * ctx)2791 calc_max_dummy_fbo_size(struct zink_context *ctx)
2792 {
2793 unsigned size = MAX2(ctx->fb_state.width, ctx->fb_state.height);
2794 return size ? size : MIN2(256, zink_screen(ctx->base.screen)->info.props.limits.maxImageDimension2D);
2795 }
2796
2797 static unsigned
begin_rendering(struct zink_context * ctx,bool check_msaa_expand)2798 begin_rendering(struct zink_context *ctx, bool check_msaa_expand)
2799 {
2800 unsigned clear_buffers = 0;
2801 ctx->gfx_pipeline_state.render_pass = NULL;
2802 zink_update_vk_sample_locations(ctx);
2803 bool has_swapchain = zink_render_update_swapchain(ctx);
2804 if (has_swapchain)
2805 zink_render_fixup_swapchain(ctx);
2806 bool has_depth = false;
2807 bool has_stencil = false;
2808 bool changed_layout = false;
2809 bool changed_size = false;
2810 bool zsbuf_used = zink_is_zsbuf_used(ctx);
2811 bool has_msrtss = zink_screen(ctx->base.screen)->info.have_EXT_multisampled_render_to_single_sampled;
2812 bool use_tc_info = !ctx->blitting && ctx->track_renderpasses;
2813 uint32_t msaa_expand_mask = 0;
2814
2815 if (ctx->rp_changed || ctx->rp_layout_changed || (!ctx->in_rp && ctx->rp_loadop_changed)) {
2816 /* init imageviews, base loadOp, formats */
2817 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2818 struct zink_surface *surf = zink_csurface(ctx->fb_state.cbufs[i]);
2819 if (!surf)
2820 continue;
2821
2822 if (!zink_resource(surf->base.texture)->valid)
2823 ctx->dynamic_fb.attachments[i].loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
2824 else
2825 ctx->dynamic_fb.attachments[i].loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
2826 if (use_tc_info) {
2827 /* can't skip stores if this is not a winsys resolve */
2828 if ((!ctx->dynamic_fb.tc_info.has_resolve || ctx->fb_state.resolve) && ctx->dynamic_fb.tc_info.cbuf_invalidate & BITFIELD_BIT(i))
2829 ctx->dynamic_fb.attachments[i].storeOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
2830 else
2831 ctx->dynamic_fb.attachments[i].storeOp = VK_ATTACHMENT_STORE_OP_STORE;
2832 }
2833 if (ctx->dynamic_fb.attachments[i].loadOp == VK_ATTACHMENT_LOAD_OP_LOAD)
2834 msaa_expand_mask |= BITFIELD_BIT(i);
2835 }
2836
2837 /* unset depth and stencil info: reset below */
2838 VkImageLayout zlayout = ctx->dynamic_fb.info.pDepthAttachment ? ctx->dynamic_fb.info.pDepthAttachment->imageLayout : VK_IMAGE_LAYOUT_UNDEFINED;
2839 VkImageLayout slayout = ctx->dynamic_fb.info.pStencilAttachment ? ctx->dynamic_fb.info.pStencilAttachment->imageLayout : VK_IMAGE_LAYOUT_UNDEFINED;
2840 ctx->dynamic_fb.info.pDepthAttachment = NULL;
2841 ctx->dynamic_fb.info.pStencilAttachment = NULL;
2842
2843 if (ctx->fb_state.zsbuf && zsbuf_used) {
2844 struct zink_surface *surf = zink_csurface(ctx->fb_state.zsbuf);
2845 has_depth = util_format_has_depth(util_format_description(ctx->fb_state.zsbuf->format));
2846 has_stencil = util_format_has_stencil(util_format_description(ctx->fb_state.zsbuf->format));
2847
2848 /* depth may or may not be used but init it anyway */
2849 if (zink_resource(surf->base.texture)->valid)
2850 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
2851 else
2852 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
2853
2854 if (use_tc_info) {
2855 if (ctx->dynamic_fb.tc_info.zsbuf_invalidate)
2856 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].storeOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
2857 else
2858 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].storeOp = VK_ATTACHMENT_STORE_OP_STORE;
2859 }
2860
2861 /* maybe TODO but also not handled by legacy rp...
2862 if (ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].loadOp == VK_ATTACHMENT_LOAD_OP_LOAD)
2863 msaa_expand_mask |= BITFIELD_BIT(PIPE_MAX_COLOR_BUFS);
2864 */
2865 /* stencil may or may not be used but init it anyway */
2866 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].loadOp = ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].loadOp;
2867 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].storeOp = ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].storeOp;
2868
2869 if (has_depth) {
2870 ctx->dynamic_fb.info.pDepthAttachment = &ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS];
2871 /* stencil info only set for clears below */
2872 }
2873 if (has_stencil) {
2874 /* must be stencil-only */
2875 ctx->dynamic_fb.info.pStencilAttachment = &ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS + 1];
2876 }
2877 } else {
2878 ctx->dynamic_fb.info.pDepthAttachment = NULL;
2879 }
2880 if (zlayout != (ctx->dynamic_fb.info.pDepthAttachment ? ctx->dynamic_fb.info.pDepthAttachment->imageLayout : VK_IMAGE_LAYOUT_UNDEFINED))
2881 changed_layout = true;
2882 if (slayout != (ctx->dynamic_fb.info.pStencilAttachment ? ctx->dynamic_fb.info.pStencilAttachment->imageLayout : VK_IMAGE_LAYOUT_UNDEFINED))
2883 changed_layout = true;
2884
2885 /* similar to begin_render_pass(), but just filling in VkRenderingInfo */
2886 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2887 /* these are no-ops */
2888 if (!ctx->fb_state.cbufs[i] || !zink_fb_clear_enabled(ctx, i))
2889 continue;
2890 /* these need actual clear calls inside the rp */
2891 struct zink_framebuffer_clear_data *clear = zink_fb_clear_element(&ctx->fb_clears[i], 0);
2892 if (zink_fb_clear_needs_explicit(&ctx->fb_clears[i])) {
2893 clear_buffers |= (PIPE_CLEAR_COLOR0 << i);
2894 if (zink_fb_clear_count(&ctx->fb_clears[i]) < 2 ||
2895 zink_fb_clear_element_needs_explicit(clear))
2896 continue;
2897 }
2898 /* we now know there's one clear that can be done here */
2899 memcpy(&ctx->dynamic_fb.attachments[i].clearValue, &clear->color, sizeof(float) * 4);
2900 ctx->dynamic_fb.attachments[i].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
2901 }
2902 if (ctx->fb_state.zsbuf && zink_fb_clear_enabled(ctx, PIPE_MAX_COLOR_BUFS)) {
2903 struct zink_framebuffer_clear *fb_clear = &ctx->fb_clears[PIPE_MAX_COLOR_BUFS];
2904 struct zink_framebuffer_clear_data *clear = zink_fb_clear_element(fb_clear, 0);
2905 if (!zink_fb_clear_element_needs_explicit(clear)) {
2906 /* base zs clear info */
2907 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].clearValue.depthStencil.depth = clear->zs.depth;
2908 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].clearValue.depthStencil.stencil = clear->zs.stencil;
2909 /* always init separate stencil attachment */
2910 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].clearValue.depthStencil.stencil = clear->zs.stencil;
2911 if ((zink_fb_clear_element(fb_clear, 0)->zs.bits & PIPE_CLEAR_DEPTH))
2912 /* initiate a depth clear */
2913 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
2914 if ((zink_fb_clear_element(fb_clear, 0)->zs.bits & PIPE_CLEAR_STENCIL)) {
2915 /* use a stencil clear, also set stencil attachment */
2916 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
2917 }
2918 }
2919 }
2920 if (changed_size || changed_layout)
2921 ctx->rp_changed = true;
2922 ctx->rp_loadop_changed = false;
2923 ctx->rp_layout_changed = false;
2924 }
2925 msaa_expand_mask &= ctx->transient_attachments;
2926 if (!has_msrtss && msaa_expand_mask && check_msaa_expand) {
2927 zink_render_msaa_expand(ctx, msaa_expand_mask);
2928 return begin_rendering(ctx, false);
2929 }
2930 /* always assemble clear_buffers mask:
2931 * if a scissored clear must be triggered during glFlush,
2932 * the renderpass metadata may be unchanged (e.g., LOAD from previous rp),
2933 * but the buffer mask must still be returned
2934 */
2935 if (ctx->clears_enabled) {
2936 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2937 /* these are no-ops */
2938 if (!ctx->fb_state.cbufs[i] || !zink_fb_clear_enabled(ctx, i))
2939 continue;
2940 /* these need actual clear calls inside the rp */
2941 if (zink_fb_clear_needs_explicit(&ctx->fb_clears[i]))
2942 clear_buffers |= (PIPE_CLEAR_COLOR0 << i);
2943 }
2944 if (ctx->fb_state.zsbuf && zink_fb_clear_enabled(ctx, PIPE_MAX_COLOR_BUFS)) {
2945 struct zink_framebuffer_clear *fb_clear = &ctx->fb_clears[PIPE_MAX_COLOR_BUFS];
2946 struct zink_framebuffer_clear_data *clear = zink_fb_clear_element(fb_clear, 0);
2947 if (zink_fb_clear_needs_explicit(fb_clear)) {
2948 for (int j = !zink_fb_clear_element_needs_explicit(clear);
2949 (clear_buffers & PIPE_CLEAR_DEPTHSTENCIL) != PIPE_CLEAR_DEPTHSTENCIL && j < zink_fb_clear_count(fb_clear);
2950 j++)
2951 clear_buffers |= zink_fb_clear_element(fb_clear, j)->zs.bits;
2952 }
2953 }
2954 }
2955
2956 if (!ctx->rp_changed && ctx->in_rp)
2957 return 0;
2958 ctx->rp_changed = false;
2959
2960 /* update pipeline info id for compatibility VUs */
2961 unsigned rp_state = zink_update_rendering_info(ctx);
2962 /* validate zs VUs: attachment must be null or format must be valid */
2963 assert(!ctx->dynamic_fb.info.pDepthAttachment || ctx->gfx_pipeline_state.rendering_info.depthAttachmentFormat);
2964 assert(!ctx->dynamic_fb.info.pStencilAttachment || ctx->gfx_pipeline_state.rendering_info.stencilAttachmentFormat);
2965 bool rp_changed = ctx->gfx_pipeline_state.rp_state != rp_state;
2966 if (!rp_changed && ctx->in_rp)
2967 return 0;
2968
2969 zink_batch_no_rp(ctx);
2970 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2971 VkImageView iv = VK_NULL_HANDLE;
2972 struct zink_surface *surf = zink_csurface(ctx->fb_state.cbufs[i]);
2973 if (surf) {
2974 struct zink_surface *transient = zink_transient_surface(ctx->fb_state.cbufs[i]);
2975 if (transient && !has_msrtss) {
2976 iv = zink_prep_fb_attachment(ctx, transient, i);
2977 ctx->dynamic_fb.attachments[i].imageLayout = zink_resource(transient->base.texture)->layout;
2978 ctx->dynamic_fb.attachments[i].resolveMode = VK_RESOLVE_MODE_AVERAGE_BIT;
2979 ctx->dynamic_fb.attachments[i].resolveImageView = zink_prep_fb_attachment(ctx, surf, i);
2980 ctx->dynamic_fb.attachments[i].resolveImageLayout = zink_resource(surf->base.texture)->layout;
2981 } else {
2982 iv = zink_prep_fb_attachment(ctx, surf, i);
2983 ctx->dynamic_fb.attachments[i].imageLayout = zink_resource(surf->base.texture)->layout;
2984 ctx->dynamic_fb.attachments[i].resolveMode = VK_RESOLVE_MODE_NONE;
2985 ctx->dynamic_fb.attachments[i].resolveImageView = VK_NULL_HANDLE;
2986 ctx->dynamic_fb.attachments[i].resolveImageLayout = VK_IMAGE_LAYOUT_UNDEFINED;
2987 }
2988 if (!iv)
2989 /* dead swapchain */
2990 return 0;
2991
2992 } else {
2993 ctx->dynamic_fb.attachments[i].resolveMode = VK_RESOLVE_MODE_NONE;
2994 }
2995 ctx->dynamic_fb.attachments[i].imageView = iv;
2996 }
2997 if (ctx->fb_state.resolve && use_tc_info && ctx->dynamic_fb.tc_info.has_resolve) {
2998 struct zink_resource *res = zink_resource(ctx->fb_state.resolve);
2999 struct zink_surface *surf = zink_csurface(res->surface);
3000 if (zink_is_swapchain(res)) {
3001 if (!zink_kopper_acquire(ctx, res, UINT64_MAX))
3002 return 0;
3003 zink_surface_swapchain_update(ctx, surf);
3004 }
3005 zink_batch_resource_usage_set(ctx->bs, res, true, false);
3006 VkImageLayout layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
3007 zink_screen(ctx->base.screen)->image_barrier(ctx, res, layout, 0, 0);
3008 res->obj->unordered_read = res->obj->unordered_write = false;
3009 ctx->dynamic_fb.attachments[0].resolveMode = VK_RESOLVE_MODE_AVERAGE_BIT;
3010 ctx->dynamic_fb.attachments[0].resolveImageLayout = zink_resource(surf->base.texture)->layout;
3011 ctx->dynamic_fb.attachments[0].resolveImageView = surf->image_view;
3012 }
3013 if (has_swapchain) {
3014 ASSERTED struct zink_resource *res = zink_resource(ctx->fb_state.cbufs[0]->texture);
3015 zink_render_fixup_swapchain(ctx);
3016 if (res->use_damage) {
3017 ctx->dynamic_fb.info.renderArea = res->damage;
3018 } else {
3019 ctx->dynamic_fb.info.renderArea.offset.x = 0;
3020 ctx->dynamic_fb.info.renderArea.offset.y = 0;
3021 ctx->dynamic_fb.info.renderArea.extent.width = ctx->fb_state.width;
3022 ctx->dynamic_fb.info.renderArea.extent.height = ctx->fb_state.height;
3023 }
3024 /* clamp for late swapchain resize */
3025 if (res->base.b.width0 < ctx->dynamic_fb.info.renderArea.extent.width)
3026 ctx->dynamic_fb.info.renderArea.extent.width = res->base.b.width0;
3027 if (res->base.b.height0 < ctx->dynamic_fb.info.renderArea.extent.height)
3028 ctx->dynamic_fb.info.renderArea.extent.height = res->base.b.height0;
3029 }
3030 if (ctx->fb_state.zsbuf && zsbuf_used) {
3031 struct zink_surface *surf = zink_csurface(ctx->fb_state.zsbuf);
3032 struct zink_surface *transient = zink_transient_surface(ctx->fb_state.zsbuf);
3033 VkImageView iv;
3034 if (transient && !has_msrtss) {
3035 iv = zink_prep_fb_attachment(ctx, transient, ctx->fb_state.nr_cbufs);
3036 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].imageLayout = zink_resource(transient->base.texture)->layout;
3037 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].resolveImageView = zink_prep_fb_attachment(ctx, surf, ctx->fb_state.nr_cbufs);
3038 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].resolveImageLayout = zink_resource(surf->base.texture)->layout;
3039 } else {
3040 iv = zink_prep_fb_attachment(ctx, surf, ctx->fb_state.nr_cbufs);
3041 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].imageLayout = zink_resource(surf->base.texture)->layout;
3042 }
3043 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].imageView = iv;
3044
3045 assert(ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].imageLayout != VK_IMAGE_LAYOUT_UNDEFINED);
3046 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].imageView = iv;
3047 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].imageLayout = zink_resource(surf->base.texture)->layout;
3048 assert(ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].imageLayout != VK_IMAGE_LAYOUT_UNDEFINED);
3049 if (ctx->transient_attachments & BITFIELD_BIT(PIPE_MAX_COLOR_BUFS)) {
3050 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].resolveMode = VK_RESOLVE_MODE_SAMPLE_ZERO_BIT;
3051 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS + 1].resolveMode = VK_RESOLVE_MODE_SAMPLE_ZERO_BIT;
3052 } else {
3053 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].resolveMode = 0;
3054 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS + 1].resolveMode = 0;
3055 }
3056 }
3057 ctx->zsbuf_unused = !zsbuf_used;
3058 assert(ctx->fb_state.width >= ctx->dynamic_fb.info.renderArea.extent.width);
3059 assert(ctx->fb_state.height >= ctx->dynamic_fb.info.renderArea.extent.height);
3060 ctx->gfx_pipeline_state.dirty |= rp_changed;
3061 ctx->gfx_pipeline_state.rp_state = rp_state;
3062
3063 VkMultisampledRenderToSingleSampledInfoEXT msrtss = {
3064 VK_STRUCTURE_TYPE_MULTISAMPLED_RENDER_TO_SINGLE_SAMPLED_INFO_EXT,
3065 NULL,
3066 VK_TRUE,
3067 ctx->gfx_pipeline_state.rast_samples + 1,
3068 };
3069
3070 if (has_msrtss)
3071 ctx->dynamic_fb.info.pNext = ctx->transient_attachments ? &msrtss : NULL;
3072 VKCTX(CmdBeginRendering)(ctx->bs->cmdbuf, &ctx->dynamic_fb.info);
3073 ctx->in_rp = true;
3074 return clear_buffers;
3075 }
3076
3077 ALWAYS_INLINE static void
update_layered_rendering_state(struct zink_context * ctx)3078 update_layered_rendering_state(struct zink_context *ctx)
3079 {
3080 if (!zink_screen(ctx->base.screen)->driver_compiler_workarounds.needs_sanitised_layer)
3081 return;
3082 unsigned framebffer_is_layered = zink_framebuffer_get_num_layers(&ctx->fb_state) > 1;
3083 VKCTX(CmdPushConstants)(
3084 ctx->bs->cmdbuf,
3085 zink_screen(ctx->base.screen)->gfx_push_constant_layout,
3086 VK_SHADER_STAGE_ALL_GRAPHICS,
3087 offsetof(struct zink_gfx_push_constant, framebuffer_is_layered), sizeof(unsigned),
3088 &framebffer_is_layered);
3089 }
3090
3091 ALWAYS_INLINE static void
batch_ref_fb_surface(struct zink_context * ctx,struct pipe_surface * psurf)3092 batch_ref_fb_surface(struct zink_context *ctx, struct pipe_surface *psurf)
3093 {
3094 if (!psurf)
3095 return;
3096 zink_batch_reference_resource(ctx, zink_resource(psurf->texture));
3097 struct zink_surface *transient = zink_transient_surface(psurf);
3098 if (transient)
3099 zink_batch_reference_resource(ctx, zink_resource(transient->base.texture));
3100 }
3101
3102 void
zink_batch_rp(struct zink_context * ctx)3103 zink_batch_rp(struct zink_context *ctx)
3104 {
3105 assert(!(ctx->in_rp && ctx->rp_changed));
3106 if (!ctx->track_renderpasses && !ctx->blitting) {
3107 if (ctx->rp_tc_info_updated)
3108 zink_parse_tc_info(ctx);
3109 }
3110 if (ctx->in_rp && !ctx->rp_layout_changed)
3111 return;
3112 bool in_rp = ctx->in_rp;
3113 if (!in_rp && ctx->void_clears) {
3114 union pipe_color_union color;
3115 color.f[0] = color.f[1] = color.f[2] = 0;
3116 color.f[3] = 1.0;
3117 ctx->base.clear(&ctx->base, ctx->void_clears, NULL, &color, 0, 0);
3118 ctx->void_clears = 0;
3119 }
3120 if (!ctx->blitting) {
3121 if (ctx->rp_tc_info_updated)
3122 update_tc_info(ctx);
3123 ctx->rp_tc_info_updated = false;
3124 }
3125 bool maybe_has_query_ends = !ctx->track_renderpasses || ctx->dynamic_fb.tc_info.has_query_ends;
3126 ctx->queries_in_rp = maybe_has_query_ends;
3127 /* if possible, out-of-renderpass resume any queries that were stopped when previous rp ended */
3128 if (!ctx->queries_disabled && !maybe_has_query_ends) {
3129 zink_resume_queries(ctx);
3130 zink_query_update_gs_states(ctx);
3131 }
3132 unsigned clear_buffers;
3133 /* use renderpass for multisample-to-singlesample or fbfetch:
3134 * - msrtss is TODO
3135 * - dynamic rendering doesn't have input attachments
3136 */
3137 if (!zink_screen(ctx->base.screen)->info.have_KHR_dynamic_rendering ||
3138 (ctx->fbfetch_outputs && !zink_screen(ctx->base.screen)->info.have_KHR_dynamic_rendering_local_read))
3139 clear_buffers = zink_begin_render_pass(ctx);
3140 else
3141 clear_buffers = begin_rendering(ctx, true);
3142 assert(!ctx->rp_changed);
3143 if (ctx->unordered_blitting)
3144 ctx->bs->has_reordered_work = true;
3145 else
3146 ctx->bs->has_work = true;
3147
3148 /* update the render-passes HUD query */
3149 ctx->hud.render_passes++;
3150
3151 if (!in_rp && ctx->in_rp) {
3152 /* only hit this for valid swapchain and new renderpass */
3153 if (ctx->render_condition.query)
3154 zink_start_conditional_render(ctx);
3155 zink_clear_framebuffer(ctx, clear_buffers);
3156 if (ctx->pipeline_changed[0]) {
3157 for (unsigned i = 0; i < ctx->fb_state.nr_cbufs; i++)
3158 batch_ref_fb_surface(ctx, ctx->fb_state.cbufs[i]);
3159 batch_ref_fb_surface(ctx, ctx->fb_state.zsbuf);
3160 }
3161 }
3162 /* unable to previously determine that queries didn't split renderpasses: ensure queries start inside renderpass */
3163 if (!ctx->queries_disabled && maybe_has_query_ends) {
3164 zink_resume_queries(ctx);
3165 zink_query_update_gs_states(ctx);
3166 }
3167 }
3168
3169 void
zink_batch_no_rp_safe(struct zink_context * ctx)3170 zink_batch_no_rp_safe(struct zink_context *ctx)
3171 {
3172 if (!ctx->in_rp)
3173 return;
3174 if (ctx->render_condition.query)
3175 zink_stop_conditional_render(ctx);
3176 /* suspend all queries that were started in a renderpass
3177 * they can then be resumed upon beginning a new renderpass
3178 */
3179 if (!ctx->queries_disabled)
3180 zink_query_renderpass_suspend(ctx);
3181 if (ctx->gfx_pipeline_state.render_pass)
3182 zink_end_render_pass(ctx);
3183 else {
3184 VKCTX(CmdEndRendering)(ctx->bs->cmdbuf);
3185 ctx->in_rp = false;
3186 }
3187 assert(!ctx->in_rp);
3188 }
3189
3190 void
zink_batch_no_rp(struct zink_context * ctx)3191 zink_batch_no_rp(struct zink_context *ctx)
3192 {
3193 if (!ctx->in_rp)
3194 return;
3195 if (ctx->track_renderpasses && !ctx->blitting)
3196 tc_renderpass_info_reset(&ctx->dynamic_fb.tc_info);
3197 zink_batch_no_rp_safe(ctx);
3198 }
3199
3200 ALWAYS_INLINE static void
update_res_sampler_layouts(struct zink_context * ctx,struct zink_resource * res)3201 update_res_sampler_layouts(struct zink_context *ctx, struct zink_resource *res)
3202 {
3203 unsigned find = res->sampler_bind_count[0];
3204 for (unsigned i = 0; find && i < MESA_SHADER_COMPUTE; i++) {
3205 u_foreach_bit(slot, res->sampler_binds[i]) {
3206 /* only set layout, skip rest of update */
3207 if (ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW][i][slot] == res)
3208 ctx->di.textures[i][slot].imageLayout = zink_descriptor_util_image_layout_eval(ctx, res, false);
3209 find--;
3210 if (!find) break;
3211 }
3212 }
3213 }
3214
3215 VkImageView
zink_prep_fb_attachment(struct zink_context * ctx,struct zink_surface * surf,unsigned i)3216 zink_prep_fb_attachment(struct zink_context *ctx, struct zink_surface *surf, unsigned i)
3217 {
3218 struct zink_resource *res;
3219 if (!surf) {
3220 surf = zink_get_dummy_surface(ctx, util_logbase2_ceil(ctx->fb_state.samples));
3221 res = zink_resource(surf->base.texture);
3222 } else {
3223 res = zink_resource(surf->base.texture);
3224 zink_batch_resource_usage_set(ctx->bs, res, true, false);
3225 }
3226
3227 VkAccessFlags access;
3228 VkPipelineStageFlags pipeline;
3229 if (zink_is_swapchain(res)) {
3230 if (!zink_kopper_acquire(ctx, res, UINT64_MAX))
3231 return VK_NULL_HANDLE;
3232 zink_surface_swapchain_update(ctx, surf);
3233 if (!i)
3234 zink_update_fbfetch(ctx);
3235 }
3236 if (ctx->blitting)
3237 return surf->image_view;
3238 VkImageLayout layout;
3239 /* depth attachment is stored as the last attachment, but bitfields always use PIPE_MAX_COLOR_BUFS */
3240 int idx = i == ctx->fb_state.nr_cbufs ? PIPE_MAX_COLOR_BUFS : i;
3241 if (ctx->feedback_loops & BITFIELD_BIT(idx)) {
3242 /* reevaluate feedback loop in case layout change eliminates the loop */
3243 if (!res->sampler_bind_count[0] || (idx == PIPE_MAX_COLOR_BUFS && !zink_is_zsbuf_write(ctx)))
3244 update_feedback_loop_state(ctx, i, ctx->feedback_loops & ~BITFIELD_BIT(idx));
3245 }
3246 if (ctx->track_renderpasses) {
3247 layout = zink_tc_renderpass_info_parse(ctx, &ctx->dynamic_fb.tc_info, idx, &pipeline, &access);
3248 assert(i < ctx->fb_state.nr_cbufs || layout != VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL || !zink_fb_clear_enabled(ctx, PIPE_MAX_COLOR_BUFS));
3249 if (i == ctx->fb_state.nr_cbufs && zink_fb_clear_enabled(ctx, PIPE_MAX_COLOR_BUFS))
3250 assert(ctx->dynamic_fb.tc_info.zsbuf_clear || ctx->dynamic_fb.tc_info.zsbuf_clear_partial || ctx->dynamic_fb.tc_info.zsbuf_load);
3251 } else {
3252 if (ctx->gfx_pipeline_state.render_pass) {
3253 layout = zink_render_pass_attachment_get_barrier_info(&ctx->gfx_pipeline_state.render_pass->state.rts[i],
3254 i < ctx->fb_state.nr_cbufs, &pipeline, &access);
3255 } else {
3256 struct zink_rt_attrib rt;
3257 if (i < ctx->fb_state.nr_cbufs)
3258 zink_init_color_attachment(ctx, i, &rt);
3259 else
3260 zink_init_zs_attachment(ctx, &rt);
3261 layout = zink_render_pass_attachment_get_barrier_info(&rt, i < ctx->fb_state.nr_cbufs, &pipeline, &access);
3262 /* avoid unnecessary read-only layout change */
3263 if (layout == VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL &&
3264 res->layout == VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL &&
3265 !res->bind_count[0])
3266 layout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
3267 }
3268 }
3269 struct zink_screen *screen = zink_screen(ctx->base.screen);
3270 /*
3271 The image subresources for a storage image must be in the VK_IMAGE_LAYOUT_SHARED_PRESENT_KHR or
3272 VK_IMAGE_LAYOUT_GENERAL layout in order to access its data in a shader.
3273 - 14.1.1. Storage Image
3274 */
3275 if (res->image_bind_count[0])
3276 layout = VK_IMAGE_LAYOUT_GENERAL;
3277 else if (!screen->info.have_EXT_attachment_feedback_loop_layout &&
3278 layout == VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT)
3279 layout = VK_IMAGE_LAYOUT_GENERAL;
3280 /* some drivers don't care about zs layouts for attachments, so this saves some layout transition cycles */
3281 else if (layout != VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT &&
3282 i >= ctx->fb_state.nr_cbufs && screen->driver_workarounds.general_depth_layout)
3283 layout = VK_IMAGE_LAYOUT_GENERAL;
3284 if (res->valid || res->layout != layout)
3285 screen->image_barrier(ctx, res, layout, access, pipeline);
3286 if (!(res->aspect & VK_IMAGE_ASPECT_COLOR_BIT))
3287 ctx->zsbuf_readonly = res->layout == VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL;
3288 res->obj->unordered_read = res->obj->unordered_write = false;
3289 if (i == ctx->fb_state.nr_cbufs && res->sampler_bind_count[0])
3290 update_res_sampler_layouts(ctx, res);
3291 return surf->image_view;
3292 }
3293
3294 static uint32_t
hash_rendering_state(const void * key)3295 hash_rendering_state(const void *key)
3296 {
3297 const VkPipelineRenderingCreateInfo *info = key;
3298 uint32_t hash = 0;
3299 /*
3300 uint32_t viewMask;
3301 uint32_t colorAttachmentCount;
3302 const VkFormat* pColorAttachmentFormats;
3303 VkFormat depthAttachmentFormat;
3304 VkFormat stencilAttachmentFormat;
3305 * this data is not optimally arranged, so it must be manually hashed
3306 */
3307 hash = XXH32(&info->colorAttachmentCount, sizeof(uint32_t), hash);
3308 hash = XXH32(&info->viewMask, sizeof(uint32_t), hash);
3309 hash = XXH32(&info->depthAttachmentFormat, sizeof(uint32_t), hash);
3310 hash = XXH32(&info->stencilAttachmentFormat, sizeof(VkFormat), hash);
3311 return XXH32(info->pColorAttachmentFormats, sizeof(VkFormat) * info->colorAttachmentCount, hash);
3312 }
3313
3314 static bool
equals_rendering_state(const void * a,const void * b)3315 equals_rendering_state(const void *a, const void *b)
3316 {
3317 const VkPipelineRenderingCreateInfo *ai = a;
3318 const VkPipelineRenderingCreateInfo *bi = b;
3319 return ai->colorAttachmentCount == bi->colorAttachmentCount &&
3320 ai->depthAttachmentFormat == bi->depthAttachmentFormat &&
3321 ai->viewMask == bi->viewMask &&
3322 ai->stencilAttachmentFormat == bi->stencilAttachmentFormat &&
3323 !memcmp(ai->pColorAttachmentFormats, bi->pColorAttachmentFormats, sizeof(VkFormat) * ai->colorAttachmentCount);
3324 }
3325
3326 static uint32_t
hash_framebuffer_imageless(const void * key)3327 hash_framebuffer_imageless(const void *key)
3328 {
3329 struct zink_framebuffer_state* s = (struct zink_framebuffer_state*)key;
3330 return _mesa_hash_data(key, offsetof(struct zink_framebuffer_state, infos) + sizeof(s->infos[0]) * s->num_attachments);
3331 }
3332
3333 static bool
equals_framebuffer_imageless(const void * a,const void * b)3334 equals_framebuffer_imageless(const void *a, const void *b)
3335 {
3336 struct zink_framebuffer_state *s = (struct zink_framebuffer_state*)a;
3337 return memcmp(a, b, offsetof(struct zink_framebuffer_state, infos) + sizeof(s->infos[0]) * s->num_attachments) == 0;
3338 }
3339
3340 void
zink_init_vk_sample_locations(struct zink_context * ctx,VkSampleLocationsInfoEXT * loc)3341 zink_init_vk_sample_locations(struct zink_context *ctx, VkSampleLocationsInfoEXT *loc)
3342 {
3343 struct zink_screen *screen = zink_screen(ctx->base.screen);
3344 unsigned idx = util_logbase2_ceil(MAX2(ctx->gfx_pipeline_state.rast_samples + 1, 1));
3345 loc->sType = VK_STRUCTURE_TYPE_SAMPLE_LOCATIONS_INFO_EXT;
3346 loc->pNext = NULL;
3347 loc->sampleLocationsPerPixel = 1 << idx;
3348 loc->sampleLocationsCount = ctx->gfx_pipeline_state.rast_samples + 1;
3349 loc->sampleLocationGridSize = screen->maxSampleLocationGridSize[idx];
3350 loc->pSampleLocations = ctx->vk_sample_locations;
3351 }
3352
3353 static void
zink_evaluate_depth_buffer(struct pipe_context * pctx)3354 zink_evaluate_depth_buffer(struct pipe_context *pctx)
3355 {
3356 struct zink_context *ctx = zink_context(pctx);
3357
3358 if (!ctx->fb_state.zsbuf)
3359 return;
3360
3361 struct zink_resource *res = zink_resource(ctx->fb_state.zsbuf->texture);
3362 res->obj->needs_zs_evaluate = true;
3363 zink_init_vk_sample_locations(ctx, &res->obj->zs_evaluate);
3364 zink_batch_no_rp(ctx);
3365 }
3366
3367 static void
sync_flush(struct zink_context * ctx,struct zink_batch_state * bs)3368 sync_flush(struct zink_context *ctx, struct zink_batch_state *bs)
3369 {
3370 if (zink_screen(ctx->base.screen)->threaded_submit)
3371 util_queue_fence_wait(&bs->flush_completed);
3372 }
3373
3374 static inline VkAccessFlags
get_access_flags_for_binding(struct zink_context * ctx,enum zink_descriptor_type type,gl_shader_stage stage,unsigned idx)3375 get_access_flags_for_binding(struct zink_context *ctx, enum zink_descriptor_type type, gl_shader_stage stage, unsigned idx)
3376 {
3377 VkAccessFlags flags = 0;
3378 switch (type) {
3379 case ZINK_DESCRIPTOR_TYPE_UBO:
3380 return VK_ACCESS_UNIFORM_READ_BIT;
3381 case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW:
3382 return VK_ACCESS_SHADER_READ_BIT;
3383 case ZINK_DESCRIPTOR_TYPE_SSBO: {
3384 flags = VK_ACCESS_SHADER_READ_BIT;
3385 if (ctx->writable_ssbos[stage] & (1 << idx))
3386 flags |= VK_ACCESS_SHADER_WRITE_BIT;
3387 return flags;
3388 }
3389 case ZINK_DESCRIPTOR_TYPE_IMAGE: {
3390 struct zink_image_view *image_view = &ctx->image_views[stage][idx];
3391 if (image_view->base.access & PIPE_IMAGE_ACCESS_READ)
3392 flags |= VK_ACCESS_SHADER_READ_BIT;
3393 if (image_view->base.access & PIPE_IMAGE_ACCESS_WRITE)
3394 flags |= VK_ACCESS_SHADER_WRITE_BIT;
3395 return flags;
3396 }
3397 default:
3398 break;
3399 }
3400 unreachable("ACK");
3401 return 0;
3402 }
3403
3404 static void
update_resource_refs_for_stage(struct zink_context * ctx,gl_shader_stage stage)3405 update_resource_refs_for_stage(struct zink_context *ctx, gl_shader_stage stage)
3406 {
3407 unsigned max_slot[] = {
3408 [ZINK_DESCRIPTOR_TYPE_UBO] = ctx->di.num_ubos[stage],
3409 [ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW] = ctx->di.num_samplers[stage],
3410 [ZINK_DESCRIPTOR_TYPE_SSBO] = ctx->di.num_ssbos[stage],
3411 [ZINK_DESCRIPTOR_TYPE_IMAGE] = ctx->di.num_images[stage]
3412 };
3413 for (unsigned i = 0; i < ZINK_DESCRIPTOR_BASE_TYPES; i++) {
3414 for (unsigned j = 0; j < max_slot[i]; j++) {
3415 if (ctx->di.descriptor_res[i][stage][j]) {
3416 struct zink_resource *res = ctx->di.descriptor_res[i][stage][j];
3417 if (!res)
3418 continue;
3419 bool is_buffer = res->obj->is_buffer;
3420 bool is_write = zink_resource_access_is_write(get_access_flags_for_binding(ctx, i, stage, j));
3421 if (zink_is_swapchain(res)) {
3422 if (!zink_kopper_acquire(ctx, res, UINT64_MAX))
3423 /* technically this is a failure condition, but there's no safe way out */
3424 continue;
3425 }
3426 zink_batch_resource_usage_set(ctx->bs, res, is_write, is_buffer);
3427 if (!ctx->unordered_blitting) {
3428 if (is_write || !res->obj->is_buffer)
3429 res->obj->unordered_read = res->obj->unordered_write = false;
3430 else
3431 res->obj->unordered_read = false;
3432 }
3433 }
3434 }
3435 }
3436 }
3437
3438 void
zink_update_descriptor_refs(struct zink_context * ctx,bool compute)3439 zink_update_descriptor_refs(struct zink_context *ctx, bool compute)
3440 {
3441 if (compute) {
3442 update_resource_refs_for_stage(ctx, MESA_SHADER_COMPUTE);
3443 if (ctx->curr_compute)
3444 zink_batch_reference_program(ctx, &ctx->curr_compute->base);
3445 } else {
3446 for (unsigned i = 0; i < ZINK_GFX_SHADER_COUNT; i++)
3447 update_resource_refs_for_stage(ctx, i);
3448 unsigned vertex_buffers_enabled_mask = ctx->gfx_pipeline_state.vertex_buffers_enabled_mask;
3449 unsigned last_vbo = util_last_bit(vertex_buffers_enabled_mask);
3450 for (unsigned i = 0; i < last_vbo + 1; i++) {
3451 struct zink_resource *res = zink_resource(ctx->vertex_buffers[i].buffer.resource);
3452 if (res) {
3453 zink_batch_resource_usage_set(ctx->bs, res, false, true);
3454 if (!ctx->unordered_blitting)
3455 res->obj->unordered_read = false;
3456 }
3457 }
3458 if (ctx->curr_program)
3459 zink_batch_reference_program(ctx, &ctx->curr_program->base);
3460 }
3461 if (ctx->di.bindless_refs_dirty) {
3462 ctx->di.bindless_refs_dirty = false;
3463 for (unsigned i = 0; i < 2; i++) {
3464 util_dynarray_foreach(&ctx->di.bindless[i].resident, struct zink_bindless_descriptor*, bd) {
3465 struct zink_resource *res = zink_descriptor_surface_resource(&(*bd)->ds);
3466 zink_batch_resource_usage_set(ctx->bs, res, (*bd)->access & PIPE_IMAGE_ACCESS_WRITE, res->obj->is_buffer);
3467 if (!ctx->unordered_blitting) {
3468 if ((*bd)->access & PIPE_IMAGE_ACCESS_WRITE || !res->obj->is_buffer)
3469 res->obj->unordered_read = res->obj->unordered_write = false;
3470 else
3471 res->obj->unordered_read = false;
3472 }
3473 }
3474 }
3475 }
3476
3477 unsigned global_count = util_dynarray_num_elements(&ctx->di.global_bindings, struct zink_resource*);
3478 struct zink_resource **globals = ctx->di.global_bindings.data;
3479 for (unsigned i = 0; i < global_count; i++) {
3480 struct zink_resource *res = globals[i];
3481 if (!res)
3482 continue;
3483 zink_batch_resource_usage_set(ctx->bs, res, true, true);
3484 res->obj->unordered_read = res->obj->unordered_write = false;
3485 }
3486 }
3487
3488 static void
reapply_color_write(struct zink_context * ctx)3489 reapply_color_write(struct zink_context *ctx)
3490 {
3491 struct zink_screen *screen = zink_screen(ctx->base.screen);
3492 assert(screen->info.have_EXT_color_write_enable);
3493 const VkBool32 enables[PIPE_MAX_COLOR_BUFS] = {1, 1, 1, 1, 1, 1, 1, 1};
3494 const VkBool32 disables[PIPE_MAX_COLOR_BUFS] = {0};
3495 const unsigned max_att = MIN2(PIPE_MAX_COLOR_BUFS, screen->info.props.limits.maxColorAttachments);
3496 VKCTX(CmdSetColorWriteEnableEXT)(ctx->bs->cmdbuf, max_att, ctx->disable_color_writes ? disables : enables);
3497 VKCTX(CmdSetColorWriteEnableEXT)(ctx->bs->reordered_cmdbuf, max_att, enables);
3498 assert(screen->info.have_EXT_extended_dynamic_state);
3499 if (ctx->dsa_state)
3500 VKCTX(CmdSetDepthWriteEnable)(ctx->bs->cmdbuf, ctx->disable_color_writes ? VK_FALSE : ctx->dsa_state->hw_state.depth_write);
3501 }
3502
3503 static void
stall(struct zink_context * ctx)3504 stall(struct zink_context *ctx)
3505 {
3506 struct zink_screen *screen = zink_screen(ctx->base.screen);
3507 sync_flush(ctx, ctx->last_batch_state);
3508 zink_screen_timeline_wait(screen, ctx->last_batch_state->fence.batch_id, OS_TIMEOUT_INFINITE);
3509 zink_batch_reset_all(ctx);
3510 }
3511
3512 void
zink_reset_ds3_states(struct zink_context * ctx)3513 zink_reset_ds3_states(struct zink_context *ctx)
3514 {
3515 struct zink_screen *screen = zink_screen(ctx->base.screen);
3516 if (!screen->info.have_EXT_extended_dynamic_state3)
3517 return;
3518 if (screen->have_full_ds3)
3519 ctx->ds3_states = UINT32_MAX;
3520 else
3521 ctx->ds3_states = BITFIELD_MASK(ZINK_DS3_BLEND_A2C);
3522 if (!screen->info.dynamic_state3_feats.extendedDynamicState3AlphaToOneEnable)
3523 ctx->ds3_states &= ~BITFIELD_BIT(ZINK_DS3_BLEND_A21);
3524 if (!screen->info.dynamic_state3_feats.extendedDynamicState3LineStippleEnable)
3525 ctx->ds3_states &= ~BITFIELD_BIT(ZINK_DS3_RAST_STIPPLE_ON);
3526 if (screen->driver_workarounds.no_linestipple)
3527 ctx->ds3_states &= ~BITFIELD_BIT(ZINK_DS3_RAST_STIPPLE);
3528 }
3529
3530 static void
flush_batch(struct zink_context * ctx,bool sync)3531 flush_batch(struct zink_context *ctx, bool sync)
3532 {
3533 assert(!ctx->unordered_blitting);
3534 if (ctx->clears_enabled)
3535 /* start rp to do all the clears */
3536 zink_batch_rp(ctx);
3537 zink_batch_no_rp_safe(ctx);
3538
3539 util_queue_fence_wait(&ctx->unsync_fence);
3540 util_queue_fence_reset(&ctx->flush_fence);
3541 zink_end_batch(ctx);
3542 ctx->deferred_fence = NULL;
3543
3544 if (sync)
3545 sync_flush(ctx, ctx->bs);
3546
3547 if (ctx->bs->is_device_lost) {
3548 check_device_lost(ctx);
3549 } else {
3550 struct zink_screen *screen = zink_screen(ctx->base.screen);
3551 zink_start_batch(ctx);
3552 if (screen->info.have_EXT_transform_feedback && ctx->num_so_targets)
3553 ctx->dirty_so_targets = true;
3554 ctx->pipeline_changed[0] = ctx->pipeline_changed[1] = true;
3555 zink_select_draw_vbo(ctx);
3556 zink_select_launch_grid(ctx);
3557
3558 if (ctx->oom_stall)
3559 stall(ctx);
3560 zink_reset_ds3_states(ctx);
3561
3562 ctx->oom_flush = false;
3563 ctx->oom_stall = false;
3564 ctx->dd.bindless_bound = false;
3565 ctx->di.bindless_refs_dirty = true;
3566 ctx->sample_locations_changed = ctx->gfx_pipeline_state.sample_locations_enabled;
3567 if (zink_screen(ctx->base.screen)->info.dynamic_state2_feats.extendedDynamicState2PatchControlPoints) {
3568 VKCTX(CmdSetPatchControlPointsEXT)(ctx->bs->cmdbuf, ctx->gfx_pipeline_state.dyn_state2.vertices_per_patch);
3569 VKCTX(CmdSetPatchControlPointsEXT)(ctx->bs->reordered_cmdbuf, 1);
3570 }
3571 update_feedback_loop_dynamic_state(ctx);
3572 if (screen->info.have_EXT_color_write_enable)
3573 reapply_color_write(ctx);
3574 update_layered_rendering_state(ctx);
3575 tc_renderpass_info_reset(&ctx->dynamic_fb.tc_info);
3576 ctx->rp_tc_info_updated = true;
3577 }
3578 util_queue_fence_signal(&ctx->flush_fence);
3579 }
3580
3581 void
zink_flush_queue(struct zink_context * ctx)3582 zink_flush_queue(struct zink_context *ctx)
3583 {
3584 flush_batch(ctx, true);
3585 }
3586
3587 static bool
rebind_fb_surface(struct zink_context * ctx,struct pipe_surface ** surf,struct zink_resource * match_res)3588 rebind_fb_surface(struct zink_context *ctx, struct pipe_surface **surf, struct zink_resource *match_res)
3589 {
3590 if (!*surf)
3591 return false;
3592 struct zink_resource *surf_res = zink_resource((*surf)->texture);
3593 if ((match_res == surf_res) || surf_res->obj != zink_csurface(*surf)->obj)
3594 return zink_rebind_ctx_surface(ctx, surf);
3595 return false;
3596 }
3597
3598 static bool
rebind_fb_state(struct zink_context * ctx,struct zink_resource * match_res,bool from_set_fb)3599 rebind_fb_state(struct zink_context *ctx, struct zink_resource *match_res, bool from_set_fb)
3600 {
3601 bool rebind = false;
3602 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++)
3603 rebind |= rebind_fb_surface(ctx, &ctx->fb_state.cbufs[i], match_res);
3604 rebind |= rebind_fb_surface(ctx, &ctx->fb_state.zsbuf, match_res);
3605 return rebind;
3606 }
3607
3608 static void
unbind_fb_surface(struct zink_context * ctx,struct pipe_surface * surf,unsigned idx,bool changed)3609 unbind_fb_surface(struct zink_context *ctx, struct pipe_surface *surf, unsigned idx, bool changed)
3610 {
3611 ctx->dynamic_fb.attachments[idx].imageView = VK_NULL_HANDLE;
3612 if (!surf)
3613 return;
3614 struct zink_resource *res = zink_resource(surf->texture);
3615 if (changed) {
3616 ctx->rp_changed = true;
3617 }
3618 res->fb_bind_count--;
3619 if (!res->fb_bind_count && !res->bind_count[0])
3620 _mesa_set_remove_key(ctx->need_barriers[0], res);
3621 unsigned feedback_loops = ctx->feedback_loops;
3622 if (ctx->feedback_loops & BITFIELD_BIT(idx)) {
3623 ctx->dynamic_fb.attachments[idx].imageLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
3624 ctx->rp_layout_changed = true;
3625 }
3626 ctx->feedback_loops &= ~BITFIELD_BIT(idx);
3627 if (feedback_loops != ctx->feedback_loops) {
3628 if (idx == PIPE_MAX_COLOR_BUFS && !zink_screen(ctx->base.screen)->driver_workarounds.always_feedback_loop_zs) {
3629 if (ctx->gfx_pipeline_state.feedback_loop_zs)
3630 ctx->gfx_pipeline_state.dirty = true;
3631 ctx->gfx_pipeline_state.feedback_loop_zs = false;
3632 } else if (idx < PIPE_MAX_COLOR_BUFS && !zink_screen(ctx->base.screen)->driver_workarounds.always_feedback_loop) {
3633 if (ctx->gfx_pipeline_state.feedback_loop)
3634 ctx->gfx_pipeline_state.dirty = true;
3635 ctx->gfx_pipeline_state.feedback_loop = false;
3636 }
3637 }
3638 res->fb_binds &= ~BITFIELD_BIT(idx);
3639 /* this is called just before the resource loses a reference, so a refcount==1 means the resource will be destroyed */
3640 if (!res->fb_bind_count && res->base.b.reference.count > 1) {
3641 if (ctx->track_renderpasses && !ctx->blitting) {
3642 if (!(res->base.b.bind & PIPE_BIND_DISPLAY_TARGET) && util_format_is_depth_or_stencil(surf->format))
3643 /* assume that all depth buffers which are not swapchain images will be used for sampling to avoid splitting renderpasses */
3644 zink_screen(ctx->base.screen)->image_barrier(ctx, res, VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL, VK_ACCESS_SHADER_READ_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT);
3645 if (!zink_is_swapchain(res) && !util_format_is_depth_or_stencil(surf->format))
3646 /* assume that all color buffers which are not swapchain images will be used for sampling to avoid splitting renderpasses */
3647 zink_screen(ctx->base.screen)->image_barrier(ctx, res, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, VK_ACCESS_SHADER_READ_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT);
3648 }
3649 if (res->sampler_bind_count[0]) {
3650 update_res_sampler_layouts(ctx, res);
3651 if (res->layout != VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL && !ctx->blitting)
3652 _mesa_set_add(ctx->need_barriers[0], res);
3653 }
3654 }
3655 }
3656
3657 void
zink_set_null_fs(struct zink_context * ctx)3658 zink_set_null_fs(struct zink_context *ctx)
3659 {
3660 struct zink_screen *screen = zink_screen(ctx->base.screen);
3661 bool prev_disable_fs = ctx->disable_fs;
3662 ctx->disable_fs = ctx->rast_state && ctx->rast_state->base.rasterizer_discard &&
3663 (ctx->primitives_generated_active || (!ctx->queries_disabled && ctx->primitives_generated_suspended));
3664 struct zink_shader *zs = ctx->gfx_stages[MESA_SHADER_FRAGMENT];
3665 unsigned compact = screen->compact_descriptors ? ZINK_DESCRIPTOR_COMPACT : 0;
3666 /* can't use CWE if side effects */
3667 bool no_cwe = (zs && (zs->ssbos_used || zs->bindless || zs->num_bindings[ZINK_DESCRIPTOR_TYPE_IMAGE - compact])) ||
3668 ctx->fs_query_active || ctx->occlusion_query_active || !screen->info.have_EXT_color_write_enable;
3669 bool prev_disable_color_writes = ctx->disable_color_writes;
3670 ctx->disable_color_writes = ctx->disable_fs && !no_cwe;
3671
3672 if (ctx->disable_fs == prev_disable_fs) {
3673 /* if this is a true no-op then return */
3674 if (!ctx->disable_fs || ctx->disable_color_writes == !no_cwe)
3675 return;
3676 /* else changing disable modes */
3677 }
3678
3679 /* either of these cases requires removing the previous mode */
3680 if (!ctx->disable_fs || (prev_disable_fs && prev_disable_color_writes != !no_cwe)) {
3681 if (prev_disable_color_writes)
3682 reapply_color_write(ctx);
3683 else
3684 ctx->base.bind_fs_state(&ctx->base, ctx->saved_fs);
3685 ctx->saved_fs = NULL;
3686 /* fs/CWE reenabled, fs active, done */
3687 if (!ctx->disable_fs)
3688 return;
3689 }
3690
3691 /* always use CWE when possible */
3692 if (!no_cwe) {
3693 reapply_color_write(ctx);
3694 return;
3695 }
3696 /* otherwise need to bind a null fs */
3697 if (!ctx->null_fs) {
3698 nir_shader *nir = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, &screen->nir_options, "null_fs").shader;
3699 nir->info.separate_shader = true;
3700 ctx->null_fs = pipe_shader_from_nir(&ctx->base, nir);
3701 }
3702 ctx->saved_fs = ctx->gfx_stages[MESA_SHADER_FRAGMENT];
3703 ctx->base.bind_fs_state(&ctx->base, ctx->null_fs);
3704 }
3705
3706 static void
check_framebuffer_surface_mutable(struct pipe_context * pctx,struct pipe_surface * psurf)3707 check_framebuffer_surface_mutable(struct pipe_context *pctx, struct pipe_surface *psurf)
3708 {
3709 struct zink_context *ctx = zink_context(pctx);
3710 struct zink_ctx_surface *csurf = (struct zink_ctx_surface *)psurf;
3711 if (!csurf->needs_mutable)
3712 return;
3713 zink_resource_object_init_mutable(ctx, zink_resource(psurf->texture));
3714 struct pipe_surface *psurf2 = pctx->create_surface(pctx, psurf->texture, psurf);
3715 pipe_resource_reference(&psurf2->texture, NULL);
3716 struct zink_ctx_surface *csurf2 = (struct zink_ctx_surface *)psurf2;
3717 zink_surface_reference(zink_screen(pctx->screen), &csurf->surf, csurf2->surf);
3718 pctx->surface_destroy(pctx, psurf2);
3719 csurf->needs_mutable = false;
3720 }
3721
3722 static void
zink_set_framebuffer_state(struct pipe_context * pctx,const struct pipe_framebuffer_state * state)3723 zink_set_framebuffer_state(struct pipe_context *pctx,
3724 const struct pipe_framebuffer_state *state)
3725 {
3726 struct zink_context *ctx = zink_context(pctx);
3727 struct zink_screen *screen = zink_screen(pctx->screen);
3728 unsigned samples = state->nr_cbufs || state->zsbuf ? 0 : state->samples;
3729 unsigned w = ctx->fb_state.width;
3730 unsigned h = ctx->fb_state.height;
3731 unsigned layers = MAX2(zink_framebuffer_get_num_layers(state), 1);
3732
3733 bool flush_clears = ctx->clears_enabled &&
3734 (ctx->dynamic_fb.info.layerCount != layers ||
3735 state->width != w || state->height != h);
3736 if (ctx->clears_enabled && !flush_clears) {
3737 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
3738 if (i >= state->nr_cbufs || !ctx->fb_state.cbufs[i] || !state->cbufs[i])
3739 flush_clears |= zink_fb_clear_enabled(ctx, i);
3740 else if (zink_fb_clear_enabled(ctx, i) && ctx->fb_state.cbufs[i] != state->cbufs[i]) {
3741 struct zink_surface *a = zink_csurface(ctx->fb_state.cbufs[i]);
3742 struct zink_surface *b = zink_csurface(state->cbufs[i]);
3743 if (a == b)
3744 continue;
3745 if (!a || !b || memcmp(&a->base.u.tex, &b->base.u.tex, sizeof(b->base.u.tex)) ||
3746 a->base.texture != b->base.texture)
3747 flush_clears = true;
3748 else if (a->base.format != b->base.format)
3749 zink_fb_clear_rewrite(ctx, i, a->base.format, b->base.format);
3750 }
3751 }
3752 }
3753 if (ctx->fb_state.zsbuf != state->zsbuf)
3754 flush_clears |= zink_fb_clear_enabled(ctx, PIPE_MAX_COLOR_BUFS);
3755 if (flush_clears) {
3756 bool queries_disabled = ctx->queries_disabled;
3757 ctx->queries_disabled = true;
3758 zink_batch_rp(ctx);
3759 ctx->queries_disabled = queries_disabled;
3760 }
3761 /* need to ensure we start a new rp on next draw */
3762 zink_batch_no_rp_safe(ctx);
3763 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
3764 struct pipe_surface *psurf = ctx->fb_state.cbufs[i];
3765 if (i < state->nr_cbufs)
3766 ctx->rp_changed |= !!zink_transient_surface(psurf) != !!zink_transient_surface(state->cbufs[i]);
3767 unbind_fb_surface(ctx, psurf, i, i >= state->nr_cbufs || psurf != state->cbufs[i]);
3768 if (psurf && ctx->needs_present == zink_resource(psurf->texture))
3769 ctx->needs_present = NULL;
3770 }
3771 if (ctx->fb_state.zsbuf) {
3772 struct pipe_surface *psurf = ctx->fb_state.zsbuf;
3773 struct zink_resource *res = zink_resource(psurf->texture);
3774 bool changed = psurf != state->zsbuf;
3775 unbind_fb_surface(ctx, psurf, PIPE_MAX_COLOR_BUFS, changed);
3776 if (!changed)
3777 ctx->rp_changed |= !!zink_transient_surface(psurf) != !!zink_transient_surface(state->zsbuf);
3778 if (changed && unlikely(res->obj->needs_zs_evaluate))
3779 /* have to flush zs eval while the sample location data still exists,
3780 * so just throw some random barrier */
3781 zink_screen(ctx->base.screen)->image_barrier(ctx, res, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
3782 VK_ACCESS_SHADER_READ_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT);
3783 }
3784 /* renderpass changes if the number or types of attachments change */
3785 ctx->rp_changed |= ctx->fb_state.nr_cbufs != state->nr_cbufs;
3786 ctx->rp_changed |= !!ctx->fb_state.zsbuf != !!state->zsbuf;
3787 if (ctx->tc && screen->driver_workarounds.track_renderpasses)
3788 ctx->rp_changed |= ctx->fb_state.resolve != state->resolve;
3789 if (ctx->fb_state.nr_cbufs != state->nr_cbufs) {
3790 ctx->blend_state_changed |= screen->have_full_ds3;
3791 if (state->nr_cbufs && screen->have_full_ds3)
3792 ctx->ds3_states |= BITFIELD_BIT(ZINK_DS3_BLEND_ON) | BITFIELD_BIT(ZINK_DS3_BLEND_WRITE) | BITFIELD_BIT(ZINK_DS3_BLEND_EQ);
3793 }
3794
3795 util_copy_framebuffer_state(&ctx->fb_state, state);
3796 ctx->rp_changed |= zink_update_fbfetch(ctx);
3797 ctx->transient_attachments = 0;
3798 ctx->fb_layer_mismatch = 0;
3799
3800 ctx->dynamic_fb.info.renderArea.offset.x = 0;
3801 ctx->dynamic_fb.info.renderArea.offset.y = 0;
3802 ctx->dynamic_fb.info.renderArea.extent.width = state->width;
3803 ctx->dynamic_fb.info.renderArea.extent.height = state->height;
3804 ctx->dynamic_fb.info.colorAttachmentCount = ctx->fb_state.nr_cbufs;
3805 ctx->rp_changed |= ctx->dynamic_fb.info.layerCount != layers;
3806 ctx->dynamic_fb.info.layerCount = layers;
3807 ctx->rp_changed |= ctx->dynamic_fb.info.viewMask != state->viewmask;
3808 ctx->dynamic_fb.info.viewMask = state->viewmask;
3809 ctx->gfx_pipeline_state.rendering_info.colorAttachmentCount = ctx->fb_state.nr_cbufs;
3810
3811 ctx->void_clears = 0;
3812 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
3813 struct pipe_surface *psurf = ctx->fb_state.cbufs[i];
3814 if (psurf) {
3815 struct zink_surface *transient = zink_transient_surface(psurf);
3816 if (transient || psurf->nr_samples)
3817 ctx->transient_attachments |= BITFIELD_BIT(i);
3818 if (!samples)
3819 samples = MAX3(transient ? transient->base.nr_samples : 1, psurf->texture->nr_samples, psurf->nr_samples ? psurf->nr_samples : 1);
3820 struct zink_resource *res = zink_resource(psurf->texture);
3821 check_framebuffer_surface_mutable(pctx, psurf);
3822 if (zink_csurface(psurf)->info.layerCount > layers)
3823 ctx->fb_layer_mismatch |= BITFIELD_BIT(i);
3824 if (res->obj->dt) {
3825 /* #6274 */
3826 if (!zink_screen(ctx->base.screen)->info.have_KHR_swapchain_mutable_format &&
3827 psurf->format != res->base.b.format) {
3828 static bool warned = false;
3829 if (!warned) {
3830 mesa_loge("zink: SRGB framebuffer unsupported without KHR_swapchain_mutable_format");
3831 warned = true;
3832 }
3833 }
3834 }
3835 res->fb_bind_count++;
3836 res->fb_binds |= BITFIELD_BIT(i);
3837 batch_ref_fb_surface(ctx, ctx->fb_state.cbufs[i]);
3838 if (util_format_has_alpha1(psurf->format)) {
3839 if (!res->valid && !zink_fb_clear_full_exists(ctx, i))
3840 ctx->void_clears |= (PIPE_CLEAR_COLOR0 << i);
3841 }
3842 }
3843 }
3844 unsigned depth_bias_scale_factor = ctx->depth_bias_scale_factor;
3845 if (ctx->fb_state.zsbuf) {
3846 struct pipe_surface *psurf = ctx->fb_state.zsbuf;
3847 struct zink_surface *transient = zink_transient_surface(psurf);
3848 check_framebuffer_surface_mutable(pctx, psurf);
3849 batch_ref_fb_surface(ctx, ctx->fb_state.zsbuf);
3850 if (transient || psurf->nr_samples)
3851 ctx->transient_attachments |= BITFIELD_BIT(PIPE_MAX_COLOR_BUFS);
3852 if (!samples)
3853 samples = MAX3(transient ? transient->base.nr_samples : 1, psurf->texture->nr_samples, psurf->nr_samples ? psurf->nr_samples : 1);
3854 if (zink_csurface(psurf)->info.layerCount > layers)
3855 ctx->fb_layer_mismatch |= BITFIELD_BIT(PIPE_MAX_COLOR_BUFS);
3856 zink_resource(psurf->texture)->fb_bind_count++;
3857 zink_resource(psurf->texture)->fb_binds |= BITFIELD_BIT(PIPE_MAX_COLOR_BUFS);
3858 switch (psurf->format) {
3859 case PIPE_FORMAT_Z16_UNORM:
3860 case PIPE_FORMAT_Z16_UNORM_S8_UINT:
3861 ctx->depth_bias_scale_factor = zink_screen(ctx->base.screen)->driver_workarounds.z16_unscaled_bias;
3862 break;
3863 case PIPE_FORMAT_Z24X8_UNORM:
3864 case PIPE_FORMAT_Z24_UNORM_S8_UINT:
3865 case PIPE_FORMAT_X24S8_UINT:
3866 case PIPE_FORMAT_X8Z24_UNORM:
3867 ctx->depth_bias_scale_factor = zink_screen(ctx->base.screen)->driver_workarounds.z24_unscaled_bias;
3868 break;
3869 case PIPE_FORMAT_Z32_FLOAT:
3870 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
3871 case PIPE_FORMAT_Z32_UNORM:
3872 ctx->depth_bias_scale_factor = 1<<23;
3873 break;
3874 default:
3875 ctx->depth_bias_scale_factor = 0;
3876 }
3877 } else {
3878 ctx->depth_bias_scale_factor = 0;
3879 }
3880
3881 if (ctx->fb_state.resolve) {
3882 struct zink_resource *res = zink_resource(ctx->fb_state.resolve);
3883 if (!res->surface) {
3884 struct pipe_surface tmpl = {0};
3885 tmpl.format = res->base.b.format;
3886 zink_screen_lock_context(screen);
3887 res->surface = screen->copy_context->base.create_surface(&screen->copy_context->base, &res->base.b, &tmpl);
3888 zink_screen_unlock_context(screen);
3889 /* delete extra ref: the resource controls the surface lifetime, not the other way around */
3890 struct pipe_resource *pres = ctx->fb_state.resolve;
3891 pipe_resource_reference(&pres, NULL);
3892 }
3893 }
3894 if (depth_bias_scale_factor != ctx->depth_bias_scale_factor &&
3895 ctx->rast_state && ctx->rast_state->base.offset_units_unscaled)
3896 ctx->rast_state_changed = true;
3897 rebind_fb_state(ctx, NULL, true);
3898 ctx->fb_state.samples = MAX2(samples, 1);
3899 zink_update_framebuffer_state(ctx);
3900 if (ctx->fb_state.width != w || ctx->fb_state.height != h)
3901 ctx->scissor_changed = true;
3902
3903 uint8_t rast_samples = ctx->fb_state.samples - 1;
3904 if (rast_samples != ctx->gfx_pipeline_state.rast_samples)
3905 zink_update_fs_key_samples(ctx);
3906 if (ctx->gfx_pipeline_state.rast_samples != rast_samples) {
3907 ctx->sample_locations_changed |= ctx->gfx_pipeline_state.sample_locations_enabled;
3908 if (screen->have_full_ds3)
3909 ctx->sample_mask_changed = true;
3910 else
3911 ctx->gfx_pipeline_state.dirty = true;
3912 }
3913 ctx->gfx_pipeline_state.rast_samples = rast_samples;
3914
3915 /* this is an ideal time to oom flush since it won't split a renderpass */
3916 if (ctx->oom_flush && !ctx->unordered_blitting)
3917 flush_batch(ctx, false);
3918 else
3919 update_layered_rendering_state(ctx);
3920
3921 ctx->rp_tc_info_updated = !ctx->blitting;
3922 }
3923
3924 static void
zink_set_blend_color(struct pipe_context * pctx,const struct pipe_blend_color * color)3925 zink_set_blend_color(struct pipe_context *pctx,
3926 const struct pipe_blend_color *color)
3927 {
3928 struct zink_context *ctx = zink_context(pctx);
3929 memcpy(ctx->blend_constants, color->color, sizeof(float) * 4);
3930
3931 ctx->blend_color_changed = true;
3932 }
3933
3934 static void
zink_set_sample_mask(struct pipe_context * pctx,unsigned sample_mask)3935 zink_set_sample_mask(struct pipe_context *pctx, unsigned sample_mask)
3936 {
3937 struct zink_context *ctx = zink_context(pctx);
3938 if (ctx->gfx_pipeline_state.sample_mask == sample_mask)
3939 return;
3940 ctx->gfx_pipeline_state.sample_mask = sample_mask;
3941 if (zink_screen(pctx->screen)->have_full_ds3)
3942 ctx->sample_mask_changed = true;
3943 else
3944 ctx->gfx_pipeline_state.dirty = true;
3945 }
3946
3947 static void
zink_set_min_samples(struct pipe_context * pctx,unsigned min_samples)3948 zink_set_min_samples(struct pipe_context *pctx, unsigned min_samples)
3949 {
3950 struct zink_context *ctx = zink_context(pctx);
3951 ctx->gfx_pipeline_state.min_samples = min_samples - 1;
3952 ctx->gfx_pipeline_state.dirty = true;
3953 }
3954
3955 static void
zink_set_sample_locations(struct pipe_context * pctx,size_t size,const uint8_t * locations)3956 zink_set_sample_locations(struct pipe_context *pctx, size_t size, const uint8_t *locations)
3957 {
3958 struct zink_context *ctx = zink_context(pctx);
3959
3960 ctx->gfx_pipeline_state.sample_locations_enabled = size && locations;
3961 ctx->sample_locations_changed = ctx->gfx_pipeline_state.sample_locations_enabled;
3962 if (size > sizeof(ctx->sample_locations))
3963 size = sizeof(ctx->sample_locations);
3964
3965 if (locations)
3966 memcpy(ctx->sample_locations, locations, size);
3967 }
3968
3969 static void
zink_flush(struct pipe_context * pctx,struct pipe_fence_handle ** pfence,unsigned flags)3970 zink_flush(struct pipe_context *pctx,
3971 struct pipe_fence_handle **pfence,
3972 unsigned flags)
3973 {
3974 struct zink_context *ctx = zink_context(pctx);
3975 bool deferred = flags & PIPE_FLUSH_DEFERRED;
3976 bool deferred_fence = false;
3977 struct zink_batch_state *bs = NULL;
3978 struct zink_screen *screen = zink_screen(ctx->base.screen);
3979 VkSemaphore export_sem = VK_NULL_HANDLE;
3980
3981 /* triggering clears will force state->has_work */
3982 if (!deferred && ctx->clears_enabled) {
3983 /* if fbfetch outputs are active, disable them when flushing clears */
3984 unsigned fbfetch_outputs = ctx->fbfetch_outputs;
3985 if (fbfetch_outputs) {
3986 ctx->fbfetch_outputs = 0;
3987 ctx->rp_changed = true;
3988 }
3989 if (ctx->fb_state.zsbuf)
3990 zink_blit_barriers(ctx, NULL, zink_resource(ctx->fb_state.zsbuf->texture), false);
3991
3992 for (unsigned i = 0; i < ctx->fb_state.nr_cbufs; i++) {
3993 if (ctx->fb_state.cbufs[i])
3994 zink_blit_barriers(ctx, NULL, zink_resource(ctx->fb_state.cbufs[i]->texture), false);
3995 }
3996 ctx->blitting = true;
3997 /* start rp to do all the clears */
3998 zink_batch_rp(ctx);
3999 ctx->blitting = false;
4000 ctx->fbfetch_outputs = fbfetch_outputs;
4001 ctx->rp_changed |= fbfetch_outputs > 0;
4002 }
4003
4004 if (flags & PIPE_FLUSH_END_OF_FRAME) {
4005 p_atomic_inc(&screen->renderdoc_frame);
4006 if (ctx->needs_present && ctx->needs_present->obj->dt_idx != UINT32_MAX &&
4007 zink_is_swapchain(ctx->needs_present)) {
4008 zink_kopper_readback_update(ctx, ctx->needs_present);
4009 screen->image_barrier(ctx, ctx->needs_present, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, 0, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT);
4010 }
4011 ctx->needs_present = NULL;
4012 }
4013
4014 if (flags & PIPE_FLUSH_FENCE_FD) {
4015 assert(!deferred && pfence);
4016 const VkExportSemaphoreCreateInfo esci = {
4017 .sType = VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO,
4018 .handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT,
4019 };
4020 const VkSemaphoreCreateInfo sci = {
4021 .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
4022 .pNext = &esci,
4023 };
4024 VkResult result = VKSCR(CreateSemaphore)(screen->dev, &sci, NULL, &export_sem);
4025 if (zink_screen_handle_vkresult(screen, result)) {
4026 assert(!ctx->bs->signal_semaphore);
4027 ctx->bs->signal_semaphore = export_sem;
4028 ctx->bs->has_work = true;
4029 } else {
4030 mesa_loge("ZINK: vkCreateSemaphore failed (%s)", vk_Result_to_str(result));
4031
4032 /* let flush proceed and ensure a null sem for fence_get_fd to return -1 */
4033 export_sem = VK_NULL_HANDLE;
4034 }
4035 }
4036
4037 bool has_work = ctx->bs->has_work | ctx->bs->has_reordered_work | ctx->bs->has_unsync;
4038 if (!has_work) {
4039 if (pfence) {
4040 /* reuse last fence */
4041 bs = ctx->last_batch_state;
4042 }
4043 if (!deferred) {
4044 struct zink_batch_state *last = ctx->last_batch_state;
4045 if (last) {
4046 sync_flush(ctx, last);
4047 if (last->is_device_lost)
4048 check_device_lost(ctx);
4049 }
4050 }
4051 if (ctx->tc && !ctx->track_renderpasses)
4052 tc_driver_internal_flush_notify(ctx->tc);
4053 } else {
4054 bs = ctx->bs;
4055 if (deferred && !(flags & PIPE_FLUSH_FENCE_FD) && pfence)
4056 deferred_fence = true;
4057 else
4058 flush_batch(ctx, true);
4059 }
4060
4061 if (pfence) {
4062 struct zink_tc_fence *mfence;
4063
4064 if (flags & TC_FLUSH_ASYNC) {
4065 mfence = zink_tc_fence(*pfence);
4066 assert(mfence);
4067 } else {
4068 mfence = zink_create_tc_fence();
4069
4070 screen->base.fence_reference(&screen->base, pfence, NULL);
4071 *pfence = (struct pipe_fence_handle *)mfence;
4072 }
4073
4074 assert(!mfence->fence);
4075 mfence->fence = &bs->fence;
4076 mfence->sem = export_sem;
4077 if (bs) {
4078 mfence->submit_count = bs->usage.submit_count;
4079 util_dynarray_append(&bs->fence.mfences, struct zink_tc_fence *, mfence);
4080 }
4081 if (export_sem) {
4082 pipe_reference(NULL, &mfence->reference);
4083 util_dynarray_append(&ctx->bs->fences, struct zink_tc_fence*, mfence);
4084 }
4085
4086 if (deferred_fence) {
4087 assert(bs);
4088 mfence->deferred_ctx = pctx;
4089 assert(!ctx->deferred_fence || ctx->deferred_fence == &bs->fence);
4090 ctx->deferred_fence = &bs->fence;
4091 }
4092
4093 if (!bs || flags & TC_FLUSH_ASYNC) {
4094 if (!util_queue_fence_is_signalled(&mfence->ready))
4095 util_queue_fence_signal(&mfence->ready);
4096 }
4097 }
4098 if (bs) {
4099 if (!(flags & (PIPE_FLUSH_DEFERRED | PIPE_FLUSH_ASYNC)))
4100 sync_flush(ctx, bs);
4101 }
4102 }
4103
4104 void
zink_fence_wait(struct pipe_context * pctx)4105 zink_fence_wait(struct pipe_context *pctx)
4106 {
4107 struct zink_context *ctx = zink_context(pctx);
4108
4109 if (ctx->bs->has_work || ctx->bs->has_reordered_work || ctx->bs->has_unsync)
4110 pctx->flush(pctx, NULL, PIPE_FLUSH_HINT_FINISH);
4111 if (ctx->last_batch_state)
4112 stall(ctx);
4113 }
4114
4115 void
zink_wait_on_batch(struct zink_context * ctx,uint64_t batch_id)4116 zink_wait_on_batch(struct zink_context *ctx, uint64_t batch_id)
4117 {
4118 struct zink_batch_state *bs;
4119 if (!batch_id) {
4120 /* not submitted yet */
4121 flush_batch(ctx, true);
4122 bs = ctx->last_batch_state;
4123 assert(bs);
4124 batch_id = bs->fence.batch_id;
4125 }
4126 assert(batch_id);
4127 if (!zink_screen_timeline_wait(zink_screen(ctx->base.screen), batch_id, UINT64_MAX))
4128 check_device_lost(ctx);
4129 }
4130
4131 bool
zink_check_batch_completion(struct zink_context * ctx,uint64_t batch_id)4132 zink_check_batch_completion(struct zink_context *ctx, uint64_t batch_id)
4133 {
4134 assert(ctx->bs);
4135 if (!batch_id)
4136 /* not submitted yet */
4137 return false;
4138
4139 if (zink_screen_check_last_finished(zink_screen(ctx->base.screen), batch_id))
4140 return true;
4141
4142 bool success = zink_screen_timeline_wait(zink_screen(ctx->base.screen), batch_id, 0);
4143 if (!success)
4144 check_device_lost(ctx);
4145 return success;
4146 }
4147
4148 static void
zink_texture_barrier(struct pipe_context * pctx,unsigned flags)4149 zink_texture_barrier(struct pipe_context *pctx, unsigned flags)
4150 {
4151 struct zink_context *ctx = zink_context(pctx);
4152 VkAccessFlags dst = flags == PIPE_TEXTURE_BARRIER_FRAMEBUFFER ?
4153 VK_ACCESS_INPUT_ATTACHMENT_READ_BIT :
4154 VK_ACCESS_SHADER_READ_BIT;
4155
4156 if (!ctx->framebuffer || !ctx->framebuffer->state.num_attachments)
4157 return;
4158
4159 /* if this is a fb barrier, flush all pending clears */
4160 if (ctx->rp_clears_enabled && dst == VK_ACCESS_INPUT_ATTACHMENT_READ_BIT)
4161 zink_batch_rp(ctx);
4162
4163 /* this is not an in-renderpass barrier */
4164 if (!ctx->fbfetch_outputs)
4165 zink_batch_no_rp(ctx);
4166
4167 if (zink_screen(ctx->base.screen)->info.have_KHR_synchronization2) {
4168 VkDependencyInfo dep = {0};
4169 dep.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO;
4170 dep.dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT;
4171 dep.memoryBarrierCount = 1;
4172
4173 VkMemoryBarrier2 dmb = {0};
4174 dmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2;
4175 dmb.pNext = NULL;
4176 dmb.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
4177 dmb.dstAccessMask = dst;
4178 dmb.srcStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
4179 dmb.dstStageMask = VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT;
4180 dep.pMemoryBarriers = &dmb;
4181
4182 /* if zs fbfetch is a thing?
4183 if (ctx->fb_state.zsbuf) {
4184 const VkPipelineStageFlagBits2 depth_flags = VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT;
4185 dmb.dstAccessMask |= VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT;
4186 dmb.srcStageMask |= depth_flags;
4187 dmb.dstStageMask |= depth_flags;
4188 }
4189 */
4190 VKCTX(CmdPipelineBarrier2)(ctx->bs->cmdbuf, &dep);
4191 } else {
4192 VkMemoryBarrier bmb = {0};
4193 bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
4194 bmb.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
4195 bmb.dstAccessMask = dst;
4196 VKCTX(CmdPipelineBarrier)(
4197 ctx->bs->cmdbuf,
4198 VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
4199 VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
4200 0,
4201 1, &bmb,
4202 0, NULL,
4203 0, NULL
4204 );
4205 }
4206 ctx->bs->has_work = true;
4207 }
4208
4209 static inline void
mem_barrier(struct zink_context * ctx,VkPipelineStageFlags src_stage,VkPipelineStageFlags dst_stage,VkAccessFlags src,VkAccessFlags dst)4210 mem_barrier(struct zink_context *ctx, VkPipelineStageFlags src_stage, VkPipelineStageFlags dst_stage, VkAccessFlags src, VkAccessFlags dst)
4211 {
4212 VkMemoryBarrier mb;
4213 mb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
4214 mb.pNext = NULL;
4215 mb.srcAccessMask = src;
4216 mb.dstAccessMask = dst;
4217 zink_batch_no_rp(ctx);
4218 VKCTX(CmdPipelineBarrier)(ctx->bs->cmdbuf, src_stage, dst_stage, 0, 1, &mb, 0, NULL, 0, NULL);
4219 ctx->bs->has_work = true;
4220 }
4221
4222 void
zink_flush_memory_barrier(struct zink_context * ctx,bool is_compute)4223 zink_flush_memory_barrier(struct zink_context *ctx, bool is_compute)
4224 {
4225 const VkPipelineStageFlags gfx_flags = VK_PIPELINE_STAGE_VERTEX_SHADER_BIT |
4226 VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT |
4227 VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT |
4228 VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT |
4229 VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
4230 const VkPipelineStageFlags cs_flags = VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
4231 VkPipelineStageFlags src = ctx->last_work_was_compute ? cs_flags : gfx_flags;
4232 VkPipelineStageFlags dst = is_compute ? cs_flags : gfx_flags;
4233
4234 if (ctx->memory_barrier & (PIPE_BARRIER_TEXTURE | PIPE_BARRIER_SHADER_BUFFER | PIPE_BARRIER_IMAGE))
4235 mem_barrier(ctx, src, dst, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT);
4236
4237 if (ctx->memory_barrier & PIPE_BARRIER_CONSTANT_BUFFER)
4238 mem_barrier(ctx, src, dst,
4239 VK_ACCESS_SHADER_WRITE_BIT,
4240 VK_ACCESS_UNIFORM_READ_BIT);
4241
4242 if (ctx->memory_barrier & PIPE_BARRIER_INDIRECT_BUFFER)
4243 mem_barrier(ctx, src, VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT,
4244 VK_ACCESS_SHADER_WRITE_BIT,
4245 VK_ACCESS_INDIRECT_COMMAND_READ_BIT);
4246 if (!is_compute) {
4247 if (ctx->memory_barrier & PIPE_BARRIER_VERTEX_BUFFER)
4248 mem_barrier(ctx, gfx_flags, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT,
4249 VK_ACCESS_SHADER_WRITE_BIT,
4250 VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT);
4251
4252 if (ctx->memory_barrier & PIPE_BARRIER_INDEX_BUFFER)
4253 mem_barrier(ctx, gfx_flags, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT,
4254 VK_ACCESS_SHADER_WRITE_BIT,
4255 VK_ACCESS_INDEX_READ_BIT);
4256 if (ctx->memory_barrier & PIPE_BARRIER_FRAMEBUFFER)
4257 zink_texture_barrier(&ctx->base, 0);
4258 if (ctx->memory_barrier & PIPE_BARRIER_STREAMOUT_BUFFER)
4259 mem_barrier(ctx, VK_PIPELINE_STAGE_VERTEX_SHADER_BIT |
4260 VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT |
4261 VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT,
4262 VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT,
4263 VK_ACCESS_SHADER_READ_BIT,
4264 VK_ACCESS_TRANSFORM_FEEDBACK_WRITE_BIT_EXT |
4265 VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_WRITE_BIT_EXT);
4266 }
4267 ctx->memory_barrier = 0;
4268 }
4269
4270 static void
zink_memory_barrier(struct pipe_context * pctx,unsigned flags)4271 zink_memory_barrier(struct pipe_context *pctx, unsigned flags)
4272 {
4273 struct zink_context *ctx = zink_context(pctx);
4274
4275 flags &= ~PIPE_BARRIER_UPDATE;
4276 if (!flags)
4277 return;
4278
4279 if (flags & PIPE_BARRIER_MAPPED_BUFFER) {
4280 /* TODO: this should flush all persistent buffers in use as I think */
4281 flags &= ~PIPE_BARRIER_MAPPED_BUFFER;
4282 }
4283 ctx->memory_barrier = flags;
4284 }
4285
4286 static void
zink_flush_resource(struct pipe_context * pctx,struct pipe_resource * pres)4287 zink_flush_resource(struct pipe_context *pctx,
4288 struct pipe_resource *pres)
4289 {
4290 struct zink_context *ctx = zink_context(pctx);
4291 struct zink_resource *res = zink_resource(pres);
4292 if (res->obj->dt) {
4293 if (zink_kopper_acquired(res->obj->dt, res->obj->dt_idx) && (!ctx->clears_enabled || !res->fb_bind_count)) {
4294 zink_batch_no_rp_safe(ctx);
4295 zink_kopper_readback_update(ctx, res);
4296 zink_screen(ctx->base.screen)->image_barrier(ctx, res, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, 0, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT);
4297 zink_batch_reference_resource_rw(ctx, res, true);
4298 } else {
4299 ctx->needs_present = res;
4300 }
4301 ctx->swapchain = res;
4302 } else if (res->dmabuf)
4303 res->queue = VK_QUEUE_FAMILY_FOREIGN_EXT;
4304 }
4305
4306 static struct pipe_stream_output_target *
zink_create_stream_output_target(struct pipe_context * pctx,struct pipe_resource * pres,unsigned buffer_offset,unsigned buffer_size)4307 zink_create_stream_output_target(struct pipe_context *pctx,
4308 struct pipe_resource *pres,
4309 unsigned buffer_offset,
4310 unsigned buffer_size)
4311 {
4312 struct zink_so_target *t;
4313 t = CALLOC_STRUCT(zink_so_target);
4314 if (!t)
4315 return NULL;
4316
4317 t->counter_buffer = pipe_buffer_create(pctx->screen, PIPE_BIND_STREAM_OUTPUT, PIPE_USAGE_DEFAULT, 4);
4318 if (!t->counter_buffer) {
4319 FREE(t);
4320 return NULL;
4321 }
4322
4323 t->base.reference.count = 1;
4324 t->base.context = pctx;
4325 pipe_resource_reference(&t->base.buffer, pres);
4326 t->base.buffer_offset = buffer_offset;
4327 t->base.buffer_size = buffer_size;
4328
4329 zink_resource(t->base.buffer)->so_valid = true;
4330
4331 return &t->base;
4332 }
4333
4334 static void
zink_stream_output_target_destroy(struct pipe_context * pctx,struct pipe_stream_output_target * psot)4335 zink_stream_output_target_destroy(struct pipe_context *pctx,
4336 struct pipe_stream_output_target *psot)
4337 {
4338 struct zink_so_target *t = (struct zink_so_target *)psot;
4339 pipe_resource_reference(&t->counter_buffer, NULL);
4340 pipe_resource_reference(&t->base.buffer, NULL);
4341 FREE(t);
4342 }
4343
4344 static void
zink_set_stream_output_targets(struct pipe_context * pctx,unsigned num_targets,struct pipe_stream_output_target ** targets,const unsigned * offsets,enum mesa_prim output_prim)4345 zink_set_stream_output_targets(struct pipe_context *pctx,
4346 unsigned num_targets,
4347 struct pipe_stream_output_target **targets,
4348 const unsigned *offsets,
4349 enum mesa_prim output_prim)
4350 {
4351 struct zink_context *ctx = zink_context(pctx);
4352
4353 /* always set counter_buffer_valid=false on unbind:
4354 * - on resume (indicated by offset==-1), set counter_buffer_valid=true
4355 * - otherwise the counter buffer is invalidated
4356 */
4357
4358 if (num_targets == 0) {
4359 for (unsigned i = 0; i < ctx->num_so_targets; i++) {
4360 if (ctx->so_targets[i]) {
4361 struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer);
4362 if (so) {
4363 so->so_bind_count--;
4364 update_res_bind_count(ctx, so, false, true);
4365 }
4366 }
4367 pipe_so_target_reference(&ctx->so_targets[i], NULL);
4368 }
4369 ctx->num_so_targets = 0;
4370 } else {
4371 for (unsigned i = 0; i < num_targets; i++) {
4372 struct zink_so_target *t = zink_so_target(targets[i]);
4373 pipe_so_target_reference(&ctx->so_targets[i], targets[i]);
4374 if (!t)
4375 continue;
4376 if (offsets[0] != (unsigned)-1)
4377 t->counter_buffer_valid = false;
4378 struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer);
4379 if (so) {
4380 so->so_bind_count++;
4381 update_res_bind_count(ctx, so, false, false);
4382 }
4383 }
4384 for (unsigned i = num_targets; i < ctx->num_so_targets; i++) {
4385 if (ctx->so_targets[i]) {
4386 struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer);
4387 if (so) {
4388 so->so_bind_count--;
4389 update_res_bind_count(ctx, so, false, true);
4390 }
4391 }
4392 pipe_so_target_reference(&ctx->so_targets[i], NULL);
4393 }
4394 ctx->num_so_targets = num_targets;
4395
4396 /* TODO: possibly avoid rebinding on resume if resuming from same buffers? */
4397 ctx->dirty_so_targets = true;
4398 }
4399 }
4400
4401 void
zink_rebind_framebuffer(struct zink_context * ctx,struct zink_resource * res)4402 zink_rebind_framebuffer(struct zink_context *ctx, struct zink_resource *res)
4403 {
4404 if (!ctx->framebuffer)
4405 return;
4406 bool did_rebind = false;
4407 if (res->aspect & VK_IMAGE_ASPECT_COLOR_BIT) {
4408 for (unsigned i = 0; i < ctx->fb_state.nr_cbufs; i++) {
4409 if (!ctx->fb_state.cbufs[i] ||
4410 zink_resource(ctx->fb_state.cbufs[i]->texture) != res)
4411 continue;
4412 zink_rebind_ctx_surface(ctx, &ctx->fb_state.cbufs[i]);
4413 did_rebind = true;
4414 }
4415 } else {
4416 if (ctx->fb_state.zsbuf && zink_resource(ctx->fb_state.zsbuf->texture) != res) {
4417 zink_rebind_ctx_surface(ctx, &ctx->fb_state.zsbuf);
4418 did_rebind = true;
4419 }
4420 }
4421
4422 did_rebind |= rebind_fb_state(ctx, res, false);
4423
4424 if (!did_rebind)
4425 return;
4426
4427 zink_batch_no_rp(ctx);
4428 struct zink_framebuffer *fb = zink_get_framebuffer(ctx);
4429 ctx->fb_changed |= ctx->framebuffer != fb;
4430 ctx->framebuffer = fb;
4431 }
4432
4433 ALWAYS_INLINE static struct zink_resource *
rebind_ubo(struct zink_context * ctx,gl_shader_stage shader,unsigned slot)4434 rebind_ubo(struct zink_context *ctx, gl_shader_stage shader, unsigned slot)
4435 {
4436 struct zink_resource *res;
4437 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
4438 res = update_descriptor_state_ubo_db(ctx, shader, slot, ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_UBO][shader][slot]);
4439 } else {
4440 res = update_descriptor_state_ubo_lazy(ctx, shader, slot, ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_UBO][shader][slot]);
4441 }
4442 if (res) {
4443 res->obj->unordered_read = false;
4444 res->obj->access |= VK_ACCESS_SHADER_READ_BIT;
4445 }
4446 ctx->invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_UBO, slot, 1);
4447 return res;
4448 }
4449
4450 ALWAYS_INLINE static struct zink_resource *
rebind_ssbo(struct zink_context * ctx,gl_shader_stage shader,unsigned slot)4451 rebind_ssbo(struct zink_context *ctx, gl_shader_stage shader, unsigned slot)
4452 {
4453 const struct pipe_shader_buffer *ssbo = &ctx->ssbos[shader][slot];
4454 struct zink_resource *res = zink_resource(ssbo->buffer);
4455 if (!res)
4456 return NULL;
4457 util_range_add(&res->base.b, &res->valid_buffer_range, ssbo->buffer_offset,
4458 ssbo->buffer_offset + ssbo->buffer_size);
4459 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
4460 update_descriptor_state_ssbo_db(ctx, shader, slot, res);
4461 } else {
4462 update_descriptor_state_ssbo_lazy(ctx, shader, slot, res);
4463 }
4464 if (res) {
4465 res->obj->unordered_read = false;
4466 res->obj->access |= VK_ACCESS_SHADER_READ_BIT;
4467 if (ctx->writable_ssbos[shader] & BITFIELD_BIT(slot)) {
4468 res->obj->unordered_write = false;
4469 res->obj->access |= VK_ACCESS_SHADER_WRITE_BIT;
4470 }
4471 }
4472 ctx->invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SSBO, slot, 1);
4473 return res;
4474 }
4475
4476 ALWAYS_INLINE static struct zink_resource *
rebind_tbo(struct zink_context * ctx,gl_shader_stage shader,unsigned slot)4477 rebind_tbo(struct zink_context *ctx, gl_shader_stage shader, unsigned slot)
4478 {
4479 struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[shader][slot]);
4480 if (!sampler_view || sampler_view->base.texture->target != PIPE_BUFFER)
4481 return NULL;
4482 struct zink_resource *res = zink_resource(sampler_view->base.texture);
4483 if (zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB) {
4484 VkBufferViewCreateInfo bvci = sampler_view->buffer_view->bvci;
4485 bvci.buffer = res->obj->buffer;
4486 zink_buffer_view_reference(zink_screen(ctx->base.screen), &sampler_view->buffer_view, NULL);
4487 sampler_view->buffer_view = get_buffer_view(ctx, res, &bvci);
4488 }
4489 update_descriptor_state_sampler(ctx, shader, slot, res);
4490 if (res) {
4491 res->obj->unordered_read = false;
4492 res->obj->access |= VK_ACCESS_SHADER_READ_BIT;
4493 }
4494 ctx->invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1);
4495 return res;
4496 }
4497
4498 ALWAYS_INLINE static struct zink_resource *
rebind_ibo(struct zink_context * ctx,gl_shader_stage shader,unsigned slot)4499 rebind_ibo(struct zink_context *ctx, gl_shader_stage shader, unsigned slot)
4500 {
4501 struct zink_image_view *image_view = &ctx->image_views[shader][slot];
4502 struct zink_resource *res = zink_resource(image_view->base.resource);
4503 if (!res || res->base.b.target != PIPE_BUFFER)
4504 return NULL;
4505 VkBufferViewCreateInfo bvci;
4506 if (zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB) {
4507 bvci = image_view->buffer_view->bvci;
4508 bvci.buffer = res->obj->buffer;
4509 zink_buffer_view_reference(zink_screen(ctx->base.screen), &image_view->buffer_view, NULL);
4510 }
4511 if (!zink_resource_object_init_storage(ctx, res)) {
4512 debug_printf("couldn't create storage image!");
4513 return NULL;
4514 }
4515 if (zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB) {
4516 image_view->buffer_view = get_buffer_view(ctx, res, &bvci);
4517 assert(image_view->buffer_view);
4518 }
4519 if (res) {
4520 res->obj->unordered_read = false;
4521 res->obj->access |= VK_ACCESS_SHADER_READ_BIT;
4522 if (image_view->base.access & PIPE_IMAGE_ACCESS_WRITE) {
4523 res->obj->unordered_write = false;
4524 res->obj->access |= VK_ACCESS_SHADER_WRITE_BIT;
4525 }
4526 }
4527 util_range_add(&res->base.b, &res->valid_buffer_range, image_view->base.u.buf.offset,
4528 image_view->base.u.buf.offset + image_view->base.u.buf.size);
4529 update_descriptor_state_image(ctx, shader, slot, res);
4530 ctx->invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_IMAGE, slot, 1);
4531 return res;
4532 }
4533
4534 static unsigned
rebind_buffer(struct zink_context * ctx,struct zink_resource * res,uint32_t rebind_mask,const unsigned expected_num_rebinds)4535 rebind_buffer(struct zink_context *ctx, struct zink_resource *res, uint32_t rebind_mask, const unsigned expected_num_rebinds)
4536 {
4537 unsigned num_rebinds = 0;
4538 bool has_write = false;
4539
4540 if (!zink_resource_has_binds(res))
4541 return 0;
4542
4543 assert(!res->bindless[1]); //TODO
4544 if ((rebind_mask & BITFIELD_BIT(TC_BINDING_STREAMOUT_BUFFER)) || (!rebind_mask && res->so_bind_count && ctx->num_so_targets)) {
4545 for (unsigned i = 0; i < ctx->num_so_targets; i++) {
4546 if (ctx->so_targets[i]) {
4547 struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer);
4548 if (so && so == res) {
4549 ctx->dirty_so_targets = true;
4550 num_rebinds++;
4551 }
4552 }
4553 }
4554 rebind_mask &= ~BITFIELD_BIT(TC_BINDING_STREAMOUT_BUFFER);
4555 }
4556 if (expected_num_rebinds && num_rebinds >= expected_num_rebinds && !rebind_mask)
4557 goto end;
4558
4559 if ((rebind_mask & BITFIELD_BIT(TC_BINDING_VERTEX_BUFFER)) || (!rebind_mask && res->vbo_bind_mask)) {
4560 u_foreach_bit(slot, res->vbo_bind_mask) {
4561 if (ctx->vertex_buffers[slot].buffer.resource != &res->base.b) //wrong context
4562 goto end;
4563 res->obj->access |= VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT;
4564 res->obj->access_stage |= VK_PIPELINE_STAGE_VERTEX_INPUT_BIT;
4565 res->obj->unordered_read = false;
4566 num_rebinds++;
4567 }
4568 rebind_mask &= ~BITFIELD_BIT(TC_BINDING_VERTEX_BUFFER);
4569 ctx->vertex_buffers_dirty = true;
4570 }
4571 if (expected_num_rebinds && num_rebinds >= expected_num_rebinds && !rebind_mask)
4572 goto end;
4573
4574 const uint32_t ubo_mask = rebind_mask ?
4575 rebind_mask & BITFIELD_RANGE(TC_BINDING_UBO_VS, MESA_SHADER_STAGES) :
4576 ((res->ubo_bind_count[0] ? BITFIELD_RANGE(TC_BINDING_UBO_VS, (MESA_SHADER_STAGES - 1)) : 0) |
4577 (res->ubo_bind_count[1] ? BITFIELD_BIT(TC_BINDING_UBO_CS) : 0));
4578 u_foreach_bit(shader, ubo_mask >> TC_BINDING_UBO_VS) {
4579 u_foreach_bit(slot, res->ubo_bind_mask[shader]) {
4580 if (&res->base.b != ctx->ubos[shader][slot].buffer) //wrong context
4581 goto end;
4582 rebind_ubo(ctx, shader, slot);
4583 num_rebinds++;
4584 }
4585 }
4586 rebind_mask &= ~BITFIELD_RANGE(TC_BINDING_UBO_VS, MESA_SHADER_STAGES);
4587 if (expected_num_rebinds && num_rebinds >= expected_num_rebinds && !rebind_mask)
4588 goto end;
4589
4590 const unsigned ssbo_mask = rebind_mask ?
4591 rebind_mask & BITFIELD_RANGE(TC_BINDING_SSBO_VS, MESA_SHADER_STAGES) :
4592 BITFIELD_RANGE(TC_BINDING_SSBO_VS, MESA_SHADER_STAGES);
4593 u_foreach_bit(shader, ssbo_mask >> TC_BINDING_SSBO_VS) {
4594 u_foreach_bit(slot, res->ssbo_bind_mask[shader]) {
4595 struct pipe_shader_buffer *ssbo = &ctx->ssbos[shader][slot];
4596 if (&res->base.b != ssbo->buffer) //wrong context
4597 goto end;
4598 rebind_ssbo(ctx, shader, slot);
4599 has_write |= (ctx->writable_ssbos[shader] & BITFIELD64_BIT(slot)) != 0;
4600 num_rebinds++;
4601 }
4602 }
4603 rebind_mask &= ~BITFIELD_RANGE(TC_BINDING_SSBO_VS, MESA_SHADER_STAGES);
4604 if (expected_num_rebinds && num_rebinds >= expected_num_rebinds && !rebind_mask)
4605 goto end;
4606 const unsigned sampler_mask = rebind_mask ?
4607 rebind_mask & BITFIELD_RANGE(TC_BINDING_SAMPLERVIEW_VS, MESA_SHADER_STAGES) :
4608 BITFIELD_RANGE(TC_BINDING_SAMPLERVIEW_VS, MESA_SHADER_STAGES);
4609 u_foreach_bit(shader, sampler_mask >> TC_BINDING_SAMPLERVIEW_VS) {
4610 u_foreach_bit(slot, res->sampler_binds[shader]) {
4611 struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[shader][slot]);
4612 if (&res->base.b != sampler_view->base.texture) //wrong context
4613 goto end;
4614 rebind_tbo(ctx, shader, slot);
4615 num_rebinds++;
4616 }
4617 }
4618 rebind_mask &= ~BITFIELD_RANGE(TC_BINDING_SAMPLERVIEW_VS, MESA_SHADER_STAGES);
4619 if (expected_num_rebinds && num_rebinds >= expected_num_rebinds && !rebind_mask)
4620 goto end;
4621
4622 const unsigned image_mask = rebind_mask ?
4623 rebind_mask & BITFIELD_RANGE(TC_BINDING_IMAGE_VS, MESA_SHADER_STAGES) :
4624 BITFIELD_RANGE(TC_BINDING_IMAGE_VS, MESA_SHADER_STAGES);
4625 unsigned num_image_rebinds_remaining = rebind_mask ? expected_num_rebinds - num_rebinds : res->image_bind_count[0] + res->image_bind_count[1];
4626 u_foreach_bit(shader, image_mask >> TC_BINDING_IMAGE_VS) {
4627 for (unsigned slot = 0; num_image_rebinds_remaining && slot < ctx->di.num_images[shader]; slot++) {
4628 struct zink_resource *cres = ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_IMAGE][shader][slot];
4629 if (res != cres)
4630 continue;
4631
4632 rebind_ibo(ctx, shader, slot);
4633 const struct zink_image_view *image_view = &ctx->image_views[shader][slot];
4634 has_write |= (image_view->base.access & PIPE_IMAGE_ACCESS_WRITE) != 0;
4635 num_image_rebinds_remaining--;
4636 num_rebinds++;
4637 }
4638 }
4639 end:
4640 if (num_rebinds)
4641 zink_batch_resource_usage_set(ctx->bs, res, has_write, true);
4642 return num_rebinds;
4643 }
4644
4645 void
zink_copy_buffer(struct zink_context * ctx,struct zink_resource * dst,struct zink_resource * src,unsigned dst_offset,unsigned src_offset,unsigned size)4646 zink_copy_buffer(struct zink_context *ctx, struct zink_resource *dst, struct zink_resource *src,
4647 unsigned dst_offset, unsigned src_offset, unsigned size)
4648 {
4649 VkBufferCopy region;
4650 region.srcOffset = src_offset;
4651 region.dstOffset = dst_offset;
4652 region.size = size;
4653
4654 struct pipe_box box;
4655 u_box_3d((int)src_offset, 0, 0, (int)size, 0, 0, &box);
4656 /* must barrier if something wrote the valid buffer range */
4657 bool valid_write = zink_check_valid_buffer_src_access(ctx, src, src_offset, size);
4658 bool unordered_src = !valid_write && !zink_check_unordered_transfer_access(src, 0, &box);
4659 zink_screen(ctx->base.screen)->buffer_barrier(ctx, src, VK_ACCESS_TRANSFER_READ_BIT, 0);
4660 bool unordered_dst = zink_resource_buffer_transfer_dst_barrier(ctx, dst, dst_offset, size);
4661 bool can_unorder = unordered_dst && unordered_src && !ctx->no_reorder;
4662 VkCommandBuffer cmdbuf = can_unorder ? ctx->bs->reordered_cmdbuf : zink_get_cmdbuf(ctx, src, dst);
4663 ctx->bs->has_reordered_work |= can_unorder;
4664 zink_batch_reference_resource_rw(ctx, src, false);
4665 zink_batch_reference_resource_rw(ctx, dst, true);
4666 if (unlikely(zink_debug & ZINK_DEBUG_SYNC)) {
4667 VkMemoryBarrier mb;
4668 mb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
4669 mb.pNext = NULL;
4670 mb.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
4671 mb.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT;
4672 VKCTX(CmdPipelineBarrier)(cmdbuf,
4673 VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
4674 VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
4675 0, 1, &mb, 0, NULL, 0, NULL);
4676 }
4677 bool marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "copy_buffer(%d)", size);
4678 VKCTX(CmdCopyBuffer)(cmdbuf, src->obj->buffer, dst->obj->buffer, 1, ®ion);
4679 zink_cmd_debug_marker_end(ctx, cmdbuf, marker);
4680 }
4681
4682 void
zink_copy_image_buffer(struct zink_context * ctx,struct zink_resource * dst,struct zink_resource * src,unsigned dst_level,unsigned dstx,unsigned dsty,unsigned dstz,unsigned src_level,const struct pipe_box * src_box,enum pipe_map_flags map_flags)4683 zink_copy_image_buffer(struct zink_context *ctx, struct zink_resource *dst, struct zink_resource *src,
4684 unsigned dst_level, unsigned dstx, unsigned dsty, unsigned dstz,
4685 unsigned src_level, const struct pipe_box *src_box, enum pipe_map_flags map_flags)
4686 {
4687 struct zink_resource *img = dst->base.b.target == PIPE_BUFFER ? src : dst;
4688 struct zink_resource *use_img = img;
4689 struct zink_resource *buf = dst->base.b.target == PIPE_BUFFER ? dst : src;
4690 bool needs_present_readback = false;
4691
4692 bool buf2img = buf == src;
4693 bool unsync = !!(map_flags & PIPE_MAP_UNSYNCHRONIZED);
4694 if (unsync) {
4695 util_queue_fence_wait(&ctx->flush_fence);
4696 util_queue_fence_reset(&ctx->unsync_fence);
4697 }
4698
4699 if (buf2img) {
4700 if (zink_is_swapchain(img)) {
4701 if (!zink_kopper_acquire(ctx, img, UINT64_MAX))
4702 return;
4703 }
4704 struct pipe_box box = *src_box;
4705 box.x = dstx;
4706 box.y = dsty;
4707 box.z = dstz;
4708 zink_resource_image_transfer_dst_barrier(ctx, img, dst_level, &box, unsync);
4709 if (!unsync)
4710 zink_screen(ctx->base.screen)->buffer_barrier(ctx, buf, VK_ACCESS_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT);
4711 } else {
4712 assert(!(map_flags & PIPE_MAP_UNSYNCHRONIZED));
4713 if (zink_is_swapchain(img))
4714 needs_present_readback = zink_kopper_acquire_readback(ctx, img, &use_img);
4715 zink_screen(ctx->base.screen)->image_barrier(ctx, use_img, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, 0, 0);
4716 zink_resource_buffer_transfer_dst_barrier(ctx, buf, dstx, src_box->width);
4717 }
4718
4719 VkBufferImageCopy region = {0};
4720 region.bufferOffset = buf2img ? src_box->x : dstx;
4721 region.bufferRowLength = 0;
4722 region.bufferImageHeight = 0;
4723 region.imageSubresource.mipLevel = buf2img ? dst_level : src_level;
4724 enum pipe_texture_target img_target = img->base.b.target;
4725 if (img->need_2D)
4726 img_target = img_target == PIPE_TEXTURE_1D ? PIPE_TEXTURE_2D : PIPE_TEXTURE_2D_ARRAY;
4727 switch (img_target) {
4728 case PIPE_TEXTURE_CUBE:
4729 case PIPE_TEXTURE_CUBE_ARRAY:
4730 case PIPE_TEXTURE_2D_ARRAY:
4731 case PIPE_TEXTURE_1D_ARRAY:
4732 /* these use layer */
4733 region.imageSubresource.baseArrayLayer = buf2img ? dstz : src_box->z;
4734 region.imageSubresource.layerCount = src_box->depth;
4735 region.imageOffset.z = 0;
4736 region.imageExtent.depth = 1;
4737 break;
4738 case PIPE_TEXTURE_3D:
4739 /* this uses depth */
4740 region.imageSubresource.baseArrayLayer = 0;
4741 region.imageSubresource.layerCount = 1;
4742 region.imageOffset.z = buf2img ? dstz : src_box->z;
4743 region.imageExtent.depth = src_box->depth;
4744 break;
4745 default:
4746 /* these must only copy one layer */
4747 region.imageSubresource.baseArrayLayer = 0;
4748 region.imageSubresource.layerCount = 1;
4749 region.imageOffset.z = 0;
4750 region.imageExtent.depth = 1;
4751 }
4752 region.imageOffset.x = buf2img ? dstx : src_box->x;
4753 region.imageOffset.y = buf2img ? dsty : src_box->y;
4754
4755 region.imageExtent.width = src_box->width;
4756 region.imageExtent.height = src_box->height;
4757
4758 VkCommandBuffer cmdbuf = unsync ?
4759 ctx->bs->unsynchronized_cmdbuf :
4760 /* never promote to unordered if swapchain was acquired */
4761 needs_present_readback ?
4762 ctx->bs->cmdbuf :
4763 buf2img ? zink_get_cmdbuf(ctx, buf, use_img) : zink_get_cmdbuf(ctx, use_img, buf);
4764 zink_batch_reference_resource_rw(ctx, use_img, buf2img);
4765 zink_batch_reference_resource_rw(ctx, buf, !buf2img);
4766 if (unsync) {
4767 ctx->bs->has_unsync = true;
4768 use_img->obj->unsync_access = true;
4769 }
4770
4771 /* we're using u_transfer_helper_deinterleave, which means we'll be getting PIPE_MAP_* usage
4772 * to indicate whether to copy either the depth or stencil aspects
4773 */
4774 unsigned aspects = 0;
4775 if (map_flags) {
4776 assert((map_flags & (PIPE_MAP_DEPTH_ONLY | PIPE_MAP_STENCIL_ONLY)) !=
4777 (PIPE_MAP_DEPTH_ONLY | PIPE_MAP_STENCIL_ONLY));
4778 if (map_flags & PIPE_MAP_DEPTH_ONLY)
4779 aspects = VK_IMAGE_ASPECT_DEPTH_BIT;
4780 else if (map_flags & PIPE_MAP_STENCIL_ONLY)
4781 aspects = VK_IMAGE_ASPECT_STENCIL_BIT;
4782 }
4783 if (!aspects)
4784 aspects = img->aspect;
4785 if (unlikely(zink_debug & ZINK_DEBUG_SYNC)) {
4786 VkMemoryBarrier mb;
4787 mb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
4788 mb.pNext = NULL;
4789 mb.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
4790 mb.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT;
4791 VKCTX(CmdPipelineBarrier)(cmdbuf,
4792 VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
4793 VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
4794 0, 1, &mb, 0, NULL, 0, NULL);
4795 }
4796 while (aspects) {
4797 int aspect = 1 << u_bit_scan(&aspects);
4798 region.imageSubresource.aspectMask = aspect;
4799
4800 /* MSAA transfers should have already been handled by U_TRANSFER_HELPER_MSAA_MAP, since
4801 * there's no way to resolve using this interface:
4802 *
4803 * srcImage must have a sample count equal to VK_SAMPLE_COUNT_1_BIT
4804 * - vkCmdCopyImageToBuffer spec
4805 *
4806 * dstImage must have a sample count equal to VK_SAMPLE_COUNT_1_BIT
4807 * - vkCmdCopyBufferToImage spec
4808 */
4809 assert(img->base.b.nr_samples <= 1);
4810 bool marker;
4811 if (buf2img) {
4812 marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "copy_buffer2image(%s, %dx%dx%d)",
4813 util_format_short_name(dst->base.b.format),
4814 region.imageExtent.width,
4815 region.imageExtent.height,
4816 MAX2(region.imageSubresource.layerCount, region.imageExtent.depth));
4817 VKCTX(CmdCopyBufferToImage)(cmdbuf, buf->obj->buffer, use_img->obj->image, use_img->layout, 1, ®ion);
4818 } else {
4819 marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "copy_image2buffer(%s, %dx%dx%d)",
4820 util_format_short_name(src->base.b.format),
4821 region.imageExtent.width,
4822 region.imageExtent.height,
4823 MAX2(region.imageSubresource.layerCount, region.imageExtent.depth));
4824 VKCTX(CmdCopyImageToBuffer)(cmdbuf, use_img->obj->image, use_img->layout, buf->obj->buffer, 1, ®ion);
4825 }
4826 zink_cmd_debug_marker_end(ctx, cmdbuf, marker);
4827 }
4828 if (unsync)
4829 util_queue_fence_signal(&ctx->unsync_fence);
4830 if (needs_present_readback) {
4831 assert(!unsync);
4832 if (buf2img) {
4833 img->obj->unordered_write = false;
4834 buf->obj->unordered_read = false;
4835 } else {
4836 img->obj->unordered_read = false;
4837 buf->obj->unordered_write = false;
4838 }
4839 zink_kopper_present_readback(ctx, img);
4840 }
4841
4842 if (ctx->oom_flush && !ctx->in_rp && !ctx->unordered_blitting)
4843 flush_batch(ctx, false);
4844 }
4845
4846 static void
zink_resource_copy_region(struct pipe_context * pctx,struct pipe_resource * pdst,unsigned dst_level,unsigned dstx,unsigned dsty,unsigned dstz,struct pipe_resource * psrc,unsigned src_level,const struct pipe_box * src_box)4847 zink_resource_copy_region(struct pipe_context *pctx,
4848 struct pipe_resource *pdst,
4849 unsigned dst_level, unsigned dstx, unsigned dsty, unsigned dstz,
4850 struct pipe_resource *psrc,
4851 unsigned src_level, const struct pipe_box *src_box)
4852 {
4853 struct zink_resource *dst = zink_resource(pdst);
4854 struct zink_resource *src = zink_resource(psrc);
4855 struct zink_context *ctx = zink_context(pctx);
4856 if (dst->base.b.target != PIPE_BUFFER && src->base.b.target != PIPE_BUFFER) {
4857 VkImageCopy region;
4858 /* fill struct holes */
4859 memset(®ion, 0, sizeof(region));
4860 if (util_format_get_num_planes(src->base.b.format) == 1 &&
4861 util_format_get_num_planes(dst->base.b.format) == 1) {
4862 /* If neither the calling command’s srcImage nor the calling command’s dstImage
4863 * has a multi-planar image format then the aspectMask member of srcSubresource
4864 * and dstSubresource must match
4865 *
4866 * -VkImageCopy spec
4867 */
4868 assert(src->aspect == dst->aspect);
4869 } else
4870 unreachable("planar formats not yet handled");
4871
4872
4873 region.srcSubresource.aspectMask = src->aspect;
4874 region.srcSubresource.mipLevel = src_level;
4875 enum pipe_texture_target src_target = src->base.b.target;
4876 if (src->need_2D)
4877 src_target = src_target == PIPE_TEXTURE_1D ? PIPE_TEXTURE_2D : PIPE_TEXTURE_2D_ARRAY;
4878 switch (src_target) {
4879 case PIPE_TEXTURE_CUBE:
4880 case PIPE_TEXTURE_CUBE_ARRAY:
4881 case PIPE_TEXTURE_2D_ARRAY:
4882 case PIPE_TEXTURE_1D_ARRAY:
4883 /* these use layer */
4884 region.srcSubresource.baseArrayLayer = src_box->z;
4885 region.srcSubresource.layerCount = src_box->depth;
4886 region.srcOffset.z = 0;
4887 region.extent.depth = 1;
4888 break;
4889 case PIPE_TEXTURE_3D:
4890 /* this uses depth */
4891 region.srcSubresource.baseArrayLayer = 0;
4892 region.srcSubresource.layerCount = 1;
4893 region.srcOffset.z = src_box->z;
4894 region.extent.depth = src_box->depth;
4895 break;
4896 default:
4897 /* these must only copy one layer */
4898 region.srcSubresource.baseArrayLayer = 0;
4899 region.srcSubresource.layerCount = 1;
4900 region.srcOffset.z = 0;
4901 region.extent.depth = 1;
4902 }
4903
4904 region.srcOffset.x = src_box->x;
4905 region.srcOffset.y = src_box->y;
4906
4907 region.dstSubresource.aspectMask = dst->aspect;
4908 region.dstSubresource.mipLevel = dst_level;
4909 enum pipe_texture_target dst_target = dst->base.b.target;
4910 if (dst->need_2D)
4911 dst_target = dst_target == PIPE_TEXTURE_1D ? PIPE_TEXTURE_2D : PIPE_TEXTURE_2D_ARRAY;
4912 switch (dst_target) {
4913 case PIPE_TEXTURE_CUBE:
4914 case PIPE_TEXTURE_CUBE_ARRAY:
4915 case PIPE_TEXTURE_2D_ARRAY:
4916 case PIPE_TEXTURE_1D_ARRAY:
4917 /* these use layer */
4918 region.dstSubresource.baseArrayLayer = dstz;
4919 region.dstSubresource.layerCount = src_box->depth;
4920 region.dstOffset.z = 0;
4921 break;
4922 case PIPE_TEXTURE_3D:
4923 /* this uses depth */
4924 region.dstSubresource.baseArrayLayer = 0;
4925 region.dstSubresource.layerCount = 1;
4926 region.dstOffset.z = dstz;
4927 break;
4928 default:
4929 /* these must only copy one layer */
4930 region.dstSubresource.baseArrayLayer = 0;
4931 region.dstSubresource.layerCount = 1;
4932 region.dstOffset.z = 0;
4933 }
4934
4935 region.dstOffset.x = dstx;
4936 region.dstOffset.y = dsty;
4937 region.extent.width = src_box->width;
4938 region.extent.height = src_box->height;
4939
4940 /* ignore no-op copies */
4941 if (src == dst &&
4942 !memcmp(®ion.dstOffset, ®ion.srcOffset, sizeof(region.srcOffset)) &&
4943 !memcmp(®ion.dstSubresource, ®ion.srcSubresource, sizeof(region.srcSubresource)))
4944 return;
4945
4946 zink_fb_clears_apply_or_discard(ctx, pdst, (struct u_rect){dstx, dstx + src_box->width, dsty, dsty + src_box->height}, false);
4947 zink_fb_clears_apply_region(ctx, psrc, zink_rect_from_box(src_box));
4948
4949 zink_resource_setup_transfer_layouts(ctx, src, dst);
4950 VkCommandBuffer cmdbuf = zink_get_cmdbuf(ctx, src, dst);
4951 zink_batch_reference_resource_rw(ctx, src, false);
4952 zink_batch_reference_resource_rw(ctx, dst, true);
4953
4954 if (unlikely(zink_debug & ZINK_DEBUG_SYNC)) {
4955 VkMemoryBarrier mb;
4956 mb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
4957 mb.pNext = NULL;
4958 mb.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
4959 mb.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT;
4960 VKCTX(CmdPipelineBarrier)(cmdbuf,
4961 VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
4962 VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
4963 0, 1, &mb, 0, NULL, 0, NULL);
4964 }
4965 bool marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "copy_image(%s->%s, %dx%dx%d)",
4966 util_format_short_name(psrc->format),
4967 util_format_short_name(pdst->format),
4968 region.extent.width,
4969 region.extent.height,
4970 MAX2(region.srcSubresource.layerCount, region.extent.depth));
4971 VKCTX(CmdCopyImage)(cmdbuf, src->obj->image, src->layout,
4972 dst->obj->image, dst->layout,
4973 1, ®ion);
4974 zink_cmd_debug_marker_end(ctx, cmdbuf, marker);
4975 } else if (dst->base.b.target == PIPE_BUFFER &&
4976 src->base.b.target == PIPE_BUFFER) {
4977 zink_copy_buffer(ctx, dst, src, dstx, src_box->x, src_box->width);
4978 } else
4979 zink_copy_image_buffer(ctx, dst, src, dst_level, dstx, dsty, dstz, src_level, src_box, 0);
4980 if (ctx->oom_flush && !ctx->in_rp && !ctx->unordered_blitting)
4981 flush_batch(ctx, false);
4982 }
4983
4984 static bool
zink_resource_commit(struct pipe_context * pctx,struct pipe_resource * pres,unsigned level,struct pipe_box * box,bool commit)4985 zink_resource_commit(struct pipe_context *pctx, struct pipe_resource *pres, unsigned level, struct pipe_box *box, bool commit)
4986 {
4987 struct zink_context *ctx = zink_context(pctx);
4988 struct zink_resource *res = zink_resource(pres);
4989
4990 /* if any current usage exists, flush the queue */
4991 if (zink_resource_has_unflushed_usage(res))
4992 zink_flush_queue(ctx);
4993
4994 bool ret = zink_bo_commit(ctx, res, level, box, commit, &ctx->bs->sparse_semaphore);
4995 if (ret) {
4996 zink_batch_reference_resource_rw(ctx, res, true);
4997 ctx->bs->has_work = true;
4998 } else {
4999 check_device_lost(ctx);
5000 }
5001
5002 return ret;
5003 }
5004
5005 static void
rebind_image(struct zink_context * ctx,struct zink_resource * res)5006 rebind_image(struct zink_context *ctx, struct zink_resource *res)
5007 {
5008 assert(!ctx->blitting);
5009 if (res->fb_binds)
5010 zink_rebind_framebuffer(ctx, res);
5011 if (!zink_resource_has_binds(res))
5012 return;
5013 for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) {
5014 if (res->sampler_binds[i]) {
5015 for (unsigned j = 0; j < ctx->di.num_sampler_views[i]; j++) {
5016 struct zink_sampler_view *sv = zink_sampler_view(ctx->sampler_views[i][j]);
5017 if (sv && sv->base.texture == &res->base.b) {
5018 struct pipe_surface *psurf = &sv->image_view->base;
5019 zink_rebind_surface(ctx, &psurf);
5020 sv->image_view = zink_surface(psurf);
5021 ctx->invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, j, 1);
5022 update_descriptor_state_sampler(ctx, i, j, res);
5023 }
5024 }
5025 }
5026 if (!res->image_bind_count[i == MESA_SHADER_COMPUTE])
5027 continue;
5028 for (unsigned j = 0; j < ctx->di.num_images[i]; j++) {
5029 if (zink_resource(ctx->image_views[i][j].base.resource) == res) {
5030 ctx->invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_IMAGE, j, 1);
5031 update_descriptor_state_image(ctx, i, j, res);
5032 _mesa_set_add(ctx->need_barriers[i == MESA_SHADER_COMPUTE], res);
5033 }
5034 }
5035 }
5036 }
5037
5038 bool
zink_resource_rebind(struct zink_context * ctx,struct zink_resource * res)5039 zink_resource_rebind(struct zink_context *ctx, struct zink_resource *res)
5040 {
5041 if (res->base.b.target == PIPE_BUFFER) {
5042 /* force counter buffer reset */
5043 res->so_valid = false;
5044 return rebind_buffer(ctx, res, 0, 0) == res->bind_count[0] + res->bind_count[1];
5045 }
5046 rebind_image(ctx, res);
5047 return false;
5048 }
5049
5050 void
zink_rebind_all_buffers(struct zink_context * ctx)5051 zink_rebind_all_buffers(struct zink_context *ctx)
5052 {
5053 ctx->vertex_buffers_dirty = ctx->gfx_pipeline_state.vertex_buffers_enabled_mask > 0;
5054 ctx->dirty_so_targets = ctx->num_so_targets > 0;
5055 if (ctx->num_so_targets)
5056 zink_screen(ctx->base.screen)->buffer_barrier(ctx, zink_resource(ctx->dummy_xfb_buffer),
5057 VK_ACCESS_TRANSFORM_FEEDBACK_WRITE_BIT_EXT, VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT);
5058 for (unsigned shader = MESA_SHADER_VERTEX; shader < MESA_SHADER_STAGES; shader++) {
5059 for (unsigned slot = 0; slot < ctx->di.num_ubos[shader]; slot++) {
5060 struct zink_resource *res = rebind_ubo(ctx, shader, slot);
5061 if (res)
5062 zink_batch_resource_usage_set(ctx->bs, res, false, true);
5063 }
5064 for (unsigned slot = 0; slot < ctx->di.num_sampler_views[shader]; slot++) {
5065 struct zink_resource *res = rebind_tbo(ctx, shader, slot);
5066 if (res)
5067 zink_batch_resource_usage_set(ctx->bs, res, false, true);
5068 }
5069 for (unsigned slot = 0; slot < ctx->di.num_ssbos[shader]; slot++) {
5070 struct zink_resource *res = rebind_ssbo(ctx, shader, slot);
5071 if (res)
5072 zink_batch_resource_usage_set(ctx->bs, res, (ctx->writable_ssbos[shader] & BITFIELD64_BIT(slot)) != 0, true);
5073 }
5074 for (unsigned slot = 0; slot < ctx->di.num_images[shader]; slot++) {
5075 struct zink_resource *res = rebind_ibo(ctx, shader, slot);
5076 if (res)
5077 zink_batch_resource_usage_set(ctx->bs, res, (ctx->image_views[shader][slot].base.access & PIPE_IMAGE_ACCESS_WRITE) != 0, true);
5078 }
5079 }
5080 }
5081
5082 void
zink_rebind_all_images(struct zink_context * ctx)5083 zink_rebind_all_images(struct zink_context *ctx)
5084 {
5085 assert(!ctx->blitting);
5086 rebind_fb_state(ctx, NULL, false);
5087 for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) {
5088 for (unsigned j = 0; j < ctx->di.num_sampler_views[i]; j++) {
5089 struct zink_sampler_view *sv = zink_sampler_view(ctx->sampler_views[i][j]);
5090 if (!sv || !sv->image_view || sv->image_view->base.texture->target == PIPE_BUFFER)
5091 continue;
5092 struct zink_resource *res = zink_resource(sv->image_view->base.texture);
5093 if (res->obj != sv->image_view->obj) {
5094 struct pipe_surface *psurf = &sv->image_view->base;
5095 zink_rebind_surface(ctx, &psurf);
5096 sv->image_view = zink_surface(psurf);
5097 ctx->invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, j, 1);
5098 update_descriptor_state_sampler(ctx, i, j, res);
5099 }
5100 }
5101 for (unsigned j = 0; j < ctx->di.num_images[i]; j++) {
5102 struct zink_image_view *image_view = &ctx->image_views[i][j];
5103 struct zink_resource *res = zink_resource(image_view->base.resource);
5104 if (!res || res->base.b.target == PIPE_BUFFER)
5105 continue;
5106 if (ctx->image_views[i][j].surface->obj != res->obj) {
5107 zink_surface_reference(zink_screen(ctx->base.screen), &image_view->surface, NULL);
5108 image_view->surface = create_image_surface(ctx, &image_view->base, i == MESA_SHADER_COMPUTE);
5109 ctx->invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_IMAGE, j, 1);
5110 update_descriptor_state_image(ctx, i, j, res);
5111 _mesa_set_add(ctx->need_barriers[i == MESA_SHADER_COMPUTE], res);
5112 }
5113 }
5114 }
5115 }
5116
5117 static void
zink_context_replace_buffer_storage(struct pipe_context * pctx,struct pipe_resource * dst,struct pipe_resource * src,unsigned num_rebinds,uint32_t rebind_mask,uint32_t delete_buffer_id)5118 zink_context_replace_buffer_storage(struct pipe_context *pctx, struct pipe_resource *dst,
5119 struct pipe_resource *src, unsigned num_rebinds,
5120 uint32_t rebind_mask, uint32_t delete_buffer_id)
5121 {
5122 struct zink_resource *d = zink_resource(dst);
5123 struct zink_resource *s = zink_resource(src);
5124 struct zink_context *ctx = zink_context(pctx);
5125 struct zink_screen *screen = zink_screen(pctx->screen);
5126
5127 assert(d->internal_format == s->internal_format);
5128 assert(d->obj);
5129 assert(s->obj);
5130 util_idalloc_mt_free(&screen->buffer_ids, delete_buffer_id);
5131 zink_batch_reference_resource(ctx, d);
5132 /* don't be too creative */
5133 zink_resource_object_reference(screen, &d->obj, s->obj);
5134 d->valid_buffer_range = s->valid_buffer_range;
5135 s->real_buffer_range = &d->valid_buffer_range;
5136 zink_resource_copies_reset(d);
5137 /* force counter buffer reset */
5138 d->so_valid = false;
5139 /* FIXME: tc buffer sharedness tracking */
5140 if (!num_rebinds) {
5141 num_rebinds = d->bind_count[0] + d->bind_count[1];
5142 rebind_mask = 0;
5143 }
5144 if (num_rebinds && rebind_buffer(ctx, d, rebind_mask, num_rebinds) < num_rebinds)
5145 ctx->buffer_rebind_counter = p_atomic_inc_return(&screen->buffer_rebind_counter);
5146 }
5147
5148 static bool
zink_context_is_resource_busy(struct pipe_screen * pscreen,struct pipe_resource * pres,unsigned usage)5149 zink_context_is_resource_busy(struct pipe_screen *pscreen, struct pipe_resource *pres, unsigned usage)
5150 {
5151 struct zink_screen *screen = zink_screen(pscreen);
5152 struct zink_resource *res = zink_resource(pres);
5153 uint32_t check_usage = 0;
5154 if (usage & PIPE_MAP_UNSYNCHRONIZED && (!res->obj->unsync_access || zink_is_swapchain(res)))
5155 return true;
5156 if (usage & PIPE_MAP_READ)
5157 check_usage |= ZINK_RESOURCE_ACCESS_WRITE;
5158 if (usage & PIPE_MAP_WRITE)
5159 check_usage |= ZINK_RESOURCE_ACCESS_RW;
5160 return !zink_resource_usage_check_completion(screen, res, check_usage);
5161 }
5162
5163 static void
zink_emit_string_marker(struct pipe_context * pctx,const char * string,int len)5164 zink_emit_string_marker(struct pipe_context *pctx,
5165 const char *string, int len)
5166 {
5167 struct zink_screen *screen = zink_screen(pctx->screen);
5168 struct zink_context *ctx = zink_context(pctx);
5169
5170 /* make sure string is nul-terminated */
5171 char buf[512], *temp = NULL;
5172 if (len < ARRAY_SIZE(buf)) {
5173 memcpy(buf, string, len);
5174 buf[len] = '\0';
5175 string = buf;
5176 } else
5177 string = temp = strndup(string, len);
5178
5179 VkDebugUtilsLabelEXT label = {
5180 VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT, NULL,
5181 string,
5182 { 0 }
5183 };
5184 screen->vk.CmdInsertDebugUtilsLabelEXT(ctx->bs->cmdbuf, &label);
5185 free(temp);
5186 }
5187
5188 struct pipe_surface *
zink_get_dummy_pipe_surface(struct zink_context * ctx,int samples_index)5189 zink_get_dummy_pipe_surface(struct zink_context *ctx, int samples_index)
5190 {
5191 unsigned size = calc_max_dummy_fbo_size(ctx);
5192 bool needs_null_init = false;
5193 if (ctx->dummy_surface[samples_index]) {
5194 /* delete old surface if ETOOSMALL */
5195 struct zink_resource *res = zink_resource(ctx->dummy_surface[samples_index]->texture);
5196 if (res->base.b.width0 > size || res->base.b.height0 > size) {
5197 pipe_surface_release(&ctx->base, &ctx->dummy_surface[samples_index]);
5198 needs_null_init = !samples_index && ctx->di.null_fbfetch_init;
5199 if (!samples_index)
5200 ctx->di.null_fbfetch_init = false;
5201 }
5202 }
5203 if (!ctx->dummy_surface[samples_index]) {
5204 ctx->dummy_surface[samples_index] = zink_surface_create_null(ctx, PIPE_TEXTURE_2D, size, size, BITFIELD_BIT(samples_index));
5205 assert(ctx->dummy_surface[samples_index]);
5206 /* This is possibly used with imageLoad which according to GL spec must return 0 */
5207 if (!samples_index) {
5208 union pipe_color_union color = {0};
5209 struct pipe_box box;
5210 u_box_2d(0, 0, size, size, &box);
5211 ctx->base.clear_texture(&ctx->base, ctx->dummy_surface[samples_index]->texture, 0, &box, &color);
5212 }
5213 }
5214 if (needs_null_init)
5215 init_null_fbfetch(ctx);
5216 return ctx->dummy_surface[samples_index];
5217 }
5218
5219 struct zink_surface *
zink_get_dummy_surface(struct zink_context * ctx,int samples_index)5220 zink_get_dummy_surface(struct zink_context *ctx, int samples_index)
5221 {
5222 return zink_csurface(zink_get_dummy_pipe_surface(ctx, samples_index));
5223
5224 }
5225
5226 static void
zink_tc_parse_dsa(void * state,struct tc_renderpass_info * info)5227 zink_tc_parse_dsa(void *state, struct tc_renderpass_info *info)
5228 {
5229 struct zink_depth_stencil_alpha_state *cso = state;
5230 info->zsbuf_write_dsa |= (cso->hw_state.depth_write || cso->hw_state.stencil_test);
5231 info->zsbuf_read_dsa |= (cso->hw_state.depth_test || cso->hw_state.stencil_test);
5232 /* TODO: if zsbuf fbfetch is ever supported */
5233 }
5234
5235 static void
zink_tc_parse_fs(void * state,struct tc_renderpass_info * info)5236 zink_tc_parse_fs(void *state, struct tc_renderpass_info *info)
5237 {
5238 struct zink_shader *zs = state;
5239 info->zsbuf_write_fs |= zs->info.outputs_written & (BITFIELD64_BIT(FRAG_RESULT_DEPTH) | BITFIELD64_BIT(FRAG_RESULT_STENCIL));
5240 /* TODO: if >1 fbfetch attachment is ever supported */
5241 info->cbuf_fbfetch |= zs->info.fs.uses_fbfetch_output ? BITFIELD_BIT(0) : 0;
5242 }
5243
5244 void
zink_parse_tc_info(struct zink_context * ctx)5245 zink_parse_tc_info(struct zink_context *ctx)
5246 {
5247 struct tc_renderpass_info *info = &ctx->dynamic_fb.tc_info;
5248 /* reset cso info first */
5249 info->data16[2] = 0;
5250 if (ctx->gfx_stages[MESA_SHADER_FRAGMENT])
5251 zink_tc_parse_fs(ctx->gfx_stages[MESA_SHADER_FRAGMENT], info);
5252 if (ctx->dsa_state)
5253 zink_tc_parse_dsa(ctx->dsa_state, info);
5254 if (ctx->zsbuf_unused == zink_is_zsbuf_used(ctx))
5255 ctx->rp_layout_changed = true;
5256 }
5257
5258 struct pipe_context *
zink_context_create(struct pipe_screen * pscreen,void * priv,unsigned flags)5259 zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags)
5260 {
5261 struct zink_screen *screen = zink_screen(pscreen);
5262 struct zink_context *ctx = rzalloc(NULL, struct zink_context);
5263 bool is_copy_only = (flags & ZINK_CONTEXT_COPY_ONLY) > 0;
5264 bool is_compute_only = (flags & PIPE_CONTEXT_COMPUTE_ONLY) > 0;
5265 bool is_robust = (flags & PIPE_CONTEXT_ROBUST_BUFFER_ACCESS) > 0;
5266 if (!ctx)
5267 goto fail;
5268
5269 ctx->flags = flags;
5270 ctx->pipeline_changed[0] = ctx->pipeline_changed[1] = true;
5271 ctx->gfx_pipeline_state.dirty = true;
5272 ctx->gfx_pipeline_state.dyn_state2.vertices_per_patch = 1;
5273 ctx->gfx_pipeline_state.uses_dynamic_stride = screen->info.have_EXT_extended_dynamic_state ||
5274 screen->info.have_EXT_vertex_input_dynamic_state;
5275 #if defined(MVK_VERSION)
5276 ctx->gfx_pipeline_state.uses_dynamic_stride = ctx->gfx_pipeline_state.uses_dynamic_stride && screen->have_dynamic_state_vertex_input_binding_stride;
5277 #endif
5278 ctx->compute_pipeline_state.dirty = true;
5279 ctx->fb_changed = ctx->rp_changed = true;
5280 ctx->sample_mask_changed = true;
5281 ctx->gfx_pipeline_state.gfx_prim_mode = MESA_PRIM_COUNT;
5282 ctx->gfx_pipeline_state.shader_rast_prim = MESA_PRIM_COUNT;
5283 ctx->gfx_pipeline_state.rast_prim = MESA_PRIM_COUNT;
5284
5285 zink_init_draw_functions(ctx, screen);
5286 zink_init_grid_functions(ctx);
5287
5288 ctx->base.screen = pscreen;
5289 ctx->base.priv = priv;
5290
5291 ctx->base.destroy = zink_context_destroy;
5292 ctx->base.set_debug_callback = zink_set_debug_callback;
5293 ctx->base.get_device_reset_status = zink_get_device_reset_status;
5294 ctx->base.set_device_reset_callback = zink_set_device_reset_callback;
5295
5296 zink_context_state_init(&ctx->base);
5297
5298 ctx->base.create_sampler_state = zink_create_sampler_state;
5299 ctx->base.bind_sampler_states = screen->info.have_EXT_non_seamless_cube_map ? zink_bind_sampler_states : zink_bind_sampler_states_nonseamless;
5300 ctx->base.delete_sampler_state = zink_delete_sampler_state;
5301
5302 ctx->base.create_sampler_view = zink_create_sampler_view;
5303 ctx->base.set_sampler_views = zink_set_sampler_views;
5304 ctx->base.sampler_view_destroy = zink_sampler_view_destroy;
5305 ctx->base.get_sample_position = zink_get_sample_position;
5306 ctx->base.set_sample_locations = zink_set_sample_locations;
5307
5308 zink_program_init(ctx);
5309
5310 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
5311 ctx->base.set_constant_buffer = zink_set_constant_buffer_db;
5312 ctx->base.set_shader_buffers = zink_set_shader_buffers_db;
5313 } else {
5314 ctx->base.set_constant_buffer = zink_set_constant_buffer_lazy;
5315 ctx->base.set_shader_buffers = zink_set_shader_buffers_lazy;
5316 }
5317 ctx->base.set_polygon_stipple = zink_set_polygon_stipple;
5318 ctx->base.set_vertex_buffers = zink_set_vertex_buffers;
5319 ctx->base.set_viewport_states = zink_set_viewport_states;
5320 ctx->base.set_scissor_states = zink_set_scissor_states;
5321 ctx->base.set_inlinable_constants = zink_set_inlinable_constants;
5322 ctx->base.set_shader_images = zink_set_shader_images;
5323 ctx->base.set_framebuffer_state = zink_set_framebuffer_state;
5324 ctx->base.set_stencil_ref = zink_set_stencil_ref;
5325 ctx->base.set_clip_state = zink_set_clip_state;
5326 ctx->base.set_blend_color = zink_set_blend_color;
5327 ctx->base.set_tess_state = zink_set_tess_state;
5328 ctx->base.set_patch_vertices = zink_set_patch_vertices;
5329
5330 ctx->base.set_min_samples = zink_set_min_samples;
5331 ctx->gfx_pipeline_state.min_samples = 0;
5332 ctx->base.set_sample_mask = zink_set_sample_mask;
5333 ctx->gfx_pipeline_state.sample_mask = UINT32_MAX;
5334
5335 ctx->base.clear = zink_clear;
5336 ctx->base.clear_texture = screen->info.have_KHR_dynamic_rendering ? zink_clear_texture_dynamic : zink_clear_texture;
5337 ctx->base.clear_buffer = zink_clear_buffer;
5338 ctx->base.clear_render_target = zink_clear_render_target;
5339 ctx->base.clear_depth_stencil = zink_clear_depth_stencil;
5340
5341 ctx->base.create_fence_fd = zink_create_fence_fd;
5342 ctx->base.fence_server_sync = zink_fence_server_sync;
5343 ctx->base.fence_server_signal = zink_fence_server_signal;
5344 ctx->base.flush = zink_flush;
5345 ctx->base.memory_barrier = zink_memory_barrier;
5346 ctx->base.texture_barrier = zink_texture_barrier;
5347 ctx->base.evaluate_depth_buffer = zink_evaluate_depth_buffer;
5348
5349 ctx->base.resource_commit = zink_resource_commit;
5350 ctx->base.resource_copy_region = zink_resource_copy_region;
5351 ctx->base.blit = zink_blit;
5352 ctx->base.create_stream_output_target = zink_create_stream_output_target;
5353 ctx->base.stream_output_target_destroy = zink_stream_output_target_destroy;
5354
5355 ctx->base.set_stream_output_targets = zink_set_stream_output_targets;
5356 ctx->base.flush_resource = zink_flush_resource;
5357 if (screen->info.have_KHR_buffer_device_address)
5358 ctx->base.set_global_binding = zink_set_global_binding;
5359
5360 ctx->base.emit_string_marker = zink_emit_string_marker;
5361
5362 zink_context_surface_init(&ctx->base);
5363 zink_context_resource_init(&ctx->base);
5364 zink_context_query_init(&ctx->base);
5365
5366 util_queue_fence_init(&ctx->flush_fence);
5367 util_queue_fence_init(&ctx->unsync_fence);
5368
5369 list_inithead(&ctx->query_pools);
5370 _mesa_set_init(&ctx->update_barriers[0][0], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
5371 _mesa_set_init(&ctx->update_barriers[1][0], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
5372 _mesa_set_init(&ctx->update_barriers[0][1], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
5373 _mesa_set_init(&ctx->update_barriers[1][1], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
5374 ctx->need_barriers[0] = &ctx->update_barriers[0][0];
5375 ctx->need_barriers[1] = &ctx->update_barriers[1][0];
5376
5377 slab_create_child(&ctx->transfer_pool, &screen->transfer_pool);
5378 slab_create_child(&ctx->transfer_pool_unsync, &screen->transfer_pool);
5379
5380 ctx->base.stream_uploader = u_upload_create_default(&ctx->base);
5381 ctx->base.const_uploader = u_upload_create_default(&ctx->base);
5382 for (int i = 0; i < ARRAY_SIZE(ctx->fb_clears); i++)
5383 util_dynarray_init(&ctx->fb_clears[i].clears, ctx);
5384
5385 if (!is_copy_only) {
5386 ctx->blitter = util_blitter_create(&ctx->base);
5387 if (!ctx->blitter)
5388 goto fail;
5389 if (screen->driver_workarounds.inconsistent_interpolation)
5390 ctx->blitter->draw_rectangle = zink_draw_rectangle;
5391 }
5392
5393 zink_set_last_vertex_key(ctx)->last_vertex_stage = true;
5394 ctx->gfx_pipeline_state.shader_keys.last_vertex.key.vs_base.last_vertex_stage = true;
5395 zink_set_tcs_key_patches(ctx, 1);
5396 if (!screen->optimal_keys) {
5397 ctx->gfx_pipeline_state.shader_keys.key[MESA_SHADER_VERTEX].size = sizeof(struct zink_vs_key_base);
5398 ctx->gfx_pipeline_state.shader_keys.key[MESA_SHADER_TESS_EVAL].size = sizeof(struct zink_vs_key_base);
5399 ctx->gfx_pipeline_state.shader_keys.key[MESA_SHADER_TESS_CTRL].size = sizeof(struct zink_tcs_key);
5400 ctx->gfx_pipeline_state.shader_keys.key[MESA_SHADER_GEOMETRY].size = sizeof(struct zink_gs_key);
5401 ctx->gfx_pipeline_state.shader_keys.key[MESA_SHADER_FRAGMENT].size = sizeof(struct zink_fs_key);
5402
5403 /* this condition must be updated if new fields are added to zink_cs_key */
5404 if (screen->driver_compiler_workarounds.lower_robustImageAccess2)
5405 ctx->compute_pipeline_state.key.size = sizeof(struct zink_cs_key);
5406
5407 if (is_robust && screen->driver_compiler_workarounds.lower_robustImageAccess2) {
5408 ctx->compute_pipeline_state.key.key.cs.robust_access = true;
5409 for (gl_shader_stage pstage = MESA_SHADER_VERTEX; pstage < MESA_SHADER_FRAGMENT; pstage++)
5410 ctx->gfx_pipeline_state.shader_keys.key[pstage].key.vs_base.robust_access = true;
5411 ctx->gfx_pipeline_state.shader_keys.key[MESA_SHADER_FRAGMENT].key.fs.robust_access = true;
5412 }
5413 }
5414 _mesa_hash_table_init(&ctx->framebuffer_cache, ctx, hash_framebuffer_imageless, equals_framebuffer_imageless);
5415 if (!zink_init_render_pass(ctx))
5416 goto fail;
5417 for (unsigned i = 0; i < ARRAY_SIZE(ctx->rendering_state_cache); i++)
5418 _mesa_set_init(&ctx->rendering_state_cache[i], ctx, hash_rendering_state, equals_rendering_state);
5419 ctx->dynamic_fb.info.pColorAttachments = ctx->dynamic_fb.attachments;
5420 ctx->dynamic_fb.info.sType = VK_STRUCTURE_TYPE_RENDERING_INFO;
5421 for (unsigned i = 0; i < ARRAY_SIZE(ctx->dynamic_fb.attachments); i++) {
5422 VkRenderingAttachmentInfo *att = &ctx->dynamic_fb.attachments[i];
5423 att->sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO;
5424 att->imageLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
5425 att->resolveImageLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
5426 att->storeOp = VK_ATTACHMENT_STORE_OP_STORE;
5427 }
5428 ctx->gfx_pipeline_state.rendering_info.sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO;
5429 ctx->gfx_pipeline_state.rendering_info.pColorAttachmentFormats = ctx->gfx_pipeline_state.rendering_formats;
5430 ctx->gfx_pipeline_state.feedback_loop = screen->driver_workarounds.always_feedback_loop;
5431 ctx->gfx_pipeline_state.feedback_loop_zs = screen->driver_workarounds.always_feedback_loop_zs;
5432
5433 const uint32_t data[] = {0};
5434 if (!is_copy_only) {
5435 ctx->dummy_vertex_buffer = pipe_buffer_create(&screen->base,
5436 PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_SHADER_IMAGE, PIPE_USAGE_IMMUTABLE, sizeof(data));
5437 if (!ctx->dummy_vertex_buffer)
5438 goto fail;
5439 ctx->dummy_xfb_buffer = pipe_buffer_create(&screen->base,
5440 PIPE_BIND_STREAM_OUTPUT, PIPE_USAGE_IMMUTABLE, sizeof(data));
5441 if (!ctx->dummy_xfb_buffer)
5442 goto fail;
5443 }
5444 if (!is_copy_only) {
5445 VkBufferViewCreateInfo bvci = create_bvci(ctx, zink_resource(ctx->dummy_vertex_buffer), PIPE_FORMAT_R8G8B8A8_UNORM, 0, sizeof(data));
5446 ctx->dummy_bufferview = get_buffer_view(ctx, zink_resource(ctx->dummy_vertex_buffer), &bvci);
5447 if (!ctx->dummy_bufferview)
5448 goto fail;
5449
5450 if (!zink_descriptors_init(ctx))
5451 goto fail;
5452 }
5453
5454 if (!is_copy_only && !is_compute_only) {
5455 ctx->base.create_texture_handle = zink_create_texture_handle;
5456 ctx->base.delete_texture_handle = zink_delete_texture_handle;
5457 ctx->base.make_texture_handle_resident = zink_make_texture_handle_resident;
5458 ctx->base.create_image_handle = zink_create_image_handle;
5459 ctx->base.delete_image_handle = zink_delete_image_handle;
5460 ctx->base.make_image_handle_resident = zink_make_image_handle_resident;
5461 for (unsigned i = 0; i < 2; i++) {
5462 _mesa_hash_table_init(&ctx->di.bindless[i].img_handles, ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
5463 _mesa_hash_table_init(&ctx->di.bindless[i].tex_handles, ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
5464
5465 /* allocate 1024 slots and reserve slot 0 */
5466 util_idalloc_init(&ctx->di.bindless[i].tex_slots, ZINK_MAX_BINDLESS_HANDLES);
5467 util_idalloc_alloc(&ctx->di.bindless[i].tex_slots);
5468 util_idalloc_init(&ctx->di.bindless[i].img_slots, ZINK_MAX_BINDLESS_HANDLES);
5469 util_idalloc_alloc(&ctx->di.bindless[i].img_slots);
5470 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
5471 ctx->di.bindless[i].db.buffer_infos = malloc(sizeof(VkDescriptorAddressInfoEXT) * ZINK_MAX_BINDLESS_HANDLES);
5472 if (!ctx->di.bindless[i].db.buffer_infos) {
5473 mesa_loge("ZINK: failed to allocate ctx->di.bindless[%d].db.buffer_infos!",i);
5474 goto fail;
5475 }
5476 for (unsigned j = 0; j < ZINK_MAX_BINDLESS_HANDLES; j++) {
5477 ctx->di.bindless[i].db.buffer_infos[j].sType = VK_STRUCTURE_TYPE_DESCRIPTOR_ADDRESS_INFO_EXT;
5478 ctx->di.bindless[i].db.buffer_infos[j].pNext = NULL;
5479 }
5480 } else {
5481 ctx->di.bindless[i].t.buffer_infos = malloc(sizeof(VkBufferView) * ZINK_MAX_BINDLESS_HANDLES);
5482 if (!ctx->di.bindless[i].t.buffer_infos) {
5483 mesa_loge("ZINK: failed to allocate ctx->di.bindless[%d].t.buffer_infos!",i);
5484 goto fail;
5485 }
5486 }
5487 ctx->di.bindless[i].img_infos = malloc(sizeof(VkDescriptorImageInfo) * ZINK_MAX_BINDLESS_HANDLES);
5488 if (!ctx->di.bindless[i].img_infos) {
5489 mesa_loge("ZINK: failed to allocate ctx->di.bindless[%d].img_infos!",i);
5490 goto fail;
5491 }
5492 util_dynarray_init(&ctx->di.bindless[i].updates, NULL);
5493 util_dynarray_init(&ctx->di.bindless[i].resident, NULL);
5494 }
5495 }
5496
5497 zink_start_batch(ctx);
5498 if (!ctx->bs)
5499 goto fail;
5500
5501 if (screen->compact_descriptors)
5502 ctx->invalidate_descriptor_state = zink_context_invalidate_descriptor_state_compact;
5503 else
5504 ctx->invalidate_descriptor_state = zink_context_invalidate_descriptor_state;
5505 if (!is_copy_only && !is_compute_only) {
5506 pipe_buffer_write_nooverlap(&ctx->base, ctx->dummy_vertex_buffer, 0, sizeof(data), data);
5507 pipe_buffer_write_nooverlap(&ctx->base, ctx->dummy_xfb_buffer, 0, sizeof(data), data);
5508 if (screen->info.have_EXT_color_write_enable)
5509 reapply_color_write(ctx);
5510
5511 /* set on startup just to avoid validation errors if a draw comes through without
5512 * a tess shader later
5513 */
5514 if (screen->info.dynamic_state2_feats.extendedDynamicState2PatchControlPoints) {
5515 VKCTX(CmdSetPatchControlPointsEXT)(ctx->bs->cmdbuf, 1);
5516 VKCTX(CmdSetPatchControlPointsEXT)(ctx->bs->reordered_cmdbuf, 1);
5517 }
5518 }
5519 if (!is_copy_only) {
5520 for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) {
5521 /* need to update these based on screen config for null descriptors */
5522 for (unsigned j = 0; j < ARRAY_SIZE(ctx->di.t.ubos[i]); j++) {
5523 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
5524 update_descriptor_state_ubo_db(ctx, i, j, NULL);
5525 ctx->di.db.ubos[i][j].sType = VK_STRUCTURE_TYPE_DESCRIPTOR_ADDRESS_INFO_EXT;
5526 } else {
5527 update_descriptor_state_ubo_lazy(ctx, i, j, NULL);
5528 }
5529 }
5530 for (unsigned j = 0; j < ARRAY_SIZE(ctx->di.textures[i]); j++) {
5531 update_descriptor_state_sampler(ctx, i, j, NULL);
5532 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB)
5533 ctx->di.db.tbos[i][j].sType = VK_STRUCTURE_TYPE_DESCRIPTOR_ADDRESS_INFO_EXT;
5534 }
5535 for (unsigned j = 0; j < ARRAY_SIZE(ctx->di.t.ssbos[i]); j++) {
5536 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
5537 update_descriptor_state_ssbo_db(ctx, i, j, NULL);
5538 ctx->di.db.ssbos[i][j].sType = VK_STRUCTURE_TYPE_DESCRIPTOR_ADDRESS_INFO_EXT;
5539 } else {
5540 update_descriptor_state_ssbo_lazy(ctx, i, j, NULL);
5541 }
5542 }
5543 for (unsigned j = 0; j < ARRAY_SIZE(ctx->di.images[i]); j++) {
5544 update_descriptor_state_image(ctx, i, j, NULL);
5545 if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB)
5546 ctx->di.db.texel_images[i][j].sType = VK_STRUCTURE_TYPE_DESCRIPTOR_ADDRESS_INFO_EXT;
5547 }
5548 }
5549
5550 p_atomic_inc(&screen->base.num_contexts);
5551 }
5552
5553 zink_select_draw_vbo(ctx);
5554 zink_select_launch_grid(ctx);
5555
5556 if (!is_copy_only && zink_debug & ZINK_DEBUG_SHADERDB) {
5557 if (!screen->info.have_EXT_vertex_input_dynamic_state) {
5558 struct pipe_vertex_element velems[32] = {0};
5559 for (unsigned i = 0; i < ARRAY_SIZE(velems); i++)
5560 velems[i].src_format = PIPE_FORMAT_R8G8B8_UNORM;
5561 void *state = ctx->base.create_vertex_elements_state(&ctx->base, ARRAY_SIZE(velems), velems);
5562 ctx->base.bind_vertex_elements_state(&ctx->base, state);
5563 }
5564 ctx->gfx_pipeline_state.sample_mask = BITFIELD_MASK(32);
5565 struct pipe_framebuffer_state fb = {0};
5566 fb.cbufs[0] = zink_get_dummy_pipe_surface(ctx, 0);
5567 fb.nr_cbufs = 1;
5568 fb.width = fb.height = 256;
5569 ctx->base.set_framebuffer_state(&ctx->base, &fb);
5570 ctx->disable_fs = true;
5571 struct pipe_depth_stencil_alpha_state dsa = {0};
5572 void *state = ctx->base.create_depth_stencil_alpha_state(&ctx->base, &dsa);
5573 ctx->base.bind_depth_stencil_alpha_state(&ctx->base, state);
5574
5575 struct pipe_blend_state blend = {
5576 .rt[0].colormask = 0xF
5577 };
5578
5579 void *blend_state = ctx->base.create_blend_state(&ctx->base, &blend);
5580 ctx->base.bind_blend_state(&ctx->base, blend_state);
5581
5582 zink_batch_rp(ctx);
5583 }
5584
5585 if (!is_compute_only && zink_debug & ZINK_DEBUG_NOREORDER)
5586 ctx->no_reorder = true;
5587
5588 if (!(flags & PIPE_CONTEXT_PREFER_THREADED) || flags & PIPE_CONTEXT_COMPUTE_ONLY) {
5589 return &ctx->base;
5590 }
5591
5592 struct threaded_context *tc = (struct threaded_context*)threaded_context_create(&ctx->base, &screen->transfer_pool,
5593 zink_context_replace_buffer_storage,
5594 &(struct threaded_context_options){
5595 .create_fence = zink_create_tc_fence_for_tc,
5596 .is_resource_busy = zink_context_is_resource_busy,
5597 .driver_calls_flush_notify = !screen->driver_workarounds.track_renderpasses,
5598 .unsynchronized_get_device_reset_status = true,
5599 .unsynchronized_texture_subdata = true,
5600 .parse_renderpass_info = screen->driver_workarounds.track_renderpasses,
5601 .dsa_parse = zink_tc_parse_dsa,
5602 .fs_parse = zink_tc_parse_fs,
5603 },
5604 &ctx->tc);
5605
5606 if (tc && (struct zink_context*)tc != ctx) {
5607 ctx->track_renderpasses = screen->driver_workarounds.track_renderpasses;
5608 threaded_context_init_bytes_mapped_limit(tc, 4);
5609 ctx->base.set_context_param = zink_set_context_param;
5610 }
5611
5612 return (struct pipe_context*)tc;
5613
5614 fail:
5615 if (ctx)
5616 zink_context_destroy(&ctx->base);
5617 return NULL;
5618 }
5619
5620 struct zink_context *
zink_tc_context_unwrap(struct pipe_context * pctx,bool threaded)5621 zink_tc_context_unwrap(struct pipe_context *pctx, bool threaded)
5622 {
5623 /* need to get the actual zink_context, not the threaded context */
5624 if (threaded)
5625 pctx = threaded_context_unwrap_sync(pctx);
5626 pctx = trace_get_possibly_threaded_context(pctx);
5627 return zink_context(pctx);
5628 }
5629
5630
5631 static bool
add_implicit_feedback_loop(struct zink_context * ctx,struct zink_resource * res)5632 add_implicit_feedback_loop(struct zink_context *ctx, struct zink_resource *res)
5633 {
5634 /* can only feedback loop with fb+sampler bind; image bind must be GENERAL */
5635 if (!res->fb_bind_count || !res->sampler_bind_count[0] || res->image_bind_count[0])
5636 return false;
5637 if (!(res->aspect & VK_IMAGE_ASPECT_COLOR_BIT) && !zink_is_zsbuf_write(ctx))
5638 /* if zsbuf isn't used then it effectively has no fb binds */
5639 /* if zsbuf isn't written to then it'll be fine with read-only access */
5640 return false;
5641
5642 /* build attachment array for miplevel/layer checks */
5643 struct pipe_surface *psurfs[PIPE_MAX_COLOR_BUFS];
5644 unsigned surf_idx = 0;
5645 u_foreach_bit(idx, res->fb_binds) {
5646 struct pipe_surface *psurf;
5647 if (idx == PIPE_MAX_COLOR_BUFS)
5648 psurf = ctx->fb_state.zsbuf;
5649 else
5650 psurf = ctx->fb_state.cbufs[idx];
5651 if (psurf->texture == &res->base.b)
5652 psurfs[surf_idx++] = psurf;
5653 }
5654
5655 bool is_feedback = false;
5656 /* avoid false positives when a texture is bound but not used */
5657 u_foreach_bit(vkstage, res->gfx_barrier) {
5658 VkPipelineStageFlags vkstagebit = BITFIELD_BIT(vkstage);
5659 if (vkstagebit < VK_PIPELINE_STAGE_VERTEX_SHADER_BIT || vkstagebit > VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT)
5660 continue;
5661 /* in-range VkPipelineStageFlagBits can be converted to VkShaderStageFlags with a bitshift */
5662 gl_shader_stage stage = vk_to_mesa_shader_stage((VkShaderStageFlagBits)(vkstagebit >> 3));
5663 /* check shader texture usage against resource's sampler binds */
5664 uint32_t texuse = res->sampler_binds[stage] & ctx->gfx_stages[stage]->info.textures_used[0];
5665 if (!ctx->gfx_stages[stage] || !texuse)
5666 continue;
5667
5668 /* check miplevel/layer: a feedback loop only exists if these overlap between fb and sampler */
5669 u_foreach_bit(slot, texuse) {
5670 struct pipe_sampler_view *sv = ctx->sampler_views[stage][slot];
5671
5672 for (unsigned i = 0; i < surf_idx; i++) {
5673 if (sv->u.tex.first_level > psurfs[i]->u.tex.level || sv->u.tex.last_level < psurfs[i]->u.tex.level)
5674 continue;
5675 if (sv->u.tex.first_layer > psurfs[i]->u.tex.last_layer || sv->u.tex.last_layer < psurfs[i]->u.tex.first_layer)
5676 continue;
5677 is_feedback = true;
5678 break;
5679 }
5680 if (is_feedback)
5681 break;
5682 }
5683 }
5684 if (!is_feedback)
5685 return false;
5686 if (ctx->feedback_loops & res->fb_binds)
5687 /* already added */
5688 return true;
5689 /* new feedback loop detected */
5690 if (res->aspect == VK_IMAGE_ASPECT_COLOR_BIT) {
5691 if (!ctx->gfx_pipeline_state.feedback_loop)
5692 ctx->gfx_pipeline_state.dirty = true;
5693 ctx->gfx_pipeline_state.feedback_loop = true;
5694 } else {
5695 if (!ctx->gfx_pipeline_state.feedback_loop_zs)
5696 ctx->gfx_pipeline_state.dirty = true;
5697 ctx->gfx_pipeline_state.feedback_loop_zs = true;
5698 }
5699 ctx->rp_layout_changed = true;
5700 ctx->feedback_loops |= res->fb_binds;
5701 u_foreach_bit(idx, res->fb_binds) {
5702 if (zink_screen(ctx->base.screen)->info.have_EXT_attachment_feedback_loop_layout)
5703 ctx->dynamic_fb.attachments[idx].imageLayout = VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT;
5704 else
5705 ctx->dynamic_fb.attachments[idx].imageLayout = VK_IMAGE_LAYOUT_GENERAL;
5706 }
5707 update_feedback_loop_dynamic_state(ctx);
5708 return true;
5709 }
5710
5711 void
zink_update_barriers(struct zink_context * ctx,bool is_compute,struct pipe_resource * index,struct pipe_resource * indirect,struct pipe_resource * indirect_draw_count)5712 zink_update_barriers(struct zink_context *ctx, bool is_compute,
5713 struct pipe_resource *index, struct pipe_resource *indirect, struct pipe_resource *indirect_draw_count)
5714 {
5715 assert(!ctx->blitting);
5716 if (!ctx->need_barriers[is_compute]->entries)
5717 return;
5718 struct set *need_barriers = ctx->need_barriers[is_compute];
5719 ctx->barrier_set_idx[is_compute] = !ctx->barrier_set_idx[is_compute];
5720 ctx->need_barriers[is_compute] = &ctx->update_barriers[is_compute][ctx->barrier_set_idx[is_compute]];
5721 ASSERTED bool check_rp = ctx->in_rp && ctx->dynamic_fb.tc_info.zsbuf_invalidate;
5722 set_foreach(need_barriers, he) {
5723 struct zink_resource *res = (struct zink_resource *)he->key;
5724 if (res->bind_count[is_compute]) {
5725 VkPipelineStageFlagBits pipeline = is_compute ? VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT : res->gfx_barrier;
5726 if (res->base.b.target == PIPE_BUFFER)
5727 zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, res->barrier_access[is_compute], pipeline);
5728 else {
5729 bool is_feedback = is_compute ? false : add_implicit_feedback_loop(ctx, res);
5730 VkImageLayout layout = zink_descriptor_util_image_layout_eval(ctx, res, is_compute);
5731 /* GENERAL is only used for feedback loops and storage image binds */
5732 if (is_feedback || layout != VK_IMAGE_LAYOUT_GENERAL || res->image_bind_count[is_compute])
5733 zink_screen(ctx->base.screen)->image_barrier(ctx, res, layout, res->barrier_access[is_compute], pipeline);
5734 assert(!check_rp || check_rp == ctx->in_rp);
5735 if (is_feedback)
5736 update_res_sampler_layouts(ctx, res);
5737 }
5738 if (zink_resource_access_is_write(res->barrier_access[is_compute]) ||
5739 // TODO: figure out a way to link up layouts between unordered and main cmdbuf
5740 res->base.b.target != PIPE_BUFFER)
5741 res->obj->unordered_write = false;
5742 res->obj->unordered_read = false;
5743 /* always barrier on draw if this resource has either multiple image write binds or
5744 * image write binds and image read binds
5745 */
5746 if (res->write_bind_count[is_compute] && res->bind_count[is_compute] > 1)
5747 _mesa_set_add_pre_hashed(ctx->need_barriers[is_compute], he->hash, res);
5748 }
5749 _mesa_set_remove(need_barriers, he);
5750 if (!need_barriers->entries)
5751 break;
5752 }
5753 }
5754
5755 /**
5756 * Emits a debug marker in the cmd stream to be captured by perfetto during
5757 * execution on the GPU.
5758 */
5759 bool
zink_cmd_debug_marker_begin(struct zink_context * ctx,VkCommandBuffer cmdbuf,const char * fmt,...)5760 zink_cmd_debug_marker_begin(struct zink_context *ctx, VkCommandBuffer cmdbuf, const char *fmt, ...)
5761 {
5762 if (!zink_tracing)
5763 return false;
5764
5765 char *name;
5766 va_list va;
5767 va_start(va, fmt);
5768 int ret = vasprintf(&name, fmt, va);
5769 va_end(va);
5770
5771 if (ret == -1)
5772 return false;
5773
5774 VkDebugUtilsLabelEXT info = { 0 };
5775 info.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT;
5776 info.pLabelName = name;
5777
5778 VKCTX(CmdBeginDebugUtilsLabelEXT)(cmdbuf ? cmdbuf : ctx->bs->cmdbuf, &info);
5779
5780 free(name);
5781 return true;
5782 }
5783
5784 void
zink_cmd_debug_marker_end(struct zink_context * ctx,VkCommandBuffer cmdbuf,bool emitted)5785 zink_cmd_debug_marker_end(struct zink_context *ctx, VkCommandBuffer cmdbuf, bool emitted)
5786 {
5787 if (emitted)
5788 VKCTX(CmdEndDebugUtilsLabelEXT)(cmdbuf);
5789 }
5790