1 /*
2 * Copyright © 2023 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Mike Blumenkrantz <michael.blumenkrantz@gmail.com>
25 */
26
27 #include "zink_batch.h"
28 #include "zink_context.h"
29 #include "zink_descriptors.h"
30 #include "zink_resource.h"
31 #include "zink_screen.h"
32
33
34 static VkAccessFlags
access_src_flags(VkImageLayout layout)35 access_src_flags(VkImageLayout layout)
36 {
37 switch (layout) {
38 case VK_IMAGE_LAYOUT_UNDEFINED:
39 return VK_ACCESS_NONE;
40
41 case VK_IMAGE_LAYOUT_GENERAL:
42 return VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
43
44 case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL:
45 case VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT:
46 return VK_ACCESS_COLOR_ATTACHMENT_READ_BIT;
47 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL:
48 return VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT;
49
50 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL:
51 case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL:
52 return VK_ACCESS_SHADER_READ_BIT;
53
54 case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL:
55 return VK_ACCESS_TRANSFER_READ_BIT;
56
57 case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL:
58 return VK_ACCESS_TRANSFER_WRITE_BIT;
59
60 case VK_IMAGE_LAYOUT_PREINITIALIZED:
61 return VK_ACCESS_HOST_WRITE_BIT;
62
63 case VK_IMAGE_LAYOUT_PRESENT_SRC_KHR:
64 return VK_ACCESS_NONE;
65
66 default:
67 unreachable("unexpected layout");
68 }
69 }
70
71 static VkAccessFlags
access_dst_flags(VkImageLayout layout)72 access_dst_flags(VkImageLayout layout)
73 {
74 switch (layout) {
75 case VK_IMAGE_LAYOUT_UNDEFINED:
76 return VK_ACCESS_NONE;
77
78 case VK_IMAGE_LAYOUT_GENERAL:
79 return VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
80
81 case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL:
82 case VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT:
83 return VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
84 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL:
85 return VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
86
87 case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL:
88 return VK_ACCESS_SHADER_READ_BIT;
89
90 case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL:
91 return VK_ACCESS_TRANSFER_READ_BIT;
92
93 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL:
94 return VK_ACCESS_SHADER_READ_BIT;
95
96 case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL:
97 return VK_ACCESS_TRANSFER_WRITE_BIT;
98
99 case VK_IMAGE_LAYOUT_PRESENT_SRC_KHR:
100 return VK_ACCESS_NONE;
101
102 default:
103 unreachable("unexpected layout");
104 }
105 }
106
107 static VkPipelineStageFlags
pipeline_dst_stage(VkImageLayout layout)108 pipeline_dst_stage(VkImageLayout layout)
109 {
110 switch (layout) {
111 case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL:
112 return VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
113 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL:
114 return VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT;
115
116 case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL:
117 return VK_PIPELINE_STAGE_TRANSFER_BIT;
118 case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL:
119 return VK_PIPELINE_STAGE_TRANSFER_BIT;
120
121 case VK_IMAGE_LAYOUT_GENERAL:
122 return VK_PIPELINE_STAGE_ALL_COMMANDS_BIT;
123
124 case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL:
125 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL:
126 return VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
127
128 default:
129 return VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT;
130 }
131 }
132
133 #define ALL_READ_ACCESS_FLAGS \
134 (VK_ACCESS_INDIRECT_COMMAND_READ_BIT | \
135 VK_ACCESS_INDEX_READ_BIT | \
136 VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | \
137 VK_ACCESS_UNIFORM_READ_BIT | \
138 VK_ACCESS_INPUT_ATTACHMENT_READ_BIT | \
139 VK_ACCESS_SHADER_READ_BIT | \
140 VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | \
141 VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | \
142 VK_ACCESS_TRANSFER_READ_BIT |\
143 VK_ACCESS_HOST_READ_BIT |\
144 VK_ACCESS_MEMORY_READ_BIT |\
145 VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_READ_BIT_EXT |\
146 VK_ACCESS_CONDITIONAL_RENDERING_READ_BIT_EXT |\
147 VK_ACCESS_COLOR_ATTACHMENT_READ_NONCOHERENT_BIT_EXT |\
148 VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_KHR |\
149 VK_ACCESS_FRAGMENT_SHADING_RATE_ATTACHMENT_READ_BIT_KHR |\
150 VK_ACCESS_FRAGMENT_DENSITY_MAP_READ_BIT_EXT |\
151 VK_ACCESS_COMMAND_PREPROCESS_READ_BIT_NV |\
152 VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_KHR |\
153 VK_ACCESS_ACCELERATION_STRUCTURE_WRITE_BIT_KHR)
154
155
156 bool
zink_resource_access_is_write(VkAccessFlags flags)157 zink_resource_access_is_write(VkAccessFlags flags)
158 {
159 return (flags & ~ALL_READ_ACCESS_FLAGS) > 0;
160 }
161
162 static bool
zink_resource_image_needs_barrier(struct zink_resource * res,VkImageLayout new_layout,VkAccessFlags flags,VkPipelineStageFlags pipeline)163 zink_resource_image_needs_barrier(struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
164 {
165 return res->layout != new_layout || (res->obj->access_stage & pipeline) != pipeline ||
166 (res->obj->access & flags) != flags ||
167 zink_resource_access_is_write(res->obj->access) ||
168 zink_resource_access_is_write(flags);
169 }
170
171 void
zink_resource_image_barrier_init(VkImageMemoryBarrier * imb,struct zink_resource * res,VkImageLayout new_layout,VkAccessFlags flags,VkPipelineStageFlags pipeline)172 zink_resource_image_barrier_init(VkImageMemoryBarrier *imb, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
173 {
174 if (!pipeline)
175 pipeline = pipeline_dst_stage(new_layout);
176 if (!flags)
177 flags = access_dst_flags(new_layout);
178
179 VkImageSubresourceRange isr = {
180 res->aspect,
181 0, VK_REMAINING_MIP_LEVELS,
182 0, VK_REMAINING_ARRAY_LAYERS
183 };
184 *imb = VkImageMemoryBarrier {
185 VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
186 NULL,
187 res->obj->access ? res->obj->access : access_src_flags(res->layout),
188 flags,
189 res->layout,
190 new_layout,
191 VK_QUEUE_FAMILY_IGNORED,
192 VK_QUEUE_FAMILY_IGNORED,
193 res->obj->image,
194 isr
195 };
196 }
197
198 void
zink_resource_image_barrier2_init(VkImageMemoryBarrier2 * imb,struct zink_resource * res,VkImageLayout new_layout,VkAccessFlags flags,VkPipelineStageFlags pipeline)199 zink_resource_image_barrier2_init(VkImageMemoryBarrier2 *imb, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
200 {
201 if (!pipeline)
202 pipeline = pipeline_dst_stage(new_layout);
203 if (!flags)
204 flags = access_dst_flags(new_layout);
205
206 VkImageSubresourceRange isr = {
207 res->aspect,
208 0, VK_REMAINING_MIP_LEVELS,
209 0, VK_REMAINING_ARRAY_LAYERS
210 };
211 *imb = VkImageMemoryBarrier2 {
212 VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
213 NULL,
214 res->obj->access_stage ? res->obj->access_stage : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
215 res->obj->access ? res->obj->access : access_src_flags(res->layout),
216 pipeline,
217 flags,
218 res->layout,
219 new_layout,
220 VK_QUEUE_FAMILY_IGNORED,
221 VK_QUEUE_FAMILY_IGNORED,
222 res->obj->image,
223 isr
224 };
225 }
226
227 static inline bool
is_shader_pipline_stage(VkPipelineStageFlags pipeline)228 is_shader_pipline_stage(VkPipelineStageFlags pipeline)
229 {
230 return pipeline & GFX_SHADER_BITS;
231 }
232
233 static void
resource_check_defer_buffer_barrier(struct zink_context * ctx,struct zink_resource * res,VkPipelineStageFlags pipeline)234 resource_check_defer_buffer_barrier(struct zink_context *ctx, struct zink_resource *res, VkPipelineStageFlags pipeline)
235 {
236 assert(res->obj->is_buffer);
237 if (res->bind_count[0] - res->so_bind_count > 0) {
238 if ((res->vbo_bind_mask && !(pipeline & VK_PIPELINE_STAGE_VERTEX_INPUT_BIT)) ||
239 (util_bitcount(res->vbo_bind_mask) != res->bind_count[0] && !is_shader_pipline_stage(pipeline)))
240 /* gfx rebind */
241 _mesa_set_add(ctx->need_barriers[0], res);
242 }
243 if (res->bind_count[1] && !(pipeline & VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT))
244 /* compute rebind */
245 _mesa_set_add(ctx->need_barriers[1], res);
246 }
247
248 static inline bool
unordered_res_exec(const struct zink_context * ctx,const struct zink_resource * res,bool is_write)249 unordered_res_exec(const struct zink_context *ctx, const struct zink_resource *res, bool is_write)
250 {
251 /* if all usage is unordered, keep unordered */
252 if (res->obj->unordered_read && res->obj->unordered_write)
253 return true;
254 /* if testing write access but have any ordered read access, cannot promote */
255 if (is_write && zink_batch_usage_matches(res->obj->bo->reads.u, ctx->batch.state) && !res->obj->unordered_read)
256 return false;
257 /* if write access is unordered or nonexistent, always promote */
258 return res->obj->unordered_write || !zink_batch_usage_matches(res->obj->bo->writes.u, ctx->batch.state);
259 }
260
261 static ALWAYS_INLINE bool
check_unordered_exec(struct zink_context * ctx,struct zink_resource * res,bool is_write)262 check_unordered_exec(struct zink_context *ctx, struct zink_resource *res, bool is_write)
263 {
264 if (res) {
265 if (!res->obj->is_buffer) {
266 /* TODO: figure out how to link up unordered layout -> ordered layout and delete this conditionals */
267 if (zink_resource_usage_is_unflushed(res) && !res->obj->unordered_read && !res->obj->unordered_write)
268 return false;
269 }
270 return unordered_res_exec(ctx, res, is_write);
271 }
272 return true;
273 }
274
275 VkCommandBuffer
zink_get_cmdbuf(struct zink_context * ctx,struct zink_resource * src,struct zink_resource * dst)276 zink_get_cmdbuf(struct zink_context *ctx, struct zink_resource *src, struct zink_resource *dst)
277 {
278 bool unordered_exec = (zink_debug & ZINK_DEBUG_NOREORDER) == 0;
279
280 unordered_exec &= check_unordered_exec(ctx, src, false) &&
281 check_unordered_exec(ctx, dst, true);
282
283 if (src)
284 src->obj->unordered_read = unordered_exec;
285 if (dst)
286 dst->obj->unordered_write = unordered_exec;
287
288 if (!unordered_exec || ctx->unordered_blitting)
289 zink_batch_no_rp(ctx);
290
291 if (unordered_exec) {
292 ctx->batch.state->has_barriers = true;
293 ctx->batch.has_work = true;
294 return ctx->batch.state->reordered_cmdbuf;
295 }
296 return ctx->batch.state->cmdbuf;
297 }
298
299 static void
resource_check_defer_image_barrier(struct zink_context * ctx,struct zink_resource * res,VkImageLayout layout,VkPipelineStageFlags pipeline)300 resource_check_defer_image_barrier(struct zink_context *ctx, struct zink_resource *res, VkImageLayout layout, VkPipelineStageFlags pipeline)
301 {
302 assert(!res->obj->is_buffer);
303 assert(!ctx->blitting);
304
305 bool is_compute = pipeline == VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
306 /* if this is a non-shader barrier and there are binds, always queue a shader barrier */
307 bool is_shader = is_shader_pipline_stage(pipeline);
308 if ((is_shader || !res->bind_count[is_compute]) &&
309 /* if no layout change is needed between gfx and compute, do nothing */
310 !res->bind_count[!is_compute] && (!is_compute || !res->fb_bind_count))
311 return;
312
313 if (res->bind_count[!is_compute] && is_shader) {
314 /* if the layout is the same between gfx and compute, do nothing */
315 if (layout == zink_descriptor_util_image_layout_eval(ctx, res, !is_compute))
316 return;
317 }
318 /* queue a layout change if a layout change will be needed */
319 if (res->bind_count[!is_compute])
320 _mesa_set_add(ctx->need_barriers[!is_compute], res);
321 /* also queue a layout change if this is a non-shader layout */
322 if (res->bind_count[is_compute] && !is_shader)
323 _mesa_set_add(ctx->need_barriers[is_compute], res);
324 }
325
326 enum barrier_type {
327 barrier_default,
328 barrier_KHR_synchronzation2
329 };
330
331 template <barrier_type BARRIER_API>
332 struct emit_memory_barrier {
for_imageemit_memory_barrier333 static void for_image(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout,
334 VkAccessFlags flags, VkPipelineStageFlags pipeline, bool completed, VkCommandBuffer cmdbuf,
335 bool *queue_import)
336 {
337 VkImageMemoryBarrier imb;
338 zink_resource_image_barrier_init(&imb, res, new_layout, flags, pipeline);
339 if (!res->obj->access_stage || completed)
340 imb.srcAccessMask = 0;
341 if (res->obj->needs_zs_evaluate)
342 imb.pNext = &res->obj->zs_evaluate;
343 res->obj->needs_zs_evaluate = false;
344 if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) {
345 imb.srcQueueFamilyIndex = res->queue;
346 imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue;
347 res->queue = VK_QUEUE_FAMILY_IGNORED;
348 *queue_import = true;
349 }
350 VKCTX(CmdPipelineBarrier)(
351 cmdbuf,
352 res->obj->access_stage ? res->obj->access_stage : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
353 pipeline,
354 0,
355 0, NULL,
356 0, NULL,
357 1, &imb
358 );
359 }
360
for_bufferemit_memory_barrier361 static void for_buffer(struct zink_context *ctx, struct zink_resource *res,
362 VkPipelineStageFlags pipeline,
363 VkAccessFlags flags,
364 bool unordered,
365 bool usage_matches,
366 VkPipelineStageFlags stages,
367 VkCommandBuffer cmdbuf)
368 {
369 VkMemoryBarrier bmb;
370 bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
371 bmb.pNext = NULL;
372 if (unordered) {
373 stages = usage_matches ? res->obj->unordered_access_stage : stages;
374 bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access;
375 } else {
376 bmb.srcAccessMask = res->obj->access;
377 }
378 VKCTX(CmdPipelineBarrier)(
379 cmdbuf,
380 stages,
381 pipeline,
382 0,
383 1, &bmb,
384 0, NULL,
385 0, NULL);
386 }
387 };
388
389
390 template <>
391 struct emit_memory_barrier<barrier_KHR_synchronzation2> {
for_imageemit_memory_barrier392 static void for_image(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout,
393 VkAccessFlags flags, VkPipelineStageFlags pipeline, bool completed, VkCommandBuffer cmdbuf,
394 bool *queue_import)
395 {
396 VkImageMemoryBarrier2 imb;
397 zink_resource_image_barrier2_init(&imb, res, new_layout, flags, pipeline);
398 if (!res->obj->access_stage || completed)
399 imb.srcAccessMask = 0;
400 if (res->obj->needs_zs_evaluate)
401 imb.pNext = &res->obj->zs_evaluate;
402 res->obj->needs_zs_evaluate = false;
403 if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) {
404 imb.srcQueueFamilyIndex = res->queue;
405 imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue;
406 res->queue = VK_QUEUE_FAMILY_IGNORED;
407 *queue_import = true;
408 }
409 VkDependencyInfo dep = {
410 VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
411 NULL,
412 0,
413 0,
414 NULL,
415 0,
416 NULL,
417 1,
418 &imb
419 };
420 VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep);
421 }
422
for_bufferemit_memory_barrier423 static void for_buffer(struct zink_context *ctx, struct zink_resource *res,
424 VkPipelineStageFlags pipeline,
425 VkAccessFlags flags,
426 bool unordered,
427 bool usage_matches,
428 VkPipelineStageFlags stages,
429 VkCommandBuffer cmdbuf)
430 {
431 VkMemoryBarrier2 bmb;
432 bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2;
433 bmb.pNext = NULL;
434 if (unordered) {
435 bmb.srcStageMask = usage_matches ? res->obj->unordered_access_stage : stages;
436 bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access;
437 } else {
438 bmb.srcStageMask = stages;
439 bmb.srcAccessMask = res->obj->access;
440 }
441 bmb.dstStageMask = pipeline;
442 bmb.dstAccessMask = flags;
443 VkDependencyInfo dep = {
444 VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
445 NULL,
446 0,
447 1,
448 &bmb,
449 0,
450 NULL,
451 0,
452 NULL
453 };
454 VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep);
455 }
456 };
457
458 template <bool UNSYNCHRONIZED>
459 struct update_unordered_access_and_get_cmdbuf {
460 /* use base template to make the cases for true and false more explicite below */
461 };
462
463 template <>
464 struct update_unordered_access_and_get_cmdbuf<true> {
applyupdate_unordered_access_and_get_cmdbuf465 static VkCommandBuffer apply(struct zink_context *ctx, struct zink_resource *res, bool usage_matches, bool is_write)
466 {
467 assert(!usage_matches);
468 res->obj->unordered_write = true;
469 res->obj->unordered_read = true;
470 ctx->batch.state->has_unsync = true;
471 return ctx->batch.state->unsynchronized_cmdbuf;
472 }
473 };
474
475 template <>
476 struct update_unordered_access_and_get_cmdbuf<false> {
applyupdate_unordered_access_and_get_cmdbuf477 static VkCommandBuffer apply(struct zink_context *ctx, struct zink_resource *res, bool usage_matches, bool is_write)
478 {
479 VkCommandBuffer cmdbuf;
480 if (!usage_matches) {
481 res->obj->unordered_write = true;
482 if (is_write || zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, ZINK_RESOURCE_ACCESS_RW))
483 res->obj->unordered_read = true;
484 }
485 if (zink_resource_usage_matches(res, ctx->batch.state) && !ctx->unordered_blitting &&
486 /* if current batch usage exists with ordered non-transfer access, never promote
487 * this avoids layout dsync
488 */
489 (!res->obj->unordered_read || !res->obj->unordered_write)) {
490 cmdbuf = ctx->batch.state->cmdbuf;
491 res->obj->unordered_write = false;
492 res->obj->unordered_read = false;
493 /* it's impossible to detect this from the caller
494 * there should be no valid case where this barrier can occur inside a renderpass
495 */
496 zink_batch_no_rp(ctx);
497 } else {
498 cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL);
499 /* force subsequent barriers to be ordered to avoid layout desync */
500 if (cmdbuf != ctx->batch.state->reordered_cmdbuf) {
501 res->obj->unordered_write = false;
502 res->obj->unordered_read = false;
503 }
504 }
505 return cmdbuf;
506 }
507 };
508
509 template <barrier_type BARRIER_API, bool UNSYNCHRONIZED>
510 void
zink_resource_image_barrier(struct zink_context * ctx,struct zink_resource * res,VkImageLayout new_layout,VkAccessFlags flags,VkPipelineStageFlags pipeline)511 zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
512 {
513 if (!pipeline)
514 pipeline = pipeline_dst_stage(new_layout);
515 if (!flags)
516 flags = access_dst_flags(new_layout);
517
518 bool is_write = zink_resource_access_is_write(flags);
519 if (is_write && zink_is_swapchain(res))
520 zink_kopper_set_readback_needs_update(res);
521 if (!res->obj->needs_zs_evaluate && !zink_resource_image_needs_barrier(res, new_layout, flags, pipeline) &&
522 (res->queue == zink_screen(ctx->base.screen)->gfx_queue || res->queue == VK_QUEUE_FAMILY_IGNORED))
523 return;
524 enum zink_resource_access rw = is_write ? ZINK_RESOURCE_ACCESS_RW : ZINK_RESOURCE_ACCESS_WRITE;
525 bool completed = zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, rw);
526 bool usage_matches = !completed && zink_resource_usage_matches(res, ctx->batch.state);
527 VkCommandBuffer cmdbuf = update_unordered_access_and_get_cmdbuf<UNSYNCHRONIZED>::apply(ctx, res, usage_matches, is_write);
528
529 assert(new_layout);
530 bool marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "image_barrier(%s->%s)", vk_ImageLayout_to_str(res->layout), vk_ImageLayout_to_str(new_layout));
531 bool queue_import = false;
532 emit_memory_barrier<BARRIER_API>::for_image(ctx, res, new_layout, flags, pipeline, completed, cmdbuf, &queue_import);
533 zink_cmd_debug_marker_end(ctx, cmdbuf, marker);
534
535 if (!UNSYNCHRONIZED)
536 resource_check_defer_image_barrier(ctx, res, new_layout, pipeline);
537
538 if (is_write)
539 res->obj->last_write = flags;
540
541 res->obj->access = flags;
542 res->obj->access_stage = pipeline;
543 res->layout = new_layout;
544
545 if (new_layout != VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL)
546 zink_resource_copies_reset(res);
547
548 if (res->obj->exportable)
549 simple_mtx_lock(&ctx->batch.state->exportable_lock);
550 if (res->obj->dt) {
551 struct kopper_displaytarget *cdt = res->obj->dt;
552 if (cdt->swapchain->num_acquires && res->obj->dt_idx != UINT32_MAX) {
553 cdt->swapchain->images[res->obj->dt_idx].layout = res->layout;
554 }
555 } else if (res->obj->exportable) {
556 struct pipe_resource *pres = NULL;
557 bool found = false;
558 _mesa_set_search_or_add(&ctx->batch.state->dmabuf_exports, res, &found);
559 if (!found) {
560 pipe_resource_reference(&pres, &res->base.b);
561 }
562 }
563 if (res->obj->exportable && queue_import) {
564 for (struct zink_resource *r = res; r; r = zink_resource(r->base.b.next)) {
565 VkSemaphore sem = zink_screen_export_dmabuf_semaphore(zink_screen(ctx->base.screen), r);
566 if (sem)
567 util_dynarray_append(&ctx->batch.state->fd_wait_semaphores, VkSemaphore, sem);
568 }
569 }
570 if (res->obj->exportable)
571 simple_mtx_unlock(&ctx->batch.state->exportable_lock);
572 }
573
574 bool
zink_check_unordered_transfer_access(struct zink_resource * res,unsigned level,const struct pipe_box * box)575 zink_check_unordered_transfer_access(struct zink_resource *res, unsigned level, const struct pipe_box *box)
576 {
577 /* always barrier against previous non-transfer writes */
578 bool non_transfer_write = res->obj->last_write && res->obj->last_write != VK_ACCESS_TRANSFER_WRITE_BIT;
579 /* must barrier if clobbering a previous write */
580 bool transfer_clobber = res->obj->last_write == VK_ACCESS_TRANSFER_WRITE_BIT && zink_resource_copy_box_intersects(res, level, box);
581 return non_transfer_write || transfer_clobber;
582 }
583
584 bool
zink_check_valid_buffer_src_access(struct zink_context * ctx,struct zink_resource * res,unsigned offset,unsigned size)585 zink_check_valid_buffer_src_access(struct zink_context *ctx, struct zink_resource *res, unsigned offset, unsigned size)
586 {
587 return res->obj->access && util_ranges_intersect(&res->valid_buffer_range, offset, offset + size) && !unordered_res_exec(ctx, res, false);
588 }
589
590 void
zink_resource_image_transfer_dst_barrier(struct zink_context * ctx,struct zink_resource * res,unsigned level,const struct pipe_box * box,bool unsync)591 zink_resource_image_transfer_dst_barrier(struct zink_context *ctx, struct zink_resource *res, unsigned level, const struct pipe_box *box, bool unsync)
592 {
593 if (res->obj->copies_need_reset)
594 zink_resource_copies_reset(res);
595 /* skip TRANSFER_DST barrier if no intersection from previous copies */
596 if (res->layout != VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL ||
597 zink_screen(ctx->base.screen)->driver_workarounds.broken_cache_semantics ||
598 zink_check_unordered_transfer_access(res, level, box)) {
599 if (unsync)
600 zink_screen(ctx->base.screen)->image_barrier_unsync(ctx, res, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT);
601 else
602 zink_screen(ctx->base.screen)->image_barrier(ctx, res, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT);
603 } else {
604 res->obj->access = VK_ACCESS_TRANSFER_WRITE_BIT;
605 res->obj->last_write = VK_ACCESS_TRANSFER_WRITE_BIT;
606 res->obj->access_stage = VK_PIPELINE_STAGE_TRANSFER_BIT;
607 }
608 zink_resource_copy_box_add(ctx, res, level, box);
609 }
610
611 bool
zink_resource_buffer_transfer_dst_barrier(struct zink_context * ctx,struct zink_resource * res,unsigned offset,unsigned size)612 zink_resource_buffer_transfer_dst_barrier(struct zink_context *ctx, struct zink_resource *res, unsigned offset, unsigned size)
613 {
614 if (res->obj->copies_need_reset)
615 zink_resource_copies_reset(res);
616 bool unordered = true;
617 struct pipe_box box = {(int)offset, 0, 0, (int)size, 0, 0};
618 bool can_unordered_write = unordered_res_exec(ctx, res, true);
619 /* must barrier if something read the valid buffer range */
620 bool valid_read = (res->obj->access || res->obj->unordered_access) &&
621 util_ranges_intersect(&res->valid_buffer_range, offset, offset + size) && !can_unordered_write;
622 if (valid_read || zink_screen(ctx->base.screen)->driver_workarounds.broken_cache_semantics || zink_check_unordered_transfer_access(res, 0, &box)) {
623 zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT);
624 unordered = res->obj->unordered_write;
625 } else {
626 res->obj->unordered_access = VK_ACCESS_TRANSFER_WRITE_BIT;
627 res->obj->last_write = VK_ACCESS_TRANSFER_WRITE_BIT;
628 res->obj->unordered_access_stage = VK_PIPELINE_STAGE_TRANSFER_BIT;
629
630 ctx->batch.state->unordered_write_access |= VK_ACCESS_TRANSFER_WRITE_BIT;
631 ctx->batch.state->unordered_write_stages |= VK_PIPELINE_STAGE_TRANSFER_BIT;
632 if (!zink_resource_usage_matches(res, ctx->batch.state)) {
633 res->obj->access = VK_ACCESS_TRANSFER_WRITE_BIT;
634 res->obj->access_stage = VK_PIPELINE_STAGE_TRANSFER_BIT;
635 res->obj->ordered_access_is_copied = true;
636 }
637 }
638 zink_resource_copy_box_add(ctx, res, 0, &box);
639 /* this return value implies that the caller could do an unordered op on this resource */
640 return unordered;
641 }
642
643 VkPipelineStageFlags
zink_pipeline_flags_from_stage(VkShaderStageFlagBits stage)644 zink_pipeline_flags_from_stage(VkShaderStageFlagBits stage)
645 {
646 switch (stage) {
647 case VK_SHADER_STAGE_VERTEX_BIT:
648 return VK_PIPELINE_STAGE_VERTEX_SHADER_BIT;
649 case VK_SHADER_STAGE_FRAGMENT_BIT:
650 return VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
651 case VK_SHADER_STAGE_GEOMETRY_BIT:
652 return VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT;
653 case VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT:
654 return VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT;
655 case VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT:
656 return VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT;
657 case VK_SHADER_STAGE_COMPUTE_BIT:
658 return VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
659 default:
660 unreachable("unknown shader stage bit");
661 }
662 }
663
664 ALWAYS_INLINE static VkPipelineStageFlags
pipeline_access_stage(VkAccessFlags flags)665 pipeline_access_stage(VkAccessFlags flags)
666 {
667 if (flags & (VK_ACCESS_UNIFORM_READ_BIT |
668 VK_ACCESS_SHADER_READ_BIT |
669 VK_ACCESS_SHADER_WRITE_BIT))
670 return VK_PIPELINE_STAGE_VERTEX_SHADER_BIT |
671 VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT |
672 VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT |
673 VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT |
674 VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT |
675 VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
676 return VK_PIPELINE_STAGE_TRANSFER_BIT;
677 }
678
679 ALWAYS_INLINE static bool
buffer_needs_barrier(struct zink_resource * res,VkAccessFlags flags,VkPipelineStageFlags pipeline,bool unordered)680 buffer_needs_barrier(struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline, bool unordered)
681 {
682 return zink_resource_access_is_write(unordered ? res->obj->unordered_access : res->obj->access) ||
683 zink_resource_access_is_write(flags) ||
684 ((unordered ? res->obj->unordered_access_stage : res->obj->access_stage) & pipeline) != pipeline ||
685 ((unordered ? res->obj->unordered_access : res->obj->access) & flags) != flags;
686 }
687
688
689
690 template <barrier_type BARRIER_API>
691 void
zink_resource_buffer_barrier(struct zink_context * ctx,struct zink_resource * res,VkAccessFlags flags,VkPipelineStageFlags pipeline)692 zink_resource_buffer_barrier(struct zink_context *ctx, struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline)
693 {
694 if (!pipeline)
695 pipeline = pipeline_access_stage(flags);
696
697 bool is_write = zink_resource_access_is_write(flags);
698 enum zink_resource_access rw = is_write ? ZINK_RESOURCE_ACCESS_RW : ZINK_RESOURCE_ACCESS_WRITE;
699 bool completed = zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, rw);
700 bool usage_matches = !completed && zink_resource_usage_matches(res, ctx->batch.state);
701 if (!usage_matches) {
702 res->obj->unordered_write = true;
703 if (is_write || zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, ZINK_RESOURCE_ACCESS_RW))
704 res->obj->unordered_read = true;
705 }
706 bool unordered_usage_matches = res->obj->unordered_access && usage_matches;
707 bool unordered = unordered_res_exec(ctx, res, is_write);
708 if (!buffer_needs_barrier(res, flags, pipeline, unordered))
709 return;
710 if (completed) {
711 /* reset access on complete */
712 res->obj->access = VK_ACCESS_NONE;
713 res->obj->access_stage = VK_PIPELINE_STAGE_NONE;
714 res->obj->last_write = VK_ACCESS_NONE;
715 } else if (unordered && unordered_usage_matches && res->obj->ordered_access_is_copied) {
716 /* always reset propagated access to avoid weirdness */
717 res->obj->access = VK_ACCESS_NONE;
718 res->obj->access_stage = VK_PIPELINE_STAGE_NONE;
719 } else if (!unordered && !unordered_usage_matches) {
720 /* reset unordered access on first ordered barrier */
721 res->obj->unordered_access = VK_ACCESS_NONE;
722 res->obj->unordered_access_stage = VK_PIPELINE_STAGE_NONE;
723 }
724 if (!usage_matches) {
725 /* reset unordered on first new cmdbuf barrier */
726 res->obj->unordered_access = VK_ACCESS_NONE;
727 res->obj->unordered_access_stage = VK_PIPELINE_STAGE_NONE;
728 res->obj->ordered_access_is_copied = false;
729 }
730 /* unordered barriers can be skipped when:
731 * - there is no current-batch unordered access AND previous batch usage is not write access
732 * - there is current-batch unordered access AND the unordered access is not write access
733 */
734 bool can_skip_unordered = !unordered ? false : !zink_resource_access_is_write(!unordered_usage_matches ? res->obj->access : res->obj->unordered_access);
735 /* ordered barriers can be skipped if both:
736 * - there is no current access
737 * - there is no current-batch unordered access
738 */
739 bool can_skip_ordered = unordered ? false : (!res->obj->access && !unordered_usage_matches);
740 if (zink_debug & ZINK_DEBUG_NOREORDER)
741 can_skip_unordered = can_skip_ordered = false;
742
743 if (!can_skip_unordered && !can_skip_ordered) {
744 VkCommandBuffer cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL);
745 bool marker = false;
746 if (unlikely(zink_tracing)) {
747 char buf[4096];
748 zink_string_vkflags_unroll(buf, sizeof(buf), flags, (zink_vkflags_func)vk_AccessFlagBits_to_str);
749 marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "buffer_barrier(%s)", buf);
750 }
751
752 VkPipelineStageFlags stages = res->obj->access_stage ? res->obj->access_stage : pipeline_access_stage(res->obj->access);;
753 emit_memory_barrier<BARRIER_API>::for_buffer(ctx, res, pipeline, flags, unordered,usage_matches, stages, cmdbuf);
754
755 zink_cmd_debug_marker_end(ctx, cmdbuf, marker);
756 }
757
758 resource_check_defer_buffer_barrier(ctx, res, pipeline);
759
760 if (is_write)
761 res->obj->last_write = flags;
762 if (unordered) {
763 /* these should get automatically emitted during submission */
764 res->obj->unordered_access = flags;
765 res->obj->unordered_access_stage = pipeline;
766 if (is_write) {
767 ctx->batch.state->unordered_write_access |= flags;
768 ctx->batch.state->unordered_write_stages |= pipeline;
769 }
770 }
771 if (!unordered || !usage_matches || res->obj->ordered_access_is_copied) {
772 res->obj->access = flags;
773 res->obj->access_stage = pipeline;
774 res->obj->ordered_access_is_copied = unordered;
775 }
776 if (pipeline != VK_PIPELINE_STAGE_TRANSFER_BIT && is_write)
777 zink_resource_copies_reset(res);
778 }
779
780 void
zink_synchronization_init(struct zink_screen * screen)781 zink_synchronization_init(struct zink_screen *screen)
782 {
783 if (screen->info.have_vulkan13 || screen->info.have_KHR_synchronization2) {
784 screen->buffer_barrier = zink_resource_buffer_barrier<barrier_KHR_synchronzation2>;
785 screen->image_barrier = zink_resource_image_barrier<barrier_KHR_synchronzation2, false>;
786 screen->image_barrier_unsync = zink_resource_image_barrier<barrier_KHR_synchronzation2, true>;
787 } else {
788 screen->buffer_barrier = zink_resource_buffer_barrier<barrier_default>;
789 screen->image_barrier = zink_resource_image_barrier<barrier_default, false>;
790 screen->image_barrier_unsync = zink_resource_image_barrier<barrier_default, true>;
791 }
792 }
793