1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "anv_private.h"
25 #include "genxml/gen80_pack.h"
26
27 static bool
lookup_blorp_shader(struct blorp_batch * batch,const void * key,uint32_t key_size,uint32_t * kernel_out,void * prog_data_out)28 lookup_blorp_shader(struct blorp_batch *batch,
29 const void *key, uint32_t key_size,
30 uint32_t *kernel_out, void *prog_data_out)
31 {
32 struct blorp_context *blorp = batch->blorp;
33 struct anv_device *device = blorp->driver_ctx;
34
35 struct anv_shader_bin *bin =
36 anv_device_search_for_kernel(device, device->internal_cache,
37 key, key_size, NULL);
38 if (!bin)
39 return false;
40
41 /* The cache already has a reference and it's not going anywhere so there
42 * is no need to hold a second reference.
43 */
44 anv_shader_bin_unref(device, bin);
45
46 *kernel_out = bin->kernel.offset;
47 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
48
49 return true;
50 }
51
52 static bool
upload_blorp_shader(struct blorp_batch * batch,uint32_t stage,const void * key,uint32_t key_size,const void * kernel,uint32_t kernel_size,const void * prog_data,uint32_t prog_data_size,uint32_t * kernel_out,void * prog_data_out)53 upload_blorp_shader(struct blorp_batch *batch, uint32_t stage,
54 const void *key, uint32_t key_size,
55 const void *kernel, uint32_t kernel_size,
56 const void *prog_data,
57 uint32_t prog_data_size,
58 uint32_t *kernel_out, void *prog_data_out)
59 {
60 struct blorp_context *blorp = batch->blorp;
61 struct anv_device *device = blorp->driver_ctx;
62
63 struct anv_pipeline_bind_map empty_bind_map = {};
64 struct anv_push_descriptor_info empty_push_desc_info = {};
65 struct anv_shader_upload_params upload_params = {
66 .stage = stage,
67 .key_data = key,
68 .key_size = key_size,
69 .kernel_data = kernel,
70 .kernel_size = kernel_size,
71 .prog_data = prog_data,
72 .prog_data_size = prog_data_size,
73 .bind_map = &empty_bind_map,
74 .push_desc_info = &empty_push_desc_info,
75 };
76
77 struct anv_shader_bin *bin =
78 anv_device_upload_kernel(device, device->internal_cache, &upload_params);
79
80 if (!bin)
81 return false;
82
83 /* The cache already has a reference and it's not going anywhere so there
84 * is no need to hold a second reference.
85 */
86 anv_shader_bin_unref(device, bin);
87
88 *kernel_out = bin->kernel.offset;
89 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
90
91 return true;
92 }
93
94 static void
upload_dynamic_state(struct blorp_context * context,const void * data,uint32_t size,uint32_t alignment,enum blorp_dynamic_state name)95 upload_dynamic_state(struct blorp_context *context,
96 const void *data, uint32_t size,
97 uint32_t alignment, enum blorp_dynamic_state name)
98 {
99 struct anv_device *device = context->driver_ctx;
100
101 device->blorp.dynamic_states[name] =
102 anv_state_pool_emit_data(&device->dynamic_state_pool,
103 size, alignment, data);
104 }
105
106 void
anv_device_init_blorp(struct anv_device * device)107 anv_device_init_blorp(struct anv_device *device)
108 {
109 const struct blorp_config config = {
110 .use_mesh_shading = device->vk.enabled_extensions.EXT_mesh_shader,
111 .use_unrestricted_depth_range =
112 device->vk.enabled_extensions.EXT_depth_range_unrestricted,
113 .use_cached_dynamic_states = true,
114 };
115
116 blorp_init_brw(&device->blorp.context, device, &device->isl_dev,
117 device->physical->compiler, &config);
118 device->blorp.context.lookup_shader = lookup_blorp_shader;
119 device->blorp.context.upload_shader = upload_blorp_shader;
120 device->blorp.context.enable_tbimr = device->physical->instance->enable_tbimr;
121 device->blorp.context.exec = anv_genX(device->info, blorp_exec);
122 device->blorp.context.upload_dynamic_state = upload_dynamic_state;
123
124 anv_genX(device->info, blorp_init_dynamic_states)(&device->blorp.context);
125 }
126
127 void
anv_device_finish_blorp(struct anv_device * device)128 anv_device_finish_blorp(struct anv_device *device)
129 {
130 #ifdef HAVE_VALGRIND
131 /* We only need to free these to prevent valgrind errors. The backing
132 * BO will go away in a couple of lines so we don't actually leak.
133 */
134 for (uint32_t i = 0; i < ARRAY_SIZE(device->blorp.dynamic_states); i++) {
135 anv_state_pool_free(&device->dynamic_state_pool,
136 device->blorp.dynamic_states[i]);
137 }
138 #endif
139 blorp_finish(&device->blorp.context);
140 }
141
142 static void
anv_blorp_batch_init(struct anv_cmd_buffer * cmd_buffer,struct blorp_batch * batch,enum blorp_batch_flags flags)143 anv_blorp_batch_init(struct anv_cmd_buffer *cmd_buffer,
144 struct blorp_batch *batch, enum blorp_batch_flags flags)
145 {
146 VkQueueFlags queue_flags = cmd_buffer->queue_family->queueFlags;
147
148 if (queue_flags & VK_QUEUE_GRAPHICS_BIT) {
149 /* blorp runs on render engine by default */
150 } else if (queue_flags & VK_QUEUE_COMPUTE_BIT) {
151 flags |= BLORP_BATCH_USE_COMPUTE;
152 } else if (queue_flags & VK_QUEUE_TRANSFER_BIT) {
153 flags |= BLORP_BATCH_USE_BLITTER;
154 } else {
155 unreachable("unknown queue family");
156 }
157
158 /* Can't have both flags at the same time. */
159 assert((flags & BLORP_BATCH_USE_BLITTER) == 0 ||
160 (flags & BLORP_BATCH_USE_COMPUTE) == 0);
161
162 blorp_batch_init(&cmd_buffer->device->blorp.context, batch, cmd_buffer, flags);
163 }
164
165 static void
anv_blorp_batch_finish(struct blorp_batch * batch)166 anv_blorp_batch_finish(struct blorp_batch *batch)
167 {
168 blorp_batch_finish(batch);
169 }
170
171 static isl_surf_usage_flags_t
get_usage_flag_for_cmd_buffer(const struct anv_cmd_buffer * cmd_buffer,bool is_dest,bool is_depth,bool protected)172 get_usage_flag_for_cmd_buffer(const struct anv_cmd_buffer *cmd_buffer,
173 bool is_dest, bool is_depth, bool protected)
174 {
175 isl_surf_usage_flags_t usage;
176
177 switch (cmd_buffer->queue_family->engine_class) {
178 case INTEL_ENGINE_CLASS_RENDER:
179 if (is_dest) {
180 /* Make the blorp operation match the MOCS used in
181 * cmd_buffer_emit_depth_stencil()
182 */
183 if (is_depth)
184 usage = ISL_SURF_USAGE_DEPTH_BIT;
185 else
186 usage = ISL_SURF_USAGE_RENDER_TARGET_BIT;
187 } else {
188 usage = ISL_SURF_USAGE_TEXTURE_BIT;
189 }
190 break;
191 case INTEL_ENGINE_CLASS_COMPUTE:
192 usage = is_dest ? ISL_SURF_USAGE_STORAGE_BIT :
193 ISL_SURF_USAGE_TEXTURE_BIT;
194 break;
195 case INTEL_ENGINE_CLASS_COPY:
196 usage = is_dest ? ISL_SURF_USAGE_BLITTER_DST_BIT :
197 ISL_SURF_USAGE_BLITTER_SRC_BIT;
198 break;
199 default:
200 unreachable("Unhandled engine class");
201 }
202
203 if (protected)
204 usage |= ISL_SURF_USAGE_PROTECTED_BIT;
205
206 return usage;
207 }
208
209 static void
get_blorp_surf_for_anv_address(struct anv_cmd_buffer * cmd_buffer,struct anv_address address,uint32_t width,uint32_t height,uint32_t row_pitch,enum isl_format format,bool is_dest,bool protected,struct blorp_surf * blorp_surf,struct isl_surf * isl_surf)210 get_blorp_surf_for_anv_address(struct anv_cmd_buffer *cmd_buffer,
211 struct anv_address address,
212 uint32_t width, uint32_t height,
213 uint32_t row_pitch, enum isl_format format,
214 bool is_dest, bool protected,
215 struct blorp_surf *blorp_surf,
216 struct isl_surf *isl_surf)
217 {
218 bool ok UNUSED;
219 isl_surf_usage_flags_t usage =
220 get_usage_flag_for_cmd_buffer(cmd_buffer, is_dest, false, protected);
221
222 *blorp_surf = (struct blorp_surf) {
223 .surf = isl_surf,
224 .addr = {
225 .buffer = address.bo,
226 .offset = address.offset,
227 .mocs = anv_mocs(cmd_buffer->device, address.bo, usage),
228 },
229 };
230
231 ok = isl_surf_init(&cmd_buffer->device->isl_dev, isl_surf,
232 .dim = ISL_SURF_DIM_2D,
233 .format = format,
234 .width = width,
235 .height = height,
236 .depth = 1,
237 .levels = 1,
238 .array_len = 1,
239 .samples = 1,
240 .row_pitch_B = row_pitch,
241 .usage = usage,
242 .tiling_flags = ISL_TILING_LINEAR_BIT);
243 assert(ok);
244 }
245
246 static void
get_blorp_surf_for_anv_buffer(struct anv_cmd_buffer * cmd_buffer,struct anv_buffer * buffer,uint64_t offset,uint32_t width,uint32_t height,uint32_t row_pitch,enum isl_format format,bool is_dest,struct blorp_surf * blorp_surf,struct isl_surf * isl_surf)247 get_blorp_surf_for_anv_buffer(struct anv_cmd_buffer *cmd_buffer,
248 struct anv_buffer *buffer, uint64_t offset,
249 uint32_t width, uint32_t height,
250 uint32_t row_pitch, enum isl_format format,
251 bool is_dest,
252 struct blorp_surf *blorp_surf,
253 struct isl_surf *isl_surf)
254 {
255 get_blorp_surf_for_anv_address(cmd_buffer,
256 anv_address_add(buffer->address, offset),
257 width, height, row_pitch, format,
258 is_dest, anv_buffer_is_protected(buffer),
259 blorp_surf, isl_surf);
260 }
261
262 /* Pick something high enough that it won't be used in core and low enough it
263 * will never map to an extension.
264 */
265 #define ANV_IMAGE_LAYOUT_EXPLICIT_AUX (VkImageLayout)10000000
266
267 static struct blorp_address
anv_to_blorp_address(struct anv_address addr)268 anv_to_blorp_address(struct anv_address addr)
269 {
270 return (struct blorp_address) {
271 .buffer = addr.bo,
272 .offset = addr.offset,
273 };
274 }
275
276 static void
get_blorp_surf_for_anv_image(const struct anv_cmd_buffer * cmd_buffer,const struct anv_image * image,VkImageAspectFlags aspect,VkImageUsageFlags usage,VkImageLayout layout,enum isl_aux_usage aux_usage,enum isl_format view_fmt,struct blorp_surf * blorp_surf)277 get_blorp_surf_for_anv_image(const struct anv_cmd_buffer *cmd_buffer,
278 const struct anv_image *image,
279 VkImageAspectFlags aspect,
280 VkImageUsageFlags usage,
281 VkImageLayout layout,
282 enum isl_aux_usage aux_usage,
283 enum isl_format view_fmt,
284 struct blorp_surf *blorp_surf)
285 {
286 const struct anv_device *device = cmd_buffer->device;
287 const uint32_t plane = anv_image_aspect_to_plane(image, aspect);
288
289 if (layout != ANV_IMAGE_LAYOUT_EXPLICIT_AUX) {
290 assert(usage != 0);
291 aux_usage = anv_layout_to_aux_usage(device->info, image,
292 aspect, usage, layout,
293 cmd_buffer->queue_family->queueFlags);
294 }
295
296 const bool is_dest = usage & VK_IMAGE_USAGE_TRANSFER_DST_BIT;
297 isl_surf_usage_flags_t isl_usage =
298 get_usage_flag_for_cmd_buffer(cmd_buffer, is_dest,
299 aspect & VK_IMAGE_ASPECT_DEPTH_BIT,
300 anv_image_is_protected(image));
301 const struct anv_surface *surface = &image->planes[plane].primary_surface;
302 const struct anv_address address =
303 anv_image_address(image, &surface->memory_range);
304
305 *blorp_surf = (struct blorp_surf) {
306 .surf = &surface->isl,
307 .addr = {
308 .buffer = address.bo,
309 .offset = address.offset,
310 .mocs = anv_mocs(device, address.bo, isl_usage),
311 },
312 };
313
314 if (aux_usage != ISL_AUX_USAGE_NONE) {
315 const struct anv_surface *aux_surface = &image->planes[plane].aux_surface;
316 const struct anv_address aux_address =
317 anv_image_address(image, &aux_surface->memory_range);
318
319 blorp_surf->aux_usage = aux_usage;
320 blorp_surf->aux_surf = &aux_surface->isl;
321
322 if (!anv_address_is_null(aux_address)) {
323 blorp_surf->aux_addr = (struct blorp_address) {
324 .buffer = aux_address.bo,
325 .offset = aux_address.offset,
326 .mocs = anv_mocs(device, aux_address.bo, isl_usage),
327 };
328 }
329
330 const struct anv_address clear_color_addr =
331 anv_image_get_clear_color_addr(device, image, view_fmt, aspect,
332 !is_dest);
333 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr);
334
335 if (aspect & VK_IMAGE_ASPECT_DEPTH_BIT)
336 blorp_surf->clear_color = anv_image_hiz_clear_value(image);
337 }
338 }
339
340 static void
copy_image(struct anv_cmd_buffer * cmd_buffer,struct blorp_batch * batch,struct anv_image * src_image,VkImageLayout src_image_layout,struct anv_image * dst_image,VkImageLayout dst_image_layout,const VkImageCopy2 * region)341 copy_image(struct anv_cmd_buffer *cmd_buffer,
342 struct blorp_batch *batch,
343 struct anv_image *src_image,
344 VkImageLayout src_image_layout,
345 struct anv_image *dst_image,
346 VkImageLayout dst_image_layout,
347 const VkImageCopy2 *region)
348 {
349 VkOffset3D srcOffset =
350 vk_image_sanitize_offset(&src_image->vk, region->srcOffset);
351 VkOffset3D dstOffset =
352 vk_image_sanitize_offset(&dst_image->vk, region->dstOffset);
353 VkExtent3D extent =
354 vk_image_sanitize_extent(&src_image->vk, region->extent);
355
356 const uint32_t dst_level = region->dstSubresource.mipLevel;
357 unsigned dst_base_layer, layer_count;
358 if (dst_image->vk.image_type == VK_IMAGE_TYPE_3D) {
359 dst_base_layer = region->dstOffset.z;
360 layer_count = region->extent.depth;
361 } else {
362 dst_base_layer = region->dstSubresource.baseArrayLayer;
363 layer_count = vk_image_subresource_layer_count(&dst_image->vk,
364 ®ion->dstSubresource);
365 }
366
367 const uint32_t src_level = region->srcSubresource.mipLevel;
368 unsigned src_base_layer;
369 if (src_image->vk.image_type == VK_IMAGE_TYPE_3D) {
370 src_base_layer = region->srcOffset.z;
371 } else {
372 src_base_layer = region->srcSubresource.baseArrayLayer;
373 assert(layer_count ==
374 vk_image_subresource_layer_count(&src_image->vk,
375 ®ion->srcSubresource));
376 }
377
378 VkImageAspectFlags src_mask = region->srcSubresource.aspectMask,
379 dst_mask = region->dstSubresource.aspectMask;
380
381 assert(anv_image_aspects_compatible(src_mask, dst_mask));
382
383 if (util_bitcount(src_mask) > 1) {
384 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_mask) {
385 enum isl_format src_format, dst_format;
386 int plane = anv_image_aspect_to_plane(src_image, 1UL << aspect_bit);
387 blorp_copy_get_formats(&cmd_buffer->device->isl_dev,
388 &src_image->planes[plane].primary_surface.isl,
389 &dst_image->planes[plane].primary_surface.isl,
390 &src_format, &dst_format);
391
392 struct blorp_surf src_surf, dst_surf;
393 get_blorp_surf_for_anv_image(cmd_buffer,
394 src_image, 1UL << aspect_bit,
395 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
396 src_image_layout, ISL_AUX_USAGE_NONE,
397 src_format, &src_surf);
398 get_blorp_surf_for_anv_image(cmd_buffer,
399 dst_image, 1UL << aspect_bit,
400 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
401 dst_image_layout, ISL_AUX_USAGE_NONE,
402 dst_format, &dst_surf);
403 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
404 1UL << aspect_bit,
405 dst_surf.aux_usage, dst_level,
406 dst_base_layer, layer_count);
407
408 for (unsigned i = 0; i < layer_count; i++) {
409 blorp_copy(batch, &src_surf, src_level, src_base_layer + i,
410 &dst_surf, dst_level, dst_base_layer + i,
411 srcOffset.x, srcOffset.y,
412 dstOffset.x, dstOffset.y,
413 extent.width, extent.height);
414 }
415 }
416 } else {
417 /* This case handles the ycbcr images, aspect mask are compatible but
418 * don't need to be the same.
419 */
420 enum isl_format src_format, dst_format;
421 int s_plane = anv_image_aspect_to_plane(src_image, src_mask);
422 int d_plane = anv_image_aspect_to_plane(dst_image, dst_mask);
423 blorp_copy_get_formats(&cmd_buffer->device->isl_dev,
424 &src_image->planes[s_plane].primary_surface.isl,
425 &dst_image->planes[d_plane].primary_surface.isl,
426 &src_format, &dst_format);
427
428 struct blorp_surf src_surf, dst_surf;
429 get_blorp_surf_for_anv_image(cmd_buffer, src_image, src_mask,
430 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
431 src_image_layout, ISL_AUX_USAGE_NONE,
432 src_format, &src_surf);
433 get_blorp_surf_for_anv_image(cmd_buffer, dst_image, dst_mask,
434 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
435 dst_image_layout, ISL_AUX_USAGE_NONE,
436 dst_format, &dst_surf);
437 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, dst_mask,
438 dst_surf.aux_usage, dst_level,
439 dst_base_layer, layer_count);
440
441 for (unsigned i = 0; i < layer_count; i++) {
442 blorp_copy(batch, &src_surf, src_level, src_base_layer + i,
443 &dst_surf, dst_level, dst_base_layer + i,
444 srcOffset.x, srcOffset.y,
445 dstOffset.x, dstOffset.y,
446 extent.width, extent.height);
447 }
448 }
449 }
450
451 static struct anv_state
record_main_rcs_cmd_buffer_done(struct anv_cmd_buffer * cmd_buffer)452 record_main_rcs_cmd_buffer_done(struct anv_cmd_buffer *cmd_buffer)
453 {
454 const struct intel_device_info *info = cmd_buffer->device->info;
455
456 const VkResult result = anv_cmd_buffer_ensure_rcs_companion(cmd_buffer);
457 if (result != VK_SUCCESS) {
458 anv_batch_set_error(&cmd_buffer->batch, result);
459 return ANV_STATE_NULL;
460 }
461
462 assert(cmd_buffer->companion_rcs_cmd_buffer != NULL);
463
464 /* Re-emit the aux table register in every command buffer. This way we're
465 * ensured that we have the table even if this command buffer doesn't
466 * initialize any images.
467 */
468 if (cmd_buffer->device->info->has_aux_map) {
469 anv_add_pending_pipe_bits(cmd_buffer->companion_rcs_cmd_buffer,
470 ANV_PIPE_AUX_TABLE_INVALIDATE_BIT,
471 "new cmd buffer with aux-tt");
472 }
473
474 return anv_genX(info, cmd_buffer_begin_companion_rcs_syncpoint)(cmd_buffer);
475 }
476
477 static void
end_main_rcs_cmd_buffer_done(struct anv_cmd_buffer * cmd_buffer,struct anv_state syncpoint)478 end_main_rcs_cmd_buffer_done(struct anv_cmd_buffer *cmd_buffer,
479 struct anv_state syncpoint)
480 {
481 const struct intel_device_info *info = cmd_buffer->device->info;
482 anv_genX(info, cmd_buffer_end_companion_rcs_syncpoint)(cmd_buffer,
483 syncpoint);
484 }
485
486 static bool
anv_blorp_blitter_execute_on_companion(struct anv_cmd_buffer * cmd_buffer,struct anv_image * image,const VkCopyBufferToImageInfo2 * pCopyBufferToImageInfo,const VkCopyImageToBufferInfo2 * pCopyImageToBufferInfo)487 anv_blorp_blitter_execute_on_companion(struct anv_cmd_buffer *cmd_buffer,
488 struct anv_image *image,
489 const VkCopyBufferToImageInfo2* pCopyBufferToImageInfo,
490 const VkCopyImageToBufferInfo2* pCopyImageToBufferInfo)
491 {
492 if (!anv_cmd_buffer_is_blitter_queue(cmd_buffer))
493 return false;
494
495 assert((pCopyBufferToImageInfo && !pCopyImageToBufferInfo) ||
496 (pCopyImageToBufferInfo && !pCopyBufferToImageInfo));
497
498 bool blorp_execute_on_companion = false;
499 VkImageAspectFlags aspect_mask = VK_IMAGE_ASPECT_NONE;
500 const uint32_t region_count = pCopyBufferToImageInfo ?
501 pCopyBufferToImageInfo->regionCount :
502 pCopyImageToBufferInfo->regionCount;
503
504 for (unsigned r = 0; r < region_count &&
505 !blorp_execute_on_companion; r++) {
506 if (pCopyBufferToImageInfo) {
507 aspect_mask =
508 pCopyBufferToImageInfo->pRegions[r].imageSubresource.aspectMask;
509 } else {
510 aspect_mask =
511 pCopyImageToBufferInfo->pRegions[r].imageSubresource.aspectMask;
512 }
513
514 enum isl_format linear_format =
515 anv_get_isl_format(cmd_buffer->device->physical, image->vk.format,
516 aspect_mask, VK_IMAGE_TILING_LINEAR);
517 const struct isl_format_layout *linear_fmtl =
518 isl_format_get_layout(linear_format);
519
520 switch (linear_fmtl->bpb) {
521 case 96:
522 /* We can only support linear mode for 96bpp on blitter engine. */
523 blorp_execute_on_companion |=
524 image->vk.tiling != VK_IMAGE_TILING_LINEAR;
525 break;
526 default:
527 blorp_execute_on_companion |= linear_fmtl->bpb % 3 == 0;
528 break;
529 }
530 }
531
532 return blorp_execute_on_companion;
533 }
534
535 static bool
anv_blorp_execute_on_companion(struct anv_cmd_buffer * cmd_buffer,struct anv_image * dst_image)536 anv_blorp_execute_on_companion(struct anv_cmd_buffer *cmd_buffer,
537 struct anv_image *dst_image)
538 {
539 const struct intel_device_info *devinfo = cmd_buffer->device->info;
540
541 /* MSAA images have to be dealt with on the companion RCS command buffer
542 * for both CCS && BCS engines.
543 */
544 if ((anv_cmd_buffer_is_blitter_queue(cmd_buffer) ||
545 anv_cmd_buffer_is_compute_queue(cmd_buffer)) &&
546 dst_image->vk.samples > 1)
547 return true;
548
549 /* Emulation of formats is done through a compute shader, so we need
550 * the companion command buffer for the BCS engine.
551 */
552 if (anv_cmd_buffer_is_blitter_queue(cmd_buffer) &&
553 dst_image->emu_plane_format != VK_FORMAT_UNDEFINED)
554 return true;
555
556 /* HSD 14021541470:
557 * The compression pairing bit on blitter engine is not programmed correctly
558 * for stencil resources. Fallback to RCS engine for performing a copy to
559 * workaround the issue.
560 */
561 if (anv_cmd_buffer_is_blitter_queue(cmd_buffer) &&
562 (devinfo->verx10 == 125) &&
563 (dst_image->vk.aspects & VK_IMAGE_ASPECT_STENCIL_BIT)) {
564 const uint32_t plane =
565 anv_image_aspect_to_plane(dst_image, VK_IMAGE_ASPECT_STENCIL_BIT);
566
567 if (isl_aux_usage_has_compression(dst_image->planes[plane].aux_usage))
568 return true;
569 }
570
571 return false;
572 }
573
anv_CmdCopyImage2(VkCommandBuffer commandBuffer,const VkCopyImageInfo2 * pCopyImageInfo)574 void anv_CmdCopyImage2(
575 VkCommandBuffer commandBuffer,
576 const VkCopyImageInfo2* pCopyImageInfo)
577 {
578 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
579 ANV_FROM_HANDLE(anv_image, src_image, pCopyImageInfo->srcImage);
580 ANV_FROM_HANDLE(anv_image, dst_image, pCopyImageInfo->dstImage);
581
582 struct anv_cmd_buffer *main_cmd_buffer = cmd_buffer;
583 UNUSED struct anv_state rcs_done = ANV_STATE_NULL;
584
585 if (anv_blorp_execute_on_companion(cmd_buffer, dst_image)) {
586 rcs_done = record_main_rcs_cmd_buffer_done(cmd_buffer);
587 cmd_buffer = cmd_buffer->companion_rcs_cmd_buffer;
588 }
589
590 struct blorp_batch batch;
591 anv_blorp_batch_init(cmd_buffer, &batch, 0);
592
593 for (unsigned r = 0; r < pCopyImageInfo->regionCount; r++) {
594 copy_image(cmd_buffer, &batch,
595 src_image, pCopyImageInfo->srcImageLayout,
596 dst_image, pCopyImageInfo->dstImageLayout,
597 &pCopyImageInfo->pRegions[r]);
598 }
599
600 anv_blorp_batch_finish(&batch);
601
602 if (dst_image->emu_plane_format != VK_FORMAT_UNDEFINED) {
603 assert(!anv_cmd_buffer_is_blitter_queue(cmd_buffer));
604 const enum anv_pipe_bits pipe_bits =
605 anv_cmd_buffer_is_compute_queue(cmd_buffer) ?
606 ANV_PIPE_HDC_PIPELINE_FLUSH_BIT :
607 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
608 anv_add_pending_pipe_bits(cmd_buffer, pipe_bits,
609 "Copy flush before astc emu");
610
611 for (unsigned r = 0; r < pCopyImageInfo->regionCount; r++) {
612 const VkImageCopy2 *region = &pCopyImageInfo->pRegions[r];
613 const VkOffset3D block_offset = vk_image_offset_to_elements(
614 &dst_image->vk, region->dstOffset);
615 const VkExtent3D block_extent = vk_image_extent_to_elements(
616 &src_image->vk, region->extent);
617 anv_astc_emu_process(cmd_buffer, dst_image,
618 pCopyImageInfo->dstImageLayout,
619 ®ion->dstSubresource,
620 block_offset, block_extent);
621 }
622 }
623
624 if (rcs_done.alloc_size)
625 end_main_rcs_cmd_buffer_done(main_cmd_buffer, rcs_done);
626 }
627
628 static enum isl_format
isl_format_for_size(unsigned size_B)629 isl_format_for_size(unsigned size_B)
630 {
631 /* Prefer 32-bit per component formats for CmdFillBuffer */
632 switch (size_B) {
633 case 1: return ISL_FORMAT_R8_UINT;
634 case 2: return ISL_FORMAT_R16_UINT;
635 case 3: return ISL_FORMAT_R8G8B8_UINT;
636 case 4: return ISL_FORMAT_R32_UINT;
637 case 6: return ISL_FORMAT_R16G16B16_UINT;
638 case 8: return ISL_FORMAT_R32G32_UINT;
639 case 12: return ISL_FORMAT_R32G32B32_UINT;
640 case 16: return ISL_FORMAT_R32G32B32A32_UINT;
641 default:
642 unreachable("Unknown format size");
643 }
644 }
645
646 static void
copy_buffer_to_image(struct anv_cmd_buffer * cmd_buffer,struct blorp_batch * batch,struct anv_buffer * anv_buffer,struct anv_image * anv_image,VkImageLayout image_layout,const VkBufferImageCopy2 * region,bool buffer_to_image)647 copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer,
648 struct blorp_batch *batch,
649 struct anv_buffer *anv_buffer,
650 struct anv_image *anv_image,
651 VkImageLayout image_layout,
652 const VkBufferImageCopy2* region,
653 bool buffer_to_image)
654 {
655 struct {
656 struct blorp_surf surf;
657 struct isl_surf *isl_surf;
658 enum isl_format copy_format;
659 uint32_t level;
660 VkOffset3D offset;
661 } image, buffer, *src, *dst;
662
663 struct isl_surf buffer_isl_surf;
664 buffer.isl_surf = &buffer_isl_surf;
665 buffer.level = 0;
666 buffer.offset = (VkOffset3D) { 0, 0, 0 };
667
668 if (buffer_to_image) {
669 src = &buffer;
670 dst = ℑ
671 } else {
672 src = ℑ
673 dst = &buffer;
674 }
675
676 const VkImageAspectFlags aspect = region->imageSubresource.aspectMask;
677 const unsigned plane = anv_image_aspect_to_plane(anv_image, aspect);
678 image.isl_surf = &anv_image->planes[plane].primary_surface.isl;
679 image.offset =
680 vk_image_sanitize_offset(&anv_image->vk, region->imageOffset);
681 image.level = region->imageSubresource.mipLevel;
682
683 VkExtent3D extent =
684 vk_image_sanitize_extent(&anv_image->vk, region->imageExtent);
685 if (anv_image->vk.image_type != VK_IMAGE_TYPE_3D) {
686 image.offset.z = region->imageSubresource.baseArrayLayer;
687 extent.depth =
688 vk_image_subresource_layer_count(&anv_image->vk,
689 ®ion->imageSubresource);
690 }
691
692 const enum isl_format linear_format =
693 anv_get_isl_format(cmd_buffer->device->physical, anv_image->vk.format,
694 aspect, VK_IMAGE_TILING_LINEAR);
695
696 const struct vk_image_buffer_layout buffer_layout =
697 vk_image_buffer_copy_layout(&anv_image->vk, region);
698
699 get_blorp_surf_for_anv_buffer(cmd_buffer,
700 anv_buffer, region->bufferOffset,
701 extent.width, extent.height,
702 buffer_layout.row_stride_B, linear_format,
703 false, &buffer.surf, &buffer_isl_surf);
704
705 blorp_copy_get_formats(&cmd_buffer->device->isl_dev,
706 src->isl_surf, dst->isl_surf,
707 &src->copy_format, &dst->copy_format);
708
709 get_blorp_surf_for_anv_image(cmd_buffer, anv_image, aspect,
710 buffer_to_image ?
711 VK_IMAGE_USAGE_TRANSFER_DST_BIT :
712 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
713 image_layout, ISL_AUX_USAGE_NONE,
714 image.copy_format, &image.surf);
715
716 if (&image == dst) {
717 anv_cmd_buffer_mark_image_written(cmd_buffer, anv_image,
718 aspect, dst->surf.aux_usage,
719 dst->level,
720 dst->offset.z, extent.depth);
721 }
722
723 for (unsigned z = 0; z < extent.depth; z++) {
724 blorp_copy(batch, &src->surf, src->level, src->offset.z,
725 &dst->surf, dst->level, dst->offset.z,
726 src->offset.x, src->offset.y, dst->offset.x, dst->offset.y,
727 extent.width, extent.height);
728
729 image.offset.z++;
730 buffer.surf.addr.offset += buffer_layout.image_stride_B;
731 }
732 }
733
anv_CmdCopyBufferToImage2(VkCommandBuffer commandBuffer,const VkCopyBufferToImageInfo2 * pCopyBufferToImageInfo)734 void anv_CmdCopyBufferToImage2(
735 VkCommandBuffer commandBuffer,
736 const VkCopyBufferToImageInfo2* pCopyBufferToImageInfo)
737 {
738 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
739 ANV_FROM_HANDLE(anv_buffer, src_buffer, pCopyBufferToImageInfo->srcBuffer);
740 ANV_FROM_HANDLE(anv_image, dst_image, pCopyBufferToImageInfo->dstImage);
741
742 struct anv_cmd_buffer *main_cmd_buffer = cmd_buffer;
743 UNUSED struct anv_state rcs_done = ANV_STATE_NULL;
744
745 bool blorp_execute_on_companion =
746 anv_blorp_execute_on_companion(cmd_buffer, dst_image);
747
748 /* Check if any one of the aspects is incompatible with the blitter engine,
749 * if true, use the companion RCS command buffer for blit operation since 3
750 * component formats are not supported natively except 96bpb on the blitter.
751 */
752 blorp_execute_on_companion |=
753 anv_blorp_blitter_execute_on_companion(cmd_buffer, dst_image,
754 pCopyBufferToImageInfo, NULL);
755
756 if (blorp_execute_on_companion) {
757 rcs_done = record_main_rcs_cmd_buffer_done(cmd_buffer);
758 cmd_buffer = cmd_buffer->companion_rcs_cmd_buffer;
759 }
760
761 struct blorp_batch batch;
762 anv_blorp_batch_init(cmd_buffer, &batch, 0);
763
764 for (unsigned r = 0; r < pCopyBufferToImageInfo->regionCount; r++) {
765 copy_buffer_to_image(cmd_buffer, &batch, src_buffer, dst_image,
766 pCopyBufferToImageInfo->dstImageLayout,
767 &pCopyBufferToImageInfo->pRegions[r], true);
768 }
769
770 anv_blorp_batch_finish(&batch);
771
772 if (dst_image->emu_plane_format != VK_FORMAT_UNDEFINED) {
773 assert(!anv_cmd_buffer_is_blitter_queue(cmd_buffer));
774 const enum anv_pipe_bits pipe_bits =
775 anv_cmd_buffer_is_compute_queue(cmd_buffer) ?
776 ANV_PIPE_HDC_PIPELINE_FLUSH_BIT :
777 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
778 anv_add_pending_pipe_bits(cmd_buffer, pipe_bits,
779 "Copy flush before astc emu");
780
781 for (unsigned r = 0; r < pCopyBufferToImageInfo->regionCount; r++) {
782 const VkBufferImageCopy2 *region =
783 &pCopyBufferToImageInfo->pRegions[r];
784 const VkOffset3D block_offset = vk_image_offset_to_elements(
785 &dst_image->vk, region->imageOffset);
786 const VkExtent3D block_extent = vk_image_extent_to_elements(
787 &dst_image->vk, region->imageExtent);
788 anv_astc_emu_process(cmd_buffer, dst_image,
789 pCopyBufferToImageInfo->dstImageLayout,
790 ®ion->imageSubresource,
791 block_offset, block_extent);
792 }
793 }
794
795 if (rcs_done.alloc_size)
796 end_main_rcs_cmd_buffer_done(main_cmd_buffer, rcs_done);
797 }
798
799 static void
anv_add_buffer_write_pending_bits(struct anv_cmd_buffer * cmd_buffer,const char * reason)800 anv_add_buffer_write_pending_bits(struct anv_cmd_buffer *cmd_buffer,
801 const char *reason)
802 {
803 const struct intel_device_info *devinfo = cmd_buffer->device->info;
804
805 if (anv_cmd_buffer_is_blitter_queue(cmd_buffer))
806 return;
807
808 cmd_buffer->state.queries.buffer_write_bits |=
809 (cmd_buffer->state.current_pipeline ==
810 cmd_buffer->device->physical->gpgpu_pipeline_value) ?
811 ANV_QUERY_COMPUTE_WRITES_PENDING_BITS :
812 ANV_QUERY_RENDER_TARGET_WRITES_PENDING_BITS(devinfo);
813 }
814
anv_CmdCopyImageToBuffer2(VkCommandBuffer commandBuffer,const VkCopyImageToBufferInfo2 * pCopyImageToBufferInfo)815 void anv_CmdCopyImageToBuffer2(
816 VkCommandBuffer commandBuffer,
817 const VkCopyImageToBufferInfo2* pCopyImageToBufferInfo)
818 {
819 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
820 ANV_FROM_HANDLE(anv_image, src_image, pCopyImageToBufferInfo->srcImage);
821 ANV_FROM_HANDLE(anv_buffer, dst_buffer, pCopyImageToBufferInfo->dstBuffer);
822
823 UNUSED struct anv_cmd_buffer *main_cmd_buffer = cmd_buffer;
824 UNUSED struct anv_state rcs_done = ANV_STATE_NULL;
825
826 bool blorp_execute_on_companion =
827 anv_blorp_execute_on_companion(cmd_buffer, src_image);
828
829 /* Check if any one of the aspects is incompatible with the blitter engine,
830 * if true, use the companion RCS command buffer for blit operation since 3
831 * component formats are not supported natively except 96bpb on the blitter.
832 */
833 blorp_execute_on_companion |=
834 anv_blorp_blitter_execute_on_companion(cmd_buffer, src_image, NULL,
835 pCopyImageToBufferInfo);
836
837 if (blorp_execute_on_companion) {
838 rcs_done = record_main_rcs_cmd_buffer_done(cmd_buffer);
839 cmd_buffer = cmd_buffer->companion_rcs_cmd_buffer;
840 }
841
842 struct blorp_batch batch;
843 anv_blorp_batch_init(cmd_buffer, &batch, 0);
844
845 for (unsigned r = 0; r < pCopyImageToBufferInfo->regionCount; r++) {
846 copy_buffer_to_image(cmd_buffer, &batch, dst_buffer, src_image,
847 pCopyImageToBufferInfo->srcImageLayout,
848 &pCopyImageToBufferInfo->pRegions[r], false);
849 }
850
851 anv_add_buffer_write_pending_bits(cmd_buffer, "after copy image to buffer");
852
853 anv_blorp_batch_finish(&batch);
854
855 if (rcs_done.alloc_size)
856 end_main_rcs_cmd_buffer_done(main_cmd_buffer, rcs_done);
857 }
858
859 static bool
flip_coords(unsigned * src0,unsigned * src1,unsigned * dst0,unsigned * dst1)860 flip_coords(unsigned *src0, unsigned *src1, unsigned *dst0, unsigned *dst1)
861 {
862 bool flip = false;
863 if (*src0 > *src1) {
864 unsigned tmp = *src0;
865 *src0 = *src1;
866 *src1 = tmp;
867 flip = !flip;
868 }
869
870 if (*dst0 > *dst1) {
871 unsigned tmp = *dst0;
872 *dst0 = *dst1;
873 *dst1 = tmp;
874 flip = !flip;
875 }
876
877 return flip;
878 }
879
880 static void
blit_image(struct anv_cmd_buffer * cmd_buffer,struct blorp_batch * batch,struct anv_image * src_image,VkImageLayout src_image_layout,struct anv_image * dst_image,VkImageLayout dst_image_layout,const VkImageBlit2 * region,VkFilter filter)881 blit_image(struct anv_cmd_buffer *cmd_buffer,
882 struct blorp_batch *batch,
883 struct anv_image *src_image,
884 VkImageLayout src_image_layout,
885 struct anv_image *dst_image,
886 VkImageLayout dst_image_layout,
887 const VkImageBlit2 *region,
888 VkFilter filter)
889 {
890 const VkImageSubresourceLayers *src_res = ®ion->srcSubresource;
891 const VkImageSubresourceLayers *dst_res = ®ion->dstSubresource;
892
893 struct blorp_surf src, dst;
894
895 enum blorp_filter blorp_filter;
896 switch (filter) {
897 case VK_FILTER_NEAREST:
898 blorp_filter = BLORP_FILTER_NEAREST;
899 break;
900 case VK_FILTER_LINEAR:
901 blorp_filter = BLORP_FILTER_BILINEAR;
902 break;
903 default:
904 unreachable("Invalid filter");
905 }
906
907 assert(anv_image_aspects_compatible(src_res->aspectMask,
908 dst_res->aspectMask));
909
910 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_res->aspectMask) {
911 VkFormat src_vk_format =
912 src_image->emu_plane_format != VK_FORMAT_UNDEFINED ?
913 src_image->emu_plane_format : src_image->vk.format;
914
915 struct anv_format_plane src_format =
916 anv_get_format_aspect(cmd_buffer->device->physical, src_vk_format,
917 1U << aspect_bit, src_image->vk.tiling);
918 struct anv_format_plane dst_format =
919 anv_get_format_aspect(cmd_buffer->device->physical, dst_image->vk.format,
920 1U << aspect_bit, dst_image->vk.tiling);
921
922 get_blorp_surf_for_anv_image(cmd_buffer,
923 src_image, 1U << aspect_bit,
924 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
925 src_image_layout, ISL_AUX_USAGE_NONE,
926 src_format.isl_format, &src);
927 get_blorp_surf_for_anv_image(cmd_buffer,
928 dst_image, 1U << aspect_bit,
929 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
930 dst_image_layout, ISL_AUX_USAGE_NONE,
931 dst_format.isl_format, &dst);
932
933 if (src_image->emu_plane_format != VK_FORMAT_UNDEFINED) {
934 /* redirect src to the hidden plane */
935 const uint32_t plane = src_image->n_planes;
936 const struct anv_surface *surface =
937 &src_image->planes[plane].primary_surface;
938 const struct anv_address address =
939 anv_image_address(src_image, &surface->memory_range);
940 src.surf = &surface->isl,
941 src.addr.offset = address.offset;
942 }
943
944 unsigned dst_start, dst_end;
945 if (dst_image->vk.image_type == VK_IMAGE_TYPE_3D) {
946 assert(dst_res->baseArrayLayer == 0);
947 dst_start = region->dstOffsets[0].z;
948 dst_end = region->dstOffsets[1].z;
949 } else {
950 dst_start = dst_res->baseArrayLayer;
951 dst_end = dst_start +
952 vk_image_subresource_layer_count(&dst_image->vk, dst_res);
953 }
954
955 unsigned src_start, src_end;
956 if (src_image->vk.image_type == VK_IMAGE_TYPE_3D) {
957 assert(src_res->baseArrayLayer == 0);
958 src_start = region->srcOffsets[0].z;
959 src_end = region->srcOffsets[1].z;
960 } else {
961 src_start = src_res->baseArrayLayer;
962 src_end = src_start +
963 vk_image_subresource_layer_count(&src_image->vk, src_res);
964 }
965
966 bool flip_z = flip_coords(&src_start, &src_end, &dst_start, &dst_end);
967 const unsigned num_layers = dst_end - dst_start;
968 float src_z_step = (float)(src_end - src_start) / (float)num_layers;
969
970 /* There is no interpolation to the pixel center during rendering, so
971 * add the 0.5 offset ourselves here. */
972 float depth_center_offset = 0;
973 if (src_image->vk.image_type == VK_IMAGE_TYPE_3D)
974 depth_center_offset = 0.5 / num_layers * (src_end - src_start);
975
976 if (flip_z) {
977 src_start = src_end;
978 src_z_step *= -1;
979 depth_center_offset *= -1;
980 }
981
982 unsigned src_x0 = region->srcOffsets[0].x;
983 unsigned src_x1 = region->srcOffsets[1].x;
984 unsigned dst_x0 = region->dstOffsets[0].x;
985 unsigned dst_x1 = region->dstOffsets[1].x;
986 bool flip_x = flip_coords(&src_x0, &src_x1, &dst_x0, &dst_x1);
987
988 unsigned src_y0 = region->srcOffsets[0].y;
989 unsigned src_y1 = region->srcOffsets[1].y;
990 unsigned dst_y0 = region->dstOffsets[0].y;
991 unsigned dst_y1 = region->dstOffsets[1].y;
992 bool flip_y = flip_coords(&src_y0, &src_y1, &dst_y0, &dst_y1);
993
994 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
995 1U << aspect_bit,
996 dst.aux_usage,
997 dst_res->mipLevel,
998 dst_start, num_layers);
999
1000 for (unsigned i = 0; i < num_layers; i++) {
1001 unsigned dst_z = dst_start + i;
1002 float src_z = src_start + i * src_z_step + depth_center_offset;
1003
1004 blorp_blit(batch, &src, src_res->mipLevel, src_z,
1005 src_format.isl_format, src_format.swizzle,
1006 &dst, dst_res->mipLevel, dst_z,
1007 dst_format.isl_format, dst_format.swizzle,
1008 src_x0, src_y0, src_x1, src_y1,
1009 dst_x0, dst_y0, dst_x1, dst_y1,
1010 blorp_filter, flip_x, flip_y);
1011 }
1012 }
1013 }
1014
anv_CmdBlitImage2(VkCommandBuffer commandBuffer,const VkBlitImageInfo2 * pBlitImageInfo)1015 void anv_CmdBlitImage2(
1016 VkCommandBuffer commandBuffer,
1017 const VkBlitImageInfo2* pBlitImageInfo)
1018 {
1019 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1020 ANV_FROM_HANDLE(anv_image, src_image, pBlitImageInfo->srcImage);
1021 ANV_FROM_HANDLE(anv_image, dst_image, pBlitImageInfo->dstImage);
1022
1023 struct blorp_batch batch;
1024 anv_blorp_batch_init(cmd_buffer, &batch, 0);
1025
1026 for (unsigned r = 0; r < pBlitImageInfo->regionCount; r++) {
1027 blit_image(cmd_buffer, &batch,
1028 src_image, pBlitImageInfo->srcImageLayout,
1029 dst_image, pBlitImageInfo->dstImageLayout,
1030 &pBlitImageInfo->pRegions[r], pBlitImageInfo->filter);
1031 }
1032
1033 anv_blorp_batch_finish(&batch);
1034 }
1035
1036 /**
1037 * Returns the greatest common divisor of a and b that is a power of two.
1038 */
1039 static uint64_t
gcd_pow2_u64(uint64_t a,uint64_t b)1040 gcd_pow2_u64(uint64_t a, uint64_t b)
1041 {
1042 assert(a > 0 || b > 0);
1043
1044 unsigned a_log2 = ffsll(a) - 1;
1045 unsigned b_log2 = ffsll(b) - 1;
1046
1047 /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
1048 * case, the MIN2() will take the other one. If both are 0 then we will
1049 * hit the assert above.
1050 */
1051 return 1 << MIN2(a_log2, b_log2);
1052 }
1053
1054 /* This is maximum possible width/height our HW can handle */
1055 #define MAX_SURFACE_DIM (1ull << 14)
1056
1057 static void
copy_buffer(struct anv_device * device,struct blorp_batch * batch,struct anv_buffer * src_buffer,struct anv_buffer * dst_buffer,const VkBufferCopy2 * region)1058 copy_buffer(struct anv_device *device,
1059 struct blorp_batch *batch,
1060 struct anv_buffer *src_buffer,
1061 struct anv_buffer *dst_buffer,
1062 const VkBufferCopy2 *region)
1063 {
1064 struct blorp_address src = {
1065 .buffer = src_buffer->address.bo,
1066 .offset = src_buffer->address.offset + region->srcOffset,
1067 .mocs = anv_mocs(device, src_buffer->address.bo,
1068 blorp_batch_isl_copy_usage(batch, false /* is_dest */,
1069 anv_buffer_is_protected(src_buffer))),
1070 };
1071 struct blorp_address dst = {
1072 .buffer = dst_buffer->address.bo,
1073 .offset = dst_buffer->address.offset + region->dstOffset,
1074 .mocs = anv_mocs(device, dst_buffer->address.bo,
1075 blorp_batch_isl_copy_usage(batch, true /* is_dest */,
1076 anv_buffer_is_protected(dst_buffer))),
1077 };
1078
1079 blorp_buffer_copy(batch, src, dst, region->size);
1080 }
1081
1082 void
anv_cmd_copy_addr(struct anv_cmd_buffer * cmd_buffer,struct anv_address src_addr,struct anv_address dst_addr,uint64_t size)1083 anv_cmd_copy_addr(struct anv_cmd_buffer *cmd_buffer,
1084 struct anv_address src_addr,
1085 struct anv_address dst_addr,
1086 uint64_t size)
1087 {
1088 struct anv_device *device = cmd_buffer->device;
1089
1090 struct blorp_batch batch;
1091 anv_blorp_batch_init(cmd_buffer, &batch,
1092 cmd_buffer->state.current_pipeline ==
1093 cmd_buffer->device->physical->gpgpu_pipeline_value ?
1094 BLORP_BATCH_USE_COMPUTE : 0);
1095
1096 struct blorp_address src = {
1097 .buffer = src_addr.bo,
1098 .offset = src_addr.offset,
1099 .mocs = anv_mocs(device, src_addr.bo,
1100 blorp_batch_isl_copy_usage(&batch, false /* is_dest */,
1101 false)),
1102 };
1103
1104 struct blorp_address dst = {
1105 .buffer = dst_addr.bo,
1106 .offset = dst_addr.offset,
1107 .mocs = anv_mocs(device, dst_addr.bo,
1108 blorp_batch_isl_copy_usage(&batch, true /* is_dest */,
1109 false)),
1110 };
1111
1112 blorp_buffer_copy(&batch, src, dst, size);
1113
1114 anv_add_buffer_write_pending_bits(cmd_buffer, "after copy buffer");
1115 anv_blorp_batch_finish(&batch);
1116 }
1117
anv_CmdCopyBuffer2(VkCommandBuffer commandBuffer,const VkCopyBufferInfo2 * pCopyBufferInfo)1118 void anv_CmdCopyBuffer2(
1119 VkCommandBuffer commandBuffer,
1120 const VkCopyBufferInfo2* pCopyBufferInfo)
1121 {
1122 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1123 ANV_FROM_HANDLE(anv_buffer, src_buffer, pCopyBufferInfo->srcBuffer);
1124 ANV_FROM_HANDLE(anv_buffer, dst_buffer, pCopyBufferInfo->dstBuffer);
1125
1126 struct blorp_batch batch;
1127 anv_blorp_batch_init(cmd_buffer, &batch,
1128 cmd_buffer->state.current_pipeline ==
1129 cmd_buffer->device->physical->gpgpu_pipeline_value ?
1130 BLORP_BATCH_USE_COMPUTE : 0);
1131
1132 for (unsigned r = 0; r < pCopyBufferInfo->regionCount; r++) {
1133 copy_buffer(cmd_buffer->device, &batch, src_buffer, dst_buffer,
1134 &pCopyBufferInfo->pRegions[r]);
1135 }
1136
1137 anv_add_buffer_write_pending_bits(cmd_buffer, "after copy buffer");
1138
1139 anv_blorp_batch_finish(&batch);
1140 }
1141
1142 void
anv_cmd_buffer_update_addr(struct anv_cmd_buffer * cmd_buffer,struct anv_address address,VkDeviceSize dstOffset,VkDeviceSize dataSize,const void * pData,bool is_protected)1143 anv_cmd_buffer_update_addr(
1144 struct anv_cmd_buffer* cmd_buffer,
1145 struct anv_address address,
1146 VkDeviceSize dstOffset,
1147 VkDeviceSize dataSize,
1148 const void* pData,
1149 bool is_protected)
1150 {
1151 struct blorp_batch batch;
1152 anv_blorp_batch_init(cmd_buffer, &batch,
1153 cmd_buffer->state.current_pipeline ==
1154 cmd_buffer->device->physical->gpgpu_pipeline_value ?
1155 BLORP_BATCH_USE_COMPUTE : 0);
1156
1157 /* We can't quite grab a full block because the state stream needs a
1158 * little data at the top to build its linked list.
1159 */
1160 const uint32_t max_update_size =
1161 cmd_buffer->device->dynamic_state_pool.block_size - 64;
1162
1163 assert(max_update_size < MAX_SURFACE_DIM * 4);
1164
1165 /* We're about to read data that was written from the CPU. Flush the
1166 * texture cache so we don't get anything stale.
1167 */
1168 anv_add_pending_pipe_bits(cmd_buffer,
1169 ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT,
1170 "before UpdateBuffer");
1171
1172 while (dataSize) {
1173 const uint32_t copy_size = MIN2(dataSize, max_update_size);
1174
1175 struct anv_state tmp_data =
1176 anv_cmd_buffer_alloc_temporary_state(cmd_buffer, copy_size, 64);
1177 struct anv_address tmp_addr =
1178 anv_cmd_buffer_temporary_state_address(cmd_buffer, tmp_data);
1179
1180 memcpy(tmp_data.map, pData, copy_size);
1181
1182 struct blorp_address src = {
1183 .buffer = tmp_addr.bo,
1184 .offset = tmp_addr.offset,
1185 .mocs = anv_mocs(cmd_buffer->device, NULL,
1186 get_usage_flag_for_cmd_buffer(cmd_buffer,
1187 false /* is_dest */,
1188 false /* is_depth */,
1189 false /* protected */)),
1190 };
1191 struct blorp_address dst = {
1192 .buffer = address.bo,
1193 .offset = address.offset + dstOffset,
1194 .mocs = anv_mocs(cmd_buffer->device, address.bo,
1195 get_usage_flag_for_cmd_buffer(
1196 cmd_buffer,
1197 true /* is_dest */,
1198 false /* is_depth */,
1199 is_protected)),
1200 };
1201
1202 blorp_buffer_copy(&batch, src, dst, copy_size);
1203
1204 dataSize -= copy_size;
1205 dstOffset += copy_size;
1206 pData = (void *)pData + copy_size;
1207 }
1208
1209 anv_add_buffer_write_pending_bits(cmd_buffer, "update buffer");
1210
1211 anv_blorp_batch_finish(&batch);
1212 }
1213
anv_CmdUpdateBuffer(VkCommandBuffer commandBuffer,VkBuffer dstBuffer,VkDeviceSize dstOffset,VkDeviceSize dataSize,const void * pData)1214 void anv_CmdUpdateBuffer(
1215 VkCommandBuffer commandBuffer,
1216 VkBuffer dstBuffer,
1217 VkDeviceSize dstOffset,
1218 VkDeviceSize dataSize,
1219 const void* pData)
1220 {
1221 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1222 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
1223
1224 anv_cmd_buffer_update_addr(cmd_buffer, dst_buffer->address,
1225 dstOffset, dataSize, pData,
1226 anv_buffer_is_protected(dst_buffer));
1227 }
1228
1229 void
anv_cmd_buffer_fill_area(struct anv_cmd_buffer * cmd_buffer,struct anv_address address,VkDeviceSize size,uint32_t data,bool protected)1230 anv_cmd_buffer_fill_area(struct anv_cmd_buffer *cmd_buffer,
1231 struct anv_address address,
1232 VkDeviceSize size,
1233 uint32_t data,
1234 bool protected)
1235 {
1236 struct blorp_surf surf;
1237 struct isl_surf isl_surf;
1238
1239 struct blorp_batch batch;
1240 anv_blorp_batch_init(cmd_buffer, &batch,
1241 cmd_buffer->state.current_pipeline ==
1242 cmd_buffer->device->physical->gpgpu_pipeline_value ?
1243 BLORP_BATCH_USE_COMPUTE : 0);
1244
1245 /* First, we compute the biggest format that can be used with the
1246 * given offsets and size.
1247 */
1248 int bs = 16;
1249 uint64_t offset = address.offset;
1250 bs = gcd_pow2_u64(bs, offset);
1251 bs = gcd_pow2_u64(bs, size);
1252 enum isl_format isl_format = isl_format_for_size(bs);
1253
1254 union isl_color_value color = {
1255 .u32 = { data, data, data, data },
1256 };
1257
1258 const uint64_t max_fill_size = MAX_SURFACE_DIM * MAX_SURFACE_DIM * bs;
1259 while (size >= max_fill_size) {
1260 get_blorp_surf_for_anv_address(cmd_buffer,
1261 (struct anv_address) {
1262 .bo = address.bo, .offset = offset,
1263 },
1264 MAX_SURFACE_DIM, MAX_SURFACE_DIM,
1265 MAX_SURFACE_DIM * bs, isl_format,
1266 true /* is_dest */, protected,
1267 &surf, &isl_surf);
1268
1269 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
1270 0, 0, 1, 0, 0, MAX_SURFACE_DIM, MAX_SURFACE_DIM,
1271 color, 0 /* color_write_disable */);
1272 size -= max_fill_size;
1273 offset += max_fill_size;
1274 }
1275
1276 uint64_t height = size / (MAX_SURFACE_DIM * bs);
1277 assert(height < MAX_SURFACE_DIM);
1278 if (height != 0) {
1279 const uint64_t rect_fill_size = height * MAX_SURFACE_DIM * bs;
1280 get_blorp_surf_for_anv_address(cmd_buffer,
1281 (struct anv_address) {
1282 .bo = address.bo, .offset = offset,
1283 },
1284 MAX_SURFACE_DIM, height,
1285 MAX_SURFACE_DIM * bs, isl_format,
1286 true /* is_dest */, protected,
1287 &surf, &isl_surf);
1288
1289 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
1290 0, 0, 1, 0, 0, MAX_SURFACE_DIM, height,
1291 color, 0 /* color_write_disable */);
1292 size -= rect_fill_size;
1293 offset += rect_fill_size;
1294 }
1295
1296 if (size != 0) {
1297 const uint32_t width = size / bs;
1298 get_blorp_surf_for_anv_address(cmd_buffer,
1299 (struct anv_address) {
1300 .bo = address.bo, .offset = offset,
1301 },
1302 width, 1,
1303 width * bs, isl_format,
1304 true /* is_dest */, protected,
1305 &surf, &isl_surf);
1306
1307 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
1308 0, 0, 1, 0, 0, width, 1,
1309 color, 0 /* color_write_disable */);
1310 }
1311
1312 anv_blorp_batch_finish(&batch);
1313 }
1314
1315 void
anv_cmd_fill_buffer_addr(VkCommandBuffer commandBuffer,VkDeviceAddress dstAddr,VkDeviceSize fillSize,uint32_t data)1316 anv_cmd_fill_buffer_addr(VkCommandBuffer commandBuffer,
1317 VkDeviceAddress dstAddr,
1318 VkDeviceSize fillSize,
1319 uint32_t data)
1320 {
1321 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1322
1323 anv_cmd_buffer_fill_area(cmd_buffer, anv_address_from_u64(dstAddr),
1324 fillSize, data, false);
1325
1326 anv_add_buffer_write_pending_bits(cmd_buffer, "after fill buffer");
1327 }
1328
anv_CmdFillBuffer(VkCommandBuffer commandBuffer,VkBuffer dstBuffer,VkDeviceSize dstOffset,VkDeviceSize fillSize,uint32_t data)1329 void anv_CmdFillBuffer(
1330 VkCommandBuffer commandBuffer,
1331 VkBuffer dstBuffer,
1332 VkDeviceSize dstOffset,
1333 VkDeviceSize fillSize,
1334 uint32_t data)
1335 {
1336 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1337 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
1338
1339 fillSize = vk_buffer_range(&dst_buffer->vk, dstOffset, fillSize);
1340
1341 /* From the Vulkan spec:
1342 *
1343 * "size is the number of bytes to fill, and must be either a multiple
1344 * of 4, or VK_WHOLE_SIZE to fill the range from offset to the end of
1345 * the buffer. If VK_WHOLE_SIZE is used and the remaining size of the
1346 * buffer is not a multiple of 4, then the nearest smaller multiple is
1347 * used."
1348 */
1349 fillSize &= ~3ull;
1350
1351 anv_cmd_buffer_fill_area(cmd_buffer,
1352 anv_address_add(dst_buffer->address, dstOffset),
1353 fillSize, data,
1354 anv_buffer_is_protected(dst_buffer));
1355
1356 anv_add_buffer_write_pending_bits(cmd_buffer, "after fill buffer");
1357 }
1358
1359 static void
exec_ccs_op(struct anv_cmd_buffer * cmd_buffer,struct blorp_batch * batch,const struct anv_image * image,enum isl_format format,struct isl_swizzle swizzle,VkImageAspectFlagBits aspect,uint32_t level,uint32_t base_layer,uint32_t layer_count,enum isl_aux_op ccs_op,union isl_color_value * clear_value)1360 exec_ccs_op(struct anv_cmd_buffer *cmd_buffer,
1361 struct blorp_batch *batch,
1362 const struct anv_image *image,
1363 enum isl_format format, struct isl_swizzle swizzle,
1364 VkImageAspectFlagBits aspect, uint32_t level,
1365 uint32_t base_layer, uint32_t layer_count,
1366 enum isl_aux_op ccs_op, union isl_color_value *clear_value)
1367 {
1368 assert(image->vk.aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
1369 assert(image->vk.samples == 1);
1370 assert(level < anv_image_aux_levels(image, aspect));
1371 /* Multi-LOD YcBcR is not allowed */
1372 assert(image->n_planes == 1 || level == 0);
1373 assert(base_layer + layer_count <=
1374 anv_image_aux_layers(image, aspect, level));
1375
1376 const uint32_t plane = anv_image_aspect_to_plane(image, aspect);
1377
1378 struct blorp_surf surf;
1379 get_blorp_surf_for_anv_image(cmd_buffer, image, aspect,
1380 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
1381 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1382 image->planes[plane].aux_usage,
1383 format, &surf);
1384
1385 uint32_t level_width = u_minify(surf.surf->logical_level0_px.w, level);
1386 uint32_t level_height = u_minify(surf.surf->logical_level0_px.h, level);
1387
1388 /* Blorp will store the clear color for us if we provide the clear color
1389 * address and we are doing a fast clear. So we save the clear value into
1390 * the blorp surface.
1391 */
1392 if (clear_value)
1393 surf.clear_color = *clear_value;
1394
1395 switch (ccs_op) {
1396 case ISL_AUX_OP_FAST_CLEAR:
1397 blorp_fast_clear(batch, &surf, format, swizzle,
1398 level, base_layer, layer_count,
1399 0, 0, level_width, level_height);
1400 break;
1401 case ISL_AUX_OP_FULL_RESOLVE:
1402 case ISL_AUX_OP_PARTIAL_RESOLVE: {
1403 /* Wa_1508744258: Enable RHWO optimization for resolves */
1404 const bool enable_rhwo_opt =
1405 intel_needs_workaround(cmd_buffer->device->info, 1508744258);
1406
1407 if (enable_rhwo_opt)
1408 cmd_buffer->state.pending_rhwo_optimization_enabled = true;
1409
1410 blorp_ccs_resolve(batch, &surf, level, base_layer, layer_count,
1411 format, ccs_op);
1412
1413 if (enable_rhwo_opt)
1414 cmd_buffer->state.pending_rhwo_optimization_enabled = false;
1415 break;
1416 }
1417 case ISL_AUX_OP_AMBIGUATE:
1418 for (uint32_t a = 0; a < layer_count; a++) {
1419 const uint32_t layer = base_layer + a;
1420 blorp_ccs_ambiguate(batch, &surf, level, layer);
1421 }
1422 break;
1423 default:
1424 unreachable("Unsupported CCS operation");
1425 }
1426 }
1427
1428 static void
exec_mcs_op(struct anv_cmd_buffer * cmd_buffer,struct blorp_batch * batch,const struct anv_image * image,enum isl_format format,struct isl_swizzle swizzle,VkImageAspectFlagBits aspect,uint32_t base_layer,uint32_t layer_count,enum isl_aux_op mcs_op,union isl_color_value * clear_value)1429 exec_mcs_op(struct anv_cmd_buffer *cmd_buffer,
1430 struct blorp_batch *batch,
1431 const struct anv_image *image,
1432 enum isl_format format, struct isl_swizzle swizzle,
1433 VkImageAspectFlagBits aspect,
1434 uint32_t base_layer, uint32_t layer_count,
1435 enum isl_aux_op mcs_op, union isl_color_value *clear_value)
1436 {
1437 assert(image->vk.aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1438 assert(image->vk.samples > 1);
1439 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, 0));
1440
1441 /* Multisampling with multi-planar formats is not supported */
1442 assert(image->n_planes == 1);
1443
1444 struct blorp_surf surf;
1445 get_blorp_surf_for_anv_image(cmd_buffer, image, aspect,
1446 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
1447 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1448 image->planes[0].aux_usage, format, &surf);
1449
1450 /* Blorp will store the clear color for us if we provide the clear color
1451 * address and we are doing a fast clear. So we save the clear value into
1452 * the blorp surface.
1453 */
1454 if (clear_value)
1455 surf.clear_color = *clear_value;
1456
1457 switch (mcs_op) {
1458 case ISL_AUX_OP_FAST_CLEAR:
1459 blorp_fast_clear(batch, &surf, format, swizzle,
1460 0, base_layer, layer_count,
1461 0, 0, image->vk.extent.width, image->vk.extent.height);
1462 break;
1463 case ISL_AUX_OP_PARTIAL_RESOLVE:
1464 blorp_mcs_partial_resolve(batch, &surf, format,
1465 base_layer, layer_count);
1466 break;
1467 case ISL_AUX_OP_AMBIGUATE:
1468 blorp_mcs_ambiguate(batch, &surf, base_layer, layer_count);
1469 break;
1470 case ISL_AUX_OP_FULL_RESOLVE:
1471 default:
1472 unreachable("Unsupported MCS operation");
1473 }
1474 }
1475
anv_CmdClearColorImage(VkCommandBuffer commandBuffer,VkImage _image,VkImageLayout imageLayout,const VkClearColorValue * pColor,uint32_t rangeCount,const VkImageSubresourceRange * pRanges)1476 void anv_CmdClearColorImage(
1477 VkCommandBuffer commandBuffer,
1478 VkImage _image,
1479 VkImageLayout imageLayout,
1480 const VkClearColorValue* pColor,
1481 uint32_t rangeCount,
1482 const VkImageSubresourceRange* pRanges)
1483 {
1484 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1485 ANV_FROM_HANDLE(anv_image, image, _image);
1486
1487 struct anv_cmd_buffer *main_cmd_buffer = cmd_buffer;
1488 UNUSED struct anv_state rcs_done = ANV_STATE_NULL;
1489
1490 if (anv_blorp_execute_on_companion(cmd_buffer, image)) {
1491 rcs_done = record_main_rcs_cmd_buffer_done(cmd_buffer);
1492 cmd_buffer = cmd_buffer->companion_rcs_cmd_buffer;
1493 }
1494
1495 struct blorp_batch batch;
1496 anv_blorp_batch_init(cmd_buffer, &batch, 0);
1497
1498 struct anv_format_plane src_format =
1499 anv_get_format_aspect(cmd_buffer->device->physical, image->vk.format,
1500 VK_IMAGE_ASPECT_COLOR_BIT, image->vk.tiling);
1501 struct blorp_surf surf;
1502 get_blorp_surf_for_anv_image(cmd_buffer, image,
1503 VK_IMAGE_ASPECT_COLOR_BIT,
1504 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
1505 imageLayout, ISL_AUX_USAGE_NONE,
1506 src_format.isl_format, &surf);
1507
1508 union isl_color_value clear_color = vk_to_isl_color(*pColor);
1509
1510
1511 for (unsigned r = 0; r < rangeCount; r++) {
1512 assert(pRanges[r].aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
1513
1514 uint32_t level_count =
1515 vk_image_subresource_level_count(&image->vk, &pRanges[r]);
1516
1517 for (uint32_t i = 0; i < level_count; i++) {
1518 const unsigned level = pRanges[r].baseMipLevel + i;
1519 const VkExtent3D level_extent =
1520 vk_image_mip_level_extent(&image->vk, level);
1521
1522 VkClearRect clear_rect = {};
1523 clear_rect.rect.extent.width = level_extent.width;
1524 clear_rect.rect.extent.height = level_extent.height;
1525 if (image->vk.image_type == VK_IMAGE_TYPE_3D) {
1526 clear_rect.baseArrayLayer = 0;
1527 clear_rect.layerCount = level_extent.depth;
1528 } else {
1529 clear_rect.baseArrayLayer = pRanges[r].baseArrayLayer;
1530 clear_rect.layerCount =
1531 vk_image_subresource_layer_count(&image->vk, &pRanges[r]);
1532 }
1533
1534 if (anv_can_fast_clear_color(cmd_buffer, image, level, &clear_rect,
1535 imageLayout, src_format.isl_format,
1536 clear_color)) {
1537 assert(level == 0);
1538 assert(clear_rect.baseArrayLayer == 0);
1539 if (image->vk.samples == 1) {
1540 exec_ccs_op(cmd_buffer, &batch, image, src_format.isl_format,
1541 src_format.swizzle, VK_IMAGE_ASPECT_COLOR_BIT,
1542 0, 0, 1, ISL_AUX_OP_FAST_CLEAR, &clear_color);
1543 } else {
1544 exec_mcs_op(cmd_buffer, &batch, image, src_format.isl_format,
1545 src_format.swizzle, VK_IMAGE_ASPECT_COLOR_BIT,
1546 0, 1, ISL_AUX_OP_FAST_CLEAR, &clear_color);
1547 }
1548
1549 if (cmd_buffer->device->info->ver < 20) {
1550 anv_cmd_buffer_mark_image_fast_cleared(cmd_buffer, image,
1551 src_format.isl_format,
1552 src_format.swizzle,
1553 clear_color);
1554 }
1555
1556 clear_rect.baseArrayLayer++;
1557 if (--clear_rect.layerCount == 0)
1558 continue;
1559 }
1560
1561 anv_cmd_buffer_mark_image_written(cmd_buffer, image,
1562 pRanges[r].aspectMask,
1563 surf.aux_usage, level,
1564 clear_rect.baseArrayLayer,
1565 clear_rect.layerCount);
1566
1567 blorp_clear(&batch, &surf,
1568 src_format.isl_format, src_format.swizzle, level,
1569 clear_rect.baseArrayLayer,
1570 clear_rect.layerCount,
1571 clear_rect.rect.offset.x,
1572 clear_rect.rect.offset.y,
1573 clear_rect.rect.extent.width,
1574 clear_rect.rect.extent.height,
1575 clear_color, 0 /* color_write_disable */);
1576 }
1577 }
1578
1579 anv_blorp_batch_finish(&batch);
1580
1581 if (rcs_done.alloc_size)
1582 end_main_rcs_cmd_buffer_done(main_cmd_buffer, rcs_done);
1583 }
1584
anv_CmdClearDepthStencilImage(VkCommandBuffer commandBuffer,VkImage image_h,VkImageLayout imageLayout,const VkClearDepthStencilValue * pDepthStencil,uint32_t rangeCount,const VkImageSubresourceRange * pRanges)1585 void anv_CmdClearDepthStencilImage(
1586 VkCommandBuffer commandBuffer,
1587 VkImage image_h,
1588 VkImageLayout imageLayout,
1589 const VkClearDepthStencilValue* pDepthStencil,
1590 uint32_t rangeCount,
1591 const VkImageSubresourceRange* pRanges)
1592 {
1593 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1594 ANV_FROM_HANDLE(anv_image, image, image_h);
1595
1596 struct blorp_batch batch;
1597 anv_blorp_batch_init(cmd_buffer, &batch, 0);
1598 assert((batch.flags & BLORP_BATCH_USE_COMPUTE) == 0);
1599
1600 struct blorp_surf depth, stencil;
1601 if (image->vk.aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1602 get_blorp_surf_for_anv_image(cmd_buffer,
1603 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1604 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
1605 imageLayout, ISL_AUX_USAGE_NONE,
1606 ISL_FORMAT_UNSUPPORTED, &depth);
1607 } else {
1608 memset(&depth, 0, sizeof(depth));
1609 }
1610
1611 if (image->vk.aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1612 get_blorp_surf_for_anv_image(cmd_buffer,
1613 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1614 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
1615 imageLayout, ISL_AUX_USAGE_NONE,
1616 ISL_FORMAT_UNSUPPORTED, &stencil);
1617 } else {
1618 memset(&stencil, 0, sizeof(stencil));
1619 }
1620
1621 for (unsigned r = 0; r < rangeCount; r++) {
1622 if (pRanges[r].aspectMask == 0)
1623 continue;
1624
1625 bool clear_depth = pRanges[r].aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
1626 bool clear_stencil = pRanges[r].aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
1627
1628 unsigned base_layer = pRanges[r].baseArrayLayer;
1629 uint32_t layer_count =
1630 vk_image_subresource_layer_count(&image->vk, &pRanges[r]);
1631 uint32_t level_count =
1632 vk_image_subresource_level_count(&image->vk, &pRanges[r]);
1633
1634 for (uint32_t i = 0; i < level_count; i++) {
1635 const unsigned level = pRanges[r].baseMipLevel + i;
1636 const unsigned level_width = u_minify(image->vk.extent.width, level);
1637 const unsigned level_height = u_minify(image->vk.extent.height, level);
1638
1639 if (image->vk.image_type == VK_IMAGE_TYPE_3D)
1640 layer_count = u_minify(image->vk.extent.depth, level);
1641
1642 blorp_clear_depth_stencil(&batch, &depth, &stencil,
1643 level, base_layer, layer_count,
1644 0, 0, level_width, level_height,
1645 clear_depth, pDepthStencil->depth,
1646 clear_stencil ? 0xff : 0,
1647 pDepthStencil->stencil);
1648 }
1649 }
1650
1651 anv_blorp_batch_finish(&batch);
1652 }
1653
1654 VkResult
anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer * cmd_buffer,uint32_t num_entries,uint32_t * state_offset,struct anv_state * bt_state)1655 anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer *cmd_buffer,
1656 uint32_t num_entries,
1657 uint32_t *state_offset,
1658 struct anv_state *bt_state)
1659 {
1660 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
1661 state_offset);
1662 if (bt_state->map == NULL) {
1663 /* We ran out of space. Grab a new binding table block. */
1664 VkResult result = anv_cmd_buffer_new_binding_table_block(cmd_buffer);
1665 if (result != VK_SUCCESS)
1666 return result;
1667
1668 /* Re-emit state base addresses so we get the new surface state base
1669 * address before we start emitting binding tables etc.
1670 */
1671 anv_cmd_buffer_emit_bt_pool_base_address(cmd_buffer);
1672
1673 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
1674 state_offset);
1675 assert(bt_state->map != NULL);
1676 }
1677
1678 return VK_SUCCESS;
1679 }
1680
1681 static VkResult
binding_table_for_surface_state(struct anv_cmd_buffer * cmd_buffer,struct anv_state surface_state,uint32_t * bt_offset)1682 binding_table_for_surface_state(struct anv_cmd_buffer *cmd_buffer,
1683 struct anv_state surface_state,
1684 uint32_t *bt_offset)
1685 {
1686 uint32_t state_offset;
1687 struct anv_state bt_state;
1688
1689 VkResult result =
1690 anv_cmd_buffer_alloc_blorp_binding_table(cmd_buffer, 1, &state_offset,
1691 &bt_state);
1692 if (result != VK_SUCCESS)
1693 return result;
1694
1695 uint32_t *bt_map = bt_state.map;
1696 bt_map[0] = surface_state.offset + state_offset;
1697
1698 *bt_offset = bt_state.offset;
1699 return VK_SUCCESS;
1700 }
1701
1702 static bool
can_fast_clear_color_att(struct anv_cmd_buffer * cmd_buffer,struct blorp_batch * batch,const struct anv_attachment * att,const VkClearAttachment * attachment,uint32_t rectCount,const VkClearRect * pRects)1703 can_fast_clear_color_att(struct anv_cmd_buffer *cmd_buffer,
1704 struct blorp_batch *batch,
1705 const struct anv_attachment *att,
1706 const VkClearAttachment *attachment,
1707 uint32_t rectCount, const VkClearRect *pRects)
1708 {
1709 union isl_color_value clear_color =
1710 vk_to_isl_color(attachment->clearValue.color);
1711
1712 if (INTEL_DEBUG(DEBUG_NO_FAST_CLEAR))
1713 return false;
1714
1715 /* We don't support fast clearing with conditional rendering at the
1716 * moment. All the tracking done around fast clears (clear color updates
1717 * and fast-clear type updates) happens unconditionally.
1718 */
1719 if (batch->flags & BLORP_BATCH_PREDICATE_ENABLE)
1720 return false;
1721
1722 if (rectCount > 1) {
1723 anv_perf_warn(VK_LOG_OBJS(&cmd_buffer->device->vk.base),
1724 "Fast clears for vkCmdClearAttachments supported only for rectCount == 1");
1725 return false;
1726 }
1727
1728 /* We only support fast-clearing a single layer */
1729 if (pRects[0].layerCount > 1)
1730 return false;
1731
1732 bool is_multiview = cmd_buffer->state.gfx.view_mask != 0;
1733 if (is_multiview && (cmd_buffer->state.gfx.view_mask != 1))
1734 return false;
1735
1736 return anv_can_fast_clear_color(cmd_buffer, att->iview->image,
1737 att->iview->vk.base_mip_level,
1738 pRects, att->layout,
1739 att->iview->planes[0].isl.format,
1740 clear_color);
1741 }
1742
1743 static void
clear_color_attachment(struct anv_cmd_buffer * cmd_buffer,struct blorp_batch * batch,const VkClearAttachment * attachment,uint32_t rectCount,const VkClearRect * pRects)1744 clear_color_attachment(struct anv_cmd_buffer *cmd_buffer,
1745 struct blorp_batch *batch,
1746 const VkClearAttachment *attachment,
1747 uint32_t rectCount, const VkClearRect *pRects)
1748 {
1749 struct anv_cmd_graphics_state *gfx = &cmd_buffer->state.gfx;
1750 const uint32_t att_idx = attachment->colorAttachment;
1751 assert(att_idx < gfx->color_att_count);
1752 const struct anv_attachment *att = &gfx->color_att[att_idx];
1753
1754 if (att->vk_format == VK_FORMAT_UNDEFINED)
1755 return;
1756
1757 union isl_color_value clear_color =
1758 vk_to_isl_color(attachment->clearValue.color);
1759
1760 const struct anv_image_view *iview = att->iview;
1761 if (iview &&
1762 can_fast_clear_color_att(cmd_buffer, batch, att,
1763 attachment, rectCount, pRects)) {
1764 if (iview->image->vk.samples == 1) {
1765 exec_ccs_op(cmd_buffer, batch, iview->image,
1766 iview->planes[0].isl.format,
1767 iview->planes[0].isl.swizzle,
1768 VK_IMAGE_ASPECT_COLOR_BIT,
1769 0, 0, 1, ISL_AUX_OP_FAST_CLEAR,
1770 &clear_color);
1771 } else {
1772 exec_mcs_op(cmd_buffer, batch, iview->image,
1773 iview->planes[0].isl.format,
1774 iview->planes[0].isl.swizzle,
1775 VK_IMAGE_ASPECT_COLOR_BIT,
1776 0, 1, ISL_AUX_OP_FAST_CLEAR,
1777 &clear_color);
1778 }
1779
1780 if (cmd_buffer->device->info->ver < 20) {
1781 anv_cmd_buffer_mark_image_fast_cleared(cmd_buffer, iview->image,
1782 iview->planes[0].isl.format,
1783 iview->planes[0].isl.swizzle,
1784 clear_color);
1785 anv_cmd_buffer_load_clear_color(cmd_buffer, att->surface_state.state,
1786 iview);
1787 }
1788 return;
1789 }
1790
1791 uint32_t binding_table;
1792 VkResult result =
1793 binding_table_for_surface_state(cmd_buffer, att->surface_state.state,
1794 &binding_table);
1795 if (result != VK_SUCCESS)
1796 return;
1797
1798 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1799 if (gfx->view_mask) {
1800 u_foreach_bit(view_idx, gfx->view_mask) {
1801 for (uint32_t r = 0; r < rectCount; ++r) {
1802 const VkOffset2D offset = pRects[r].rect.offset;
1803 const VkExtent2D extent = pRects[r].rect.extent;
1804 blorp_clear_attachments(batch, binding_table,
1805 ISL_FORMAT_UNSUPPORTED,
1806 gfx->samples,
1807 view_idx, 1,
1808 offset.x, offset.y,
1809 offset.x + extent.width,
1810 offset.y + extent.height,
1811 true, clear_color, false, 0.0f, 0, 0);
1812 }
1813 }
1814 return;
1815 }
1816
1817 for (uint32_t r = 0; r < rectCount; ++r) {
1818 const VkOffset2D offset = pRects[r].rect.offset;
1819 const VkExtent2D extent = pRects[r].rect.extent;
1820 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1821 blorp_clear_attachments(batch, binding_table,
1822 ISL_FORMAT_UNSUPPORTED,
1823 gfx->samples,
1824 pRects[r].baseArrayLayer,
1825 pRects[r].layerCount,
1826 offset.x, offset.y,
1827 offset.x + extent.width, offset.y + extent.height,
1828 true, clear_color, false, 0.0f, 0, 0);
1829 }
1830 }
1831
1832 static void
anv_fast_clear_depth_stencil(struct anv_cmd_buffer * cmd_buffer,struct blorp_batch * batch,const struct anv_image * image,VkImageAspectFlags aspects,uint32_t level,uint32_t base_layer,uint32_t layer_count,VkRect2D area,const VkClearDepthStencilValue * clear_value)1833 anv_fast_clear_depth_stencil(struct anv_cmd_buffer *cmd_buffer,
1834 struct blorp_batch *batch,
1835 const struct anv_image *image,
1836 VkImageAspectFlags aspects,
1837 uint32_t level,
1838 uint32_t base_layer, uint32_t layer_count,
1839 VkRect2D area,
1840 const VkClearDepthStencilValue *clear_value)
1841 {
1842 assert(image->vk.aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1843 VK_IMAGE_ASPECT_STENCIL_BIT));
1844
1845 struct blorp_surf depth = {};
1846 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1847 const uint32_t plane =
1848 anv_image_aspect_to_plane(image, VK_IMAGE_ASPECT_DEPTH_BIT);
1849 assert(base_layer + layer_count <=
1850 anv_image_aux_layers(image, VK_IMAGE_ASPECT_DEPTH_BIT, level));
1851 get_blorp_surf_for_anv_image(cmd_buffer,
1852 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1853 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1854 image->planes[plane].aux_usage,
1855 ISL_FORMAT_UNSUPPORTED, &depth);
1856 }
1857
1858 struct blorp_surf stencil = {};
1859 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1860 const uint32_t plane =
1861 anv_image_aspect_to_plane(image, VK_IMAGE_ASPECT_STENCIL_BIT);
1862 get_blorp_surf_for_anv_image(cmd_buffer,
1863 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1864 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1865 image->planes[plane].aux_usage,
1866 ISL_FORMAT_UNSUPPORTED, &stencil);
1867 }
1868
1869 /* From the Sky Lake PRM Volume 7, "Depth Buffer Clear":
1870 *
1871 * "The following is required when performing a depth buffer clear with
1872 * using the WM_STATE or 3DSTATE_WM:
1873 *
1874 * * If other rendering operations have preceded this clear, a
1875 * PIPE_CONTROL with depth cache flush enabled, Depth Stall bit
1876 * enabled must be issued before the rectangle primitive used for
1877 * the depth buffer clear operation.
1878 * * [...]"
1879 *
1880 * Even though the PRM only says that this is required if using 3DSTATE_WM
1881 * and a 3DPRIMITIVE, the GPU appears to also need this to avoid occasional
1882 * hangs when doing a clear with WM_HZ_OP.
1883 */
1884 anv_add_pending_pipe_bits(cmd_buffer,
1885 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT |
1886 ANV_PIPE_DEPTH_STALL_BIT,
1887 "before clear hiz");
1888
1889 if ((aspects & VK_IMAGE_ASPECT_DEPTH_BIT) &&
1890 depth.aux_usage == ISL_AUX_USAGE_HIZ_CCS_WT) {
1891 /* From Bspec 47010 (Depth Buffer Clear):
1892 *
1893 * Since the fast clear cycles to CCS are not cached in TileCache,
1894 * any previous depth buffer writes to overlapping pixels must be
1895 * flushed out of TileCache before a succeeding Depth Buffer Clear.
1896 * This restriction only applies to Depth Buffer with write-thru
1897 * enabled, since fast clears to CCS only occur for write-thru mode.
1898 *
1899 * There may have been a write to this depth buffer. Flush it from the
1900 * tile cache just in case.
1901 *
1902 * Set CS stall bit to guarantee that the fast clear starts the execution
1903 * after the tile cache flush completed.
1904 *
1905 * There is no Bspec requirement to flush the data cache but the
1906 * experiment shows that flusing the data cache helps to resolve the
1907 * corruption.
1908 */
1909 unsigned wa_flush = cmd_buffer->device->info->verx10 >= 125 ?
1910 ANV_PIPE_DATA_CACHE_FLUSH_BIT : 0;
1911 anv_add_pending_pipe_bits(cmd_buffer,
1912 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT |
1913 ANV_PIPE_CS_STALL_BIT |
1914 ANV_PIPE_TILE_CACHE_FLUSH_BIT |
1915 wa_flush,
1916 "before clear hiz_ccs_wt");
1917 }
1918
1919 blorp_hiz_clear_depth_stencil(batch, &depth, &stencil,
1920 level, base_layer, layer_count,
1921 area.offset.x, area.offset.y,
1922 area.offset.x + area.extent.width,
1923 area.offset.y + area.extent.height,
1924 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1925 clear_value->depth,
1926 aspects & VK_IMAGE_ASPECT_STENCIL_BIT,
1927 clear_value->stencil);
1928
1929 /* From the SKL PRM, Depth Buffer Clear:
1930 *
1931 * "Depth Buffer Clear Workaround
1932 *
1933 * Depth buffer clear pass using any of the methods (WM_STATE,
1934 * 3DSTATE_WM or 3DSTATE_WM_HZ_OP) must be followed by a PIPE_CONTROL
1935 * command with DEPTH_STALL bit and Depth FLUSH bits “set” before
1936 * starting to render. DepthStall and DepthFlush are not needed between
1937 * consecutive depth clear passes nor is it required if the depth-clear
1938 * pass was done with “full_surf_clear” bit set in the
1939 * 3DSTATE_WM_HZ_OP."
1940 *
1941 * Even though the PRM provides a bunch of conditions under which this is
1942 * supposedly unnecessary, we choose to perform the flush unconditionally
1943 * just to be safe.
1944 *
1945 * From Bspec 46959, a programming note applicable to Gfx12+:
1946 *
1947 * "Since HZ_OP has to be sent twice (first time set the clear/resolve state
1948 * and 2nd time to clear the state), and HW internally flushes the depth
1949 * cache on HZ_OP, there is no need to explicitly send a Depth Cache flush
1950 * after Clear or Resolve."
1951 */
1952 if (cmd_buffer->device->info->verx10 < 120) {
1953 anv_add_pending_pipe_bits(cmd_buffer,
1954 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT |
1955 ANV_PIPE_DEPTH_STALL_BIT,
1956 "after clear hiz");
1957 }
1958 }
1959
1960 static bool
can_hiz_clear_att(struct anv_cmd_buffer * cmd_buffer,struct blorp_batch * batch,const struct anv_attachment * ds_att,const VkClearAttachment * attachment,uint32_t rectCount,const VkClearRect * pRects)1961 can_hiz_clear_att(struct anv_cmd_buffer *cmd_buffer,
1962 struct blorp_batch *batch,
1963 const struct anv_attachment *ds_att,
1964 const VkClearAttachment *attachment,
1965 uint32_t rectCount, const VkClearRect *pRects)
1966 {
1967 if (INTEL_DEBUG(DEBUG_NO_FAST_CLEAR))
1968 return false;
1969
1970 /* From Bspec's section MI_PREDICATE:
1971 *
1972 * "The MI_PREDICATE command is used to control the Predicate state bit,
1973 * which in turn can be used to enable/disable the processing of
1974 * 3DPRIMITIVE commands."
1975 *
1976 * Also from BDW/CHV Bspec's 3DSTATE_WM_HZ_OP programming notes:
1977 *
1978 * "This command does NOT support predication from the use of the
1979 * MI_PREDICATE register. To predicate depth clears and resolves on you
1980 * must fall back to using the 3D_PRIMITIVE or GPGPU_WALKER commands."
1981 *
1982 * Since BLORP's predication is currently dependent on MI_PREDICATE, fall
1983 * back to the slow depth clear path when the BLORP_BATCH_PREDICATE_ENABLE
1984 * flag is set.
1985 */
1986 if (batch->flags & BLORP_BATCH_PREDICATE_ENABLE)
1987 return false;
1988
1989 if (rectCount > 1) {
1990 anv_perf_warn(VK_LOG_OBJS(&cmd_buffer->device->vk.base),
1991 "Fast clears for vkCmdClearAttachments supported only for rectCount == 1");
1992 return false;
1993 }
1994
1995 /* When the BLORP_BATCH_NO_EMIT_DEPTH_STENCIL flag is set, BLORP can only
1996 * clear the first slice of the currently configured depth/stencil view.
1997 */
1998 assert(batch->flags & BLORP_BATCH_NO_EMIT_DEPTH_STENCIL);
1999 if (pRects[0].layerCount > 1 || pRects[0].baseArrayLayer > 0)
2000 return false;
2001
2002 return anv_can_hiz_clear_ds_view(cmd_buffer->device, ds_att->iview,
2003 ds_att->layout,
2004 attachment->aspectMask,
2005 attachment->clearValue.depthStencil.depth,
2006 pRects->rect,
2007 cmd_buffer->queue_family->queueFlags);
2008 }
2009
2010 static void
clear_depth_stencil_attachment(struct anv_cmd_buffer * cmd_buffer,struct blorp_batch * batch,const VkClearAttachment * attachment,uint32_t rectCount,const VkClearRect * pRects)2011 clear_depth_stencil_attachment(struct anv_cmd_buffer *cmd_buffer,
2012 struct blorp_batch *batch,
2013 const VkClearAttachment *attachment,
2014 uint32_t rectCount, const VkClearRect *pRects)
2015 {
2016 static const union isl_color_value color_value = { .u32 = { 0, } };
2017 struct anv_cmd_graphics_state *gfx = &cmd_buffer->state.gfx;
2018 const struct anv_attachment *d_att = &gfx->depth_att;
2019 const struct anv_attachment *s_att = &gfx->stencil_att;
2020 if (d_att->vk_format == VK_FORMAT_UNDEFINED &&
2021 s_att->vk_format == VK_FORMAT_UNDEFINED)
2022 return;
2023
2024 const struct anv_attachment *ds_att = d_att->iview ? d_att : s_att;
2025 if (ds_att->iview &&
2026 can_hiz_clear_att(cmd_buffer, batch, ds_att, attachment, rectCount, pRects)) {
2027 anv_fast_clear_depth_stencil(cmd_buffer, batch, ds_att->iview->image,
2028 attachment->aspectMask,
2029 ds_att->iview->planes[0].isl.base_level,
2030 ds_att->iview->planes[0].isl.base_array_layer,
2031 pRects[0].layerCount, pRects->rect,
2032 &attachment->clearValue.depthStencil);
2033 return;
2034 }
2035
2036 bool clear_depth = attachment->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
2037 bool clear_stencil = attachment->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
2038
2039 enum isl_format depth_format = ISL_FORMAT_UNSUPPORTED;
2040 if (d_att->vk_format != VK_FORMAT_UNDEFINED) {
2041 depth_format = anv_get_isl_format(cmd_buffer->device->physical,
2042 d_att->vk_format,
2043 VK_IMAGE_ASPECT_DEPTH_BIT,
2044 VK_IMAGE_TILING_OPTIMAL);
2045 }
2046
2047 uint32_t binding_table;
2048 VkResult result =
2049 binding_table_for_surface_state(cmd_buffer,
2050 gfx->null_surface_state,
2051 &binding_table);
2052 if (result != VK_SUCCESS)
2053 return;
2054
2055 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
2056 if (gfx->view_mask) {
2057 u_foreach_bit(view_idx, gfx->view_mask) {
2058 for (uint32_t r = 0; r < rectCount; ++r) {
2059 const VkOffset2D offset = pRects[r].rect.offset;
2060 const VkExtent2D extent = pRects[r].rect.extent;
2061 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
2062 blorp_clear_attachments(batch, binding_table,
2063 depth_format,
2064 gfx->samples,
2065 view_idx, 1,
2066 offset.x, offset.y,
2067 offset.x + extent.width,
2068 offset.y + extent.height,
2069 false, color_value,
2070 clear_depth, value.depth,
2071 clear_stencil ? 0xff : 0, value.stencil);
2072 }
2073 }
2074 return;
2075 }
2076
2077 for (uint32_t r = 0; r < rectCount; ++r) {
2078 const VkOffset2D offset = pRects[r].rect.offset;
2079 const VkExtent2D extent = pRects[r].rect.extent;
2080 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
2081 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
2082 blorp_clear_attachments(batch, binding_table,
2083 depth_format,
2084 gfx->samples,
2085 pRects[r].baseArrayLayer,
2086 pRects[r].layerCount,
2087 offset.x, offset.y,
2088 offset.x + extent.width, offset.y + extent.height,
2089 false, color_value,
2090 clear_depth, value.depth,
2091 clear_stencil ? 0xff : 0, value.stencil);
2092 }
2093 }
2094
anv_CmdClearAttachments(VkCommandBuffer commandBuffer,uint32_t attachmentCount,const VkClearAttachment * pAttachments,uint32_t rectCount,const VkClearRect * pRects)2095 void anv_CmdClearAttachments(
2096 VkCommandBuffer commandBuffer,
2097 uint32_t attachmentCount,
2098 const VkClearAttachment* pAttachments,
2099 uint32_t rectCount,
2100 const VkClearRect* pRects)
2101 {
2102 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
2103
2104 /* Because this gets called within a render pass, we tell blorp not to
2105 * trash our depth and stencil buffers.
2106 */
2107 struct blorp_batch batch;
2108 enum blorp_batch_flags flags = BLORP_BATCH_NO_EMIT_DEPTH_STENCIL;
2109 if (cmd_buffer->state.conditional_render_enabled) {
2110 anv_cmd_emit_conditional_render_predicate(cmd_buffer);
2111 flags |= BLORP_BATCH_PREDICATE_ENABLE;
2112 }
2113 anv_blorp_batch_init(cmd_buffer, &batch, flags);
2114
2115 for (uint32_t a = 0; a < attachmentCount; ++a) {
2116 if (pAttachments[a].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
2117 assert(pAttachments[a].aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
2118 clear_color_attachment(cmd_buffer, &batch,
2119 &pAttachments[a],
2120 rectCount, pRects);
2121 } else {
2122 clear_depth_stencil_attachment(cmd_buffer, &batch,
2123 &pAttachments[a],
2124 rectCount, pRects);
2125 }
2126 }
2127
2128 anv_blorp_batch_finish(&batch);
2129 }
2130
2131 static void
anv_image_msaa_resolve(struct anv_cmd_buffer * cmd_buffer,const struct anv_image * src_image,enum isl_format src_format_override,enum isl_aux_usage src_aux_usage,uint32_t src_level,uint32_t src_base_layer,const struct anv_image * dst_image,enum isl_format dst_format_override,enum isl_aux_usage dst_aux_usage,uint32_t dst_level,uint32_t dst_base_layer,VkImageAspectFlagBits aspect,uint32_t src_x,uint32_t src_y,uint32_t dst_x,uint32_t dst_y,uint32_t width,uint32_t height,uint32_t layer_count,enum blorp_filter filter)2132 anv_image_msaa_resolve(struct anv_cmd_buffer *cmd_buffer,
2133 const struct anv_image *src_image,
2134 enum isl_format src_format_override,
2135 enum isl_aux_usage src_aux_usage,
2136 uint32_t src_level, uint32_t src_base_layer,
2137 const struct anv_image *dst_image,
2138 enum isl_format dst_format_override,
2139 enum isl_aux_usage dst_aux_usage,
2140 uint32_t dst_level, uint32_t dst_base_layer,
2141 VkImageAspectFlagBits aspect,
2142 uint32_t src_x, uint32_t src_y,
2143 uint32_t dst_x, uint32_t dst_y,
2144 uint32_t width, uint32_t height,
2145 uint32_t layer_count,
2146 enum blorp_filter filter)
2147 {
2148 struct blorp_batch batch;
2149 anv_blorp_batch_init(cmd_buffer, &batch, 0);
2150 assert((batch.flags & BLORP_BATCH_USE_COMPUTE) == 0);
2151
2152 assert(src_image->vk.image_type == VK_IMAGE_TYPE_2D);
2153 assert(src_image->vk.samples > 1);
2154 assert(dst_image->vk.image_type == VK_IMAGE_TYPE_2D);
2155 assert(dst_image->vk.samples == 1);
2156
2157 struct blorp_surf src_surf, dst_surf;
2158 get_blorp_surf_for_anv_image(cmd_buffer, src_image, aspect,
2159 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
2160 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
2161 src_aux_usage, src_format_override,
2162 &src_surf);
2163 get_blorp_surf_for_anv_image(cmd_buffer, dst_image, aspect,
2164 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
2165 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
2166 dst_aux_usage, dst_format_override,
2167 &dst_surf);
2168 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
2169 aspect, dst_aux_usage,
2170 dst_level, dst_base_layer, layer_count);
2171
2172 if (filter == BLORP_FILTER_NONE) {
2173 /* If no explicit filter is provided, then it's implied by the type of
2174 * the source image.
2175 */
2176 if ((src_surf.surf->usage & ISL_SURF_USAGE_DEPTH_BIT) ||
2177 (src_surf.surf->usage & ISL_SURF_USAGE_STENCIL_BIT) ||
2178 isl_format_has_int_channel(src_surf.surf->format)) {
2179 filter = BLORP_FILTER_SAMPLE_0;
2180 } else {
2181 filter = BLORP_FILTER_AVERAGE;
2182 }
2183 }
2184
2185 for (uint32_t l = 0; l < layer_count; l++) {
2186 blorp_blit(&batch,
2187 &src_surf, src_level, src_base_layer + l,
2188 src_format_override, ISL_SWIZZLE_IDENTITY,
2189 &dst_surf, dst_level, dst_base_layer + l,
2190 dst_format_override, ISL_SWIZZLE_IDENTITY,
2191 src_x, src_y, src_x + width, src_y + height,
2192 dst_x, dst_y, dst_x + width, dst_y + height,
2193 filter, false, false);
2194 }
2195
2196 anv_blorp_batch_finish(&batch);
2197 }
2198
2199 static enum blorp_filter
vk_to_blorp_resolve_mode(VkResolveModeFlagBits vk_mode)2200 vk_to_blorp_resolve_mode(VkResolveModeFlagBits vk_mode)
2201 {
2202 switch (vk_mode) {
2203 case VK_RESOLVE_MODE_SAMPLE_ZERO_BIT:
2204 return BLORP_FILTER_SAMPLE_0;
2205 case VK_RESOLVE_MODE_AVERAGE_BIT:
2206 return BLORP_FILTER_AVERAGE;
2207 case VK_RESOLVE_MODE_MIN_BIT:
2208 return BLORP_FILTER_MIN_SAMPLE;
2209 case VK_RESOLVE_MODE_MAX_BIT:
2210 return BLORP_FILTER_MAX_SAMPLE;
2211 default:
2212 return BLORP_FILTER_NONE;
2213 }
2214 }
2215
2216 void
anv_attachment_msaa_resolve(struct anv_cmd_buffer * cmd_buffer,const struct anv_attachment * att,VkImageLayout layout,VkImageAspectFlagBits aspect)2217 anv_attachment_msaa_resolve(struct anv_cmd_buffer *cmd_buffer,
2218 const struct anv_attachment *att,
2219 VkImageLayout layout,
2220 VkImageAspectFlagBits aspect)
2221 {
2222 struct anv_cmd_graphics_state *gfx = &cmd_buffer->state.gfx;
2223 const struct anv_image_view *src_iview = att->iview;
2224 const struct anv_image_view *dst_iview = att->resolve_iview;
2225
2226 enum isl_aux_usage src_aux_usage =
2227 anv_layout_to_aux_usage(cmd_buffer->device->info,
2228 src_iview->image, aspect,
2229 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
2230 layout,
2231 cmd_buffer->queue_family->queueFlags);
2232
2233 enum isl_aux_usage dst_aux_usage =
2234 anv_layout_to_aux_usage(cmd_buffer->device->info,
2235 dst_iview->image, aspect,
2236 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
2237 att->resolve_layout,
2238 cmd_buffer->queue_family->queueFlags);
2239
2240 enum blorp_filter filter = vk_to_blorp_resolve_mode(att->resolve_mode);
2241
2242 /* Depth/stencil should not use their view format for resolve because they
2243 * go in pairs.
2244 */
2245 enum isl_format src_format = ISL_FORMAT_UNSUPPORTED;
2246 enum isl_format dst_format = ISL_FORMAT_UNSUPPORTED;
2247 if (!(aspect & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT))) {
2248 src_format = src_iview->planes[0].isl.format;
2249 dst_format = dst_iview->planes[0].isl.format;
2250 }
2251
2252 const VkRect2D render_area = gfx->render_area;
2253 if (gfx->view_mask == 0) {
2254 anv_image_msaa_resolve(cmd_buffer,
2255 src_iview->image, src_format, src_aux_usage,
2256 src_iview->planes[0].isl.base_level,
2257 src_iview->planes[0].isl.base_array_layer,
2258 dst_iview->image, dst_format, dst_aux_usage,
2259 dst_iview->planes[0].isl.base_level,
2260 dst_iview->planes[0].isl.base_array_layer,
2261 aspect,
2262 render_area.offset.x, render_area.offset.y,
2263 render_area.offset.x, render_area.offset.y,
2264 render_area.extent.width,
2265 render_area.extent.height,
2266 gfx->layer_count, filter);
2267 } else {
2268 uint32_t res_view_mask = gfx->view_mask;
2269 while (res_view_mask) {
2270 int i = u_bit_scan(&res_view_mask);
2271
2272 anv_image_msaa_resolve(cmd_buffer,
2273 src_iview->image, src_format, src_aux_usage,
2274 src_iview->planes[0].isl.base_level,
2275 src_iview->planes[0].isl.base_array_layer + i,
2276 dst_iview->image, dst_format, dst_aux_usage,
2277 dst_iview->planes[0].isl.base_level,
2278 dst_iview->planes[0].isl.base_array_layer + i,
2279 aspect,
2280 render_area.offset.x, render_area.offset.y,
2281 render_area.offset.x, render_area.offset.y,
2282 render_area.extent.width,
2283 render_area.extent.height,
2284 1, filter);
2285 }
2286 }
2287 }
2288
2289 static void
resolve_image(struct anv_cmd_buffer * cmd_buffer,struct anv_image * src_image,VkImageLayout src_image_layout,struct anv_image * dst_image,VkImageLayout dst_image_layout,const VkImageResolve2 * region)2290 resolve_image(struct anv_cmd_buffer *cmd_buffer,
2291 struct anv_image *src_image,
2292 VkImageLayout src_image_layout,
2293 struct anv_image *dst_image,
2294 VkImageLayout dst_image_layout,
2295 const VkImageResolve2 *region)
2296 {
2297 assert(region->srcSubresource.aspectMask == region->dstSubresource.aspectMask);
2298 assert(vk_image_subresource_layer_count(&src_image->vk, ®ion->srcSubresource) ==
2299 vk_image_subresource_layer_count(&dst_image->vk, ®ion->dstSubresource));
2300
2301 const uint32_t layer_count =
2302 vk_image_subresource_layer_count(&dst_image->vk, ®ion->dstSubresource);
2303
2304 anv_foreach_image_aspect_bit(aspect_bit, src_image,
2305 region->srcSubresource.aspectMask) {
2306 enum isl_aux_usage src_aux_usage =
2307 anv_layout_to_aux_usage(cmd_buffer->device->info, src_image,
2308 (1 << aspect_bit),
2309 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
2310 src_image_layout,
2311 cmd_buffer->queue_family->queueFlags);
2312 enum isl_aux_usage dst_aux_usage =
2313 anv_layout_to_aux_usage(cmd_buffer->device->info, dst_image,
2314 (1 << aspect_bit),
2315 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
2316 dst_image_layout,
2317 cmd_buffer->queue_family->queueFlags);
2318
2319 anv_image_msaa_resolve(cmd_buffer,
2320 src_image, ISL_FORMAT_UNSUPPORTED, src_aux_usage,
2321 region->srcSubresource.mipLevel,
2322 region->srcSubresource.baseArrayLayer,
2323 dst_image, ISL_FORMAT_UNSUPPORTED, dst_aux_usage,
2324 region->dstSubresource.mipLevel,
2325 region->dstSubresource.baseArrayLayer,
2326 (1 << aspect_bit),
2327 region->srcOffset.x,
2328 region->srcOffset.y,
2329 region->dstOffset.x,
2330 region->dstOffset.y,
2331 region->extent.width,
2332 region->extent.height,
2333 layer_count, BLORP_FILTER_NONE);
2334 }
2335 }
2336
anv_CmdResolveImage2(VkCommandBuffer commandBuffer,const VkResolveImageInfo2 * pResolveImageInfo)2337 void anv_CmdResolveImage2(
2338 VkCommandBuffer commandBuffer,
2339 const VkResolveImageInfo2* pResolveImageInfo)
2340 {
2341 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
2342 ANV_FROM_HANDLE(anv_image, src_image, pResolveImageInfo->srcImage);
2343 ANV_FROM_HANDLE(anv_image, dst_image, pResolveImageInfo->dstImage);
2344
2345 for (uint32_t r = 0; r < pResolveImageInfo->regionCount; r++) {
2346 resolve_image(cmd_buffer,
2347 src_image, pResolveImageInfo->srcImageLayout,
2348 dst_image, pResolveImageInfo->dstImageLayout,
2349 &pResolveImageInfo->pRegions[r]);
2350 }
2351 }
2352
2353 void
anv_image_clear_color(struct anv_cmd_buffer * cmd_buffer,const struct anv_image * image,VkImageAspectFlagBits aspect,enum isl_aux_usage aux_usage,enum isl_format format,struct isl_swizzle swizzle,uint32_t level,uint32_t base_layer,uint32_t layer_count,VkRect2D area,union isl_color_value clear_color)2354 anv_image_clear_color(struct anv_cmd_buffer *cmd_buffer,
2355 const struct anv_image *image,
2356 VkImageAspectFlagBits aspect,
2357 enum isl_aux_usage aux_usage,
2358 enum isl_format format, struct isl_swizzle swizzle,
2359 uint32_t level, uint32_t base_layer, uint32_t layer_count,
2360 VkRect2D area, union isl_color_value clear_color)
2361 {
2362 assert(image->vk.aspects == VK_IMAGE_ASPECT_COLOR_BIT);
2363
2364 /* We don't support planar images with multisampling yet */
2365 assert(image->n_planes == 1);
2366
2367 struct blorp_batch batch;
2368 anv_blorp_batch_init(cmd_buffer, &batch, 0);
2369
2370 struct blorp_surf surf;
2371 get_blorp_surf_for_anv_image(cmd_buffer, image, aspect,
2372 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
2373 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
2374 aux_usage, format, &surf);
2375 anv_cmd_buffer_mark_image_written(cmd_buffer, image, aspect, aux_usage,
2376 level, base_layer, layer_count);
2377
2378 blorp_clear(&batch, &surf, format, anv_swizzle_for_render(swizzle),
2379 level, base_layer, layer_count,
2380 area.offset.x, area.offset.y,
2381 area.offset.x + area.extent.width,
2382 area.offset.y + area.extent.height,
2383 clear_color, 0 /* color_write_disable */);
2384
2385 anv_blorp_batch_finish(&batch);
2386 }
2387
2388 void
anv_image_clear_depth_stencil(struct anv_cmd_buffer * cmd_buffer,const struct anv_image * image,VkImageAspectFlags aspects,enum isl_aux_usage depth_aux_usage,uint32_t level,uint32_t base_layer,uint32_t layer_count,VkRect2D area,const VkClearDepthStencilValue * clear_value)2389 anv_image_clear_depth_stencil(struct anv_cmd_buffer *cmd_buffer,
2390 const struct anv_image *image,
2391 VkImageAspectFlags aspects,
2392 enum isl_aux_usage depth_aux_usage,
2393 uint32_t level,
2394 uint32_t base_layer, uint32_t layer_count,
2395 VkRect2D area,
2396 const VkClearDepthStencilValue *clear_value)
2397 {
2398 assert(image->vk.aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
2399 VK_IMAGE_ASPECT_STENCIL_BIT));
2400 assert(layer_count > 0);
2401
2402 struct blorp_batch batch;
2403 anv_blorp_batch_init(cmd_buffer, &batch, 0);
2404 assert((batch.flags & BLORP_BATCH_USE_COMPUTE) == 0);
2405
2406 struct blorp_surf depth = {};
2407 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
2408 get_blorp_surf_for_anv_image(cmd_buffer,
2409 image, VK_IMAGE_ASPECT_DEPTH_BIT,
2410 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
2411 depth_aux_usage, ISL_FORMAT_UNSUPPORTED,
2412 &depth);
2413 }
2414
2415 struct blorp_surf stencil = {};
2416 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
2417 const uint32_t plane =
2418 anv_image_aspect_to_plane(image, VK_IMAGE_ASPECT_STENCIL_BIT);
2419 get_blorp_surf_for_anv_image(cmd_buffer,
2420 image, VK_IMAGE_ASPECT_STENCIL_BIT,
2421 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
2422 image->planes[plane].aux_usage,
2423 ISL_FORMAT_UNSUPPORTED, &stencil);
2424 }
2425
2426 /* Blorp may choose to clear stencil using RGBA32_UINT for better
2427 * performance. If it does this, we need to flush it out of the depth
2428 * cache before rendering to it.
2429 */
2430 anv_add_pending_pipe_bits(cmd_buffer,
2431 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT |
2432 ANV_PIPE_END_OF_PIPE_SYNC_BIT,
2433 "before clear DS");
2434
2435 blorp_clear_depth_stencil(&batch, &depth, &stencil,
2436 level, base_layer, layer_count,
2437 area.offset.x, area.offset.y,
2438 area.offset.x + area.extent.width,
2439 area.offset.y + area.extent.height,
2440 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
2441 clear_value->depth,
2442 (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) ? 0xff : 0,
2443 clear_value->stencil);
2444
2445 /* Blorp may choose to clear stencil using RGBA32_UINT for better
2446 * performance. If it does this, we need to flush it out of the render
2447 * cache before someone starts trying to do stencil on it.
2448 */
2449 anv_add_pending_pipe_bits(cmd_buffer,
2450 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT |
2451 ANV_PIPE_END_OF_PIPE_SYNC_BIT,
2452 "after clear DS");
2453
2454 anv_blorp_batch_finish(&batch);
2455 }
2456
2457 void
anv_image_hiz_op(struct anv_cmd_buffer * cmd_buffer,const struct anv_image * image,VkImageAspectFlagBits aspect,uint32_t level,uint32_t base_layer,uint32_t layer_count,enum isl_aux_op hiz_op)2458 anv_image_hiz_op(struct anv_cmd_buffer *cmd_buffer,
2459 const struct anv_image *image,
2460 VkImageAspectFlagBits aspect, uint32_t level,
2461 uint32_t base_layer, uint32_t layer_count,
2462 enum isl_aux_op hiz_op)
2463 {
2464 assert(aspect == VK_IMAGE_ASPECT_DEPTH_BIT);
2465 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, level));
2466 const uint32_t plane = anv_image_aspect_to_plane(image, aspect);
2467 assert(plane == 0);
2468
2469 struct blorp_batch batch;
2470 anv_blorp_batch_init(cmd_buffer, &batch, 0);
2471 assert((batch.flags & BLORP_BATCH_USE_COMPUTE) == 0);
2472
2473 struct blorp_surf surf;
2474 get_blorp_surf_for_anv_image(cmd_buffer,
2475 image, VK_IMAGE_ASPECT_DEPTH_BIT,
2476 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
2477 image->planes[plane].aux_usage,
2478 ISL_FORMAT_UNSUPPORTED, &surf);
2479
2480 blorp_hiz_op(&batch, &surf, level, base_layer, layer_count, hiz_op);
2481
2482 anv_blorp_batch_finish(&batch);
2483 }
2484
2485 void
anv_image_hiz_clear(struct anv_cmd_buffer * cmd_buffer,const struct anv_image * image,VkImageAspectFlags aspects,uint32_t level,uint32_t base_layer,uint32_t layer_count,VkRect2D area,const VkClearDepthStencilValue * clear_value)2486 anv_image_hiz_clear(struct anv_cmd_buffer *cmd_buffer,
2487 const struct anv_image *image,
2488 VkImageAspectFlags aspects,
2489 uint32_t level,
2490 uint32_t base_layer, uint32_t layer_count,
2491 VkRect2D area,
2492 const VkClearDepthStencilValue *clear_value)
2493 {
2494 struct blorp_batch batch;
2495 anv_blorp_batch_init(cmd_buffer, &batch, 0);
2496 assert((batch.flags & BLORP_BATCH_USE_COMPUTE) == 0);
2497
2498 anv_fast_clear_depth_stencil(cmd_buffer, &batch, image, aspects, level,
2499 base_layer, layer_count, area, clear_value);
2500
2501 anv_blorp_batch_finish(&batch);
2502 }
2503
2504 void
anv_image_mcs_op(struct anv_cmd_buffer * cmd_buffer,const struct anv_image * image,enum isl_format format,struct isl_swizzle swizzle,VkImageAspectFlagBits aspect,uint32_t base_layer,uint32_t layer_count,enum isl_aux_op mcs_op,union isl_color_value * clear_value,bool predicate)2505 anv_image_mcs_op(struct anv_cmd_buffer *cmd_buffer,
2506 const struct anv_image *image,
2507 enum isl_format format, struct isl_swizzle swizzle,
2508 VkImageAspectFlagBits aspect,
2509 uint32_t base_layer, uint32_t layer_count,
2510 enum isl_aux_op mcs_op, union isl_color_value *clear_value,
2511 bool predicate)
2512 {
2513 struct blorp_batch batch;
2514 anv_blorp_batch_init(cmd_buffer, &batch,
2515 BLORP_BATCH_PREDICATE_ENABLE * predicate);
2516 assert((batch.flags & BLORP_BATCH_USE_COMPUTE) == 0);
2517
2518 exec_mcs_op(cmd_buffer, &batch, image, format, swizzle, aspect,
2519 base_layer, layer_count, mcs_op, clear_value);
2520
2521 anv_blorp_batch_finish(&batch);
2522 }
2523
2524 void
anv_image_ccs_op(struct anv_cmd_buffer * cmd_buffer,const struct anv_image * image,enum isl_format format,struct isl_swizzle swizzle,VkImageAspectFlagBits aspect,uint32_t level,uint32_t base_layer,uint32_t layer_count,enum isl_aux_op ccs_op,union isl_color_value * clear_value,bool predicate)2525 anv_image_ccs_op(struct anv_cmd_buffer *cmd_buffer,
2526 const struct anv_image *image,
2527 enum isl_format format, struct isl_swizzle swizzle,
2528 VkImageAspectFlagBits aspect, uint32_t level,
2529 uint32_t base_layer, uint32_t layer_count,
2530 enum isl_aux_op ccs_op, union isl_color_value *clear_value,
2531 bool predicate)
2532 {
2533 struct blorp_batch batch;
2534 anv_blorp_batch_init(cmd_buffer, &batch,
2535 BLORP_BATCH_PREDICATE_ENABLE * predicate);
2536 assert((batch.flags & BLORP_BATCH_USE_COMPUTE) == 0);
2537
2538 exec_ccs_op(cmd_buffer, &batch, image, format, swizzle, aspect, level,
2539 base_layer, layer_count, ccs_op, clear_value);
2540
2541 anv_blorp_batch_finish(&batch);
2542 }
2543