1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include <assert.h>
25 #include <stdbool.h>
26 #include <string.h>
27 #include <unistd.h>
28 #include <fcntl.h>
29 #include <sys/mman.h>
30 #include "drm-uapi/drm_fourcc.h"
31
32 #include "anv_private.h"
33 #include "common/intel_aux_map.h"
34 #include "util/u_debug.h"
35 #include "vk_util.h"
36 #include "util/u_math.h"
37
38 #include "vk_format.h"
39 #include "av1_tables.h"
40
41 #define ANV_OFFSET_IMPLICIT UINT64_MAX
42
43 static const enum isl_surf_dim
44 vk_to_isl_surf_dim[] = {
45 [VK_IMAGE_TYPE_1D] = ISL_SURF_DIM_1D,
46 [VK_IMAGE_TYPE_2D] = ISL_SURF_DIM_2D,
47 [VK_IMAGE_TYPE_3D] = ISL_SURF_DIM_3D,
48 };
49
50 static uint64_t MUST_CHECK UNUSED
memory_range_end(struct anv_image_memory_range memory_range)51 memory_range_end(struct anv_image_memory_range memory_range)
52 {
53 assert(anv_is_aligned(memory_range.offset, memory_range.alignment));
54 return memory_range.offset + memory_range.size;
55 }
56
57 /**
58 * Get binding for VkImagePlaneMemoryRequirementsInfo,
59 * VkBindImagePlaneMemoryInfo and VkDeviceImageMemoryRequirements.
60 */
61 enum anv_image_memory_binding
anv_image_aspect_to_binding(struct anv_image * image,VkImageAspectFlags aspect)62 anv_image_aspect_to_binding(struct anv_image *image,
63 VkImageAspectFlags aspect)
64 {
65 uint32_t plane = 0;
66
67 assert(image->disjoint);
68
69 if (image->vk.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
70 /* Spec requires special aspects for modifier images. */
71 assert(aspect == VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT ||
72 aspect == VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT ||
73 aspect == VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT ||
74 aspect == VK_IMAGE_ASPECT_MEMORY_PLANE_3_BIT_EXT);
75
76 /* We don't advertise DISJOINT for modifiers with aux, and therefore we
77 * don't handle queries of the modifier's "aux plane" here.
78 */
79 assert(!isl_drm_modifier_has_aux(image->vk.drm_format_mod));
80
81 switch(aspect) {
82 case VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT: plane = 0; break;
83 case VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT: plane = 1; break;
84 case VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT: plane = 2; break;
85 case VK_IMAGE_ASPECT_MEMORY_PLANE_3_BIT_EXT: plane = 3; break;
86 }
87 } else {
88 plane = anv_image_aspect_to_plane(image, aspect);
89 }
90
91 return ANV_IMAGE_MEMORY_BINDING_PLANE_0 + plane;
92 }
93
94 /**
95 * Extend the memory binding's range by appending a new memory range with `size`
96 * and `alignment` at `offset`. Return the appended range.
97 *
98 * Offset is ignored if ANV_OFFSET_IMPLICIT.
99 *
100 * The given binding must not be ANV_IMAGE_MEMORY_BINDING_MAIN. The function
101 * converts to MAIN as needed.
102 */
103 static VkResult MUST_CHECK
image_binding_grow(const struct anv_device * device,struct anv_image * image,enum anv_image_memory_binding binding,uint64_t offset,uint64_t size,uint32_t alignment,struct anv_image_memory_range * out_range)104 image_binding_grow(const struct anv_device *device,
105 struct anv_image *image,
106 enum anv_image_memory_binding binding,
107 uint64_t offset,
108 uint64_t size,
109 uint32_t alignment,
110 struct anv_image_memory_range *out_range)
111 {
112 /* We overwrite 'offset' but need to remember if it was implicit. */
113 const bool has_implicit_offset = (offset == ANV_OFFSET_IMPLICIT);
114
115 assert(size > 0);
116 assert(util_is_power_of_two_or_zero(alignment));
117
118 switch (binding) {
119 case ANV_IMAGE_MEMORY_BINDING_MAIN:
120 /* The caller must not pre-translate BINDING_PLANE_i to BINDING_MAIN. */
121 unreachable("ANV_IMAGE_MEMORY_BINDING_MAIN");
122 case ANV_IMAGE_MEMORY_BINDING_PLANE_0:
123 case ANV_IMAGE_MEMORY_BINDING_PLANE_1:
124 case ANV_IMAGE_MEMORY_BINDING_PLANE_2:
125 if (!image->disjoint)
126 binding = ANV_IMAGE_MEMORY_BINDING_MAIN;
127 break;
128 case ANV_IMAGE_MEMORY_BINDING_PRIVATE:
129 assert(offset == ANV_OFFSET_IMPLICIT);
130 break;
131 case ANV_IMAGE_MEMORY_BINDING_END:
132 unreachable("ANV_IMAGE_MEMORY_BINDING_END");
133 }
134
135 struct anv_image_memory_range *container =
136 &image->bindings[binding].memory_range;
137
138 if (has_implicit_offset) {
139 offset = align64(container->offset + container->size, alignment);
140 } else {
141 /* Offset must be validated because it comes from
142 * VkImageDrmFormatModifierExplicitCreateInfoEXT.
143 */
144 if (unlikely(!anv_is_aligned(offset, alignment))) {
145 return vk_errorf(device,
146 VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT,
147 "VkImageDrmFormatModifierExplicitCreateInfoEXT::"
148 "pPlaneLayouts[]::offset is misaligned");
149 }
150 }
151
152 /* Surfaces can be added out of memory-order. Track the end of each memory
153 * plane to update the binding size properly.
154 */
155 uint64_t memory_range_end;
156 if (__builtin_add_overflow(offset, size, &memory_range_end)) {
157 if (has_implicit_offset) {
158 assert(!"overflow");
159 return vk_errorf(device, VK_ERROR_UNKNOWN,
160 "internal error: overflow in %s", __func__);
161 } else {
162 return vk_errorf(device,
163 VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT,
164 "VkImageDrmFormatModifierExplicitCreateInfoEXT::"
165 "pPlaneLayouts[]::offset is too large");
166 }
167 }
168
169 container->size = MAX2(container->size, memory_range_end);
170 container->alignment = MAX2(container->alignment, alignment);
171
172 *out_range = (struct anv_image_memory_range) {
173 .binding = binding,
174 .alignment = alignment,
175 .size = size,
176 .offset = offset,
177 };
178
179 return VK_SUCCESS;
180 }
181
182 /**
183 * Adjust range 'a' to contain range 'b'.
184 *
185 * For simplicity's sake, the offset of 'a' must be 0 and remains 0.
186 * If 'a' and 'b' target different bindings, then no merge occurs.
187 */
188 static void
memory_range_merge(struct anv_image_memory_range * a,const struct anv_image_memory_range b)189 memory_range_merge(struct anv_image_memory_range *a,
190 const struct anv_image_memory_range b)
191 {
192 if (b.size == 0)
193 return;
194
195 if (a->binding != b.binding)
196 return;
197
198 assert(a->offset == 0);
199 assert(anv_is_aligned(a->offset, a->alignment));
200 assert(anv_is_aligned(b.offset, b.alignment));
201
202 a->alignment = MAX2(a->alignment, b.alignment);
203 a->size = MAX2(a->size, b.offset + b.size);
204 }
205
206 isl_surf_usage_flags_t
anv_image_choose_isl_surf_usage(struct anv_physical_device * device,VkImageCreateFlags vk_create_flags,VkImageUsageFlags vk_usage,isl_surf_usage_flags_t isl_extra_usage,VkImageAspectFlagBits aspect,VkImageCompressionFlagsEXT comp_flags)207 anv_image_choose_isl_surf_usage(struct anv_physical_device *device,
208 VkImageCreateFlags vk_create_flags,
209 VkImageUsageFlags vk_usage,
210 isl_surf_usage_flags_t isl_extra_usage,
211 VkImageAspectFlagBits aspect,
212 VkImageCompressionFlagsEXT comp_flags)
213 {
214 isl_surf_usage_flags_t isl_usage = isl_extra_usage;
215
216 /* On platform like MTL, we choose to allocate additional CCS memory at the
217 * back of the VkDeviceMemory objects since different images can share the
218 * AUX-TT PTE because the HW doesn't care about the image format in the
219 * PTE. That means we can always ignore the AUX-TT alignment requirement
220 * from an ISL point of view.
221 */
222 if (device->alloc_aux_tt_mem)
223 isl_usage |= ISL_SURF_USAGE_NO_AUX_TT_ALIGNMENT_BIT;
224
225 if (vk_usage & VK_IMAGE_USAGE_SAMPLED_BIT)
226 isl_usage |= ISL_SURF_USAGE_TEXTURE_BIT;
227
228 if (vk_usage & VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT)
229 isl_usage |= ISL_SURF_USAGE_TEXTURE_BIT;
230
231 if (vk_usage & VK_IMAGE_USAGE_STORAGE_BIT)
232 isl_usage |= ISL_SURF_USAGE_STORAGE_BIT;
233
234 if (vk_usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT)
235 isl_usage |= ISL_SURF_USAGE_RENDER_TARGET_BIT;
236
237 if (vk_usage & VK_IMAGE_USAGE_FRAGMENT_SHADING_RATE_ATTACHMENT_BIT_KHR)
238 isl_usage |= ISL_SURF_USAGE_CPB_BIT;
239
240 /* TODO: consider whether compression with sparse is workable. */
241 if (vk_create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT)
242 isl_usage |= ISL_SURF_USAGE_SPARSE_BIT |
243 ISL_SURF_USAGE_DISABLE_AUX_BIT;
244
245 if (vk_usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR ||
246 vk_usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR ||
247 vk_usage & VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR ||
248 vk_usage & VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR)
249 isl_usage |= ISL_SURF_USAGE_VIDEO_DECODE_BIT;
250
251 /* We disable aux surfaces for host read/write images so that we can update
252 * the main surface without caring about the auxiliary surface.
253 */
254 if (vk_usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT)
255 isl_usage |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
256
257 if (vk_create_flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT)
258 isl_usage |= ISL_SURF_USAGE_CUBE_BIT;
259
260 if (vk_create_flags & (VK_IMAGE_CREATE_2D_VIEW_COMPATIBLE_BIT_EXT |
261 VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT))
262 isl_usage |= ISL_SURF_USAGE_2D_3D_COMPATIBLE_BIT;
263
264 if (vk_create_flags & VK_IMAGE_CREATE_PROTECTED_BIT)
265 isl_usage |= ISL_SURF_USAGE_PROTECTED_BIT;
266
267 /* Even if we're only using it for transfer operations, clears to depth and
268 * stencil images happen as depth and stencil so they need the right ISL
269 * usage bits or else things will fall apart.
270 */
271 switch (aspect) {
272 case VK_IMAGE_ASPECT_DEPTH_BIT:
273 isl_usage |= ISL_SURF_USAGE_DEPTH_BIT;
274 break;
275 case VK_IMAGE_ASPECT_STENCIL_BIT:
276 isl_usage |= ISL_SURF_USAGE_STENCIL_BIT;
277 break;
278 case VK_IMAGE_ASPECT_COLOR_BIT:
279 case VK_IMAGE_ASPECT_PLANE_0_BIT:
280 case VK_IMAGE_ASPECT_PLANE_1_BIT:
281 case VK_IMAGE_ASPECT_PLANE_2_BIT:
282 break;
283 default:
284 unreachable("bad VkImageAspect");
285 }
286
287 if (vk_usage & VK_IMAGE_USAGE_TRANSFER_SRC_BIT) {
288 /* blorp implements transfers by sampling from the source image. */
289 isl_usage |= ISL_SURF_USAGE_TEXTURE_BIT;
290 }
291
292 if (vk_usage & VK_IMAGE_USAGE_TRANSFER_DST_BIT &&
293 aspect == VK_IMAGE_ASPECT_COLOR_BIT) {
294 /* blorp implements transfers by rendering into the destination image.
295 * Only request this with color images, as we deal with depth/stencil
296 * formats differently. */
297 isl_usage |= ISL_SURF_USAGE_RENDER_TARGET_BIT;
298 }
299
300 if (comp_flags & VK_IMAGE_COMPRESSION_DISABLED_EXT)
301 isl_usage |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
302
303 return isl_usage;
304 }
305
306 static isl_tiling_flags_t
choose_isl_tiling_flags(const struct intel_device_info * devinfo,const struct anv_image * image,const struct anv_image_create_info * anv_info,const struct isl_drm_modifier_info * isl_mod_info)307 choose_isl_tiling_flags(const struct intel_device_info *devinfo,
308 const struct anv_image *image,
309 const struct anv_image_create_info *anv_info,
310 const struct isl_drm_modifier_info *isl_mod_info)
311 {
312 const VkImageCreateInfo *base_info = anv_info->vk_info;
313 isl_tiling_flags_t flags = 0;
314
315 assert((isl_mod_info != NULL) ==
316 (base_info->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT));
317
318 switch (base_info->tiling) {
319 default:
320 unreachable("bad VkImageTiling");
321 case VK_IMAGE_TILING_OPTIMAL:
322 flags = ISL_TILING_ANY_MASK;
323 break;
324 case VK_IMAGE_TILING_LINEAR:
325 flags = ISL_TILING_LINEAR_BIT;
326 break;
327 case VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT:
328 flags = 1 << isl_mod_info->tiling;
329 }
330
331 if (anv_info->isl_tiling_flags) {
332 assert(isl_mod_info == NULL);
333 flags &= anv_info->isl_tiling_flags;
334 }
335
336 if (image->vk.wsi_legacy_scanout) {
337 isl_tiling_flags_t legacy_mask = ISL_TILING_LINEAR_BIT;
338 if (devinfo->has_tiling_uapi)
339 legacy_mask |= ISL_TILING_X_BIT;
340 flags &= legacy_mask;
341 }
342
343 assert(flags);
344
345 return flags;
346 }
347
348 /**
349 * Add the surface to the binding at the given offset.
350 *
351 * \see image_binding_grow()
352 */
353 static VkResult MUST_CHECK
add_surface(struct anv_device * device,struct anv_image * image,struct anv_surface * surf,enum anv_image_memory_binding binding,uint64_t offset)354 add_surface(struct anv_device *device,
355 struct anv_image *image,
356 struct anv_surface *surf,
357 enum anv_image_memory_binding binding,
358 uint64_t offset)
359 {
360 /* isl surface must be initialized */
361 assert(surf->isl.size_B > 0);
362
363 return image_binding_grow(device, image, binding, offset,
364 surf->isl.size_B,
365 surf->isl.alignment_B,
366 &surf->memory_range);
367 }
368
369 static bool
image_may_use_r32_view(VkImageCreateFlags create_flags,VkFormat vk_format,const VkImageFormatListCreateInfo * fmt_list)370 image_may_use_r32_view(VkImageCreateFlags create_flags,
371 VkFormat vk_format,
372 const VkImageFormatListCreateInfo *fmt_list)
373 {
374 if (vk_format == VK_FORMAT_R32_SINT ||
375 vk_format == VK_FORMAT_R32_UINT ||
376 vk_format == VK_FORMAT_R32_SFLOAT)
377 return true;
378
379 if (!(create_flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT))
380 return false;
381
382 if (fmt_list) {
383 for (uint32_t i = 0; i < fmt_list->viewFormatCount; i++) {
384 if (fmt_list->pViewFormats[i] == VK_FORMAT_R32_SINT ||
385 fmt_list->pViewFormats[i] == VK_FORMAT_R32_UINT ||
386 fmt_list->pViewFormats[i] == VK_FORMAT_R32_SFLOAT)
387 return true;
388 }
389
390 return false;
391 }
392
393 return vk_format_get_blocksizebits(vk_format) == 32;
394 }
395
396 static bool
formats_ccs_e_compatible(const struct intel_device_info * devinfo,VkImageCreateFlags create_flags,VkImageAspectFlagBits aspect,enum isl_format format,VkImageTiling vk_tiling,const VkImageFormatListCreateInfo * fmt_list)397 formats_ccs_e_compatible(const struct intel_device_info *devinfo,
398 VkImageCreateFlags create_flags,
399 VkImageAspectFlagBits aspect,
400 enum isl_format format, VkImageTiling vk_tiling,
401 const VkImageFormatListCreateInfo *fmt_list)
402 {
403 if (!anv_format_supports_ccs_e(devinfo, format))
404 return false;
405
406 /* For images created without MUTABLE_FORMAT_BIT set, we know that they will
407 * always be used with the original format. In particular, they will always
408 * be used with a format that supports color compression.
409 */
410 if (!(create_flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT))
411 return true;
412
413 if (!fmt_list || fmt_list->viewFormatCount == 0)
414 return false;
415
416 for (uint32_t i = 0; i < fmt_list->viewFormatCount; i++) {
417 if (fmt_list->pViewFormats[i] == VK_FORMAT_UNDEFINED)
418 continue;
419
420 enum isl_format view_format =
421 anv_get_isl_format(devinfo, fmt_list->pViewFormats[i],
422 aspect, vk_tiling);
423
424 if (!isl_formats_are_ccs_e_compatible(devinfo, format, view_format))
425 return false;
426 }
427
428 return true;
429 }
430
431 bool
anv_format_supports_ccs_e(const struct intel_device_info * devinfo,const enum isl_format format)432 anv_format_supports_ccs_e(const struct intel_device_info *devinfo,
433 const enum isl_format format)
434 {
435 /* CCS_E for YCRCB_NORMAL and YCRCB_SWAP_UV is not currently supported by
436 * ANV so leave it disabled for now.
437 */
438 if (isl_format_is_yuv(format))
439 return false;
440
441 return isl_format_supports_ccs_e(devinfo, format);
442 }
443
444 bool
anv_formats_ccs_e_compatible(const struct intel_device_info * devinfo,VkImageCreateFlags create_flags,VkFormat vk_format,VkImageTiling vk_tiling,VkImageUsageFlags vk_usage,const VkImageFormatListCreateInfo * fmt_list)445 anv_formats_ccs_e_compatible(const struct intel_device_info *devinfo,
446 VkImageCreateFlags create_flags,
447 VkFormat vk_format, VkImageTiling vk_tiling,
448 VkImageUsageFlags vk_usage,
449 const VkImageFormatListCreateInfo *fmt_list)
450 {
451 u_foreach_bit(b, vk_format_aspects(vk_format)) {
452 VkImageAspectFlagBits aspect = 1 << b;
453 enum isl_format format =
454 anv_get_isl_format(devinfo, vk_format, aspect, vk_tiling);
455
456 if (!formats_ccs_e_compatible(devinfo, create_flags, aspect,
457 format, vk_tiling, fmt_list))
458 return false;
459 }
460
461 if ((vk_usage & VK_IMAGE_USAGE_STORAGE_BIT) &&
462 vk_tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
463 assert(vk_format_aspects(vk_format) == VK_IMAGE_ASPECT_COLOR_BIT);
464 if (devinfo->ver == 12) {
465 /* From the TGL Bspec 44930 (r47128):
466 *
467 * "Memory atomic operation on compressed data is not supported
468 * in Gen12 E2E compression. Result of such operation is
469 * undefined.
470 *
471 * Software should ensure at the time of the Atomic operation
472 * the surface is resolved (uncompressed) state."
473 *
474 * On gfx12.0, compression is not supported with atomic
475 * operations. On gfx12.5, the support is there, but it's slow
476 * (see HSD 1406337848).
477 */
478 if (image_may_use_r32_view(create_flags, vk_format, fmt_list))
479 return false;
480
481 } else if (devinfo->ver <= 11) {
482 /* Storage accesses are not supported on compressed surfaces. */
483 return false;
484 }
485 }
486
487 return true;
488 }
489
490 /**
491 * For color images that have an auxiliary surface, request allocation for an
492 * additional buffer that mainly stores fast-clear values. Use of this buffer
493 * allows us to access the image's subresources while being aware of their
494 * fast-clear values in non-trivial cases (e.g., outside of a render pass in
495 * which a fast clear has occurred).
496 *
497 * In order to avoid having multiple clear colors for a single plane of an
498 * image (hence a single RENDER_SURFACE_STATE), we only allow fast-clears on
499 * the first slice (level 0, layer 0). At the time of our testing (Jan 17,
500 * 2018), there were no known applications which would benefit from fast-
501 * clearing more than just the first slice.
502 *
503 * The fast clear portion of the image is laid out in the following order:
504 *
505 * * 1 or 4 dwords (depending on hardware generation) for the clear color
506 * * 1 dword for the anv_fast_clear_type of the clear color
507 * * On gfx9+, 1 dword per level and layer of the image (3D levels count
508 * multiple layers) in level-major order for compression state.
509 *
510 * For the purpose of discoverability, the algorithm used to manage
511 * compression and fast-clears is described here:
512 *
513 * * On a transition from UNDEFINED or PREINITIALIZED to a defined layout,
514 * all of the values in the fast clear portion of the image are initialized
515 * to default values.
516 *
517 * * On fast-clear, the clear value is written into surface state and also
518 * into the buffer and the fast clear type is set appropriately. Both
519 * setting the fast-clear value in the buffer and setting the fast-clear
520 * type happen from the GPU using MI commands.
521 *
522 * * Whenever a render or blorp operation is performed with CCS_E, we call
523 * genX(cmd_buffer_mark_image_written) to set the compression state to
524 * true (which is represented by UINT32_MAX).
525 *
526 * * On pipeline barrier transitions, the worst-case transition is computed
527 * from the image layouts. The command streamer inspects the fast clear
528 * type and compression state dwords and constructs a predicate. The
529 * worst-case resolve is performed with the given predicate and the fast
530 * clear and compression state is set accordingly.
531 *
532 * See anv_layout_to_aux_usage and anv_layout_to_fast_clear_type functions for
533 * details on exactly what is allowed in what layouts.
534 *
535 * On gfx7-9, we do not have a concept of indirect clear colors in hardware.
536 * In order to deal with this, we have to do some clear color management.
537 *
538 * * For LOAD_OP_LOAD at the top of a renderpass, we have to copy the clear
539 * value from the buffer into the surface state with MI commands.
540 *
541 * * For any blorp operations, we pass the address to the clear value into
542 * blorp and it knows to copy the clear color.
543 */
544 static VkResult MUST_CHECK
add_aux_state_tracking_buffer(struct anv_device * device,struct anv_image * image,uint64_t state_offset,uint32_t plane)545 add_aux_state_tracking_buffer(struct anv_device *device,
546 struct anv_image *image,
547 uint64_t state_offset,
548 uint32_t plane)
549 {
550 assert(image && device);
551 /* Xe2+ platforms don't use aux tracking buffers. We shouldn't get here. */
552 assert(device->info->ver < 20);
553 assert(image->planes[plane].aux_usage != ISL_AUX_USAGE_NONE &&
554 image->vk.aspects & (VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV |
555 VK_IMAGE_ASPECT_DEPTH_BIT));
556
557 unsigned clear_color_state_size;
558 if (device->info->ver >= 11) {
559 /* When importing an image from another source with a drm modifier that
560 * supports clear color, the clear color values are in a 32-byte struct
561 * defined in drm_fourcc.h. The fast clear type and compression state
562 * are not defined in these drm_fourcc.h, so there won't be memory
563 * allocated for these extra meta data by the source.
564 *
565 * We use the last 2 dwords of the clear color struct's memory to store
566 * the fast clear type and the first compression state, so the driver
567 * doesn't assume the extra size or need another allocation later.
568 *
569 * So far, the 2 stolen dwords are either not used in the clear color
570 * struct or for features not enabled. There should be no side effect to
571 * the hardware and destinations of images exported by this driver.
572 *
573 * Images with multiple levels or layers are not supported by drm
574 * modifiers, so we don't have to apply the above approach or face a
575 * bigger shortage from multiple compression states. We just apply the
576 * approach to all cases to keep the design unified.
577 *
578 * As a result, the state starts 8 bytes lower than where it should be.
579 */
580 assert(device->isl_dev.ss.clear_color_state_size == 32);
581 clear_color_state_size = (image->num_view_formats - 1) * 64 + 32 - 8;
582 } else {
583 /* When sampling or rendering with an sRGB format, HW expects the clear
584 * color to be in two different color spaces - sRGB in the former and
585 * linear in the latter. Allocate twice the space to support either
586 * access.
587 */
588 assert(device->isl_dev.ss.clear_value_size == 16);
589 clear_color_state_size = image->num_view_formats * 16 * 2;
590 }
591
592 /* Clear color and fast clear type */
593 unsigned state_size = clear_color_state_size + 4;
594
595 /* We only need to track compression on CCS_E surfaces. */
596 if (isl_aux_usage_has_ccs_e(image->planes[plane].aux_usage)) {
597 if (image->vk.image_type == VK_IMAGE_TYPE_3D) {
598 for (uint32_t l = 0; l < image->vk.mip_levels; l++)
599 state_size += u_minify(image->vk.extent.depth, l) * 4;
600 } else {
601 state_size += image->vk.mip_levels * image->vk.array_layers * 4;
602 }
603 }
604
605 enum anv_image_memory_binding binding =
606 ANV_IMAGE_MEMORY_BINDING_PLANE_0 + plane;
607
608 const struct isl_drm_modifier_info *mod_info =
609 isl_drm_modifier_get_info(image->vk.drm_format_mod);
610
611 /* If an auxiliary surface is used for an externally-shareable image,
612 * we have to hide this from the memory of the image since other
613 * processes with access to the memory may not be aware of it or of
614 * its current state. So put that auxiliary data into a separate
615 * buffer (ANV_IMAGE_MEMORY_BINDING_PRIVATE).
616 *
617 * But when the image is created with a drm modifier that supports
618 * clear color, it will be exported along with main surface.
619 */
620 if (anv_image_is_externally_shared(image) &&
621 !mod_info->supports_clear_color)
622 binding = ANV_IMAGE_MEMORY_BINDING_PRIVATE;
623
624 /* The indirect clear color BO requires 64B-alignment on gfx11+. If we're
625 * using a modifier with clear color, then some kernels might require a 4k
626 * alignment.
627 */
628 const uint32_t clear_color_alignment =
629 (mod_info && mod_info->supports_clear_color) ? 4096 : 64;
630
631 return image_binding_grow(device, image, binding,
632 state_offset, state_size, clear_color_alignment,
633 &image->planes[plane].fast_clear_memory_range);
634 }
635
636 static VkResult MUST_CHECK
add_compression_control_buffer(struct anv_device * device,struct anv_image * image,uint32_t plane,uint32_t binding,uint64_t offset)637 add_compression_control_buffer(struct anv_device *device,
638 struct anv_image *image,
639 uint32_t plane,
640 uint32_t binding,
641 uint64_t offset)
642 {
643 assert(device->info->has_aux_map);
644
645 return image_binding_grow(device, image, binding, offset,
646 image->planes[plane].primary_surface.isl.size_B /
647 INTEL_AUX_MAP_MAIN_SIZE_SCALEDOWN,
648 INTEL_AUX_MAP_META_ALIGNMENT_B,
649 &image->planes[plane].compr_ctrl_memory_range);
650 }
651
652 static bool
want_hiz_wt_for_image(const struct intel_device_info * devinfo,const struct anv_image * image)653 want_hiz_wt_for_image(const struct intel_device_info *devinfo,
654 const struct anv_image *image)
655 {
656 /* Gen12 only supports single-sampled while Gen20+ supports
657 * multi-sampled images.
658 */
659 if (devinfo->ver < 20 && image->vk.samples > 1)
660 return false;
661
662 if ((image->vk.usage & (VK_IMAGE_USAGE_SAMPLED_BIT |
663 VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT)) == 0)
664 return false;
665
666 /* If this image has the maximum number of samples supported by
667 * running platform and will be used as a texture, put the HiZ surface
668 * in write-through mode so that we can sample from it.
669 *
670 * TODO: This is a heuristic trade-off; we haven't tuned it at all.
671 */
672 return true;
673 }
674
675 /**
676 * The return code indicates whether creation of the VkImage should continue
677 * or fail, not whether the creation of the aux surface succeeded. If the aux
678 * surface is not required (for example, by neither hardware nor DRM format
679 * modifier), then this may return VK_SUCCESS when creation of the aux surface
680 * fails.
681 *
682 * @param offset See add_surface()
683 */
684 static VkResult
add_aux_surface_if_supported(struct anv_device * device,struct anv_image * image,uint32_t plane,struct anv_format_plane plane_format,const VkImageFormatListCreateInfo * fmt_list,uint64_t offset,uint32_t stride,uint64_t aux_state_offset)685 add_aux_surface_if_supported(struct anv_device *device,
686 struct anv_image *image,
687 uint32_t plane,
688 struct anv_format_plane plane_format,
689 const VkImageFormatListCreateInfo *fmt_list,
690 uint64_t offset,
691 uint32_t stride,
692 uint64_t aux_state_offset)
693 {
694 VkImageAspectFlags aspect = plane_format.aspect;
695 VkResult result;
696 bool ok;
697
698 /* The aux surface must not be already added. */
699 assert(!anv_surface_is_valid(&image->planes[plane].aux_surface));
700
701 if (image->planes[plane].primary_surface.isl.usage &
702 ISL_SURF_USAGE_DISABLE_AUX_BIT)
703 return VK_SUCCESS;
704
705 uint32_t binding;
706 if (image->vk.drm_format_mod == DRM_FORMAT_MOD_INVALID ||
707 isl_drm_modifier_has_aux(image->vk.drm_format_mod)) {
708 binding = ANV_IMAGE_MEMORY_BINDING_PLANE_0 + plane;
709 } else {
710 binding = ANV_IMAGE_MEMORY_BINDING_PRIVATE;
711 }
712
713 if (aspect == VK_IMAGE_ASPECT_DEPTH_BIT) {
714 /* We don't advertise that depth buffers could be used as storage
715 * images.
716 */
717 assert(!(image->vk.usage & VK_IMAGE_USAGE_STORAGE_BIT));
718
719 ok = isl_surf_get_hiz_surf(&device->isl_dev,
720 &image->planes[plane].primary_surface.isl,
721 &image->planes[plane].aux_surface.isl);
722 if (!ok)
723 return VK_SUCCESS;
724
725 if (!isl_surf_supports_ccs(&device->isl_dev,
726 &image->planes[plane].primary_surface.isl,
727 &image->planes[plane].aux_surface.isl)) {
728 image->planes[plane].aux_usage = ISL_AUX_USAGE_HIZ;
729 } else if (want_hiz_wt_for_image(device->info, image)) {
730 assert(device->info->ver >= 12);
731 image->planes[plane].aux_usage = ISL_AUX_USAGE_HIZ_CCS_WT;
732 } else {
733 assert(device->info->ver >= 12);
734 image->planes[plane].aux_usage = ISL_AUX_USAGE_HIZ_CCS;
735 }
736
737 result = add_surface(device, image, &image->planes[plane].aux_surface,
738 binding, ANV_OFFSET_IMPLICIT);
739 if (result != VK_SUCCESS)
740 return result;
741
742 if (anv_image_plane_uses_aux_map(device, image, plane)) {
743 result = add_compression_control_buffer(device, image, plane,
744 binding,
745 ANV_OFFSET_IMPLICIT);
746 if (result != VK_SUCCESS)
747 return result;
748 }
749
750 if (device->info->ver == 12 &&
751 image->planes[plane].aux_usage == ISL_AUX_USAGE_HIZ_CCS_WT) {
752 return add_aux_state_tracking_buffer(device, image, aux_state_offset,
753 plane);
754 }
755 } else if (aspect == VK_IMAGE_ASPECT_STENCIL_BIT) {
756 if (!isl_surf_supports_ccs(&device->isl_dev,
757 &image->planes[plane].primary_surface.isl,
758 NULL))
759 return VK_SUCCESS;
760
761 image->planes[plane].aux_usage = ISL_AUX_USAGE_STC_CCS;
762
763 if (device->info->has_aux_map) {
764 result = add_compression_control_buffer(device, image, plane,
765 binding,
766 ANV_OFFSET_IMPLICIT);
767 if (result != VK_SUCCESS)
768 return result;
769 }
770 } else if ((aspect & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) && image->vk.samples == 1) {
771
772 if (device->info->has_flat_ccs || device->info->has_aux_map) {
773 ok = isl_surf_supports_ccs(&device->isl_dev,
774 &image->planes[plane].primary_surface.isl,
775 NULL);
776 } else {
777 ok = isl_surf_get_ccs_surf(&device->isl_dev,
778 &image->planes[plane].primary_surface.isl,
779 &image->planes[plane].aux_surface.isl,
780 stride);
781 }
782 if (!ok)
783 return VK_SUCCESS;
784
785 /* Choose aux usage. */
786 if (device->info->verx10 == 125 && !device->physical->disable_fcv) {
787 /* FCV is enabled via 3DSTATE_3D_MODE. We'd expect plain CCS_E to
788 * perform better because it allows for non-zero fast clear colors,
789 * but we've run into regressions in several benchmarks (F1 22 and
790 * RDR2) when trying to enable it. When non-zero clear colors are
791 * enabled, we've observed many partial resolves. We haven't yet
792 * root-caused what layout transitions are causing these resolves,
793 * so in the meantime, we choose to reduce our clear color support.
794 * With only zero clear colors being supported, we might as well
795 * turn on FCV.
796 */
797 image->planes[plane].aux_usage = ISL_AUX_USAGE_FCV_CCS_E;
798 } else if (intel_needs_workaround(device->info, 1607794140)) {
799 /* FCV is permanently enabled on this hardware. */
800 assert(device->info->verx10 == 120);
801 image->planes[plane].aux_usage = ISL_AUX_USAGE_FCV_CCS_E;
802 } else if (device->info->ver >= 12) {
803 /* Support for CCS_E was already checked for in anv_image_init(). */
804 image->planes[plane].aux_usage = ISL_AUX_USAGE_CCS_E;
805 } else if (anv_formats_ccs_e_compatible(device->info,
806 image->vk.create_flags,
807 image->vk.format,
808 image->vk.tiling,
809 image->vk.usage, fmt_list)) {
810 image->planes[plane].aux_usage = ISL_AUX_USAGE_CCS_E;
811 } else {
812 image->planes[plane].aux_usage = ISL_AUX_USAGE_CCS_D;
813 }
814
815 if (device->info->has_flat_ccs) {
816 result = VK_SUCCESS;
817 } else if (device->info->has_aux_map) {
818 result = add_compression_control_buffer(device, image, plane,
819 binding, offset);
820 } else {
821 result = add_surface(device, image,
822 &image->planes[plane].aux_surface, binding,
823 offset);
824 }
825 if (result != VK_SUCCESS)
826 return result;
827
828 if (device->info->ver <= 12)
829 return add_aux_state_tracking_buffer(device, image, aux_state_offset,
830 plane);
831 } else if ((aspect & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) && image->vk.samples > 1) {
832 assert(!(image->vk.usage & VK_IMAGE_USAGE_STORAGE_BIT));
833 ok = isl_surf_get_mcs_surf(&device->isl_dev,
834 &image->planes[plane].primary_surface.isl,
835 &image->planes[plane].aux_surface.isl);
836 if (!ok)
837 return VK_SUCCESS;
838
839 if (isl_surf_supports_ccs(&device->isl_dev,
840 &image->planes[plane].primary_surface.isl,
841 &image->planes[plane].aux_surface.isl)) {
842 image->planes[plane].aux_usage = ISL_AUX_USAGE_MCS_CCS;
843 } else {
844 image->planes[plane].aux_usage = ISL_AUX_USAGE_MCS;
845 }
846
847 result = add_surface(device, image, &image->planes[plane].aux_surface,
848 binding, ANV_OFFSET_IMPLICIT);
849 if (result != VK_SUCCESS)
850 return result;
851
852 if (anv_image_plane_uses_aux_map(device, image, plane)) {
853 result = add_compression_control_buffer(device, image, plane,
854 binding,
855 ANV_OFFSET_IMPLICIT);
856 if (result != VK_SUCCESS)
857 return result;
858 }
859
860 if (device->info->ver <= 12)
861 return add_aux_state_tracking_buffer(device, image, aux_state_offset,
862 plane);
863 }
864
865 return VK_SUCCESS;
866 }
867
868 static VkResult
add_video_buffers(struct anv_device * device,struct anv_image * image,const struct VkVideoProfileListInfoKHR * profile_list,bool independent_profile)869 add_video_buffers(struct anv_device *device,
870 struct anv_image *image,
871 const struct VkVideoProfileListInfoKHR *profile_list,
872 bool independent_profile)
873 {
874 ASSERTED VkResult ok;
875 unsigned size = 0;
876
877 if (independent_profile) {
878 /* Takes the worst case */
879 unsigned w_mb = DIV_ROUND_UP(image->vk.extent.width, ANV_MB_WIDTH);
880 unsigned h_mb = DIV_ROUND_UP(image->vk.extent.height, ANV_MB_HEIGHT);
881 size = w_mb * h_mb * 128;
882 } else {
883 size = anv_video_get_image_mv_size(device, image, profile_list);
884 }
885
886 if (size == 0)
887 return VK_SUCCESS;
888
889 ok = image_binding_grow(device, image, ANV_IMAGE_MEMORY_BINDING_PRIVATE,
890 ANV_OFFSET_IMPLICIT, size, 65536, &image->vid_dmv_top_surface);
891 if (ok != VK_SUCCESS)
892 return ok;
893
894 /* Doesn't work for av1 without provided profiles */
895 if (!independent_profile) {
896 for (unsigned i = 0; i < profile_list->profileCount; i++) {
897 if (profile_list->pProfiles[i].videoCodecOperation == VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR) {
898 ok = image_binding_grow(device, image, ANV_IMAGE_MEMORY_BINDING_PRIVATE,
899 ANV_OFFSET_IMPLICIT, av1_cdf_max_num_bytes, 4096, &image->av1_cdf_table);
900 }
901 }
902 }
903
904 return ok;
905 }
906
907 /**
908 * Initialize the anv_image::*_surface selected by \a aspect. Then update the
909 * image's memory requirements (that is, the image's size and alignment).
910 *
911 * @param offset See add_surface()
912 */
913 static VkResult
add_primary_surface(struct anv_device * device,struct anv_image * image,uint32_t plane,struct anv_format_plane plane_format,uint64_t offset,uint32_t stride,isl_tiling_flags_t isl_tiling_flags,isl_surf_usage_flags_t isl_usage)914 add_primary_surface(struct anv_device *device,
915 struct anv_image *image,
916 uint32_t plane,
917 struct anv_format_plane plane_format,
918 uint64_t offset,
919 uint32_t stride,
920 isl_tiling_flags_t isl_tiling_flags,
921 isl_surf_usage_flags_t isl_usage)
922 {
923 struct anv_surface *anv_surf = &image->planes[plane].primary_surface;
924 bool ok;
925
926 uint32_t width = image->vk.extent.width;
927 uint32_t height = image->vk.extent.height;
928 const struct vk_format_ycbcr_info *ycbcr_info =
929 vk_format_get_ycbcr_info(image->vk.format);
930 if (ycbcr_info) {
931 assert(plane < ycbcr_info->n_planes);
932 width /= ycbcr_info->planes[plane].denominator_scales[0];
933 height /= ycbcr_info->planes[plane].denominator_scales[1];
934 }
935
936 ok = isl_surf_init(&device->isl_dev, &anv_surf->isl,
937 .dim = vk_to_isl_surf_dim[image->vk.image_type],
938 .format = plane_format.isl_format,
939 .width = width,
940 .height = height,
941 .depth = image->vk.extent.depth,
942 .levels = image->vk.mip_levels,
943 .array_len = image->vk.array_layers,
944 .samples = image->vk.samples,
945 .min_alignment_B = 0,
946 .row_pitch_B = stride,
947 .usage = isl_usage,
948 .tiling_flags = isl_tiling_flags);
949
950 if (!ok) {
951 /* TODO: Should return
952 * VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT in come cases.
953 */
954 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
955 }
956
957 image->planes[plane].aux_usage = ISL_AUX_USAGE_NONE;
958
959 return add_surface(device, image, anv_surf,
960 ANV_IMAGE_MEMORY_BINDING_PLANE_0 + plane, offset);
961 }
962
963 #ifndef NDEBUG
964 static bool MUST_CHECK
memory_range_is_aligned(struct anv_image_memory_range memory_range)965 memory_range_is_aligned(struct anv_image_memory_range memory_range)
966 {
967 return anv_is_aligned(memory_range.offset, memory_range.alignment);
968 }
969
970 static bool MUST_CHECK
memory_ranges_equal(struct anv_image_memory_range a,struct anv_image_memory_range b)971 memory_ranges_equal(struct anv_image_memory_range a,
972 struct anv_image_memory_range b)
973 {
974 return a.binding == b.binding &&
975 a.alignment == b.alignment &&
976 a.size == b.size &&
977 a.offset == b.offset;
978 }
979 #endif
980
981 struct check_memory_range_params {
982 struct anv_image_memory_range *accum_ranges;
983 const struct anv_surface *test_surface;
984 const struct anv_image_memory_range *test_range;
985 enum anv_image_memory_binding expect_binding;
986 };
987
988 #define check_memory_range(...) \
989 check_memory_range_s(&(struct check_memory_range_params) { __VA_ARGS__ })
990
991 static void UNUSED
check_memory_range_s(const struct check_memory_range_params * p)992 check_memory_range_s(const struct check_memory_range_params *p)
993 {
994 assert((p->test_surface == NULL) != (p->test_range == NULL));
995
996 const struct anv_image_memory_range *test_range =
997 p->test_range ?: &p->test_surface->memory_range;
998
999 struct anv_image_memory_range *accum_range =
1000 &p->accum_ranges[p->expect_binding];
1001
1002 assert(test_range->binding == p->expect_binding);
1003 assert(test_range->offset >= memory_range_end(*accum_range));
1004 assert(memory_range_is_aligned(*test_range));
1005
1006 if (p->test_surface) {
1007 assert(anv_surface_is_valid(p->test_surface));
1008 assert(p->test_surface->memory_range.alignment ==
1009 p->test_surface->isl.alignment_B);
1010 }
1011
1012 memory_range_merge(accum_range, *test_range);
1013 }
1014
1015 /**
1016 * Validate the image's memory bindings *after* all its surfaces and memory
1017 * ranges are final.
1018 *
1019 * For simplicity's sake, we do not validate free-form layout of the image's
1020 * memory bindings. We validate the layout described in the comments of struct
1021 * anv_image.
1022 */
1023 static void
check_memory_bindings(const struct anv_device * device,const struct anv_image * image)1024 check_memory_bindings(const struct anv_device *device,
1025 const struct anv_image *image)
1026 {
1027 #if MESA_DEBUG
1028 /* As we inspect each part of the image, we merge the part's memory range
1029 * into these accumulation ranges.
1030 */
1031 struct anv_image_memory_range accum_ranges[ANV_IMAGE_MEMORY_BINDING_END];
1032 for (int i = 0; i < ANV_IMAGE_MEMORY_BINDING_END; ++i) {
1033 accum_ranges[i] = (struct anv_image_memory_range) {
1034 .binding = i,
1035 };
1036 }
1037
1038 for (uint32_t p = 0; p < image->n_planes; ++p) {
1039 const struct anv_image_plane *plane = &image->planes[p];
1040
1041 /* The binding that must contain the plane's primary surface. */
1042 const enum anv_image_memory_binding primary_binding = image->disjoint
1043 ? ANV_IMAGE_MEMORY_BINDING_PLANE_0 + p
1044 : ANV_IMAGE_MEMORY_BINDING_MAIN;
1045
1046 /* Aliasing is incompatible with the private binding because it does not
1047 * live in a VkDeviceMemory. The exception is either swapchain images or
1048 * that the private binding is for a video motion vector buffer.
1049 */
1050 assert(!(image->vk.create_flags & VK_IMAGE_CREATE_ALIAS_BIT) ||
1051 image->from_wsi ||
1052 (plane->primary_surface.isl.usage & ISL_SURF_USAGE_VIDEO_DECODE_BIT) ||
1053 image->bindings[ANV_IMAGE_MEMORY_BINDING_PRIVATE].memory_range.size == 0);
1054
1055 /* Check primary surface */
1056 check_memory_range(accum_ranges,
1057 .test_surface = &plane->primary_surface,
1058 .expect_binding = primary_binding);
1059
1060 /* Check aux_surface */
1061 const struct anv_image_memory_range *aux_mem_range =
1062 anv_image_get_aux_memory_range(image, p);
1063 if (aux_mem_range->size > 0) {
1064 enum anv_image_memory_binding binding = primary_binding;
1065
1066 /* If an auxiliary surface is used for an externally-shareable image,
1067 * we have to hide this from the memory of the image since other
1068 * processes with access to the memory may not be aware of it or of
1069 * its current state. So put that auxiliary data into a separate
1070 * buffer (ANV_IMAGE_MEMORY_BINDING_PRIVATE).
1071 */
1072 if (anv_image_is_externally_shared(image) &&
1073 !isl_drm_modifier_has_aux(image->vk.drm_format_mod)) {
1074 binding = ANV_IMAGE_MEMORY_BINDING_PRIVATE;
1075 }
1076
1077 /* Display hardware requires that the aux surface start at
1078 * a higher address than the primary surface. The 3D hardware
1079 * doesn't care, but we enforce the display requirement in case
1080 * the image is sent to display.
1081 */
1082 check_memory_range(accum_ranges,
1083 .test_range = aux_mem_range,
1084 .expect_binding = binding);
1085 }
1086
1087 /* Check fast clear state */
1088 if (plane->fast_clear_memory_range.size > 0) {
1089 enum anv_image_memory_binding binding = primary_binding;
1090
1091 /* If an auxiliary surface is used for an externally-shareable image,
1092 * we have to hide this from the memory of the image since other
1093 * processes with access to the memory may not be aware of it or of
1094 * its current state. So put that auxiliary data into a separate
1095 * buffer (ANV_IMAGE_MEMORY_BINDING_PRIVATE).
1096 *
1097 * But when the image is created with a drm modifier that supports
1098 * clear color, it will be exported along with main surface.
1099 */
1100 if (anv_image_is_externally_shared(image)
1101 && !isl_drm_modifier_get_info(image->vk.drm_format_mod)->supports_clear_color) {
1102 binding = ANV_IMAGE_MEMORY_BINDING_PRIVATE;
1103 }
1104
1105 /* The indirect clear color BO requires 64B-alignment on gfx11+. */
1106 assert(plane->fast_clear_memory_range.alignment % 64 == 0);
1107 check_memory_range(accum_ranges,
1108 .test_range = &plane->fast_clear_memory_range,
1109 .expect_binding = binding);
1110 }
1111 }
1112 #endif
1113 }
1114
1115 /**
1116 * Check that the fully-initialized anv_image is compatible with its DRM format
1117 * modifier.
1118 *
1119 * Checking compatibility at the end of image creation is prudent, not
1120 * superfluous, because usage of modifiers triggers numerous special cases
1121 * throughout queries and image creation, and because
1122 * vkGetPhysicalDeviceImageFormatProperties2 has difficulty detecting all
1123 * incompatibilities.
1124 *
1125 * Return VK_ERROR_UNKNOWN if the incompatibility is difficult to detect in
1126 * vkGetPhysicalDeviceImageFormatProperties2. Otherwise, assert fail.
1127 *
1128 * Ideally, if vkGetPhysicalDeviceImageFormatProperties2() succeeds with a given
1129 * modifier, then vkCreateImage() produces an image that is compatible with the
1130 * modifier. However, it is difficult to reconcile the two functions to agree
1131 * due to their complexity. For example, isl_surf_get_ccs_surf() may
1132 * unexpectedly fail in vkCreateImage(), eliminating the image's aux surface
1133 * even when the modifier requires one. (Maybe we should reconcile the two
1134 * functions despite the difficulty).
1135 */
1136 static VkResult MUST_CHECK
check_drm_format_mod(const struct anv_device * device,const struct anv_image * image)1137 check_drm_format_mod(const struct anv_device *device,
1138 const struct anv_image *image)
1139 {
1140 /* Image must have a modifier if and only if it has modifier tiling. */
1141 assert((image->vk.drm_format_mod != DRM_FORMAT_MOD_INVALID) ==
1142 (image->vk.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT));
1143
1144 if (image->vk.drm_format_mod == DRM_FORMAT_MOD_INVALID)
1145 return VK_SUCCESS;
1146
1147 const struct isl_drm_modifier_info *isl_mod_info =
1148 isl_drm_modifier_get_info(image->vk.drm_format_mod);
1149
1150 /* Driver must support the modifier. */
1151 assert(isl_drm_modifier_get_score(device->info, isl_mod_info->modifier));
1152
1153 /* Enforced by us, not the Vulkan spec. */
1154 assert(image->vk.image_type == VK_IMAGE_TYPE_2D);
1155 assert(!(image->vk.aspects & VK_IMAGE_ASPECT_DEPTH_BIT));
1156 assert(!(image->vk.aspects & VK_IMAGE_ASPECT_STENCIL_BIT));
1157 assert(image->vk.mip_levels == 1);
1158 assert(image->vk.array_layers == 1);
1159 assert(image->vk.samples == 1);
1160
1161 for (int i = 0; i < image->n_planes; ++i) {
1162 const struct anv_image_plane *plane = &image->planes[i];
1163 ASSERTED const struct isl_format_layout *isl_layout =
1164 isl_format_get_layout(plane->primary_surface.isl.format);
1165
1166 /* Enforced by us, not the Vulkan spec. */
1167 assert(isl_layout->txc == ISL_TXC_NONE);
1168 assert(isl_layout->colorspace == ISL_COLORSPACE_LINEAR ||
1169 isl_layout->colorspace == ISL_COLORSPACE_SRGB);
1170
1171 if (isl_drm_modifier_has_aux(isl_mod_info->modifier)) {
1172 /* Reject DISJOINT for consistency with the GL driver. */
1173 assert(!image->disjoint);
1174
1175 /* The modifier's required aux usage mandates the image's aux usage.
1176 * The inverse, however, does not hold; if the modifier has no aux
1177 * usage, then we may enable a private aux surface.
1178 */
1179 if ((isl_mod_info->supports_media_compression &&
1180 plane->aux_usage != ISL_AUX_USAGE_MC) ||
1181 (isl_mod_info->supports_render_compression &&
1182 !isl_aux_usage_has_ccs_e(plane->aux_usage))) {
1183 return vk_errorf(device, VK_ERROR_UNKNOWN,
1184 "image with modifier unexpectedly has wrong aux "
1185 "usage");
1186 }
1187 }
1188 }
1189
1190 return VK_SUCCESS;
1191 }
1192
1193 /**
1194 * Use when the app does not provide
1195 * VkImageDrmFormatModifierExplicitCreateInfoEXT.
1196 */
1197 static VkResult MUST_CHECK
add_all_surfaces_implicit_layout(struct anv_device * device,struct anv_image * image,const VkImageFormatListCreateInfo * format_list_info,uint32_t stride,isl_tiling_flags_t isl_tiling_flags,isl_surf_usage_flags_t isl_extra_usage_flags)1198 add_all_surfaces_implicit_layout(
1199 struct anv_device *device,
1200 struct anv_image *image,
1201 const VkImageFormatListCreateInfo *format_list_info,
1202 uint32_t stride,
1203 isl_tiling_flags_t isl_tiling_flags,
1204 isl_surf_usage_flags_t isl_extra_usage_flags)
1205 {
1206 const struct intel_device_info *devinfo = device->info;
1207 VkResult result;
1208
1209 const struct vk_format_ycbcr_info *ycbcr_info =
1210 vk_format_get_ycbcr_info(image->vk.format);
1211 if (ycbcr_info)
1212 assert(ycbcr_info->n_planes == image->n_planes);
1213
1214 unsigned num_aspects = 0;
1215 VkImageAspectFlagBits aspects[3];
1216 u_foreach_bit(b, image->vk.aspects) {
1217 assert(num_aspects < 3);
1218 aspects[num_aspects++] = 1 << b;
1219 }
1220 assert(num_aspects == image->n_planes);
1221
1222 /* The Android hardware buffer YV12 format has the planes ordered as Y-Cr-Cb,
1223 * while Vulkan expects VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM to be in Y-Cb-Cr.
1224 * Adjust the order we add the ISL surfaces accordingly so the implicit
1225 * offset gets calculated correctly.
1226 */
1227 if (image->from_ahb && image->vk.format == VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM) {
1228 assert(num_aspects == 3);
1229 assert(aspects[1] == VK_IMAGE_ASPECT_PLANE_1_BIT);
1230 assert(aspects[2] == VK_IMAGE_ASPECT_PLANE_2_BIT);
1231 aspects[1] = VK_IMAGE_ASPECT_PLANE_2_BIT;
1232 aspects[2] = VK_IMAGE_ASPECT_PLANE_1_BIT;
1233 }
1234
1235 for (unsigned i = 0; i < num_aspects; i++) {
1236 VkImageAspectFlagBits aspect = aspects[i];
1237 const uint32_t plane = anv_image_aspect_to_plane(image, aspect);
1238 const struct anv_format_plane plane_format =
1239 anv_get_format_plane(devinfo, image->vk.format, plane, image->vk.tiling);
1240
1241 enum isl_format isl_fmt = plane_format.isl_format;
1242 assert(isl_fmt != ISL_FORMAT_UNSUPPORTED);
1243
1244 uint32_t plane_stride = stride * isl_format_get_layout(isl_fmt)->bpb / 8;
1245 if (ycbcr_info)
1246 plane_stride /= ycbcr_info->planes[plane].denominator_scales[0];
1247
1248 VkImageUsageFlags vk_usage = vk_image_usage(&image->vk, aspect);
1249 isl_surf_usage_flags_t isl_usage =
1250 anv_image_choose_isl_surf_usage(device->physical,
1251 image->vk.create_flags, vk_usage,
1252 isl_extra_usage_flags, aspect,
1253 image->vk.compr_flags);
1254
1255 result = add_primary_surface(device, image, plane, plane_format,
1256 ANV_OFFSET_IMPLICIT, plane_stride,
1257 isl_tiling_flags, isl_usage);
1258 if (result != VK_SUCCESS)
1259 return result;
1260
1261 result = add_aux_surface_if_supported(device, image, plane, plane_format,
1262 format_list_info,
1263 ANV_OFFSET_IMPLICIT, plane_stride,
1264 ANV_OFFSET_IMPLICIT);
1265 if (result != VK_SUCCESS)
1266 return result;
1267 }
1268
1269 return VK_SUCCESS;
1270 }
1271
1272 /**
1273 * Use when the app provides VkImageDrmFormatModifierExplicitCreateInfoEXT.
1274 */
1275 static VkResult
add_all_surfaces_explicit_layout(struct anv_device * device,struct anv_image * image,const VkImageFormatListCreateInfo * format_list_info,const VkImageDrmFormatModifierExplicitCreateInfoEXT * drm_info,isl_tiling_flags_t isl_tiling_flags,isl_surf_usage_flags_t isl_extra_usage_flags)1276 add_all_surfaces_explicit_layout(
1277 struct anv_device *device,
1278 struct anv_image *image,
1279 const VkImageFormatListCreateInfo *format_list_info,
1280 const VkImageDrmFormatModifierExplicitCreateInfoEXT *drm_info,
1281 isl_tiling_flags_t isl_tiling_flags,
1282 isl_surf_usage_flags_t isl_extra_usage_flags)
1283 {
1284 const struct intel_device_info *devinfo = device->info;
1285 const uint32_t mod_plane_count = drm_info->drmFormatModifierPlaneCount;
1286 const bool mod_has_aux =
1287 isl_drm_modifier_has_aux(drm_info->drmFormatModifier);
1288 VkResult result;
1289
1290 /* Currently there is no way to properly map memory planes to format planes
1291 * and aux planes due to the lack of defined ABI for external multi-planar
1292 * images.
1293 */
1294 if (image->n_planes == 1)
1295 assert(image->vk.aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1296 else
1297 assert(!(image->vk.aspects & ~VK_IMAGE_ASPECT_PLANES_BITS_ANV));
1298
1299 if (mod_has_aux) {
1300 assert(image->n_planes == 1);
1301
1302 /* About valid usage in the Vulkan spec:
1303 *
1304 * Unlike vanilla vkCreateImage, which produces undefined behavior on user
1305 * error, here the spec requires the implementation to return
1306 * VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT if the app provides
1307 * a bad plane layout. However, the spec does require
1308 * drmFormatModifierPlaneCount to be valid.
1309 *
1310 * Most validation of plane layout occurs in add_surface().
1311 */
1312 uint32_t n_mod_planes =
1313 isl_drm_modifier_get_plane_count(devinfo,
1314 drm_info->drmFormatModifier,
1315 image->n_planes);
1316 assert(n_mod_planes == mod_plane_count);
1317 } else {
1318 assert(image->n_planes == mod_plane_count);
1319 }
1320
1321 /* Reject special values in the app-provided plane layouts. */
1322 for (uint32_t i = 0; i < mod_plane_count; ++i) {
1323 if (drm_info->pPlaneLayouts[i].rowPitch == 0) {
1324 return vk_errorf(device,
1325 VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT,
1326 "VkImageDrmFormatModifierExplicitCreateInfoEXT::"
1327 "pPlaneLayouts[%u]::rowPitch is 0", i);
1328 }
1329
1330 if (drm_info->pPlaneLayouts[i].offset == ANV_OFFSET_IMPLICIT) {
1331 return vk_errorf(device,
1332 VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT,
1333 "VkImageDrmFormatModifierExplicitCreateInfoEXT::"
1334 "pPlaneLayouts[%u]::offset is %" PRIu64,
1335 i, ANV_OFFSET_IMPLICIT);
1336 }
1337 }
1338
1339 u_foreach_bit(b, image->vk.aspects) {
1340 const VkImageAspectFlagBits aspect = 1 << b;
1341 const uint32_t plane = anv_image_aspect_to_plane(image, aspect);
1342 const struct anv_format_plane format_plane =
1343 anv_get_format_plane(devinfo, image->vk.format, plane, image->vk.tiling);
1344 const VkSubresourceLayout *primary_layout = &drm_info->pPlaneLayouts[plane];
1345
1346 result = add_primary_surface(device, image, plane,
1347 format_plane,
1348 primary_layout->offset,
1349 primary_layout->rowPitch,
1350 isl_tiling_flags,
1351 isl_extra_usage_flags);
1352 if (result != VK_SUCCESS)
1353 return result;
1354
1355 if (mod_has_aux) {
1356 const VkSubresourceLayout flat_ccs_layout = {
1357 .offset = ANV_OFFSET_IMPLICIT,
1358 };
1359
1360 const VkSubresourceLayout *aux_layout;
1361
1362 uint64_t aux_state_offset = ANV_OFFSET_IMPLICIT;
1363
1364 /* We already asserted on image->n_planes == 1 when mod_has_aux is
1365 * true above, so the indexes of aux and clear color are just hard-
1366 * coded without ambiguity.
1367 */
1368 if (devinfo->has_flat_ccs) {
1369 aux_layout = &flat_ccs_layout;
1370 if (isl_drm_modifier_get_info(
1371 drm_info->drmFormatModifier)->supports_clear_color) {
1372 aux_state_offset = drm_info->pPlaneLayouts[1].offset;
1373 }
1374 } else {
1375 aux_layout = &drm_info->pPlaneLayouts[1];
1376 if (isl_drm_modifier_get_info(
1377 drm_info->drmFormatModifier)->supports_clear_color) {
1378 aux_state_offset = drm_info->pPlaneLayouts[2].offset;
1379 }
1380 }
1381
1382 result = add_aux_surface_if_supported(device, image, plane,
1383 format_plane,
1384 format_list_info,
1385 aux_layout->offset,
1386 aux_layout->rowPitch,
1387 aux_state_offset);
1388 if (result != VK_SUCCESS)
1389 return result;
1390
1391 assert(isl_aux_usage_has_ccs(image->planes[plane].aux_usage));
1392 if (aux_state_offset != ANV_OFFSET_IMPLICIT) {
1393 assert(image->planes[plane].fast_clear_memory_range.size <=
1394 device->isl_dev.ss.clear_color_state_size);
1395 }
1396 }
1397 }
1398
1399 return VK_SUCCESS;
1400 }
1401
1402 static const struct isl_drm_modifier_info *
choose_drm_format_mod(const struct anv_physical_device * device,uint32_t modifier_count,const uint64_t * modifiers)1403 choose_drm_format_mod(const struct anv_physical_device *device,
1404 uint32_t modifier_count, const uint64_t *modifiers)
1405 {
1406 uint64_t best_mod = UINT64_MAX;
1407 uint32_t best_score = 0;
1408
1409 for (uint32_t i = 0; i < modifier_count; ++i) {
1410 uint32_t score = isl_drm_modifier_get_score(&device->info, modifiers[i]);
1411 if (score > best_score) {
1412 best_mod = modifiers[i];
1413 best_score = score;
1414 }
1415 }
1416
1417 if (best_score > 0)
1418 return isl_drm_modifier_get_info(best_mod);
1419 else
1420 return NULL;
1421 }
1422
1423 static VkImageUsageFlags
anv_image_create_usage(const VkImageCreateInfo * pCreateInfo,VkImageUsageFlags usage)1424 anv_image_create_usage(const VkImageCreateInfo *pCreateInfo,
1425 VkImageUsageFlags usage)
1426 {
1427 /* Add TRANSFER_SRC usage for multisample attachment images. This is
1428 * because we might internally use the TRANSFER_SRC layout on them for
1429 * blorp operations associated with resolving those into other attachments
1430 * at the end of a subpass.
1431 *
1432 * Without this additional usage, we compute an incorrect AUX state in
1433 * anv_layout_to_aux_state().
1434 */
1435 if (pCreateInfo->samples > VK_SAMPLE_COUNT_1_BIT &&
1436 (usage & (VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
1437 VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT)))
1438 usage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
1439 return usage;
1440 }
1441
1442 static VkResult MUST_CHECK
alloc_private_binding(struct anv_device * device,struct anv_image * image,const VkImageCreateInfo * create_info)1443 alloc_private_binding(struct anv_device *device,
1444 struct anv_image *image,
1445 const VkImageCreateInfo *create_info)
1446 {
1447 struct anv_image_binding *binding =
1448 &image->bindings[ANV_IMAGE_MEMORY_BINDING_PRIVATE];
1449
1450 if (binding->memory_range.size == 0)
1451 return VK_SUCCESS;
1452
1453 const VkImageSwapchainCreateInfoKHR *swapchain_info =
1454 vk_find_struct_const(create_info->pNext, IMAGE_SWAPCHAIN_CREATE_INFO_KHR);
1455
1456 if (swapchain_info && swapchain_info->swapchain != VK_NULL_HANDLE) {
1457 /* The image will be bound to swapchain memory. */
1458 return VK_SUCCESS;
1459 }
1460
1461 VkResult result = anv_device_alloc_bo(device, "image-binding-private",
1462 binding->memory_range.size, 0, 0,
1463 &binding->address.bo);
1464 if (result == VK_SUCCESS) {
1465 pthread_mutex_lock(&device->mutex);
1466 list_addtail(&image->link, &device->image_private_objects);
1467 pthread_mutex_unlock(&device->mutex);
1468 }
1469
1470 return result;
1471 }
1472
1473 static void
anv_image_finish_sparse_bindings(struct anv_image * image)1474 anv_image_finish_sparse_bindings(struct anv_image *image)
1475 {
1476 struct anv_device *device =
1477 container_of(image->vk.base.device, struct anv_device, vk);
1478
1479 assert(anv_image_is_sparse(image));
1480
1481 for (int i = 0; i < ANV_IMAGE_MEMORY_BINDING_END; i++) {
1482 struct anv_image_binding *b = &image->bindings[i];
1483
1484 if (b->sparse_data.size != 0) {
1485 assert(b->memory_range.size == b->sparse_data.size);
1486 assert(b->address.offset == b->sparse_data.address);
1487 anv_free_sparse_bindings(device, &b->sparse_data);
1488 }
1489 }
1490 }
1491
1492 static VkResult MUST_CHECK
anv_image_init_sparse_bindings(struct anv_image * image,const struct anv_image_create_info * create_info)1493 anv_image_init_sparse_bindings(struct anv_image *image,
1494 const struct anv_image_create_info *create_info)
1495 {
1496 struct anv_device *device =
1497 container_of(image->vk.base.device, struct anv_device, vk);
1498 VkResult result;
1499
1500 assert(anv_image_is_sparse(image));
1501
1502 enum anv_bo_alloc_flags alloc_flags = 0;
1503 uint64_t explicit_address = 0;
1504 if (image->vk.create_flags & VK_IMAGE_CREATE_DESCRIPTOR_BUFFER_CAPTURE_REPLAY_BIT_EXT) {
1505 alloc_flags |= ANV_BO_ALLOC_FIXED_ADDRESS;
1506
1507 const VkOpaqueCaptureDescriptorDataCreateInfoEXT *opaque_info =
1508 vk_find_struct_const(create_info->vk_info->pNext,
1509 OPAQUE_CAPTURE_DESCRIPTOR_DATA_CREATE_INFO_EXT);
1510 if (opaque_info)
1511 explicit_address = *((const uint64_t *)opaque_info->opaqueCaptureDescriptorData);
1512 }
1513
1514 for (int i = 0; i < ANV_IMAGE_MEMORY_BINDING_END; i++) {
1515 struct anv_image_binding *b = &image->bindings[i];
1516
1517 if (b->memory_range.size != 0) {
1518 assert(b->sparse_data.size == 0);
1519
1520 /* From the spec, Custom Sparse Image Block Shapes section:
1521 * "... the size in bytes of the custom sparse image block shape
1522 * will be reported in VkMemoryRequirements::alignment."
1523 *
1524 * ISL should have set this for us, so just assert it here.
1525 */
1526 assert(b->memory_range.alignment == ANV_SPARSE_BLOCK_SIZE);
1527 assert(b->memory_range.size % ANV_SPARSE_BLOCK_SIZE == 0);
1528
1529 result = anv_init_sparse_bindings(device,
1530 b->memory_range.size,
1531 &b->sparse_data,
1532 alloc_flags,
1533 explicit_address,
1534 &b->address);
1535 if (result != VK_SUCCESS) {
1536 anv_image_finish_sparse_bindings(image);
1537 return result;
1538 }
1539 }
1540 }
1541
1542 return VK_SUCCESS;
1543 }
1544
1545 bool
anv_image_view_formats_incomplete(const struct anv_image * image)1546 anv_image_view_formats_incomplete(const struct anv_image *image)
1547 {
1548 /* See mark_image_view_formats_incomplete(). */
1549 return image->num_view_formats < ARRAY_SIZE(image->view_formats) &&
1550 image->view_formats[image->num_view_formats] ==
1551 ISL_FORMAT_UNSUPPORTED;
1552 }
1553
1554 static void
mark_image_view_formats_incomplete(struct anv_image * image)1555 mark_image_view_formats_incomplete(struct anv_image *image)
1556 {
1557 /* We need to reserve space to insert the token for an incomplete list. Use
1558 * up all the space except for the first entry. This helps various code
1559 * paths that depend on the list to have decent fall-back behavior. For
1560 * examples, see add_aux_state_tracking_buffer() and
1561 * set_image_clear_color().
1562 */
1563 assert(image->num_view_formats >= 1);
1564 image->num_view_formats = 1;
1565
1566 /* Replace the first unused entry with the token for an incomplete list. */
1567 image->view_formats[image->num_view_formats] = ISL_FORMAT_UNSUPPORTED;
1568 assert(anv_image_view_formats_incomplete(image));
1569 }
1570
1571 static void
add_image_view_format(struct anv_image * image,enum isl_format view_format)1572 add_image_view_format(struct anv_image *image, enum isl_format view_format)
1573 {
1574 /* If this list can't be completed, reject all formats. */
1575 if (anv_image_view_formats_incomplete(image))
1576 return;
1577
1578 /* Reject invalid formats. */
1579 if (view_format == ISL_FORMAT_UNSUPPORTED)
1580 return;
1581
1582 /* Reject duplicate formats. */
1583 for (int i = 0; i < image->num_view_formats; i++)
1584 if (view_format == image->view_formats[i])
1585 return;
1586
1587 if (image->num_view_formats == ARRAY_SIZE(image->view_formats)) {
1588 /* The array is full, so we can't add any more entries. */
1589 mark_image_view_formats_incomplete(image);
1590 } else {
1591 /* Add to the end of the array. */
1592 image->view_formats[image->num_view_formats++] = view_format;
1593 }
1594 }
1595
1596 VkResult
anv_image_init(struct anv_device * device,struct anv_image * image,const struct anv_image_create_info * create_info)1597 anv_image_init(struct anv_device *device, struct anv_image *image,
1598 const struct anv_image_create_info *create_info)
1599 {
1600 const VkImageCreateInfo *pCreateInfo = create_info->vk_info;
1601 const struct VkImageDrmFormatModifierExplicitCreateInfoEXT *mod_explicit_info = NULL;
1602 const struct isl_drm_modifier_info *isl_mod_info = NULL;
1603 VkResult r;
1604
1605 vk_image_init(&device->vk, &image->vk, pCreateInfo);
1606
1607 image->vk.usage = anv_image_create_usage(pCreateInfo, image->vk.usage);
1608 image->vk.stencil_usage =
1609 anv_image_create_usage(pCreateInfo, image->vk.stencil_usage);
1610
1611 isl_surf_usage_flags_t isl_extra_usage_flags =
1612 create_info->isl_extra_usage_flags;
1613 if (pCreateInfo->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
1614 assert(!image->vk.wsi_legacy_scanout);
1615 mod_explicit_info =
1616 vk_find_struct_const(pCreateInfo->pNext,
1617 IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT);
1618 if (mod_explicit_info) {
1619 isl_mod_info = isl_drm_modifier_get_info(mod_explicit_info->drmFormatModifier);
1620 } else {
1621 const struct VkImageDrmFormatModifierListCreateInfoEXT *mod_list_info =
1622 vk_find_struct_const(pCreateInfo->pNext,
1623 IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
1624 isl_mod_info = choose_drm_format_mod(device->physical,
1625 mod_list_info->drmFormatModifierCount,
1626 mod_list_info->pDrmFormatModifiers);
1627 }
1628
1629 assert(isl_mod_info);
1630 assert(image->vk.drm_format_mod == DRM_FORMAT_MOD_INVALID);
1631 image->vk.drm_format_mod = isl_mod_info->modifier;
1632
1633 if (isl_drm_modifier_needs_display_layout(image->vk.drm_format_mod))
1634 isl_extra_usage_flags |= ISL_SURF_USAGE_DISPLAY_BIT;
1635
1636 /* Disable compression on gen12+ if the selected/requested modifier
1637 * doesn't support it. Prior to that we can use a private binding for
1638 * the aux surface and it should be transparent to users.
1639 */
1640 if (device->info->ver >= 12 &&
1641 !isl_drm_modifier_has_aux(image->vk.drm_format_mod)) {
1642 isl_extra_usage_flags |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
1643 }
1644 }
1645
1646 for (int i = 0; i < ANV_IMAGE_MEMORY_BINDING_END; ++i) {
1647 image->bindings[i] = (struct anv_image_binding) {
1648 .memory_range = { .binding = i },
1649 };
1650 }
1651
1652 /* In case of AHardwareBuffer import, we don't know the layout yet */
1653 if (image->vk.external_handle_types &
1654 VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID) {
1655 image->from_ahb = true;
1656 #if DETECT_OS_ANDROID
1657 image->vk.ahb_format = anv_ahb_format_for_vk_format(image->vk.format);
1658 #endif
1659 return VK_SUCCESS;
1660 }
1661
1662 image->n_planes = anv_get_format_planes(image->vk.format);
1663
1664 #ifdef VK_USE_PLATFORM_ANDROID_KHR
1665 /* In the case of gralloc-backed swap chain image, we don't know the
1666 * layout yet.
1667 */
1668 if (vk_find_struct_const(pCreateInfo->pNext,
1669 IMAGE_SWAPCHAIN_CREATE_INFO_KHR) != NULL)
1670 return VK_SUCCESS;
1671 #endif
1672
1673 image->from_wsi =
1674 vk_find_struct_const(pCreateInfo->pNext, WSI_IMAGE_CREATE_INFO_MESA) != NULL;
1675
1676 /* The Vulkan 1.2.165 glossary says:
1677 *
1678 * A disjoint image consists of multiple disjoint planes, and is created
1679 * with the VK_IMAGE_CREATE_DISJOINT_BIT bit set.
1680 */
1681 image->disjoint = image->n_planes > 1 &&
1682 (pCreateInfo->flags & VK_IMAGE_CREATE_DISJOINT_BIT);
1683
1684 if (anv_is_format_emulated(device->physical, pCreateInfo->format)) {
1685 assert(image->n_planes == 1 &&
1686 vk_format_is_compressed(image->vk.format));
1687 assert(!(image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT));
1688
1689 image->emu_plane_format =
1690 anv_get_emulation_format(device->physical, image->vk.format);
1691
1692 /* for fetching the raw copmressed data and storing the decompressed
1693 * data
1694 */
1695 image->vk.create_flags |=
1696 VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT |
1697 VK_IMAGE_CREATE_BLOCK_TEXEL_VIEW_COMPATIBLE_BIT;
1698 if (image->vk.image_type == VK_IMAGE_TYPE_3D)
1699 image->vk.create_flags |= VK_IMAGE_CREATE_2D_VIEW_COMPATIBLE_BIT_EXT;
1700 image->vk.usage |=
1701 VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT;
1702
1703 /* TODO: enable compression on emulation plane */
1704 isl_extra_usage_flags |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
1705 }
1706
1707 /* Disable aux if image supports export without modifiers. */
1708 if (image->vk.external_handle_types != 0 &&
1709 image->vk.tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT)
1710 isl_extra_usage_flags |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
1711
1712 if (device->queue_count > 1) {
1713 /* Notify ISL that the app may access this image from different engines.
1714 * Note that parallel access to the surface will occur regardless of the
1715 * sharing mode.
1716 */
1717 isl_extra_usage_flags |= ISL_SURF_USAGE_MULTI_ENGINE_PAR_BIT;
1718
1719 /* If the resource is created with the CONCURRENT sharing mode, we can't
1720 * support compression because we aren't allowed barriers in order to
1721 * construct the main surface data with FULL_RESOLVE/PARTIAL_RESOLVE.
1722 */
1723 if (image->vk.sharing_mode == VK_SHARING_MODE_CONCURRENT)
1724 isl_extra_usage_flags |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
1725 }
1726
1727 /* Aux is pointless if it will never be used as an attachment. */
1728 if (vk_format_is_depth_or_stencil(image->vk.format) &&
1729 !(image->vk.usage & VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT))
1730 isl_extra_usage_flags |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
1731
1732 /* TODO: Adjust blorp for multi-LOD HiZ surface on Gen9. */
1733 if (vk_format_has_depth(image->vk.format) &&
1734 image->vk.mip_levels > 1 && device->info->ver == 9) {
1735 anv_perf_warn(VK_LOG_OBJS(&image->vk.base), "Enable multi-LOD HiZ");
1736 isl_extra_usage_flags |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
1737 }
1738
1739 /* Mark WSI images with the right surf usage. */
1740 if (image->from_wsi)
1741 isl_extra_usage_flags |= ISL_SURF_USAGE_DISPLAY_BIT;
1742
1743 const isl_tiling_flags_t isl_tiling_flags =
1744 choose_isl_tiling_flags(device->info, image, create_info, isl_mod_info);
1745
1746 const VkImageFormatListCreateInfo *fmt_list =
1747 vk_find_struct_const(pCreateInfo->pNext,
1748 IMAGE_FORMAT_LIST_CREATE_INFO);
1749
1750 if ((image->vk.aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) &&
1751 image->vk.samples == 1) {
1752 if (image->n_planes != 1) {
1753 /* Multiplanar images seem to hit a sampler bug with CCS and R16G16
1754 * format. (Putting the clear state a page/4096bytes further fixes
1755 * the issue).
1756 */
1757 isl_extra_usage_flags |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
1758 }
1759
1760 if ((image->vk.create_flags & VK_IMAGE_CREATE_ALIAS_BIT) &&
1761 !image->from_wsi) {
1762 /* The image may alias a plane of a multiplanar image. Above we ban
1763 * CCS on multiplanar images.
1764 *
1765 * We must also reject aliasing of any image that uses
1766 * ANV_IMAGE_MEMORY_BINDING_PRIVATE. Since we're already rejecting
1767 * all aliasing here, there's no need to further analyze if the image
1768 * needs a private binding.
1769 */
1770 isl_extra_usage_flags |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
1771 }
1772
1773 if (device->info->ver >= 12 &&
1774 !anv_formats_ccs_e_compatible(device->info, image->vk.create_flags,
1775 image->vk.format, image->vk.tiling,
1776 image->vk.usage, fmt_list)) {
1777 /* CCS_E is the only aux-mode supported for single sampled color
1778 * surfaces on gfx12+. If we can't support it, we should configure
1779 * the main surface without aux support.
1780 */
1781 isl_extra_usage_flags |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
1782 }
1783 }
1784
1785 /* Fill out the list of view formats. */
1786 const enum isl_format image_format =
1787 anv_get_format_plane(device->info, image->vk.format, 0,
1788 image->vk.tiling).isl_format;
1789 add_image_view_format(image, image_format);
1790
1791 if (image->vk.usage & (VK_IMAGE_USAGE_TRANSFER_DST_BIT |
1792 VK_IMAGE_USAGE_TRANSFER_SRC_BIT)) {
1793 if (vk_format_is_color(image->vk.format)) {
1794 const enum isl_format blorp_copy_format =
1795 blorp_copy_get_color_format(&device->isl_dev, image_format);
1796 add_image_view_format(image, blorp_copy_format);
1797 } else {
1798 /* We don't have a helper for depth-stencil formats. */
1799 mark_image_view_formats_incomplete(image);
1800 }
1801 }
1802
1803 if (image->vk.create_flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) {
1804 if (!fmt_list || fmt_list->viewFormatCount == 0) {
1805 /* Without a format list provided, we must assume all compatible
1806 * formats. Instead of adding them all, mark our list as incomplete.
1807 */
1808 mark_image_view_formats_incomplete(image);
1809 } else {
1810 for (uint32_t i = 0; i < fmt_list->viewFormatCount; i++) {
1811 const enum isl_format fmt_list_format =
1812 anv_get_format_plane(device->info,
1813 fmt_list->pViewFormats[i], 0,
1814 image->vk.tiling).isl_format;
1815 add_image_view_format(image, fmt_list_format);
1816 }
1817 }
1818 }
1819
1820 if (isl_mod_info && isl_mod_info->supports_clear_color) {
1821 if (image->num_view_formats > 1) {
1822 /* We use the number of view formats to determine the number of
1823 * CLEAR_COLOR structures to append to the image. For an imported
1824 * dmabuf supporting clear colors, we're limited to a single such
1825 * struct. So, mark the view format list as incomplete because doing
1826 * so shrinks the list size to one.
1827 */
1828 mark_image_view_formats_incomplete(image);
1829 }
1830 assert(image->num_view_formats == 1);
1831 }
1832
1833 if (mod_explicit_info) {
1834 r = add_all_surfaces_explicit_layout(device, image, fmt_list,
1835 mod_explicit_info, isl_tiling_flags,
1836 isl_extra_usage_flags);
1837 } else {
1838 r = add_all_surfaces_implicit_layout(device, image, fmt_list, create_info->stride,
1839 isl_tiling_flags,
1840 isl_extra_usage_flags);
1841 }
1842
1843 if (r != VK_SUCCESS)
1844 goto fail;
1845
1846 if (image->emu_plane_format != VK_FORMAT_UNDEFINED) {
1847 const struct intel_device_info *devinfo = device->info;
1848 const uint32_t plane = image->n_planes;
1849 const struct anv_format_plane plane_format = anv_get_format_plane(
1850 devinfo, image->emu_plane_format, 0, image->vk.tiling);
1851
1852 isl_surf_usage_flags_t isl_usage = anv_image_choose_isl_surf_usage(
1853 device->physical, image->vk.create_flags, image->vk.usage,
1854 isl_extra_usage_flags, VK_IMAGE_ASPECT_COLOR_BIT,
1855 image->vk.compr_flags);
1856
1857 r = add_primary_surface(device, image, plane, plane_format,
1858 ANV_OFFSET_IMPLICIT, 0,
1859 isl_tiling_flags, isl_usage);
1860 if (r != VK_SUCCESS)
1861 goto fail;
1862 }
1863
1864 const VkVideoProfileListInfoKHR *video_profile =
1865 vk_find_struct_const(pCreateInfo->pNext,
1866 VIDEO_PROFILE_LIST_INFO_KHR);
1867
1868 bool independent_video_profile =
1869 pCreateInfo->flags & VK_IMAGE_CREATE_VIDEO_PROFILE_INDEPENDENT_BIT_KHR;
1870
1871 if (video_profile || independent_video_profile) {
1872 r = add_video_buffers(device, image, video_profile, independent_video_profile);
1873 if (r != VK_SUCCESS)
1874 goto fail;
1875 }
1876
1877 if (!create_info->no_private_binding_alloc) {
1878 r = alloc_private_binding(device, image, pCreateInfo);
1879 if (r != VK_SUCCESS)
1880 goto fail;
1881 }
1882
1883 check_memory_bindings(device, image);
1884
1885 r = check_drm_format_mod(device, image);
1886 if (r != VK_SUCCESS)
1887 goto fail;
1888
1889 if (anv_image_is_sparse(image)) {
1890 r = anv_image_init_sparse_bindings(image, create_info);
1891 if (r != VK_SUCCESS)
1892 goto fail;
1893 }
1894
1895 return VK_SUCCESS;
1896
1897 fail:
1898 vk_image_finish(&image->vk);
1899 return r;
1900 }
1901
1902 void
anv_image_finish(struct anv_image * image)1903 anv_image_finish(struct anv_image *image)
1904 {
1905 struct anv_device *device =
1906 container_of(image->vk.base.device, struct anv_device, vk);
1907
1908 if (anv_image_is_sparse(image))
1909 anv_image_finish_sparse_bindings(image);
1910
1911 /* Unmap a CCS so that if the bound region of the image is rebound to
1912 * another image, the AUX tables will be cleared to allow for a new
1913 * mapping.
1914 */
1915 for (int p = 0; p < image->n_planes; ++p) {
1916 if (image->planes[p].aux_tt.mapped) {
1917 intel_aux_map_del_mapping(device->aux_map_ctx,
1918 image->planes[p].aux_tt.addr,
1919 image->planes[p].aux_tt.size);
1920 }
1921 }
1922
1923 if (image->from_gralloc) {
1924 assert(!image->disjoint);
1925 assert(image->n_planes == 1);
1926 assert(image->planes[0].primary_surface.memory_range.binding ==
1927 ANV_IMAGE_MEMORY_BINDING_MAIN);
1928 assert(image->bindings[ANV_IMAGE_MEMORY_BINDING_MAIN].address.bo != NULL);
1929 anv_device_release_bo(device, image->bindings[ANV_IMAGE_MEMORY_BINDING_MAIN].address.bo);
1930 }
1931
1932 for (uint32_t b = 0; b < ARRAY_SIZE(image->bindings); b++) {
1933 if (image->bindings[b].host_map != NULL) {
1934 anv_device_unmap_bo(device,
1935 image->bindings[b].address.bo,
1936 image->bindings[b].host_map,
1937 image->bindings[b].map_size,
1938 false /* replace */);
1939 }
1940 }
1941
1942 struct anv_bo *private_bo = image->bindings[ANV_IMAGE_MEMORY_BINDING_PRIVATE].address.bo;
1943 if (private_bo) {
1944 pthread_mutex_lock(&device->mutex);
1945 list_del(&image->link);
1946 pthread_mutex_unlock(&device->mutex);
1947 anv_device_release_bo(device, private_bo);
1948 }
1949
1950 vk_image_finish(&image->vk);
1951 }
1952
1953 static struct anv_image *
anv_swapchain_get_image(VkSwapchainKHR swapchain,uint32_t index)1954 anv_swapchain_get_image(VkSwapchainKHR swapchain,
1955 uint32_t index)
1956 {
1957 VkImage image = wsi_common_get_image(swapchain, index);
1958 return anv_image_from_handle(image);
1959 }
1960
1961 static VkResult
anv_image_init_from_create_info(struct anv_device * device,struct anv_image * image,const VkImageCreateInfo * pCreateInfo,bool no_private_binding_alloc)1962 anv_image_init_from_create_info(struct anv_device *device,
1963 struct anv_image *image,
1964 const VkImageCreateInfo *pCreateInfo,
1965 bool no_private_binding_alloc)
1966 {
1967 if (pCreateInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) {
1968 VkResult result =
1969 anv_sparse_image_check_support(device->physical,
1970 pCreateInfo->flags,
1971 pCreateInfo->tiling,
1972 pCreateInfo->samples,
1973 pCreateInfo->imageType,
1974 pCreateInfo->format);
1975 if (result != VK_SUCCESS)
1976 return result;
1977 }
1978
1979 const VkNativeBufferANDROID *gralloc_info =
1980 vk_find_struct_const(pCreateInfo->pNext, NATIVE_BUFFER_ANDROID);
1981 if (gralloc_info)
1982 return anv_image_init_from_gralloc(device, image, pCreateInfo,
1983 gralloc_info);
1984
1985 struct anv_image_create_info create_info = {
1986 .vk_info = pCreateInfo,
1987 .no_private_binding_alloc = no_private_binding_alloc,
1988 };
1989
1990 return anv_image_init(device, image, &create_info);
1991 }
1992
anv_CreateImage(VkDevice _device,const VkImageCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkImage * pImage)1993 VkResult anv_CreateImage(
1994 VkDevice _device,
1995 const VkImageCreateInfo* pCreateInfo,
1996 const VkAllocationCallbacks* pAllocator,
1997 VkImage* pImage)
1998 {
1999 ANV_FROM_HANDLE(anv_device, device, _device);
2000
2001 if ((device->physical->sparse_type == ANV_SPARSE_TYPE_NOT_SUPPORTED) &&
2002 INTEL_DEBUG(DEBUG_SPARSE) &&
2003 pCreateInfo->flags & (VK_IMAGE_CREATE_SPARSE_BINDING_BIT |
2004 VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT |
2005 VK_IMAGE_CREATE_SPARSE_ALIASED_BIT))
2006 fprintf(stderr, "=== %s %s:%d flags:0x%08x\n", __func__, __FILE__,
2007 __LINE__, pCreateInfo->flags);
2008
2009 #ifndef VK_USE_PLATFORM_ANDROID_KHR
2010 /* Skip the WSI common swapchain creation here on Android. Similar to ahw,
2011 * this case is handled by a partial image init and then resolved when the
2012 * image is bound and gralloc info is passed.
2013 */
2014 const VkImageSwapchainCreateInfoKHR *swapchain_info =
2015 vk_find_struct_const(pCreateInfo->pNext, IMAGE_SWAPCHAIN_CREATE_INFO_KHR);
2016 if (swapchain_info && swapchain_info->swapchain != VK_NULL_HANDLE) {
2017 return wsi_common_create_swapchain_image(&device->physical->wsi_device,
2018 pCreateInfo,
2019 swapchain_info->swapchain,
2020 pImage);
2021 }
2022 #endif
2023
2024 struct anv_image *image =
2025 vk_object_zalloc(&device->vk, pAllocator, sizeof(*image),
2026 VK_OBJECT_TYPE_IMAGE);
2027 if (!image)
2028 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
2029
2030 VkResult result = anv_image_init_from_create_info(device, image,
2031 pCreateInfo,
2032 false);
2033 if (result != VK_SUCCESS) {
2034 vk_object_free(&device->vk, pAllocator, image);
2035 return result;
2036 }
2037
2038 ANV_RMV(image_create, device, false, image);
2039
2040 *pImage = anv_image_to_handle(image);
2041
2042 return result;
2043 }
2044
2045 void
anv_DestroyImage(VkDevice _device,VkImage _image,const VkAllocationCallbacks * pAllocator)2046 anv_DestroyImage(VkDevice _device, VkImage _image,
2047 const VkAllocationCallbacks *pAllocator)
2048 {
2049 ANV_FROM_HANDLE(anv_device, device, _device);
2050 ANV_FROM_HANDLE(anv_image, image, _image);
2051
2052 if (!image)
2053 return;
2054
2055 ANV_RMV(image_destroy, device, image);
2056
2057 assert(&device->vk == image->vk.base.device);
2058 anv_image_finish(image);
2059
2060 vk_free2(&device->vk.alloc, pAllocator, image);
2061 }
2062
2063 /* We are binding AHardwareBuffer. Get a description, resolve the
2064 * format and prepare anv_image properly.
2065 */
2066 static void
resolve_ahw_image(struct anv_device * device,struct anv_image * image,struct anv_device_memory * mem)2067 resolve_ahw_image(struct anv_device *device,
2068 struct anv_image *image,
2069 struct anv_device_memory *mem)
2070 {
2071 #if DETECT_OS_ANDROID && ANDROID_API_LEVEL >= 26
2072 assert(mem->vk.ahardware_buffer);
2073 AHardwareBuffer_Desc desc;
2074 AHardwareBuffer_describe(mem->vk.ahardware_buffer, &desc);
2075 VkResult result;
2076
2077 /* Check tiling. */
2078 enum isl_tiling tiling;
2079 const native_handle_t *handle =
2080 AHardwareBuffer_getNativeHandle(mem->vk.ahardware_buffer);
2081 struct u_gralloc_buffer_handle gr_handle = {
2082 .handle = handle,
2083 .hal_format = desc.format,
2084 .pixel_stride = desc.stride,
2085 };
2086 result = anv_android_get_tiling(device, &gr_handle, &tiling);
2087 assert(result == VK_SUCCESS);
2088 isl_tiling_flags_t isl_tiling_flags = (1u << tiling);
2089
2090 /* Check format. */
2091 VkFormat vk_format = vk_format_from_android(desc.format, desc.usage);
2092 assert(vk_format != VK_FORMAT_UNDEFINED);
2093
2094 /* Now we are able to fill anv_image fields properly and create
2095 * isl_surface for it.
2096 */
2097 vk_image_set_format(&image->vk, vk_format);
2098 image->n_planes = anv_get_format_planes(image->vk.format);
2099
2100 result = add_all_surfaces_implicit_layout(device, image, NULL, desc.stride,
2101 isl_tiling_flags,
2102 ISL_SURF_USAGE_DISABLE_AUX_BIT);
2103 assert(result == VK_SUCCESS);
2104 #endif
2105 }
2106
2107 static void
resolve_anb_image(struct anv_device * device,struct anv_image * image,const VkNativeBufferANDROID * gralloc_info)2108 resolve_anb_image(struct anv_device *device,
2109 struct anv_image *image,
2110 const VkNativeBufferANDROID *gralloc_info)
2111 {
2112 #if DETECT_OS_ANDROID && ANDROID_API_LEVEL >= 29
2113 VkResult result;
2114
2115 /* Check tiling. */
2116 enum isl_tiling tiling;
2117 struct u_gralloc_buffer_handle gr_handle = {
2118 .handle = gralloc_info->handle,
2119 .hal_format = gralloc_info->format,
2120 .pixel_stride = gralloc_info->stride,
2121 };
2122 result = anv_android_get_tiling(device, &gr_handle, &tiling);
2123 assert(result == VK_SUCCESS);
2124
2125 isl_tiling_flags_t isl_tiling_flags = (1u << tiling);
2126
2127 /* Now we are able to fill anv_image fields properly and create
2128 * isl_surface for it.
2129 */
2130 result = add_all_surfaces_implicit_layout(device, image, NULL, gralloc_info->stride,
2131 isl_tiling_flags,
2132 ISL_SURF_USAGE_DISABLE_AUX_BIT);
2133 assert(result == VK_SUCCESS);
2134 #endif
2135 }
2136
2137 static bool
anv_image_is_pat_compressible(struct anv_device * device,struct anv_image * image)2138 anv_image_is_pat_compressible(struct anv_device *device, struct anv_image *image)
2139 {
2140 if (INTEL_DEBUG(DEBUG_NO_CCS))
2141 return false;
2142
2143 if (device->info->ver < 20)
2144 return false;
2145
2146 /*
2147 * Be aware that Vulkan spec requires that Images with some properties
2148 * always returns the same memory types, so this function also needs to
2149 * have the same return for the same set of properties.
2150 *
2151 * For images created with a color format, the memoryTypeBits member is
2152 * identical for all VkImage objects created with the same combination
2153 * of values for the tiling member, the
2154 * VK_IMAGE_CREATE_SPARSE_BINDING_BIT bit and
2155 * VK_IMAGE_CREATE_PROTECTED_BIT bit of the flags member, the
2156 * VK_IMAGE_CREATE_SPLIT_INSTANCE_BIND_REGIONS_BIT bit of the flags
2157 * member, the VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT bit of the usage
2158 * member if the
2159 * VkPhysicalDeviceHostImageCopyPropertiesEXT::identicalMemoryTypeRequirements
2160 * property is VK_FALSE, handleTypes member of
2161 * VkExternalMemoryImageCreateInfo, and the
2162 * VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT of the usage member in the
2163 * VkImageCreateInfo structure passed to vkCreateImage.
2164 *
2165 * For images created with a depth/stencil format, the memoryTypeBits
2166 * member is identical for all VkImage objects created with the same
2167 * combination of values for the format member, the tiling member, the
2168 * VK_IMAGE_CREATE_SPARSE_BINDING_BIT bit and
2169 * VK_IMAGE_CREATE_PROTECTED_BIT bit of the flags member, the
2170 * VK_IMAGE_CREATE_SPLIT_INSTANCE_BIND_REGIONS_BIT bit of the flags
2171 * member, the VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT bit of the usage
2172 * member if the
2173 * VkPhysicalDeviceHostImageCopyPropertiesEXT::identicalMemoryTypeRequirements
2174 * property is VK_FALSE, handleTypes member of
2175 * VkExternalMemoryImageCreateInfo, and the
2176 * VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT of the usage member in the
2177 * VkImageCreateInfo structure passed to vkCreateImage.
2178 */
2179
2180 /* There are no compression-enabled modifiers on Xe2, and all legacy
2181 * modifiers are not defined with compression. We simply disable
2182 * compression on all modifiers.
2183 *
2184 * We disable this in anv_AllocateMemory() as well.
2185 */
2186 if (image->vk.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT)
2187 return false;
2188
2189 /* Host accessed images cannot be compressed. */
2190 if (image->vk.usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT)
2191 return false;
2192
2193 return true;
2194 }
2195
2196 void
anv_image_get_memory_requirements(struct anv_device * device,struct anv_image * image,VkImageAspectFlags aspects,VkMemoryRequirements2 * pMemoryRequirements)2197 anv_image_get_memory_requirements(struct anv_device *device,
2198 struct anv_image *image,
2199 VkImageAspectFlags aspects,
2200 VkMemoryRequirements2 *pMemoryRequirements)
2201 {
2202 /* The Vulkan spec (git aaed022) says:
2203 *
2204 * memoryTypeBits is a bitfield and contains one bit set for every
2205 * supported memory type for the resource. The bit `1<<i` is set if and
2206 * only if the memory type `i` in the VkPhysicalDeviceMemoryProperties
2207 * structure for the physical device is supported.
2208 */
2209 uint32_t memory_types;
2210
2211 if (image->vk.create_flags & VK_IMAGE_CREATE_PROTECTED_BIT) {
2212 memory_types = device->physical->memory.protected_mem_types;
2213 } else {
2214 memory_types = device->physical->memory.default_buffer_mem_types;
2215 if (anv_image_is_pat_compressible(device, image))
2216 memory_types |= device->physical->memory.compressed_mem_types;
2217 }
2218
2219 if (image->vk.usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT) {
2220 /* Remove non host visible heaps from the types for host transfers on
2221 * non ReBAR devices
2222 */
2223 if (device->physical->has_small_bar) {
2224 for (uint32_t i = 0; i < device->physical->memory.type_count; i++) {
2225 if (!(device->physical->memory.types[i].propertyFlags &
2226 VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT))
2227 memory_types &= ~BITFIELD_BIT(i);
2228 }
2229 }
2230 }
2231
2232 vk_foreach_struct(ext, pMemoryRequirements->pNext) {
2233 switch (ext->sType) {
2234 case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS: {
2235 VkMemoryDedicatedRequirements *requirements = (void *)ext;
2236 if (image->vk.wsi_legacy_scanout ||
2237 image->from_ahb ||
2238 (isl_drm_modifier_has_aux(image->vk.drm_format_mod) &&
2239 anv_image_uses_aux_map(device, image))) {
2240 /* If we need to set the tiling for external consumers or the
2241 * modifier involves AUX tables, we need a dedicated allocation.
2242 *
2243 * See also anv_AllocateMemory.
2244 */
2245 requirements->prefersDedicatedAllocation = true;
2246 requirements->requiresDedicatedAllocation = true;
2247 } else {
2248 requirements->prefersDedicatedAllocation = false;
2249 requirements->requiresDedicatedAllocation = false;
2250 }
2251 break;
2252 }
2253
2254 default:
2255 vk_debug_ignored_stype(ext->sType);
2256 break;
2257 }
2258 }
2259
2260 /* If the image is disjoint, then we must return the memory requirements for
2261 * the single plane specified in VkImagePlaneMemoryRequirementsInfo. If
2262 * non-disjoint, then exactly one set of memory requirements exists for the
2263 * whole image.
2264 *
2265 * This is enforced by the Valid Usage for VkImageMemoryRequirementsInfo2,
2266 * which requires that the app provide VkImagePlaneMemoryRequirementsInfo if
2267 * and only if the image is disjoint (that is, multi-planar format and
2268 * VK_IMAGE_CREATE_DISJOINT_BIT).
2269 */
2270 enum anv_image_memory_binding binding;
2271 if (image->disjoint) {
2272 assert(util_bitcount(aspects) == 1);
2273 assert(aspects & image->vk.aspects);
2274 binding = anv_image_aspect_to_binding(image, aspects);
2275 } else {
2276 assert(aspects == image->vk.aspects);
2277 binding = ANV_IMAGE_MEMORY_BINDING_MAIN;
2278 }
2279
2280 pMemoryRequirements->memoryRequirements = (VkMemoryRequirements) {
2281 .size = image->bindings[binding].memory_range.size,
2282 .alignment = image->bindings[binding].memory_range.alignment,
2283 .memoryTypeBits = memory_types,
2284 };
2285 }
2286
anv_GetImageMemoryRequirements2(VkDevice _device,const VkImageMemoryRequirementsInfo2 * pInfo,VkMemoryRequirements2 * pMemoryRequirements)2287 void anv_GetImageMemoryRequirements2(
2288 VkDevice _device,
2289 const VkImageMemoryRequirementsInfo2* pInfo,
2290 VkMemoryRequirements2* pMemoryRequirements)
2291 {
2292 ANV_FROM_HANDLE(anv_device, device, _device);
2293 ANV_FROM_HANDLE(anv_image, image, pInfo->image);
2294
2295 VkImageAspectFlags aspects = image->vk.aspects;
2296
2297 vk_foreach_struct_const(ext, pInfo->pNext) {
2298 switch (ext->sType) {
2299 case VK_STRUCTURE_TYPE_IMAGE_PLANE_MEMORY_REQUIREMENTS_INFO: {
2300 assert(image->disjoint);
2301 const VkImagePlaneMemoryRequirementsInfo *plane_reqs =
2302 (const VkImagePlaneMemoryRequirementsInfo *) ext;
2303 aspects = plane_reqs->planeAspect;
2304 break;
2305 }
2306
2307 default:
2308 vk_debug_ignored_stype(ext->sType);
2309 break;
2310 }
2311 }
2312
2313 anv_image_get_memory_requirements(device, image, aspects,
2314 pMemoryRequirements);
2315 }
2316
anv_GetDeviceImageMemoryRequirements(VkDevice _device,const VkDeviceImageMemoryRequirements * pInfo,VkMemoryRequirements2 * pMemoryRequirements)2317 void anv_GetDeviceImageMemoryRequirements(
2318 VkDevice _device,
2319 const VkDeviceImageMemoryRequirements* pInfo,
2320 VkMemoryRequirements2* pMemoryRequirements)
2321 {
2322 ANV_FROM_HANDLE(anv_device, device, _device);
2323 struct anv_image image = { 0 };
2324
2325 if ((device->physical->sparse_type == ANV_SPARSE_TYPE_NOT_SUPPORTED) &&
2326 INTEL_DEBUG(DEBUG_SPARSE) &&
2327 pInfo->pCreateInfo->flags & (VK_IMAGE_CREATE_SPARSE_BINDING_BIT |
2328 VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT |
2329 VK_IMAGE_CREATE_SPARSE_ALIASED_BIT))
2330 fprintf(stderr, "=== %s %s:%d flags:0x%08x\n", __func__, __FILE__,
2331 __LINE__, pInfo->pCreateInfo->flags);
2332
2333 ASSERTED VkResult result =
2334 anv_image_init_from_create_info(device, &image, pInfo->pCreateInfo, true);
2335 assert(result == VK_SUCCESS);
2336
2337 VkImageAspectFlags aspects =
2338 image.disjoint ? pInfo->planeAspect : image.vk.aspects;
2339
2340 anv_image_get_memory_requirements(device, &image, aspects,
2341 pMemoryRequirements);
2342 anv_image_finish(&image);
2343 }
2344
2345 static void
anv_image_get_sparse_memory_requirements(struct anv_device * device,struct anv_image * image,VkImageAspectFlags aspects,uint32_t * pSparseMemoryRequirementCount,VkSparseImageMemoryRequirements2 * pSparseMemoryRequirements)2346 anv_image_get_sparse_memory_requirements(
2347 struct anv_device *device,
2348 struct anv_image *image,
2349 VkImageAspectFlags aspects,
2350 uint32_t *pSparseMemoryRequirementCount,
2351 VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
2352 {
2353 VK_OUTARRAY_MAKE_TYPED(VkSparseImageMemoryRequirements2, reqs,
2354 pSparseMemoryRequirements,
2355 pSparseMemoryRequirementCount);
2356
2357 /* From the spec:
2358 * "The sparse image must have been created using the
2359 * VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT flag to retrieve valid sparse
2360 * image memory requirements."
2361 */
2362 if (!(image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT))
2363 return;
2364
2365 VkSparseImageMemoryRequirements ds_mem_reqs = {};
2366 VkSparseImageMemoryRequirements2 *ds_reqs_ptr = NULL;
2367
2368 u_foreach_bit(b, aspects) {
2369 VkImageAspectFlagBits aspect = 1 << b;
2370 const uint32_t plane = anv_image_aspect_to_plane(image, aspect);
2371 struct isl_surf *surf = &image->planes[plane].primary_surface.isl;
2372
2373 VkSparseImageFormatProperties format_props =
2374 anv_sparse_calc_image_format_properties(device->physical, aspect,
2375 image->vk.image_type,
2376 image->vk.samples, surf);
2377
2378 uint32_t miptail_first_lod;
2379 VkDeviceSize miptail_size, miptail_offset, miptail_stride;
2380 anv_sparse_calc_miptail_properties(device, image, aspect,
2381 &miptail_first_lod, &miptail_size,
2382 &miptail_offset, &miptail_stride);
2383
2384 VkSparseImageMemoryRequirements mem_reqs = {
2385 .formatProperties = format_props,
2386 .imageMipTailFirstLod = miptail_first_lod,
2387 .imageMipTailSize = miptail_size,
2388 .imageMipTailOffset = miptail_offset,
2389 .imageMipTailStride = miptail_stride,
2390 };
2391
2392 /* If both depth and stencil are the same, unify them if possible. */
2393 if (aspect & (VK_IMAGE_ASPECT_DEPTH_BIT |
2394 VK_IMAGE_ASPECT_STENCIL_BIT)) {
2395 if (!ds_reqs_ptr) {
2396 ds_mem_reqs = mem_reqs;
2397 } else if (ds_mem_reqs.formatProperties.imageGranularity.width ==
2398 mem_reqs.formatProperties.imageGranularity.width &&
2399 ds_mem_reqs.formatProperties.imageGranularity.height ==
2400 mem_reqs.formatProperties.imageGranularity.height &&
2401 ds_mem_reqs.formatProperties.imageGranularity.depth ==
2402 mem_reqs.formatProperties.imageGranularity.depth &&
2403 ds_mem_reqs.imageMipTailFirstLod ==
2404 mem_reqs.imageMipTailFirstLod &&
2405 ds_mem_reqs.imageMipTailSize ==
2406 mem_reqs.imageMipTailSize &&
2407 ds_mem_reqs.imageMipTailOffset ==
2408 mem_reqs.imageMipTailOffset &&
2409 ds_mem_reqs.imageMipTailStride ==
2410 mem_reqs.imageMipTailStride) {
2411 ds_reqs_ptr->memoryRequirements.formatProperties.aspectMask |=
2412 aspect;
2413 continue;
2414 }
2415 }
2416
2417 vk_outarray_append_typed(VkSparseImageMemoryRequirements2, &reqs, r) {
2418 r->memoryRequirements = mem_reqs;
2419 if (aspect & (VK_IMAGE_ASPECT_DEPTH_BIT |
2420 VK_IMAGE_ASPECT_STENCIL_BIT))
2421 ds_reqs_ptr = r;
2422 }
2423 }
2424 }
2425
anv_GetImageSparseMemoryRequirements2(VkDevice _device,const VkImageSparseMemoryRequirementsInfo2 * pInfo,uint32_t * pSparseMemoryRequirementCount,VkSparseImageMemoryRequirements2 * pSparseMemoryRequirements)2426 void anv_GetImageSparseMemoryRequirements2(
2427 VkDevice _device,
2428 const VkImageSparseMemoryRequirementsInfo2* pInfo,
2429 uint32_t* pSparseMemoryRequirementCount,
2430 VkSparseImageMemoryRequirements2* pSparseMemoryRequirements)
2431 {
2432 ANV_FROM_HANDLE(anv_device, device, _device);
2433 ANV_FROM_HANDLE(anv_image, image, pInfo->image);
2434
2435 if (!anv_sparse_residency_is_enabled(device)) {
2436 if ((device->physical->sparse_type == ANV_SPARSE_TYPE_NOT_SUPPORTED) &&
2437 INTEL_DEBUG(DEBUG_SPARSE))
2438 fprintf(stderr, "=== [%s:%d] [%s]\n", __FILE__, __LINE__, __func__);
2439
2440 *pSparseMemoryRequirementCount = 0;
2441 return;
2442 }
2443
2444 anv_image_get_sparse_memory_requirements(device, image, image->vk.aspects,
2445 pSparseMemoryRequirementCount,
2446 pSparseMemoryRequirements);
2447 }
2448
anv_GetDeviceImageSparseMemoryRequirements(VkDevice _device,const VkDeviceImageMemoryRequirements * pInfo,uint32_t * pSparseMemoryRequirementCount,VkSparseImageMemoryRequirements2 * pSparseMemoryRequirements)2449 void anv_GetDeviceImageSparseMemoryRequirements(
2450 VkDevice _device,
2451 const VkDeviceImageMemoryRequirements* pInfo,
2452 uint32_t* pSparseMemoryRequirementCount,
2453 VkSparseImageMemoryRequirements2* pSparseMemoryRequirements)
2454 {
2455 ANV_FROM_HANDLE(anv_device, device, _device);
2456 struct anv_image image = { 0 };
2457
2458 if (!anv_sparse_residency_is_enabled(device)) {
2459 if ((device->physical->sparse_type == ANV_SPARSE_TYPE_NOT_SUPPORTED) &&
2460 INTEL_DEBUG(DEBUG_SPARSE))
2461 fprintf(stderr, "=== [%s:%d] [%s]\n", __FILE__, __LINE__, __func__);
2462
2463 *pSparseMemoryRequirementCount = 0;
2464 return;
2465 }
2466
2467 /* This function is similar to anv_GetDeviceImageMemoryRequirements, in
2468 * which it actually creates an image, gets the properties and then
2469 * destroys the image.
2470 *
2471 * We could one day refactor things to allow us to gather the properties
2472 * without having to actually create the image, maybe by reworking ISL to
2473 * separate creation from parameter computing.
2474 */
2475 VkResult result =
2476 anv_image_init_from_create_info(device, &image, pInfo->pCreateInfo,
2477 true /* no_private_binding_alloc */);
2478 if (result != VK_SUCCESS) {
2479 *pSparseMemoryRequirementCount = 0;
2480 return;
2481 }
2482
2483 /* The spec says:
2484 * "planeAspect is a VkImageAspectFlagBits value specifying the aspect
2485 * corresponding to the image plane to query. This parameter is ignored
2486 * unless pCreateInfo::tiling is VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT,
2487 * or pCreateInfo::flags has VK_IMAGE_CREATE_DISJOINT_BIT set."
2488 */
2489 VkImageAspectFlags aspects =
2490 (pInfo->pCreateInfo->flags & VK_IMAGE_CREATE_DISJOINT_BIT) ||
2491 (pInfo->pCreateInfo->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT)
2492 ? pInfo->planeAspect : image.vk.aspects;
2493
2494 anv_image_get_sparse_memory_requirements(device, &image, aspects,
2495 pSparseMemoryRequirementCount,
2496 pSparseMemoryRequirements);
2497
2498 anv_image_finish(&image);
2499 }
2500
2501 static bool
anv_image_map_aux_tt(struct anv_device * device,struct anv_image * image,uint32_t plane)2502 anv_image_map_aux_tt(struct anv_device *device,
2503 struct anv_image *image, uint32_t plane)
2504 {
2505 const struct anv_address main_addr = anv_image_address(
2506 image, &image->planes[plane].primary_surface.memory_range);
2507 struct anv_bo *bo = main_addr.bo;
2508 assert(bo != NULL);
2509
2510 /* If the additional memory padding was added at the end of the BO for CCS
2511 * data, map this region at the granularity of the main/CCS pages.
2512 *
2513 * Otherwise the image should have additional CCS data at the computed
2514 * offset.
2515 */
2516 if (device->physical->alloc_aux_tt_mem &&
2517 (bo->alloc_flags & ANV_BO_ALLOC_AUX_CCS)) {
2518 uint64_t main_aux_alignment =
2519 intel_aux_map_get_alignment(device->aux_map_ctx);
2520 assert(bo->offset % main_aux_alignment == 0);
2521 const struct anv_address start_addr = (struct anv_address) {
2522 .bo = bo,
2523 .offset = ROUND_DOWN_TO(main_addr.offset, main_aux_alignment),
2524 };
2525 const struct anv_address aux_addr = (struct anv_address) {
2526 .bo = bo,
2527 .offset = bo->ccs_offset +
2528 intel_aux_main_to_aux_offset(device->aux_map_ctx,
2529 start_addr.offset),
2530 };
2531 const struct isl_surf *surf = &image->planes[plane].primary_surface.isl;
2532 const uint64_t format_bits =
2533 intel_aux_map_format_bits_for_isl_surf(surf);
2534 /* Make sure to have the mapping cover the entire image from the aux
2535 * aligned start.
2536 */
2537 const uint64_t main_size = align(
2538 (main_addr.offset - start_addr.offset) + surf->size_B,
2539 main_aux_alignment);
2540
2541 if (intel_aux_map_add_mapping(device->aux_map_ctx,
2542 anv_address_physical(start_addr),
2543 anv_address_physical(aux_addr),
2544 main_size, format_bits)) {
2545 image->planes[plane].aux_tt.mapped = true;
2546 image->planes[plane].aux_tt.addr = anv_address_physical(start_addr);
2547 image->planes[plane].aux_tt.size = main_size;
2548 return true;
2549 }
2550 } else {
2551 if (anv_address_allows_aux_map(device, main_addr)) {
2552 const struct anv_address aux_addr =
2553 anv_image_address(image,
2554 &image->planes[plane].compr_ctrl_memory_range);
2555 const struct isl_surf *surf =
2556 &image->planes[plane].primary_surface.isl;
2557 const uint64_t format_bits =
2558 intel_aux_map_format_bits_for_isl_surf(surf);
2559 if (intel_aux_map_add_mapping(device->aux_map_ctx,
2560 anv_address_physical(main_addr),
2561 anv_address_physical(aux_addr),
2562 surf->size_B, format_bits)) {
2563 image->planes[plane].aux_tt.mapped = true;
2564 image->planes[plane].aux_tt.addr = anv_address_physical(main_addr);
2565 image->planes[plane].aux_tt.size = surf->size_B;
2566 return true;
2567 }
2568 }
2569 }
2570
2571 return false;
2572
2573 }
2574
2575 static VkResult
anv_image_bind_address(struct anv_device * device,struct anv_image * image,enum anv_image_memory_binding binding,struct anv_address address)2576 anv_image_bind_address(struct anv_device *device,
2577 struct anv_image *image,
2578 enum anv_image_memory_binding binding,
2579 struct anv_address address)
2580 {
2581 image->bindings[binding].address = address;
2582
2583 /* Map bindings for images with host transfer usage, so that we don't have
2584 * to map/unmap things at every host operation. We map cached, that means
2585 * that the copy operations need to cflush on platforms that have no
2586 * host_cache+host_coherent memory types.
2587 */
2588 if (image->vk.usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT) {
2589 uint64_t offset = image->bindings[binding].address.offset +
2590 image->bindings[binding].memory_range.offset;
2591 uint64_t map_offset, map_size;
2592 anv_sanitize_map_params(device, offset,
2593 image->bindings[binding].memory_range.size,
2594 &map_offset, &map_size);
2595
2596 VkResult result = anv_device_map_bo(device,
2597 image->bindings[binding].address.bo,
2598 map_offset, map_size,
2599 NULL /* placed_addr */,
2600 &image->bindings[binding].host_map);
2601 if (result != VK_SUCCESS)
2602 return result;
2603
2604 image->bindings[binding].map_delta = (offset - map_offset);
2605 image->bindings[binding].map_size = map_size;
2606 }
2607
2608 ANV_RMV(image_bind, device, image, binding);
2609
2610 return VK_SUCCESS;
2611 }
2612
2613 static VkResult
anv_bind_image_memory(struct anv_device * device,const VkBindImageMemoryInfo * bind_info)2614 anv_bind_image_memory(struct anv_device *device,
2615 const VkBindImageMemoryInfo *bind_info)
2616 {
2617 ANV_FROM_HANDLE(anv_device_memory, mem, bind_info->memory);
2618 ANV_FROM_HANDLE(anv_image, image, bind_info->image);
2619 bool did_bind = false;
2620 VkResult result = VK_SUCCESS;
2621
2622 const VkBindMemoryStatusKHR *bind_status =
2623 vk_find_struct_const(bind_info->pNext, BIND_MEMORY_STATUS_KHR);
2624
2625 assert(!anv_image_is_sparse(image));
2626
2627 /* Resolve will alter the image's aspects, do this first. */
2628 if (mem && mem->vk.ahardware_buffer)
2629 resolve_ahw_image(device, image, mem);
2630
2631 vk_foreach_struct_const(s, bind_info->pNext) {
2632 switch (s->sType) {
2633 case VK_STRUCTURE_TYPE_BIND_IMAGE_PLANE_MEMORY_INFO: {
2634 const VkBindImagePlaneMemoryInfo *plane_info =
2635 (const VkBindImagePlaneMemoryInfo *) s;
2636
2637 /* Workaround for possible spec bug.
2638 *
2639 * Unlike VkImagePlaneMemoryRequirementsInfo, which requires that
2640 * the image be disjoint (that is, multi-planar format and
2641 * VK_IMAGE_CREATE_DISJOINT_BIT), VkBindImagePlaneMemoryInfo allows
2642 * the image to be non-disjoint and requires only that the image
2643 * have the DISJOINT flag. In this case, regardless of the value of
2644 * VkImagePlaneMemoryRequirementsInfo::planeAspect, the behavior is
2645 * the same as if VkImagePlaneMemoryRequirementsInfo were omitted.
2646 */
2647 if (!image->disjoint)
2648 break;
2649
2650 enum anv_image_memory_binding binding =
2651 anv_image_aspect_to_binding(image, plane_info->planeAspect);
2652
2653 anv_image_bind_address(device, image, binding,
2654 (struct anv_address) {
2655 .bo = mem->bo,
2656 .offset = bind_info->memoryOffset,
2657 });
2658
2659 did_bind = true;
2660 break;
2661 }
2662 case VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_SWAPCHAIN_INFO_KHR: {
2663 /* Ignore this struct on Android, we cannot access swapchain
2664 * structures there.
2665 */
2666 #ifndef VK_USE_PLATFORM_ANDROID_KHR
2667 const VkBindImageMemorySwapchainInfoKHR *swapchain_info =
2668 (const VkBindImageMemorySwapchainInfoKHR *) s;
2669 struct anv_image *swapchain_image =
2670 anv_swapchain_get_image(swapchain_info->swapchain,
2671 swapchain_info->imageIndex);
2672 assert(swapchain_image);
2673 assert(image->vk.aspects == swapchain_image->vk.aspects);
2674 assert(mem == NULL);
2675
2676 for (int j = 0; j < ARRAY_SIZE(image->bindings); ++j) {
2677 assert(memory_ranges_equal(image->bindings[j].memory_range,
2678 swapchain_image->bindings[j].memory_range));
2679 if (image->bindings[j].memory_range.size != 0) {
2680 anv_image_bind_address(device, image, j,
2681 swapchain_image->bindings[j].address);
2682 }
2683 }
2684
2685 /* We must bump the private binding's bo's refcount because, unlike the other
2686 * bindings, its lifetime is not application-managed.
2687 */
2688 struct anv_bo *private_bo =
2689 image->bindings[ANV_IMAGE_MEMORY_BINDING_PRIVATE].address.bo;
2690 if (private_bo)
2691 anv_bo_ref(private_bo);
2692
2693 did_bind = true;
2694 #endif
2695 break;
2696 }
2697 #pragma GCC diagnostic push
2698 #pragma GCC diagnostic ignored "-Wswitch"
2699 case VK_STRUCTURE_TYPE_NATIVE_BUFFER_ANDROID: {
2700 const VkNativeBufferANDROID *gralloc_info =
2701 (const VkNativeBufferANDROID *)s;
2702 result = anv_image_bind_from_gralloc(device, image, gralloc_info);
2703 if (result != VK_SUCCESS)
2704 return result;
2705
2706 resolve_anb_image(device, image, gralloc_info);
2707 did_bind = true;
2708 break;
2709 }
2710 #pragma GCC diagnostic pop
2711 default:
2712 vk_debug_ignored_stype(s->sType);
2713 break;
2714 }
2715 }
2716
2717 if (!did_bind) {
2718 assert(!image->disjoint);
2719 result = anv_image_bind_address(device, image,
2720 ANV_IMAGE_MEMORY_BINDING_MAIN,
2721 (struct anv_address) {
2722 .bo = mem->bo,
2723 .offset = bind_info->memoryOffset,
2724 });
2725 did_bind = true;
2726 }
2727
2728 /* Now that we have the BO, finalize CCS setup. */
2729 for (int p = 0; p < image->n_planes; ++p) {
2730 enum anv_image_memory_binding binding =
2731 image->planes[p].primary_surface.memory_range.binding;
2732 const struct anv_bo *bo =
2733 image->bindings[binding].address.bo;
2734
2735 if (!bo || !isl_aux_usage_has_ccs(image->planes[p].aux_usage))
2736 continue;
2737
2738 /* Do nothing if flat CCS requirements are satisfied.
2739 *
2740 * Also, assume that imported BOs with a modifier including
2741 * CCS live only in local memory. Otherwise the exporter should
2742 * have failed the creation of the BO.
2743 */
2744 if (device->info->has_flat_ccs &&
2745 (anv_bo_is_vram_only(bo) ||
2746 (bo->alloc_flags & ANV_BO_ALLOC_IMPORTED)))
2747 continue;
2748
2749 /* If the AUX-TT mapping succeeds, there is nothing else to do. */
2750 if (device->info->has_aux_map && anv_image_map_aux_tt(device, image, p))
2751 continue;
2752
2753 /* Do nothing except for gfx12. There are no special requirements. */
2754 if (device->info->ver != 12)
2755 continue;
2756
2757 /* The plane's BO cannot support CCS, disable compression on it. */
2758 assert(!isl_drm_modifier_has_aux(image->vk.drm_format_mod));
2759
2760 anv_perf_warn(VK_LOG_OBJS(&image->vk.base),
2761 "BO lacks CCS support. Disabling the CCS aux usage.");
2762
2763 if (image->planes[p].aux_usage == ISL_AUX_USAGE_MCS_CCS) {
2764 assert(image->planes[p].aux_surface.memory_range.size);
2765 image->planes[p].aux_usage = ISL_AUX_USAGE_MCS;
2766 } else if (image->planes[p].aux_usage == ISL_AUX_USAGE_HIZ_CCS ||
2767 image->planes[p].aux_usage == ISL_AUX_USAGE_HIZ_CCS_WT) {
2768 assert(image->planes[p].aux_surface.memory_range.size);
2769 image->planes[p].aux_usage = ISL_AUX_USAGE_HIZ;
2770 } else {
2771 assert(image->planes[p].aux_usage == ISL_AUX_USAGE_CCS_E ||
2772 image->planes[p].aux_usage == ISL_AUX_USAGE_FCV_CCS_E ||
2773 image->planes[p].aux_usage == ISL_AUX_USAGE_STC_CCS);
2774 image->planes[p].aux_usage = ISL_AUX_USAGE_NONE;
2775 }
2776 }
2777
2778 if (bind_status)
2779 *bind_status->pResult = result;
2780
2781 return result;
2782 }
2783
anv_BindImageMemory2(VkDevice _device,uint32_t bindInfoCount,const VkBindImageMemoryInfo * pBindInfos)2784 VkResult anv_BindImageMemory2(
2785 VkDevice _device,
2786 uint32_t bindInfoCount,
2787 const VkBindImageMemoryInfo* pBindInfos)
2788 {
2789 ANV_FROM_HANDLE(anv_device, device, _device);
2790 VkResult result = VK_SUCCESS;
2791
2792 for (uint32_t i = 0; i < bindInfoCount; i++) {
2793 VkResult res = anv_bind_image_memory(device, &pBindInfos[i]);
2794 if (result == VK_SUCCESS && res != VK_SUCCESS)
2795 result = res;
2796 }
2797
2798 return result;
2799 }
2800
2801 static void
anv_get_image_subresource_layout(struct anv_device * device,const struct anv_image * image,const VkImageSubresource2KHR * subresource,VkSubresourceLayout2KHR * layout)2802 anv_get_image_subresource_layout(struct anv_device *device,
2803 const struct anv_image *image,
2804 const VkImageSubresource2KHR *subresource,
2805 VkSubresourceLayout2KHR *layout)
2806 {
2807 const struct isl_surf *isl_surf = NULL;
2808 const struct anv_image_memory_range *mem_range;
2809 uint64_t row_pitch_B;
2810
2811 assert(__builtin_popcount(subresource->imageSubresource.aspectMask) == 1);
2812
2813 /* The Vulkan spec requires that aspectMask be
2814 * VK_IMAGE_ASPECT_MEMORY_PLANE_i_BIT_EXT if tiling is
2815 * VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT.
2816 *
2817 * For swapchain images, the Vulkan spec says that every swapchain image has
2818 * tiling VK_IMAGE_TILING_OPTIMAL, but we may choose
2819 * VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT internally. Vulkan doesn't allow
2820 * vkGetImageSubresourceLayout for images with VK_IMAGE_TILING_OPTIMAL,
2821 * therefore it's invalid for the application to call this on a swapchain
2822 * image. The WSI code, however, knows when it has internally created
2823 * a swapchain image with VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT,
2824 * so it _should_ correctly use VK_IMAGE_ASPECT_MEMORY_PLANE_* in that case.
2825 * But it incorrectly uses VK_IMAGE_ASPECT_PLANE_*, so we have a temporary
2826 * workaround.
2827 *
2828 * https://gitlab.freedesktop.org/mesa/mesa/-/issues/10176
2829 */
2830 if (image->vk.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
2831 /* TODO(chadv): Drop this workaround when WSI gets fixed. */
2832 uint32_t mem_plane;
2833 switch (subresource->imageSubresource.aspectMask) {
2834 case VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT:
2835 case VK_IMAGE_ASPECT_PLANE_0_BIT:
2836 mem_plane = 0;
2837 break;
2838 case VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT:
2839 case VK_IMAGE_ASPECT_PLANE_1_BIT:
2840 mem_plane = 1;
2841 break;
2842 case VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT:
2843 case VK_IMAGE_ASPECT_PLANE_2_BIT:
2844 mem_plane = 2;
2845 break;
2846 default:
2847 unreachable("bad VkImageAspectFlags");
2848 }
2849
2850 if (isl_drm_modifier_plane_is_clear_color(image->vk.drm_format_mod,
2851 mem_plane)) {
2852 assert(image->n_planes == 1);
2853
2854 mem_range = &image->planes[0].fast_clear_memory_range;
2855 row_pitch_B = ISL_DRM_CC_PLANE_PITCH_B;
2856 } else if (mem_plane == 1 &&
2857 image->planes[0].compr_ctrl_memory_range.size > 0) {
2858 assert(image->n_planes == 1);
2859 assert(isl_drm_modifier_has_aux(image->vk.drm_format_mod));
2860
2861 mem_range = &image->planes[0].compr_ctrl_memory_range;
2862 row_pitch_B = image->planes[0].primary_surface.isl.row_pitch_B /
2863 INTEL_AUX_MAP_MAIN_PITCH_SCALEDOWN;
2864 } else if (mem_plane == 1 &&
2865 image->planes[0].aux_surface.memory_range.size > 0) {
2866 assert(image->n_planes == 1);
2867 assert(image->vk.drm_format_mod == I915_FORMAT_MOD_Y_TILED_CCS);
2868
2869 mem_range = &image->planes[0].aux_surface.memory_range;
2870 row_pitch_B = image->planes[0].aux_surface.isl.row_pitch_B;
2871 } else {
2872 assert(mem_plane < image->n_planes);
2873
2874 mem_range = &image->planes[mem_plane].primary_surface.memory_range;
2875 row_pitch_B =
2876 image->planes[mem_plane].primary_surface.isl.row_pitch_B;
2877
2878 isl_surf = &image->planes[mem_plane].primary_surface.isl;
2879 }
2880
2881 /* If the memory binding differs between the primary plane and the
2882 * specified memory plane, the returned offset will be incorrect.
2883 */
2884 assert(mem_range->binding ==
2885 image->planes[0].primary_surface.memory_range.binding);
2886
2887 /* We are working with a non-arrayed 2D image. */
2888 assert(image->vk.image_type == VK_IMAGE_TYPE_2D);
2889 assert(image->vk.array_layers == 1);
2890 } else {
2891 const uint32_t plane =
2892 anv_image_aspect_to_plane(image,
2893 subresource->imageSubresource.aspectMask);
2894 isl_surf = &image->planes[plane].primary_surface.isl;
2895 mem_range = &image->planes[plane].primary_surface.memory_range;
2896 row_pitch_B = isl_surf->row_pitch_B;
2897 }
2898
2899 const uint32_t level = subresource->imageSubresource.mipLevel;
2900 if (isl_surf) {
2901 /* ISL tries to give us a single layer but the Vulkan API expect the
2902 * entire 3D size.
2903 */
2904 const uint32_t layer = subresource->imageSubresource.arrayLayer;
2905 const uint32_t z = u_minify(isl_surf->logical_level0_px.d, level) - 1;
2906 uint64_t z0_start_tile_B, z0_end_tile_B;
2907 uint64_t zX_start_tile_B, zX_end_tile_B;
2908 isl_surf_get_image_range_B_tile(isl_surf, level, layer, 0,
2909 &z0_start_tile_B, &z0_end_tile_B);
2910 isl_surf_get_image_range_B_tile(isl_surf, level, layer, z,
2911 &zX_start_tile_B, &zX_end_tile_B);
2912
2913 layout->subresourceLayout.offset = mem_range->offset + z0_start_tile_B;
2914 layout->subresourceLayout.size = zX_end_tile_B - z0_start_tile_B;
2915 layout->subresourceLayout.rowPitch = row_pitch_B;
2916 layout->subresourceLayout.depthPitch =
2917 isl_surf_get_array_pitch(isl_surf);
2918 layout->subresourceLayout.arrayPitch =
2919 isl_surf_get_array_pitch(isl_surf);
2920 } else {
2921 layout->subresourceLayout.offset = mem_range->offset;
2922 layout->subresourceLayout.size = mem_range->size;
2923 layout->subresourceLayout.rowPitch = row_pitch_B;
2924 /* Not a surface so those fields don't make sense */
2925 layout->subresourceLayout.depthPitch = 0;
2926 layout->subresourceLayout.arrayPitch = 0;
2927 }
2928
2929 VkSubresourceHostMemcpySizeEXT *host_memcpy_size =
2930 vk_find_struct(layout->pNext, SUBRESOURCE_HOST_MEMCPY_SIZE_EXT);
2931 if (host_memcpy_size) {
2932 if (!isl_surf) {
2933 host_memcpy_size->size = 0;
2934 } else if (anv_image_can_host_memcpy(image)) {
2935 host_memcpy_size->size = layout->subresourceLayout.size;
2936 } else {
2937 /* If we cannot do straight memcpy of the image, compute a linear
2938 * size. This will be the format in which we store the data.
2939 */
2940 struct isl_surf lin_surf;
2941 bool ok =
2942 isl_surf_init(&device->physical->isl_dev, &lin_surf,
2943 .dim = isl_surf->dim,
2944 .format = isl_surf->format,
2945 .width = u_minify(
2946 isl_surf->logical_level0_px.w, level),
2947 .height = u_minify(
2948 isl_surf->logical_level0_px.h, level),
2949 .depth = u_minify(
2950 isl_surf->logical_level0_px.d, level),
2951 .array_len = 1,
2952 .levels = 1,
2953 .samples = isl_surf->samples,
2954 .tiling_flags = ISL_TILING_LINEAR_BIT);
2955 assert(ok);
2956 host_memcpy_size->size = lin_surf.size_B;
2957 }
2958 }
2959
2960 VkImageCompressionPropertiesEXT *comp_props =
2961 vk_find_struct(layout->pNext, IMAGE_COMPRESSION_PROPERTIES_EXT);
2962 if (comp_props) {
2963 comp_props->imageCompressionFixedRateFlags =
2964 VK_IMAGE_COMPRESSION_FIXED_RATE_NONE_EXT;
2965 comp_props->imageCompressionFlags = VK_IMAGE_COMPRESSION_DISABLED_EXT;
2966 for (uint32_t p = 0; p < image->n_planes; p++) {
2967 if (image->planes[p].aux_usage != ISL_AUX_USAGE_NONE) {
2968 comp_props->imageCompressionFlags = VK_IMAGE_COMPRESSION_DEFAULT_EXT;
2969 break;
2970 }
2971 }
2972 }
2973 }
2974
anv_GetDeviceImageSubresourceLayoutKHR(VkDevice _device,const VkDeviceImageSubresourceInfoKHR * pInfo,VkSubresourceLayout2KHR * pLayout)2975 void anv_GetDeviceImageSubresourceLayoutKHR(
2976 VkDevice _device,
2977 const VkDeviceImageSubresourceInfoKHR* pInfo,
2978 VkSubresourceLayout2KHR* pLayout)
2979 {
2980 ANV_FROM_HANDLE(anv_device, device, _device);
2981
2982 struct anv_image image = { 0 };
2983
2984 if (anv_image_init_from_create_info(device, &image, pInfo->pCreateInfo,
2985 true) != VK_SUCCESS) {
2986 pLayout->subresourceLayout = (VkSubresourceLayout) { 0, };
2987 return;
2988 }
2989
2990 anv_get_image_subresource_layout(device, &image, pInfo->pSubresource, pLayout);
2991 }
2992
anv_GetImageSubresourceLayout2KHR(VkDevice _device,VkImage _image,const VkImageSubresource2KHR * pSubresource,VkSubresourceLayout2KHR * pLayout)2993 void anv_GetImageSubresourceLayout2KHR(
2994 VkDevice _device,
2995 VkImage _image,
2996 const VkImageSubresource2KHR* pSubresource,
2997 VkSubresourceLayout2KHR* pLayout)
2998 {
2999 ANV_FROM_HANDLE(anv_device, device, _device);
3000 ANV_FROM_HANDLE(anv_image, image, _image);
3001
3002 anv_get_image_subresource_layout(device, image, pSubresource, pLayout);
3003 }
3004
3005 static VkImageUsageFlags
anv_image_flags_filter_for_queue(VkImageUsageFlags usages,VkQueueFlagBits queue_flags)3006 anv_image_flags_filter_for_queue(VkImageUsageFlags usages,
3007 VkQueueFlagBits queue_flags)
3008 {
3009 /* Eliminate graphics usages if the queue is not graphics capable */
3010 if (!(queue_flags & VK_QUEUE_GRAPHICS_BIT)) {
3011 usages &= ~(VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
3012 VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT |
3013 VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT |
3014 VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT |
3015 VK_IMAGE_USAGE_FRAGMENT_DENSITY_MAP_BIT_EXT |
3016 VK_IMAGE_USAGE_FRAGMENT_SHADING_RATE_ATTACHMENT_BIT_KHR |
3017 VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT);
3018 }
3019
3020 /* Eliminate sampling & storage usages if the queue is neither graphics nor
3021 * compute capable
3022 */
3023 if (!(queue_flags & (VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT))) {
3024 usages &= ~(VK_IMAGE_USAGE_SAMPLED_BIT |
3025 VK_IMAGE_USAGE_STORAGE_BIT);
3026 }
3027
3028 /* Eliminate transfer usages if the queue is neither transfer, compute or
3029 * graphics capable
3030 */
3031 if (!(queue_flags & (VK_QUEUE_TRANSFER_BIT |
3032 VK_QUEUE_COMPUTE_BIT |
3033 VK_QUEUE_GRAPHICS_BIT))) {
3034 usages &= ~(VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
3035 VK_IMAGE_USAGE_TRANSFER_DST_BIT);
3036 }
3037
3038 return usages;
3039 }
3040
3041 /**
3042 * This function returns the assumed isl_aux_state for a given VkImageLayout.
3043 * Because Vulkan image layouts don't map directly to isl_aux_state enums, the
3044 * returned enum is the assumed worst case.
3045 *
3046 * @param devinfo The device information of the Intel GPU.
3047 * @param image The image that may contain a collection of buffers.
3048 * @param aspect The aspect of the image to be accessed.
3049 * @param layout The current layout of the image aspect(s).
3050 *
3051 * @return The primary buffer that should be used for the given layout.
3052 */
3053 enum isl_aux_state ATTRIBUTE_PURE
anv_layout_to_aux_state(const struct intel_device_info * const devinfo,const struct anv_image * const image,const VkImageAspectFlagBits aspect,const VkImageLayout layout,const VkQueueFlagBits queue_flags)3054 anv_layout_to_aux_state(const struct intel_device_info * const devinfo,
3055 const struct anv_image * const image,
3056 const VkImageAspectFlagBits aspect,
3057 const VkImageLayout layout,
3058 const VkQueueFlagBits queue_flags)
3059 {
3060 /* Validate the inputs. */
3061
3062 /* The devinfo is needed as the optimal buffer varies across generations. */
3063 assert(devinfo != NULL);
3064
3065 /* The layout of a NULL image is not properly defined. */
3066 assert(image != NULL);
3067
3068 /* The aspect must be exactly one of the image aspects. */
3069 assert(util_bitcount(aspect) == 1 && (aspect & image->vk.aspects));
3070
3071 /* Determine the optimal buffer. */
3072
3073 const uint32_t plane = anv_image_aspect_to_plane(image, aspect);
3074
3075 /* If we don't have an aux buffer then aux state makes no sense */
3076 const enum isl_aux_usage aux_usage = image->planes[plane].aux_usage;
3077 assert(aux_usage != ISL_AUX_USAGE_NONE);
3078
3079 /* All images that use an auxiliary surface are required to be tiled. */
3080 assert(image->planes[plane].primary_surface.isl.tiling != ISL_TILING_LINEAR);
3081
3082 /* Handle a few special cases */
3083 switch (layout) {
3084 /* Invalid layouts */
3085 case VK_IMAGE_LAYOUT_MAX_ENUM:
3086 unreachable("Invalid image layout.");
3087
3088 /* Undefined layouts
3089 *
3090 * The pre-initialized layout is equivalent to the undefined layout for
3091 * optimally-tiled images. We can only do color compression (CCS or HiZ)
3092 * on tiled images.
3093 */
3094 case VK_IMAGE_LAYOUT_UNDEFINED:
3095 case VK_IMAGE_LAYOUT_PREINITIALIZED:
3096 return ISL_AUX_STATE_AUX_INVALID;
3097
3098 case VK_IMAGE_LAYOUT_PRESENT_SRC_KHR: {
3099 assert(image->vk.aspects == VK_IMAGE_ASPECT_COLOR_BIT);
3100
3101 enum isl_aux_state aux_state =
3102 isl_drm_modifier_get_default_aux_state(image->vk.drm_format_mod);
3103
3104 switch (aux_state) {
3105 case ISL_AUX_STATE_AUX_INVALID:
3106 /* The modifier does not support compression. But, if we arrived
3107 * here, then we have enabled compression on it anyway, in which case
3108 * we must resolve the aux surface before we release ownership to the
3109 * presentation engine (because, having no modifier, the presentation
3110 * engine will not be aware of the aux surface). The presentation
3111 * engine will not access the aux surface (because it is unware of
3112 * it), and so the aux surface will still be resolved when we
3113 * re-acquire ownership.
3114 *
3115 * Therefore, at ownership transfers in either direction, there does
3116 * exist an aux surface despite the lack of modifier and its state is
3117 * pass-through.
3118 */
3119 return ISL_AUX_STATE_PASS_THROUGH;
3120 case ISL_AUX_STATE_COMPRESSED_CLEAR:
3121 return ISL_AUX_STATE_COMPRESSED_CLEAR;
3122 case ISL_AUX_STATE_COMPRESSED_NO_CLEAR:
3123 return ISL_AUX_STATE_COMPRESSED_NO_CLEAR;
3124 default:
3125 unreachable("unexpected isl_aux_state");
3126 }
3127 }
3128
3129 default:
3130 break;
3131 }
3132
3133 const bool read_only = vk_image_layout_is_read_only(layout, aspect);
3134
3135 const VkImageUsageFlags image_aspect_usage =
3136 anv_image_flags_filter_for_queue(
3137 vk_image_usage(&image->vk, aspect), queue_flags);
3138 const VkImageUsageFlags usage =
3139 vk_image_layout_to_usage_flags(layout, aspect) & image_aspect_usage;
3140
3141 bool aux_supported = true;
3142 bool clear_supported = isl_aux_usage_has_fast_clears(aux_usage);
3143
3144 if ((usage & (VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT |
3145 VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT)) &&
3146 !read_only) {
3147 /* This image could be used as both an input attachment and a render
3148 * target (depth, stencil, or color) at the same time and this can cause
3149 * corruption.
3150 *
3151 * We currently only disable aux in this way for depth even though we
3152 * disable it for color in GL.
3153 *
3154 * TODO: Should we be disabling this in more cases?
3155 */
3156 if (aspect == VK_IMAGE_ASPECT_DEPTH_BIT && devinfo->ver <= 9) {
3157 aux_supported = false;
3158 clear_supported = false;
3159 }
3160 }
3161
3162 if (usage & (VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
3163 VK_IMAGE_USAGE_SAMPLED_BIT |
3164 VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT)) {
3165 switch (aux_usage) {
3166 case ISL_AUX_USAGE_HIZ:
3167 if (!anv_can_sample_with_hiz(devinfo, image)) {
3168 aux_supported = false;
3169 clear_supported = false;
3170 }
3171 break;
3172
3173 case ISL_AUX_USAGE_HIZ_CCS:
3174 aux_supported = false;
3175 clear_supported = false;
3176 break;
3177
3178 case ISL_AUX_USAGE_HIZ_CCS_WT:
3179 break;
3180
3181 case ISL_AUX_USAGE_CCS_D:
3182 aux_supported = false;
3183 clear_supported = false;
3184 break;
3185
3186 case ISL_AUX_USAGE_MCS:
3187 case ISL_AUX_USAGE_MCS_CCS:
3188 if (!anv_can_sample_mcs_with_clear(devinfo, image))
3189 clear_supported = false;
3190 break;
3191
3192 case ISL_AUX_USAGE_CCS_E:
3193 case ISL_AUX_USAGE_FCV_CCS_E:
3194 case ISL_AUX_USAGE_STC_CCS:
3195 break;
3196
3197 default:
3198 unreachable("Unsupported aux usage");
3199 }
3200 }
3201
3202 switch (aux_usage) {
3203 case ISL_AUX_USAGE_HIZ:
3204 case ISL_AUX_USAGE_HIZ_CCS:
3205 case ISL_AUX_USAGE_HIZ_CCS_WT:
3206 if (aux_supported) {
3207 assert(clear_supported);
3208 return ISL_AUX_STATE_COMPRESSED_CLEAR;
3209 } else if (read_only) {
3210 return ISL_AUX_STATE_RESOLVED;
3211 } else {
3212 return ISL_AUX_STATE_AUX_INVALID;
3213 }
3214
3215 case ISL_AUX_USAGE_CCS_D:
3216 /* We only support clear in exactly one state */
3217 if (layout == VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL ||
3218 layout == VK_IMAGE_LAYOUT_ATTACHMENT_OPTIMAL) {
3219 assert(aux_supported);
3220 assert(clear_supported);
3221 return ISL_AUX_STATE_PARTIAL_CLEAR;
3222 } else {
3223 return ISL_AUX_STATE_PASS_THROUGH;
3224 }
3225
3226 case ISL_AUX_USAGE_CCS_E:
3227 case ISL_AUX_USAGE_FCV_CCS_E:
3228 if (aux_supported) {
3229 assert(clear_supported);
3230 return ISL_AUX_STATE_COMPRESSED_CLEAR;
3231 } else {
3232 return ISL_AUX_STATE_PASS_THROUGH;
3233 }
3234
3235 case ISL_AUX_USAGE_MCS:
3236 case ISL_AUX_USAGE_MCS_CCS:
3237 assert(aux_supported);
3238 if (clear_supported) {
3239 return ISL_AUX_STATE_COMPRESSED_CLEAR;
3240 } else {
3241 return ISL_AUX_STATE_COMPRESSED_NO_CLEAR;
3242 }
3243
3244 case ISL_AUX_USAGE_STC_CCS:
3245 assert(aux_supported);
3246 assert(!clear_supported);
3247 return ISL_AUX_STATE_COMPRESSED_NO_CLEAR;
3248
3249 default:
3250 unreachable("Unsupported aux usage");
3251 }
3252 }
3253
3254 /**
3255 * This function determines the optimal buffer to use for a given
3256 * VkImageLayout and other pieces of information needed to make that
3257 * determination. This does not determine the optimal buffer to use
3258 * during a resolve operation.
3259 *
3260 * @param devinfo The device information of the Intel GPU.
3261 * @param image The image that may contain a collection of buffers.
3262 * @param aspect The aspect of the image to be accessed.
3263 * @param usage The usage which describes how the image will be accessed.
3264 * @param layout The current layout of the image aspect(s).
3265 *
3266 * @return The primary buffer that should be used for the given layout.
3267 */
3268 enum isl_aux_usage ATTRIBUTE_PURE
anv_layout_to_aux_usage(const struct intel_device_info * const devinfo,const struct anv_image * const image,const VkImageAspectFlagBits aspect,const VkImageUsageFlagBits usage,const VkImageLayout layout,const VkQueueFlagBits queue_flags)3269 anv_layout_to_aux_usage(const struct intel_device_info * const devinfo,
3270 const struct anv_image * const image,
3271 const VkImageAspectFlagBits aspect,
3272 const VkImageUsageFlagBits usage,
3273 const VkImageLayout layout,
3274 const VkQueueFlagBits queue_flags)
3275 {
3276 const uint32_t plane = anv_image_aspect_to_plane(image, aspect);
3277
3278 /* If there is no auxiliary surface allocated, we must use the one and only
3279 * main buffer.
3280 */
3281 if (image->planes[plane].aux_usage == ISL_AUX_USAGE_NONE)
3282 return ISL_AUX_USAGE_NONE;
3283
3284 enum isl_aux_state aux_state =
3285 anv_layout_to_aux_state(devinfo, image, aspect, layout, queue_flags);
3286
3287 switch (aux_state) {
3288 case ISL_AUX_STATE_CLEAR:
3289 unreachable("We never use this state");
3290
3291 case ISL_AUX_STATE_PARTIAL_CLEAR:
3292 assert(image->vk.aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
3293 assert(image->planes[plane].aux_usage == ISL_AUX_USAGE_CCS_D);
3294 assert(image->vk.samples == 1);
3295 return ISL_AUX_USAGE_CCS_D;
3296
3297 case ISL_AUX_STATE_COMPRESSED_CLEAR:
3298 case ISL_AUX_STATE_COMPRESSED_NO_CLEAR:
3299 return image->planes[plane].aux_usage;
3300
3301 case ISL_AUX_STATE_RESOLVED:
3302 /* We can only use RESOLVED in read-only layouts because any write will
3303 * either land us in AUX_INVALID or COMPRESSED_NO_CLEAR. We can do
3304 * writes in PASS_THROUGH without destroying it so that is allowed.
3305 */
3306 assert(vk_image_layout_is_read_only(layout, aspect));
3307 assert(util_is_power_of_two_or_zero(usage));
3308 if (usage == VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT) {
3309 /* If we have valid HiZ data and are using the image as a read-only
3310 * depth/stencil attachment, we should enable HiZ so that we can get
3311 * faster depth testing.
3312 */
3313 return image->planes[plane].aux_usage;
3314 } else {
3315 return ISL_AUX_USAGE_NONE;
3316 }
3317
3318 case ISL_AUX_STATE_PASS_THROUGH:
3319 case ISL_AUX_STATE_AUX_INVALID:
3320 return ISL_AUX_USAGE_NONE;
3321 }
3322
3323 unreachable("Invalid isl_aux_state");
3324 }
3325
3326 /**
3327 * This function returns the level of unresolved fast-clear support of the
3328 * given image in the given VkImageLayout.
3329 *
3330 * @param devinfo The device information of the Intel GPU.
3331 * @param image The image that may contain a collection of buffers.
3332 * @param aspect The aspect of the image to be accessed.
3333 * @param usage The usage which describes how the image will be accessed.
3334 * @param layout The current layout of the image aspect(s).
3335 */
3336 enum anv_fast_clear_type ATTRIBUTE_PURE
anv_layout_to_fast_clear_type(const struct intel_device_info * const devinfo,const struct anv_image * const image,const VkImageAspectFlagBits aspect,const VkImageLayout layout,const VkQueueFlagBits queue_flags)3337 anv_layout_to_fast_clear_type(const struct intel_device_info * const devinfo,
3338 const struct anv_image * const image,
3339 const VkImageAspectFlagBits aspect,
3340 const VkImageLayout layout,
3341 const VkQueueFlagBits queue_flags)
3342 {
3343 if (INTEL_DEBUG(DEBUG_NO_FAST_CLEAR))
3344 return ANV_FAST_CLEAR_NONE;
3345
3346 const uint32_t plane = anv_image_aspect_to_plane(image, aspect);
3347
3348 /* If there is no auxiliary surface allocated, there are no fast-clears */
3349 if (image->planes[plane].aux_usage == ISL_AUX_USAGE_NONE)
3350 return ANV_FAST_CLEAR_NONE;
3351
3352 /* Xe2+ platforms don't have fast clear type and can always support
3353 * arbitrary fast-clear values.
3354 */
3355 if (devinfo->ver >= 20)
3356 return ANV_FAST_CLEAR_ANY;
3357
3358 enum isl_aux_state aux_state =
3359 anv_layout_to_aux_state(devinfo, image, aspect, layout, queue_flags);
3360
3361 const VkImageUsageFlags layout_usage =
3362 vk_image_layout_to_usage_flags(layout, aspect) & image->vk.usage;
3363
3364 const struct isl_drm_modifier_info *isl_mod_info =
3365 image->vk.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT ?
3366 isl_drm_modifier_get_info(image->vk.drm_format_mod) : NULL;
3367
3368 switch (aux_state) {
3369 case ISL_AUX_STATE_CLEAR:
3370 unreachable("We never use this state");
3371
3372 case ISL_AUX_STATE_PARTIAL_CLEAR:
3373 case ISL_AUX_STATE_COMPRESSED_CLEAR:
3374
3375 /* Generally, enabling non-zero fast-clears is dependent on knowing which
3376 * formats will be used with the surface. So, disable them if we lack
3377 * this knowledge.
3378 *
3379 * For dmabufs with clear color modifiers, we already restrict
3380 * problematic accesses for the clear color during the negotiation
3381 * phase. So, don't restrict clear color support in this case.
3382 */
3383 if (anv_image_view_formats_incomplete(image) &&
3384 !(isl_mod_info && isl_mod_info->supports_clear_color)) {
3385 return ANV_FAST_CLEAR_DEFAULT_VALUE;
3386 }
3387
3388 /* On gfx12, the FCV feature may convert a block of fragment shader
3389 * outputs to fast-clears. If this image has multiple subresources,
3390 * restrict the clear color to zero to keep the fast cleared blocks in
3391 * sync.
3392 */
3393 if (image->planes[plane].aux_usage == ISL_AUX_USAGE_FCV_CCS_E &&
3394 (image->vk.mip_levels > 1 ||
3395 image->vk.array_layers > 1 ||
3396 image->vk.extent.depth > 1)) {
3397 return ANV_FAST_CLEAR_DEFAULT_VALUE;
3398 }
3399
3400 /* On gfx9, we only load clear colors for attachments and for BLORP
3401 * surfaces. Outside of those surfaces, we can only support the default
3402 * clear value of zero.
3403 */
3404 if (devinfo->ver == 9 &&
3405 (layout_usage & (VK_IMAGE_USAGE_SAMPLED_BIT |
3406 VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT))) {
3407 return ANV_FAST_CLEAR_DEFAULT_VALUE;
3408 }
3409
3410 return ANV_FAST_CLEAR_ANY;
3411
3412 case ISL_AUX_STATE_COMPRESSED_NO_CLEAR:
3413 case ISL_AUX_STATE_RESOLVED:
3414 case ISL_AUX_STATE_PASS_THROUGH:
3415 case ISL_AUX_STATE_AUX_INVALID:
3416 return ANV_FAST_CLEAR_NONE;
3417 }
3418
3419 unreachable("Invalid isl_aux_state");
3420 }
3421
3422 bool
anv_can_fast_clear_color(const struct anv_cmd_buffer * cmd_buffer,const struct anv_image * image,unsigned level,const struct VkClearRect * clear_rect,VkImageLayout layout,enum isl_format view_format,union isl_color_value clear_color)3423 anv_can_fast_clear_color(const struct anv_cmd_buffer *cmd_buffer,
3424 const struct anv_image *image,
3425 unsigned level,
3426 const struct VkClearRect *clear_rect,
3427 VkImageLayout layout,
3428 enum isl_format view_format,
3429 union isl_color_value clear_color)
3430 {
3431 if (INTEL_DEBUG(DEBUG_NO_FAST_CLEAR))
3432 return false;
3433
3434 /* We only have fast-clears implemented for the render engine. */
3435 if (cmd_buffer->queue_family->engine_class != INTEL_ENGINE_CLASS_RENDER)
3436 return false;
3437
3438 /* Start by getting the fast clear type. We use the first subpass
3439 * layout here because we don't want to fast-clear if the first subpass
3440 * to use the attachment can't handle fast-clears.
3441 */
3442 enum anv_fast_clear_type fast_clear_type =
3443 anv_layout_to_fast_clear_type(cmd_buffer->device->info, image,
3444 VK_IMAGE_ASPECT_COLOR_BIT, layout,
3445 cmd_buffer->queue_family->queueFlags);
3446 switch (fast_clear_type) {
3447 case ANV_FAST_CLEAR_NONE:
3448 return false;
3449 case ANV_FAST_CLEAR_DEFAULT_VALUE:
3450 if (!isl_color_value_is_zero(clear_color, view_format))
3451 return false;
3452 break;
3453 case ANV_FAST_CLEAR_ANY:
3454 break;
3455 }
3456
3457 /* Potentially, we could do partial fast-clears but doing so has crazy
3458 * alignment restrictions. It's easier to just restrict to full size
3459 * fast clears for now.
3460 */
3461 if (clear_rect->rect.offset.x != 0 ||
3462 clear_rect->rect.offset.y != 0 ||
3463 clear_rect->rect.extent.width != image->vk.extent.width ||
3464 clear_rect->rect.extent.height != image->vk.extent.height)
3465 return false;
3466
3467 /* We only allow fast clears to the first slice of an image (level 0,
3468 * layer 0) and only for the entire slice. This guarantees us that, at
3469 * any given time, there is only one clear color on any given image at
3470 * any given time. At the time of our testing (Jan 17, 2018), there
3471 * were no known applications which would benefit from fast-clearing
3472 * more than just the first slice.
3473 */
3474 if (level > 0) {
3475 anv_perf_warn(VK_LOG_OBJS(&image->vk.base),
3476 "level > 0. Not fast clearing.");
3477 return false;
3478 }
3479
3480 if (clear_rect->baseArrayLayer > 0) {
3481 anv_perf_warn(VK_LOG_OBJS(&image->vk.base),
3482 "baseArrayLayer > 0. Not fast clearing.");
3483 return false;
3484 }
3485
3486
3487 if (clear_rect->layerCount > 1) {
3488 anv_perf_warn(VK_LOG_OBJS(&image->vk.base),
3489 "layerCount > 1. Only fast-clearing the first slice");
3490 }
3491
3492 /* Wa_18020603990 - slow clear surfaces up to 256x256, 32bpp. */
3493 if (intel_needs_workaround(cmd_buffer->device->info, 18020603990)) {
3494 const struct anv_surface *anv_surf = &image->planes->primary_surface;
3495 if (isl_format_get_layout(anv_surf->isl.format)->bpb <= 32 &&
3496 anv_surf->isl.logical_level0_px.w <= 256 &&
3497 anv_surf->isl.logical_level0_px.h <= 256)
3498 return false;
3499 }
3500
3501 /* On gfx12.0, CCS fast clears don't seem to cover the correct portion of
3502 * the aux buffer when the pitch is not 512B-aligned.
3503 */
3504 if (cmd_buffer->device->info->verx10 == 120 &&
3505 image->planes->primary_surface.isl.samples == 1 &&
3506 image->planes->primary_surface.isl.row_pitch_B % 512) {
3507 anv_perf_warn(VK_LOG_OBJS(&image->vk.base),
3508 "Pitch not 512B-aligned. Slow clearing surface.");
3509 return false;
3510 }
3511
3512 /* Wa_16021232440: Disable fast clear when height is 16k */
3513 if (intel_needs_workaround(cmd_buffer->device->info, 16021232440) &&
3514 image->vk.extent.height == 16 * 1024) {
3515 return false;
3516 }
3517
3518 /* The fast-clear preamble and/or postamble flushes are more expensive than
3519 * the flushes performed by BLORP during slow clears. Use a heuristic to
3520 * determine if the cost of the flushes are worth fast-clearing. See
3521 * genX(cmd_buffer_update_color_aux_op)() and blorp_exec_on_render().
3522 * TODO: Tune for Xe2
3523 */
3524 if (cmd_buffer->device->info->verx10 <= 125 &&
3525 cmd_buffer->num_independent_clears >= 16 &&
3526 cmd_buffer->num_independent_clears >
3527 cmd_buffer->num_dependent_clears * 2) {
3528 anv_perf_warn(VK_LOG_OBJS(&image->vk.base),
3529 "Not enough back-to-back fast-clears. Slow clearing.");
3530 return false;
3531 }
3532
3533 return true;
3534 }
3535
3536 /**
3537 * This function determines if the layout & usage of an image can have
3538 * untracked aux writes. When we see a transition that matches this criteria,
3539 * we need to mark the image as compressed written so that our predicated
3540 * resolves work properly.
3541 *
3542 * @param devinfo The device information of the Intel GPU.
3543 * @param image The image that may contain a collection of buffers.
3544 * @param aspect The aspect of the image to be accessed.
3545 * @param layout The current layout of the image aspect(s).
3546 */
3547 bool
anv_layout_has_untracked_aux_writes(const struct intel_device_info * const devinfo,const struct anv_image * const image,const VkImageAspectFlagBits aspect,const VkImageLayout layout,const VkQueueFlagBits queue_flags)3548 anv_layout_has_untracked_aux_writes(const struct intel_device_info * const devinfo,
3549 const struct anv_image * const image,
3550 const VkImageAspectFlagBits aspect,
3551 const VkImageLayout layout,
3552 const VkQueueFlagBits queue_flags)
3553 {
3554 const VkImageUsageFlags image_aspect_usage =
3555 vk_image_usage(&image->vk, aspect);
3556 const VkImageUsageFlags usage =
3557 vk_image_layout_to_usage_flags(layout, aspect) & image_aspect_usage;
3558
3559 /* Storage is the only usage where we do not write the image through a
3560 * render target but through a descriptor. Since VK_EXT_descriptor_indexing
3561 * and the update-after-bind feature, it has become impossible to track
3562 * writes to images in descriptor at the command buffer build time. So it's
3563 * not possible to mark an image as compressed like we do in
3564 * genX_cmd_buffer.c(EndRendering) or anv_blorp.c for all transfer
3565 * operations.
3566 */
3567 if (!(usage & VK_IMAGE_USAGE_STORAGE_BIT))
3568 return false;
3569
3570 /* No AUX, no writes to the AUX surface :) */
3571 const uint32_t plane = anv_image_aspect_to_plane(image, aspect);
3572 const enum isl_aux_usage aux_usage = image->planes[plane].aux_usage;
3573 if (aux_usage == ISL_AUX_USAGE_NONE)
3574 return false;
3575
3576 return true;
3577 }
3578
anv_GetRenderingAreaGranularityKHR(VkDevice _device,const VkRenderingAreaInfoKHR * pRenderingAreaInfo,VkExtent2D * pGranularity)3579 void anv_GetRenderingAreaGranularityKHR(
3580 VkDevice _device,
3581 const VkRenderingAreaInfoKHR* pRenderingAreaInfo,
3582 VkExtent2D* pGranularity)
3583 {
3584 *pGranularity = (VkExtent2D) {
3585 .width = 1,
3586 .height = 1,
3587 };
3588 }
3589