• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright © 2015 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  */
23 
24 #include <assert.h>
25 #include <stdbool.h>
26 #include <string.h>
27 #include <unistd.h>
28 #include <fcntl.h>
29 #include <sys/mman.h>
30 #include "drm-uapi/drm_fourcc.h"
31 
32 #include "anv_private.h"
33 #include "common/intel_aux_map.h"
34 #include "util/u_debug.h"
35 #include "vk_util.h"
36 #include "util/u_math.h"
37 
38 #include "vk_format.h"
39 #include "av1_tables.h"
40 
41 #define ANV_OFFSET_IMPLICIT UINT64_MAX
42 
43 static const enum isl_surf_dim
44 vk_to_isl_surf_dim[] = {
45    [VK_IMAGE_TYPE_1D] = ISL_SURF_DIM_1D,
46    [VK_IMAGE_TYPE_2D] = ISL_SURF_DIM_2D,
47    [VK_IMAGE_TYPE_3D] = ISL_SURF_DIM_3D,
48 };
49 
50 static uint64_t MUST_CHECK UNUSED
memory_range_end(struct anv_image_memory_range memory_range)51 memory_range_end(struct anv_image_memory_range memory_range)
52 {
53    assert(anv_is_aligned(memory_range.offset, memory_range.alignment));
54    return memory_range.offset + memory_range.size;
55 }
56 
57 /**
58  * Get binding for VkImagePlaneMemoryRequirementsInfo,
59  * VkBindImagePlaneMemoryInfo and VkDeviceImageMemoryRequirements.
60  */
61 enum anv_image_memory_binding
anv_image_aspect_to_binding(struct anv_image * image,VkImageAspectFlags aspect)62 anv_image_aspect_to_binding(struct anv_image *image,
63                             VkImageAspectFlags aspect)
64 {
65    uint32_t plane = 0;
66 
67    assert(image->disjoint);
68 
69    if (image->vk.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
70       /* Spec requires special aspects for modifier images. */
71       assert(aspect == VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT ||
72              aspect == VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT ||
73              aspect == VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT ||
74              aspect == VK_IMAGE_ASPECT_MEMORY_PLANE_3_BIT_EXT);
75 
76       /* We don't advertise DISJOINT for modifiers with aux, and therefore we
77        * don't handle queries of the modifier's "aux plane" here.
78        */
79       assert(!isl_drm_modifier_has_aux(image->vk.drm_format_mod));
80 
81       switch(aspect) {
82          case VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT: plane = 0; break;
83          case VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT: plane = 1; break;
84          case VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT: plane = 2; break;
85          case VK_IMAGE_ASPECT_MEMORY_PLANE_3_BIT_EXT: plane = 3; break;
86       }
87    } else {
88       plane = anv_image_aspect_to_plane(image, aspect);
89    }
90 
91    return ANV_IMAGE_MEMORY_BINDING_PLANE_0 + plane;
92 }
93 
94 /**
95  * Extend the memory binding's range by appending a new memory range with `size`
96  * and `alignment` at `offset`. Return the appended range.
97  *
98  * Offset is ignored if ANV_OFFSET_IMPLICIT.
99  *
100  * The given binding must not be ANV_IMAGE_MEMORY_BINDING_MAIN. The function
101  * converts to MAIN as needed.
102  */
103 static VkResult MUST_CHECK
image_binding_grow(const struct anv_device * device,struct anv_image * image,enum anv_image_memory_binding binding,uint64_t offset,uint64_t size,uint32_t alignment,struct anv_image_memory_range * out_range)104 image_binding_grow(const struct anv_device *device,
105                    struct anv_image *image,
106                    enum anv_image_memory_binding binding,
107                    uint64_t offset,
108                    uint64_t size,
109                    uint32_t alignment,
110                    struct anv_image_memory_range *out_range)
111 {
112    /* We overwrite 'offset' but need to remember if it was implicit. */
113    const bool has_implicit_offset = (offset == ANV_OFFSET_IMPLICIT);
114 
115    assert(size > 0);
116    assert(util_is_power_of_two_or_zero(alignment));
117 
118    switch (binding) {
119    case ANV_IMAGE_MEMORY_BINDING_MAIN:
120       /* The caller must not pre-translate BINDING_PLANE_i to BINDING_MAIN. */
121       unreachable("ANV_IMAGE_MEMORY_BINDING_MAIN");
122    case ANV_IMAGE_MEMORY_BINDING_PLANE_0:
123    case ANV_IMAGE_MEMORY_BINDING_PLANE_1:
124    case ANV_IMAGE_MEMORY_BINDING_PLANE_2:
125       if (!image->disjoint)
126          binding = ANV_IMAGE_MEMORY_BINDING_MAIN;
127       break;
128    case ANV_IMAGE_MEMORY_BINDING_PRIVATE:
129       assert(offset == ANV_OFFSET_IMPLICIT);
130       break;
131    case ANV_IMAGE_MEMORY_BINDING_END:
132       unreachable("ANV_IMAGE_MEMORY_BINDING_END");
133    }
134 
135    struct anv_image_memory_range *container =
136       &image->bindings[binding].memory_range;
137 
138    if (has_implicit_offset) {
139       offset = align64(container->offset + container->size, alignment);
140    } else {
141       /* Offset must be validated because it comes from
142        * VkImageDrmFormatModifierExplicitCreateInfoEXT.
143        */
144       if (unlikely(!anv_is_aligned(offset, alignment))) {
145          return vk_errorf(device,
146                           VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT,
147                           "VkImageDrmFormatModifierExplicitCreateInfoEXT::"
148                           "pPlaneLayouts[]::offset is misaligned");
149       }
150    }
151 
152    /* Surfaces can be added out of memory-order. Track the end of each memory
153     * plane to update the binding size properly.
154     */
155    uint64_t memory_range_end;
156    if (__builtin_add_overflow(offset, size, &memory_range_end)) {
157       if (has_implicit_offset) {
158          assert(!"overflow");
159          return vk_errorf(device, VK_ERROR_UNKNOWN,
160                           "internal error: overflow in %s", __func__);
161       } else {
162          return vk_errorf(device,
163                           VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT,
164                           "VkImageDrmFormatModifierExplicitCreateInfoEXT::"
165                           "pPlaneLayouts[]::offset is too large");
166       }
167    }
168 
169    container->size = MAX2(container->size, memory_range_end);
170    container->alignment = MAX2(container->alignment, alignment);
171 
172    *out_range = (struct anv_image_memory_range) {
173       .binding = binding,
174       .alignment = alignment,
175       .size = size,
176       .offset = offset,
177    };
178 
179    return VK_SUCCESS;
180 }
181 
182 /**
183  * Adjust range 'a' to contain range 'b'.
184  *
185  * For simplicity's sake, the offset of 'a' must be 0 and remains 0.
186  * If 'a' and 'b' target different bindings, then no merge occurs.
187  */
188 static void
memory_range_merge(struct anv_image_memory_range * a,const struct anv_image_memory_range b)189 memory_range_merge(struct anv_image_memory_range *a,
190                    const struct anv_image_memory_range b)
191 {
192    if (b.size == 0)
193       return;
194 
195    if (a->binding != b.binding)
196       return;
197 
198    assert(a->offset == 0);
199    assert(anv_is_aligned(a->offset, a->alignment));
200    assert(anv_is_aligned(b.offset, b.alignment));
201 
202    a->alignment = MAX2(a->alignment, b.alignment);
203    a->size = MAX2(a->size, b.offset + b.size);
204 }
205 
206 isl_surf_usage_flags_t
anv_image_choose_isl_surf_usage(struct anv_physical_device * device,VkImageCreateFlags vk_create_flags,VkImageUsageFlags vk_usage,isl_surf_usage_flags_t isl_extra_usage,VkImageAspectFlagBits aspect,VkImageCompressionFlagsEXT comp_flags)207 anv_image_choose_isl_surf_usage(struct anv_physical_device *device,
208                                 VkImageCreateFlags vk_create_flags,
209                                 VkImageUsageFlags vk_usage,
210                                 isl_surf_usage_flags_t isl_extra_usage,
211                                 VkImageAspectFlagBits aspect,
212                                 VkImageCompressionFlagsEXT comp_flags)
213 {
214    isl_surf_usage_flags_t isl_usage = isl_extra_usage;
215 
216    /* On platform like MTL, we choose to allocate additional CCS memory at the
217     * back of the VkDeviceMemory objects since different images can share the
218     * AUX-TT PTE because the HW doesn't care about the image format in the
219     * PTE. That means we can always ignore the AUX-TT alignment requirement
220     * from an ISL point of view.
221     */
222    if (device->alloc_aux_tt_mem)
223       isl_usage |= ISL_SURF_USAGE_NO_AUX_TT_ALIGNMENT_BIT;
224 
225    if (vk_usage & VK_IMAGE_USAGE_SAMPLED_BIT)
226       isl_usage |= ISL_SURF_USAGE_TEXTURE_BIT;
227 
228    if (vk_usage & VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT)
229       isl_usage |= ISL_SURF_USAGE_TEXTURE_BIT;
230 
231    if (vk_usage & VK_IMAGE_USAGE_STORAGE_BIT)
232       isl_usage |= ISL_SURF_USAGE_STORAGE_BIT;
233 
234    if (vk_usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT)
235       isl_usage |= ISL_SURF_USAGE_RENDER_TARGET_BIT;
236 
237    if (vk_usage & VK_IMAGE_USAGE_FRAGMENT_SHADING_RATE_ATTACHMENT_BIT_KHR)
238       isl_usage |= ISL_SURF_USAGE_CPB_BIT;
239 
240    /* TODO: consider whether compression with sparse is workable. */
241    if (vk_create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT)
242       isl_usage |= ISL_SURF_USAGE_SPARSE_BIT |
243                    ISL_SURF_USAGE_DISABLE_AUX_BIT;
244 
245    if (vk_usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR ||
246        vk_usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR ||
247        vk_usage & VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR ||
248        vk_usage & VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR)
249       isl_usage |= ISL_SURF_USAGE_VIDEO_DECODE_BIT;
250 
251    /* We disable aux surfaces for host read/write images so that we can update
252     * the main surface without caring about the auxiliary surface.
253     */
254    if (vk_usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT)
255       isl_usage |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
256 
257    if (vk_create_flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT)
258       isl_usage |= ISL_SURF_USAGE_CUBE_BIT;
259 
260    if (vk_create_flags & (VK_IMAGE_CREATE_2D_VIEW_COMPATIBLE_BIT_EXT |
261                           VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT))
262       isl_usage |= ISL_SURF_USAGE_2D_3D_COMPATIBLE_BIT;
263 
264    if (vk_create_flags & VK_IMAGE_CREATE_PROTECTED_BIT)
265       isl_usage |= ISL_SURF_USAGE_PROTECTED_BIT;
266 
267    /* Even if we're only using it for transfer operations, clears to depth and
268     * stencil images happen as depth and stencil so they need the right ISL
269     * usage bits or else things will fall apart.
270     */
271    switch (aspect) {
272    case VK_IMAGE_ASPECT_DEPTH_BIT:
273       isl_usage |= ISL_SURF_USAGE_DEPTH_BIT;
274       break;
275    case VK_IMAGE_ASPECT_STENCIL_BIT:
276       isl_usage |= ISL_SURF_USAGE_STENCIL_BIT;
277       break;
278    case VK_IMAGE_ASPECT_COLOR_BIT:
279    case VK_IMAGE_ASPECT_PLANE_0_BIT:
280    case VK_IMAGE_ASPECT_PLANE_1_BIT:
281    case VK_IMAGE_ASPECT_PLANE_2_BIT:
282       break;
283    default:
284       unreachable("bad VkImageAspect");
285    }
286 
287    if (vk_usage & VK_IMAGE_USAGE_TRANSFER_SRC_BIT) {
288       /* blorp implements transfers by sampling from the source image. */
289       isl_usage |= ISL_SURF_USAGE_TEXTURE_BIT;
290    }
291 
292    if (vk_usage & VK_IMAGE_USAGE_TRANSFER_DST_BIT &&
293        aspect == VK_IMAGE_ASPECT_COLOR_BIT) {
294       /* blorp implements transfers by rendering into the destination image.
295        * Only request this with color images, as we deal with depth/stencil
296        * formats differently. */
297       isl_usage |= ISL_SURF_USAGE_RENDER_TARGET_BIT;
298    }
299 
300    if (comp_flags & VK_IMAGE_COMPRESSION_DISABLED_EXT)
301       isl_usage |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
302 
303    return isl_usage;
304 }
305 
306 static isl_tiling_flags_t
choose_isl_tiling_flags(const struct intel_device_info * devinfo,const struct anv_image * image,const struct anv_image_create_info * anv_info,const struct isl_drm_modifier_info * isl_mod_info)307 choose_isl_tiling_flags(const struct intel_device_info *devinfo,
308                         const struct anv_image *image,
309                         const struct anv_image_create_info *anv_info,
310                         const struct isl_drm_modifier_info *isl_mod_info)
311 {
312    const VkImageCreateInfo *base_info = anv_info->vk_info;
313    isl_tiling_flags_t flags = 0;
314 
315    assert((isl_mod_info != NULL) ==
316           (base_info->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT));
317 
318    switch (base_info->tiling) {
319    default:
320       unreachable("bad VkImageTiling");
321    case VK_IMAGE_TILING_OPTIMAL:
322       flags = ISL_TILING_ANY_MASK;
323       break;
324    case VK_IMAGE_TILING_LINEAR:
325       flags = ISL_TILING_LINEAR_BIT;
326       break;
327    case VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT:
328       flags = 1 << isl_mod_info->tiling;
329    }
330 
331    if (anv_info->isl_tiling_flags) {
332       assert(isl_mod_info == NULL);
333       flags &= anv_info->isl_tiling_flags;
334    }
335 
336    if (image->vk.wsi_legacy_scanout) {
337       isl_tiling_flags_t legacy_mask = ISL_TILING_LINEAR_BIT;
338       if (devinfo->has_tiling_uapi)
339          legacy_mask |= ISL_TILING_X_BIT;
340       flags &= legacy_mask;
341    }
342 
343    assert(flags);
344 
345    return flags;
346 }
347 
348 /**
349  * Add the surface to the binding at the given offset.
350  *
351  * \see image_binding_grow()
352  */
353 static VkResult MUST_CHECK
add_surface(struct anv_device * device,struct anv_image * image,struct anv_surface * surf,enum anv_image_memory_binding binding,uint64_t offset)354 add_surface(struct anv_device *device,
355             struct anv_image *image,
356             struct anv_surface *surf,
357             enum anv_image_memory_binding binding,
358             uint64_t offset)
359 {
360    /* isl surface must be initialized */
361    assert(surf->isl.size_B > 0);
362 
363    return image_binding_grow(device, image, binding, offset,
364                              surf->isl.size_B,
365                              surf->isl.alignment_B,
366                              &surf->memory_range);
367 }
368 
369 static bool
image_may_use_r32_view(VkImageCreateFlags create_flags,VkFormat vk_format,const VkImageFormatListCreateInfo * fmt_list)370 image_may_use_r32_view(VkImageCreateFlags create_flags,
371                        VkFormat vk_format,
372                        const VkImageFormatListCreateInfo *fmt_list)
373 {
374    if (vk_format == VK_FORMAT_R32_SINT ||
375        vk_format == VK_FORMAT_R32_UINT ||
376        vk_format == VK_FORMAT_R32_SFLOAT)
377       return true;
378 
379    if (!(create_flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT))
380       return false;
381 
382    if (fmt_list) {
383       for (uint32_t i = 0; i < fmt_list->viewFormatCount; i++) {
384          if (fmt_list->pViewFormats[i] == VK_FORMAT_R32_SINT ||
385              fmt_list->pViewFormats[i] == VK_FORMAT_R32_UINT ||
386              fmt_list->pViewFormats[i] == VK_FORMAT_R32_SFLOAT)
387             return true;
388       }
389 
390       return false;
391    }
392 
393    return vk_format_get_blocksizebits(vk_format) == 32;
394 }
395 
396 static bool
formats_ccs_e_compatible(const struct intel_device_info * devinfo,VkImageCreateFlags create_flags,VkImageAspectFlagBits aspect,enum isl_format format,VkImageTiling vk_tiling,const VkImageFormatListCreateInfo * fmt_list)397 formats_ccs_e_compatible(const struct intel_device_info *devinfo,
398                          VkImageCreateFlags create_flags,
399                          VkImageAspectFlagBits aspect,
400                          enum isl_format format, VkImageTiling vk_tiling,
401                          const VkImageFormatListCreateInfo *fmt_list)
402 {
403    if (!anv_format_supports_ccs_e(devinfo, format))
404       return false;
405 
406    /* For images created without MUTABLE_FORMAT_BIT set, we know that they will
407     * always be used with the original format. In particular, they will always
408     * be used with a format that supports color compression.
409     */
410    if (!(create_flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT))
411       return true;
412 
413    if (!fmt_list || fmt_list->viewFormatCount == 0)
414       return false;
415 
416    for (uint32_t i = 0; i < fmt_list->viewFormatCount; i++) {
417       if (fmt_list->pViewFormats[i] == VK_FORMAT_UNDEFINED)
418          continue;
419 
420       enum isl_format view_format =
421          anv_get_isl_format(devinfo, fmt_list->pViewFormats[i],
422                             aspect, vk_tiling);
423 
424       if (!isl_formats_are_ccs_e_compatible(devinfo, format, view_format))
425          return false;
426    }
427 
428    return true;
429 }
430 
431 bool
anv_format_supports_ccs_e(const struct intel_device_info * devinfo,const enum isl_format format)432 anv_format_supports_ccs_e(const struct intel_device_info *devinfo,
433                           const enum isl_format format)
434 {
435    /* CCS_E for YCRCB_NORMAL and YCRCB_SWAP_UV is not currently supported by
436     * ANV so leave it disabled for now.
437     */
438    if (isl_format_is_yuv(format))
439       return false;
440 
441    return isl_format_supports_ccs_e(devinfo, format);
442 }
443 
444 bool
anv_formats_ccs_e_compatible(const struct intel_device_info * devinfo,VkImageCreateFlags create_flags,VkFormat vk_format,VkImageTiling vk_tiling,VkImageUsageFlags vk_usage,const VkImageFormatListCreateInfo * fmt_list)445 anv_formats_ccs_e_compatible(const struct intel_device_info *devinfo,
446                              VkImageCreateFlags create_flags,
447                              VkFormat vk_format, VkImageTiling vk_tiling,
448                              VkImageUsageFlags vk_usage,
449                              const VkImageFormatListCreateInfo *fmt_list)
450 {
451    u_foreach_bit(b, vk_format_aspects(vk_format)) {
452       VkImageAspectFlagBits aspect = 1 << b;
453       enum isl_format format =
454          anv_get_isl_format(devinfo, vk_format, aspect, vk_tiling);
455 
456       if (!formats_ccs_e_compatible(devinfo, create_flags, aspect,
457                                     format, vk_tiling, fmt_list))
458          return false;
459    }
460 
461    if ((vk_usage & VK_IMAGE_USAGE_STORAGE_BIT) &&
462        vk_tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
463       assert(vk_format_aspects(vk_format) == VK_IMAGE_ASPECT_COLOR_BIT);
464       if (devinfo->ver == 12) {
465          /* From the TGL Bspec 44930 (r47128):
466           *
467           *    "Memory atomic operation on compressed data is not supported
468           *     in Gen12 E2E compression. Result of such operation is
469           *     undefined.
470           *
471           *     Software should ensure at the time of the Atomic operation
472           *     the surface is resolved (uncompressed) state."
473           *
474           * On gfx12.0, compression is not supported with atomic
475           * operations. On gfx12.5, the support is there, but it's slow
476           * (see HSD 1406337848).
477           */
478          if (image_may_use_r32_view(create_flags, vk_format, fmt_list))
479             return false;
480 
481       } else if (devinfo->ver <= 11) {
482          /* Storage accesses are not supported on compressed surfaces. */
483          return false;
484       }
485    }
486 
487    return true;
488 }
489 
490 /**
491  * For color images that have an auxiliary surface, request allocation for an
492  * additional buffer that mainly stores fast-clear values. Use of this buffer
493  * allows us to access the image's subresources while being aware of their
494  * fast-clear values in non-trivial cases (e.g., outside of a render pass in
495  * which a fast clear has occurred).
496  *
497  * In order to avoid having multiple clear colors for a single plane of an
498  * image (hence a single RENDER_SURFACE_STATE), we only allow fast-clears on
499  * the first slice (level 0, layer 0).  At the time of our testing (Jan 17,
500  * 2018), there were no known applications which would benefit from fast-
501  * clearing more than just the first slice.
502  *
503  * The fast clear portion of the image is laid out in the following order:
504  *
505  *  * 1 or 4 dwords (depending on hardware generation) for the clear color
506  *  * 1 dword for the anv_fast_clear_type of the clear color
507  *  * On gfx9+, 1 dword per level and layer of the image (3D levels count
508  *    multiple layers) in level-major order for compression state.
509  *
510  * For the purpose of discoverability, the algorithm used to manage
511  * compression and fast-clears is described here:
512  *
513  *  * On a transition from UNDEFINED or PREINITIALIZED to a defined layout,
514  *    all of the values in the fast clear portion of the image are initialized
515  *    to default values.
516  *
517  *  * On fast-clear, the clear value is written into surface state and also
518  *    into the buffer and the fast clear type is set appropriately.  Both
519  *    setting the fast-clear value in the buffer and setting the fast-clear
520  *    type happen from the GPU using MI commands.
521  *
522  *  * Whenever a render or blorp operation is performed with CCS_E, we call
523  *    genX(cmd_buffer_mark_image_written) to set the compression state to
524  *    true (which is represented by UINT32_MAX).
525  *
526  *  * On pipeline barrier transitions, the worst-case transition is computed
527  *    from the image layouts.  The command streamer inspects the fast clear
528  *    type and compression state dwords and constructs a predicate.  The
529  *    worst-case resolve is performed with the given predicate and the fast
530  *    clear and compression state is set accordingly.
531  *
532  * See anv_layout_to_aux_usage and anv_layout_to_fast_clear_type functions for
533  * details on exactly what is allowed in what layouts.
534  *
535  * On gfx7-9, we do not have a concept of indirect clear colors in hardware.
536  * In order to deal with this, we have to do some clear color management.
537  *
538  *  * For LOAD_OP_LOAD at the top of a renderpass, we have to copy the clear
539  *    value from the buffer into the surface state with MI commands.
540  *
541  *  * For any blorp operations, we pass the address to the clear value into
542  *    blorp and it knows to copy the clear color.
543  */
544 static VkResult MUST_CHECK
add_aux_state_tracking_buffer(struct anv_device * device,struct anv_image * image,uint64_t state_offset,uint32_t plane)545 add_aux_state_tracking_buffer(struct anv_device *device,
546                               struct anv_image *image,
547                               uint64_t state_offset,
548                               uint32_t plane)
549 {
550    assert(image && device);
551    /* Xe2+ platforms don't use aux tracking buffers. We shouldn't get here. */
552    assert(device->info->ver < 20);
553    assert(image->planes[plane].aux_usage != ISL_AUX_USAGE_NONE &&
554           image->vk.aspects & (VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV |
555                                VK_IMAGE_ASPECT_DEPTH_BIT));
556 
557    unsigned clear_color_state_size;
558    if (device->info->ver >= 11) {
559       /* When importing an image from another source with a drm modifier that
560        * supports clear color, the clear color values are in a 32-byte struct
561        * defined in drm_fourcc.h. The fast clear type and compression state
562        * are not defined in these drm_fourcc.h, so there won't be memory
563        * allocated for these extra meta data by the source.
564        *
565        * We use the last 2 dwords of the clear color struct's memory to store
566        * the fast clear type and the first compression state, so the driver
567        * doesn't assume the extra size or need another allocation later.
568        *
569        * So far, the 2 stolen dwords are either not used in the clear color
570        * struct or for features not enabled. There should be no side effect to
571        * the hardware and destinations of images exported by this driver.
572        *
573        * Images with multiple levels or layers are not supported by drm
574        * modifiers, so we don't have to apply the above approach or face a
575        * bigger shortage from multiple compression states. We just apply the
576        * approach to all cases to keep the design unified.
577        *
578        * As a result, the state starts 8 bytes lower than where it should be.
579        */
580       assert(device->isl_dev.ss.clear_color_state_size == 32);
581       clear_color_state_size = (image->num_view_formats - 1) * 64 + 32 - 8;
582    } else {
583       /* When sampling or rendering with an sRGB format, HW expects the clear
584        * color to be in two different color spaces - sRGB in the former and
585        * linear in the latter. Allocate twice the space to support either
586        * access.
587        */
588       assert(device->isl_dev.ss.clear_value_size == 16);
589       clear_color_state_size = image->num_view_formats * 16 * 2;
590    }
591 
592    /* Clear color and fast clear type */
593    unsigned state_size = clear_color_state_size + 4;
594 
595    /* We only need to track compression on CCS_E surfaces. */
596    if (isl_aux_usage_has_ccs_e(image->planes[plane].aux_usage)) {
597       if (image->vk.image_type == VK_IMAGE_TYPE_3D) {
598          for (uint32_t l = 0; l < image->vk.mip_levels; l++)
599             state_size += u_minify(image->vk.extent.depth, l) * 4;
600       } else {
601          state_size += image->vk.mip_levels * image->vk.array_layers * 4;
602       }
603    }
604 
605    enum anv_image_memory_binding binding =
606       ANV_IMAGE_MEMORY_BINDING_PLANE_0 + plane;
607 
608    const struct isl_drm_modifier_info *mod_info =
609       isl_drm_modifier_get_info(image->vk.drm_format_mod);
610 
611    /* If an auxiliary surface is used for an externally-shareable image,
612     * we have to hide this from the memory of the image since other
613     * processes with access to the memory may not be aware of it or of
614     * its current state. So put that auxiliary data into a separate
615     * buffer (ANV_IMAGE_MEMORY_BINDING_PRIVATE).
616     *
617     * But when the image is created with a drm modifier that supports
618     * clear color, it will be exported along with main surface.
619     */
620    if (anv_image_is_externally_shared(image) &&
621        !mod_info->supports_clear_color)
622       binding = ANV_IMAGE_MEMORY_BINDING_PRIVATE;
623 
624    /* The indirect clear color BO requires 64B-alignment on gfx11+. If we're
625     * using a modifier with clear color, then some kernels might require a 4k
626     * alignment.
627     */
628    const uint32_t clear_color_alignment =
629       (mod_info && mod_info->supports_clear_color) ? 4096 : 64;
630 
631    return image_binding_grow(device, image, binding,
632                              state_offset, state_size, clear_color_alignment,
633                              &image->planes[plane].fast_clear_memory_range);
634 }
635 
636 static VkResult MUST_CHECK
add_compression_control_buffer(struct anv_device * device,struct anv_image * image,uint32_t plane,uint32_t binding,uint64_t offset)637 add_compression_control_buffer(struct anv_device *device,
638                                struct anv_image *image,
639                                uint32_t plane,
640                                uint32_t binding,
641                                uint64_t offset)
642 {
643    assert(device->info->has_aux_map);
644 
645    return image_binding_grow(device, image, binding, offset,
646                              image->planes[plane].primary_surface.isl.size_B /
647                              INTEL_AUX_MAP_MAIN_SIZE_SCALEDOWN,
648                              INTEL_AUX_MAP_META_ALIGNMENT_B,
649                              &image->planes[plane].compr_ctrl_memory_range);
650 }
651 
652 static bool
want_hiz_wt_for_image(const struct intel_device_info * devinfo,const struct anv_image * image)653 want_hiz_wt_for_image(const struct intel_device_info *devinfo,
654                       const struct anv_image *image)
655 {
656    /* Gen12 only supports single-sampled while Gen20+ supports
657     * multi-sampled images.
658     */
659    if (devinfo->ver < 20 && image->vk.samples > 1)
660       return false;
661 
662    if ((image->vk.usage & (VK_IMAGE_USAGE_SAMPLED_BIT |
663                            VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT)) == 0)
664       return false;
665 
666    /* If this image has the maximum number of samples supported by
667     * running platform and will be used as a texture, put the HiZ surface
668     * in write-through mode so that we can sample from it.
669     *
670     * TODO: This is a heuristic trade-off; we haven't tuned it at all.
671     */
672    return true;
673 }
674 
675 /**
676  * The return code indicates whether creation of the VkImage should continue
677  * or fail, not whether the creation of the aux surface succeeded.  If the aux
678  * surface is not required (for example, by neither hardware nor DRM format
679  * modifier), then this may return VK_SUCCESS when creation of the aux surface
680  * fails.
681  *
682  * @param offset See add_surface()
683  */
684 static VkResult
add_aux_surface_if_supported(struct anv_device * device,struct anv_image * image,uint32_t plane,struct anv_format_plane plane_format,const VkImageFormatListCreateInfo * fmt_list,uint64_t offset,uint32_t stride,uint64_t aux_state_offset)685 add_aux_surface_if_supported(struct anv_device *device,
686                              struct anv_image *image,
687                              uint32_t plane,
688                              struct anv_format_plane plane_format,
689                              const VkImageFormatListCreateInfo *fmt_list,
690                              uint64_t offset,
691                              uint32_t stride,
692                              uint64_t aux_state_offset)
693 {
694    VkImageAspectFlags aspect = plane_format.aspect;
695    VkResult result;
696    bool ok;
697 
698    /* The aux surface must not be already added. */
699    assert(!anv_surface_is_valid(&image->planes[plane].aux_surface));
700 
701    if (image->planes[plane].primary_surface.isl.usage &
702        ISL_SURF_USAGE_DISABLE_AUX_BIT)
703       return VK_SUCCESS;
704 
705    uint32_t binding;
706    if (image->vk.drm_format_mod == DRM_FORMAT_MOD_INVALID ||
707        isl_drm_modifier_has_aux(image->vk.drm_format_mod)) {
708       binding = ANV_IMAGE_MEMORY_BINDING_PLANE_0 + plane;
709    } else {
710       binding = ANV_IMAGE_MEMORY_BINDING_PRIVATE;
711    }
712 
713    if (aspect == VK_IMAGE_ASPECT_DEPTH_BIT) {
714       /* We don't advertise that depth buffers could be used as storage
715        * images.
716        */
717        assert(!(image->vk.usage & VK_IMAGE_USAGE_STORAGE_BIT));
718 
719       ok = isl_surf_get_hiz_surf(&device->isl_dev,
720                                  &image->planes[plane].primary_surface.isl,
721                                  &image->planes[plane].aux_surface.isl);
722       if (!ok)
723          return VK_SUCCESS;
724 
725       if (!isl_surf_supports_ccs(&device->isl_dev,
726                                  &image->planes[plane].primary_surface.isl,
727                                  &image->planes[plane].aux_surface.isl)) {
728          image->planes[plane].aux_usage = ISL_AUX_USAGE_HIZ;
729       } else if (want_hiz_wt_for_image(device->info, image)) {
730          assert(device->info->ver >= 12);
731          image->planes[plane].aux_usage = ISL_AUX_USAGE_HIZ_CCS_WT;
732       } else {
733          assert(device->info->ver >= 12);
734          image->planes[plane].aux_usage = ISL_AUX_USAGE_HIZ_CCS;
735       }
736 
737       result = add_surface(device, image, &image->planes[plane].aux_surface,
738                            binding, ANV_OFFSET_IMPLICIT);
739       if (result != VK_SUCCESS)
740          return result;
741 
742       if (anv_image_plane_uses_aux_map(device, image, plane)) {
743          result = add_compression_control_buffer(device, image, plane,
744                                                  binding,
745                                                  ANV_OFFSET_IMPLICIT);
746          if (result != VK_SUCCESS)
747             return result;
748       }
749 
750       if (device->info->ver == 12 &&
751           image->planes[plane].aux_usage == ISL_AUX_USAGE_HIZ_CCS_WT) {
752          return add_aux_state_tracking_buffer(device, image, aux_state_offset,
753                                               plane);
754       }
755    } else if (aspect == VK_IMAGE_ASPECT_STENCIL_BIT) {
756       if (!isl_surf_supports_ccs(&device->isl_dev,
757                                  &image->planes[plane].primary_surface.isl,
758                                  NULL))
759          return VK_SUCCESS;
760 
761       image->planes[plane].aux_usage = ISL_AUX_USAGE_STC_CCS;
762 
763       if (device->info->has_aux_map) {
764          result = add_compression_control_buffer(device, image, plane,
765                                                  binding,
766                                                  ANV_OFFSET_IMPLICIT);
767          if (result != VK_SUCCESS)
768             return result;
769       }
770    } else if ((aspect & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) && image->vk.samples == 1) {
771 
772       if (device->info->has_flat_ccs || device->info->has_aux_map) {
773          ok = isl_surf_supports_ccs(&device->isl_dev,
774                                     &image->planes[plane].primary_surface.isl,
775                                     NULL);
776       } else {
777          ok = isl_surf_get_ccs_surf(&device->isl_dev,
778                                     &image->planes[plane].primary_surface.isl,
779                                     &image->planes[plane].aux_surface.isl,
780                                     stride);
781       }
782       if (!ok)
783          return VK_SUCCESS;
784 
785       /* Choose aux usage. */
786       if (device->info->verx10 == 125 && !device->physical->disable_fcv) {
787          /* FCV is enabled via 3DSTATE_3D_MODE. We'd expect plain CCS_E to
788           * perform better because it allows for non-zero fast clear colors,
789           * but we've run into regressions in several benchmarks (F1 22 and
790           * RDR2) when trying to enable it. When non-zero clear colors are
791           * enabled, we've observed many partial resolves. We haven't yet
792           * root-caused what layout transitions are causing these resolves,
793           * so in the meantime, we choose to reduce our clear color support.
794           * With only zero clear colors being supported, we might as well
795           * turn on FCV.
796           */
797          image->planes[plane].aux_usage = ISL_AUX_USAGE_FCV_CCS_E;
798       } else if (intel_needs_workaround(device->info, 1607794140)) {
799          /* FCV is permanently enabled on this hardware. */
800          assert(device->info->verx10 == 120);
801          image->planes[plane].aux_usage = ISL_AUX_USAGE_FCV_CCS_E;
802       } else if (device->info->ver >= 12) {
803          /* Support for CCS_E was already checked for in anv_image_init(). */
804          image->planes[plane].aux_usage = ISL_AUX_USAGE_CCS_E;
805       } else if (anv_formats_ccs_e_compatible(device->info,
806                                               image->vk.create_flags,
807                                               image->vk.format,
808                                               image->vk.tiling,
809                                               image->vk.usage, fmt_list)) {
810          image->planes[plane].aux_usage = ISL_AUX_USAGE_CCS_E;
811       } else {
812          image->planes[plane].aux_usage = ISL_AUX_USAGE_CCS_D;
813       }
814 
815       if (device->info->has_flat_ccs) {
816          result = VK_SUCCESS;
817       } else if (device->info->has_aux_map) {
818          result = add_compression_control_buffer(device, image, plane,
819                                                  binding, offset);
820       } else {
821          result = add_surface(device, image,
822                               &image->planes[plane].aux_surface, binding,
823                               offset);
824       }
825       if (result != VK_SUCCESS)
826          return result;
827 
828       if (device->info->ver <= 12)
829          return add_aux_state_tracking_buffer(device, image, aux_state_offset,
830                                               plane);
831    } else if ((aspect & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) && image->vk.samples > 1) {
832       assert(!(image->vk.usage & VK_IMAGE_USAGE_STORAGE_BIT));
833       ok = isl_surf_get_mcs_surf(&device->isl_dev,
834                                  &image->planes[plane].primary_surface.isl,
835                                  &image->planes[plane].aux_surface.isl);
836       if (!ok)
837          return VK_SUCCESS;
838 
839       if (isl_surf_supports_ccs(&device->isl_dev,
840                                 &image->planes[plane].primary_surface.isl,
841                                 &image->planes[plane].aux_surface.isl)) {
842          image->planes[plane].aux_usage = ISL_AUX_USAGE_MCS_CCS;
843       } else {
844          image->planes[plane].aux_usage = ISL_AUX_USAGE_MCS;
845       }
846 
847       result = add_surface(device, image, &image->planes[plane].aux_surface,
848                            binding, ANV_OFFSET_IMPLICIT);
849       if (result != VK_SUCCESS)
850          return result;
851 
852       if (anv_image_plane_uses_aux_map(device, image, plane)) {
853          result = add_compression_control_buffer(device, image, plane,
854                                                  binding,
855                                                  ANV_OFFSET_IMPLICIT);
856          if (result != VK_SUCCESS)
857             return result;
858       }
859 
860       if (device->info->ver <= 12)
861          return add_aux_state_tracking_buffer(device, image, aux_state_offset,
862                                               plane);
863    }
864 
865    return VK_SUCCESS;
866 }
867 
868 static VkResult
add_video_buffers(struct anv_device * device,struct anv_image * image,const struct VkVideoProfileListInfoKHR * profile_list,bool independent_profile)869 add_video_buffers(struct anv_device *device,
870                   struct anv_image *image,
871                   const struct VkVideoProfileListInfoKHR *profile_list,
872                   bool independent_profile)
873 {
874    ASSERTED VkResult ok;
875    unsigned size = 0;
876 
877    if (independent_profile) {
878       /* Takes the worst case */
879       unsigned w_mb = DIV_ROUND_UP(image->vk.extent.width, ANV_MB_WIDTH);
880       unsigned h_mb = DIV_ROUND_UP(image->vk.extent.height, ANV_MB_HEIGHT);
881       size = w_mb * h_mb * 128;
882    } else {
883       size = anv_video_get_image_mv_size(device, image, profile_list);
884    }
885 
886    if (size == 0)
887       return VK_SUCCESS;
888 
889    ok = image_binding_grow(device, image, ANV_IMAGE_MEMORY_BINDING_PRIVATE,
890                            ANV_OFFSET_IMPLICIT, size, 65536, &image->vid_dmv_top_surface);
891    if (ok != VK_SUCCESS)
892       return ok;
893 
894    /* Doesn't work for av1 without provided profiles */
895    if (!independent_profile) {
896       for (unsigned i = 0; i < profile_list->profileCount; i++) {
897          if (profile_list->pProfiles[i].videoCodecOperation == VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR) {
898             ok = image_binding_grow(device, image, ANV_IMAGE_MEMORY_BINDING_PRIVATE,
899                                     ANV_OFFSET_IMPLICIT, av1_cdf_max_num_bytes, 4096, &image->av1_cdf_table);
900          }
901       }
902    }
903 
904    return ok;
905 }
906 
907 /**
908  * Initialize the anv_image::*_surface selected by \a aspect. Then update the
909  * image's memory requirements (that is, the image's size and alignment).
910  *
911  * @param offset See add_surface()
912  */
913 static VkResult
add_primary_surface(struct anv_device * device,struct anv_image * image,uint32_t plane,struct anv_format_plane plane_format,uint64_t offset,uint32_t stride,isl_tiling_flags_t isl_tiling_flags,isl_surf_usage_flags_t isl_usage)914 add_primary_surface(struct anv_device *device,
915                     struct anv_image *image,
916                     uint32_t plane,
917                     struct anv_format_plane plane_format,
918                     uint64_t offset,
919                     uint32_t stride,
920                     isl_tiling_flags_t isl_tiling_flags,
921                     isl_surf_usage_flags_t isl_usage)
922 {
923    struct anv_surface *anv_surf = &image->planes[plane].primary_surface;
924    bool ok;
925 
926    uint32_t width = image->vk.extent.width;
927    uint32_t height = image->vk.extent.height;
928    const struct vk_format_ycbcr_info *ycbcr_info =
929       vk_format_get_ycbcr_info(image->vk.format);
930    if (ycbcr_info) {
931       assert(plane < ycbcr_info->n_planes);
932       width /= ycbcr_info->planes[plane].denominator_scales[0];
933       height /= ycbcr_info->planes[plane].denominator_scales[1];
934    }
935 
936    ok = isl_surf_init(&device->isl_dev, &anv_surf->isl,
937       .dim = vk_to_isl_surf_dim[image->vk.image_type],
938       .format = plane_format.isl_format,
939       .width = width,
940       .height = height,
941       .depth = image->vk.extent.depth,
942       .levels = image->vk.mip_levels,
943       .array_len = image->vk.array_layers,
944       .samples = image->vk.samples,
945       .min_alignment_B = 0,
946       .row_pitch_B = stride,
947       .usage = isl_usage,
948       .tiling_flags = isl_tiling_flags);
949 
950    if (!ok) {
951       /* TODO: Should return
952        * VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT in come cases.
953        */
954       return VK_ERROR_OUT_OF_DEVICE_MEMORY;
955    }
956 
957    image->planes[plane].aux_usage = ISL_AUX_USAGE_NONE;
958 
959    return add_surface(device, image, anv_surf,
960                       ANV_IMAGE_MEMORY_BINDING_PLANE_0 + plane, offset);
961 }
962 
963 #ifndef NDEBUG
964 static bool MUST_CHECK
memory_range_is_aligned(struct anv_image_memory_range memory_range)965 memory_range_is_aligned(struct anv_image_memory_range memory_range)
966 {
967    return anv_is_aligned(memory_range.offset, memory_range.alignment);
968 }
969 
970 static bool MUST_CHECK
memory_ranges_equal(struct anv_image_memory_range a,struct anv_image_memory_range b)971 memory_ranges_equal(struct anv_image_memory_range a,
972                     struct anv_image_memory_range b)
973 {
974    return a.binding == b.binding &&
975           a.alignment == b.alignment &&
976           a.size == b.size &&
977           a.offset == b.offset;
978 }
979 #endif
980 
981 struct check_memory_range_params {
982    struct anv_image_memory_range *accum_ranges;
983    const struct anv_surface *test_surface;
984    const struct anv_image_memory_range *test_range;
985    enum anv_image_memory_binding expect_binding;
986 };
987 
988 #define check_memory_range(...) \
989    check_memory_range_s(&(struct check_memory_range_params) { __VA_ARGS__ })
990 
991 static void UNUSED
check_memory_range_s(const struct check_memory_range_params * p)992 check_memory_range_s(const struct check_memory_range_params *p)
993 {
994    assert((p->test_surface == NULL) != (p->test_range == NULL));
995 
996    const struct anv_image_memory_range *test_range =
997       p->test_range ?: &p->test_surface->memory_range;
998 
999    struct anv_image_memory_range *accum_range =
1000       &p->accum_ranges[p->expect_binding];
1001 
1002    assert(test_range->binding == p->expect_binding);
1003    assert(test_range->offset >= memory_range_end(*accum_range));
1004    assert(memory_range_is_aligned(*test_range));
1005 
1006    if (p->test_surface) {
1007       assert(anv_surface_is_valid(p->test_surface));
1008       assert(p->test_surface->memory_range.alignment ==
1009              p->test_surface->isl.alignment_B);
1010    }
1011 
1012    memory_range_merge(accum_range, *test_range);
1013 }
1014 
1015 /**
1016  * Validate the image's memory bindings *after* all its surfaces and memory
1017  * ranges are final.
1018  *
1019  * For simplicity's sake, we do not validate free-form layout of the image's
1020  * memory bindings. We validate the layout described in the comments of struct
1021  * anv_image.
1022  */
1023 static void
check_memory_bindings(const struct anv_device * device,const struct anv_image * image)1024 check_memory_bindings(const struct anv_device *device,
1025                      const struct anv_image *image)
1026 {
1027 #if MESA_DEBUG
1028    /* As we inspect each part of the image, we merge the part's memory range
1029     * into these accumulation ranges.
1030     */
1031    struct anv_image_memory_range accum_ranges[ANV_IMAGE_MEMORY_BINDING_END];
1032    for (int i = 0; i < ANV_IMAGE_MEMORY_BINDING_END; ++i) {
1033       accum_ranges[i] = (struct anv_image_memory_range) {
1034          .binding = i,
1035       };
1036    }
1037 
1038    for (uint32_t p = 0; p < image->n_planes; ++p) {
1039       const struct anv_image_plane *plane = &image->planes[p];
1040 
1041       /* The binding that must contain the plane's primary surface. */
1042       const enum anv_image_memory_binding primary_binding = image->disjoint
1043          ? ANV_IMAGE_MEMORY_BINDING_PLANE_0 + p
1044          : ANV_IMAGE_MEMORY_BINDING_MAIN;
1045 
1046       /* Aliasing is incompatible with the private binding because it does not
1047        * live in a VkDeviceMemory.  The exception is either swapchain images or
1048        * that the private binding is for a video motion vector buffer.
1049        */
1050       assert(!(image->vk.create_flags & VK_IMAGE_CREATE_ALIAS_BIT) ||
1051              image->from_wsi ||
1052              (plane->primary_surface.isl.usage & ISL_SURF_USAGE_VIDEO_DECODE_BIT) ||
1053              image->bindings[ANV_IMAGE_MEMORY_BINDING_PRIVATE].memory_range.size == 0);
1054 
1055       /* Check primary surface */
1056       check_memory_range(accum_ranges,
1057                          .test_surface = &plane->primary_surface,
1058                          .expect_binding = primary_binding);
1059 
1060       /* Check aux_surface */
1061       const struct anv_image_memory_range *aux_mem_range =
1062          anv_image_get_aux_memory_range(image, p);
1063       if (aux_mem_range->size > 0) {
1064          enum anv_image_memory_binding binding = primary_binding;
1065 
1066          /* If an auxiliary surface is used for an externally-shareable image,
1067           * we have to hide this from the memory of the image since other
1068           * processes with access to the memory may not be aware of it or of
1069           * its current state. So put that auxiliary data into a separate
1070           * buffer (ANV_IMAGE_MEMORY_BINDING_PRIVATE).
1071           */
1072          if (anv_image_is_externally_shared(image) &&
1073              !isl_drm_modifier_has_aux(image->vk.drm_format_mod)) {
1074             binding = ANV_IMAGE_MEMORY_BINDING_PRIVATE;
1075          }
1076 
1077          /* Display hardware requires that the aux surface start at
1078           * a higher address than the primary surface. The 3D hardware
1079           * doesn't care, but we enforce the display requirement in case
1080           * the image is sent to display.
1081           */
1082          check_memory_range(accum_ranges,
1083                             .test_range = aux_mem_range,
1084                             .expect_binding = binding);
1085       }
1086 
1087       /* Check fast clear state */
1088       if (plane->fast_clear_memory_range.size > 0) {
1089          enum anv_image_memory_binding binding = primary_binding;
1090 
1091          /* If an auxiliary surface is used for an externally-shareable image,
1092           * we have to hide this from the memory of the image since other
1093           * processes with access to the memory may not be aware of it or of
1094           * its current state. So put that auxiliary data into a separate
1095           * buffer (ANV_IMAGE_MEMORY_BINDING_PRIVATE).
1096           *
1097           * But when the image is created with a drm modifier that supports
1098           * clear color, it will be exported along with main surface.
1099           */
1100          if (anv_image_is_externally_shared(image)
1101              && !isl_drm_modifier_get_info(image->vk.drm_format_mod)->supports_clear_color) {
1102             binding = ANV_IMAGE_MEMORY_BINDING_PRIVATE;
1103          }
1104 
1105          /* The indirect clear color BO requires 64B-alignment on gfx11+. */
1106          assert(plane->fast_clear_memory_range.alignment % 64 == 0);
1107          check_memory_range(accum_ranges,
1108                             .test_range = &plane->fast_clear_memory_range,
1109                             .expect_binding = binding);
1110       }
1111    }
1112 #endif
1113 }
1114 
1115 /**
1116  * Check that the fully-initialized anv_image is compatible with its DRM format
1117  * modifier.
1118  *
1119  * Checking compatibility at the end of image creation is prudent, not
1120  * superfluous, because usage of modifiers triggers numerous special cases
1121  * throughout queries and image creation, and because
1122  * vkGetPhysicalDeviceImageFormatProperties2 has difficulty detecting all
1123  * incompatibilities.
1124  *
1125  * Return VK_ERROR_UNKNOWN if the incompatibility is difficult to detect in
1126  * vkGetPhysicalDeviceImageFormatProperties2.  Otherwise, assert fail.
1127  *
1128  * Ideally, if vkGetPhysicalDeviceImageFormatProperties2() succeeds with a given
1129  * modifier, then vkCreateImage() produces an image that is compatible with the
1130  * modifier. However, it is difficult to reconcile the two functions to agree
1131  * due to their complexity. For example, isl_surf_get_ccs_surf() may
1132  * unexpectedly fail in vkCreateImage(), eliminating the image's aux surface
1133  * even when the modifier requires one. (Maybe we should reconcile the two
1134  * functions despite the difficulty).
1135  */
1136 static VkResult MUST_CHECK
check_drm_format_mod(const struct anv_device * device,const struct anv_image * image)1137 check_drm_format_mod(const struct anv_device *device,
1138                      const struct anv_image *image)
1139 {
1140    /* Image must have a modifier if and only if it has modifier tiling. */
1141    assert((image->vk.drm_format_mod != DRM_FORMAT_MOD_INVALID) ==
1142           (image->vk.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT));
1143 
1144    if (image->vk.drm_format_mod == DRM_FORMAT_MOD_INVALID)
1145       return VK_SUCCESS;
1146 
1147    const struct isl_drm_modifier_info *isl_mod_info =
1148       isl_drm_modifier_get_info(image->vk.drm_format_mod);
1149 
1150    /* Driver must support the modifier. */
1151    assert(isl_drm_modifier_get_score(device->info, isl_mod_info->modifier));
1152 
1153    /* Enforced by us, not the Vulkan spec. */
1154    assert(image->vk.image_type == VK_IMAGE_TYPE_2D);
1155    assert(!(image->vk.aspects & VK_IMAGE_ASPECT_DEPTH_BIT));
1156    assert(!(image->vk.aspects & VK_IMAGE_ASPECT_STENCIL_BIT));
1157    assert(image->vk.mip_levels == 1);
1158    assert(image->vk.array_layers == 1);
1159    assert(image->vk.samples == 1);
1160 
1161    for (int i = 0; i < image->n_planes; ++i) {
1162       const struct anv_image_plane *plane = &image->planes[i];
1163       ASSERTED const struct isl_format_layout *isl_layout =
1164          isl_format_get_layout(plane->primary_surface.isl.format);
1165 
1166       /* Enforced by us, not the Vulkan spec. */
1167       assert(isl_layout->txc == ISL_TXC_NONE);
1168       assert(isl_layout->colorspace == ISL_COLORSPACE_LINEAR ||
1169              isl_layout->colorspace == ISL_COLORSPACE_SRGB);
1170 
1171       if (isl_drm_modifier_has_aux(isl_mod_info->modifier)) {
1172          /* Reject DISJOINT for consistency with the GL driver. */
1173          assert(!image->disjoint);
1174 
1175          /* The modifier's required aux usage mandates the image's aux usage.
1176           * The inverse, however, does not hold; if the modifier has no aux
1177           * usage, then we may enable a private aux surface.
1178           */
1179          if ((isl_mod_info->supports_media_compression &&
1180               plane->aux_usage != ISL_AUX_USAGE_MC) ||
1181              (isl_mod_info->supports_render_compression &&
1182               !isl_aux_usage_has_ccs_e(plane->aux_usage))) {
1183             return vk_errorf(device, VK_ERROR_UNKNOWN,
1184                              "image with modifier unexpectedly has wrong aux "
1185                              "usage");
1186          }
1187       }
1188    }
1189 
1190    return VK_SUCCESS;
1191 }
1192 
1193 /**
1194  * Use when the app does not provide
1195  * VkImageDrmFormatModifierExplicitCreateInfoEXT.
1196  */
1197 static VkResult MUST_CHECK
add_all_surfaces_implicit_layout(struct anv_device * device,struct anv_image * image,const VkImageFormatListCreateInfo * format_list_info,uint32_t stride,isl_tiling_flags_t isl_tiling_flags,isl_surf_usage_flags_t isl_extra_usage_flags)1198 add_all_surfaces_implicit_layout(
1199    struct anv_device *device,
1200    struct anv_image *image,
1201    const VkImageFormatListCreateInfo *format_list_info,
1202    uint32_t stride,
1203    isl_tiling_flags_t isl_tiling_flags,
1204    isl_surf_usage_flags_t isl_extra_usage_flags)
1205 {
1206    const struct intel_device_info *devinfo = device->info;
1207    VkResult result;
1208 
1209    const struct vk_format_ycbcr_info *ycbcr_info =
1210       vk_format_get_ycbcr_info(image->vk.format);
1211    if (ycbcr_info)
1212       assert(ycbcr_info->n_planes == image->n_planes);
1213 
1214    unsigned num_aspects = 0;
1215    VkImageAspectFlagBits aspects[3];
1216    u_foreach_bit(b, image->vk.aspects) {
1217       assert(num_aspects < 3);
1218       aspects[num_aspects++] = 1 << b;
1219    }
1220    assert(num_aspects == image->n_planes);
1221 
1222    /* The Android hardware buffer YV12 format has the planes ordered as Y-Cr-Cb,
1223     * while Vulkan expects VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM to be in Y-Cb-Cr.
1224     * Adjust the order we add the ISL surfaces accordingly so the implicit
1225     * offset gets calculated correctly.
1226     */
1227    if (image->from_ahb && image->vk.format == VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM) {
1228       assert(num_aspects == 3);
1229       assert(aspects[1] == VK_IMAGE_ASPECT_PLANE_1_BIT);
1230       assert(aspects[2] == VK_IMAGE_ASPECT_PLANE_2_BIT);
1231       aspects[1] = VK_IMAGE_ASPECT_PLANE_2_BIT;
1232       aspects[2] = VK_IMAGE_ASPECT_PLANE_1_BIT;
1233    }
1234 
1235    for (unsigned i = 0; i < num_aspects; i++) {
1236       VkImageAspectFlagBits aspect = aspects[i];
1237       const uint32_t plane = anv_image_aspect_to_plane(image, aspect);
1238       const  struct anv_format_plane plane_format =
1239          anv_get_format_plane(devinfo, image->vk.format, plane, image->vk.tiling);
1240 
1241       enum isl_format isl_fmt = plane_format.isl_format;
1242       assert(isl_fmt != ISL_FORMAT_UNSUPPORTED);
1243 
1244       uint32_t plane_stride = stride * isl_format_get_layout(isl_fmt)->bpb / 8;
1245       if (ycbcr_info)
1246          plane_stride /= ycbcr_info->planes[plane].denominator_scales[0];
1247 
1248       VkImageUsageFlags vk_usage = vk_image_usage(&image->vk, aspect);
1249       isl_surf_usage_flags_t isl_usage =
1250          anv_image_choose_isl_surf_usage(device->physical,
1251                                          image->vk.create_flags, vk_usage,
1252                                          isl_extra_usage_flags, aspect,
1253                                          image->vk.compr_flags);
1254 
1255       result = add_primary_surface(device, image, plane, plane_format,
1256                                    ANV_OFFSET_IMPLICIT, plane_stride,
1257                                    isl_tiling_flags, isl_usage);
1258       if (result != VK_SUCCESS)
1259          return result;
1260 
1261       result = add_aux_surface_if_supported(device, image, plane, plane_format,
1262                                             format_list_info,
1263                                             ANV_OFFSET_IMPLICIT, plane_stride,
1264                                             ANV_OFFSET_IMPLICIT);
1265       if (result != VK_SUCCESS)
1266          return result;
1267    }
1268 
1269    return VK_SUCCESS;
1270 }
1271 
1272 /**
1273  * Use when the app provides VkImageDrmFormatModifierExplicitCreateInfoEXT.
1274  */
1275 static VkResult
add_all_surfaces_explicit_layout(struct anv_device * device,struct anv_image * image,const VkImageFormatListCreateInfo * format_list_info,const VkImageDrmFormatModifierExplicitCreateInfoEXT * drm_info,isl_tiling_flags_t isl_tiling_flags,isl_surf_usage_flags_t isl_extra_usage_flags)1276 add_all_surfaces_explicit_layout(
1277    struct anv_device *device,
1278    struct anv_image *image,
1279    const VkImageFormatListCreateInfo *format_list_info,
1280    const VkImageDrmFormatModifierExplicitCreateInfoEXT *drm_info,
1281    isl_tiling_flags_t isl_tiling_flags,
1282    isl_surf_usage_flags_t isl_extra_usage_flags)
1283 {
1284    const struct intel_device_info *devinfo = device->info;
1285    const uint32_t mod_plane_count = drm_info->drmFormatModifierPlaneCount;
1286    const bool mod_has_aux =
1287       isl_drm_modifier_has_aux(drm_info->drmFormatModifier);
1288    VkResult result;
1289 
1290    /* Currently there is no way to properly map memory planes to format planes
1291     * and aux planes due to the lack of defined ABI for external multi-planar
1292     * images.
1293     */
1294    if (image->n_planes == 1)
1295       assert(image->vk.aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1296    else
1297       assert(!(image->vk.aspects & ~VK_IMAGE_ASPECT_PLANES_BITS_ANV));
1298 
1299    if (mod_has_aux) {
1300       assert(image->n_planes == 1);
1301 
1302       /* About valid usage in the Vulkan spec:
1303        *
1304        * Unlike vanilla vkCreateImage, which produces undefined behavior on user
1305        * error, here the spec requires the implementation to return
1306        * VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT if the app provides
1307        * a bad plane layout. However, the spec does require
1308        * drmFormatModifierPlaneCount to be valid.
1309        *
1310        * Most validation of plane layout occurs in add_surface().
1311        */
1312       uint32_t n_mod_planes =
1313          isl_drm_modifier_get_plane_count(devinfo,
1314                                           drm_info->drmFormatModifier,
1315                                           image->n_planes);
1316       assert(n_mod_planes == mod_plane_count);
1317    } else {
1318       assert(image->n_planes == mod_plane_count);
1319    }
1320 
1321    /* Reject special values in the app-provided plane layouts. */
1322    for (uint32_t i = 0; i < mod_plane_count; ++i) {
1323       if (drm_info->pPlaneLayouts[i].rowPitch == 0) {
1324          return vk_errorf(device,
1325                           VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT,
1326                           "VkImageDrmFormatModifierExplicitCreateInfoEXT::"
1327                           "pPlaneLayouts[%u]::rowPitch is 0", i);
1328       }
1329 
1330       if (drm_info->pPlaneLayouts[i].offset == ANV_OFFSET_IMPLICIT) {
1331          return vk_errorf(device,
1332                           VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT,
1333                           "VkImageDrmFormatModifierExplicitCreateInfoEXT::"
1334                           "pPlaneLayouts[%u]::offset is %" PRIu64,
1335                           i, ANV_OFFSET_IMPLICIT);
1336       }
1337    }
1338 
1339    u_foreach_bit(b, image->vk.aspects) {
1340       const VkImageAspectFlagBits aspect = 1 << b;
1341       const uint32_t plane = anv_image_aspect_to_plane(image, aspect);
1342       const struct anv_format_plane format_plane =
1343          anv_get_format_plane(devinfo, image->vk.format, plane, image->vk.tiling);
1344       const VkSubresourceLayout *primary_layout = &drm_info->pPlaneLayouts[plane];
1345 
1346       result = add_primary_surface(device, image, plane,
1347                                    format_plane,
1348                                    primary_layout->offset,
1349                                    primary_layout->rowPitch,
1350                                    isl_tiling_flags,
1351                                    isl_extra_usage_flags);
1352       if (result != VK_SUCCESS)
1353          return result;
1354 
1355       if (mod_has_aux) {
1356          const VkSubresourceLayout flat_ccs_layout = {
1357             .offset = ANV_OFFSET_IMPLICIT,
1358          };
1359 
1360          const VkSubresourceLayout *aux_layout;
1361 
1362          uint64_t aux_state_offset = ANV_OFFSET_IMPLICIT;
1363 
1364          /* We already asserted on image->n_planes == 1 when mod_has_aux is
1365           * true above, so the indexes of aux and clear color are just hard-
1366           * coded without ambiguity.
1367           */
1368          if (devinfo->has_flat_ccs) {
1369             aux_layout = &flat_ccs_layout;
1370             if (isl_drm_modifier_get_info(
1371                   drm_info->drmFormatModifier)->supports_clear_color) {
1372                aux_state_offset = drm_info->pPlaneLayouts[1].offset;
1373             }
1374          } else {
1375             aux_layout = &drm_info->pPlaneLayouts[1];
1376             if (isl_drm_modifier_get_info(
1377                   drm_info->drmFormatModifier)->supports_clear_color) {
1378                aux_state_offset = drm_info->pPlaneLayouts[2].offset;
1379             }
1380          }
1381 
1382          result = add_aux_surface_if_supported(device, image, plane,
1383                                                format_plane,
1384                                                format_list_info,
1385                                                aux_layout->offset,
1386                                                aux_layout->rowPitch,
1387                                                aux_state_offset);
1388          if (result != VK_SUCCESS)
1389             return result;
1390 
1391          assert(isl_aux_usage_has_ccs(image->planes[plane].aux_usage));
1392          if (aux_state_offset != ANV_OFFSET_IMPLICIT) {
1393             assert(image->planes[plane].fast_clear_memory_range.size <=
1394                    device->isl_dev.ss.clear_color_state_size);
1395          }
1396       }
1397    }
1398 
1399    return VK_SUCCESS;
1400 }
1401 
1402 static const struct isl_drm_modifier_info *
choose_drm_format_mod(const struct anv_physical_device * device,uint32_t modifier_count,const uint64_t * modifiers)1403 choose_drm_format_mod(const struct anv_physical_device *device,
1404                       uint32_t modifier_count, const uint64_t *modifiers)
1405 {
1406    uint64_t best_mod = UINT64_MAX;
1407    uint32_t best_score = 0;
1408 
1409    for (uint32_t i = 0; i < modifier_count; ++i) {
1410       uint32_t score = isl_drm_modifier_get_score(&device->info, modifiers[i]);
1411       if (score > best_score) {
1412          best_mod = modifiers[i];
1413          best_score = score;
1414       }
1415    }
1416 
1417    if (best_score > 0)
1418       return isl_drm_modifier_get_info(best_mod);
1419    else
1420       return NULL;
1421 }
1422 
1423 static VkImageUsageFlags
anv_image_create_usage(const VkImageCreateInfo * pCreateInfo,VkImageUsageFlags usage)1424 anv_image_create_usage(const VkImageCreateInfo *pCreateInfo,
1425                        VkImageUsageFlags usage)
1426 {
1427    /* Add TRANSFER_SRC usage for multisample attachment images. This is
1428     * because we might internally use the TRANSFER_SRC layout on them for
1429     * blorp operations associated with resolving those into other attachments
1430     * at the end of a subpass.
1431     *
1432     * Without this additional usage, we compute an incorrect AUX state in
1433     * anv_layout_to_aux_state().
1434     */
1435    if (pCreateInfo->samples > VK_SAMPLE_COUNT_1_BIT &&
1436        (usage & (VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
1437                  VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT)))
1438       usage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
1439    return usage;
1440 }
1441 
1442 static VkResult MUST_CHECK
alloc_private_binding(struct anv_device * device,struct anv_image * image,const VkImageCreateInfo * create_info)1443 alloc_private_binding(struct anv_device *device,
1444                       struct anv_image *image,
1445                       const VkImageCreateInfo *create_info)
1446 {
1447    struct anv_image_binding *binding =
1448       &image->bindings[ANV_IMAGE_MEMORY_BINDING_PRIVATE];
1449 
1450    if (binding->memory_range.size == 0)
1451       return VK_SUCCESS;
1452 
1453    const VkImageSwapchainCreateInfoKHR *swapchain_info =
1454       vk_find_struct_const(create_info->pNext, IMAGE_SWAPCHAIN_CREATE_INFO_KHR);
1455 
1456    if (swapchain_info && swapchain_info->swapchain != VK_NULL_HANDLE) {
1457       /* The image will be bound to swapchain memory. */
1458       return VK_SUCCESS;
1459    }
1460 
1461    VkResult result = anv_device_alloc_bo(device, "image-binding-private",
1462                                          binding->memory_range.size, 0, 0,
1463                                          &binding->address.bo);
1464    if (result == VK_SUCCESS) {
1465       pthread_mutex_lock(&device->mutex);
1466       list_addtail(&image->link, &device->image_private_objects);
1467       pthread_mutex_unlock(&device->mutex);
1468    }
1469 
1470    return result;
1471 }
1472 
1473 static void
anv_image_finish_sparse_bindings(struct anv_image * image)1474 anv_image_finish_sparse_bindings(struct anv_image *image)
1475 {
1476    struct anv_device *device =
1477       container_of(image->vk.base.device, struct anv_device, vk);
1478 
1479    assert(anv_image_is_sparse(image));
1480 
1481    for (int i = 0; i < ANV_IMAGE_MEMORY_BINDING_END; i++) {
1482       struct anv_image_binding *b = &image->bindings[i];
1483 
1484       if (b->sparse_data.size != 0) {
1485          assert(b->memory_range.size == b->sparse_data.size);
1486          assert(b->address.offset == b->sparse_data.address);
1487          anv_free_sparse_bindings(device, &b->sparse_data);
1488       }
1489    }
1490 }
1491 
1492 static VkResult MUST_CHECK
anv_image_init_sparse_bindings(struct anv_image * image,const struct anv_image_create_info * create_info)1493 anv_image_init_sparse_bindings(struct anv_image *image,
1494                                const struct anv_image_create_info *create_info)
1495 {
1496    struct anv_device *device =
1497       container_of(image->vk.base.device, struct anv_device, vk);
1498    VkResult result;
1499 
1500    assert(anv_image_is_sparse(image));
1501 
1502    enum anv_bo_alloc_flags alloc_flags = 0;
1503    uint64_t explicit_address = 0;
1504    if (image->vk.create_flags & VK_IMAGE_CREATE_DESCRIPTOR_BUFFER_CAPTURE_REPLAY_BIT_EXT) {
1505       alloc_flags |= ANV_BO_ALLOC_FIXED_ADDRESS;
1506 
1507       const VkOpaqueCaptureDescriptorDataCreateInfoEXT *opaque_info =
1508          vk_find_struct_const(create_info->vk_info->pNext,
1509                               OPAQUE_CAPTURE_DESCRIPTOR_DATA_CREATE_INFO_EXT);
1510       if (opaque_info)
1511          explicit_address = *((const uint64_t *)opaque_info->opaqueCaptureDescriptorData);
1512    }
1513 
1514    for (int i = 0; i < ANV_IMAGE_MEMORY_BINDING_END; i++) {
1515       struct anv_image_binding *b = &image->bindings[i];
1516 
1517       if (b->memory_range.size != 0) {
1518          assert(b->sparse_data.size == 0);
1519 
1520          /* From the spec, Custom Sparse Image Block Shapes section:
1521           *   "... the size in bytes of the custom sparse image block shape
1522           *    will be reported in VkMemoryRequirements::alignment."
1523           *
1524           * ISL should have set this for us, so just assert it here.
1525           */
1526          assert(b->memory_range.alignment == ANV_SPARSE_BLOCK_SIZE);
1527          assert(b->memory_range.size % ANV_SPARSE_BLOCK_SIZE == 0);
1528 
1529          result = anv_init_sparse_bindings(device,
1530                                            b->memory_range.size,
1531                                            &b->sparse_data,
1532                                            alloc_flags,
1533                                            explicit_address,
1534                                            &b->address);
1535          if (result != VK_SUCCESS) {
1536             anv_image_finish_sparse_bindings(image);
1537             return result;
1538          }
1539       }
1540    }
1541 
1542    return VK_SUCCESS;
1543 }
1544 
1545 bool
anv_image_view_formats_incomplete(const struct anv_image * image)1546 anv_image_view_formats_incomplete(const struct anv_image *image)
1547 {
1548    /* See mark_image_view_formats_incomplete(). */
1549    return image->num_view_formats < ARRAY_SIZE(image->view_formats) &&
1550           image->view_formats[image->num_view_formats] ==
1551           ISL_FORMAT_UNSUPPORTED;
1552 }
1553 
1554 static void
mark_image_view_formats_incomplete(struct anv_image * image)1555 mark_image_view_formats_incomplete(struct anv_image *image)
1556 {
1557    /* We need to reserve space to insert the token for an incomplete list. Use
1558     * up all the space except for the first entry. This helps various code
1559     * paths that depend on the list to have decent fall-back behavior. For
1560     * examples, see add_aux_state_tracking_buffer() and
1561     * set_image_clear_color().
1562     */
1563    assert(image->num_view_formats >= 1);
1564    image->num_view_formats = 1;
1565 
1566    /* Replace the first unused entry with the token for an incomplete list. */
1567    image->view_formats[image->num_view_formats] = ISL_FORMAT_UNSUPPORTED;
1568    assert(anv_image_view_formats_incomplete(image));
1569 }
1570 
1571 static void
add_image_view_format(struct anv_image * image,enum isl_format view_format)1572 add_image_view_format(struct anv_image *image, enum isl_format view_format)
1573 {
1574    /* If this list can't be completed, reject all formats. */
1575    if (anv_image_view_formats_incomplete(image))
1576       return;
1577 
1578    /* Reject invalid formats. */
1579    if (view_format == ISL_FORMAT_UNSUPPORTED)
1580       return;
1581 
1582    /* Reject duplicate formats. */
1583    for (int i = 0; i < image->num_view_formats; i++)
1584       if (view_format == image->view_formats[i])
1585          return;
1586 
1587    if (image->num_view_formats == ARRAY_SIZE(image->view_formats)) {
1588       /* The array is full, so we can't add any more entries. */
1589       mark_image_view_formats_incomplete(image);
1590    } else {
1591       /* Add to the end of the array. */
1592       image->view_formats[image->num_view_formats++] = view_format;
1593    }
1594 }
1595 
1596 VkResult
anv_image_init(struct anv_device * device,struct anv_image * image,const struct anv_image_create_info * create_info)1597 anv_image_init(struct anv_device *device, struct anv_image *image,
1598                const struct anv_image_create_info *create_info)
1599 {
1600    const VkImageCreateInfo *pCreateInfo = create_info->vk_info;
1601    const struct VkImageDrmFormatModifierExplicitCreateInfoEXT *mod_explicit_info = NULL;
1602    const struct isl_drm_modifier_info *isl_mod_info = NULL;
1603    VkResult r;
1604 
1605    vk_image_init(&device->vk, &image->vk, pCreateInfo);
1606 
1607    image->vk.usage = anv_image_create_usage(pCreateInfo, image->vk.usage);
1608    image->vk.stencil_usage =
1609       anv_image_create_usage(pCreateInfo, image->vk.stencil_usage);
1610 
1611    isl_surf_usage_flags_t isl_extra_usage_flags =
1612       create_info->isl_extra_usage_flags;
1613    if (pCreateInfo->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
1614       assert(!image->vk.wsi_legacy_scanout);
1615       mod_explicit_info =
1616          vk_find_struct_const(pCreateInfo->pNext,
1617                               IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT);
1618       if (mod_explicit_info) {
1619          isl_mod_info = isl_drm_modifier_get_info(mod_explicit_info->drmFormatModifier);
1620       } else {
1621          const struct VkImageDrmFormatModifierListCreateInfoEXT *mod_list_info =
1622             vk_find_struct_const(pCreateInfo->pNext,
1623                                  IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
1624          isl_mod_info = choose_drm_format_mod(device->physical,
1625                                               mod_list_info->drmFormatModifierCount,
1626                                               mod_list_info->pDrmFormatModifiers);
1627       }
1628 
1629       assert(isl_mod_info);
1630       assert(image->vk.drm_format_mod == DRM_FORMAT_MOD_INVALID);
1631       image->vk.drm_format_mod = isl_mod_info->modifier;
1632 
1633       if (isl_drm_modifier_needs_display_layout(image->vk.drm_format_mod))
1634          isl_extra_usage_flags |= ISL_SURF_USAGE_DISPLAY_BIT;
1635 
1636       /* Disable compression on gen12+ if the selected/requested modifier
1637        * doesn't support it. Prior to that we can use a private binding for
1638        * the aux surface and it should be transparent to users.
1639        */
1640       if (device->info->ver >= 12 &&
1641           !isl_drm_modifier_has_aux(image->vk.drm_format_mod)) {
1642          isl_extra_usage_flags |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
1643       }
1644    }
1645 
1646    for (int i = 0; i < ANV_IMAGE_MEMORY_BINDING_END; ++i) {
1647       image->bindings[i] = (struct anv_image_binding) {
1648          .memory_range = { .binding = i },
1649       };
1650    }
1651 
1652    /* In case of AHardwareBuffer import, we don't know the layout yet */
1653    if (image->vk.external_handle_types &
1654        VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID) {
1655       image->from_ahb = true;
1656 #if DETECT_OS_ANDROID
1657       image->vk.ahb_format = anv_ahb_format_for_vk_format(image->vk.format);
1658 #endif
1659       return VK_SUCCESS;
1660    }
1661 
1662    image->n_planes = anv_get_format_planes(image->vk.format);
1663 
1664 #ifdef VK_USE_PLATFORM_ANDROID_KHR
1665    /* In the case of gralloc-backed swap chain image, we don't know the
1666     * layout yet.
1667     */
1668    if (vk_find_struct_const(pCreateInfo->pNext,
1669                             IMAGE_SWAPCHAIN_CREATE_INFO_KHR) != NULL)
1670       return VK_SUCCESS;
1671 #endif
1672 
1673    image->from_wsi =
1674       vk_find_struct_const(pCreateInfo->pNext, WSI_IMAGE_CREATE_INFO_MESA) != NULL;
1675 
1676    /* The Vulkan 1.2.165 glossary says:
1677     *
1678     *    A disjoint image consists of multiple disjoint planes, and is created
1679     *    with the VK_IMAGE_CREATE_DISJOINT_BIT bit set.
1680     */
1681    image->disjoint = image->n_planes > 1 &&
1682                      (pCreateInfo->flags & VK_IMAGE_CREATE_DISJOINT_BIT);
1683 
1684    if (anv_is_format_emulated(device->physical, pCreateInfo->format)) {
1685       assert(image->n_planes == 1 &&
1686              vk_format_is_compressed(image->vk.format));
1687       assert(!(image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT));
1688 
1689       image->emu_plane_format =
1690          anv_get_emulation_format(device->physical, image->vk.format);
1691 
1692       /* for fetching the raw copmressed data and storing the decompressed
1693        * data
1694        */
1695       image->vk.create_flags |=
1696          VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT |
1697          VK_IMAGE_CREATE_BLOCK_TEXEL_VIEW_COMPATIBLE_BIT;
1698       if (image->vk.image_type == VK_IMAGE_TYPE_3D)
1699          image->vk.create_flags |= VK_IMAGE_CREATE_2D_VIEW_COMPATIBLE_BIT_EXT;
1700       image->vk.usage |=
1701          VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT;
1702 
1703       /* TODO: enable compression on emulation plane */
1704       isl_extra_usage_flags |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
1705    }
1706 
1707    /* Disable aux if image supports export without modifiers. */
1708    if (image->vk.external_handle_types != 0 &&
1709        image->vk.tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT)
1710       isl_extra_usage_flags |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
1711 
1712    if (device->queue_count > 1) {
1713       /* Notify ISL that the app may access this image from different engines.
1714        * Note that parallel access to the surface will occur regardless of the
1715        * sharing mode.
1716        */
1717       isl_extra_usage_flags |= ISL_SURF_USAGE_MULTI_ENGINE_PAR_BIT;
1718 
1719       /* If the resource is created with the CONCURRENT sharing mode, we can't
1720        * support compression because we aren't allowed barriers in order to
1721        * construct the main surface data with FULL_RESOLVE/PARTIAL_RESOLVE.
1722        */
1723       if (image->vk.sharing_mode == VK_SHARING_MODE_CONCURRENT)
1724          isl_extra_usage_flags |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
1725    }
1726 
1727    /* Aux is pointless if it will never be used as an attachment. */
1728    if (vk_format_is_depth_or_stencil(image->vk.format) &&
1729        !(image->vk.usage & VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT))
1730       isl_extra_usage_flags |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
1731 
1732    /* TODO: Adjust blorp for multi-LOD HiZ surface on Gen9. */
1733    if (vk_format_has_depth(image->vk.format) &&
1734        image->vk.mip_levels > 1 && device->info->ver == 9) {
1735       anv_perf_warn(VK_LOG_OBJS(&image->vk.base), "Enable multi-LOD HiZ");
1736       isl_extra_usage_flags |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
1737    }
1738 
1739    /* Mark WSI images with the right surf usage. */
1740    if (image->from_wsi)
1741       isl_extra_usage_flags |= ISL_SURF_USAGE_DISPLAY_BIT;
1742 
1743    const isl_tiling_flags_t isl_tiling_flags =
1744       choose_isl_tiling_flags(device->info, image, create_info, isl_mod_info);
1745 
1746    const VkImageFormatListCreateInfo *fmt_list =
1747       vk_find_struct_const(pCreateInfo->pNext,
1748                            IMAGE_FORMAT_LIST_CREATE_INFO);
1749 
1750    if ((image->vk.aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) &&
1751        image->vk.samples == 1) {
1752       if (image->n_planes != 1) {
1753          /* Multiplanar images seem to hit a sampler bug with CCS and R16G16
1754           * format. (Putting the clear state a page/4096bytes further fixes
1755           * the issue).
1756           */
1757          isl_extra_usage_flags |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
1758       }
1759 
1760       if ((image->vk.create_flags & VK_IMAGE_CREATE_ALIAS_BIT) &&
1761           !image->from_wsi) {
1762          /* The image may alias a plane of a multiplanar image. Above we ban
1763           * CCS on multiplanar images.
1764           *
1765           * We must also reject aliasing of any image that uses
1766           * ANV_IMAGE_MEMORY_BINDING_PRIVATE. Since we're already rejecting
1767           * all aliasing here, there's no need to further analyze if the image
1768           * needs a private binding.
1769           */
1770          isl_extra_usage_flags |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
1771       }
1772 
1773       if (device->info->ver >= 12 &&
1774           !anv_formats_ccs_e_compatible(device->info, image->vk.create_flags,
1775                                         image->vk.format, image->vk.tiling,
1776                                         image->vk.usage, fmt_list)) {
1777          /* CCS_E is the only aux-mode supported for single sampled color
1778           * surfaces on gfx12+. If we can't support it, we should configure
1779           * the main surface without aux support.
1780           */
1781          isl_extra_usage_flags |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
1782       }
1783    }
1784 
1785    /* Fill out the list of view formats. */
1786    const enum isl_format image_format =
1787       anv_get_format_plane(device->info, image->vk.format, 0,
1788                            image->vk.tiling).isl_format;
1789    add_image_view_format(image, image_format);
1790 
1791    if (image->vk.usage & (VK_IMAGE_USAGE_TRANSFER_DST_BIT |
1792                           VK_IMAGE_USAGE_TRANSFER_SRC_BIT)) {
1793       if (vk_format_is_color(image->vk.format)) {
1794          const enum isl_format blorp_copy_format =
1795             blorp_copy_get_color_format(&device->isl_dev, image_format);
1796          add_image_view_format(image, blorp_copy_format);
1797       } else {
1798          /* We don't have a helper for depth-stencil formats. */
1799          mark_image_view_formats_incomplete(image);
1800       }
1801    }
1802 
1803    if (image->vk.create_flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) {
1804       if (!fmt_list || fmt_list->viewFormatCount == 0) {
1805          /* Without a format list provided, we must assume all compatible
1806           * formats. Instead of adding them all, mark our list as incomplete.
1807           */
1808          mark_image_view_formats_incomplete(image);
1809       } else {
1810          for (uint32_t i = 0; i < fmt_list->viewFormatCount; i++) {
1811             const enum isl_format fmt_list_format =
1812                anv_get_format_plane(device->info,
1813                                     fmt_list->pViewFormats[i], 0,
1814                                     image->vk.tiling).isl_format;
1815             add_image_view_format(image, fmt_list_format);
1816          }
1817       }
1818    }
1819 
1820    if (isl_mod_info && isl_mod_info->supports_clear_color) {
1821       if (image->num_view_formats > 1) {
1822          /* We use the number of view formats to determine the number of
1823           * CLEAR_COLOR structures to append to the image. For an imported
1824           * dmabuf supporting clear colors, we're limited to a single such
1825           * struct. So, mark the view format list as incomplete because doing
1826           * so shrinks the list size to one.
1827           */
1828          mark_image_view_formats_incomplete(image);
1829       }
1830       assert(image->num_view_formats == 1);
1831    }
1832 
1833    if (mod_explicit_info) {
1834       r = add_all_surfaces_explicit_layout(device, image, fmt_list,
1835                                            mod_explicit_info, isl_tiling_flags,
1836                                            isl_extra_usage_flags);
1837    } else {
1838       r = add_all_surfaces_implicit_layout(device, image, fmt_list, create_info->stride,
1839                                            isl_tiling_flags,
1840                                            isl_extra_usage_flags);
1841    }
1842 
1843    if (r != VK_SUCCESS)
1844       goto fail;
1845 
1846    if (image->emu_plane_format != VK_FORMAT_UNDEFINED) {
1847       const struct intel_device_info *devinfo = device->info;
1848       const uint32_t plane = image->n_planes;
1849       const struct anv_format_plane plane_format = anv_get_format_plane(
1850             devinfo, image->emu_plane_format, 0, image->vk.tiling);
1851 
1852       isl_surf_usage_flags_t isl_usage = anv_image_choose_isl_surf_usage(
1853          device->physical, image->vk.create_flags, image->vk.usage,
1854          isl_extra_usage_flags, VK_IMAGE_ASPECT_COLOR_BIT,
1855          image->vk.compr_flags);
1856 
1857       r = add_primary_surface(device, image, plane, plane_format,
1858                               ANV_OFFSET_IMPLICIT, 0,
1859                               isl_tiling_flags, isl_usage);
1860       if (r != VK_SUCCESS)
1861          goto fail;
1862    }
1863 
1864    const VkVideoProfileListInfoKHR *video_profile =
1865       vk_find_struct_const(pCreateInfo->pNext,
1866                            VIDEO_PROFILE_LIST_INFO_KHR);
1867 
1868    bool independent_video_profile =
1869       pCreateInfo->flags & VK_IMAGE_CREATE_VIDEO_PROFILE_INDEPENDENT_BIT_KHR;
1870 
1871    if (video_profile || independent_video_profile) {
1872       r = add_video_buffers(device, image, video_profile, independent_video_profile);
1873       if (r != VK_SUCCESS)
1874          goto fail;
1875    }
1876 
1877    if (!create_info->no_private_binding_alloc) {
1878       r = alloc_private_binding(device, image, pCreateInfo);
1879       if (r != VK_SUCCESS)
1880          goto fail;
1881    }
1882 
1883    check_memory_bindings(device, image);
1884 
1885    r = check_drm_format_mod(device, image);
1886    if (r != VK_SUCCESS)
1887       goto fail;
1888 
1889    if (anv_image_is_sparse(image)) {
1890       r = anv_image_init_sparse_bindings(image, create_info);
1891       if (r != VK_SUCCESS)
1892          goto fail;
1893    }
1894 
1895    return VK_SUCCESS;
1896 
1897 fail:
1898    vk_image_finish(&image->vk);
1899    return r;
1900 }
1901 
1902 void
anv_image_finish(struct anv_image * image)1903 anv_image_finish(struct anv_image *image)
1904 {
1905    struct anv_device *device =
1906       container_of(image->vk.base.device, struct anv_device, vk);
1907 
1908    if (anv_image_is_sparse(image))
1909       anv_image_finish_sparse_bindings(image);
1910 
1911    /* Unmap a CCS so that if the bound region of the image is rebound to
1912     * another image, the AUX tables will be cleared to allow for a new
1913     * mapping.
1914     */
1915    for (int p = 0; p < image->n_planes; ++p) {
1916       if (image->planes[p].aux_tt.mapped) {
1917          intel_aux_map_del_mapping(device->aux_map_ctx,
1918                                    image->planes[p].aux_tt.addr,
1919                                    image->planes[p].aux_tt.size);
1920       }
1921    }
1922 
1923    if (image->from_gralloc) {
1924       assert(!image->disjoint);
1925       assert(image->n_planes == 1);
1926       assert(image->planes[0].primary_surface.memory_range.binding ==
1927              ANV_IMAGE_MEMORY_BINDING_MAIN);
1928       assert(image->bindings[ANV_IMAGE_MEMORY_BINDING_MAIN].address.bo != NULL);
1929       anv_device_release_bo(device, image->bindings[ANV_IMAGE_MEMORY_BINDING_MAIN].address.bo);
1930    }
1931 
1932    for (uint32_t b = 0; b < ARRAY_SIZE(image->bindings); b++) {
1933       if (image->bindings[b].host_map != NULL) {
1934          anv_device_unmap_bo(device,
1935                              image->bindings[b].address.bo,
1936                              image->bindings[b].host_map,
1937                              image->bindings[b].map_size,
1938                              false /* replace */);
1939       }
1940    }
1941 
1942    struct anv_bo *private_bo = image->bindings[ANV_IMAGE_MEMORY_BINDING_PRIVATE].address.bo;
1943    if (private_bo) {
1944       pthread_mutex_lock(&device->mutex);
1945       list_del(&image->link);
1946       pthread_mutex_unlock(&device->mutex);
1947       anv_device_release_bo(device, private_bo);
1948    }
1949 
1950    vk_image_finish(&image->vk);
1951 }
1952 
1953 static struct anv_image *
anv_swapchain_get_image(VkSwapchainKHR swapchain,uint32_t index)1954 anv_swapchain_get_image(VkSwapchainKHR swapchain,
1955                         uint32_t index)
1956 {
1957    VkImage image = wsi_common_get_image(swapchain, index);
1958    return anv_image_from_handle(image);
1959 }
1960 
1961 static VkResult
anv_image_init_from_create_info(struct anv_device * device,struct anv_image * image,const VkImageCreateInfo * pCreateInfo,bool no_private_binding_alloc)1962 anv_image_init_from_create_info(struct anv_device *device,
1963                                 struct anv_image *image,
1964                                 const VkImageCreateInfo *pCreateInfo,
1965                                 bool no_private_binding_alloc)
1966 {
1967    if (pCreateInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) {
1968       VkResult result =
1969          anv_sparse_image_check_support(device->physical,
1970                                         pCreateInfo->flags,
1971                                         pCreateInfo->tiling,
1972                                         pCreateInfo->samples,
1973                                         pCreateInfo->imageType,
1974                                         pCreateInfo->format);
1975       if (result != VK_SUCCESS)
1976          return result;
1977    }
1978 
1979    const VkNativeBufferANDROID *gralloc_info =
1980       vk_find_struct_const(pCreateInfo->pNext, NATIVE_BUFFER_ANDROID);
1981    if (gralloc_info)
1982       return anv_image_init_from_gralloc(device, image, pCreateInfo,
1983                                          gralloc_info);
1984 
1985    struct anv_image_create_info create_info = {
1986       .vk_info = pCreateInfo,
1987       .no_private_binding_alloc = no_private_binding_alloc,
1988    };
1989 
1990    return anv_image_init(device, image, &create_info);
1991 }
1992 
anv_CreateImage(VkDevice _device,const VkImageCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkImage * pImage)1993 VkResult anv_CreateImage(
1994     VkDevice                                    _device,
1995     const VkImageCreateInfo*                    pCreateInfo,
1996     const VkAllocationCallbacks*                pAllocator,
1997     VkImage*                                    pImage)
1998 {
1999    ANV_FROM_HANDLE(anv_device, device, _device);
2000 
2001    if ((device->physical->sparse_type == ANV_SPARSE_TYPE_NOT_SUPPORTED) &&
2002        INTEL_DEBUG(DEBUG_SPARSE) &&
2003        pCreateInfo->flags & (VK_IMAGE_CREATE_SPARSE_BINDING_BIT |
2004                              VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT |
2005                              VK_IMAGE_CREATE_SPARSE_ALIASED_BIT))
2006       fprintf(stderr, "=== %s %s:%d flags:0x%08x\n", __func__, __FILE__,
2007               __LINE__, pCreateInfo->flags);
2008 
2009 #ifndef VK_USE_PLATFORM_ANDROID_KHR
2010    /* Skip the WSI common swapchain creation here on Android. Similar to ahw,
2011     * this case is handled by a partial image init and then resolved when the
2012     * image is bound and gralloc info is passed.
2013     */
2014    const VkImageSwapchainCreateInfoKHR *swapchain_info =
2015       vk_find_struct_const(pCreateInfo->pNext, IMAGE_SWAPCHAIN_CREATE_INFO_KHR);
2016    if (swapchain_info && swapchain_info->swapchain != VK_NULL_HANDLE) {
2017       return wsi_common_create_swapchain_image(&device->physical->wsi_device,
2018                                                pCreateInfo,
2019                                                swapchain_info->swapchain,
2020                                                pImage);
2021    }
2022 #endif
2023 
2024    struct anv_image *image =
2025       vk_object_zalloc(&device->vk, pAllocator, sizeof(*image),
2026                        VK_OBJECT_TYPE_IMAGE);
2027    if (!image)
2028       return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
2029 
2030    VkResult result = anv_image_init_from_create_info(device, image,
2031                                                      pCreateInfo,
2032                                                      false);
2033    if (result != VK_SUCCESS) {
2034       vk_object_free(&device->vk, pAllocator, image);
2035       return result;
2036    }
2037 
2038    ANV_RMV(image_create, device, false, image);
2039 
2040    *pImage = anv_image_to_handle(image);
2041 
2042    return result;
2043 }
2044 
2045 void
anv_DestroyImage(VkDevice _device,VkImage _image,const VkAllocationCallbacks * pAllocator)2046 anv_DestroyImage(VkDevice _device, VkImage _image,
2047                  const VkAllocationCallbacks *pAllocator)
2048 {
2049    ANV_FROM_HANDLE(anv_device, device, _device);
2050    ANV_FROM_HANDLE(anv_image, image, _image);
2051 
2052    if (!image)
2053       return;
2054 
2055    ANV_RMV(image_destroy, device, image);
2056 
2057    assert(&device->vk == image->vk.base.device);
2058    anv_image_finish(image);
2059 
2060    vk_free2(&device->vk.alloc, pAllocator, image);
2061 }
2062 
2063 /* We are binding AHardwareBuffer. Get a description, resolve the
2064  * format and prepare anv_image properly.
2065  */
2066 static void
resolve_ahw_image(struct anv_device * device,struct anv_image * image,struct anv_device_memory * mem)2067 resolve_ahw_image(struct anv_device *device,
2068                   struct anv_image *image,
2069                   struct anv_device_memory *mem)
2070 {
2071 #if DETECT_OS_ANDROID && ANDROID_API_LEVEL >= 26
2072    assert(mem->vk.ahardware_buffer);
2073    AHardwareBuffer_Desc desc;
2074    AHardwareBuffer_describe(mem->vk.ahardware_buffer, &desc);
2075    VkResult result;
2076 
2077    /* Check tiling. */
2078    enum isl_tiling tiling;
2079    const native_handle_t *handle =
2080       AHardwareBuffer_getNativeHandle(mem->vk.ahardware_buffer);
2081    struct u_gralloc_buffer_handle gr_handle = {
2082       .handle = handle,
2083       .hal_format = desc.format,
2084       .pixel_stride = desc.stride,
2085    };
2086    result = anv_android_get_tiling(device, &gr_handle, &tiling);
2087    assert(result == VK_SUCCESS);
2088    isl_tiling_flags_t isl_tiling_flags = (1u << tiling);
2089 
2090    /* Check format. */
2091    VkFormat vk_format = vk_format_from_android(desc.format, desc.usage);
2092    assert(vk_format != VK_FORMAT_UNDEFINED);
2093 
2094    /* Now we are able to fill anv_image fields properly and create
2095     * isl_surface for it.
2096     */
2097    vk_image_set_format(&image->vk, vk_format);
2098    image->n_planes = anv_get_format_planes(image->vk.format);
2099 
2100    result = add_all_surfaces_implicit_layout(device, image, NULL, desc.stride,
2101                                              isl_tiling_flags,
2102                                              ISL_SURF_USAGE_DISABLE_AUX_BIT);
2103    assert(result == VK_SUCCESS);
2104 #endif
2105 }
2106 
2107 static void
resolve_anb_image(struct anv_device * device,struct anv_image * image,const VkNativeBufferANDROID * gralloc_info)2108 resolve_anb_image(struct anv_device *device,
2109                   struct anv_image *image,
2110                   const VkNativeBufferANDROID *gralloc_info)
2111 {
2112 #if DETECT_OS_ANDROID && ANDROID_API_LEVEL >= 29
2113    VkResult result;
2114 
2115    /* Check tiling. */
2116    enum isl_tiling tiling;
2117    struct u_gralloc_buffer_handle gr_handle = {
2118       .handle = gralloc_info->handle,
2119       .hal_format = gralloc_info->format,
2120       .pixel_stride = gralloc_info->stride,
2121    };
2122    result = anv_android_get_tiling(device, &gr_handle, &tiling);
2123    assert(result == VK_SUCCESS);
2124 
2125    isl_tiling_flags_t isl_tiling_flags = (1u << tiling);
2126 
2127    /* Now we are able to fill anv_image fields properly and create
2128     * isl_surface for it.
2129     */
2130    result = add_all_surfaces_implicit_layout(device, image, NULL, gralloc_info->stride,
2131                                              isl_tiling_flags,
2132                                              ISL_SURF_USAGE_DISABLE_AUX_BIT);
2133    assert(result == VK_SUCCESS);
2134 #endif
2135 }
2136 
2137 static bool
anv_image_is_pat_compressible(struct anv_device * device,struct anv_image * image)2138 anv_image_is_pat_compressible(struct anv_device *device, struct anv_image *image)
2139 {
2140    if (INTEL_DEBUG(DEBUG_NO_CCS))
2141       return false;
2142 
2143    if (device->info->ver < 20)
2144       return false;
2145 
2146    /*
2147     * Be aware that Vulkan spec requires that Images with some properties
2148     * always returns the same memory types, so this function also needs to
2149     * have the same return for the same set of properties.
2150     *
2151     *    For images created with a color format, the memoryTypeBits member is
2152     *    identical for all VkImage objects created with the same combination
2153     *    of values for the tiling member, the
2154     *    VK_IMAGE_CREATE_SPARSE_BINDING_BIT bit and
2155     *    VK_IMAGE_CREATE_PROTECTED_BIT bit of the flags member, the
2156     *    VK_IMAGE_CREATE_SPLIT_INSTANCE_BIND_REGIONS_BIT bit of the flags
2157     *    member, the VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT bit of the usage
2158     *    member if the
2159     *    VkPhysicalDeviceHostImageCopyPropertiesEXT::identicalMemoryTypeRequirements
2160     *    property is VK_FALSE, handleTypes member of
2161     *    VkExternalMemoryImageCreateInfo, and the
2162     *    VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT of the usage member in the
2163     *    VkImageCreateInfo structure passed to vkCreateImage.
2164     *
2165     *    For images created with a depth/stencil format, the memoryTypeBits
2166     *    member is identical for all VkImage objects created with the same
2167     *    combination of values for the format member, the tiling member, the
2168     *    VK_IMAGE_CREATE_SPARSE_BINDING_BIT bit and
2169     *    VK_IMAGE_CREATE_PROTECTED_BIT bit of the flags member, the
2170     *    VK_IMAGE_CREATE_SPLIT_INSTANCE_BIND_REGIONS_BIT bit of the flags
2171     *    member, the VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT bit of the usage
2172     *    member if the
2173     *    VkPhysicalDeviceHostImageCopyPropertiesEXT::identicalMemoryTypeRequirements
2174     *    property is VK_FALSE, handleTypes member of
2175     *    VkExternalMemoryImageCreateInfo, and the
2176     *    VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT of the usage member in the
2177     *    VkImageCreateInfo structure passed to vkCreateImage.
2178     */
2179 
2180    /* There are no compression-enabled modifiers on Xe2, and all legacy
2181     * modifiers are not defined with compression. We simply disable
2182     * compression on all modifiers.
2183     *
2184     * We disable this in anv_AllocateMemory() as well.
2185     */
2186    if (image->vk.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT)
2187       return false;
2188 
2189    /* Host accessed images cannot be compressed. */
2190    if (image->vk.usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT)
2191       return false;
2192 
2193    return true;
2194 }
2195 
2196 void
anv_image_get_memory_requirements(struct anv_device * device,struct anv_image * image,VkImageAspectFlags aspects,VkMemoryRequirements2 * pMemoryRequirements)2197 anv_image_get_memory_requirements(struct anv_device *device,
2198                                   struct anv_image *image,
2199                                   VkImageAspectFlags aspects,
2200                                   VkMemoryRequirements2 *pMemoryRequirements)
2201 {
2202    /* The Vulkan spec (git aaed022) says:
2203     *
2204     *    memoryTypeBits is a bitfield and contains one bit set for every
2205     *    supported memory type for the resource. The bit `1<<i` is set if and
2206     *    only if the memory type `i` in the VkPhysicalDeviceMemoryProperties
2207     *    structure for the physical device is supported.
2208     */
2209    uint32_t memory_types;
2210 
2211    if (image->vk.create_flags & VK_IMAGE_CREATE_PROTECTED_BIT) {
2212       memory_types = device->physical->memory.protected_mem_types;
2213    } else {
2214       memory_types = device->physical->memory.default_buffer_mem_types;
2215       if (anv_image_is_pat_compressible(device, image))
2216          memory_types |= device->physical->memory.compressed_mem_types;
2217    }
2218 
2219    if (image->vk.usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT) {
2220       /* Remove non host visible heaps from the types for host transfers on
2221        * non ReBAR devices
2222        */
2223       if (device->physical->has_small_bar) {
2224          for (uint32_t i = 0; i < device->physical->memory.type_count; i++) {
2225             if (!(device->physical->memory.types[i].propertyFlags &
2226                   VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT))
2227                memory_types &= ~BITFIELD_BIT(i);
2228          }
2229       }
2230    }
2231 
2232    vk_foreach_struct(ext, pMemoryRequirements->pNext) {
2233       switch (ext->sType) {
2234       case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS: {
2235          VkMemoryDedicatedRequirements *requirements = (void *)ext;
2236          if (image->vk.wsi_legacy_scanout ||
2237              image->from_ahb ||
2238              (isl_drm_modifier_has_aux(image->vk.drm_format_mod) &&
2239               anv_image_uses_aux_map(device, image))) {
2240             /* If we need to set the tiling for external consumers or the
2241              * modifier involves AUX tables, we need a dedicated allocation.
2242              *
2243              * See also anv_AllocateMemory.
2244              */
2245             requirements->prefersDedicatedAllocation = true;
2246             requirements->requiresDedicatedAllocation = true;
2247          } else {
2248             requirements->prefersDedicatedAllocation = false;
2249             requirements->requiresDedicatedAllocation = false;
2250          }
2251          break;
2252       }
2253 
2254       default:
2255          vk_debug_ignored_stype(ext->sType);
2256          break;
2257       }
2258    }
2259 
2260    /* If the image is disjoint, then we must return the memory requirements for
2261     * the single plane specified in VkImagePlaneMemoryRequirementsInfo. If
2262     * non-disjoint, then exactly one set of memory requirements exists for the
2263     * whole image.
2264     *
2265     * This is enforced by the Valid Usage for VkImageMemoryRequirementsInfo2,
2266     * which requires that the app provide VkImagePlaneMemoryRequirementsInfo if
2267     * and only if the image is disjoint (that is, multi-planar format and
2268     * VK_IMAGE_CREATE_DISJOINT_BIT).
2269     */
2270    enum anv_image_memory_binding binding;
2271    if (image->disjoint) {
2272       assert(util_bitcount(aspects) == 1);
2273       assert(aspects & image->vk.aspects);
2274       binding = anv_image_aspect_to_binding(image, aspects);
2275    } else {
2276       assert(aspects == image->vk.aspects);
2277       binding = ANV_IMAGE_MEMORY_BINDING_MAIN;
2278    }
2279 
2280    pMemoryRequirements->memoryRequirements = (VkMemoryRequirements) {
2281       .size = image->bindings[binding].memory_range.size,
2282       .alignment = image->bindings[binding].memory_range.alignment,
2283       .memoryTypeBits = memory_types,
2284    };
2285 }
2286 
anv_GetImageMemoryRequirements2(VkDevice _device,const VkImageMemoryRequirementsInfo2 * pInfo,VkMemoryRequirements2 * pMemoryRequirements)2287 void anv_GetImageMemoryRequirements2(
2288     VkDevice                                    _device,
2289     const VkImageMemoryRequirementsInfo2*       pInfo,
2290     VkMemoryRequirements2*                      pMemoryRequirements)
2291 {
2292    ANV_FROM_HANDLE(anv_device, device, _device);
2293    ANV_FROM_HANDLE(anv_image, image, pInfo->image);
2294 
2295    VkImageAspectFlags aspects = image->vk.aspects;
2296 
2297    vk_foreach_struct_const(ext, pInfo->pNext) {
2298       switch (ext->sType) {
2299       case VK_STRUCTURE_TYPE_IMAGE_PLANE_MEMORY_REQUIREMENTS_INFO: {
2300          assert(image->disjoint);
2301          const VkImagePlaneMemoryRequirementsInfo *plane_reqs =
2302             (const VkImagePlaneMemoryRequirementsInfo *) ext;
2303          aspects = plane_reqs->planeAspect;
2304          break;
2305       }
2306 
2307       default:
2308          vk_debug_ignored_stype(ext->sType);
2309          break;
2310       }
2311    }
2312 
2313    anv_image_get_memory_requirements(device, image, aspects,
2314                                      pMemoryRequirements);
2315 }
2316 
anv_GetDeviceImageMemoryRequirements(VkDevice _device,const VkDeviceImageMemoryRequirements * pInfo,VkMemoryRequirements2 * pMemoryRequirements)2317 void anv_GetDeviceImageMemoryRequirements(
2318     VkDevice                                    _device,
2319     const VkDeviceImageMemoryRequirements*   pInfo,
2320     VkMemoryRequirements2*                      pMemoryRequirements)
2321 {
2322    ANV_FROM_HANDLE(anv_device, device, _device);
2323    struct anv_image image = { 0 };
2324 
2325    if ((device->physical->sparse_type == ANV_SPARSE_TYPE_NOT_SUPPORTED) &&
2326        INTEL_DEBUG(DEBUG_SPARSE) &&
2327        pInfo->pCreateInfo->flags & (VK_IMAGE_CREATE_SPARSE_BINDING_BIT |
2328                                     VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT |
2329                                     VK_IMAGE_CREATE_SPARSE_ALIASED_BIT))
2330       fprintf(stderr, "=== %s %s:%d flags:0x%08x\n", __func__, __FILE__,
2331               __LINE__, pInfo->pCreateInfo->flags);
2332 
2333    ASSERTED VkResult result =
2334       anv_image_init_from_create_info(device, &image, pInfo->pCreateInfo, true);
2335    assert(result == VK_SUCCESS);
2336 
2337    VkImageAspectFlags aspects =
2338       image.disjoint ? pInfo->planeAspect : image.vk.aspects;
2339 
2340    anv_image_get_memory_requirements(device, &image, aspects,
2341                                      pMemoryRequirements);
2342    anv_image_finish(&image);
2343 }
2344 
2345 static void
anv_image_get_sparse_memory_requirements(struct anv_device * device,struct anv_image * image,VkImageAspectFlags aspects,uint32_t * pSparseMemoryRequirementCount,VkSparseImageMemoryRequirements2 * pSparseMemoryRequirements)2346 anv_image_get_sparse_memory_requirements(
2347       struct anv_device *device,
2348       struct anv_image *image,
2349       VkImageAspectFlags aspects,
2350       uint32_t *pSparseMemoryRequirementCount,
2351       VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
2352 {
2353    VK_OUTARRAY_MAKE_TYPED(VkSparseImageMemoryRequirements2, reqs,
2354                           pSparseMemoryRequirements,
2355                           pSparseMemoryRequirementCount);
2356 
2357    /* From the spec:
2358     *   "The sparse image must have been created using the
2359     *    VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT flag to retrieve valid sparse
2360     *    image memory requirements."
2361     */
2362    if (!(image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT))
2363       return;
2364 
2365    VkSparseImageMemoryRequirements ds_mem_reqs = {};
2366    VkSparseImageMemoryRequirements2 *ds_reqs_ptr = NULL;
2367 
2368    u_foreach_bit(b, aspects) {
2369       VkImageAspectFlagBits aspect = 1 << b;
2370       const uint32_t plane = anv_image_aspect_to_plane(image, aspect);
2371       struct isl_surf *surf = &image->planes[plane].primary_surface.isl;
2372 
2373       VkSparseImageFormatProperties format_props =
2374          anv_sparse_calc_image_format_properties(device->physical, aspect,
2375                                                  image->vk.image_type,
2376                                                  image->vk.samples, surf);
2377 
2378       uint32_t miptail_first_lod;
2379       VkDeviceSize miptail_size, miptail_offset, miptail_stride;
2380       anv_sparse_calc_miptail_properties(device, image, aspect,
2381                                          &miptail_first_lod, &miptail_size,
2382                                          &miptail_offset, &miptail_stride);
2383 
2384       VkSparseImageMemoryRequirements mem_reqs = {
2385          .formatProperties = format_props,
2386          .imageMipTailFirstLod = miptail_first_lod,
2387          .imageMipTailSize = miptail_size,
2388          .imageMipTailOffset = miptail_offset,
2389          .imageMipTailStride = miptail_stride,
2390       };
2391 
2392       /* If both depth and stencil are the same, unify them if possible. */
2393       if (aspect & (VK_IMAGE_ASPECT_DEPTH_BIT |
2394                     VK_IMAGE_ASPECT_STENCIL_BIT)) {
2395          if (!ds_reqs_ptr) {
2396             ds_mem_reqs = mem_reqs;
2397          } else if (ds_mem_reqs.formatProperties.imageGranularity.width ==
2398                        mem_reqs.formatProperties.imageGranularity.width &&
2399                     ds_mem_reqs.formatProperties.imageGranularity.height ==
2400                        mem_reqs.formatProperties.imageGranularity.height &&
2401                     ds_mem_reqs.formatProperties.imageGranularity.depth ==
2402                        mem_reqs.formatProperties.imageGranularity.depth &&
2403                     ds_mem_reqs.imageMipTailFirstLod ==
2404                        mem_reqs.imageMipTailFirstLod &&
2405                     ds_mem_reqs.imageMipTailSize ==
2406                        mem_reqs.imageMipTailSize &&
2407                     ds_mem_reqs.imageMipTailOffset ==
2408                        mem_reqs.imageMipTailOffset &&
2409                     ds_mem_reqs.imageMipTailStride ==
2410                        mem_reqs.imageMipTailStride) {
2411             ds_reqs_ptr->memoryRequirements.formatProperties.aspectMask |=
2412                aspect;
2413             continue;
2414          }
2415       }
2416 
2417       vk_outarray_append_typed(VkSparseImageMemoryRequirements2, &reqs, r) {
2418          r->memoryRequirements = mem_reqs;
2419          if (aspect & (VK_IMAGE_ASPECT_DEPTH_BIT |
2420                        VK_IMAGE_ASPECT_STENCIL_BIT))
2421             ds_reqs_ptr = r;
2422       }
2423    }
2424 }
2425 
anv_GetImageSparseMemoryRequirements2(VkDevice _device,const VkImageSparseMemoryRequirementsInfo2 * pInfo,uint32_t * pSparseMemoryRequirementCount,VkSparseImageMemoryRequirements2 * pSparseMemoryRequirements)2426 void anv_GetImageSparseMemoryRequirements2(
2427     VkDevice                                    _device,
2428     const VkImageSparseMemoryRequirementsInfo2* pInfo,
2429     uint32_t*                                   pSparseMemoryRequirementCount,
2430     VkSparseImageMemoryRequirements2*           pSparseMemoryRequirements)
2431 {
2432    ANV_FROM_HANDLE(anv_device, device, _device);
2433    ANV_FROM_HANDLE(anv_image, image, pInfo->image);
2434 
2435    if (!anv_sparse_residency_is_enabled(device)) {
2436       if ((device->physical->sparse_type == ANV_SPARSE_TYPE_NOT_SUPPORTED) &&
2437           INTEL_DEBUG(DEBUG_SPARSE))
2438          fprintf(stderr, "=== [%s:%d] [%s]\n", __FILE__, __LINE__, __func__);
2439 
2440       *pSparseMemoryRequirementCount = 0;
2441       return;
2442    }
2443 
2444    anv_image_get_sparse_memory_requirements(device, image, image->vk.aspects,
2445                                             pSparseMemoryRequirementCount,
2446                                             pSparseMemoryRequirements);
2447 }
2448 
anv_GetDeviceImageSparseMemoryRequirements(VkDevice _device,const VkDeviceImageMemoryRequirements * pInfo,uint32_t * pSparseMemoryRequirementCount,VkSparseImageMemoryRequirements2 * pSparseMemoryRequirements)2449 void anv_GetDeviceImageSparseMemoryRequirements(
2450     VkDevice                                    _device,
2451     const VkDeviceImageMemoryRequirements*      pInfo,
2452     uint32_t*                                   pSparseMemoryRequirementCount,
2453     VkSparseImageMemoryRequirements2*           pSparseMemoryRequirements)
2454 {
2455    ANV_FROM_HANDLE(anv_device, device, _device);
2456    struct anv_image image = { 0 };
2457 
2458    if (!anv_sparse_residency_is_enabled(device)) {
2459       if ((device->physical->sparse_type == ANV_SPARSE_TYPE_NOT_SUPPORTED) &&
2460           INTEL_DEBUG(DEBUG_SPARSE))
2461          fprintf(stderr, "=== [%s:%d] [%s]\n", __FILE__, __LINE__, __func__);
2462 
2463       *pSparseMemoryRequirementCount = 0;
2464       return;
2465    }
2466 
2467    /* This function is similar to anv_GetDeviceImageMemoryRequirements, in
2468     * which it actually creates an image, gets the properties and then
2469     * destroys the image.
2470     *
2471     * We could one day refactor things to allow us to gather the properties
2472     * without having to actually create the image, maybe by reworking ISL to
2473     * separate creation from parameter computing.
2474     */
2475    VkResult result =
2476       anv_image_init_from_create_info(device, &image, pInfo->pCreateInfo,
2477                                       true /* no_private_binding_alloc */);
2478    if (result != VK_SUCCESS) {
2479       *pSparseMemoryRequirementCount = 0;
2480       return;
2481    }
2482 
2483    /* The spec says:
2484     *  "planeAspect is a VkImageAspectFlagBits value specifying the aspect
2485     *   corresponding to the image plane to query. This parameter is ignored
2486     *   unless pCreateInfo::tiling is VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT,
2487     *   or pCreateInfo::flags has VK_IMAGE_CREATE_DISJOINT_BIT set."
2488     */
2489    VkImageAspectFlags aspects =
2490       (pInfo->pCreateInfo->flags & VK_IMAGE_CREATE_DISJOINT_BIT) ||
2491       (pInfo->pCreateInfo->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT)
2492          ? pInfo->planeAspect : image.vk.aspects;
2493 
2494    anv_image_get_sparse_memory_requirements(device, &image, aspects,
2495                                             pSparseMemoryRequirementCount,
2496                                             pSparseMemoryRequirements);
2497 
2498    anv_image_finish(&image);
2499 }
2500 
2501 static bool
anv_image_map_aux_tt(struct anv_device * device,struct anv_image * image,uint32_t plane)2502 anv_image_map_aux_tt(struct anv_device *device,
2503                      struct anv_image *image, uint32_t plane)
2504 {
2505    const struct anv_address main_addr = anv_image_address(
2506       image, &image->planes[plane].primary_surface.memory_range);
2507    struct anv_bo *bo = main_addr.bo;
2508    assert(bo != NULL);
2509 
2510    /* If the additional memory padding was added at the end of the BO for CCS
2511     * data, map this region at the granularity of the main/CCS pages.
2512     *
2513     * Otherwise the image should have additional CCS data at the computed
2514     * offset.
2515     */
2516    if (device->physical->alloc_aux_tt_mem &&
2517        (bo->alloc_flags & ANV_BO_ALLOC_AUX_CCS)) {
2518       uint64_t main_aux_alignment =
2519          intel_aux_map_get_alignment(device->aux_map_ctx);
2520       assert(bo->offset % main_aux_alignment == 0);
2521       const struct anv_address start_addr = (struct anv_address) {
2522          .bo = bo,
2523          .offset = ROUND_DOWN_TO(main_addr.offset, main_aux_alignment),
2524       };
2525       const struct anv_address aux_addr = (struct anv_address) {
2526          .bo = bo,
2527          .offset = bo->ccs_offset +
2528                    intel_aux_main_to_aux_offset(device->aux_map_ctx,
2529                                                 start_addr.offset),
2530       };
2531       const struct isl_surf *surf = &image->planes[plane].primary_surface.isl;
2532       const uint64_t format_bits =
2533          intel_aux_map_format_bits_for_isl_surf(surf);
2534       /* Make sure to have the mapping cover the entire image from the aux
2535        * aligned start.
2536        */
2537       const uint64_t main_size = align(
2538          (main_addr.offset - start_addr.offset) + surf->size_B,
2539          main_aux_alignment);
2540 
2541       if (intel_aux_map_add_mapping(device->aux_map_ctx,
2542                                     anv_address_physical(start_addr),
2543                                     anv_address_physical(aux_addr),
2544                                     main_size, format_bits)) {
2545          image->planes[plane].aux_tt.mapped = true;
2546          image->planes[plane].aux_tt.addr = anv_address_physical(start_addr);
2547          image->planes[plane].aux_tt.size = main_size;
2548          return true;
2549       }
2550    } else {
2551       if (anv_address_allows_aux_map(device, main_addr)) {
2552          const struct anv_address aux_addr =
2553             anv_image_address(image,
2554                               &image->planes[plane].compr_ctrl_memory_range);
2555          const struct isl_surf *surf =
2556             &image->planes[plane].primary_surface.isl;
2557          const uint64_t format_bits =
2558             intel_aux_map_format_bits_for_isl_surf(surf);
2559          if (intel_aux_map_add_mapping(device->aux_map_ctx,
2560                                        anv_address_physical(main_addr),
2561                                        anv_address_physical(aux_addr),
2562                                        surf->size_B, format_bits)) {
2563             image->planes[plane].aux_tt.mapped = true;
2564             image->planes[plane].aux_tt.addr = anv_address_physical(main_addr);
2565             image->planes[plane].aux_tt.size = surf->size_B;
2566             return true;
2567          }
2568       }
2569    }
2570 
2571    return false;
2572 
2573 }
2574 
2575 static VkResult
anv_image_bind_address(struct anv_device * device,struct anv_image * image,enum anv_image_memory_binding binding,struct anv_address address)2576 anv_image_bind_address(struct anv_device *device,
2577                         struct anv_image *image,
2578                         enum anv_image_memory_binding binding,
2579                         struct anv_address address)
2580 {
2581    image->bindings[binding].address = address;
2582 
2583    /* Map bindings for images with host transfer usage, so that we don't have
2584     * to map/unmap things at every host operation. We map cached, that means
2585     * that the copy operations need to cflush on platforms that have no
2586     * host_cache+host_coherent memory types.
2587     */
2588    if (image->vk.usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT) {
2589       uint64_t offset = image->bindings[binding].address.offset +
2590                         image->bindings[binding].memory_range.offset;
2591       uint64_t map_offset, map_size;
2592       anv_sanitize_map_params(device, offset,
2593                               image->bindings[binding].memory_range.size,
2594                               &map_offset, &map_size);
2595 
2596       VkResult result = anv_device_map_bo(device,
2597                                           image->bindings[binding].address.bo,
2598                                           map_offset, map_size,
2599                                           NULL /* placed_addr */,
2600                                           &image->bindings[binding].host_map);
2601       if (result != VK_SUCCESS)
2602          return result;
2603 
2604       image->bindings[binding].map_delta = (offset - map_offset);
2605       image->bindings[binding].map_size = map_size;
2606    }
2607 
2608    ANV_RMV(image_bind, device, image, binding);
2609 
2610    return VK_SUCCESS;
2611 }
2612 
2613 static VkResult
anv_bind_image_memory(struct anv_device * device,const VkBindImageMemoryInfo * bind_info)2614 anv_bind_image_memory(struct anv_device *device,
2615                       const VkBindImageMemoryInfo *bind_info)
2616 {
2617    ANV_FROM_HANDLE(anv_device_memory, mem, bind_info->memory);
2618    ANV_FROM_HANDLE(anv_image, image, bind_info->image);
2619    bool did_bind = false;
2620    VkResult result = VK_SUCCESS;
2621 
2622    const VkBindMemoryStatusKHR *bind_status =
2623       vk_find_struct_const(bind_info->pNext, BIND_MEMORY_STATUS_KHR);
2624 
2625    assert(!anv_image_is_sparse(image));
2626 
2627    /* Resolve will alter the image's aspects, do this first. */
2628    if (mem && mem->vk.ahardware_buffer)
2629       resolve_ahw_image(device, image, mem);
2630 
2631    vk_foreach_struct_const(s, bind_info->pNext) {
2632       switch (s->sType) {
2633       case VK_STRUCTURE_TYPE_BIND_IMAGE_PLANE_MEMORY_INFO: {
2634          const VkBindImagePlaneMemoryInfo *plane_info =
2635             (const VkBindImagePlaneMemoryInfo *) s;
2636 
2637          /* Workaround for possible spec bug.
2638           *
2639           * Unlike VkImagePlaneMemoryRequirementsInfo, which requires that
2640           * the image be disjoint (that is, multi-planar format and
2641           * VK_IMAGE_CREATE_DISJOINT_BIT), VkBindImagePlaneMemoryInfo allows
2642           * the image to be non-disjoint and requires only that the image
2643           * have the DISJOINT flag. In this case, regardless of the value of
2644           * VkImagePlaneMemoryRequirementsInfo::planeAspect, the behavior is
2645           * the same as if VkImagePlaneMemoryRequirementsInfo were omitted.
2646           */
2647          if (!image->disjoint)
2648             break;
2649 
2650          enum anv_image_memory_binding binding =
2651             anv_image_aspect_to_binding(image, plane_info->planeAspect);
2652 
2653          anv_image_bind_address(device, image, binding,
2654                                 (struct anv_address) {
2655                                    .bo = mem->bo,
2656                                    .offset = bind_info->memoryOffset,
2657                                 });
2658 
2659          did_bind = true;
2660          break;
2661       }
2662       case VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_SWAPCHAIN_INFO_KHR: {
2663          /* Ignore this struct on Android, we cannot access swapchain
2664           * structures there.
2665           */
2666 #ifndef VK_USE_PLATFORM_ANDROID_KHR
2667          const VkBindImageMemorySwapchainInfoKHR *swapchain_info =
2668             (const VkBindImageMemorySwapchainInfoKHR *) s;
2669          struct anv_image *swapchain_image =
2670             anv_swapchain_get_image(swapchain_info->swapchain,
2671                                     swapchain_info->imageIndex);
2672          assert(swapchain_image);
2673          assert(image->vk.aspects == swapchain_image->vk.aspects);
2674          assert(mem == NULL);
2675 
2676          for (int j = 0; j < ARRAY_SIZE(image->bindings); ++j) {
2677             assert(memory_ranges_equal(image->bindings[j].memory_range,
2678                                        swapchain_image->bindings[j].memory_range));
2679             if (image->bindings[j].memory_range.size != 0) {
2680                anv_image_bind_address(device, image, j,
2681                                       swapchain_image->bindings[j].address);
2682             }
2683          }
2684 
2685          /* We must bump the private binding's bo's refcount because, unlike the other
2686           * bindings, its lifetime is not application-managed.
2687           */
2688          struct anv_bo *private_bo =
2689             image->bindings[ANV_IMAGE_MEMORY_BINDING_PRIVATE].address.bo;
2690          if (private_bo)
2691             anv_bo_ref(private_bo);
2692 
2693          did_bind = true;
2694 #endif
2695          break;
2696       }
2697 #pragma GCC diagnostic push
2698 #pragma GCC diagnostic ignored "-Wswitch"
2699       case VK_STRUCTURE_TYPE_NATIVE_BUFFER_ANDROID: {
2700          const VkNativeBufferANDROID *gralloc_info =
2701             (const VkNativeBufferANDROID *)s;
2702          result = anv_image_bind_from_gralloc(device, image, gralloc_info);
2703          if (result != VK_SUCCESS)
2704             return result;
2705 
2706          resolve_anb_image(device, image, gralloc_info);
2707          did_bind = true;
2708          break;
2709       }
2710 #pragma GCC diagnostic pop
2711       default:
2712          vk_debug_ignored_stype(s->sType);
2713          break;
2714       }
2715    }
2716 
2717    if (!did_bind) {
2718       assert(!image->disjoint);
2719       result = anv_image_bind_address(device, image,
2720                                       ANV_IMAGE_MEMORY_BINDING_MAIN,
2721                                       (struct anv_address) {
2722                                          .bo = mem->bo,
2723                                          .offset = bind_info->memoryOffset,
2724                                       });
2725       did_bind = true;
2726    }
2727 
2728    /* Now that we have the BO, finalize CCS setup. */
2729    for (int p = 0; p < image->n_planes; ++p) {
2730       enum anv_image_memory_binding binding =
2731          image->planes[p].primary_surface.memory_range.binding;
2732       const struct anv_bo *bo =
2733          image->bindings[binding].address.bo;
2734 
2735       if (!bo || !isl_aux_usage_has_ccs(image->planes[p].aux_usage))
2736          continue;
2737 
2738       /* Do nothing if flat CCS requirements are satisfied.
2739        *
2740        * Also, assume that imported BOs with a modifier including
2741        * CCS live only in local memory. Otherwise the exporter should
2742        * have failed the creation of the BO.
2743        */
2744       if (device->info->has_flat_ccs &&
2745           (anv_bo_is_vram_only(bo) ||
2746            (bo->alloc_flags & ANV_BO_ALLOC_IMPORTED)))
2747          continue;
2748 
2749       /* If the AUX-TT mapping succeeds, there is nothing else to do. */
2750       if (device->info->has_aux_map && anv_image_map_aux_tt(device, image, p))
2751          continue;
2752 
2753       /* Do nothing except for gfx12. There are no special requirements. */
2754       if (device->info->ver != 12)
2755          continue;
2756 
2757       /* The plane's BO cannot support CCS, disable compression on it. */
2758       assert(!isl_drm_modifier_has_aux(image->vk.drm_format_mod));
2759 
2760       anv_perf_warn(VK_LOG_OBJS(&image->vk.base),
2761                     "BO lacks CCS support. Disabling the CCS aux usage.");
2762 
2763       if (image->planes[p].aux_usage == ISL_AUX_USAGE_MCS_CCS) {
2764          assert(image->planes[p].aux_surface.memory_range.size);
2765          image->planes[p].aux_usage = ISL_AUX_USAGE_MCS;
2766       } else if (image->planes[p].aux_usage == ISL_AUX_USAGE_HIZ_CCS ||
2767                  image->planes[p].aux_usage == ISL_AUX_USAGE_HIZ_CCS_WT) {
2768          assert(image->planes[p].aux_surface.memory_range.size);
2769          image->planes[p].aux_usage = ISL_AUX_USAGE_HIZ;
2770       } else {
2771          assert(image->planes[p].aux_usage == ISL_AUX_USAGE_CCS_E ||
2772                 image->planes[p].aux_usage == ISL_AUX_USAGE_FCV_CCS_E ||
2773                 image->planes[p].aux_usage == ISL_AUX_USAGE_STC_CCS);
2774          image->planes[p].aux_usage = ISL_AUX_USAGE_NONE;
2775       }
2776    }
2777 
2778    if (bind_status)
2779       *bind_status->pResult = result;
2780 
2781    return result;
2782 }
2783 
anv_BindImageMemory2(VkDevice _device,uint32_t bindInfoCount,const VkBindImageMemoryInfo * pBindInfos)2784 VkResult anv_BindImageMemory2(
2785     VkDevice                                    _device,
2786     uint32_t                                    bindInfoCount,
2787     const VkBindImageMemoryInfo*                pBindInfos)
2788 {
2789    ANV_FROM_HANDLE(anv_device, device, _device);
2790    VkResult result = VK_SUCCESS;
2791 
2792    for (uint32_t i = 0; i < bindInfoCount; i++) {
2793       VkResult res = anv_bind_image_memory(device, &pBindInfos[i]);
2794       if (result == VK_SUCCESS && res != VK_SUCCESS)
2795          result = res;
2796    }
2797 
2798    return result;
2799 }
2800 
2801 static void
anv_get_image_subresource_layout(struct anv_device * device,const struct anv_image * image,const VkImageSubresource2KHR * subresource,VkSubresourceLayout2KHR * layout)2802 anv_get_image_subresource_layout(struct anv_device *device,
2803                                  const struct anv_image *image,
2804                                  const VkImageSubresource2KHR *subresource,
2805                                  VkSubresourceLayout2KHR *layout)
2806 {
2807    const struct isl_surf *isl_surf = NULL;
2808    const struct anv_image_memory_range *mem_range;
2809    uint64_t row_pitch_B;
2810 
2811    assert(__builtin_popcount(subresource->imageSubresource.aspectMask) == 1);
2812 
2813    /* The Vulkan spec requires that aspectMask be
2814     * VK_IMAGE_ASPECT_MEMORY_PLANE_i_BIT_EXT if tiling is
2815     * VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT.
2816     *
2817     * For swapchain images, the Vulkan spec says that every swapchain image has
2818     * tiling VK_IMAGE_TILING_OPTIMAL, but we may choose
2819     * VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT internally.  Vulkan doesn't allow
2820     * vkGetImageSubresourceLayout for images with VK_IMAGE_TILING_OPTIMAL,
2821     * therefore it's invalid for the application to call this on a swapchain
2822     * image.  The WSI code, however, knows when it has internally created
2823     * a swapchain image with VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT,
2824     * so it _should_ correctly use VK_IMAGE_ASPECT_MEMORY_PLANE_* in that case.
2825     * But it incorrectly uses VK_IMAGE_ASPECT_PLANE_*, so we have a temporary
2826     * workaround.
2827     *
2828     * https://gitlab.freedesktop.org/mesa/mesa/-/issues/10176
2829     */
2830    if (image->vk.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
2831       /* TODO(chadv): Drop this workaround when WSI gets fixed. */
2832       uint32_t mem_plane;
2833       switch (subresource->imageSubresource.aspectMask) {
2834       case VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT:
2835       case VK_IMAGE_ASPECT_PLANE_0_BIT:
2836          mem_plane = 0;
2837          break;
2838       case VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT:
2839       case VK_IMAGE_ASPECT_PLANE_1_BIT:
2840          mem_plane = 1;
2841          break;
2842       case VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT:
2843       case VK_IMAGE_ASPECT_PLANE_2_BIT:
2844          mem_plane = 2;
2845          break;
2846       default:
2847          unreachable("bad VkImageAspectFlags");
2848       }
2849 
2850       if (isl_drm_modifier_plane_is_clear_color(image->vk.drm_format_mod,
2851                                                 mem_plane)) {
2852          assert(image->n_planes == 1);
2853 
2854          mem_range = &image->planes[0].fast_clear_memory_range;
2855          row_pitch_B = ISL_DRM_CC_PLANE_PITCH_B;
2856       } else if (mem_plane == 1 &&
2857                  image->planes[0].compr_ctrl_memory_range.size > 0) {
2858          assert(image->n_planes == 1);
2859          assert(isl_drm_modifier_has_aux(image->vk.drm_format_mod));
2860 
2861          mem_range = &image->planes[0].compr_ctrl_memory_range;
2862          row_pitch_B = image->planes[0].primary_surface.isl.row_pitch_B /
2863                        INTEL_AUX_MAP_MAIN_PITCH_SCALEDOWN;
2864       } else if (mem_plane == 1 &&
2865                  image->planes[0].aux_surface.memory_range.size > 0) {
2866          assert(image->n_planes == 1);
2867          assert(image->vk.drm_format_mod == I915_FORMAT_MOD_Y_TILED_CCS);
2868 
2869          mem_range = &image->planes[0].aux_surface.memory_range;
2870          row_pitch_B = image->planes[0].aux_surface.isl.row_pitch_B;
2871       } else {
2872          assert(mem_plane < image->n_planes);
2873 
2874          mem_range = &image->planes[mem_plane].primary_surface.memory_range;
2875          row_pitch_B =
2876             image->planes[mem_plane].primary_surface.isl.row_pitch_B;
2877 
2878          isl_surf = &image->planes[mem_plane].primary_surface.isl;
2879       }
2880 
2881       /* If the memory binding differs between the primary plane and the
2882        * specified memory plane, the returned offset will be incorrect.
2883        */
2884       assert(mem_range->binding ==
2885              image->planes[0].primary_surface.memory_range.binding);
2886 
2887       /* We are working with a non-arrayed 2D image. */
2888       assert(image->vk.image_type == VK_IMAGE_TYPE_2D);
2889       assert(image->vk.array_layers == 1);
2890    } else {
2891       const uint32_t plane =
2892          anv_image_aspect_to_plane(image,
2893                                    subresource->imageSubresource.aspectMask);
2894       isl_surf = &image->planes[plane].primary_surface.isl;
2895       mem_range = &image->planes[plane].primary_surface.memory_range;
2896       row_pitch_B = isl_surf->row_pitch_B;
2897    }
2898 
2899    const uint32_t level = subresource->imageSubresource.mipLevel;
2900    if (isl_surf) {
2901       /* ISL tries to give us a single layer but the Vulkan API expect the
2902        * entire 3D size.
2903        */
2904       const uint32_t layer = subresource->imageSubresource.arrayLayer;
2905       const uint32_t z = u_minify(isl_surf->logical_level0_px.d, level) - 1;
2906       uint64_t z0_start_tile_B, z0_end_tile_B;
2907       uint64_t zX_start_tile_B, zX_end_tile_B;
2908       isl_surf_get_image_range_B_tile(isl_surf, level, layer, 0,
2909                                       &z0_start_tile_B, &z0_end_tile_B);
2910       isl_surf_get_image_range_B_tile(isl_surf, level, layer, z,
2911                                       &zX_start_tile_B, &zX_end_tile_B);
2912 
2913       layout->subresourceLayout.offset = mem_range->offset + z0_start_tile_B;
2914       layout->subresourceLayout.size = zX_end_tile_B - z0_start_tile_B;
2915       layout->subresourceLayout.rowPitch = row_pitch_B;
2916       layout->subresourceLayout.depthPitch =
2917          isl_surf_get_array_pitch(isl_surf);
2918       layout->subresourceLayout.arrayPitch =
2919          isl_surf_get_array_pitch(isl_surf);
2920    } else {
2921       layout->subresourceLayout.offset = mem_range->offset;
2922       layout->subresourceLayout.size = mem_range->size;
2923       layout->subresourceLayout.rowPitch = row_pitch_B;
2924       /* Not a surface so those fields don't make sense */
2925       layout->subresourceLayout.depthPitch = 0;
2926       layout->subresourceLayout.arrayPitch = 0;
2927    }
2928 
2929    VkSubresourceHostMemcpySizeEXT *host_memcpy_size =
2930       vk_find_struct(layout->pNext, SUBRESOURCE_HOST_MEMCPY_SIZE_EXT);
2931    if (host_memcpy_size) {
2932       if (!isl_surf) {
2933          host_memcpy_size->size = 0;
2934       } else if (anv_image_can_host_memcpy(image)) {
2935          host_memcpy_size->size = layout->subresourceLayout.size;
2936       } else {
2937          /* If we cannot do straight memcpy of the image, compute a linear
2938           * size. This will be the format in which we store the data.
2939           */
2940          struct isl_surf lin_surf;
2941          bool ok =
2942             isl_surf_init(&device->physical->isl_dev, &lin_surf,
2943                           .dim          = isl_surf->dim,
2944                           .format       = isl_surf->format,
2945                           .width        = u_minify(
2946                              isl_surf->logical_level0_px.w, level),
2947                           .height       = u_minify(
2948                              isl_surf->logical_level0_px.h, level),
2949                           .depth        = u_minify(
2950                              isl_surf->logical_level0_px.d, level),
2951                           .array_len    = 1,
2952                           .levels       = 1,
2953                           .samples      = isl_surf->samples,
2954                           .tiling_flags = ISL_TILING_LINEAR_BIT);
2955          assert(ok);
2956          host_memcpy_size->size = lin_surf.size_B;
2957       }
2958    }
2959 
2960    VkImageCompressionPropertiesEXT *comp_props =
2961       vk_find_struct(layout->pNext, IMAGE_COMPRESSION_PROPERTIES_EXT);
2962    if (comp_props) {
2963       comp_props->imageCompressionFixedRateFlags =
2964          VK_IMAGE_COMPRESSION_FIXED_RATE_NONE_EXT;
2965       comp_props->imageCompressionFlags = VK_IMAGE_COMPRESSION_DISABLED_EXT;
2966       for (uint32_t p = 0; p < image->n_planes; p++) {
2967          if (image->planes[p].aux_usage != ISL_AUX_USAGE_NONE) {
2968             comp_props->imageCompressionFlags = VK_IMAGE_COMPRESSION_DEFAULT_EXT;
2969             break;
2970          }
2971       }
2972    }
2973 }
2974 
anv_GetDeviceImageSubresourceLayoutKHR(VkDevice _device,const VkDeviceImageSubresourceInfoKHR * pInfo,VkSubresourceLayout2KHR * pLayout)2975 void anv_GetDeviceImageSubresourceLayoutKHR(
2976     VkDevice                                    _device,
2977     const VkDeviceImageSubresourceInfoKHR*      pInfo,
2978     VkSubresourceLayout2KHR*                    pLayout)
2979 {
2980    ANV_FROM_HANDLE(anv_device, device, _device);
2981 
2982    struct anv_image image = { 0 };
2983 
2984    if (anv_image_init_from_create_info(device, &image, pInfo->pCreateInfo,
2985                                        true) != VK_SUCCESS) {
2986       pLayout->subresourceLayout = (VkSubresourceLayout) { 0, };
2987       return;
2988    }
2989 
2990    anv_get_image_subresource_layout(device, &image, pInfo->pSubresource, pLayout);
2991 }
2992 
anv_GetImageSubresourceLayout2KHR(VkDevice _device,VkImage _image,const VkImageSubresource2KHR * pSubresource,VkSubresourceLayout2KHR * pLayout)2993 void anv_GetImageSubresourceLayout2KHR(
2994     VkDevice                                    _device,
2995     VkImage                                     _image,
2996     const VkImageSubresource2KHR*               pSubresource,
2997     VkSubresourceLayout2KHR*                    pLayout)
2998 {
2999    ANV_FROM_HANDLE(anv_device, device, _device);
3000    ANV_FROM_HANDLE(anv_image, image, _image);
3001 
3002    anv_get_image_subresource_layout(device, image, pSubresource, pLayout);
3003 }
3004 
3005 static VkImageUsageFlags
anv_image_flags_filter_for_queue(VkImageUsageFlags usages,VkQueueFlagBits queue_flags)3006 anv_image_flags_filter_for_queue(VkImageUsageFlags usages,
3007                                  VkQueueFlagBits queue_flags)
3008 {
3009    /* Eliminate graphics usages if the queue is not graphics capable */
3010    if (!(queue_flags & VK_QUEUE_GRAPHICS_BIT)) {
3011       usages &= ~(VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
3012                   VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT |
3013                   VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT |
3014                   VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT |
3015                   VK_IMAGE_USAGE_FRAGMENT_DENSITY_MAP_BIT_EXT |
3016                   VK_IMAGE_USAGE_FRAGMENT_SHADING_RATE_ATTACHMENT_BIT_KHR |
3017                   VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT);
3018    }
3019 
3020    /* Eliminate sampling & storage usages if the queue is neither graphics nor
3021     * compute capable
3022     */
3023    if (!(queue_flags & (VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT))) {
3024       usages &= ~(VK_IMAGE_USAGE_SAMPLED_BIT |
3025                   VK_IMAGE_USAGE_STORAGE_BIT);
3026    }
3027 
3028    /* Eliminate transfer usages if the queue is neither transfer, compute or
3029     * graphics capable
3030     */
3031    if (!(queue_flags & (VK_QUEUE_TRANSFER_BIT |
3032                         VK_QUEUE_COMPUTE_BIT |
3033                         VK_QUEUE_GRAPHICS_BIT))) {
3034       usages &= ~(VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
3035                   VK_IMAGE_USAGE_TRANSFER_DST_BIT);
3036    }
3037 
3038    return usages;
3039 }
3040 
3041 /**
3042  * This function returns the assumed isl_aux_state for a given VkImageLayout.
3043  * Because Vulkan image layouts don't map directly to isl_aux_state enums, the
3044  * returned enum is the assumed worst case.
3045  *
3046  * @param devinfo The device information of the Intel GPU.
3047  * @param image The image that may contain a collection of buffers.
3048  * @param aspect The aspect of the image to be accessed.
3049  * @param layout The current layout of the image aspect(s).
3050  *
3051  * @return The primary buffer that should be used for the given layout.
3052  */
3053 enum isl_aux_state ATTRIBUTE_PURE
anv_layout_to_aux_state(const struct intel_device_info * const devinfo,const struct anv_image * const image,const VkImageAspectFlagBits aspect,const VkImageLayout layout,const VkQueueFlagBits queue_flags)3054 anv_layout_to_aux_state(const struct intel_device_info * const devinfo,
3055                         const struct anv_image * const image,
3056                         const VkImageAspectFlagBits aspect,
3057                         const VkImageLayout layout,
3058                         const VkQueueFlagBits queue_flags)
3059 {
3060    /* Validate the inputs. */
3061 
3062    /* The devinfo is needed as the optimal buffer varies across generations. */
3063    assert(devinfo != NULL);
3064 
3065    /* The layout of a NULL image is not properly defined. */
3066    assert(image != NULL);
3067 
3068    /* The aspect must be exactly one of the image aspects. */
3069    assert(util_bitcount(aspect) == 1 && (aspect & image->vk.aspects));
3070 
3071    /* Determine the optimal buffer. */
3072 
3073    const uint32_t plane = anv_image_aspect_to_plane(image, aspect);
3074 
3075    /* If we don't have an aux buffer then aux state makes no sense */
3076    const enum isl_aux_usage aux_usage = image->planes[plane].aux_usage;
3077    assert(aux_usage != ISL_AUX_USAGE_NONE);
3078 
3079    /* All images that use an auxiliary surface are required to be tiled. */
3080    assert(image->planes[plane].primary_surface.isl.tiling != ISL_TILING_LINEAR);
3081 
3082    /* Handle a few special cases */
3083    switch (layout) {
3084    /* Invalid layouts */
3085    case VK_IMAGE_LAYOUT_MAX_ENUM:
3086       unreachable("Invalid image layout.");
3087 
3088    /* Undefined layouts
3089     *
3090     * The pre-initialized layout is equivalent to the undefined layout for
3091     * optimally-tiled images.  We can only do color compression (CCS or HiZ)
3092     * on tiled images.
3093     */
3094    case VK_IMAGE_LAYOUT_UNDEFINED:
3095    case VK_IMAGE_LAYOUT_PREINITIALIZED:
3096       return ISL_AUX_STATE_AUX_INVALID;
3097 
3098    case VK_IMAGE_LAYOUT_PRESENT_SRC_KHR: {
3099       assert(image->vk.aspects == VK_IMAGE_ASPECT_COLOR_BIT);
3100 
3101       enum isl_aux_state aux_state =
3102          isl_drm_modifier_get_default_aux_state(image->vk.drm_format_mod);
3103 
3104       switch (aux_state) {
3105       case ISL_AUX_STATE_AUX_INVALID:
3106          /* The modifier does not support compression. But, if we arrived
3107           * here, then we have enabled compression on it anyway, in which case
3108           * we must resolve the aux surface before we release ownership to the
3109           * presentation engine (because, having no modifier, the presentation
3110           * engine will not be aware of the aux surface). The presentation
3111           * engine will not access the aux surface (because it is unware of
3112           * it), and so the aux surface will still be resolved when we
3113           * re-acquire ownership.
3114           *
3115           * Therefore, at ownership transfers in either direction, there does
3116           * exist an aux surface despite the lack of modifier and its state is
3117           * pass-through.
3118           */
3119          return ISL_AUX_STATE_PASS_THROUGH;
3120       case ISL_AUX_STATE_COMPRESSED_CLEAR:
3121          return ISL_AUX_STATE_COMPRESSED_CLEAR;
3122       case ISL_AUX_STATE_COMPRESSED_NO_CLEAR:
3123          return ISL_AUX_STATE_COMPRESSED_NO_CLEAR;
3124       default:
3125          unreachable("unexpected isl_aux_state");
3126       }
3127    }
3128 
3129    default:
3130       break;
3131    }
3132 
3133    const bool read_only = vk_image_layout_is_read_only(layout, aspect);
3134 
3135    const VkImageUsageFlags image_aspect_usage =
3136       anv_image_flags_filter_for_queue(
3137          vk_image_usage(&image->vk, aspect), queue_flags);
3138    const VkImageUsageFlags usage =
3139       vk_image_layout_to_usage_flags(layout, aspect) & image_aspect_usage;
3140 
3141    bool aux_supported = true;
3142    bool clear_supported = isl_aux_usage_has_fast_clears(aux_usage);
3143 
3144    if ((usage & (VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT |
3145                  VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT)) &&
3146        !read_only) {
3147       /* This image could be used as both an input attachment and a render
3148        * target (depth, stencil, or color) at the same time and this can cause
3149        * corruption.
3150        *
3151        * We currently only disable aux in this way for depth even though we
3152        * disable it for color in GL.
3153        *
3154        * TODO: Should we be disabling this in more cases?
3155        */
3156       if (aspect == VK_IMAGE_ASPECT_DEPTH_BIT && devinfo->ver <= 9) {
3157          aux_supported = false;
3158          clear_supported = false;
3159       }
3160    }
3161 
3162    if (usage & (VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
3163                 VK_IMAGE_USAGE_SAMPLED_BIT |
3164                 VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT)) {
3165       switch (aux_usage) {
3166       case ISL_AUX_USAGE_HIZ:
3167          if (!anv_can_sample_with_hiz(devinfo, image)) {
3168             aux_supported = false;
3169             clear_supported = false;
3170          }
3171          break;
3172 
3173       case ISL_AUX_USAGE_HIZ_CCS:
3174          aux_supported = false;
3175          clear_supported = false;
3176          break;
3177 
3178       case ISL_AUX_USAGE_HIZ_CCS_WT:
3179          break;
3180 
3181       case ISL_AUX_USAGE_CCS_D:
3182          aux_supported = false;
3183          clear_supported = false;
3184          break;
3185 
3186       case ISL_AUX_USAGE_MCS:
3187       case ISL_AUX_USAGE_MCS_CCS:
3188          if (!anv_can_sample_mcs_with_clear(devinfo, image))
3189             clear_supported = false;
3190          break;
3191 
3192       case ISL_AUX_USAGE_CCS_E:
3193       case ISL_AUX_USAGE_FCV_CCS_E:
3194       case ISL_AUX_USAGE_STC_CCS:
3195          break;
3196 
3197       default:
3198          unreachable("Unsupported aux usage");
3199       }
3200    }
3201 
3202    switch (aux_usage) {
3203    case ISL_AUX_USAGE_HIZ:
3204    case ISL_AUX_USAGE_HIZ_CCS:
3205    case ISL_AUX_USAGE_HIZ_CCS_WT:
3206       if (aux_supported) {
3207          assert(clear_supported);
3208          return ISL_AUX_STATE_COMPRESSED_CLEAR;
3209       } else if (read_only) {
3210          return ISL_AUX_STATE_RESOLVED;
3211       } else {
3212          return ISL_AUX_STATE_AUX_INVALID;
3213       }
3214 
3215    case ISL_AUX_USAGE_CCS_D:
3216       /* We only support clear in exactly one state */
3217       if (layout == VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL ||
3218           layout == VK_IMAGE_LAYOUT_ATTACHMENT_OPTIMAL) {
3219          assert(aux_supported);
3220          assert(clear_supported);
3221          return ISL_AUX_STATE_PARTIAL_CLEAR;
3222       } else {
3223          return ISL_AUX_STATE_PASS_THROUGH;
3224       }
3225 
3226    case ISL_AUX_USAGE_CCS_E:
3227    case ISL_AUX_USAGE_FCV_CCS_E:
3228       if (aux_supported) {
3229          assert(clear_supported);
3230          return ISL_AUX_STATE_COMPRESSED_CLEAR;
3231       } else {
3232          return ISL_AUX_STATE_PASS_THROUGH;
3233       }
3234 
3235    case ISL_AUX_USAGE_MCS:
3236    case ISL_AUX_USAGE_MCS_CCS:
3237       assert(aux_supported);
3238       if (clear_supported) {
3239          return ISL_AUX_STATE_COMPRESSED_CLEAR;
3240       } else {
3241          return ISL_AUX_STATE_COMPRESSED_NO_CLEAR;
3242       }
3243 
3244    case ISL_AUX_USAGE_STC_CCS:
3245       assert(aux_supported);
3246       assert(!clear_supported);
3247       return ISL_AUX_STATE_COMPRESSED_NO_CLEAR;
3248 
3249    default:
3250       unreachable("Unsupported aux usage");
3251    }
3252 }
3253 
3254 /**
3255  * This function determines the optimal buffer to use for a given
3256  * VkImageLayout and other pieces of information needed to make that
3257  * determination. This does not determine the optimal buffer to use
3258  * during a resolve operation.
3259  *
3260  * @param devinfo The device information of the Intel GPU.
3261  * @param image The image that may contain a collection of buffers.
3262  * @param aspect The aspect of the image to be accessed.
3263  * @param usage The usage which describes how the image will be accessed.
3264  * @param layout The current layout of the image aspect(s).
3265  *
3266  * @return The primary buffer that should be used for the given layout.
3267  */
3268 enum isl_aux_usage ATTRIBUTE_PURE
anv_layout_to_aux_usage(const struct intel_device_info * const devinfo,const struct anv_image * const image,const VkImageAspectFlagBits aspect,const VkImageUsageFlagBits usage,const VkImageLayout layout,const VkQueueFlagBits queue_flags)3269 anv_layout_to_aux_usage(const struct intel_device_info * const devinfo,
3270                         const struct anv_image * const image,
3271                         const VkImageAspectFlagBits aspect,
3272                         const VkImageUsageFlagBits usage,
3273                         const VkImageLayout layout,
3274                         const VkQueueFlagBits queue_flags)
3275 {
3276    const uint32_t plane = anv_image_aspect_to_plane(image, aspect);
3277 
3278    /* If there is no auxiliary surface allocated, we must use the one and only
3279     * main buffer.
3280     */
3281    if (image->planes[plane].aux_usage == ISL_AUX_USAGE_NONE)
3282       return ISL_AUX_USAGE_NONE;
3283 
3284    enum isl_aux_state aux_state =
3285       anv_layout_to_aux_state(devinfo, image, aspect, layout, queue_flags);
3286 
3287    switch (aux_state) {
3288    case ISL_AUX_STATE_CLEAR:
3289       unreachable("We never use this state");
3290 
3291    case ISL_AUX_STATE_PARTIAL_CLEAR:
3292       assert(image->vk.aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
3293       assert(image->planes[plane].aux_usage == ISL_AUX_USAGE_CCS_D);
3294       assert(image->vk.samples == 1);
3295       return ISL_AUX_USAGE_CCS_D;
3296 
3297    case ISL_AUX_STATE_COMPRESSED_CLEAR:
3298    case ISL_AUX_STATE_COMPRESSED_NO_CLEAR:
3299       return image->planes[plane].aux_usage;
3300 
3301    case ISL_AUX_STATE_RESOLVED:
3302       /* We can only use RESOLVED in read-only layouts because any write will
3303        * either land us in AUX_INVALID or COMPRESSED_NO_CLEAR.  We can do
3304        * writes in PASS_THROUGH without destroying it so that is allowed.
3305        */
3306       assert(vk_image_layout_is_read_only(layout, aspect));
3307       assert(util_is_power_of_two_or_zero(usage));
3308       if (usage == VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT) {
3309          /* If we have valid HiZ data and are using the image as a read-only
3310           * depth/stencil attachment, we should enable HiZ so that we can get
3311           * faster depth testing.
3312           */
3313          return image->planes[plane].aux_usage;
3314       } else {
3315          return ISL_AUX_USAGE_NONE;
3316       }
3317 
3318    case ISL_AUX_STATE_PASS_THROUGH:
3319    case ISL_AUX_STATE_AUX_INVALID:
3320       return ISL_AUX_USAGE_NONE;
3321    }
3322 
3323    unreachable("Invalid isl_aux_state");
3324 }
3325 
3326 /**
3327  * This function returns the level of unresolved fast-clear support of the
3328  * given image in the given VkImageLayout.
3329  *
3330  * @param devinfo The device information of the Intel GPU.
3331  * @param image The image that may contain a collection of buffers.
3332  * @param aspect The aspect of the image to be accessed.
3333  * @param usage The usage which describes how the image will be accessed.
3334  * @param layout The current layout of the image aspect(s).
3335  */
3336 enum anv_fast_clear_type ATTRIBUTE_PURE
anv_layout_to_fast_clear_type(const struct intel_device_info * const devinfo,const struct anv_image * const image,const VkImageAspectFlagBits aspect,const VkImageLayout layout,const VkQueueFlagBits queue_flags)3337 anv_layout_to_fast_clear_type(const struct intel_device_info * const devinfo,
3338                               const struct anv_image * const image,
3339                               const VkImageAspectFlagBits aspect,
3340                               const VkImageLayout layout,
3341                               const VkQueueFlagBits queue_flags)
3342 {
3343    if (INTEL_DEBUG(DEBUG_NO_FAST_CLEAR))
3344       return ANV_FAST_CLEAR_NONE;
3345 
3346    const uint32_t plane = anv_image_aspect_to_plane(image, aspect);
3347 
3348    /* If there is no auxiliary surface allocated, there are no fast-clears */
3349    if (image->planes[plane].aux_usage == ISL_AUX_USAGE_NONE)
3350       return ANV_FAST_CLEAR_NONE;
3351 
3352    /* Xe2+ platforms don't have fast clear type and can always support
3353     * arbitrary fast-clear values.
3354     */
3355    if (devinfo->ver >= 20)
3356       return ANV_FAST_CLEAR_ANY;
3357 
3358    enum isl_aux_state aux_state =
3359       anv_layout_to_aux_state(devinfo, image, aspect, layout, queue_flags);
3360 
3361    const VkImageUsageFlags layout_usage =
3362       vk_image_layout_to_usage_flags(layout, aspect) & image->vk.usage;
3363 
3364    const struct isl_drm_modifier_info *isl_mod_info =
3365       image->vk.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT ?
3366       isl_drm_modifier_get_info(image->vk.drm_format_mod) : NULL;
3367 
3368    switch (aux_state) {
3369    case ISL_AUX_STATE_CLEAR:
3370       unreachable("We never use this state");
3371 
3372    case ISL_AUX_STATE_PARTIAL_CLEAR:
3373    case ISL_AUX_STATE_COMPRESSED_CLEAR:
3374 
3375       /* Generally, enabling non-zero fast-clears is dependent on knowing which
3376        * formats will be used with the surface. So, disable them if we lack
3377        * this knowledge.
3378        *
3379        * For dmabufs with clear color modifiers, we already restrict
3380        * problematic accesses for the clear color during the negotiation
3381        * phase. So, don't restrict clear color support in this case.
3382        */
3383       if (anv_image_view_formats_incomplete(image) &&
3384           !(isl_mod_info && isl_mod_info->supports_clear_color)) {
3385          return ANV_FAST_CLEAR_DEFAULT_VALUE;
3386       }
3387 
3388       /* On gfx12, the FCV feature may convert a block of fragment shader
3389        * outputs to fast-clears. If this image has multiple subresources,
3390        * restrict the clear color to zero to keep the fast cleared blocks in
3391        * sync.
3392        */
3393       if (image->planes[plane].aux_usage == ISL_AUX_USAGE_FCV_CCS_E &&
3394           (image->vk.mip_levels > 1 ||
3395            image->vk.array_layers > 1 ||
3396            image->vk.extent.depth > 1)) {
3397          return ANV_FAST_CLEAR_DEFAULT_VALUE;
3398       }
3399 
3400       /* On gfx9, we only load clear colors for attachments and for BLORP
3401        * surfaces. Outside of those surfaces, we can only support the default
3402        * clear value of zero.
3403        */
3404       if (devinfo->ver == 9 &&
3405           (layout_usage & (VK_IMAGE_USAGE_SAMPLED_BIT |
3406                            VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT))) {
3407          return ANV_FAST_CLEAR_DEFAULT_VALUE;
3408       }
3409 
3410       return ANV_FAST_CLEAR_ANY;
3411 
3412    case ISL_AUX_STATE_COMPRESSED_NO_CLEAR:
3413    case ISL_AUX_STATE_RESOLVED:
3414    case ISL_AUX_STATE_PASS_THROUGH:
3415    case ISL_AUX_STATE_AUX_INVALID:
3416       return ANV_FAST_CLEAR_NONE;
3417    }
3418 
3419    unreachable("Invalid isl_aux_state");
3420 }
3421 
3422 bool
anv_can_fast_clear_color(const struct anv_cmd_buffer * cmd_buffer,const struct anv_image * image,unsigned level,const struct VkClearRect * clear_rect,VkImageLayout layout,enum isl_format view_format,union isl_color_value clear_color)3423 anv_can_fast_clear_color(const struct anv_cmd_buffer *cmd_buffer,
3424                          const struct anv_image *image,
3425                          unsigned level,
3426                          const struct VkClearRect *clear_rect,
3427                          VkImageLayout layout,
3428                          enum isl_format view_format,
3429                          union isl_color_value clear_color)
3430 {
3431    if (INTEL_DEBUG(DEBUG_NO_FAST_CLEAR))
3432       return false;
3433 
3434    /* We only have fast-clears implemented for the render engine. */
3435    if (cmd_buffer->queue_family->engine_class != INTEL_ENGINE_CLASS_RENDER)
3436       return false;
3437 
3438    /* Start by getting the fast clear type.  We use the first subpass
3439     * layout here because we don't want to fast-clear if the first subpass
3440     * to use the attachment can't handle fast-clears.
3441     */
3442    enum anv_fast_clear_type fast_clear_type =
3443       anv_layout_to_fast_clear_type(cmd_buffer->device->info, image,
3444                                     VK_IMAGE_ASPECT_COLOR_BIT, layout,
3445                                     cmd_buffer->queue_family->queueFlags);
3446    switch (fast_clear_type) {
3447    case ANV_FAST_CLEAR_NONE:
3448       return false;
3449    case ANV_FAST_CLEAR_DEFAULT_VALUE:
3450       if (!isl_color_value_is_zero(clear_color, view_format))
3451          return false;
3452       break;
3453    case ANV_FAST_CLEAR_ANY:
3454       break;
3455    }
3456 
3457    /* Potentially, we could do partial fast-clears but doing so has crazy
3458     * alignment restrictions.  It's easier to just restrict to full size
3459     * fast clears for now.
3460     */
3461    if (clear_rect->rect.offset.x != 0 ||
3462        clear_rect->rect.offset.y != 0 ||
3463        clear_rect->rect.extent.width != image->vk.extent.width ||
3464        clear_rect->rect.extent.height != image->vk.extent.height)
3465       return false;
3466 
3467    /* We only allow fast clears to the first slice of an image (level 0,
3468     * layer 0) and only for the entire slice.  This guarantees us that, at
3469     * any given time, there is only one clear color on any given image at
3470     * any given time.  At the time of our testing (Jan 17, 2018), there
3471     * were no known applications which would benefit from fast-clearing
3472     * more than just the first slice.
3473     */
3474    if (level > 0) {
3475       anv_perf_warn(VK_LOG_OBJS(&image->vk.base),
3476                     "level > 0.  Not fast clearing.");
3477       return false;
3478    }
3479 
3480    if (clear_rect->baseArrayLayer > 0) {
3481       anv_perf_warn(VK_LOG_OBJS(&image->vk.base),
3482                     "baseArrayLayer > 0.  Not fast clearing.");
3483       return false;
3484    }
3485 
3486 
3487    if (clear_rect->layerCount > 1) {
3488       anv_perf_warn(VK_LOG_OBJS(&image->vk.base),
3489                     "layerCount > 1.  Only fast-clearing the first slice");
3490    }
3491 
3492    /* Wa_18020603990 - slow clear surfaces up to 256x256, 32bpp. */
3493    if (intel_needs_workaround(cmd_buffer->device->info, 18020603990)) {
3494       const struct anv_surface *anv_surf = &image->planes->primary_surface;
3495       if (isl_format_get_layout(anv_surf->isl.format)->bpb <= 32 &&
3496           anv_surf->isl.logical_level0_px.w <= 256 &&
3497           anv_surf->isl.logical_level0_px.h <= 256)
3498          return false;
3499    }
3500 
3501    /* On gfx12.0, CCS fast clears don't seem to cover the correct portion of
3502     * the aux buffer when the pitch is not 512B-aligned.
3503     */
3504    if (cmd_buffer->device->info->verx10 == 120 &&
3505        image->planes->primary_surface.isl.samples == 1 &&
3506        image->planes->primary_surface.isl.row_pitch_B % 512) {
3507       anv_perf_warn(VK_LOG_OBJS(&image->vk.base),
3508                     "Pitch not 512B-aligned. Slow clearing surface.");
3509       return false;
3510    }
3511 
3512    /* Wa_16021232440: Disable fast clear when height is 16k */
3513    if (intel_needs_workaround(cmd_buffer->device->info, 16021232440) &&
3514        image->vk.extent.height == 16 * 1024) {
3515       return false;
3516    }
3517 
3518    /* The fast-clear preamble and/or postamble flushes are more expensive than
3519     * the flushes performed by BLORP during slow clears. Use a heuristic to
3520     * determine if the cost of the flushes are worth fast-clearing. See
3521     * genX(cmd_buffer_update_color_aux_op)() and blorp_exec_on_render().
3522     * TODO: Tune for Xe2
3523     */
3524    if (cmd_buffer->device->info->verx10 <= 125 &&
3525        cmd_buffer->num_independent_clears >= 16 &&
3526        cmd_buffer->num_independent_clears >
3527        cmd_buffer->num_dependent_clears * 2) {
3528       anv_perf_warn(VK_LOG_OBJS(&image->vk.base),
3529                     "Not enough back-to-back fast-clears. Slow clearing.");
3530       return false;
3531    }
3532 
3533    return true;
3534 }
3535 
3536 /**
3537  * This function determines if the layout & usage of an image can have
3538  * untracked aux writes. When we see a transition that matches this criteria,
3539  * we need to mark the image as compressed written so that our predicated
3540  * resolves work properly.
3541  *
3542  * @param devinfo The device information of the Intel GPU.
3543  * @param image The image that may contain a collection of buffers.
3544  * @param aspect The aspect of the image to be accessed.
3545  * @param layout The current layout of the image aspect(s).
3546  */
3547 bool
anv_layout_has_untracked_aux_writes(const struct intel_device_info * const devinfo,const struct anv_image * const image,const VkImageAspectFlagBits aspect,const VkImageLayout layout,const VkQueueFlagBits queue_flags)3548 anv_layout_has_untracked_aux_writes(const struct intel_device_info * const devinfo,
3549                                     const struct anv_image * const image,
3550                                     const VkImageAspectFlagBits aspect,
3551                                     const VkImageLayout layout,
3552                                     const VkQueueFlagBits queue_flags)
3553 {
3554    const VkImageUsageFlags image_aspect_usage =
3555       vk_image_usage(&image->vk, aspect);
3556    const VkImageUsageFlags usage =
3557       vk_image_layout_to_usage_flags(layout, aspect) & image_aspect_usage;
3558 
3559    /* Storage is the only usage where we do not write the image through a
3560     * render target but through a descriptor. Since VK_EXT_descriptor_indexing
3561     * and the update-after-bind feature, it has become impossible to track
3562     * writes to images in descriptor at the command buffer build time. So it's
3563     * not possible to mark an image as compressed like we do in
3564     * genX_cmd_buffer.c(EndRendering) or anv_blorp.c for all transfer
3565     * operations.
3566     */
3567    if (!(usage & VK_IMAGE_USAGE_STORAGE_BIT))
3568       return false;
3569 
3570    /* No AUX, no writes to the AUX surface :) */
3571    const uint32_t plane = anv_image_aspect_to_plane(image, aspect);
3572    const enum isl_aux_usage aux_usage = image->planes[plane].aux_usage;
3573    if (aux_usage == ISL_AUX_USAGE_NONE)
3574       return false;
3575 
3576    return true;
3577 }
3578 
anv_GetRenderingAreaGranularityKHR(VkDevice _device,const VkRenderingAreaInfoKHR * pRenderingAreaInfo,VkExtent2D * pGranularity)3579 void anv_GetRenderingAreaGranularityKHR(
3580     VkDevice                                    _device,
3581     const VkRenderingAreaInfoKHR*               pRenderingAreaInfo,
3582     VkExtent2D*                                 pGranularity)
3583 {
3584    *pGranularity = (VkExtent2D) {
3585       .width = 1,
3586       .height = 1,
3587    };
3588 }
3589