1 /*
2 * Copyright 2024 Valve Corporation
3 * Copyright 2024 Alyssa Rosenzweig
4 * Copyright 2022-2023 Collabora Ltd. and Red Hat Inc.
5 * SPDX-License-Identifier: MIT
6 */
7 #include "hk_image.h"
8 #include "asahi/layout/layout.h"
9 #include "drm-uapi/drm_fourcc.h"
10 #include "util/bitscan.h"
11 #include "util/format/u_format.h"
12 #include "util/format/u_formats.h"
13 #include "util/macros.h"
14 #include "util/u_math.h"
15 #include "vulkan/vulkan_core.h"
16
17 #include "hk_device.h"
18 #include "hk_device_memory.h"
19 #include "hk_entrypoints.h"
20 #include "hk_physical_device.h"
21
22 #include "vk_format.h"
23
24 /* Minimum alignment encodable for our descriptors. The hardware texture/PBE
25 * descriptors require 16-byte alignment. Our software PBE atomic descriptor
26 * requires 128-byte alignment, but we could relax that one if we wanted.
27 */
28 #define HK_PLANE_ALIGN_B 128
29
30 static VkFormatFeatureFlags2
hk_get_image_plane_format_features(struct hk_physical_device * pdev,VkFormat vk_format,VkImageTiling tiling)31 hk_get_image_plane_format_features(struct hk_physical_device *pdev,
32 VkFormat vk_format, VkImageTiling tiling)
33 {
34 VkFormatFeatureFlags2 features = 0;
35
36 /* Conformance fails with these optional formats. Just drop them for now.
37 * TODO: Investigate later if we have a use case.
38 */
39 switch (vk_format) {
40 case VK_FORMAT_A1B5G5R5_UNORM_PACK16_KHR:
41 case VK_FORMAT_A8_UNORM_KHR:
42 return 0;
43 default:
44 break;
45 }
46
47 enum pipe_format p_format = hk_format_to_pipe_format(vk_format);
48 if (p_format == PIPE_FORMAT_NONE)
49 return 0;
50
51 /* NPOT formats only supported for texel buffers */
52 if (!util_is_power_of_two_nonzero(util_format_get_blocksize(p_format)))
53 return 0;
54
55 if (util_format_is_compressed(p_format)) {
56 /* Linear block-compressed images are all sorts of problematic, not sure
57 * if AGX even supports them. Don't try.
58 */
59 if (tiling != VK_IMAGE_TILING_OPTIMAL)
60 return 0;
61
62 /* XXX: Conformance fails, e.g.:
63 * dEQP-VK.pipeline.monolithic.sampler.view_type.2d.format.etc2_r8g8b8a1_unorm_block.mipmap.linear.lod.select_bias_3_7
64 *
65 * I suspect ail bug with mipmapping of compressed :-/
66 */
67 switch (util_format_description(p_format)->layout) {
68 case UTIL_FORMAT_LAYOUT_ETC:
69 case UTIL_FORMAT_LAYOUT_ASTC:
70 return 0;
71 default:
72 break;
73 }
74 }
75
76 if (ail_pixel_format[p_format].texturable) {
77 features |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT;
78 features |= VK_FORMAT_FEATURE_2_BLIT_SRC_BIT;
79
80 /* We can sample integer formats but it doesn't make sense to linearly
81 * filter them.
82 */
83 if (!util_format_is_pure_integer(p_format)) {
84 features |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_FILTER_LINEAR_BIT;
85 }
86
87 if (vk_format_has_depth(vk_format)) {
88 features |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_DEPTH_COMPARISON_BIT;
89 }
90 }
91
92 if (ail_pixel_format[p_format].renderable) {
93 /* For now, disable snorm rendering due to nir_lower_blend bugs.
94 *
95 * TODO: revisit.
96 */
97 if (!util_format_is_snorm(p_format)) {
98 features |= VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT;
99 features |= VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BLEND_BIT;
100 }
101
102 features |= VK_FORMAT_FEATURE_2_BLIT_DST_BIT;
103 features |= VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT |
104 VK_FORMAT_FEATURE_2_STORAGE_WRITE_WITHOUT_FORMAT_BIT |
105 VK_FORMAT_FEATURE_2_STORAGE_READ_WITHOUT_FORMAT_BIT;
106 }
107
108 if (vk_format_is_depth_or_stencil(vk_format)) {
109 if (!(p_format == PIPE_FORMAT_Z32_FLOAT ||
110 p_format == PIPE_FORMAT_S8_UINT ||
111 p_format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT ||
112 p_format == PIPE_FORMAT_Z16_UNORM) ||
113 tiling == VK_IMAGE_TILING_LINEAR)
114 return 0;
115
116 features |= VK_FORMAT_FEATURE_2_DEPTH_STENCIL_ATTACHMENT_BIT;
117 }
118
119 /* Our image atomic lowering doesn't bother to handle linear */
120 if ((p_format == PIPE_FORMAT_R32_UINT || p_format == PIPE_FORMAT_R32_SINT) &&
121 tiling == VK_IMAGE_TILING_OPTIMAL) {
122
123 features |= VK_FORMAT_FEATURE_2_STORAGE_IMAGE_ATOMIC_BIT;
124 }
125
126 if (features != 0) {
127 features |= VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT;
128 features |= VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT;
129 features |= VK_FORMAT_FEATURE_2_HOST_IMAGE_TRANSFER_BIT_EXT;
130 }
131
132 return features;
133 }
134
135 VkFormatFeatureFlags2
hk_get_image_format_features(struct hk_physical_device * pdev,VkFormat vk_format,VkImageTiling tiling)136 hk_get_image_format_features(struct hk_physical_device *pdev,
137 VkFormat vk_format, VkImageTiling tiling)
138 {
139 const struct vk_format_ycbcr_info *ycbcr_info =
140 vk_format_get_ycbcr_info(vk_format);
141 if (ycbcr_info == NULL)
142 return hk_get_image_plane_format_features(pdev, vk_format, tiling);
143
144 /* For multi-plane, we get the feature flags of each plane separately,
145 * then take their intersection as the overall format feature flags
146 */
147 VkFormatFeatureFlags2 features = ~0ull;
148 bool cosited_chroma = false;
149 for (uint8_t plane = 0; plane < ycbcr_info->n_planes; plane++) {
150 const struct vk_format_ycbcr_plane *plane_info =
151 &ycbcr_info->planes[plane];
152 features &=
153 hk_get_image_plane_format_features(pdev, plane_info->format, tiling);
154 if (plane_info->denominator_scales[0] > 1 ||
155 plane_info->denominator_scales[1] > 1)
156 cosited_chroma = true;
157 }
158 if (features == 0)
159 return 0;
160
161 /* Uh... We really should be able to sample from YCbCr */
162 assert(features & VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT);
163 assert(features & VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_FILTER_LINEAR_BIT);
164
165 /* These aren't allowed for YCbCr formats */
166 features &=
167 ~(VK_FORMAT_FEATURE_2_BLIT_SRC_BIT | VK_FORMAT_FEATURE_2_BLIT_DST_BIT |
168 VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT |
169 VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BLEND_BIT |
170 VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT);
171
172 /* This is supported on all YCbCr formats */
173 features |=
174 VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_YCBCR_CONVERSION_LINEAR_FILTER_BIT;
175
176 if (ycbcr_info->n_planes > 1) {
177 /* DISJOINT_BIT implies that each plane has its own separate binding,
178 * while SEPARATE_RECONSTRUCTION_FILTER_BIT implies that luma and chroma
179 * each have their own, separate filters, so these two bits make sense
180 * for multi-planar formats only.
181 *
182 * For MIDPOINT_CHROMA_SAMPLES_BIT, NVIDIA HW on single-plane interleaved
183 * YCbCr defaults to COSITED_EVEN, which is inaccurate and fails tests.
184 * This can be fixed with a NIR tweak but for now, we only enable this bit
185 * for multi-plane formats. See Issue #9525 on the mesa/main tracker.
186 */
187 features |=
188 VK_FORMAT_FEATURE_DISJOINT_BIT |
189 VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_YCBCR_CONVERSION_SEPARATE_RECONSTRUCTION_FILTER_BIT |
190 VK_FORMAT_FEATURE_2_MIDPOINT_CHROMA_SAMPLES_BIT;
191 }
192
193 if (cosited_chroma)
194 features |= VK_FORMAT_FEATURE_COSITED_CHROMA_SAMPLES_BIT;
195
196 return features;
197 }
198
199 static VkFormatFeatureFlags2
vk_image_usage_to_format_features(VkImageUsageFlagBits usage_flag)200 vk_image_usage_to_format_features(VkImageUsageFlagBits usage_flag)
201 {
202 assert(util_bitcount(usage_flag) == 1);
203 switch (usage_flag) {
204 case VK_IMAGE_USAGE_TRANSFER_SRC_BIT:
205 return VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT |
206 VK_FORMAT_FEATURE_BLIT_SRC_BIT;
207 case VK_IMAGE_USAGE_TRANSFER_DST_BIT:
208 return VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT |
209 VK_FORMAT_FEATURE_BLIT_DST_BIT;
210 case VK_IMAGE_USAGE_SAMPLED_BIT:
211 return VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT;
212 case VK_IMAGE_USAGE_STORAGE_BIT:
213 return VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT;
214 case VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT:
215 return VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT;
216 case VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT:
217 return VK_FORMAT_FEATURE_2_DEPTH_STENCIL_ATTACHMENT_BIT;
218 default:
219 return 0;
220 }
221 }
222
223 static bool
hk_can_compress(const struct agx_device * dev,VkFormat format,unsigned plane,unsigned width,unsigned height,unsigned samples,VkImageCreateFlagBits flags,VkImageUsageFlagBits usage,const void * pNext)224 hk_can_compress(const struct agx_device *dev, VkFormat format, unsigned plane,
225 unsigned width, unsigned height, unsigned samples,
226 VkImageCreateFlagBits flags, VkImageUsageFlagBits usage,
227 const void *pNext)
228 {
229 const struct vk_format_ycbcr_info *ycbcr_info =
230 vk_format_get_ycbcr_info(format);
231
232 if (ycbcr_info) {
233 format = ycbcr_info->planes[plane].format;
234 width /= ycbcr_info->planes[plane].denominator_scales[0];
235 height /= ycbcr_info->planes[plane].denominator_scales[0];
236 } else if (format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
237 format = (plane == 0) ? VK_FORMAT_D32_SFLOAT : VK_FORMAT_S8_UINT;
238 }
239
240 /* Allow disabling compression for debugging */
241 if (dev->debug & AGX_DBG_NOCOMPRESS)
242 return false;
243
244 /* Image compression is not (yet?) supported with host image copies,
245 * although the vendor driver does support something similar if I recall.
246 * Compression is not supported in hardware for storage images or mutable
247 * formats in general.
248 *
249 * Feedback loops are problematic with compression. The GL driver bans them.
250 * Interestingly, the relevant CTS tests pass on G13G and G14C, but not on
251 * G13D. For now, conservatively ban compression with feedback loops.
252 */
253 if (usage &
254 (VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT | VK_IMAGE_USAGE_STORAGE_BIT |
255 VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT)) {
256
257 perf_debug_dev(
258 dev, "No compression: incompatible usage -%s%s%s",
259 (usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT) ? " host-transfer" : "",
260 (usage & VK_IMAGE_USAGE_STORAGE_BIT) ? " storage" : "",
261 (usage & VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT)
262 ? " feedback-loop"
263 : "");
264 return false;
265 }
266
267 enum pipe_format p_format = hk_format_to_pipe_format(format);
268
269 /* Check for format compatibility if mutability is enabled. */
270 if (flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) {
271 const struct VkImageFormatListCreateInfo *format_list =
272 (void *)vk_find_struct_const(pNext, IMAGE_FORMAT_LIST_CREATE_INFO);
273
274 if (!format_list || format_list->viewFormatCount == 0)
275 return false;
276
277 for (unsigned i = 0; i < format_list->viewFormatCount; ++i) {
278 if (format_list->pViewFormats[i] == VK_FORMAT_UNDEFINED)
279 continue;
280
281 enum pipe_format view_format =
282 hk_format_to_pipe_format(format_list->pViewFormats[i]);
283
284 if (!ail_formats_compatible(p_format, view_format)) {
285 perf_debug_dev(dev, "No compression: incompatible image view");
286 return false;
287 }
288 }
289 }
290
291 if (!ail_can_compress(p_format, width, height, samples)) {
292 perf_debug_dev(dev, "No compression: invalid layout %s %ux%ux%u",
293 util_format_short_name(p_format), width, height, samples);
294 return false;
295 }
296
297 return true;
298 }
299
300 bool
hk_can_compress_format(const struct agx_device * dev,VkFormat format)301 hk_can_compress_format(const struct agx_device *dev, VkFormat format)
302 {
303 /* Check compressability of a sufficiently large image of the same
304 * format, since we don't have dimensions here. This is lossy for
305 * small images, but that's ok.
306 *
307 * Likewise, we do not set flags as flags only disable compression.
308 */
309 return hk_can_compress(dev, format, 0, 64, 64, 1, 0, 0, NULL);
310 }
311
312 VKAPI_ATTR VkResult VKAPI_CALL
hk_GetPhysicalDeviceImageFormatProperties2(VkPhysicalDevice physicalDevice,const VkPhysicalDeviceImageFormatInfo2 * pImageFormatInfo,VkImageFormatProperties2 * pImageFormatProperties)313 hk_GetPhysicalDeviceImageFormatProperties2(
314 VkPhysicalDevice physicalDevice,
315 const VkPhysicalDeviceImageFormatInfo2 *pImageFormatInfo,
316 VkImageFormatProperties2 *pImageFormatProperties)
317 {
318 VK_FROM_HANDLE(hk_physical_device, pdev, physicalDevice);
319
320 const VkPhysicalDeviceExternalImageFormatInfo *external_info =
321 vk_find_struct_const(pImageFormatInfo->pNext,
322 PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO);
323
324 /* Initialize to zero in case we return VK_ERROR_FORMAT_NOT_SUPPORTED */
325 memset(&pImageFormatProperties->imageFormatProperties, 0,
326 sizeof(pImageFormatProperties->imageFormatProperties));
327
328 const struct vk_format_ycbcr_info *ycbcr_info =
329 vk_format_get_ycbcr_info(pImageFormatInfo->format);
330
331 /* For the purposes of these checks, we don't care about all the extra
332 * YCbCr features and we just want the accumulation of features available
333 * to all planes of the given format.
334 */
335 VkFormatFeatureFlags2 features;
336 if (ycbcr_info == NULL) {
337 features = hk_get_image_plane_format_features(
338 pdev, pImageFormatInfo->format, pImageFormatInfo->tiling);
339 } else {
340 features = ~0ull;
341 assert(ycbcr_info->n_planes > 0);
342 for (uint8_t plane = 0; plane < ycbcr_info->n_planes; plane++) {
343 const VkFormat plane_format = ycbcr_info->planes[plane].format;
344 features &= hk_get_image_plane_format_features(
345 pdev, plane_format, pImageFormatInfo->tiling);
346 }
347 }
348 if (features == 0)
349 return VK_ERROR_FORMAT_NOT_SUPPORTED;
350
351 if (pImageFormatInfo->tiling == VK_IMAGE_TILING_LINEAR &&
352 pImageFormatInfo->type != VK_IMAGE_TYPE_2D)
353 return VK_ERROR_FORMAT_NOT_SUPPORTED;
354
355 if (ycbcr_info && pImageFormatInfo->type != VK_IMAGE_TYPE_2D)
356 return VK_ERROR_FORMAT_NOT_SUPPORTED;
357
358 /* From the Vulkan 1.3.279 spec:
359 *
360 * VUID-VkImageCreateInfo-tiling-04121
361 *
362 * "If tiling is VK_IMAGE_TILING_LINEAR, flags must not contain
363 * VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT"
364 *
365 * VUID-VkImageCreateInfo-imageType-00970
366 *
367 * "If imageType is VK_IMAGE_TYPE_1D, flags must not contain
368 * VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT"
369 */
370 if (pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT &&
371 (pImageFormatInfo->type == VK_IMAGE_TYPE_1D ||
372 pImageFormatInfo->tiling == VK_IMAGE_TILING_LINEAR))
373 return VK_ERROR_FORMAT_NOT_SUPPORTED;
374
375 /* From the Vulkan 1.3.279 spec:
376 *
377 * VUID-VkImageCreateInfo-flags-09403
378 *
379 * "If flags contains VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT, flags
380 * must not include VK_IMAGE_CREATE_SPARSE_ALIASED_BIT,
381 * VK_IMAGE_CREATE_SPARSE_BINDING_BIT, or
382 * VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT"
383 */
384 if ((pImageFormatInfo->flags & VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT) &&
385 (pImageFormatInfo->flags & (VK_IMAGE_CREATE_SPARSE_ALIASED_BIT |
386 VK_IMAGE_CREATE_SPARSE_BINDING_BIT |
387 VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT)))
388 return VK_ERROR_FORMAT_NOT_SUPPORTED;
389
390 /* We don't yet support sparse, but it shouldn't be too hard */
391 if (pImageFormatInfo->flags & (VK_IMAGE_CREATE_SPARSE_ALIASED_BIT |
392 VK_IMAGE_CREATE_SPARSE_BINDING_BIT |
393 VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT))
394 return VK_ERROR_FORMAT_NOT_SUPPORTED;
395
396 const uint32_t max_dim = 16384;
397 VkExtent3D maxExtent;
398 uint32_t maxArraySize;
399 switch (pImageFormatInfo->type) {
400 case VK_IMAGE_TYPE_1D:
401 maxExtent = (VkExtent3D){max_dim, 1, 1};
402 maxArraySize = 2048;
403 break;
404 case VK_IMAGE_TYPE_2D:
405 maxExtent = (VkExtent3D){max_dim, max_dim, 1};
406 maxArraySize = 2048;
407 break;
408 case VK_IMAGE_TYPE_3D:
409 maxExtent = (VkExtent3D){max_dim, max_dim, max_dim};
410 maxArraySize = 1;
411 break;
412 default:
413 unreachable("Invalid image type");
414 }
415 if (pImageFormatInfo->tiling == VK_IMAGE_TILING_LINEAR)
416 maxArraySize = 1;
417
418 assert(util_is_power_of_two_nonzero(max_dim));
419 uint32_t maxMipLevels = util_logbase2(max_dim) + 1;
420 if (ycbcr_info != NULL || pImageFormatInfo->tiling == VK_IMAGE_TILING_LINEAR)
421 maxMipLevels = 1;
422
423 VkSampleCountFlags sampleCounts = VK_SAMPLE_COUNT_1_BIT;
424 if (pImageFormatInfo->tiling == VK_IMAGE_TILING_OPTIMAL &&
425 pImageFormatInfo->type == VK_IMAGE_TYPE_2D && ycbcr_info == NULL &&
426 (features & (VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT |
427 VK_FORMAT_FEATURE_2_DEPTH_STENCIL_ATTACHMENT_BIT)) &&
428 !(pImageFormatInfo->flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT)) {
429
430 sampleCounts =
431 VK_SAMPLE_COUNT_1_BIT | VK_SAMPLE_COUNT_2_BIT | VK_SAMPLE_COUNT_4_BIT;
432 }
433
434 /* From the Vulkan 1.2.199 spec:
435 *
436 * "VK_IMAGE_CREATE_EXTENDED_USAGE_BIT specifies that the image can be
437 * created with usage flags that are not supported for the format the
438 * image is created with but are supported for at least one format a
439 * VkImageView created from the image can have."
440 *
441 * If VK_IMAGE_CREATE_EXTENDED_USAGE_BIT is set, views can be created with
442 * different usage than the image so we can't always filter on usage.
443 * There is one exception to this below for storage.
444 */
445 const VkImageUsageFlags image_usage = pImageFormatInfo->usage;
446 VkImageUsageFlags view_usage = image_usage;
447 if (pImageFormatInfo->flags & VK_IMAGE_CREATE_EXTENDED_USAGE_BIT)
448 view_usage = 0;
449
450 if (view_usage & VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT) {
451 if (!(features & (VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT |
452 VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT))) {
453 return VK_ERROR_FORMAT_NOT_SUPPORTED;
454 }
455 }
456
457 u_foreach_bit(b, view_usage) {
458 VkFormatFeatureFlags2 usage_features =
459 vk_image_usage_to_format_features(1 << b);
460 if (usage_features && !(features & usage_features))
461 return VK_ERROR_FORMAT_NOT_SUPPORTED;
462 }
463
464 const VkExternalMemoryProperties *ext_mem_props = NULL;
465 if (external_info != NULL && external_info->handleType != 0) {
466 bool tiling_has_explicit_layout;
467 switch (pImageFormatInfo->tiling) {
468 case VK_IMAGE_TILING_LINEAR:
469 case VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT:
470 tiling_has_explicit_layout = true;
471 break;
472 case VK_IMAGE_TILING_OPTIMAL:
473 tiling_has_explicit_layout = false;
474 break;
475 default:
476 unreachable("Unsupported VkImageTiling");
477 }
478
479 switch (external_info->handleType) {
480 case VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT:
481 /* No special restrictions */
482 if (tiling_has_explicit_layout) {
483 /* With an explicit memory layout, we don't care which type of
484 * fd the image belongs too. Both OPAQUE_FD and DMA_BUF are
485 * interchangeable here.
486 */
487 ext_mem_props = &hk_dma_buf_mem_props;
488 } else {
489 ext_mem_props = &hk_opaque_fd_mem_props;
490 }
491 break;
492
493 case VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT:
494 if (!tiling_has_explicit_layout) {
495 return vk_errorf(pdev, VK_ERROR_FORMAT_NOT_SUPPORTED,
496 "VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT "
497 "requires VK_IMAGE_TILING_LINEAR or "
498 "VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT");
499 }
500 ext_mem_props = &hk_dma_buf_mem_props;
501 break;
502
503 default:
504 /* From the Vulkan 1.3.256 spec:
505 *
506 * "If handleType is not compatible with the [parameters] in
507 * VkPhysicalDeviceImageFormatInfo2, then
508 * vkGetPhysicalDeviceImageFormatProperties2 returns
509 * VK_ERROR_FORMAT_NOT_SUPPORTED."
510 */
511 return vk_errorf(pdev, VK_ERROR_FORMAT_NOT_SUPPORTED,
512 "unsupported VkExternalMemoryTypeFlagBits 0x%x",
513 external_info->handleType);
514 }
515 }
516
517 const unsigned plane_count =
518 vk_format_get_plane_count(pImageFormatInfo->format);
519
520 /* From the Vulkan 1.3.259 spec, VkImageCreateInfo:
521 *
522 * VUID-VkImageCreateInfo-imageCreateFormatFeatures-02260
523 *
524 * "If format is a multi-planar format, and if imageCreateFormatFeatures
525 * (as defined in Image Creation Limits) does not contain
526 * VK_FORMAT_FEATURE_DISJOINT_BIT, then flags must not contain
527 * VK_IMAGE_CREATE_DISJOINT_BIT"
528 *
529 * This is satisfied trivially because we support DISJOINT on all
530 * multi-plane formats. Also,
531 *
532 * VUID-VkImageCreateInfo-format-01577
533 *
534 * "If format is not a multi-planar format, and flags does not include
535 * VK_IMAGE_CREATE_ALIAS_BIT, flags must not contain
536 * VK_IMAGE_CREATE_DISJOINT_BIT"
537 */
538 if (plane_count == 1 &&
539 !(pImageFormatInfo->flags & VK_IMAGE_CREATE_ALIAS_BIT) &&
540 (pImageFormatInfo->flags & VK_IMAGE_CREATE_DISJOINT_BIT))
541 return VK_ERROR_FORMAT_NOT_SUPPORTED;
542
543 if (ycbcr_info &&
544 ((pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) ||
545 (pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT)))
546 return VK_ERROR_FORMAT_NOT_SUPPORTED;
547
548 pImageFormatProperties->imageFormatProperties = (VkImageFormatProperties){
549 .maxExtent = maxExtent,
550 .maxMipLevels = maxMipLevels,
551 .maxArrayLayers = maxArraySize,
552 .sampleCounts = sampleCounts,
553 .maxResourceSize = UINT32_MAX, /* TODO */
554 };
555
556 vk_foreach_struct(s, pImageFormatProperties->pNext) {
557 switch (s->sType) {
558 case VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES: {
559 VkExternalImageFormatProperties *p = (void *)s;
560 /* From the Vulkan 1.3.256 spec:
561 *
562 * "If handleType is 0, vkGetPhysicalDeviceImageFormatProperties2
563 * will behave as if VkPhysicalDeviceExternalImageFormatInfo was
564 * not present, and VkExternalImageFormatProperties will be
565 * ignored."
566 *
567 * This is true if and only if ext_mem_props == NULL
568 */
569 if (ext_mem_props != NULL)
570 p->externalMemoryProperties = *ext_mem_props;
571 break;
572 }
573 case VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_IMAGE_FORMAT_PROPERTIES: {
574 VkSamplerYcbcrConversionImageFormatProperties *ycbcr_props = (void *)s;
575 ycbcr_props->combinedImageSamplerDescriptorCount = plane_count;
576 break;
577 }
578 case VK_STRUCTURE_TYPE_HOST_IMAGE_COPY_DEVICE_PERFORMANCE_QUERY_EXT: {
579 VkHostImageCopyDevicePerformanceQueryEXT *hic_props = (void *)s;
580
581 hic_props->optimalDeviceAccess = hic_props->identicalMemoryLayout =
582 !(pImageFormatInfo->tiling == VK_IMAGE_TILING_OPTIMAL &&
583 hk_can_compress_format(&pdev->dev, pImageFormatInfo->format));
584 break;
585 }
586 default:
587 vk_debug_ignored_stype(s->sType);
588 break;
589 }
590 }
591
592 return VK_SUCCESS;
593 }
594
595 static VkSparseImageFormatProperties
hk_fill_sparse_image_fmt_props(VkImageAspectFlags aspects)596 hk_fill_sparse_image_fmt_props(VkImageAspectFlags aspects)
597 {
598 /* TODO */
599 return (VkSparseImageFormatProperties){
600 .aspectMask = aspects,
601 .flags = VK_SPARSE_IMAGE_FORMAT_SINGLE_MIPTAIL_BIT,
602 .imageGranularity =
603 {
604 .width = 1,
605 .height = 1,
606 .depth = 1,
607 },
608 };
609 }
610
611 VKAPI_ATTR void VKAPI_CALL
hk_GetPhysicalDeviceSparseImageFormatProperties2(VkPhysicalDevice physicalDevice,const VkPhysicalDeviceSparseImageFormatInfo2 * pFormatInfo,uint32_t * pPropertyCount,VkSparseImageFormatProperties2 * pProperties)612 hk_GetPhysicalDeviceSparseImageFormatProperties2(
613 VkPhysicalDevice physicalDevice,
614 const VkPhysicalDeviceSparseImageFormatInfo2 *pFormatInfo,
615 uint32_t *pPropertyCount, VkSparseImageFormatProperties2 *pProperties)
616 {
617 VkResult result;
618
619 /* Check if the given format info is valid first before returning sparse
620 * props. The easiest way to do this is to just call
621 * hk_GetPhysicalDeviceImageFormatProperties2()
622 */
623 const VkPhysicalDeviceImageFormatInfo2 img_fmt_info = {
624 .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
625 .format = pFormatInfo->format,
626 .type = pFormatInfo->type,
627 .tiling = pFormatInfo->tiling,
628 .usage = pFormatInfo->usage,
629 .flags = VK_IMAGE_CREATE_SPARSE_BINDING_BIT |
630 VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT,
631 };
632
633 VkImageFormatProperties2 img_fmt_props2 = {
634 .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
635 .pNext = NULL,
636 };
637
638 result = hk_GetPhysicalDeviceImageFormatProperties2(
639 physicalDevice, &img_fmt_info, &img_fmt_props2);
640 if (result != VK_SUCCESS) {
641 *pPropertyCount = 0;
642 return;
643 }
644
645 const VkImageFormatProperties *props = &img_fmt_props2.imageFormatProperties;
646 if (!(pFormatInfo->samples & props->sampleCounts)) {
647 *pPropertyCount = 0;
648 return;
649 }
650
651 VK_OUTARRAY_MAKE_TYPED(VkSparseImageFormatProperties2, out, pProperties,
652 pPropertyCount);
653
654 VkImageAspectFlags aspects = vk_format_aspects(pFormatInfo->format);
655
656 vk_outarray_append_typed(VkSparseImageFormatProperties2, &out, props)
657 {
658 props->properties = hk_fill_sparse_image_fmt_props(aspects);
659 }
660 }
661
662 static bool
hk_can_compress_create_info(struct hk_device * dev,unsigned plane,const VkImageCreateInfo * info)663 hk_can_compress_create_info(struct hk_device *dev, unsigned plane,
664 const VkImageCreateInfo *info)
665 {
666 return hk_can_compress(&dev->dev, info->format, plane, info->extent.width,
667 info->extent.height, info->samples, info->flags,
668 info->usage, info->pNext);
669 }
670
671 static enum ail_tiling
hk_map_tiling(struct hk_device * dev,const VkImageCreateInfo * info,unsigned plane,uint64_t modifier)672 hk_map_tiling(struct hk_device *dev, const VkImageCreateInfo *info,
673 unsigned plane, uint64_t modifier)
674 {
675 switch (info->tiling) {
676 case VK_IMAGE_TILING_LINEAR:
677 return AIL_TILING_LINEAR;
678
679 case VK_IMAGE_TILING_OPTIMAL:
680 if (hk_can_compress_create_info(dev, plane, info)) {
681 return AIL_TILING_TWIDDLED_COMPRESSED;
682 } else {
683 return AIL_TILING_TWIDDLED;
684 }
685
686 case VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT:
687 return ail_drm_modifier_to_tiling(modifier);
688
689 default:
690 unreachable("invalid tiling");
691 }
692 }
693
694 static uint32_t
modifier_get_score(uint64_t mod)695 modifier_get_score(uint64_t mod)
696 {
697 switch (mod) {
698 case DRM_FORMAT_MOD_APPLE_TWIDDLED_COMPRESSED:
699 return 10;
700
701 case DRM_FORMAT_MOD_APPLE_TWIDDLED:
702 return 5;
703
704 case DRM_FORMAT_MOD_LINEAR:
705 return 1;
706
707 default:
708 return 0;
709 }
710 }
711
712 static uint64_t
choose_drm_format_mod(struct hk_device * dev,uint8_t plane_count,const VkImageCreateInfo * info,uint32_t modifier_count,const uint64_t * modifiers)713 choose_drm_format_mod(struct hk_device *dev, uint8_t plane_count,
714 const VkImageCreateInfo *info, uint32_t modifier_count,
715 const uint64_t *modifiers)
716 {
717 uint64_t best_mod = UINT64_MAX;
718 uint32_t best_score = 0;
719 bool can_compress = true;
720
721 for (uint8_t plane = 0; plane < plane_count; plane++) {
722 if (!hk_can_compress_create_info(dev, plane, info))
723 can_compress = false;
724 }
725
726 for (uint32_t i = 0; i < modifier_count; ++i) {
727 if (!can_compress &&
728 modifiers[i] == DRM_FORMAT_MOD_APPLE_TWIDDLED_COMPRESSED)
729 continue;
730
731 uint32_t score = modifier_get_score(modifiers[i]);
732 if (score > best_score) {
733 best_mod = modifiers[i];
734 best_score = score;
735 }
736 }
737
738 if (best_score > 0)
739 return best_mod;
740 else
741 return DRM_FORMAT_MOD_INVALID;
742 }
743
744 static VkResult
hk_image_init(struct hk_device * dev,struct hk_image * image,const VkImageCreateInfo * pCreateInfo)745 hk_image_init(struct hk_device *dev, struct hk_image *image,
746 const VkImageCreateInfo *pCreateInfo)
747 {
748 vk_image_init(&dev->vk, &image->vk, pCreateInfo);
749
750 if ((image->vk.usage & (VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
751 VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT)) &&
752 image->vk.samples > 1) {
753 image->vk.usage |= VK_IMAGE_USAGE_SAMPLED_BIT;
754 image->vk.stencil_usage |= VK_IMAGE_USAGE_SAMPLED_BIT;
755 }
756
757 if (image->vk.usage & VK_IMAGE_USAGE_TRANSFER_SRC_BIT)
758 image->vk.usage |= VK_IMAGE_USAGE_SAMPLED_BIT;
759 if (image->vk.usage & VK_IMAGE_USAGE_TRANSFER_DST_BIT)
760 image->vk.usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
761
762 image->plane_count = vk_format_get_plane_count(pCreateInfo->format);
763 image->disjoint = image->plane_count > 1 &&
764 (pCreateInfo->flags & VK_IMAGE_CREATE_DISJOINT_BIT);
765
766 /* We do not support interleaved depth/stencil. Instead, we decompose to
767 * a depth plane and a stencil plane.
768 */
769 if (image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
770 image->plane_count = 2;
771 }
772
773 if (image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) {
774 /* Sparse multiplane is not supported. Sparse depth/stencil not supported
775 * on G13 so we're fine there too.
776 */
777 assert(image->plane_count == 1);
778 }
779
780 const struct VkImageDrmFormatModifierExplicitCreateInfoEXT
781 *mod_explicit_info = NULL;
782
783 if (pCreateInfo->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
784 assert(!image->vk.wsi_legacy_scanout);
785 mod_explicit_info = vk_find_struct_const(
786 pCreateInfo->pNext,
787 IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT);
788
789 uint64_t modifier = DRM_FORMAT_MOD_INVALID;
790
791 if (mod_explicit_info) {
792 modifier = mod_explicit_info->drmFormatModifier;
793 } else {
794 const struct VkImageDrmFormatModifierListCreateInfoEXT *mod_list_info =
795 vk_find_struct_const(
796 pCreateInfo->pNext,
797 IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
798
799 modifier = choose_drm_format_mod(dev, image->plane_count, pCreateInfo,
800 mod_list_info->drmFormatModifierCount,
801 mod_list_info->pDrmFormatModifiers);
802 }
803
804 assert(modifier != DRM_FORMAT_MOD_INVALID);
805 assert(image->vk.drm_format_mod == DRM_FORMAT_MOD_INVALID);
806 image->vk.drm_format_mod = modifier;
807 }
808
809 const struct vk_format_ycbcr_info *ycbcr_info =
810 vk_format_get_ycbcr_info(pCreateInfo->format);
811 for (uint8_t plane = 0; plane < image->plane_count; plane++) {
812 VkFormat format =
813 ycbcr_info ? ycbcr_info->planes[plane].format : pCreateInfo->format;
814
815 if (format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
816 format = (plane == 0) ? VK_FORMAT_D32_SFLOAT : VK_FORMAT_S8_UINT;
817 }
818
819 const uint8_t width_scale =
820 ycbcr_info ? ycbcr_info->planes[plane].denominator_scales[0] : 1;
821 const uint8_t height_scale =
822 ycbcr_info ? ycbcr_info->planes[plane].denominator_scales[1] : 1;
823
824 enum ail_tiling tiling =
825 hk_map_tiling(dev, pCreateInfo, plane, image->vk.drm_format_mod);
826
827 image->planes[plane].layout = (struct ail_layout){
828 .tiling = tiling,
829 .mipmapped_z = pCreateInfo->imageType == VK_IMAGE_TYPE_3D,
830 .format = hk_format_to_pipe_format(format),
831
832 .width_px = pCreateInfo->extent.width / width_scale,
833 .height_px = pCreateInfo->extent.height / height_scale,
834 .depth_px = MAX2(pCreateInfo->extent.depth, pCreateInfo->arrayLayers),
835
836 .levels = pCreateInfo->mipLevels,
837 .sample_count_sa = pCreateInfo->samples,
838 .writeable_image = tiling != AIL_TILING_TWIDDLED_COMPRESSED,
839
840 /* TODO: Maybe optimize this, our GL driver doesn't bother though */
841 .renderable = true,
842 };
843
844 ail_make_miptree(&image->planes[plane].layout);
845 }
846
847 return VK_SUCCESS;
848 }
849
850 static VkResult
hk_image_plane_alloc_vma(struct hk_device * dev,struct hk_image_plane * plane,VkImageCreateFlags create_flags)851 hk_image_plane_alloc_vma(struct hk_device *dev, struct hk_image_plane *plane,
852 VkImageCreateFlags create_flags)
853 {
854 const bool sparse_bound = create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT;
855 const bool sparse_resident =
856 create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT;
857 assert(sparse_bound || !sparse_resident);
858
859 if (sparse_bound) {
860 plane->vma_size_B = plane->layout.size_B;
861 #if 0
862 plane->addr = nouveau_ws_alloc_vma(dev->ws_dev, 0, plane->vma_size_B,
863 plane->layout.align_B,
864 false, sparse_resident);
865 #endif
866 if (plane->addr == 0) {
867 return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY,
868 "Sparse VMA allocation failed");
869 }
870 }
871
872 return VK_SUCCESS;
873 }
874
875 static void
hk_image_plane_finish(struct hk_device * dev,struct hk_image_plane * plane,VkImageCreateFlags create_flags,const VkAllocationCallbacks * pAllocator)876 hk_image_plane_finish(struct hk_device *dev, struct hk_image_plane *plane,
877 VkImageCreateFlags create_flags,
878 const VkAllocationCallbacks *pAllocator)
879 {
880 if (plane->vma_size_B) {
881 #if 0
882 const bool sparse_resident =
883 create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT;
884
885 agx_bo_unbind_vma(dev->ws_dev, plane->addr, plane->vma_size_B);
886 nouveau_ws_free_vma(dev->ws_dev, plane->addr, plane->vma_size_B,
887 false, sparse_resident);
888 #endif
889 }
890 }
891
892 static void
hk_image_finish(struct hk_device * dev,struct hk_image * image,const VkAllocationCallbacks * pAllocator)893 hk_image_finish(struct hk_device *dev, struct hk_image *image,
894 const VkAllocationCallbacks *pAllocator)
895 {
896 for (uint8_t plane = 0; plane < image->plane_count; plane++) {
897 hk_image_plane_finish(dev, &image->planes[plane], image->vk.create_flags,
898 pAllocator);
899 }
900
901 vk_image_finish(&image->vk);
902 }
903
904 VKAPI_ATTR VkResult VKAPI_CALL
hk_CreateImage(VkDevice _device,const VkImageCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkImage * pImage)905 hk_CreateImage(VkDevice _device, const VkImageCreateInfo *pCreateInfo,
906 const VkAllocationCallbacks *pAllocator, VkImage *pImage)
907 {
908 VK_FROM_HANDLE(hk_device, dev, _device);
909 struct hk_physical_device *pdev = hk_device_physical(dev);
910 struct hk_image *image;
911 VkResult result;
912
913 #ifdef HK_USE_WSI_PLATFORM
914 /* Ignore swapchain creation info on Android. Since we don't have an
915 * implementation in Mesa, we're guaranteed to access an Android object
916 * incorrectly.
917 */
918 const VkImageSwapchainCreateInfoKHR *swapchain_info =
919 vk_find_struct_const(pCreateInfo->pNext, IMAGE_SWAPCHAIN_CREATE_INFO_KHR);
920 if (swapchain_info && swapchain_info->swapchain != VK_NULL_HANDLE) {
921 return wsi_common_create_swapchain_image(
922 &pdev->wsi_device, pCreateInfo, swapchain_info->swapchain, pImage);
923 }
924 #endif
925
926 image = vk_zalloc2(&dev->vk.alloc, pAllocator, sizeof(*image), 8,
927 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
928 if (!image)
929 return vk_error(dev, VK_ERROR_OUT_OF_HOST_MEMORY);
930
931 result = hk_image_init(dev, image, pCreateInfo);
932 if (result != VK_SUCCESS) {
933 vk_free2(&dev->vk.alloc, pAllocator, image);
934 return result;
935 }
936
937 for (uint8_t plane = 0; plane < image->plane_count; plane++) {
938 result = hk_image_plane_alloc_vma(dev, &image->planes[plane],
939 image->vk.create_flags);
940 if (result != VK_SUCCESS) {
941 hk_image_finish(dev, image, pAllocator);
942 vk_free2(&dev->vk.alloc, pAllocator, image);
943 return result;
944 }
945 }
946
947 *pImage = hk_image_to_handle(image);
948
949 return VK_SUCCESS;
950 }
951
952 VKAPI_ATTR void VKAPI_CALL
hk_DestroyImage(VkDevice device,VkImage _image,const VkAllocationCallbacks * pAllocator)953 hk_DestroyImage(VkDevice device, VkImage _image,
954 const VkAllocationCallbacks *pAllocator)
955 {
956 VK_FROM_HANDLE(hk_device, dev, device);
957 VK_FROM_HANDLE(hk_image, image, _image);
958
959 if (!image)
960 return;
961
962 hk_image_finish(dev, image, pAllocator);
963 vk_free2(&dev->vk.alloc, pAllocator, image);
964 }
965
966 static void
hk_image_plane_add_req(struct hk_image_plane * plane,uint64_t * size_B,uint32_t * align_B)967 hk_image_plane_add_req(struct hk_image_plane *plane, uint64_t *size_B,
968 uint32_t *align_B)
969 {
970 assert(util_is_power_of_two_or_zero64(*align_B));
971 assert(util_is_power_of_two_or_zero64(HK_PLANE_ALIGN_B));
972
973 *align_B = MAX2(*align_B, HK_PLANE_ALIGN_B);
974 *size_B = align64(*size_B, HK_PLANE_ALIGN_B);
975 *size_B += plane->layout.size_B;
976 }
977
978 static void
hk_get_image_memory_requirements(struct hk_device * dev,struct hk_image * image,VkImageAspectFlags aspects,VkMemoryRequirements2 * pMemoryRequirements)979 hk_get_image_memory_requirements(struct hk_device *dev, struct hk_image *image,
980 VkImageAspectFlags aspects,
981 VkMemoryRequirements2 *pMemoryRequirements)
982 {
983 struct hk_physical_device *pdev = hk_device_physical(dev);
984 uint32_t memory_types = (1 << pdev->mem_type_count) - 1;
985
986 // TODO hope for the best?
987
988 uint64_t size_B = 0;
989 uint32_t align_B = 0;
990 if (image->disjoint) {
991 uint8_t plane = hk_image_aspects_to_plane(image, aspects);
992 hk_image_plane_add_req(&image->planes[plane], &size_B, &align_B);
993 } else {
994 for (unsigned plane = 0; plane < image->plane_count; plane++)
995 hk_image_plane_add_req(&image->planes[plane], &size_B, &align_B);
996 }
997
998 pMemoryRequirements->memoryRequirements.memoryTypeBits = memory_types;
999 pMemoryRequirements->memoryRequirements.alignment = align_B;
1000 pMemoryRequirements->memoryRequirements.size = size_B;
1001
1002 vk_foreach_struct_const(ext, pMemoryRequirements->pNext) {
1003 switch (ext->sType) {
1004 case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS: {
1005 VkMemoryDedicatedRequirements *dedicated = (void *)ext;
1006 dedicated->prefersDedicatedAllocation = false;
1007 dedicated->requiresDedicatedAllocation = false;
1008 break;
1009 }
1010 default:
1011 vk_debug_ignored_stype(ext->sType);
1012 break;
1013 }
1014 }
1015 }
1016
1017 VKAPI_ATTR void VKAPI_CALL
hk_GetImageMemoryRequirements2(VkDevice device,const VkImageMemoryRequirementsInfo2 * pInfo,VkMemoryRequirements2 * pMemoryRequirements)1018 hk_GetImageMemoryRequirements2(VkDevice device,
1019 const VkImageMemoryRequirementsInfo2 *pInfo,
1020 VkMemoryRequirements2 *pMemoryRequirements)
1021 {
1022 VK_FROM_HANDLE(hk_device, dev, device);
1023 VK_FROM_HANDLE(hk_image, image, pInfo->image);
1024
1025 const VkImagePlaneMemoryRequirementsInfo *plane_info =
1026 vk_find_struct_const(pInfo->pNext, IMAGE_PLANE_MEMORY_REQUIREMENTS_INFO);
1027 const VkImageAspectFlags aspects =
1028 image->disjoint ? plane_info->planeAspect : image->vk.aspects;
1029
1030 hk_get_image_memory_requirements(dev, image, aspects, pMemoryRequirements);
1031 }
1032
1033 VKAPI_ATTR void VKAPI_CALL
hk_GetDeviceImageMemoryRequirements(VkDevice device,const VkDeviceImageMemoryRequirements * pInfo,VkMemoryRequirements2 * pMemoryRequirements)1034 hk_GetDeviceImageMemoryRequirements(VkDevice device,
1035 const VkDeviceImageMemoryRequirements *pInfo,
1036 VkMemoryRequirements2 *pMemoryRequirements)
1037 {
1038 VK_FROM_HANDLE(hk_device, dev, device);
1039 ASSERTED VkResult result;
1040 struct hk_image image = {0};
1041
1042 result = hk_image_init(dev, &image, pInfo->pCreateInfo);
1043 assert(result == VK_SUCCESS);
1044
1045 const VkImageAspectFlags aspects =
1046 image.disjoint ? pInfo->planeAspect : image.vk.aspects;
1047
1048 hk_get_image_memory_requirements(dev, &image, aspects, pMemoryRequirements);
1049
1050 hk_image_finish(dev, &image, NULL);
1051 }
1052
1053 static VkSparseImageMemoryRequirements
hk_fill_sparse_image_memory_reqs(const struct ail_layout * layout,VkImageAspectFlags aspects)1054 hk_fill_sparse_image_memory_reqs(const struct ail_layout *layout,
1055 VkImageAspectFlags aspects)
1056 {
1057 VkSparseImageFormatProperties sparse_format_props =
1058 hk_fill_sparse_image_fmt_props(aspects);
1059
1060 // assert(layout->mip_tail_first_lod <= layout->num_levels);
1061 VkSparseImageMemoryRequirements sparse_memory_reqs = {
1062 .formatProperties = sparse_format_props,
1063 .imageMipTailFirstLod = 0, // layout->mip_tail_first_lod,
1064 .imageMipTailStride = 0,
1065 };
1066
1067 sparse_memory_reqs.imageMipTailSize = layout->size_B;
1068 sparse_memory_reqs.imageMipTailOffset = 0;
1069 return sparse_memory_reqs;
1070 }
1071
1072 static void
hk_get_image_sparse_memory_requirements(struct hk_device * dev,struct hk_image * image,VkImageAspectFlags aspects,uint32_t * pSparseMemoryRequirementCount,VkSparseImageMemoryRequirements2 * pSparseMemoryRequirements)1073 hk_get_image_sparse_memory_requirements(
1074 struct hk_device *dev, struct hk_image *image, VkImageAspectFlags aspects,
1075 uint32_t *pSparseMemoryRequirementCount,
1076 VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
1077 {
1078 VK_OUTARRAY_MAKE_TYPED(VkSparseImageMemoryRequirements2, out,
1079 pSparseMemoryRequirements,
1080 pSparseMemoryRequirementCount);
1081
1082 /* From the Vulkan 1.3.279 spec:
1083 *
1084 * "The sparse image must have been created using the
1085 * VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT flag to retrieve valid sparse
1086 * image memory requirements."
1087 */
1088 if (!(image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT))
1089 return;
1090
1091 /* We don't support multiplane sparse for now */
1092 if (image->plane_count > 1)
1093 return;
1094
1095 vk_outarray_append_typed(VkSparseImageMemoryRequirements2, &out, reqs)
1096 {
1097 reqs->memoryRequirements =
1098 hk_fill_sparse_image_memory_reqs(&image->planes[0].layout, aspects);
1099 };
1100 }
1101
1102 VKAPI_ATTR void VKAPI_CALL
hk_GetImageSparseMemoryRequirements2(VkDevice device,const VkImageSparseMemoryRequirementsInfo2 * pInfo,uint32_t * pSparseMemoryRequirementCount,VkSparseImageMemoryRequirements2 * pSparseMemoryRequirements)1103 hk_GetImageSparseMemoryRequirements2(
1104 VkDevice device, const VkImageSparseMemoryRequirementsInfo2 *pInfo,
1105 uint32_t *pSparseMemoryRequirementCount,
1106 VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
1107 {
1108 VK_FROM_HANDLE(hk_device, dev, device);
1109 VK_FROM_HANDLE(hk_image, image, pInfo->image);
1110
1111 const VkImageAspectFlags aspects = image->vk.aspects;
1112
1113 hk_get_image_sparse_memory_requirements(dev, image, aspects,
1114 pSparseMemoryRequirementCount,
1115 pSparseMemoryRequirements);
1116 }
1117
1118 VKAPI_ATTR void VKAPI_CALL
hk_GetDeviceImageSparseMemoryRequirements(VkDevice device,const VkDeviceImageMemoryRequirements * pInfo,uint32_t * pSparseMemoryRequirementCount,VkSparseImageMemoryRequirements2 * pSparseMemoryRequirements)1119 hk_GetDeviceImageSparseMemoryRequirements(
1120 VkDevice device, const VkDeviceImageMemoryRequirements *pInfo,
1121 uint32_t *pSparseMemoryRequirementCount,
1122 VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
1123 {
1124 VK_FROM_HANDLE(hk_device, dev, device);
1125 ASSERTED VkResult result;
1126 struct hk_image image = {0};
1127
1128 result = hk_image_init(dev, &image, pInfo->pCreateInfo);
1129 assert(result == VK_SUCCESS);
1130
1131 const VkImageAspectFlags aspects =
1132 image.disjoint ? pInfo->planeAspect : image.vk.aspects;
1133
1134 hk_get_image_sparse_memory_requirements(dev, &image, aspects,
1135 pSparseMemoryRequirementCount,
1136 pSparseMemoryRequirements);
1137
1138 hk_image_finish(dev, &image, NULL);
1139 }
1140
1141 static void
hk_get_image_subresource_layout(UNUSED struct hk_device * dev,struct hk_image * image,const VkImageSubresource2KHR * pSubresource,VkSubresourceLayout2KHR * pLayout)1142 hk_get_image_subresource_layout(UNUSED struct hk_device *dev,
1143 struct hk_image *image,
1144 const VkImageSubresource2KHR *pSubresource,
1145 VkSubresourceLayout2KHR *pLayout)
1146 {
1147 const VkImageSubresource *isr = &pSubresource->imageSubresource;
1148
1149 const uint8_t p = hk_image_aspects_to_plane(image, isr->aspectMask);
1150 const struct hk_image_plane *plane = &image->planes[p];
1151
1152 uint64_t offset_B = 0;
1153 if (!image->disjoint) {
1154 uint32_t align_B = 0;
1155 for (unsigned plane = 0; plane < p; plane++)
1156 hk_image_plane_add_req(&image->planes[plane], &offset_B, &align_B);
1157 }
1158 offset_B +=
1159 ail_get_layer_level_B(&plane->layout, isr->arrayLayer, isr->mipLevel);
1160
1161 bool is_3d = image->vk.image_type == VK_IMAGE_TYPE_3D;
1162
1163 pLayout->subresourceLayout = (VkSubresourceLayout){
1164 .offset = offset_B,
1165 .size = ail_get_level_size_B(&plane->layout, isr->mipLevel),
1166
1167 /* From the spec:
1168 *
1169 * It is legal to call vkGetImageSubresourceLayout2KHR with a image
1170 * created with tiling equal to VK_IMAGE_TILING_OPTIMAL, but the
1171 * members of VkSubresourceLayout2KHR::subresourceLayout will have
1172 * undefined values in this case.
1173 *
1174 * So don't collapse with mips.
1175 */
1176 .rowPitch = isr->mipLevel
1177 ? 0
1178 : ail_get_wsi_stride_B(&plane->layout, isr->mipLevel),
1179 .arrayPitch = is_3d ? 0 : plane->layout.layer_stride_B,
1180 .depthPitch = is_3d ? plane->layout.layer_stride_B : 0,
1181 };
1182
1183 VkSubresourceHostMemcpySizeEXT *memcpy_size =
1184 vk_find_struct(pLayout, SUBRESOURCE_HOST_MEMCPY_SIZE_EXT);
1185 if (memcpy_size) {
1186 memcpy_size->size = pLayout->subresourceLayout.size;
1187 }
1188 }
1189
1190 VKAPI_ATTR void VKAPI_CALL
hk_GetImageSubresourceLayout2KHR(VkDevice device,VkImage _image,const VkImageSubresource2KHR * pSubresource,VkSubresourceLayout2KHR * pLayout)1191 hk_GetImageSubresourceLayout2KHR(VkDevice device, VkImage _image,
1192 const VkImageSubresource2KHR *pSubresource,
1193 VkSubresourceLayout2KHR *pLayout)
1194 {
1195 VK_FROM_HANDLE(hk_device, dev, device);
1196 VK_FROM_HANDLE(hk_image, image, _image);
1197
1198 hk_get_image_subresource_layout(dev, image, pSubresource, pLayout);
1199 }
1200
1201 VKAPI_ATTR void VKAPI_CALL
hk_GetDeviceImageSubresourceLayoutKHR(VkDevice device,const VkDeviceImageSubresourceInfoKHR * pInfo,VkSubresourceLayout2KHR * pLayout)1202 hk_GetDeviceImageSubresourceLayoutKHR(
1203 VkDevice device, const VkDeviceImageSubresourceInfoKHR *pInfo,
1204 VkSubresourceLayout2KHR *pLayout)
1205 {
1206 VK_FROM_HANDLE(hk_device, dev, device);
1207 ASSERTED VkResult result;
1208 struct hk_image image = {0};
1209
1210 result = hk_image_init(dev, &image, pInfo->pCreateInfo);
1211 assert(result == VK_SUCCESS);
1212
1213 hk_get_image_subresource_layout(dev, &image, pInfo->pSubresource, pLayout);
1214
1215 hk_image_finish(dev, &image, NULL);
1216 }
1217
1218 static void
hk_image_plane_bind(struct hk_device * dev,struct hk_image_plane * plane,struct hk_device_memory * mem,uint64_t * offset_B)1219 hk_image_plane_bind(struct hk_device *dev, struct hk_image_plane *plane,
1220 struct hk_device_memory *mem, uint64_t *offset_B)
1221 {
1222 *offset_B = align64(*offset_B, HK_PLANE_ALIGN_B);
1223
1224 if (plane->vma_size_B) {
1225 #if 0
1226 agx_bo_bind_vma(dev->ws_dev,
1227 mem->bo,
1228 plane->addr,
1229 plane->vma_size_B,
1230 *offset_B,
1231 plane->nil.pte_kind);
1232 #endif
1233 unreachable("todo");
1234 } else {
1235 plane->addr = mem->bo->va->addr + *offset_B;
1236 plane->map = agx_bo_map(mem->bo) + *offset_B;
1237 plane->rem = mem->bo->size - (*offset_B);
1238 }
1239
1240 *offset_B += plane->layout.size_B;
1241 }
1242
1243 VKAPI_ATTR VkResult VKAPI_CALL
hk_BindImageMemory2(VkDevice device,uint32_t bindInfoCount,const VkBindImageMemoryInfo * pBindInfos)1244 hk_BindImageMemory2(VkDevice device, uint32_t bindInfoCount,
1245 const VkBindImageMemoryInfo *pBindInfos)
1246 {
1247 VK_FROM_HANDLE(hk_device, dev, device);
1248 for (uint32_t i = 0; i < bindInfoCount; ++i) {
1249 VK_FROM_HANDLE(hk_device_memory, mem, pBindInfos[i].memory);
1250 VK_FROM_HANDLE(hk_image, image, pBindInfos[i].image);
1251
1252 /* Ignore this struct on Android, we cannot access swapchain structures
1253 * there. */
1254 #ifdef HK_USE_WSI_PLATFORM
1255 const VkBindImageMemorySwapchainInfoKHR *swapchain_info =
1256 vk_find_struct_const(pBindInfos[i].pNext,
1257 BIND_IMAGE_MEMORY_SWAPCHAIN_INFO_KHR);
1258
1259 if (swapchain_info && swapchain_info->swapchain != VK_NULL_HANDLE) {
1260 VkImage _wsi_image = wsi_common_get_image(swapchain_info->swapchain,
1261 swapchain_info->imageIndex);
1262 VK_FROM_HANDLE(hk_image, wsi_img, _wsi_image);
1263
1264 assert(image->plane_count == 1);
1265 assert(wsi_img->plane_count == 1);
1266
1267 struct hk_image_plane *plane = &image->planes[0];
1268 struct hk_image_plane *swapchain_plane = &wsi_img->planes[0];
1269
1270 /* Copy memory binding information from swapchain image to the current
1271 * image's plane. */
1272 plane->addr = swapchain_plane->addr;
1273 continue;
1274 }
1275 #endif
1276
1277 uint64_t offset_B = pBindInfos[i].memoryOffset;
1278 if (image->disjoint) {
1279 const VkBindImagePlaneMemoryInfo *plane_info = vk_find_struct_const(
1280 pBindInfos[i].pNext, BIND_IMAGE_PLANE_MEMORY_INFO);
1281 uint8_t plane =
1282 hk_image_aspects_to_plane(image, plane_info->planeAspect);
1283 hk_image_plane_bind(dev, &image->planes[plane], mem, &offset_B);
1284 } else {
1285 for (unsigned plane = 0; plane < image->plane_count; plane++) {
1286 hk_image_plane_bind(dev, &image->planes[plane], mem, &offset_B);
1287 }
1288 }
1289
1290 const VkBindMemoryStatusKHR *status =
1291 vk_find_struct_const(pBindInfos[i].pNext, BIND_MEMORY_STATUS_KHR);
1292 if (status != NULL && status->pResult != NULL)
1293 *status->pResult = VK_SUCCESS;
1294 }
1295
1296 return VK_SUCCESS;
1297 }
1298
1299 static uint32_t
hk_plane_index(VkFormat format,VkImageAspectFlags aspect_mask)1300 hk_plane_index(VkFormat format, VkImageAspectFlags aspect_mask)
1301 {
1302 switch (aspect_mask) {
1303 default:
1304 assert(aspect_mask != VK_IMAGE_ASPECT_MEMORY_PLANE_3_BIT_EXT);
1305 return 0;
1306 case VK_IMAGE_ASPECT_PLANE_1_BIT:
1307 case VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT:
1308 return 1;
1309 case VK_IMAGE_ASPECT_PLANE_2_BIT:
1310 case VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT:
1311 return 2;
1312 case VK_IMAGE_ASPECT_STENCIL_BIT:
1313 return format == VK_FORMAT_D32_SFLOAT_S8_UINT;
1314 }
1315 }
1316
1317 static void
hk_copy_memory_to_image(struct hk_device * device,struct hk_image * dst_image,const VkMemoryToImageCopyEXT * info,bool copy_memcpy)1318 hk_copy_memory_to_image(struct hk_device *device, struct hk_image *dst_image,
1319 const VkMemoryToImageCopyEXT *info, bool copy_memcpy)
1320 {
1321 unsigned plane =
1322 hk_plane_index(dst_image->vk.format, info->imageSubresource.aspectMask);
1323 const struct ail_layout *layout = &dst_image->planes[plane].layout;
1324
1325 VkOffset3D offset = info->imageOffset;
1326 VkExtent3D extent = info->imageExtent;
1327 uint32_t src_width = info->memoryRowLength ?: extent.width;
1328 uint32_t src_height = info->memoryImageHeight ?: extent.height;
1329
1330 uint32_t blocksize_B = util_format_get_blocksize(layout->format);
1331 uint32_t src_pitch = src_width * blocksize_B;
1332
1333 unsigned start_layer = (dst_image->vk.image_type == VK_IMAGE_TYPE_3D)
1334 ? offset.z
1335 : info->imageSubresource.baseArrayLayer;
1336 uint32_t layers =
1337 MAX2(extent.depth, vk_image_subresource_layer_count(
1338 &dst_image->vk, &info->imageSubresource));
1339
1340 unsigned level = info->imageSubresource.mipLevel;
1341 uint32_t image_offset = ail_get_layer_level_B(layout, start_layer, level);
1342 uint32_t dst_layer_stride = layout->layer_stride_B;
1343 uint32_t src_layer_stride = copy_memcpy
1344 ? ail_get_level_size_B(layout, level)
1345 : (src_width * src_height * blocksize_B);
1346 bool tiled = ail_is_level_twiddled_uncompressed(
1347 layout, info->imageSubresource.mipLevel);
1348
1349 const char *src =
1350 (const char *)info->pHostPointer + start_layer * dst_layer_stride;
1351 char *dst = (char *)dst_image->planes[plane].map + image_offset;
1352 for (unsigned layer = 0; layer < layers;
1353 layer++, src += src_layer_stride, dst += dst_layer_stride) {
1354 if (copy_memcpy) {
1355 memcpy(dst, src, ail_get_level_size_B(layout, level));
1356 } else if (!tiled) {
1357 uint32_t dst_pitch = ail_get_linear_stride_B(layout, level);
1358 /*TODO:comp*/
1359 for (unsigned y = 0; y < extent.height; y++) {
1360 memcpy(dst + dst_pitch * (y + offset.y) + offset.x * blocksize_B,
1361 src + src_pitch * y, extent.width * blocksize_B);
1362 }
1363 } else {
1364 ail_tile(dst, (void *)src, layout, level, src_pitch, offset.x,
1365 offset.y, extent.width, extent.height);
1366 }
1367 }
1368 }
1369
1370 VKAPI_ATTR VkResult VKAPI_CALL
hk_CopyMemoryToImageEXT(VkDevice _device,const VkCopyMemoryToImageInfoEXT * info)1371 hk_CopyMemoryToImageEXT(VkDevice _device,
1372 const VkCopyMemoryToImageInfoEXT *info)
1373 {
1374 VK_FROM_HANDLE(hk_device, device, _device);
1375 VK_FROM_HANDLE(hk_image, dst_image, info->dstImage);
1376
1377 for (unsigned i = 0; i < info->regionCount; i++) {
1378 hk_copy_memory_to_image(device, dst_image, &info->pRegions[i],
1379 info->flags & VK_HOST_IMAGE_COPY_MEMCPY_EXT);
1380 }
1381
1382 return VK_SUCCESS;
1383 }
1384
1385 static void
hk_copy_image_to_memory(struct hk_device * device,struct hk_image * src_image,const VkImageToMemoryCopyEXT * info,bool copy_memcpy)1386 hk_copy_image_to_memory(struct hk_device *device, struct hk_image *src_image,
1387 const VkImageToMemoryCopyEXT *info, bool copy_memcpy)
1388 {
1389 unsigned plane =
1390 hk_plane_index(src_image->vk.format, info->imageSubresource.aspectMask);
1391 const struct ail_layout *layout = &src_image->planes[plane].layout;
1392
1393 VkOffset3D offset = info->imageOffset;
1394 VkExtent3D extent = info->imageExtent;
1395 uint32_t dst_width = info->memoryRowLength ?: extent.width;
1396 uint32_t dst_height = info->memoryImageHeight ?: extent.height;
1397
1398 #if 0
1399 copy_compressed(src_image->vk.format, &offset, &extent, &dst_width,
1400 &dst_height);
1401 #endif
1402
1403 uint32_t blocksize_B = util_format_get_blocksize(layout->format);
1404 uint32_t dst_pitch = dst_width * blocksize_B;
1405
1406 unsigned start_layer = (src_image->vk.image_type == VK_IMAGE_TYPE_3D)
1407 ? offset.z
1408 : info->imageSubresource.baseArrayLayer;
1409 uint32_t layers =
1410 MAX2(extent.depth, vk_image_subresource_layer_count(
1411 &src_image->vk, &info->imageSubresource));
1412 unsigned level = info->imageSubresource.mipLevel;
1413
1414 uint32_t image_offset = ail_get_layer_level_B(layout, start_layer, level);
1415 uint32_t src_layer_stride = layout->layer_stride_B;
1416 uint32_t dst_layer_stride = copy_memcpy
1417 ? ail_get_level_size_B(layout, level)
1418 : (dst_width * dst_height * blocksize_B);
1419
1420 bool tiled = ail_is_level_twiddled_uncompressed(
1421 layout, info->imageSubresource.mipLevel);
1422
1423 const char *src = (const char *)src_image->planes[plane].map + image_offset;
1424 char *dst = (char *)info->pHostPointer + start_layer * dst_layer_stride;
1425 for (unsigned layer = 0; layer < layers;
1426 layer++, src += src_layer_stride, dst += dst_layer_stride) {
1427
1428 if (copy_memcpy) {
1429 memcpy(dst, src, dst_layer_stride);
1430 } else if (!tiled) {
1431 /* TODO: comp */
1432 uint32_t src_pitch = ail_get_linear_stride_B(layout, level);
1433 for (unsigned y = 0; y < extent.height; y++) {
1434 memcpy(dst + dst_pitch * y,
1435 src + src_pitch * (y + offset.y) + offset.x * blocksize_B,
1436 extent.width * blocksize_B);
1437 }
1438 } else {
1439 ail_detile((void *)src, dst, layout, info->imageSubresource.mipLevel,
1440 dst_pitch, offset.x, offset.y, extent.width, extent.height);
1441 }
1442 }
1443 }
1444
1445 VKAPI_ATTR VkResult VKAPI_CALL
hk_CopyImageToMemoryEXT(VkDevice _device,const VkCopyImageToMemoryInfoEXT * info)1446 hk_CopyImageToMemoryEXT(VkDevice _device,
1447 const VkCopyImageToMemoryInfoEXT *info)
1448 {
1449 VK_FROM_HANDLE(hk_device, device, _device);
1450 VK_FROM_HANDLE(hk_image, image, info->srcImage);
1451
1452 for (unsigned i = 0; i < info->regionCount; i++) {
1453 hk_copy_image_to_memory(device, image, &info->pRegions[i],
1454 info->flags & VK_HOST_IMAGE_COPY_MEMCPY_EXT);
1455 }
1456
1457 return VK_SUCCESS;
1458 }
1459
1460 static void
hk_copy_image_to_image_cpu(struct hk_device * device,struct hk_image * src_image,struct hk_image * dst_image,const VkImageCopy2 * info,bool copy_memcpy)1461 hk_copy_image_to_image_cpu(struct hk_device *device, struct hk_image *src_image,
1462 struct hk_image *dst_image, const VkImageCopy2 *info,
1463 bool copy_memcpy)
1464 {
1465 unsigned src_plane =
1466 hk_plane_index(src_image->vk.format, info->srcSubresource.aspectMask);
1467 unsigned dst_plane =
1468 hk_plane_index(dst_image->vk.format, info->dstSubresource.aspectMask);
1469
1470 const struct ail_layout *src_layout = &src_image->planes[src_plane].layout;
1471 const struct ail_layout *dst_layout = &dst_image->planes[dst_plane].layout;
1472
1473 VkOffset3D src_offset = info->srcOffset;
1474 VkOffset3D dst_offset = info->dstOffset;
1475 VkExtent3D extent = info->extent;
1476 uint32_t layers_to_copy = MAX2(
1477 info->extent.depth,
1478 vk_image_subresource_layer_count(&src_image->vk, &info->srcSubresource));
1479
1480 /* See comment above. */
1481 #if 0
1482 copy_compressed(src_image->vk.format, &src_offset, &extent, NULL, NULL);
1483 copy_compressed(dst_image->vk.format, &dst_offset, NULL, NULL, NULL);
1484 #endif
1485
1486 unsigned src_start_layer = (src_image->vk.image_type == VK_IMAGE_TYPE_3D)
1487 ? src_offset.z
1488 : info->srcSubresource.baseArrayLayer;
1489 unsigned dst_start_layer = (dst_image->vk.image_type == VK_IMAGE_TYPE_3D)
1490 ? dst_offset.z
1491 : info->dstSubresource.baseArrayLayer;
1492
1493 uint32_t src_layer_stride = src_layout->layer_stride_B;
1494 uint32_t dst_layer_stride = dst_layout->layer_stride_B;
1495
1496 uint32_t dst_block_B = util_format_get_blocksize(dst_layout->format);
1497 uint32_t src_block_B = util_format_get_blocksize(src_layout->format);
1498
1499 uint32_t src_image_offset = ail_get_layer_level_B(
1500 src_layout, src_start_layer, info->srcSubresource.mipLevel);
1501 uint32_t dst_image_offset = ail_get_layer_level_B(
1502 dst_layout, dst_start_layer, info->dstSubresource.mipLevel);
1503
1504 bool src_tiled = ail_is_level_twiddled_uncompressed(
1505 src_layout, info->srcSubresource.mipLevel);
1506 bool dst_tiled = ail_is_level_twiddled_uncompressed(
1507 dst_layout, info->dstSubresource.mipLevel);
1508
1509 const char *src =
1510 (const char *)src_image->planes[src_plane].map + src_image_offset;
1511 char *dst = (char *)dst_image->planes[dst_plane].map + dst_image_offset;
1512 for (unsigned layer = 0; layer < layers_to_copy;
1513 layer++, src += src_layer_stride, dst += dst_layer_stride) {
1514
1515 if (copy_memcpy) {
1516 uint32_t src_size =
1517 ail_get_level_size_B(src_layout, info->srcSubresource.mipLevel);
1518 uint32_t dst_size =
1519 ail_get_level_size_B(dst_layout, info->dstSubresource.mipLevel);
1520
1521 assert(src_size == dst_size);
1522 memcpy(dst, src, src_size);
1523 } else if (!src_tiled && !dst_tiled) {
1524 /* TODO comp */
1525 uint32_t src_pitch =
1526 ail_get_linear_stride_B(src_layout, info->srcSubresource.mipLevel);
1527
1528 uint32_t dst_pitch =
1529 ail_get_linear_stride_B(dst_layout, info->dstSubresource.mipLevel);
1530
1531 for (unsigned y = 0; y < extent.height; y++) {
1532 memcpy(dst + dst_pitch * (y + dst_offset.y) +
1533 dst_offset.x * dst_block_B,
1534 src + src_pitch * (y + src_offset.y) +
1535 src_offset.x * src_block_B,
1536 extent.width * src_block_B);
1537 }
1538 } else if (!src_tiled) {
1539 unreachable("todo");
1540 #if 0
1541 fdl6_memcpy_linear_to_tiled(
1542 dst_offset.x, dst_offset.y, extent.width, extent.height, dst,
1543 src + src_pitch * src_offset.y + src_offset.x * src_layout->cpp,
1544 dst_layout, info->dstSubresource.mipLevel, src_pitch,
1545 &device->physical_device->ubwc_config);
1546 #endif
1547 } else if (!dst_tiled) {
1548 unreachable("todo");
1549 #if 0
1550 fdl6_memcpy_tiled_to_linear(
1551 src_offset.x, src_offset.y, extent.width, extent.height,
1552 dst + dst_pitch * dst_offset.y + dst_offset.x * dst_layout->cpp,
1553 src, src_layout, info->dstSubresource.mipLevel, dst_pitch,
1554 &device->physical_device->ubwc_config);
1555 #endif
1556 } else {
1557 /* Work tile-by-tile, holding the unswizzled tile in a temporary
1558 * buffer.
1559 */
1560 char temp_tile[16384];
1561
1562 unsigned src_level = info->srcSubresource.mipLevel;
1563 unsigned dst_level = info->dstSubresource.mipLevel;
1564 uint32_t block_width = src_layout->tilesize_el[src_level].width_el;
1565 uint32_t block_height = src_layout->tilesize_el[src_level].height_el;
1566 uint32_t temp_pitch = block_width * src_block_B;
1567 ;
1568
1569 for (unsigned by = src_offset.y / block_height;
1570 by * block_height < src_offset.y + extent.height; by++) {
1571 uint32_t src_y_start = MAX2(src_offset.y, by * block_height);
1572 uint32_t dst_y_start = src_y_start - src_offset.y + dst_offset.y;
1573 uint32_t height =
1574 MIN2((by + 1) * block_height, src_offset.y + extent.height) -
1575 src_y_start;
1576 for (unsigned bx = src_offset.x / block_width;
1577 bx * block_width < src_offset.x + extent.width; bx++) {
1578 uint32_t src_x_start = MAX2(src_offset.x, bx * block_width);
1579 uint32_t dst_x_start = src_x_start - src_offset.x + dst_offset.x;
1580 uint32_t width =
1581 MIN2((bx + 1) * block_width, src_offset.x + extent.width) -
1582 src_x_start;
1583
1584 ail_detile((void *)src, temp_tile, src_layout, src_level,
1585 temp_pitch, src_x_start, src_y_start, width, height);
1586 ail_tile(dst, temp_tile, dst_layout, dst_level, temp_pitch,
1587 dst_x_start, dst_y_start, width, height);
1588 }
1589 }
1590 }
1591 }
1592 }
1593
1594 VKAPI_ATTR VkResult VKAPI_CALL
hk_CopyImageToImageEXT(VkDevice _device,const VkCopyImageToImageInfoEXT * pCopyImageToImageInfo)1595 hk_CopyImageToImageEXT(VkDevice _device,
1596 const VkCopyImageToImageInfoEXT *pCopyImageToImageInfo)
1597 {
1598 VK_FROM_HANDLE(hk_device, device, _device);
1599 VK_FROM_HANDLE(hk_image, src_image, pCopyImageToImageInfo->srcImage);
1600 VK_FROM_HANDLE(hk_image, dst_image, pCopyImageToImageInfo->dstImage);
1601 bool copy_memcpy =
1602 pCopyImageToImageInfo->flags & VK_HOST_IMAGE_COPY_MEMCPY_EXT;
1603
1604 for (uint32_t i = 0; i < pCopyImageToImageInfo->regionCount; ++i) {
1605 if (src_image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
1606 VkImageCopy2 info = pCopyImageToImageInfo->pRegions[i];
1607 u_foreach_bit(b, info.dstSubresource.aspectMask) {
1608 info.srcSubresource.aspectMask = BITFIELD_BIT(b);
1609 info.dstSubresource.aspectMask = BITFIELD_BIT(b);
1610 hk_copy_image_to_image_cpu(device, src_image, dst_image, &info,
1611 copy_memcpy);
1612 }
1613 continue;
1614 }
1615
1616 hk_copy_image_to_image_cpu(device, src_image, dst_image,
1617 pCopyImageToImageInfo->pRegions + i,
1618 copy_memcpy);
1619 }
1620
1621 return VK_SUCCESS;
1622 }
1623
1624 VKAPI_ATTR VkResult VKAPI_CALL
hk_TransitionImageLayoutEXT(VkDevice device,uint32_t transitionCount,const VkHostImageLayoutTransitionInfoEXT * transitions)1625 hk_TransitionImageLayoutEXT(
1626 VkDevice device, uint32_t transitionCount,
1627 const VkHostImageLayoutTransitionInfoEXT *transitions)
1628 {
1629 /* We don't do anything with layouts so this should be a no-op */
1630 return VK_SUCCESS;
1631 }
1632