1 /*
2 * Copyright © 2022 Collabora Ltd. and Red Hat Inc.
3 * SPDX-License-Identifier: MIT
4 */
5 #include "nvk_image.h"
6
7 #include "nvk_device.h"
8 #include "nvk_device_memory.h"
9 #include "nvk_entrypoints.h"
10 #include "nvk_format.h"
11 #include "nvk_physical_device.h"
12 #include "nvkmd/nvkmd.h"
13
14 #include "util/detect_os.h"
15 #include "vk_android.h"
16 #include "vk_enum_to_str.h"
17 #include "vk_format.h"
18 #include "nil.h"
19 #include "vk_enum_defines.h"
20 #include "vk_format.h"
21
22 #include "clb097.h"
23 #include "clb197.h"
24 #include "clc097.h"
25 #include "clc597.h"
26
27 static VkFormatFeatureFlags2
nvk_get_image_plane_format_features(struct nvk_physical_device * pdev,VkFormat vk_format,VkImageTiling tiling,uint64_t drm_format_mod)28 nvk_get_image_plane_format_features(struct nvk_physical_device *pdev,
29 VkFormat vk_format, VkImageTiling tiling,
30 uint64_t drm_format_mod)
31 {
32 VkFormatFeatureFlags2 features = 0;
33
34 if (tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT &&
35 drm_format_mod != DRM_FORMAT_MOD_LINEAR &&
36 !fourcc_mod_is_vendor(drm_format_mod, NVIDIA))
37 return 0;
38
39 enum pipe_format p_format = nvk_format_to_pipe_format(vk_format);
40 if (p_format == PIPE_FORMAT_NONE)
41 return 0;
42
43 /* You can't tile a non-power-of-two */
44 if (!util_is_power_of_two_nonzero(util_format_get_blocksize(p_format)))
45 return 0;
46
47 if (nil_format_supports_texturing(&pdev->info, p_format)) {
48 features |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT;
49 features |= VK_FORMAT_FEATURE_2_BLIT_SRC_BIT;
50 }
51
52 if (nil_format_supports_filtering(&pdev->info, p_format)) {
53 features |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_FILTER_LINEAR_BIT;
54 if (pdev->info.cls_eng3d >= MAXWELL_B)
55 features |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_FILTER_MINMAX_BIT;
56 }
57
58 /* TODO: VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_DEPTH_COMPARISON_BIT */
59 if (vk_format_has_depth(vk_format)) {
60 features |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_DEPTH_COMPARISON_BIT;
61 }
62
63 if (nil_format_supports_color_targets(&pdev->info, p_format) &&
64 tiling != VK_IMAGE_TILING_LINEAR) {
65 features |= VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT;
66 if (nil_format_supports_blending(&pdev->info, p_format))
67 features |= VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BLEND_BIT;
68 features |= VK_FORMAT_FEATURE_2_BLIT_DST_BIT;
69 }
70
71 if (vk_format_is_depth_or_stencil(vk_format)) {
72 if (!nil_format_supports_depth_stencil(&pdev->info, p_format) ||
73 tiling == VK_IMAGE_TILING_LINEAR)
74 return 0;
75
76 features |= VK_FORMAT_FEATURE_2_DEPTH_STENCIL_ATTACHMENT_BIT;
77 }
78
79 if (nil_format_supports_storage(&pdev->info, p_format)) {
80 features |= VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT |
81 VK_FORMAT_FEATURE_2_STORAGE_WRITE_WITHOUT_FORMAT_BIT;
82 if (pdev->info.cls_eng3d >= MAXWELL_A)
83 features |= VK_FORMAT_FEATURE_2_STORAGE_READ_WITHOUT_FORMAT_BIT;
84 }
85
86 if (nvk_format_supports_atomics(&pdev->info, p_format))
87 features |= VK_FORMAT_FEATURE_2_STORAGE_IMAGE_ATOMIC_BIT;
88
89 if (p_format == PIPE_FORMAT_R8_UINT && tiling == VK_IMAGE_TILING_OPTIMAL)
90 features |= VK_FORMAT_FEATURE_FRAGMENT_SHADING_RATE_ATTACHMENT_BIT_KHR;
91
92 if (features != 0) {
93 features |= VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT;
94 features |= VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT;
95 if (!vk_format_is_depth_or_stencil(vk_format))
96 features |= VK_FORMAT_FEATURE_2_HOST_IMAGE_TRANSFER_BIT_EXT;
97 }
98
99 return features;
100 }
101
102 VkFormatFeatureFlags2
nvk_get_image_format_features(struct nvk_physical_device * pdev,VkFormat vk_format,VkImageTiling tiling,uint64_t drm_format_mod)103 nvk_get_image_format_features(struct nvk_physical_device *pdev,
104 VkFormat vk_format, VkImageTiling tiling,
105 uint64_t drm_format_mod)
106 {
107 const struct vk_format_ycbcr_info *ycbcr_info =
108 vk_format_get_ycbcr_info(vk_format);
109 if (ycbcr_info == NULL) {
110 return nvk_get_image_plane_format_features(pdev, vk_format, tiling,
111 drm_format_mod);
112 }
113
114 /* For multi-plane, we get the feature flags of each plane separately,
115 * then take their intersection as the overall format feature flags
116 */
117 VkFormatFeatureFlags2 features = ~0ull;
118 bool cosited_chroma = false;
119 for (uint8_t plane = 0; plane < ycbcr_info->n_planes; plane++) {
120 const struct vk_format_ycbcr_plane *plane_info = &ycbcr_info->planes[plane];
121 features &= nvk_get_image_plane_format_features(pdev, plane_info->format,
122 tiling, drm_format_mod);
123 if (plane_info->denominator_scales[0] > 1 ||
124 plane_info->denominator_scales[1] > 1)
125 cosited_chroma = true;
126 }
127 if (features == 0)
128 return 0;
129
130 /* Uh... We really should be able to sample from YCbCr */
131 assert(features & VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT);
132 assert(features & VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_FILTER_LINEAR_BIT);
133
134 /* These aren't allowed for YCbCr formats */
135 features &= ~(VK_FORMAT_FEATURE_2_BLIT_SRC_BIT |
136 VK_FORMAT_FEATURE_2_BLIT_DST_BIT |
137 VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT |
138 VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BLEND_BIT |
139 VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT);
140
141 /* This is supported on all YCbCr formats */
142 features |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_YCBCR_CONVERSION_LINEAR_FILTER_BIT;
143
144 if (ycbcr_info->n_planes > 1) {
145 /* DISJOINT_BIT implies that each plane has its own separate binding,
146 * while SEPARATE_RECONSTRUCTION_FILTER_BIT implies that luma and chroma
147 * each have their own, separate filters, so these two bits make sense
148 * for multi-planar formats only.
149 *
150 * For MIDPOINT_CHROMA_SAMPLES_BIT, NVIDIA HW on single-plane interleaved
151 * YCbCr defaults to COSITED_EVEN, which is inaccurate and fails tests.
152 * This can be fixed with a NIR tweak but for now, we only enable this bit
153 * for multi-plane formats. See Issue #9525 on the mesa/main tracker.
154 */
155 features |= VK_FORMAT_FEATURE_DISJOINT_BIT |
156 VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_YCBCR_CONVERSION_SEPARATE_RECONSTRUCTION_FILTER_BIT |
157 VK_FORMAT_FEATURE_2_MIDPOINT_CHROMA_SAMPLES_BIT;
158 }
159
160 if (cosited_chroma)
161 features |= VK_FORMAT_FEATURE_COSITED_CHROMA_SAMPLES_BIT;
162
163 return features;
164 }
165
166 void
nvk_get_drm_format_modifier_properties_list(struct nvk_physical_device * pdev,VkFormat vk_format,VkBaseOutStructure * ext)167 nvk_get_drm_format_modifier_properties_list(struct nvk_physical_device *pdev,
168 VkFormat vk_format,
169 VkBaseOutStructure *ext)
170 {
171 assert(ext->sType == VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT ||
172 ext->sType == VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_2_EXT);
173
174 /* The two top-level data structures are the same. It's only when
175 * you get to walking the actual list of modifier properties that
176 * they differ.
177 */
178 VkDrmFormatModifierPropertiesListEXT *p = (void *)ext;
179
180 /* We don't support modifiers for YCbCr images */
181 if (vk_format_get_ycbcr_info(vk_format) != NULL) {
182 p->drmFormatModifierCount = 0;
183 return;
184 }
185
186 /* Check that we actually support the format so we don't try to query
187 * modifiers for formats NIL doesn't support.
188 */
189 const VkFormatFeatureFlags2 tiled_features =
190 nvk_get_image_plane_format_features(pdev, vk_format,
191 VK_IMAGE_TILING_OPTIMAL,
192 DRM_FORMAT_MOD_INVALID);
193 if (tiled_features == 0) {
194 p->drmFormatModifierCount = 0;
195 return;
196 }
197
198 uint64_t mods[NIL_MAX_DRM_FORMAT_MODS];
199 size_t mod_count = NIL_MAX_DRM_FORMAT_MODS;
200 enum pipe_format p_format = nvk_format_to_pipe_format(vk_format);
201 nil_drm_format_mods_for_format(&pdev->info, nil_format(p_format),
202 &mod_count, &mods);
203 if (mod_count == 0) {
204 p->drmFormatModifierCount = 0;
205 return;
206 }
207
208 switch (ext->sType) {
209 case VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT: {
210 VK_OUTARRAY_MAKE_TYPED(VkDrmFormatModifierPropertiesEXT, out,
211 p->pDrmFormatModifierProperties,
212 &p->drmFormatModifierCount);
213
214 for (uint32_t i = 0; i < mod_count; i++) {
215 const VkFormatFeatureFlags2 features2 =
216 nvk_get_image_format_features(pdev, vk_format,
217 VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT,
218 mods[i]);
219 if (features2 != 0) {
220 vk_outarray_append_typed(VkDrmFormatModifierPropertiesEXT, &out, mp) {
221 mp->drmFormatModifier = mods[i];
222 mp->drmFormatModifierPlaneCount = 1;
223 mp->drmFormatModifierTilingFeatures =
224 vk_format_features2_to_features(features2);
225 }
226 }
227 }
228 break;
229 }
230
231 case VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_2_EXT: {
232 VkDrmFormatModifierPropertiesList2EXT *p2 = (void *)p;
233 VK_OUTARRAY_MAKE_TYPED(VkDrmFormatModifierProperties2EXT, out,
234 p2->pDrmFormatModifierProperties,
235 &p2->drmFormatModifierCount);
236
237 for (uint32_t i = 0; i < mod_count; i++) {
238 const VkFormatFeatureFlags2 features2 =
239 nvk_get_image_format_features(pdev, vk_format,
240 VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT,
241 mods[i]);
242 if (features2 != 0) {
243 vk_outarray_append_typed(VkDrmFormatModifierProperties2EXT, &out, mp) {
244 mp->drmFormatModifier = mods[i];
245 mp->drmFormatModifierPlaneCount = 1;
246 mp->drmFormatModifierTilingFeatures = features2;
247 }
248 }
249 }
250 break;
251 }
252
253 default:
254 unreachable("Invalid structure type");
255 }
256 }
257
258 static VkFormatFeatureFlags2
vk_image_usage_to_format_features(VkImageUsageFlagBits usage_flag)259 vk_image_usage_to_format_features(VkImageUsageFlagBits usage_flag)
260 {
261 assert(util_bitcount(usage_flag) == 1);
262 switch (usage_flag) {
263 case VK_IMAGE_USAGE_TRANSFER_SRC_BIT:
264 return VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT |
265 VK_FORMAT_FEATURE_BLIT_SRC_BIT;
266 case VK_IMAGE_USAGE_TRANSFER_DST_BIT:
267 return VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT |
268 VK_FORMAT_FEATURE_BLIT_DST_BIT;
269 case VK_IMAGE_USAGE_SAMPLED_BIT:
270 return VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT;
271 case VK_IMAGE_USAGE_STORAGE_BIT:
272 return VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT;
273 case VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT:
274 return VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT;
275 case VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT:
276 return VK_FORMAT_FEATURE_2_DEPTH_STENCIL_ATTACHMENT_BIT;
277 case VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT:
278 return VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT |
279 VK_FORMAT_FEATURE_2_DEPTH_STENCIL_ATTACHMENT_BIT;
280 case VK_IMAGE_USAGE_FRAGMENT_SHADING_RATE_ATTACHMENT_BIT_KHR:
281 return VK_FORMAT_FEATURE_FRAGMENT_SHADING_RATE_ATTACHMENT_BIT_KHR;
282 default:
283 return 0;
284 }
285 }
286
287 uint32_t
nvk_image_max_dimension(const struct nv_device_info * info,VkImageType image_type)288 nvk_image_max_dimension(const struct nv_device_info *info,
289 VkImageType image_type)
290 {
291 switch (image_type) {
292 case VK_IMAGE_TYPE_1D:
293 case VK_IMAGE_TYPE_2D:
294 return info->cls_eng3d >= PASCAL_A ? 0x8000 : 0x4000;
295 case VK_IMAGE_TYPE_3D:
296 return 0x4000;
297 default:
298 unreachable("Invalid image type");
299 }
300 }
301
302 static uint64_t
get_explicit_drm_format_mod(const void * pNext)303 get_explicit_drm_format_mod(const void *pNext)
304 {
305 const VkPhysicalDeviceImageDrmFormatModifierInfoEXT *drm_format_mod_info =
306 vk_find_struct_const(pNext,
307 PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT);
308 if (drm_format_mod_info)
309 return drm_format_mod_info->drmFormatModifier;
310 else
311 return DRM_FORMAT_MOD_INVALID;
312 }
313
314 VKAPI_ATTR VkResult VKAPI_CALL
nvk_GetPhysicalDeviceImageFormatProperties2(VkPhysicalDevice physicalDevice,const VkPhysicalDeviceImageFormatInfo2 * pImageFormatInfo,VkImageFormatProperties2 * pImageFormatProperties)315 nvk_GetPhysicalDeviceImageFormatProperties2(
316 VkPhysicalDevice physicalDevice,
317 const VkPhysicalDeviceImageFormatInfo2 *pImageFormatInfo,
318 VkImageFormatProperties2 *pImageFormatProperties)
319 {
320 VK_FROM_HANDLE(nvk_physical_device, pdev, physicalDevice);
321
322 const VkPhysicalDeviceExternalImageFormatInfo *external_info =
323 vk_find_struct_const(pImageFormatInfo->pNext,
324 PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO);
325
326 /* Initialize to zero in case we return VK_ERROR_FORMAT_NOT_SUPPORTED */
327 memset(&pImageFormatProperties->imageFormatProperties, 0,
328 sizeof(pImageFormatProperties->imageFormatProperties));
329
330 uint64_t drm_format_mod =
331 get_explicit_drm_format_mod(pImageFormatInfo->pNext);
332 const struct vk_format_ycbcr_info *ycbcr_info =
333 vk_format_get_ycbcr_info(pImageFormatInfo->format);
334
335 /* For the purposes of these checks, we don't care about all the extra
336 * YCbCr features and we just want the accumulation of features available
337 * to all planes of the given format.
338 */
339 VkFormatFeatureFlags2 features;
340 if (ycbcr_info == NULL) {
341 features = nvk_get_image_plane_format_features(
342 pdev, pImageFormatInfo->format, pImageFormatInfo->tiling,
343 drm_format_mod);
344 } else {
345 features = ~0ull;
346 assert(ycbcr_info->n_planes > 0);
347 for (uint8_t plane = 0; plane < ycbcr_info->n_planes; plane++) {
348 const VkFormat plane_format = ycbcr_info->planes[plane].format;
349 features &= nvk_get_image_plane_format_features(
350 pdev, plane_format, pImageFormatInfo->tiling, drm_format_mod);
351 }
352 }
353
354 if (features == 0)
355 return VK_ERROR_FORMAT_NOT_SUPPORTED;
356
357 if (pImageFormatInfo->tiling == VK_IMAGE_TILING_LINEAR &&
358 pImageFormatInfo->type != VK_IMAGE_TYPE_2D)
359 return VK_ERROR_FORMAT_NOT_SUPPORTED;
360
361 if (ycbcr_info && pImageFormatInfo->type != VK_IMAGE_TYPE_2D)
362 return VK_ERROR_FORMAT_NOT_SUPPORTED;
363
364 /* Maxwell B and earlier don't support sparse residency */
365 if ((pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) &&
366 pdev->info.cls_eng3d < MAXWELL_B)
367 return VK_ERROR_FORMAT_NOT_SUPPORTED;
368
369 /* Don't allow sparse on D32S8 cube maps. The hardware doesn't seem to
370 * handle these correctly and hard-faults instead of the expected soft
371 * fault when there's sparse VA.
372 */
373 if (pImageFormatInfo->format == VK_FORMAT_D32_SFLOAT_S8_UINT &&
374 (pImageFormatInfo->flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) &&
375 (pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) &&
376 pdev->info.cls_eng3d < TURING_A)
377 return VK_ERROR_FORMAT_NOT_SUPPORTED;
378
379 /* From the Vulkan 1.3.279 spec:
380 *
381 * VUID-VkImageCreateInfo-tiling-04121
382 *
383 * "If tiling is VK_IMAGE_TILING_LINEAR, flags must not contain
384 * VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT"
385 *
386 * VUID-VkImageCreateInfo-imageType-00970
387 *
388 * "If imageType is VK_IMAGE_TYPE_1D, flags must not contain
389 * VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT"
390 */
391 if ((pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) &&
392 (pImageFormatInfo->type == VK_IMAGE_TYPE_1D ||
393 pImageFormatInfo->tiling == VK_IMAGE_TILING_LINEAR))
394 return VK_ERROR_FORMAT_NOT_SUPPORTED;
395
396 /* From the Vulkan 1.3.279 spec:
397 *
398 * VUID-VkImageCreateInfo-flags-09403
399 *
400 * "If flags contains VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT, flags
401 * must not include VK_IMAGE_CREATE_SPARSE_ALIASED_BIT,
402 * VK_IMAGE_CREATE_SPARSE_BINDING_BIT, or
403 * VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT"
404 */
405 if ((pImageFormatInfo->flags & VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT) &&
406 (pImageFormatInfo->flags & (VK_IMAGE_CREATE_SPARSE_ALIASED_BIT |
407 VK_IMAGE_CREATE_SPARSE_BINDING_BIT |
408 VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT)))
409 return VK_ERROR_FORMAT_NOT_SUPPORTED;
410
411 if (pImageFormatInfo->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT &&
412 pImageFormatInfo->type != VK_IMAGE_TYPE_2D)
413 return VK_ERROR_FORMAT_NOT_SUPPORTED;
414
415 const uint32_t max_dim =
416 nvk_image_max_dimension(&pdev->info, VK_IMAGE_TYPE_1D);
417 VkExtent3D maxExtent;
418 uint32_t maxArraySize;
419 switch (pImageFormatInfo->type) {
420 case VK_IMAGE_TYPE_1D:
421 maxExtent = (VkExtent3D) { max_dim, 1, 1 };
422 maxArraySize = 2048;
423 break;
424 case VK_IMAGE_TYPE_2D:
425 maxExtent = (VkExtent3D) { max_dim, max_dim, 1 };
426 maxArraySize = 2048;
427 break;
428 case VK_IMAGE_TYPE_3D:
429 maxExtent = (VkExtent3D) { max_dim, max_dim, max_dim };
430 maxArraySize = 1;
431 break;
432 default:
433 unreachable("Invalid image type");
434 }
435 if (pImageFormatInfo->tiling == VK_IMAGE_TILING_LINEAR)
436 maxArraySize = 1;
437
438 assert(util_is_power_of_two_nonzero(max_dim));
439 uint32_t maxMipLevels = util_logbase2(max_dim) + 1;
440 if (ycbcr_info != NULL || pImageFormatInfo->tiling == VK_IMAGE_TILING_LINEAR)
441 maxMipLevels = 1;
442
443 if (pImageFormatInfo->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
444 maxArraySize = 1;
445 maxMipLevels = 1;
446 }
447
448 VkSampleCountFlags sampleCounts = VK_SAMPLE_COUNT_1_BIT;
449 if (pImageFormatInfo->tiling == VK_IMAGE_TILING_OPTIMAL &&
450 pImageFormatInfo->type == VK_IMAGE_TYPE_2D &&
451 ycbcr_info == NULL &&
452 (features & (VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT |
453 VK_FORMAT_FEATURE_2_DEPTH_STENCIL_ATTACHMENT_BIT)) &&
454 !(pImageFormatInfo->flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT)) {
455 sampleCounts = VK_SAMPLE_COUNT_1_BIT |
456 VK_SAMPLE_COUNT_2_BIT |
457 VK_SAMPLE_COUNT_4_BIT |
458 VK_SAMPLE_COUNT_8_BIT;
459 }
460
461 /* From the Vulkan 1.2.199 spec:
462 *
463 * "VK_IMAGE_CREATE_EXTENDED_USAGE_BIT specifies that the image can be
464 * created with usage flags that are not supported for the format the
465 * image is created with but are supported for at least one format a
466 * VkImageView created from the image can have."
467 *
468 * If VK_IMAGE_CREATE_EXTENDED_USAGE_BIT is set, views can be created with
469 * different usage than the image so we can't always filter on usage.
470 * There is one exception to this below for storage.
471 */
472 const VkImageUsageFlags image_usage = pImageFormatInfo->usage;
473 VkImageUsageFlags view_usage = image_usage;
474 if (pImageFormatInfo->flags & VK_IMAGE_CREATE_EXTENDED_USAGE_BIT)
475 view_usage = 0;
476
477 u_foreach_bit(b, view_usage) {
478 VkFormatFeatureFlags2 usage_features =
479 vk_image_usage_to_format_features(1 << b);
480 if (usage_features && !(features & usage_features))
481 return VK_ERROR_FORMAT_NOT_SUPPORTED;
482 }
483
484 const VkExternalMemoryProperties *ext_mem_props = NULL;
485 if (external_info != NULL && external_info->handleType != 0) {
486 bool tiling_has_explicit_layout;
487 switch (pImageFormatInfo->tiling) {
488 case VK_IMAGE_TILING_LINEAR:
489 case VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT:
490 tiling_has_explicit_layout = true;
491 break;
492 case VK_IMAGE_TILING_OPTIMAL:
493 tiling_has_explicit_layout = false;
494 break;
495 default:
496 unreachable("Unsupported VkImageTiling");
497 }
498
499 switch (external_info->handleType) {
500 case VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT:
501 /* No special restrictions */
502 if (tiling_has_explicit_layout) {
503 /* With an explicit memory layout, we don't care which type of
504 * fd the image belongs too. Both OPAQUE_FD and DMA_BUF are
505 * interchangeable here.
506 */
507 ext_mem_props = &nvk_dma_buf_mem_props;
508 } else {
509 ext_mem_props = &nvk_opaque_fd_mem_props;
510 }
511 break;
512
513 case VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT:
514 if (!tiling_has_explicit_layout) {
515 return vk_errorf(pdev, VK_ERROR_FORMAT_NOT_SUPPORTED,
516 "VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT "
517 "requires VK_IMAGE_TILING_LINEAR or "
518 "VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT");
519 }
520 ext_mem_props = &nvk_dma_buf_mem_props;
521 break;
522
523 default:
524 /* From the Vulkan 1.3.256 spec:
525 *
526 * "If handleType is not compatible with the [parameters] in
527 * VkPhysicalDeviceImageFormatInfo2, then
528 * vkGetPhysicalDeviceImageFormatProperties2 returns
529 * VK_ERROR_FORMAT_NOT_SUPPORTED."
530 */
531 return vk_errorf(pdev, VK_ERROR_FORMAT_NOT_SUPPORTED,
532 "unsupported VkExternalMemoryHandleTypeFlagBits: %s ",
533 vk_ExternalMemoryHandleTypeFlagBits_to_str(external_info->handleType));
534 }
535 }
536
537 const unsigned plane_count =
538 vk_format_get_plane_count(pImageFormatInfo->format);
539
540 /* From the Vulkan 1.3.259 spec, VkImageCreateInfo:
541 *
542 * VUID-VkImageCreateInfo-imageCreateFormatFeatures-02260
543 *
544 * "If format is a multi-planar format, and if imageCreateFormatFeatures
545 * (as defined in Image Creation Limits) does not contain
546 * VK_FORMAT_FEATURE_DISJOINT_BIT, then flags must not contain
547 * VK_IMAGE_CREATE_DISJOINT_BIT"
548 *
549 * This is satisfied trivially because we support DISJOINT on all
550 * multi-plane formats. Also,
551 *
552 * VUID-VkImageCreateInfo-format-01577
553 *
554 * "If format is not a multi-planar format, and flags does not include
555 * VK_IMAGE_CREATE_ALIAS_BIT, flags must not contain
556 * VK_IMAGE_CREATE_DISJOINT_BIT"
557 */
558 if (plane_count == 1 &&
559 !(pImageFormatInfo->flags & VK_IMAGE_CREATE_ALIAS_BIT) &&
560 (pImageFormatInfo->flags & VK_IMAGE_CREATE_DISJOINT_BIT))
561 return VK_ERROR_FORMAT_NOT_SUPPORTED;
562
563 if (ycbcr_info &&
564 ((pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) ||
565 (pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT)))
566 return VK_ERROR_FORMAT_NOT_SUPPORTED;
567
568 if ((pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) &&
569 (pImageFormatInfo->usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT))
570 return VK_ERROR_FORMAT_NOT_SUPPORTED;
571
572 pImageFormatProperties->imageFormatProperties = (VkImageFormatProperties) {
573 .maxExtent = maxExtent,
574 .maxMipLevels = maxMipLevels,
575 .maxArrayLayers = maxArraySize,
576 .sampleCounts = sampleCounts,
577 .maxResourceSize = UINT32_MAX, /* TODO */
578 };
579
580 vk_foreach_struct(s, pImageFormatProperties->pNext) {
581 switch (s->sType) {
582 case VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES: {
583 VkExternalImageFormatProperties *p = (void *)s;
584 /* From the Vulkan 1.3.256 spec:
585 *
586 * "If handleType is 0, vkGetPhysicalDeviceImageFormatProperties2
587 * will behave as if VkPhysicalDeviceExternalImageFormatInfo was
588 * not present, and VkExternalImageFormatProperties will be
589 * ignored."
590 *
591 * This is true if and only if ext_mem_props == NULL
592 */
593 if (ext_mem_props != NULL)
594 p->externalMemoryProperties = *ext_mem_props;
595 break;
596 }
597 case VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_IMAGE_FORMAT_PROPERTIES: {
598 VkSamplerYcbcrConversionImageFormatProperties *ycbcr_props = (void *) s;
599 ycbcr_props->combinedImageSamplerDescriptorCount = plane_count;
600 break;
601 }
602 case VK_STRUCTURE_TYPE_HOST_IMAGE_COPY_DEVICE_PERFORMANCE_QUERY_EXT: {
603 VkHostImageCopyDevicePerformanceQueryEXT *host_props = (void *) s;
604 host_props->optimalDeviceAccess = true;
605 host_props->identicalMemoryLayout = true;
606 break;
607 }
608 default:
609 vk_debug_ignored_stype(s->sType);
610 break;
611 }
612 }
613
614 return VK_SUCCESS;
615 }
616
617 static enum nil_image_dim
vk_image_type_to_nil_dim(VkImageType type)618 vk_image_type_to_nil_dim(VkImageType type)
619 {
620 switch (type) {
621 case VK_IMAGE_TYPE_1D: return NIL_IMAGE_DIM_1D;
622 case VK_IMAGE_TYPE_2D: return NIL_IMAGE_DIM_2D;
623 case VK_IMAGE_TYPE_3D: return NIL_IMAGE_DIM_3D;
624 default:
625 unreachable("Invalid image type");
626 }
627 }
628
629 static VkSparseImageFormatProperties
nvk_fill_sparse_image_fmt_props(VkImageAspectFlags aspects,const enum pipe_format format,const enum nil_image_dim dim,const enum nil_sample_layout sample_layout)630 nvk_fill_sparse_image_fmt_props(VkImageAspectFlags aspects,
631 const enum pipe_format format,
632 const enum nil_image_dim dim,
633 const enum nil_sample_layout sample_layout)
634 {
635 struct nil_Extent4D_Pixels sparse_block_extent_px =
636 nil_sparse_block_extent_px(nil_format(format), dim, sample_layout);
637
638 assert(sparse_block_extent_px.array_len == 1);
639
640 VkSparseImageFormatProperties sparse_format_props = {
641 .aspectMask = aspects,
642 .flags = VK_SPARSE_IMAGE_FORMAT_SINGLE_MIPTAIL_BIT,
643 .imageGranularity = {
644 .width = sparse_block_extent_px.width,
645 .height = sparse_block_extent_px.height,
646 .depth = sparse_block_extent_px.depth,
647 },
648 };
649
650 return sparse_format_props;
651 }
652
653 VKAPI_ATTR void VKAPI_CALL
nvk_GetPhysicalDeviceSparseImageFormatProperties2(VkPhysicalDevice physicalDevice,const VkPhysicalDeviceSparseImageFormatInfo2 * pFormatInfo,uint32_t * pPropertyCount,VkSparseImageFormatProperties2 * pProperties)654 nvk_GetPhysicalDeviceSparseImageFormatProperties2(
655 VkPhysicalDevice physicalDevice,
656 const VkPhysicalDeviceSparseImageFormatInfo2* pFormatInfo,
657 uint32_t *pPropertyCount,
658 VkSparseImageFormatProperties2 *pProperties)
659 {
660 VkResult result;
661
662 /* Check if the given format info is valid first before returning sparse
663 * props. The easiest way to do this is to just call
664 * nvk_GetPhysicalDeviceImageFormatProperties2()
665 */
666 const VkPhysicalDeviceImageFormatInfo2 img_fmt_info = {
667 .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
668 .format = pFormatInfo->format,
669 .type = pFormatInfo->type,
670 .tiling = pFormatInfo->tiling,
671 .usage = pFormatInfo->usage,
672 .flags = VK_IMAGE_CREATE_SPARSE_BINDING_BIT |
673 VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT,
674 };
675
676 VkImageFormatProperties2 img_fmt_props2 = {
677 .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
678 .pNext = NULL,
679 };
680
681 result = nvk_GetPhysicalDeviceImageFormatProperties2(physicalDevice,
682 &img_fmt_info,
683 &img_fmt_props2);
684 if (result != VK_SUCCESS) {
685 *pPropertyCount = 0;
686 return;
687 }
688
689 const VkImageFormatProperties *props = &img_fmt_props2.imageFormatProperties;
690 if (!(pFormatInfo->samples & props->sampleCounts)) {
691 *pPropertyCount = 0;
692 return;
693 }
694
695 VK_OUTARRAY_MAKE_TYPED(VkSparseImageFormatProperties2, out,
696 pProperties, pPropertyCount);
697
698 VkImageAspectFlags aspects = vk_format_aspects(pFormatInfo->format);
699 const enum pipe_format pipe_format =
700 nvk_format_to_pipe_format(pFormatInfo->format);
701 const enum nil_image_dim dim = vk_image_type_to_nil_dim(pFormatInfo->type);
702 const enum nil_sample_layout sample_layout =
703 nil_choose_sample_layout(pFormatInfo->samples);
704
705 vk_outarray_append_typed(VkSparseImageFormatProperties2, &out, props) {
706 props->properties = nvk_fill_sparse_image_fmt_props(aspects, pipe_format,
707 dim, sample_layout);
708 }
709 }
710
711 static VkResult
nvk_image_init(struct nvk_device * dev,struct nvk_image * image,const VkImageCreateInfo * pCreateInfo)712 nvk_image_init(struct nvk_device *dev,
713 struct nvk_image *image,
714 const VkImageCreateInfo *pCreateInfo)
715 {
716 struct nvk_physical_device *pdev = nvk_device_physical(dev);
717
718 vk_image_init(&dev->vk, &image->vk, pCreateInfo);
719
720 if ((image->vk.usage & (VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
721 VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT)) &&
722 image->vk.samples > 1) {
723 image->vk.usage |= VK_IMAGE_USAGE_SAMPLED_BIT;
724 image->vk.stencil_usage |= VK_IMAGE_USAGE_SAMPLED_BIT;
725 }
726
727 if (image->vk.usage & VK_IMAGE_USAGE_TRANSFER_SRC_BIT)
728 image->vk.usage |= VK_IMAGE_USAGE_SAMPLED_BIT;
729 if (image->vk.usage & VK_IMAGE_USAGE_TRANSFER_DST_BIT)
730 image->vk.usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
731
732 nil_image_usage_flags usage = 0;
733 if (pCreateInfo->tiling == VK_IMAGE_TILING_LINEAR)
734 usage |= NIL_IMAGE_USAGE_LINEAR_BIT;
735 if (pCreateInfo->flags & VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT)
736 usage |= NIL_IMAGE_USAGE_2D_VIEW_BIT;
737 if (pCreateInfo->flags & VK_IMAGE_CREATE_2D_VIEW_COMPATIBLE_BIT_EXT)
738 usage |= NIL_IMAGE_USAGE_2D_VIEW_BIT;
739
740 /* In order to be able to clear 3D depth/stencil images, we need to bind
741 * them as 2D arrays. Fortunately, 3D depth/stencil shouldn't be common.
742 */
743 if ((image->vk.aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
744 VK_IMAGE_ASPECT_STENCIL_BIT)) &&
745 pCreateInfo->imageType == VK_IMAGE_TYPE_3D)
746 usage |= NIL_IMAGE_USAGE_2D_VIEW_BIT;
747
748 image->plane_count = vk_format_get_plane_count(pCreateInfo->format);
749 image->disjoint = image->plane_count > 1 &&
750 (pCreateInfo->flags & VK_IMAGE_CREATE_DISJOINT_BIT);
751
752 if (image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) {
753 /* Sparse multiplane is not supported */
754 assert(image->plane_count == 1);
755 usage |= NIL_IMAGE_USAGE_SPARSE_RESIDENCY_BIT;
756 }
757
758 uint32_t explicit_row_stride_B = 0;
759
760 /* This section is removed by the optimizer for non-ANDROID builds */
761 if (vk_image_is_android_native_buffer(&image->vk)) {
762 VkImageDrmFormatModifierExplicitCreateInfoEXT eci;
763 VkSubresourceLayout a_plane_layouts[4];
764 VkResult result = vk_android_get_anb_layout(
765 pCreateInfo, &eci, a_plane_layouts, 4);
766 if (result != VK_SUCCESS)
767 return result;
768
769 image->vk.drm_format_mod = eci.drmFormatModifier;
770 explicit_row_stride_B = eci.pPlaneLayouts[0].rowPitch;
771 }
772
773 if (image->vk.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
774 /* Modifiers are not supported with YCbCr */
775 assert(image->plane_count == 1);
776
777 const struct VkImageDrmFormatModifierExplicitCreateInfoEXT *mod_explicit_info =
778 vk_find_struct_const(pCreateInfo->pNext,
779 IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT);
780 if (mod_explicit_info) {
781 image->vk.drm_format_mod = mod_explicit_info->drmFormatModifier;
782 /* Normally with explicit modifiers, the client specifies all strides,
783 * however in our case, we can only really make use of this in the linear
784 * case, and we can only create 2D non-array linear images, so ultimately
785 * we only care about the row stride.
786 */
787 explicit_row_stride_B = mod_explicit_info->pPlaneLayouts->rowPitch;
788 } else {
789 const struct VkImageDrmFormatModifierListCreateInfoEXT *mod_list_info =
790 vk_find_struct_const(pCreateInfo->pNext,
791 IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
792
793 enum pipe_format p_format =
794 nvk_format_to_pipe_format(pCreateInfo->format);
795 image->vk.drm_format_mod =
796 nil_select_best_drm_format_mod(&pdev->info, nil_format(p_format),
797 mod_list_info->drmFormatModifierCount,
798 mod_list_info->pDrmFormatModifiers);
799 assert(image->vk.drm_format_mod != DRM_FORMAT_MOD_INVALID);
800 }
801
802 if (image->vk.drm_format_mod == DRM_FORMAT_MOD_LINEAR) {
803 /* We only have one shadow plane per nvk_image */
804 assert(image->plane_count == 1);
805
806 struct nil_image_init_info tiled_shadow_nil_info = {
807 .dim = vk_image_type_to_nil_dim(pCreateInfo->imageType),
808 .format = nil_format(nvk_format_to_pipe_format(image->vk.format)),
809 .modifier = DRM_FORMAT_MOD_INVALID,
810 .extent_px = {
811 .width = pCreateInfo->extent.width,
812 .height = pCreateInfo->extent.height,
813 .depth = pCreateInfo->extent.depth,
814 .array_len = pCreateInfo->arrayLayers,
815 },
816 .levels = pCreateInfo->mipLevels,
817 .samples = pCreateInfo->samples,
818 .usage = usage & ~NIL_IMAGE_USAGE_LINEAR_BIT,
819 .explicit_row_stride_B = 0,
820 };
821 image->linear_tiled_shadow.nil =
822 nil_image_new(&pdev->info, &tiled_shadow_nil_info);
823 }
824 }
825
826 const struct vk_format_ycbcr_info *ycbcr_info =
827 vk_format_get_ycbcr_info(pCreateInfo->format);
828 for (uint8_t plane = 0; plane < image->plane_count; plane++) {
829 VkFormat format = ycbcr_info ?
830 ycbcr_info->planes[plane].format : pCreateInfo->format;
831 const uint8_t width_scale = ycbcr_info ?
832 ycbcr_info->planes[plane].denominator_scales[0] : 1;
833 const uint8_t height_scale = ycbcr_info ?
834 ycbcr_info->planes[plane].denominator_scales[1] : 1;
835 struct nil_image_init_info nil_info = {
836 .dim = vk_image_type_to_nil_dim(pCreateInfo->imageType),
837 .format = nil_format(nvk_format_to_pipe_format(format)),
838 .modifier = image->vk.drm_format_mod,
839 .extent_px = {
840 .width = pCreateInfo->extent.width / width_scale,
841 .height = pCreateInfo->extent.height / height_scale,
842 .depth = pCreateInfo->extent.depth,
843 .array_len = pCreateInfo->arrayLayers,
844 },
845 .levels = pCreateInfo->mipLevels,
846 .samples = pCreateInfo->samples,
847 .usage = usage,
848 .explicit_row_stride_B = explicit_row_stride_B,
849 };
850
851 image->planes[plane].nil = nil_image_new(&pdev->info, &nil_info);
852 }
853
854 if (image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
855 struct nil_image_init_info stencil_nil_info = {
856 .dim = vk_image_type_to_nil_dim(pCreateInfo->imageType),
857 .format = nil_format(PIPE_FORMAT_R32_UINT),
858 .modifier = DRM_FORMAT_MOD_INVALID,
859 .extent_px = {
860 .width = pCreateInfo->extent.width,
861 .height = pCreateInfo->extent.height,
862 .depth = pCreateInfo->extent.depth,
863 .array_len = pCreateInfo->arrayLayers,
864 },
865 .levels = pCreateInfo->mipLevels,
866 .samples = pCreateInfo->samples,
867 .usage = usage,
868 .explicit_row_stride_B = 0,
869 };
870
871 image->stencil_copy_temp.nil =
872 nil_image_new(&pdev->info, &stencil_nil_info);
873 }
874
875 return VK_SUCCESS;
876 }
877
878 static void
nvk_image_plane_size_align_B(struct nvk_device * dev,const struct nvk_image * image,const struct nvk_image_plane * plane,uint64_t * size_B_out,uint64_t * align_B_out)879 nvk_image_plane_size_align_B(struct nvk_device *dev,
880 const struct nvk_image *image,
881 const struct nvk_image_plane *plane,
882 uint64_t *size_B_out, uint64_t *align_B_out)
883 {
884 struct nvk_physical_device *pdev = nvk_device_physical(dev);
885 const bool sparse_bound =
886 image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT;
887
888 assert(util_is_power_of_two_or_zero64(plane->nil.align_B));
889 if (sparse_bound || plane->nil.pte_kind) {
890 *align_B_out = MAX2(plane->nil.align_B, pdev->nvkmd->bind_align_B);
891 } else {
892 *align_B_out = plane->nil.align_B;
893 }
894 *size_B_out = align64(plane->nil.size_B, *align_B_out);
895 }
896
897 static VkResult
nvk_image_plane_alloc_va(struct nvk_device * dev,const struct nvk_image * image,struct nvk_image_plane * plane)898 nvk_image_plane_alloc_va(struct nvk_device *dev,
899 const struct nvk_image *image,
900 struct nvk_image_plane *plane)
901 {
902 VkResult result;
903
904 const bool sparse_bound =
905 image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT;
906 const bool sparse_resident =
907 image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT;
908 assert(sparse_bound || !sparse_resident);
909
910 if (sparse_bound || plane->nil.pte_kind) {
911 enum nvkmd_va_flags va_flags = 0;
912 if (sparse_resident)
913 va_flags |= NVKMD_VA_SPARSE;
914
915 uint64_t va_size_B, va_align_B;
916 nvk_image_plane_size_align_B(dev, image, plane, &va_size_B, &va_align_B);
917
918 result = nvkmd_dev_alloc_va(dev->nvkmd, &dev->vk.base,
919 va_flags, plane->nil.pte_kind,
920 va_size_B, va_align_B,
921 0 /* fixed_addr */, &plane->va);
922 if (result != VK_SUCCESS)
923 return result;
924
925 plane->addr = plane->va->addr;
926 }
927
928 return VK_SUCCESS;
929 }
930
931 static void
nvk_image_plane_finish(struct nvk_device * dev,struct nvk_image_plane * plane,VkImageCreateFlags create_flags,const VkAllocationCallbacks * pAllocator)932 nvk_image_plane_finish(struct nvk_device *dev,
933 struct nvk_image_plane *plane,
934 VkImageCreateFlags create_flags,
935 const VkAllocationCallbacks *pAllocator)
936 {
937 if (plane->va != NULL)
938 nvkmd_va_free(plane->va);
939 }
940
941 static void
nvk_image_finish(struct nvk_device * dev,struct nvk_image * image,const VkAllocationCallbacks * pAllocator)942 nvk_image_finish(struct nvk_device *dev, struct nvk_image *image,
943 const VkAllocationCallbacks *pAllocator)
944 {
945 for (uint8_t plane = 0; plane < image->plane_count; plane++) {
946 nvk_image_plane_finish(dev, &image->planes[plane],
947 image->vk.create_flags, pAllocator);
948 }
949
950 if (image->stencil_copy_temp.nil.size_B > 0) {
951 nvk_image_plane_finish(dev, &image->stencil_copy_temp,
952 image->vk.create_flags, pAllocator);
953 }
954
955 assert(image->linear_tiled_shadow.va == NULL);
956 if (image->linear_tiled_shadow_mem != NULL)
957 nvkmd_mem_unref(image->linear_tiled_shadow_mem);
958
959 vk_image_finish(&image->vk);
960 }
961
962 VKAPI_ATTR VkResult VKAPI_CALL
nvk_CreateImage(VkDevice _device,const VkImageCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkImage * pImage)963 nvk_CreateImage(VkDevice _device,
964 const VkImageCreateInfo *pCreateInfo,
965 const VkAllocationCallbacks *pAllocator,
966 VkImage *pImage)
967 {
968 VK_FROM_HANDLE(nvk_device, dev, _device);
969 struct nvk_physical_device *pdev = nvk_device_physical(dev);
970 struct nvk_image *image;
971 VkResult result;
972
973 #ifdef NVK_USE_WSI_PLATFORM
974 /* Ignore swapchain creation info on Android. Since we don't have an
975 * implementation in Mesa, we're guaranteed to access an Android object
976 * incorrectly.
977 */
978 const VkImageSwapchainCreateInfoKHR *swapchain_info =
979 vk_find_struct_const(pCreateInfo->pNext, IMAGE_SWAPCHAIN_CREATE_INFO_KHR);
980 if (swapchain_info && swapchain_info->swapchain != VK_NULL_HANDLE) {
981 return wsi_common_create_swapchain_image(&pdev->wsi_device,
982 pCreateInfo,
983 swapchain_info->swapchain,
984 pImage);
985 }
986 #endif
987
988 image = vk_zalloc2(&dev->vk.alloc, pAllocator, sizeof(*image), 8,
989 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
990 if (!image)
991 return vk_error(dev, VK_ERROR_OUT_OF_HOST_MEMORY);
992
993 result = nvk_image_init(dev, image, pCreateInfo);
994 if (result != VK_SUCCESS) {
995 vk_free2(&dev->vk.alloc, pAllocator, image);
996 return result;
997 }
998
999 for (uint8_t plane = 0; plane < image->plane_count; plane++) {
1000 result = nvk_image_plane_alloc_va(dev, image, &image->planes[plane]);
1001 if (result != VK_SUCCESS) {
1002 nvk_image_finish(dev, image, pAllocator);
1003 vk_free2(&dev->vk.alloc, pAllocator, image);
1004 return result;
1005 }
1006 }
1007
1008 if (image->stencil_copy_temp.nil.size_B > 0) {
1009 result = nvk_image_plane_alloc_va(dev, image, &image->stencil_copy_temp);
1010 if (result != VK_SUCCESS) {
1011 nvk_image_finish(dev, image, pAllocator);
1012 vk_free2(&dev->vk.alloc, pAllocator, image);
1013 return result;
1014 }
1015 }
1016
1017 if (image->linear_tiled_shadow.nil.size_B > 0) {
1018 struct nvk_image_plane *shadow = &image->linear_tiled_shadow;
1019 result = nvkmd_dev_alloc_tiled_mem(dev->nvkmd, &dev->vk.base,
1020 shadow->nil.size_B, shadow->nil.align_B,
1021 shadow->nil.pte_kind, shadow->nil.tile_mode,
1022 NVKMD_MEM_LOCAL,
1023 &image->linear_tiled_shadow_mem);
1024 if (result != VK_SUCCESS) {
1025 nvk_image_finish(dev, image, pAllocator);
1026 vk_free2(&dev->vk.alloc, pAllocator, image);
1027 return result;
1028 }
1029 shadow->addr = image->linear_tiled_shadow_mem->va->addr;
1030 }
1031
1032 /* This section is removed by the optimizer for non-ANDROID builds */
1033 if (vk_image_is_android_native_buffer(&image->vk)) {
1034 result = vk_android_import_anb(&dev->vk, pCreateInfo, pAllocator,
1035 &image->vk);
1036 if (result != VK_SUCCESS) {
1037 nvk_image_finish(dev, image, pAllocator);
1038 vk_free2(&dev->vk.alloc, pAllocator, image);
1039 return result;
1040 }
1041 }
1042
1043 *pImage = nvk_image_to_handle(image);
1044
1045 return VK_SUCCESS;
1046 }
1047
1048 VKAPI_ATTR void VKAPI_CALL
nvk_DestroyImage(VkDevice device,VkImage _image,const VkAllocationCallbacks * pAllocator)1049 nvk_DestroyImage(VkDevice device,
1050 VkImage _image,
1051 const VkAllocationCallbacks *pAllocator)
1052 {
1053 VK_FROM_HANDLE(nvk_device, dev, device);
1054 VK_FROM_HANDLE(nvk_image, image, _image);
1055
1056 if (!image)
1057 return;
1058
1059 nvk_image_finish(dev, image, pAllocator);
1060 vk_free2(&dev->vk.alloc, pAllocator, image);
1061 }
1062
1063 static void
nvk_image_plane_add_req(struct nvk_device * dev,const struct nvk_image * image,const struct nvk_image_plane * plane,uint64_t * size_B,uint32_t * align_B)1064 nvk_image_plane_add_req(struct nvk_device *dev,
1065 const struct nvk_image *image,
1066 const struct nvk_image_plane *plane,
1067 uint64_t *size_B, uint32_t *align_B)
1068 {
1069 assert(util_is_power_of_two_or_zero64(*align_B));
1070 uint64_t plane_size_B, plane_align_B;
1071 nvk_image_plane_size_align_B(dev, image, plane,
1072 &plane_size_B, &plane_align_B);
1073
1074 *align_B = MAX2(*align_B, plane_align_B);
1075 *size_B = align64(*size_B, plane_align_B);
1076 *size_B += plane_size_B;
1077 }
1078
1079 static void
nvk_get_image_memory_requirements(struct nvk_device * dev,struct nvk_image * image,VkImageAspectFlags aspects,VkMemoryRequirements2 * pMemoryRequirements)1080 nvk_get_image_memory_requirements(struct nvk_device *dev,
1081 struct nvk_image *image,
1082 VkImageAspectFlags aspects,
1083 VkMemoryRequirements2 *pMemoryRequirements)
1084 {
1085 struct nvk_physical_device *pdev = nvk_device_physical(dev);
1086 uint32_t memory_types = (1 << pdev->mem_type_count) - 1;
1087
1088 /* Remove non host visible heaps from the types for host image copy in case
1089 * of potential issues. This should be removed when we get ReBAR.
1090 */
1091 if (image->vk.usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT) {
1092 struct nvk_physical_device *pdev = nvk_device_physical(dev);
1093 for (uint32_t i = 0; i < pdev->mem_type_count; i++) {
1094 if (!(pdev->mem_types[i].propertyFlags &
1095 VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT))
1096 memory_types &= ~BITFIELD_BIT(i);
1097 }
1098 }
1099
1100 // TODO hope for the best?
1101
1102 uint64_t size_B = 0;
1103 uint32_t align_B = 0;
1104 if (image->disjoint) {
1105 uint8_t plane = nvk_image_memory_aspects_to_plane(image, aspects);
1106 nvk_image_plane_add_req(dev, image, &image->planes[plane],
1107 &size_B, &align_B);
1108 } else {
1109 for (unsigned plane = 0; plane < image->plane_count; plane++) {
1110 nvk_image_plane_add_req(dev, image, &image->planes[plane],
1111 &size_B, &align_B);
1112 }
1113 }
1114
1115 if (image->stencil_copy_temp.nil.size_B > 0) {
1116 nvk_image_plane_add_req(dev, image, &image->stencil_copy_temp,
1117 &size_B, &align_B);
1118 }
1119
1120 pMemoryRequirements->memoryRequirements.memoryTypeBits = memory_types;
1121 pMemoryRequirements->memoryRequirements.alignment = align_B;
1122 pMemoryRequirements->memoryRequirements.size = size_B;
1123
1124 vk_foreach_struct_const(ext, pMemoryRequirements->pNext) {
1125 switch (ext->sType) {
1126 case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS: {
1127 VkMemoryDedicatedRequirements *dedicated = (void *)ext;
1128 dedicated->prefersDedicatedAllocation =
1129 image->vk.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
1130 dedicated->requiresDedicatedAllocation =
1131 image->vk.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
1132 break;
1133 }
1134 default:
1135 vk_debug_ignored_stype(ext->sType);
1136 break;
1137 }
1138 }
1139 }
1140
1141 VKAPI_ATTR void VKAPI_CALL
nvk_GetImageMemoryRequirements2(VkDevice device,const VkImageMemoryRequirementsInfo2 * pInfo,VkMemoryRequirements2 * pMemoryRequirements)1142 nvk_GetImageMemoryRequirements2(VkDevice device,
1143 const VkImageMemoryRequirementsInfo2 *pInfo,
1144 VkMemoryRequirements2 *pMemoryRequirements)
1145 {
1146 VK_FROM_HANDLE(nvk_device, dev, device);
1147 VK_FROM_HANDLE(nvk_image, image, pInfo->image);
1148
1149 const VkImagePlaneMemoryRequirementsInfo *plane_info =
1150 vk_find_struct_const(pInfo->pNext, IMAGE_PLANE_MEMORY_REQUIREMENTS_INFO);
1151 const VkImageAspectFlags aspects =
1152 image->disjoint ? plane_info->planeAspect : image->vk.aspects;
1153
1154 nvk_get_image_memory_requirements(dev, image, aspects,
1155 pMemoryRequirements);
1156 }
1157
1158 VKAPI_ATTR void VKAPI_CALL
nvk_GetDeviceImageMemoryRequirements(VkDevice device,const VkDeviceImageMemoryRequirements * pInfo,VkMemoryRequirements2 * pMemoryRequirements)1159 nvk_GetDeviceImageMemoryRequirements(VkDevice device,
1160 const VkDeviceImageMemoryRequirements *pInfo,
1161 VkMemoryRequirements2 *pMemoryRequirements)
1162 {
1163 VK_FROM_HANDLE(nvk_device, dev, device);
1164 ASSERTED VkResult result;
1165 struct nvk_image image = {0};
1166
1167 result = nvk_image_init(dev, &image, pInfo->pCreateInfo);
1168 assert(result == VK_SUCCESS);
1169
1170 const VkImageAspectFlags aspects =
1171 image.disjoint ? pInfo->planeAspect : image.vk.aspects;
1172
1173 nvk_get_image_memory_requirements(dev, &image, aspects,
1174 pMemoryRequirements);
1175
1176 nvk_image_finish(dev, &image, NULL);
1177 }
1178
1179 static VkSparseImageMemoryRequirements
nvk_fill_sparse_image_memory_reqs(const struct nil_image * nil,const struct nil_image * stencil_tmp,VkImageAspectFlags aspects)1180 nvk_fill_sparse_image_memory_reqs(const struct nil_image *nil,
1181 const struct nil_image *stencil_tmp,
1182 VkImageAspectFlags aspects)
1183 {
1184 VkSparseImageFormatProperties sparse_format_props =
1185 nvk_fill_sparse_image_fmt_props(aspects, nil->format.p_format,
1186 nil->dim, nil->sample_layout);
1187
1188 assert(nil->mip_tail_first_lod <= nil->num_levels);
1189 VkSparseImageMemoryRequirements sparse_memory_reqs = {
1190 .formatProperties = sparse_format_props,
1191 .imageMipTailFirstLod = nil->mip_tail_first_lod,
1192 .imageMipTailStride = 0,
1193 };
1194
1195 if (nil->mip_tail_first_lod == 0) {
1196 sparse_memory_reqs.imageMipTailSize = nil->size_B;
1197 sparse_memory_reqs.imageMipTailOffset = 0;
1198 } else if (nil->mip_tail_first_lod < nil->num_levels) {
1199 sparse_memory_reqs.imageMipTailSize =
1200 nil_image_mip_tail_size_B(nil) * nil->extent_px.array_len;
1201 sparse_memory_reqs.imageMipTailOffset = NVK_MIP_TAIL_START_OFFSET;
1202 } else {
1203 sparse_memory_reqs.imageMipTailSize = 0;
1204 sparse_memory_reqs.imageMipTailOffset = NVK_MIP_TAIL_START_OFFSET;
1205 }
1206
1207 if (stencil_tmp != NULL)
1208 sparse_memory_reqs.imageMipTailSize += stencil_tmp->size_B;
1209
1210 return sparse_memory_reqs;
1211 }
1212
1213 static void
nvk_get_image_sparse_memory_requirements(struct nvk_device * dev,struct nvk_image * image,VkImageAspectFlags aspects,uint32_t * pSparseMemoryRequirementCount,VkSparseImageMemoryRequirements2 * pSparseMemoryRequirements)1214 nvk_get_image_sparse_memory_requirements(
1215 struct nvk_device *dev,
1216 struct nvk_image *image,
1217 VkImageAspectFlags aspects,
1218 uint32_t *pSparseMemoryRequirementCount,
1219 VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
1220 {
1221 VK_OUTARRAY_MAKE_TYPED(VkSparseImageMemoryRequirements2, out,
1222 pSparseMemoryRequirements,
1223 pSparseMemoryRequirementCount);
1224
1225 /* From the Vulkan 1.3.279 spec:
1226 *
1227 * "The sparse image must have been created using the
1228 * VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT flag to retrieve valid sparse
1229 * image memory requirements."
1230 */
1231 if (!(image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT))
1232 return;
1233
1234 /* We don't support multiplane sparse for now */
1235 if (image->plane_count > 1)
1236 return;
1237
1238 const struct nil_image *stencil_tmp = NULL;
1239 if (image->stencil_copy_temp.nil.size_B > 0)
1240 stencil_tmp = &image->stencil_copy_temp.nil;
1241
1242 vk_outarray_append_typed(VkSparseImageMemoryRequirements2, &out, reqs) {
1243 reqs->memoryRequirements =
1244 nvk_fill_sparse_image_memory_reqs(&image->planes[0].nil,
1245 stencil_tmp, aspects);
1246 };
1247 }
1248
1249 VKAPI_ATTR void VKAPI_CALL
nvk_GetImageSparseMemoryRequirements2(VkDevice device,const VkImageSparseMemoryRequirementsInfo2 * pInfo,uint32_t * pSparseMemoryRequirementCount,VkSparseImageMemoryRequirements2 * pSparseMemoryRequirements)1250 nvk_GetImageSparseMemoryRequirements2(
1251 VkDevice device,
1252 const VkImageSparseMemoryRequirementsInfo2* pInfo,
1253 uint32_t* pSparseMemoryRequirementCount,
1254 VkSparseImageMemoryRequirements2* pSparseMemoryRequirements)
1255 {
1256 VK_FROM_HANDLE(nvk_device, dev, device);
1257 VK_FROM_HANDLE(nvk_image, image, pInfo->image);
1258
1259 const VkImageAspectFlags aspects = image->vk.aspects;
1260
1261 nvk_get_image_sparse_memory_requirements(dev, image, aspects,
1262 pSparseMemoryRequirementCount,
1263 pSparseMemoryRequirements);
1264 }
1265
1266 VKAPI_ATTR void VKAPI_CALL
nvk_GetDeviceImageSparseMemoryRequirements(VkDevice device,const VkDeviceImageMemoryRequirements * pInfo,uint32_t * pSparseMemoryRequirementCount,VkSparseImageMemoryRequirements2 * pSparseMemoryRequirements)1267 nvk_GetDeviceImageSparseMemoryRequirements(
1268 VkDevice device,
1269 const VkDeviceImageMemoryRequirements* pInfo,
1270 uint32_t *pSparseMemoryRequirementCount,
1271 VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
1272 {
1273 VK_FROM_HANDLE(nvk_device, dev, device);
1274 ASSERTED VkResult result;
1275 struct nvk_image image = {0};
1276
1277 result = nvk_image_init(dev, &image, pInfo->pCreateInfo);
1278 assert(result == VK_SUCCESS);
1279
1280 const VkImageAspectFlags aspects =
1281 image.disjoint ? pInfo->planeAspect : image.vk.aspects;
1282
1283 nvk_get_image_sparse_memory_requirements(dev, &image, aspects,
1284 pSparseMemoryRequirementCount,
1285 pSparseMemoryRequirements);
1286
1287 nvk_image_finish(dev, &image, NULL);
1288 }
1289
1290 static void
nvk_get_image_subresource_layout(struct nvk_device * dev,struct nvk_image * image,const VkImageSubresource2KHR * pSubresource,VkSubresourceLayout2KHR * pLayout)1291 nvk_get_image_subresource_layout(struct nvk_device *dev,
1292 struct nvk_image *image,
1293 const VkImageSubresource2KHR *pSubresource,
1294 VkSubresourceLayout2KHR *pLayout)
1295 {
1296 const VkImageSubresource *isr = &pSubresource->imageSubresource;
1297
1298 const uint8_t p = nvk_image_memory_aspects_to_plane(image, isr->aspectMask);
1299 const struct nvk_image_plane *plane = &image->planes[p];
1300
1301 uint64_t offset_B = 0;
1302 if (!image->disjoint) {
1303 uint32_t align_B = 0;
1304 for (unsigned plane = 0; plane < p; plane++) {
1305 nvk_image_plane_add_req(dev, image, &image->planes[plane],
1306 &offset_B, &align_B);
1307 }
1308 }
1309 offset_B += nil_image_level_layer_offset_B(&plane->nil, isr->mipLevel,
1310 isr->arrayLayer);
1311
1312 VkSubresourceHostMemcpySizeEXT *host_memcpy_size =
1313 vk_find_struct(pLayout->pNext, SUBRESOURCE_HOST_MEMCPY_SIZE_EXT);
1314 if (host_memcpy_size) {
1315 host_memcpy_size->size =
1316 nil_image_level_layer_size_B(&plane->nil, isr->mipLevel) *
1317 plane->nil.extent_px.array_len;
1318 }
1319
1320 pLayout->subresourceLayout = (VkSubresourceLayout) {
1321 .offset = offset_B,
1322 .size = nil_image_level_size_B(&plane->nil, isr->mipLevel),
1323 .rowPitch = plane->nil.levels[isr->mipLevel].row_stride_B,
1324 .arrayPitch = plane->nil.array_stride_B,
1325 .depthPitch = nil_image_level_depth_stride_B(&plane->nil, isr->mipLevel),
1326 };
1327 }
1328
1329 VKAPI_ATTR void VKAPI_CALL
nvk_GetImageSubresourceLayout2KHR(VkDevice device,VkImage _image,const VkImageSubresource2KHR * pSubresource,VkSubresourceLayout2KHR * pLayout)1330 nvk_GetImageSubresourceLayout2KHR(VkDevice device,
1331 VkImage _image,
1332 const VkImageSubresource2KHR *pSubresource,
1333 VkSubresourceLayout2KHR *pLayout)
1334 {
1335 VK_FROM_HANDLE(nvk_device, dev, device);
1336 VK_FROM_HANDLE(nvk_image, image, _image);
1337
1338 nvk_get_image_subresource_layout(dev, image, pSubresource, pLayout);
1339 }
1340
1341 VKAPI_ATTR void VKAPI_CALL
nvk_GetDeviceImageSubresourceLayoutKHR(VkDevice device,const VkDeviceImageSubresourceInfoKHR * pInfo,VkSubresourceLayout2KHR * pLayout)1342 nvk_GetDeviceImageSubresourceLayoutKHR(
1343 VkDevice device,
1344 const VkDeviceImageSubresourceInfoKHR *pInfo,
1345 VkSubresourceLayout2KHR *pLayout)
1346 {
1347 VK_FROM_HANDLE(nvk_device, dev, device);
1348 ASSERTED VkResult result;
1349 struct nvk_image image = {0};
1350
1351 result = nvk_image_init(dev, &image, pInfo->pCreateInfo);
1352 assert(result == VK_SUCCESS);
1353
1354 nvk_get_image_subresource_layout(dev, &image, pInfo->pSubresource, pLayout);
1355
1356 nvk_image_finish(dev, &image, NULL);
1357 }
1358
1359 static VkResult
nvk_image_plane_bind(struct nvk_device * dev,struct nvk_image * image,struct nvk_image_plane * plane,struct nvk_device_memory * mem,uint64_t * offset_B)1360 nvk_image_plane_bind(struct nvk_device *dev,
1361 struct nvk_image *image,
1362 struct nvk_image_plane *plane,
1363 struct nvk_device_memory *mem,
1364 uint64_t *offset_B)
1365 {
1366 uint64_t plane_size_B, plane_align_B;
1367 nvk_image_plane_size_align_B(dev, image, plane,
1368 &plane_size_B, &plane_align_B);
1369 *offset_B = align64(*offset_B, plane_align_B);
1370
1371 if (plane->va != NULL) {
1372 VkResult result = nvkmd_va_bind_mem(plane->va, &image->vk.base, 0,
1373 mem->mem, *offset_B,
1374 plane->va->size_B);
1375 if (result != VK_SUCCESS)
1376 return result;
1377 } else {
1378 assert(plane->nil.pte_kind == 0);
1379 plane->addr = mem->mem->va->addr + *offset_B;
1380 }
1381
1382 if (image->vk.usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT) {
1383 plane->host_mem = mem;
1384 plane->host_offset = *offset_B;
1385 }
1386
1387 *offset_B += plane_size_B;
1388
1389 return VK_SUCCESS;
1390 }
1391
1392 static VkResult
nvk_bind_image_memory(struct nvk_device * dev,const VkBindImageMemoryInfo * info)1393 nvk_bind_image_memory(struct nvk_device *dev,
1394 const VkBindImageMemoryInfo *info)
1395 {
1396 VK_FROM_HANDLE(nvk_device_memory, mem, info->memory);
1397 VK_FROM_HANDLE(nvk_image, image, info->image);
1398 VkResult result;
1399
1400 #if DETECT_OS_ANDROID
1401 const VkNativeBufferANDROID *anb_info =
1402 vk_find_struct_const(info->pNext, NATIVE_BUFFER_ANDROID);
1403 if (anb_info != NULL && anb_info->handle != NULL) {
1404 /* We do the actual bind the end of CreateImage() */
1405 assert(mem == NULL);
1406 return VK_SUCCESS;
1407 }
1408 #endif
1409
1410 /* Ignore this struct on Android, we cannot access swapchain structures there. */
1411 #ifdef NVK_USE_WSI_PLATFORM
1412 const VkBindImageMemorySwapchainInfoKHR *swapchain_info =
1413 vk_find_struct_const(info->pNext, BIND_IMAGE_MEMORY_SWAPCHAIN_INFO_KHR);
1414
1415 if (swapchain_info && swapchain_info->swapchain != VK_NULL_HANDLE) {
1416 VkImage _wsi_image = wsi_common_get_image(swapchain_info->swapchain,
1417 swapchain_info->imageIndex);
1418 VK_FROM_HANDLE(nvk_image, wsi_img, _wsi_image);
1419
1420 assert(image->plane_count == 1);
1421 assert(wsi_img->plane_count == 1);
1422
1423 struct nvk_image_plane *plane = &image->planes[0];
1424 struct nvk_image_plane *swapchain_plane = &wsi_img->planes[0];
1425
1426 /* Copy memory binding information from swapchain image to the current image's plane. */
1427 plane->addr = swapchain_plane->addr;
1428
1429 return VK_SUCCESS;
1430 }
1431 #endif
1432
1433 uint64_t offset_B = info->memoryOffset;
1434 if (image->disjoint) {
1435 const VkBindImagePlaneMemoryInfo *plane_info =
1436 vk_find_struct_const(info->pNext, BIND_IMAGE_PLANE_MEMORY_INFO);
1437 const uint8_t plane =
1438 nvk_image_memory_aspects_to_plane(image, plane_info->planeAspect);
1439 result = nvk_image_plane_bind(dev, image, &image->planes[plane],
1440 mem, &offset_B);
1441 if (result != VK_SUCCESS)
1442 return result;
1443 } else {
1444 for (unsigned plane = 0; plane < image->plane_count; plane++) {
1445 result = nvk_image_plane_bind(dev, image, &image->planes[plane],
1446 mem, &offset_B);
1447 if (result != VK_SUCCESS)
1448 return result;
1449 }
1450 }
1451
1452 if (image->stencil_copy_temp.nil.size_B > 0) {
1453 result = nvk_image_plane_bind(dev, image, &image->stencil_copy_temp,
1454 mem, &offset_B);
1455 if (result != VK_SUCCESS)
1456 return result;
1457 }
1458
1459 return VK_SUCCESS;
1460 }
1461
1462 VKAPI_ATTR VkResult VKAPI_CALL
nvk_BindImageMemory2(VkDevice device,uint32_t bindInfoCount,const VkBindImageMemoryInfo * pBindInfos)1463 nvk_BindImageMemory2(VkDevice device,
1464 uint32_t bindInfoCount,
1465 const VkBindImageMemoryInfo *pBindInfos)
1466 {
1467 VK_FROM_HANDLE(nvk_device, dev, device);
1468 VkResult first_error_or_success = VK_SUCCESS;
1469
1470 for (uint32_t i = 0; i < bindInfoCount; ++i) {
1471 VkResult result = nvk_bind_image_memory(dev, &pBindInfos[i]);
1472
1473 const VkBindMemoryStatusKHR *status =
1474 vk_find_struct_const(pBindInfos[i].pNext, BIND_MEMORY_STATUS_KHR);
1475 if (status != NULL && status->pResult != NULL)
1476 *status->pResult = VK_SUCCESS;
1477
1478 if (first_error_or_success == VK_SUCCESS)
1479 first_error_or_success = result;
1480 }
1481
1482 return first_error_or_success;
1483 }
1484
1485
1486 static VkResult
queue_image_plane_bind(struct nvk_queue * queue,const struct nvk_image_plane * plane,const VkSparseImageMemoryBind * bind)1487 queue_image_plane_bind(struct nvk_queue *queue,
1488 const struct nvk_image_plane *plane,
1489 const VkSparseImageMemoryBind *bind)
1490 {
1491 VK_FROM_HANDLE(nvk_device_memory, mem, bind->memory);
1492 uint64_t image_bind_offset_B;
1493
1494 const uint64_t mem_bind_offset_B = bind->memoryOffset;
1495 const uint32_t layer = bind->subresource.arrayLayer;
1496 const uint32_t level = bind->subresource.mipLevel;
1497
1498 const struct nil_tiling plane_tiling = plane->nil.levels[level].tiling;
1499 const uint32_t tile_size_B = nil_tiling_size_B(&plane_tiling);
1500
1501 const struct nil_Extent4D_Pixels bind_extent_px = {
1502 .width = bind->extent.width,
1503 .height = bind->extent.height,
1504 .depth = bind->extent.depth,
1505 .array_len = 1,
1506 };
1507 const struct nil_Offset4D_Pixels bind_offset_px = {
1508 .x = bind->offset.x,
1509 .y = bind->offset.y,
1510 .z = bind->offset.z,
1511 .a = layer,
1512 };
1513
1514 const struct nil_Extent4D_Pixels level_extent_px =
1515 nil_image_level_extent_px(&plane->nil, level);
1516 const struct nil_Extent4D_Tiles level_extent_tl =
1517 nil_extent4d_px_to_tl(level_extent_px, &plane_tiling,
1518 plane->nil.format,
1519 plane->nil.sample_layout);
1520
1521 /* Convert the extent and offset to tiles */
1522 const struct nil_Extent4D_Tiles bind_extent_tl =
1523 nil_extent4d_px_to_tl(bind_extent_px, &plane_tiling,
1524 plane->nil.format,
1525 plane->nil.sample_layout);
1526 const struct nil_Offset4D_Tiles bind_offset_tl =
1527 nil_offset4d_px_to_tl(bind_offset_px, &plane_tiling,
1528 plane->nil.format,
1529 plane->nil.sample_layout);
1530
1531 image_bind_offset_B =
1532 nil_image_level_layer_offset_B(&plane->nil, level, layer);
1533
1534 /* We can only bind contiguous ranges, so we'll split the image into rows
1535 * of tiles that are guaranteed to be contiguous, and bind in terms of
1536 * these rows
1537 */
1538
1539 /* First, get the size of the bind. Since we have the extent in terms of
1540 * tiles already, we just need to multiply that by the tile size to get
1541 * the size in bytes
1542 */
1543 uint64_t row_bind_size_B = bind_extent_tl.width * tile_size_B;
1544
1545 const uint32_t nvkmd_bind_count = bind_extent_tl.depth *
1546 bind_extent_tl.height;
1547 STACK_ARRAY(struct nvkmd_ctx_bind, nvkmd_binds, nvkmd_bind_count);
1548 uint32_t nvkmd_bind_idx = 0;
1549
1550 /* Second, start walking the binding region in units of tiles, starting
1551 * from the third dimension
1552 */
1553 for (uint32_t z_tl = 0; z_tl < bind_extent_tl.depth; z_tl++) {
1554 /* Start walking the rows to be bound */
1555 for (uint32_t y_tl = 0; y_tl < bind_extent_tl.height; y_tl++) {
1556 /* For the bind offset, get a memory offset to the start of the row
1557 * in terms of the bind extent
1558 */
1559 const uint64_t mem_row_start_tl =
1560 y_tl * bind_extent_tl.width +
1561 z_tl * bind_extent_tl.width * bind_extent_tl.height;
1562
1563 const uint32_t image_x_tl = bind_offset_tl.x;
1564 const uint32_t image_y_tl = bind_offset_tl.y + y_tl;
1565 const uint32_t image_z_tl = bind_offset_tl.z + z_tl;
1566
1567 /* The image offset is calculated in terms of the level extent */
1568 const uint64_t image_row_start_tl =
1569 image_x_tl +
1570 image_y_tl * level_extent_tl.width +
1571 image_z_tl * level_extent_tl.width * level_extent_tl.height;
1572
1573 nvkmd_binds[nvkmd_bind_idx++] = (struct nvkmd_ctx_bind) {
1574 .op = mem ? NVKMD_BIND_OP_BIND : NVKMD_BIND_OP_UNBIND,
1575 .va = plane->va,
1576 .va_offset_B = image_bind_offset_B +
1577 image_row_start_tl * tile_size_B,
1578 .mem = mem ? mem->mem : NULL,
1579 .mem_offset_B = mem_bind_offset_B +
1580 mem_row_start_tl * tile_size_B,
1581 .range_B = row_bind_size_B,
1582 };
1583 }
1584 }
1585
1586 assert(nvkmd_bind_idx == nvkmd_bind_count);
1587 VkResult result = nvkmd_ctx_bind(queue->bind_ctx, &queue->vk.base,
1588 nvkmd_bind_count, nvkmd_binds);
1589
1590 STACK_ARRAY_FINISH(nvkmd_binds);
1591
1592 return result;
1593 }
1594
1595 VkResult
nvk_queue_image_bind(struct nvk_queue * queue,const VkSparseImageMemoryBindInfo * bind_info)1596 nvk_queue_image_bind(struct nvk_queue *queue,
1597 const VkSparseImageMemoryBindInfo *bind_info)
1598 {
1599 VK_FROM_HANDLE(nvk_image, image, bind_info->image);
1600 VkResult result;
1601
1602 /* Sparse residency with multiplane is currently not supported */
1603 assert(image->plane_count == 1);
1604
1605 for (unsigned i = 0; i < bind_info->bindCount; i++) {
1606 result = queue_image_plane_bind(queue, &image->planes[0],
1607 &bind_info->pBinds[i]);
1608 if (result != VK_SUCCESS)
1609 return result;
1610 }
1611
1612 return VK_SUCCESS;
1613 }
1614
1615 static bool
next_opaque_bind_plane(const VkSparseMemoryBind * bind,uint64_t size_B,uint32_t align_B,uint64_t * plane_offset_B,uint64_t * mem_offset_B,uint64_t * bind_size_B,uint64_t * image_plane_offset_B_iter)1616 next_opaque_bind_plane(const VkSparseMemoryBind *bind,
1617 uint64_t size_B, uint32_t align_B,
1618 uint64_t *plane_offset_B,
1619 uint64_t *mem_offset_B,
1620 uint64_t *bind_size_B,
1621 uint64_t *image_plane_offset_B_iter)
1622 {
1623 /* Figure out the offset to thise plane and increment _iter up-front so
1624 * that we're free to early return elsewhere in the function.
1625 */
1626 *image_plane_offset_B_iter = align64(*image_plane_offset_B_iter, align_B);
1627 const uint64_t image_plane_offset_B = *image_plane_offset_B_iter;
1628 *image_plane_offset_B_iter += size_B;
1629
1630 /* Offset into the image or image mip tail, as appropriate */
1631 uint64_t bind_offset_B = bind->resourceOffset;
1632 if (bind_offset_B >= NVK_MIP_TAIL_START_OFFSET)
1633 bind_offset_B -= NVK_MIP_TAIL_START_OFFSET;
1634
1635 if (bind_offset_B < image_plane_offset_B) {
1636 /* The offset of the plane within the bind */
1637 const uint64_t bind_plane_offset_B =
1638 image_plane_offset_B - bind_offset_B;
1639
1640 /* If this plane lies above the bound range, skip this plane */
1641 if (bind_plane_offset_B >= bind->size)
1642 return false;
1643
1644 *plane_offset_B = 0;
1645 *mem_offset_B = bind->memoryOffset + bind_plane_offset_B;
1646 *bind_size_B = MIN2(bind->size - bind_plane_offset_B, size_B);
1647 } else {
1648 /* The offset of the bind within the plane */
1649 const uint64_t plane_bind_offset_B =
1650 bind_offset_B - image_plane_offset_B;
1651
1652 /* If this plane lies below the bound range, skip this plane */
1653 if (plane_bind_offset_B >= size_B)
1654 return false;
1655
1656 *plane_offset_B = plane_bind_offset_B;
1657 *mem_offset_B = bind->memoryOffset;
1658 *bind_size_B = MIN2(bind->size, size_B - plane_bind_offset_B);
1659 }
1660
1661 return true;
1662 }
1663
1664 static VkResult
queue_image_plane_opaque_bind(struct nvk_queue * queue,struct nvk_image * image,struct nvk_image_plane * plane,const VkSparseMemoryBind * bind,uint64_t * image_plane_offset_B)1665 queue_image_plane_opaque_bind(struct nvk_queue *queue,
1666 struct nvk_image *image,
1667 struct nvk_image_plane *plane,
1668 const VkSparseMemoryBind *bind,
1669 uint64_t *image_plane_offset_B)
1670 {
1671 uint64_t plane_size_B, plane_align_B;
1672 nvk_image_plane_size_align_B(nvk_queue_device(queue), image, plane,
1673 &plane_size_B, &plane_align_B);
1674
1675 uint64_t plane_offset_B, mem_offset_B, bind_size_B;
1676 if (!next_opaque_bind_plane(bind, plane_size_B, plane_align_B,
1677 &plane_offset_B, &mem_offset_B, &bind_size_B,
1678 image_plane_offset_B))
1679 return VK_SUCCESS;
1680
1681 VK_FROM_HANDLE(nvk_device_memory, mem, bind->memory);
1682
1683 assert(plane_offset_B + bind_size_B <= plane->va->size_B);
1684 assert(!mem || mem_offset_B + bind_size_B <= mem->vk.size);
1685
1686 const struct nvkmd_ctx_bind nvkmd_bind = {
1687 .op = mem ? NVKMD_BIND_OP_BIND : NVKMD_BIND_OP_UNBIND,
1688 .va = plane->va,
1689 .va_offset_B = plane_offset_B,
1690 .mem = mem ? mem->mem : NULL,
1691 .mem_offset_B = mem_offset_B,
1692 .range_B = bind_size_B,
1693 };
1694 return nvkmd_ctx_bind(queue->bind_ctx, &queue->vk.base, 1, &nvkmd_bind);
1695 }
1696
1697 static VkResult
queue_image_plane_bind_mip_tail(struct nvk_queue * queue,struct nvk_image * image,struct nvk_image_plane * plane,const VkSparseMemoryBind * bind,uint64_t * image_plane_offset_B)1698 queue_image_plane_bind_mip_tail(struct nvk_queue *queue,
1699 struct nvk_image *image,
1700 struct nvk_image_plane *plane,
1701 const VkSparseMemoryBind *bind,
1702 uint64_t *image_plane_offset_B)
1703 {
1704 uint64_t plane_size_B, plane_align_B;
1705 nvk_image_plane_size_align_B(nvk_queue_device(queue), image, plane,
1706 &plane_size_B, &plane_align_B);
1707
1708 const uint64_t mip_tail_offset_B =
1709 nil_image_mip_tail_offset_B(&plane->nil);
1710 const uint64_t mip_tail_size_B =
1711 nil_image_mip_tail_size_B(&plane->nil);
1712 const uint64_t mip_tail_stride_B = plane->nil.array_stride_B;
1713
1714 const uint64_t whole_mip_tail_size_B =
1715 mip_tail_size_B * plane->nil.extent_px.array_len;
1716
1717 uint64_t plane_offset_B, mem_offset_B, bind_size_B;
1718 if (!next_opaque_bind_plane(bind, whole_mip_tail_size_B, plane_align_B,
1719 &plane_offset_B, &mem_offset_B, &bind_size_B,
1720 image_plane_offset_B))
1721 return VK_SUCCESS;
1722
1723 VK_FROM_HANDLE(nvk_device_memory, mem, bind->memory);
1724
1725 /* Range within the virtual mip_tail space */
1726 const uint64_t mip_bind_start_B = plane_offset_B;
1727 const uint64_t mip_bind_end_B = mip_bind_start_B + bind_size_B;
1728
1729 /* Range of array slices covered by this bind */
1730 const uint32_t start_a = mip_bind_start_B / mip_tail_size_B;
1731 const uint32_t end_a = DIV_ROUND_UP(mip_bind_end_B, mip_tail_size_B);
1732
1733 const uint32_t nvkmd_bind_count = end_a - start_a;
1734 STACK_ARRAY(struct nvkmd_ctx_bind, nvkmd_binds, nvkmd_bind_count);
1735 uint32_t nvkmd_bind_idx = 0;
1736
1737 for (uint32_t a = start_a; a < end_a; a++) {
1738 /* Range within the virtual mip_tail space of this array slice */
1739 const uint64_t a_mip_bind_start_B =
1740 MAX2(a * mip_tail_size_B, mip_bind_start_B);
1741 const uint64_t a_mip_bind_end_B =
1742 MIN2((a + 1) * mip_tail_size_B, mip_bind_end_B);
1743
1744 /* Offset and range within this mip_tail slice */
1745 const uint64_t a_offset_B = a_mip_bind_start_B - a * mip_tail_size_B;
1746 const uint64_t a_range_B = a_mip_bind_end_B - a_mip_bind_start_B;
1747
1748 /* Offset within the current bind operation */
1749 const uint64_t a_bind_offset_B =
1750 a_mip_bind_start_B - mip_bind_start_B;
1751
1752 /* Offset within the image */
1753 const uint64_t a_image_offset_B =
1754 mip_tail_offset_B + (a * mip_tail_stride_B) + a_offset_B;
1755
1756 nvkmd_binds[nvkmd_bind_idx++] = (struct nvkmd_ctx_bind) {
1757 .op = mem ? NVKMD_BIND_OP_BIND : NVKMD_BIND_OP_UNBIND,
1758 .va = plane->va,
1759 .va_offset_B = a_image_offset_B,
1760 .mem = mem ? mem->mem : NULL,
1761 .mem_offset_B = mem_offset_B + a_bind_offset_B,
1762 .range_B = a_range_B,
1763 };
1764 }
1765
1766 assert(nvkmd_bind_idx == nvkmd_bind_count);
1767 VkResult result = nvkmd_ctx_bind(queue->bind_ctx, &queue->vk.base,
1768 nvkmd_bind_count, nvkmd_binds);
1769
1770 STACK_ARRAY_FINISH(nvkmd_binds);
1771
1772 return result;
1773 }
1774
1775 VkResult
nvk_queue_image_opaque_bind(struct nvk_queue * queue,const VkSparseImageOpaqueMemoryBindInfo * bind_info)1776 nvk_queue_image_opaque_bind(struct nvk_queue *queue,
1777 const VkSparseImageOpaqueMemoryBindInfo *bind_info)
1778 {
1779 VK_FROM_HANDLE(nvk_image, image, bind_info->image);
1780 VkResult result;
1781
1782 for (unsigned i = 0; i < bind_info->bindCount; i++) {
1783 const VkSparseMemoryBind *bind = &bind_info->pBinds[i];
1784
1785 uint64_t image_plane_offset_B = 0;
1786 for (unsigned plane = 0; plane < image->plane_count; plane++) {
1787 if (bind->resourceOffset >= NVK_MIP_TAIL_START_OFFSET) {
1788 result = queue_image_plane_bind_mip_tail(queue, image,
1789 &image->planes[plane],
1790 bind,
1791 &image_plane_offset_B);
1792 } else {
1793 result = queue_image_plane_opaque_bind(queue, image,
1794 &image->planes[plane],
1795 bind,
1796 &image_plane_offset_B);
1797 }
1798 if (result != VK_SUCCESS)
1799 return result;
1800 }
1801 if (image->stencil_copy_temp.nil.size_B > 0) {
1802 result = queue_image_plane_opaque_bind(queue, image,
1803 &image->stencil_copy_temp,
1804 bind,
1805 &image_plane_offset_B);
1806 if (result != VK_SUCCESS)
1807 return result;
1808 }
1809 }
1810
1811 return VK_SUCCESS;
1812 }
1813
1814 VKAPI_ATTR VkResult VKAPI_CALL
nvk_GetImageOpaqueCaptureDescriptorDataEXT(VkDevice _device,const VkImageCaptureDescriptorDataInfoEXT * pInfo,void * pData)1815 nvk_GetImageOpaqueCaptureDescriptorDataEXT(
1816 VkDevice _device,
1817 const VkImageCaptureDescriptorDataInfoEXT *pInfo,
1818 void *pData)
1819 {
1820 return VK_SUCCESS;
1821 }
1822