1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 * SPDX-License-Identifier: MIT
5 *
6 * based in part on anv driver which is:
7 * Copyright © 2015 Intel Corporation
8 */
9
10 #include "tu_image.h"
11
12 #include "fdl/fd6_format_table.h"
13
14 #include "util/debug.h"
15 #include "util/format/u_format.h"
16 #include "vk_util.h"
17 #include "drm-uapi/drm_fourcc.h"
18
19 #include "tu_android.h"
20 #include "tu_cs.h"
21 #include "tu_descriptor_set.h"
22 #include "tu_device.h"
23 #include "tu_formats.h"
24
25 uint32_t
tu6_plane_count(VkFormat format)26 tu6_plane_count(VkFormat format)
27 {
28 switch (format) {
29 default:
30 return 1;
31 case VK_FORMAT_G8_B8R8_2PLANE_420_UNORM:
32 case VK_FORMAT_D32_SFLOAT_S8_UINT:
33 return 2;
34 case VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM:
35 return 3;
36 }
37 }
38
39 enum pipe_format
tu6_plane_format(VkFormat format,uint32_t plane)40 tu6_plane_format(VkFormat format, uint32_t plane)
41 {
42 switch (format) {
43 case VK_FORMAT_G8_B8R8_2PLANE_420_UNORM:
44 return plane ? PIPE_FORMAT_R8G8_UNORM : PIPE_FORMAT_Y8_UNORM;
45 case VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM:
46 return PIPE_FORMAT_R8_UNORM;
47 case VK_FORMAT_D32_SFLOAT_S8_UINT:
48 return plane ? PIPE_FORMAT_S8_UINT : PIPE_FORMAT_Z32_FLOAT;
49 default:
50 return tu_vk_format_to_pipe_format(format);
51 }
52 }
53
54 uint32_t
tu6_plane_index(VkFormat format,VkImageAspectFlags aspect_mask)55 tu6_plane_index(VkFormat format, VkImageAspectFlags aspect_mask)
56 {
57 switch (aspect_mask) {
58 default:
59 assert(aspect_mask != VK_IMAGE_ASPECT_MEMORY_PLANE_3_BIT_EXT);
60 return 0;
61 case VK_IMAGE_ASPECT_PLANE_1_BIT:
62 case VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT:
63 return 1;
64 case VK_IMAGE_ASPECT_PLANE_2_BIT:
65 case VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT:
66 return 2;
67 case VK_IMAGE_ASPECT_STENCIL_BIT:
68 return format == VK_FORMAT_D32_SFLOAT_S8_UINT;
69 }
70 }
71
72 enum pipe_format
tu_format_for_aspect(enum pipe_format format,VkImageAspectFlags aspect_mask)73 tu_format_for_aspect(enum pipe_format format, VkImageAspectFlags aspect_mask)
74 {
75 switch (format) {
76 case PIPE_FORMAT_Z24_UNORM_S8_UINT:
77 if (aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT)
78 return PIPE_FORMAT_Z24_UNORM_S8_UINT_AS_R8G8B8A8;
79 if (aspect_mask & VK_IMAGE_ASPECT_STENCIL_BIT) {
80 if (aspect_mask & VK_IMAGE_ASPECT_DEPTH_BIT)
81 return PIPE_FORMAT_Z24_UNORM_S8_UINT;
82 else
83 return PIPE_FORMAT_X24S8_UINT;
84 } else {
85 return PIPE_FORMAT_Z24X8_UNORM;
86 }
87 case PIPE_FORMAT_Z24X8_UNORM:
88 if (aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT)
89 return PIPE_FORMAT_Z24_UNORM_S8_UINT_AS_R8G8B8A8;
90 return PIPE_FORMAT_Z24X8_UNORM;
91 default:
92 return format;
93 }
94 }
95
96 static bool
tu_is_r8g8(enum pipe_format format)97 tu_is_r8g8(enum pipe_format format)
98 {
99 return (util_format_get_blocksize(format) == 2) &&
100 (util_format_get_nr_components(format) == 2);
101 }
102
103 static bool
tu_is_r8g8_compatible(enum pipe_format format)104 tu_is_r8g8_compatible(enum pipe_format format)
105 {
106 return (util_format_get_blocksize(format) == 2) &&
107 !util_format_is_depth_or_stencil(format);
108 }
109
110 void
tu_cs_image_ref(struct tu_cs * cs,const struct fdl6_view * iview,uint32_t layer)111 tu_cs_image_ref(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer)
112 {
113 tu_cs_emit(cs, iview->PITCH);
114 tu_cs_emit(cs, iview->layer_size >> 6);
115 tu_cs_emit_qw(cs, iview->base_addr + iview->layer_size * layer);
116 }
117
118 void
tu_cs_image_stencil_ref(struct tu_cs * cs,const struct tu_image_view * iview,uint32_t layer)119 tu_cs_image_stencil_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer)
120 {
121 tu_cs_emit(cs, iview->stencil_PITCH);
122 tu_cs_emit(cs, iview->stencil_layer_size >> 6);
123 tu_cs_emit_qw(cs, iview->stencil_base_addr + iview->stencil_layer_size * layer);
124 }
125
126 void
tu_cs_image_depth_ref(struct tu_cs * cs,const struct tu_image_view * iview,uint32_t layer)127 tu_cs_image_depth_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer)
128 {
129 tu_cs_emit(cs, iview->depth_PITCH);
130 tu_cs_emit(cs, iview->depth_layer_size >> 6);
131 tu_cs_emit_qw(cs, iview->depth_base_addr + iview->depth_layer_size * layer);
132 }
133
134 void
tu_cs_image_ref_2d(struct tu_cs * cs,const struct fdl6_view * iview,uint32_t layer,bool src)135 tu_cs_image_ref_2d(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer, bool src)
136 {
137 tu_cs_emit_qw(cs, iview->base_addr + iview->layer_size * layer);
138 /* SP_PS_2D_SRC_PITCH has shifted pitch field */
139 tu_cs_emit(cs, iview->PITCH << (src ? 9 : 0));
140 }
141
142 void
tu_cs_image_flag_ref(struct tu_cs * cs,const struct fdl6_view * iview,uint32_t layer)143 tu_cs_image_flag_ref(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer)
144 {
145 tu_cs_emit_qw(cs, iview->ubwc_addr + iview->ubwc_layer_size * layer);
146 tu_cs_emit(cs, iview->FLAG_BUFFER_PITCH);
147 }
148
149 static void
tu_image_view_init(struct tu_device * device,struct tu_image_view * iview,const VkImageViewCreateInfo * pCreateInfo,bool has_z24uint_s8uint)150 tu_image_view_init(struct tu_device *device,
151 struct tu_image_view *iview,
152 const VkImageViewCreateInfo *pCreateInfo,
153 bool has_z24uint_s8uint)
154 {
155 TU_FROM_HANDLE(tu_image, image, pCreateInfo->image);
156 const VkImageSubresourceRange *range = &pCreateInfo->subresourceRange;
157 VkFormat vk_format = pCreateInfo->format;
158 VkImageAspectFlagBits aspect_mask = pCreateInfo->subresourceRange.aspectMask;
159
160 const struct VkSamplerYcbcrConversionInfo *ycbcr_conversion =
161 vk_find_struct_const(pCreateInfo->pNext, SAMPLER_YCBCR_CONVERSION_INFO);
162 const struct tu_sampler_ycbcr_conversion *conversion = ycbcr_conversion ?
163 tu_sampler_ycbcr_conversion_from_handle(ycbcr_conversion->conversion) : NULL;
164
165 vk_image_view_init(&device->vk, &iview->vk, false, pCreateInfo);
166
167 iview->image = image;
168
169 const struct fdl_layout *layouts[3];
170
171 layouts[0] = &image->layout[tu6_plane_index(image->vk.format, aspect_mask)];
172
173 enum pipe_format format;
174 if (aspect_mask != VK_IMAGE_ASPECT_COLOR_BIT)
175 format = tu6_plane_format(vk_format, tu6_plane_index(vk_format, aspect_mask));
176 else
177 format = tu_vk_format_to_pipe_format(vk_format);
178
179 if (image->vk.format == VK_FORMAT_G8_B8R8_2PLANE_420_UNORM &&
180 aspect_mask == VK_IMAGE_ASPECT_PLANE_0_BIT) {
181 if (vk_format == VK_FORMAT_R8_UNORM) {
182 /* The 0'th plane of this format has a different UBWC compression. */
183 format = PIPE_FORMAT_Y8_UNORM;
184 } else {
185 /* If the user wants to reinterpret this plane, then they should've
186 * set MUTABLE_FORMAT_BIT which should disable UBWC and tiling.
187 */
188 assert(!layouts[0]->ubwc);
189 }
190 }
191
192 if (aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT &&
193 (vk_format == VK_FORMAT_G8_B8R8_2PLANE_420_UNORM ||
194 vk_format == VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM)) {
195 layouts[1] = &image->layout[1];
196 layouts[2] = &image->layout[2];
197 }
198
199 struct fdl_view_args args = {};
200 args.iova = image->iova;
201 args.base_array_layer = range->baseArrayLayer;
202 args.base_miplevel = range->baseMipLevel;
203 args.layer_count = vk_image_subresource_layer_count(&image->vk, range);
204 args.level_count = vk_image_subresource_level_count(&image->vk, range);
205 args.min_lod_clamp = iview->vk.min_lod;
206 args.format = tu_format_for_aspect(format, aspect_mask);
207 vk_component_mapping_to_pipe_swizzle(pCreateInfo->components, args.swiz);
208 if (conversion) {
209 unsigned char conversion_swiz[4], create_swiz[4];
210 memcpy(create_swiz, args.swiz, sizeof(create_swiz));
211 vk_component_mapping_to_pipe_swizzle(conversion->components,
212 conversion_swiz);
213 util_format_compose_swizzles(create_swiz, conversion_swiz, args.swiz);
214 }
215
216 switch (pCreateInfo->viewType) {
217 case VK_IMAGE_VIEW_TYPE_1D:
218 case VK_IMAGE_VIEW_TYPE_1D_ARRAY:
219 args.type = FDL_VIEW_TYPE_1D;
220 break;
221 case VK_IMAGE_VIEW_TYPE_2D:
222 case VK_IMAGE_VIEW_TYPE_2D_ARRAY:
223 args.type = FDL_VIEW_TYPE_2D;
224 break;
225 case VK_IMAGE_VIEW_TYPE_CUBE:
226 case VK_IMAGE_VIEW_TYPE_CUBE_ARRAY:
227 args.type = FDL_VIEW_TYPE_CUBE;
228 break;
229 case VK_IMAGE_VIEW_TYPE_3D:
230 args.type = FDL_VIEW_TYPE_3D;
231 break;
232 default:
233 unreachable("unknown view type");
234 }
235
236 STATIC_ASSERT((unsigned)VK_CHROMA_LOCATION_COSITED_EVEN == (unsigned)FDL_CHROMA_LOCATION_COSITED_EVEN);
237 STATIC_ASSERT((unsigned)VK_CHROMA_LOCATION_MIDPOINT == (unsigned)FDL_CHROMA_LOCATION_MIDPOINT);
238 if (conversion) {
239 args.chroma_offsets[0] = (enum fdl_chroma_location) conversion->chroma_offsets[0];
240 args.chroma_offsets[1] = (enum fdl_chroma_location) conversion->chroma_offsets[1];
241 }
242
243 fdl6_view_init(&iview->view, layouts, &args, has_z24uint_s8uint);
244
245 if (image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
246 struct fdl_layout *layout = &image->layout[0];
247 iview->depth_base_addr = image->iova +
248 fdl_surface_offset(layout, range->baseMipLevel, range->baseArrayLayer);
249 iview->depth_layer_size = fdl_layer_stride(layout, range->baseMipLevel);
250 iview->depth_PITCH = A6XX_RB_DEPTH_BUFFER_PITCH(fdl_pitch(layout, range->baseMipLevel)).value;
251
252 layout = &image->layout[1];
253 iview->stencil_base_addr = image->iova +
254 fdl_surface_offset(layout, range->baseMipLevel, range->baseArrayLayer);
255 iview->stencil_layer_size = fdl_layer_stride(layout, range->baseMipLevel);
256 iview->stencil_PITCH = A6XX_RB_STENCIL_BUFFER_PITCH(fdl_pitch(layout, range->baseMipLevel)).value;
257 }
258 }
259
260 bool
tiling_possible(VkFormat format)261 tiling_possible(VkFormat format)
262 {
263 if (format == VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM ||
264 format == VK_FORMAT_G8B8G8R8_422_UNORM ||
265 format == VK_FORMAT_B8G8R8G8_422_UNORM)
266 return false;
267
268 return true;
269 }
270
271 bool
ubwc_possible(VkFormat format,VkImageType type,VkImageUsageFlags usage,VkImageUsageFlags stencil_usage,const struct fd_dev_info * info,VkSampleCountFlagBits samples,bool use_z24uint_s8uint)272 ubwc_possible(VkFormat format, VkImageType type, VkImageUsageFlags usage,
273 VkImageUsageFlags stencil_usage, const struct fd_dev_info *info,
274 VkSampleCountFlagBits samples, bool use_z24uint_s8uint)
275 {
276 /* no UBWC with compressed formats, E5B9G9R9, S8_UINT
277 * (S8_UINT because separate stencil doesn't have UBWC-enable bit)
278 */
279 if (vk_format_is_compressed(format) ||
280 format == VK_FORMAT_E5B9G9R9_UFLOAT_PACK32 ||
281 format == VK_FORMAT_S8_UINT)
282 return false;
283
284 /* In copy_format, we treat snorm as unorm to avoid clamping. But snorm
285 * and unorm are UBWC incompatible for special values such as all 0's or
286 * all 1's. Disable UBWC for snorm.
287 */
288 if (vk_format_is_snorm(format))
289 return false;
290
291 if (!info->a6xx.has_8bpp_ubwc &&
292 (format == VK_FORMAT_R8_UNORM ||
293 format == VK_FORMAT_R8_SNORM ||
294 format == VK_FORMAT_R8_UINT ||
295 format == VK_FORMAT_R8_SINT ||
296 format == VK_FORMAT_R8_SRGB))
297 return false;
298
299 if (type == VK_IMAGE_TYPE_3D) {
300 tu_finishme("UBWC with 3D textures");
301 return false;
302 }
303
304 /* Disable UBWC for storage images.
305 *
306 * The closed GL driver skips UBWC for storage images (and additionally
307 * uses linear for writeonly images). We seem to have image tiling working
308 * in freedreno in general, so turnip matches that. freedreno also enables
309 * UBWC on images, but it's not really tested due to the lack of
310 * UBWC-enabled mipmaps in freedreno currently. Just match the closed GL
311 * behavior of no UBWC.
312 */
313 if ((usage | stencil_usage) & VK_IMAGE_USAGE_STORAGE_BIT)
314 return false;
315
316 /* Disable UBWC for D24S8 on A630 in some cases
317 *
318 * VK_IMAGE_ASPECT_STENCIL_BIT image view requires to be able to sample
319 * from the stencil component as UINT, however no format allows this
320 * on a630 (the special FMT6_Z24_UINT_S8_UINT format is missing)
321 *
322 * It must be sampled as FMT6_8_8_8_8_UINT, which is not UBWC-compatible
323 *
324 * If we wish to get the border colors correct without knowing the format
325 * when creating the sampler, we also have to use the A630 workaround.
326 *
327 * Additionally, the special AS_R8G8B8A8 format is broken without UBWC,
328 * so we have to fallback to 8_8_8_8_UNORM when UBWC is disabled
329 */
330 if (!use_z24uint_s8uint &&
331 format == VK_FORMAT_D24_UNORM_S8_UINT &&
332 (stencil_usage & (VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT)))
333 return false;
334
335 if (!info->a6xx.has_z24uint_s8uint && samples > VK_SAMPLE_COUNT_1_BIT)
336 return false;
337
338 return true;
339 }
340
341 static VkResult
tu_image_init(struct tu_device * device,struct tu_image * image,const VkImageCreateInfo * pCreateInfo,uint64_t modifier,const VkSubresourceLayout * plane_layouts)342 tu_image_init(struct tu_device *device, struct tu_image *image,
343 const VkImageCreateInfo *pCreateInfo, uint64_t modifier,
344 const VkSubresourceLayout *plane_layouts)
345 {
346 vk_image_init(&device->vk, &image->vk, pCreateInfo);
347 image->vk.drm_format_mod = modifier;
348
349 enum a6xx_tile_mode tile_mode = TILE6_3;
350 bool ubwc_enabled = true;
351
352 /* use linear tiling if requested */
353 if (pCreateInfo->tiling == VK_IMAGE_TILING_LINEAR || modifier == DRM_FORMAT_MOD_LINEAR) {
354 tile_mode = TILE6_LINEAR;
355 ubwc_enabled = false;
356 }
357
358 /* Force linear tiling for formats with "fake" optimalTilingFeatures */
359 if (!tiling_possible(image->vk.format)) {
360 tile_mode = TILE6_LINEAR;
361 ubwc_enabled = false;
362 }
363
364 /* No sense in tiling a 1D image, you'd just waste space and cache locality. */
365 if (pCreateInfo->imageType == VK_IMAGE_TYPE_1D) {
366 tile_mode = TILE6_LINEAR;
367 ubwc_enabled = false;
368 }
369
370 enum pipe_format format =
371 tu_vk_format_to_pipe_format(image->vk.format);
372 /* Whether a view of the image with an R8G8 format could be made. */
373 bool has_r8g8 = tu_is_r8g8(format);
374
375 /* Mutable images can be reinterpreted as any other compatible format.
376 * This is a problem with UBWC (compression for different formats is different),
377 * but also tiling ("swap" affects how tiled formats are stored in memory)
378 * Depth and stencil formats cannot be reintepreted as another format, and
379 * cannot be linear with sysmem rendering, so don't fall back for those.
380 *
381 * TODO:
382 * - if the fmt_list contains only formats which are swapped, but compatible
383 * with each other (B8G8R8A8_UNORM and B8G8R8A8_UINT for example), then
384 * tiling is still possible
385 * - figure out which UBWC compressions are compatible to keep it enabled
386 */
387 if ((pCreateInfo->flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) &&
388 !vk_format_is_depth_or_stencil(image->vk.format)) {
389 const VkImageFormatListCreateInfo *fmt_list =
390 vk_find_struct_const(pCreateInfo->pNext, IMAGE_FORMAT_LIST_CREATE_INFO);
391 bool may_be_swapped = true;
392 /* Whether a view of the image with a non-R8G8 but R8G8 compatible format
393 * could be made.
394 */
395 bool has_r8g8_compatible = false;
396 if (fmt_list) {
397 may_be_swapped = false;
398 has_r8g8_compatible = !has_r8g8 && tu_is_r8g8_compatible(format);
399 for (uint32_t i = 0; i < fmt_list->viewFormatCount; i++) {
400 enum pipe_format format =
401 tu_vk_format_to_pipe_format(fmt_list->pViewFormats[i]);
402 bool is_r8g8 = tu_is_r8g8(format);
403 has_r8g8 = has_r8g8 || is_r8g8;
404 has_r8g8_compatible = has_r8g8_compatible ||
405 (!is_r8g8 && tu_is_r8g8_compatible(format));
406
407 if (tu6_format_texture(format, TILE6_LINEAR).swap) {
408 may_be_swapped = true;
409 break;
410 }
411 }
412 } else {
413 /* If there is no format list it could be reinterpreted as
414 * any compatible format.
415 */
416 has_r8g8 = tu_is_r8g8_compatible(format);
417 has_r8g8_compatible = has_r8g8;
418 }
419
420 if (may_be_swapped)
421 tile_mode = TILE6_LINEAR;
422
423 /* R8G8 have a different block width/height and height alignment from other
424 * formats that would normally be compatible (like R16), and so if we are
425 * trying to, for example, sample R16 as R8G8 we need to demote to linear.
426 */
427 if (has_r8g8 && has_r8g8_compatible)
428 tile_mode = TILE6_LINEAR;
429
430 ubwc_enabled = false;
431 }
432
433 if (!ubwc_possible(image->vk.format, pCreateInfo->imageType,
434 pCreateInfo->usage, image->vk.stencil_usage,
435 device->physical_device->info, pCreateInfo->samples,
436 device->use_z24uint_s8uint))
437 ubwc_enabled = false;
438
439 /* expect UBWC enabled if we asked for it */
440 if (modifier == DRM_FORMAT_MOD_QCOM_COMPRESSED)
441 assert(ubwc_enabled);
442 else if (device->physical_device->instance->debug_flags & TU_DEBUG_NOUBWC)
443 ubwc_enabled = false;
444
445 /* Non-UBWC tiled R8G8 is probably buggy since media formats are always
446 * either linear or UBWC. There is no simple test to reproduce the bug.
447 * However it was observed in the wild leading to an unrecoverable hang
448 * on a650/a660.
449 */
450 if (has_r8g8 && tile_mode == TILE6_3 && !ubwc_enabled) {
451 tile_mode = TILE6_LINEAR;
452 }
453
454 for (uint32_t i = 0; i < tu6_plane_count(image->vk.format); i++) {
455 struct fdl_layout *layout = &image->layout[i];
456 enum pipe_format format = tu6_plane_format(image->vk.format, i);
457 uint32_t width0 = pCreateInfo->extent.width;
458 uint32_t height0 = pCreateInfo->extent.height;
459
460 if (i > 0) {
461 switch (image->vk.format) {
462 case VK_FORMAT_G8_B8R8_2PLANE_420_UNORM:
463 case VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM:
464 /* half width/height on chroma planes */
465 width0 = (width0 + 1) >> 1;
466 height0 = (height0 + 1) >> 1;
467 break;
468 case VK_FORMAT_D32_SFLOAT_S8_UINT:
469 /* no UBWC for separate stencil */
470 ubwc_enabled = false;
471 break;
472 default:
473 break;
474 }
475 }
476
477 struct fdl_explicit_layout plane_layout;
478
479 if (plane_layouts) {
480 /* only expect simple 2D images for now */
481 if (pCreateInfo->mipLevels != 1 ||
482 pCreateInfo->arrayLayers != 1 ||
483 pCreateInfo->extent.depth != 1)
484 return vk_error(device, VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT);
485
486 plane_layout.offset = plane_layouts[i].offset;
487 plane_layout.pitch = plane_layouts[i].rowPitch;
488 /* note: use plane_layouts[0].arrayPitch to support array formats */
489 }
490
491 layout->tile_mode = tile_mode;
492 layout->ubwc = ubwc_enabled;
493
494 if (!fdl6_layout(layout, format,
495 pCreateInfo->samples,
496 width0, height0,
497 pCreateInfo->extent.depth,
498 pCreateInfo->mipLevels,
499 pCreateInfo->arrayLayers,
500 pCreateInfo->imageType == VK_IMAGE_TYPE_3D,
501 plane_layouts ? &plane_layout : NULL)) {
502 assert(plane_layouts); /* can only fail with explicit layout */
503 return vk_error(device, VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT);
504 }
505
506 if (device->instance->debug_flags & TU_DEBUG_LAYOUT)
507 fdl_dump_layout(layout);
508
509 /* fdl6_layout can't take explicit offset without explicit pitch
510 * add offset manually for extra layouts for planes
511 */
512 if (!plane_layouts && i > 0) {
513 uint32_t offset = ALIGN_POT(image->total_size, 4096);
514 for (int i = 0; i < pCreateInfo->mipLevels; i++) {
515 layout->slices[i].offset += offset;
516 layout->ubwc_slices[i].offset += offset;
517 }
518 layout->size += offset;
519 }
520
521 image->total_size = MAX2(image->total_size, layout->size);
522 }
523
524 const struct util_format_description *desc = util_format_description(image->layout[0].format);
525 if (util_format_has_depth(desc) && !(device->instance->debug_flags & TU_DEBUG_NOLRZ))
526 {
527 /* Depth plane is the first one */
528 struct fdl_layout *layout = &image->layout[0];
529 unsigned width = layout->width0;
530 unsigned height = layout->height0;
531
532 /* LRZ buffer is super-sampled */
533 switch (layout->nr_samples) {
534 case 4:
535 width *= 2;
536 FALLTHROUGH;
537 case 2:
538 height *= 2;
539 break;
540 default:
541 break;
542 }
543
544 unsigned lrz_pitch = align(DIV_ROUND_UP(width, 8), 32);
545 unsigned lrz_height = align(DIV_ROUND_UP(height, 8), 16);
546
547 image->lrz_height = lrz_height;
548 image->lrz_pitch = lrz_pitch;
549 image->lrz_offset = image->total_size;
550 unsigned lrz_size = lrz_pitch * lrz_height * 2;
551 image->total_size += lrz_size;
552
553 unsigned nblocksx = DIV_ROUND_UP(DIV_ROUND_UP(width, 8), 16);
554 unsigned nblocksy = DIV_ROUND_UP(DIV_ROUND_UP(height, 8), 4);
555
556 /* Fast-clear buffer is 1bit/block */
557 image->lrz_fc_size = DIV_ROUND_UP(nblocksx * nblocksy, 8);
558
559 /* Fast-clear buffer cannot be larger than 512 bytes (HW limitation) */
560 bool has_lrz_fc = image->lrz_fc_size <= 512 &&
561 device->physical_device->info->a6xx.enable_lrz_fast_clear &&
562 !unlikely(device->physical_device->instance->debug_flags & TU_DEBUG_NOLRZFC);
563
564 if (has_lrz_fc || device->physical_device->info->a6xx.has_lrz_dir_tracking) {
565 image->lrz_fc_offset = image->total_size;
566 image->total_size += 512;
567
568 if (device->physical_device->info->a6xx.has_lrz_dir_tracking) {
569 /* Direction tracking uses 1 byte */
570 image->total_size += 1;
571 /* GRAS_LRZ_DEPTH_VIEW needs 5 bytes: 4 for view data and 1 for padding */
572 image->total_size += 5;
573 }
574 }
575
576 if (!has_lrz_fc) {
577 image->lrz_fc_size = 0;
578 }
579 } else {
580 image->lrz_height = 0;
581 }
582
583 return VK_SUCCESS;
584 }
585
586 VKAPI_ATTR VkResult VKAPI_CALL
tu_CreateImage(VkDevice _device,const VkImageCreateInfo * pCreateInfo,const VkAllocationCallbacks * alloc,VkImage * pImage)587 tu_CreateImage(VkDevice _device,
588 const VkImageCreateInfo *pCreateInfo,
589 const VkAllocationCallbacks *alloc,
590 VkImage *pImage)
591 {
592 uint64_t modifier = DRM_FORMAT_MOD_INVALID;
593 const VkSubresourceLayout *plane_layouts = NULL;
594
595 TU_FROM_HANDLE(tu_device, device, _device);
596 struct tu_image *image =
597 vk_object_zalloc(&device->vk, alloc, sizeof(*image), VK_OBJECT_TYPE_IMAGE);
598
599 if (!image)
600 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
601
602 if (pCreateInfo->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
603 const VkImageDrmFormatModifierListCreateInfoEXT *mod_info =
604 vk_find_struct_const(pCreateInfo->pNext,
605 IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
606 const VkImageDrmFormatModifierExplicitCreateInfoEXT *drm_explicit_info =
607 vk_find_struct_const(pCreateInfo->pNext,
608 IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT);
609
610 assert(mod_info || drm_explicit_info);
611
612 if (mod_info) {
613 modifier = DRM_FORMAT_MOD_LINEAR;
614 for (unsigned i = 0; i < mod_info->drmFormatModifierCount; i++) {
615 if (mod_info->pDrmFormatModifiers[i] == DRM_FORMAT_MOD_QCOM_COMPRESSED)
616 modifier = DRM_FORMAT_MOD_QCOM_COMPRESSED;
617 }
618 } else {
619 modifier = drm_explicit_info->drmFormatModifier;
620 assert(modifier == DRM_FORMAT_MOD_LINEAR ||
621 modifier == DRM_FORMAT_MOD_QCOM_COMPRESSED);
622 plane_layouts = drm_explicit_info->pPlaneLayouts;
623 }
624 } else {
625 const struct wsi_image_create_info *wsi_info =
626 vk_find_struct_const(pCreateInfo->pNext, WSI_IMAGE_CREATE_INFO_MESA);
627 if (wsi_info && wsi_info->scanout)
628 modifier = DRM_FORMAT_MOD_LINEAR;
629 }
630
631 #ifdef ANDROID
632 const VkNativeBufferANDROID *gralloc_info =
633 vk_find_struct_const(pCreateInfo->pNext, NATIVE_BUFFER_ANDROID);
634 int dma_buf;
635 if (gralloc_info) {
636 VkResult result = tu_gralloc_info(device, gralloc_info, &dma_buf, &modifier);
637 if (result != VK_SUCCESS)
638 return result;
639 }
640 #endif
641
642 VkResult result = tu_image_init(device, image, pCreateInfo, modifier,
643 plane_layouts);
644 if (result != VK_SUCCESS) {
645 vk_object_free(&device->vk, alloc, image);
646 return result;
647 }
648
649 *pImage = tu_image_to_handle(image);
650
651 #ifdef ANDROID
652 if (gralloc_info)
653 return tu_import_memory_from_gralloc_handle(_device, dma_buf, alloc,
654 *pImage);
655 #endif
656 return VK_SUCCESS;
657 }
658
659 VKAPI_ATTR void VKAPI_CALL
tu_DestroyImage(VkDevice _device,VkImage _image,const VkAllocationCallbacks * pAllocator)660 tu_DestroyImage(VkDevice _device,
661 VkImage _image,
662 const VkAllocationCallbacks *pAllocator)
663 {
664 TU_FROM_HANDLE(tu_device, device, _device);
665 TU_FROM_HANDLE(tu_image, image, _image);
666
667 if (!image)
668 return;
669
670 #ifdef ANDROID
671 if (image->owned_memory != VK_NULL_HANDLE)
672 tu_FreeMemory(_device, image->owned_memory, pAllocator);
673 #endif
674
675 vk_object_free(&device->vk, pAllocator, image);
676 }
677
678 static void
tu_get_image_memory_requirements(struct tu_image * image,VkMemoryRequirements2 * pMemoryRequirements)679 tu_get_image_memory_requirements(struct tu_image *image,
680 VkMemoryRequirements2 *pMemoryRequirements)
681 {
682 pMemoryRequirements->memoryRequirements = (VkMemoryRequirements) {
683 .memoryTypeBits = 1,
684 .alignment = image->layout[0].base_align,
685 .size = image->total_size
686 };
687
688 vk_foreach_struct(ext, pMemoryRequirements->pNext) {
689 switch (ext->sType) {
690 case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS: {
691 VkMemoryDedicatedRequirements *req =
692 (VkMemoryDedicatedRequirements *) ext;
693 req->requiresDedicatedAllocation =
694 image->vk.external_handle_types != 0;
695 req->prefersDedicatedAllocation = req->requiresDedicatedAllocation;
696 break;
697 }
698 default:
699 break;
700 }
701 }
702 }
703
704 VKAPI_ATTR void VKAPI_CALL
tu_GetImageMemoryRequirements2(VkDevice device,const VkImageMemoryRequirementsInfo2 * pInfo,VkMemoryRequirements2 * pMemoryRequirements)705 tu_GetImageMemoryRequirements2(VkDevice device,
706 const VkImageMemoryRequirementsInfo2 *pInfo,
707 VkMemoryRequirements2 *pMemoryRequirements)
708 {
709 TU_FROM_HANDLE(tu_image, image, pInfo->image);
710
711 tu_get_image_memory_requirements(image, pMemoryRequirements);
712 }
713
714 VKAPI_ATTR void VKAPI_CALL
tu_GetImageSparseMemoryRequirements2(VkDevice device,const VkImageSparseMemoryRequirementsInfo2 * pInfo,uint32_t * pSparseMemoryRequirementCount,VkSparseImageMemoryRequirements2 * pSparseMemoryRequirements)715 tu_GetImageSparseMemoryRequirements2(
716 VkDevice device,
717 const VkImageSparseMemoryRequirementsInfo2 *pInfo,
718 uint32_t *pSparseMemoryRequirementCount,
719 VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
720 {
721 tu_stub();
722 }
723
724 VKAPI_ATTR void VKAPI_CALL
tu_GetDeviceImageMemoryRequirements(VkDevice _device,const VkDeviceImageMemoryRequirements * pInfo,VkMemoryRequirements2 * pMemoryRequirements)725 tu_GetDeviceImageMemoryRequirements(
726 VkDevice _device,
727 const VkDeviceImageMemoryRequirements *pInfo,
728 VkMemoryRequirements2 *pMemoryRequirements)
729 {
730 TU_FROM_HANDLE(tu_device, device, _device);
731
732 struct tu_image image = {0};
733
734 tu_image_init(device, &image, pInfo->pCreateInfo, DRM_FORMAT_MOD_INVALID,
735 NULL);
736
737 tu_get_image_memory_requirements(&image, pMemoryRequirements);
738 }
739
740 VKAPI_ATTR void VKAPI_CALL
tu_GetDeviceImageSparseMemoryRequirements(VkDevice device,const VkDeviceImageMemoryRequirements * pInfo,uint32_t * pSparseMemoryRequirementCount,VkSparseImageMemoryRequirements2 * pSparseMemoryRequirements)741 tu_GetDeviceImageSparseMemoryRequirements(
742 VkDevice device,
743 const VkDeviceImageMemoryRequirements *pInfo,
744 uint32_t *pSparseMemoryRequirementCount,
745 VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
746 {
747 tu_stub();
748 }
749
750 VKAPI_ATTR void VKAPI_CALL
tu_GetImageSubresourceLayout(VkDevice _device,VkImage _image,const VkImageSubresource * pSubresource,VkSubresourceLayout * pLayout)751 tu_GetImageSubresourceLayout(VkDevice _device,
752 VkImage _image,
753 const VkImageSubresource *pSubresource,
754 VkSubresourceLayout *pLayout)
755 {
756 TU_FROM_HANDLE(tu_image, image, _image);
757
758 struct fdl_layout *layout =
759 &image->layout[tu6_plane_index(image->vk.format, pSubresource->aspectMask)];
760 const struct fdl_slice *slice = layout->slices + pSubresource->mipLevel;
761
762 pLayout->offset =
763 fdl_surface_offset(layout, pSubresource->mipLevel, pSubresource->arrayLayer);
764 pLayout->rowPitch = fdl_pitch(layout, pSubresource->mipLevel);
765 pLayout->arrayPitch = fdl_layer_stride(layout, pSubresource->mipLevel);
766 pLayout->depthPitch = slice->size0;
767 pLayout->size = pLayout->depthPitch * layout->depth0;
768
769 if (fdl_ubwc_enabled(layout, pSubresource->mipLevel)) {
770 /* UBWC starts at offset 0 */
771 pLayout->offset = 0;
772 /* UBWC scanout won't match what the kernel wants if we have levels/layers */
773 assert(image->vk.mip_levels == 1 && image->vk.array_layers == 1);
774 }
775 }
776
777 VKAPI_ATTR VkResult VKAPI_CALL
tu_CreateImageView(VkDevice _device,const VkImageViewCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkImageView * pView)778 tu_CreateImageView(VkDevice _device,
779 const VkImageViewCreateInfo *pCreateInfo,
780 const VkAllocationCallbacks *pAllocator,
781 VkImageView *pView)
782 {
783 TU_FROM_HANDLE(tu_device, device, _device);
784 struct tu_image_view *view;
785
786 view = vk_object_alloc(&device->vk, pAllocator, sizeof(*view),
787 VK_OBJECT_TYPE_IMAGE_VIEW);
788 if (view == NULL)
789 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
790
791 tu_image_view_init(device, view, pCreateInfo, device->use_z24uint_s8uint);
792
793 *pView = tu_image_view_to_handle(view);
794
795 return VK_SUCCESS;
796 }
797
798 VKAPI_ATTR void VKAPI_CALL
tu_DestroyImageView(VkDevice _device,VkImageView _iview,const VkAllocationCallbacks * pAllocator)799 tu_DestroyImageView(VkDevice _device,
800 VkImageView _iview,
801 const VkAllocationCallbacks *pAllocator)
802 {
803 TU_FROM_HANDLE(tu_device, device, _device);
804 TU_FROM_HANDLE(tu_image_view, iview, _iview);
805
806 if (!iview)
807 return;
808
809 vk_object_free(&device->vk, pAllocator, iview);
810 }
811
812 void
tu_buffer_view_init(struct tu_buffer_view * view,struct tu_device * device,const VkBufferViewCreateInfo * pCreateInfo)813 tu_buffer_view_init(struct tu_buffer_view *view,
814 struct tu_device *device,
815 const VkBufferViewCreateInfo *pCreateInfo)
816 {
817 TU_FROM_HANDLE(tu_buffer, buffer, pCreateInfo->buffer);
818
819 view->buffer = buffer;
820
821 uint32_t range;
822 if (pCreateInfo->range == VK_WHOLE_SIZE)
823 range = buffer->size - pCreateInfo->offset;
824 else
825 range = pCreateInfo->range;
826
827 uint8_t swiz[4] = { PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y, PIPE_SWIZZLE_Z,
828 PIPE_SWIZZLE_W };
829
830 fdl6_buffer_view_init(
831 view->descriptor, tu_vk_format_to_pipe_format(pCreateInfo->format),
832 swiz, buffer->iova + pCreateInfo->offset, range);
833 }
834
835 VKAPI_ATTR VkResult VKAPI_CALL
tu_CreateBufferView(VkDevice _device,const VkBufferViewCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkBufferView * pView)836 tu_CreateBufferView(VkDevice _device,
837 const VkBufferViewCreateInfo *pCreateInfo,
838 const VkAllocationCallbacks *pAllocator,
839 VkBufferView *pView)
840 {
841 TU_FROM_HANDLE(tu_device, device, _device);
842 struct tu_buffer_view *view;
843
844 view = vk_object_alloc(&device->vk, pAllocator, sizeof(*view),
845 VK_OBJECT_TYPE_BUFFER_VIEW);
846 if (!view)
847 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
848
849 tu_buffer_view_init(view, device, pCreateInfo);
850
851 *pView = tu_buffer_view_to_handle(view);
852
853 return VK_SUCCESS;
854 }
855
856 VKAPI_ATTR void VKAPI_CALL
tu_DestroyBufferView(VkDevice _device,VkBufferView bufferView,const VkAllocationCallbacks * pAllocator)857 tu_DestroyBufferView(VkDevice _device,
858 VkBufferView bufferView,
859 const VkAllocationCallbacks *pAllocator)
860 {
861 TU_FROM_HANDLE(tu_device, device, _device);
862 TU_FROM_HANDLE(tu_buffer_view, view, bufferView);
863
864 if (!view)
865 return;
866
867 vk_object_free(&device->vk, pAllocator, view);
868 }
869