1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * based in part on anv driver which is:
6 * Copyright © 2015 Intel Corporation
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the "Software"),
10 * to deal in the Software without restriction, including without limitation
11 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 * and/or sell copies of the Software, and to permit persons to whom the
13 * Software is furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the next
16 * paragraph) shall be included in all copies or substantial portions of the
17 * Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25 * DEALINGS IN THE SOFTWARE.
26 */
27
28 #include "tu_private.h"
29
30 #include "util/debug.h"
31 #include "util/u_atomic.h"
32 #include "util/format/u_format.h"
33 #include "vk_format.h"
34 #include "vk_util.h"
35 #include "drm-uapi/drm_fourcc.h"
36
37 #include "tu_cs.h"
38
39 static uint32_t
tu6_plane_count(VkFormat format)40 tu6_plane_count(VkFormat format)
41 {
42 switch (format) {
43 default:
44 return 1;
45 case VK_FORMAT_G8_B8R8_2PLANE_420_UNORM:
46 case VK_FORMAT_D32_SFLOAT_S8_UINT:
47 return 2;
48 case VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM:
49 return 3;
50 }
51 }
52
53 static VkFormat
tu6_plane_format(VkFormat format,uint32_t plane)54 tu6_plane_format(VkFormat format, uint32_t plane)
55 {
56 switch (format) {
57 case VK_FORMAT_G8_B8R8_2PLANE_420_UNORM:
58 /* note: with UBWC, and Y plane UBWC is different from R8_UNORM */
59 return plane ? VK_FORMAT_R8G8_UNORM : VK_FORMAT_R8_UNORM;
60 case VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM:
61 return VK_FORMAT_R8_UNORM;
62 case VK_FORMAT_D32_SFLOAT_S8_UINT:
63 return plane ? VK_FORMAT_S8_UINT : VK_FORMAT_D32_SFLOAT;
64 default:
65 return format;
66 }
67 }
68
69 static uint32_t
tu6_plane_index(VkFormat format,VkImageAspectFlags aspect_mask)70 tu6_plane_index(VkFormat format, VkImageAspectFlags aspect_mask)
71 {
72 switch (aspect_mask) {
73 default:
74 return 0;
75 case VK_IMAGE_ASPECT_PLANE_1_BIT:
76 return 1;
77 case VK_IMAGE_ASPECT_PLANE_2_BIT:
78 return 2;
79 case VK_IMAGE_ASPECT_STENCIL_BIT:
80 return format == VK_FORMAT_D32_SFLOAT_S8_UINT;
81 }
82 }
83
84 static void
compose_swizzle(unsigned char * swiz,const VkComponentMapping * mapping)85 compose_swizzle(unsigned char *swiz, const VkComponentMapping *mapping)
86 {
87 unsigned char src_swiz[4] = { swiz[0], swiz[1], swiz[2], swiz[3] };
88 VkComponentSwizzle vk_swiz[4] = {
89 mapping->r, mapping->g, mapping->b, mapping->a
90 };
91 for (int i = 0; i < 4; i++) {
92 switch (vk_swiz[i]) {
93 case VK_COMPONENT_SWIZZLE_IDENTITY:
94 swiz[i] = src_swiz[i];
95 break;
96 case VK_COMPONENT_SWIZZLE_R...VK_COMPONENT_SWIZZLE_A:
97 swiz[i] = src_swiz[vk_swiz[i] - VK_COMPONENT_SWIZZLE_R];
98 break;
99 case VK_COMPONENT_SWIZZLE_ZERO:
100 swiz[i] = A6XX_TEX_ZERO;
101 break;
102 case VK_COMPONENT_SWIZZLE_ONE:
103 swiz[i] = A6XX_TEX_ONE;
104 break;
105 default:
106 unreachable("unexpected swizzle");
107 }
108 }
109 }
110
111 static uint32_t
tu6_texswiz(const VkComponentMapping * comps,const struct tu_sampler_ycbcr_conversion * conversion,VkFormat format,VkImageAspectFlagBits aspect_mask,bool limited_z24s8)112 tu6_texswiz(const VkComponentMapping *comps,
113 const struct tu_sampler_ycbcr_conversion *conversion,
114 VkFormat format,
115 VkImageAspectFlagBits aspect_mask,
116 bool limited_z24s8)
117 {
118 unsigned char swiz[4] = {
119 A6XX_TEX_X, A6XX_TEX_Y, A6XX_TEX_Z, A6XX_TEX_W,
120 };
121
122 switch (format) {
123 case VK_FORMAT_G8B8G8R8_422_UNORM:
124 case VK_FORMAT_B8G8R8G8_422_UNORM:
125 case VK_FORMAT_G8_B8R8_2PLANE_420_UNORM:
126 case VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM:
127 swiz[0] = A6XX_TEX_Z;
128 swiz[1] = A6XX_TEX_X;
129 swiz[2] = A6XX_TEX_Y;
130 break;
131 case VK_FORMAT_BC1_RGB_UNORM_BLOCK:
132 case VK_FORMAT_BC1_RGB_SRGB_BLOCK:
133 /* same hardware format is used for BC1_RGB / BC1_RGBA */
134 swiz[3] = A6XX_TEX_ONE;
135 break;
136 case VK_FORMAT_D24_UNORM_S8_UINT:
137 if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT) {
138 if (limited_z24s8) {
139 /* using FMT6_8_8_8_8_UINT */
140 swiz[0] = A6XX_TEX_W;
141 swiz[1] = A6XX_TEX_ZERO;
142 } else {
143 /* using FMT6_Z24_UINT_S8_UINT */
144 swiz[0] = A6XX_TEX_Y;
145 swiz[1] = A6XX_TEX_ZERO;
146 }
147 }
148 default:
149 break;
150 }
151
152 compose_swizzle(swiz, comps);
153 if (conversion)
154 compose_swizzle(swiz, &conversion->components);
155
156 return A6XX_TEX_CONST_0_SWIZ_X(swiz[0]) |
157 A6XX_TEX_CONST_0_SWIZ_Y(swiz[1]) |
158 A6XX_TEX_CONST_0_SWIZ_Z(swiz[2]) |
159 A6XX_TEX_CONST_0_SWIZ_W(swiz[3]);
160 }
161
162 void
tu_cs_image_ref(struct tu_cs * cs,const struct tu_image_view * iview,uint32_t layer)163 tu_cs_image_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer)
164 {
165 tu_cs_emit(cs, iview->PITCH);
166 tu_cs_emit(cs, iview->layer_size >> 6);
167 tu_cs_emit_qw(cs, iview->base_addr + iview->layer_size * layer);
168 }
169
170 void
tu_cs_image_stencil_ref(struct tu_cs * cs,const struct tu_image_view * iview,uint32_t layer)171 tu_cs_image_stencil_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer)
172 {
173 tu_cs_emit(cs, iview->stencil_PITCH);
174 tu_cs_emit(cs, iview->stencil_layer_size >> 6);
175 tu_cs_emit_qw(cs, iview->stencil_base_addr + iview->stencil_layer_size * layer);
176 }
177
178 void
tu_cs_image_ref_2d(struct tu_cs * cs,const struct tu_image_view * iview,uint32_t layer,bool src)179 tu_cs_image_ref_2d(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer, bool src)
180 {
181 tu_cs_emit_qw(cs, iview->base_addr + iview->layer_size * layer);
182 /* SP_PS_2D_SRC_PITCH has shifted pitch field */
183 tu_cs_emit(cs, iview->PITCH << (src ? 9 : 0));
184 }
185
186 void
tu_cs_image_flag_ref(struct tu_cs * cs,const struct tu_image_view * iview,uint32_t layer)187 tu_cs_image_flag_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer)
188 {
189 tu_cs_emit_qw(cs, iview->ubwc_addr + iview->ubwc_layer_size * layer);
190 tu_cs_emit(cs, iview->FLAG_BUFFER_PITCH);
191 }
192
193 void
tu_image_view_init(struct tu_image_view * iview,const VkImageViewCreateInfo * pCreateInfo,bool limited_z24s8)194 tu_image_view_init(struct tu_image_view *iview,
195 const VkImageViewCreateInfo *pCreateInfo,
196 bool limited_z24s8)
197 {
198 TU_FROM_HANDLE(tu_image, image, pCreateInfo->image);
199 const VkImageSubresourceRange *range = &pCreateInfo->subresourceRange;
200 VkFormat format = pCreateInfo->format;
201 VkImageAspectFlagBits aspect_mask = pCreateInfo->subresourceRange.aspectMask;
202
203 const struct VkSamplerYcbcrConversionInfo *ycbcr_conversion =
204 vk_find_struct_const(pCreateInfo->pNext, SAMPLER_YCBCR_CONVERSION_INFO);
205 const struct tu_sampler_ycbcr_conversion *conversion = ycbcr_conversion ?
206 tu_sampler_ycbcr_conversion_from_handle(ycbcr_conversion->conversion) : NULL;
207
208 iview->image = image;
209
210 memset(iview->descriptor, 0, sizeof(iview->descriptor));
211
212 struct fdl_layout *layout =
213 &image->layout[tu6_plane_index(image->vk_format, aspect_mask)];
214
215 uint32_t width = u_minify(layout->width0, range->baseMipLevel);
216 uint32_t height = u_minify(layout->height0, range->baseMipLevel);
217 uint32_t storage_depth = tu_get_layerCount(image, range);
218 if (pCreateInfo->viewType == VK_IMAGE_VIEW_TYPE_3D) {
219 storage_depth = u_minify(image->layout[0].depth0, range->baseMipLevel);
220 }
221
222 uint32_t depth = storage_depth;
223 if (pCreateInfo->viewType == VK_IMAGE_VIEW_TYPE_CUBE ||
224 pCreateInfo->viewType == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY) {
225 /* Cubes are treated as 2D arrays for storage images, so only divide the
226 * depth by 6 for the texture descriptor.
227 */
228 depth /= 6;
229 }
230
231 uint64_t base_addr = image->bo->iova + image->bo_offset +
232 fdl_surface_offset(layout, range->baseMipLevel, range->baseArrayLayer);
233 uint64_t ubwc_addr = image->bo->iova + image->bo_offset +
234 fdl_ubwc_offset(layout, range->baseMipLevel, range->baseArrayLayer);
235
236 uint32_t pitch = fdl_pitch(layout, range->baseMipLevel);
237 uint32_t ubwc_pitch = fdl_ubwc_pitch(layout, range->baseMipLevel);
238 uint32_t layer_size = fdl_layer_stride(layout, range->baseMipLevel);
239
240 if (aspect_mask != VK_IMAGE_ASPECT_COLOR_BIT)
241 format = tu6_plane_format(format, tu6_plane_index(format, aspect_mask));
242
243 struct tu_native_format fmt = tu6_format_texture(format, layout->tile_mode);
244 /* note: freedreno layout assumes no TILE_ALL bit for non-UBWC color formats
245 * this means smaller mipmap levels have a linear tile mode.
246 * Depth/stencil formats have non-linear tile mode.
247 */
248 fmt.tile_mode = fdl_tile_mode(layout, range->baseMipLevel);
249
250 bool ubwc_enabled = fdl_ubwc_enabled(layout, range->baseMipLevel);
251
252 bool is_d24s8 = (format == VK_FORMAT_D24_UNORM_S8_UINT ||
253 format == VK_FORMAT_X8_D24_UNORM_PACK32);
254
255 if (is_d24s8 && ubwc_enabled)
256 fmt.fmt = FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8;
257
258 unsigned fmt_tex = fmt.fmt;
259 if (is_d24s8) {
260 if (aspect_mask & VK_IMAGE_ASPECT_DEPTH_BIT)
261 fmt_tex = FMT6_Z24_UNORM_S8_UINT;
262 if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT)
263 fmt_tex = limited_z24s8 ? FMT6_8_8_8_8_UINT : FMT6_Z24_UINT_S8_UINT;
264 /* TODO: also use this format with storage descriptor ? */
265 }
266
267 iview->descriptor[0] =
268 A6XX_TEX_CONST_0_TILE_MODE(fmt.tile_mode) |
269 COND(vk_format_is_srgb(format), A6XX_TEX_CONST_0_SRGB) |
270 A6XX_TEX_CONST_0_FMT(fmt_tex) |
271 A6XX_TEX_CONST_0_SAMPLES(tu_msaa_samples(layout->nr_samples)) |
272 A6XX_TEX_CONST_0_SWAP(fmt.swap) |
273 tu6_texswiz(&pCreateInfo->components, conversion, format, aspect_mask, limited_z24s8) |
274 A6XX_TEX_CONST_0_MIPLVLS(tu_get_levelCount(image, range) - 1);
275 iview->descriptor[1] = A6XX_TEX_CONST_1_WIDTH(width) | A6XX_TEX_CONST_1_HEIGHT(height);
276 iview->descriptor[2] =
277 A6XX_TEX_CONST_2_PITCHALIGN(layout->pitchalign - 6) |
278 A6XX_TEX_CONST_2_PITCH(pitch) |
279 A6XX_TEX_CONST_2_TYPE(tu6_tex_type(pCreateInfo->viewType, false));
280 iview->descriptor[3] = A6XX_TEX_CONST_3_ARRAY_PITCH(layer_size);
281 iview->descriptor[4] = base_addr;
282 iview->descriptor[5] = (base_addr >> 32) | A6XX_TEX_CONST_5_DEPTH(depth);
283
284 if (layout->tile_all)
285 iview->descriptor[3] |= A6XX_TEX_CONST_3_TILE_ALL;
286
287 if (format == VK_FORMAT_G8_B8R8_2PLANE_420_UNORM ||
288 format == VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM) {
289 /* chroma offset re-uses MIPLVLS bits */
290 assert(tu_get_levelCount(image, range) == 1);
291 if (conversion) {
292 if (conversion->chroma_offsets[0] == VK_CHROMA_LOCATION_MIDPOINT)
293 iview->descriptor[0] |= A6XX_TEX_CONST_0_CHROMA_MIDPOINT_X;
294 if (conversion->chroma_offsets[1] == VK_CHROMA_LOCATION_MIDPOINT)
295 iview->descriptor[0] |= A6XX_TEX_CONST_0_CHROMA_MIDPOINT_Y;
296 }
297
298 uint64_t base_addr[3];
299
300 iview->descriptor[3] |= A6XX_TEX_CONST_3_TILE_ALL;
301 if (ubwc_enabled) {
302 iview->descriptor[3] |= A6XX_TEX_CONST_3_FLAG;
303 /* no separate ubwc base, image must have the expected layout */
304 for (uint32_t i = 0; i < 3; i++) {
305 base_addr[i] = image->bo->iova + image->bo_offset +
306 fdl_ubwc_offset(&image->layout[i], range->baseMipLevel, range->baseArrayLayer);
307 }
308 } else {
309 for (uint32_t i = 0; i < 3; i++) {
310 base_addr[i] = image->bo->iova + image->bo_offset +
311 fdl_surface_offset(&image->layout[i], range->baseMipLevel, range->baseArrayLayer);
312 }
313 }
314
315 iview->descriptor[4] = base_addr[0];
316 iview->descriptor[5] |= base_addr[0] >> 32;
317 iview->descriptor[6] =
318 A6XX_TEX_CONST_6_PLANE_PITCH(fdl_pitch(&image->layout[1], range->baseMipLevel));
319 iview->descriptor[7] = base_addr[1];
320 iview->descriptor[8] = base_addr[1] >> 32;
321 iview->descriptor[9] = base_addr[2];
322 iview->descriptor[10] = base_addr[2] >> 32;
323
324 assert(pCreateInfo->viewType != VK_IMAGE_VIEW_TYPE_3D);
325 return;
326 }
327
328 if (ubwc_enabled) {
329 uint32_t block_width, block_height;
330 fdl6_get_ubwc_blockwidth(layout, &block_width, &block_height);
331
332 iview->descriptor[3] |= A6XX_TEX_CONST_3_FLAG;
333 iview->descriptor[7] = ubwc_addr;
334 iview->descriptor[8] = ubwc_addr >> 32;
335 iview->descriptor[9] |= A6XX_TEX_CONST_9_FLAG_BUFFER_ARRAY_PITCH(layout->ubwc_layer_size >> 2);
336 iview->descriptor[10] |=
337 A6XX_TEX_CONST_10_FLAG_BUFFER_PITCH(ubwc_pitch) |
338 A6XX_TEX_CONST_10_FLAG_BUFFER_LOGW(util_logbase2_ceil(DIV_ROUND_UP(width, block_width))) |
339 A6XX_TEX_CONST_10_FLAG_BUFFER_LOGH(util_logbase2_ceil(DIV_ROUND_UP(height, block_height)));
340 }
341
342 if (pCreateInfo->viewType == VK_IMAGE_VIEW_TYPE_3D) {
343 iview->descriptor[3] |=
344 A6XX_TEX_CONST_3_MIN_LAYERSZ(layout->slices[image->level_count - 1].size0);
345 }
346
347 iview->SP_PS_2D_SRC_INFO = A6XX_SP_PS_2D_SRC_INFO(
348 .color_format = fmt.fmt,
349 .tile_mode = fmt.tile_mode,
350 .color_swap = fmt.swap,
351 .flags = ubwc_enabled,
352 .srgb = vk_format_is_srgb(format),
353 .samples = tu_msaa_samples(layout->nr_samples),
354 .samples_average = layout->nr_samples > 1 &&
355 !vk_format_is_int(format) &&
356 !vk_format_is_depth_or_stencil(format),
357 .unk20 = 1,
358 .unk22 = 1).value;
359 iview->SP_PS_2D_SRC_SIZE =
360 A6XX_SP_PS_2D_SRC_SIZE(.width = width, .height = height).value;
361
362 /* note: these have same encoding for MRT and 2D (except 2D PITCH src) */
363 iview->PITCH = A6XX_RB_DEPTH_BUFFER_PITCH(pitch).value;
364 iview->FLAG_BUFFER_PITCH = A6XX_RB_DEPTH_FLAG_BUFFER_PITCH(
365 .pitch = ubwc_pitch, .array_pitch = layout->ubwc_layer_size >> 2).value;
366
367 iview->base_addr = base_addr;
368 iview->ubwc_addr = ubwc_addr;
369 iview->layer_size = layer_size;
370 iview->ubwc_layer_size = layout->ubwc_layer_size;
371
372 /* Don't set fields that are only used for attachments/blit dest if COLOR
373 * is unsupported.
374 */
375 if (!(fmt.supported & FMT_COLOR))
376 return;
377
378 struct tu_native_format cfmt = tu6_format_color(format, layout->tile_mode);
379 cfmt.tile_mode = fmt.tile_mode;
380
381 if (is_d24s8 && ubwc_enabled)
382 cfmt.fmt = FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8;
383
384 memset(iview->storage_descriptor, 0, sizeof(iview->storage_descriptor));
385
386 iview->storage_descriptor[0] =
387 A6XX_IBO_0_FMT(fmt.fmt) |
388 A6XX_IBO_0_TILE_MODE(fmt.tile_mode);
389 iview->storage_descriptor[1] =
390 A6XX_IBO_1_WIDTH(width) |
391 A6XX_IBO_1_HEIGHT(height);
392 iview->storage_descriptor[2] =
393 A6XX_IBO_2_PITCH(pitch) |
394 A6XX_IBO_2_TYPE(tu6_tex_type(pCreateInfo->viewType, true));
395 iview->storage_descriptor[3] = A6XX_IBO_3_ARRAY_PITCH(layer_size);
396
397 iview->storage_descriptor[4] = base_addr;
398 iview->storage_descriptor[5] = (base_addr >> 32) | A6XX_IBO_5_DEPTH(storage_depth);
399
400 if (ubwc_enabled) {
401 iview->storage_descriptor[3] |= A6XX_IBO_3_FLAG | A6XX_IBO_3_UNK27;
402 iview->storage_descriptor[7] |= ubwc_addr;
403 iview->storage_descriptor[8] |= ubwc_addr >> 32;
404 iview->storage_descriptor[9] = A6XX_IBO_9_FLAG_BUFFER_ARRAY_PITCH(layout->ubwc_layer_size >> 2);
405 iview->storage_descriptor[10] =
406 A6XX_IBO_10_FLAG_BUFFER_PITCH(ubwc_pitch);
407 }
408
409 iview->extent.width = width;
410 iview->extent.height = height;
411 iview->need_y2_align =
412 (fmt.tile_mode == TILE6_LINEAR && range->baseMipLevel != image->level_count - 1);
413
414 iview->ubwc_enabled = ubwc_enabled;
415
416 iview->RB_MRT_BUF_INFO = A6XX_RB_MRT_BUF_INFO(0,
417 .color_tile_mode = cfmt.tile_mode,
418 .color_format = cfmt.fmt,
419 .color_swap = cfmt.swap).value;
420
421 iview->SP_FS_MRT_REG = A6XX_SP_FS_MRT_REG(0,
422 .color_format = cfmt.fmt,
423 .color_sint = vk_format_is_sint(format),
424 .color_uint = vk_format_is_uint(format)).value;
425
426 iview->RB_2D_DST_INFO = A6XX_RB_2D_DST_INFO(
427 .color_format = cfmt.fmt,
428 .tile_mode = cfmt.tile_mode,
429 .color_swap = cfmt.swap,
430 .flags = ubwc_enabled,
431 .srgb = vk_format_is_srgb(format)).value;
432
433 iview->RB_BLIT_DST_INFO = A6XX_RB_BLIT_DST_INFO(
434 .tile_mode = cfmt.tile_mode,
435 .samples = tu_msaa_samples(layout->nr_samples),
436 .color_format = cfmt.fmt,
437 .color_swap = cfmt.swap,
438 .flags = ubwc_enabled).value;
439
440 if (image->vk_format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
441 layout = &image->layout[1];
442 iview->stencil_base_addr = image->bo->iova + image->bo_offset +
443 fdl_surface_offset(layout, range->baseMipLevel, range->baseArrayLayer);
444 iview->stencil_layer_size = fdl_layer_stride(layout, range->baseMipLevel);
445 iview->stencil_PITCH = A6XX_RB_STENCIL_BUFFER_PITCH(fdl_pitch(layout, range->baseMipLevel)).value;
446 }
447 }
448
449 bool
ubwc_possible(VkFormat format,VkImageType type,VkImageUsageFlags usage,bool limited_z24s8)450 ubwc_possible(VkFormat format, VkImageType type, VkImageUsageFlags usage, bool limited_z24s8)
451 {
452 /* no UBWC with compressed formats, E5B9G9R9, S8_UINT
453 * (S8_UINT because separate stencil doesn't have UBWC-enable bit)
454 */
455 if (vk_format_is_compressed(format) ||
456 format == VK_FORMAT_E5B9G9R9_UFLOAT_PACK32 ||
457 format == VK_FORMAT_S8_UINT)
458 return false;
459
460 if (type == VK_IMAGE_TYPE_3D) {
461 tu_finishme("UBWC with 3D textures");
462 return false;
463 }
464
465 /* Disable UBWC for storage images.
466 *
467 * The closed GL driver skips UBWC for storage images (and additionally
468 * uses linear for writeonly images). We seem to have image tiling working
469 * in freedreno in general, so turnip matches that. freedreno also enables
470 * UBWC on images, but it's not really tested due to the lack of
471 * UBWC-enabled mipmaps in freedreno currently. Just match the closed GL
472 * behavior of no UBWC.
473 */
474 if (usage & VK_IMAGE_USAGE_STORAGE_BIT)
475 return false;
476
477 /* Disable UBWC for D24S8 on A630 in some cases
478 *
479 * VK_IMAGE_ASPECT_STENCIL_BIT image view requires to be able to sample
480 * from the stencil component as UINT, however no format allows this
481 * on a630 (the special FMT6_Z24_UINT_S8_UINT format is missing)
482 *
483 * It must be sampled as FMT6_8_8_8_8_UINT, which is not UBWC-compatible
484 *
485 * Additionally, the special AS_R8G8B8A8 format is broken without UBWC,
486 * so we have to fallback to 8_8_8_8_UNORM when UBWC is disabled
487 */
488 if (limited_z24s8 &&
489 format == VK_FORMAT_D24_UNORM_S8_UINT &&
490 (usage & (VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT)))
491 return false;
492
493 return true;
494 }
495
496 VkResult
tu_CreateImage(VkDevice _device,const VkImageCreateInfo * pCreateInfo,const VkAllocationCallbacks * alloc,VkImage * pImage)497 tu_CreateImage(VkDevice _device,
498 const VkImageCreateInfo *pCreateInfo,
499 const VkAllocationCallbacks *alloc,
500 VkImage *pImage)
501 {
502 TU_FROM_HANDLE(tu_device, device, _device);
503 uint64_t modifier = DRM_FORMAT_MOD_INVALID;
504 const VkSubresourceLayout *plane_layouts = NULL;
505 struct tu_image *image;
506
507 if (pCreateInfo->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
508 const VkImageDrmFormatModifierListCreateInfoEXT *mod_info =
509 vk_find_struct_const(pCreateInfo->pNext,
510 IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
511 const VkImageDrmFormatModifierExplicitCreateInfoEXT *drm_explicit_info =
512 vk_find_struct_const(pCreateInfo->pNext,
513 IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT);
514
515 assert(mod_info || drm_explicit_info);
516
517 if (mod_info) {
518 modifier = DRM_FORMAT_MOD_LINEAR;
519 for (unsigned i = 0; i < mod_info->drmFormatModifierCount; i++) {
520 if (mod_info->pDrmFormatModifiers[i] == DRM_FORMAT_MOD_QCOM_COMPRESSED)
521 modifier = DRM_FORMAT_MOD_QCOM_COMPRESSED;
522 }
523 } else {
524 modifier = drm_explicit_info->drmFormatModifier;
525 assert(modifier == DRM_FORMAT_MOD_LINEAR ||
526 modifier == DRM_FORMAT_MOD_QCOM_COMPRESSED);
527 plane_layouts = drm_explicit_info->pPlaneLayouts;
528 }
529 } else {
530 const struct wsi_image_create_info *wsi_info =
531 vk_find_struct_const(pCreateInfo->pNext, WSI_IMAGE_CREATE_INFO_MESA);
532 if (wsi_info && wsi_info->scanout)
533 modifier = DRM_FORMAT_MOD_LINEAR;
534 }
535
536 #ifdef ANDROID
537 const VkNativeBufferANDROID *gralloc_info =
538 vk_find_struct_const(pCreateInfo->pNext, NATIVE_BUFFER_ANDROID);
539 int dma_buf;
540 if (gralloc_info) {
541 VkResult result = tu_gralloc_info(device, gralloc_info, &dma_buf, &modifier);
542 if (result != VK_SUCCESS)
543 return result;
544 }
545 #endif
546
547 image = vk_object_zalloc(&device->vk, alloc, sizeof(*image),
548 VK_OBJECT_TYPE_IMAGE);
549 if (!image)
550 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
551
552 image->vk_format = pCreateInfo->format;
553 image->level_count = pCreateInfo->mipLevels;
554 image->layer_count = pCreateInfo->arrayLayers;
555
556 enum a6xx_tile_mode tile_mode = TILE6_3;
557 bool ubwc_enabled =
558 !(device->physical_device->instance->debug_flags & TU_DEBUG_NOUBWC);
559
560 /* use linear tiling if requested */
561 if (pCreateInfo->tiling == VK_IMAGE_TILING_LINEAR || modifier == DRM_FORMAT_MOD_LINEAR) {
562 tile_mode = TILE6_LINEAR;
563 ubwc_enabled = false;
564 }
565
566 /* Mutable images can be reinterpreted as any other compatible format.
567 * This is a problem with UBWC (compression for different formats is different),
568 * but also tiling ("swap" affects how tiled formats are stored in memory)
569 * Depth and stencil formats cannot be reintepreted as another format, and
570 * cannot be linear with sysmem rendering, so don't fall back for those.
571 *
572 * TODO:
573 * - if the fmt_list contains only formats which are swapped, but compatible
574 * with each other (B8G8R8A8_UNORM and B8G8R8A8_UINT for example), then
575 * tiling is still possible
576 * - figure out which UBWC compressions are compatible to keep it enabled
577 */
578 if ((pCreateInfo->flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) &&
579 !vk_format_is_depth_or_stencil(image->vk_format)) {
580 const VkImageFormatListCreateInfo *fmt_list =
581 vk_find_struct_const(pCreateInfo->pNext, IMAGE_FORMAT_LIST_CREATE_INFO);
582 bool may_be_swapped = true;
583 if (fmt_list) {
584 may_be_swapped = false;
585 for (uint32_t i = 0; i < fmt_list->viewFormatCount; i++) {
586 if (tu6_format_color(fmt_list->pViewFormats[i], TILE6_LINEAR).swap) {
587 may_be_swapped = true;
588 break;
589 }
590 }
591 }
592 if (may_be_swapped)
593 tile_mode = TILE6_LINEAR;
594 ubwc_enabled = false;
595 }
596
597 if (!ubwc_possible(image->vk_format, pCreateInfo->imageType, pCreateInfo->usage,
598 device->physical_device->limited_z24s8))
599 ubwc_enabled = false;
600
601 /* expect UBWC enabled if we asked for it */
602 assert(modifier != DRM_FORMAT_MOD_QCOM_COMPRESSED || ubwc_enabled);
603
604 for (uint32_t i = 0; i < tu6_plane_count(image->vk_format); i++) {
605 struct fdl_layout *layout = &image->layout[i];
606 VkFormat format = tu6_plane_format(image->vk_format, i);
607 uint32_t width0 = pCreateInfo->extent.width;
608 uint32_t height0 = pCreateInfo->extent.height;
609
610 if (i > 0) {
611 switch (image->vk_format) {
612 case VK_FORMAT_G8_B8R8_2PLANE_420_UNORM:
613 case VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM:
614 /* half width/height on chroma planes */
615 width0 = (width0 + 1) >> 1;
616 height0 = (height0 + 1) >> 1;
617 break;
618 case VK_FORMAT_D32_SFLOAT_S8_UINT:
619 /* no UBWC for separate stencil */
620 ubwc_enabled = false;
621 break;
622 default:
623 break;
624 }
625 }
626
627 struct fdl_explicit_layout plane_layout;
628
629 if (plane_layouts) {
630 /* only expect simple 2D images for now */
631 if (pCreateInfo->mipLevels != 1 ||
632 pCreateInfo->arrayLayers != 1 ||
633 pCreateInfo->extent.depth != 1)
634 goto invalid_layout;
635
636 plane_layout.offset = plane_layouts[i].offset;
637 plane_layout.pitch = plane_layouts[i].rowPitch;
638 /* note: use plane_layouts[0].arrayPitch to support array formats */
639 }
640
641 layout->tile_mode = tile_mode;
642 layout->ubwc = ubwc_enabled;
643
644 if (!fdl6_layout(layout, vk_format_to_pipe_format(format),
645 pCreateInfo->samples,
646 width0, height0,
647 pCreateInfo->extent.depth,
648 pCreateInfo->mipLevels,
649 pCreateInfo->arrayLayers,
650 pCreateInfo->imageType == VK_IMAGE_TYPE_3D,
651 plane_layouts ? &plane_layout : NULL)) {
652 assert(plane_layouts); /* can only fail with explicit layout */
653 goto invalid_layout;
654 }
655
656 /* fdl6_layout can't take explicit offset without explicit pitch
657 * add offset manually for extra layouts for planes
658 */
659 if (!plane_layouts && i > 0) {
660 uint32_t offset = ALIGN_POT(image->total_size, 4096);
661 for (int i = 0; i < pCreateInfo->mipLevels; i++) {
662 layout->slices[i].offset += offset;
663 layout->ubwc_slices[i].offset += offset;
664 }
665 layout->size += offset;
666 }
667
668 image->total_size = MAX2(image->total_size, layout->size);
669 }
670
671 const struct util_format_description *desc = util_format_description(image->layout[0].format);
672 if (util_format_has_depth(desc) && !(device->instance->debug_flags & TU_DEBUG_NOLRZ))
673 {
674 /* Depth plane is the first one */
675 struct fdl_layout *layout = &image->layout[0];
676 unsigned width = layout->width0;
677 unsigned height = layout->height0;
678
679 /* LRZ buffer is super-sampled */
680 switch (layout->nr_samples) {
681 case 4:
682 width *= 2;
683 /* fallthru */
684 case 2:
685 height *= 2;
686 break;
687 default:
688 break;
689 }
690
691 unsigned lrz_pitch = align(DIV_ROUND_UP(width, 8), 32);
692 unsigned lrz_height = align(DIV_ROUND_UP(height, 8), 16);
693
694 image->lrz_height = lrz_height;
695 image->lrz_pitch = lrz_pitch;
696 image->lrz_offset = image->total_size;
697 unsigned lrz_size = lrz_pitch * lrz_height * 2;
698 image->total_size += lrz_size;
699 }
700
701 *pImage = tu_image_to_handle(image);
702
703 #ifdef ANDROID
704 if (gralloc_info)
705 return tu_import_memory_from_gralloc_handle(_device, dma_buf, alloc, *pImage);
706 #endif
707 return VK_SUCCESS;
708
709 invalid_layout:
710 vk_object_free(&device->vk, alloc, image);
711 return vk_error(device->instance, VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT);
712 }
713
714 void
tu_DestroyImage(VkDevice _device,VkImage _image,const VkAllocationCallbacks * pAllocator)715 tu_DestroyImage(VkDevice _device,
716 VkImage _image,
717 const VkAllocationCallbacks *pAllocator)
718 {
719 TU_FROM_HANDLE(tu_device, device, _device);
720 TU_FROM_HANDLE(tu_image, image, _image);
721
722 if (!image)
723 return;
724
725 #ifdef ANDROID
726 if (image->owned_memory != VK_NULL_HANDLE)
727 tu_FreeMemory(_device, image->owned_memory, pAllocator);
728 #endif
729
730 vk_object_free(&device->vk, pAllocator, image);
731 }
732
733 void
tu_GetImageSubresourceLayout(VkDevice _device,VkImage _image,const VkImageSubresource * pSubresource,VkSubresourceLayout * pLayout)734 tu_GetImageSubresourceLayout(VkDevice _device,
735 VkImage _image,
736 const VkImageSubresource *pSubresource,
737 VkSubresourceLayout *pLayout)
738 {
739 TU_FROM_HANDLE(tu_image, image, _image);
740
741 struct fdl_layout *layout =
742 &image->layout[tu6_plane_index(image->vk_format, pSubresource->aspectMask)];
743 const struct fdl_slice *slice = layout->slices + pSubresource->mipLevel;
744
745 pLayout->offset =
746 fdl_surface_offset(layout, pSubresource->mipLevel, pSubresource->arrayLayer);
747 pLayout->rowPitch = fdl_pitch(layout, pSubresource->mipLevel);
748 pLayout->arrayPitch = fdl_layer_stride(layout, pSubresource->mipLevel);
749 pLayout->depthPitch = slice->size0;
750 pLayout->size = pLayout->depthPitch * layout->depth0;
751
752 if (fdl_ubwc_enabled(layout, pSubresource->mipLevel)) {
753 /* UBWC starts at offset 0 */
754 pLayout->offset = 0;
755 /* UBWC scanout won't match what the kernel wants if we have levels/layers */
756 assert(image->level_count == 1 && image->layer_count == 1);
757 }
758 }
759
tu_GetImageDrmFormatModifierPropertiesEXT(VkDevice device,VkImage _image,VkImageDrmFormatModifierPropertiesEXT * pProperties)760 VkResult tu_GetImageDrmFormatModifierPropertiesEXT(
761 VkDevice device,
762 VkImage _image,
763 VkImageDrmFormatModifierPropertiesEXT* pProperties)
764 {
765 TU_FROM_HANDLE(tu_image, image, _image);
766
767 /* TODO invent a modifier for tiled but not UBWC buffers */
768
769 if (!image->layout[0].tile_mode)
770 pProperties->drmFormatModifier = DRM_FORMAT_MOD_LINEAR;
771 else if (image->layout[0].ubwc_layer_size)
772 pProperties->drmFormatModifier = DRM_FORMAT_MOD_QCOM_COMPRESSED;
773 else
774 pProperties->drmFormatModifier = DRM_FORMAT_MOD_INVALID;
775
776 return VK_SUCCESS;
777 }
778
779
780 VkResult
tu_CreateImageView(VkDevice _device,const VkImageViewCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkImageView * pView)781 tu_CreateImageView(VkDevice _device,
782 const VkImageViewCreateInfo *pCreateInfo,
783 const VkAllocationCallbacks *pAllocator,
784 VkImageView *pView)
785 {
786 TU_FROM_HANDLE(tu_device, device, _device);
787 struct tu_image_view *view;
788
789 view = vk_object_alloc(&device->vk, pAllocator, sizeof(*view),
790 VK_OBJECT_TYPE_IMAGE_VIEW);
791 if (view == NULL)
792 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
793
794 tu_image_view_init(view, pCreateInfo, device->physical_device->limited_z24s8);
795
796 *pView = tu_image_view_to_handle(view);
797
798 return VK_SUCCESS;
799 }
800
801 void
tu_DestroyImageView(VkDevice _device,VkImageView _iview,const VkAllocationCallbacks * pAllocator)802 tu_DestroyImageView(VkDevice _device,
803 VkImageView _iview,
804 const VkAllocationCallbacks *pAllocator)
805 {
806 TU_FROM_HANDLE(tu_device, device, _device);
807 TU_FROM_HANDLE(tu_image_view, iview, _iview);
808
809 if (!iview)
810 return;
811
812 vk_object_free(&device->vk, pAllocator, iview);
813 }
814
815 void
tu_buffer_view_init(struct tu_buffer_view * view,struct tu_device * device,const VkBufferViewCreateInfo * pCreateInfo)816 tu_buffer_view_init(struct tu_buffer_view *view,
817 struct tu_device *device,
818 const VkBufferViewCreateInfo *pCreateInfo)
819 {
820 TU_FROM_HANDLE(tu_buffer, buffer, pCreateInfo->buffer);
821
822 view->buffer = buffer;
823
824 enum VkFormat vfmt = pCreateInfo->format;
825 enum pipe_format pfmt = vk_format_to_pipe_format(vfmt);
826 const struct tu_native_format fmt = tu6_format_texture(vfmt, TILE6_LINEAR);
827
828 uint32_t range;
829 if (pCreateInfo->range == VK_WHOLE_SIZE)
830 range = buffer->size - pCreateInfo->offset;
831 else
832 range = pCreateInfo->range;
833 uint32_t elements = range / util_format_get_blocksize(pfmt);
834
835 static const VkComponentMapping components = {
836 .r = VK_COMPONENT_SWIZZLE_R,
837 .g = VK_COMPONENT_SWIZZLE_G,
838 .b = VK_COMPONENT_SWIZZLE_B,
839 .a = VK_COMPONENT_SWIZZLE_A,
840 };
841
842 uint64_t iova = tu_buffer_iova(buffer) + pCreateInfo->offset;
843
844 memset(&view->descriptor, 0, sizeof(view->descriptor));
845
846 view->descriptor[0] =
847 A6XX_TEX_CONST_0_TILE_MODE(TILE6_LINEAR) |
848 A6XX_TEX_CONST_0_SWAP(fmt.swap) |
849 A6XX_TEX_CONST_0_FMT(fmt.fmt) |
850 A6XX_TEX_CONST_0_MIPLVLS(0) |
851 tu6_texswiz(&components, NULL, vfmt, VK_IMAGE_ASPECT_COLOR_BIT, false);
852 COND(vk_format_is_srgb(vfmt), A6XX_TEX_CONST_0_SRGB);
853 view->descriptor[1] =
854 A6XX_TEX_CONST_1_WIDTH(elements & MASK(15)) |
855 A6XX_TEX_CONST_1_HEIGHT(elements >> 15);
856 view->descriptor[2] =
857 A6XX_TEX_CONST_2_UNK4 |
858 A6XX_TEX_CONST_2_UNK31;
859 view->descriptor[4] = iova;
860 view->descriptor[5] = iova >> 32;
861 }
862
863 VkResult
tu_CreateBufferView(VkDevice _device,const VkBufferViewCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkBufferView * pView)864 tu_CreateBufferView(VkDevice _device,
865 const VkBufferViewCreateInfo *pCreateInfo,
866 const VkAllocationCallbacks *pAllocator,
867 VkBufferView *pView)
868 {
869 TU_FROM_HANDLE(tu_device, device, _device);
870 struct tu_buffer_view *view;
871
872 view = vk_object_alloc(&device->vk, pAllocator, sizeof(*view),
873 VK_OBJECT_TYPE_BUFFER_VIEW);
874 if (!view)
875 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
876
877 tu_buffer_view_init(view, device, pCreateInfo);
878
879 *pView = tu_buffer_view_to_handle(view);
880
881 return VK_SUCCESS;
882 }
883
884 void
tu_DestroyBufferView(VkDevice _device,VkBufferView bufferView,const VkAllocationCallbacks * pAllocator)885 tu_DestroyBufferView(VkDevice _device,
886 VkBufferView bufferView,
887 const VkAllocationCallbacks *pAllocator)
888 {
889 TU_FROM_HANDLE(tu_device, device, _device);
890 TU_FROM_HANDLE(tu_buffer_view, view, bufferView);
891
892 if (!view)
893 return;
894
895 vk_object_free(&device->vk, pAllocator, view);
896 }
897