1 /*
2 * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
3 * Copyright 2018 Advanced Micro Devices, Inc.
4 *
5 * SPDX-License-Identifier: MIT
6 */
7
8 #include "drm-uapi/drm_fourcc.h"
9 #include "si_pipe.h"
10 #include "si_query.h"
11 #include "sid.h"
12 #include "frontend/drm_driver.h"
13 #include "util/format/u_format.h"
14 #include "util/os_time.h"
15 #include "util/u_log.h"
16 #include "util/u_memory.h"
17 #include "util/u_pack_color.h"
18 #include "util/u_resource.h"
19 #include "util/u_surface.h"
20 #include "util/u_transfer.h"
21
22 #include <errno.h>
23 #include <inttypes.h>
24
25 #include "amd/addrlib/inc/addrinterface.h"
26 #include "ac_formats.h"
27
28 static enum radeon_surf_mode si_choose_tiling(struct si_screen *sscreen,
29 const struct pipe_resource *templ,
30 bool tc_compatible_htile);
31
32 static bool si_texture_is_aux_plane(const struct pipe_resource *resource);
33
34 /* Same as resource_copy_region, except that both upsampling and downsampling are allowed. */
si_copy_region_with_blit(struct pipe_context * pipe,struct pipe_resource * dst,unsigned dst_level,unsigned dst_sample,unsigned dstx,unsigned dsty,unsigned dstz,struct pipe_resource * src,unsigned src_level,const struct pipe_box * src_box)35 static void si_copy_region_with_blit(struct pipe_context *pipe, struct pipe_resource *dst,
36 unsigned dst_level, unsigned dst_sample, unsigned dstx, unsigned dsty,
37 unsigned dstz, struct pipe_resource *src, unsigned src_level,
38 const struct pipe_box *src_box)
39 {
40 struct pipe_blit_info blit;
41
42 memset(&blit, 0, sizeof(blit));
43 blit.src.resource = src;
44 blit.src.format = src->format;
45 blit.src.level = src_level;
46 blit.src.box = *src_box;
47 blit.dst.resource = dst;
48 blit.dst.format = dst->format;
49 blit.dst.level = dst_level;
50 blit.dst.box.x = dstx;
51 blit.dst.box.y = dsty;
52 blit.dst.box.z = dstz;
53 blit.dst.box.width = src_box->width;
54 blit.dst.box.height = src_box->height;
55 blit.dst.box.depth = src_box->depth;
56 blit.mask = util_format_get_mask(dst->format);
57 blit.filter = PIPE_TEX_FILTER_NEAREST;
58 blit.dst_sample = dst_sample;
59
60 if (blit.mask) {
61 /* Only the gfx blit handles dst_sample. */
62 if (dst_sample)
63 si_gfx_blit(pipe, &blit);
64 else
65 pipe->blit(pipe, &blit);
66 }
67 }
68
69 /* Copy all planes of multi-plane texture */
si_copy_multi_plane_texture(struct pipe_context * ctx,struct pipe_resource * dst,unsigned dst_level,unsigned dstx,unsigned dsty,unsigned dstz,struct pipe_resource * src,unsigned src_level,const struct pipe_box * src_box)70 static bool si_copy_multi_plane_texture(struct pipe_context *ctx, struct pipe_resource *dst,
71 unsigned dst_level, unsigned dstx, unsigned dsty, unsigned dstz,
72 struct pipe_resource *src, unsigned src_level,
73 const struct pipe_box *src_box)
74 {
75 unsigned i, dx, dy;
76 struct si_texture *src_tex = (struct si_texture *)src;
77 struct si_texture *dst_tex = (struct si_texture *)dst;
78 struct pipe_box sbox;
79
80 if (src_tex->multi_plane_format == PIPE_FORMAT_NONE || src_tex->plane_index != 0)
81 return false;
82
83 assert(src_tex->multi_plane_format == dst_tex->multi_plane_format);
84 assert(dst_tex->plane_index == 0 && src_tex->num_planes == dst_tex->num_planes);
85
86 sbox = *src_box;
87
88 for (i = 0; i < src_tex->num_planes && src && dst; ++i) {
89 dx = util_format_get_plane_width(src_tex->multi_plane_format, i, dstx);
90 dy = util_format_get_plane_height(src_tex->multi_plane_format, i, dsty);
91 sbox.x = util_format_get_plane_width(src_tex->multi_plane_format, i, src_box->x);
92 sbox.y = util_format_get_plane_height(src_tex->multi_plane_format, i, src_box->y);
93 sbox.width = util_format_get_plane_width(src_tex->multi_plane_format, i, src_box->width);
94 sbox.height = util_format_get_plane_height(src_tex->multi_plane_format, i, src_box->height);
95
96 si_resource_copy_region(ctx, dst, dst_level, dx, dy, dstz, src, src_level, &sbox);
97
98 src = src->next;
99 dst = dst->next;
100 }
101
102 return true;
103 }
104
105 /* Copy from a full GPU texture to a transfer's staging one. */
si_copy_to_staging_texture(struct pipe_context * ctx,struct si_transfer * stransfer)106 static void si_copy_to_staging_texture(struct pipe_context *ctx, struct si_transfer *stransfer)
107 {
108 struct pipe_transfer *transfer = (struct pipe_transfer *)stransfer;
109 struct pipe_resource *dst = &stransfer->staging->b.b;
110 struct pipe_resource *src = transfer->resource;
111 /* level means sample_index - 1 with MSAA. Used by texture uploads. */
112 unsigned src_level = src->nr_samples > 1 ? 0 : transfer->level;
113
114 if (src->nr_samples > 1 || ((struct si_texture *)src)->is_depth) {
115 si_copy_region_with_blit(ctx, dst, 0, 0, 0, 0, 0, src, src_level, &transfer->box);
116 return;
117 }
118
119 if (si_copy_multi_plane_texture(ctx, dst, 0, 0, 0, 0, src, src_level, &transfer->box))
120 return;
121
122 si_resource_copy_region(ctx, dst, 0, 0, 0, 0, src, src_level, &transfer->box);
123 }
124
125 /* Copy from a transfer's staging texture to a full GPU one. */
si_copy_from_staging_texture(struct pipe_context * ctx,struct si_transfer * stransfer)126 static void si_copy_from_staging_texture(struct pipe_context *ctx, struct si_transfer *stransfer)
127 {
128 struct pipe_transfer *transfer = (struct pipe_transfer *)stransfer;
129 struct pipe_resource *dst = transfer->resource;
130 struct pipe_resource *src = &stransfer->staging->b.b;
131 struct pipe_box sbox;
132
133 u_box_3d(0, 0, 0, transfer->box.width, transfer->box.height, transfer->box.depth, &sbox);
134
135 if (dst->nr_samples > 1 || ((struct si_texture *)dst)->is_depth) {
136 unsigned dst_level = dst->nr_samples > 1 ? 0 : transfer->level;
137 unsigned dst_sample = dst->nr_samples > 1 ? transfer->level : 0;
138
139 si_copy_region_with_blit(ctx, dst, dst_level, dst_sample, transfer->box.x, transfer->box.y,
140 transfer->box.z, src, 0, &sbox);
141 return;
142 }
143
144 if (si_copy_multi_plane_texture(ctx, dst, transfer->level, transfer->box.x, transfer->box.y,
145 transfer->box.z, src, 0, &sbox))
146 return;
147
148 if (util_format_is_compressed(dst->format)) {
149 sbox.width = util_format_get_nblocksx(dst->format, sbox.width);
150 sbox.height = util_format_get_nblocksx(dst->format, sbox.height);
151 }
152
153 si_resource_copy_region(ctx, dst, transfer->level, transfer->box.x, transfer->box.y,
154 transfer->box.z, src, 0, &sbox);
155 }
156
si_texture_get_offset(struct si_screen * sscreen,struct si_texture * tex,unsigned level,const struct pipe_box * box,unsigned * stride,uintptr_t * layer_stride)157 static uint64_t si_texture_get_offset(struct si_screen *sscreen, struct si_texture *tex,
158 unsigned level, const struct pipe_box *box, unsigned *stride,
159 uintptr_t *layer_stride)
160 {
161 if (sscreen->info.gfx_level >= GFX9) {
162 unsigned pitch;
163 if (tex->surface.is_linear) {
164 pitch = tex->surface.u.gfx9.pitch[level];
165 } else {
166 pitch = tex->surface.u.gfx9.surf_pitch;
167 }
168
169 *stride = pitch * tex->surface.bpe;
170 *layer_stride = tex->surface.u.gfx9.surf_slice_size;
171
172 if (!box)
173 return 0;
174
175 /* Each texture is an array of slices. Each slice is an array
176 * of mipmap levels. */
177 return tex->surface.u.gfx9.surf_offset + box->z * tex->surface.u.gfx9.surf_slice_size +
178 tex->surface.u.gfx9.offset[level] +
179 (box->y / tex->surface.blk_h * (uint64_t)pitch + box->x / tex->surface.blk_w) *
180 tex->surface.bpe;
181 } else {
182 *stride = tex->surface.u.legacy.level[level].nblk_x * tex->surface.bpe;
183 assert((uint64_t)tex->surface.u.legacy.level[level].slice_size_dw * 4 <= UINT_MAX);
184 *layer_stride = (uint64_t)tex->surface.u.legacy.level[level].slice_size_dw * 4;
185
186 if (!box)
187 return (uint64_t)tex->surface.u.legacy.level[level].offset_256B * 256;
188
189 /* Each texture is an array of mipmap levels. Each level is
190 * an array of slices. */
191 return (uint64_t)tex->surface.u.legacy.level[level].offset_256B * 256 +
192 box->z * (uint64_t)tex->surface.u.legacy.level[level].slice_size_dw * 4 +
193 (box->y / tex->surface.blk_h * tex->surface.u.legacy.level[level].nblk_x +
194 box->x / tex->surface.blk_w) *
195 tex->surface.bpe;
196 }
197 }
198
si_init_surface(struct si_screen * sscreen,struct radeon_surf * surface,const struct pipe_resource * ptex,enum radeon_surf_mode array_mode,uint64_t modifier,bool is_imported,bool is_scanout,bool is_flushed_depth,bool tc_compatible_htile)199 static int si_init_surface(struct si_screen *sscreen, struct radeon_surf *surface,
200 const struct pipe_resource *ptex, enum radeon_surf_mode array_mode,
201 uint64_t modifier, bool is_imported, bool is_scanout,
202 bool is_flushed_depth, bool tc_compatible_htile)
203 {
204 const struct util_format_description *desc = util_format_description(ptex->format);
205 bool is_depth = util_format_has_depth(desc);
206 bool is_stencil = util_format_has_stencil(desc);
207 int r;
208 unsigned bpe;
209 uint64_t flags = 0;
210
211 if (!is_flushed_depth && ptex->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) {
212 bpe = 4; /* stencil is allocated separately */
213 } else {
214 bpe = util_format_get_blocksize(ptex->format);
215 assert(util_is_power_of_two_or_zero(bpe));
216 }
217
218 if (sscreen->info.gfx_level >= GFX12) {
219 if (!is_flushed_depth && is_depth) {
220 flags |= RADEON_SURF_ZBUFFER;
221 if (is_stencil)
222 flags |= RADEON_SURF_SBUFFER;
223
224 if (sscreen->debug_flags & DBG(NO_HYPERZ) ||
225 ptex->flags & PIPE_RESOURCE_FLAG_SPARSE)
226 flags |= RADEON_SURF_NO_HTILE;
227 }
228
229 /* The kernel code translating tiling flags into a modifier was wrong
230 * until .58, so don't set these attributes for older versions.
231 */
232 bool supports_display_dcc = sscreen->info.drm_minor >= 58;
233 if (!is_imported && (!(ptex->bind & PIPE_BIND_SCANOUT) || supports_display_dcc)) {
234 enum pipe_format format = util_format_get_depth_only(ptex->format);
235
236 /* These should be set for both color and Z/S. */
237 surface->u.gfx9.color.dcc_number_type = ac_get_cb_number_type(format);
238 surface->u.gfx9.color.dcc_data_format = ac_get_cb_format(sscreen->info.gfx_level, format);
239 }
240
241 if (modifier == DRM_FORMAT_MOD_INVALID &&
242 (ptex->bind & PIPE_BIND_CONST_BW ||
243 ptex->bind & PIPE_BIND_PROTECTED ||
244 ptex->bind & PIPE_BIND_USE_FRONT_RENDERING ||
245 sscreen->debug_flags & DBG(NO_DCC) ||
246 (ptex->bind & PIPE_BIND_SCANOUT && sscreen->debug_flags & DBG(NO_DISPLAY_DCC))))
247 flags |= RADEON_SURF_DISABLE_DCC;
248 } else {
249 /* Gfx6-11 */
250 if (!is_flushed_depth && is_depth) {
251 flags |= RADEON_SURF_ZBUFFER;
252
253 if ((sscreen->debug_flags & DBG(NO_HYPERZ)) ||
254 (ptex->bind & PIPE_BIND_SHARED) || is_imported) {
255 flags |= RADEON_SURF_NO_HTILE;
256 } else if (tc_compatible_htile &&
257 (sscreen->info.gfx_level >= GFX9 || array_mode == RADEON_SURF_MODE_2D)) {
258 /* TC-compatible HTILE only supports Z32_FLOAT.
259 * GFX9 also supports Z16_UNORM.
260 * On GFX8, promote Z16 to Z32. DB->CB copies will convert
261 * the format for transfers.
262 */
263 if (sscreen->info.gfx_level == GFX8)
264 bpe = 4;
265
266 flags |= RADEON_SURF_TC_COMPATIBLE_HTILE;
267 }
268
269 if (is_stencil)
270 flags |= RADEON_SURF_SBUFFER;
271 }
272
273 /* Disable DCC? (it can't be disabled if modifiers are used) */
274 if (sscreen->info.gfx_level >= GFX8 && modifier == DRM_FORMAT_MOD_INVALID && !is_imported) {
275 /* Global options that disable DCC. */
276 if (ptex->nr_samples >= 2 && sscreen->debug_flags & DBG(NO_DCC_MSAA))
277 flags |= RADEON_SURF_DISABLE_DCC;
278
279 /* Shared textures must always set up DCC. If it's not present, it will be disabled by
280 * si_get_opaque_metadata later.
281 */
282 if (!is_imported && sscreen->debug_flags & DBG(NO_DCC))
283 flags |= RADEON_SURF_DISABLE_DCC;
284
285 /* R9G9B9E5 isn't supported for rendering by older generations. */
286 if (sscreen->info.gfx_level < GFX10_3 &&
287 ptex->format == PIPE_FORMAT_R9G9B9E5_FLOAT)
288 flags |= RADEON_SURF_DISABLE_DCC;
289
290 /* If constant (non-data-dependent) format is requested, disable DCC: */
291 if (ptex->bind & PIPE_BIND_CONST_BW)
292 flags |= RADEON_SURF_DISABLE_DCC;
293
294 if (ptex->bind & PIPE_BIND_USE_FRONT_RENDERING)
295 flags |= RADEON_SURF_DISABLE_DCC;
296
297 switch (sscreen->info.gfx_level) {
298 case GFX8:
299 /* Stoney: 128bpp MSAA textures randomly fail piglit tests with DCC. */
300 if (sscreen->info.family == CHIP_STONEY && bpe == 16 && ptex->nr_samples >= 2)
301 flags |= RADEON_SURF_DISABLE_DCC;
302
303 /* DCC clear for 4x and 8x MSAA array textures unimplemented. */
304 if (ptex->nr_storage_samples >= 4 && ptex->array_size > 1)
305 flags |= RADEON_SURF_DISABLE_DCC;
306 break;
307
308 case GFX9:
309 /* DCC MSAA fails this on Raven:
310 * https://www.khronos.org/registry/webgl/sdk/tests/deqp/functional/gles3/fbomultisample.2_samples.html
311 * and this on Picasso:
312 * https://www.khronos.org/registry/webgl/sdk/tests/deqp/functional/gles3/fbomultisample.4_samples.html
313 */
314 if (sscreen->info.family == CHIP_RAVEN && ptex->nr_storage_samples >= 2 && bpe < 4)
315 flags |= RADEON_SURF_DISABLE_DCC;
316
317 /* Vega10 fails these 2x and 4x MSAA tests with DCC:
318 * piglit/bin/ext_framebuffer_multisample-formats 2 GL_EXT_texture_snorm
319 * piglit/bin/ext_framebuffer_multisample-formats 4 GL_EXT_texture_snorm
320 */
321 if ((ptex->nr_storage_samples == 2 || ptex->nr_storage_samples == 4) && bpe <= 2 &&
322 util_format_is_snorm(ptex->format))
323 flags |= RADEON_SURF_DISABLE_DCC;
324
325 /* Vega10 fails these MSAA tests with DCC:
326 * piglit/bin/ext_framebuffer_multisample-formats 2 GL_ARB_texture_float
327 * piglit/bin/ext_framebuffer_multisample-formats 2 GL_ARB_texture_rg-float
328 */
329 if (ptex->nr_storage_samples == 2 && bpe == 2 && util_format_is_float(ptex->format))
330 flags |= RADEON_SURF_DISABLE_DCC;
331
332 /* We allow S8_UINT as a color format, and piglit/draw-pixels fails if we enable DCC. */
333 if (ptex->format == PIPE_FORMAT_S8_UINT)
334 flags |= RADEON_SURF_DISABLE_DCC;
335 break;
336
337 case GFX10:
338 case GFX10_3:
339 if (ptex->nr_storage_samples >= 2 && !sscreen->options.dcc_msaa)
340 flags |= RADEON_SURF_DISABLE_DCC;
341
342 /* Navi10 fails these MSAA tests with DCC:
343 * piglit/bin/arb_sample_shading-samplemask 2 all all
344 * piglit/bin/arb_sample_shading-samplemask 4 all all
345 * piglit/bin/ext_framebuffer_multisample-formats 2 GL_ARB_texture_float
346 * piglit/bin/ext_framebuffer_multisample-formats 2 GL_EXT_texture_integer
347 */
348 if (sscreen->info.gfx_level == GFX10 &&
349 (ptex->nr_storage_samples == 2 || ptex->nr_storage_samples == 4))
350 flags |= RADEON_SURF_DISABLE_DCC;
351 break;
352
353 case GFX11:
354 case GFX11_5:
355 break;
356
357 default:
358 assert(0);
359 }
360 }
361
362 if (sscreen->debug_flags & DBG(NO_FMASK))
363 flags |= RADEON_SURF_NO_FMASK;
364
365 if (ptex->flags & PIPE_RESOURCE_FLAG_SPARSE) {
366 flags |= RADEON_SURF_NO_FMASK |
367 RADEON_SURF_NO_HTILE |
368 RADEON_SURF_DISABLE_DCC;
369 }
370 }
371
372 if (is_scanout) {
373 /* This should catch bugs in gallium users setting incorrect flags. */
374 assert(ptex->nr_samples <= 1 && ptex->depth0 == 1 &&
375 ptex->last_level == 0 && !(flags & RADEON_SURF_Z_OR_SBUFFER));
376
377 flags |= RADEON_SURF_SCANOUT;
378 }
379
380 if (ptex->bind & PIPE_BIND_SHARED)
381 flags |= RADEON_SURF_SHAREABLE;
382
383 if (is_imported)
384 flags |= RADEON_SURF_IMPORTED | RADEON_SURF_SHAREABLE;
385
386 if (ptex->flags & PIPE_RESOURCE_FLAG_SPARSE)
387 flags |= RADEON_SURF_PRT;
388
389 if (ptex->bind & (PIPE_BIND_VIDEO_DECODE_DPB | PIPE_BIND_VIDEO_ENCODE_DPB))
390 flags |= RADEON_SURF_VIDEO_REFERENCE;
391
392 surface->modifier = modifier;
393
394 r = sscreen->ws->surface_init(sscreen->ws, &sscreen->info, ptex, flags, bpe, array_mode,
395 surface);
396 if (r) {
397 return r;
398 }
399
400 return 0;
401 }
402
si_eliminate_fast_color_clear(struct si_context * sctx,struct si_texture * tex,bool * ctx_flushed)403 void si_eliminate_fast_color_clear(struct si_context *sctx, struct si_texture *tex,
404 bool *ctx_flushed)
405 {
406 struct pipe_context *ctx = &sctx->b;
407
408 unsigned n = sctx->num_decompress_calls;
409 ctx->flush_resource(ctx, &tex->buffer.b.b);
410
411 /* Flush only if any fast clear elimination took place. */
412 bool flushed = false;
413 if (n != sctx->num_decompress_calls)
414 {
415 ctx->flush(ctx, NULL, 0);
416 flushed = true;
417 }
418 if (ctx_flushed)
419 *ctx_flushed = flushed;
420 }
421
si_texture_discard_cmask(struct si_screen * sscreen,struct si_texture * tex)422 void si_texture_discard_cmask(struct si_screen *sscreen, struct si_texture *tex)
423 {
424 if (!tex->cmask_buffer)
425 return;
426
427 assert(tex->buffer.b.b.nr_samples <= 1);
428
429 /* Disable CMASK. */
430 tex->cmask_base_address_reg = tex->buffer.gpu_address >> 8;
431 tex->dirty_level_mask = 0;
432
433 tex->cb_color_info &= ~S_028C70_FAST_CLEAR(1);
434
435 if (tex->cmask_buffer != &tex->buffer)
436 si_resource_reference(&tex->cmask_buffer, NULL);
437
438 tex->cmask_buffer = NULL;
439
440 /* Notify all contexts about the change. */
441 p_atomic_inc(&sscreen->dirty_tex_counter);
442 p_atomic_inc(&sscreen->compressed_colortex_counter);
443 }
444
si_can_disable_dcc(struct si_texture * tex)445 static bool si_can_disable_dcc(struct si_texture *tex)
446 {
447 /* We can't disable DCC if it can be written by another process. */
448 return !tex->is_depth &&
449 tex->surface.meta_offset &&
450 (!tex->buffer.b.is_shared ||
451 !(tex->buffer.external_usage & PIPE_HANDLE_USAGE_FRAMEBUFFER_WRITE)) &&
452 !ac_modifier_has_dcc(tex->surface.modifier);
453 }
454
si_texture_discard_dcc(struct si_screen * sscreen,struct si_texture * tex)455 static bool si_texture_discard_dcc(struct si_screen *sscreen, struct si_texture *tex)
456 {
457 if (!si_can_disable_dcc(tex))
458 return false;
459
460 /* Disable DCC. */
461 ac_surface_zero_dcc_fields(&tex->surface);
462
463 /* Notify all contexts about the change. */
464 p_atomic_inc(&sscreen->dirty_tex_counter);
465 return true;
466 }
467
468 /**
469 * Disable DCC for the texture. (first decompress, then discard metadata).
470 *
471 * There is unresolved multi-context synchronization issue between
472 * screen::aux_context and the current context. If applications do this with
473 * multiple contexts, it's already undefined behavior for them and we don't
474 * have to worry about that. The scenario is:
475 *
476 * If context 1 disables DCC and context 2 has queued commands that write
477 * to the texture via CB with DCC enabled, and the order of operations is
478 * as follows:
479 * context 2 queues draw calls rendering to the texture, but doesn't flush
480 * context 1 disables DCC and flushes
481 * context 1 & 2 reset descriptors and FB state
482 * context 2 flushes (new compressed tiles written by the draw calls)
483 * context 1 & 2 read garbage, because DCC is disabled, yet there are
484 * compressed tiled
485 *
486 * \param sctx the current context if you have one, or sscreen->aux_context
487 * if you don't.
488 */
si_texture_disable_dcc(struct si_context * sctx,struct si_texture * tex)489 bool si_texture_disable_dcc(struct si_context *sctx, struct si_texture *tex)
490 {
491 struct si_screen *sscreen = sctx->screen;
492
493 if (!sctx->has_graphics)
494 return si_texture_discard_dcc(sscreen, tex);
495
496 if (!si_can_disable_dcc(tex))
497 return false;
498
499 /* Decompress DCC. */
500 si_decompress_dcc(sctx, tex);
501 sctx->b.flush(&sctx->b, NULL, 0);
502
503 return si_texture_discard_dcc(sscreen, tex);
504 }
505
si_reallocate_texture_inplace(struct si_context * sctx,struct si_texture * tex,unsigned new_bind_flag,bool invalidate_storage)506 static void si_reallocate_texture_inplace(struct si_context *sctx, struct si_texture *tex,
507 unsigned new_bind_flag, bool invalidate_storage)
508 {
509 struct pipe_screen *screen = sctx->b.screen;
510 struct si_texture *new_tex;
511 struct pipe_resource templ = tex->buffer.b.b;
512 unsigned i;
513
514 templ.bind |= new_bind_flag;
515
516 if (tex->buffer.b.is_shared || tex->num_planes > 1)
517 return;
518
519 if (new_bind_flag == PIPE_BIND_LINEAR) {
520 if (tex->surface.is_linear)
521 return;
522
523 /* This fails with MSAA, depth, and compressed textures. */
524 if (si_choose_tiling(sctx->screen, &templ, false) != RADEON_SURF_MODE_LINEAR_ALIGNED)
525 return;
526 }
527
528 /* Inherit the modifier from the old texture. */
529 if (tex->surface.modifier != DRM_FORMAT_MOD_INVALID && screen->resource_create_with_modifiers)
530 new_tex = (struct si_texture *)screen->resource_create_with_modifiers(screen, &templ,
531 &tex->surface.modifier, 1);
532 else
533 new_tex = (struct si_texture *)screen->resource_create(screen, &templ);
534
535 if (!new_tex)
536 return;
537
538 /* Copy the pixels to the new texture. */
539 if (!invalidate_storage) {
540 for (i = 0; i <= templ.last_level; i++) {
541 struct pipe_box box;
542
543 u_box_3d(0, 0, 0, u_minify(templ.width0, i), u_minify(templ.height0, i),
544 util_num_layers(&templ, i), &box);
545
546 si_resource_copy_region(&sctx->b, &new_tex->buffer.b.b,
547 i, 0, 0, 0, &tex->buffer.b.b, i, &box);
548 }
549 }
550
551 if (new_bind_flag == PIPE_BIND_LINEAR) {
552 si_texture_discard_cmask(sctx->screen, tex);
553 si_texture_discard_dcc(sctx->screen, tex);
554 }
555
556 /* Replace the structure fields of tex. */
557 tex->buffer.b.b.bind = templ.bind;
558 radeon_bo_reference(sctx->screen->ws, &tex->buffer.buf, new_tex->buffer.buf);
559 tex->buffer.gpu_address = new_tex->buffer.gpu_address;
560 tex->buffer.bo_size = new_tex->buffer.bo_size;
561 tex->buffer.bo_alignment_log2 = new_tex->buffer.bo_alignment_log2;
562 tex->buffer.domains = new_tex->buffer.domains;
563 tex->buffer.flags = new_tex->buffer.flags;
564
565 tex->surface = new_tex->surface;
566 si_texture_reference(&tex->flushed_depth_texture, new_tex->flushed_depth_texture);
567
568 tex->surface.fmask_offset = new_tex->surface.fmask_offset;
569 tex->surface.cmask_offset = new_tex->surface.cmask_offset;
570 tex->cmask_base_address_reg = new_tex->cmask_base_address_reg;
571
572 if (tex->cmask_buffer == &tex->buffer)
573 tex->cmask_buffer = NULL;
574 else
575 si_resource_reference(&tex->cmask_buffer, NULL);
576
577 if (new_tex->cmask_buffer == &new_tex->buffer)
578 tex->cmask_buffer = &tex->buffer;
579 else
580 si_resource_reference(&tex->cmask_buffer, new_tex->cmask_buffer);
581
582 tex->surface.meta_offset = new_tex->surface.meta_offset;
583 tex->cb_color_info = new_tex->cb_color_info;
584 memcpy(tex->color_clear_value, new_tex->color_clear_value, sizeof(tex->color_clear_value));
585 tex->last_msaa_resolve_target_micro_mode = new_tex->last_msaa_resolve_target_micro_mode;
586
587 memcpy(tex->depth_clear_value, new_tex->depth_clear_value, sizeof(tex->depth_clear_value));
588 tex->dirty_level_mask = new_tex->dirty_level_mask;
589 tex->stencil_dirty_level_mask = new_tex->stencil_dirty_level_mask;
590 tex->db_render_format = new_tex->db_render_format;
591 memcpy(tex->stencil_clear_value, new_tex->stencil_clear_value, sizeof(tex->stencil_clear_value));
592 tex->tc_compatible_htile = new_tex->tc_compatible_htile;
593 tex->depth_cleared_level_mask_once = new_tex->depth_cleared_level_mask_once;
594 tex->stencil_cleared_level_mask_once = new_tex->stencil_cleared_level_mask_once;
595 tex->upgraded_depth = new_tex->upgraded_depth;
596 tex->db_compatible = new_tex->db_compatible;
597 tex->can_sample_z = new_tex->can_sample_z;
598 tex->can_sample_s = new_tex->can_sample_s;
599
600 tex->displayable_dcc_dirty = new_tex->displayable_dcc_dirty;
601
602 if (new_bind_flag == PIPE_BIND_LINEAR) {
603 assert(!tex->surface.meta_offset);
604 assert(!tex->cmask_buffer);
605 assert(!tex->surface.fmask_size);
606 assert(!tex->is_depth);
607 }
608
609 si_texture_reference(&new_tex, NULL);
610
611 p_atomic_inc(&sctx->screen->dirty_tex_counter);
612 }
613
si_set_tex_bo_metadata(struct si_screen * sscreen,struct si_texture * tex)614 static void si_set_tex_bo_metadata(struct si_screen *sscreen, struct si_texture *tex)
615 {
616 struct pipe_resource *res = &tex->buffer.b.b;
617 struct radeon_bo_metadata md;
618
619 memset(&md, 0, sizeof(md));
620
621 assert(tex->surface.fmask_size == 0);
622
623 static const unsigned char swizzle[] = {PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y, PIPE_SWIZZLE_Z,
624 PIPE_SWIZZLE_W};
625 bool is_array = util_texture_is_array(res->target);
626 uint32_t desc[8];
627
628 si_make_texture_descriptor(sscreen, tex, true, res->target,
629 tex->is_depth ? tex->db_render_format : res->format, swizzle, 0,
630 res->last_level, 0, is_array ? res->array_size - 1 : 0, res->width0,
631 res->height0, res->depth0, true, desc, NULL);
632 si_set_mutable_tex_desc_fields(sscreen, tex, &tex->surface.u.legacy.level[0], 0, 0,
633 tex->surface.blk_w, false, 0, desc);
634
635 ac_surface_compute_umd_metadata(&sscreen->info, &tex->surface,
636 tex->buffer.b.b.last_level + 1,
637 desc, &md.size_metadata, md.metadata,
638 sscreen->debug_flags & DBG(EXTRA_METADATA));
639 sscreen->ws->buffer_set_metadata(sscreen->ws, tex->buffer.buf, &md, &tex->surface);
640 }
641
si_displayable_dcc_needs_explicit_flush(struct si_texture * tex)642 static bool si_displayable_dcc_needs_explicit_flush(struct si_texture *tex)
643 {
644 struct si_screen *sscreen = (struct si_screen *)tex->buffer.b.b.screen;
645
646 if (sscreen->info.gfx_level <= GFX8)
647 return false;
648
649 /* With modifiers and > 1 planes any applications will know that they
650 * cannot do frontbuffer rendering with the texture. */
651 if (ac_surface_get_nplanes(&tex->surface) > 1)
652 return false;
653
654 return tex->surface.is_displayable && tex->surface.meta_offset;
655 }
656
si_resource_get_param(struct pipe_screen * screen,struct pipe_context * context,struct pipe_resource * resource,unsigned plane,unsigned layer,unsigned level,enum pipe_resource_param param,unsigned handle_usage,uint64_t * value)657 static bool si_resource_get_param(struct pipe_screen *screen, struct pipe_context *context,
658 struct pipe_resource *resource, unsigned plane, unsigned layer,
659 unsigned level,
660 enum pipe_resource_param param, unsigned handle_usage,
661 uint64_t *value)
662 {
663 while (plane && resource->next && !si_texture_is_aux_plane(resource->next)) {
664 --plane;
665 resource = resource->next;
666 }
667
668 struct si_screen *sscreen = (struct si_screen *)screen;
669 struct si_texture *tex = (struct si_texture *)resource;
670 struct winsys_handle whandle;
671
672 switch (param) {
673 case PIPE_RESOURCE_PARAM_NPLANES:
674 if (resource->target == PIPE_BUFFER)
675 *value = 1;
676 else if (tex->num_planes > 1)
677 *value = tex->num_planes;
678 else
679 *value = ac_surface_get_nplanes(&tex->surface);
680 return true;
681
682 case PIPE_RESOURCE_PARAM_STRIDE:
683 if (resource->target == PIPE_BUFFER)
684 *value = 0;
685 else
686 *value = ac_surface_get_plane_stride(sscreen->info.gfx_level,
687 &tex->surface, plane, level);
688 return true;
689
690 case PIPE_RESOURCE_PARAM_OFFSET:
691 if (resource->target == PIPE_BUFFER) {
692 *value = 0;
693 } else {
694 uint64_t level_offset = 0;
695 if (sscreen->info.gfx_level >= GFX9 && tex->surface.is_linear)
696 level_offset = tex->surface.u.gfx9.offset[level];
697 *value = ac_surface_get_plane_offset(sscreen->info.gfx_level,
698 &tex->surface, plane, layer) + level_offset;
699 }
700 return true;
701
702 case PIPE_RESOURCE_PARAM_MODIFIER:
703 *value = tex->surface.modifier;
704 return true;
705
706 case PIPE_RESOURCE_PARAM_HANDLE_TYPE_SHARED:
707 case PIPE_RESOURCE_PARAM_HANDLE_TYPE_KMS:
708 case PIPE_RESOURCE_PARAM_HANDLE_TYPE_FD:
709 memset(&whandle, 0, sizeof(whandle));
710
711 if (param == PIPE_RESOURCE_PARAM_HANDLE_TYPE_SHARED)
712 whandle.type = WINSYS_HANDLE_TYPE_SHARED;
713 else if (param == PIPE_RESOURCE_PARAM_HANDLE_TYPE_KMS)
714 whandle.type = WINSYS_HANDLE_TYPE_KMS;
715 else if (param == PIPE_RESOURCE_PARAM_HANDLE_TYPE_FD)
716 whandle.type = WINSYS_HANDLE_TYPE_FD;
717
718 if (!screen->resource_get_handle(screen, context, resource, &whandle, handle_usage))
719 return false;
720
721 *value = whandle.handle;
722 return true;
723 case PIPE_RESOURCE_PARAM_LAYER_STRIDE:
724 break;
725 }
726 return false;
727 }
728
si_texture_get_info(struct pipe_screen * screen,struct pipe_resource * resource,unsigned * pstride,unsigned * poffset)729 static void si_texture_get_info(struct pipe_screen *screen, struct pipe_resource *resource,
730 unsigned *pstride, unsigned *poffset)
731 {
732 uint64_t value;
733
734 if (pstride) {
735 si_resource_get_param(screen, NULL, resource, 0, 0, 0, PIPE_RESOURCE_PARAM_STRIDE, 0, &value);
736 *pstride = value;
737 }
738
739 if (poffset) {
740 si_resource_get_param(screen, NULL, resource, 0, 0, 0, PIPE_RESOURCE_PARAM_OFFSET, 0, &value);
741 *poffset = value;
742 }
743 }
744
si_texture_get_handle(struct pipe_screen * screen,struct pipe_context * ctx,struct pipe_resource * resource,struct winsys_handle * whandle,unsigned usage)745 static bool si_texture_get_handle(struct pipe_screen *screen, struct pipe_context *ctx,
746 struct pipe_resource *resource, struct winsys_handle *whandle,
747 unsigned usage)
748 {
749 struct si_screen *sscreen = (struct si_screen *)screen;
750 struct si_context *sctx;
751 struct si_resource *res = si_resource(resource);
752 struct si_texture *tex = (struct si_texture *)resource;
753 bool update_metadata = false;
754 unsigned stride, offset, slice_size;
755 uint64_t modifier = DRM_FORMAT_MOD_INVALID;
756 bool flush = false;
757
758 ctx = threaded_context_unwrap_sync(ctx);
759 sctx = ctx ? (struct si_context *)ctx : si_get_aux_context(&sscreen->aux_context.general);
760
761 if (resource->target != PIPE_BUFFER) {
762 unsigned plane = whandle->plane;
763
764 /* Individual planes are chained pipe_resource instances. */
765 while (plane && resource->next && !si_texture_is_aux_plane(resource->next)) {
766 resource = resource->next;
767 --plane;
768 }
769
770 res = si_resource(resource);
771 tex = (struct si_texture *)resource;
772
773 /* This is not supported now, but it might be required for OpenCL
774 * interop in the future.
775 */
776 if (resource->nr_samples > 1 || tex->is_depth) {
777 if (!ctx)
778 si_put_aux_context_flush(&sscreen->aux_context.general);
779 return false;
780 }
781
782 whandle->size = tex->buffer.bo_size;
783
784 if (plane) {
785 if (!ctx)
786 si_put_aux_context_flush(&sscreen->aux_context.general);
787 whandle->offset = ac_surface_get_plane_offset(sscreen->info.gfx_level,
788 &tex->surface, plane, 0);
789 whandle->stride = ac_surface_get_plane_stride(sscreen->info.gfx_level,
790 &tex->surface, plane, 0);
791 whandle->modifier = tex->surface.modifier;
792 return sscreen->ws->buffer_get_handle(sscreen->ws, res->buf, whandle);
793 }
794
795 /* Move a suballocated texture into a non-suballocated allocation. */
796 if (sscreen->ws->buffer_is_suballocated(res->buf) || tex->surface.tile_swizzle ||
797 (tex->buffer.flags & RADEON_FLAG_NO_INTERPROCESS_SHARING &&
798 sscreen->info.has_local_buffers)) {
799 assert(!res->b.is_shared);
800 si_reallocate_texture_inplace(sctx, tex, PIPE_BIND_SHARED, false);
801 flush = true;
802 assert(res->b.b.bind & PIPE_BIND_SHARED);
803 assert(res->flags & RADEON_FLAG_NO_SUBALLOC);
804 assert(!(res->flags & RADEON_FLAG_NO_INTERPROCESS_SHARING));
805 assert(tex->surface.tile_swizzle == 0);
806 }
807
808 const bool debug_disable_dcc = sscreen->debug_flags & DBG(NO_EXPORTED_DCC);
809 /* Since shader image stores don't support DCC on GFX9 and older,
810 * disable it for external clients that want write access.
811 */
812 const bool shader_write = sscreen->info.gfx_level <= GFX9 &&
813 usage & PIPE_HANDLE_USAGE_SHADER_WRITE &&
814 !tex->is_depth &&
815 tex->surface.meta_offset;
816 /* Another reason to disable display dcc is front buffer rendering.
817 * This can happens with Xorg. If the ddx driver uses GBM_BO_USE_FRONT_RENDERING,
818 * there's nothing to do because the texture is not using DCC.
819 * If the flag isn't set, we have to infer it to get correct rendering.
820 */
821 const bool front_buffer_rendering = !(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) &&
822 tex->buffer.b.b.bind & PIPE_BIND_SCANOUT;
823
824 /* If display dcc requires a retiling step, drop dcc. */
825 const bool explicit_flush = !(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) &&
826 si_displayable_dcc_needs_explicit_flush(tex);
827
828 if (debug_disable_dcc || shader_write || front_buffer_rendering || explicit_flush) {
829 if (sscreen->info.gfx_level >= GFX12) {
830 si_reallocate_texture_inplace(sctx, tex, PIPE_BIND_CONST_BW, false);
831 update_metadata = true;
832 } else if (si_texture_disable_dcc(sctx, tex)) {
833 update_metadata = true;
834 /* si_texture_disable_dcc flushes the context */
835 flush = false;
836 }
837 }
838
839 if (!(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) &&
840 (tex->cmask_buffer || (!tex->is_depth && tex->surface.meta_offset))) {
841 /* Eliminate fast clear (both CMASK and DCC) */
842 bool flushed;
843 si_eliminate_fast_color_clear(sctx, tex, &flushed);
844 /* eliminate_fast_color_clear sometimes flushes the context */
845 flush = !flushed;
846
847 /* Disable CMASK if flush_resource isn't going
848 * to be called.
849 */
850 if (tex->cmask_buffer)
851 si_texture_discard_cmask(sscreen, tex);
852 }
853
854 /* Set metadata. */
855 if ((!res->b.is_shared || update_metadata) && whandle->offset == 0)
856 si_set_tex_bo_metadata(sscreen, tex);
857
858 if (sscreen->info.gfx_level >= GFX9) {
859 slice_size = tex->surface.u.gfx9.surf_slice_size;
860 } else {
861 slice_size = (uint64_t)tex->surface.u.legacy.level[0].slice_size_dw * 4;
862 }
863
864 modifier = tex->surface.modifier;
865 } else {
866 tc_buffer_disable_cpu_storage(&res->b.b);
867
868 /* Buffer exports are for the OpenCL interop. */
869 /* Move a suballocated buffer into a non-suballocated allocation. */
870 if (sscreen->ws->buffer_is_suballocated(res->buf) ||
871 /* A DMABUF export always fails if the BO is local. */
872 (tex->buffer.flags & RADEON_FLAG_NO_INTERPROCESS_SHARING &&
873 sscreen->info.has_local_buffers)) {
874 assert(!res->b.is_shared);
875
876 /* Allocate a new buffer with PIPE_BIND_SHARED. */
877 struct pipe_resource templ = res->b.b;
878 templ.bind |= PIPE_BIND_SHARED;
879
880 struct pipe_resource *newb = screen->resource_create(screen, &templ);
881 if (!newb) {
882 if (!ctx)
883 si_put_aux_context_flush(&sscreen->aux_context.general);
884 return false;
885 }
886
887 /* Copy the old buffer contents to the new one. */
888 struct pipe_box box;
889 u_box_1d(0, newb->width0, &box);
890 sctx->b.resource_copy_region(&sctx->b, newb, 0, 0, 0, 0, &res->b.b, 0, &box);
891 flush = true;
892 /* Move the new buffer storage to the old pipe_resource. */
893 si_replace_buffer_storage(&sctx->b, &res->b.b, newb, 0, 0, 0);
894 pipe_resource_reference(&newb, NULL);
895
896 assert(res->b.b.bind & PIPE_BIND_SHARED);
897 assert(res->flags & RADEON_FLAG_NO_SUBALLOC);
898 }
899
900 /* Buffers */
901 slice_size = 0;
902 }
903
904 si_texture_get_info(screen, resource, &stride, &offset);
905
906 if (res->b.is_shared) {
907 /* USAGE_EXPLICIT_FLUSH must be cleared if at least one user
908 * doesn't set it.
909 */
910 res->external_usage |= usage & ~PIPE_HANDLE_USAGE_EXPLICIT_FLUSH;
911 if (!(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH))
912 res->external_usage &= ~PIPE_HANDLE_USAGE_EXPLICIT_FLUSH;
913 } else {
914 res->b.is_shared = true;
915 res->external_usage = usage;
916 }
917
918 if (flush && ctx)
919 sctx->b.flush(&sctx->b, NULL, 0);
920 if (!ctx)
921 si_put_aux_context_flush(&sscreen->aux_context.general);
922
923 whandle->stride = stride;
924 whandle->offset = offset + slice_size * whandle->layer;
925 whandle->modifier = modifier;
926
927 return sscreen->ws->buffer_get_handle(sscreen->ws, res->buf, whandle);
928 }
929
si_print_texture_info(struct si_screen * sscreen,struct si_texture * tex,struct u_log_context * log)930 void si_print_texture_info(struct si_screen *sscreen, struct si_texture *tex,
931 struct u_log_context *log)
932 {
933 int i;
934 FILE *f;
935 char *surf_info = NULL;
936 size_t surf_info_size;
937
938 /* Common parameters. */
939 u_log_printf(log,
940 " Info: npix_x=%u, npix_y=%u, npix_z=%u, "
941 "array_size=%u, last_level=%u, nsamples=%u",
942 tex->buffer.b.b.width0, tex->buffer.b.b.height0,
943 tex->buffer.b.b.depth0, tex->buffer.b.b.array_size,
944 tex->buffer.b.b.last_level, tex->buffer.b.b.nr_samples);
945
946 if (tex->is_depth && tex->surface.meta_offset)
947 u_log_printf(log, ", tc_compatible_htile=%u", tex->tc_compatible_htile);
948
949 u_log_printf(log, ", %s\n",
950 util_format_short_name(tex->buffer.b.b.format));
951
952 f = open_memstream(&surf_info, &surf_info_size);
953 if (!f)
954 return;
955 ac_surface_print_info(f, &sscreen->info, &tex->surface);
956 fclose(f);
957 u_log_printf(log, "%s", surf_info);
958 free(surf_info);
959
960 if (sscreen->info.gfx_level >= GFX9) {
961 return;
962 }
963
964 if (!tex->is_depth && tex->surface.meta_offset) {
965 for (i = 0; i <= tex->buffer.b.b.last_level; i++)
966 u_log_printf(log,
967 " DCCLevel[%i]: enabled=%u, offset=%u, "
968 "fast_clear_size=%u\n",
969 i, i < tex->surface.num_meta_levels, tex->surface.u.legacy.color.dcc_level[i].dcc_offset,
970 tex->surface.u.legacy.color.dcc_level[i].dcc_fast_clear_size);
971 }
972
973 for (i = 0; i <= tex->buffer.b.b.last_level; i++)
974 u_log_printf(log,
975 " Level[%i]: offset=%" PRIu64 ", slice_size=%" PRIu64 ", "
976 "npix_x=%u, npix_y=%u, npix_z=%u, nblk_x=%u, nblk_y=%u, "
977 "mode=%u, tiling_index = %u\n",
978 i, (uint64_t)tex->surface.u.legacy.level[i].offset_256B * 256,
979 (uint64_t)tex->surface.u.legacy.level[i].slice_size_dw * 4,
980 u_minify(tex->buffer.b.b.width0, i), u_minify(tex->buffer.b.b.height0, i),
981 u_minify(tex->buffer.b.b.depth0, i), tex->surface.u.legacy.level[i].nblk_x,
982 tex->surface.u.legacy.level[i].nblk_y, tex->surface.u.legacy.level[i].mode,
983 tex->surface.u.legacy.tiling_index[i]);
984
985 if (tex->surface.has_stencil) {
986 for (i = 0; i <= tex->buffer.b.b.last_level; i++) {
987 u_log_printf(log,
988 " StencilLevel[%i]: offset=%" PRIu64 ", "
989 "slice_size=%" PRIu64 ", npix_x=%u, "
990 "npix_y=%u, npix_z=%u, nblk_x=%u, nblk_y=%u, "
991 "mode=%u, tiling_index = %u\n",
992 i, (uint64_t)tex->surface.u.legacy.zs.stencil_level[i].offset_256B * 256,
993 (uint64_t)tex->surface.u.legacy.zs.stencil_level[i].slice_size_dw * 4,
994 u_minify(tex->buffer.b.b.width0, i), u_minify(tex->buffer.b.b.height0, i),
995 u_minify(tex->buffer.b.b.depth0, i),
996 tex->surface.u.legacy.zs.stencil_level[i].nblk_x,
997 tex->surface.u.legacy.zs.stencil_level[i].nblk_y,
998 tex->surface.u.legacy.zs.stencil_level[i].mode,
999 tex->surface.u.legacy.zs.stencil_tiling_index[i]);
1000 }
1001 }
1002 }
1003
print_debug_tex(struct si_screen * sscreen,struct si_texture * tex)1004 static void print_debug_tex(struct si_screen *sscreen, struct si_texture *tex)
1005 {
1006 if (sscreen->debug_flags & DBG(TEX)) {
1007 puts("Texture:");
1008 struct u_log_context log;
1009 u_log_context_init(&log);
1010 si_print_texture_info(sscreen, tex, &log);
1011 u_log_new_page_print(&log, stdout);
1012 fflush(stdout);
1013 u_log_context_destroy(&log);
1014 }
1015 }
1016
1017 /**
1018 * Common function for si_texture_create and si_texture_from_handle.
1019 *
1020 * \param screen screen
1021 * \param base resource template
1022 * \param surface radeon_surf
1023 * \param plane0 if a non-zero plane is being created, this is the first plane
1024 * \param imported_buf from si_texture_from_handle
1025 * \param offset offset for non-zero planes or imported buffers
1026 * \param alloc_size the size to allocate if plane0 != NULL
1027 * \param alignment alignment for the allocation
1028 */
si_texture_create_object(struct pipe_screen * screen,const struct pipe_resource * base,const struct radeon_surf * surface,const struct si_texture * plane0,struct pb_buffer_lean * imported_buf,uint64_t offset,unsigned pitch_in_bytes,uint64_t alloc_size,unsigned alignment)1029 static struct si_texture *si_texture_create_object(struct pipe_screen *screen,
1030 const struct pipe_resource *base,
1031 const struct radeon_surf *surface,
1032 const struct si_texture *plane0,
1033 struct pb_buffer_lean *imported_buf,
1034 uint64_t offset, unsigned pitch_in_bytes,
1035 uint64_t alloc_size, unsigned alignment)
1036 {
1037 struct si_texture *tex;
1038 struct si_resource *resource;
1039 struct si_screen *sscreen = (struct si_screen *)screen;
1040
1041 if (!sscreen->info.has_3d_cube_border_color_mipmap &&
1042 (base->last_level > 0 ||
1043 base->target == PIPE_TEXTURE_3D ||
1044 base->target == PIPE_TEXTURE_CUBE)) {
1045 assert(0);
1046 return NULL;
1047 }
1048
1049 tex = CALLOC_STRUCT_CL(si_texture);
1050 if (!tex)
1051 goto error;
1052
1053 resource = &tex->buffer;
1054 resource->b.b = *base;
1055 pipe_reference_init(&resource->b.b.reference, 1);
1056 resource->b.b.screen = screen;
1057
1058 /* don't include stencil-only formats which we don't support for rendering */
1059 tex->is_depth = util_format_has_depth(util_format_description(tex->buffer.b.b.format));
1060 tex->surface = *surface;
1061
1062 if (!ac_surface_override_offset_stride(&sscreen->info, &tex->surface,
1063 tex->buffer.b.b.array_size,
1064 tex->buffer.b.b.last_level + 1,
1065 offset, pitch_in_bytes / tex->surface.bpe))
1066 goto error;
1067
1068 if (plane0) {
1069 /* The buffer is shared with the first plane. */
1070 resource->bo_size = plane0->buffer.bo_size;
1071 resource->bo_alignment_log2 = plane0->buffer.bo_alignment_log2;
1072 resource->flags = plane0->buffer.flags;
1073 resource->domains = plane0->buffer.domains;
1074
1075 radeon_bo_reference(sscreen->ws, &resource->buf, plane0->buffer.buf);
1076 resource->gpu_address = plane0->buffer.gpu_address;
1077 } else if (!(surface->flags & RADEON_SURF_IMPORTED)) {
1078 if (base->flags & PIPE_RESOURCE_FLAG_SPARSE)
1079 resource->b.b.flags |= PIPE_RESOURCE_FLAG_UNMAPPABLE;
1080 if (base->bind & PIPE_BIND_PRIME_BLIT_DST)
1081 resource->b.b.flags |= SI_RESOURCE_FLAG_GL2_BYPASS;
1082
1083 /* Create the backing buffer. */
1084 si_init_resource_fields(sscreen, resource, alloc_size, alignment);
1085
1086 /* GFX12: Image descriptors always set COMPRESSION_EN=1, so this is the only thing that
1087 * disables DCC in the driver.
1088 */
1089 if (sscreen->info.gfx_level >= GFX12 &&
1090 resource->domains & RADEON_DOMAIN_VRAM &&
1091 surface->u.gfx9.gfx12_enable_dcc)
1092 resource->flags |= RADEON_FLAG_GFX12_ALLOW_DCC;
1093
1094 if (!si_alloc_resource(sscreen, resource))
1095 goto error;
1096 } else {
1097 resource->buf = imported_buf;
1098 resource->gpu_address = sscreen->ws->buffer_get_virtual_address(resource->buf);
1099 resource->bo_size = imported_buf->size;
1100 resource->bo_alignment_log2 = imported_buf->alignment_log2;
1101 resource->domains = sscreen->ws->buffer_get_initial_domain(resource->buf);
1102 if (sscreen->ws->buffer_get_flags)
1103 resource->flags = sscreen->ws->buffer_get_flags(resource->buf);
1104 }
1105
1106 if (sscreen->debug_flags & DBG(VM)) {
1107 fprintf(stderr,
1108 "VM start=0x%" PRIX64 " end=0x%" PRIX64
1109 " | Texture %ix%ix%i, %i levels, %i samples, %s | Flags: ",
1110 tex->buffer.gpu_address, tex->buffer.gpu_address + tex->buffer.buf->size,
1111 base->width0, base->height0, util_num_layers(base, 0), base->last_level + 1,
1112 base->nr_samples ? base->nr_samples : 1, util_format_short_name(base->format));
1113 si_res_print_flags(tex->buffer.flags);
1114 fprintf(stderr, "\n");
1115 }
1116
1117 if (sscreen->info.gfx_level >= GFX12) {
1118 print_debug_tex(sscreen, tex);
1119 if (tex->is_depth) {
1120 /* Z24 is no longer supported. We should use Z32_FLOAT instead. */
1121 if (base->format == PIPE_FORMAT_Z16_UNORM) {
1122 tex->db_render_format = base->format;
1123 } else {
1124 tex->db_render_format = PIPE_FORMAT_Z32_FLOAT;
1125 tex->upgraded_depth = base->format != PIPE_FORMAT_Z32_FLOAT &&
1126 base->format != PIPE_FORMAT_Z32_FLOAT_S8X24_UINT;
1127 }
1128
1129 tex->db_compatible = true;
1130 tex->can_sample_z = true;
1131 tex->can_sample_s = true;
1132 }
1133
1134 /* Always set BO metadata - required for programming DCC fields for GFX12 SDMA in the kernel.
1135 * If the texture is suballocated, this will overwrite the metadata for all suballocations,
1136 * but there is nothing we can do about that.
1137 */
1138 si_set_tex_bo_metadata(sscreen, tex);
1139 return tex;
1140 }
1141
1142 /* Everything below is for GFX6-11. */
1143
1144 /* Use 1.0 as the default clear value to get optimal ZRANGE_PRECISION if we don't
1145 * get a fast clear.
1146 */
1147 for (unsigned i = 0; i < ARRAY_SIZE(tex->depth_clear_value); i++)
1148 tex->depth_clear_value[i] = 1.0;
1149
1150 if (tex->surface.flags & RADEON_SURF_TC_COMPATIBLE_HTILE) {
1151 assert(sscreen->info.gfx_level < GFX12);
1152
1153 /* On GFX8, HTILE uses different tiling depending on the TC_COMPATIBLE_HTILE
1154 * setting, so we have to enable it if we enabled it at allocation.
1155 *
1156 * GFX9+ and later use the same tiling for both, so TC-compatible HTILE can be
1157 * enabled on demand.
1158 */
1159 tex->tc_compatible_htile = sscreen->info.gfx_level == GFX8 ||
1160 /* Mipmapping always starts TC-compatible. */
1161 (sscreen->info.gfx_level >= GFX9 &&
1162 tex->buffer.b.b.last_level > 0);
1163 }
1164
1165 print_debug_tex(sscreen, tex);
1166
1167 /* TC-compatible HTILE:
1168 * - GFX8 only supports Z32_FLOAT.
1169 * - GFX9 only supports Z32_FLOAT and Z16_UNORM. */
1170 if (tex->surface.flags & RADEON_SURF_TC_COMPATIBLE_HTILE) {
1171 if (sscreen->info.gfx_level >= GFX9 && base->format == PIPE_FORMAT_Z16_UNORM)
1172 tex->db_render_format = base->format;
1173 else {
1174 tex->db_render_format = PIPE_FORMAT_Z32_FLOAT;
1175 tex->upgraded_depth = base->format != PIPE_FORMAT_Z32_FLOAT &&
1176 base->format != PIPE_FORMAT_Z32_FLOAT_S8X24_UINT;
1177 }
1178 } else {
1179 tex->db_render_format = base->format;
1180 }
1181
1182 /* Applies to GCN. */
1183 tex->last_msaa_resolve_target_micro_mode = tex->surface.micro_tile_mode;
1184
1185 if (tex->is_depth) {
1186 tex->htile_stencil_disabled = !tex->surface.has_stencil;
1187
1188 if (sscreen->info.gfx_level >= GFX9) {
1189 tex->can_sample_z = true;
1190 tex->can_sample_s = true;
1191
1192 /* Stencil texturing with HTILE doesn't work
1193 * with mipmapping on Navi10-14. */
1194 if (sscreen->info.gfx_level == GFX10 && base->last_level > 0)
1195 tex->htile_stencil_disabled = true;
1196 } else {
1197 tex->can_sample_z = !tex->surface.u.legacy.depth_adjusted;
1198 tex->can_sample_s = !tex->surface.u.legacy.stencil_adjusted;
1199
1200 /* GFX8 must keep stencil enabled because it can't use Z-only TC-compatible
1201 * HTILE because of a hw bug. This has only a small effect on performance
1202 * because we lose a little bit of Z precision in order to make space for
1203 * stencil in HTILE.
1204 */
1205 if (sscreen->info.gfx_level == GFX8 &&
1206 tex->surface.flags & RADEON_SURF_TC_COMPATIBLE_HTILE)
1207 tex->htile_stencil_disabled = false;
1208 }
1209
1210 tex->db_compatible = surface->flags & RADEON_SURF_ZBUFFER;
1211 } else {
1212 if (tex->surface.cmask_offset) {
1213 assert(sscreen->info.gfx_level < GFX11);
1214 tex->cb_color_info |= S_028C70_FAST_CLEAR(1);
1215 tex->cmask_buffer = &tex->buffer;
1216 }
1217 }
1218
1219 /* Prepare metadata clears. */
1220 struct si_clear_info clears[4];
1221 unsigned num_clears = 0;
1222
1223 if (tex->cmask_buffer) {
1224 /* Initialize the cmask to 0xCC (= compressed state). */
1225 assert(num_clears < ARRAY_SIZE(clears));
1226 si_init_buffer_clear(&clears[num_clears++], &tex->cmask_buffer->b.b,
1227 tex->surface.cmask_offset, tex->surface.cmask_size,
1228 0xCCCCCCCC);
1229 }
1230 if (tex->is_depth && tex->surface.meta_offset) {
1231 uint32_t clear_value = 0;
1232
1233 if (sscreen->info.gfx_level >= GFX9 || tex->tc_compatible_htile)
1234 clear_value = 0x0000030F;
1235
1236 assert(num_clears < ARRAY_SIZE(clears));
1237 si_init_buffer_clear(&clears[num_clears++], &tex->buffer.b.b, tex->surface.meta_offset,
1238 tex->surface.meta_size, clear_value);
1239 }
1240
1241 /* Initialize DCC only if the texture is not being imported. */
1242 if (!(surface->flags & RADEON_SURF_IMPORTED) && !tex->is_depth && tex->surface.meta_offset) {
1243 /* Clear DCC to black for all tiles with DCC enabled.
1244 *
1245 * This fixes corruption in 3DMark Slingshot Extreme, which
1246 * uses uninitialized textures, causing corruption.
1247 */
1248 if (tex->surface.num_meta_levels == tex->buffer.b.b.last_level + 1 &&
1249 tex->buffer.b.b.nr_samples <= 2) {
1250 /* Simple case - all tiles have DCC enabled. */
1251 assert(num_clears < ARRAY_SIZE(clears));
1252 si_init_buffer_clear(&clears[num_clears++], &tex->buffer.b.b, tex->surface.meta_offset,
1253 tex->surface.meta_size, DCC_CLEAR_0000);
1254 } else if (sscreen->info.gfx_level >= GFX9) {
1255 /* Clear to uncompressed. Clearing this to black is complicated. */
1256 assert(num_clears < ARRAY_SIZE(clears));
1257 si_init_buffer_clear(&clears[num_clears++], &tex->buffer.b.b, tex->surface.meta_offset,
1258 tex->surface.meta_size, DCC_UNCOMPRESSED);
1259 } else {
1260 /* GFX8: Initialize mipmap levels and multisamples separately. */
1261 if (tex->buffer.b.b.nr_samples >= 2) {
1262 /* Clearing this to black is complicated. */
1263 assert(num_clears < ARRAY_SIZE(clears));
1264 si_init_buffer_clear(&clears[num_clears++], &tex->buffer.b.b, tex->surface.meta_offset,
1265 tex->surface.meta_size, DCC_UNCOMPRESSED);
1266 } else {
1267 /* Clear the enabled mipmap levels to black. */
1268 unsigned size = 0;
1269
1270 for (unsigned i = 0; i < tex->surface.num_meta_levels; i++) {
1271 if (!tex->surface.u.legacy.color.dcc_level[i].dcc_fast_clear_size)
1272 break;
1273
1274 size = tex->surface.u.legacy.color.dcc_level[i].dcc_offset +
1275 tex->surface.u.legacy.color.dcc_level[i].dcc_fast_clear_size;
1276 }
1277
1278 /* Mipmap levels with DCC. */
1279 if (size) {
1280 assert(num_clears < ARRAY_SIZE(clears));
1281 si_init_buffer_clear(&clears[num_clears++], &tex->buffer.b.b, tex->surface.meta_offset, size,
1282 DCC_CLEAR_0000);
1283 }
1284 /* Mipmap levels without DCC. */
1285 if (size != tex->surface.meta_size) {
1286 assert(num_clears < ARRAY_SIZE(clears));
1287 si_init_buffer_clear(&clears[num_clears++], &tex->buffer.b.b, tex->surface.meta_offset + size,
1288 tex->surface.meta_size - size, DCC_UNCOMPRESSED);
1289 }
1290 }
1291 }
1292 }
1293
1294 /* Initialize displayable DCC that requires the retile blit. */
1295 if (tex->surface.display_dcc_offset && !(surface->flags & RADEON_SURF_IMPORTED)) {
1296 /* Uninitialized DCC can hang the display hw.
1297 * Clear to white to indicate that. */
1298 assert(num_clears < ARRAY_SIZE(clears));
1299 si_init_buffer_clear(&clears[num_clears++], &tex->buffer.b.b, tex->surface.display_dcc_offset,
1300 tex->surface.u.gfx9.color.display_dcc_size,
1301 sscreen->info.gfx_level >= GFX11 ? GFX11_DCC_CLEAR_1111_UNORM
1302 : GFX8_DCC_CLEAR_1111);
1303 }
1304
1305 /* Execute the clears. */
1306 if (num_clears) {
1307 struct si_context *sctx = si_get_aux_context(&sscreen->aux_context.compute_resource_init);
1308
1309 si_execute_clears(sctx, clears, num_clears, false);
1310 si_put_aux_context_flush(&sscreen->aux_context.compute_resource_init);
1311 }
1312
1313 /* Initialize the CMASK base register value. */
1314 tex->cmask_base_address_reg = (tex->buffer.gpu_address + tex->surface.cmask_offset) >> 8;
1315
1316 return tex;
1317
1318 error:
1319 FREE_CL(tex);
1320 return NULL;
1321 }
1322
si_choose_tiling(struct si_screen * sscreen,const struct pipe_resource * templ,bool tc_compatible_htile)1323 static enum radeon_surf_mode si_choose_tiling(struct si_screen *sscreen,
1324 const struct pipe_resource *templ,
1325 bool tc_compatible_htile)
1326 {
1327 const struct util_format_description *desc = util_format_description(templ->format);
1328 bool is_depth_stencil = util_format_is_depth_or_stencil(templ->format) &&
1329 !(templ->flags & SI_RESOURCE_FLAG_FLUSHED_DEPTH);
1330
1331 /* MSAA resources must be 2D tiled. */
1332 if (templ->nr_samples > 1)
1333 return RADEON_SURF_MODE_2D;
1334
1335 /* Transfer resources should be linear. */
1336 if (templ->flags & SI_RESOURCE_FLAG_FORCE_LINEAR)
1337 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1338
1339 /* Avoid Z/S decompress blits by forcing TC-compatible HTILE on GFX8,
1340 * which requires 2D tiling.
1341 */
1342 if (sscreen->info.gfx_level == GFX8 && tc_compatible_htile)
1343 return RADEON_SURF_MODE_2D;
1344
1345 /* Handle common candidates for the linear mode.
1346 * Compressed textures and DB surfaces must always be tiled.
1347 */
1348 if (!is_depth_stencil && !util_format_is_compressed(templ->format)) {
1349 if (sscreen->debug_flags & DBG(NO_TILING) ||
1350 (templ->bind & PIPE_BIND_SCANOUT && sscreen->debug_flags & DBG(NO_DISPLAY_TILING)))
1351 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1352
1353 /* Tiling doesn't work with the 422 (SUBSAMPLED) formats. */
1354 if (desc->layout == UTIL_FORMAT_LAYOUT_SUBSAMPLED)
1355 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1356
1357 /* Cursors are linear on AMD GCN.
1358 * (XXX double-check, maybe also use RADEON_SURF_SCANOUT) */
1359 if (templ->bind & PIPE_BIND_CURSOR)
1360 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1361
1362 if (templ->bind & PIPE_BIND_LINEAR)
1363 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1364
1365 /* Textures with a very small height are recommended to be linear. */
1366 if (templ->target == PIPE_TEXTURE_1D || templ->target == PIPE_TEXTURE_1D_ARRAY ||
1367 /* Only very thin and long 2D textures should benefit from
1368 * linear_aligned. */
1369 templ->height0 <= 2)
1370 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1371
1372 /* Textures likely to be mapped often. */
1373 if (templ->usage == PIPE_USAGE_STAGING || templ->usage == PIPE_USAGE_STREAM)
1374 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1375 }
1376
1377 /* Make small textures 1D tiled. */
1378 if (templ->width0 <= 16 || templ->height0 <= 16 || (sscreen->debug_flags & DBG(NO_2D_TILING)))
1379 return RADEON_SURF_MODE_1D;
1380
1381 /* The allocator will switch to 1D if needed. */
1382 return RADEON_SURF_MODE_2D;
1383 }
1384
1385 static struct pipe_resource *
si_texture_create_with_modifier(struct pipe_screen * screen,const struct pipe_resource * templ,uint64_t modifier)1386 si_texture_create_with_modifier(struct pipe_screen *screen,
1387 const struct pipe_resource *templ,
1388 uint64_t modifier)
1389 {
1390 struct si_screen *sscreen = (struct si_screen *)screen;
1391 bool is_zs = util_format_is_depth_or_stencil(templ->format);
1392
1393 if (templ->nr_samples >= 2) {
1394 /* This is hackish (overwriting the const pipe_resource template),
1395 * but should be harmless and gallium frontends can also see
1396 * the overridden number of samples in the created pipe_resource.
1397 */
1398 if (is_zs && sscreen->eqaa_force_z_samples) {
1399 ((struct pipe_resource *)templ)->nr_samples =
1400 ((struct pipe_resource *)templ)->nr_storage_samples = sscreen->eqaa_force_z_samples;
1401 } else if (!is_zs && sscreen->eqaa_force_color_samples) {
1402 ((struct pipe_resource *)templ)->nr_samples = sscreen->eqaa_force_coverage_samples;
1403 ((struct pipe_resource *)templ)->nr_storage_samples = sscreen->eqaa_force_color_samples;
1404 }
1405 }
1406
1407 bool is_flushed_depth = templ->flags & SI_RESOURCE_FLAG_FLUSHED_DEPTH ||
1408 templ->flags & SI_RESOURCE_FLAG_FORCE_LINEAR;
1409 bool tc_compatible_htile = is_zs && !is_flushed_depth &&
1410 !(sscreen->debug_flags & DBG(NO_HYPERZ)) &&
1411 sscreen->info.has_tc_compatible_htile;
1412
1413 enum radeon_surf_mode tile_mode = si_choose_tiling(sscreen, templ, tc_compatible_htile);
1414
1415 /* This allocates textures with multiple planes like NV12 in 1 buffer. */
1416 enum
1417 {
1418 SI_TEXTURE_MAX_PLANES = 3
1419 };
1420 struct radeon_surf surface[SI_TEXTURE_MAX_PLANES] = {};
1421 struct pipe_resource plane_templ[SI_TEXTURE_MAX_PLANES];
1422 uint64_t plane_offset[SI_TEXTURE_MAX_PLANES] = {};
1423 uint64_t total_size = 0;
1424 unsigned max_alignment = 0;
1425 unsigned num_planes = util_format_get_num_planes(templ->format);
1426 assert(num_planes <= SI_TEXTURE_MAX_PLANES);
1427
1428 /* Compute texture or plane layouts and offsets. */
1429 for (unsigned i = 0; i < num_planes; i++) {
1430 plane_templ[i] = *templ;
1431 plane_templ[i].format = util_format_get_plane_format(templ->format, i);
1432 plane_templ[i].width0 = util_format_get_plane_width(templ->format, i, templ->width0);
1433 plane_templ[i].height0 = util_format_get_plane_height(templ->format, i, templ->height0);
1434
1435 /* Multi-plane allocations need PIPE_BIND_SHARED, because we can't
1436 * reallocate the storage to add PIPE_BIND_SHARED, because it's
1437 * shared by 3 pipe_resources.
1438 */
1439 if (num_planes > 1)
1440 plane_templ[i].bind |= PIPE_BIND_SHARED;
1441 /* Setting metadata on suballocated buffers is impossible. So use PIPE_BIND_CUSTOM to
1442 * request a non-suballocated buffer.
1443 */
1444 if (!is_zs && sscreen->debug_flags & DBG(EXTRA_METADATA))
1445 plane_templ[i].bind |= PIPE_BIND_CUSTOM;
1446
1447 if (si_init_surface(sscreen, &surface[i], &plane_templ[i], tile_mode, modifier,
1448 false, plane_templ[i].bind & PIPE_BIND_SCANOUT,
1449 is_flushed_depth, tc_compatible_htile))
1450 return NULL;
1451
1452 plane_templ[i].nr_sparse_levels = surface[i].first_mip_tail_level;
1453
1454 plane_offset[i] = align64(total_size, 1 << surface[i].surf_alignment_log2);
1455 total_size = plane_offset[i] + surface[i].total_size;
1456 max_alignment = MAX2(max_alignment, 1 << surface[i].surf_alignment_log2);
1457 }
1458
1459 struct si_texture *plane0 = NULL, *last_plane = NULL;
1460
1461 for (unsigned i = 0; i < num_planes; i++) {
1462 struct si_texture *tex =
1463 si_texture_create_object(screen, &plane_templ[i], &surface[i], plane0, NULL,
1464 plane_offset[i], 0, total_size, max_alignment);
1465 if (!tex) {
1466 si_texture_reference(&plane0, NULL);
1467 return NULL;
1468 }
1469
1470 tex->plane_index = i;
1471 tex->num_planes = num_planes;
1472
1473 if (!plane0) {
1474 plane0 = last_plane = tex;
1475 } else {
1476 last_plane->buffer.b.b.next = &tex->buffer.b.b;
1477 last_plane = tex;
1478 }
1479 if (i == 0 && !is_zs && tex->surface.fmask_size == 0 &&
1480 sscreen->debug_flags & DBG(EXTRA_METADATA))
1481 si_set_tex_bo_metadata(sscreen, tex);
1482 }
1483
1484 if (num_planes >= 2)
1485 plane0->multi_plane_format = templ->format;
1486
1487 return (struct pipe_resource *)plane0;
1488 }
1489
si_texture_create(struct pipe_screen * screen,const struct pipe_resource * templ)1490 struct pipe_resource *si_texture_create(struct pipe_screen *screen,
1491 const struct pipe_resource *templ)
1492 {
1493 return si_texture_create_with_modifier(screen, templ, DRM_FORMAT_MOD_INVALID);
1494 }
1495
si_texture_commit(struct si_context * ctx,struct si_resource * res,unsigned level,struct pipe_box * box,bool commit)1496 bool si_texture_commit(struct si_context *ctx, struct si_resource *res, unsigned level,
1497 struct pipe_box *box, bool commit)
1498 {
1499 struct si_texture *tex = (struct si_texture *)res;
1500 struct radeon_surf *surface = &tex->surface;
1501 enum pipe_format format = res->b.b.format;
1502 unsigned blks = util_format_get_blocksize(format);
1503 unsigned samples = MAX2(1, res->b.b.nr_samples);
1504
1505 assert(ctx->gfx_level >= GFX9);
1506
1507 unsigned row_pitch = surface->u.gfx9.prt_level_pitch[level] *
1508 surface->prt_tile_height * surface->prt_tile_depth * blks * samples;
1509 uint64_t depth_pitch = surface->u.gfx9.surf_slice_size * surface->prt_tile_depth;
1510
1511 unsigned x = box->x / surface->prt_tile_width;
1512 unsigned y = box->y / surface->prt_tile_height;
1513 unsigned z = box->z / surface->prt_tile_depth;
1514
1515 unsigned w = DIV_ROUND_UP(box->width, surface->prt_tile_width);
1516 unsigned h = DIV_ROUND_UP(box->height, surface->prt_tile_height);
1517 unsigned d = DIV_ROUND_UP(box->depth, surface->prt_tile_depth);
1518
1519 /* Align to tile block base, for levels in mip tail whose offset is inside
1520 * a tile block.
1521 */
1522 uint64_t level_base = ROUND_DOWN_TO(surface->u.gfx9.prt_level_offset[level],
1523 RADEON_SPARSE_PAGE_SIZE);
1524 uint64_t commit_base = level_base +
1525 x * RADEON_SPARSE_PAGE_SIZE + y * (uint64_t)row_pitch + z * depth_pitch;
1526
1527 uint64_t size = (uint64_t)w * RADEON_SPARSE_PAGE_SIZE;
1528 for (int i = 0; i < d; i++) {
1529 uint64_t base = commit_base + i * depth_pitch;
1530 for (int j = 0; j < h; j++) {
1531 uint64_t offset = base + j * row_pitch;
1532 if (!ctx->ws->buffer_commit(ctx->ws, res->buf, offset, size, commit))
1533 return false;
1534 }
1535 }
1536
1537 return true;
1538 }
1539
si_query_dmabuf_modifiers(struct pipe_screen * screen,enum pipe_format format,int max,uint64_t * modifiers,unsigned int * external_only,int * count)1540 static void si_query_dmabuf_modifiers(struct pipe_screen *screen,
1541 enum pipe_format format,
1542 int max,
1543 uint64_t *modifiers,
1544 unsigned int *external_only,
1545 int *count)
1546 {
1547 struct si_screen *sscreen = (struct si_screen *)screen;
1548
1549 unsigned ac_mod_count = max;
1550 ac_get_supported_modifiers(&sscreen->info, &(struct ac_modifier_options) {
1551 .dcc = !(sscreen->debug_flags & (DBG(NO_DCC) | DBG(NO_EXPORTED_DCC))),
1552 /* Do not support DCC with retiling yet. This needs explicit
1553 * resource flushes, but the app has no way to promise doing
1554 * flushes with modifiers. */
1555 .dcc_retile = !(sscreen->debug_flags & DBG(NO_DCC)),
1556 }, format, &ac_mod_count, max ? modifiers : NULL);
1557 if (max && external_only) {
1558 for (unsigned i = 0; i < ac_mod_count; ++i)
1559 external_only[i] = util_format_is_yuv(format);
1560 }
1561 *count = ac_mod_count;
1562 }
1563
1564 static bool
si_is_dmabuf_modifier_supported(struct pipe_screen * screen,uint64_t modifier,enum pipe_format format,bool * external_only)1565 si_is_dmabuf_modifier_supported(struct pipe_screen *screen,
1566 uint64_t modifier,
1567 enum pipe_format format,
1568 bool *external_only)
1569 {
1570 int allowed_mod_count;
1571 si_query_dmabuf_modifiers(screen, format, 0, NULL, NULL, &allowed_mod_count);
1572
1573 uint64_t *allowed_modifiers = (uint64_t *)calloc(allowed_mod_count, sizeof(uint64_t));
1574 if (!allowed_modifiers)
1575 return false;
1576
1577 unsigned *external_array = NULL;
1578 if (external_only) {
1579 external_array = (unsigned *)calloc(allowed_mod_count, sizeof(unsigned));
1580 if (!external_array) {
1581 free(allowed_modifiers);
1582 return false;
1583 }
1584 }
1585
1586 si_query_dmabuf_modifiers(screen, format, allowed_mod_count, allowed_modifiers,
1587 external_array, &allowed_mod_count);
1588
1589 bool supported = false;
1590 for (int i = 0; i < allowed_mod_count && !supported; ++i) {
1591 if (allowed_modifiers[i] != modifier)
1592 continue;
1593
1594 supported = true;
1595 if (external_only)
1596 *external_only = external_array[i];
1597 }
1598
1599 free(allowed_modifiers);
1600 free(external_array);
1601 return supported;
1602 }
1603
1604 static unsigned
si_get_dmabuf_modifier_planes(struct pipe_screen * pscreen,uint64_t modifier,enum pipe_format format)1605 si_get_dmabuf_modifier_planes(struct pipe_screen *pscreen, uint64_t modifier,
1606 enum pipe_format format)
1607 {
1608 unsigned planes = util_format_get_num_planes(format);
1609
1610 if (AMD_FMT_MOD_GET(TILE_VERSION, modifier) < AMD_FMT_MOD_TILE_VER_GFX12) {
1611 if (IS_AMD_FMT_MOD(modifier) && planes == 1) {
1612 if (AMD_FMT_MOD_GET(DCC_RETILE, modifier))
1613 return 3;
1614 else if (AMD_FMT_MOD_GET(DCC, modifier))
1615 return 2;
1616 else
1617 return 1;
1618 }
1619 }
1620
1621 return planes;
1622 }
1623
1624 static bool
si_modifier_supports_resource(struct pipe_screen * screen,uint64_t modifier,const struct pipe_resource * templ)1625 si_modifier_supports_resource(struct pipe_screen *screen,
1626 uint64_t modifier,
1627 const struct pipe_resource *templ)
1628 {
1629 struct si_screen *sscreen = (struct si_screen *)screen;
1630 uint32_t max_width, max_height;
1631
1632 if (((templ->bind & PIPE_BIND_LINEAR) || sscreen->debug_flags & DBG(NO_TILING)) &&
1633 modifier != DRM_FORMAT_MOD_LINEAR)
1634 return false;
1635
1636 if ((templ->bind & PIPE_BIND_USE_FRONT_RENDERING) && ac_modifier_has_dcc(modifier))
1637 return false;
1638
1639 /* Protected content doesn't support DCC on GFX12. */
1640 if (sscreen->info.gfx_level >= GFX12 && templ->bind & PIPE_BIND_PROTECTED &&
1641 IS_AMD_FMT_MOD(modifier) &&
1642 AMD_FMT_MOD_GET(TILE_VERSION, modifier) >= AMD_FMT_MOD_TILE_VER_GFX12 &&
1643 AMD_FMT_MOD_GET(DCC, modifier))
1644 return false;
1645
1646 ac_modifier_max_extent(&sscreen->info, modifier, &max_width, &max_height);
1647 return templ->width0 <= max_width && templ->height0 <= max_height;
1648 }
1649
1650 static struct pipe_resource *
si_texture_create_with_modifiers(struct pipe_screen * screen,const struct pipe_resource * templ,const uint64_t * modifiers,int modifier_count)1651 si_texture_create_with_modifiers(struct pipe_screen *screen,
1652 const struct pipe_resource *templ,
1653 const uint64_t *modifiers,
1654 int modifier_count)
1655 {
1656 /* Buffers with modifiers make zero sense. */
1657 assert(templ->target != PIPE_BUFFER);
1658
1659 /* Select modifier. */
1660 int allowed_mod_count;
1661 si_query_dmabuf_modifiers(screen, templ->format, 0, NULL, NULL, &allowed_mod_count);
1662
1663 uint64_t *allowed_modifiers = (uint64_t *)calloc(allowed_mod_count, sizeof(uint64_t));
1664 if (!allowed_modifiers) {
1665 return NULL;
1666 }
1667
1668 /* This does not take external_only into account. We assume it is the same for all modifiers. */
1669 si_query_dmabuf_modifiers(screen, templ->format, allowed_mod_count, allowed_modifiers, NULL, &allowed_mod_count);
1670
1671 uint64_t modifier = DRM_FORMAT_MOD_INVALID;
1672
1673 /* Try to find the first allowed modifier that is in the application provided
1674 * list. We assume that the allowed modifiers are ordered in descending
1675 * preference in the list provided by si_query_dmabuf_modifiers. */
1676 for (int i = 0; i < allowed_mod_count; ++i) {
1677 bool found = false;
1678 for (int j = 0; j < modifier_count && !found; ++j)
1679 if (modifiers[j] == allowed_modifiers[i] && si_modifier_supports_resource(screen, modifiers[j], templ))
1680 found = true;
1681
1682 if (found) {
1683 modifier = allowed_modifiers[i];
1684 break;
1685 }
1686 }
1687
1688 free(allowed_modifiers);
1689
1690 if (modifier == DRM_FORMAT_MOD_INVALID) {
1691 return NULL;
1692 }
1693 return si_texture_create_with_modifier(screen, templ, modifier);
1694 }
1695
si_texture_is_aux_plane(const struct pipe_resource * resource)1696 static bool si_texture_is_aux_plane(const struct pipe_resource *resource)
1697 {
1698 return resource->flags & SI_RESOURCE_AUX_PLANE;
1699 }
1700
si_texture_from_winsys_buffer(struct si_screen * sscreen,const struct pipe_resource * templ,struct pb_buffer_lean * buf,unsigned stride,uint64_t offset,uint64_t modifier,unsigned usage,bool dedicated)1701 static struct pipe_resource *si_texture_from_winsys_buffer(struct si_screen *sscreen,
1702 const struct pipe_resource *templ,
1703 struct pb_buffer_lean *buf, unsigned stride,
1704 uint64_t offset, uint64_t modifier,
1705 unsigned usage, bool dedicated)
1706 {
1707 struct radeon_surf surface = {};
1708 struct radeon_bo_metadata metadata = {};
1709 uint32_t md_version, md_flags;
1710 struct si_texture *tex;
1711 int r;
1712
1713 /* Ignore metadata for non-zero planes. */
1714 if (offset != 0)
1715 dedicated = false;
1716
1717 if (dedicated) {
1718 sscreen->ws->buffer_get_metadata(sscreen->ws, buf, &metadata, &surface);
1719
1720 /* Refuse to import texture allocated with a overriden gfx family since
1721 * the data will be garbage.
1722 */
1723 md_version = metadata.metadata[0] & 0xffff;
1724 md_flags = metadata.metadata[0] >> 16;
1725
1726 if (metadata.mode != RADEON_SURF_MODE_LINEAR_ALIGNED &&
1727 modifier == DRM_FORMAT_MOD_INVALID &&
1728 md_version >= 3 &&
1729 md_flags & (1u << AC_SURF_METADATA_FLAG_FAMILY_OVERRIDEN_BIT)) {
1730 fprintf(stderr, "si_texture_from_winsys_buffer: fail texture import due to "
1731 "AC_SURF_METADATA_FLAG_FAMILY_OVERRIDEN_BIT being set.\n");
1732 return NULL;
1733 }
1734 } else {
1735 /**
1736 * The bo metadata is unset for un-dedicated images. So we fall
1737 * back to linear. See answer to question 5 of the
1738 * VK_KHX_external_memory spec for some details.
1739 *
1740 * It is possible that this case isn't going to work if the
1741 * surface pitch isn't correctly aligned by default.
1742 *
1743 * In order to support it correctly we require multi-image
1744 * metadata to be synchronized between radv and radeonsi. The
1745 * semantics of associating multiple image metadata to a memory
1746 * object on the vulkan export side are not concretely defined
1747 * either.
1748 *
1749 * All the use cases we are aware of at the moment for memory
1750 * objects use dedicated allocations. So lets keep the initial
1751 * implementation simple.
1752 *
1753 * A possible alternative is to attempt to reconstruct the
1754 * tiling information when the TexParameter TEXTURE_TILING_EXT
1755 * is set.
1756 */
1757 metadata.mode = RADEON_SURF_MODE_LINEAR_ALIGNED;
1758 }
1759
1760 r = si_init_surface(sscreen, &surface, templ, metadata.mode, modifier, true,
1761 surface.flags & RADEON_SURF_SCANOUT, false, false);
1762 if (r)
1763 return NULL;
1764
1765 /* This is a hack to skip alignment checking for 3D textures */
1766 if (templ->target == PIPE_TEXTURE_3D)
1767 stride = 0;
1768
1769 tex = si_texture_create_object(&sscreen->b, templ, &surface, NULL, buf,
1770 offset, stride, 0, 0);
1771 if (!tex)
1772 return NULL;
1773
1774 tex->buffer.b.is_shared = true;
1775 tex->buffer.external_usage = usage;
1776 tex->num_planes = 1;
1777 if (tex->buffer.flags & RADEON_FLAG_ENCRYPTED)
1778 tex->buffer.b.b.bind |= PIPE_BIND_PROTECTED;
1779
1780 /* Account for multiple planes with lowered yuv import. */
1781 struct pipe_resource *next_plane = tex->buffer.b.b.next;
1782 while (next_plane && !si_texture_is_aux_plane(next_plane)) {
1783 struct si_texture *next_tex = (struct si_texture *)next_plane;
1784 ++next_tex->num_planes;
1785 ++tex->num_planes;
1786 next_plane = next_plane->next;
1787 }
1788
1789 unsigned nplanes = ac_surface_get_nplanes(&tex->surface);
1790 unsigned plane = 1;
1791 while (next_plane) {
1792 struct si_auxiliary_texture *ptex = (struct si_auxiliary_texture *)next_plane;
1793 if (plane >= nplanes || ptex->buffer != tex->buffer.buf ||
1794 ptex->offset != ac_surface_get_plane_offset(sscreen->info.gfx_level,
1795 &tex->surface, plane, 0) ||
1796 ptex->stride != ac_surface_get_plane_stride(sscreen->info.gfx_level,
1797 &tex->surface, plane, 0)) {
1798 si_texture_reference(&tex, NULL);
1799 return NULL;
1800 }
1801 ++plane;
1802 next_plane = next_plane->next;
1803 }
1804
1805 if (plane != nplanes && tex->num_planes == 1) {
1806 si_texture_reference(&tex, NULL);
1807 return NULL;
1808 }
1809
1810 if (!ac_surface_apply_umd_metadata(&sscreen->info, &tex->surface,
1811 tex->buffer.b.b.nr_storage_samples,
1812 tex->buffer.b.b.last_level + 1,
1813 metadata.size_metadata,
1814 metadata.metadata)) {
1815 si_texture_reference(&tex, NULL);
1816 return NULL;
1817 }
1818
1819 if (ac_surface_get_plane_offset(sscreen->info.gfx_level, &tex->surface, 0, 0) +
1820 tex->surface.total_size > buf->size) {
1821 si_texture_reference(&tex, NULL);
1822 return NULL;
1823 }
1824
1825 /* Displayable DCC requires an explicit flush. */
1826 if (dedicated && offset == 0 && !(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) &&
1827 si_displayable_dcc_needs_explicit_flush(tex)) {
1828 /* TODO: do we need to decompress DCC? */
1829 if (si_texture_discard_dcc(sscreen, tex)) {
1830 /* Update BO metadata after disabling DCC. */
1831 si_set_tex_bo_metadata(sscreen, tex);
1832 }
1833 }
1834
1835 assert(tex->surface.tile_swizzle == 0);
1836 return &tex->buffer.b.b;
1837 }
1838
si_texture_from_handle(struct pipe_screen * screen,const struct pipe_resource * templ,struct winsys_handle * whandle,unsigned usage)1839 static struct pipe_resource *si_texture_from_handle(struct pipe_screen *screen,
1840 const struct pipe_resource *templ,
1841 struct winsys_handle *whandle, unsigned usage)
1842 {
1843 struct si_screen *sscreen = (struct si_screen *)screen;
1844 struct pb_buffer_lean *buf = NULL;
1845
1846 buf = sscreen->ws->buffer_from_handle(sscreen->ws, whandle,
1847 sscreen->info.max_alignment,
1848 templ->bind & PIPE_BIND_PRIME_BLIT_DST);
1849 if (!buf)
1850 return NULL;
1851
1852 if (templ->target == PIPE_BUFFER)
1853 return si_buffer_from_winsys_buffer(screen, templ, buf, 0);
1854
1855 if (whandle->plane >= util_format_get_num_planes(whandle->format)) {
1856 struct si_auxiliary_texture *tex = CALLOC_STRUCT_CL(si_auxiliary_texture);
1857 if (!tex)
1858 return NULL;
1859 tex->b.b = *templ;
1860 tex->b.b.flags |= SI_RESOURCE_AUX_PLANE;
1861 tex->stride = whandle->stride;
1862 tex->offset = whandle->offset;
1863 tex->buffer = buf;
1864 pipe_reference_init(&tex->b.b.reference, 1);
1865 tex->b.b.screen = screen;
1866
1867 return &tex->b.b;
1868 }
1869
1870 return si_texture_from_winsys_buffer(sscreen, templ, buf, whandle->stride, whandle->offset,
1871 whandle->modifier, usage, true);
1872 }
1873
si_init_flushed_depth_texture(struct pipe_context * ctx,struct pipe_resource * texture)1874 bool si_init_flushed_depth_texture(struct pipe_context *ctx, struct pipe_resource *texture)
1875 {
1876 struct si_texture *tex = (struct si_texture *)texture;
1877 struct pipe_resource resource;
1878 enum pipe_format pipe_format = texture->format;
1879
1880 assert(!tex->flushed_depth_texture);
1881
1882 if (!tex->can_sample_z && tex->can_sample_s) {
1883 switch (pipe_format) {
1884 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
1885 /* Save memory by not allocating the S plane. */
1886 pipe_format = PIPE_FORMAT_Z32_FLOAT;
1887 break;
1888 case PIPE_FORMAT_Z24_UNORM_S8_UINT:
1889 case PIPE_FORMAT_S8_UINT_Z24_UNORM:
1890 /* Save memory bandwidth by not copying the
1891 * stencil part during flush.
1892 *
1893 * This potentially increases memory bandwidth
1894 * if an application uses both Z and S texturing
1895 * simultaneously (a flushed Z24S8 texture
1896 * would be stored compactly), but how often
1897 * does that really happen?
1898 */
1899 pipe_format = PIPE_FORMAT_Z24X8_UNORM;
1900 break;
1901 default:;
1902 }
1903 } else if (!tex->can_sample_s && tex->can_sample_z) {
1904 assert(util_format_has_stencil(util_format_description(pipe_format)));
1905
1906 /* DB->CB copies to an 8bpp surface don't work. */
1907 pipe_format = PIPE_FORMAT_X24S8_UINT;
1908 }
1909
1910 memset(&resource, 0, sizeof(resource));
1911 resource.target = texture->target;
1912 resource.format = pipe_format;
1913 resource.width0 = texture->width0;
1914 resource.height0 = texture->height0;
1915 resource.depth0 = texture->depth0;
1916 resource.array_size = texture->array_size;
1917 resource.last_level = texture->last_level;
1918 resource.nr_samples = texture->nr_samples;
1919 resource.nr_storage_samples = texture->nr_storage_samples;
1920 resource.usage = PIPE_USAGE_DEFAULT;
1921 resource.bind = texture->bind & ~PIPE_BIND_DEPTH_STENCIL;
1922 resource.flags = texture->flags | SI_RESOURCE_FLAG_FLUSHED_DEPTH;
1923
1924 tex->flushed_depth_texture =
1925 (struct si_texture *)ctx->screen->resource_create(ctx->screen, &resource);
1926 if (!tex->flushed_depth_texture) {
1927 PRINT_ERR("failed to create temporary texture to hold flushed depth\n");
1928 return false;
1929 }
1930 return true;
1931 }
1932
1933 /**
1934 * Initialize the pipe_resource descriptor to be of the same size as the box,
1935 * which is supposed to hold a subregion of the texture "orig" at the given
1936 * mipmap level.
1937 */
si_init_temp_resource_from_box(struct pipe_resource * res,struct pipe_resource * orig,const struct pipe_box * box,unsigned level,unsigned usage,unsigned flags)1938 static void si_init_temp_resource_from_box(struct pipe_resource *res, struct pipe_resource *orig,
1939 const struct pipe_box *box, unsigned level,
1940 unsigned usage, unsigned flags)
1941 {
1942 struct si_texture *tex = (struct si_texture *)orig;
1943 enum pipe_format orig_format = tex->multi_plane_format != PIPE_FORMAT_NONE ?
1944 tex->multi_plane_format : orig->format;
1945
1946 memset(res, 0, sizeof(*res));
1947 res->format = orig_format;
1948 res->width0 = box->width;
1949 res->height0 = box->height;
1950 res->depth0 = 1;
1951 res->array_size = 1;
1952 res->usage = usage;
1953 res->flags = flags;
1954
1955 if (flags & SI_RESOURCE_FLAG_FORCE_LINEAR && util_format_is_compressed(orig_format)) {
1956 /* Transfer resources are allocated with linear tiling, which is
1957 * not supported for compressed formats.
1958 */
1959 unsigned blocksize = util_format_get_blocksize(orig_format);
1960
1961 if (blocksize == 8) {
1962 res->format = PIPE_FORMAT_R16G16B16A16_UINT;
1963 } else {
1964 assert(blocksize == 16);
1965 res->format = PIPE_FORMAT_R32G32B32A32_UINT;
1966 }
1967
1968 res->width0 = util_format_get_nblocksx(orig_format, box->width);
1969 res->height0 = util_format_get_nblocksy(orig_format, box->height);
1970 }
1971
1972 /* We must set the correct texture target and dimensions for a 3D box. */
1973 if (box->depth > 1 && util_max_layer(orig, level) > 0) {
1974 res->target = PIPE_TEXTURE_2D_ARRAY;
1975 res->array_size = box->depth;
1976 } else {
1977 res->target = PIPE_TEXTURE_2D;
1978 }
1979 }
1980
si_can_invalidate_texture(struct si_screen * sscreen,struct si_texture * tex,unsigned transfer_usage,const struct pipe_box * box)1981 static bool si_can_invalidate_texture(struct si_screen *sscreen, struct si_texture *tex,
1982 unsigned transfer_usage, const struct pipe_box *box)
1983 {
1984 return !tex->buffer.b.is_shared && !(tex->surface.flags & RADEON_SURF_IMPORTED) &&
1985 !(transfer_usage & PIPE_MAP_READ) && tex->buffer.b.b.last_level == 0 &&
1986 util_texrange_covers_whole_level(&tex->buffer.b.b, 0, box->x, box->y, box->z, box->width,
1987 box->height, box->depth);
1988 }
1989
si_texture_invalidate_storage(struct si_context * sctx,struct si_texture * tex)1990 static void si_texture_invalidate_storage(struct si_context *sctx, struct si_texture *tex)
1991 {
1992 struct si_screen *sscreen = sctx->screen;
1993
1994 /* There is no point in discarding depth and tiled buffers. */
1995 assert(!tex->is_depth);
1996 assert(tex->surface.is_linear);
1997
1998 /* Reallocate the buffer in the same pipe_resource. */
1999 si_alloc_resource(sscreen, &tex->buffer);
2000
2001 /* Initialize the CMASK base address (needed even without CMASK). */
2002 tex->cmask_base_address_reg = (tex->buffer.gpu_address + tex->surface.cmask_offset) >> 8;
2003
2004 p_atomic_inc(&sscreen->dirty_tex_counter);
2005
2006 sctx->num_alloc_tex_transfer_bytes += tex->surface.total_size;
2007 }
2008
si_texture_transfer_map(struct pipe_context * ctx,struct pipe_resource * texture,unsigned level,unsigned usage,const struct pipe_box * box,struct pipe_transfer ** ptransfer)2009 static void *si_texture_transfer_map(struct pipe_context *ctx, struct pipe_resource *texture,
2010 unsigned level, unsigned usage, const struct pipe_box *box,
2011 struct pipe_transfer **ptransfer)
2012 {
2013 struct si_context *sctx = (struct si_context *)ctx;
2014 struct si_texture *tex = (struct si_texture *)texture;
2015 struct si_transfer *trans;
2016 struct si_resource *buf;
2017 uint64_t offset = 0;
2018 char *map;
2019 bool use_staging_texture = tex->buffer.flags & RADEON_FLAG_ENCRYPTED;
2020 unsigned real_level = texture->nr_samples > 1 ? 0 : level;
2021
2022 assert(texture->target != PIPE_BUFFER);
2023 assert(!(texture->flags & SI_RESOURCE_FLAG_FORCE_LINEAR));
2024 assert(box->width && box->height && box->depth);
2025
2026 if (tex->buffer.b.b.flags & SI_RESOURCE_AUX_PLANE)
2027 return NULL;
2028
2029 if ((tex->buffer.flags & RADEON_FLAG_ENCRYPTED) && usage & PIPE_MAP_READ)
2030 return NULL;
2031
2032 if (tex->is_depth || tex->buffer.flags & RADEON_FLAG_SPARSE) {
2033 /* Depth and sparse textures use staging unconditionally. */
2034 use_staging_texture = true;
2035 } else {
2036 /* Degrade the tile mode if we get too many transfers on APUs.
2037 * On dGPUs, the staging texture is always faster.
2038 * Only count uploads that are at least 4x4 pixels large.
2039 */
2040 if (!sctx->screen->info.has_dedicated_vram && real_level == 0 && box->width >= 4 &&
2041 box->height >= 4 && p_atomic_inc_return(&tex->num_level0_transfers) == 10) {
2042 bool can_invalidate = si_can_invalidate_texture(sctx->screen, tex, usage, box);
2043
2044 si_reallocate_texture_inplace(sctx, tex, PIPE_BIND_LINEAR, can_invalidate);
2045 }
2046
2047 /* Tiled textures need to be converted into a linear texture for CPU
2048 * access. The staging texture is always linear and is placed in GART.
2049 *
2050 * dGPU use a staging texture for VRAM, so that we don't map it and
2051 * don't relocate it to GTT.
2052 *
2053 * Reading from VRAM or GTT WC is slow, always use the staging
2054 * texture in this case.
2055 *
2056 * Use the staging texture for uploads if the underlying BO
2057 * is busy.
2058 */
2059 if (!tex->surface.is_linear || (tex->buffer.flags & RADEON_FLAG_ENCRYPTED) ||
2060 (tex->buffer.domains & RADEON_DOMAIN_VRAM && sctx->screen->info.has_dedicated_vram))
2061 use_staging_texture = true;
2062 else if (usage & PIPE_MAP_READ)
2063 use_staging_texture =
2064 tex->buffer.domains & RADEON_DOMAIN_VRAM || tex->buffer.flags & RADEON_FLAG_GTT_WC;
2065 /* Write & linear only: */
2066 else if (si_cs_is_buffer_referenced(sctx, tex->buffer.buf, RADEON_USAGE_READWRITE) ||
2067 !sctx->ws->buffer_wait(sctx->ws, tex->buffer.buf, 0,
2068 RADEON_USAGE_READWRITE | RADEON_USAGE_DISALLOW_SLOW_REPLY)) {
2069 /* It's busy. */
2070 if (si_can_invalidate_texture(sctx->screen, tex, usage, box))
2071 si_texture_invalidate_storage(sctx, tex);
2072 else
2073 use_staging_texture = true;
2074 }
2075 }
2076
2077 trans = CALLOC_STRUCT(si_transfer);
2078 if (!trans)
2079 return NULL;
2080 pipe_resource_reference(&trans->b.b.resource, texture);
2081 trans->b.b.level = level;
2082 trans->b.b.usage = usage;
2083 trans->b.b.box = *box;
2084
2085 if (use_staging_texture) {
2086 struct pipe_resource resource;
2087 struct si_texture *staging;
2088 unsigned bo_usage = usage & PIPE_MAP_READ ? PIPE_USAGE_STAGING : PIPE_USAGE_STREAM;
2089 unsigned bo_flags = SI_RESOURCE_FLAG_FORCE_LINEAR | SI_RESOURCE_FLAG_DRIVER_INTERNAL;
2090
2091 si_init_temp_resource_from_box(&resource, texture, box, real_level, bo_usage,
2092 bo_flags);
2093
2094 /* Since depth-stencil textures don't support linear tiling,
2095 * blit from ZS to color and vice versa. u_blitter will do
2096 * the packing for these formats.
2097 */
2098 if (tex->is_depth)
2099 resource.format = util_blitter_get_color_format_for_zs(resource.format);
2100
2101 /* Create the temporary texture. */
2102 staging = (struct si_texture *)ctx->screen->resource_create(ctx->screen, &resource);
2103 if (!staging) {
2104 PRINT_ERR("failed to create temporary texture to hold untiled copy\n");
2105 goto fail_trans;
2106 }
2107 trans->staging = &staging->buffer;
2108
2109 /* Just get the strides. */
2110 si_texture_get_offset(sctx->screen, staging, 0, NULL, &trans->b.b.stride,
2111 &trans->b.b.layer_stride);
2112
2113 if (usage & PIPE_MAP_READ)
2114 si_copy_to_staging_texture(ctx, trans);
2115 else
2116 usage |= PIPE_MAP_UNSYNCHRONIZED;
2117
2118 buf = trans->staging;
2119 } else {
2120 /* the resource is mapped directly */
2121 offset = si_texture_get_offset(sctx->screen, tex, real_level, box, &trans->b.b.stride,
2122 &trans->b.b.layer_stride);
2123 buf = &tex->buffer;
2124 }
2125
2126 /* Always unmap texture CPU mappings on 32-bit architectures, so that
2127 * we don't run out of the CPU address space.
2128 */
2129 if (sizeof(void *) == 4)
2130 usage |= RADEON_MAP_TEMPORARY;
2131
2132 if (!(map = si_buffer_map(sctx, buf, usage)))
2133 goto fail_trans;
2134
2135 *ptransfer = &trans->b.b;
2136 return map + offset;
2137
2138 fail_trans:
2139 si_resource_reference(&trans->staging, NULL);
2140 pipe_resource_reference(&trans->b.b.resource, NULL);
2141 FREE(trans);
2142 return NULL;
2143 }
2144
si_texture_transfer_unmap(struct pipe_context * ctx,struct pipe_transfer * transfer)2145 static void si_texture_transfer_unmap(struct pipe_context *ctx, struct pipe_transfer *transfer)
2146 {
2147 struct si_context *sctx = (struct si_context *)ctx;
2148 struct si_transfer *stransfer = (struct si_transfer *)transfer;
2149 struct pipe_resource *texture = transfer->resource;
2150 struct si_texture *tex = (struct si_texture *)texture;
2151
2152 /* Always unmap texture CPU mappings on 32-bit architectures, so that
2153 * we don't run out of the CPU address space.
2154 */
2155 if (sizeof(void *) == 4) {
2156 struct si_resource *buf = stransfer->staging ? stransfer->staging : &tex->buffer;
2157
2158 sctx->ws->buffer_unmap(sctx->ws, buf->buf);
2159 }
2160
2161 if ((transfer->usage & PIPE_MAP_WRITE) && stransfer->staging)
2162 si_copy_from_staging_texture(ctx, stransfer);
2163
2164 if (stransfer->staging) {
2165 sctx->num_alloc_tex_transfer_bytes += stransfer->staging->buf->size;
2166 si_resource_reference(&stransfer->staging, NULL);
2167 }
2168
2169 /* Heuristic for {upload, draw, upload, draw, ..}:
2170 *
2171 * Flush the gfx IB if we've allocated too much texture storage.
2172 *
2173 * The idea is that we don't want to build IBs that use too much
2174 * memory and put pressure on the kernel memory manager and we also
2175 * want to make temporary and invalidated buffers go idle ASAP to
2176 * decrease the total memory usage or make them reusable. The memory
2177 * usage will be slightly higher than given here because of the buffer
2178 * cache in the winsys.
2179 *
2180 * The result is that the kernel memory manager is never a bottleneck.
2181 */
2182 if (sctx->num_alloc_tex_transfer_bytes > (uint64_t)sctx->screen->info.gart_size_kb * 1024 / 4) {
2183 si_flush_gfx_cs(sctx, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
2184 sctx->num_alloc_tex_transfer_bytes = 0;
2185 }
2186
2187 pipe_resource_reference(&transfer->resource, NULL);
2188 FREE(transfer);
2189 }
2190
2191 /* Return if it's allowed to reinterpret one format as another with DCC enabled.
2192 */
vi_dcc_formats_compatible(struct si_screen * sscreen,enum pipe_format format1,enum pipe_format format2)2193 bool vi_dcc_formats_compatible(struct si_screen *sscreen, enum pipe_format format1,
2194 enum pipe_format format2)
2195 {
2196 const struct util_format_description *desc1, *desc2;
2197
2198 /* All formats are compatible on GFX11. */
2199 if (sscreen->info.gfx_level >= GFX11)
2200 return true;
2201
2202 /* No format change - exit early. */
2203 if (format1 == format2)
2204 return true;
2205
2206 format1 = ac_simplify_cb_format(format1);
2207 format2 = ac_simplify_cb_format(format2);
2208
2209 /* Check again after format adjustments. */
2210 if (format1 == format2)
2211 return true;
2212
2213 desc1 = util_format_description(format1);
2214 desc2 = util_format_description(format2);
2215
2216 if (desc1->layout != UTIL_FORMAT_LAYOUT_PLAIN || desc2->layout != UTIL_FORMAT_LAYOUT_PLAIN)
2217 return false;
2218
2219 /* Float and non-float are totally incompatible. */
2220 if ((desc1->channel[0].type == UTIL_FORMAT_TYPE_FLOAT) !=
2221 (desc2->channel[0].type == UTIL_FORMAT_TYPE_FLOAT))
2222 return false;
2223
2224 /* Channel sizes must match across DCC formats.
2225 * Comparing just the first 2 channels should be enough.
2226 */
2227 if (desc1->channel[0].size != desc2->channel[0].size ||
2228 (desc1->nr_channels >= 2 && desc1->channel[1].size != desc2->channel[1].size))
2229 return false;
2230
2231 /* Everything below is not needed if the driver never uses the DCC
2232 * clear code with the value of 1.
2233 */
2234
2235 /* If the clear values are all 1 or all 0, this constraint can be
2236 * ignored. */
2237 if (ac_alpha_is_on_msb(&sscreen->info, format1) != ac_alpha_is_on_msb(&sscreen->info, format2))
2238 return false;
2239
2240 /* Channel types must match if the clear value of 1 is used.
2241 * The type categories are only float, signed, unsigned.
2242 * NORM and INT are always compatible.
2243 */
2244 if (desc1->channel[0].type != desc2->channel[0].type ||
2245 (desc1->nr_channels >= 2 && desc1->channel[1].type != desc2->channel[1].type))
2246 return false;
2247
2248 return true;
2249 }
2250
vi_dcc_formats_are_incompatible(struct pipe_resource * tex,unsigned level,enum pipe_format view_format)2251 bool vi_dcc_formats_are_incompatible(struct pipe_resource *tex, unsigned level,
2252 enum pipe_format view_format)
2253 {
2254 struct si_texture *stex = (struct si_texture *)tex;
2255
2256 return vi_dcc_enabled(stex, level) &&
2257 !vi_dcc_formats_compatible(si_screen(tex->screen), tex->format, view_format);
2258 }
2259
2260 /* This can't be merged with the above function, because
2261 * vi_dcc_formats_compatible should be called only when DCC is enabled. */
vi_disable_dcc_if_incompatible_format(struct si_context * sctx,struct pipe_resource * tex,unsigned level,enum pipe_format view_format)2262 void vi_disable_dcc_if_incompatible_format(struct si_context *sctx, struct pipe_resource *tex,
2263 unsigned level, enum pipe_format view_format)
2264 {
2265 struct si_texture *stex = (struct si_texture *)tex;
2266
2267 if (vi_dcc_formats_are_incompatible(tex, level, view_format))
2268 if (!si_texture_disable_dcc(sctx, stex))
2269 si_decompress_dcc(sctx, stex);
2270 }
2271
si_create_surface(struct pipe_context * pipe,struct pipe_resource * tex,const struct pipe_surface * templ)2272 static struct pipe_surface *si_create_surface(struct pipe_context *pipe, struct pipe_resource *tex,
2273 const struct pipe_surface *templ)
2274 {
2275 unsigned level = templ->u.tex.level;
2276 unsigned width = u_minify(tex->width0, level);
2277 unsigned height = u_minify(tex->height0, level);
2278 unsigned width0 = tex->width0;
2279 unsigned height0 = tex->height0;
2280
2281 if (tex->target != PIPE_BUFFER && templ->format != tex->format) {
2282 const struct util_format_description *tex_desc = util_format_description(tex->format);
2283 const struct util_format_description *templ_desc = util_format_description(templ->format);
2284
2285 assert(tex_desc->block.bits == templ_desc->block.bits);
2286
2287 /* Adjust size of surface if and only if the block width or
2288 * height is changed. */
2289 if (tex_desc->block.width != templ_desc->block.width ||
2290 tex_desc->block.height != templ_desc->block.height) {
2291 unsigned nblks_x = util_format_get_nblocksx(tex->format, width);
2292 unsigned nblks_y = util_format_get_nblocksy(tex->format, height);
2293
2294 width = nblks_x * templ_desc->block.width;
2295 height = nblks_y * templ_desc->block.height;
2296
2297 width0 = util_format_get_nblocksx(tex->format, width0);
2298 height0 = util_format_get_nblocksy(tex->format, height0);
2299 }
2300 }
2301
2302 struct si_surface *surface = CALLOC_STRUCT(si_surface);
2303
2304 if (!surface)
2305 return NULL;
2306
2307 assert(templ->u.tex.first_layer <= util_max_layer(tex, templ->u.tex.level));
2308 assert(templ->u.tex.last_layer <= util_max_layer(tex, templ->u.tex.level));
2309
2310 pipe_reference_init(&surface->base.reference, 1);
2311 pipe_resource_reference(&surface->base.texture, tex);
2312 surface->base.context = pipe;
2313 surface->base.format = templ->format;
2314 surface->base.width = width;
2315 surface->base.height = height;
2316 surface->base.u = templ->u;
2317
2318 surface->width0 = width0;
2319 surface->height0 = height0;
2320
2321 surface->dcc_incompatible =
2322 tex->target != PIPE_BUFFER &&
2323 vi_dcc_formats_are_incompatible(tex, templ->u.tex.level, templ->format);
2324 return &surface->base;
2325 }
2326
si_surface_destroy(struct pipe_context * pipe,struct pipe_surface * surface)2327 static void si_surface_destroy(struct pipe_context *pipe, struct pipe_surface *surface)
2328 {
2329 pipe_resource_reference(&surface->texture, NULL);
2330 FREE(surface);
2331 }
2332
2333 static struct pipe_memory_object *
si_memobj_from_handle(struct pipe_screen * screen,struct winsys_handle * whandle,bool dedicated)2334 si_memobj_from_handle(struct pipe_screen *screen, struct winsys_handle *whandle, bool dedicated)
2335 {
2336 struct si_screen *sscreen = (struct si_screen *)screen;
2337 struct si_memory_object *memobj = CALLOC_STRUCT(si_memory_object);
2338 struct pb_buffer_lean *buf = NULL;
2339
2340 if (!memobj)
2341 return NULL;
2342
2343 buf = sscreen->ws->buffer_from_handle(sscreen->ws, whandle, sscreen->info.max_alignment, false);
2344 if (!buf) {
2345 free(memobj);
2346 return NULL;
2347 }
2348
2349 memobj->b.dedicated = dedicated;
2350 memobj->buf = buf;
2351 memobj->stride = whandle->stride;
2352
2353 return (struct pipe_memory_object *)memobj;
2354 }
2355
si_memobj_destroy(struct pipe_screen * screen,struct pipe_memory_object * _memobj)2356 static void si_memobj_destroy(struct pipe_screen *screen, struct pipe_memory_object *_memobj)
2357 {
2358 struct si_memory_object *memobj = (struct si_memory_object *)_memobj;
2359
2360 radeon_bo_reference(((struct si_screen*)screen)->ws, &memobj->buf, NULL);
2361 free(memobj);
2362 }
2363
si_resource_from_memobj(struct pipe_screen * screen,const struct pipe_resource * templ,struct pipe_memory_object * _memobj,uint64_t offset)2364 static struct pipe_resource *si_resource_from_memobj(struct pipe_screen *screen,
2365 const struct pipe_resource *templ,
2366 struct pipe_memory_object *_memobj,
2367 uint64_t offset)
2368 {
2369 struct si_screen *sscreen = (struct si_screen *)screen;
2370 struct si_memory_object *memobj = (struct si_memory_object *)_memobj;
2371 struct pipe_resource *res;
2372
2373 if (templ->target == PIPE_BUFFER)
2374 res = si_buffer_from_winsys_buffer(screen, templ, memobj->buf, offset);
2375 else
2376 res = si_texture_from_winsys_buffer(sscreen, templ, memobj->buf,
2377 memobj->stride,
2378 offset, DRM_FORMAT_MOD_INVALID,
2379 PIPE_HANDLE_USAGE_FRAMEBUFFER_WRITE | PIPE_HANDLE_USAGE_SHADER_WRITE,
2380 memobj->b.dedicated);
2381
2382 if (!res)
2383 return NULL;
2384
2385 /* si_texture_from_winsys_buffer doesn't increment refcount of
2386 * memobj->buf, so increment it here.
2387 */
2388 struct pb_buffer_lean *buf = NULL;
2389 radeon_bo_reference(sscreen->ws, &buf, memobj->buf);
2390 return res;
2391 }
2392
si_check_resource_capability(struct pipe_screen * screen,struct pipe_resource * resource,unsigned bind)2393 static bool si_check_resource_capability(struct pipe_screen *screen, struct pipe_resource *resource,
2394 unsigned bind)
2395 {
2396 struct si_texture *tex = (struct si_texture *)resource;
2397
2398 /* Buffers only support the linear flag. */
2399 if (resource->target == PIPE_BUFFER)
2400 return (bind & ~PIPE_BIND_LINEAR) == 0;
2401
2402 if (bind & PIPE_BIND_LINEAR && !tex->surface.is_linear)
2403 return false;
2404
2405 if (bind & PIPE_BIND_SCANOUT && !tex->surface.is_displayable)
2406 return false;
2407
2408 /* TODO: PIPE_BIND_CURSOR - do we care? */
2409 return true;
2410 }
2411
si_get_sparse_texture_virtual_page_size(struct pipe_screen * screen,enum pipe_texture_target target,bool multi_sample,enum pipe_format format,unsigned offset,unsigned size,int * x,int * y,int * z)2412 static int si_get_sparse_texture_virtual_page_size(struct pipe_screen *screen,
2413 enum pipe_texture_target target,
2414 bool multi_sample,
2415 enum pipe_format format,
2416 unsigned offset, unsigned size,
2417 int *x, int *y, int *z)
2418 {
2419 struct si_screen *sscreen = (struct si_screen *)screen;
2420
2421 /* Only support one type of page size. */
2422 if (offset != 0)
2423 return 0;
2424
2425 static const int page_size_2d[][3] = {
2426 { 256, 256, 1 }, /* 8bpp */
2427 { 256, 128, 1 }, /* 16bpp */
2428 { 128, 128, 1 }, /* 32bpp */
2429 { 128, 64, 1 }, /* 64bpp */
2430 { 64, 64, 1 }, /* 128bpp */
2431 };
2432 static const int page_size_3d[][3] = {
2433 { 64, 32, 32 }, /* 8bpp */
2434 { 32, 32, 32 }, /* 16bpp */
2435 { 32, 32, 16 }, /* 32bpp */
2436 { 32, 16, 16 }, /* 64bpp */
2437 { 16, 16, 16 }, /* 128bpp */
2438 };
2439
2440 const int (*page_sizes)[3];
2441
2442 /* Supported targets. */
2443 switch (target) {
2444 case PIPE_TEXTURE_2D:
2445 case PIPE_TEXTURE_CUBE:
2446 case PIPE_TEXTURE_RECT:
2447 case PIPE_TEXTURE_2D_ARRAY:
2448 case PIPE_TEXTURE_CUBE_ARRAY:
2449 page_sizes = page_size_2d;
2450 break;
2451 case PIPE_TEXTURE_3D:
2452 page_sizes = page_size_3d;
2453 break;
2454 default:
2455 return 0;
2456 }
2457
2458 /* ARB_sparse_texture2 need to query supported virtual page x/y/z without
2459 * knowing the actual sample count. So we need to return a fixed virtual page
2460 * x/y/z for all sample count which means the virtual page size can not be fixed
2461 * to 64KB.
2462 *
2463 * Only enabled for GFX9. GFX10+ removed MS texture support. By specification
2464 * ARB_sparse_texture2 need MS texture support, but we relax it by just return
2465 * no page size for GFX10+ to keep shader query capbility.
2466 */
2467 if (multi_sample && sscreen->info.gfx_level != GFX9)
2468 return 0;
2469
2470 /* Unsupported formats. */
2471 /* TODO: support these formats. */
2472 if (util_format_is_depth_or_stencil(format) ||
2473 util_format_get_num_planes(format) > 1 ||
2474 util_format_is_compressed(format))
2475 return 0;
2476
2477 int blk_size = util_format_get_blocksize(format);
2478 /* We don't support any non-power-of-two bpp formats, so
2479 * pipe_screen->is_format_supported() should already filter out these formats.
2480 */
2481 assert(util_is_power_of_two_nonzero(blk_size));
2482
2483 if (size) {
2484 unsigned index = util_logbase2(blk_size);
2485 if (x) *x = page_sizes[index][0];
2486 if (y) *y = page_sizes[index][1];
2487 if (z) *z = page_sizes[index][2];
2488 }
2489
2490 return 1;
2491 }
2492
si_init_screen_texture_functions(struct si_screen * sscreen)2493 void si_init_screen_texture_functions(struct si_screen *sscreen)
2494 {
2495 sscreen->b.resource_from_handle = si_texture_from_handle;
2496 sscreen->b.resource_get_handle = si_texture_get_handle;
2497 sscreen->b.resource_get_param = si_resource_get_param;
2498 sscreen->b.resource_get_info = si_texture_get_info;
2499 sscreen->b.resource_from_memobj = si_resource_from_memobj;
2500 sscreen->b.memobj_create_from_handle = si_memobj_from_handle;
2501 sscreen->b.memobj_destroy = si_memobj_destroy;
2502 sscreen->b.check_resource_capability = si_check_resource_capability;
2503 sscreen->b.get_sparse_texture_virtual_page_size =
2504 si_get_sparse_texture_virtual_page_size;
2505
2506 /* By not setting it the frontend will fall back to non-modifier create,
2507 * which works around some applications using modifiers that are not
2508 * allowed in combination with lack of error reporting in
2509 * gbm_dri_surface_create */
2510 if (sscreen->info.gfx_level >= GFX9 && sscreen->info.kernel_has_modifiers) {
2511 sscreen->b.resource_create_with_modifiers = si_texture_create_with_modifiers;
2512 sscreen->b.query_dmabuf_modifiers = si_query_dmabuf_modifiers;
2513 sscreen->b.is_dmabuf_modifier_supported = si_is_dmabuf_modifier_supported;
2514 sscreen->b.get_dmabuf_modifier_planes = si_get_dmabuf_modifier_planes;
2515 }
2516 }
2517
si_init_context_texture_functions(struct si_context * sctx)2518 void si_init_context_texture_functions(struct si_context *sctx)
2519 {
2520 sctx->b.texture_map = si_texture_transfer_map;
2521 sctx->b.texture_unmap = si_texture_transfer_unmap;
2522 sctx->b.create_surface = si_create_surface;
2523 sctx->b.surface_destroy = si_surface_destroy;
2524 }
2525