1 /*
2 * Copyright (C) 2008 VMware, Inc.
3 * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org>
4 * Copyright (C) 2014-2017 Broadcom
5 * Copyright (C) 2018-2019 Alyssa Rosenzweig
6 * Copyright (C) 2019 Collabora, Ltd.
7 * Copyright (C) 2023 Amazon.com, Inc. or its affiliates
8 *
9 * Permission is hereby granted, free of charge, to any person obtaining a
10 * copy of this software and associated documentation files (the "Software"),
11 * to deal in the Software without restriction, including without limitation
12 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
13 * and/or sell copies of the Software, and to permit persons to whom the
14 * Software is furnished to do so, subject to the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the next
17 * paragraph) shall be included in all copies or substantial portions of the
18 * Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26 * SOFTWARE.
27 *
28 * Authors (Collabora):
29 * Tomeu Vizoso <tomeu.vizoso@collabora.com>
30 * Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
31 *
32 */
33
34 #include <fcntl.h>
35 #include <xf86drm.h>
36 #include "drm-uapi/drm_fourcc.h"
37
38 #include "frontend/winsys_handle.h"
39 #include "util/format/u_format.h"
40 #include "util/u_debug_image.h"
41 #include "util/u_drm.h"
42 #include "util/u_gen_mipmap.h"
43 #include "util/u_memory.h"
44 #include "util/u_resource.h"
45 #include "util/u_surface.h"
46 #include "util/u_transfer.h"
47 #include "util/u_transfer_helper.h"
48
49 #include "decode.h"
50 #include "pan_bo.h"
51 #include "pan_context.h"
52 #include "pan_resource.h"
53 #include "pan_screen.h"
54 #include "pan_tiling.h"
55 #include "pan_util.h"
56
57 static void
panfrost_clear_depth_stencil(struct pipe_context * pipe,struct pipe_surface * dst,unsigned clear_flags,double depth,unsigned stencil,unsigned dstx,unsigned dsty,unsigned width,unsigned height,bool render_condition_enabled)58 panfrost_clear_depth_stencil(struct pipe_context *pipe,
59 struct pipe_surface *dst, unsigned clear_flags,
60 double depth, unsigned stencil, unsigned dstx,
61 unsigned dsty, unsigned width, unsigned height,
62 bool render_condition_enabled)
63 {
64 struct panfrost_context *ctx = pan_context(pipe);
65
66 if (render_condition_enabled && !panfrost_render_condition_check(ctx))
67 return;
68
69 panfrost_blitter_save(
70 ctx, render_condition_enabled ? PAN_RENDER_COND : PAN_RENDER_BASE);
71 util_blitter_clear_depth_stencil(ctx->blitter, dst, clear_flags, depth,
72 stencil, dstx, dsty, width, height);
73 }
74
75 static void
panfrost_clear_render_target(struct pipe_context * pipe,struct pipe_surface * dst,const union pipe_color_union * color,unsigned dstx,unsigned dsty,unsigned width,unsigned height,bool render_condition_enabled)76 panfrost_clear_render_target(struct pipe_context *pipe,
77 struct pipe_surface *dst,
78 const union pipe_color_union *color, unsigned dstx,
79 unsigned dsty, unsigned width, unsigned height,
80 bool render_condition_enabled)
81 {
82 struct panfrost_context *ctx = pan_context(pipe);
83
84 if (render_condition_enabled && !panfrost_render_condition_check(ctx))
85 return;
86
87 panfrost_blitter_save(
88 ctx, render_condition_enabled ? PAN_RENDER_COND : PAN_RENDER_BASE);
89 util_blitter_clear_render_target(ctx->blitter, dst, color, dstx, dsty, width,
90 height);
91 }
92
93 static struct pipe_resource *
panfrost_resource_from_handle(struct pipe_screen * pscreen,const struct pipe_resource * templat,struct winsys_handle * whandle,unsigned usage)94 panfrost_resource_from_handle(struct pipe_screen *pscreen,
95 const struct pipe_resource *templat,
96 struct winsys_handle *whandle, unsigned usage)
97 {
98 struct panfrost_device *dev = pan_device(pscreen);
99 struct panfrost_resource *rsc;
100 struct pipe_resource *prsc;
101
102 assert(whandle->type == WINSYS_HANDLE_TYPE_FD);
103
104 rsc = CALLOC_STRUCT(panfrost_resource);
105 if (!rsc)
106 return NULL;
107
108 prsc = &rsc->base;
109
110 *prsc = *templat;
111
112 pipe_reference_init(&prsc->reference, 1);
113 prsc->screen = pscreen;
114
115 uint64_t mod = whandle->modifier == DRM_FORMAT_MOD_INVALID
116 ? DRM_FORMAT_MOD_LINEAR
117 : whandle->modifier;
118 enum mali_texture_dimension dim =
119 panfrost_translate_texture_dimension(templat->target);
120 struct pan_image_explicit_layout explicit_layout = {
121 .offset = whandle->offset,
122 .row_stride =
123 panfrost_from_legacy_stride(whandle->stride, templat->format, mod),
124 };
125
126 rsc->image.layout = (struct pan_image_layout){
127 .modifier = mod,
128 .format = templat->format,
129 .dim = dim,
130 .width = prsc->width0,
131 .height = prsc->height0,
132 .depth = prsc->depth0,
133 .array_size = prsc->array_size,
134 .nr_samples = MAX2(prsc->nr_samples, 1),
135 .nr_slices = 1,
136 };
137
138 bool valid =
139 pan_image_layout_init(dev->arch, &rsc->image.layout, &explicit_layout);
140
141 if (!valid) {
142 FREE(rsc);
143 return NULL;
144 }
145
146 rsc->bo = panfrost_bo_import(dev, whandle->handle);
147 /* Sometimes an import can fail e.g. on an invalid buffer fd, out of
148 * memory space to mmap it etc.
149 */
150 if (!rsc->bo) {
151 FREE(rsc);
152 return NULL;
153 }
154
155 rsc->image.data.base = rsc->bo->ptr.gpu;
156 rsc->modifier_constant = true;
157
158 BITSET_SET(rsc->valid.data, 0);
159 panfrost_resource_set_damage_region(pscreen, &rsc->base, 0, NULL);
160
161 if (dev->ro) {
162 rsc->scanout =
163 renderonly_create_gpu_import_for_resource(prsc, dev->ro, NULL);
164 /* failure is expected in some cases.. */
165 }
166
167 return prsc;
168 }
169
170 static bool
panfrost_resource_get_handle(struct pipe_screen * pscreen,struct pipe_context * ctx,struct pipe_resource * pt,struct winsys_handle * handle,unsigned usage)171 panfrost_resource_get_handle(struct pipe_screen *pscreen,
172 struct pipe_context *ctx, struct pipe_resource *pt,
173 struct winsys_handle *handle, unsigned usage)
174 {
175 struct panfrost_device *dev = pan_device(pscreen);
176 struct panfrost_resource *rsrc;
177 struct renderonly_scanout *scanout;
178 struct pipe_resource *plane_res =
179 util_resource_at_index(pt, handle->plane);
180
181 if (!plane_res)
182 return false;
183
184 rsrc = pan_resource(plane_res);
185 scanout = rsrc->scanout;
186
187 handle->modifier = rsrc->image.layout.modifier;
188 rsrc->modifier_constant = true;
189
190 if (handle->type == WINSYS_HANDLE_TYPE_KMS && dev->ro) {
191 return renderonly_get_handle(scanout, handle);
192 } else if (handle->type == WINSYS_HANDLE_TYPE_KMS) {
193 handle->handle = panfrost_bo_handle(rsrc->bo);
194 } else if (handle->type == WINSYS_HANDLE_TYPE_FD) {
195 int fd = panfrost_bo_export(rsrc->bo);
196
197 if (fd < 0)
198 return false;
199
200 handle->handle = fd;
201 } else {
202 /* Other handle types not supported */
203 return false;
204 }
205
206 handle->stride = panfrost_get_legacy_stride(&rsrc->image.layout, 0);
207 handle->offset = rsrc->image.layout.slices[0].offset;
208 return true;
209 }
210
211 static bool
panfrost_resource_get_param(struct pipe_screen * pscreen,struct pipe_context * pctx,struct pipe_resource * prsc,unsigned plane,unsigned layer,unsigned level,enum pipe_resource_param param,unsigned usage,uint64_t * value)212 panfrost_resource_get_param(struct pipe_screen *pscreen,
213 struct pipe_context *pctx,
214 struct pipe_resource *prsc, unsigned plane,
215 unsigned layer, unsigned level,
216 enum pipe_resource_param param, unsigned usage,
217 uint64_t *value)
218 {
219 struct pipe_resource *plane_res = util_resource_at_index(prsc, plane);
220 struct panfrost_resource *rsrc = pan_resource(plane_res);
221
222 switch (param) {
223 case PIPE_RESOURCE_PARAM_STRIDE:
224 *value = panfrost_get_legacy_stride(&rsrc->image.layout, level);
225 return true;
226 case PIPE_RESOURCE_PARAM_OFFSET:
227 *value = rsrc->image.layout.slices[level].offset;
228 return true;
229 case PIPE_RESOURCE_PARAM_MODIFIER:
230 *value = rsrc->image.layout.modifier;
231 return true;
232 case PIPE_RESOURCE_PARAM_NPLANES:
233 *value = util_resource_num(prsc);
234 return true;
235 default:
236 return false;
237 }
238 }
239
240 static void
panfrost_flush_resource(struct pipe_context * pctx,struct pipe_resource * prsc)241 panfrost_flush_resource(struct pipe_context *pctx, struct pipe_resource *prsc)
242 {
243 /* TODO */
244 }
245
246 static struct pipe_surface *
panfrost_create_surface(struct pipe_context * pipe,struct pipe_resource * pt,const struct pipe_surface * surf_tmpl)247 panfrost_create_surface(struct pipe_context *pipe, struct pipe_resource *pt,
248 const struct pipe_surface *surf_tmpl)
249 {
250 struct pipe_surface *ps = NULL;
251
252 ps = CALLOC_STRUCT(pipe_surface);
253
254 if (ps) {
255 pipe_reference_init(&ps->reference, 1);
256 pipe_resource_reference(&ps->texture, pt);
257 ps->context = pipe;
258 ps->format = surf_tmpl->format;
259
260 if (pt->target != PIPE_BUFFER) {
261 assert(surf_tmpl->u.tex.level <= pt->last_level);
262 ps->width = u_minify(pt->width0, surf_tmpl->u.tex.level);
263 ps->height = u_minify(pt->height0, surf_tmpl->u.tex.level);
264 ps->nr_samples = surf_tmpl->nr_samples;
265 ps->u.tex.level = surf_tmpl->u.tex.level;
266 ps->u.tex.first_layer = surf_tmpl->u.tex.first_layer;
267 ps->u.tex.last_layer = surf_tmpl->u.tex.last_layer;
268 } else {
269 /* setting width as number of elements should get us correct
270 * renderbuffer width */
271 ps->width =
272 surf_tmpl->u.buf.last_element - surf_tmpl->u.buf.first_element + 1;
273 ps->height = pt->height0;
274 ps->u.buf.first_element = surf_tmpl->u.buf.first_element;
275 ps->u.buf.last_element = surf_tmpl->u.buf.last_element;
276 assert(ps->u.buf.first_element <= ps->u.buf.last_element);
277 assert(ps->u.buf.last_element < ps->width);
278 }
279 }
280
281 return ps;
282 }
283
284 static void
panfrost_surface_destroy(struct pipe_context * pipe,struct pipe_surface * surf)285 panfrost_surface_destroy(struct pipe_context *pipe, struct pipe_surface *surf)
286 {
287 assert(surf->texture);
288 pipe_resource_reference(&surf->texture, NULL);
289 free(surf);
290 }
291
292 static inline bool
panfrost_is_2d(const struct panfrost_resource * pres)293 panfrost_is_2d(const struct panfrost_resource *pres)
294 {
295 return (pres->base.target == PIPE_TEXTURE_2D) ||
296 (pres->base.target == PIPE_TEXTURE_RECT);
297 }
298
299 /* Based on the usage, determine if it makes sense to use u-inteleaved tiling.
300 * We only have routines to tile 2D textures of sane bpps. On the hardware
301 * level, not all usages are valid for tiling. Finally, if the app is hinting
302 * that the contents frequently change, tiling will be a loss.
303 *
304 * On platforms where it is supported, AFBC is even better. */
305
306 static bool
panfrost_should_afbc(struct panfrost_device * dev,const struct panfrost_resource * pres,enum pipe_format fmt)307 panfrost_should_afbc(struct panfrost_device *dev,
308 const struct panfrost_resource *pres, enum pipe_format fmt)
309 {
310 /* AFBC resources may be rendered to, textured from, or shared across
311 * processes, but may not be used as e.g buffers */
312 const unsigned valid_binding =
313 PIPE_BIND_DEPTH_STENCIL | PIPE_BIND_RENDER_TARGET | PIPE_BIND_BLENDABLE |
314 PIPE_BIND_SAMPLER_VIEW | PIPE_BIND_DISPLAY_TARGET | PIPE_BIND_SCANOUT |
315 PIPE_BIND_SHARED;
316
317 if (pres->base.bind & ~valid_binding)
318 return false;
319
320 /* AFBC support is optional */
321 if (!dev->has_afbc)
322 return false;
323
324 /* AFBC<-->staging is expensive */
325 if (pres->base.usage == PIPE_USAGE_STREAM)
326 return false;
327
328 /* If constant (non-data-dependent) format is requested, don't AFBC: */
329 if (pres->base.bind & PIPE_BIND_CONST_BW)
330 return false;
331
332 /* Only a small selection of formats are AFBC'able */
333 if (!panfrost_format_supports_afbc(dev->arch, fmt))
334 return false;
335
336 /* AFBC does not support layered (GLES3 style) multisampling. Use
337 * EXT_multisampled_render_to_texture instead */
338 if (pres->base.nr_samples > 1)
339 return false;
340
341 switch (pres->base.target) {
342 case PIPE_TEXTURE_2D:
343 case PIPE_TEXTURE_RECT:
344 case PIPE_TEXTURE_2D_ARRAY:
345 case PIPE_TEXTURE_CUBE:
346 case PIPE_TEXTURE_CUBE_ARRAY:
347 break;
348
349 case PIPE_TEXTURE_3D:
350 /* 3D AFBC is only supported on Bifrost v7+. It's supposed to
351 * be supported on Midgard but it doesn't seem to work */
352 if (dev->arch != 7)
353 return false;
354
355 break;
356
357 default:
358 return false;
359 }
360
361 /* For one tile, AFBC is a loss compared to u-interleaved */
362 if (pres->base.width0 <= 16 && pres->base.height0 <= 16)
363 return false;
364
365 /* Otherwise, we'd prefer AFBC as it is dramatically more efficient
366 * than linear or usually even u-interleaved */
367 return true;
368 }
369
370 /*
371 * For a resource we want to use AFBC with, should we use AFBC with tiled
372 * headers? On GPUs that support it, this is believed to be beneficial for
373 * images that are at least 128x128.
374 */
375 static bool
panfrost_should_tile_afbc(const struct panfrost_device * dev,const struct panfrost_resource * pres)376 panfrost_should_tile_afbc(const struct panfrost_device *dev,
377 const struct panfrost_resource *pres)
378 {
379 return panfrost_afbc_can_tile(dev->arch) && pres->base.width0 >= 128 &&
380 pres->base.height0 >= 128 && !(dev->debug & PAN_DBG_FORCE_PACK);
381 }
382
383 bool
panfrost_should_pack_afbc(struct panfrost_device * dev,const struct panfrost_resource * prsrc)384 panfrost_should_pack_afbc(struct panfrost_device *dev,
385 const struct panfrost_resource *prsrc)
386 {
387 const unsigned valid_binding = PIPE_BIND_DEPTH_STENCIL |
388 PIPE_BIND_RENDER_TARGET |
389 PIPE_BIND_SAMPLER_VIEW;
390
391 return panfrost_afbc_can_pack(prsrc->base.format) && panfrost_is_2d(prsrc) &&
392 drm_is_afbc(prsrc->image.layout.modifier) &&
393 (prsrc->image.layout.modifier & AFBC_FORMAT_MOD_SPARSE) &&
394 !(prsrc->image.layout.modifier & AFBC_FORMAT_MOD_SPLIT) &&
395 (prsrc->base.bind & ~valid_binding) == 0 &&
396 !prsrc->modifier_constant && prsrc->base.array_size == 1 &&
397 prsrc->base.width0 >= 32 && prsrc->base.height0 >= 32 ;
398 }
399
400 static bool
panfrost_should_tile(struct panfrost_device * dev,const struct panfrost_resource * pres,enum pipe_format fmt)401 panfrost_should_tile(struct panfrost_device *dev,
402 const struct panfrost_resource *pres, enum pipe_format fmt)
403 {
404 const unsigned valid_binding =
405 PIPE_BIND_DEPTH_STENCIL | PIPE_BIND_RENDER_TARGET | PIPE_BIND_BLENDABLE |
406 PIPE_BIND_SAMPLER_VIEW | PIPE_BIND_DISPLAY_TARGET | PIPE_BIND_SCANOUT |
407 PIPE_BIND_SHARED;
408
409 /* The purpose of tiling is improving locality in both X- and
410 * Y-directions. If there is only a single pixel in either direction,
411 * tiling does not make sense; using a linear layout instead is optimal
412 * for both memory usage and performance.
413 */
414 if (MIN2(pres->base.width0, pres->base.height0) < 2)
415 return false;
416
417 bool can_tile = (pres->base.target != PIPE_BUFFER) &&
418 ((pres->base.bind & ~valid_binding) == 0);
419
420 return can_tile && (pres->base.usage != PIPE_USAGE_STREAM);
421 }
422
423 static bool
panfrost_should_afrc(struct panfrost_device * dev,const struct panfrost_resource * pres,enum pipe_format fmt)424 panfrost_should_afrc(struct panfrost_device *dev,
425 const struct panfrost_resource *pres, enum pipe_format fmt)
426 {
427 const unsigned valid_binding = PIPE_BIND_RENDER_TARGET |
428 PIPE_BIND_BLENDABLE | PIPE_BIND_SAMPLER_VIEW |
429 PIPE_BIND_DISPLAY_TARGET | PIPE_BIND_SHARED;
430
431 if (pres->base.bind & ~valid_binding)
432 return false;
433
434 /* AFRC support is optional */
435 if (!dev->has_afrc)
436 return false;
437
438 /* AFRC<-->staging is expensive */
439 if (pres->base.usage == PIPE_USAGE_STREAM)
440 return false;
441
442 /* Only a small selection of formats are AFRC'able */
443 if (!panfrost_format_supports_afrc(fmt))
444 return false;
445
446 /* AFRC does not support layered (GLES3 style) multisampling. Use
447 * EXT_multisampled_render_to_texture instead */
448 if (pres->base.nr_samples > 1)
449 return false;
450
451 switch (pres->base.target) {
452 case PIPE_TEXTURE_2D:
453 case PIPE_TEXTURE_RECT:
454 case PIPE_TEXTURE_2D_ARRAY:
455 case PIPE_TEXTURE_CUBE:
456 case PIPE_TEXTURE_CUBE_ARRAY:
457 case PIPE_TEXTURE_3D:
458 break;
459
460 default:
461 return false;
462 }
463
464 return true;
465 }
466
467 static uint64_t
panfrost_best_modifier(struct pipe_screen * pscreen,const struct panfrost_resource * pres,enum pipe_format fmt)468 panfrost_best_modifier(struct pipe_screen *pscreen,
469 const struct panfrost_resource *pres,
470 enum pipe_format fmt)
471 {
472 struct panfrost_screen *screen = pan_screen(pscreen);
473 struct panfrost_device *dev = pan_device(pscreen);
474
475 /* Force linear textures when debugging tiling/compression */
476 if (unlikely(dev->debug & PAN_DBG_LINEAR))
477 return DRM_FORMAT_MOD_LINEAR;
478
479 int afrc_rate = screen->force_afrc_rate;
480 if (afrc_rate < 0)
481 afrc_rate = pres->base.compression_rate;
482 if (afrc_rate > PIPE_COMPRESSION_FIXED_RATE_NONE &&
483 panfrost_should_afrc(dev, pres, fmt)) {
484 /* It's not really possible to decide on a global AFRC-rate,
485 * because the set of valid AFRC rates varies from format to
486 * format. So instead, treat this as a minimum rate, and search
487 * for the next valid one.
488 */
489 for (int i = afrc_rate; i < 12; ++i) {
490 if (panfrost_afrc_get_modifiers(fmt, i, 0, NULL)) {
491 afrc_rate = i;
492 break;
493 }
494 }
495 }
496
497 if (afrc_rate != PIPE_COMPRESSION_FIXED_RATE_NONE &&
498 panfrost_should_afrc(dev, pres, fmt)) {
499 uint64_t mod;
500 unsigned num_mods = 0;
501
502 STATIC_ASSERT(PIPE_COMPRESSION_FIXED_RATE_DEFAULT == PAN_AFRC_RATE_DEFAULT);
503 num_mods = panfrost_afrc_get_modifiers(fmt, afrc_rate, 1, &mod);
504 if (num_mods > 0) {
505 return mod;
506 }
507 }
508
509 if (panfrost_should_afbc(dev, pres, fmt)) {
510 uint64_t afbc = AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | AFBC_FORMAT_MOD_SPARSE;
511
512 if (panfrost_afbc_can_ytr(pres->base.format))
513 afbc |= AFBC_FORMAT_MOD_YTR;
514
515 if (panfrost_should_tile_afbc(dev, pres))
516 afbc |= AFBC_FORMAT_MOD_TILED | AFBC_FORMAT_MOD_SC;
517
518 return DRM_FORMAT_MOD_ARM_AFBC(afbc);
519 } else if (panfrost_should_tile(dev, pres, fmt))
520 return DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED;
521 else
522 return DRM_FORMAT_MOD_LINEAR;
523 }
524
525 static bool
panfrost_should_checksum(const struct panfrost_device * dev,const struct panfrost_resource * pres)526 panfrost_should_checksum(const struct panfrost_device *dev,
527 const struct panfrost_resource *pres)
528 {
529 /* When checksumming is enabled, the tile data must fit in the
530 * size of the writeback buffer, so don't checksum formats
531 * that use too much space. */
532
533 unsigned bytes_per_pixel_max = (dev->arch == 6) ? 6 : 4;
534
535 unsigned bytes_per_pixel = MAX2(pres->base.nr_samples, 1) *
536 util_format_get_blocksize(pres->base.format);
537
538 return pres->base.bind & PIPE_BIND_RENDER_TARGET && panfrost_is_2d(pres) &&
539 bytes_per_pixel <= bytes_per_pixel_max &&
540 pres->base.last_level == 0 && !(dev->debug & PAN_DBG_NO_CRC);
541 }
542
543 static void
panfrost_resource_setup(struct pipe_screen * screen,struct panfrost_resource * pres,uint64_t modifier,enum pipe_format fmt)544 panfrost_resource_setup(struct pipe_screen *screen,
545 struct panfrost_resource *pres, uint64_t modifier,
546 enum pipe_format fmt)
547 {
548 struct panfrost_device *dev = pan_device(screen);
549 uint64_t chosen_mod = modifier != DRM_FORMAT_MOD_INVALID
550 ? modifier
551 : panfrost_best_modifier(screen, pres, fmt);
552 enum mali_texture_dimension dim =
553 panfrost_translate_texture_dimension(pres->base.target);
554
555 /* We can only switch tiled->linear if the resource isn't already
556 * linear and if we control the modifier */
557 pres->modifier_constant = !(chosen_mod != DRM_FORMAT_MOD_LINEAR &&
558 modifier == DRM_FORMAT_MOD_INVALID);
559
560 /* Z32_S8X24 variants are actually stored in 2 planes (one per
561 * component), we have to adjust the format on the first plane.
562 */
563 if (fmt == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT)
564 fmt = PIPE_FORMAT_Z32_FLOAT;
565
566 pres->image.layout = (struct pan_image_layout){
567 .modifier = chosen_mod,
568 .format = fmt,
569 .dim = dim,
570 .width = pres->base.width0,
571 .height = pres->base.height0,
572 .depth = pres->base.depth0,
573 .array_size = pres->base.array_size,
574 .nr_samples = MAX2(pres->base.nr_samples, 1),
575 .nr_slices = pres->base.last_level + 1,
576 .crc = panfrost_should_checksum(dev, pres),
577 };
578
579 /* Update the compression rate with the correct value as we
580 * want the real bitrate and not DEFAULT */
581 pres->base.compression_rate = panfrost_afrc_get_rate(fmt, chosen_mod);
582
583 ASSERTED bool valid =
584 pan_image_layout_init(dev->arch, &pres->image.layout, NULL);
585 assert(valid);
586 }
587
588 static int
panfrost_resource_init_afbc_headers(struct panfrost_resource * pres)589 panfrost_resource_init_afbc_headers(struct panfrost_resource *pres)
590 {
591 if (panfrost_bo_mmap(pres->bo))
592 return -1;
593
594 unsigned nr_samples = MAX2(pres->base.nr_samples, 1);
595
596 for (unsigned i = 0; i < pres->base.array_size; ++i) {
597 for (unsigned l = 0; l <= pres->base.last_level; ++l) {
598 struct pan_image_slice_layout *slice = &pres->image.layout.slices[l];
599
600 for (unsigned s = 0; s < nr_samples; ++s) {
601 void *ptr = pres->bo->ptr.cpu +
602 (i * pres->image.layout.array_stride) + slice->offset +
603 (s * slice->afbc.surface_stride);
604
605 /* Zero-ed AFBC headers seem to encode a plain
606 * black. Let's use this pattern to keep the
607 * initialization simple.
608 */
609 memset(ptr, 0, slice->afbc.header_size);
610 }
611 }
612 }
613 return 0;
614 }
615
616 void
panfrost_resource_set_damage_region(struct pipe_screen * screen,struct pipe_resource * res,unsigned int nrects,const struct pipe_box * rects)617 panfrost_resource_set_damage_region(struct pipe_screen *screen,
618 struct pipe_resource *res,
619 unsigned int nrects,
620 const struct pipe_box *rects)
621 {
622 struct panfrost_device *dev = pan_device(screen);
623 struct panfrost_resource *pres = pan_resource(res);
624 struct pipe_scissor_state *damage_extent = &pres->damage.extent;
625 unsigned int i;
626
627 /* Partial updates are implemented with a tile enable map only on v5.
628 * Later architectures have a more efficient method of implementing
629 * partial updates (frame shaders), while earlier architectures lack
630 * tile enable maps altogether.
631 */
632 if (dev->arch == 5 && nrects > 1) {
633 if (!pres->damage.tile_map.data) {
634 pres->damage.tile_map.stride =
635 ALIGN_POT(DIV_ROUND_UP(res->width0, 32 * 8), 64);
636 pres->damage.tile_map.size =
637 pres->damage.tile_map.stride * DIV_ROUND_UP(res->height0, 32);
638 pres->damage.tile_map.data = malloc(pres->damage.tile_map.size);
639 }
640
641 memset(pres->damage.tile_map.data, 0, pres->damage.tile_map.size);
642 pres->damage.tile_map.enable = true;
643 } else {
644 pres->damage.tile_map.enable = false;
645 }
646
647 /* Track the damage extent: the quad including all damage regions. Will
648 * be used restrict the rendering area */
649
650 damage_extent->minx = 0xffff;
651 damage_extent->miny = 0xffff;
652
653 unsigned enable_count = 0;
654
655 for (i = 0; i < nrects; i++) {
656 int x = rects[i].x, w = rects[i].width, h = rects[i].height;
657 int y = res->height0 - (rects[i].y + h);
658
659 damage_extent->minx = MIN2(damage_extent->minx, x);
660 damage_extent->miny = MIN2(damage_extent->miny, y);
661 damage_extent->maxx = MAX2(damage_extent->maxx, MIN2(x + w, res->width0));
662 damage_extent->maxy =
663 MAX2(damage_extent->maxy, MIN2(y + h, res->height0));
664
665 if (!pres->damage.tile_map.enable)
666 continue;
667
668 unsigned t_x_start = x / 32;
669 unsigned t_x_end = (x + w - 1) / 32;
670 unsigned t_y_start = y / 32;
671 unsigned t_y_end = (y + h - 1) / 32;
672
673 for (unsigned t_y = t_y_start; t_y <= t_y_end; t_y++) {
674 for (unsigned t_x = t_x_start; t_x <= t_x_end; t_x++) {
675 unsigned b = (t_y * pres->damage.tile_map.stride * 8) + t_x;
676
677 if (BITSET_TEST(pres->damage.tile_map.data, b))
678 continue;
679
680 BITSET_SET(pres->damage.tile_map.data, b);
681 enable_count++;
682 }
683 }
684 }
685
686 if (nrects == 0) {
687 damage_extent->minx = 0;
688 damage_extent->miny = 0;
689 damage_extent->maxx = res->width0;
690 damage_extent->maxy = res->height0;
691 }
692
693 if (pres->damage.tile_map.enable) {
694 unsigned t_x_start = damage_extent->minx / 32;
695 unsigned t_x_end = damage_extent->maxx / 32;
696 unsigned t_y_start = damage_extent->miny / 32;
697 unsigned t_y_end = damage_extent->maxy / 32;
698 unsigned tile_count =
699 (t_x_end - t_x_start + 1) * (t_y_end - t_y_start + 1);
700
701 /* Don't bother passing a tile-enable-map if the amount of
702 * tiles to reload is to close to the total number of tiles.
703 */
704 if (tile_count - enable_count < 10)
705 pres->damage.tile_map.enable = false;
706 }
707 }
708
709 struct pipe_resource *
panfrost_resource_create_with_modifier(struct pipe_screen * screen,const struct pipe_resource * template,uint64_t modifier)710 panfrost_resource_create_with_modifier(struct pipe_screen *screen,
711 const struct pipe_resource *template,
712 uint64_t modifier)
713 {
714 struct panfrost_device *dev = pan_device(screen);
715
716 struct panfrost_resource *so = CALLOC_STRUCT(panfrost_resource);
717
718 if (!so)
719 return NULL;
720
721 so->base = *template;
722 so->base.screen = screen;
723
724 pipe_reference_init(&so->base.reference, 1);
725
726 util_range_init(&so->valid_buffer_range);
727
728 if (template->bind & PAN_BIND_SHARED_MASK) {
729 /* For compatibility with older consumers that may not be
730 * modifiers aware, treat INVALID as LINEAR for shared
731 * resources.
732 */
733 if (modifier == DRM_FORMAT_MOD_INVALID)
734 modifier = DRM_FORMAT_MOD_LINEAR;
735
736 /* At any rate, we can't change the modifier later for shared
737 * resources, since we have no way to propagate the modifier
738 * change.
739 */
740 so->modifier_constant = true;
741 }
742
743 panfrost_resource_setup(screen, so, modifier, template->format);
744
745 /* Guess a label based on the bind */
746 unsigned bind = template->bind;
747 const char *label = (bind & PIPE_BIND_INDEX_BUFFER) ? "Index buffer"
748 : (bind & PIPE_BIND_SCANOUT) ? "Scanout"
749 : (bind & PIPE_BIND_DISPLAY_TARGET) ? "Display target"
750 : (bind & PIPE_BIND_SHARED) ? "Shared resource"
751 : (bind & PIPE_BIND_RENDER_TARGET) ? "Render target"
752 : (bind & PIPE_BIND_DEPTH_STENCIL)
753 ? "Depth/stencil buffer"
754 : (bind & PIPE_BIND_SAMPLER_VIEW) ? "Texture"
755 : (bind & PIPE_BIND_VERTEX_BUFFER) ? "Vertex buffer"
756 : (bind & PIPE_BIND_CONSTANT_BUFFER) ? "Constant buffer"
757 : (bind & PIPE_BIND_GLOBAL) ? "Global memory"
758 : (bind & PIPE_BIND_SHADER_BUFFER) ? "Shader buffer"
759 : (bind & PIPE_BIND_SHADER_IMAGE) ? "Shader image"
760 : "Other resource";
761
762 if (dev->ro && (template->bind & PIPE_BIND_SCANOUT)) {
763 struct winsys_handle handle;
764 struct pan_block_size blocksize =
765 panfrost_renderblock_size(modifier, template->format);
766
767 /* Block-based texture formats are only used for texture
768 * compression (not framebuffer compression!), which doesn't
769 * make sense to share across processes.
770 */
771 assert(util_format_get_blockwidth(template->format) == 1);
772 assert(util_format_get_blockheight(template->format) == 1);
773
774 /* Present a resource with similar dimensions that, if allocated
775 * as a linear image, is big enough to fit the resource in the
776 * actual layout. For linear images, this is a no-op. For 16x16
777 * tiling, this aligns the dimensions to 16x16.
778 *
779 * For AFBC, this aligns the width to the superblock width (as
780 * expected) and adds extra rows to account for the header. This
781 * is a bit of a lie, but it's the best we can do with dumb
782 * buffers, which are extremely not meant for AFBC. And yet this
783 * has to work anyway...
784 *
785 * Moral of the story: if you're reading this comment, that
786 * means you're working on WSI and so it's already too late for
787 * you. I'm sorry.
788 */
789 unsigned width = ALIGN_POT(template->width0, blocksize.width);
790 unsigned stride = ALIGN_POT(template->width0, blocksize.width) *
791 util_format_get_blocksize(template->format);
792 unsigned size = so->image.layout.data_size;
793 unsigned effective_rows = DIV_ROUND_UP(size, stride);
794
795 struct pipe_resource scanout_tmpl = {
796 .target = so->base.target,
797 .format = template->format,
798 .width0 = width,
799 .height0 = effective_rows,
800 .depth0 = 1,
801 .array_size = 1,
802 };
803
804 so->scanout =
805 renderonly_scanout_for_resource(&scanout_tmpl, dev->ro, &handle);
806
807 if (!so->scanout) {
808 mesa_loge("Failed to create scanout resource\n");
809 FREE(so);
810 return NULL;
811 }
812 assert(handle.type == WINSYS_HANDLE_TYPE_FD);
813 so->bo = panfrost_bo_import(dev, handle.handle);
814 close(handle.handle);
815
816 if (!so->bo) {
817 FREE(so);
818 return NULL;
819 }
820
821 so->image.data.base = so->bo->ptr.gpu;
822 } else {
823 /* We create a BO immediately but don't bother mapping, since we don't
824 * care to map e.g. FBOs which the CPU probably won't touch */
825 uint32_t flags = PAN_BO_DELAY_MMAP;
826
827 /* If the resource is never exported, we can make the BO private. */
828 if (template->bind & PIPE_BIND_SHARED)
829 flags |= PAN_BO_SHAREABLE;
830
831 so->bo =
832 panfrost_bo_create(dev, so->image.layout.data_size, flags, label);
833
834 if (!so->bo) {
835 FREE(so);
836 return NULL;
837 }
838
839 so->image.data.base = so->bo->ptr.gpu;
840
841 so->constant_stencil = true;
842 }
843
844 if (drm_is_afbc(so->image.layout.modifier)) {
845 if (panfrost_resource_init_afbc_headers(so)) {
846 FREE(so);
847 return NULL;
848 }
849 }
850
851 panfrost_resource_set_damage_region(screen, &so->base, 0, NULL);
852
853 if (template->bind & PIPE_BIND_INDEX_BUFFER)
854 so->index_cache = CALLOC_STRUCT(panfrost_minmax_cache);
855
856 return (struct pipe_resource *)so;
857 }
858
859 /* Default is to create a resource as don't care */
860
861 static struct pipe_resource *
panfrost_resource_create(struct pipe_screen * screen,const struct pipe_resource * template)862 panfrost_resource_create(struct pipe_screen *screen,
863 const struct pipe_resource *template)
864 {
865 return panfrost_resource_create_with_modifier(screen, template,
866 DRM_FORMAT_MOD_INVALID);
867 }
868
869 /* If no modifier is specified, we'll choose. Otherwise, the order of
870 * preference is compressed, tiled, linear. */
871
872 static struct pipe_resource *
panfrost_resource_create_with_modifiers(struct pipe_screen * screen,const struct pipe_resource * template,const uint64_t * modifiers,int count)873 panfrost_resource_create_with_modifiers(struct pipe_screen *screen,
874 const struct pipe_resource *template,
875 const uint64_t *modifiers, int count)
876 {
877 for (unsigned i = 0; i < PAN_MODIFIER_COUNT; ++i) {
878 if (drm_find_modifier(pan_best_modifiers[i], modifiers, count)) {
879 return panfrost_resource_create_with_modifier(screen, template,
880 pan_best_modifiers[i]);
881 }
882 }
883
884 /* If we didn't find one, app specified invalid */
885 assert(count == 1 && modifiers[0] == DRM_FORMAT_MOD_INVALID);
886 return panfrost_resource_create(screen, template);
887 }
888
889 static void
panfrost_resource_destroy(struct pipe_screen * screen,struct pipe_resource * pt)890 panfrost_resource_destroy(struct pipe_screen *screen, struct pipe_resource *pt)
891 {
892 struct panfrost_device *dev = pan_device(screen);
893 struct panfrost_resource *rsrc = (struct panfrost_resource *)pt;
894
895 if (rsrc->scanout)
896 renderonly_scanout_destroy(rsrc->scanout, dev->ro);
897
898 if (rsrc->bo)
899 panfrost_bo_unreference(rsrc->bo);
900
901 free(rsrc->index_cache);
902 free(rsrc->damage.tile_map.data);
903
904 util_range_destroy(&rsrc->valid_buffer_range);
905 free(rsrc);
906 }
907
908 /* Most of the time we can do CPU-side transfers, but sometimes we need to use
909 * the 3D pipe for this. Let's wrap u_blitter to blit to/from staging textures.
910 * Code adapted from freedreno */
911
912 static struct panfrost_resource *
pan_alloc_staging(struct panfrost_context * ctx,struct panfrost_resource * rsc,unsigned level,const struct pipe_box * box)913 pan_alloc_staging(struct panfrost_context *ctx, struct panfrost_resource *rsc,
914 unsigned level, const struct pipe_box *box)
915 {
916 struct pipe_context *pctx = &ctx->base;
917 struct pipe_resource tmpl = rsc->base;
918
919 tmpl.width0 = box->width;
920 tmpl.height0 = box->height;
921 /* for array textures, box->depth is the array_size, otherwise
922 * for 3d textures, it is the depth:
923 */
924 if (tmpl.array_size > 1) {
925 if (tmpl.target == PIPE_TEXTURE_CUBE)
926 tmpl.target = PIPE_TEXTURE_2D_ARRAY;
927 tmpl.array_size = box->depth;
928 tmpl.depth0 = 1;
929 } else {
930 tmpl.array_size = 1;
931 tmpl.depth0 = box->depth;
932 }
933 tmpl.last_level = 0;
934 tmpl.bind |= PIPE_BIND_LINEAR;
935 tmpl.bind &= ~PAN_BIND_SHARED_MASK;
936 tmpl.compression_rate = PIPE_COMPRESSION_FIXED_RATE_NONE;
937
938 struct pipe_resource *pstaging =
939 pctx->screen->resource_create(pctx->screen, &tmpl);
940 if (!pstaging)
941 return NULL;
942
943 return pan_resource(pstaging);
944 }
945
946 static void
pan_blit_from_staging(struct pipe_context * pctx,struct panfrost_transfer * trans)947 pan_blit_from_staging(struct pipe_context *pctx,
948 struct panfrost_transfer *trans)
949 {
950 struct pipe_resource *dst = trans->base.resource;
951 struct pipe_blit_info blit = {0};
952
953 blit.dst.resource = dst;
954 blit.dst.format = dst->format;
955 blit.dst.level = trans->base.level;
956 blit.dst.box = trans->base.box;
957 blit.src.resource = trans->staging.rsrc;
958 blit.src.format = trans->staging.rsrc->format;
959 blit.src.level = 0;
960 blit.src.box = trans->staging.box;
961 blit.mask = util_format_get_mask(blit.src.format);
962 blit.filter = PIPE_TEX_FILTER_NEAREST;
963
964 panfrost_blit_no_afbc_legalization(pctx, &blit);
965 }
966
967 static void
pan_blit_to_staging(struct pipe_context * pctx,struct panfrost_transfer * trans)968 pan_blit_to_staging(struct pipe_context *pctx, struct panfrost_transfer *trans)
969 {
970 struct pipe_resource *src = trans->base.resource;
971 struct pipe_blit_info blit = {0};
972
973 blit.src.resource = src;
974 blit.src.format = src->format;
975 blit.src.level = trans->base.level;
976 blit.src.box = trans->base.box;
977 blit.dst.resource = trans->staging.rsrc;
978 blit.dst.format = trans->staging.rsrc->format;
979 blit.dst.level = 0;
980 blit.dst.box = trans->staging.box;
981 blit.mask = util_format_get_mask(blit.dst.format);
982 blit.filter = PIPE_TEX_FILTER_NEAREST;
983
984 panfrost_blit_no_afbc_legalization(pctx, &blit);
985 }
986
987 static void
panfrost_load_tiled_images(struct panfrost_transfer * transfer,struct panfrost_resource * rsrc)988 panfrost_load_tiled_images(struct panfrost_transfer *transfer,
989 struct panfrost_resource *rsrc)
990 {
991 struct pipe_transfer *ptrans = &transfer->base;
992 unsigned level = ptrans->level;
993
994 /* If the requested level of the image is uninitialized, it's not
995 * necessary to copy it. Leave the result unintiialized too.
996 */
997 if (!BITSET_TEST(rsrc->valid.data, level))
998 return;
999
1000 struct panfrost_bo *bo = rsrc->bo;
1001 unsigned stride = panfrost_get_layer_stride(&rsrc->image.layout, level);
1002
1003 /* Otherwise, load each layer separately, required to load from 3D and
1004 * array textures.
1005 */
1006 for (unsigned z = 0; z < ptrans->box.depth; ++z) {
1007 void *dst = transfer->map + (ptrans->layer_stride * z);
1008 uint8_t *map = bo->ptr.cpu + rsrc->image.layout.slices[level].offset +
1009 (z + ptrans->box.z) * stride;
1010
1011 panfrost_load_tiled_image(dst, map, ptrans->box.x, ptrans->box.y,
1012 ptrans->box.width, ptrans->box.height,
1013 ptrans->stride,
1014 rsrc->image.layout.slices[level].row_stride,
1015 rsrc->image.layout.format);
1016 }
1017 }
1018
1019 #if MESA_DEBUG
1020
1021 static unsigned
get_superblock_size(uint32_t * hdr,unsigned uncompressed_size)1022 get_superblock_size(uint32_t *hdr, unsigned uncompressed_size)
1023 {
1024 /* AFBC superblock layout 0 */
1025 unsigned body_base_ptr_len = 32;
1026 unsigned nr_subblocks = 16;
1027 unsigned sz_len = 6; /* bits */
1028 unsigned mask = (1 << sz_len) - 1;
1029 unsigned size = 0;
1030
1031 /* Sum up all of the subblock sizes */
1032 for (int i = 0; i < nr_subblocks; i++) {
1033 unsigned bitoffset = body_base_ptr_len + (i * sz_len);
1034 unsigned start = bitoffset / 32;
1035 unsigned end = (bitoffset + (sz_len - 1)) / 32;
1036 unsigned offset = bitoffset % 32;
1037 unsigned subblock_size;
1038
1039 if (start != end)
1040 subblock_size = (hdr[start] >> offset) | (hdr[end] << (32 - offset));
1041 else
1042 subblock_size = hdr[start] >> offset;
1043 subblock_size = (subblock_size == 1) ? uncompressed_size : subblock_size;
1044 size += subblock_size & mask;
1045
1046 if (i == 0 && size == 0)
1047 return 0;
1048 }
1049
1050 return size;
1051 }
1052
1053 static void
dump_block(struct panfrost_resource * rsrc,uint32_t idx)1054 dump_block(struct panfrost_resource *rsrc, uint32_t idx)
1055 {
1056 panfrost_bo_wait(rsrc->bo, INT64_MAX, false);
1057
1058 uint8_t *ptr = rsrc->bo->ptr.cpu;
1059 uint32_t *header = (uint32_t *)(ptr + (idx * AFBC_HEADER_BYTES_PER_TILE));
1060 uint32_t body_base_ptr = header[0];
1061 uint32_t *body = (uint32_t *)(ptr + body_base_ptr);
1062 struct pan_block_size block_sz =
1063 panfrost_afbc_subblock_size(rsrc->image.layout.modifier);
1064 unsigned pixel_sz = util_format_get_blocksize(rsrc->base.format);
1065 unsigned uncompressed_size = pixel_sz * block_sz.width * block_sz.height;
1066 unsigned size = get_superblock_size(header, uncompressed_size);
1067
1068 fprintf(stderr, " Header: %08x %08x %08x %08x (size: %u bytes)\n",
1069 header[0], header[1], header[2], header[3], size);
1070 if (size > 0) {
1071 fprintf(stderr, " Body: %08x %08x %08x %08x\n", body[0], body[1],
1072 body[2], body[3]);
1073 } else {
1074 uint8_t *comp = (uint8_t *)(header + 2);
1075 fprintf(stderr, " Color: 0x%02x%02x%02x%02x\n", comp[0], comp[1],
1076 comp[2], comp[3]);
1077 }
1078 fprintf(stderr, "\n");
1079 }
1080
1081 void
pan_dump_resource(struct panfrost_context * ctx,struct panfrost_resource * rsc)1082 pan_dump_resource(struct panfrost_context *ctx, struct panfrost_resource *rsc)
1083 {
1084 struct pipe_context *pctx = &ctx->base;
1085 struct pipe_resource tmpl = rsc->base;
1086 struct pipe_resource *plinear = NULL;
1087 struct panfrost_resource *linear = rsc;
1088 struct pipe_blit_info blit = {0};
1089 struct pipe_box box;
1090 char buffer[1024];
1091
1092 if (rsc->image.layout.modifier != DRM_FORMAT_MOD_LINEAR) {
1093 tmpl.bind |= PIPE_BIND_LINEAR;
1094 tmpl.bind &= ~PAN_BIND_SHARED_MASK;
1095
1096 plinear = pctx->screen->resource_create(pctx->screen, &tmpl);
1097 u_box_2d(0, 0, rsc->base.width0, rsc->base.height0, &box);
1098
1099 blit.src.resource = &rsc->base;
1100 blit.src.format = rsc->base.format;
1101 blit.src.level = 0;
1102 blit.src.box = box;
1103 blit.dst.resource = plinear;
1104 blit.dst.format = rsc->base.format;
1105 blit.dst.level = 0;
1106 blit.dst.box = box;
1107 blit.mask = util_format_get_mask(blit.dst.format);
1108 blit.filter = PIPE_TEX_FILTER_NEAREST;
1109
1110 panfrost_blit(pctx, &blit);
1111
1112 linear = pan_resource(plinear);
1113 }
1114
1115 panfrost_flush_writer(ctx, linear, "dump image");
1116 panfrost_bo_wait(linear->bo, INT64_MAX, false);
1117
1118 if (!panfrost_bo_mmap(linear->bo)) {
1119 static unsigned frame_count = 0;
1120 frame_count++;
1121 snprintf(buffer, sizeof(buffer), "dump_image.%04d", frame_count);
1122
1123 debug_dump_image(buffer, rsc->base.format, 0 /* UNUSED */, rsc->base.width0,
1124 rsc->base.height0,
1125 linear->image.layout.slices[0].row_stride,
1126 linear->bo->ptr.cpu);
1127 } else {
1128 mesa_loge("failed to mmap, not dumping resource");
1129 }
1130
1131 if (plinear)
1132 pipe_resource_reference(&plinear, NULL);
1133 }
1134
1135 #endif
1136
1137 /* Get scan-order index from (x, y) position when blocks are
1138 * arranged in z-order in 8x8 tiles */
1139 static unsigned
get_morton_index(unsigned x,unsigned y,unsigned stride)1140 get_morton_index(unsigned x, unsigned y, unsigned stride)
1141 {
1142 unsigned i = ((x << 0) & 1) | ((y << 1) & 2) | ((x << 1) & 4) |
1143 ((y << 2) & 8) | ((x << 2) & 16) | ((y << 3) & 32);
1144 return (((y & ~7) * stride) + ((x & ~7) << 3)) + i;
1145 }
1146
1147 static void
panfrost_store_tiled_images(struct panfrost_transfer * transfer,struct panfrost_resource * rsrc)1148 panfrost_store_tiled_images(struct panfrost_transfer *transfer,
1149 struct panfrost_resource *rsrc)
1150 {
1151 struct panfrost_bo *bo = rsrc->bo;
1152 struct pipe_transfer *ptrans = &transfer->base;
1153 unsigned level = ptrans->level;
1154 unsigned stride = panfrost_get_layer_stride(&rsrc->image.layout, level);
1155
1156 /* Otherwise, store each layer separately, required to store to 3D and
1157 * array textures.
1158 */
1159 for (unsigned z = 0; z < ptrans->box.depth; ++z) {
1160 void *src = transfer->map + (ptrans->layer_stride * z);
1161 uint8_t *map = bo->ptr.cpu + rsrc->image.layout.slices[level].offset +
1162 (z + ptrans->box.z) * stride;
1163
1164 panfrost_store_tiled_image(map, src, ptrans->box.x, ptrans->box.y,
1165 ptrans->box.width, ptrans->box.height,
1166 rsrc->image.layout.slices[level].row_stride,
1167 ptrans->stride, rsrc->image.layout.format);
1168 }
1169 }
1170
1171 static bool
panfrost_box_covers_resource(const struct pipe_resource * resource,const struct pipe_box * box)1172 panfrost_box_covers_resource(const struct pipe_resource *resource,
1173 const struct pipe_box *box)
1174 {
1175 return resource->last_level == 0 &&
1176 util_texrange_covers_whole_level(resource, 0, box->x, box->y, box->z,
1177 box->width, box->height, box->depth);
1178 }
1179
1180 static bool
panfrost_can_discard(struct pipe_resource * resource,const struct pipe_box * box,unsigned usage)1181 panfrost_can_discard(struct pipe_resource *resource, const struct pipe_box *box,
1182 unsigned usage)
1183 {
1184 struct panfrost_resource *rsrc = pan_resource(resource);
1185
1186 return ((usage & PIPE_MAP_DISCARD_RANGE) &&
1187 !(usage & PIPE_MAP_UNSYNCHRONIZED) &&
1188 !(resource->flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) &&
1189 panfrost_box_covers_resource(resource, box) &&
1190 !(rsrc->bo->flags & PAN_BO_SHARED));
1191 }
1192
1193 static void *
panfrost_ptr_map(struct pipe_context * pctx,struct pipe_resource * resource,unsigned level,unsigned usage,const struct pipe_box * box,struct pipe_transfer ** out_transfer)1194 panfrost_ptr_map(struct pipe_context *pctx, struct pipe_resource *resource,
1195 unsigned level,
1196 unsigned usage, /* a combination of PIPE_MAP_x */
1197 const struct pipe_box *box,
1198 struct pipe_transfer **out_transfer)
1199 {
1200 struct panfrost_context *ctx = pan_context(pctx);
1201 struct panfrost_device *dev = pan_device(pctx->screen);
1202 struct panfrost_resource *rsrc = pan_resource(resource);
1203 enum pipe_format format = rsrc->image.layout.format;
1204 int bytes_per_block = util_format_get_blocksize(format);
1205 struct panfrost_bo *bo = rsrc->bo;
1206
1207 /* Can't map tiled/compressed directly */
1208 if ((usage & PIPE_MAP_DIRECTLY) &&
1209 rsrc->image.layout.modifier != DRM_FORMAT_MOD_LINEAR)
1210 return NULL;
1211
1212 struct panfrost_transfer *transfer = rzalloc(pctx, struct panfrost_transfer);
1213 transfer->base.level = level;
1214 transfer->base.usage = usage;
1215 transfer->base.box = *box;
1216
1217 pipe_resource_reference(&transfer->base.resource, resource);
1218 *out_transfer = &transfer->base;
1219
1220 if (usage & PIPE_MAP_WRITE)
1221 rsrc->constant_stencil = false;
1222
1223 /* We don't have s/w routines for AFBC/AFRC, so use a staging texture */
1224 if (drm_is_afbc(rsrc->image.layout.modifier) ||
1225 drm_is_afrc(rsrc->image.layout.modifier)) {
1226 struct panfrost_resource *staging =
1227 pan_alloc_staging(ctx, rsrc, level, box);
1228 assert(staging);
1229
1230 /* Staging resources have one LOD: level 0. Query the strides
1231 * on this LOD.
1232 */
1233 transfer->base.stride = staging->image.layout.slices[0].row_stride;
1234 transfer->base.layer_stride =
1235 panfrost_get_layer_stride(&staging->image.layout, 0);
1236
1237 transfer->staging.rsrc = &staging->base;
1238
1239 transfer->staging.box = *box;
1240 transfer->staging.box.x = 0;
1241 transfer->staging.box.y = 0;
1242 transfer->staging.box.z = 0;
1243
1244 assert(transfer->staging.rsrc != NULL);
1245
1246 bool valid = BITSET_TEST(rsrc->valid.data, level);
1247
1248 if ((usage & PIPE_MAP_READ) &&
1249 (valid || panfrost_any_batch_writes_rsrc(ctx, rsrc))) {
1250 pan_blit_to_staging(pctx, transfer);
1251 panfrost_flush_writer(ctx, staging, "AFBC/AFRC tex read staging blit");
1252 panfrost_bo_wait(staging->bo, INT64_MAX, false);
1253 }
1254
1255 if (panfrost_bo_mmap(staging->bo))
1256 return NULL;
1257
1258 return staging->bo->ptr.cpu;
1259 }
1260
1261 bool already_mapped = bo->ptr.cpu != NULL;
1262
1263 /* If we haven't already mmaped, now's the time */
1264 if (panfrost_bo_mmap(bo))
1265 return NULL;
1266
1267 if (dev->debug & (PAN_DBG_TRACE | PAN_DBG_SYNC)) {
1268 pandecode_inject_mmap(dev->decode_ctx, bo->ptr.gpu, bo->ptr.cpu,
1269 panfrost_bo_size(bo), NULL);
1270 }
1271
1272 /* Upgrade writes to uninitialized ranges to UNSYNCHRONIZED */
1273 if ((usage & PIPE_MAP_WRITE) && resource->target == PIPE_BUFFER &&
1274 !util_ranges_intersect(&rsrc->valid_buffer_range, box->x,
1275 box->x + box->width)) {
1276
1277 usage |= PIPE_MAP_UNSYNCHRONIZED;
1278 }
1279
1280 /* Upgrade DISCARD_RANGE to WHOLE_RESOURCE if the whole resource is
1281 * being mapped.
1282 */
1283 if (panfrost_can_discard(resource, box, usage)) {
1284 usage |= PIPE_MAP_DISCARD_WHOLE_RESOURCE;
1285 }
1286
1287 bool create_new_bo = usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE;
1288 bool copy_resource = false;
1289
1290 if (!(usage & PIPE_MAP_UNSYNCHRONIZED) &&
1291 !(resource->flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) &&
1292 (usage & PIPE_MAP_WRITE) && panfrost_any_batch_reads_rsrc(ctx, rsrc)) {
1293 /* When a resource to be modified is already being used by a
1294 * pending batch, it is often faster to copy the whole BO than
1295 * to flush and split the frame in two.
1296 */
1297
1298 panfrost_flush_writer(ctx, rsrc, "Shadow resource creation");
1299 panfrost_bo_wait(bo, INT64_MAX, false);
1300
1301 create_new_bo = true;
1302 copy_resource = !(usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE);
1303 }
1304
1305 /* Shadowing with separate stencil may require additional accounting.
1306 * Bail in these exotic cases.
1307 */
1308 if (rsrc->separate_stencil) {
1309 create_new_bo = false;
1310 copy_resource = false;
1311 }
1312
1313 if (create_new_bo &&
1314 (!(resource->flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) ||
1315 !already_mapped)) {
1316 /* Make sure we re-emit any descriptors using this resource */
1317 panfrost_dirty_state_all(ctx);
1318
1319 /* If the BO is used by one of the pending batches or if it's
1320 * not ready yet (still accessed by one of the already flushed
1321 * batches), we try to allocate a new one to avoid waiting.
1322 */
1323 if (panfrost_any_batch_reads_rsrc(ctx, rsrc) ||
1324 !panfrost_bo_wait(bo, 0, true)) {
1325 /* We want the BO to be MMAPed. */
1326 uint32_t flags = bo->flags & ~PAN_BO_DELAY_MMAP;
1327 struct panfrost_bo *newbo = NULL;
1328
1329 /* When the BO has been imported/exported, we can't
1330 * replace it by another one, otherwise the
1331 * importer/exporter wouldn't see the change we're
1332 * doing to it.
1333 */
1334 if (!(bo->flags & PAN_BO_SHARED)) {
1335 newbo =
1336 panfrost_bo_create(dev, panfrost_bo_size(bo), flags, bo->label);
1337 }
1338
1339 if (newbo) {
1340 if (copy_resource) {
1341 memcpy(newbo->ptr.cpu, rsrc->bo->ptr.cpu, panfrost_bo_size(bo));
1342 }
1343
1344 /* Swap the pointers, dropping a reference to
1345 * the old BO which is no long referenced from
1346 * the resource.
1347 */
1348 panfrost_bo_unreference(rsrc->bo);
1349 rsrc->bo = newbo;
1350 rsrc->image.data.base = newbo->ptr.gpu;
1351
1352 if (!copy_resource && drm_is_afbc(rsrc->image.layout.modifier)) {
1353 if (panfrost_resource_init_afbc_headers(rsrc))
1354 return NULL;
1355 }
1356
1357 bo = newbo;
1358 } else {
1359 /* Allocation failed or was impossible, let's
1360 * fall back on a flush+wait.
1361 */
1362 panfrost_flush_batches_accessing_rsrc(
1363 ctx, rsrc, "Resource access with high memory pressure");
1364 panfrost_bo_wait(bo, INT64_MAX, true);
1365 }
1366 }
1367 } else if (!(usage & PIPE_MAP_UNSYNCHRONIZED)) {
1368 if (usage & PIPE_MAP_WRITE) {
1369 panfrost_flush_batches_accessing_rsrc(ctx, rsrc, "Synchronized write");
1370 panfrost_bo_wait(bo, INT64_MAX, true);
1371 } else if (usage & PIPE_MAP_READ) {
1372 panfrost_flush_writer(ctx, rsrc, "Synchronized read");
1373 panfrost_bo_wait(bo, INT64_MAX, false);
1374 }
1375 }
1376
1377 /* For access to compressed textures, we want the (x, y, w, h)
1378 * region-of-interest in blocks, not pixels. Then we compute the stride
1379 * between rows of blocks as the width in blocks times the width per
1380 * block, etc.
1381 */
1382 struct pipe_box box_blocks;
1383 u_box_pixels_to_blocks(&box_blocks, box, format);
1384
1385 if (rsrc->image.layout.modifier ==
1386 DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) {
1387 transfer->base.stride = box_blocks.width * bytes_per_block;
1388 transfer->base.layer_stride = transfer->base.stride * box_blocks.height;
1389 transfer->map =
1390 ralloc_size(transfer, transfer->base.layer_stride * box->depth);
1391
1392 if (usage & PIPE_MAP_READ)
1393 panfrost_load_tiled_images(transfer, rsrc);
1394
1395 return transfer->map;
1396 } else {
1397 assert(rsrc->image.layout.modifier == DRM_FORMAT_MOD_LINEAR);
1398
1399 /* Direct, persistent writes create holes in time for
1400 * caching... I don't know if this is actually possible but we
1401 * should still get it right */
1402
1403 unsigned dpw = PIPE_MAP_DIRECTLY | PIPE_MAP_WRITE | PIPE_MAP_PERSISTENT;
1404
1405 if ((usage & dpw) == dpw && rsrc->index_cache)
1406 return NULL;
1407
1408 transfer->base.stride = rsrc->image.layout.slices[level].row_stride;
1409 transfer->base.layer_stride =
1410 panfrost_get_layer_stride(&rsrc->image.layout, level);
1411
1412 /* By mapping direct-write, we're implicitly already
1413 * initialized (maybe), so be conservative */
1414
1415 if (usage & PIPE_MAP_WRITE) {
1416 BITSET_SET(rsrc->valid.data, level);
1417 panfrost_minmax_cache_invalidate(
1418 rsrc->index_cache, transfer->base.box.x, transfer->base.box.width);
1419 }
1420
1421 return bo->ptr.cpu + rsrc->image.layout.slices[level].offset +
1422 box->z * transfer->base.layer_stride +
1423 box_blocks.y * rsrc->image.layout.slices[level].row_stride +
1424 box_blocks.x * bytes_per_block;
1425 }
1426 }
1427
1428 void
pan_resource_modifier_convert(struct panfrost_context * ctx,struct panfrost_resource * rsrc,uint64_t modifier,bool copy_resource,const char * reason)1429 pan_resource_modifier_convert(struct panfrost_context *ctx,
1430 struct panfrost_resource *rsrc, uint64_t modifier,
1431 bool copy_resource, const char *reason)
1432 {
1433 assert(!rsrc->modifier_constant);
1434
1435 struct pipe_resource *tmp_prsrc = panfrost_resource_create_with_modifier(
1436 ctx->base.screen, &rsrc->base, modifier);
1437 struct panfrost_resource *tmp_rsrc = pan_resource(tmp_prsrc);
1438
1439 if (copy_resource) {
1440 struct pipe_blit_info blit = {
1441 .dst.resource = &tmp_rsrc->base,
1442 .dst.format = tmp_rsrc->base.format,
1443 .src.resource = &rsrc->base,
1444 .src.format = rsrc->base.format,
1445 .mask = util_format_get_mask(tmp_rsrc->base.format),
1446 .filter = PIPE_TEX_FILTER_NEAREST,
1447 };
1448
1449 /* data_valid is not valid until flushed */
1450 panfrost_flush_writer(ctx, rsrc, "AFBC/AFRC decompressing blit");
1451
1452 for (int i = 0; i <= rsrc->base.last_level; i++) {
1453 if (BITSET_TEST(rsrc->valid.data, i)) {
1454 blit.dst.level = blit.src.level = i;
1455
1456 u_box_3d(0, 0, 0, u_minify(rsrc->base.width0, i),
1457 u_minify(rsrc->base.height0, i),
1458 util_num_layers(&rsrc->base, i), &blit.dst.box);
1459 blit.src.box = blit.dst.box;
1460
1461 panfrost_blit_no_afbc_legalization(&ctx->base, &blit);
1462 }
1463 }
1464
1465 /* we lose track of tmp_rsrc after this point, and the BO migration
1466 * (from tmp_rsrc to rsrc) doesn't transfer the last_writer to rsrc
1467 */
1468 panfrost_flush_writer(ctx, tmp_rsrc, "AFBC/AFRC decompressing blit");
1469 }
1470
1471 panfrost_bo_unreference(rsrc->bo);
1472
1473 rsrc->bo = tmp_rsrc->bo;
1474 rsrc->image.data.base = rsrc->bo->ptr.gpu;
1475 panfrost_bo_reference(rsrc->bo);
1476
1477 panfrost_resource_setup(ctx->base.screen, rsrc, modifier,
1478 tmp_rsrc->base.format);
1479 /* panfrost_resource_setup will force the modifier to stay constant when
1480 * called with a specific modifier. We don't want that here, we want to
1481 * be able to convert back to another modifier if needed */
1482 rsrc->modifier_constant = false;
1483 pipe_resource_reference(&tmp_prsrc, NULL);
1484 perf_debug(ctx, "resource_modifier_convert required due to: %s", reason);
1485 }
1486
1487 /* Validate that an AFBC/AFRC resource may be used as a particular format. If it
1488 * may not, decompress it on the fly. Failure to do so can produce wrong results
1489 * or invalid data faults when sampling or rendering to AFBC */
1490
1491 void
pan_legalize_format(struct panfrost_context * ctx,struct panfrost_resource * rsrc,enum pipe_format format,bool write,bool discard)1492 pan_legalize_format(struct panfrost_context *ctx,
1493 struct panfrost_resource *rsrc, enum pipe_format format,
1494 bool write, bool discard)
1495 {
1496 struct panfrost_device *dev = pan_device(ctx->base.screen);
1497 enum pipe_format old_format = rsrc->base.format;
1498 enum pipe_format new_format = format;
1499 bool compatible = true;
1500
1501 if (!drm_is_afbc(rsrc->image.layout.modifier) &&
1502 !drm_is_afrc(rsrc->image.layout.modifier))
1503 return;
1504
1505 if (drm_is_afbc(rsrc->image.layout.modifier)) {
1506 compatible = (panfrost_afbc_format(dev->arch, old_format) ==
1507 panfrost_afbc_format(dev->arch, new_format));
1508 } else if (drm_is_afrc(rsrc->image.layout.modifier)) {
1509 struct pan_afrc_format_info old_info =
1510 panfrost_afrc_get_format_info(old_format);
1511 struct pan_afrc_format_info new_info =
1512 panfrost_afrc_get_format_info(new_format);
1513 compatible = !memcmp(&old_info, &new_info, sizeof(old_info));
1514 }
1515
1516 if (!compatible) {
1517 pan_resource_modifier_convert(
1518 ctx, rsrc, DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED, !discard,
1519 drm_is_afbc(rsrc->image.layout.modifier)
1520 ? "Reinterpreting AFBC surface as incompatible format"
1521 : "Reinterpreting AFRC surface as incompatible format");
1522 return;
1523 }
1524
1525 /* Can't write to AFBC-P resources */
1526 if (write && drm_is_afbc(rsrc->image.layout.modifier) &&
1527 (rsrc->image.layout.modifier & AFBC_FORMAT_MOD_SPARSE) == 0) {
1528 pan_resource_modifier_convert(
1529 ctx, rsrc, rsrc->image.layout.modifier | AFBC_FORMAT_MOD_SPARSE,
1530 !discard, "Legalizing resource to allow writing");
1531 }
1532 }
1533
1534 static bool
panfrost_should_linear_convert(struct panfrost_context * ctx,struct panfrost_resource * prsrc,struct pipe_transfer * transfer)1535 panfrost_should_linear_convert(struct panfrost_context *ctx,
1536 struct panfrost_resource *prsrc,
1537 struct pipe_transfer *transfer)
1538 {
1539 if (prsrc->modifier_constant)
1540 return false;
1541
1542 /* Overwriting the entire resource indicates streaming, for which
1543 * linear layout is most efficient due to the lack of expensive
1544 * conversion.
1545 *
1546 * For now we just switch to linear after a number of complete
1547 * overwrites to keep things simple, but we could do better.
1548 *
1549 * This mechanism is only implemented for 2D resources. This suffices
1550 * for video players, its intended use case.
1551 */
1552
1553 bool entire_overwrite = panfrost_is_2d(prsrc) &&
1554 prsrc->base.last_level == 0 &&
1555 transfer->box.width == prsrc->base.width0 &&
1556 transfer->box.height == prsrc->base.height0 &&
1557 transfer->box.x == 0 && transfer->box.y == 0;
1558
1559 if (entire_overwrite)
1560 ++prsrc->modifier_updates;
1561
1562 if (prsrc->modifier_updates >= LAYOUT_CONVERT_THRESHOLD) {
1563 perf_debug(ctx, "Transitioning to linear due to streaming usage");
1564 return true;
1565 } else {
1566 return false;
1567 }
1568 }
1569
1570 struct panfrost_bo *
panfrost_get_afbc_superblock_sizes(struct panfrost_context * ctx,struct panfrost_resource * rsrc,unsigned first_level,unsigned last_level,unsigned * out_offsets)1571 panfrost_get_afbc_superblock_sizes(struct panfrost_context *ctx,
1572 struct panfrost_resource *rsrc,
1573 unsigned first_level, unsigned last_level,
1574 unsigned *out_offsets)
1575 {
1576 struct panfrost_screen *screen = pan_screen(ctx->base.screen);
1577 struct panfrost_device *dev = pan_device(ctx->base.screen);
1578 struct panfrost_batch *batch;
1579 struct panfrost_bo *bo;
1580 unsigned metadata_size = 0;
1581
1582 for (int level = first_level; level <= last_level; ++level) {
1583 struct pan_image_slice_layout *slice = &rsrc->image.layout.slices[level];
1584 unsigned sz = slice->afbc.nr_blocks * sizeof(struct pan_afbc_block_info);
1585 out_offsets[level - first_level] = metadata_size;
1586 metadata_size += sz;
1587 }
1588
1589 panfrost_flush_batches_accessing_rsrc(ctx, rsrc, "AFBC before size flush");
1590 batch = panfrost_get_fresh_batch_for_fbo(ctx, "AFBC superblock sizes");
1591 bo = panfrost_bo_create(dev, metadata_size, 0, "AFBC superblock sizes");
1592 assert(bo);
1593
1594 for (int level = first_level; level <= last_level; ++level) {
1595 unsigned offset = out_offsets[level - first_level];
1596 screen->vtbl.afbc_size(batch, rsrc, bo, offset, level);
1597 }
1598
1599 panfrost_flush_batches_accessing_rsrc(ctx, rsrc, "AFBC after size flush");
1600
1601 return bo;
1602 }
1603
1604 void
panfrost_pack_afbc(struct panfrost_context * ctx,struct panfrost_resource * prsrc)1605 panfrost_pack_afbc(struct panfrost_context *ctx,
1606 struct panfrost_resource *prsrc)
1607 {
1608 struct panfrost_screen *screen = pan_screen(ctx->base.screen);
1609 struct panfrost_device *dev = pan_device(ctx->base.screen);
1610 struct panfrost_bo *metadata_bo;
1611 unsigned metadata_offsets[PIPE_MAX_TEXTURE_LEVELS];
1612
1613 assert(prsrc->base.array_size == 1);
1614
1615 uint64_t src_modifier = prsrc->image.layout.modifier;
1616 uint64_t dst_modifier =
1617 src_modifier & ~(AFBC_FORMAT_MOD_TILED | AFBC_FORMAT_MOD_SPARSE);
1618 bool is_tiled = src_modifier & AFBC_FORMAT_MOD_TILED;
1619 unsigned last_level = prsrc->base.last_level;
1620 struct pan_image_slice_layout slice_infos[PIPE_MAX_TEXTURE_LEVELS] = {0};
1621 unsigned total_size = 0;
1622
1623 /* It doesn't make sense to pack everything if we need to unpack right
1624 * away to upload data to another level */
1625 for (int i = 0; i <= last_level; i++) {
1626 if (!BITSET_TEST(prsrc->valid.data, i))
1627 return;
1628 }
1629
1630 metadata_bo = panfrost_get_afbc_superblock_sizes(ctx, prsrc, 0, last_level,
1631 metadata_offsets);
1632 panfrost_bo_wait(metadata_bo, INT64_MAX, false);
1633
1634 for (unsigned level = 0; level <= last_level; ++level) {
1635 struct pan_image_slice_layout *src_slice =
1636 &prsrc->image.layout.slices[level];
1637 struct pan_image_slice_layout *dst_slice = &slice_infos[level];
1638
1639 unsigned width = u_minify(prsrc->base.width0, level);
1640 unsigned height = u_minify(prsrc->base.height0, level);
1641 unsigned src_stride =
1642 pan_afbc_stride_blocks(src_modifier, src_slice->row_stride);
1643 unsigned dst_stride =
1644 DIV_ROUND_UP(width, panfrost_afbc_superblock_width(dst_modifier));
1645 unsigned dst_height =
1646 DIV_ROUND_UP(height, panfrost_afbc_superblock_height(dst_modifier));
1647
1648 uint32_t offset = 0;
1649 struct pan_afbc_block_info *meta =
1650 metadata_bo->ptr.cpu + metadata_offsets[level];
1651
1652 for (unsigned y = 0, i = 0; y < dst_height; ++y) {
1653 for (unsigned x = 0; x < dst_stride; ++x, ++i) {
1654 unsigned idx = is_tiled ? get_morton_index(x, y, src_stride) : i;
1655 uint32_t size = meta[idx].size;
1656 meta[idx].offset = offset; /* write the start offset */
1657 offset += size;
1658 }
1659 }
1660
1661 total_size = ALIGN_POT(total_size, pan_slice_align(dst_modifier));
1662 {
1663 dst_slice->afbc.stride = dst_stride;
1664 dst_slice->afbc.nr_blocks = dst_stride * dst_height;
1665 dst_slice->afbc.header_size =
1666 ALIGN_POT(dst_stride * dst_height * AFBC_HEADER_BYTES_PER_TILE,
1667 pan_afbc_body_align(dev->arch, dst_modifier));
1668 dst_slice->afbc.body_size = offset;
1669 dst_slice->afbc.surface_stride = dst_slice->afbc.header_size + offset;
1670
1671 dst_slice->offset = total_size;
1672 dst_slice->row_stride = dst_stride * AFBC_HEADER_BYTES_PER_TILE;
1673 dst_slice->surface_stride = dst_slice->afbc.surface_stride;
1674 dst_slice->size = dst_slice->afbc.surface_stride;
1675
1676 /* We can't write to AFBC-packed resource, so there is no reason to
1677 * keep CRC data around */
1678 dst_slice->crc.offset = 0;
1679 dst_slice->crc.size = 0;
1680 }
1681 total_size += dst_slice->afbc.surface_stride;
1682 }
1683
1684 unsigned new_size = ALIGN_POT(total_size, 4096); // FIXME
1685 unsigned old_size = panfrost_bo_size(prsrc->bo);
1686 unsigned ratio = 100 * new_size / old_size;
1687
1688 if (ratio > screen->max_afbc_packing_ratio) {
1689 panfrost_bo_unreference(metadata_bo);
1690 return;
1691 }
1692 perf_debug(ctx, "%i%%: %i KB -> %i KB\n", ratio, old_size / 1024,
1693 new_size / 1024);
1694
1695 struct panfrost_bo *dst =
1696 panfrost_bo_create(dev, new_size, 0, "AFBC compact texture");
1697 assert(dst);
1698 struct panfrost_batch *batch =
1699 panfrost_get_fresh_batch_for_fbo(ctx, "AFBC compaction");
1700
1701 for (unsigned level = 0; level <= last_level; ++level) {
1702 struct pan_image_slice_layout *slice = &slice_infos[level];
1703 screen->vtbl.afbc_pack(batch, prsrc, dst, slice, metadata_bo,
1704 metadata_offsets[level], level);
1705 prsrc->image.layout.slices[level] = *slice;
1706 }
1707 prsrc->image.layout.array_stride = new_size;
1708 prsrc->image.layout.data_size = new_size;
1709
1710 panfrost_flush_batches_accessing_rsrc(ctx, prsrc, "AFBC compaction flush");
1711
1712 prsrc->image.layout.modifier = dst_modifier;
1713 panfrost_bo_unreference(prsrc->bo);
1714 prsrc->bo = dst;
1715 prsrc->image.data.base = dst->ptr.gpu;
1716 prsrc->image.layout.crc = false;
1717 prsrc->valid.crc = false;
1718 panfrost_bo_unreference(metadata_bo);
1719 }
1720
1721 static void
panfrost_ptr_unmap(struct pipe_context * pctx,struct pipe_transfer * transfer)1722 panfrost_ptr_unmap(struct pipe_context *pctx, struct pipe_transfer *transfer)
1723 {
1724 /* Gallium expects writeback here, so we tile */
1725
1726 struct panfrost_context *ctx = pan_context(pctx);
1727 struct pipe_screen *screen = ctx->base.screen;
1728 struct panfrost_transfer *trans = pan_transfer(transfer);
1729 struct panfrost_resource *prsrc =
1730 (struct panfrost_resource *)transfer->resource;
1731 struct panfrost_device *dev = pan_device(pctx->screen);
1732
1733 if (transfer->usage & PIPE_MAP_WRITE)
1734 prsrc->valid.crc = false;
1735
1736 /* AFBC/AFRC will use a staging resource. `initialized` will be set when
1737 * the fragment job is created; this is deferred to prevent useless surface
1738 * reloads that can cascade into DATA_INVALID_FAULTs due to reading
1739 * malformed AFBC/AFRC data if uninitialized */
1740
1741 if (trans->staging.rsrc) {
1742 if (transfer->usage & PIPE_MAP_WRITE) {
1743 if (panfrost_should_linear_convert(ctx, prsrc, transfer)) {
1744
1745 panfrost_bo_unreference(prsrc->bo);
1746
1747 panfrost_resource_setup(screen, prsrc, DRM_FORMAT_MOD_LINEAR,
1748 prsrc->image.layout.format);
1749
1750 prsrc->bo = pan_resource(trans->staging.rsrc)->bo;
1751 prsrc->image.data.base = prsrc->bo->ptr.gpu;
1752 panfrost_bo_reference(prsrc->bo);
1753 } else {
1754 bool discard = panfrost_can_discard(&prsrc->base, &transfer->box,
1755 transfer->usage);
1756 pan_legalize_format(ctx, prsrc, prsrc->image.layout.format, true,
1757 discard);
1758 pan_blit_from_staging(pctx, trans);
1759 panfrost_flush_batches_accessing_rsrc(
1760 ctx, pan_resource(trans->staging.rsrc),
1761 "AFBC write staging blit");
1762
1763 if (pan_screen(pctx->screen)->force_afbc_packing) {
1764 if (panfrost_should_pack_afbc(dev, prsrc))
1765 panfrost_pack_afbc(ctx, prsrc);
1766 }
1767 }
1768 }
1769
1770 pipe_resource_reference(&trans->staging.rsrc, NULL);
1771 }
1772
1773 /* Tiling will occur in software from a staging cpu buffer */
1774 if (trans->map) {
1775 struct panfrost_bo *bo = prsrc->bo;
1776
1777 if (transfer->usage & PIPE_MAP_WRITE) {
1778 BITSET_SET(prsrc->valid.data, transfer->level);
1779
1780 if (prsrc->image.layout.modifier ==
1781 DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) {
1782 if (panfrost_should_linear_convert(ctx, prsrc, transfer)) {
1783 panfrost_resource_setup(screen, prsrc, DRM_FORMAT_MOD_LINEAR,
1784 prsrc->image.layout.format);
1785 if (prsrc->image.layout.data_size > panfrost_bo_size(bo)) {
1786 const char *label = bo->label;
1787 panfrost_bo_unreference(bo);
1788 bo = prsrc->bo = panfrost_bo_create(
1789 dev, prsrc->image.layout.data_size, 0, label);
1790 prsrc->image.data.base = prsrc->bo->ptr.gpu;
1791 assert(bo);
1792 }
1793
1794 util_copy_rect(
1795 bo->ptr.cpu + prsrc->image.layout.slices[0].offset,
1796 prsrc->base.format, prsrc->image.layout.slices[0].row_stride,
1797 0, 0, transfer->box.width, transfer->box.height, trans->map,
1798 transfer->stride, 0, 0);
1799 } else {
1800 panfrost_store_tiled_images(trans, prsrc);
1801 }
1802 }
1803 }
1804 }
1805
1806 util_range_add(&prsrc->base, &prsrc->valid_buffer_range, transfer->box.x,
1807 transfer->box.x + transfer->box.width);
1808
1809 if (transfer->usage & PIPE_MAP_WRITE) {
1810 panfrost_minmax_cache_invalidate(prsrc->index_cache, transfer->box.x,
1811 transfer->box.width);
1812 }
1813
1814 /* Derefence the resource */
1815 pipe_resource_reference(&transfer->resource, NULL);
1816
1817 /* Transfer itself is RALLOCed at the moment */
1818 ralloc_free(transfer);
1819 }
1820
1821 static void
panfrost_ptr_flush_region(struct pipe_context * pctx,struct pipe_transfer * transfer,const struct pipe_box * box)1822 panfrost_ptr_flush_region(struct pipe_context *pctx,
1823 struct pipe_transfer *transfer,
1824 const struct pipe_box *box)
1825 {
1826 struct panfrost_resource *rsc = pan_resource(transfer->resource);
1827
1828 if (transfer->resource->target == PIPE_BUFFER) {
1829 util_range_add(&rsc->base, &rsc->valid_buffer_range,
1830 transfer->box.x + box->x,
1831 transfer->box.x + box->x + box->width);
1832 } else {
1833 BITSET_SET(rsc->valid.data, transfer->level);
1834 }
1835 }
1836
1837 static void
panfrost_invalidate_resource(struct pipe_context * pctx,struct pipe_resource * prsrc)1838 panfrost_invalidate_resource(struct pipe_context *pctx,
1839 struct pipe_resource *prsrc)
1840 {
1841 struct panfrost_context *ctx = pan_context(pctx);
1842 struct panfrost_batch *batch = panfrost_get_batch_for_fbo(ctx);
1843 struct panfrost_resource *rsrc = pan_resource(prsrc);
1844
1845 if (!batch) {
1846 mesa_loge("panfrost_invalidate_resource failed");
1847 return;
1848 }
1849
1850 rsrc->constant_stencil = true;
1851
1852 /* Handle the glInvalidateFramebuffer case */
1853 if (batch->key.zsbuf && batch->key.zsbuf->texture == prsrc)
1854 batch->resolve &= ~PIPE_CLEAR_DEPTHSTENCIL;
1855
1856 for (unsigned i = 0; i < batch->key.nr_cbufs; ++i) {
1857 struct pipe_surface *surf = batch->key.cbufs[i];
1858
1859 if (surf && surf->texture == prsrc)
1860 batch->resolve &= ~(PIPE_CLEAR_COLOR0 << i);
1861 }
1862 }
1863
1864 static enum pipe_format
panfrost_resource_get_internal_format(struct pipe_resource * rsrc)1865 panfrost_resource_get_internal_format(struct pipe_resource *rsrc)
1866 {
1867 struct panfrost_resource *prsrc = (struct panfrost_resource *)rsrc;
1868 return prsrc->image.layout.format;
1869 }
1870
1871 void
panfrost_set_image_view_planes(struct pan_image_view * iview,struct pipe_resource * texture)1872 panfrost_set_image_view_planes(struct pan_image_view *iview,
1873 struct pipe_resource *texture)
1874 {
1875 struct panfrost_resource *prsrc_plane = (struct panfrost_resource *)texture;
1876
1877 for (int i = 0; i < MAX_IMAGE_PLANES && prsrc_plane; i++) {
1878 iview->planes[i] = &prsrc_plane->image;
1879 prsrc_plane = (struct panfrost_resource *)prsrc_plane->base.next;
1880 }
1881 }
1882
1883 static bool
panfrost_generate_mipmap(struct pipe_context * pctx,struct pipe_resource * prsrc,enum pipe_format format,unsigned base_level,unsigned last_level,unsigned first_layer,unsigned last_layer)1884 panfrost_generate_mipmap(struct pipe_context *pctx, struct pipe_resource *prsrc,
1885 enum pipe_format format, unsigned base_level,
1886 unsigned last_level, unsigned first_layer,
1887 unsigned last_layer)
1888 {
1889 struct panfrost_resource *rsrc = pan_resource(prsrc);
1890
1891 perf_debug(pan_context(pctx), "Unoptimized mipmap generation");
1892
1893 /* Generating a mipmap invalidates the written levels, so make that
1894 * explicit so we don't try to wallpaper them back and end up with
1895 * u_blitter recursion */
1896
1897 assert(rsrc->bo);
1898 for (unsigned l = base_level + 1; l <= last_level; ++l)
1899 BITSET_CLEAR(rsrc->valid.data, l);
1900
1901 /* Beyond that, we just delegate the hard stuff. */
1902
1903 bool blit_res =
1904 util_gen_mipmap(pctx, prsrc, format, base_level, last_level, first_layer,
1905 last_layer, PIPE_TEX_FILTER_LINEAR);
1906
1907 return blit_res;
1908 }
1909
1910 static void
panfrost_resource_set_stencil(struct pipe_resource * prsrc,struct pipe_resource * stencil)1911 panfrost_resource_set_stencil(struct pipe_resource *prsrc,
1912 struct pipe_resource *stencil)
1913 {
1914 pan_resource(prsrc)->separate_stencil = pan_resource(stencil);
1915 }
1916
1917 static struct pipe_resource *
panfrost_resource_get_stencil(struct pipe_resource * prsrc)1918 panfrost_resource_get_stencil(struct pipe_resource *prsrc)
1919 {
1920 if (!pan_resource(prsrc)->separate_stencil)
1921 return NULL;
1922
1923 return &pan_resource(prsrc)->separate_stencil->base;
1924 }
1925
1926 static const struct u_transfer_vtbl transfer_vtbl = {
1927 .resource_create = panfrost_resource_create,
1928 .resource_destroy = panfrost_resource_destroy,
1929 .transfer_map = panfrost_ptr_map,
1930 .transfer_unmap = panfrost_ptr_unmap,
1931 .transfer_flush_region = panfrost_ptr_flush_region,
1932 .get_internal_format = panfrost_resource_get_internal_format,
1933 .set_stencil = panfrost_resource_set_stencil,
1934 .get_stencil = panfrost_resource_get_stencil,
1935 };
1936
1937 void
panfrost_resource_screen_init(struct pipe_screen * pscreen)1938 panfrost_resource_screen_init(struct pipe_screen *pscreen)
1939 {
1940 pscreen->resource_create_with_modifiers =
1941 panfrost_resource_create_with_modifiers;
1942 pscreen->resource_create = u_transfer_helper_resource_create;
1943 pscreen->resource_destroy = u_transfer_helper_resource_destroy;
1944 pscreen->resource_from_handle = panfrost_resource_from_handle;
1945 pscreen->resource_get_handle = panfrost_resource_get_handle;
1946 pscreen->resource_get_param = panfrost_resource_get_param;
1947 pscreen->transfer_helper = u_transfer_helper_create(
1948 &transfer_vtbl,
1949 U_TRANSFER_HELPER_SEPARATE_Z32S8 | U_TRANSFER_HELPER_MSAA_MAP);
1950 }
1951 void
panfrost_resource_screen_destroy(struct pipe_screen * pscreen)1952 panfrost_resource_screen_destroy(struct pipe_screen *pscreen)
1953 {
1954 u_transfer_helper_destroy(pscreen->transfer_helper);
1955 }
1956
1957 void
panfrost_resource_context_init(struct pipe_context * pctx)1958 panfrost_resource_context_init(struct pipe_context *pctx)
1959 {
1960 pctx->buffer_map = u_transfer_helper_transfer_map;
1961 pctx->buffer_unmap = u_transfer_helper_transfer_unmap;
1962 pctx->texture_map = u_transfer_helper_transfer_map;
1963 pctx->texture_unmap = u_transfer_helper_transfer_unmap;
1964 pctx->create_surface = panfrost_create_surface;
1965 pctx->surface_destroy = panfrost_surface_destroy;
1966 pctx->resource_copy_region = util_resource_copy_region;
1967 pctx->blit = panfrost_blit;
1968 pctx->generate_mipmap = panfrost_generate_mipmap;
1969 pctx->flush_resource = panfrost_flush_resource;
1970 pctx->invalidate_resource = panfrost_invalidate_resource;
1971 pctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;
1972 pctx->buffer_subdata = u_default_buffer_subdata;
1973 pctx->texture_subdata = u_default_texture_subdata;
1974 pctx->clear_buffer = u_default_clear_buffer;
1975 pctx->clear_render_target = panfrost_clear_render_target;
1976 pctx->clear_depth_stencil = panfrost_clear_depth_stencil;
1977 }
1978