1 /*
2 * Copyright (C) 2008 VMware, Inc.
3 * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org>
4 * Copyright (C) 2014-2017 Broadcom
5 * Copyright (C) 2018-2019 Alyssa Rosenzweig
6 * Copyright (C) 2019 Collabora, Ltd.
7 * Copyright (C) 2023 Amazon.com, Inc. or its affiliates
8 *
9 * Permission is hereby granted, free of charge, to any person obtaining a
10 * copy of this software and associated documentation files (the "Software"),
11 * to deal in the Software without restriction, including without limitation
12 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
13 * and/or sell copies of the Software, and to permit persons to whom the
14 * Software is furnished to do so, subject to the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the next
17 * paragraph) shall be included in all copies or substantial portions of the
18 * Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26 * SOFTWARE.
27 *
28 * Authors (Collabora):
29 * Tomeu Vizoso <tomeu.vizoso@collabora.com>
30 * Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
31 *
32 */
33
34 #include <fcntl.h>
35 #include <xf86drm.h>
36 #include "drm-uapi/drm_fourcc.h"
37
38 #include "frontend/winsys_handle.h"
39 #include "util/format/u_format.h"
40 #include "util/u_debug_image.h"
41 #include "util/u_drm.h"
42 #include "util/u_gen_mipmap.h"
43 #include "util/u_memory.h"
44 #include "util/u_resource.h"
45 #include "util/u_surface.h"
46 #include "util/u_transfer.h"
47 #include "util/u_transfer_helper.h"
48
49 #include "decode.h"
50 #include "pan_bo.h"
51 #include "pan_context.h"
52 #include "pan_resource.h"
53 #include "pan_screen.h"
54 #include "pan_tiling.h"
55 #include "pan_util.h"
56
57 static void
panfrost_clear_depth_stencil(struct pipe_context * pipe,struct pipe_surface * dst,unsigned clear_flags,double depth,unsigned stencil,unsigned dstx,unsigned dsty,unsigned width,unsigned height,bool render_condition_enabled)58 panfrost_clear_depth_stencil(struct pipe_context *pipe,
59 struct pipe_surface *dst, unsigned clear_flags,
60 double depth, unsigned stencil, unsigned dstx,
61 unsigned dsty, unsigned width, unsigned height,
62 bool render_condition_enabled)
63 {
64 struct panfrost_context *ctx = pan_context(pipe);
65
66 if (render_condition_enabled && !panfrost_render_condition_check(ctx))
67 return;
68
69 panfrost_blitter_save(
70 ctx, render_condition_enabled ? PAN_RENDER_COND : PAN_RENDER_BASE);
71 util_blitter_clear_depth_stencil(ctx->blitter, dst, clear_flags, depth,
72 stencil, dstx, dsty, width, height);
73 }
74
75 static void
panfrost_clear_render_target(struct pipe_context * pipe,struct pipe_surface * dst,const union pipe_color_union * color,unsigned dstx,unsigned dsty,unsigned width,unsigned height,bool render_condition_enabled)76 panfrost_clear_render_target(struct pipe_context *pipe,
77 struct pipe_surface *dst,
78 const union pipe_color_union *color, unsigned dstx,
79 unsigned dsty, unsigned width, unsigned height,
80 bool render_condition_enabled)
81 {
82 struct panfrost_context *ctx = pan_context(pipe);
83
84 if (render_condition_enabled && !panfrost_render_condition_check(ctx))
85 return;
86
87 panfrost_blitter_save(
88 ctx, render_condition_enabled ? PAN_RENDER_COND : PAN_RENDER_BASE);
89 util_blitter_clear_render_target(ctx->blitter, dst, color, dstx, dsty, width,
90 height);
91 }
92
93 static struct pipe_resource *
panfrost_resource_from_handle(struct pipe_screen * pscreen,const struct pipe_resource * templat,struct winsys_handle * whandle,unsigned usage)94 panfrost_resource_from_handle(struct pipe_screen *pscreen,
95 const struct pipe_resource *templat,
96 struct winsys_handle *whandle, unsigned usage)
97 {
98 struct panfrost_device *dev = pan_device(pscreen);
99 struct panfrost_resource *rsc;
100 struct pipe_resource *prsc;
101
102 assert(whandle->type == WINSYS_HANDLE_TYPE_FD);
103
104 rsc = CALLOC_STRUCT(panfrost_resource);
105 if (!rsc)
106 return NULL;
107
108 prsc = &rsc->base;
109
110 *prsc = *templat;
111
112 pipe_reference_init(&prsc->reference, 1);
113 prsc->screen = pscreen;
114
115 uint64_t mod = whandle->modifier == DRM_FORMAT_MOD_INVALID
116 ? DRM_FORMAT_MOD_LINEAR
117 : whandle->modifier;
118 enum mali_texture_dimension dim =
119 panfrost_translate_texture_dimension(templat->target);
120 struct pan_image_explicit_layout explicit_layout = {
121 .offset = whandle->offset,
122 .row_stride =
123 panfrost_from_legacy_stride(whandle->stride, templat->format, mod),
124 };
125
126 rsc->image.layout = (struct pan_image_layout){
127 .modifier = mod,
128 .format = templat->format,
129 .dim = dim,
130 .width = prsc->width0,
131 .height = prsc->height0,
132 .depth = prsc->depth0,
133 .array_size = prsc->array_size,
134 .nr_samples = MAX2(prsc->nr_samples, 1),
135 .nr_slices = 1,
136 };
137
138 bool valid =
139 pan_image_layout_init(dev->arch, &rsc->image.layout, &explicit_layout);
140
141 if (!valid) {
142 FREE(rsc);
143 return NULL;
144 }
145
146 rsc->bo = panfrost_bo_import(dev, whandle->handle);
147 /* Sometimes an import can fail e.g. on an invalid buffer fd, out of
148 * memory space to mmap it etc.
149 */
150 if (!rsc->bo) {
151 FREE(rsc);
152 return NULL;
153 }
154
155 rsc->image.data.base = rsc->bo->ptr.gpu;
156 rsc->modifier_constant = true;
157
158 BITSET_SET(rsc->valid.data, 0);
159 panfrost_resource_set_damage_region(pscreen, &rsc->base, 0, NULL);
160
161 if (dev->ro) {
162 rsc->scanout =
163 renderonly_create_gpu_import_for_resource(prsc, dev->ro, NULL);
164 /* failure is expected in some cases.. */
165 }
166
167 return prsc;
168 }
169
170 static bool
panfrost_resource_get_handle(struct pipe_screen * pscreen,struct pipe_context * ctx,struct pipe_resource * pt,struct winsys_handle * handle,unsigned usage)171 panfrost_resource_get_handle(struct pipe_screen *pscreen,
172 struct pipe_context *ctx, struct pipe_resource *pt,
173 struct winsys_handle *handle, unsigned usage)
174 {
175 struct panfrost_device *dev = pan_device(pscreen);
176 struct panfrost_resource *rsrc;
177 struct renderonly_scanout *scanout;
178 struct pipe_resource *cur = pt;
179
180 /* Even though panfrost doesn't support multi-planar formats, we
181 * can get here through GBM, which does. Walk the list of planes
182 * to find the right one.
183 */
184 for (int i = 0; i < handle->plane; i++) {
185 cur = cur->next;
186 if (!cur)
187 return false;
188 }
189 rsrc = pan_resource(cur);
190 scanout = rsrc->scanout;
191
192 handle->modifier = rsrc->image.layout.modifier;
193 rsrc->modifier_constant = true;
194
195 if (handle->type == WINSYS_HANDLE_TYPE_KMS && dev->ro) {
196 return renderonly_get_handle(scanout, handle);
197 } else if (handle->type == WINSYS_HANDLE_TYPE_KMS) {
198 handle->handle = panfrost_bo_handle(rsrc->bo);
199 } else if (handle->type == WINSYS_HANDLE_TYPE_FD) {
200 int fd = panfrost_bo_export(rsrc->bo);
201
202 if (fd < 0)
203 return false;
204
205 handle->handle = fd;
206 } else {
207 /* Other handle types not supported */
208 return false;
209 }
210
211 handle->stride = panfrost_get_legacy_stride(&rsrc->image.layout, 0);
212 handle->offset = rsrc->image.layout.slices[0].offset;
213 return true;
214 }
215
216 static bool
panfrost_resource_get_param(struct pipe_screen * pscreen,struct pipe_context * pctx,struct pipe_resource * prsc,unsigned plane,unsigned layer,unsigned level,enum pipe_resource_param param,unsigned usage,uint64_t * value)217 panfrost_resource_get_param(struct pipe_screen *pscreen,
218 struct pipe_context *pctx,
219 struct pipe_resource *prsc, unsigned plane,
220 unsigned layer, unsigned level,
221 enum pipe_resource_param param, unsigned usage,
222 uint64_t *value)
223 {
224 struct panfrost_resource *rsrc =
225 (struct panfrost_resource *)util_resource_at_index(prsc, plane);
226
227 switch (param) {
228 case PIPE_RESOURCE_PARAM_STRIDE:
229 *value = panfrost_get_legacy_stride(&rsrc->image.layout, level);
230 return true;
231 case PIPE_RESOURCE_PARAM_OFFSET:
232 *value = rsrc->image.layout.slices[level].offset;
233 return true;
234 case PIPE_RESOURCE_PARAM_MODIFIER:
235 *value = rsrc->image.layout.modifier;
236 return true;
237 case PIPE_RESOURCE_PARAM_NPLANES:
238 *value = util_resource_num(prsc);
239 return true;
240 default:
241 return false;
242 }
243 }
244
245 static void
panfrost_flush_resource(struct pipe_context * pctx,struct pipe_resource * prsc)246 panfrost_flush_resource(struct pipe_context *pctx, struct pipe_resource *prsc)
247 {
248 /* TODO */
249 }
250
251 static struct pipe_surface *
panfrost_create_surface(struct pipe_context * pipe,struct pipe_resource * pt,const struct pipe_surface * surf_tmpl)252 panfrost_create_surface(struct pipe_context *pipe, struct pipe_resource *pt,
253 const struct pipe_surface *surf_tmpl)
254 {
255 struct pipe_surface *ps = NULL;
256
257 ps = CALLOC_STRUCT(pipe_surface);
258
259 if (ps) {
260 pipe_reference_init(&ps->reference, 1);
261 pipe_resource_reference(&ps->texture, pt);
262 ps->context = pipe;
263 ps->format = surf_tmpl->format;
264
265 if (pt->target != PIPE_BUFFER) {
266 assert(surf_tmpl->u.tex.level <= pt->last_level);
267 ps->width = u_minify(pt->width0, surf_tmpl->u.tex.level);
268 ps->height = u_minify(pt->height0, surf_tmpl->u.tex.level);
269 ps->nr_samples = surf_tmpl->nr_samples;
270 ps->u.tex.level = surf_tmpl->u.tex.level;
271 ps->u.tex.first_layer = surf_tmpl->u.tex.first_layer;
272 ps->u.tex.last_layer = surf_tmpl->u.tex.last_layer;
273 } else {
274 /* setting width as number of elements should get us correct
275 * renderbuffer width */
276 ps->width =
277 surf_tmpl->u.buf.last_element - surf_tmpl->u.buf.first_element + 1;
278 ps->height = pt->height0;
279 ps->u.buf.first_element = surf_tmpl->u.buf.first_element;
280 ps->u.buf.last_element = surf_tmpl->u.buf.last_element;
281 assert(ps->u.buf.first_element <= ps->u.buf.last_element);
282 assert(ps->u.buf.last_element < ps->width);
283 }
284 }
285
286 return ps;
287 }
288
289 static void
panfrost_surface_destroy(struct pipe_context * pipe,struct pipe_surface * surf)290 panfrost_surface_destroy(struct pipe_context *pipe, struct pipe_surface *surf)
291 {
292 assert(surf->texture);
293 pipe_resource_reference(&surf->texture, NULL);
294 free(surf);
295 }
296
297 static inline bool
panfrost_is_2d(const struct panfrost_resource * pres)298 panfrost_is_2d(const struct panfrost_resource *pres)
299 {
300 return (pres->base.target == PIPE_TEXTURE_2D) ||
301 (pres->base.target == PIPE_TEXTURE_RECT);
302 }
303
304 /* Based on the usage, determine if it makes sense to use u-inteleaved tiling.
305 * We only have routines to tile 2D textures of sane bpps. On the hardware
306 * level, not all usages are valid for tiling. Finally, if the app is hinting
307 * that the contents frequently change, tiling will be a loss.
308 *
309 * On platforms where it is supported, AFBC is even better. */
310
311 static bool
panfrost_should_afbc(struct panfrost_device * dev,const struct panfrost_resource * pres,enum pipe_format fmt)312 panfrost_should_afbc(struct panfrost_device *dev,
313 const struct panfrost_resource *pres, enum pipe_format fmt)
314 {
315 /* AFBC resources may be rendered to, textured from, or shared across
316 * processes, but may not be used as e.g buffers */
317 const unsigned valid_binding =
318 PIPE_BIND_DEPTH_STENCIL | PIPE_BIND_RENDER_TARGET | PIPE_BIND_BLENDABLE |
319 PIPE_BIND_SAMPLER_VIEW | PIPE_BIND_DISPLAY_TARGET | PIPE_BIND_SCANOUT |
320 PIPE_BIND_SHARED;
321
322 if (pres->base.bind & ~valid_binding)
323 return false;
324
325 /* AFBC support is optional */
326 if (!dev->has_afbc)
327 return false;
328
329 /* AFBC<-->staging is expensive */
330 if (pres->base.usage == PIPE_USAGE_STREAM)
331 return false;
332
333 /* If constant (non-data-dependent) format is requested, don't AFBC: */
334 if (pres->base.bind & PIPE_BIND_CONST_BW)
335 return false;
336
337 /* Only a small selection of formats are AFBC'able */
338 if (!panfrost_format_supports_afbc(dev->arch, fmt))
339 return false;
340
341 /* AFBC does not support layered (GLES3 style) multisampling. Use
342 * EXT_multisampled_render_to_texture instead */
343 if (pres->base.nr_samples > 1)
344 return false;
345
346 switch (pres->base.target) {
347 case PIPE_TEXTURE_2D:
348 case PIPE_TEXTURE_RECT:
349 case PIPE_TEXTURE_2D_ARRAY:
350 case PIPE_TEXTURE_CUBE:
351 case PIPE_TEXTURE_CUBE_ARRAY:
352 break;
353
354 case PIPE_TEXTURE_3D:
355 /* 3D AFBC is only supported on Bifrost v7+. It's supposed to
356 * be supported on Midgard but it doesn't seem to work */
357 if (dev->arch != 7)
358 return false;
359
360 break;
361
362 default:
363 return false;
364 }
365
366 /* For one tile, AFBC is a loss compared to u-interleaved */
367 if (pres->base.width0 <= 16 && pres->base.height0 <= 16)
368 return false;
369
370 /* Otherwise, we'd prefer AFBC as it is dramatically more efficient
371 * than linear or usually even u-interleaved */
372 return true;
373 }
374
375 /*
376 * For a resource we want to use AFBC with, should we use AFBC with tiled
377 * headers? On GPUs that support it, this is believed to be beneficial for
378 * images that are at least 128x128.
379 */
380 static bool
panfrost_should_tile_afbc(const struct panfrost_device * dev,const struct panfrost_resource * pres)381 panfrost_should_tile_afbc(const struct panfrost_device *dev,
382 const struct panfrost_resource *pres)
383 {
384 return panfrost_afbc_can_tile(dev->arch) && pres->base.width0 >= 128 &&
385 pres->base.height0 >= 128 && !(dev->debug & PAN_DBG_FORCE_PACK);
386 }
387
388 bool
panfrost_should_pack_afbc(struct panfrost_device * dev,const struct panfrost_resource * prsrc)389 panfrost_should_pack_afbc(struct panfrost_device *dev,
390 const struct panfrost_resource *prsrc)
391 {
392 const unsigned valid_binding = PIPE_BIND_DEPTH_STENCIL |
393 PIPE_BIND_RENDER_TARGET |
394 PIPE_BIND_SAMPLER_VIEW;
395
396 return panfrost_afbc_can_pack(prsrc->base.format) && panfrost_is_2d(prsrc) &&
397 drm_is_afbc(prsrc->image.layout.modifier) &&
398 (prsrc->image.layout.modifier & AFBC_FORMAT_MOD_SPARSE) &&
399 (prsrc->base.bind & ~valid_binding) == 0 &&
400 !prsrc->modifier_constant && prsrc->base.width0 >= 32 &&
401 prsrc->base.height0 >= 32;
402 }
403
404 static bool
panfrost_should_tile(struct panfrost_device * dev,const struct panfrost_resource * pres,enum pipe_format fmt)405 panfrost_should_tile(struct panfrost_device *dev,
406 const struct panfrost_resource *pres, enum pipe_format fmt)
407 {
408 const unsigned valid_binding =
409 PIPE_BIND_DEPTH_STENCIL | PIPE_BIND_RENDER_TARGET | PIPE_BIND_BLENDABLE |
410 PIPE_BIND_SAMPLER_VIEW | PIPE_BIND_DISPLAY_TARGET | PIPE_BIND_SCANOUT |
411 PIPE_BIND_SHARED;
412
413 /* The purpose of tiling is improving locality in both X- and
414 * Y-directions. If there is only a single pixel in either direction,
415 * tiling does not make sense; using a linear layout instead is optimal
416 * for both memory usage and performance.
417 */
418 if (MIN2(pres->base.width0, pres->base.height0) < 2)
419 return false;
420
421 bool can_tile = (pres->base.target != PIPE_BUFFER) &&
422 ((pres->base.bind & ~valid_binding) == 0);
423
424 return can_tile && (pres->base.usage != PIPE_USAGE_STREAM);
425 }
426
427 static uint64_t
panfrost_best_modifier(struct panfrost_device * dev,const struct panfrost_resource * pres,enum pipe_format fmt)428 panfrost_best_modifier(struct panfrost_device *dev,
429 const struct panfrost_resource *pres,
430 enum pipe_format fmt)
431 {
432 /* Force linear textures when debugging tiling/compression */
433 if (unlikely(dev->debug & PAN_DBG_LINEAR))
434 return DRM_FORMAT_MOD_LINEAR;
435
436 if (panfrost_should_afbc(dev, pres, fmt)) {
437 uint64_t afbc = AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | AFBC_FORMAT_MOD_SPARSE;
438
439 if (panfrost_afbc_can_ytr(pres->base.format))
440 afbc |= AFBC_FORMAT_MOD_YTR;
441
442 if (panfrost_should_tile_afbc(dev, pres))
443 afbc |= AFBC_FORMAT_MOD_TILED | AFBC_FORMAT_MOD_SC;
444
445 return DRM_FORMAT_MOD_ARM_AFBC(afbc);
446 } else if (panfrost_should_tile(dev, pres, fmt))
447 return DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED;
448 else
449 return DRM_FORMAT_MOD_LINEAR;
450 }
451
452 static bool
panfrost_should_checksum(const struct panfrost_device * dev,const struct panfrost_resource * pres)453 panfrost_should_checksum(const struct panfrost_device *dev,
454 const struct panfrost_resource *pres)
455 {
456 /* Checksumming is disabled by default due to fundamental unsoundness */
457 if (!(dev->debug & PAN_DBG_CRC))
458 return false;
459
460 /* When checksumming is enabled, the tile data must fit in the
461 * size of the writeback buffer, so don't checksum formats
462 * that use too much space. */
463
464 unsigned bytes_per_pixel_max = (dev->arch == 6) ? 6 : 4;
465
466 unsigned bytes_per_pixel = MAX2(pres->base.nr_samples, 1) *
467 util_format_get_blocksize(pres->base.format);
468
469 return pres->base.bind & PIPE_BIND_RENDER_TARGET && panfrost_is_2d(pres) &&
470 bytes_per_pixel <= bytes_per_pixel_max && pres->base.last_level == 0;
471 }
472
473 static void
panfrost_resource_setup(struct panfrost_device * dev,struct panfrost_resource * pres,uint64_t modifier,enum pipe_format fmt)474 panfrost_resource_setup(struct panfrost_device *dev,
475 struct panfrost_resource *pres, uint64_t modifier,
476 enum pipe_format fmt)
477 {
478 uint64_t chosen_mod = modifier != DRM_FORMAT_MOD_INVALID
479 ? modifier
480 : panfrost_best_modifier(dev, pres, fmt);
481 enum mali_texture_dimension dim =
482 panfrost_translate_texture_dimension(pres->base.target);
483
484 /* We can only switch tiled->linear if the resource isn't already
485 * linear and if we control the modifier */
486 pres->modifier_constant = !(chosen_mod != DRM_FORMAT_MOD_LINEAR &&
487 modifier == DRM_FORMAT_MOD_INVALID);
488
489 /* Z32_S8X24 variants are actually stored in 2 planes (one per
490 * component), we have to adjust the format on the first plane.
491 */
492 if (fmt == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT)
493 fmt = PIPE_FORMAT_Z32_FLOAT;
494
495 pres->image.layout = (struct pan_image_layout){
496 .modifier = chosen_mod,
497 .format = fmt,
498 .dim = dim,
499 .width = pres->base.width0,
500 .height = pres->base.height0,
501 .depth = pres->base.depth0,
502 .array_size = pres->base.array_size,
503 .nr_samples = MAX2(pres->base.nr_samples, 1),
504 .nr_slices = pres->base.last_level + 1,
505 .crc = panfrost_should_checksum(dev, pres),
506 };
507
508 ASSERTED bool valid =
509 pan_image_layout_init(dev->arch, &pres->image.layout, NULL);
510 assert(valid);
511 }
512
513 static void
panfrost_resource_init_afbc_headers(struct panfrost_resource * pres)514 panfrost_resource_init_afbc_headers(struct panfrost_resource *pres)
515 {
516 panfrost_bo_mmap(pres->bo);
517
518 unsigned nr_samples = MAX2(pres->base.nr_samples, 1);
519
520 for (unsigned i = 0; i < pres->base.array_size; ++i) {
521 for (unsigned l = 0; l <= pres->base.last_level; ++l) {
522 struct pan_image_slice_layout *slice = &pres->image.layout.slices[l];
523
524 for (unsigned s = 0; s < nr_samples; ++s) {
525 void *ptr = pres->bo->ptr.cpu +
526 (i * pres->image.layout.array_stride) + slice->offset +
527 (s * slice->afbc.surface_stride);
528
529 /* Zero-ed AFBC headers seem to encode a plain
530 * black. Let's use this pattern to keep the
531 * initialization simple.
532 */
533 memset(ptr, 0, slice->afbc.header_size);
534 }
535 }
536 }
537 }
538
539 void
panfrost_resource_set_damage_region(struct pipe_screen * screen,struct pipe_resource * res,unsigned int nrects,const struct pipe_box * rects)540 panfrost_resource_set_damage_region(struct pipe_screen *screen,
541 struct pipe_resource *res,
542 unsigned int nrects,
543 const struct pipe_box *rects)
544 {
545 struct panfrost_device *dev = pan_device(screen);
546 struct panfrost_resource *pres = pan_resource(res);
547 struct pipe_scissor_state *damage_extent = &pres->damage.extent;
548 unsigned int i;
549
550 /* Partial updates are implemented with a tile enable map only on v5.
551 * Later architectures have a more efficient method of implementing
552 * partial updates (frame shaders), while earlier architectures lack
553 * tile enable maps altogether.
554 */
555 if (dev->arch == 5 && nrects > 1) {
556 if (!pres->damage.tile_map.data) {
557 pres->damage.tile_map.stride =
558 ALIGN_POT(DIV_ROUND_UP(res->width0, 32 * 8), 64);
559 pres->damage.tile_map.size =
560 pres->damage.tile_map.stride * DIV_ROUND_UP(res->height0, 32);
561 pres->damage.tile_map.data = malloc(pres->damage.tile_map.size);
562 }
563
564 memset(pres->damage.tile_map.data, 0, pres->damage.tile_map.size);
565 pres->damage.tile_map.enable = true;
566 } else {
567 pres->damage.tile_map.enable = false;
568 }
569
570 /* Track the damage extent: the quad including all damage regions. Will
571 * be used restrict the rendering area */
572
573 damage_extent->minx = 0xffff;
574 damage_extent->miny = 0xffff;
575
576 unsigned enable_count = 0;
577
578 for (i = 0; i < nrects; i++) {
579 int x = rects[i].x, w = rects[i].width, h = rects[i].height;
580 int y = res->height0 - (rects[i].y + h);
581
582 damage_extent->minx = MIN2(damage_extent->minx, x);
583 damage_extent->miny = MIN2(damage_extent->miny, y);
584 damage_extent->maxx = MAX2(damage_extent->maxx, MIN2(x + w, res->width0));
585 damage_extent->maxy =
586 MAX2(damage_extent->maxy, MIN2(y + h, res->height0));
587
588 if (!pres->damage.tile_map.enable)
589 continue;
590
591 unsigned t_x_start = x / 32;
592 unsigned t_x_end = (x + w - 1) / 32;
593 unsigned t_y_start = y / 32;
594 unsigned t_y_end = (y + h - 1) / 32;
595
596 for (unsigned t_y = t_y_start; t_y <= t_y_end; t_y++) {
597 for (unsigned t_x = t_x_start; t_x <= t_x_end; t_x++) {
598 unsigned b = (t_y * pres->damage.tile_map.stride * 8) + t_x;
599
600 if (BITSET_TEST(pres->damage.tile_map.data, b))
601 continue;
602
603 BITSET_SET(pres->damage.tile_map.data, b);
604 enable_count++;
605 }
606 }
607 }
608
609 if (nrects == 0) {
610 damage_extent->minx = 0;
611 damage_extent->miny = 0;
612 damage_extent->maxx = res->width0;
613 damage_extent->maxy = res->height0;
614 }
615
616 if (pres->damage.tile_map.enable) {
617 unsigned t_x_start = damage_extent->minx / 32;
618 unsigned t_x_end = damage_extent->maxx / 32;
619 unsigned t_y_start = damage_extent->miny / 32;
620 unsigned t_y_end = damage_extent->maxy / 32;
621 unsigned tile_count =
622 (t_x_end - t_x_start + 1) * (t_y_end - t_y_start + 1);
623
624 /* Don't bother passing a tile-enable-map if the amount of
625 * tiles to reload is to close to the total number of tiles.
626 */
627 if (tile_count - enable_count < 10)
628 pres->damage.tile_map.enable = false;
629 }
630 }
631
632 struct pipe_resource *
panfrost_resource_create_with_modifier(struct pipe_screen * screen,const struct pipe_resource * template,uint64_t modifier)633 panfrost_resource_create_with_modifier(struct pipe_screen *screen,
634 const struct pipe_resource *template,
635 uint64_t modifier)
636 {
637 struct panfrost_device *dev = pan_device(screen);
638
639 struct panfrost_resource *so = CALLOC_STRUCT(panfrost_resource);
640 so->base = *template;
641 so->base.screen = screen;
642
643 pipe_reference_init(&so->base.reference, 1);
644
645 util_range_init(&so->valid_buffer_range);
646
647 if (template->bind & PAN_BIND_SHARED_MASK) {
648 /* For compatibility with older consumers that may not be
649 * modifiers aware, treat INVALID as LINEAR for shared
650 * resources.
651 */
652 if (modifier == DRM_FORMAT_MOD_INVALID)
653 modifier = DRM_FORMAT_MOD_LINEAR;
654
655 /* At any rate, we can't change the modifier later for shared
656 * resources, since we have no way to propagate the modifier
657 * change.
658 */
659 so->modifier_constant = true;
660 }
661
662 panfrost_resource_setup(dev, so, modifier, template->format);
663
664 /* Guess a label based on the bind */
665 unsigned bind = template->bind;
666 const char *label = (bind & PIPE_BIND_INDEX_BUFFER) ? "Index buffer"
667 : (bind & PIPE_BIND_SCANOUT) ? "Scanout"
668 : (bind & PIPE_BIND_DISPLAY_TARGET) ? "Display target"
669 : (bind & PIPE_BIND_SHARED) ? "Shared resource"
670 : (bind & PIPE_BIND_RENDER_TARGET) ? "Render target"
671 : (bind & PIPE_BIND_DEPTH_STENCIL)
672 ? "Depth/stencil buffer"
673 : (bind & PIPE_BIND_SAMPLER_VIEW) ? "Texture"
674 : (bind & PIPE_BIND_VERTEX_BUFFER) ? "Vertex buffer"
675 : (bind & PIPE_BIND_CONSTANT_BUFFER) ? "Constant buffer"
676 : (bind & PIPE_BIND_GLOBAL) ? "Global memory"
677 : (bind & PIPE_BIND_SHADER_BUFFER) ? "Shader buffer"
678 : (bind & PIPE_BIND_SHADER_IMAGE) ? "Shader image"
679 : "Other resource";
680
681 if (dev->ro && (template->bind & PIPE_BIND_SCANOUT)) {
682 struct winsys_handle handle;
683 struct pan_block_size blocksize =
684 panfrost_block_size(modifier, template->format);
685
686 /* Block-based texture formats are only used for texture
687 * compression (not framebuffer compression!), which doesn't
688 * make sense to share across processes.
689 */
690 assert(util_format_get_blockwidth(template->format) == 1);
691 assert(util_format_get_blockheight(template->format) == 1);
692
693 /* Present a resource with similar dimensions that, if allocated
694 * as a linear image, is big enough to fit the resource in the
695 * actual layout. For linear images, this is a no-op. For 16x16
696 * tiling, this aligns the dimensions to 16x16.
697 *
698 * For AFBC, this aligns the width to the superblock width (as
699 * expected) and adds extra rows to account for the header. This
700 * is a bit of a lie, but it's the best we can do with dumb
701 * buffers, which are extremely not meant for AFBC. And yet this
702 * has to work anyway...
703 *
704 * Moral of the story: if you're reading this comment, that
705 * means you're working on WSI and so it's already too late for
706 * you. I'm sorry.
707 */
708 unsigned width = ALIGN_POT(template->width0, blocksize.width);
709 unsigned stride = ALIGN_POT(template->width0, blocksize.width) *
710 util_format_get_blocksize(template->format);
711 unsigned size = so->image.layout.data_size;
712 unsigned effective_rows = DIV_ROUND_UP(size, stride);
713
714 struct pipe_resource scanout_tmpl = {
715 .target = so->base.target,
716 .format = template->format,
717 .width0 = width,
718 .height0 = effective_rows,
719 .depth0 = 1,
720 .array_size = 1,
721 };
722
723 so->scanout =
724 renderonly_scanout_for_resource(&scanout_tmpl, dev->ro, &handle);
725
726 if (!so->scanout) {
727 fprintf(stderr, "Failed to create scanout resource\n");
728 free(so);
729 return NULL;
730 }
731 assert(handle.type == WINSYS_HANDLE_TYPE_FD);
732 so->bo = panfrost_bo_import(dev, handle.handle);
733 so->image.data.base = so->bo->ptr.gpu;
734 close(handle.handle);
735
736 if (!so->bo) {
737 free(so);
738 return NULL;
739 }
740 } else {
741 /* We create a BO immediately but don't bother mapping, since we don't
742 * care to map e.g. FBOs which the CPU probably won't touch */
743 uint32_t flags = PAN_BO_DELAY_MMAP;
744
745 /* If the resource is never exported, we can make the BO private. */
746 if (template->bind & PIPE_BIND_SHARED)
747 flags |= PAN_BO_SHAREABLE;
748
749 so->bo =
750 panfrost_bo_create(dev, so->image.layout.data_size, flags, label);
751 so->image.data.base = so->bo->ptr.gpu;
752
753 so->constant_stencil = true;
754 }
755
756 if (drm_is_afbc(so->image.layout.modifier))
757 panfrost_resource_init_afbc_headers(so);
758
759 panfrost_resource_set_damage_region(screen, &so->base, 0, NULL);
760
761 if (template->bind & PIPE_BIND_INDEX_BUFFER)
762 so->index_cache = CALLOC_STRUCT(panfrost_minmax_cache);
763
764 return (struct pipe_resource *)so;
765 }
766
767 /* Default is to create a resource as don't care */
768
769 static struct pipe_resource *
panfrost_resource_create(struct pipe_screen * screen,const struct pipe_resource * template)770 panfrost_resource_create(struct pipe_screen *screen,
771 const struct pipe_resource *template)
772 {
773 return panfrost_resource_create_with_modifier(screen, template,
774 DRM_FORMAT_MOD_INVALID);
775 }
776
777 /* If no modifier is specified, we'll choose. Otherwise, the order of
778 * preference is compressed, tiled, linear. */
779
780 static struct pipe_resource *
panfrost_resource_create_with_modifiers(struct pipe_screen * screen,const struct pipe_resource * template,const uint64_t * modifiers,int count)781 panfrost_resource_create_with_modifiers(struct pipe_screen *screen,
782 const struct pipe_resource *template,
783 const uint64_t *modifiers, int count)
784 {
785 for (unsigned i = 0; i < PAN_MODIFIER_COUNT; ++i) {
786 if (drm_find_modifier(pan_best_modifiers[i], modifiers, count)) {
787 return panfrost_resource_create_with_modifier(screen, template,
788 pan_best_modifiers[i]);
789 }
790 }
791
792 /* If we didn't find one, app specified invalid */
793 assert(count == 1 && modifiers[0] == DRM_FORMAT_MOD_INVALID);
794 return panfrost_resource_create(screen, template);
795 }
796
797 static void
panfrost_resource_destroy(struct pipe_screen * screen,struct pipe_resource * pt)798 panfrost_resource_destroy(struct pipe_screen *screen, struct pipe_resource *pt)
799 {
800 struct panfrost_device *dev = pan_device(screen);
801 struct panfrost_resource *rsrc = (struct panfrost_resource *)pt;
802
803 if (rsrc->scanout)
804 renderonly_scanout_destroy(rsrc->scanout, dev->ro);
805
806 if (rsrc->bo)
807 panfrost_bo_unreference(rsrc->bo);
808
809 free(rsrc->index_cache);
810 free(rsrc->damage.tile_map.data);
811
812 util_range_destroy(&rsrc->valid_buffer_range);
813 free(rsrc);
814 }
815
816 /* Most of the time we can do CPU-side transfers, but sometimes we need to use
817 * the 3D pipe for this. Let's wrap u_blitter to blit to/from staging textures.
818 * Code adapted from freedreno */
819
820 static struct panfrost_resource *
pan_alloc_staging(struct panfrost_context * ctx,struct panfrost_resource * rsc,unsigned level,const struct pipe_box * box)821 pan_alloc_staging(struct panfrost_context *ctx, struct panfrost_resource *rsc,
822 unsigned level, const struct pipe_box *box)
823 {
824 struct pipe_context *pctx = &ctx->base;
825 struct pipe_resource tmpl = rsc->base;
826
827 tmpl.width0 = box->width;
828 tmpl.height0 = box->height;
829 /* for array textures, box->depth is the array_size, otherwise
830 * for 3d textures, it is the depth:
831 */
832 if (tmpl.array_size > 1) {
833 if (tmpl.target == PIPE_TEXTURE_CUBE)
834 tmpl.target = PIPE_TEXTURE_2D_ARRAY;
835 tmpl.array_size = box->depth;
836 tmpl.depth0 = 1;
837 } else {
838 tmpl.array_size = 1;
839 tmpl.depth0 = box->depth;
840 }
841 tmpl.last_level = 0;
842 tmpl.bind |= PIPE_BIND_LINEAR;
843 tmpl.bind &= ~PAN_BIND_SHARED_MASK;
844
845 struct pipe_resource *pstaging =
846 pctx->screen->resource_create(pctx->screen, &tmpl);
847 if (!pstaging)
848 return NULL;
849
850 return pan_resource(pstaging);
851 }
852
853 static enum pipe_format
pan_blit_format(enum pipe_format fmt)854 pan_blit_format(enum pipe_format fmt)
855 {
856 const struct util_format_description *desc;
857 desc = util_format_description(fmt);
858
859 /* This must be an emulated format (using u_transfer_helper) as if it
860 * was real RGTC we wouldn't have used AFBC and needed a blit. */
861 if (desc->layout == UTIL_FORMAT_LAYOUT_RGTC)
862 fmt = PIPE_FORMAT_R8G8B8A8_UNORM;
863
864 return fmt;
865 }
866
867 static void
pan_blit_from_staging(struct pipe_context * pctx,struct panfrost_transfer * trans)868 pan_blit_from_staging(struct pipe_context *pctx,
869 struct panfrost_transfer *trans)
870 {
871 struct pipe_resource *dst = trans->base.resource;
872 struct pipe_blit_info blit = {0};
873
874 blit.dst.resource = dst;
875 blit.dst.format = dst->format;
876 blit.dst.level = trans->base.level;
877 blit.dst.box = trans->base.box;
878 blit.src.resource = trans->staging.rsrc;
879 blit.src.format = trans->staging.rsrc->format;
880 blit.src.level = 0;
881 blit.src.box = trans->staging.box;
882 blit.mask = util_format_get_mask(blit.src.format);
883 blit.filter = PIPE_TEX_FILTER_NEAREST;
884
885 panfrost_blit_no_afbc_legalization(pctx, &blit);
886 }
887
888 static void
pan_blit_to_staging(struct pipe_context * pctx,struct panfrost_transfer * trans)889 pan_blit_to_staging(struct pipe_context *pctx, struct panfrost_transfer *trans)
890 {
891 struct pipe_resource *src = trans->base.resource;
892 struct pipe_blit_info blit = {0};
893
894 blit.src.resource = src;
895 blit.src.format = src->format;
896 blit.src.level = trans->base.level;
897 blit.src.box = trans->base.box;
898 blit.dst.resource = trans->staging.rsrc;
899 blit.dst.format = trans->staging.rsrc->format;
900 blit.dst.level = 0;
901 blit.dst.box = trans->staging.box;
902 blit.mask = util_format_get_mask(blit.dst.format);
903 blit.filter = PIPE_TEX_FILTER_NEAREST;
904
905 panfrost_blit_no_afbc_legalization(pctx, &blit);
906 }
907
908 static void
panfrost_load_tiled_images(struct panfrost_transfer * transfer,struct panfrost_resource * rsrc)909 panfrost_load_tiled_images(struct panfrost_transfer *transfer,
910 struct panfrost_resource *rsrc)
911 {
912 struct pipe_transfer *ptrans = &transfer->base;
913 unsigned level = ptrans->level;
914
915 /* If the requested level of the image is uninitialized, it's not
916 * necessary to copy it. Leave the result unintiialized too.
917 */
918 if (!BITSET_TEST(rsrc->valid.data, level))
919 return;
920
921 struct panfrost_bo *bo = rsrc->bo;
922 unsigned stride = panfrost_get_layer_stride(&rsrc->image.layout, level);
923
924 /* Otherwise, load each layer separately, required to load from 3D and
925 * array textures.
926 */
927 for (unsigned z = 0; z < ptrans->box.depth; ++z) {
928 void *dst = transfer->map + (ptrans->layer_stride * z);
929 uint8_t *map = bo->ptr.cpu + rsrc->image.layout.slices[level].offset +
930 (z + ptrans->box.z) * stride;
931
932 panfrost_load_tiled_image(dst, map, ptrans->box.x, ptrans->box.y,
933 ptrans->box.width, ptrans->box.height,
934 ptrans->stride,
935 rsrc->image.layout.slices[level].row_stride,
936 rsrc->image.layout.format);
937 }
938 }
939
940 #ifdef DEBUG
941
942 static unsigned
get_superblock_size(uint32_t * hdr,unsigned uncompressed_size)943 get_superblock_size(uint32_t *hdr, unsigned uncompressed_size)
944 {
945 /* AFBC superblock layout 0 */
946 unsigned body_base_ptr_len = 32;
947 unsigned nr_subblocks = 16;
948 unsigned sz_len = 6; /* bits */
949 unsigned mask = (1 << sz_len) - 1;
950 unsigned size = 0;
951
952 /* Sum up all of the subblock sizes */
953 for (int i = 0; i < nr_subblocks; i++) {
954 unsigned bitoffset = body_base_ptr_len + (i * sz_len);
955 unsigned start = bitoffset / 32;
956 unsigned end = (bitoffset + (sz_len - 1)) / 32;
957 unsigned offset = bitoffset % 32;
958 unsigned subblock_size;
959
960 if (start != end)
961 subblock_size = (hdr[start] >> offset) | (hdr[end] << (32 - offset));
962 else
963 subblock_size = hdr[start] >> offset;
964 subblock_size = (subblock_size == 1) ? uncompressed_size : subblock_size;
965 size += subblock_size & mask;
966
967 if (i == 0 && size == 0)
968 return 0;
969 }
970
971 return size;
972 }
973
974 static void
dump_block(struct panfrost_resource * rsrc,uint32_t idx)975 dump_block(struct panfrost_resource *rsrc, uint32_t idx)
976 {
977 panfrost_bo_wait(rsrc->bo, INT64_MAX, false);
978
979 uint8_t *ptr = rsrc->bo->ptr.cpu;
980 uint32_t *header = (uint32_t *)(ptr + (idx * AFBC_HEADER_BYTES_PER_TILE));
981 uint32_t body_base_ptr = header[0];
982 uint32_t *body = (uint32_t *)(ptr + body_base_ptr);
983 struct pan_block_size block_sz =
984 panfrost_afbc_subblock_size(rsrc->image.layout.modifier);
985 unsigned pixel_sz = util_format_get_blocksize(rsrc->base.format);
986 unsigned uncompressed_size = pixel_sz * block_sz.width * block_sz.height;
987 unsigned size = get_superblock_size(header, uncompressed_size);
988
989 fprintf(stderr, " Header: %08x %08x %08x %08x (size: %u bytes)\n",
990 header[0], header[1], header[2], header[3], size);
991 if (size > 0) {
992 fprintf(stderr, " Body: %08x %08x %08x %08x\n", body[0], body[1],
993 body[2], body[3]);
994 } else {
995 uint8_t *comp = (uint8_t *)(header + 2);
996 fprintf(stderr, " Color: 0x%02x%02x%02x%02x\n", comp[0], comp[1],
997 comp[2], comp[3]);
998 }
999 fprintf(stderr, "\n");
1000 }
1001
1002 void
pan_dump_resource(struct panfrost_context * ctx,struct panfrost_resource * rsc)1003 pan_dump_resource(struct panfrost_context *ctx, struct panfrost_resource *rsc)
1004 {
1005 struct pipe_context *pctx = &ctx->base;
1006 struct pipe_resource tmpl = rsc->base;
1007 struct pipe_resource *plinear = NULL;
1008 struct panfrost_resource *linear = rsc;
1009 struct pipe_blit_info blit = {0};
1010 struct pipe_box box;
1011 char buffer[1024];
1012
1013 if (rsc->image.layout.modifier != DRM_FORMAT_MOD_LINEAR) {
1014 tmpl.bind |= PIPE_BIND_LINEAR;
1015 tmpl.bind &= ~PAN_BIND_SHARED_MASK;
1016
1017 plinear = pctx->screen->resource_create(pctx->screen, &tmpl);
1018 u_box_2d(0, 0, rsc->base.width0, rsc->base.height0, &box);
1019
1020 blit.src.resource = &rsc->base;
1021 blit.src.format = rsc->base.format;
1022 blit.src.level = 0;
1023 blit.src.box = box;
1024 blit.dst.resource = plinear;
1025 blit.dst.format = rsc->base.format;
1026 blit.dst.level = 0;
1027 blit.dst.box = box;
1028 blit.mask = util_format_get_mask(blit.dst.format);
1029 blit.filter = PIPE_TEX_FILTER_NEAREST;
1030
1031 panfrost_blit(pctx, &blit);
1032
1033 linear = pan_resource(plinear);
1034 }
1035
1036 panfrost_flush_writer(ctx, linear, "dump image");
1037 panfrost_bo_wait(linear->bo, INT64_MAX, false);
1038 panfrost_bo_mmap(linear->bo);
1039
1040 static unsigned frame_count = 0;
1041 frame_count++;
1042 snprintf(buffer, sizeof(buffer), "dump_image.%04d", frame_count);
1043
1044 debug_dump_image(buffer, rsc->base.format, 0 /* UNUSED */, rsc->base.width0,
1045 rsc->base.height0,
1046 linear->image.layout.slices[0].row_stride,
1047 linear->bo->ptr.cpu);
1048
1049 if (plinear)
1050 pipe_resource_reference(&plinear, NULL);
1051 }
1052
1053 #endif
1054
1055 /* Get scan-order index from (x, y) position when blocks are
1056 * arranged in z-order in 8x8 tiles */
1057 static unsigned
get_morton_index(unsigned x,unsigned y,unsigned stride)1058 get_morton_index(unsigned x, unsigned y, unsigned stride)
1059 {
1060 unsigned i = ((x << 0) & 1) | ((y << 1) & 2) | ((x << 1) & 4) |
1061 ((y << 2) & 8) | ((x << 2) & 16) | ((y << 3) & 32);
1062 return (((y & ~7) * stride) + ((x & ~7) << 3)) + i;
1063 }
1064
1065 static void
panfrost_store_tiled_images(struct panfrost_transfer * transfer,struct panfrost_resource * rsrc)1066 panfrost_store_tiled_images(struct panfrost_transfer *transfer,
1067 struct panfrost_resource *rsrc)
1068 {
1069 struct panfrost_bo *bo = rsrc->bo;
1070 struct pipe_transfer *ptrans = &transfer->base;
1071 unsigned level = ptrans->level;
1072 unsigned stride = panfrost_get_layer_stride(&rsrc->image.layout, level);
1073
1074 /* Otherwise, store each layer separately, required to store to 3D and
1075 * array textures.
1076 */
1077 for (unsigned z = 0; z < ptrans->box.depth; ++z) {
1078 void *src = transfer->map + (ptrans->layer_stride * z);
1079 uint8_t *map = bo->ptr.cpu + rsrc->image.layout.slices[level].offset +
1080 (z + ptrans->box.z) * stride;
1081
1082 panfrost_store_tiled_image(map, src, ptrans->box.x, ptrans->box.y,
1083 ptrans->box.width, ptrans->box.height,
1084 rsrc->image.layout.slices[level].row_stride,
1085 ptrans->stride, rsrc->image.layout.format);
1086 }
1087 }
1088
1089 static bool
panfrost_box_covers_resource(const struct pipe_resource * resource,const struct pipe_box * box)1090 panfrost_box_covers_resource(const struct pipe_resource *resource,
1091 const struct pipe_box *box)
1092 {
1093 return resource->last_level == 0 &&
1094 util_texrange_covers_whole_level(resource, 0, box->x, box->y, box->z,
1095 box->width, box->height, box->depth);
1096 }
1097
1098 static bool
panfrost_can_discard(struct pipe_resource * resource,const struct pipe_box * box,unsigned usage)1099 panfrost_can_discard(struct pipe_resource *resource, const struct pipe_box *box,
1100 unsigned usage)
1101 {
1102 struct panfrost_resource *rsrc = pan_resource(resource);
1103
1104 return ((usage & PIPE_MAP_DISCARD_RANGE) &&
1105 !(usage & PIPE_MAP_UNSYNCHRONIZED) &&
1106 !(resource->flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) &&
1107 panfrost_box_covers_resource(resource, box) &&
1108 !(rsrc->bo->flags & PAN_BO_SHARED));
1109 }
1110
1111 static void *
panfrost_ptr_map(struct pipe_context * pctx,struct pipe_resource * resource,unsigned level,unsigned usage,const struct pipe_box * box,struct pipe_transfer ** out_transfer)1112 panfrost_ptr_map(struct pipe_context *pctx, struct pipe_resource *resource,
1113 unsigned level,
1114 unsigned usage, /* a combination of PIPE_MAP_x */
1115 const struct pipe_box *box,
1116 struct pipe_transfer **out_transfer)
1117 {
1118 struct panfrost_context *ctx = pan_context(pctx);
1119 struct panfrost_device *dev = pan_device(pctx->screen);
1120 struct panfrost_resource *rsrc = pan_resource(resource);
1121 enum pipe_format format = rsrc->image.layout.format;
1122 int bytes_per_block = util_format_get_blocksize(format);
1123 struct panfrost_bo *bo = rsrc->bo;
1124
1125 /* Can't map tiled/compressed directly */
1126 if ((usage & PIPE_MAP_DIRECTLY) &&
1127 rsrc->image.layout.modifier != DRM_FORMAT_MOD_LINEAR)
1128 return NULL;
1129
1130 struct panfrost_transfer *transfer = rzalloc(pctx, struct panfrost_transfer);
1131 transfer->base.level = level;
1132 transfer->base.usage = usage;
1133 transfer->base.box = *box;
1134
1135 pipe_resource_reference(&transfer->base.resource, resource);
1136 *out_transfer = &transfer->base;
1137
1138 if (usage & PIPE_MAP_WRITE)
1139 rsrc->constant_stencil = false;
1140
1141 /* We don't have s/w routines for AFBC, so use a staging texture */
1142 if (drm_is_afbc(rsrc->image.layout.modifier)) {
1143 struct panfrost_resource *staging =
1144 pan_alloc_staging(ctx, rsrc, level, box);
1145 assert(staging);
1146
1147 /* Staging resources have one LOD: level 0. Query the strides
1148 * on this LOD.
1149 */
1150 transfer->base.stride = staging->image.layout.slices[0].row_stride;
1151 transfer->base.layer_stride =
1152 panfrost_get_layer_stride(&staging->image.layout, 0);
1153
1154 transfer->staging.rsrc = &staging->base;
1155
1156 transfer->staging.box = *box;
1157 transfer->staging.box.x = 0;
1158 transfer->staging.box.y = 0;
1159 transfer->staging.box.z = 0;
1160
1161 assert(transfer->staging.rsrc != NULL);
1162
1163 bool valid = BITSET_TEST(rsrc->valid.data, level);
1164
1165 if ((usage & PIPE_MAP_READ) &&
1166 (valid || panfrost_any_batch_writes_rsrc(ctx, rsrc))) {
1167 pan_blit_to_staging(pctx, transfer);
1168 panfrost_flush_writer(ctx, staging, "AFBC read staging blit");
1169 panfrost_bo_wait(staging->bo, INT64_MAX, false);
1170 }
1171
1172 panfrost_bo_mmap(staging->bo);
1173 return staging->bo->ptr.cpu;
1174 }
1175
1176 /* If we haven't already mmaped, now's the time */
1177 panfrost_bo_mmap(bo);
1178
1179 if (dev->debug & (PAN_DBG_TRACE | PAN_DBG_SYNC)) {
1180 pandecode_inject_mmap(dev->decode_ctx, bo->ptr.gpu, bo->ptr.cpu,
1181 panfrost_bo_size(bo), NULL);
1182 }
1183
1184 /* Upgrade writes to uninitialized ranges to UNSYNCHRONIZED */
1185 if ((usage & PIPE_MAP_WRITE) && resource->target == PIPE_BUFFER &&
1186 !util_ranges_intersect(&rsrc->valid_buffer_range, box->x,
1187 box->x + box->width)) {
1188
1189 usage |= PIPE_MAP_UNSYNCHRONIZED;
1190 }
1191
1192 /* Upgrade DISCARD_RANGE to WHOLE_RESOURCE if the whole resource is
1193 * being mapped.
1194 */
1195 if (panfrost_can_discard(resource, box, usage)) {
1196 usage |= PIPE_MAP_DISCARD_WHOLE_RESOURCE;
1197 }
1198
1199 bool create_new_bo = usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE;
1200 bool copy_resource = false;
1201
1202 if (!create_new_bo && !(usage & PIPE_MAP_UNSYNCHRONIZED) &&
1203 !(resource->flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) &&
1204 (usage & PIPE_MAP_WRITE) && panfrost_any_batch_reads_rsrc(ctx, rsrc)) {
1205 /* When a resource to be modified is already being used by a
1206 * pending batch, it is often faster to copy the whole BO than
1207 * to flush and split the frame in two.
1208 */
1209
1210 panfrost_flush_writer(ctx, rsrc, "Shadow resource creation");
1211 panfrost_bo_wait(bo, INT64_MAX, false);
1212
1213 create_new_bo = true;
1214 copy_resource = !(usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE);
1215 }
1216
1217 /* Shadowing with separate stencil may require additional accounting.
1218 * Bail in these exotic cases.
1219 */
1220 if (rsrc->separate_stencil) {
1221 create_new_bo = false;
1222 copy_resource = false;
1223 }
1224
1225 if (create_new_bo) {
1226 /* Make sure we re-emit any descriptors using this resource */
1227 panfrost_dirty_state_all(ctx);
1228
1229 /* If the BO is used by one of the pending batches or if it's
1230 * not ready yet (still accessed by one of the already flushed
1231 * batches), we try to allocate a new one to avoid waiting.
1232 */
1233 if (panfrost_any_batch_reads_rsrc(ctx, rsrc) ||
1234 !panfrost_bo_wait(bo, 0, true)) {
1235 /* We want the BO to be MMAPed. */
1236 uint32_t flags = bo->flags & ~PAN_BO_DELAY_MMAP;
1237 struct panfrost_bo *newbo = NULL;
1238
1239 /* When the BO has been imported/exported, we can't
1240 * replace it by another one, otherwise the
1241 * importer/exporter wouldn't see the change we're
1242 * doing to it.
1243 */
1244 if (!(bo->flags & PAN_BO_SHARED)) {
1245 newbo =
1246 panfrost_bo_create(dev, panfrost_bo_size(bo), flags, bo->label);
1247 }
1248
1249 if (newbo) {
1250 if (copy_resource) {
1251 memcpy(newbo->ptr.cpu, rsrc->bo->ptr.cpu, panfrost_bo_size(bo));
1252 }
1253
1254 /* Swap the pointers, dropping a reference to
1255 * the old BO which is no long referenced from
1256 * the resource.
1257 */
1258 panfrost_bo_unreference(rsrc->bo);
1259 rsrc->bo = newbo;
1260 rsrc->image.data.base = newbo->ptr.gpu;
1261
1262 if (!copy_resource && drm_is_afbc(rsrc->image.layout.modifier))
1263 panfrost_resource_init_afbc_headers(rsrc);
1264
1265 bo = newbo;
1266 } else {
1267 /* Allocation failed or was impossible, let's
1268 * fall back on a flush+wait.
1269 */
1270 panfrost_flush_batches_accessing_rsrc(
1271 ctx, rsrc, "Resource access with high memory pressure");
1272 panfrost_bo_wait(bo, INT64_MAX, true);
1273 }
1274 }
1275 } else if (!(usage & PIPE_MAP_UNSYNCHRONIZED)) {
1276 if (usage & PIPE_MAP_WRITE) {
1277 panfrost_flush_batches_accessing_rsrc(ctx, rsrc, "Synchronized write");
1278 panfrost_bo_wait(bo, INT64_MAX, true);
1279 } else if (usage & PIPE_MAP_READ) {
1280 panfrost_flush_writer(ctx, rsrc, "Synchronized read");
1281 panfrost_bo_wait(bo, INT64_MAX, false);
1282 }
1283 }
1284
1285 /* For access to compressed textures, we want the (x, y, w, h)
1286 * region-of-interest in blocks, not pixels. Then we compute the stride
1287 * between rows of blocks as the width in blocks times the width per
1288 * block, etc.
1289 */
1290 struct pipe_box box_blocks;
1291 u_box_pixels_to_blocks(&box_blocks, box, format);
1292
1293 if (rsrc->image.layout.modifier ==
1294 DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) {
1295 transfer->base.stride = box_blocks.width * bytes_per_block;
1296 transfer->base.layer_stride = transfer->base.stride * box_blocks.height;
1297 transfer->map =
1298 ralloc_size(transfer, transfer->base.layer_stride * box->depth);
1299
1300 if (usage & PIPE_MAP_READ)
1301 panfrost_load_tiled_images(transfer, rsrc);
1302
1303 return transfer->map;
1304 } else {
1305 assert(rsrc->image.layout.modifier == DRM_FORMAT_MOD_LINEAR);
1306
1307 /* Direct, persistent writes create holes in time for
1308 * caching... I don't know if this is actually possible but we
1309 * should still get it right */
1310
1311 unsigned dpw = PIPE_MAP_DIRECTLY | PIPE_MAP_WRITE | PIPE_MAP_PERSISTENT;
1312
1313 if ((usage & dpw) == dpw && rsrc->index_cache)
1314 return NULL;
1315
1316 transfer->base.stride = rsrc->image.layout.slices[level].row_stride;
1317 transfer->base.layer_stride =
1318 panfrost_get_layer_stride(&rsrc->image.layout, level);
1319
1320 /* By mapping direct-write, we're implicitly already
1321 * initialized (maybe), so be conservative */
1322
1323 if (usage & PIPE_MAP_WRITE) {
1324 BITSET_SET(rsrc->valid.data, level);
1325 panfrost_minmax_cache_invalidate(rsrc->index_cache, &transfer->base);
1326 }
1327
1328 return bo->ptr.cpu + rsrc->image.layout.slices[level].offset +
1329 box->z * transfer->base.layer_stride +
1330 box_blocks.y * rsrc->image.layout.slices[level].row_stride +
1331 box_blocks.x * bytes_per_block;
1332 }
1333 }
1334
1335 void
pan_resource_modifier_convert(struct panfrost_context * ctx,struct panfrost_resource * rsrc,uint64_t modifier,bool copy_resource,const char * reason)1336 pan_resource_modifier_convert(struct panfrost_context *ctx,
1337 struct panfrost_resource *rsrc, uint64_t modifier,
1338 bool copy_resource, const char *reason)
1339 {
1340 assert(!rsrc->modifier_constant);
1341
1342 struct pipe_resource *tmp_prsrc = panfrost_resource_create_with_modifier(
1343 ctx->base.screen, &rsrc->base, modifier);
1344 struct panfrost_resource *tmp_rsrc = pan_resource(tmp_prsrc);
1345
1346 if (copy_resource) {
1347 struct pipe_blit_info blit = {
1348 .dst.resource = &tmp_rsrc->base,
1349 .dst.format = tmp_rsrc->base.format,
1350 .src.resource = &rsrc->base,
1351 .src.format = rsrc->base.format,
1352 .mask = util_format_get_mask(tmp_rsrc->base.format),
1353 .filter = PIPE_TEX_FILTER_NEAREST,
1354 };
1355
1356 /* data_valid is not valid until flushed */
1357 panfrost_flush_writer(ctx, rsrc, "AFBC decompressing blit");
1358
1359 for (int i = 0; i <= rsrc->base.last_level; i++) {
1360 if (BITSET_TEST(rsrc->valid.data, i)) {
1361 blit.dst.level = blit.src.level = i;
1362
1363 u_box_3d(0, 0, 0, u_minify(rsrc->base.width0, i),
1364 u_minify(rsrc->base.height0, i),
1365 util_num_layers(&rsrc->base, i), &blit.dst.box);
1366 blit.src.box = blit.dst.box;
1367
1368 panfrost_blit_no_afbc_legalization(&ctx->base, &blit);
1369 }
1370 }
1371
1372 /* we lose track of tmp_rsrc after this point, and the BO migration
1373 * (from tmp_rsrc to rsrc) doesn't transfer the last_writer to rsrc
1374 */
1375 panfrost_flush_writer(ctx, tmp_rsrc, "AFBC decompressing blit");
1376 }
1377
1378 panfrost_bo_unreference(rsrc->bo);
1379
1380 rsrc->bo = tmp_rsrc->bo;
1381 rsrc->image.data.base = rsrc->bo->ptr.gpu;
1382 panfrost_bo_reference(rsrc->bo);
1383
1384 panfrost_resource_setup(pan_device(ctx->base.screen), rsrc, modifier,
1385 tmp_rsrc->base.format);
1386 /* panfrost_resource_setup will force the modifier to stay constant when
1387 * called with a specific modifier. We don't want that here, we want to
1388 * be able to convert back to another modifier if needed */
1389 rsrc->modifier_constant = false;
1390 pipe_resource_reference(&tmp_prsrc, NULL);
1391 }
1392
1393 /* Validate that an AFBC resource may be used as a particular format. If it may
1394 * not, decompress it on the fly. Failure to do so can produce wrong results or
1395 * invalid data faults when sampling or rendering to AFBC */
1396
1397 void
pan_legalize_afbc_format(struct panfrost_context * ctx,struct panfrost_resource * rsrc,enum pipe_format format,bool write,bool discard)1398 pan_legalize_afbc_format(struct panfrost_context *ctx,
1399 struct panfrost_resource *rsrc,
1400 enum pipe_format format, bool write, bool discard)
1401 {
1402 struct panfrost_device *dev = pan_device(ctx->base.screen);
1403
1404 if (!drm_is_afbc(rsrc->image.layout.modifier))
1405 return;
1406
1407 if (panfrost_afbc_format(dev->arch, rsrc->base.format) !=
1408 panfrost_afbc_format(dev->arch, format)) {
1409 pan_resource_modifier_convert(
1410 ctx, rsrc, DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED, !discard,
1411 "Reinterpreting AFBC surface as incompatible format");
1412 return;
1413 }
1414
1415 if (write && (rsrc->image.layout.modifier & AFBC_FORMAT_MOD_SPARSE) == 0)
1416 pan_resource_modifier_convert(
1417 ctx, rsrc, rsrc->image.layout.modifier | AFBC_FORMAT_MOD_SPARSE,
1418 !discard, "Legalizing resource to allow writing");
1419 }
1420
1421 static bool
panfrost_should_linear_convert(struct panfrost_device * dev,struct panfrost_resource * prsrc,struct pipe_transfer * transfer)1422 panfrost_should_linear_convert(struct panfrost_device *dev,
1423 struct panfrost_resource *prsrc,
1424 struct pipe_transfer *transfer)
1425 {
1426 if (prsrc->modifier_constant)
1427 return false;
1428
1429 /* Overwriting the entire resource indicates streaming, for which
1430 * linear layout is most efficient due to the lack of expensive
1431 * conversion.
1432 *
1433 * For now we just switch to linear after a number of complete
1434 * overwrites to keep things simple, but we could do better.
1435 *
1436 * This mechanism is only implemented for 2D resources. This suffices
1437 * for video players, its intended use case.
1438 */
1439
1440 bool entire_overwrite = panfrost_is_2d(prsrc) &&
1441 prsrc->base.last_level == 0 &&
1442 transfer->box.width == prsrc->base.width0 &&
1443 transfer->box.height == prsrc->base.height0 &&
1444 transfer->box.x == 0 && transfer->box.y == 0;
1445
1446 if (entire_overwrite)
1447 ++prsrc->modifier_updates;
1448
1449 if (prsrc->modifier_updates >= LAYOUT_CONVERT_THRESHOLD) {
1450 perf_debug(dev, "Transitioning to linear due to streaming usage");
1451 return true;
1452 } else {
1453 return false;
1454 }
1455 }
1456
1457 struct panfrost_bo *
panfrost_get_afbc_superblock_sizes(struct panfrost_context * ctx,struct panfrost_resource * rsrc,unsigned first_level,unsigned last_level,unsigned * out_offsets)1458 panfrost_get_afbc_superblock_sizes(struct panfrost_context *ctx,
1459 struct panfrost_resource *rsrc,
1460 unsigned first_level, unsigned last_level,
1461 unsigned *out_offsets)
1462 {
1463 struct panfrost_screen *screen = pan_screen(ctx->base.screen);
1464 struct panfrost_device *dev = pan_device(ctx->base.screen);
1465 struct panfrost_batch *batch;
1466 struct panfrost_bo *bo;
1467 unsigned metadata_size = 0;
1468
1469 for (int level = first_level; level <= last_level; ++level) {
1470 struct pan_image_slice_layout *slice = &rsrc->image.layout.slices[level];
1471 unsigned sz = slice->afbc.nr_blocks * sizeof(struct pan_afbc_block_info);
1472 out_offsets[level - first_level] = metadata_size;
1473 metadata_size += sz;
1474 }
1475
1476 panfrost_flush_batches_accessing_rsrc(ctx, rsrc, "AFBC before size flush");
1477 batch = panfrost_get_fresh_batch_for_fbo(ctx, "AFBC superblock sizes");
1478 bo = panfrost_bo_create(dev, metadata_size, 0, "AFBC superblock sizes");
1479
1480 for (int level = first_level; level <= last_level; ++level) {
1481 unsigned offset = out_offsets[level - first_level];
1482 screen->vtbl.afbc_size(batch, rsrc, bo, offset, level);
1483 }
1484
1485 panfrost_flush_batches_accessing_rsrc(ctx, rsrc, "AFBC after size flush");
1486
1487 return bo;
1488 }
1489
1490 void
panfrost_pack_afbc(struct panfrost_context * ctx,struct panfrost_resource * prsrc)1491 panfrost_pack_afbc(struct panfrost_context *ctx,
1492 struct panfrost_resource *prsrc)
1493 {
1494 struct panfrost_screen *screen = pan_screen(ctx->base.screen);
1495 struct panfrost_device *dev = pan_device(ctx->base.screen);
1496 struct panfrost_bo *metadata_bo;
1497 unsigned metadata_offsets[PIPE_MAX_TEXTURE_LEVELS];
1498
1499 uint64_t src_modifier = prsrc->image.layout.modifier;
1500 uint64_t dst_modifier =
1501 src_modifier & ~(AFBC_FORMAT_MOD_TILED | AFBC_FORMAT_MOD_SPARSE);
1502 bool is_tiled = src_modifier & AFBC_FORMAT_MOD_TILED;
1503 unsigned last_level = prsrc->base.last_level;
1504 struct pan_image_slice_layout slice_infos[PIPE_MAX_TEXTURE_LEVELS] = {0};
1505 unsigned total_size = 0;
1506
1507 /* It doesn't make sense to pack everything if we need to unpack right
1508 * away to upload data to another level */
1509 for (int i = 0; i <= last_level; i++) {
1510 if (!BITSET_TEST(prsrc->valid.data, i))
1511 return;
1512 }
1513
1514 metadata_bo = panfrost_get_afbc_superblock_sizes(ctx, prsrc, 0, last_level,
1515 metadata_offsets);
1516 panfrost_bo_wait(metadata_bo, INT64_MAX, false);
1517
1518 for (unsigned level = 0; level <= last_level; ++level) {
1519 struct pan_image_slice_layout *src_slice =
1520 &prsrc->image.layout.slices[level];
1521 struct pan_image_slice_layout *dst_slice = &slice_infos[level];
1522
1523 unsigned width = u_minify(prsrc->base.width0, level);
1524 unsigned height = u_minify(prsrc->base.height0, level);
1525 unsigned src_stride =
1526 pan_afbc_stride_blocks(src_modifier, src_slice->row_stride);
1527 unsigned dst_stride =
1528 DIV_ROUND_UP(width, panfrost_afbc_superblock_width(dst_modifier));
1529 unsigned dst_height =
1530 DIV_ROUND_UP(height, panfrost_afbc_superblock_height(dst_modifier));
1531
1532 uint32_t offset = 0;
1533 struct pan_afbc_block_info *meta =
1534 metadata_bo->ptr.cpu + metadata_offsets[level];
1535
1536 for (unsigned y = 0, i = 0; y < dst_height; ++y) {
1537 for (unsigned x = 0; x < dst_stride; ++x, ++i) {
1538 unsigned idx = is_tiled ? get_morton_index(x, y, src_stride) : i;
1539 uint32_t size = meta[idx].size;
1540 meta[idx].offset = offset; /* write the start offset */
1541 offset += size;
1542 }
1543 }
1544
1545 total_size = ALIGN_POT(total_size, pan_slice_align(dst_modifier));
1546 {
1547 dst_slice->afbc.stride = dst_stride;
1548 dst_slice->afbc.nr_blocks = dst_stride * dst_height;
1549 dst_slice->afbc.header_size =
1550 ALIGN_POT(dst_stride * dst_height * AFBC_HEADER_BYTES_PER_TILE,
1551 pan_afbc_body_align(dst_modifier));
1552 dst_slice->afbc.body_size = offset;
1553 dst_slice->afbc.surface_stride = dst_slice->afbc.header_size + offset;
1554
1555 dst_slice->offset = total_size;
1556 dst_slice->row_stride = dst_stride * AFBC_HEADER_BYTES_PER_TILE;
1557 dst_slice->surface_stride = dst_slice->afbc.surface_stride;
1558 dst_slice->size = dst_slice->afbc.surface_stride;
1559 }
1560 total_size += dst_slice->afbc.surface_stride;
1561 }
1562
1563 unsigned new_size = ALIGN_POT(total_size, 4096); // FIXME
1564 unsigned old_size = panfrost_bo_size(prsrc->bo);
1565 unsigned ratio = 100 * new_size / old_size;
1566
1567 if (ratio > screen->max_afbc_packing_ratio)
1568 return;
1569
1570 perf_debug(dev, "%i%%: %i KB -> %i KB\n", ratio, old_size / 1024,
1571 new_size / 1024);
1572
1573 struct panfrost_bo *dst =
1574 panfrost_bo_create(dev, new_size, 0, "AFBC compact texture");
1575 struct panfrost_batch *batch =
1576 panfrost_get_fresh_batch_for_fbo(ctx, "AFBC compaction");
1577
1578 for (unsigned level = 0; level <= last_level; ++level) {
1579 struct pan_image_slice_layout *slice = &slice_infos[level];
1580 screen->vtbl.afbc_pack(batch, prsrc, dst, slice, metadata_bo,
1581 metadata_offsets[level], level);
1582 prsrc->image.layout.slices[level] = *slice;
1583 }
1584
1585 panfrost_flush_batches_accessing_rsrc(ctx, prsrc, "AFBC compaction flush");
1586
1587 prsrc->image.layout.modifier = dst_modifier;
1588 panfrost_bo_unreference(prsrc->bo);
1589 prsrc->bo = dst;
1590 prsrc->image.data.base = dst->ptr.gpu;
1591 panfrost_bo_unreference(metadata_bo);
1592 }
1593
1594 static void
panfrost_ptr_unmap(struct pipe_context * pctx,struct pipe_transfer * transfer)1595 panfrost_ptr_unmap(struct pipe_context *pctx, struct pipe_transfer *transfer)
1596 {
1597 /* Gallium expects writeback here, so we tile */
1598
1599 struct panfrost_context *ctx = pan_context(pctx);
1600 struct panfrost_transfer *trans = pan_transfer(transfer);
1601 struct panfrost_resource *prsrc =
1602 (struct panfrost_resource *)transfer->resource;
1603 struct panfrost_device *dev = pan_device(pctx->screen);
1604
1605 if (transfer->usage & PIPE_MAP_WRITE)
1606 prsrc->valid.crc = false;
1607
1608 /* AFBC will use a staging resource. `initialized` will be set when the
1609 * fragment job is created; this is deferred to prevent useless surface
1610 * reloads that can cascade into DATA_INVALID_FAULTs due to reading
1611 * malformed AFBC data if uninitialized */
1612
1613 if (trans->staging.rsrc) {
1614 if (transfer->usage & PIPE_MAP_WRITE) {
1615 if (panfrost_should_linear_convert(dev, prsrc, transfer)) {
1616
1617 panfrost_bo_unreference(prsrc->bo);
1618
1619 panfrost_resource_setup(dev, prsrc, DRM_FORMAT_MOD_LINEAR,
1620 prsrc->image.layout.format);
1621
1622 prsrc->bo = pan_resource(trans->staging.rsrc)->bo;
1623 prsrc->image.data.base = prsrc->bo->ptr.gpu;
1624 panfrost_bo_reference(prsrc->bo);
1625 } else {
1626 bool discard = panfrost_can_discard(&prsrc->base, &transfer->box,
1627 transfer->usage);
1628 pan_legalize_afbc_format(ctx, prsrc, prsrc->image.layout.format,
1629 true, discard);
1630 pan_blit_from_staging(pctx, trans);
1631 panfrost_flush_batches_accessing_rsrc(
1632 ctx, pan_resource(trans->staging.rsrc),
1633 "AFBC write staging blit");
1634
1635 if (dev->debug & PAN_DBG_FORCE_PACK) {
1636 if (panfrost_should_pack_afbc(dev, prsrc))
1637 panfrost_pack_afbc(ctx, prsrc);
1638 }
1639 }
1640 }
1641
1642 pipe_resource_reference(&trans->staging.rsrc, NULL);
1643 }
1644
1645 /* Tiling will occur in software from a staging cpu buffer */
1646 if (trans->map) {
1647 struct panfrost_bo *bo = prsrc->bo;
1648
1649 if (transfer->usage & PIPE_MAP_WRITE) {
1650 BITSET_SET(prsrc->valid.data, transfer->level);
1651
1652 if (prsrc->image.layout.modifier ==
1653 DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) {
1654 if (panfrost_should_linear_convert(dev, prsrc, transfer)) {
1655 panfrost_resource_setup(dev, prsrc, DRM_FORMAT_MOD_LINEAR,
1656 prsrc->image.layout.format);
1657 if (prsrc->image.layout.data_size > panfrost_bo_size(bo)) {
1658 const char *label = bo->label;
1659 panfrost_bo_unreference(bo);
1660 bo = prsrc->bo = panfrost_bo_create(
1661 dev, prsrc->image.layout.data_size, 0, label);
1662 prsrc->image.data.base = prsrc->bo->ptr.gpu;
1663 assert(bo);
1664 }
1665
1666 util_copy_rect(
1667 bo->ptr.cpu + prsrc->image.layout.slices[0].offset,
1668 prsrc->base.format, prsrc->image.layout.slices[0].row_stride,
1669 0, 0, transfer->box.width, transfer->box.height, trans->map,
1670 transfer->stride, 0, 0);
1671 } else {
1672 panfrost_store_tiled_images(trans, prsrc);
1673 }
1674 }
1675 }
1676 }
1677
1678 util_range_add(&prsrc->base, &prsrc->valid_buffer_range, transfer->box.x,
1679 transfer->box.x + transfer->box.width);
1680
1681 panfrost_minmax_cache_invalidate(prsrc->index_cache, transfer);
1682
1683 /* Derefence the resource */
1684 pipe_resource_reference(&transfer->resource, NULL);
1685
1686 /* Transfer itself is RALLOCed at the moment */
1687 ralloc_free(transfer);
1688 }
1689
1690 static void
panfrost_ptr_flush_region(struct pipe_context * pctx,struct pipe_transfer * transfer,const struct pipe_box * box)1691 panfrost_ptr_flush_region(struct pipe_context *pctx,
1692 struct pipe_transfer *transfer,
1693 const struct pipe_box *box)
1694 {
1695 struct panfrost_resource *rsc = pan_resource(transfer->resource);
1696
1697 if (transfer->resource->target == PIPE_BUFFER) {
1698 util_range_add(&rsc->base, &rsc->valid_buffer_range,
1699 transfer->box.x + box->x,
1700 transfer->box.x + box->x + box->width);
1701 } else {
1702 BITSET_SET(rsc->valid.data, transfer->level);
1703 }
1704 }
1705
1706 static void
panfrost_invalidate_resource(struct pipe_context * pctx,struct pipe_resource * prsrc)1707 panfrost_invalidate_resource(struct pipe_context *pctx,
1708 struct pipe_resource *prsrc)
1709 {
1710 struct panfrost_context *ctx = pan_context(pctx);
1711 struct panfrost_batch *batch = panfrost_get_batch_for_fbo(ctx);
1712 struct panfrost_resource *rsrc = pan_resource(prsrc);
1713
1714 rsrc->constant_stencil = true;
1715
1716 /* Handle the glInvalidateFramebuffer case */
1717 if (batch->key.zsbuf && batch->key.zsbuf->texture == prsrc)
1718 batch->resolve &= ~PIPE_CLEAR_DEPTHSTENCIL;
1719
1720 for (unsigned i = 0; i < batch->key.nr_cbufs; ++i) {
1721 struct pipe_surface *surf = batch->key.cbufs[i];
1722
1723 if (surf && surf->texture == prsrc)
1724 batch->resolve &= ~(PIPE_CLEAR_COLOR0 << i);
1725 }
1726 }
1727
1728 static enum pipe_format
panfrost_resource_get_internal_format(struct pipe_resource * rsrc)1729 panfrost_resource_get_internal_format(struct pipe_resource *rsrc)
1730 {
1731 struct panfrost_resource *prsrc = (struct panfrost_resource *)rsrc;
1732 return prsrc->image.layout.format;
1733 }
1734
1735 void
panfrost_set_image_view_planes(struct pan_image_view * iview,struct pipe_resource * texture)1736 panfrost_set_image_view_planes(struct pan_image_view *iview,
1737 struct pipe_resource *texture)
1738 {
1739 struct panfrost_resource *prsrc_plane = (struct panfrost_resource *)texture;
1740
1741 for (int i = 0; i < MAX_IMAGE_PLANES && prsrc_plane; i++) {
1742 iview->planes[i] = &prsrc_plane->image;
1743 prsrc_plane = (struct panfrost_resource *)prsrc_plane->base.next;
1744 }
1745 }
1746
1747 static bool
panfrost_generate_mipmap(struct pipe_context * pctx,struct pipe_resource * prsrc,enum pipe_format format,unsigned base_level,unsigned last_level,unsigned first_layer,unsigned last_layer)1748 panfrost_generate_mipmap(struct pipe_context *pctx, struct pipe_resource *prsrc,
1749 enum pipe_format format, unsigned base_level,
1750 unsigned last_level, unsigned first_layer,
1751 unsigned last_layer)
1752 {
1753 struct panfrost_resource *rsrc = pan_resource(prsrc);
1754
1755 perf_debug_ctx(pan_context(pctx), "Unoptimized mipmap generation");
1756
1757 /* Generating a mipmap invalidates the written levels, so make that
1758 * explicit so we don't try to wallpaper them back and end up with
1759 * u_blitter recursion */
1760
1761 assert(rsrc->bo);
1762 for (unsigned l = base_level + 1; l <= last_level; ++l)
1763 BITSET_CLEAR(rsrc->valid.data, l);
1764
1765 /* Beyond that, we just delegate the hard stuff. */
1766
1767 bool blit_res =
1768 util_gen_mipmap(pctx, prsrc, format, base_level, last_level, first_layer,
1769 last_layer, PIPE_TEX_FILTER_LINEAR);
1770
1771 return blit_res;
1772 }
1773
1774 static void
panfrost_resource_set_stencil(struct pipe_resource * prsrc,struct pipe_resource * stencil)1775 panfrost_resource_set_stencil(struct pipe_resource *prsrc,
1776 struct pipe_resource *stencil)
1777 {
1778 pan_resource(prsrc)->separate_stencil = pan_resource(stencil);
1779 }
1780
1781 static struct pipe_resource *
panfrost_resource_get_stencil(struct pipe_resource * prsrc)1782 panfrost_resource_get_stencil(struct pipe_resource *prsrc)
1783 {
1784 if (!pan_resource(prsrc)->separate_stencil)
1785 return NULL;
1786
1787 return &pan_resource(prsrc)->separate_stencil->base;
1788 }
1789
1790 static const struct u_transfer_vtbl transfer_vtbl = {
1791 .resource_create = panfrost_resource_create,
1792 .resource_destroy = panfrost_resource_destroy,
1793 .transfer_map = panfrost_ptr_map,
1794 .transfer_unmap = panfrost_ptr_unmap,
1795 .transfer_flush_region = panfrost_ptr_flush_region,
1796 .get_internal_format = panfrost_resource_get_internal_format,
1797 .set_stencil = panfrost_resource_set_stencil,
1798 .get_stencil = panfrost_resource_get_stencil,
1799 };
1800
1801 void
panfrost_resource_screen_init(struct pipe_screen * pscreen)1802 panfrost_resource_screen_init(struct pipe_screen *pscreen)
1803 {
1804 pscreen->resource_create_with_modifiers =
1805 panfrost_resource_create_with_modifiers;
1806 pscreen->resource_create = u_transfer_helper_resource_create;
1807 pscreen->resource_destroy = u_transfer_helper_resource_destroy;
1808 pscreen->resource_from_handle = panfrost_resource_from_handle;
1809 pscreen->resource_get_handle = panfrost_resource_get_handle;
1810 pscreen->resource_get_param = panfrost_resource_get_param;
1811 pscreen->transfer_helper = u_transfer_helper_create(
1812 &transfer_vtbl,
1813 U_TRANSFER_HELPER_SEPARATE_Z32S8 | U_TRANSFER_HELPER_MSAA_MAP);
1814 }
1815 void
panfrost_resource_screen_destroy(struct pipe_screen * pscreen)1816 panfrost_resource_screen_destroy(struct pipe_screen *pscreen)
1817 {
1818 u_transfer_helper_destroy(pscreen->transfer_helper);
1819 }
1820
1821 void
panfrost_resource_context_init(struct pipe_context * pctx)1822 panfrost_resource_context_init(struct pipe_context *pctx)
1823 {
1824 pctx->buffer_map = u_transfer_helper_transfer_map;
1825 pctx->buffer_unmap = u_transfer_helper_transfer_unmap;
1826 pctx->texture_map = u_transfer_helper_transfer_map;
1827 pctx->texture_unmap = u_transfer_helper_transfer_unmap;
1828 pctx->create_surface = panfrost_create_surface;
1829 pctx->surface_destroy = panfrost_surface_destroy;
1830 pctx->resource_copy_region = util_resource_copy_region;
1831 pctx->blit = panfrost_blit;
1832 pctx->generate_mipmap = panfrost_generate_mipmap;
1833 pctx->flush_resource = panfrost_flush_resource;
1834 pctx->invalidate_resource = panfrost_invalidate_resource;
1835 pctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;
1836 pctx->buffer_subdata = u_default_buffer_subdata;
1837 pctx->texture_subdata = u_default_texture_subdata;
1838 pctx->clear_buffer = u_default_clear_buffer;
1839 pctx->clear_render_target = panfrost_clear_render_target;
1840 pctx->clear_depth_stencil = panfrost_clear_depth_stencil;
1841 }
1842