1 /*
2 * Copyright (C) 2008 VMware, Inc.
3 * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org>
4 * Copyright (C) 2014-2017 Broadcom
5 * Copyright (C) 2018-2019 Alyssa Rosenzweig
6 * Copyright (C) 2019 Collabora, Ltd.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the "Software"),
10 * to deal in the Software without restriction, including without limitation
11 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 * and/or sell copies of the Software, and to permit persons to whom the
13 * Software is furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the next
16 * paragraph) shall be included in all copies or substantial portions of the
17 * Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25 * SOFTWARE.
26 *
27 * Authors (Collabora):
28 * Tomeu Vizoso <tomeu.vizoso@collabora.com>
29 * Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
30 *
31 */
32
33 #include <xf86drm.h>
34 #include <fcntl.h>
35 #include "drm-uapi/drm_fourcc.h"
36
37 #include "frontend/winsys_handle.h"
38 #include "util/format/u_format.h"
39 #include "util/u_memory.h"
40 #include "util/u_surface.h"
41 #include "util/u_transfer.h"
42 #include "util/u_transfer_helper.h"
43 #include "util/u_gen_mipmap.h"
44 #include "util/u_drm.h"
45
46 #include "pan_bo.h"
47 #include "pan_context.h"
48 #include "pan_screen.h"
49 #include "pan_resource.h"
50 #include "pan_util.h"
51 #include "pan_tiling.h"
52 #include "decode.h"
53 #include "panfrost-quirks.h"
54
55 static bool
56 panfrost_should_checksum(const struct panfrost_device *dev, const struct panfrost_resource *pres);
57
58 static struct pipe_resource *
panfrost_resource_from_handle(struct pipe_screen * pscreen,const struct pipe_resource * templat,struct winsys_handle * whandle,unsigned usage)59 panfrost_resource_from_handle(struct pipe_screen *pscreen,
60 const struct pipe_resource *templat,
61 struct winsys_handle *whandle,
62 unsigned usage)
63 {
64 struct panfrost_device *dev = pan_device(pscreen);
65 struct panfrost_resource *rsc;
66 struct pipe_resource *prsc;
67
68 assert(whandle->type == WINSYS_HANDLE_TYPE_FD);
69
70 rsc = CALLOC_STRUCT(panfrost_resource);
71 if (!rsc)
72 return NULL;
73
74 prsc = &rsc->base;
75
76 *prsc = *templat;
77
78 pipe_reference_init(&prsc->reference, 1);
79 prsc->screen = pscreen;
80
81 uint64_t mod = whandle->modifier == DRM_FORMAT_MOD_INVALID ?
82 DRM_FORMAT_MOD_LINEAR : whandle->modifier;
83 enum mali_texture_dimension dim =
84 panfrost_translate_texture_dimension(templat->target);
85 enum pan_image_crc_mode crc_mode =
86 panfrost_should_checksum(dev, rsc) ?
87 PAN_IMAGE_CRC_OOB : PAN_IMAGE_CRC_NONE;
88 struct pan_image_explicit_layout explicit_layout = {
89 .offset = whandle->offset,
90 .line_stride = whandle->stride,
91 };
92
93 bool valid = pan_image_layout_init(dev, &rsc->image.layout, mod,
94 templat->format, dim,
95 prsc->width0, prsc->height0,
96 prsc->depth0, prsc->array_size,
97 MAX2(prsc->nr_samples, 1), 1,
98 crc_mode, &explicit_layout);
99
100 if (!valid) {
101 FREE(rsc);
102 return NULL;
103 }
104
105 rsc->image.data.bo = panfrost_bo_import(dev, whandle->handle);
106 /* Sometimes an import can fail e.g. on an invalid buffer fd, out of
107 * memory space to mmap it etc.
108 */
109 if (!rsc->image.data.bo) {
110 FREE(rsc);
111 return NULL;
112 }
113 if (rsc->image.layout.crc_mode == PAN_IMAGE_CRC_OOB)
114 rsc->image.crc.bo = panfrost_bo_create(dev, rsc->image.layout.crc_size, 0, "CRC data");
115
116 rsc->modifier_constant = true;
117
118 BITSET_SET(rsc->valid.data, 0);
119 panfrost_resource_set_damage_region(pscreen, &rsc->base, 0, NULL);
120
121 if (dev->ro) {
122 rsc->scanout =
123 renderonly_create_gpu_import_for_resource(prsc, dev->ro, NULL);
124 /* failure is expected in some cases.. */
125 }
126
127 return prsc;
128 }
129
130 static bool
panfrost_resource_get_handle(struct pipe_screen * pscreen,struct pipe_context * ctx,struct pipe_resource * pt,struct winsys_handle * handle,unsigned usage)131 panfrost_resource_get_handle(struct pipe_screen *pscreen,
132 struct pipe_context *ctx,
133 struct pipe_resource *pt,
134 struct winsys_handle *handle,
135 unsigned usage)
136 {
137 struct panfrost_device *dev = pan_device(pscreen);
138 struct panfrost_resource *rsrc = (struct panfrost_resource *) pt;
139 struct renderonly_scanout *scanout = rsrc->scanout;
140
141 handle->modifier = rsrc->image.layout.modifier;
142 rsrc->modifier_constant = true;
143
144 if (handle->type == WINSYS_HANDLE_TYPE_SHARED) {
145 return false;
146 } else if (handle->type == WINSYS_HANDLE_TYPE_KMS) {
147 if (dev->ro) {
148 return renderonly_get_handle(scanout, handle);
149 } else {
150 handle->handle = rsrc->image.data.bo->gem_handle;
151 handle->stride = rsrc->image.layout.slices[0].line_stride;
152 handle->offset = rsrc->image.layout.slices[0].offset;
153 return true;
154 }
155 } else if (handle->type == WINSYS_HANDLE_TYPE_FD) {
156 if (scanout) {
157 struct drm_prime_handle args = {
158 .handle = scanout->handle,
159 .flags = DRM_CLOEXEC,
160 };
161
162 int ret = drmIoctl(dev->ro->kms_fd, DRM_IOCTL_PRIME_HANDLE_TO_FD, &args);
163 if (ret == -1)
164 return false;
165
166 handle->stride = scanout->stride;
167 handle->handle = args.fd;
168
169 return true;
170 } else {
171 int fd = panfrost_bo_export(rsrc->image.data.bo);
172
173 if (fd < 0)
174 return false;
175
176 handle->handle = fd;
177 handle->stride = rsrc->image.layout.slices[0].line_stride;
178 handle->offset = rsrc->image.layout.slices[0].offset;
179 return true;
180 }
181 }
182
183 return false;
184 }
185
186 static bool
panfrost_resource_get_param(struct pipe_screen * pscreen,struct pipe_context * pctx,struct pipe_resource * prsc,unsigned plane,unsigned layer,unsigned level,enum pipe_resource_param param,unsigned usage,uint64_t * value)187 panfrost_resource_get_param(struct pipe_screen *pscreen,
188 struct pipe_context *pctx, struct pipe_resource *prsc,
189 unsigned plane, unsigned layer, unsigned level,
190 enum pipe_resource_param param,
191 unsigned usage, uint64_t *value)
192 {
193 struct panfrost_resource *rsrc = (struct panfrost_resource *) prsc;
194
195 switch (param) {
196 case PIPE_RESOURCE_PARAM_STRIDE:
197 *value = rsrc->image.layout.slices[level].line_stride;
198 return true;
199 case PIPE_RESOURCE_PARAM_OFFSET:
200 *value = rsrc->image.layout.slices[level].offset;
201 return true;
202 case PIPE_RESOURCE_PARAM_MODIFIER:
203 *value = rsrc->image.layout.modifier;
204 return true;
205 default:
206 return false;
207 }
208 }
209
210 static void
panfrost_flush_resource(struct pipe_context * pctx,struct pipe_resource * prsc)211 panfrost_flush_resource(struct pipe_context *pctx, struct pipe_resource *prsc)
212 {
213 /* TODO */
214 }
215
216 static struct pipe_surface *
panfrost_create_surface(struct pipe_context * pipe,struct pipe_resource * pt,const struct pipe_surface * surf_tmpl)217 panfrost_create_surface(struct pipe_context *pipe,
218 struct pipe_resource *pt,
219 const struct pipe_surface *surf_tmpl)
220 {
221 struct panfrost_context *ctx = pan_context(pipe);
222 struct pipe_surface *ps = NULL;
223
224 pan_legalize_afbc_format(ctx, pan_resource(pt), surf_tmpl->format);
225
226 ps = CALLOC_STRUCT(pipe_surface);
227
228 if (ps) {
229 pipe_reference_init(&ps->reference, 1);
230 pipe_resource_reference(&ps->texture, pt);
231 ps->context = pipe;
232 ps->format = surf_tmpl->format;
233
234 if (pt->target != PIPE_BUFFER) {
235 assert(surf_tmpl->u.tex.level <= pt->last_level);
236 ps->width = u_minify(pt->width0, surf_tmpl->u.tex.level);
237 ps->height = u_minify(pt->height0, surf_tmpl->u.tex.level);
238 ps->nr_samples = surf_tmpl->nr_samples;
239 ps->u.tex.level = surf_tmpl->u.tex.level;
240 ps->u.tex.first_layer = surf_tmpl->u.tex.first_layer;
241 ps->u.tex.last_layer = surf_tmpl->u.tex.last_layer;
242 } else {
243 /* setting width as number of elements should get us correct renderbuffer width */
244 ps->width = surf_tmpl->u.buf.last_element - surf_tmpl->u.buf.first_element + 1;
245 ps->height = pt->height0;
246 ps->u.buf.first_element = surf_tmpl->u.buf.first_element;
247 ps->u.buf.last_element = surf_tmpl->u.buf.last_element;
248 assert(ps->u.buf.first_element <= ps->u.buf.last_element);
249 assert(ps->u.buf.last_element < ps->width);
250 }
251 }
252
253 return ps;
254 }
255
256 static void
panfrost_surface_destroy(struct pipe_context * pipe,struct pipe_surface * surf)257 panfrost_surface_destroy(struct pipe_context *pipe,
258 struct pipe_surface *surf)
259 {
260 assert(surf->texture);
261 pipe_resource_reference(&surf->texture, NULL);
262 free(surf);
263 }
264
265 static struct pipe_resource *
panfrost_create_scanout_res(struct pipe_screen * screen,const struct pipe_resource * template,uint64_t modifier)266 panfrost_create_scanout_res(struct pipe_screen *screen,
267 const struct pipe_resource *template,
268 uint64_t modifier)
269 {
270 struct panfrost_device *dev = pan_device(screen);
271 struct renderonly_scanout *scanout;
272 struct winsys_handle handle;
273 struct pipe_resource *res;
274 struct pipe_resource scanout_templat = *template;
275
276 /* Tiled formats need to be tile aligned */
277 if (modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) {
278 scanout_templat.width0 = ALIGN_POT(template->width0, 16);
279 scanout_templat.height0 = ALIGN_POT(template->height0, 16);
280 }
281
282 /* AFBC formats need a header. Thankfully we don't care about the
283 * stride so we can just use wonky dimensions as long as the right
284 * number of bytes are allocated at the end of the day... this implies
285 * that stride/pitch is invalid for AFBC buffers */
286
287 if (drm_is_afbc(modifier)) {
288 /* Space for the header. We need to keep vaguely similar
289 * dimensions because... reasons... to allocate with renderonly
290 * as a dumb buffer. To do so, after the usual 16x16 alignment,
291 * we add on extra rows for the header. The order of operations
292 * matters here, the extra rows of padding can in fact be
293 * needed and missing them can lead to faults. */
294
295 unsigned header_size = panfrost_afbc_header_size(
296 template->width0, template->height0);
297
298 unsigned pitch = ALIGN_POT(template->width0, 16) *
299 util_format_get_blocksize(template->format);
300
301 unsigned header_rows =
302 DIV_ROUND_UP(header_size, pitch);
303
304 scanout_templat.width0 = ALIGN_POT(template->width0, 16);
305 scanout_templat.height0 = ALIGN_POT(template->height0, 16) + header_rows;
306 }
307
308 scanout = renderonly_scanout_for_resource(&scanout_templat,
309 dev->ro, &handle);
310 if (!scanout)
311 return NULL;
312
313 assert(handle.type == WINSYS_HANDLE_TYPE_FD);
314 handle.modifier = modifier;
315 res = screen->resource_from_handle(screen, template, &handle,
316 PIPE_HANDLE_USAGE_FRAMEBUFFER_WRITE);
317 close(handle.handle);
318 if (!res)
319 return NULL;
320
321 struct panfrost_resource *pres = pan_resource(res);
322
323 pres->scanout = scanout;
324
325 return res;
326 }
327
328 static inline bool
panfrost_is_2d(const struct panfrost_resource * pres)329 panfrost_is_2d(const struct panfrost_resource *pres)
330 {
331 return (pres->base.target == PIPE_TEXTURE_2D)
332 || (pres->base.target == PIPE_TEXTURE_RECT);
333 }
334
335 /* Based on the usage, determine if it makes sense to use u-inteleaved tiling.
336 * We only have routines to tile 2D textures of sane bpps. On the hardware
337 * level, not all usages are valid for tiling. Finally, if the app is hinting
338 * that the contents frequently change, tiling will be a loss.
339 *
340 * On platforms where it is supported, AFBC is even better. */
341
342 static bool
panfrost_should_afbc(struct panfrost_device * dev,const struct panfrost_resource * pres,enum pipe_format fmt)343 panfrost_should_afbc(struct panfrost_device *dev,
344 const struct panfrost_resource *pres,
345 enum pipe_format fmt)
346 {
347 /* AFBC resources may be rendered to, textured from, or shared across
348 * processes, but may not be used as e.g buffers */
349 const unsigned valid_binding =
350 PIPE_BIND_DEPTH_STENCIL |
351 PIPE_BIND_RENDER_TARGET |
352 PIPE_BIND_BLENDABLE |
353 PIPE_BIND_SAMPLER_VIEW |
354 PIPE_BIND_DISPLAY_TARGET |
355 PIPE_BIND_SCANOUT |
356 PIPE_BIND_SHARED;
357
358 if (pres->base.bind & ~valid_binding)
359 return false;
360
361 /* AFBC support is optional */
362 if (!dev->has_afbc)
363 return false;
364
365 /* AFBC<-->staging is expensive */
366 if (pres->base.usage == PIPE_USAGE_STREAM)
367 return false;
368
369 /* Only a small selection of formats are AFBC'able */
370 if (!panfrost_format_supports_afbc(dev, fmt))
371 return false;
372
373 /* AFBC does not support layered (GLES3 style) multisampling. Use
374 * EXT_multisampled_render_to_texture instead */
375 if (pres->base.nr_samples > 1)
376 return false;
377
378 switch (pres->base.target) {
379 case PIPE_TEXTURE_2D:
380 case PIPE_TEXTURE_2D_ARRAY:
381 case PIPE_TEXTURE_RECT:
382 break;
383
384 case PIPE_TEXTURE_3D:
385 /* 3D AFBC is only supported on Bifrost v7+. It's supposed to
386 * be supported on Midgard but it doesn't seem to work */
387 if (dev->arch < 7)
388 return false;
389
390 break;
391
392 default:
393 return false;
394 }
395
396 /* For one tile, AFBC is a loss compared to u-interleaved */
397 if (pres->base.width0 <= 16 && pres->base.height0 <= 16)
398 return false;
399
400 /* Otherwise, we'd prefer AFBC as it is dramatically more efficient
401 * than linear or usually even u-interleaved */
402 return true;
403 }
404
405 static bool
panfrost_should_tile(struct panfrost_device * dev,const struct panfrost_resource * pres,enum pipe_format fmt)406 panfrost_should_tile(struct panfrost_device *dev,
407 const struct panfrost_resource *pres,
408 enum pipe_format fmt)
409 {
410 const unsigned valid_binding =
411 PIPE_BIND_DEPTH_STENCIL |
412 PIPE_BIND_RENDER_TARGET |
413 PIPE_BIND_BLENDABLE |
414 PIPE_BIND_SAMPLER_VIEW |
415 PIPE_BIND_DISPLAY_TARGET |
416 PIPE_BIND_SCANOUT |
417 PIPE_BIND_SHARED;
418
419 unsigned bpp = util_format_get_blocksizebits(fmt);
420
421 bool is_sane_bpp =
422 bpp == 8 || bpp == 16 || bpp == 24 || bpp == 32 ||
423 bpp == 64 || bpp == 128;
424
425 bool can_tile = panfrost_is_2d(pres)
426 && is_sane_bpp
427 && ((pres->base.bind & ~valid_binding) == 0);
428
429 return can_tile && (pres->base.usage != PIPE_USAGE_STREAM);
430 }
431
432 static uint64_t
panfrost_best_modifier(struct panfrost_device * dev,const struct panfrost_resource * pres,enum pipe_format fmt)433 panfrost_best_modifier(struct panfrost_device *dev,
434 const struct panfrost_resource *pres,
435 enum pipe_format fmt)
436 {
437 /* Force linear textures when debugging tiling/compression */
438 if (unlikely(dev->debug & PAN_DBG_LINEAR))
439 return DRM_FORMAT_MOD_LINEAR;
440
441 if (panfrost_should_afbc(dev, pres, fmt)) {
442 uint64_t afbc =
443 AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 |
444 AFBC_FORMAT_MOD_SPARSE;
445
446 if (panfrost_afbc_can_ytr(pres->base.format))
447 afbc |= AFBC_FORMAT_MOD_YTR;
448
449 return DRM_FORMAT_MOD_ARM_AFBC(afbc);
450 } else if (panfrost_should_tile(dev, pres, fmt))
451 return DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED;
452 else
453 return DRM_FORMAT_MOD_LINEAR;
454 }
455
456 static bool
panfrost_should_checksum(const struct panfrost_device * dev,const struct panfrost_resource * pres)457 panfrost_should_checksum(const struct panfrost_device *dev, const struct panfrost_resource *pres)
458 {
459 /* When checksumming is enabled, the tile data must fit in the
460 * size of the writeback buffer, so don't checksum formats
461 * that use too much space. */
462
463 unsigned bytes_per_pixel_max = (dev->arch == 6) ? 6 : 4;
464
465 unsigned bytes_per_pixel = MAX2(pres->base.nr_samples, 1) *
466 util_format_get_blocksize(pres->base.format);
467
468 return pres->base.bind & PIPE_BIND_RENDER_TARGET &&
469 panfrost_is_2d(pres) &&
470 bytes_per_pixel <= bytes_per_pixel_max &&
471 pres->base.last_level == 0 &&
472 !(dev->debug & PAN_DBG_NO_CRC);
473 }
474
475 static void
panfrost_resource_setup(struct panfrost_device * dev,struct panfrost_resource * pres,uint64_t modifier,enum pipe_format fmt)476 panfrost_resource_setup(struct panfrost_device *dev,
477 struct panfrost_resource *pres,
478 uint64_t modifier, enum pipe_format fmt)
479 {
480 uint64_t chosen_mod = modifier != DRM_FORMAT_MOD_INVALID ?
481 modifier : panfrost_best_modifier(dev, pres, fmt);
482 enum pan_image_crc_mode crc_mode =
483 panfrost_should_checksum(dev, pres) ?
484 PAN_IMAGE_CRC_INBAND : PAN_IMAGE_CRC_NONE;
485 enum mali_texture_dimension dim =
486 panfrost_translate_texture_dimension(pres->base.target);
487
488 /* We can only switch tiled->linear if the resource isn't already
489 * linear and if we control the modifier */
490 pres->modifier_constant =
491 !(chosen_mod != DRM_FORMAT_MOD_LINEAR &&
492 modifier == DRM_FORMAT_MOD_INVALID);
493
494 /* Z32_S8X24 variants are actually stored in 2 planes (one per
495 * component), we have to adjust the format on the first plane.
496 */
497 if (fmt == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT)
498 fmt = PIPE_FORMAT_Z32_FLOAT;
499
500 ASSERTED bool valid =
501 pan_image_layout_init(dev, &pres->image.layout,
502 chosen_mod, fmt, dim,
503 pres->base.width0,
504 pres->base.height0,
505 pres->base.depth0,
506 pres->base.array_size,
507 MAX2(pres->base.nr_samples, 1),
508 pres->base.last_level + 1,
509 crc_mode, NULL);
510 assert(valid);
511 }
512
513 static void
panfrost_resource_init_afbc_headers(struct panfrost_resource * pres)514 panfrost_resource_init_afbc_headers(struct panfrost_resource *pres)
515 {
516 panfrost_bo_mmap(pres->image.data.bo);
517
518 unsigned nr_samples = MAX2(pres->base.nr_samples, 1);
519
520 for (unsigned i = 0; i < pres->base.array_size; ++i) {
521 for (unsigned l = 0; l <= pres->base.last_level; ++l) {
522 struct pan_image_slice_layout *slice = &pres->image.layout.slices[l];
523
524 for (unsigned s = 0; s < nr_samples; ++s) {
525 void *ptr = pres->image.data.bo->ptr.cpu +
526 (i * pres->image.layout.array_stride) +
527 slice->offset +
528 (s * slice->afbc.surface_stride);
529
530 /* Zero-ed AFBC headers seem to encode a plain
531 * black. Let's use this pattern to keep the
532 * initialization simple.
533 */
534 memset(ptr, 0, slice->afbc.header_size);
535 }
536 }
537 }
538 }
539
540 void
panfrost_resource_set_damage_region(struct pipe_screen * screen,struct pipe_resource * res,unsigned int nrects,const struct pipe_box * rects)541 panfrost_resource_set_damage_region(struct pipe_screen *screen,
542 struct pipe_resource *res,
543 unsigned int nrects,
544 const struct pipe_box *rects)
545 {
546 struct panfrost_device *dev = pan_device(screen);
547 struct panfrost_resource *pres = pan_resource(res);
548 struct pipe_scissor_state *damage_extent = &pres->damage.extent;
549 unsigned int i;
550
551 if (!pan_is_bifrost(dev) && !(dev->quirks & NO_TILE_ENABLE_MAP) &&
552 nrects > 1) {
553 if (!pres->damage.tile_map.data) {
554 pres->damage.tile_map.stride =
555 ALIGN_POT(DIV_ROUND_UP(res->width0, 32 * 8), 64);
556 pres->damage.tile_map.size =
557 pres->damage.tile_map.stride *
558 DIV_ROUND_UP(res->height0, 32);
559 pres->damage.tile_map.data =
560 malloc(pres->damage.tile_map.size);
561 }
562
563 memset(pres->damage.tile_map.data, 0, pres->damage.tile_map.size);
564 pres->damage.tile_map.enable = true;
565 } else {
566 pres->damage.tile_map.enable = false;
567 }
568
569 /* Track the damage extent: the quad including all damage regions. Will
570 * be used restrict the rendering area */
571
572 damage_extent->minx = 0xffff;
573 damage_extent->miny = 0xffff;
574
575 unsigned enable_count = 0;
576
577 for (i = 0; i < nrects; i++) {
578 int x = rects[i].x, w = rects[i].width, h = rects[i].height;
579 int y = res->height0 - (rects[i].y + h);
580
581 damage_extent->minx = MIN2(damage_extent->minx, x);
582 damage_extent->miny = MIN2(damage_extent->miny, y);
583 damage_extent->maxx = MAX2(damage_extent->maxx,
584 MIN2(x + w, res->width0));
585 damage_extent->maxy = MAX2(damage_extent->maxy,
586 MIN2(y + h, res->height0));
587
588 if (!pres->damage.tile_map.enable)
589 continue;
590
591 unsigned t_x_start = x / 32;
592 unsigned t_x_end = (x + w - 1) / 32;
593 unsigned t_y_start = y / 32;
594 unsigned t_y_end = (y + h - 1) / 32;
595
596 for (unsigned t_y = t_y_start; t_y <= t_y_end; t_y++) {
597 for (unsigned t_x = t_x_start; t_x <= t_x_end; t_x++) {
598 unsigned b = (t_y * pres->damage.tile_map.stride * 8) + t_x;
599
600 if (BITSET_TEST(pres->damage.tile_map.data, b))
601 continue;
602
603 BITSET_SET(pres->damage.tile_map.data, b);
604 enable_count++;
605 }
606 }
607 }
608
609 if (nrects == 0) {
610 damage_extent->minx = 0;
611 damage_extent->miny = 0;
612 damage_extent->maxx = res->width0;
613 damage_extent->maxy = res->height0;
614 }
615
616 if (pres->damage.tile_map.enable) {
617 unsigned t_x_start = damage_extent->minx / 32;
618 unsigned t_x_end = damage_extent->maxx / 32;
619 unsigned t_y_start = damage_extent->miny / 32;
620 unsigned t_y_end = damage_extent->maxy / 32;
621 unsigned tile_count = (t_x_end - t_x_start + 1) *
622 (t_y_end - t_y_start + 1);
623
624 /* Don't bother passing a tile-enable-map if the amount of
625 * tiles to reload is to close to the total number of tiles.
626 */
627 if (tile_count - enable_count < 10)
628 pres->damage.tile_map.enable = false;
629 }
630
631 }
632
633 static struct pipe_resource *
panfrost_resource_create_with_modifier(struct pipe_screen * screen,const struct pipe_resource * template,uint64_t modifier)634 panfrost_resource_create_with_modifier(struct pipe_screen *screen,
635 const struct pipe_resource *template,
636 uint64_t modifier)
637 {
638 struct panfrost_device *dev = pan_device(screen);
639
640 if (dev->ro && (template->bind &
641 (PIPE_BIND_DISPLAY_TARGET | PIPE_BIND_SCANOUT | PIPE_BIND_SHARED)))
642 return panfrost_create_scanout_res(screen, template, modifier);
643
644 struct panfrost_resource *so = CALLOC_STRUCT(panfrost_resource);
645 so->base = *template;
646 so->base.screen = screen;
647
648 pipe_reference_init(&so->base.reference, 1);
649
650 util_range_init(&so->valid_buffer_range);
651
652 panfrost_resource_setup(dev, so, modifier, template->format);
653
654 /* Guess a label based on the bind */
655 unsigned bind = template->bind;
656 const char *label =
657 (bind & PIPE_BIND_INDEX_BUFFER) ? "Index buffer" :
658 (bind & PIPE_BIND_SCANOUT) ? "Scanout" :
659 (bind & PIPE_BIND_DISPLAY_TARGET) ? "Display target" :
660 (bind & PIPE_BIND_SHARED) ? "Shared resource" :
661 (bind & PIPE_BIND_RENDER_TARGET) ? "Render target" :
662 (bind & PIPE_BIND_DEPTH_STENCIL) ? "Depth/stencil buffer" :
663 (bind & PIPE_BIND_SAMPLER_VIEW) ? "Texture" :
664 (bind & PIPE_BIND_VERTEX_BUFFER) ? "Vertex buffer" :
665 (bind & PIPE_BIND_CONSTANT_BUFFER) ? "Constant buffer" :
666 (bind & PIPE_BIND_GLOBAL) ? "Global memory" :
667 (bind & PIPE_BIND_SHADER_BUFFER) ? "Shader buffer" :
668 (bind & PIPE_BIND_SHADER_IMAGE) ? "Shader image" :
669 "Other resource";
670
671 /* We create a BO immediately but don't bother mapping, since we don't
672 * care to map e.g. FBOs which the CPU probably won't touch */
673 so->image.data.bo =
674 panfrost_bo_create(dev, so->image.layout.data_size, PAN_BO_DELAY_MMAP, label);
675
676 if (drm_is_afbc(so->image.layout.modifier))
677 panfrost_resource_init_afbc_headers(so);
678
679 panfrost_resource_set_damage_region(screen, &so->base, 0, NULL);
680
681 if (template->bind & PIPE_BIND_INDEX_BUFFER)
682 so->index_cache = CALLOC_STRUCT(panfrost_minmax_cache);
683
684 return (struct pipe_resource *)so;
685 }
686
687 /* Default is to create a resource as don't care */
688
689 static struct pipe_resource *
panfrost_resource_create(struct pipe_screen * screen,const struct pipe_resource * template)690 panfrost_resource_create(struct pipe_screen *screen,
691 const struct pipe_resource *template)
692 {
693 return panfrost_resource_create_with_modifier(screen, template,
694 DRM_FORMAT_MOD_INVALID);
695 }
696
697 /* If no modifier is specified, we'll choose. Otherwise, the order of
698 * preference is compressed, tiled, linear. */
699
700 static struct pipe_resource *
panfrost_resource_create_with_modifiers(struct pipe_screen * screen,const struct pipe_resource * template,const uint64_t * modifiers,int count)701 panfrost_resource_create_with_modifiers(struct pipe_screen *screen,
702 const struct pipe_resource *template,
703 const uint64_t *modifiers, int count)
704 {
705 for (unsigned i = 0; i < PAN_MODIFIER_COUNT; ++i) {
706 if (drm_find_modifier(pan_best_modifiers[i], modifiers, count)) {
707 return panfrost_resource_create_with_modifier(screen, template,
708 pan_best_modifiers[i]);
709 }
710 }
711
712 /* If we didn't find one, app specified invalid */
713 assert(count == 1 && modifiers[0] == DRM_FORMAT_MOD_INVALID);
714 return panfrost_resource_create(screen, template);
715 }
716
717 static void
panfrost_resource_destroy(struct pipe_screen * screen,struct pipe_resource * pt)718 panfrost_resource_destroy(struct pipe_screen *screen,
719 struct pipe_resource *pt)
720 {
721 struct panfrost_device *dev = pan_device(screen);
722 struct panfrost_resource *rsrc = (struct panfrost_resource *) pt;
723
724 if (rsrc->scanout)
725 renderonly_scanout_destroy(rsrc->scanout, dev->ro);
726
727 if (rsrc->image.data.bo)
728 panfrost_bo_unreference(rsrc->image.data.bo);
729
730 if (rsrc->image.crc.bo)
731 panfrost_bo_unreference(rsrc->image.crc.bo);
732
733 free(rsrc->index_cache);
734 free(rsrc->damage.tile_map.data);
735
736 util_range_destroy(&rsrc->valid_buffer_range);
737 free(rsrc);
738 }
739
740 /* Most of the time we can do CPU-side transfers, but sometimes we need to use
741 * the 3D pipe for this. Let's wrap u_blitter to blit to/from staging textures.
742 * Code adapted from freedreno */
743
744 static struct panfrost_resource *
pan_alloc_staging(struct panfrost_context * ctx,struct panfrost_resource * rsc,unsigned level,const struct pipe_box * box)745 pan_alloc_staging(struct panfrost_context *ctx, struct panfrost_resource *rsc,
746 unsigned level, const struct pipe_box *box)
747 {
748 struct pipe_context *pctx = &ctx->base;
749 struct pipe_resource tmpl = rsc->base;
750
751 tmpl.width0 = box->width;
752 tmpl.height0 = box->height;
753 /* for array textures, box->depth is the array_size, otherwise
754 * for 3d textures, it is the depth:
755 */
756 if (tmpl.array_size > 1) {
757 if (tmpl.target == PIPE_TEXTURE_CUBE)
758 tmpl.target = PIPE_TEXTURE_2D_ARRAY;
759 tmpl.array_size = box->depth;
760 tmpl.depth0 = 1;
761 } else {
762 tmpl.array_size = 1;
763 tmpl.depth0 = box->depth;
764 }
765 tmpl.last_level = 0;
766 tmpl.bind |= PIPE_BIND_LINEAR;
767 tmpl.bind &= ~(PIPE_BIND_DISPLAY_TARGET | PIPE_BIND_SCANOUT | PIPE_BIND_SHARED);
768
769 struct pipe_resource *pstaging =
770 pctx->screen->resource_create(pctx->screen, &tmpl);
771 if (!pstaging)
772 return NULL;
773
774 return pan_resource(pstaging);
775 }
776
777 static enum pipe_format
pan_blit_format(enum pipe_format fmt)778 pan_blit_format(enum pipe_format fmt)
779 {
780 const struct util_format_description *desc;
781 desc = util_format_description(fmt);
782
783 /* This must be an emulated format (using u_transfer_helper) as if it
784 * was real RGTC we wouldn't have used AFBC and needed a blit. */
785 if (desc->layout == UTIL_FORMAT_LAYOUT_RGTC)
786 fmt = PIPE_FORMAT_R8G8B8A8_UNORM;
787
788 return fmt;
789 }
790
791 static void
pan_blit_from_staging(struct pipe_context * pctx,struct panfrost_transfer * trans)792 pan_blit_from_staging(struct pipe_context *pctx, struct panfrost_transfer *trans)
793 {
794 struct pipe_resource *dst = trans->base.resource;
795 struct pipe_blit_info blit = {0};
796
797 blit.dst.resource = dst;
798 blit.dst.format = pan_blit_format(dst->format);
799 blit.dst.level = trans->base.level;
800 blit.dst.box = trans->base.box;
801 blit.src.resource = trans->staging.rsrc;
802 blit.src.format = pan_blit_format(trans->staging.rsrc->format);
803 blit.src.level = 0;
804 blit.src.box = trans->staging.box;
805 blit.mask = util_format_get_mask(blit.src.format);
806 blit.filter = PIPE_TEX_FILTER_NEAREST;
807
808 panfrost_blit(pctx, &blit);
809 }
810
811 static void
pan_blit_to_staging(struct pipe_context * pctx,struct panfrost_transfer * trans)812 pan_blit_to_staging(struct pipe_context *pctx, struct panfrost_transfer *trans)
813 {
814 struct pipe_resource *src = trans->base.resource;
815 struct pipe_blit_info blit = {0};
816
817 blit.src.resource = src;
818 blit.src.format = pan_blit_format(src->format);
819 blit.src.level = trans->base.level;
820 blit.src.box = trans->base.box;
821 blit.dst.resource = trans->staging.rsrc;
822 blit.dst.format = pan_blit_format(trans->staging.rsrc->format);
823 blit.dst.level = 0;
824 blit.dst.box = trans->staging.box;
825 blit.mask = util_format_get_mask(blit.dst.format);
826 blit.filter = PIPE_TEX_FILTER_NEAREST;
827
828 panfrost_blit(pctx, &blit);
829 }
830
831 static void *
panfrost_ptr_map(struct pipe_context * pctx,struct pipe_resource * resource,unsigned level,unsigned usage,const struct pipe_box * box,struct pipe_transfer ** out_transfer)832 panfrost_ptr_map(struct pipe_context *pctx,
833 struct pipe_resource *resource,
834 unsigned level,
835 unsigned usage, /* a combination of PIPE_MAP_x */
836 const struct pipe_box *box,
837 struct pipe_transfer **out_transfer)
838 {
839 struct panfrost_context *ctx = pan_context(pctx);
840 struct panfrost_device *dev = pan_device(pctx->screen);
841 struct panfrost_resource *rsrc = pan_resource(resource);
842 int bytes_per_pixel = util_format_get_blocksize(rsrc->image.layout.format);
843 struct panfrost_bo *bo = rsrc->image.data.bo;
844
845 /* Can't map tiled/compressed directly */
846 if ((usage & PIPE_MAP_DIRECTLY) && rsrc->image.layout.modifier != DRM_FORMAT_MOD_LINEAR)
847 return NULL;
848
849 struct panfrost_transfer *transfer = rzalloc(pctx, struct panfrost_transfer);
850 transfer->base.level = level;
851 transfer->base.usage = usage;
852 transfer->base.box = *box;
853
854 pipe_resource_reference(&transfer->base.resource, resource);
855 *out_transfer = &transfer->base;
856
857 /* We don't have s/w routines for AFBC, so use a staging texture */
858 if (drm_is_afbc(rsrc->image.layout.modifier)) {
859 struct panfrost_resource *staging = pan_alloc_staging(ctx, rsrc, level, box);
860 assert(staging);
861
862 /* Staging resources have one LOD: level 0. Query the strides
863 * on this LOD.
864 */
865 transfer->base.stride = staging->image.layout.slices[0].line_stride;
866 transfer->base.layer_stride =
867 panfrost_get_layer_stride(&staging->image.layout, 0);
868
869 transfer->staging.rsrc = &staging->base;
870
871 transfer->staging.box = *box;
872 transfer->staging.box.x = 0;
873 transfer->staging.box.y = 0;
874 transfer->staging.box.z = 0;
875
876 assert(transfer->staging.rsrc != NULL);
877
878 bool valid = BITSET_TEST(rsrc->valid.data, level);
879
880 if ((usage & PIPE_MAP_READ) && (valid || rsrc->track.nr_writers > 0)) {
881 pan_blit_to_staging(pctx, transfer);
882 panfrost_flush_writer(ctx, staging, "AFBC read staging blit");
883 panfrost_bo_wait(staging->image.data.bo, INT64_MAX, false);
884 }
885
886 panfrost_bo_mmap(staging->image.data.bo);
887 return staging->image.data.bo->ptr.cpu;
888 }
889
890 /* If we haven't already mmaped, now's the time */
891 panfrost_bo_mmap(bo);
892
893 if (dev->debug & (PAN_DBG_TRACE | PAN_DBG_SYNC))
894 pandecode_inject_mmap(bo->ptr.gpu, bo->ptr.cpu, bo->size, NULL);
895
896 bool create_new_bo = usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE;
897 bool copy_resource = false;
898
899 if (!create_new_bo &&
900 !(usage & PIPE_MAP_UNSYNCHRONIZED) &&
901 (usage & PIPE_MAP_WRITE) &&
902 !(resource->target == PIPE_BUFFER
903 && !util_ranges_intersect(&rsrc->valid_buffer_range, box->x, box->x + box->width)) &&
904 rsrc->track.nr_users > 0) {
905
906 /* When a resource to be modified is already being used by a
907 * pending batch, it is often faster to copy the whole BO than
908 * to flush and split the frame in two.
909 */
910
911 panfrost_flush_writer(ctx, rsrc, "Shadow resource creation");
912 panfrost_bo_wait(bo, INT64_MAX, false);
913
914 create_new_bo = true;
915 copy_resource = true;
916 }
917
918 if (create_new_bo) {
919 /* If the BO is used by one of the pending batches or if it's
920 * not ready yet (still accessed by one of the already flushed
921 * batches), we try to allocate a new one to avoid waiting.
922 */
923 if (rsrc->track.nr_users > 0 ||
924 !panfrost_bo_wait(bo, 0, true)) {
925 /* We want the BO to be MMAPed. */
926 uint32_t flags = bo->flags & ~PAN_BO_DELAY_MMAP;
927 struct panfrost_bo *newbo = NULL;
928
929 /* When the BO has been imported/exported, we can't
930 * replace it by another one, otherwise the
931 * importer/exporter wouldn't see the change we're
932 * doing to it.
933 */
934 if (!(bo->flags & PAN_BO_SHARED))
935 newbo = panfrost_bo_create(dev, bo->size,
936 flags, bo->label);
937
938 if (newbo) {
939 if (copy_resource)
940 memcpy(newbo->ptr.cpu, rsrc->image.data.bo->ptr.cpu, bo->size);
941
942 panfrost_bo_unreference(bo);
943 rsrc->image.data.bo = newbo;
944
945 if (!copy_resource &&
946 drm_is_afbc(rsrc->image.layout.modifier))
947 panfrost_resource_init_afbc_headers(rsrc);
948
949 bo = newbo;
950 } else {
951 /* Allocation failed or was impossible, let's
952 * fall back on a flush+wait.
953 */
954 panfrost_flush_batches_accessing_rsrc(ctx, rsrc,
955 "Resource access with high memory pressure");
956 panfrost_bo_wait(bo, INT64_MAX, true);
957 }
958 }
959 } else if ((usage & PIPE_MAP_WRITE)
960 && resource->target == PIPE_BUFFER
961 && !util_ranges_intersect(&rsrc->valid_buffer_range, box->x, box->x + box->width)) {
962 /* No flush for writes to uninitialized */
963 } else if (!(usage & PIPE_MAP_UNSYNCHRONIZED)) {
964 if (usage & PIPE_MAP_WRITE) {
965 panfrost_flush_batches_accessing_rsrc(ctx, rsrc, "Synchronized write");
966 panfrost_bo_wait(bo, INT64_MAX, true);
967 } else if (usage & PIPE_MAP_READ) {
968 panfrost_flush_writer(ctx, rsrc, "Synchronized read");
969 panfrost_bo_wait(bo, INT64_MAX, false);
970 }
971 }
972
973 if (rsrc->image.layout.modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) {
974 transfer->base.stride = box->width * bytes_per_pixel;
975 transfer->base.layer_stride = transfer->base.stride * box->height;
976 transfer->map = ralloc_size(transfer, transfer->base.layer_stride * box->depth);
977 assert(box->depth == 1);
978
979 if ((usage & PIPE_MAP_READ) && BITSET_TEST(rsrc->valid.data, level)) {
980 panfrost_load_tiled_image(
981 transfer->map,
982 bo->ptr.cpu + rsrc->image.layout.slices[level].offset,
983 box->x, box->y, box->width, box->height,
984 transfer->base.stride,
985 rsrc->image.layout.slices[level].line_stride,
986 rsrc->image.layout.format);
987 }
988
989 return transfer->map;
990 } else {
991 assert (rsrc->image.layout.modifier == DRM_FORMAT_MOD_LINEAR);
992
993 /* Direct, persistent writes create holes in time for
994 * caching... I don't know if this is actually possible but we
995 * should still get it right */
996
997 unsigned dpw = PIPE_MAP_DIRECTLY | PIPE_MAP_WRITE | PIPE_MAP_PERSISTENT;
998
999 if ((usage & dpw) == dpw && rsrc->index_cache)
1000 return NULL;
1001
1002 transfer->base.stride = rsrc->image.layout.slices[level].line_stride;
1003 transfer->base.layer_stride =
1004 panfrost_get_layer_stride(&rsrc->image.layout, level);
1005
1006 /* By mapping direct-write, we're implicitly already
1007 * initialized (maybe), so be conservative */
1008
1009 if (usage & PIPE_MAP_WRITE) {
1010 BITSET_SET(rsrc->valid.data, level);
1011 panfrost_minmax_cache_invalidate(rsrc->index_cache, &transfer->base);
1012 }
1013
1014 return bo->ptr.cpu
1015 + rsrc->image.layout.slices[level].offset
1016 + transfer->base.box.z * transfer->base.layer_stride
1017 + transfer->base.box.y * rsrc->image.layout.slices[level].line_stride
1018 + transfer->base.box.x * bytes_per_pixel;
1019 }
1020 }
1021
1022 void
pan_resource_modifier_convert(struct panfrost_context * ctx,struct panfrost_resource * rsrc,uint64_t modifier,const char * reason)1023 pan_resource_modifier_convert(struct panfrost_context *ctx,
1024 struct panfrost_resource *rsrc,
1025 uint64_t modifier, const char *reason)
1026 {
1027 assert(!rsrc->modifier_constant);
1028
1029 perf_debug_ctx(ctx, "Disabling AFBC with a blit. Reason: %s", reason);
1030
1031 struct pipe_resource *tmp_prsrc =
1032 panfrost_resource_create_with_modifier(
1033 ctx->base.screen, &rsrc->base, modifier);
1034 struct panfrost_resource *tmp_rsrc = pan_resource(tmp_prsrc);
1035 enum pipe_format blit_fmt = pan_blit_format(tmp_rsrc->base.format);
1036
1037 unsigned depth = rsrc->base.target == PIPE_TEXTURE_3D ?
1038 rsrc->base.depth0 : rsrc->base.array_size;
1039
1040 struct pipe_box box =
1041 { 0, 0, 0, rsrc->base.width0, rsrc->base.height0, depth };
1042
1043 struct pipe_blit_info blit = {
1044 .dst.resource = &tmp_rsrc->base,
1045 .dst.format = blit_fmt,
1046 .dst.box = box,
1047 .src.resource = &rsrc->base,
1048 .src.format = pan_blit_format(rsrc->base.format),
1049 .src.box = box,
1050 .mask = util_format_get_mask(blit_fmt),
1051 .filter = PIPE_TEX_FILTER_NEAREST
1052 };
1053
1054 for (int i = 0; i <= rsrc->base.last_level; i++) {
1055 if (BITSET_TEST(rsrc->valid.data, i)) {
1056 blit.dst.level = blit.src.level = i;
1057 panfrost_blit(&ctx->base, &blit);
1058 }
1059 }
1060
1061 panfrost_bo_unreference(rsrc->image.data.bo);
1062 if (rsrc->image.crc.bo)
1063 panfrost_bo_unreference(rsrc->image.crc.bo);
1064
1065 rsrc->image.data.bo = tmp_rsrc->image.data.bo;
1066 panfrost_bo_reference(rsrc->image.data.bo);
1067
1068 panfrost_resource_setup(pan_device(ctx->base.screen), rsrc, modifier,
1069 blit.dst.format);
1070 pipe_resource_reference(&tmp_prsrc, NULL);
1071 }
1072
1073 /* Validate that an AFBC resource may be used as a particular format. If it may
1074 * not, decompress it on the fly. Failure to do so can produce wrong results or
1075 * invalid data faults when sampling or rendering to AFBC */
1076
1077 void
pan_legalize_afbc_format(struct panfrost_context * ctx,struct panfrost_resource * rsrc,enum pipe_format format)1078 pan_legalize_afbc_format(struct panfrost_context *ctx,
1079 struct panfrost_resource *rsrc,
1080 enum pipe_format format)
1081 {
1082 struct panfrost_device *dev = pan_device(ctx->base.screen);
1083
1084 if (!drm_is_afbc(rsrc->image.layout.modifier))
1085 return;
1086
1087 if (panfrost_afbc_format(dev, pan_blit_format(rsrc->base.format)) ==
1088 panfrost_afbc_format(dev, pan_blit_format(format)))
1089 return;
1090
1091 pan_resource_modifier_convert(ctx, rsrc,
1092 DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED,
1093 "Reinterpreting AFBC surface as incompatible format");
1094 }
1095
1096 static bool
panfrost_should_linear_convert(struct panfrost_device * dev,struct panfrost_resource * prsrc,struct pipe_transfer * transfer)1097 panfrost_should_linear_convert(struct panfrost_device *dev,
1098 struct panfrost_resource *prsrc,
1099 struct pipe_transfer *transfer)
1100 {
1101 if (prsrc->modifier_constant)
1102 return false;
1103
1104 /* Overwriting the entire resource indicates streaming, for which
1105 * linear layout is most efficient due to the lack of expensive
1106 * conversion.
1107 *
1108 * For now we just switch to linear after a number of complete
1109 * overwrites to keep things simple, but we could do better.
1110 */
1111
1112 unsigned depth = prsrc->base.target == PIPE_TEXTURE_3D ?
1113 prsrc->base.depth0 : prsrc->base.array_size;
1114 bool entire_overwrite =
1115 prsrc->base.last_level == 0 &&
1116 transfer->box.width == prsrc->base.width0 &&
1117 transfer->box.height == prsrc->base.height0 &&
1118 transfer->box.depth == depth &&
1119 transfer->box.x == 0 &&
1120 transfer->box.y == 0 &&
1121 transfer->box.z == 0;
1122
1123 if (entire_overwrite)
1124 ++prsrc->modifier_updates;
1125
1126 if (prsrc->modifier_updates >= LAYOUT_CONVERT_THRESHOLD) {
1127 perf_debug(dev, "Transitioning to linear due to streaming usage");
1128 return true;
1129 } else {
1130 return false;
1131 }
1132 }
1133
1134 static void
panfrost_ptr_unmap(struct pipe_context * pctx,struct pipe_transfer * transfer)1135 panfrost_ptr_unmap(struct pipe_context *pctx,
1136 struct pipe_transfer *transfer)
1137 {
1138 /* Gallium expects writeback here, so we tile */
1139
1140 struct panfrost_transfer *trans = pan_transfer(transfer);
1141 struct panfrost_resource *prsrc = (struct panfrost_resource *) transfer->resource;
1142 struct panfrost_device *dev = pan_device(pctx->screen);
1143
1144 if (transfer->usage & PIPE_MAP_WRITE)
1145 prsrc->valid.crc = false;
1146
1147 /* AFBC will use a staging resource. `initialized` will be set when the
1148 * fragment job is created; this is deferred to prevent useless surface
1149 * reloads that can cascade into DATA_INVALID_FAULTs due to reading
1150 * malformed AFBC data if uninitialized */
1151
1152 if (trans->staging.rsrc) {
1153 if (transfer->usage & PIPE_MAP_WRITE) {
1154 if (panfrost_should_linear_convert(dev, prsrc, transfer)) {
1155
1156 panfrost_bo_unreference(prsrc->image.data.bo);
1157 if (prsrc->image.crc.bo)
1158 panfrost_bo_unreference(prsrc->image.crc.bo);
1159
1160 panfrost_resource_setup(dev, prsrc, DRM_FORMAT_MOD_LINEAR,
1161 prsrc->image.layout.format);
1162
1163 prsrc->image.data.bo = pan_resource(trans->staging.rsrc)->image.data.bo;
1164 panfrost_bo_reference(prsrc->image.data.bo);
1165 } else {
1166 pan_blit_from_staging(pctx, trans);
1167 panfrost_flush_batches_accessing_rsrc(pan_context(pctx),
1168 pan_resource(trans->staging.rsrc),
1169 "AFBC write staging blit");
1170 }
1171 }
1172
1173 pipe_resource_reference(&trans->staging.rsrc, NULL);
1174 }
1175
1176 /* Tiling will occur in software from a staging cpu buffer */
1177 if (trans->map) {
1178 struct panfrost_bo *bo = prsrc->image.data.bo;
1179
1180 if (transfer->usage & PIPE_MAP_WRITE) {
1181 BITSET_SET(prsrc->valid.data, transfer->level);
1182
1183 if (prsrc->image.layout.modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) {
1184 assert(transfer->box.depth == 1);
1185
1186 if (panfrost_should_linear_convert(dev, prsrc, transfer)) {
1187 panfrost_resource_setup(dev, prsrc, DRM_FORMAT_MOD_LINEAR,
1188 prsrc->image.layout.format);
1189 if (prsrc->image.layout.data_size > bo->size) {
1190 const char *label = bo->label;
1191 panfrost_bo_unreference(bo);
1192 bo = prsrc->image.data.bo =
1193 panfrost_bo_create(dev, prsrc->image.layout.data_size, 0, label);
1194 assert(bo);
1195 }
1196
1197 util_copy_rect(
1198 bo->ptr.cpu + prsrc->image.layout.slices[0].offset,
1199 prsrc->base.format,
1200 prsrc->image.layout.slices[0].line_stride,
1201 0, 0,
1202 transfer->box.width,
1203 transfer->box.height,
1204 trans->map,
1205 transfer->stride,
1206 0, 0);
1207 } else {
1208 panfrost_store_tiled_image(
1209 bo->ptr.cpu + prsrc->image.layout.slices[transfer->level].offset,
1210 trans->map,
1211 transfer->box.x, transfer->box.y,
1212 transfer->box.width, transfer->box.height,
1213 prsrc->image.layout.slices[transfer->level].line_stride,
1214 transfer->stride,
1215 prsrc->image.layout.format);
1216 }
1217 }
1218 }
1219 }
1220
1221
1222 util_range_add(&prsrc->base, &prsrc->valid_buffer_range,
1223 transfer->box.x,
1224 transfer->box.x + transfer->box.width);
1225
1226 panfrost_minmax_cache_invalidate(prsrc->index_cache, transfer);
1227
1228 /* Derefence the resource */
1229 pipe_resource_reference(&transfer->resource, NULL);
1230
1231 /* Transfer itself is RALLOCed at the moment */
1232 ralloc_free(transfer);
1233 }
1234
1235 static void
panfrost_ptr_flush_region(struct pipe_context * pctx,struct pipe_transfer * transfer,const struct pipe_box * box)1236 panfrost_ptr_flush_region(struct pipe_context *pctx,
1237 struct pipe_transfer *transfer,
1238 const struct pipe_box *box)
1239 {
1240 struct panfrost_resource *rsc = pan_resource(transfer->resource);
1241
1242 if (transfer->resource->target == PIPE_BUFFER) {
1243 util_range_add(&rsc->base, &rsc->valid_buffer_range,
1244 transfer->box.x + box->x,
1245 transfer->box.x + box->x + box->width);
1246 } else {
1247 BITSET_SET(rsc->valid.data, transfer->level);
1248 }
1249 }
1250
1251 static void
panfrost_invalidate_resource(struct pipe_context * pctx,struct pipe_resource * prsrc)1252 panfrost_invalidate_resource(struct pipe_context *pctx, struct pipe_resource *prsrc)
1253 {
1254 struct panfrost_context *ctx = pan_context(pctx);
1255 struct panfrost_batch *batch = panfrost_get_batch_for_fbo(ctx);
1256
1257 /* Handle the glInvalidateFramebuffer case */
1258 if (batch->key.zsbuf && batch->key.zsbuf->texture == prsrc)
1259 batch->resolve &= ~PIPE_CLEAR_DEPTHSTENCIL;
1260
1261 for (unsigned i = 0; i < batch->key.nr_cbufs; ++i) {
1262 struct pipe_surface *surf = batch->key.cbufs[i];
1263
1264 if (surf && surf->texture == prsrc)
1265 batch->resolve &= ~(PIPE_CLEAR_COLOR0 << i);
1266 }
1267 }
1268
1269 static enum pipe_format
panfrost_resource_get_internal_format(struct pipe_resource * rsrc)1270 panfrost_resource_get_internal_format(struct pipe_resource *rsrc)
1271 {
1272 struct panfrost_resource *prsrc = (struct panfrost_resource *) rsrc;
1273 return prsrc->image.layout.format;
1274 }
1275
1276 static bool
panfrost_generate_mipmap(struct pipe_context * pctx,struct pipe_resource * prsrc,enum pipe_format format,unsigned base_level,unsigned last_level,unsigned first_layer,unsigned last_layer)1277 panfrost_generate_mipmap(
1278 struct pipe_context *pctx,
1279 struct pipe_resource *prsrc,
1280 enum pipe_format format,
1281 unsigned base_level,
1282 unsigned last_level,
1283 unsigned first_layer,
1284 unsigned last_layer)
1285 {
1286 struct panfrost_resource *rsrc = pan_resource(prsrc);
1287
1288 /* Generating a mipmap invalidates the written levels, so make that
1289 * explicit so we don't try to wallpaper them back and end up with
1290 * u_blitter recursion */
1291
1292 assert(rsrc->image.data.bo);
1293 for (unsigned l = base_level + 1; l <= last_level; ++l)
1294 BITSET_CLEAR(rsrc->valid.data, l);
1295
1296 /* Beyond that, we just delegate the hard stuff. */
1297
1298 bool blit_res = util_gen_mipmap(
1299 pctx, prsrc, format,
1300 base_level, last_level,
1301 first_layer, last_layer,
1302 PIPE_TEX_FILTER_LINEAR);
1303
1304 return blit_res;
1305 }
1306
1307 static void
panfrost_resource_set_stencil(struct pipe_resource * prsrc,struct pipe_resource * stencil)1308 panfrost_resource_set_stencil(struct pipe_resource *prsrc,
1309 struct pipe_resource *stencil)
1310 {
1311 pan_resource(prsrc)->separate_stencil = pan_resource(stencil);
1312 }
1313
1314 static struct pipe_resource *
panfrost_resource_get_stencil(struct pipe_resource * prsrc)1315 panfrost_resource_get_stencil(struct pipe_resource *prsrc)
1316 {
1317 if (!pan_resource(prsrc)->separate_stencil)
1318 return NULL;
1319
1320 return &pan_resource(prsrc)->separate_stencil->base;
1321 }
1322
1323 static const struct u_transfer_vtbl transfer_vtbl = {
1324 .resource_create = panfrost_resource_create,
1325 .resource_destroy = panfrost_resource_destroy,
1326 .transfer_map = panfrost_ptr_map,
1327 .transfer_unmap = panfrost_ptr_unmap,
1328 .transfer_flush_region = panfrost_ptr_flush_region,
1329 .get_internal_format = panfrost_resource_get_internal_format,
1330 .set_stencil = panfrost_resource_set_stencil,
1331 .get_stencil = panfrost_resource_get_stencil,
1332 };
1333
1334 void
panfrost_resource_screen_init(struct pipe_screen * pscreen)1335 panfrost_resource_screen_init(struct pipe_screen *pscreen)
1336 {
1337 struct panfrost_device *dev = pan_device(pscreen);
1338
1339 bool fake_rgtc = !panfrost_supports_compressed_format(dev, MALI_BC4_UNORM);
1340
1341 pscreen->resource_create_with_modifiers =
1342 panfrost_resource_create_with_modifiers;
1343 pscreen->resource_create = u_transfer_helper_resource_create;
1344 pscreen->resource_destroy = u_transfer_helper_resource_destroy;
1345 pscreen->resource_from_handle = panfrost_resource_from_handle;
1346 pscreen->resource_get_handle = panfrost_resource_get_handle;
1347 pscreen->resource_get_param = panfrost_resource_get_param;
1348 pscreen->transfer_helper = u_transfer_helper_create(&transfer_vtbl,
1349 true, false,
1350 fake_rgtc, true);
1351 }
1352 void
panfrost_resource_screen_destroy(struct pipe_screen * pscreen)1353 panfrost_resource_screen_destroy(struct pipe_screen *pscreen)
1354 {
1355 u_transfer_helper_destroy(pscreen->transfer_helper);
1356 }
1357
1358 void
panfrost_resource_context_init(struct pipe_context * pctx)1359 panfrost_resource_context_init(struct pipe_context *pctx)
1360 {
1361 pctx->buffer_map = u_transfer_helper_transfer_map;
1362 pctx->buffer_unmap = u_transfer_helper_transfer_unmap;
1363 pctx->texture_map = u_transfer_helper_transfer_map;
1364 pctx->texture_unmap = u_transfer_helper_transfer_unmap;
1365 pctx->create_surface = panfrost_create_surface;
1366 pctx->surface_destroy = panfrost_surface_destroy;
1367 pctx->resource_copy_region = util_resource_copy_region;
1368 pctx->blit = panfrost_blit;
1369 pctx->generate_mipmap = panfrost_generate_mipmap;
1370 pctx->flush_resource = panfrost_flush_resource;
1371 pctx->invalidate_resource = panfrost_invalidate_resource;
1372 pctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;
1373 pctx->buffer_subdata = u_default_buffer_subdata;
1374 pctx->texture_subdata = u_default_texture_subdata;
1375 }
1376