1 /*
2 * Copyright © 2014-2017 Broadcom
3 * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org>
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 * IN THE SOFTWARE.
23 */
24
25 #include "pipe/p_defines.h"
26 #include "util/u_memory.h"
27 #include "util/format/u_format.h"
28 #include "util/u_inlines.h"
29 #include "util/u_surface.h"
30 #include "util/u_transfer_helper.h"
31 #include "util/u_upload_mgr.h"
32 #include "util/format/u_format_zs.h"
33 #include "util/u_drm.h"
34
35 #include "drm-uapi/drm_fourcc.h"
36 #include "v3d_screen.h"
37 #include "v3d_context.h"
38 #include "v3d_resource.h"
39 #include "broadcom/cle/v3d_packet_v33_pack.h"
40
41 static void
v3d_debug_resource_layout(struct v3d_resource * rsc,const char * caller)42 v3d_debug_resource_layout(struct v3d_resource *rsc, const char *caller)
43 {
44 if (!(unlikely(V3D_DEBUG & V3D_DEBUG_SURFACE)))
45 return;
46
47 struct pipe_resource *prsc = &rsc->base;
48
49 if (prsc->target == PIPE_BUFFER) {
50 fprintf(stderr,
51 "rsc %s %p (format %s), %dx%d buffer @0x%08x-0x%08x\n",
52 caller, rsc,
53 util_format_short_name(prsc->format),
54 prsc->width0, prsc->height0,
55 rsc->bo->offset,
56 rsc->bo->offset + rsc->bo->size - 1);
57 return;
58 }
59
60 static const char *const tiling_descriptions[] = {
61 [V3D_TILING_RASTER] = "R",
62 [V3D_TILING_LINEARTILE] = "LT",
63 [V3D_TILING_UBLINEAR_1_COLUMN] = "UB1",
64 [V3D_TILING_UBLINEAR_2_COLUMN] = "UB2",
65 [V3D_TILING_UIF_NO_XOR] = "UIF",
66 [V3D_TILING_UIF_XOR] = "UIF^",
67 };
68
69 for (int i = 0; i <= prsc->last_level; i++) {
70 struct v3d_resource_slice *slice = &rsc->slices[i];
71
72 int level_width = slice->stride / rsc->cpp;
73 int level_height = slice->padded_height;
74 int level_depth =
75 u_minify(util_next_power_of_two(prsc->depth0), i);
76
77 fprintf(stderr,
78 "rsc %s %p (format %s), %dx%d: "
79 "level %d (%s) %dx%dx%d -> %dx%dx%d, stride %d@0x%08x\n",
80 caller, rsc,
81 util_format_short_name(prsc->format),
82 prsc->width0, prsc->height0,
83 i, tiling_descriptions[slice->tiling],
84 u_minify(prsc->width0, i),
85 u_minify(prsc->height0, i),
86 u_minify(prsc->depth0, i),
87 level_width,
88 level_height,
89 level_depth,
90 slice->stride,
91 rsc->bo->offset + slice->offset);
92 }
93 }
94
95 static bool
v3d_resource_bo_alloc(struct v3d_resource * rsc)96 v3d_resource_bo_alloc(struct v3d_resource *rsc)
97 {
98 struct pipe_resource *prsc = &rsc->base;
99 struct pipe_screen *pscreen = prsc->screen;
100 struct v3d_bo *bo;
101
102 bo = v3d_bo_alloc(v3d_screen(pscreen), rsc->size, "resource");
103 if (bo) {
104 v3d_bo_unreference(&rsc->bo);
105 rsc->bo = bo;
106 rsc->serial_id++;
107 v3d_debug_resource_layout(rsc, "alloc");
108 return true;
109 } else {
110 return false;
111 }
112 }
113
114 static void
v3d_resource_transfer_unmap(struct pipe_context * pctx,struct pipe_transfer * ptrans)115 v3d_resource_transfer_unmap(struct pipe_context *pctx,
116 struct pipe_transfer *ptrans)
117 {
118 struct v3d_context *v3d = v3d_context(pctx);
119 struct v3d_transfer *trans = v3d_transfer(ptrans);
120
121 if (trans->map) {
122 struct v3d_resource *rsc = v3d_resource(ptrans->resource);
123 struct v3d_resource_slice *slice = &rsc->slices[ptrans->level];
124
125 if (ptrans->usage & PIPE_MAP_WRITE) {
126 for (int z = 0; z < ptrans->box.depth; z++) {
127 void *dst = rsc->bo->map +
128 v3d_layer_offset(&rsc->base,
129 ptrans->level,
130 ptrans->box.z + z);
131 v3d_store_tiled_image(dst,
132 slice->stride,
133 (trans->map +
134 ptrans->stride *
135 ptrans->box.height * z),
136 ptrans->stride,
137 slice->tiling, rsc->cpp,
138 slice->padded_height,
139 &ptrans->box);
140 }
141 }
142 free(trans->map);
143 }
144
145 pipe_resource_reference(&ptrans->resource, NULL);
146 slab_free(&v3d->transfer_pool, ptrans);
147 }
148
149 static void
rebind_sampler_views(struct v3d_context * v3d,struct v3d_resource * rsc)150 rebind_sampler_views(struct v3d_context *v3d,
151 struct v3d_resource *rsc)
152 {
153 for (int st = 0; st < PIPE_SHADER_TYPES; st++) {
154 struct v3d_texture_stateobj *tex = v3d->tex + st;
155
156 for (unsigned i = 0; i < tex->num_textures; i++) {
157 struct pipe_sampler_view *psview = tex->textures[i];
158
159 if (psview->texture != &rsc->base)
160 continue;
161
162 struct v3d_sampler_view *sview =
163 v3d_sampler_view(psview);
164
165 v3d_create_texture_shader_state_bo(v3d, sview);
166
167 v3d_flag_dirty_sampler_state(v3d, st);
168 }
169 }
170 }
171
172 static void
v3d_map_usage_prep(struct pipe_context * pctx,struct pipe_resource * prsc,unsigned usage)173 v3d_map_usage_prep(struct pipe_context *pctx,
174 struct pipe_resource *prsc,
175 unsigned usage)
176 {
177 struct v3d_context *v3d = v3d_context(pctx);
178 struct v3d_resource *rsc = v3d_resource(prsc);
179
180 if (usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE) {
181 if (v3d_resource_bo_alloc(rsc)) {
182 /* If it might be bound as one of our vertex buffers
183 * or UBOs, make sure we re-emit vertex buffer state
184 * or uniforms.
185 */
186 if (prsc->bind & PIPE_BIND_VERTEX_BUFFER)
187 v3d->dirty |= V3D_DIRTY_VTXBUF;
188 if (prsc->bind & PIPE_BIND_CONSTANT_BUFFER)
189 v3d->dirty |= V3D_DIRTY_CONSTBUF;
190 /* Since we are changing the texture BO we need to
191 * update any bound samplers to point to the new
192 * BO. Notice we can have samplers that are not
193 * currently bound to the state that won't be
194 * updated. These will be fixed when they are bound in
195 * v3d_set_sampler_views.
196 */
197 if (prsc->bind & PIPE_BIND_SAMPLER_VIEW)
198 rebind_sampler_views(v3d, rsc);
199 } else {
200 /* If we failed to reallocate, flush users so that we
201 * don't violate any syncing requirements.
202 */
203 v3d_flush_jobs_reading_resource(v3d, prsc,
204 V3D_FLUSH_DEFAULT,
205 false);
206 }
207 } else if (!(usage & PIPE_MAP_UNSYNCHRONIZED)) {
208 /* If we're writing and the buffer is being used by the CL, we
209 * have to flush the CL first. If we're only reading, we need
210 * to flush if the CL has written our buffer.
211 */
212 if (usage & PIPE_MAP_WRITE) {
213 v3d_flush_jobs_reading_resource(v3d, prsc,
214 V3D_FLUSH_ALWAYS,
215 false);
216 } else {
217 v3d_flush_jobs_writing_resource(v3d, prsc,
218 V3D_FLUSH_ALWAYS,
219 false);
220 }
221 }
222
223 if (usage & PIPE_MAP_WRITE) {
224 rsc->writes++;
225 rsc->initialized_buffers = ~0;
226 }
227 }
228
229 static void *
v3d_resource_transfer_map(struct pipe_context * pctx,struct pipe_resource * prsc,unsigned level,unsigned usage,const struct pipe_box * box,struct pipe_transfer ** pptrans)230 v3d_resource_transfer_map(struct pipe_context *pctx,
231 struct pipe_resource *prsc,
232 unsigned level, unsigned usage,
233 const struct pipe_box *box,
234 struct pipe_transfer **pptrans)
235 {
236 struct v3d_context *v3d = v3d_context(pctx);
237 struct v3d_resource *rsc = v3d_resource(prsc);
238 struct v3d_transfer *trans;
239 struct pipe_transfer *ptrans;
240 enum pipe_format format = prsc->format;
241 char *buf;
242
243 /* MSAA maps should have been handled by u_transfer_helper. */
244 assert(prsc->nr_samples <= 1);
245
246 /* Upgrade DISCARD_RANGE to WHOLE_RESOURCE if the whole resource is
247 * being mapped.
248 */
249 if ((usage & PIPE_MAP_DISCARD_RANGE) &&
250 !(usage & PIPE_MAP_UNSYNCHRONIZED) &&
251 !(prsc->flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) &&
252 prsc->last_level == 0 &&
253 prsc->width0 == box->width &&
254 prsc->height0 == box->height &&
255 prsc->depth0 == box->depth &&
256 prsc->array_size == 1 &&
257 rsc->bo->private) {
258 usage |= PIPE_MAP_DISCARD_WHOLE_RESOURCE;
259 }
260
261 v3d_map_usage_prep(pctx, prsc, usage);
262
263 trans = slab_zalloc(&v3d->transfer_pool);
264 if (!trans)
265 return NULL;
266
267 /* XXX: Handle DONTBLOCK, DISCARD_RANGE, PERSISTENT, COHERENT. */
268
269 ptrans = &trans->base;
270
271 pipe_resource_reference(&ptrans->resource, prsc);
272 ptrans->level = level;
273 ptrans->usage = usage;
274 ptrans->box = *box;
275
276 /* Note that the current kernel implementation is synchronous, so no
277 * need to do syncing stuff here yet.
278 */
279
280 if (usage & PIPE_MAP_UNSYNCHRONIZED)
281 buf = v3d_bo_map_unsynchronized(rsc->bo);
282 else
283 buf = v3d_bo_map(rsc->bo);
284 if (!buf) {
285 fprintf(stderr, "Failed to map bo\n");
286 goto fail;
287 }
288
289 *pptrans = ptrans;
290
291 /* Our load/store routines work on entire compressed blocks. */
292 u_box_pixels_to_blocks(&ptrans->box, &ptrans->box, format);
293
294 struct v3d_resource_slice *slice = &rsc->slices[level];
295 if (rsc->tiled) {
296 /* No direct mappings of tiled, since we need to manually
297 * tile/untile.
298 */
299 if (usage & PIPE_MAP_DIRECTLY)
300 return NULL;
301
302 ptrans->stride = ptrans->box.width * rsc->cpp;
303 ptrans->layer_stride = ptrans->stride * ptrans->box.height;
304
305 trans->map = malloc(ptrans->layer_stride * ptrans->box.depth);
306
307 if (usage & PIPE_MAP_READ) {
308 for (int z = 0; z < ptrans->box.depth; z++) {
309 void *src = rsc->bo->map +
310 v3d_layer_offset(&rsc->base,
311 ptrans->level,
312 ptrans->box.z + z);
313 v3d_load_tiled_image((trans->map +
314 ptrans->stride *
315 ptrans->box.height * z),
316 ptrans->stride,
317 src,
318 slice->stride,
319 slice->tiling, rsc->cpp,
320 slice->padded_height,
321 &ptrans->box);
322 }
323 }
324 return trans->map;
325 } else {
326 ptrans->stride = slice->stride;
327 ptrans->layer_stride = rsc->cube_map_stride;
328
329 return buf + slice->offset +
330 ptrans->box.y * ptrans->stride +
331 ptrans->box.x * rsc->cpp +
332 ptrans->box.z * rsc->cube_map_stride;
333 }
334
335
336 fail:
337 v3d_resource_transfer_unmap(pctx, ptrans);
338 return NULL;
339 }
340
341 static void
v3d_texture_subdata(struct pipe_context * pctx,struct pipe_resource * prsc,unsigned level,unsigned usage,const struct pipe_box * box,const void * data,unsigned stride,unsigned layer_stride)342 v3d_texture_subdata(struct pipe_context *pctx,
343 struct pipe_resource *prsc,
344 unsigned level,
345 unsigned usage,
346 const struct pipe_box *box,
347 const void *data,
348 unsigned stride,
349 unsigned layer_stride)
350 {
351 struct v3d_resource *rsc = v3d_resource(prsc);
352 struct v3d_resource_slice *slice = &rsc->slices[level];
353
354 /* For a direct mapping, we can just take the u_transfer path. */
355 if (!rsc->tiled) {
356 return u_default_texture_subdata(pctx, prsc, level, usage, box,
357 data, stride, layer_stride);
358 }
359
360 /* Otherwise, map and store the texture data directly into the tiled
361 * texture. Note that gallium's texture_subdata may be called with
362 * obvious usage flags missing!
363 */
364 v3d_map_usage_prep(pctx, prsc, usage | (PIPE_MAP_WRITE |
365 PIPE_MAP_DISCARD_RANGE));
366
367 void *buf;
368 if (usage & PIPE_MAP_UNSYNCHRONIZED)
369 buf = v3d_bo_map_unsynchronized(rsc->bo);
370 else
371 buf = v3d_bo_map(rsc->bo);
372
373 for (int i = 0; i < box->depth; i++) {
374 v3d_store_tiled_image(buf +
375 v3d_layer_offset(&rsc->base,
376 level,
377 box->z + i),
378 slice->stride,
379 (void *)data + layer_stride * i,
380 stride,
381 slice->tiling, rsc->cpp, slice->padded_height,
382 box);
383 }
384 }
385
386 static void
v3d_resource_destroy(struct pipe_screen * pscreen,struct pipe_resource * prsc)387 v3d_resource_destroy(struct pipe_screen *pscreen,
388 struct pipe_resource *prsc)
389 {
390 struct v3d_screen *screen = v3d_screen(pscreen);
391 struct v3d_resource *rsc = v3d_resource(prsc);
392
393 if (rsc->scanout)
394 renderonly_scanout_destroy(rsc->scanout, screen->ro);
395
396 v3d_bo_unreference(&rsc->bo);
397 free(rsc);
398 }
399
400 static uint64_t
v3d_resource_modifier(struct v3d_resource * rsc)401 v3d_resource_modifier(struct v3d_resource *rsc)
402 {
403 if (rsc->tiled) {
404 /* A shared tiled buffer should always be allocated as UIF,
405 * not UBLINEAR or LT.
406 */
407 assert(rsc->slices[0].tiling == V3D_TILING_UIF_XOR ||
408 rsc->slices[0].tiling == V3D_TILING_UIF_NO_XOR);
409 return DRM_FORMAT_MOD_BROADCOM_UIF;
410 } else {
411 return DRM_FORMAT_MOD_LINEAR;
412 }
413 }
414
415 static bool
v3d_resource_get_handle(struct pipe_screen * pscreen,struct pipe_context * pctx,struct pipe_resource * prsc,struct winsys_handle * whandle,unsigned usage)416 v3d_resource_get_handle(struct pipe_screen *pscreen,
417 struct pipe_context *pctx,
418 struct pipe_resource *prsc,
419 struct winsys_handle *whandle,
420 unsigned usage)
421 {
422 struct v3d_screen *screen = v3d_screen(pscreen);
423 struct v3d_resource *rsc = v3d_resource(prsc);
424 struct v3d_bo *bo = rsc->bo;
425
426 whandle->stride = rsc->slices[0].stride;
427 whandle->offset = 0;
428 whandle->modifier = v3d_resource_modifier(rsc);
429
430 /* If we're passing some reference to our BO out to some other part of
431 * the system, then we can't do any optimizations about only us being
432 * the ones seeing it (like BO caching).
433 */
434 bo->private = false;
435
436 switch (whandle->type) {
437 case WINSYS_HANDLE_TYPE_SHARED:
438 return v3d_bo_flink(bo, &whandle->handle);
439 case WINSYS_HANDLE_TYPE_KMS:
440 if (screen->ro) {
441 if (renderonly_get_handle(rsc->scanout, whandle)) {
442 whandle->stride = rsc->slices[0].stride;
443 return true;
444 }
445 return false;
446 }
447 whandle->handle = bo->handle;
448 return true;
449 case WINSYS_HANDLE_TYPE_FD:
450 whandle->handle = v3d_bo_get_dmabuf(bo);
451 return whandle->handle != -1;
452 }
453
454 return false;
455 }
456
457 static bool
v3d_resource_get_param(struct pipe_screen * pscreen,struct pipe_context * pctx,struct pipe_resource * prsc,unsigned plane,unsigned layer,unsigned level,enum pipe_resource_param param,unsigned usage,uint64_t * value)458 v3d_resource_get_param(struct pipe_screen *pscreen,
459 struct pipe_context *pctx, struct pipe_resource *prsc,
460 unsigned plane, unsigned layer, unsigned level,
461 enum pipe_resource_param param,
462 unsigned usage, uint64_t *value)
463 {
464 struct v3d_resource *rsc = v3d_resource(prsc);
465
466 switch (param) {
467 case PIPE_RESOURCE_PARAM_STRIDE:
468 *value = rsc->slices[level].stride;
469 return true;
470 case PIPE_RESOURCE_PARAM_OFFSET:
471 *value = 0;
472 return true;
473 case PIPE_RESOURCE_PARAM_MODIFIER:
474 *value = v3d_resource_modifier(rsc);
475 return true;
476 default:
477 return false;
478 }
479 }
480
481 #define PAGE_UB_ROWS (V3D_UIFCFG_PAGE_SIZE / V3D_UIFBLOCK_ROW_SIZE)
482 #define PAGE_UB_ROWS_TIMES_1_5 ((PAGE_UB_ROWS * 3) >> 1)
483 #define PAGE_CACHE_UB_ROWS (V3D_PAGE_CACHE_SIZE / V3D_UIFBLOCK_ROW_SIZE)
484 #define PAGE_CACHE_MINUS_1_5_UB_ROWS (PAGE_CACHE_UB_ROWS - PAGE_UB_ROWS_TIMES_1_5)
485
486 /**
487 * Computes the HW's UIFblock padding for a given height/cpp.
488 *
489 * The goal of the padding is to keep pages of the same color (bank number) at
490 * least half a page away from each other vertically when crossing between
491 * between columns of UIF blocks.
492 */
493 static uint32_t
v3d_get_ub_pad(struct v3d_resource * rsc,uint32_t height)494 v3d_get_ub_pad(struct v3d_resource *rsc, uint32_t height)
495 {
496 uint32_t utile_h = v3d_utile_height(rsc->cpp);
497 uint32_t uif_block_h = utile_h * 2;
498 uint32_t height_ub = height / uif_block_h;
499
500 uint32_t height_offset_in_pc = height_ub % PAGE_CACHE_UB_ROWS;
501
502 /* For the perfectly-aligned-for-UIF-XOR case, don't add any pad. */
503 if (height_offset_in_pc == 0)
504 return 0;
505
506 /* Try padding up to where we're offset by at least half a page. */
507 if (height_offset_in_pc < PAGE_UB_ROWS_TIMES_1_5) {
508 /* If we fit entirely in the page cache, don't pad. */
509 if (height_ub < PAGE_CACHE_UB_ROWS)
510 return 0;
511 else
512 return PAGE_UB_ROWS_TIMES_1_5 - height_offset_in_pc;
513 }
514
515 /* If we're close to being aligned to page cache size, then round up
516 * and rely on XOR.
517 */
518 if (height_offset_in_pc > PAGE_CACHE_MINUS_1_5_UB_ROWS)
519 return PAGE_CACHE_UB_ROWS - height_offset_in_pc;
520
521 /* Otherwise, we're far enough away (top and bottom) to not need any
522 * padding.
523 */
524 return 0;
525 }
526
527 static void
v3d_setup_slices(struct v3d_resource * rsc,uint32_t winsys_stride,bool uif_top)528 v3d_setup_slices(struct v3d_resource *rsc, uint32_t winsys_stride,
529 bool uif_top)
530 {
531 struct pipe_resource *prsc = &rsc->base;
532 uint32_t width = prsc->width0;
533 uint32_t height = prsc->height0;
534 uint32_t depth = prsc->depth0;
535 /* Note that power-of-two padding is based on level 1. These are not
536 * equivalent to just util_next_power_of_two(dimension), because at a
537 * level 0 dimension of 9, the level 1 power-of-two padded value is 4,
538 * not 8.
539 */
540 uint32_t pot_width = 2 * util_next_power_of_two(u_minify(width, 1));
541 uint32_t pot_height = 2 * util_next_power_of_two(u_minify(height, 1));
542 uint32_t pot_depth = 2 * util_next_power_of_two(u_minify(depth, 1));
543 uint32_t offset = 0;
544 uint32_t utile_w = v3d_utile_width(rsc->cpp);
545 uint32_t utile_h = v3d_utile_height(rsc->cpp);
546 uint32_t uif_block_w = utile_w * 2;
547 uint32_t uif_block_h = utile_h * 2;
548 uint32_t block_width = util_format_get_blockwidth(prsc->format);
549 uint32_t block_height = util_format_get_blockheight(prsc->format);
550 bool msaa = prsc->nr_samples > 1;
551
552 /* MSAA textures/renderbuffers are always laid out as single-level
553 * UIF.
554 */
555 uif_top |= msaa;
556
557 /* Check some easy mistakes to make in a resource_create() call that
558 * will break our setup.
559 */
560 assert(prsc->array_size != 0);
561 assert(prsc->depth0 != 0);
562
563 for (int i = prsc->last_level; i >= 0; i--) {
564 struct v3d_resource_slice *slice = &rsc->slices[i];
565
566 uint32_t level_width, level_height, level_depth;
567 if (i < 2) {
568 level_width = u_minify(width, i);
569 level_height = u_minify(height, i);
570 } else {
571 level_width = u_minify(pot_width, i);
572 level_height = u_minify(pot_height, i);
573 }
574 if (i < 1)
575 level_depth = u_minify(depth, i);
576 else
577 level_depth = u_minify(pot_depth, i);
578
579 if (msaa) {
580 level_width *= 2;
581 level_height *= 2;
582 }
583
584 level_width = DIV_ROUND_UP(level_width, block_width);
585 level_height = DIV_ROUND_UP(level_height, block_height);
586
587 if (!rsc->tiled) {
588 slice->tiling = V3D_TILING_RASTER;
589 if (prsc->target == PIPE_TEXTURE_1D)
590 level_width = align(level_width, 64 / rsc->cpp);
591 } else {
592 if ((i != 0 || !uif_top) &&
593 (level_width <= utile_w ||
594 level_height <= utile_h)) {
595 slice->tiling = V3D_TILING_LINEARTILE;
596 level_width = align(level_width, utile_w);
597 level_height = align(level_height, utile_h);
598 } else if ((i != 0 || !uif_top) &&
599 level_width <= uif_block_w) {
600 slice->tiling = V3D_TILING_UBLINEAR_1_COLUMN;
601 level_width = align(level_width, uif_block_w);
602 level_height = align(level_height, uif_block_h);
603 } else if ((i != 0 || !uif_top) &&
604 level_width <= 2 * uif_block_w) {
605 slice->tiling = V3D_TILING_UBLINEAR_2_COLUMN;
606 level_width = align(level_width, 2 * uif_block_w);
607 level_height = align(level_height, uif_block_h);
608 } else {
609 /* We align the width to a 4-block column of
610 * UIF blocks, but we only align height to UIF
611 * blocks.
612 */
613 level_width = align(level_width,
614 4 * uif_block_w);
615 level_height = align(level_height,
616 uif_block_h);
617
618 slice->ub_pad = v3d_get_ub_pad(rsc,
619 level_height);
620 level_height += slice->ub_pad * uif_block_h;
621
622 /* If the padding set us to to be aligned to
623 * the page cache size, then the HW will use
624 * the XOR bit on odd columns to get us
625 * perfectly misaligned
626 */
627 if ((level_height / uif_block_h) %
628 (V3D_PAGE_CACHE_SIZE /
629 V3D_UIFBLOCK_ROW_SIZE) == 0) {
630 slice->tiling = V3D_TILING_UIF_XOR;
631 } else {
632 slice->tiling = V3D_TILING_UIF_NO_XOR;
633 }
634 }
635 }
636
637 slice->offset = offset;
638 if (winsys_stride)
639 slice->stride = winsys_stride;
640 else
641 slice->stride = level_width * rsc->cpp;
642 slice->padded_height = level_height;
643 slice->size = level_height * slice->stride;
644
645 uint32_t slice_total_size = slice->size * level_depth;
646
647 /* The HW aligns level 1's base to a page if any of level 1 or
648 * below could be UIF XOR. The lower levels then inherit the
649 * alignment for as long as necessary, thanks to being power of
650 * two aligned.
651 */
652 if (i == 1 &&
653 level_width > 4 * uif_block_w &&
654 level_height > PAGE_CACHE_MINUS_1_5_UB_ROWS * uif_block_h) {
655 slice_total_size = align(slice_total_size,
656 V3D_UIFCFG_PAGE_SIZE);
657 }
658
659 offset += slice_total_size;
660
661 }
662 rsc->size = offset;
663
664 /* UIF/UBLINEAR levels need to be aligned to UIF-blocks, and LT only
665 * needs to be aligned to utile boundaries. Since tiles are laid out
666 * from small to big in memory, we need to align the later UIF slices
667 * to UIF blocks, if they were preceded by non-UIF-block-aligned LT
668 * slices.
669 *
670 * We additionally align to 4k, which improves UIF XOR performance.
671 */
672 uint32_t page_align_offset = (align(rsc->slices[0].offset, 4096) -
673 rsc->slices[0].offset);
674 if (page_align_offset) {
675 rsc->size += page_align_offset;
676 for (int i = 0; i <= prsc->last_level; i++)
677 rsc->slices[i].offset += page_align_offset;
678 }
679
680 /* Arrays and cube textures have a stride which is the distance from
681 * one full mipmap tree to the next (64b aligned). For 3D textures,
682 * we need to program the stride between slices of miplevel 0.
683 */
684 if (prsc->target != PIPE_TEXTURE_3D) {
685 rsc->cube_map_stride = align(rsc->slices[0].offset +
686 rsc->slices[0].size, 64);
687 rsc->size += rsc->cube_map_stride * (prsc->array_size - 1);
688 } else {
689 rsc->cube_map_stride = rsc->slices[0].size;
690 }
691 }
692
693 uint32_t
v3d_layer_offset(struct pipe_resource * prsc,uint32_t level,uint32_t layer)694 v3d_layer_offset(struct pipe_resource *prsc, uint32_t level, uint32_t layer)
695 {
696 struct v3d_resource *rsc = v3d_resource(prsc);
697 struct v3d_resource_slice *slice = &rsc->slices[level];
698
699 if (prsc->target == PIPE_TEXTURE_3D)
700 return slice->offset + layer * slice->size;
701 else
702 return slice->offset + layer * rsc->cube_map_stride;
703 }
704
705 static struct v3d_resource *
v3d_resource_setup(struct pipe_screen * pscreen,const struct pipe_resource * tmpl)706 v3d_resource_setup(struct pipe_screen *pscreen,
707 const struct pipe_resource *tmpl)
708 {
709 struct v3d_screen *screen = v3d_screen(pscreen);
710 struct v3d_resource *rsc = CALLOC_STRUCT(v3d_resource);
711 if (!rsc)
712 return NULL;
713 struct pipe_resource *prsc = &rsc->base;
714
715 *prsc = *tmpl;
716
717 pipe_reference_init(&prsc->reference, 1);
718 prsc->screen = pscreen;
719
720 if (prsc->nr_samples <= 1 ||
721 screen->devinfo.ver >= 40 ||
722 util_format_is_depth_or_stencil(prsc->format)) {
723 rsc->cpp = util_format_get_blocksize(prsc->format);
724 if (screen->devinfo.ver < 40 && prsc->nr_samples > 1)
725 rsc->cpp *= prsc->nr_samples;
726 } else {
727 assert(v3d_rt_format_supported(&screen->devinfo, prsc->format));
728 uint32_t output_image_format =
729 v3d_get_rt_format(&screen->devinfo, prsc->format);
730 uint32_t internal_type;
731 uint32_t internal_bpp;
732 v3d_get_internal_type_bpp_for_output_format(&screen->devinfo,
733 output_image_format,
734 &internal_type,
735 &internal_bpp);
736 switch (internal_bpp) {
737 case V3D_INTERNAL_BPP_32:
738 rsc->cpp = 4;
739 break;
740 case V3D_INTERNAL_BPP_64:
741 rsc->cpp = 8;
742 break;
743 case V3D_INTERNAL_BPP_128:
744 rsc->cpp = 16;
745 break;
746 }
747 }
748
749 rsc->serial_id++;
750
751 assert(rsc->cpp);
752
753 return rsc;
754 }
755
756 static struct pipe_resource *
v3d_resource_create_with_modifiers(struct pipe_screen * pscreen,const struct pipe_resource * tmpl,const uint64_t * modifiers,int count)757 v3d_resource_create_with_modifiers(struct pipe_screen *pscreen,
758 const struct pipe_resource *tmpl,
759 const uint64_t *modifiers,
760 int count)
761 {
762 struct v3d_screen *screen = v3d_screen(pscreen);
763
764 bool linear_ok = drm_find_modifier(DRM_FORMAT_MOD_LINEAR, modifiers, count);
765 struct v3d_resource *rsc = v3d_resource_setup(pscreen, tmpl);
766 struct pipe_resource *prsc = &rsc->base;
767 /* Use a tiled layout if we can, for better 3D performance. */
768 bool should_tile = true;
769
770 assert(tmpl->target != PIPE_BUFFER ||
771 (tmpl->format == PIPE_FORMAT_NONE ||
772 util_format_get_blocksize(tmpl->format) == 1));
773
774 /* VBOs/PBOs/Texture Buffer Objects are untiled (and 1 height). */
775 if (tmpl->target == PIPE_BUFFER)
776 should_tile = false;
777
778 /* Cursors are always linear, and the user can request linear as well.
779 */
780 if (tmpl->bind & (PIPE_BIND_LINEAR | PIPE_BIND_CURSOR))
781 should_tile = false;
782
783 /* 1D and 1D_ARRAY textures are always raster-order. */
784 if (tmpl->target == PIPE_TEXTURE_1D ||
785 tmpl->target == PIPE_TEXTURE_1D_ARRAY)
786 should_tile = false;
787
788 /* Scanout BOs for simulator need to be linear for interaction with
789 * i965.
790 */
791 if (using_v3d_simulator &&
792 tmpl->bind & (PIPE_BIND_SHARED | PIPE_BIND_SCANOUT))
793 should_tile = false;
794
795 /* If using the old-school SCANOUT flag, we don't know what the screen
796 * might support other than linear. Just force linear.
797 */
798 if (tmpl->bind & PIPE_BIND_SCANOUT)
799 should_tile = false;
800
801 /* No user-specified modifier; determine our own. */
802 if (count == 1 && modifiers[0] == DRM_FORMAT_MOD_INVALID) {
803 linear_ok = true;
804 rsc->tiled = should_tile;
805 } else if (should_tile &&
806 drm_find_modifier(DRM_FORMAT_MOD_BROADCOM_UIF,
807 modifiers, count)) {
808 rsc->tiled = true;
809 } else if (linear_ok) {
810 rsc->tiled = false;
811 } else {
812 fprintf(stderr, "Unsupported modifier requested\n");
813 goto fail;
814 }
815
816 rsc->internal_format = prsc->format;
817
818 v3d_setup_slices(rsc, 0, tmpl->bind & PIPE_BIND_SHARED);
819
820 if (screen->ro && (tmpl->bind & PIPE_BIND_SCANOUT)) {
821 struct winsys_handle handle;
822 struct pipe_resource scanout_tmpl = {
823 .target = prsc->target,
824 .format = PIPE_FORMAT_RGBA8888_UNORM,
825 .width0 = 1024, /* one page */
826 .height0 = align(rsc->size, 4096) / 4096,
827 .depth0 = 1,
828 .array_size = 1,
829 };
830
831 rsc->scanout =
832 renderonly_scanout_for_resource(&scanout_tmpl,
833 screen->ro,
834 &handle);
835
836 if (!rsc->scanout) {
837 fprintf(stderr, "Failed to create scanout resource\n");
838 goto fail;
839 }
840 assert(handle.type == WINSYS_HANDLE_TYPE_FD);
841 rsc->bo = v3d_bo_open_dmabuf(screen, handle.handle);
842 close(handle.handle);
843
844 if (!rsc->bo)
845 goto fail;
846
847 v3d_debug_resource_layout(rsc, "renderonly");
848
849 return prsc;
850 } else {
851 if (!v3d_resource_bo_alloc(rsc))
852 goto fail;
853 }
854
855 return prsc;
856 fail:
857 v3d_resource_destroy(pscreen, prsc);
858 return NULL;
859 }
860
861 struct pipe_resource *
v3d_resource_create(struct pipe_screen * pscreen,const struct pipe_resource * tmpl)862 v3d_resource_create(struct pipe_screen *pscreen,
863 const struct pipe_resource *tmpl)
864 {
865 const uint64_t mod = DRM_FORMAT_MOD_INVALID;
866 return v3d_resource_create_with_modifiers(pscreen, tmpl, &mod, 1);
867 }
868
869 static struct pipe_resource *
v3d_resource_from_handle(struct pipe_screen * pscreen,const struct pipe_resource * tmpl,struct winsys_handle * whandle,unsigned usage)870 v3d_resource_from_handle(struct pipe_screen *pscreen,
871 const struct pipe_resource *tmpl,
872 struct winsys_handle *whandle,
873 unsigned usage)
874 {
875 struct v3d_screen *screen = v3d_screen(pscreen);
876 struct v3d_resource *rsc = v3d_resource_setup(pscreen, tmpl);
877 struct pipe_resource *prsc = &rsc->base;
878 struct v3d_resource_slice *slice = &rsc->slices[0];
879
880 if (!rsc)
881 return NULL;
882
883 switch (whandle->modifier) {
884 case DRM_FORMAT_MOD_LINEAR:
885 rsc->tiled = false;
886 break;
887 case DRM_FORMAT_MOD_BROADCOM_UIF:
888 rsc->tiled = true;
889 break;
890 case DRM_FORMAT_MOD_INVALID:
891 rsc->tiled = screen->ro == NULL;
892 break;
893 default:
894 switch(fourcc_mod_broadcom_mod(whandle->modifier)) {
895 case DRM_FORMAT_MOD_BROADCOM_SAND128:
896 rsc->tiled = false;
897 rsc->sand_col128_stride =
898 fourcc_mod_broadcom_param(whandle->modifier);
899 break;
900 default:
901 fprintf(stderr,
902 "Attempt to import unsupported modifier 0x%llx\n",
903 (long long)whandle->modifier);
904 goto fail;
905 }
906 }
907
908 switch (whandle->type) {
909 case WINSYS_HANDLE_TYPE_SHARED:
910 rsc->bo = v3d_bo_open_name(screen, whandle->handle);
911 break;
912 case WINSYS_HANDLE_TYPE_FD:
913 rsc->bo = v3d_bo_open_dmabuf(screen, whandle->handle);
914 break;
915 default:
916 fprintf(stderr,
917 "Attempt to import unsupported handle type %d\n",
918 whandle->type);
919 goto fail;
920 }
921
922 if (!rsc->bo)
923 goto fail;
924
925 rsc->internal_format = prsc->format;
926
927 v3d_setup_slices(rsc, whandle->stride, true);
928 v3d_debug_resource_layout(rsc, "import");
929
930 if (whandle->offset != 0) {
931 if (rsc->tiled) {
932 fprintf(stderr,
933 "Attempt to import unsupported winsys offset %u\n",
934 whandle->offset);
935 goto fail;
936 }
937 rsc->slices[0].offset += whandle->offset;
938
939 if (rsc->slices[0].offset + rsc->slices[0].size >
940 rsc->bo->size) {
941 fprintf(stderr, "Attempt to import "
942 "with overflowing offset (%d + %d > %d)\n",
943 whandle->offset,
944 rsc->slices[0].size,
945 rsc->bo->size);
946 goto fail;
947 }
948 }
949
950 if (screen->ro) {
951 /* Make sure that renderonly has a handle to our buffer in the
952 * display's fd, so that a later renderonly_get_handle()
953 * returns correct handles or GEM names.
954 */
955 rsc->scanout =
956 renderonly_create_gpu_import_for_resource(prsc,
957 screen->ro,
958 NULL);
959 }
960
961 if (rsc->tiled && whandle->stride != slice->stride) {
962 static bool warned = false;
963 if (!warned) {
964 warned = true;
965 fprintf(stderr,
966 "Attempting to import %dx%d %s with "
967 "unsupported stride %d instead of %d\n",
968 prsc->width0, prsc->height0,
969 util_format_short_name(prsc->format),
970 whandle->stride,
971 slice->stride);
972 }
973 goto fail;
974 } else if (!rsc->tiled) {
975 slice->stride = whandle->stride;
976 }
977
978 return prsc;
979
980 fail:
981 v3d_resource_destroy(pscreen, prsc);
982 return NULL;
983 }
984
985 void
v3d_update_shadow_texture(struct pipe_context * pctx,struct pipe_sampler_view * pview)986 v3d_update_shadow_texture(struct pipe_context *pctx,
987 struct pipe_sampler_view *pview)
988 {
989 struct v3d_context *v3d = v3d_context(pctx);
990 struct v3d_sampler_view *view = v3d_sampler_view(pview);
991 struct v3d_resource *shadow = v3d_resource(view->texture);
992 struct v3d_resource *orig = v3d_resource(pview->texture);
993
994 assert(view->texture != pview->texture);
995
996 if (shadow->writes == orig->writes && orig->bo->private)
997 return;
998
999 perf_debug("Updating %dx%d@%d shadow for linear texture\n",
1000 orig->base.width0, orig->base.height0,
1001 pview->u.tex.first_level);
1002
1003 for (int i = 0; i <= shadow->base.last_level; i++) {
1004 unsigned width = u_minify(shadow->base.width0, i);
1005 unsigned height = u_minify(shadow->base.height0, i);
1006 struct pipe_blit_info info = {
1007 .dst = {
1008 .resource = &shadow->base,
1009 .level = i,
1010 .box = {
1011 .x = 0,
1012 .y = 0,
1013 .z = 0,
1014 .width = width,
1015 .height = height,
1016 .depth = 1,
1017 },
1018 .format = shadow->base.format,
1019 },
1020 .src = {
1021 .resource = &orig->base,
1022 .level = pview->u.tex.first_level + i,
1023 .box = {
1024 .x = 0,
1025 .y = 0,
1026 .z = 0,
1027 .width = width,
1028 .height = height,
1029 .depth = 1,
1030 },
1031 .format = orig->base.format,
1032 },
1033 .mask = util_format_get_mask(orig->base.format),
1034 };
1035 pctx->blit(pctx, &info);
1036 }
1037
1038 shadow->writes = orig->writes;
1039 }
1040
1041 static struct pipe_surface *
v3d_create_surface(struct pipe_context * pctx,struct pipe_resource * ptex,const struct pipe_surface * surf_tmpl)1042 v3d_create_surface(struct pipe_context *pctx,
1043 struct pipe_resource *ptex,
1044 const struct pipe_surface *surf_tmpl)
1045 {
1046 struct v3d_context *v3d = v3d_context(pctx);
1047 struct v3d_screen *screen = v3d->screen;
1048 struct v3d_surface *surface = CALLOC_STRUCT(v3d_surface);
1049 struct v3d_resource *rsc = v3d_resource(ptex);
1050
1051 if (!surface)
1052 return NULL;
1053
1054 struct pipe_surface *psurf = &surface->base;
1055 unsigned level = surf_tmpl->u.tex.level;
1056 struct v3d_resource_slice *slice = &rsc->slices[level];
1057
1058 pipe_reference_init(&psurf->reference, 1);
1059 pipe_resource_reference(&psurf->texture, ptex);
1060
1061 psurf->context = pctx;
1062 psurf->format = surf_tmpl->format;
1063 psurf->width = u_minify(ptex->width0, level);
1064 psurf->height = u_minify(ptex->height0, level);
1065 psurf->u.tex.level = level;
1066 psurf->u.tex.first_layer = surf_tmpl->u.tex.first_layer;
1067 psurf->u.tex.last_layer = surf_tmpl->u.tex.last_layer;
1068
1069 surface->offset = v3d_layer_offset(ptex, level,
1070 psurf->u.tex.first_layer);
1071 surface->tiling = slice->tiling;
1072
1073 surface->format = v3d_get_rt_format(&screen->devinfo, psurf->format);
1074
1075 const struct util_format_description *desc =
1076 util_format_description(psurf->format);
1077
1078 surface->swap_rb = (desc->swizzle[0] == PIPE_SWIZZLE_Z &&
1079 psurf->format != PIPE_FORMAT_B5G6R5_UNORM);
1080
1081 if (util_format_is_depth_or_stencil(psurf->format)) {
1082 switch (psurf->format) {
1083 case PIPE_FORMAT_Z16_UNORM:
1084 surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_16;
1085 break;
1086 case PIPE_FORMAT_Z32_FLOAT:
1087 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
1088 surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_32F;
1089 break;
1090 default:
1091 surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_24;
1092 }
1093 } else {
1094 uint32_t bpp, type;
1095 v3d_get_internal_type_bpp_for_output_format(&screen->devinfo,
1096 surface->format,
1097 &type, &bpp);
1098 surface->internal_type = type;
1099 surface->internal_bpp = bpp;
1100 }
1101
1102 if (surface->tiling == V3D_TILING_UIF_NO_XOR ||
1103 surface->tiling == V3D_TILING_UIF_XOR) {
1104 surface->padded_height_of_output_image_in_uif_blocks =
1105 (slice->padded_height /
1106 (2 * v3d_utile_height(rsc->cpp)));
1107 }
1108
1109 if (rsc->separate_stencil) {
1110 surface->separate_stencil =
1111 v3d_create_surface(pctx, &rsc->separate_stencil->base,
1112 surf_tmpl);
1113 }
1114
1115 return &surface->base;
1116 }
1117
1118 static void
v3d_surface_destroy(struct pipe_context * pctx,struct pipe_surface * psurf)1119 v3d_surface_destroy(struct pipe_context *pctx, struct pipe_surface *psurf)
1120 {
1121 struct v3d_surface *surf = v3d_surface(psurf);
1122
1123 if (surf->separate_stencil)
1124 pipe_surface_reference(&surf->separate_stencil, NULL);
1125
1126 pipe_resource_reference(&psurf->texture, NULL);
1127 FREE(psurf);
1128 }
1129
1130 static void
v3d_flush_resource(struct pipe_context * pctx,struct pipe_resource * resource)1131 v3d_flush_resource(struct pipe_context *pctx, struct pipe_resource *resource)
1132 {
1133 /* All calls to flush_resource are followed by a flush of the context,
1134 * so there's nothing to do.
1135 */
1136 }
1137
1138 static enum pipe_format
v3d_resource_get_internal_format(struct pipe_resource * prsc)1139 v3d_resource_get_internal_format(struct pipe_resource *prsc)
1140 {
1141 return v3d_resource(prsc)->internal_format;
1142 }
1143
1144 static void
v3d_resource_set_stencil(struct pipe_resource * prsc,struct pipe_resource * stencil)1145 v3d_resource_set_stencil(struct pipe_resource *prsc,
1146 struct pipe_resource *stencil)
1147 {
1148 v3d_resource(prsc)->separate_stencil = v3d_resource(stencil);
1149 }
1150
1151 static struct pipe_resource *
v3d_resource_get_stencil(struct pipe_resource * prsc)1152 v3d_resource_get_stencil(struct pipe_resource *prsc)
1153 {
1154 struct v3d_resource *rsc = v3d_resource(prsc);
1155
1156 return &rsc->separate_stencil->base;
1157 }
1158
1159 static const struct u_transfer_vtbl transfer_vtbl = {
1160 .resource_create = v3d_resource_create,
1161 .resource_destroy = v3d_resource_destroy,
1162 .transfer_map = v3d_resource_transfer_map,
1163 .transfer_unmap = v3d_resource_transfer_unmap,
1164 .transfer_flush_region = u_default_transfer_flush_region,
1165 .get_internal_format = v3d_resource_get_internal_format,
1166 .set_stencil = v3d_resource_set_stencil,
1167 .get_stencil = v3d_resource_get_stencil,
1168 };
1169
1170 void
v3d_resource_screen_init(struct pipe_screen * pscreen)1171 v3d_resource_screen_init(struct pipe_screen *pscreen)
1172 {
1173 pscreen->resource_create_with_modifiers =
1174 v3d_resource_create_with_modifiers;
1175 pscreen->resource_create = u_transfer_helper_resource_create;
1176 pscreen->resource_from_handle = v3d_resource_from_handle;
1177 pscreen->resource_get_handle = v3d_resource_get_handle;
1178 pscreen->resource_get_param = v3d_resource_get_param;
1179 pscreen->resource_destroy = u_transfer_helper_resource_destroy;
1180 pscreen->transfer_helper = u_transfer_helper_create(&transfer_vtbl,
1181 true, false,
1182 true, true,
1183 false);
1184 }
1185
1186 void
v3d_resource_context_init(struct pipe_context * pctx)1187 v3d_resource_context_init(struct pipe_context *pctx)
1188 {
1189 pctx->buffer_map = u_transfer_helper_transfer_map;
1190 pctx->texture_map = u_transfer_helper_transfer_map;
1191 pctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;
1192 pctx->buffer_unmap = u_transfer_helper_transfer_unmap;
1193 pctx->texture_unmap = u_transfer_helper_transfer_unmap;
1194 pctx->buffer_subdata = u_default_buffer_subdata;
1195 pctx->texture_subdata = v3d_texture_subdata;
1196 pctx->create_surface = v3d_create_surface;
1197 pctx->surface_destroy = v3d_surface_destroy;
1198 pctx->resource_copy_region = util_resource_copy_region;
1199 pctx->blit = v3d_blit;
1200 pctx->generate_mipmap = v3d_generate_mipmap;
1201 pctx->flush_resource = v3d_flush_resource;
1202 }
1203