1 /*
2 * Copyright © 2014-2017 Broadcom
3 * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org>
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 * IN THE SOFTWARE.
23 */
24
25 #include "pipe/p_defines.h"
26 #include "util/u_memory.h"
27 #include "util/format/u_format.h"
28 #include "util/u_inlines.h"
29 #include "util/u_resource.h"
30 #include "util/u_surface.h"
31 #include "util/u_transfer_helper.h"
32 #include "util/u_upload_mgr.h"
33 #include "util/format/u_format_zs.h"
34 #include "util/u_drm.h"
35
36 #include "drm-uapi/drm_fourcc.h"
37 #include "v3d_screen.h"
38 #include "v3d_context.h"
39 #include "v3d_resource.h"
40 /* The packets used here the same across V3D versions. */
41 #include "broadcom/cle/v3d_packet_v42_pack.h"
42
43 static void
v3d_debug_resource_layout(struct v3d_resource * rsc,const char * caller)44 v3d_debug_resource_layout(struct v3d_resource *rsc, const char *caller)
45 {
46 if (!V3D_DBG(SURFACE))
47 return;
48
49 struct pipe_resource *prsc = &rsc->base;
50
51 if (prsc->target == PIPE_BUFFER) {
52 fprintf(stderr,
53 "rsc %s %p (format %s), %dx%d buffer @0x%08x-0x%08x\n",
54 caller, rsc,
55 util_format_short_name(prsc->format),
56 prsc->width0, prsc->height0,
57 rsc->bo->offset,
58 rsc->bo->offset + rsc->bo->size - 1);
59 return;
60 }
61
62 static const char *const tiling_descriptions[] = {
63 [V3D_TILING_RASTER] = "R",
64 [V3D_TILING_LINEARTILE] = "LT",
65 [V3D_TILING_UBLINEAR_1_COLUMN] = "UB1",
66 [V3D_TILING_UBLINEAR_2_COLUMN] = "UB2",
67 [V3D_TILING_UIF_NO_XOR] = "UIF",
68 [V3D_TILING_UIF_XOR] = "UIF^",
69 };
70
71 for (int i = 0; i <= prsc->last_level; i++) {
72 struct v3d_resource_slice *slice = &rsc->slices[i];
73
74 int level_width = slice->stride / rsc->cpp;
75 int level_height = slice->padded_height;
76 int level_depth =
77 u_minify(util_next_power_of_two(prsc->depth0), i);
78
79 fprintf(stderr,
80 "rsc %s %p (format %s), %dx%d: "
81 "level %d (%s) %dx%dx%d -> %dx%dx%d, stride %d@0x%08x\n",
82 caller, rsc,
83 util_format_short_name(prsc->format),
84 prsc->width0, prsc->height0,
85 i, tiling_descriptions[slice->tiling],
86 u_minify(prsc->width0, i),
87 u_minify(prsc->height0, i),
88 u_minify(prsc->depth0, i),
89 level_width,
90 level_height,
91 level_depth,
92 slice->stride,
93 rsc->bo->offset + slice->offset);
94 }
95 }
96
97 static bool
v3d_resource_bo_alloc(struct v3d_resource * rsc)98 v3d_resource_bo_alloc(struct v3d_resource *rsc)
99 {
100 struct pipe_resource *prsc = &rsc->base;
101 struct pipe_screen *pscreen = prsc->screen;
102 struct v3d_bo *bo;
103
104 /* Buffers may be read using ldunifa, which prefetches the next
105 * 4 bytes after a read. If the buffer's size is exactly a multiple
106 * of a page size and the shader reads the last 4 bytes with ldunifa
107 * the prefetching would read out of bounds and cause an MMU error,
108 * so we allocate extra space to avoid kernel error spamming.
109 */
110 uint32_t size = rsc->size;
111 if (rsc->base.target == PIPE_BUFFER && (size % 4096 == 0))
112 size += 4;
113
114 bo = v3d_bo_alloc(v3d_screen(pscreen), size, "resource");
115 if (bo) {
116 v3d_bo_unreference(&rsc->bo);
117 rsc->bo = bo;
118 rsc->serial_id++;
119 v3d_debug_resource_layout(rsc, "alloc");
120 return true;
121 } else {
122 return false;
123 }
124 }
125
126 static void
v3d_resource_transfer_unmap(struct pipe_context * pctx,struct pipe_transfer * ptrans)127 v3d_resource_transfer_unmap(struct pipe_context *pctx,
128 struct pipe_transfer *ptrans)
129 {
130 struct v3d_context *v3d = v3d_context(pctx);
131 struct v3d_transfer *trans = v3d_transfer(ptrans);
132
133 if (trans->map) {
134 struct v3d_resource *rsc = v3d_resource(ptrans->resource);
135 struct v3d_resource_slice *slice = &rsc->slices[ptrans->level];
136
137 if (ptrans->usage & PIPE_MAP_WRITE) {
138 for (int z = 0; z < ptrans->box.depth; z++) {
139 void *dst = rsc->bo->map +
140 v3d_layer_offset(&rsc->base,
141 ptrans->level,
142 ptrans->box.z + z);
143 v3d_store_tiled_image(dst,
144 slice->stride,
145 (trans->map +
146 ptrans->stride *
147 ptrans->box.height * z),
148 ptrans->stride,
149 slice->tiling, rsc->cpp,
150 slice->padded_height,
151 &ptrans->box);
152 }
153 }
154 free(trans->map);
155 }
156
157 pipe_resource_reference(&ptrans->resource, NULL);
158 slab_free(&v3d->transfer_pool, ptrans);
159 }
160
161 static void
rebind_sampler_views(struct v3d_context * v3d,struct v3d_resource * rsc)162 rebind_sampler_views(struct v3d_context *v3d,
163 struct v3d_resource *rsc)
164 {
165 for (int st = 0; st < PIPE_SHADER_TYPES; st++) {
166 struct v3d_texture_stateobj *tex = v3d->tex + st;
167
168 for (unsigned i = 0; i < tex->num_textures; i++) {
169 struct pipe_sampler_view *psview = tex->textures[i];
170
171 if (psview->texture != &rsc->base)
172 continue;
173
174 struct v3d_sampler_view *sview =
175 v3d_sampler_view(psview);
176 struct v3d_device_info *devinfo =
177 &v3d->screen->devinfo;
178
179 v3d_X(devinfo, create_texture_shader_state_bo)(v3d, sview);
180
181 v3d_flag_dirty_sampler_state(v3d, st);
182 }
183 }
184 }
185
186 static void
v3d_map_usage_prep(struct pipe_context * pctx,struct pipe_resource * prsc,unsigned usage)187 v3d_map_usage_prep(struct pipe_context *pctx,
188 struct pipe_resource *prsc,
189 unsigned usage)
190 {
191 struct v3d_context *v3d = v3d_context(pctx);
192 struct v3d_resource *rsc = v3d_resource(prsc);
193
194 if (usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE) {
195 if (v3d_resource_bo_alloc(rsc)) {
196 /* If it might be bound as one of our vertex buffers
197 * or UBOs, make sure we re-emit vertex buffer state
198 * or uniforms.
199 */
200 if (prsc->bind & PIPE_BIND_VERTEX_BUFFER)
201 v3d->dirty |= V3D_DIRTY_VTXBUF;
202 if (prsc->bind & PIPE_BIND_CONSTANT_BUFFER)
203 v3d->dirty |= V3D_DIRTY_CONSTBUF;
204 /* Since we are changing the texture BO we need to
205 * update any bound samplers to point to the new
206 * BO. Notice we can have samplers that are not
207 * currently bound to the state that won't be
208 * updated. These will be fixed when they are bound in
209 * v3d_set_sampler_views.
210 */
211 if (prsc->bind & PIPE_BIND_SAMPLER_VIEW)
212 rebind_sampler_views(v3d, rsc);
213 } else {
214 /* If we failed to reallocate, flush users so that we
215 * don't violate any syncing requirements.
216 */
217 v3d_flush_jobs_reading_resource(v3d, prsc,
218 V3D_FLUSH_DEFAULT,
219 false);
220 }
221 } else if (!(usage & PIPE_MAP_UNSYNCHRONIZED)) {
222 /* If we're writing and the buffer is being used by the CL, we
223 * have to flush the CL first. If we're only reading, we need
224 * to flush if the CL has written our buffer.
225 */
226 if (usage & PIPE_MAP_WRITE) {
227 v3d_flush_jobs_reading_resource(v3d, prsc,
228 V3D_FLUSH_ALWAYS,
229 false);
230 } else {
231 v3d_flush_jobs_writing_resource(v3d, prsc,
232 V3D_FLUSH_ALWAYS,
233 false);
234 }
235 }
236
237 if (usage & PIPE_MAP_WRITE) {
238 rsc->writes++;
239 rsc->graphics_written = true;
240 rsc->initialized_buffers = ~0;
241 }
242 }
243
244 static void *
v3d_resource_transfer_map(struct pipe_context * pctx,struct pipe_resource * prsc,unsigned level,unsigned usage,const struct pipe_box * box,struct pipe_transfer ** pptrans)245 v3d_resource_transfer_map(struct pipe_context *pctx,
246 struct pipe_resource *prsc,
247 unsigned level, unsigned usage,
248 const struct pipe_box *box,
249 struct pipe_transfer **pptrans)
250 {
251 struct v3d_context *v3d = v3d_context(pctx);
252 struct v3d_resource *rsc = v3d_resource(prsc);
253 struct v3d_transfer *trans;
254 struct pipe_transfer *ptrans;
255 enum pipe_format format = prsc->format;
256 char *buf;
257
258 /* MSAA maps should have been handled by u_transfer_helper. */
259 assert(prsc->nr_samples <= 1);
260
261 /* Upgrade DISCARD_RANGE to WHOLE_RESOURCE if the whole resource is
262 * being mapped.
263 */
264 if ((usage & PIPE_MAP_DISCARD_RANGE) &&
265 !(usage & PIPE_MAP_UNSYNCHRONIZED) &&
266 !(prsc->flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) &&
267 prsc->last_level == 0 &&
268 prsc->width0 == box->width &&
269 prsc->height0 == box->height &&
270 prsc->depth0 == box->depth &&
271 prsc->array_size == 1 &&
272 rsc->bo->private) {
273 usage |= PIPE_MAP_DISCARD_WHOLE_RESOURCE;
274 }
275
276 v3d_map_usage_prep(pctx, prsc, usage);
277
278 trans = slab_zalloc(&v3d->transfer_pool);
279 if (!trans)
280 return NULL;
281
282 /* XXX: Handle DONTBLOCK, DISCARD_RANGE, PERSISTENT, COHERENT. */
283
284 ptrans = &trans->base;
285
286 pipe_resource_reference(&ptrans->resource, prsc);
287 ptrans->level = level;
288 ptrans->usage = usage;
289 ptrans->box = *box;
290
291 /* Note that the current kernel implementation is synchronous, so no
292 * need to do syncing stuff here yet.
293 */
294
295 if (usage & PIPE_MAP_UNSYNCHRONIZED)
296 buf = v3d_bo_map_unsynchronized(rsc->bo);
297 else
298 buf = v3d_bo_map(rsc->bo);
299 if (!buf) {
300 fprintf(stderr, "Failed to map bo\n");
301 goto fail;
302 }
303
304 *pptrans = ptrans;
305
306 /* Our load/store routines work on entire compressed blocks. */
307 u_box_pixels_to_blocks(&ptrans->box, &ptrans->box, format);
308
309 struct v3d_resource_slice *slice = &rsc->slices[level];
310 if (rsc->tiled) {
311 /* No direct mappings of tiled, since we need to manually
312 * tile/untile.
313 */
314 if (usage & PIPE_MAP_DIRECTLY)
315 return NULL;
316
317 ptrans->stride = ptrans->box.width * rsc->cpp;
318 ptrans->layer_stride = ptrans->stride * ptrans->box.height;
319
320 trans->map = malloc(ptrans->layer_stride * ptrans->box.depth);
321
322 if (usage & PIPE_MAP_READ) {
323 for (int z = 0; z < ptrans->box.depth; z++) {
324 void *src = rsc->bo->map +
325 v3d_layer_offset(&rsc->base,
326 ptrans->level,
327 ptrans->box.z + z);
328 v3d_load_tiled_image((trans->map +
329 ptrans->stride *
330 ptrans->box.height * z),
331 ptrans->stride,
332 src,
333 slice->stride,
334 slice->tiling, rsc->cpp,
335 slice->padded_height,
336 &ptrans->box);
337 }
338 }
339 return trans->map;
340 } else {
341 ptrans->stride = slice->stride;
342 ptrans->layer_stride = rsc->cube_map_stride;
343
344 return buf + slice->offset +
345 ptrans->box.y * ptrans->stride +
346 ptrans->box.x * rsc->cpp +
347 ptrans->box.z * rsc->cube_map_stride;
348 }
349
350
351 fail:
352 v3d_resource_transfer_unmap(pctx, ptrans);
353 return NULL;
354 }
355
356 static void
v3d_texture_subdata(struct pipe_context * pctx,struct pipe_resource * prsc,unsigned level,unsigned usage,const struct pipe_box * box,const void * data,unsigned stride,uintptr_t layer_stride)357 v3d_texture_subdata(struct pipe_context *pctx,
358 struct pipe_resource *prsc,
359 unsigned level,
360 unsigned usage,
361 const struct pipe_box *box,
362 const void *data,
363 unsigned stride,
364 uintptr_t layer_stride)
365 {
366 struct v3d_resource *rsc = v3d_resource(prsc);
367 struct v3d_resource_slice *slice = &rsc->slices[level];
368
369 /* For a direct mapping, we can just take the u_transfer path. */
370 if (!rsc->tiled) {
371 return u_default_texture_subdata(pctx, prsc, level, usage, box,
372 data, stride, layer_stride);
373 }
374
375 /* Otherwise, map and store the texture data directly into the tiled
376 * texture. Note that gallium's texture_subdata may be called with
377 * obvious usage flags missing!
378 */
379 v3d_map_usage_prep(pctx, prsc, usage | (PIPE_MAP_WRITE |
380 PIPE_MAP_DISCARD_RANGE));
381
382 void *buf;
383 if (usage & PIPE_MAP_UNSYNCHRONIZED)
384 buf = v3d_bo_map_unsynchronized(rsc->bo);
385 else
386 buf = v3d_bo_map(rsc->bo);
387
388 for (int i = 0; i < box->depth; i++) {
389 v3d_store_tiled_image(buf +
390 v3d_layer_offset(&rsc->base,
391 level,
392 box->z + i),
393 slice->stride,
394 (void *)data + layer_stride * i,
395 stride,
396 slice->tiling, rsc->cpp, slice->padded_height,
397 box);
398 }
399 }
400
401 static void
v3d_resource_destroy(struct pipe_screen * pscreen,struct pipe_resource * prsc)402 v3d_resource_destroy(struct pipe_screen *pscreen,
403 struct pipe_resource *prsc)
404 {
405 struct v3d_screen *screen = v3d_screen(pscreen);
406 struct v3d_resource *rsc = v3d_resource(prsc);
407
408 if (rsc->scanout)
409 renderonly_scanout_destroy(rsc->scanout, screen->ro);
410
411 v3d_bo_unreference(&rsc->bo);
412 free(rsc);
413 }
414
415 static uint64_t
v3d_resource_modifier(struct v3d_resource * rsc)416 v3d_resource_modifier(struct v3d_resource *rsc)
417 {
418 if (rsc->tiled) {
419 /* A shared tiled buffer should always be allocated as UIF,
420 * not UBLINEAR or LT.
421 */
422 assert(rsc->slices[0].tiling == V3D_TILING_UIF_XOR ||
423 rsc->slices[0].tiling == V3D_TILING_UIF_NO_XOR);
424 return DRM_FORMAT_MOD_BROADCOM_UIF;
425 } else {
426 return DRM_FORMAT_MOD_LINEAR;
427 }
428 }
429
430 static bool
v3d_resource_get_handle(struct pipe_screen * pscreen,struct pipe_context * pctx,struct pipe_resource * prsc,struct winsys_handle * whandle,unsigned usage)431 v3d_resource_get_handle(struct pipe_screen *pscreen,
432 struct pipe_context *pctx,
433 struct pipe_resource *prsc,
434 struct winsys_handle *whandle,
435 unsigned usage)
436 {
437 struct v3d_screen *screen = v3d_screen(pscreen);
438 struct v3d_resource *rsc = v3d_resource(prsc);
439 struct v3d_bo *bo = rsc->bo;
440
441 whandle->stride = rsc->slices[0].stride;
442 whandle->offset = 0;
443 whandle->modifier = v3d_resource_modifier(rsc);
444
445 /* If we're passing some reference to our BO out to some other part of
446 * the system, then we can't do any optimizations about only us being
447 * the ones seeing it (like BO caching).
448 */
449 bo->private = false;
450
451 switch (whandle->type) {
452 case WINSYS_HANDLE_TYPE_SHARED:
453 return v3d_bo_flink(bo, &whandle->handle);
454 case WINSYS_HANDLE_TYPE_KMS:
455 if (screen->ro) {
456 if (renderonly_get_handle(rsc->scanout, whandle)) {
457 whandle->stride = rsc->slices[0].stride;
458 return true;
459 }
460 return false;
461 }
462 whandle->handle = bo->handle;
463 return true;
464 case WINSYS_HANDLE_TYPE_FD:
465 whandle->handle = v3d_bo_get_dmabuf(bo);
466 return whandle->handle != -1;
467 }
468
469 return false;
470 }
471
472 static bool
v3d_resource_get_param(struct pipe_screen * pscreen,struct pipe_context * pctx,struct pipe_resource * prsc,unsigned plane,unsigned layer,unsigned level,enum pipe_resource_param param,unsigned usage,uint64_t * value)473 v3d_resource_get_param(struct pipe_screen *pscreen,
474 struct pipe_context *pctx, struct pipe_resource *prsc,
475 unsigned plane, unsigned layer, unsigned level,
476 enum pipe_resource_param param,
477 unsigned usage, uint64_t *value)
478 {
479 struct v3d_resource *rsc =
480 (struct v3d_resource *)util_resource_at_index(prsc, plane);
481
482 switch (param) {
483 case PIPE_RESOURCE_PARAM_STRIDE:
484 *value = rsc->slices[level].stride;
485 return true;
486 case PIPE_RESOURCE_PARAM_OFFSET:
487 *value = rsc->slices[level].offset;
488 return true;
489 case PIPE_RESOURCE_PARAM_MODIFIER:
490 *value = v3d_resource_modifier(rsc);
491 return true;
492 case PIPE_RESOURCE_PARAM_NPLANES:
493 *value = util_resource_num(prsc);
494 return true;
495 default:
496 return false;
497 }
498 }
499
500 #define PAGE_UB_ROWS (V3D_UIFCFG_PAGE_SIZE / V3D_UIFBLOCK_ROW_SIZE)
501 #define PAGE_UB_ROWS_TIMES_1_5 ((PAGE_UB_ROWS * 3) >> 1)
502 #define PAGE_CACHE_UB_ROWS (V3D_PAGE_CACHE_SIZE / V3D_UIFBLOCK_ROW_SIZE)
503 #define PAGE_CACHE_MINUS_1_5_UB_ROWS (PAGE_CACHE_UB_ROWS - PAGE_UB_ROWS_TIMES_1_5)
504
505 /**
506 * Computes the HW's UIFblock padding for a given height/cpp.
507 *
508 * The goal of the padding is to keep pages of the same color (bank number) at
509 * least half a page away from each other vertically when crossing between
510 * between columns of UIF blocks.
511 */
512 static uint32_t
v3d_get_ub_pad(struct v3d_resource * rsc,uint32_t height)513 v3d_get_ub_pad(struct v3d_resource *rsc, uint32_t height)
514 {
515 uint32_t utile_h = v3d_utile_height(rsc->cpp);
516 uint32_t uif_block_h = utile_h * 2;
517 uint32_t height_ub = height / uif_block_h;
518
519 uint32_t height_offset_in_pc = height_ub % PAGE_CACHE_UB_ROWS;
520
521 /* For the perfectly-aligned-for-UIF-XOR case, don't add any pad. */
522 if (height_offset_in_pc == 0)
523 return 0;
524
525 /* Try padding up to where we're offset by at least half a page. */
526 if (height_offset_in_pc < PAGE_UB_ROWS_TIMES_1_5) {
527 /* If we fit entirely in the page cache, don't pad. */
528 if (height_ub < PAGE_CACHE_UB_ROWS)
529 return 0;
530 else
531 return PAGE_UB_ROWS_TIMES_1_5 - height_offset_in_pc;
532 }
533
534 /* If we're close to being aligned to page cache size, then round up
535 * and rely on XOR.
536 */
537 if (height_offset_in_pc > PAGE_CACHE_MINUS_1_5_UB_ROWS)
538 return PAGE_CACHE_UB_ROWS - height_offset_in_pc;
539
540 /* Otherwise, we're far enough away (top and bottom) to not need any
541 * padding.
542 */
543 return 0;
544 }
545
546 /**
547 * Computes the dimension with required padding for mip levels.
548 *
549 * This padding is required for width and height dimensions when the mip
550 * level is greater than 1, and for the depth dimension when the mip level
551 * is greater than 0. This function expects to be passed a mip level >= 1.
552 *
553 * Note: Hardware documentation seems to suggest that the third argument
554 * should be the utile dimensions, but through testing it was found that
555 * the block dimension should be used instead.
556 */
557 static uint32_t
v3d_get_dimension_mpad(uint32_t dimension,uint32_t level,uint32_t block_dimension)558 v3d_get_dimension_mpad(uint32_t dimension, uint32_t level, uint32_t block_dimension)
559 {
560 assert(level >= 1);
561 uint32_t pot_dim = u_minify(dimension, 1);
562 pot_dim = util_next_power_of_two(DIV_ROUND_UP(pot_dim, block_dimension));
563 uint32_t padded_dim = block_dimension * pot_dim;
564 return u_minify(padded_dim, level - 1);
565 }
566
567 static void
v3d_setup_slices(struct v3d_resource * rsc,uint32_t winsys_stride,bool uif_top)568 v3d_setup_slices(struct v3d_resource *rsc, uint32_t winsys_stride,
569 bool uif_top)
570 {
571 struct pipe_resource *prsc = &rsc->base;
572 uint32_t width = prsc->width0;
573 uint32_t height = prsc->height0;
574 uint32_t depth = prsc->depth0;
575 uint32_t offset = 0;
576 uint32_t utile_w = v3d_utile_width(rsc->cpp);
577 uint32_t utile_h = v3d_utile_height(rsc->cpp);
578 uint32_t uif_block_w = utile_w * 2;
579 uint32_t uif_block_h = utile_h * 2;
580 uint32_t block_width = util_format_get_blockwidth(prsc->format);
581 uint32_t block_height = util_format_get_blockheight(prsc->format);
582
583 /* Note that power-of-two padding is based on level 1. These are not
584 * equivalent to just util_next_power_of_two(dimension), because at a
585 * level 0 dimension of 9, the level 1 power-of-two padded value is 4,
586 * not 8. Additionally the pot padding is based on the block size.
587 */
588 uint32_t pot_width = 2 * v3d_get_dimension_mpad(width,
589 1,
590 block_width);
591 uint32_t pot_height = 2 * v3d_get_dimension_mpad(height,
592 1,
593 block_height);
594 uint32_t pot_depth = 2 * v3d_get_dimension_mpad(depth,
595 1,
596 1);
597 bool msaa = prsc->nr_samples > 1;
598
599 /* MSAA textures/renderbuffers are always laid out as single-level
600 * UIF.
601 */
602 uif_top |= msaa;
603
604 /* Check some easy mistakes to make in a resource_create() call that
605 * will break our setup.
606 */
607 assert(prsc->array_size != 0);
608 assert(prsc->depth0 != 0);
609
610 for (int i = prsc->last_level; i >= 0; i--) {
611 struct v3d_resource_slice *slice = &rsc->slices[i];
612
613 uint32_t level_width, level_height, level_depth;
614 if (i < 2) {
615 level_width = u_minify(width, i);
616 level_height = u_minify(height, i);
617 } else {
618 level_width = u_minify(pot_width, i);
619 level_height = u_minify(pot_height, i);
620 }
621 if (i < 1)
622 level_depth = u_minify(depth, i);
623 else
624 level_depth = u_minify(pot_depth, i);
625
626 if (msaa) {
627 level_width *= 2;
628 level_height *= 2;
629 }
630
631 level_width = DIV_ROUND_UP(level_width, block_width);
632 level_height = DIV_ROUND_UP(level_height, block_height);
633
634 if (!rsc->tiled) {
635 slice->tiling = V3D_TILING_RASTER;
636 if (prsc->target == PIPE_TEXTURE_1D ||
637 prsc->target == PIPE_TEXTURE_1D_ARRAY)
638 level_width = align(level_width, 64 / rsc->cpp);
639 } else {
640 if ((i != 0 || !uif_top) &&
641 (level_width <= utile_w ||
642 level_height <= utile_h)) {
643 slice->tiling = V3D_TILING_LINEARTILE;
644 level_width = align(level_width, utile_w);
645 level_height = align(level_height, utile_h);
646 } else if ((i != 0 || !uif_top) &&
647 level_width <= uif_block_w) {
648 slice->tiling = V3D_TILING_UBLINEAR_1_COLUMN;
649 level_width = align(level_width, uif_block_w);
650 level_height = align(level_height, uif_block_h);
651 } else if ((i != 0 || !uif_top) &&
652 level_width <= 2 * uif_block_w) {
653 slice->tiling = V3D_TILING_UBLINEAR_2_COLUMN;
654 level_width = align(level_width, 2 * uif_block_w);
655 level_height = align(level_height, uif_block_h);
656 } else {
657 /* We align the width to a 4-block column of
658 * UIF blocks, but we only align height to UIF
659 * blocks.
660 */
661 level_width = align(level_width,
662 4 * uif_block_w);
663 level_height = align(level_height,
664 uif_block_h);
665
666 slice->ub_pad = v3d_get_ub_pad(rsc,
667 level_height);
668 level_height += slice->ub_pad * uif_block_h;
669
670 /* If the padding set us to to be aligned to
671 * the page cache size, then the HW will use
672 * the XOR bit on odd columns to get us
673 * perfectly misaligned
674 */
675 if ((level_height / uif_block_h) %
676 (V3D_PAGE_CACHE_SIZE /
677 V3D_UIFBLOCK_ROW_SIZE) == 0) {
678 slice->tiling = V3D_TILING_UIF_XOR;
679 } else {
680 slice->tiling = V3D_TILING_UIF_NO_XOR;
681 }
682 }
683 }
684
685 slice->offset = offset;
686 if (winsys_stride)
687 slice->stride = winsys_stride;
688 else
689 slice->stride = level_width * rsc->cpp;
690 slice->padded_height = level_height;
691 slice->size = level_height * slice->stride;
692
693 uint32_t slice_total_size = slice->size * level_depth;
694
695 /* The HW aligns level 1's base to a page if any of level 1 or
696 * below could be UIF XOR. The lower levels then inherit the
697 * alignment for as long as necessary, thanks to being power of
698 * two aligned.
699 */
700 if (i == 1 &&
701 level_width > 4 * uif_block_w &&
702 level_height > PAGE_CACHE_MINUS_1_5_UB_ROWS * uif_block_h) {
703 slice_total_size = align(slice_total_size,
704 V3D_UIFCFG_PAGE_SIZE);
705 }
706
707 offset += slice_total_size;
708
709 }
710 rsc->size = offset;
711
712 /* UIF/UBLINEAR levels need to be aligned to UIF-blocks, and LT only
713 * needs to be aligned to utile boundaries. Since tiles are laid out
714 * from small to big in memory, we need to align the later UIF slices
715 * to UIF blocks, if they were preceded by non-UIF-block-aligned LT
716 * slices.
717 *
718 * We additionally align to 4k, which improves UIF XOR performance.
719 */
720 uint32_t page_align_offset = (align(rsc->slices[0].offset, 4096) -
721 rsc->slices[0].offset);
722 if (page_align_offset) {
723 rsc->size += page_align_offset;
724 for (int i = 0; i <= prsc->last_level; i++)
725 rsc->slices[i].offset += page_align_offset;
726 }
727
728 /* Arrays and cube textures have a stride which is the distance from
729 * one full mipmap tree to the next (64b aligned). For 3D textures,
730 * we need to program the stride between slices of miplevel 0.
731 */
732 if (prsc->target != PIPE_TEXTURE_3D) {
733 rsc->cube_map_stride = align(rsc->slices[0].offset +
734 rsc->slices[0].size, 64);
735 rsc->size += rsc->cube_map_stride * (prsc->array_size - 1);
736 } else {
737 rsc->cube_map_stride = rsc->slices[0].size;
738 }
739 }
740
741 uint32_t
v3d_layer_offset(struct pipe_resource * prsc,uint32_t level,uint32_t layer)742 v3d_layer_offset(struct pipe_resource *prsc, uint32_t level, uint32_t layer)
743 {
744 struct v3d_resource *rsc = v3d_resource(prsc);
745 struct v3d_resource_slice *slice = &rsc->slices[level];
746
747 if (prsc->target == PIPE_TEXTURE_3D)
748 return slice->offset + layer * slice->size;
749 else
750 return slice->offset + layer * rsc->cube_map_stride;
751 }
752
753 static struct v3d_resource *
v3d_resource_setup(struct pipe_screen * pscreen,const struct pipe_resource * tmpl)754 v3d_resource_setup(struct pipe_screen *pscreen,
755 const struct pipe_resource *tmpl)
756 {
757 struct v3d_resource *rsc = CALLOC_STRUCT(v3d_resource);
758
759 if (!rsc)
760 return NULL;
761 struct pipe_resource *prsc = &rsc->base;
762
763 *prsc = *tmpl;
764
765 pipe_reference_init(&prsc->reference, 1);
766 prsc->screen = pscreen;
767
768 rsc->cpp = util_format_get_blocksize(prsc->format);
769 rsc->serial_id++;
770
771 assert(rsc->cpp);
772
773 return rsc;
774 }
775
776 static struct pipe_resource *
v3d_resource_create_with_modifiers(struct pipe_screen * pscreen,const struct pipe_resource * tmpl,const uint64_t * modifiers,int count)777 v3d_resource_create_with_modifiers(struct pipe_screen *pscreen,
778 const struct pipe_resource *tmpl,
779 const uint64_t *modifiers,
780 int count)
781 {
782 struct v3d_screen *screen = v3d_screen(pscreen);
783
784 bool linear_ok = drm_find_modifier(DRM_FORMAT_MOD_LINEAR, modifiers, count);
785 struct v3d_resource *rsc = v3d_resource_setup(pscreen, tmpl);
786 struct pipe_resource *prsc = &rsc->base;
787 /* Use a tiled layout if we can, for better 3D performance. */
788 bool should_tile = true;
789
790 assert(tmpl->target != PIPE_BUFFER ||
791 (tmpl->format == PIPE_FORMAT_NONE ||
792 util_format_get_blocksize(tmpl->format) == 1));
793
794 /* VBOs/PBOs/Texture Buffer Objects are untiled (and 1 height). */
795 if (tmpl->target == PIPE_BUFFER)
796 should_tile = false;
797
798 /* Cursors are always linear, and the user can request linear as well.
799 */
800 if (tmpl->bind & (PIPE_BIND_LINEAR | PIPE_BIND_CURSOR))
801 should_tile = false;
802
803 /* 1D and 1D_ARRAY textures are always raster-order. */
804 if (tmpl->target == PIPE_TEXTURE_1D ||
805 tmpl->target == PIPE_TEXTURE_1D_ARRAY)
806 should_tile = false;
807
808 /* Scanout BOs for simulator need to be linear for interaction with
809 * i965.
810 */
811 if (using_v3d_simulator &&
812 tmpl->bind & (PIPE_BIND_SHARED | PIPE_BIND_SCANOUT))
813 should_tile = false;
814
815 /* If using the old-school SCANOUT flag, we don't know what the screen
816 * might support other than linear. Just force linear.
817 */
818 if (tmpl->bind & PIPE_BIND_SCANOUT)
819 should_tile = false;
820
821 /* No user-specified modifier; determine our own. */
822 if (count == 1 && modifiers[0] == DRM_FORMAT_MOD_INVALID) {
823 linear_ok = true;
824 rsc->tiled = should_tile;
825 } else if (should_tile &&
826 drm_find_modifier(DRM_FORMAT_MOD_BROADCOM_UIF,
827 modifiers, count)) {
828 rsc->tiled = true;
829 } else if (linear_ok) {
830 rsc->tiled = false;
831 } else {
832 fprintf(stderr, "Unsupported modifier requested\n");
833 goto fail;
834 }
835
836 rsc->internal_format = prsc->format;
837
838 v3d_setup_slices(rsc, 0, tmpl->bind & PIPE_BIND_SHARED);
839
840 if (screen->ro && (tmpl->bind & PIPE_BIND_SCANOUT)) {
841 struct winsys_handle handle;
842 struct pipe_resource scanout_tmpl = {
843 .target = prsc->target,
844 .format = PIPE_FORMAT_RGBA8888_UNORM,
845 .width0 = 1024, /* one page */
846 .height0 = align(rsc->size, 4096) / 4096,
847 .depth0 = 1,
848 .array_size = 1,
849 };
850
851 rsc->scanout =
852 renderonly_scanout_for_resource(&scanout_tmpl,
853 screen->ro,
854 &handle);
855
856 if (!rsc->scanout) {
857 fprintf(stderr, "Failed to create scanout resource\n");
858 goto fail;
859 }
860 assert(handle.type == WINSYS_HANDLE_TYPE_FD);
861 rsc->bo = v3d_bo_open_dmabuf(screen, handle.handle);
862 close(handle.handle);
863
864 if (!rsc->bo)
865 goto fail;
866
867 v3d_debug_resource_layout(rsc, "renderonly");
868
869 return prsc;
870 } else {
871 if (!v3d_resource_bo_alloc(rsc))
872 goto fail;
873 }
874
875 return prsc;
876 fail:
877 v3d_resource_destroy(pscreen, prsc);
878 return NULL;
879 }
880
881 struct pipe_resource *
v3d_resource_create(struct pipe_screen * pscreen,const struct pipe_resource * tmpl)882 v3d_resource_create(struct pipe_screen *pscreen,
883 const struct pipe_resource *tmpl)
884 {
885 const uint64_t mod = DRM_FORMAT_MOD_INVALID;
886 return v3d_resource_create_with_modifiers(pscreen, tmpl, &mod, 1);
887 }
888
889 static struct pipe_resource *
v3d_resource_from_handle(struct pipe_screen * pscreen,const struct pipe_resource * tmpl,struct winsys_handle * whandle,unsigned usage)890 v3d_resource_from_handle(struct pipe_screen *pscreen,
891 const struct pipe_resource *tmpl,
892 struct winsys_handle *whandle,
893 unsigned usage)
894 {
895 struct v3d_screen *screen = v3d_screen(pscreen);
896 struct v3d_resource *rsc = v3d_resource_setup(pscreen, tmpl);
897 struct pipe_resource *prsc = &rsc->base;
898 struct v3d_resource_slice *slice = &rsc->slices[0];
899
900 if (!rsc)
901 return NULL;
902
903 switch (whandle->modifier) {
904 case DRM_FORMAT_MOD_LINEAR:
905 rsc->tiled = false;
906 break;
907 case DRM_FORMAT_MOD_BROADCOM_UIF:
908 rsc->tiled = true;
909 break;
910 case DRM_FORMAT_MOD_INVALID:
911 rsc->tiled = screen->ro == NULL;
912 break;
913 default:
914 switch(fourcc_mod_broadcom_mod(whandle->modifier)) {
915 case DRM_FORMAT_MOD_BROADCOM_SAND128:
916 rsc->tiled = false;
917 rsc->sand_col128_stride =
918 fourcc_mod_broadcom_param(whandle->modifier);
919 break;
920 default:
921 fprintf(stderr,
922 "Attempt to import unsupported modifier 0x%llx\n",
923 (long long)whandle->modifier);
924 goto fail;
925 }
926 }
927
928 switch (whandle->type) {
929 case WINSYS_HANDLE_TYPE_SHARED:
930 rsc->bo = v3d_bo_open_name(screen, whandle->handle);
931 break;
932 case WINSYS_HANDLE_TYPE_FD:
933 rsc->bo = v3d_bo_open_dmabuf(screen, whandle->handle);
934 break;
935 default:
936 fprintf(stderr,
937 "Attempt to import unsupported handle type %d\n",
938 whandle->type);
939 goto fail;
940 }
941
942 if (!rsc->bo)
943 goto fail;
944
945 rsc->internal_format = prsc->format;
946
947 v3d_setup_slices(rsc, whandle->stride, true);
948 v3d_debug_resource_layout(rsc, "import");
949
950 if (whandle->offset != 0) {
951 if (rsc->tiled) {
952 fprintf(stderr,
953 "Attempt to import unsupported winsys offset %u\n",
954 whandle->offset);
955 goto fail;
956 }
957 rsc->slices[0].offset += whandle->offset;
958
959 if (rsc->slices[0].offset + rsc->slices[0].size >
960 rsc->bo->size) {
961 fprintf(stderr, "Attempt to import "
962 "with overflowing offset (%d + %d > %d)\n",
963 whandle->offset,
964 rsc->slices[0].size,
965 rsc->bo->size);
966 goto fail;
967 }
968 }
969
970 if (screen->ro) {
971 /* Make sure that renderonly has a handle to our buffer in the
972 * display's fd, so that a later renderonly_get_handle()
973 * returns correct handles or GEM names.
974 */
975 rsc->scanout =
976 renderonly_create_gpu_import_for_resource(prsc,
977 screen->ro,
978 NULL);
979 }
980
981 if (rsc->tiled && whandle->stride != slice->stride) {
982 static bool warned = false;
983 if (!warned) {
984 warned = true;
985 fprintf(stderr,
986 "Attempting to import %dx%d %s with "
987 "unsupported stride %d instead of %d\n",
988 prsc->width0, prsc->height0,
989 util_format_short_name(prsc->format),
990 whandle->stride,
991 slice->stride);
992 }
993 goto fail;
994 } else if (!rsc->tiled) {
995 slice->stride = whandle->stride;
996 }
997
998 /* Prevent implicit clearing of the imported buffer contents. */
999 rsc->writes = 1;
1000
1001 return prsc;
1002
1003 fail:
1004 v3d_resource_destroy(pscreen, prsc);
1005 return NULL;
1006 }
1007
1008 void
v3d_update_shadow_texture(struct pipe_context * pctx,struct pipe_sampler_view * pview)1009 v3d_update_shadow_texture(struct pipe_context *pctx,
1010 struct pipe_sampler_view *pview)
1011 {
1012 struct v3d_context *v3d = v3d_context(pctx);
1013 struct v3d_sampler_view *view = v3d_sampler_view(pview);
1014 struct v3d_resource *shadow = v3d_resource(view->texture);
1015 struct v3d_resource *orig = v3d_resource(pview->texture);
1016
1017 assert(view->texture != pview->texture);
1018
1019 if (shadow->writes == orig->writes && orig->bo->private)
1020 return;
1021
1022 perf_debug("Updating %dx%d@%d shadow for linear texture\n",
1023 orig->base.width0, orig->base.height0,
1024 pview->u.tex.first_level);
1025
1026 for (int i = 0; i <= shadow->base.last_level; i++) {
1027 unsigned width = u_minify(shadow->base.width0, i);
1028 unsigned height = u_minify(shadow->base.height0, i);
1029 struct pipe_blit_info info = {
1030 .dst = {
1031 .resource = &shadow->base,
1032 .level = i,
1033 .box = {
1034 .x = 0,
1035 .y = 0,
1036 .z = 0,
1037 .width = width,
1038 .height = height,
1039 .depth = 1,
1040 },
1041 .format = shadow->base.format,
1042 },
1043 .src = {
1044 .resource = &orig->base,
1045 .level = pview->u.tex.first_level + i,
1046 .box = {
1047 .x = 0,
1048 .y = 0,
1049 .z = 0,
1050 .width = width,
1051 .height = height,
1052 .depth = 1,
1053 },
1054 .format = orig->base.format,
1055 },
1056 .mask = util_format_get_mask(orig->base.format),
1057 };
1058 pctx->blit(pctx, &info);
1059 }
1060
1061 shadow->writes = orig->writes;
1062 }
1063
1064 static struct pipe_surface *
v3d_create_surface(struct pipe_context * pctx,struct pipe_resource * ptex,const struct pipe_surface * surf_tmpl)1065 v3d_create_surface(struct pipe_context *pctx,
1066 struct pipe_resource *ptex,
1067 const struct pipe_surface *surf_tmpl)
1068 {
1069 struct v3d_context *v3d = v3d_context(pctx);
1070 struct v3d_screen *screen = v3d->screen;
1071 struct v3d_device_info *devinfo = &screen->devinfo;
1072 struct v3d_surface *surface = CALLOC_STRUCT(v3d_surface);
1073 struct v3d_resource *rsc = v3d_resource(ptex);
1074
1075 if (!surface)
1076 return NULL;
1077
1078 struct pipe_surface *psurf = &surface->base;
1079 unsigned level = surf_tmpl->u.tex.level;
1080 struct v3d_resource_slice *slice = &rsc->slices[level];
1081
1082 pipe_reference_init(&psurf->reference, 1);
1083 pipe_resource_reference(&psurf->texture, ptex);
1084
1085 psurf->context = pctx;
1086 psurf->format = surf_tmpl->format;
1087 psurf->width = u_minify(ptex->width0, level);
1088 psurf->height = u_minify(ptex->height0, level);
1089 psurf->u.tex.level = level;
1090 psurf->u.tex.first_layer = surf_tmpl->u.tex.first_layer;
1091 psurf->u.tex.last_layer = surf_tmpl->u.tex.last_layer;
1092
1093 surface->offset = v3d_layer_offset(ptex, level,
1094 psurf->u.tex.first_layer);
1095 surface->tiling = slice->tiling;
1096
1097 surface->format = v3d_get_rt_format(devinfo, psurf->format);
1098
1099 const struct util_format_description *desc =
1100 util_format_description(psurf->format);
1101
1102 surface->swap_rb = (desc->swizzle[0] == PIPE_SWIZZLE_Z &&
1103 psurf->format != PIPE_FORMAT_B5G6R5_UNORM);
1104
1105 if (util_format_is_depth_or_stencil(psurf->format)) {
1106 switch (psurf->format) {
1107 case PIPE_FORMAT_Z16_UNORM:
1108 surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_16;
1109 break;
1110 case PIPE_FORMAT_Z32_FLOAT:
1111 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
1112 surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_32F;
1113 break;
1114 default:
1115 surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_24;
1116 }
1117 } else {
1118 uint32_t bpp, type;
1119 v3d_X(devinfo, get_internal_type_bpp_for_output_format)
1120 (surface->format, &type, &bpp);
1121 surface->internal_type = type;
1122 surface->internal_bpp = bpp;
1123 }
1124
1125 if (surface->tiling == V3D_TILING_UIF_NO_XOR ||
1126 surface->tiling == V3D_TILING_UIF_XOR) {
1127 surface->padded_height_of_output_image_in_uif_blocks =
1128 (slice->padded_height /
1129 (2 * v3d_utile_height(rsc->cpp)));
1130 }
1131
1132 if (rsc->separate_stencil) {
1133 surface->separate_stencil =
1134 v3d_create_surface(pctx, &rsc->separate_stencil->base,
1135 surf_tmpl);
1136 }
1137
1138 return &surface->base;
1139 }
1140
1141 static void
v3d_surface_destroy(struct pipe_context * pctx,struct pipe_surface * psurf)1142 v3d_surface_destroy(struct pipe_context *pctx, struct pipe_surface *psurf)
1143 {
1144 struct v3d_surface *surf = v3d_surface(psurf);
1145
1146 if (surf->separate_stencil)
1147 pipe_surface_reference(&surf->separate_stencil, NULL);
1148
1149 pipe_resource_reference(&psurf->texture, NULL);
1150 FREE(psurf);
1151 }
1152
1153 static void
v3d_flush_resource(struct pipe_context * pctx,struct pipe_resource * resource)1154 v3d_flush_resource(struct pipe_context *pctx, struct pipe_resource *resource)
1155 {
1156 /* All calls to flush_resource are followed by a flush of the context,
1157 * so there's nothing to do.
1158 */
1159 }
1160
1161 static enum pipe_format
v3d_resource_get_internal_format(struct pipe_resource * prsc)1162 v3d_resource_get_internal_format(struct pipe_resource *prsc)
1163 {
1164 return v3d_resource(prsc)->internal_format;
1165 }
1166
1167 static void
v3d_resource_set_stencil(struct pipe_resource * prsc,struct pipe_resource * stencil)1168 v3d_resource_set_stencil(struct pipe_resource *prsc,
1169 struct pipe_resource *stencil)
1170 {
1171 v3d_resource(prsc)->separate_stencil = v3d_resource(stencil);
1172 }
1173
1174 static struct pipe_resource *
v3d_resource_get_stencil(struct pipe_resource * prsc)1175 v3d_resource_get_stencil(struct pipe_resource *prsc)
1176 {
1177 struct v3d_resource *rsc = v3d_resource(prsc);
1178
1179 return &rsc->separate_stencil->base;
1180 }
1181
1182 static const struct u_transfer_vtbl transfer_vtbl = {
1183 .resource_create = v3d_resource_create,
1184 .resource_destroy = v3d_resource_destroy,
1185 .transfer_map = v3d_resource_transfer_map,
1186 .transfer_unmap = v3d_resource_transfer_unmap,
1187 .transfer_flush_region = u_default_transfer_flush_region,
1188 .get_internal_format = v3d_resource_get_internal_format,
1189 .set_stencil = v3d_resource_set_stencil,
1190 .get_stencil = v3d_resource_get_stencil,
1191 };
1192
1193 void
v3d_resource_screen_init(struct pipe_screen * pscreen)1194 v3d_resource_screen_init(struct pipe_screen *pscreen)
1195 {
1196 pscreen->resource_create_with_modifiers =
1197 v3d_resource_create_with_modifiers;
1198 pscreen->resource_create = u_transfer_helper_resource_create;
1199 pscreen->resource_from_handle = v3d_resource_from_handle;
1200 pscreen->resource_get_handle = v3d_resource_get_handle;
1201 pscreen->resource_get_param = v3d_resource_get_param;
1202 pscreen->resource_destroy = u_transfer_helper_resource_destroy;
1203 pscreen->transfer_helper = u_transfer_helper_create(&transfer_vtbl,
1204 U_TRANSFER_HELPER_SEPARATE_Z32S8 |
1205 U_TRANSFER_HELPER_MSAA_MAP);
1206 }
1207
1208 void
v3d_resource_context_init(struct pipe_context * pctx)1209 v3d_resource_context_init(struct pipe_context *pctx)
1210 {
1211 pctx->buffer_map = u_transfer_helper_transfer_map;
1212 pctx->texture_map = u_transfer_helper_transfer_map;
1213 pctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;
1214 pctx->buffer_unmap = u_transfer_helper_transfer_unmap;
1215 pctx->texture_unmap = u_transfer_helper_transfer_unmap;
1216 pctx->buffer_subdata = u_default_buffer_subdata;
1217 pctx->texture_subdata = v3d_texture_subdata;
1218 pctx->create_surface = v3d_create_surface;
1219 pctx->surface_destroy = v3d_surface_destroy;
1220 pctx->resource_copy_region = util_resource_copy_region;
1221 pctx->blit = v3d_blit;
1222 pctx->generate_mipmap = v3d_generate_mipmap;
1223 pctx->flush_resource = v3d_flush_resource;
1224 }
1225