1 /*
2 * Copyright © 2017 Red Hat
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23
24 #include "pipe/p_screen.h"
25
26 #include "util/u_box.h"
27 #include "util/format/u_format.h"
28 #include "util/format/u_format_zs.h"
29 #include "util/u_inlines.h"
30 #include "util/u_transfer_helper.h"
31
32
33 struct u_transfer_helper {
34 const struct u_transfer_vtbl *vtbl;
35 bool separate_z32s8; /**< separate z32 and s8 */
36 bool separate_stencil; /**< separate stencil for all formats */
37 bool msaa_map;
38 bool z24_in_z32f; /* the z24 values are stored in a z32 - translate them. */
39 bool interleave_in_place;
40 };
41
42 /* If we need to take the path for PIPE_MAP_DEPTH/STENCIL_ONLY on the parent
43 * depth/stencil resource an interleaving those to/from a staging buffer. The
44 * other path for z/s interleave is when separate z and s resources are
45 * created at resource create time.
46 */
needs_in_place_zs_interleave(struct u_transfer_helper * helper,enum pipe_format format)47 static inline bool needs_in_place_zs_interleave(struct u_transfer_helper *helper,
48 enum pipe_format format)
49 {
50 if (!helper->interleave_in_place)
51 return false;
52 if (helper->separate_stencil && util_format_is_depth_and_stencil(format))
53 return true;
54 if (helper->separate_z32s8 && format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT)
55 return true;
56 /* this isn't interleaving, but still needs conversions on that path. */
57 if (helper->z24_in_z32f && format == PIPE_FORMAT_Z24X8_UNORM)
58 return true;
59 return false;
60 }
61
handle_transfer(struct pipe_resource * prsc)62 static inline bool handle_transfer(struct pipe_resource *prsc)
63 {
64 struct u_transfer_helper *helper = prsc->screen->transfer_helper;
65
66 if (helper->vtbl->get_internal_format) {
67 enum pipe_format internal_format =
68 helper->vtbl->get_internal_format(prsc);
69 if (internal_format != prsc->format)
70 return true;
71 }
72
73 if (helper->msaa_map && (prsc->nr_samples > 1))
74 return true;
75
76 if (needs_in_place_zs_interleave(helper, prsc->format))
77 return true;
78
79 return false;
80 }
81
82 /* The pipe_transfer ptr could either be the driver's, or u_transfer,
83 * depending on whether we are intervening or not. Check handle_transfer()
84 * before dereferencing.
85 */
86 struct u_transfer {
87 struct pipe_transfer base;
88 /* Note that in case of MSAA resolve for transfer plus z32s8
89 * we end up with stacked u_transfer's. The MSAA resolve case doesn't call
90 * helper->vtbl fxns directly, but calls back to pctx->transfer_map()/etc
91 * so the format related handling can work in conjunction with MSAA resolve.
92 */
93 struct pipe_transfer *trans; /* driver's transfer */
94 struct pipe_transfer *trans2; /* 2nd transfer for s8 stencil buffer in z32s8 */
95 void *ptr, *ptr2; /* ptr to trans, and trans2 */
96 void *staging; /* staging buffer */
97 struct pipe_resource *ss; /* staging resource for MSAA resolves */
98 };
99
100 static inline struct u_transfer *
u_transfer(struct pipe_transfer * ptrans)101 u_transfer(struct pipe_transfer *ptrans)
102 {
103 assert(handle_transfer(ptrans->resource));
104 return (struct u_transfer *)ptrans;
105 }
106
107 struct pipe_resource *
u_transfer_helper_resource_create(struct pipe_screen * pscreen,const struct pipe_resource * templ)108 u_transfer_helper_resource_create(struct pipe_screen *pscreen,
109 const struct pipe_resource *templ)
110 {
111 struct u_transfer_helper *helper = pscreen->transfer_helper;
112 enum pipe_format format = templ->format;
113 struct pipe_resource *prsc;
114
115 if (((helper->separate_stencil && util_format_is_depth_and_stencil(format)) ||
116 (format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT && helper->separate_z32s8)) &&
117 !helper->interleave_in_place) {
118 struct pipe_resource t = *templ;
119 struct pipe_resource *stencil;
120
121 t.format = util_format_get_depth_only(format);
122
123 if (t.format == PIPE_FORMAT_Z24X8_UNORM && helper->z24_in_z32f)
124 t.format = PIPE_FORMAT_Z32_FLOAT;
125
126 prsc = helper->vtbl->resource_create(pscreen, &t);
127 if (!prsc)
128 return NULL;
129
130 prsc->format = format; /* frob the format back to the "external" format */
131
132 t.format = PIPE_FORMAT_S8_UINT;
133 stencil = helper->vtbl->resource_create(pscreen, &t);
134
135 if (!stencil) {
136 helper->vtbl->resource_destroy(pscreen, prsc);
137 return NULL;
138 }
139
140 helper->vtbl->set_stencil(prsc, stencil);
141 } else if (format == PIPE_FORMAT_Z24X8_UNORM && helper->z24_in_z32f) {
142 struct pipe_resource t = *templ;
143 t.format = PIPE_FORMAT_Z32_FLOAT;
144
145 prsc = helper->vtbl->resource_create(pscreen, &t);
146 if (!prsc)
147 return NULL;
148
149 prsc->format = format; /* frob the format back to the "external" format */
150 } else {
151 /* normal case, no special handling: */
152 prsc = helper->vtbl->resource_create(pscreen, templ);
153 if (!prsc)
154 return NULL;
155 }
156
157 return prsc;
158 }
159
160 void
u_transfer_helper_resource_destroy(struct pipe_screen * pscreen,struct pipe_resource * prsc)161 u_transfer_helper_resource_destroy(struct pipe_screen *pscreen,
162 struct pipe_resource *prsc)
163 {
164 struct u_transfer_helper *helper = pscreen->transfer_helper;
165
166 if (helper->vtbl->get_stencil && !helper->interleave_in_place) {
167 struct pipe_resource *stencil = helper->vtbl->get_stencil(prsc);
168
169 pipe_resource_reference(&stencil, NULL);
170 }
171
172 helper->vtbl->resource_destroy(pscreen, prsc);
173 }
174
needs_pack(unsigned usage)175 static bool needs_pack(unsigned usage)
176 {
177 return (usage & PIPE_MAP_READ) &&
178 !(usage & (PIPE_MAP_DISCARD_WHOLE_RESOURCE | PIPE_MAP_DISCARD_RANGE));
179 }
180
181 /* In the case of transfer_map of a multi-sample resource, call back into
182 * pctx->transfer_map() to map the staging resource, to handle cases of
183 * MSAA + separate_z32s8
184 */
185 static void *
transfer_map_msaa(struct pipe_context * pctx,struct pipe_resource * prsc,unsigned level,unsigned usage,const struct pipe_box * box,struct pipe_transfer ** pptrans)186 transfer_map_msaa(struct pipe_context *pctx,
187 struct pipe_resource *prsc,
188 unsigned level, unsigned usage,
189 const struct pipe_box *box,
190 struct pipe_transfer **pptrans)
191 {
192 struct pipe_screen *pscreen = pctx->screen;
193 struct u_transfer *trans = calloc(1, sizeof(*trans));
194 if (!trans)
195 return NULL;
196 struct pipe_transfer *ptrans = &trans->base;
197
198 pipe_resource_reference(&ptrans->resource, prsc);
199 ptrans->level = level;
200 ptrans->usage = usage;
201 ptrans->box = *box;
202
203 struct pipe_resource tmpl = {
204 .target = prsc->target,
205 .format = prsc->format,
206 .width0 = box->width,
207 .height0 = box->height,
208 .depth0 = 1,
209 .array_size = 1,
210 };
211 if (util_format_is_depth_or_stencil(tmpl.format))
212 tmpl.bind |= PIPE_BIND_DEPTH_STENCIL;
213 else
214 tmpl.bind |= PIPE_BIND_RENDER_TARGET;
215 trans->ss = pscreen->resource_create(pscreen, &tmpl);
216 if (!trans->ss) {
217 free(trans);
218 return NULL;
219 }
220
221 if (needs_pack(usage)) {
222 struct pipe_blit_info blit;
223 memset(&blit, 0, sizeof(blit));
224
225 blit.src.resource = ptrans->resource;
226 blit.src.format = ptrans->resource->format;
227 blit.src.level = ptrans->level;
228 blit.src.box = *box;
229
230 blit.dst.resource = trans->ss;
231 blit.dst.format = trans->ss->format;
232 blit.dst.box.width = box->width;
233 blit.dst.box.height = box->height;
234 blit.dst.box.depth = 1;
235
236 blit.mask = util_format_get_mask(prsc->format);
237 blit.filter = PIPE_TEX_FILTER_NEAREST;
238
239 pctx->blit(pctx, &blit);
240 }
241
242 struct pipe_box map_box = *box;
243 map_box.x = 0;
244 map_box.y = 0;
245
246 void *ss_map = pctx->texture_map(pctx, trans->ss, 0, usage, &map_box,
247 &trans->trans);
248 if (!ss_map) {
249 free(trans);
250 return NULL;
251 }
252
253 ptrans->stride = trans->trans->stride;
254 *pptrans = ptrans;
255 return ss_map;
256 }
257
258 void *
u_transfer_helper_transfer_map(struct pipe_context * pctx,struct pipe_resource * prsc,unsigned level,unsigned usage,const struct pipe_box * box,struct pipe_transfer ** pptrans)259 u_transfer_helper_transfer_map(struct pipe_context *pctx,
260 struct pipe_resource *prsc,
261 unsigned level, unsigned usage,
262 const struct pipe_box *box,
263 struct pipe_transfer **pptrans)
264 {
265 struct u_transfer_helper *helper = pctx->screen->transfer_helper;
266 struct u_transfer *trans;
267 struct pipe_transfer *ptrans;
268 enum pipe_format format = prsc->format;
269 unsigned width = box->width;
270 unsigned height = box->height;
271 bool in_place_zs_interleave = needs_in_place_zs_interleave(helper, format);
272
273 if (!handle_transfer(prsc))
274 return helper->vtbl->transfer_map(pctx, prsc, level, usage, box, pptrans);
275
276 if (helper->msaa_map && (prsc->nr_samples > 1))
277 return transfer_map_msaa(pctx, prsc, level, usage, box, pptrans);
278
279 assert(box->depth == 1);
280
281 trans = calloc(1, sizeof(*trans));
282 if (!trans)
283 return NULL;
284
285 ptrans = &trans->base;
286 pipe_resource_reference(&ptrans->resource, prsc);
287 ptrans->level = level;
288 ptrans->usage = usage;
289 ptrans->box = *box;
290 ptrans->stride = util_format_get_stride(format, box->width);
291 ptrans->layer_stride = (uint64_t)ptrans->stride * box->height;
292
293 trans->staging = malloc(ptrans->layer_stride);
294 if (!trans->staging)
295 goto fail;
296
297 trans->ptr = helper->vtbl->transfer_map(pctx, prsc, level,
298 usage | (in_place_zs_interleave ? PIPE_MAP_DEPTH_ONLY : 0),
299 box, &trans->trans);
300 if (!trans->ptr)
301 goto fail;
302
303 if (util_format_is_depth_and_stencil(prsc->format)) {
304 struct pipe_resource *stencil;
305
306 if (in_place_zs_interleave)
307 stencil = prsc;
308 else
309 stencil = helper->vtbl->get_stencil(prsc);
310 trans->ptr2 = helper->vtbl->transfer_map(pctx, stencil, level,
311 usage | (in_place_zs_interleave ? PIPE_MAP_STENCIL_ONLY : 0),
312 box, &trans->trans2);
313
314 if (needs_pack(usage)) {
315 switch (prsc->format) {
316 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
317 util_format_z32_float_s8x24_uint_pack_z_float(trans->staging,
318 ptrans->stride,
319 trans->ptr,
320 trans->trans->stride,
321 width, height);
322 util_format_z32_float_s8x24_uint_pack_s_8uint(trans->staging,
323 ptrans->stride,
324 trans->ptr2,
325 trans->trans2->stride,
326 width, height);
327 break;
328 case PIPE_FORMAT_Z24_UNORM_S8_UINT:
329 if (in_place_zs_interleave) {
330 if (helper->z24_in_z32f) {
331 util_format_z24_unorm_s8_uint_pack_separate_z32(trans->staging,
332 ptrans->stride,
333 trans->ptr,
334 trans->trans->stride,
335 trans->ptr2,
336 trans->trans2->stride,
337 width, height);
338 } else {
339 util_format_z24_unorm_s8_uint_pack_separate(trans->staging,
340 ptrans->stride,
341 trans->ptr,
342 trans->trans->stride,
343 trans->ptr2,
344 trans->trans2->stride,
345 width, height);
346 }
347 } else {
348 if (helper->z24_in_z32f) {
349 util_format_z24_unorm_s8_uint_pack_z_float(trans->staging,
350 ptrans->stride,
351 trans->ptr,
352 trans->trans->stride,
353 width, height);
354 util_format_z24_unorm_s8_uint_pack_s_8uint(trans->staging,
355 ptrans->stride,
356 trans->ptr2,
357 trans->trans2->stride,
358 width, height);
359 } else {
360 util_format_z24_unorm_s8_uint_pack_separate(trans->staging,
361 ptrans->stride,
362 trans->ptr,
363 trans->trans->stride,
364 trans->ptr2,
365 trans->trans2->stride,
366 width, height);
367 }
368 }
369 break;
370 case PIPE_FORMAT_Z24X8_UNORM:
371 assert(helper->z24_in_z32f);
372 util_format_z24x8_unorm_pack_z_float(trans->staging, ptrans->stride,
373 trans->ptr, trans->trans->stride,
374 width, height);
375 break;
376 default:
377 unreachable("Unexpected format");
378 }
379 }
380 } else if (prsc->format == PIPE_FORMAT_Z24X8_UNORM) {
381 assert(helper->z24_in_z32f);
382 util_format_z24x8_unorm_pack_z_float(trans->staging, ptrans->stride,
383 trans->ptr, trans->trans->stride,
384 width, height);
385 } else {
386 unreachable("bleh");
387 }
388
389 *pptrans = ptrans;
390 return trans->staging;
391
392 fail:
393 if (trans->trans)
394 helper->vtbl->transfer_unmap(pctx, trans->trans);
395 if (trans->trans2)
396 helper->vtbl->transfer_unmap(pctx, trans->trans2);
397 pipe_resource_reference(&ptrans->resource, NULL);
398 free(trans->staging);
399 free(trans);
400 return NULL;
401 }
402
403 static void
flush_region(struct pipe_context * pctx,struct pipe_transfer * ptrans,const struct pipe_box * box)404 flush_region(struct pipe_context *pctx, struct pipe_transfer *ptrans,
405 const struct pipe_box *box)
406 {
407 struct u_transfer_helper *helper = pctx->screen->transfer_helper;
408 /* using the function here hits an assert for the deinterleave cases */
409 struct u_transfer *trans = (struct u_transfer *)ptrans;
410 enum pipe_format iformat, format = ptrans->resource->format;
411 unsigned width = box->width;
412 unsigned height = box->height;
413 void *src, *dst;
414
415 if (!(ptrans->usage & PIPE_MAP_WRITE))
416 return;
417
418 if (trans->ss) {
419 struct pipe_blit_info blit;
420 memset(&blit, 0, sizeof(blit));
421
422 blit.src.resource = trans->ss;
423 blit.src.format = trans->ss->format;
424 blit.src.box = *box;
425
426 blit.dst.resource = ptrans->resource;
427 blit.dst.format = ptrans->resource->format;
428 blit.dst.level = ptrans->level;
429
430 u_box_2d(ptrans->box.x + box->x,
431 ptrans->box.y + box->y,
432 box->width, box->height,
433 &blit.dst.box);
434
435 blit.mask = util_format_get_mask(ptrans->resource->format);
436 blit.filter = PIPE_TEX_FILTER_NEAREST;
437
438 pctx->blit(pctx, &blit);
439
440 return;
441 }
442
443 iformat = helper->vtbl->get_internal_format(ptrans->resource);
444
445 src = (uint8_t *)trans->staging +
446 (box->y * ptrans->stride) +
447 (box->x * util_format_get_blocksize(format));
448 dst = (uint8_t *)trans->ptr +
449 (box->y * trans->trans->stride) +
450 (box->x * util_format_get_blocksize(iformat));
451
452 switch (format) {
453 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
454 util_format_z32_float_s8x24_uint_unpack_z_float(dst,
455 trans->trans->stride,
456 src,
457 ptrans->stride,
458 width, height);
459 FALLTHROUGH;
460 case PIPE_FORMAT_X32_S8X24_UINT:
461 dst = (uint8_t *)trans->ptr2 +
462 (box->y * trans->trans2->stride) +
463 (box->x * util_format_get_blocksize(PIPE_FORMAT_S8_UINT));
464
465 util_format_z32_float_s8x24_uint_unpack_s_8uint(dst,
466 trans->trans2->stride,
467 src,
468 ptrans->stride,
469 width, height);
470 break;
471 case PIPE_FORMAT_Z24X8_UNORM:
472 util_format_z24x8_unorm_unpack_z_float(dst, trans->trans->stride,
473 src, ptrans->stride,
474 width, height);
475 break;
476 case PIPE_FORMAT_Z24_UNORM_S8_UINT:
477 if (helper->z24_in_z32f) {
478 util_format_z24_unorm_s8_uint_unpack_z_float(dst, trans->trans->stride,
479 src, ptrans->stride,
480 width, height);
481 } else {
482 /* just do a strided 32-bit copy for depth; s8 can become garbage x8 */
483 util_format_z32_unorm_unpack_z_32unorm(dst, trans->trans->stride,
484 src, ptrans->stride,
485 width, height);
486 }
487 FALLTHROUGH;
488 case PIPE_FORMAT_X24S8_UINT:
489 dst = (uint8_t *)trans->ptr2 +
490 (box->y * trans->trans2->stride) +
491 (box->x * util_format_get_blocksize(PIPE_FORMAT_S8_UINT));
492
493 util_format_z24_unorm_s8_uint_unpack_s_8uint(dst, trans->trans2->stride,
494 src, ptrans->stride,
495 width, height);
496 break;
497
498 default:
499 assert(!"Unexpected staging transfer type");
500 break;
501 }
502 }
503
504 void
u_transfer_helper_transfer_flush_region(struct pipe_context * pctx,struct pipe_transfer * ptrans,const struct pipe_box * box)505 u_transfer_helper_transfer_flush_region(struct pipe_context *pctx,
506 struct pipe_transfer *ptrans,
507 const struct pipe_box *box)
508 {
509 struct u_transfer_helper *helper = pctx->screen->transfer_helper;
510
511 if (handle_transfer(ptrans->resource)) {
512 struct u_transfer *trans = u_transfer(ptrans);
513
514 /* handle MSAA case, since there could be multiple levels of
515 * wrapped transfer, call pctx->transfer_flush_region()
516 * instead of helper->vtbl->transfer_flush_region()
517 */
518 if (trans->ss) {
519 pctx->transfer_flush_region(pctx, trans->trans, box);
520 flush_region(pctx, ptrans, box);
521 return;
522 }
523
524 flush_region(pctx, ptrans, box);
525
526 helper->vtbl->transfer_flush_region(pctx, trans->trans, box);
527 if (trans->trans2)
528 helper->vtbl->transfer_flush_region(pctx, trans->trans2, box);
529
530 } else {
531 helper->vtbl->transfer_flush_region(pctx, ptrans, box);
532 }
533 }
534
535 void
u_transfer_helper_transfer_unmap(struct pipe_context * pctx,struct pipe_transfer * ptrans)536 u_transfer_helper_transfer_unmap(struct pipe_context *pctx,
537 struct pipe_transfer *ptrans)
538 {
539 struct u_transfer_helper *helper = pctx->screen->transfer_helper;
540
541 if (handle_transfer(ptrans->resource)) {
542 struct u_transfer *trans = u_transfer(ptrans);
543
544 if (!(ptrans->usage & PIPE_MAP_FLUSH_EXPLICIT)) {
545 struct pipe_box box;
546 u_box_2d(0, 0, ptrans->box.width, ptrans->box.height, &box);
547 if (trans->ss)
548 pctx->transfer_flush_region(pctx, trans->trans, &box);
549 flush_region(pctx, ptrans, &box);
550 }
551
552 /* in MSAA case, there could be multiple levels of wrapping
553 * so don't call helper->vtbl->transfer_unmap() directly
554 */
555 if (trans->ss) {
556 pctx->texture_unmap(pctx, trans->trans);
557 pipe_resource_reference(&trans->ss, NULL);
558 } else {
559 helper->vtbl->transfer_unmap(pctx, trans->trans);
560 if (trans->trans2)
561 helper->vtbl->transfer_unmap(pctx, trans->trans2);
562 }
563
564 pipe_resource_reference(&ptrans->resource, NULL);
565
566 free(trans->staging);
567 free(trans);
568 } else {
569 helper->vtbl->transfer_unmap(pctx, ptrans);
570 }
571 }
572
573 struct u_transfer_helper *
u_transfer_helper_create(const struct u_transfer_vtbl * vtbl,enum u_transfer_helper_flags flags)574 u_transfer_helper_create(const struct u_transfer_vtbl *vtbl,
575 enum u_transfer_helper_flags flags)
576 {
577 struct u_transfer_helper *helper = calloc(1, sizeof(*helper));
578
579 helper->vtbl = vtbl;
580 helper->separate_z32s8 = flags & U_TRANSFER_HELPER_SEPARATE_Z32S8;
581 helper->separate_stencil = flags & U_TRANSFER_HELPER_SEPARATE_STENCIL;
582 helper->msaa_map = flags & U_TRANSFER_HELPER_MSAA_MAP;
583 helper->z24_in_z32f = flags & U_TRANSFER_HELPER_Z24_IN_Z32F;
584 helper->interleave_in_place = flags & U_TRANSFER_HELPER_INTERLEAVE_IN_PLACE;
585
586 return helper;
587 }
588
589 void
u_transfer_helper_destroy(struct u_transfer_helper * helper)590 u_transfer_helper_destroy(struct u_transfer_helper *helper)
591 {
592 free(helper);
593 }
594