1 /*
2 * Copyright (c) 2012-2017 Etnaviv Project
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sub license,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the
12 * next paragraph) shall be included in all copies or substantial portions
13 * of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 *
23 * Authors:
24 * Wladimir J. van der Laan <laanwj@gmail.com>
25 */
26
27 #include "etnaviv_rs.h"
28
29 #include "hw/common.xml.h"
30
31 #include "etnaviv_clear_blit.h"
32 #include "etnaviv_context.h"
33 #include "etnaviv_emit.h"
34 #include "etnaviv_format.h"
35 #include "etnaviv_resource.h"
36 #include "etnaviv_screen.h"
37 #include "etnaviv_surface.h"
38 #include "etnaviv_tiling.h"
39 #include "etnaviv_translate.h"
40 #include "etnaviv_util.h"
41
42 #include "pipe/p_defines.h"
43 #include "pipe/p_state.h"
44 #include "util/u_blitter.h"
45 #include "util/u_inlines.h"
46 #include "util/u_memory.h"
47 #include "util/u_surface.h"
48
49 #include "hw/common.xml.h"
50 #include "hw/state.xml.h"
51 #include "hw/state_3d.xml.h"
52
53 #include <assert.h>
54
55 void
etna_compile_rs_state(struct etna_context * ctx,struct compiled_rs_state * cs,const struct rs_state * rs)56 etna_compile_rs_state(struct etna_context *ctx, struct compiled_rs_state *cs,
57 const struct rs_state *rs)
58 {
59 memset(cs, 0, sizeof(*cs));
60
61 /* TILED and SUPERTILED layout have their strides multiplied with 4 in RS */
62 unsigned source_stride_shift = COND(rs->source_tiling != ETNA_LAYOUT_LINEAR, 2);
63 unsigned dest_stride_shift = COND(rs->dest_tiling != ETNA_LAYOUT_LINEAR, 2);
64
65 /* tiling == ETNA_LAYOUT_MULTI_TILED or ETNA_LAYOUT_MULTI_SUPERTILED? */
66 int source_multi = COND(rs->source_tiling & ETNA_LAYOUT_BIT_MULTI, 1);
67 int dest_multi = COND(rs->dest_tiling & ETNA_LAYOUT_BIT_MULTI, 1);
68
69 /* Vivante RS needs widths to be a multiple of 16 or bad things
70 * happen, such as scribbing over memory, or the GPU hanging,
71 * even for non-tiled formats. As this is serious, use abort().
72 */
73 if (rs->width & ETNA_RS_WIDTH_MASK)
74 abort();
75
76 /* TODO could just pre-generate command buffer, would simply submit to one memcpy */
77 cs->RS_CONFIG = VIVS_RS_CONFIG_SOURCE_FORMAT(rs->source_format) |
78 COND(rs->downsample_x, VIVS_RS_CONFIG_DOWNSAMPLE_X) |
79 COND(rs->downsample_y, VIVS_RS_CONFIG_DOWNSAMPLE_Y) |
80 COND(rs->source_tiling & 1, VIVS_RS_CONFIG_SOURCE_TILED) |
81 VIVS_RS_CONFIG_DEST_FORMAT(rs->dest_format) |
82 COND(rs->dest_tiling & 1, VIVS_RS_CONFIG_DEST_TILED) |
83 COND(rs->swap_rb, VIVS_RS_CONFIG_SWAP_RB) |
84 COND(rs->flip, VIVS_RS_CONFIG_FLIP);
85
86 cs->RS_SOURCE_STRIDE = (rs->source_stride << source_stride_shift) |
87 COND(rs->source_tiling & 2, VIVS_RS_SOURCE_STRIDE_TILING) |
88 COND(source_multi, VIVS_RS_SOURCE_STRIDE_MULTI);
89
90 /* Initially all pipes are set to the base address of the source and
91 * destination buffer respectively. This will be overridden below as
92 * necessary for the multi-pipe, multi-tiled case.
93 */
94 for (unsigned pipe = 0; pipe < ctx->specs.pixel_pipes; ++pipe) {
95 cs->source[pipe].bo = rs->source;
96 cs->source[pipe].offset = rs->source_offset;
97 cs->source[pipe].flags = ETNA_RELOC_READ;
98
99 cs->dest[pipe].bo = rs->dest;
100 cs->dest[pipe].offset = rs->dest_offset;
101 cs->dest[pipe].flags = ETNA_RELOC_WRITE;
102
103 cs->RS_PIPE_OFFSET[pipe] = VIVS_RS_PIPE_OFFSET_X(0) | VIVS_RS_PIPE_OFFSET_Y(0);
104 }
105
106 cs->RS_DEST_STRIDE = (rs->dest_stride << dest_stride_shift) |
107 COND(rs->dest_tiling & 2, VIVS_RS_DEST_STRIDE_TILING) |
108 COND(dest_multi, VIVS_RS_DEST_STRIDE_MULTI);
109
110 if (ctx->specs.pixel_pipes == 1 || ctx->specs.single_buffer) {
111 cs->RS_WINDOW_SIZE = VIVS_RS_WINDOW_SIZE_WIDTH(rs->width) |
112 VIVS_RS_WINDOW_SIZE_HEIGHT(rs->height);
113 } else if (ctx->specs.pixel_pipes == 2) {
114 assert((rs->height & 7) == 0); /* GPU hangs happen if height not 8-aligned */
115
116 if (source_multi)
117 cs->source[1].offset = rs->source_offset + rs->source_stride * rs->source_padded_height / 2;
118
119 if (dest_multi)
120 cs->dest[1].offset = rs->dest_offset + rs->dest_stride * rs->dest_padded_height / 2;
121
122 cs->RS_WINDOW_SIZE = VIVS_RS_WINDOW_SIZE_WIDTH(rs->width) |
123 VIVS_RS_WINDOW_SIZE_HEIGHT(rs->height / 2);
124 cs->RS_PIPE_OFFSET[1] = VIVS_RS_PIPE_OFFSET_X(0) | VIVS_RS_PIPE_OFFSET_Y(rs->height / 2);
125 } else {
126 abort();
127 }
128
129 cs->RS_DITHER[0] = rs->dither[0];
130 cs->RS_DITHER[1] = rs->dither[1];
131 cs->RS_CLEAR_CONTROL = VIVS_RS_CLEAR_CONTROL_BITS(rs->clear_bits) | rs->clear_mode;
132 cs->RS_FILL_VALUE[0] = rs->clear_value[0];
133 cs->RS_FILL_VALUE[1] = rs->clear_value[1];
134 cs->RS_FILL_VALUE[2] = rs->clear_value[2];
135 cs->RS_FILL_VALUE[3] = rs->clear_value[3];
136 cs->RS_EXTRA_CONFIG = VIVS_RS_EXTRA_CONFIG_AA(rs->aa) |
137 VIVS_RS_EXTRA_CONFIG_ENDIAN(rs->endian_mode);
138
139 /* If source the same as destination, and the hardware supports this,
140 * do an in-place resolve to fill in unrendered tiles.
141 */
142 if (ctx->specs.single_buffer && rs->source == rs->dest &&
143 rs->source_offset == rs->dest_offset &&
144 rs->source_format == rs->dest_format &&
145 rs->source_tiling == rs->dest_tiling &&
146 (rs->source_tiling & ETNA_LAYOUT_BIT_SUPER) &&
147 rs->source_stride == rs->dest_stride &&
148 !rs->downsample_x && !rs->downsample_y &&
149 !rs->swap_rb && !rs->flip &&
150 !rs->clear_mode && rs->source_padded_width) {
151 /* Total number of tiles (same as for autodisable) */
152 cs->RS_KICKER_INPLACE = rs->source_padded_width * rs->source_padded_height / 16;
153 }
154 cs->source_ts_valid = rs->source_ts_valid;
155 }
156
157 /* modify the clear bits value in the compiled RS state */
158 static void
etna_modify_rs_clearbits(struct compiled_rs_state * cs,uint32_t clear_bits)159 etna_modify_rs_clearbits(struct compiled_rs_state *cs, uint32_t clear_bits)
160 {
161 cs->RS_CLEAR_CONTROL &= ~VIVS_RS_CLEAR_CONTROL_BITS__MASK;
162 cs->RS_CLEAR_CONTROL |= VIVS_RS_CLEAR_CONTROL_BITS(clear_bits);
163 }
164
165 #define EMIT_STATE(state_name, src_value) \
166 etna_coalsence_emit(stream, &coalesce, VIVS_##state_name, src_value)
167
168 #define EMIT_STATE_FIXP(state_name, src_value) \
169 etna_coalsence_emit_fixp(stream, &coalesce, VIVS_##state_name, src_value)
170
171 #define EMIT_STATE_RELOC(state_name, src_value) \
172 etna_coalsence_emit_reloc(stream, &coalesce, VIVS_##state_name, src_value)
173
174 /* submit RS state, without any processing and no dependence on context
175 * except TS if this is a source-to-destination blit. */
176 static void
etna_submit_rs_state(struct etna_context * ctx,const struct compiled_rs_state * cs)177 etna_submit_rs_state(struct etna_context *ctx,
178 const struct compiled_rs_state *cs)
179 {
180 struct etna_screen *screen = etna_screen(ctx->base.screen);
181 struct etna_cmd_stream *stream = ctx->stream;
182 struct etna_coalesce coalesce;
183
184 if (cs->RS_KICKER_INPLACE && !cs->source_ts_valid)
185 /* Inplace resolve is no-op if TS is not configured */
186 return;
187
188 ctx->stats.rs_operations++;
189
190 if (cs->RS_KICKER_INPLACE) {
191 etna_cmd_stream_reserve(stream, 6);
192 etna_coalesce_start(stream, &coalesce);
193 /* 0/1 */ EMIT_STATE(RS_EXTRA_CONFIG, cs->RS_EXTRA_CONFIG);
194 /* 2/3 */ EMIT_STATE(RS_SOURCE_STRIDE, cs->RS_SOURCE_STRIDE);
195 /* 4/5 */ EMIT_STATE(RS_KICKER_INPLACE, cs->RS_KICKER_INPLACE);
196 etna_coalesce_end(stream, &coalesce);
197 } else if (screen->specs.pixel_pipes == 1) {
198 etna_cmd_stream_reserve(stream, 22);
199 etna_coalesce_start(stream, &coalesce);
200 /* 0/1 */ EMIT_STATE(RS_CONFIG, cs->RS_CONFIG);
201 /* 2 */ EMIT_STATE_RELOC(RS_SOURCE_ADDR, &cs->source[0]);
202 /* 3 */ EMIT_STATE(RS_SOURCE_STRIDE, cs->RS_SOURCE_STRIDE);
203 /* 4 */ EMIT_STATE_RELOC(RS_DEST_ADDR, &cs->dest[0]);
204 /* 5 */ EMIT_STATE(RS_DEST_STRIDE, cs->RS_DEST_STRIDE);
205 /* 6/7 */ EMIT_STATE(RS_WINDOW_SIZE, cs->RS_WINDOW_SIZE);
206 /* 8/9 */ EMIT_STATE(RS_DITHER(0), cs->RS_DITHER[0]);
207 /*10 */ EMIT_STATE(RS_DITHER(1), cs->RS_DITHER[1]);
208 /*11 - pad */
209 /*12/13*/ EMIT_STATE(RS_CLEAR_CONTROL, cs->RS_CLEAR_CONTROL);
210 /*14 */ EMIT_STATE(RS_FILL_VALUE(0), cs->RS_FILL_VALUE[0]);
211 /*15 */ EMIT_STATE(RS_FILL_VALUE(1), cs->RS_FILL_VALUE[1]);
212 /*16 */ EMIT_STATE(RS_FILL_VALUE(2), cs->RS_FILL_VALUE[2]);
213 /*17 */ EMIT_STATE(RS_FILL_VALUE(3), cs->RS_FILL_VALUE[3]);
214 /*18/19*/ EMIT_STATE(RS_EXTRA_CONFIG, cs->RS_EXTRA_CONFIG);
215 /*20/21*/ EMIT_STATE(RS_KICKER, 0xbeebbeeb);
216 etna_coalesce_end(stream, &coalesce);
217 } else if (screen->specs.pixel_pipes == 2) {
218 etna_cmd_stream_reserve(stream, 34); /* worst case - both pipes multi=1 */
219 etna_coalesce_start(stream, &coalesce);
220 /* 0/1 */ EMIT_STATE(RS_CONFIG, cs->RS_CONFIG);
221 /* 2/3 */ EMIT_STATE(RS_SOURCE_STRIDE, cs->RS_SOURCE_STRIDE);
222 /* 4/5 */ EMIT_STATE(RS_DEST_STRIDE, cs->RS_DEST_STRIDE);
223 /* 6/7 */ EMIT_STATE_RELOC(RS_PIPE_SOURCE_ADDR(0), &cs->source[0]);
224 if (cs->RS_SOURCE_STRIDE & VIVS_RS_SOURCE_STRIDE_MULTI) {
225 /*8 */ EMIT_STATE_RELOC(RS_PIPE_SOURCE_ADDR(1), &cs->source[1]);
226 /*9 - pad */
227 }
228 /*10/11*/ EMIT_STATE_RELOC(RS_PIPE_DEST_ADDR(0), &cs->dest[0]);
229 if (cs->RS_DEST_STRIDE & VIVS_RS_DEST_STRIDE_MULTI) {
230 /*12*/ EMIT_STATE_RELOC(RS_PIPE_DEST_ADDR(1), &cs->dest[1]);
231 /*13 - pad */
232 }
233 /*14/15*/ EMIT_STATE(RS_PIPE_OFFSET(0), cs->RS_PIPE_OFFSET[0]);
234 /*16 */ EMIT_STATE(RS_PIPE_OFFSET(1), cs->RS_PIPE_OFFSET[1]);
235 /*17 - pad */
236 /*18/19*/ EMIT_STATE(RS_WINDOW_SIZE, cs->RS_WINDOW_SIZE);
237 /*20/21*/ EMIT_STATE(RS_DITHER(0), cs->RS_DITHER[0]);
238 /*22 */ EMIT_STATE(RS_DITHER(1), cs->RS_DITHER[1]);
239 /*23 - pad */
240 /*24/25*/ EMIT_STATE(RS_CLEAR_CONTROL, cs->RS_CLEAR_CONTROL);
241 /*26 */ EMIT_STATE(RS_FILL_VALUE(0), cs->RS_FILL_VALUE[0]);
242 /*27 */ EMIT_STATE(RS_FILL_VALUE(1), cs->RS_FILL_VALUE[1]);
243 /*28 */ EMIT_STATE(RS_FILL_VALUE(2), cs->RS_FILL_VALUE[2]);
244 /*29 */ EMIT_STATE(RS_FILL_VALUE(3), cs->RS_FILL_VALUE[3]);
245 /*30/31*/ EMIT_STATE(RS_EXTRA_CONFIG, cs->RS_EXTRA_CONFIG);
246 /*32/33*/ EMIT_STATE(RS_KICKER, 0xbeebbeeb);
247 etna_coalesce_end(stream, &coalesce);
248 } else {
249 abort();
250 }
251 }
252
253 /* Generate clear command for a surface (non-fast clear case) */
254 void
etna_rs_gen_clear_surface(struct etna_context * ctx,struct etna_surface * surf,uint32_t clear_value)255 etna_rs_gen_clear_surface(struct etna_context *ctx, struct etna_surface *surf,
256 uint32_t clear_value)
257 {
258 struct etna_resource *dst = etna_resource(surf->base.texture);
259 uint32_t format = translate_rs_format(surf->base.format);
260
261 if (format == ETNA_NO_MATCH) {
262 BUG("etna_rs_gen_clear_surface: Unhandled clear fmt %s", util_format_name(surf->base.format));
263 format = RS_FORMAT_A8R8G8B8;
264 assert(0);
265 }
266
267 /* use tiled clear if width is multiple of 16 */
268 bool tiled_clear = (surf->surf.padded_width & ETNA_RS_WIDTH_MASK) == 0 &&
269 (surf->surf.padded_height & ETNA_RS_HEIGHT_MASK) == 0;
270
271 etna_compile_rs_state( ctx, &surf->clear_command, &(struct rs_state) {
272 .source_format = format,
273 .dest_format = format,
274 .dest = dst->bo,
275 .dest_offset = surf->surf.offset,
276 .dest_stride = surf->surf.stride,
277 .dest_padded_height = surf->surf.padded_height,
278 .dest_tiling = tiled_clear ? dst->layout : ETNA_LAYOUT_LINEAR,
279 .dither = {0xffffffff, 0xffffffff},
280 .width = surf->surf.padded_width, /* These must be padded to 16x4 if !LINEAR, otherwise RS will hang */
281 .height = surf->surf.padded_height,
282 .clear_value = {clear_value},
283 .clear_mode = VIVS_RS_CLEAR_CONTROL_MODE_ENABLED1,
284 .clear_bits = 0xffff
285 });
286 }
287
288 static void
etna_blit_clear_color_rs(struct pipe_context * pctx,struct pipe_surface * dst,const union pipe_color_union * color)289 etna_blit_clear_color_rs(struct pipe_context *pctx, struct pipe_surface *dst,
290 const union pipe_color_union *color)
291 {
292 struct etna_context *ctx = etna_context(pctx);
293 struct etna_surface *surf = etna_surface(dst);
294 uint32_t new_clear_value = etna_clear_blit_pack_rgba(surf->base.format, color->f);
295
296 if (surf->surf.ts_size) { /* TS: use precompiled clear command */
297 ctx->framebuffer.TS_COLOR_CLEAR_VALUE = new_clear_value;
298
299 if (VIV_FEATURE(ctx->screen, chipMinorFeatures1, AUTO_DISABLE)) {
300 /* Set number of color tiles to be filled */
301 etna_set_state(ctx->stream, VIVS_TS_COLOR_AUTO_DISABLE_COUNT,
302 surf->surf.padded_width * surf->surf.padded_height / 16);
303 ctx->framebuffer.TS_MEM_CONFIG |= VIVS_TS_MEM_CONFIG_COLOR_AUTO_DISABLE;
304 }
305
306 surf->level->ts_valid = true;
307 ctx->dirty |= ETNA_DIRTY_TS | ETNA_DIRTY_DERIVE_TS;
308 } else if (unlikely(new_clear_value != surf->level->clear_value)) { /* Queue normal RS clear for non-TS surfaces */
309 /* If clear color changed, re-generate stored command */
310 etna_rs_gen_clear_surface(ctx, surf, new_clear_value);
311 }
312
313 etna_submit_rs_state(ctx, &surf->clear_command);
314
315 surf->level->clear_value = new_clear_value;
316 resource_written(ctx, surf->base.texture);
317 etna_resource(surf->base.texture)->seqno++;
318 }
319
320 static void
etna_blit_clear_zs_rs(struct pipe_context * pctx,struct pipe_surface * dst,unsigned buffers,double depth,unsigned stencil)321 etna_blit_clear_zs_rs(struct pipe_context *pctx, struct pipe_surface *dst,
322 unsigned buffers, double depth, unsigned stencil)
323 {
324 struct etna_context *ctx = etna_context(pctx);
325 struct etna_surface *surf = etna_surface(dst);
326 uint32_t new_clear_value = translate_clear_depth_stencil(surf->base.format, depth, stencil);
327 uint32_t new_clear_bits = 0, clear_bits_depth, clear_bits_stencil;
328
329 /* Get the channels to clear */
330 switch (surf->base.format) {
331 case PIPE_FORMAT_Z16_UNORM:
332 clear_bits_depth = 0xffff;
333 clear_bits_stencil = 0;
334 break;
335 case PIPE_FORMAT_X8Z24_UNORM:
336 case PIPE_FORMAT_S8_UINT_Z24_UNORM:
337 clear_bits_depth = 0xeeee;
338 clear_bits_stencil = 0x1111;
339 break;
340 default:
341 clear_bits_depth = clear_bits_stencil = 0xffff;
342 break;
343 }
344
345 if (buffers & PIPE_CLEAR_DEPTH)
346 new_clear_bits |= clear_bits_depth;
347 if (buffers & PIPE_CLEAR_STENCIL)
348 new_clear_bits |= clear_bits_stencil;
349 /* FIXME: when tile status is enabled, this becomes more complex as
350 * we may separately clear the depth from the stencil. In this case,
351 * we want to resolve the surface, and avoid using the tile status.
352 * We may be better off recording the pending clear operation,
353 * delaying the actual clear to the first use. This way, we can merge
354 * consecutive clears together. */
355 if (surf->surf.ts_size) { /* TS: use precompiled clear command */
356 /* Set new clear depth value */
357 ctx->framebuffer.TS_DEPTH_CLEAR_VALUE = new_clear_value;
358 if (VIV_FEATURE(ctx->screen, chipMinorFeatures1, AUTO_DISABLE)) {
359 /* Set number of depth tiles to be filled */
360 etna_set_state(ctx->stream, VIVS_TS_DEPTH_AUTO_DISABLE_COUNT,
361 surf->surf.padded_width * surf->surf.padded_height / 16);
362 ctx->framebuffer.TS_MEM_CONFIG |= VIVS_TS_MEM_CONFIG_DEPTH_AUTO_DISABLE;
363 }
364
365 surf->level->ts_valid = true;
366 ctx->dirty |= ETNA_DIRTY_TS | ETNA_DIRTY_DERIVE_TS;
367 } else {
368 if (unlikely(new_clear_value != surf->level->clear_value)) { /* Queue normal RS clear for non-TS surfaces */
369 /* If clear depth value changed, re-generate stored command */
370 etna_rs_gen_clear_surface(ctx, surf, new_clear_value);
371 }
372 /* Update the channels to be cleared */
373 etna_modify_rs_clearbits(&surf->clear_command, new_clear_bits);
374 }
375
376 etna_submit_rs_state(ctx, &surf->clear_command);
377
378 surf->level->clear_value = new_clear_value;
379 resource_written(ctx, surf->base.texture);
380 etna_resource(surf->base.texture)->seqno++;
381 }
382
383 static void
etna_clear_rs(struct pipe_context * pctx,unsigned buffers,const union pipe_color_union * color,double depth,unsigned stencil)384 etna_clear_rs(struct pipe_context *pctx, unsigned buffers,
385 const union pipe_color_union *color, double depth, unsigned stencil)
386 {
387 struct etna_context *ctx = etna_context(pctx);
388
389 /* Flush color and depth cache before clearing anything.
390 * This is especially important when coming from another surface, as
391 * otherwise it may clear part of the old surface instead. */
392 etna_set_state(ctx->stream, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
393 etna_stall(ctx->stream, SYNC_RECIPIENT_RA, SYNC_RECIPIENT_PE);
394
395 /* Preparation: Flush the TS if needed. This must be done after flushing
396 * color and depth, otherwise it can result in crashes */
397 bool need_ts_flush = false;
398 if ((buffers & PIPE_CLEAR_COLOR) && ctx->framebuffer_s.nr_cbufs) {
399 struct etna_surface *surf = etna_surface(ctx->framebuffer_s.cbufs[0]);
400 if (surf->surf.ts_size)
401 need_ts_flush = true;
402 }
403 if ((buffers & PIPE_CLEAR_DEPTHSTENCIL) && ctx->framebuffer_s.zsbuf != NULL) {
404 struct etna_surface *surf = etna_surface(ctx->framebuffer_s.zsbuf);
405
406 if (surf->surf.ts_size)
407 need_ts_flush = true;
408 }
409
410 if (need_ts_flush)
411 etna_set_state(ctx->stream, VIVS_TS_FLUSH_CACHE, VIVS_TS_FLUSH_CACHE_FLUSH);
412
413 /* No need to set up the TS here as RS clear operations (in contrast to
414 * resolve and copy) do not require the TS state.
415 */
416 if (buffers & PIPE_CLEAR_COLOR) {
417 for (int idx = 0; idx < ctx->framebuffer_s.nr_cbufs; ++idx) {
418 etna_blit_clear_color_rs(pctx, ctx->framebuffer_s.cbufs[idx],
419 &color[idx]);
420 }
421 }
422
423 /* Flush the color and depth caches before each RS clear operation
424 * This fixes a hang on GC600. */
425 if (buffers & PIPE_CLEAR_DEPTHSTENCIL && buffers & PIPE_CLEAR_COLOR)
426 etna_set_state(ctx->stream, VIVS_GL_FLUSH_CACHE,
427 VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
428
429 if ((buffers & PIPE_CLEAR_DEPTHSTENCIL) && ctx->framebuffer_s.zsbuf != NULL)
430 etna_blit_clear_zs_rs(pctx, ctx->framebuffer_s.zsbuf, buffers, depth, stencil);
431
432 etna_stall(ctx->stream, SYNC_RECIPIENT_RA, SYNC_RECIPIENT_PE);
433 }
434
435 static bool
etna_manual_blit(struct etna_resource * dst,struct etna_resource_level * dst_lev,unsigned int dst_offset,struct etna_resource * src,struct etna_resource_level * src_lev,unsigned int src_offset,const struct pipe_blit_info * blit_info)436 etna_manual_blit(struct etna_resource *dst, struct etna_resource_level *dst_lev,
437 unsigned int dst_offset, struct etna_resource *src,
438 struct etna_resource_level *src_lev, unsigned int src_offset,
439 const struct pipe_blit_info *blit_info)
440 {
441 void *smap, *srow, *dmap, *drow;
442 size_t tile_size;
443
444 assert(src->layout == ETNA_LAYOUT_TILED);
445 assert(dst->layout == ETNA_LAYOUT_TILED);
446 assert(src->base.nr_samples == 0);
447 assert(dst->base.nr_samples == 0);
448
449 tile_size = util_format_get_blocksize(blit_info->src.format) * 4 * 4;
450
451 smap = etna_bo_map(src->bo);
452 if (!smap)
453 return false;
454
455 dmap = etna_bo_map(dst->bo);
456 if (!dmap)
457 return false;
458
459 srow = smap + src_offset;
460 drow = dmap + dst_offset;
461
462 etna_bo_cpu_prep(src->bo, DRM_ETNA_PREP_READ);
463 etna_bo_cpu_prep(dst->bo, DRM_ETNA_PREP_WRITE);
464
465 for (int y = 0; y < blit_info->src.box.height; y += 4) {
466 memcpy(drow, srow, tile_size * blit_info->src.box.width);
467 srow += src_lev->stride * 4;
468 drow += dst_lev->stride * 4;
469 }
470
471 etna_bo_cpu_fini(dst->bo);
472 etna_bo_cpu_fini(src->bo);
473
474 return true;
475 }
476
477 static inline size_t
etna_compute_tileoffset(const struct pipe_box * box,enum pipe_format format,size_t stride,enum etna_surface_layout layout)478 etna_compute_tileoffset(const struct pipe_box *box, enum pipe_format format,
479 size_t stride, enum etna_surface_layout layout)
480 {
481 size_t offset;
482 unsigned int x = box->x, y = box->y;
483 unsigned int blocksize = util_format_get_blocksize(format);
484
485 switch (layout) {
486 case ETNA_LAYOUT_LINEAR:
487 offset = y * stride + x * blocksize;
488 break;
489 case ETNA_LAYOUT_MULTI_TILED:
490 y >>= 1;
491 /* fall-through */
492 case ETNA_LAYOUT_TILED:
493 assert(!(x & 0x03) && !(y & 0x03));
494 offset = (y & ~0x03) * stride + blocksize * ((x & ~0x03) << 2);
495 break;
496 case ETNA_LAYOUT_MULTI_SUPERTILED:
497 y >>= 1;
498 /* fall-through */
499 case ETNA_LAYOUT_SUPER_TILED:
500 assert(!(x & 0x3f) && !(y & 0x3f));
501 offset = (y & ~0x3f) * stride + blocksize * ((x & ~0x3f) << 6);
502 break;
503 default:
504 unreachable("invalid resource layout");
505 }
506
507 return offset;
508 }
509
510 static inline void
etna_get_rs_alignment_mask(const struct etna_context * ctx,const enum etna_surface_layout layout,unsigned int * width_mask,unsigned int * height_mask)511 etna_get_rs_alignment_mask(const struct etna_context *ctx,
512 const enum etna_surface_layout layout,
513 unsigned int *width_mask, unsigned int *height_mask)
514 {
515 unsigned int h_align, w_align;
516
517 if (layout & ETNA_LAYOUT_BIT_SUPER) {
518 w_align = h_align = 64;
519 } else {
520 w_align = ETNA_RS_WIDTH_MASK + 1;
521 h_align = ETNA_RS_HEIGHT_MASK + 1;
522 }
523
524 h_align *= ctx->screen->specs.pixel_pipes;
525
526 *width_mask = w_align - 1;
527 *height_mask = h_align -1;
528 }
529
530 static bool
etna_try_rs_blit(struct pipe_context * pctx,const struct pipe_blit_info * blit_info)531 etna_try_rs_blit(struct pipe_context *pctx,
532 const struct pipe_blit_info *blit_info)
533 {
534 struct etna_context *ctx = etna_context(pctx);
535 struct etna_resource *src = etna_resource(blit_info->src.resource);
536 struct etna_resource *dst = etna_resource(blit_info->dst.resource);
537 struct compiled_rs_state copy_to_screen;
538 uint32_t ts_mem_config = 0;
539 int msaa_xscale = 1, msaa_yscale = 1;
540
541 /* Ensure that the level is valid */
542 assert(blit_info->src.level <= src->base.last_level);
543 assert(blit_info->dst.level <= dst->base.last_level);
544
545 if (!translate_samples_to_xyscale(src->base.nr_samples, &msaa_xscale, &msaa_yscale, NULL))
546 return FALSE;
547
548 /* The width/height are in pixels; they do not change as a result of
549 * multi-sampling. So, when blitting from a 4x multisampled surface
550 * to a non-multisampled surface, the width and height will be
551 * identical. As we do not support scaling, reject different sizes. */
552 if (blit_info->dst.box.width != blit_info->src.box.width ||
553 blit_info->dst.box.height != blit_info->src.box.height) {
554 DBG("scaling requested: source %dx%d destination %dx%d",
555 blit_info->src.box.width, blit_info->src.box.height,
556 blit_info->dst.box.width, blit_info->dst.box.height);
557 return FALSE;
558 }
559
560 /* No masks - RS can't copy specific channels */
561 unsigned mask = util_format_get_mask(blit_info->dst.format);
562 if ((blit_info->mask & mask) != mask) {
563 DBG("sub-mask requested: 0x%02x vs format mask 0x%02x", blit_info->mask, mask);
564 return FALSE;
565 }
566
567 unsigned src_format = etna_compatible_rs_format(blit_info->src.format);
568 unsigned dst_format = etna_compatible_rs_format(blit_info->dst.format);
569 if (translate_rs_format(src_format) == ETNA_NO_MATCH ||
570 translate_rs_format(dst_format) == ETNA_NO_MATCH ||
571 blit_info->scissor_enable ||
572 blit_info->dst.box.depth != blit_info->src.box.depth ||
573 blit_info->dst.box.depth != 1) {
574 return FALSE;
575 }
576
577 unsigned w_mask, h_mask;
578
579 etna_get_rs_alignment_mask(ctx, src->layout, &w_mask, &h_mask);
580 if ((blit_info->src.box.x & w_mask) || (blit_info->src.box.y & h_mask))
581 return FALSE;
582
583 etna_get_rs_alignment_mask(ctx, dst->layout, &w_mask, &h_mask);
584 if ((blit_info->dst.box.x & w_mask) || (blit_info->dst.box.y & h_mask))
585 return FALSE;
586
587 /* Ensure that the Z coordinate is sane */
588 if (dst->base.target != PIPE_TEXTURE_CUBE)
589 assert(blit_info->dst.box.z == 0);
590 if (src->base.target != PIPE_TEXTURE_CUBE)
591 assert(blit_info->src.box.z == 0);
592
593 assert(blit_info->src.box.z < src->base.array_size);
594 assert(blit_info->dst.box.z < dst->base.array_size);
595
596 struct etna_resource_level *src_lev = &src->levels[blit_info->src.level];
597 struct etna_resource_level *dst_lev = &dst->levels[blit_info->dst.level];
598
599 /* we may be given coordinates up to the padded width to avoid
600 * any alignment issues with different tiling formats */
601 assert((blit_info->src.box.x + blit_info->src.box.width) * msaa_xscale <= src_lev->padded_width);
602 assert((blit_info->src.box.y + blit_info->src.box.height) * msaa_yscale <= src_lev->padded_height);
603 assert(blit_info->dst.box.x + blit_info->dst.box.width <= dst_lev->padded_width);
604 assert(blit_info->dst.box.y + blit_info->dst.box.height <= dst_lev->padded_height);
605
606 unsigned src_offset = src_lev->offset +
607 blit_info->src.box.z * src_lev->layer_stride +
608 etna_compute_tileoffset(&blit_info->src.box,
609 blit_info->src.format,
610 src_lev->stride,
611 src->layout);
612 unsigned dst_offset = dst_lev->offset +
613 blit_info->dst.box.z * dst_lev->layer_stride +
614 etna_compute_tileoffset(&blit_info->dst.box,
615 blit_info->dst.format,
616 dst_lev->stride,
617 dst->layout);
618
619 if (src_lev->padded_width <= ETNA_RS_WIDTH_MASK ||
620 dst_lev->padded_width <= ETNA_RS_WIDTH_MASK ||
621 src_lev->padded_height <= ETNA_RS_HEIGHT_MASK ||
622 dst_lev->padded_height <= ETNA_RS_HEIGHT_MASK)
623 goto manual;
624
625 /* If the width is not aligned to the RS width, but is within our
626 * padding, adjust the width to suite the RS width restriction.
627 * Note: the RS width/height are converted to source samples here. */
628 unsigned int width = blit_info->src.box.width * msaa_xscale;
629 unsigned int height = blit_info->src.box.height * msaa_yscale;
630 unsigned int w_align = ETNA_RS_WIDTH_MASK + 1;
631 unsigned int h_align = (ETNA_RS_HEIGHT_MASK + 1) * ctx->specs.pixel_pipes;
632
633 if (width & (w_align - 1) && width >= src_lev->width * msaa_xscale && width >= dst_lev->width)
634 width = align(width, w_align);
635
636 if (height & (h_align - 1) && height >= src_lev->height * msaa_yscale && height >= dst_lev->height)
637 height = align(height, h_align);
638
639 /* The padded dimensions are in samples */
640 if (width > src_lev->padded_width ||
641 width > dst_lev->padded_width * msaa_xscale ||
642 height > src_lev->padded_height ||
643 height > dst_lev->padded_height * msaa_yscale ||
644 width & (w_align - 1) || height & (h_align - 1))
645 goto manual;
646
647 if (src->base.nr_samples > 1) {
648 uint32_t msaa_format = translate_msaa_format(src_format);
649 assert(msaa_format != ETNA_NO_MATCH);
650 ts_mem_config |= VIVS_TS_MEM_CONFIG_COLOR_COMPRESSION | msaa_format;
651 }
652
653 /* Always flush color and depth cache together before resolving. This works
654 * around artifacts that appear in some cases when scanning out a texture
655 * directly after it has been rendered to, such as rendering an animated web
656 * page in a QtWebEngine based WebView on GC2000. The artifacts look like
657 * the texture sampler samples zeroes instead of texture data in a small,
658 * irregular triangle in the lower right of each browser tile quad. Other
659 * attempts to avoid these artifacts, including a pipeline stall before the
660 * color flush or a TS cache flush afterwards, or flushing multiple times,
661 * with stalls before and after each flush, have shown no effect. */
662 if (src->base.bind & PIPE_BIND_RENDER_TARGET ||
663 src->base.bind & PIPE_BIND_DEPTH_STENCIL) {
664 etna_set_state(ctx->stream, VIVS_GL_FLUSH_CACHE,
665 VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
666 etna_stall(ctx->stream, SYNC_RECIPIENT_RA, SYNC_RECIPIENT_PE);
667
668 if (src->levels[blit_info->src.level].ts_size &&
669 src->levels[blit_info->src.level].ts_valid)
670 etna_set_state(ctx->stream, VIVS_TS_FLUSH_CACHE, VIVS_TS_FLUSH_CACHE_FLUSH);
671 }
672
673 /* Set up color TS to source surface before blit, if needed */
674 bool source_ts_valid = false;
675 if (src->levels[blit_info->src.level].ts_size &&
676 src->levels[blit_info->src.level].ts_valid) {
677 struct etna_reloc reloc;
678 unsigned ts_offset =
679 src_lev->ts_offset + blit_info->src.box.z * src_lev->ts_layer_stride;
680
681 etna_set_state(ctx->stream, VIVS_TS_MEM_CONFIG,
682 VIVS_TS_MEM_CONFIG_COLOR_FAST_CLEAR | ts_mem_config);
683
684 memset(&reloc, 0, sizeof(struct etna_reloc));
685 reloc.bo = src->ts_bo;
686 reloc.offset = ts_offset;
687 reloc.flags = ETNA_RELOC_READ;
688 etna_set_state_reloc(ctx->stream, VIVS_TS_COLOR_STATUS_BASE, &reloc);
689
690 memset(&reloc, 0, sizeof(struct etna_reloc));
691 reloc.bo = src->bo;
692 reloc.offset = src_lev->offset +
693 blit_info->src.box.z * src_lev->layer_stride;
694 reloc.flags = ETNA_RELOC_READ;
695 etna_set_state_reloc(ctx->stream, VIVS_TS_COLOR_SURFACE_BASE, &reloc);
696
697 etna_set_state(ctx->stream, VIVS_TS_COLOR_CLEAR_VALUE,
698 src->levels[blit_info->src.level].clear_value);
699
700 source_ts_valid = true;
701 } else {
702 etna_set_state(ctx->stream, VIVS_TS_MEM_CONFIG, ts_mem_config);
703 }
704 ctx->dirty |= ETNA_DIRTY_TS;
705
706 /* Kick off RS here */
707 etna_compile_rs_state(ctx, ©_to_screen, &(struct rs_state) {
708 .source_format = translate_rs_format(src_format),
709 .source_tiling = src->layout,
710 .source = src->bo,
711 .source_offset = src_offset,
712 .source_stride = src_lev->stride,
713 .source_padded_width = src_lev->padded_width,
714 .source_padded_height = src_lev->padded_height,
715 .source_ts_valid = source_ts_valid,
716 .dest_format = translate_rs_format(dst_format),
717 .dest_tiling = dst->layout,
718 .dest = dst->bo,
719 .dest_offset = dst_offset,
720 .dest_stride = dst_lev->stride,
721 .dest_padded_height = dst_lev->padded_height,
722 .downsample_x = msaa_xscale > 1,
723 .downsample_y = msaa_yscale > 1,
724 .swap_rb = translate_rb_src_dst_swap(src->base.format, dst->base.format),
725 .dither = {0xffffffff, 0xffffffff}, // XXX dither when going from 24 to 16 bit?
726 .clear_mode = VIVS_RS_CLEAR_CONTROL_MODE_DISABLED,
727 .width = width,
728 .height = height
729 });
730
731 etna_submit_rs_state(ctx, ©_to_screen);
732 resource_written(ctx, &dst->base);
733 dst->seqno++;
734 dst->levels[blit_info->dst.level].ts_valid = false;
735 ctx->dirty |= ETNA_DIRTY_DERIVE_TS;
736
737 return TRUE;
738
739 manual:
740 if (src->layout == ETNA_LAYOUT_TILED && dst->layout == ETNA_LAYOUT_TILED) {
741 if ((src->status & ETNA_PENDING_WRITE) ||
742 (dst->status & ETNA_PENDING_WRITE))
743 pctx->flush(pctx, NULL, 0);
744 return etna_manual_blit(dst, dst_lev, dst_offset, src, src_lev, src_offset, blit_info);
745 }
746
747 return FALSE;
748 }
749
750 static void
etna_blit_rs(struct pipe_context * pctx,const struct pipe_blit_info * blit_info)751 etna_blit_rs(struct pipe_context *pctx, const struct pipe_blit_info *blit_info)
752 {
753 /* This is a more extended version of resource_copy_region */
754 /* TODO Some cases can be handled by RS; if not, fall back to rendering or
755 * even CPU copy block of pixels from info->src to info->dst
756 * (resource, level, box, format);
757 * function is used for scaling, flipping in x and y direction (negative
758 * width/height), format conversion, mask and filter and even a scissor rectangle
759 *
760 * What can the RS do for us:
761 * convert between tiling formats (layouts)
762 * downsample 2x in x and y
763 * convert between a limited number of pixel formats
764 *
765 * For the rest, fall back to util_blitter
766 * XXX this goes wrong when source surface is supertiled. */
767 struct etna_context *ctx = etna_context(pctx);
768 struct pipe_blit_info info = *blit_info;
769
770 if (info.src.resource->nr_samples > 1 &&
771 info.dst.resource->nr_samples <= 1 &&
772 !util_format_is_depth_or_stencil(info.src.resource->format) &&
773 !util_format_is_pure_integer(info.src.resource->format)) {
774 DBG("color resolve unimplemented");
775 return;
776 }
777
778 if (etna_try_rs_blit(pctx, blit_info))
779 return;
780
781 if (util_try_blit_via_copy_region(pctx, blit_info))
782 return;
783
784 if (info.mask & PIPE_MASK_S) {
785 DBG("cannot blit stencil, skipping");
786 info.mask &= ~PIPE_MASK_S;
787 }
788
789 if (!util_blitter_is_blit_supported(ctx->blitter, &info)) {
790 DBG("blit unsupported %s -> %s",
791 util_format_short_name(info.src.resource->format),
792 util_format_short_name(info.dst.resource->format));
793 return;
794 }
795
796 etna_blit_save_state(ctx);
797 util_blitter_blit(ctx->blitter, &info);
798 }
799
800 void
etna_clear_blit_rs_init(struct pipe_context * pctx)801 etna_clear_blit_rs_init(struct pipe_context *pctx)
802 {
803 DBG("etnaviv: Using RS blit engine\n");
804 pctx->clear = etna_clear_rs;
805 pctx->blit = etna_blit_rs;
806 }
807