1 /*
2 * Copyright (c) 2011-2013 Luc Verhaegen <libv@skynet.be>
3 * Copyright (c) 2018 Alyssa Rosenzweig <alyssa@rosenzweig.io>
4 * Copyright (c) 2018 Vasily Khoruzhick <anarsoul@gmail.com>
5 * Copyright (c) 2019 Collabora, Ltd.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sub license,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
24 * DEALINGS IN THE SOFTWARE.
25 *
26 */
27
28 #include "pan_tiling.h"
29 #include <stdbool.h>
30 #include "util/macros.h"
31
32 /* This file implements software encode/decode of the tiling format used for
33 * textures and framebuffers primarily on Utgard GPUs. Names for this format
34 * include "Utgard-style tiling", "(Mali) swizzled textures", and
35 * "U-interleaved" (the former two names being used in the community
36 * Lima/Panfrost drivers; the latter name used internally at Arm).
37 * Conceptually, like any tiling scheme, the pixel reordering attempts to 2D
38 * spatial locality, to improve cache locality in both horizontal and vertical
39 * directions.
40 *
41 * This format is tiled: first, the image dimensions must be aligned to 16
42 * pixels in each axis. Once aligned, the image is divided into 16x16 tiles.
43 * This size harmonizes with other properties of the GPU; on Midgard,
44 * framebuffer tiles are logically 16x16 (this is the tile size used in
45 * Transaction Elimination and the minimum tile size used in Hierarchical
46 * Tiling). Conversely, for a standard 4 bytes-per-pixel format (like
47 * RGBA8888), 16 pixels * 4 bytes/pixel = 64 bytes, equal to the cache line
48 * size.
49 *
50 * Within each 16x16 block, the bits are reordered according to this pattern:
51 *
52 * | y3 | (x3 ^ y3) | y2 | (y2 ^ x2) | y1 | (y1 ^ x1) | y0 | (y0 ^ x0) |
53 *
54 * Basically, interleaving the X and Y bits, with XORs thrown in for every
55 * adjacent bit pair.
56 *
57 * This is cheap to implement both encode/decode in both hardware and software.
58 * In hardware, lines are simply rerouted to reorder and some XOR gates are
59 * thrown in. Software has to be a bit more clever.
60 *
61 * In software, the trick is to divide the pattern into two lines:
62 *
63 * | y3 | y3 | y2 | y2 | y1 | y1 | y0 | y0 |
64 * ^ | 0 | x3 | 0 | x2 | 0 | x1 | 0 | x0 |
65 *
66 * That is, duplicate the bits of the Y and space out the bits of the X. The
67 * top line is a function only of Y, so it can be calculated once per row and
68 * stored in a register. The bottom line is simply X with the bits spaced out.
69 * Spacing out the X is easy enough with a LUT, or by subtracting+ANDing the
70 * mask pattern (abusing carry bits).
71 *
72 * This format is also supported on Midgard GPUs, where it *can* be used for
73 * textures and framebuffers. That said, in practice it is usually as a
74 * fallback layout; Midgard introduces Arm FrameBuffer Compression, which is
75 * significantly more efficient than Utgard-style tiling and preferred for both
76 * textures and framebuffers, where possible. For unsupported texture types,
77 * for instance sRGB textures and framebuffers, this tiling scheme is used at a
78 * performance penalty, as AFBC is not compatible.
79 */
80
81 /* Given the lower 4-bits of the Y coordinate, we would like to
82 * duplicate every bit over. So instead of 0b1010, we would like
83 * 0b11001100. The idea is that for the bits in the solely Y place, we
84 * get a Y place, and the bits in the XOR place *also* get a Y. */
85
86 const uint32_t bit_duplication[16] = {
87 0b00000000,
88 0b00000011,
89 0b00001100,
90 0b00001111,
91 0b00110000,
92 0b00110011,
93 0b00111100,
94 0b00111111,
95 0b11000000,
96 0b11000011,
97 0b11001100,
98 0b11001111,
99 0b11110000,
100 0b11110011,
101 0b11111100,
102 0b11111111,
103 };
104
105 /* Space the bits out of a 4-bit nibble */
106
107 const unsigned space_4[16] = {
108 0b0000000,
109 0b0000001,
110 0b0000100,
111 0b0000101,
112 0b0010000,
113 0b0010001,
114 0b0010100,
115 0b0010101,
116 0b1000000,
117 0b1000001,
118 0b1000100,
119 0b1000101,
120 0b1010000,
121 0b1010001,
122 0b1010100,
123 0b1010101
124 };
125
126 /* The scheme uses 16x16 tiles */
127
128 #define TILE_WIDTH 16
129 #define TILE_HEIGHT 16
130 #define PIXELS_PER_TILE (TILE_WIDTH * TILE_HEIGHT)
131
132 /* We need a 128-bit type for idiomatically tiling bpp128 formats. The type must
133 * only support copies and sizeof, so emulating with a packed structure works
134 * well enough, but if there's a native 128-bit type we may we well prefer
135 * that. */
136
137 #ifdef __SIZEOF_INT128__
138 typedef __uint128_t pan_uint128_t;
139 #else
140 typedef struct {
141 uint64_t lo;
142 uint64_t hi;
143 } __attribute__((packed)) pan_uint128_t;
144 #endif
145
146 typedef struct {
147 uint16_t lo;
148 uint8_t hi;
149 } __attribute__((packed)) pan_uint24_t;
150
151 /* Optimized routine to tile an aligned (w & 0xF == 0) texture. Explanation:
152 *
153 * dest_start precomputes the offset to the beginning of the first horizontal
154 * tile we're writing to, knowing that x is 16-aligned. Tiles themselves are
155 * stored linearly, so we get the X tile number by shifting and then multiply
156 * by the bytes per tile .
157 *
158 * We iterate across the pixels we're trying to store in source-order. For each
159 * row in the destination image, we figure out which row of 16x16 block we're
160 * in, by slicing off the lower 4-bits (block_y).
161 *
162 * dest then precomputes the location of the top-left corner of the block the
163 * row starts in. In pixel coordinates (where the origin is the top-left),
164 * (block_y, 0) is the top-left corner of the leftmost tile in this row. While
165 * pixels are reordered within a block, the blocks themselves are stored
166 * linearly, so multiplying block_y by the pixel stride of the destination
167 * image equals the byte offset of that top-left corner of the block this row
168 * is in.
169 *
170 * On the other hand, the source is linear so we compute the locations of the
171 * start and end of the row in the source by a simple linear addressing.
172 *
173 * For indexing within the tile, we need to XOR with the [y3 y3 y2 y2 y1 y1 y0
174 * y0] value. Since this is constant across a row, we look it up per-row and
175 * store in expanded_y.
176 *
177 * Finally, we iterate each row in source order. In the outer loop, we iterate
178 * each 16 pixel tile. Within each tile, we iterate the 16 pixels (this should
179 * be unrolled), calculating the index within the tile and writing.
180 */
181
182 #define TILED_ACCESS_TYPE(pixel_t, shift) \
183 static ALWAYS_INLINE void \
184 panfrost_access_tiled_image_##pixel_t \
185 (void *dst, void *src, \
186 uint16_t sx, uint16_t sy, \
187 uint16_t w, uint16_t h, \
188 uint32_t dst_stride, \
189 uint32_t src_stride, \
190 bool is_store) \
191 { \
192 uint8_t *dest_start = dst + ((sx >> 4) * PIXELS_PER_TILE * sizeof(pixel_t)); \
193 for (int y = sy, src_y = 0; src_y < h; ++y, ++src_y) { \
194 uint16_t block_y = y & ~0x0f; \
195 uint8_t *dest = (uint8_t *) (dest_start + (block_y * dst_stride)); \
196 pixel_t *source = src + (src_y * src_stride); \
197 pixel_t *source_end = source + w; \
198 unsigned expanded_y = bit_duplication[y & 0xF] << shift; \
199 for (; source < source_end; dest += (PIXELS_PER_TILE << shift)) { \
200 for (uint8_t i = 0; i < 16; ++i) { \
201 unsigned index = expanded_y ^ (space_4[i] << shift); \
202 if (is_store) \
203 *((pixel_t *) (dest + index)) = *(source++); \
204 else \
205 *(source++) = *((pixel_t *) (dest + index)); \
206 } \
207 } \
208 } \
209 } \
210
211 TILED_ACCESS_TYPE(uint8_t, 0);
212 TILED_ACCESS_TYPE(uint16_t, 1);
213 TILED_ACCESS_TYPE(uint32_t, 2);
214 TILED_ACCESS_TYPE(uint64_t, 3);
215 TILED_ACCESS_TYPE(pan_uint128_t, 4);
216
217 #define TILED_UNALIGNED_TYPE(pixel_t, is_store, tile_shift) { \
218 const unsigned mask = (1 << tile_shift) - 1; \
219 for (int y = sy, src_y = 0; src_y < h; ++y, ++src_y) { \
220 unsigned block_y = y & ~mask; \
221 unsigned block_start_s = block_y * dst_stride; \
222 unsigned source_start = src_y * src_stride; \
223 unsigned expanded_y = bit_duplication[y & mask]; \
224 \
225 for (int x = sx, src_x = 0; src_x < w; ++x, ++src_x) { \
226 unsigned block_x_s = (x >> tile_shift) * (1 << (tile_shift * 2)); \
227 unsigned index = expanded_y ^ space_4[x & mask]; \
228 uint8_t *source = src + source_start + sizeof(pixel_t) * src_x; \
229 uint8_t *dest = dst + block_start_s + sizeof(pixel_t) * (block_x_s + index); \
230 \
231 pixel_t *outp = (pixel_t *) (is_store ? dest : source); \
232 pixel_t *inp = (pixel_t *) (is_store ? source : dest); \
233 *outp = *inp; \
234 } \
235 } \
236 }
237
238 #define TILED_UNALIGNED_TYPES(store, shift) { \
239 if (bpp == 8) \
240 TILED_UNALIGNED_TYPE(uint8_t, store, shift) \
241 else if (bpp == 16) \
242 TILED_UNALIGNED_TYPE(uint16_t, store, shift) \
243 else if (bpp == 24) \
244 TILED_UNALIGNED_TYPE(pan_uint24_t, store, shift) \
245 else if (bpp == 32) \
246 TILED_UNALIGNED_TYPE(uint32_t, store, shift) \
247 else if (bpp == 64) \
248 TILED_UNALIGNED_TYPE(uint64_t, store, shift) \
249 else if (bpp == 128) \
250 TILED_UNALIGNED_TYPE(pan_uint128_t, store, shift) \
251 }
252
253 static void
panfrost_access_tiled_image_generic(void * dst,void * src,unsigned sx,unsigned sy,unsigned w,unsigned h,uint32_t dst_stride,uint32_t src_stride,const struct util_format_description * desc,bool _is_store)254 panfrost_access_tiled_image_generic(void *dst, void *src,
255 unsigned sx, unsigned sy,
256 unsigned w, unsigned h,
257 uint32_t dst_stride,
258 uint32_t src_stride,
259 const struct util_format_description *desc,
260 bool _is_store)
261 {
262 unsigned bpp = desc->block.bits;
263
264 if (desc->block.width > 1) {
265 w = DIV_ROUND_UP(w, desc->block.width);
266 h = DIV_ROUND_UP(h, desc->block.height);
267
268 if (_is_store)
269 TILED_UNALIGNED_TYPES(true, 2)
270 else
271 TILED_UNALIGNED_TYPES(false, 2)
272 } else {
273 if (_is_store)
274 TILED_UNALIGNED_TYPES(true, 4)
275 else
276 TILED_UNALIGNED_TYPES(false, 4)
277 }
278 }
279
280 #define OFFSET(src, _x, _y) (void *) ((uint8_t *) src + ((_y) - orig_y) * src_stride + (((_x) - orig_x) * (bpp / 8)))
281
282 static ALWAYS_INLINE void
panfrost_access_tiled_image(void * dst,void * src,unsigned x,unsigned y,unsigned w,unsigned h,uint32_t dst_stride,uint32_t src_stride,enum pipe_format format,bool is_store)283 panfrost_access_tiled_image(void *dst, void *src,
284 unsigned x, unsigned y,
285 unsigned w, unsigned h,
286 uint32_t dst_stride,
287 uint32_t src_stride,
288 enum pipe_format format,
289 bool is_store)
290 {
291 const struct util_format_description *desc = util_format_description(format);
292
293 if (desc->block.width > 1 || desc->block.bits == 24) {
294 panfrost_access_tiled_image_generic(dst, (void *) src,
295 x, y, w, h,
296 dst_stride, src_stride, desc, is_store);
297
298 return;
299 }
300
301 unsigned bpp = desc->block.bits;
302 unsigned first_full_tile_x = DIV_ROUND_UP(x, TILE_WIDTH) * TILE_WIDTH;
303 unsigned first_full_tile_y = DIV_ROUND_UP(y, TILE_HEIGHT) * TILE_HEIGHT;
304 unsigned last_full_tile_x = ((x + w) / TILE_WIDTH) * TILE_WIDTH;
305 unsigned last_full_tile_y = ((y + h) / TILE_HEIGHT) * TILE_HEIGHT;
306
307 /* First, tile the top portion */
308
309 unsigned orig_x = x, orig_y = y;
310
311 if (first_full_tile_y != y) {
312 unsigned dist = MIN2(first_full_tile_y - y, h);
313
314 panfrost_access_tiled_image_generic(dst, OFFSET(src, x, y),
315 x, y, w, dist,
316 dst_stride, src_stride, desc, is_store);
317
318 if (dist == h)
319 return;
320
321 y += dist;
322 h -= dist;
323 }
324
325 /* Next, the bottom portion */
326 if (last_full_tile_y != (y + h)) {
327 unsigned dist = (y + h) - last_full_tile_y;
328
329 panfrost_access_tiled_image_generic(dst, OFFSET(src, x, last_full_tile_y),
330 x, last_full_tile_y, w, dist,
331 dst_stride, src_stride, desc, is_store);
332
333 h -= dist;
334 }
335
336 /* The left portion */
337 if (first_full_tile_x != x) {
338 unsigned dist = MIN2(first_full_tile_x - x, w);
339
340 panfrost_access_tiled_image_generic(dst, OFFSET(src, x, y),
341 x, y, dist, h,
342 dst_stride, src_stride, desc, is_store);
343
344 if (dist == w)
345 return;
346
347 x += dist;
348 w -= dist;
349 }
350
351 /* Finally, the right portion */
352 if (last_full_tile_x != (x + w)) {
353 unsigned dist = (x + w) - last_full_tile_x;
354
355 panfrost_access_tiled_image_generic(dst, OFFSET(src, last_full_tile_x, y),
356 last_full_tile_x, y, dist, h,
357 dst_stride, src_stride, desc, is_store);
358
359 w -= dist;
360 }
361
362 if (bpp == 8)
363 panfrost_access_tiled_image_uint8_t(dst, OFFSET(src, x, y), x, y, w, h, dst_stride, src_stride, is_store);
364 else if (bpp == 16)
365 panfrost_access_tiled_image_uint16_t(dst, OFFSET(src, x, y), x, y, w, h, dst_stride, src_stride, is_store);
366 else if (bpp == 32)
367 panfrost_access_tiled_image_uint32_t(dst, OFFSET(src, x, y), x, y, w, h, dst_stride, src_stride, is_store);
368 else if (bpp == 64)
369 panfrost_access_tiled_image_uint64_t(dst, OFFSET(src, x, y), x, y, w, h, dst_stride, src_stride, is_store);
370 else if (bpp == 128)
371 panfrost_access_tiled_image_pan_uint128_t(dst, OFFSET(src, x, y), x, y, w, h, dst_stride, src_stride, is_store);
372 }
373
374 void
panfrost_store_tiled_image(void * dst,const void * src,unsigned x,unsigned y,unsigned w,unsigned h,uint32_t dst_stride,uint32_t src_stride,enum pipe_format format)375 panfrost_store_tiled_image(void *dst, const void *src,
376 unsigned x, unsigned y,
377 unsigned w, unsigned h,
378 uint32_t dst_stride,
379 uint32_t src_stride,
380 enum pipe_format format)
381 {
382 panfrost_access_tiled_image(dst, (void *) src,
383 x, y, w, h,
384 dst_stride, src_stride, format, true);
385 }
386
387 void
panfrost_load_tiled_image(void * dst,const void * src,unsigned x,unsigned y,unsigned w,unsigned h,uint32_t dst_stride,uint32_t src_stride,enum pipe_format format)388 panfrost_load_tiled_image(void *dst, const void *src,
389 unsigned x, unsigned y,
390 unsigned w, unsigned h,
391 uint32_t dst_stride,
392 uint32_t src_stride,
393 enum pipe_format format)
394 {
395 panfrost_access_tiled_image((void *) src, dst,
396 x, y, w, h,
397 src_stride, dst_stride, format, false);
398 }
399