1 // Copyright 2011 Google Inc. All Rights Reserved.
2 //
3 // Use of this source code is governed by a BSD-style license
4 // that can be found in the COPYING file in the root of the source
5 // tree. An additional intellectual property rights grant can be found
6 // in the file PATENTS. All contributing project authors may
7 // be found in the AUTHORS file in the root of the source tree.
8 // -----------------------------------------------------------------------------
9 //
10 // NEON version of YUV to RGB upsampling functions.
11 //
12 // Author: mans@mansr.com (Mans Rullgard)
13 // Based on SSE code by: somnath@google.com (Somnath Banerjee)
14
15 #include "./dsp.h"
16
17 #if defined(WEBP_USE_NEON)
18
19 #include <assert.h>
20 #include <arm_neon.h>
21 #include <string.h>
22 #include "./neon.h"
23 #include "./yuv.h"
24
25 #ifdef FANCY_UPSAMPLING
26
27 //-----------------------------------------------------------------------------
28 // U/V upsampling
29
30 // Loads 9 pixels each from rows r1 and r2 and generates 16 pixels.
31 #define UPSAMPLE_16PIXELS(r1, r2, out) { \
32 uint8x8_t a = vld1_u8(r1); \
33 uint8x8_t b = vld1_u8(r1 + 1); \
34 uint8x8_t c = vld1_u8(r2); \
35 uint8x8_t d = vld1_u8(r2 + 1); \
36 \
37 uint16x8_t al = vshll_n_u8(a, 1); \
38 uint16x8_t bl = vshll_n_u8(b, 1); \
39 uint16x8_t cl = vshll_n_u8(c, 1); \
40 uint16x8_t dl = vshll_n_u8(d, 1); \
41 \
42 uint8x8_t diag1, diag2; \
43 uint16x8_t sl; \
44 \
45 /* a + b + c + d */ \
46 sl = vaddl_u8(a, b); \
47 sl = vaddw_u8(sl, c); \
48 sl = vaddw_u8(sl, d); \
49 \
50 al = vaddq_u16(sl, al); /* 3a + b + c + d */ \
51 bl = vaddq_u16(sl, bl); /* a + 3b + c + d */ \
52 \
53 al = vaddq_u16(al, dl); /* 3a + b + c + 3d */ \
54 bl = vaddq_u16(bl, cl); /* a + 3b + 3c + d */ \
55 \
56 diag2 = vshrn_n_u16(al, 3); \
57 diag1 = vshrn_n_u16(bl, 3); \
58 \
59 a = vrhadd_u8(a, diag1); \
60 b = vrhadd_u8(b, diag2); \
61 c = vrhadd_u8(c, diag2); \
62 d = vrhadd_u8(d, diag1); \
63 \
64 { \
65 uint8x8x2_t a_b, c_d; \
66 INIT_VECTOR2(a_b, a, b); \
67 INIT_VECTOR2(c_d, c, d); \
68 vst2_u8(out, a_b); \
69 vst2_u8(out + 32, c_d); \
70 } \
71 }
72
73 // Turn the macro into a function for reducing code-size when non-critical
Upsample16Pixels(const uint8_t * r1,const uint8_t * r2,uint8_t * out)74 static void Upsample16Pixels(const uint8_t *r1, const uint8_t *r2,
75 uint8_t *out) {
76 UPSAMPLE_16PIXELS(r1, r2, out);
77 }
78
79 #define UPSAMPLE_LAST_BLOCK(tb, bb, num_pixels, out) { \
80 uint8_t r1[9], r2[9]; \
81 memcpy(r1, (tb), (num_pixels)); \
82 memcpy(r2, (bb), (num_pixels)); \
83 /* replicate last byte */ \
84 memset(r1 + (num_pixels), r1[(num_pixels) - 1], 9 - (num_pixels)); \
85 memset(r2 + (num_pixels), r2[(num_pixels) - 1], 9 - (num_pixels)); \
86 Upsample16Pixels(r1, r2, out); \
87 }
88
89 //-----------------------------------------------------------------------------
90 // YUV->RGB conversion
91
92 // note: we represent the 33050 large constant as 32768 + 282
93 static const int16_t kCoeffs1[4] = { 19077, 26149, 6419, 13320 };
94
95 #define v255 vdup_n_u8(255)
96 #define v_0x0f vdup_n_u8(15)
97
98 #define STORE_Rgb(out, r, g, b) do { \
99 uint8x8x3_t r_g_b; \
100 INIT_VECTOR3(r_g_b, r, g, b); \
101 vst3_u8(out, r_g_b); \
102 } while (0)
103
104 #define STORE_Bgr(out, r, g, b) do { \
105 uint8x8x3_t b_g_r; \
106 INIT_VECTOR3(b_g_r, b, g, r); \
107 vst3_u8(out, b_g_r); \
108 } while (0)
109
110 #define STORE_Rgba(out, r, g, b) do { \
111 uint8x8x4_t r_g_b_v255; \
112 INIT_VECTOR4(r_g_b_v255, r, g, b, v255); \
113 vst4_u8(out, r_g_b_v255); \
114 } while (0)
115
116 #define STORE_Bgra(out, r, g, b) do { \
117 uint8x8x4_t b_g_r_v255; \
118 INIT_VECTOR4(b_g_r_v255, b, g, r, v255); \
119 vst4_u8(out, b_g_r_v255); \
120 } while (0)
121
122 #define STORE_Argb(out, r, g, b) do { \
123 uint8x8x4_t v255_r_g_b; \
124 INIT_VECTOR4(v255_r_g_b, v255, r, g, b); \
125 vst4_u8(out, v255_r_g_b); \
126 } while (0)
127
128 #if !defined(WEBP_SWAP_16BIT_CSP)
129 #define ZIP_U8(lo, hi) vzip_u8((lo), (hi))
130 #else
131 #define ZIP_U8(lo, hi) vzip_u8((hi), (lo))
132 #endif
133
134 #define STORE_Rgba4444(out, r, g, b) do { \
135 const uint8x8_t r1 = vshl_n_u8(vshr_n_u8(r, 4), 4); /* 4bits */ \
136 const uint8x8_t g1 = vshr_n_u8(g, 4); \
137 const uint8x8_t ba = vorr_u8(b, v_0x0f); \
138 const uint8x8_t rg = vorr_u8(r1, g1); \
139 const uint8x8x2_t rgba4444 = ZIP_U8(rg, ba); \
140 vst1q_u8(out, vcombine_u8(rgba4444.val[0], rgba4444.val[1])); \
141 } while (0)
142
143 #define STORE_Rgb565(out, r, g, b) do { \
144 const uint8x8_t r1 = vshl_n_u8(vshr_n_u8(r, 3), 3); /* 5bits */ \
145 const uint8x8_t g1 = vshr_n_u8(g, 5); /* upper 3bits */\
146 const uint8x8_t g2 = vshl_n_u8(vshr_n_u8(g, 2), 5); /* lower 3bits */\
147 const uint8x8_t b1 = vshr_n_u8(b, 3); /* 5bits */ \
148 const uint8x8_t rg = vorr_u8(r1, g1); \
149 const uint8x8_t gb = vorr_u8(g2, b1); \
150 const uint8x8x2_t rgb565 = ZIP_U8(rg, gb); \
151 vst1q_u8(out, vcombine_u8(rgb565.val[0], rgb565.val[1])); \
152 } while (0)
153
154 #define CONVERT8(FMT, XSTEP, N, src_y, src_uv, out, cur_x) do { \
155 int i; \
156 for (i = 0; i < N; i += 8) { \
157 const int off = ((cur_x) + i) * XSTEP; \
158 const uint8x8_t y = vld1_u8((src_y) + (cur_x) + i); \
159 const uint8x8_t u = vld1_u8((src_uv) + i + 0); \
160 const uint8x8_t v = vld1_u8((src_uv) + i + 16); \
161 const int16x8_t Y0 = vreinterpretq_s16_u16(vshll_n_u8(y, 7)); \
162 const int16x8_t U0 = vreinterpretq_s16_u16(vshll_n_u8(u, 7)); \
163 const int16x8_t V0 = vreinterpretq_s16_u16(vshll_n_u8(v, 7)); \
164 const int16x8_t Y1 = vqdmulhq_lane_s16(Y0, coeff1, 0); \
165 const int16x8_t R0 = vqdmulhq_lane_s16(V0, coeff1, 1); \
166 const int16x8_t G0 = vqdmulhq_lane_s16(U0, coeff1, 2); \
167 const int16x8_t G1 = vqdmulhq_lane_s16(V0, coeff1, 3); \
168 const int16x8_t B0 = vqdmulhq_n_s16(U0, 282); \
169 const int16x8_t R1 = vqaddq_s16(Y1, R_Rounder); \
170 const int16x8_t G2 = vqaddq_s16(Y1, G_Rounder); \
171 const int16x8_t B1 = vqaddq_s16(Y1, B_Rounder); \
172 const int16x8_t R2 = vqaddq_s16(R0, R1); \
173 const int16x8_t G3 = vqaddq_s16(G0, G1); \
174 const int16x8_t B2 = vqaddq_s16(B0, B1); \
175 const int16x8_t G4 = vqsubq_s16(G2, G3); \
176 const int16x8_t B3 = vqaddq_s16(B2, U0); \
177 const uint8x8_t R = vqshrun_n_s16(R2, YUV_FIX2); \
178 const uint8x8_t G = vqshrun_n_s16(G4, YUV_FIX2); \
179 const uint8x8_t B = vqshrun_n_s16(B3, YUV_FIX2); \
180 STORE_ ## FMT(out + off, R, G, B); \
181 } \
182 } while (0)
183
184 #define CONVERT1(FUNC, XSTEP, N, src_y, src_uv, rgb, cur_x) { \
185 int i; \
186 for (i = 0; i < N; i++) { \
187 const int off = ((cur_x) + i) * XSTEP; \
188 const int y = src_y[(cur_x) + i]; \
189 const int u = (src_uv)[i]; \
190 const int v = (src_uv)[i + 16]; \
191 FUNC(y, u, v, rgb + off); \
192 } \
193 }
194
195 #define CONVERT2RGB_8(FMT, XSTEP, top_y, bottom_y, uv, \
196 top_dst, bottom_dst, cur_x, len) { \
197 CONVERT8(FMT, XSTEP, len, top_y, uv, top_dst, cur_x); \
198 if (bottom_y != NULL) { \
199 CONVERT8(FMT, XSTEP, len, bottom_y, (uv) + 32, bottom_dst, cur_x); \
200 } \
201 }
202
203 #define CONVERT2RGB_1(FUNC, XSTEP, top_y, bottom_y, uv, \
204 top_dst, bottom_dst, cur_x, len) { \
205 CONVERT1(FUNC, XSTEP, len, top_y, uv, top_dst, cur_x); \
206 if (bottom_y != NULL) { \
207 CONVERT1(FUNC, XSTEP, len, bottom_y, (uv) + 32, bottom_dst, cur_x); \
208 } \
209 }
210
211 #define NEON_UPSAMPLE_FUNC(FUNC_NAME, FMT, XSTEP) \
212 static void FUNC_NAME(const uint8_t *top_y, const uint8_t *bottom_y, \
213 const uint8_t *top_u, const uint8_t *top_v, \
214 const uint8_t *cur_u, const uint8_t *cur_v, \
215 uint8_t *top_dst, uint8_t *bottom_dst, int len) { \
216 int block; \
217 /* 16 byte aligned array to cache reconstructed u and v */ \
218 uint8_t uv_buf[2 * 32 + 15]; \
219 uint8_t *const r_uv = (uint8_t*)((uintptr_t)(uv_buf + 15) & ~15); \
220 const int uv_len = (len + 1) >> 1; \
221 /* 9 pixels must be read-able for each block */ \
222 const int num_blocks = (uv_len - 1) >> 3; \
223 const int leftover = uv_len - num_blocks * 8; \
224 const int last_pos = 1 + 16 * num_blocks; \
225 \
226 const int u_diag = ((top_u[0] + cur_u[0]) >> 1) + 1; \
227 const int v_diag = ((top_v[0] + cur_v[0]) >> 1) + 1; \
228 \
229 const int16x4_t coeff1 = vld1_s16(kCoeffs1); \
230 const int16x8_t R_Rounder = vdupq_n_s16(-14234); \
231 const int16x8_t G_Rounder = vdupq_n_s16(8708); \
232 const int16x8_t B_Rounder = vdupq_n_s16(-17685); \
233 \
234 /* Treat the first pixel in regular way */ \
235 assert(top_y != NULL); \
236 { \
237 const int u0 = (top_u[0] + u_diag) >> 1; \
238 const int v0 = (top_v[0] + v_diag) >> 1; \
239 VP8YuvTo ## FMT(top_y[0], u0, v0, top_dst); \
240 } \
241 if (bottom_y != NULL) { \
242 const int u0 = (cur_u[0] + u_diag) >> 1; \
243 const int v0 = (cur_v[0] + v_diag) >> 1; \
244 VP8YuvTo ## FMT(bottom_y[0], u0, v0, bottom_dst); \
245 } \
246 \
247 for (block = 0; block < num_blocks; ++block) { \
248 UPSAMPLE_16PIXELS(top_u, cur_u, r_uv); \
249 UPSAMPLE_16PIXELS(top_v, cur_v, r_uv + 16); \
250 CONVERT2RGB_8(FMT, XSTEP, top_y, bottom_y, r_uv, \
251 top_dst, bottom_dst, 16 * block + 1, 16); \
252 top_u += 8; \
253 cur_u += 8; \
254 top_v += 8; \
255 cur_v += 8; \
256 } \
257 \
258 UPSAMPLE_LAST_BLOCK(top_u, cur_u, leftover, r_uv); \
259 UPSAMPLE_LAST_BLOCK(top_v, cur_v, leftover, r_uv + 16); \
260 CONVERT2RGB_1(VP8YuvTo ## FMT, XSTEP, top_y, bottom_y, r_uv, \
261 top_dst, bottom_dst, last_pos, len - last_pos); \
262 }
263
264 // NEON variants of the fancy upsampler.
265 NEON_UPSAMPLE_FUNC(UpsampleRgbLinePair, Rgb, 3)
266 NEON_UPSAMPLE_FUNC(UpsampleBgrLinePair, Bgr, 3)
267 NEON_UPSAMPLE_FUNC(UpsampleRgbaLinePair, Rgba, 4)
268 NEON_UPSAMPLE_FUNC(UpsampleBgraLinePair, Bgra, 4)
269 NEON_UPSAMPLE_FUNC(UpsampleArgbLinePair, Argb, 4)
270 NEON_UPSAMPLE_FUNC(UpsampleRgba4444LinePair, Rgba4444, 2)
271 NEON_UPSAMPLE_FUNC(UpsampleRgb565LinePair, Rgb565, 2)
272
273 //------------------------------------------------------------------------------
274 // Entry point
275
276 extern WebPUpsampleLinePairFunc WebPUpsamplers[/* MODE_LAST */];
277
278 extern void WebPInitUpsamplersNEON(void);
279
WebPInitUpsamplersNEON(void)280 WEBP_TSAN_IGNORE_FUNCTION void WebPInitUpsamplersNEON(void) {
281 WebPUpsamplers[MODE_RGB] = UpsampleRgbLinePair;
282 WebPUpsamplers[MODE_RGBA] = UpsampleRgbaLinePair;
283 WebPUpsamplers[MODE_BGR] = UpsampleBgrLinePair;
284 WebPUpsamplers[MODE_BGRA] = UpsampleBgraLinePair;
285 WebPUpsamplers[MODE_ARGB] = UpsampleArgbLinePair;
286 WebPUpsamplers[MODE_rgbA] = UpsampleRgbaLinePair;
287 WebPUpsamplers[MODE_bgrA] = UpsampleBgraLinePair;
288 WebPUpsamplers[MODE_Argb] = UpsampleArgbLinePair;
289 WebPUpsamplers[MODE_RGB_565] = UpsampleRgb565LinePair;
290 WebPUpsamplers[MODE_RGBA_4444] = UpsampleRgba4444LinePair;
291 WebPUpsamplers[MODE_rgbA_4444] = UpsampleRgba4444LinePair;
292 }
293
294 #endif // FANCY_UPSAMPLING
295
296 #endif // WEBP_USE_NEON
297
298 #if !(defined(FANCY_UPSAMPLING) && defined(WEBP_USE_NEON))
299 WEBP_DSP_INIT_STUB(WebPInitUpsamplersNEON)
300 #endif
301