1 // Copyright 2014 Google Inc. All Rights Reserved.
2 //
3 // Use of this source code is governed by a BSD-style license
4 // that can be found in the COPYING file in the root of the source
5 // tree. An additional intellectual property rights grant can be found
6 // in the file PATENTS. All contributing project authors may
7 // be found in the AUTHORS file in the root of the source tree.
8 // -----------------------------------------------------------------------------
9 //
10 // NEON variant of methods for lossless decoder
11 //
12 // Author: Skal (pascal.massimino@gmail.com)
13
14 #include "./dsp.h"
15
16 #if defined(WEBP_USE_NEON)
17
18 #include <arm_neon.h>
19
20 #include "./lossless.h"
21 #include "./neon.h"
22
23 //------------------------------------------------------------------------------
24 // Colorspace conversion functions
25
26 #if !defined(WORK_AROUND_GCC)
27 // gcc 4.6.0 had some trouble (NDK-r9) with this code. We only use it for
28 // gcc-4.8.x at least.
ConvertBGRAToRGBA(const uint32_t * src,int num_pixels,uint8_t * dst)29 static void ConvertBGRAToRGBA(const uint32_t* src,
30 int num_pixels, uint8_t* dst) {
31 const uint32_t* const end = src + (num_pixels & ~15);
32 for (; src < end; src += 16) {
33 uint8x16x4_t pixel = vld4q_u8((uint8_t*)src);
34 // swap B and R. (VSWP d0,d2 has no intrinsics equivalent!)
35 const uint8x16_t tmp = pixel.val[0];
36 pixel.val[0] = pixel.val[2];
37 pixel.val[2] = tmp;
38 vst4q_u8(dst, pixel);
39 dst += 64;
40 }
41 VP8LConvertBGRAToRGBA_C(src, num_pixels & 15, dst); // left-overs
42 }
43
ConvertBGRAToBGR(const uint32_t * src,int num_pixels,uint8_t * dst)44 static void ConvertBGRAToBGR(const uint32_t* src,
45 int num_pixels, uint8_t* dst) {
46 const uint32_t* const end = src + (num_pixels & ~15);
47 for (; src < end; src += 16) {
48 const uint8x16x4_t pixel = vld4q_u8((uint8_t*)src);
49 const uint8x16x3_t tmp = { { pixel.val[0], pixel.val[1], pixel.val[2] } };
50 vst3q_u8(dst, tmp);
51 dst += 48;
52 }
53 VP8LConvertBGRAToBGR_C(src, num_pixels & 15, dst); // left-overs
54 }
55
ConvertBGRAToRGB(const uint32_t * src,int num_pixels,uint8_t * dst)56 static void ConvertBGRAToRGB(const uint32_t* src,
57 int num_pixels, uint8_t* dst) {
58 const uint32_t* const end = src + (num_pixels & ~15);
59 for (; src < end; src += 16) {
60 const uint8x16x4_t pixel = vld4q_u8((uint8_t*)src);
61 const uint8x16x3_t tmp = { { pixel.val[2], pixel.val[1], pixel.val[0] } };
62 vst3q_u8(dst, tmp);
63 dst += 48;
64 }
65 VP8LConvertBGRAToRGB_C(src, num_pixels & 15, dst); // left-overs
66 }
67
68 #else // WORK_AROUND_GCC
69
70 // gcc-4.6.0 fallback
71
72 static const uint8_t kRGBAShuffle[8] = { 2, 1, 0, 3, 6, 5, 4, 7 };
73
ConvertBGRAToRGBA(const uint32_t * src,int num_pixels,uint8_t * dst)74 static void ConvertBGRAToRGBA(const uint32_t* src,
75 int num_pixels, uint8_t* dst) {
76 const uint32_t* const end = src + (num_pixels & ~1);
77 const uint8x8_t shuffle = vld1_u8(kRGBAShuffle);
78 for (; src < end; src += 2) {
79 const uint8x8_t pixels = vld1_u8((uint8_t*)src);
80 vst1_u8(dst, vtbl1_u8(pixels, shuffle));
81 dst += 8;
82 }
83 VP8LConvertBGRAToRGBA_C(src, num_pixels & 1, dst); // left-overs
84 }
85
86 static const uint8_t kBGRShuffle[3][8] = {
87 { 0, 1, 2, 4, 5, 6, 8, 9 },
88 { 10, 12, 13, 14, 16, 17, 18, 20 },
89 { 21, 22, 24, 25, 26, 28, 29, 30 }
90 };
91
ConvertBGRAToBGR(const uint32_t * src,int num_pixels,uint8_t * dst)92 static void ConvertBGRAToBGR(const uint32_t* src,
93 int num_pixels, uint8_t* dst) {
94 const uint32_t* const end = src + (num_pixels & ~7);
95 const uint8x8_t shuffle0 = vld1_u8(kBGRShuffle[0]);
96 const uint8x8_t shuffle1 = vld1_u8(kBGRShuffle[1]);
97 const uint8x8_t shuffle2 = vld1_u8(kBGRShuffle[2]);
98 for (; src < end; src += 8) {
99 uint8x8x4_t pixels;
100 INIT_VECTOR4(pixels,
101 vld1_u8((const uint8_t*)(src + 0)),
102 vld1_u8((const uint8_t*)(src + 2)),
103 vld1_u8((const uint8_t*)(src + 4)),
104 vld1_u8((const uint8_t*)(src + 6)));
105 vst1_u8(dst + 0, vtbl4_u8(pixels, shuffle0));
106 vst1_u8(dst + 8, vtbl4_u8(pixels, shuffle1));
107 vst1_u8(dst + 16, vtbl4_u8(pixels, shuffle2));
108 dst += 8 * 3;
109 }
110 VP8LConvertBGRAToBGR_C(src, num_pixels & 7, dst); // left-overs
111 }
112
113 static const uint8_t kRGBShuffle[3][8] = {
114 { 2, 1, 0, 6, 5, 4, 10, 9 },
115 { 8, 14, 13, 12, 18, 17, 16, 22 },
116 { 21, 20, 26, 25, 24, 30, 29, 28 }
117 };
118
ConvertBGRAToRGB(const uint32_t * src,int num_pixels,uint8_t * dst)119 static void ConvertBGRAToRGB(const uint32_t* src,
120 int num_pixels, uint8_t* dst) {
121 const uint32_t* const end = src + (num_pixels & ~7);
122 const uint8x8_t shuffle0 = vld1_u8(kRGBShuffle[0]);
123 const uint8x8_t shuffle1 = vld1_u8(kRGBShuffle[1]);
124 const uint8x8_t shuffle2 = vld1_u8(kRGBShuffle[2]);
125 for (; src < end; src += 8) {
126 uint8x8x4_t pixels;
127 INIT_VECTOR4(pixels,
128 vld1_u8((const uint8_t*)(src + 0)),
129 vld1_u8((const uint8_t*)(src + 2)),
130 vld1_u8((const uint8_t*)(src + 4)),
131 vld1_u8((const uint8_t*)(src + 6)));
132 vst1_u8(dst + 0, vtbl4_u8(pixels, shuffle0));
133 vst1_u8(dst + 8, vtbl4_u8(pixels, shuffle1));
134 vst1_u8(dst + 16, vtbl4_u8(pixels, shuffle2));
135 dst += 8 * 3;
136 }
137 VP8LConvertBGRAToRGB_C(src, num_pixels & 7, dst); // left-overs
138 }
139
140 #endif // !WORK_AROUND_GCC
141
142 //------------------------------------------------------------------------------
143 // Subtract-Green Transform
144
145 // vtbl?_u8 are marked unavailable for iOS arm64 with Xcode < 6.3, use
146 // non-standard versions there.
147 #if defined(__APPLE__) && defined(__aarch64__) && \
148 defined(__apple_build_version__) && (__apple_build_version__< 6020037)
149 #define USE_VTBLQ
150 #endif
151
152 #ifdef USE_VTBLQ
153 // 255 = byte will be zeroed
154 static const uint8_t kGreenShuffle[16] = {
155 1, 255, 1, 255, 5, 255, 5, 255, 9, 255, 9, 255, 13, 255, 13, 255
156 };
157
DoGreenShuffle(const uint8x16_t argb,const uint8x16_t shuffle)158 static WEBP_INLINE uint8x16_t DoGreenShuffle(const uint8x16_t argb,
159 const uint8x16_t shuffle) {
160 return vcombine_u8(vtbl1q_u8(argb, vget_low_u8(shuffle)),
161 vtbl1q_u8(argb, vget_high_u8(shuffle)));
162 }
163 #else // !USE_VTBLQ
164 // 255 = byte will be zeroed
165 static const uint8_t kGreenShuffle[8] = { 1, 255, 1, 255, 5, 255, 5, 255 };
166
DoGreenShuffle(const uint8x16_t argb,const uint8x8_t shuffle)167 static WEBP_INLINE uint8x16_t DoGreenShuffle(const uint8x16_t argb,
168 const uint8x8_t shuffle) {
169 return vcombine_u8(vtbl1_u8(vget_low_u8(argb), shuffle),
170 vtbl1_u8(vget_high_u8(argb), shuffle));
171 }
172 #endif // USE_VTBLQ
173
AddGreenToBlueAndRed(uint32_t * argb_data,int num_pixels)174 static void AddGreenToBlueAndRed(uint32_t* argb_data, int num_pixels) {
175 const uint32_t* const end = argb_data + (num_pixels & ~3);
176 #ifdef USE_VTBLQ
177 const uint8x16_t shuffle = vld1q_u8(kGreenShuffle);
178 #else
179 const uint8x8_t shuffle = vld1_u8(kGreenShuffle);
180 #endif
181 for (; argb_data < end; argb_data += 4) {
182 const uint8x16_t argb = vld1q_u8((uint8_t*)argb_data);
183 const uint8x16_t greens = DoGreenShuffle(argb, shuffle);
184 vst1q_u8((uint8_t*)argb_data, vaddq_u8(argb, greens));
185 }
186 // fallthrough and finish off with plain-C
187 VP8LAddGreenToBlueAndRed_C(argb_data, num_pixels & 3);
188 }
189
190 //------------------------------------------------------------------------------
191 // Color Transform
192
TransformColorInverse(const VP8LMultipliers * const m,uint32_t * argb_data,int num_pixels)193 static void TransformColorInverse(const VP8LMultipliers* const m,
194 uint32_t* argb_data, int num_pixels) {
195 // sign-extended multiplying constants, pre-shifted by 6.
196 #define CST(X) (((int16_t)(m->X << 8)) >> 6)
197 const int16_t rb[8] = {
198 CST(green_to_blue_), CST(green_to_red_),
199 CST(green_to_blue_), CST(green_to_red_),
200 CST(green_to_blue_), CST(green_to_red_),
201 CST(green_to_blue_), CST(green_to_red_)
202 };
203 const int16x8_t mults_rb = vld1q_s16(rb);
204 const int16_t b2[8] = {
205 0, CST(red_to_blue_), 0, CST(red_to_blue_),
206 0, CST(red_to_blue_), 0, CST(red_to_blue_),
207 };
208 const int16x8_t mults_b2 = vld1q_s16(b2);
209 #undef CST
210 #ifdef USE_VTBLQ
211 static const uint8_t kg0g0[16] = {
212 255, 1, 255, 1, 255, 5, 255, 5, 255, 9, 255, 9, 255, 13, 255, 13
213 };
214 const uint8x16_t shuffle = vld1q_u8(kg0g0);
215 #else
216 static const uint8_t k0g0g[8] = { 255, 1, 255, 1, 255, 5, 255, 5 };
217 const uint8x8_t shuffle = vld1_u8(k0g0g);
218 #endif
219 const uint32x4_t mask_ag = vdupq_n_u32(0xff00ff00u);
220 int i;
221 for (i = 0; i + 4 <= num_pixels; i += 4) {
222 const uint8x16_t in = vld1q_u8((uint8_t*)(argb_data + i));
223 const uint32x4_t a0g0 = vandq_u32(vreinterpretq_u32_u8(in), mask_ag);
224 // 0 g 0 g
225 const uint8x16_t greens = DoGreenShuffle(in, shuffle);
226 // x dr x db1
227 const int16x8_t A = vqdmulhq_s16(vreinterpretq_s16_u8(greens), mults_rb);
228 // x r' x b'
229 const int8x16_t B = vaddq_s8(vreinterpretq_s8_u8(in),
230 vreinterpretq_s8_s16(A));
231 // r' 0 b' 0
232 const int16x8_t C = vshlq_n_s16(vreinterpretq_s16_s8(B), 8);
233 // x db2 0 0
234 const int16x8_t D = vqdmulhq_s16(C, mults_b2);
235 // 0 x db2 0
236 const uint32x4_t E = vshrq_n_u32(vreinterpretq_u32_s16(D), 8);
237 // r' x b'' 0
238 const int8x16_t F = vaddq_s8(vreinterpretq_s8_u32(E),
239 vreinterpretq_s8_s16(C));
240 // 0 r' 0 b''
241 const uint16x8_t G = vshrq_n_u16(vreinterpretq_u16_s8(F), 8);
242 const uint32x4_t out = vorrq_u32(vreinterpretq_u32_u16(G), a0g0);
243 vst1q_u32(argb_data + i, out);
244 }
245 // Fall-back to C-version for left-overs.
246 VP8LTransformColorInverse_C(m, argb_data + i, num_pixels - i);
247 }
248
249 #undef USE_VTBLQ
250
251 //------------------------------------------------------------------------------
252 // Entry point
253
254 extern void VP8LDspInitNEON(void);
255
VP8LDspInitNEON(void)256 WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInitNEON(void) {
257 VP8LConvertBGRAToRGBA = ConvertBGRAToRGBA;
258 VP8LConvertBGRAToBGR = ConvertBGRAToBGR;
259 VP8LConvertBGRAToRGB = ConvertBGRAToRGB;
260
261 VP8LAddGreenToBlueAndRed = AddGreenToBlueAndRed;
262 VP8LTransformColorInverse = TransformColorInverse;
263 }
264
265 #else // !WEBP_USE_NEON
266
267 WEBP_DSP_INIT_STUB(VP8LDspInitNEON)
268
269 #endif // WEBP_USE_NEON
270