1 // Copyright 2016 Google Inc. All Rights Reserved.
2 //
3 // Use of this source code is governed by a BSD-style license
4 // that can be found in the COPYING file in the root of the source
5 // tree. An additional intellectual property rights grant can be found
6 // in the file PATENTS. All contributing project authors may
7 // be found in the AUTHORS file in the root of the source tree.
8 // -----------------------------------------------------------------------------
9 //
10 // MSA version of YUV to RGB upsampling functions.
11 //
12 // Author: Prashant Patil (prashant.patil@imgtec.com)
13
14 #include <string.h>
15 #include "src/dsp/dsp.h"
16
17 #if defined(WEBP_USE_MSA)
18
19 #include "src/dsp/msa_macro.h"
20 #include "src/dsp/yuv.h"
21
22 #ifdef FANCY_UPSAMPLING
23
24 #define ILVR_UW2(in, out0, out1) do { \
25 const v8i16 t0 = (v8i16)__msa_ilvr_b((v16i8)zero, (v16i8)in); \
26 out0 = (v4u32)__msa_ilvr_h((v8i16)zero, t0); \
27 out1 = (v4u32)__msa_ilvl_h((v8i16)zero, t0); \
28 } while (0)
29
30 #define ILVRL_UW4(in, out0, out1, out2, out3) do { \
31 v16u8 t0, t1; \
32 ILVRL_B2_UB(zero, in, t0, t1); \
33 ILVRL_H2_UW(zero, t0, out0, out1); \
34 ILVRL_H2_UW(zero, t1, out2, out3); \
35 } while (0)
36
37 #define MULTHI_16(in0, in1, in2, in3, cnst, out0, out1) do { \
38 const v4i32 const0 = (v4i32)__msa_fill_w(cnst * 256); \
39 v4u32 temp0, temp1, temp2, temp3; \
40 MUL4(in0, const0, in1, const0, in2, const0, in3, const0, \
41 temp0, temp1, temp2, temp3); \
42 PCKOD_H2_UH(temp1, temp0, temp3, temp2, out0, out1); \
43 } while (0)
44
45 #define MULTHI_8(in0, in1, cnst, out0) do { \
46 const v4i32 const0 = (v4i32)__msa_fill_w(cnst * 256); \
47 v4u32 temp0, temp1; \
48 MUL2(in0, const0, in1, const0, temp0, temp1); \
49 out0 = (v8u16)__msa_pckod_h((v8i16)temp1, (v8i16)temp0); \
50 } while (0)
51
52 #define CALC_R16(y0, y1, v0, v1, dst) do { \
53 const v8i16 const_a = (v8i16)__msa_fill_h(14234); \
54 const v8i16 a0 = __msa_adds_s_h((v8i16)y0, (v8i16)v0); \
55 const v8i16 a1 = __msa_adds_s_h((v8i16)y1, (v8i16)v1); \
56 v8i16 b0 = __msa_subs_s_h(a0, const_a); \
57 v8i16 b1 = __msa_subs_s_h(a1, const_a); \
58 SRAI_H2_SH(b0, b1, 6); \
59 CLIP_SH2_0_255(b0, b1); \
60 dst = (v16u8)__msa_pckev_b((v16i8)b1, (v16i8)b0); \
61 } while (0)
62
63 #define CALC_R8(y0, v0, dst) do { \
64 const v8i16 const_a = (v8i16)__msa_fill_h(14234); \
65 const v8i16 a0 = __msa_adds_s_h((v8i16)y0, (v8i16)v0); \
66 v8i16 b0 = __msa_subs_s_h(a0, const_a); \
67 b0 = SRAI_H(b0, 6); \
68 CLIP_SH_0_255(b0); \
69 dst = (v16u8)__msa_pckev_b((v16i8)b0, (v16i8)b0); \
70 } while (0)
71
72 #define CALC_G16(y0, y1, u0, u1, v0, v1, dst) do { \
73 const v8i16 const_a = (v8i16)__msa_fill_h(8708); \
74 v8i16 a0 = __msa_subs_s_h((v8i16)y0, (v8i16)u0); \
75 v8i16 a1 = __msa_subs_s_h((v8i16)y1, (v8i16)u1); \
76 const v8i16 b0 = __msa_subs_s_h(a0, (v8i16)v0); \
77 const v8i16 b1 = __msa_subs_s_h(a1, (v8i16)v1); \
78 a0 = __msa_adds_s_h(b0, const_a); \
79 a1 = __msa_adds_s_h(b1, const_a); \
80 SRAI_H2_SH(a0, a1, 6); \
81 CLIP_SH2_0_255(a0, a1); \
82 dst = (v16u8)__msa_pckev_b((v16i8)a1, (v16i8)a0); \
83 } while (0)
84
85 #define CALC_G8(y0, u0, v0, dst) do { \
86 const v8i16 const_a = (v8i16)__msa_fill_h(8708); \
87 v8i16 a0 = __msa_subs_s_h((v8i16)y0, (v8i16)u0); \
88 const v8i16 b0 = __msa_subs_s_h(a0, (v8i16)v0); \
89 a0 = __msa_adds_s_h(b0, const_a); \
90 a0 = SRAI_H(a0, 6); \
91 CLIP_SH_0_255(a0); \
92 dst = (v16u8)__msa_pckev_b((v16i8)a0, (v16i8)a0); \
93 } while (0)
94
95 #define CALC_B16(y0, y1, u0, u1, dst) do { \
96 const v8u16 const_a = (v8u16)__msa_fill_h(17685); \
97 const v8u16 a0 = __msa_adds_u_h((v8u16)y0, u0); \
98 const v8u16 a1 = __msa_adds_u_h((v8u16)y1, u1); \
99 v8u16 b0 = __msa_subs_u_h(a0, const_a); \
100 v8u16 b1 = __msa_subs_u_h(a1, const_a); \
101 SRAI_H2_UH(b0, b1, 6); \
102 CLIP_UH2_0_255(b0, b1); \
103 dst = (v16u8)__msa_pckev_b((v16i8)b1, (v16i8)b0); \
104 } while (0)
105
106 #define CALC_B8(y0, u0, dst) do { \
107 const v8u16 const_a = (v8u16)__msa_fill_h(17685); \
108 const v8u16 a0 = __msa_adds_u_h((v8u16)y0, u0); \
109 v8u16 b0 = __msa_subs_u_h(a0, const_a); \
110 b0 = SRAI_H(b0, 6); \
111 CLIP_UH_0_255(b0); \
112 dst = (v16u8)__msa_pckev_b((v16i8)b0, (v16i8)b0); \
113 } while (0)
114
115 #define CALC_RGB16(y, u, v, R, G, B) do { \
116 const v16u8 zero = { 0 }; \
117 v8u16 y0, y1, u0, u1, v0, v1; \
118 v4u32 p0, p1, p2, p3; \
119 const v16u8 in_y = LD_UB(y); \
120 const v16u8 in_u = LD_UB(u); \
121 const v16u8 in_v = LD_UB(v); \
122 ILVRL_UW4(in_y, p0, p1, p2, p3); \
123 MULTHI_16(p0, p1, p2, p3, 19077, y0, y1); \
124 ILVRL_UW4(in_v, p0, p1, p2, p3); \
125 MULTHI_16(p0, p1, p2, p3, 26149, v0, v1); \
126 CALC_R16(y0, y1, v0, v1, R); \
127 MULTHI_16(p0, p1, p2, p3, 13320, v0, v1); \
128 ILVRL_UW4(in_u, p0, p1, p2, p3); \
129 MULTHI_16(p0, p1, p2, p3, 6419, u0, u1); \
130 CALC_G16(y0, y1, u0, u1, v0, v1, G); \
131 MULTHI_16(p0, p1, p2, p3, 33050, u0, u1); \
132 CALC_B16(y0, y1, u0, u1, B); \
133 } while (0)
134
135 #define CALC_RGB8(y, u, v, R, G, B) do { \
136 const v16u8 zero = { 0 }; \
137 v8u16 y0, u0, v0; \
138 v4u32 p0, p1; \
139 const v16u8 in_y = LD_UB(y); \
140 const v16u8 in_u = LD_UB(u); \
141 const v16u8 in_v = LD_UB(v); \
142 ILVR_UW2(in_y, p0, p1); \
143 MULTHI_8(p0, p1, 19077, y0); \
144 ILVR_UW2(in_v, p0, p1); \
145 MULTHI_8(p0, p1, 26149, v0); \
146 CALC_R8(y0, v0, R); \
147 MULTHI_8(p0, p1, 13320, v0); \
148 ILVR_UW2(in_u, p0, p1); \
149 MULTHI_8(p0, p1, 6419, u0); \
150 CALC_G8(y0, u0, v0, G); \
151 MULTHI_8(p0, p1, 33050, u0); \
152 CALC_B8(y0, u0, B); \
153 } while (0)
154
155 #define STORE16_3(a0, a1, a2, dst) do { \
156 const v16u8 mask0 = { 0, 1, 16, 2, 3, 17, 4, 5, 18, 6, 7, 19, \
157 8, 9, 20, 10 }; \
158 const v16u8 mask1 = { 0, 21, 1, 2, 22, 3, 4, 23, 5, 6, 24, 7, \
159 8, 25, 9, 10 }; \
160 const v16u8 mask2 = { 26, 0, 1, 27, 2, 3, 28, 4, 5, 29, 6, 7, \
161 30, 8, 9, 31 }; \
162 v16u8 out0, out1, out2, tmp0, tmp1, tmp2; \
163 ILVRL_B2_UB(a1, a0, tmp0, tmp1); \
164 out0 = VSHF_UB(tmp0, a2, mask0); \
165 tmp2 = SLDI_UB(tmp1, tmp0, 11); \
166 out1 = VSHF_UB(tmp2, a2, mask1); \
167 tmp2 = SLDI_UB(tmp1, tmp1, 6); \
168 out2 = VSHF_UB(tmp2, a2, mask2); \
169 ST_UB(out0, dst + 0); \
170 ST_UB(out1, dst + 16); \
171 ST_UB(out2, dst + 32); \
172 } while (0)
173
174 #define STORE8_3(a0, a1, a2, dst) do { \
175 int64_t out_m; \
176 const v16u8 mask0 = { 0, 1, 16, 2, 3, 17, 4, 5, 18, 6, 7, 19, \
177 8, 9, 20, 10 }; \
178 const v16u8 mask1 = { 11, 21, 12, 13, 22, 14, 15, 23, \
179 255, 255, 255, 255, 255, 255, 255, 255 }; \
180 const v16u8 tmp0 = (v16u8)__msa_ilvr_b((v16i8)a1, (v16i8)a0); \
181 v16u8 out0, out1; \
182 VSHF_B2_UB(tmp0, a2, tmp0, a2, mask0, mask1, out0, out1); \
183 ST_UB(out0, dst); \
184 out_m = __msa_copy_s_d((v2i64)out1, 0); \
185 SD(out_m, dst + 16); \
186 } while (0)
187
188 #define STORE16_4(a0, a1, a2, a3, dst) do { \
189 v16u8 tmp0, tmp1, tmp2, tmp3; \
190 v16u8 out0, out1, out2, out3; \
191 ILVRL_B2_UB(a1, a0, tmp0, tmp1); \
192 ILVRL_B2_UB(a3, a2, tmp2, tmp3); \
193 ILVRL_H2_UB(tmp2, tmp0, out0, out1); \
194 ILVRL_H2_UB(tmp3, tmp1, out2, out3); \
195 ST_UB(out0, dst + 0); \
196 ST_UB(out1, dst + 16); \
197 ST_UB(out2, dst + 32); \
198 ST_UB(out3, dst + 48); \
199 } while (0)
200
201 #define STORE8_4(a0, a1, a2, a3, dst) do { \
202 v16u8 tmp0, tmp1, tmp2, tmp3; \
203 ILVR_B2_UB(a1, a0, a3, a2, tmp0, tmp1); \
204 ILVRL_H2_UB(tmp1, tmp0, tmp2, tmp3); \
205 ST_UB(tmp2, dst + 0); \
206 ST_UB(tmp3, dst + 16); \
207 } while (0)
208
209 #define STORE2_16(a0, a1, dst) do { \
210 v16u8 out0, out1; \
211 ILVRL_B2_UB(a1, a0, out0, out1); \
212 ST_UB(out0, dst + 0); \
213 ST_UB(out1, dst + 16); \
214 } while (0)
215
216 #define STORE2_8(a0, a1, dst) do { \
217 const v16u8 out0 = (v16u8)__msa_ilvr_b((v16i8)a1, (v16i8)a0); \
218 ST_UB(out0, dst); \
219 } while (0)
220
221 #define CALC_RGBA4444(y, u, v, out0, out1, N, dst) do { \
222 CALC_RGB##N(y, u, v, R, G, B); \
223 tmp0 = ANDI_B(R, 0xf0); \
224 tmp1 = SRAI_B(G, 4); \
225 RG = tmp0 | tmp1; \
226 tmp0 = ANDI_B(B, 0xf0); \
227 BA = ORI_B(tmp0, 0x0f); \
228 STORE2_##N(out0, out1, dst); \
229 } while (0)
230
231 #define CALC_RGB565(y, u, v, out0, out1, N, dst) do { \
232 CALC_RGB##N(y, u, v, R, G, B); \
233 tmp0 = ANDI_B(R, 0xf8); \
234 tmp1 = SRAI_B(G, 5); \
235 RG = tmp0 | tmp1; \
236 tmp0 = SLLI_B(G, 3); \
237 tmp1 = ANDI_B(tmp0, 0xe0); \
238 tmp0 = SRAI_B(B, 3); \
239 GB = tmp0 | tmp1; \
240 STORE2_##N(out0, out1, dst); \
241 } while (0)
242
Clip8(int v)243 static WEBP_INLINE int Clip8(int v) {
244 return v < 0 ? 0 : v > 255 ? 255 : v;
245 }
246
YuvToRgb(int y,int u,int v,uint8_t * const rgb)247 static void YuvToRgb(int y, int u, int v, uint8_t* const rgb) {
248 const int y1 = MultHi(y, 19077);
249 const int r1 = y1 + MultHi(v, 26149) - 14234;
250 const int g1 = y1 - MultHi(u, 6419) - MultHi(v, 13320) + 8708;
251 const int b1 = y1 + MultHi(u, 33050) - 17685;
252 rgb[0] = Clip8(r1 >> 6);
253 rgb[1] = Clip8(g1 >> 6);
254 rgb[2] = Clip8(b1 >> 6);
255 }
256
YuvToBgr(int y,int u,int v,uint8_t * const bgr)257 static void YuvToBgr(int y, int u, int v, uint8_t* const bgr) {
258 const int y1 = MultHi(y, 19077);
259 const int r1 = y1 + MultHi(v, 26149) - 14234;
260 const int g1 = y1 - MultHi(u, 6419) - MultHi(v, 13320) + 8708;
261 const int b1 = y1 + MultHi(u, 33050) - 17685;
262 bgr[0] = Clip8(b1 >> 6);
263 bgr[1] = Clip8(g1 >> 6);
264 bgr[2] = Clip8(r1 >> 6);
265 }
266
267 #if !defined(WEBP_REDUCE_CSP)
YuvToRgb565(int y,int u,int v,uint8_t * const rgb)268 static void YuvToRgb565(int y, int u, int v, uint8_t* const rgb) {
269 const int y1 = MultHi(y, 19077);
270 const int r1 = y1 + MultHi(v, 26149) - 14234;
271 const int g1 = y1 - MultHi(u, 6419) - MultHi(v, 13320) + 8708;
272 const int b1 = y1 + MultHi(u, 33050) - 17685;
273 const int r = Clip8(r1 >> 6);
274 const int g = Clip8(g1 >> 6);
275 const int b = Clip8(b1 >> 6);
276 const int rg = (r & 0xf8) | (g >> 5);
277 const int gb = ((g << 3) & 0xe0) | (b >> 3);
278 #if (WEBP_SWAP_16BIT_CSP == 1)
279 rgb[0] = gb;
280 rgb[1] = rg;
281 #else
282 rgb[0] = rg;
283 rgb[1] = gb;
284 #endif
285 }
286
YuvToRgba4444(int y,int u,int v,uint8_t * const argb)287 static void YuvToRgba4444(int y, int u, int v, uint8_t* const argb) {
288 const int y1 = MultHi(y, 19077);
289 const int r1 = y1 + MultHi(v, 26149) - 14234;
290 const int g1 = y1 - MultHi(u, 6419) - MultHi(v, 13320) + 8708;
291 const int b1 = y1 + MultHi(u, 33050) - 17685;
292 const int r = Clip8(r1 >> 6);
293 const int g = Clip8(g1 >> 6);
294 const int b = Clip8(b1 >> 6);
295 const int rg = (r & 0xf0) | (g >> 4);
296 const int ba = (b & 0xf0) | 0x0f; // overwrite the lower 4 bits
297 #if (WEBP_SWAP_16BIT_CSP == 1)
298 argb[0] = ba;
299 argb[1] = rg;
300 #else
301 argb[0] = rg;
302 argb[1] = ba;
303 #endif
304 }
305
YuvToArgb(uint8_t y,uint8_t u,uint8_t v,uint8_t * const argb)306 static void YuvToArgb(uint8_t y, uint8_t u, uint8_t v, uint8_t* const argb) {
307 argb[0] = 0xff;
308 YuvToRgb(y, u, v, argb + 1);
309 }
310 #endif // WEBP_REDUCE_CSP
311
YuvToBgra(uint8_t y,uint8_t u,uint8_t v,uint8_t * const bgra)312 static void YuvToBgra(uint8_t y, uint8_t u, uint8_t v, uint8_t* const bgra) {
313 YuvToBgr(y, u, v, bgra);
314 bgra[3] = 0xff;
315 }
316
YuvToRgba(uint8_t y,uint8_t u,uint8_t v,uint8_t * const rgba)317 static void YuvToRgba(uint8_t y, uint8_t u, uint8_t v, uint8_t* const rgba) {
318 YuvToRgb(y, u, v, rgba);
319 rgba[3] = 0xff;
320 }
321
322 #if !defined(WEBP_REDUCE_CSP)
YuvToRgbLine(const uint8_t * WEBP_RESTRICT y,const uint8_t * WEBP_RESTRICT u,const uint8_t * WEBP_RESTRICT v,uint8_t * WEBP_RESTRICT dst,int length)323 static void YuvToRgbLine(const uint8_t* WEBP_RESTRICT y,
324 const uint8_t* WEBP_RESTRICT u,
325 const uint8_t* WEBP_RESTRICT v,
326 uint8_t* WEBP_RESTRICT dst, int length) {
327 v16u8 R, G, B;
328 while (length >= 16) {
329 CALC_RGB16(y, u, v, R, G, B);
330 STORE16_3(R, G, B, dst);
331 y += 16;
332 u += 16;
333 v += 16;
334 dst += 16 * 3;
335 length -= 16;
336 }
337 if (length > 8) {
338 uint8_t temp[3 * 16] = { 0 };
339 memcpy(temp, y, length * sizeof(*temp));
340 CALC_RGB16(temp, u, v, R, G, B);
341 STORE16_3(R, G, B, temp);
342 memcpy(dst, temp, length * 3 * sizeof(*dst));
343 } else if (length > 0) {
344 uint8_t temp[3 * 8] = { 0 };
345 memcpy(temp, y, length * sizeof(*temp));
346 CALC_RGB8(temp, u, v, R, G, B);
347 STORE8_3(R, G, B, temp);
348 memcpy(dst, temp, length * 3 * sizeof(*dst));
349 }
350 }
351
YuvToBgrLine(const uint8_t * WEBP_RESTRICT y,const uint8_t * WEBP_RESTRICT u,const uint8_t * WEBP_RESTRICT v,uint8_t * WEBP_RESTRICT dst,int length)352 static void YuvToBgrLine(const uint8_t* WEBP_RESTRICT y,
353 const uint8_t* WEBP_RESTRICT u,
354 const uint8_t* WEBP_RESTRICT v,
355 uint8_t* WEBP_RESTRICT dst, int length) {
356 v16u8 R, G, B;
357 while (length >= 16) {
358 CALC_RGB16(y, u, v, R, G, B);
359 STORE16_3(B, G, R, dst);
360 y += 16;
361 u += 16;
362 v += 16;
363 dst += 16 * 3;
364 length -= 16;
365 }
366 if (length > 8) {
367 uint8_t temp[3 * 16] = { 0 };
368 memcpy(temp, y, length * sizeof(*temp));
369 CALC_RGB16(temp, u, v, R, G, B);
370 STORE16_3(B, G, R, temp);
371 memcpy(dst, temp, length * 3 * sizeof(*dst));
372 } else if (length > 0) {
373 uint8_t temp[3 * 8] = { 0 };
374 memcpy(temp, y, length * sizeof(*temp));
375 CALC_RGB8(temp, u, v, R, G, B);
376 STORE8_3(B, G, R, temp);
377 memcpy(dst, temp, length * 3 * sizeof(*dst));
378 }
379 }
380 #endif // WEBP_REDUCE_CSP
381
YuvToRgbaLine(const uint8_t * WEBP_RESTRICT y,const uint8_t * WEBP_RESTRICT u,const uint8_t * WEBP_RESTRICT v,uint8_t * WEBP_RESTRICT dst,int length)382 static void YuvToRgbaLine(const uint8_t* WEBP_RESTRICT y,
383 const uint8_t* WEBP_RESTRICT u,
384 const uint8_t* WEBP_RESTRICT v,
385 uint8_t* WEBP_RESTRICT dst, int length) {
386 v16u8 R, G, B;
387 const v16u8 A = (v16u8)__msa_ldi_b(ALPHAVAL);
388 while (length >= 16) {
389 CALC_RGB16(y, u, v, R, G, B);
390 STORE16_4(R, G, B, A, dst);
391 y += 16;
392 u += 16;
393 v += 16;
394 dst += 16 * 4;
395 length -= 16;
396 }
397 if (length > 8) {
398 uint8_t temp[4 * 16] = { 0 };
399 memcpy(temp, y, length * sizeof(*temp));
400 CALC_RGB16(&temp[0], u, v, R, G, B);
401 STORE16_4(R, G, B, A, temp);
402 memcpy(dst, temp, length * 4 * sizeof(*dst));
403 } else if (length > 0) {
404 uint8_t temp[4 * 8] = { 0 };
405 memcpy(temp, y, length * sizeof(*temp));
406 CALC_RGB8(temp, u, v, R, G, B);
407 STORE8_4(R, G, B, A, temp);
408 memcpy(dst, temp, length * 4 * sizeof(*dst));
409 }
410 }
411
YuvToBgraLine(const uint8_t * WEBP_RESTRICT y,const uint8_t * WEBP_RESTRICT u,const uint8_t * WEBP_RESTRICT v,uint8_t * WEBP_RESTRICT dst,int length)412 static void YuvToBgraLine(const uint8_t* WEBP_RESTRICT y,
413 const uint8_t* WEBP_RESTRICT u,
414 const uint8_t* WEBP_RESTRICT v,
415 uint8_t* WEBP_RESTRICT dst, int length) {
416 v16u8 R, G, B;
417 const v16u8 A = (v16u8)__msa_ldi_b(ALPHAVAL);
418 while (length >= 16) {
419 CALC_RGB16(y, u, v, R, G, B);
420 STORE16_4(B, G, R, A, dst);
421 y += 16;
422 u += 16;
423 v += 16;
424 dst += 16 * 4;
425 length -= 16;
426 }
427 if (length > 8) {
428 uint8_t temp[4 * 16] = { 0 };
429 memcpy(temp, y, length * sizeof(*temp));
430 CALC_RGB16(temp, u, v, R, G, B);
431 STORE16_4(B, G, R, A, temp);
432 memcpy(dst, temp, length * 4 * sizeof(*dst));
433 } else if (length > 0) {
434 uint8_t temp[4 * 8] = { 0 };
435 memcpy(temp, y, length * sizeof(*temp));
436 CALC_RGB8(temp, u, v, R, G, B);
437 STORE8_4(B, G, R, A, temp);
438 memcpy(dst, temp, length * 4 * sizeof(*dst));
439 }
440 }
441
442 #if !defined(WEBP_REDUCE_CSP)
YuvToArgbLine(const uint8_t * WEBP_RESTRICT y,const uint8_t * WEBP_RESTRICT u,const uint8_t * WEBP_RESTRICT v,uint8_t * WEBP_RESTRICT dst,int length)443 static void YuvToArgbLine(const uint8_t* WEBP_RESTRICT y,
444 const uint8_t* WEBP_RESTRICT u,
445 const uint8_t* WEBP_RESTRICT v,
446 uint8_t* WEBP_RESTRICT dst, int length) {
447 v16u8 R, G, B;
448 const v16u8 A = (v16u8)__msa_ldi_b(ALPHAVAL);
449 while (length >= 16) {
450 CALC_RGB16(y, u, v, R, G, B);
451 STORE16_4(A, R, G, B, dst);
452 y += 16;
453 u += 16;
454 v += 16;
455 dst += 16 * 4;
456 length -= 16;
457 }
458 if (length > 8) {
459 uint8_t temp[4 * 16] = { 0 };
460 memcpy(temp, y, length * sizeof(*temp));
461 CALC_RGB16(temp, u, v, R, G, B);
462 STORE16_4(A, R, G, B, temp);
463 memcpy(dst, temp, length * 4 * sizeof(*dst));
464 } else if (length > 0) {
465 uint8_t temp[4 * 8] = { 0 };
466 memcpy(temp, y, length * sizeof(*temp));
467 CALC_RGB8(temp, u, v, R, G, B);
468 STORE8_4(A, R, G, B, temp);
469 memcpy(dst, temp, length * 4 * sizeof(*dst));
470 }
471 }
472
YuvToRgba4444Line(const uint8_t * WEBP_RESTRICT y,const uint8_t * WEBP_RESTRICT u,const uint8_t * WEBP_RESTRICT v,uint8_t * WEBP_RESTRICT dst,int length)473 static void YuvToRgba4444Line(const uint8_t* WEBP_RESTRICT y,
474 const uint8_t* WEBP_RESTRICT u,
475 const uint8_t* WEBP_RESTRICT v,
476 uint8_t* WEBP_RESTRICT dst, int length) {
477 v16u8 R, G, B, RG, BA, tmp0, tmp1;
478 while (length >= 16) {
479 #if (WEBP_SWAP_16BIT_CSP == 1)
480 CALC_RGBA4444(y, u, v, BA, RG, 16, dst);
481 #else
482 CALC_RGBA4444(y, u, v, RG, BA, 16, dst);
483 #endif
484 y += 16;
485 u += 16;
486 v += 16;
487 dst += 16 * 2;
488 length -= 16;
489 }
490 if (length > 8) {
491 uint8_t temp[2 * 16] = { 0 };
492 memcpy(temp, y, length * sizeof(*temp));
493 #if (WEBP_SWAP_16BIT_CSP == 1)
494 CALC_RGBA4444(temp, u, v, BA, RG, 16, temp);
495 #else
496 CALC_RGBA4444(temp, u, v, RG, BA, 16, temp);
497 #endif
498 memcpy(dst, temp, length * 2 * sizeof(*dst));
499 } else if (length > 0) {
500 uint8_t temp[2 * 8] = { 0 };
501 memcpy(temp, y, length * sizeof(*temp));
502 #if (WEBP_SWAP_16BIT_CSP == 1)
503 CALC_RGBA4444(temp, u, v, BA, RG, 8, temp);
504 #else
505 CALC_RGBA4444(temp, u, v, RG, BA, 8, temp);
506 #endif
507 memcpy(dst, temp, length * 2 * sizeof(*dst));
508 }
509 }
510
YuvToRgb565Line(const uint8_t * WEBP_RESTRICT y,const uint8_t * WEBP_RESTRICT u,const uint8_t * WEBP_RESTRICT v,uint8_t * WEBP_RESTRICT dst,int length)511 static void YuvToRgb565Line(const uint8_t* WEBP_RESTRICT y,
512 const uint8_t* WEBP_RESTRICT u,
513 const uint8_t* WEBP_RESTRICT v,
514 uint8_t* WEBP_RESTRICT dst, int length) {
515 v16u8 R, G, B, RG, GB, tmp0, tmp1;
516 while (length >= 16) {
517 #if (WEBP_SWAP_16BIT_CSP == 1)
518 CALC_RGB565(y, u, v, GB, RG, 16, dst);
519 #else
520 CALC_RGB565(y, u, v, RG, GB, 16, dst);
521 #endif
522 y += 16;
523 u += 16;
524 v += 16;
525 dst += 16 * 2;
526 length -= 16;
527 }
528 if (length > 8) {
529 uint8_t temp[2 * 16] = { 0 };
530 memcpy(temp, y, length * sizeof(*temp));
531 #if (WEBP_SWAP_16BIT_CSP == 1)
532 CALC_RGB565(temp, u, v, GB, RG, 16, temp);
533 #else
534 CALC_RGB565(temp, u, v, RG, GB, 16, temp);
535 #endif
536 memcpy(dst, temp, length * 2 * sizeof(*dst));
537 } else if (length > 0) {
538 uint8_t temp[2 * 8] = { 0 };
539 memcpy(temp, y, length * sizeof(*temp));
540 #if (WEBP_SWAP_16BIT_CSP == 1)
541 CALC_RGB565(temp, u, v, GB, RG, 8, temp);
542 #else
543 CALC_RGB565(temp, u, v, RG, GB, 8, temp);
544 #endif
545 memcpy(dst, temp, length * 2 * sizeof(*dst));
546 }
547 }
548 #endif // WEBP_REDUCE_CSP
549
550 #define UPSAMPLE_32PIXELS(a, b, c, d) do { \
551 v16u8 s = __msa_aver_u_b(a, d); \
552 v16u8 t = __msa_aver_u_b(b, c); \
553 const v16u8 st = s ^ t; \
554 v16u8 ad = a ^ d; \
555 v16u8 bc = b ^ c; \
556 v16u8 t0 = ad | bc; \
557 v16u8 t1 = t0 | st; \
558 v16u8 t2 = ANDI_B(t1, 1); \
559 v16u8 t3 = __msa_aver_u_b(s, t); \
560 const v16u8 k = t3 - t2; \
561 v16u8 diag1, diag2; \
562 AVER_UB2_UB(t, k, s, k, t0, t1); \
563 bc = bc & st; \
564 ad = ad & st; \
565 t = t ^ k; \
566 s = s ^ k; \
567 t2 = bc | t; \
568 t3 = ad | s; \
569 t2 = ANDI_B(t2, 1); \
570 t3 = ANDI_B(t3, 1); \
571 SUB2(t0, t2, t1, t3, diag1, diag2); \
572 AVER_UB2_UB(a, diag1, b, diag2, t0, t1); \
573 ILVRL_B2_UB(t1, t0, a, b); \
574 if (pbot_y != NULL) { \
575 AVER_UB2_UB(c, diag2, d, diag1, t0, t1); \
576 ILVRL_B2_UB(t1, t0, c, d); \
577 } \
578 } while (0)
579
580 #define UPSAMPLE_FUNC(FUNC_NAME, FUNC, XSTEP) \
581 static void FUNC_NAME(const uint8_t* WEBP_RESTRICT top_y, \
582 const uint8_t* WEBP_RESTRICT bot_y, \
583 const uint8_t* WEBP_RESTRICT top_u, \
584 const uint8_t* WEBP_RESTRICT top_v, \
585 const uint8_t* WEBP_RESTRICT cur_u, \
586 const uint8_t* WEBP_RESTRICT cur_v, \
587 uint8_t* WEBP_RESTRICT top_dst, \
588 uint8_t* WEBP_RESTRICT bot_dst, int len) { \
589 int size = (len - 1) >> 1; \
590 uint8_t temp_u[64]; \
591 uint8_t temp_v[64]; \
592 const uint32_t tl_uv = ((top_u[0]) | ((top_v[0]) << 16)); \
593 const uint32_t l_uv = ((cur_u[0]) | ((cur_v[0]) << 16)); \
594 const uint32_t uv0 = (3 * tl_uv + l_uv + 0x00020002u) >> 2; \
595 const uint8_t* ptop_y = &top_y[1]; \
596 uint8_t* ptop_dst = top_dst + XSTEP; \
597 const uint8_t* pbot_y = &bot_y[1]; \
598 uint8_t* pbot_dst = bot_dst + XSTEP; \
599 \
600 FUNC(top_y[0], uv0 & 0xff, (uv0 >> 16), top_dst); \
601 if (bot_y != NULL) { \
602 const uint32_t uv1 = (3 * l_uv + tl_uv + 0x00020002u) >> 2; \
603 FUNC(bot_y[0], uv1 & 0xff, (uv1 >> 16), bot_dst); \
604 } \
605 while (size >= 16) { \
606 v16u8 tu0, tu1, tv0, tv1, cu0, cu1, cv0, cv1; \
607 LD_UB2(top_u, 1, tu0, tu1); \
608 LD_UB2(cur_u, 1, cu0, cu1); \
609 LD_UB2(top_v, 1, tv0, tv1); \
610 LD_UB2(cur_v, 1, cv0, cv1); \
611 UPSAMPLE_32PIXELS(tu0, tu1, cu0, cu1); \
612 UPSAMPLE_32PIXELS(tv0, tv1, cv0, cv1); \
613 ST_UB4(tu0, tu1, cu0, cu1, &temp_u[0], 16); \
614 ST_UB4(tv0, tv1, cv0, cv1, &temp_v[0], 16); \
615 FUNC##Line(ptop_y, &temp_u[ 0], &temp_v[0], ptop_dst, 32); \
616 if (bot_y != NULL) { \
617 FUNC##Line(pbot_y, &temp_u[32], &temp_v[32], pbot_dst, 32); \
618 } \
619 ptop_y += 32; \
620 pbot_y += 32; \
621 ptop_dst += XSTEP * 32; \
622 pbot_dst += XSTEP * 32; \
623 top_u += 16; \
624 top_v += 16; \
625 cur_u += 16; \
626 cur_v += 16; \
627 size -= 16; \
628 } \
629 if (size > 0) { \
630 v16u8 tu0, tu1, tv0, tv1, cu0, cu1, cv0, cv1; \
631 memcpy(&temp_u[ 0], top_u, 17 * sizeof(uint8_t)); \
632 memcpy(&temp_u[32], cur_u, 17 * sizeof(uint8_t)); \
633 memcpy(&temp_v[ 0], top_v, 17 * sizeof(uint8_t)); \
634 memcpy(&temp_v[32], cur_v, 17 * sizeof(uint8_t)); \
635 LD_UB2(&temp_u[ 0], 1, tu0, tu1); \
636 LD_UB2(&temp_u[32], 1, cu0, cu1); \
637 LD_UB2(&temp_v[ 0], 1, tv0, tv1); \
638 LD_UB2(&temp_v[32], 1, cv0, cv1); \
639 UPSAMPLE_32PIXELS(tu0, tu1, cu0, cu1); \
640 UPSAMPLE_32PIXELS(tv0, tv1, cv0, cv1); \
641 ST_UB4(tu0, tu1, cu0, cu1, &temp_u[0], 16); \
642 ST_UB4(tv0, tv1, cv0, cv1, &temp_v[0], 16); \
643 FUNC##Line(ptop_y, &temp_u[ 0], &temp_v[0], ptop_dst, size * 2); \
644 if (bot_y != NULL) { \
645 FUNC##Line(pbot_y, &temp_u[32], &temp_v[32], pbot_dst, size * 2); \
646 } \
647 top_u += size; \
648 top_v += size; \
649 cur_u += size; \
650 cur_v += size; \
651 } \
652 if (!(len & 1)) { \
653 const uint32_t t0 = ((top_u[0]) | ((top_v[0]) << 16)); \
654 const uint32_t c0 = ((cur_u[0]) | ((cur_v[0]) << 16)); \
655 const uint32_t tmp0 = (3 * t0 + c0 + 0x00020002u) >> 2; \
656 FUNC(top_y[len - 1], tmp0 & 0xff, (tmp0 >> 16), \
657 top_dst + (len - 1) * XSTEP); \
658 if (bot_y != NULL) { \
659 const uint32_t tmp1 = (3 * c0 + t0 + 0x00020002u) >> 2; \
660 FUNC(bot_y[len - 1], tmp1 & 0xff, (tmp1 >> 16), \
661 bot_dst + (len - 1) * XSTEP); \
662 } \
663 } \
664 }
665
666 UPSAMPLE_FUNC(UpsampleRgbaLinePair, YuvToRgba, 4)
667 UPSAMPLE_FUNC(UpsampleBgraLinePair, YuvToBgra, 4)
668 #if !defined(WEBP_REDUCE_CSP)
669 UPSAMPLE_FUNC(UpsampleRgbLinePair, YuvToRgb, 3)
670 UPSAMPLE_FUNC(UpsampleBgrLinePair, YuvToBgr, 3)
671 UPSAMPLE_FUNC(UpsampleArgbLinePair, YuvToArgb, 4)
672 UPSAMPLE_FUNC(UpsampleRgba4444LinePair, YuvToRgba4444, 2)
673 UPSAMPLE_FUNC(UpsampleRgb565LinePair, YuvToRgb565, 2)
674 #endif // WEBP_REDUCE_CSP
675
676 //------------------------------------------------------------------------------
677 // Entry point
678
679 extern WebPUpsampleLinePairFunc WebPUpsamplers[/* MODE_LAST */];
680
681 extern void WebPInitUpsamplersMSA(void);
682
WebPInitUpsamplersMSA(void)683 WEBP_TSAN_IGNORE_FUNCTION void WebPInitUpsamplersMSA(void) {
684 WebPUpsamplers[MODE_RGBA] = UpsampleRgbaLinePair;
685 WebPUpsamplers[MODE_BGRA] = UpsampleBgraLinePair;
686 WebPUpsamplers[MODE_rgbA] = UpsampleRgbaLinePair;
687 WebPUpsamplers[MODE_bgrA] = UpsampleBgraLinePair;
688 #if !defined(WEBP_REDUCE_CSP)
689 WebPUpsamplers[MODE_RGB] = UpsampleRgbLinePair;
690 WebPUpsamplers[MODE_BGR] = UpsampleBgrLinePair;
691 WebPUpsamplers[MODE_ARGB] = UpsampleArgbLinePair;
692 WebPUpsamplers[MODE_Argb] = UpsampleArgbLinePair;
693 WebPUpsamplers[MODE_RGB_565] = UpsampleRgb565LinePair;
694 WebPUpsamplers[MODE_RGBA_4444] = UpsampleRgba4444LinePair;
695 WebPUpsamplers[MODE_rgbA_4444] = UpsampleRgba4444LinePair;
696 #endif // WEBP_REDUCE_CSP
697 }
698
699 #endif // FANCY_UPSAMPLING
700
701 #endif // WEBP_USE_MSA
702
703 #if !(defined(FANCY_UPSAMPLING) && defined(WEBP_USE_MSA))
704 WEBP_DSP_INIT_STUB(WebPInitUpsamplersMSA)
705 #endif
706