1 // Copyright 2016 Google Inc. All Rights Reserved.
2 //
3 // Use of this source code is governed by a BSD-style license
4 // that can be found in the COPYING file in the root of the source
5 // tree. An additional intellectual property rights grant can be found
6 // in the file PATENTS. All contributing project authors may
7 // be found in the AUTHORS file in the root of the source tree.
8 // -----------------------------------------------------------------------------
9 //
10 // SSE2 code common to several files.
11 //
12 // Author: Vincent Rabaud (vrabaud@google.com)
13
14 #ifndef WEBP_DSP_COMMON_SSE2_H_
15 #define WEBP_DSP_COMMON_SSE2_H_
16
17 #ifdef __cplusplus
18 extern "C" {
19 #endif
20
21 #if defined(WEBP_USE_SSE2)
22
23 #include <emmintrin.h>
24
25 //------------------------------------------------------------------------------
26 // Quite useful macro for debugging. Left here for convenience.
27
28 #if 0
29 #include <stdio.h>
30 static WEBP_INLINE void PrintReg(const __m128i r, const char* const name,
31 int size) {
32 int n;
33 union {
34 __m128i r;
35 uint8_t i8[16];
36 uint16_t i16[8];
37 uint32_t i32[4];
38 uint64_t i64[2];
39 } tmp;
40 tmp.r = r;
41 fprintf(stderr, "%s\t: ", name);
42 if (size == 8) {
43 for (n = 0; n < 16; ++n) fprintf(stderr, "%.2x ", tmp.i8[n]);
44 } else if (size == 16) {
45 for (n = 0; n < 8; ++n) fprintf(stderr, "%.4x ", tmp.i16[n]);
46 } else if (size == 32) {
47 for (n = 0; n < 4; ++n) fprintf(stderr, "%.8x ", tmp.i32[n]);
48 } else {
49 for (n = 0; n < 2; ++n) fprintf(stderr, "%.16lx ", tmp.i64[n]);
50 }
51 fprintf(stderr, "\n");
52 }
53 #endif
54
55 //------------------------------------------------------------------------------
56 // Math functions.
57
58 // Return the sum of all the 8b in the register.
VP8HorizontalAdd8b(const __m128i * const a)59 static WEBP_INLINE int VP8HorizontalAdd8b(const __m128i* const a) {
60 const __m128i zero = _mm_setzero_si128();
61 const __m128i sad8x2 = _mm_sad_epu8(*a, zero);
62 // sum the two sads: sad8x2[0:1] + sad8x2[8:9]
63 const __m128i sum = _mm_add_epi32(sad8x2, _mm_shuffle_epi32(sad8x2, 2));
64 return _mm_cvtsi128_si32(sum);
65 }
66
67 // Transpose two 4x4 16b matrices horizontally stored in registers.
VP8Transpose_2_4x4_16b(const __m128i * const in0,const __m128i * const in1,const __m128i * const in2,const __m128i * const in3,__m128i * const out0,__m128i * const out1,__m128i * const out2,__m128i * const out3)68 static WEBP_INLINE void VP8Transpose_2_4x4_16b(
69 const __m128i* const in0, const __m128i* const in1,
70 const __m128i* const in2, const __m128i* const in3, __m128i* const out0,
71 __m128i* const out1, __m128i* const out2, __m128i* const out3) {
72 // Transpose the two 4x4.
73 // a00 a01 a02 a03 b00 b01 b02 b03
74 // a10 a11 a12 a13 b10 b11 b12 b13
75 // a20 a21 a22 a23 b20 b21 b22 b23
76 // a30 a31 a32 a33 b30 b31 b32 b33
77 const __m128i transpose0_0 = _mm_unpacklo_epi16(*in0, *in1);
78 const __m128i transpose0_1 = _mm_unpacklo_epi16(*in2, *in3);
79 const __m128i transpose0_2 = _mm_unpackhi_epi16(*in0, *in1);
80 const __m128i transpose0_3 = _mm_unpackhi_epi16(*in2, *in3);
81 // a00 a10 a01 a11 a02 a12 a03 a13
82 // a20 a30 a21 a31 a22 a32 a23 a33
83 // b00 b10 b01 b11 b02 b12 b03 b13
84 // b20 b30 b21 b31 b22 b32 b23 b33
85 const __m128i transpose1_0 = _mm_unpacklo_epi32(transpose0_0, transpose0_1);
86 const __m128i transpose1_1 = _mm_unpacklo_epi32(transpose0_2, transpose0_3);
87 const __m128i transpose1_2 = _mm_unpackhi_epi32(transpose0_0, transpose0_1);
88 const __m128i transpose1_3 = _mm_unpackhi_epi32(transpose0_2, transpose0_3);
89 // a00 a10 a20 a30 a01 a11 a21 a31
90 // b00 b10 b20 b30 b01 b11 b21 b31
91 // a02 a12 a22 a32 a03 a13 a23 a33
92 // b02 b12 a22 b32 b03 b13 b23 b33
93 *out0 = _mm_unpacklo_epi64(transpose1_0, transpose1_1);
94 *out1 = _mm_unpackhi_epi64(transpose1_0, transpose1_1);
95 *out2 = _mm_unpacklo_epi64(transpose1_2, transpose1_3);
96 *out3 = _mm_unpackhi_epi64(transpose1_2, transpose1_3);
97 // a00 a10 a20 a30 b00 b10 b20 b30
98 // a01 a11 a21 a31 b01 b11 b21 b31
99 // a02 a12 a22 a32 b02 b12 b22 b32
100 // a03 a13 a23 a33 b03 b13 b23 b33
101 }
102
103 //------------------------------------------------------------------------------
104 // Channel mixing.
105
106 // Function used several times in VP8PlanarTo24b.
107 // It samples the in buffer as follows: one every two unsigned char is stored
108 // at the beginning of the buffer, while the other half is stored at the end.
109 #define VP8PlanarTo24bHelper(IN, OUT) \
110 do { \
111 const __m128i v_mask = _mm_set1_epi16(0x00ff); \
112 /* Take one every two upper 8b values.*/ \
113 (OUT##0) = _mm_packus_epi16(_mm_and_si128((IN##0), v_mask), \
114 _mm_and_si128((IN##1), v_mask)); \
115 (OUT##1) = _mm_packus_epi16(_mm_and_si128((IN##2), v_mask), \
116 _mm_and_si128((IN##3), v_mask)); \
117 (OUT##2) = _mm_packus_epi16(_mm_and_si128((IN##4), v_mask), \
118 _mm_and_si128((IN##5), v_mask)); \
119 /* Take one every two lower 8b values.*/ \
120 (OUT##3) = _mm_packus_epi16(_mm_srli_epi16((IN##0), 8), \
121 _mm_srli_epi16((IN##1), 8)); \
122 (OUT##4) = _mm_packus_epi16(_mm_srli_epi16((IN##2), 8), \
123 _mm_srli_epi16((IN##3), 8)); \
124 (OUT##5) = _mm_packus_epi16(_mm_srli_epi16((IN##4), 8), \
125 _mm_srli_epi16((IN##5), 8)); \
126 } while (0)
127
128 // Pack the planar buffers
129 // rrrr... rrrr... gggg... gggg... bbbb... bbbb....
130 // triplet by triplet in the output buffer rgb as rgbrgbrgbrgb ...
VP8PlanarTo24b(__m128i * const in0,__m128i * const in1,__m128i * const in2,__m128i * const in3,__m128i * const in4,__m128i * const in5)131 static WEBP_INLINE void VP8PlanarTo24b(__m128i* const in0, __m128i* const in1,
132 __m128i* const in2, __m128i* const in3,
133 __m128i* const in4, __m128i* const in5) {
134 // The input is 6 registers of sixteen 8b but for the sake of explanation,
135 // let's take 6 registers of four 8b values.
136 // To pack, we will keep taking one every two 8b integer and move it
137 // around as follows:
138 // Input:
139 // r0r1r2r3 | r4r5r6r7 | g0g1g2g3 | g4g5g6g7 | b0b1b2b3 | b4b5b6b7
140 // Split the 6 registers in two sets of 3 registers: the first set as the even
141 // 8b bytes, the second the odd ones:
142 // r0r2r4r6 | g0g2g4g6 | b0b2b4b6 | r1r3r5r7 | g1g3g5g7 | b1b3b5b7
143 // Repeat the same permutations twice more:
144 // r0r4g0g4 | b0b4r1r5 | g1g5b1b5 | r2r6g2g6 | b2b6r3r7 | g3g7b3b7
145 // r0g0b0r1 | g1b1r2g2 | b2r3g3b3 | r4g4b4r5 | g5b5r6g6 | b6r7g7b7
146 __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
147 VP8PlanarTo24bHelper(*in, tmp);
148 VP8PlanarTo24bHelper(tmp, *in);
149 VP8PlanarTo24bHelper(*in, tmp);
150 // We need to do it two more times than the example as we have sixteen bytes.
151 {
152 __m128i out0, out1, out2, out3, out4, out5;
153 VP8PlanarTo24bHelper(tmp, out);
154 VP8PlanarTo24bHelper(out, *in);
155 }
156 }
157
158 #undef VP8PlanarTo24bHelper
159
160 // Convert four packed four-channel buffers like argbargbargbargb... into the
161 // split channels aaaaa ... rrrr ... gggg .... bbbbb ......
VP8L32bToPlanar(__m128i * const in0,__m128i * const in1,__m128i * const in2,__m128i * const in3)162 static WEBP_INLINE void VP8L32bToPlanar(__m128i* const in0,
163 __m128i* const in1,
164 __m128i* const in2,
165 __m128i* const in3) {
166 // Column-wise transpose.
167 const __m128i A0 = _mm_unpacklo_epi8(*in0, *in1);
168 const __m128i A1 = _mm_unpackhi_epi8(*in0, *in1);
169 const __m128i A2 = _mm_unpacklo_epi8(*in2, *in3);
170 const __m128i A3 = _mm_unpackhi_epi8(*in2, *in3);
171 const __m128i B0 = _mm_unpacklo_epi8(A0, A1);
172 const __m128i B1 = _mm_unpackhi_epi8(A0, A1);
173 const __m128i B2 = _mm_unpacklo_epi8(A2, A3);
174 const __m128i B3 = _mm_unpackhi_epi8(A2, A3);
175 // C0 = g7 g6 ... g1 g0 | b7 b6 ... b1 b0
176 // C1 = a7 a6 ... a1 a0 | r7 r6 ... r1 r0
177 const __m128i C0 = _mm_unpacklo_epi8(B0, B1);
178 const __m128i C1 = _mm_unpackhi_epi8(B0, B1);
179 const __m128i C2 = _mm_unpacklo_epi8(B2, B3);
180 const __m128i C3 = _mm_unpackhi_epi8(B2, B3);
181 // Gather the channels.
182 *in0 = _mm_unpackhi_epi64(C1, C3);
183 *in1 = _mm_unpacklo_epi64(C1, C3);
184 *in2 = _mm_unpackhi_epi64(C0, C2);
185 *in3 = _mm_unpacklo_epi64(C0, C2);
186 }
187
188 #endif // WEBP_USE_SSE2
189
190 #ifdef __cplusplus
191 } // extern "C"
192 #endif
193
194 #endif // WEBP_DSP_COMMON_SSE2_H_
195