1 // Copyright 2015 Google Inc. All Rights Reserved.
2 //
3 // Use of this source code is governed by a BSD-style license
4 // that can be found in the COPYING file in the root of the source
5 // tree. An additional intellectual property rights grant can be found
6 // in the file PATENTS. All contributing project authors may
7 // be found in the AUTHORS file in the root of the source tree.
8 // -----------------------------------------------------------------------------
9 //
10 // SSE4.1 variant of methods for lossless encoder
11 //
12 // Author: Skal (pascal.massimino@gmail.com)
13
14 #include "src/dsp/dsp.h"
15
16 #if defined(WEBP_USE_SSE41)
17 #include <assert.h>
18 #include <smmintrin.h>
19 #include "src/dsp/lossless.h"
20
21 //------------------------------------------------------------------------------
22 // Cost operations.
23
HorizontalSum_SSE41(__m128i cost)24 static WEBP_INLINE uint32_t HorizontalSum_SSE41(__m128i cost) {
25 cost = _mm_add_epi32(cost, _mm_srli_si128(cost, 8));
26 cost = _mm_add_epi32(cost, _mm_srli_si128(cost, 4));
27 return _mm_cvtsi128_si32(cost);
28 }
29
ExtraCost_SSE41(const uint32_t * const a,int length)30 static uint32_t ExtraCost_SSE41(const uint32_t* const a, int length) {
31 int i;
32 __m128i cost = _mm_set_epi32(2 * a[7], 2 * a[6], a[5], a[4]);
33 assert(length % 8 == 0);
34
35 for (i = 8; i + 8 <= length; i += 8) {
36 const int j = (i - 2) >> 1;
37 const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i]);
38 const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]);
39 const __m128i w = _mm_set_epi32(j + 3, j + 2, j + 1, j);
40 const __m128i a2 = _mm_hadd_epi32(a0, a1);
41 const __m128i mul = _mm_mullo_epi32(a2, w);
42 cost = _mm_add_epi32(mul, cost);
43 }
44 return HorizontalSum_SSE41(cost);
45 }
46
ExtraCostCombined_SSE41(const uint32_t * WEBP_RESTRICT const a,const uint32_t * WEBP_RESTRICT const b,int length)47 static uint32_t ExtraCostCombined_SSE41(const uint32_t* WEBP_RESTRICT const a,
48 const uint32_t* WEBP_RESTRICT const b,
49 int length) {
50 int i;
51 __m128i cost = _mm_add_epi32(_mm_set_epi32(2 * a[7], 2 * a[6], a[5], a[4]),
52 _mm_set_epi32(2 * b[7], 2 * b[6], b[5], b[4]));
53 assert(length % 8 == 0);
54
55 for (i = 8; i + 8 <= length; i += 8) {
56 const int j = (i - 2) >> 1;
57 const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i]);
58 const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]);
59 const __m128i b0 = _mm_loadu_si128((const __m128i*)&b[i]);
60 const __m128i b1 = _mm_loadu_si128((const __m128i*)&b[i + 4]);
61 const __m128i w = _mm_set_epi32(j + 3, j + 2, j + 1, j);
62 const __m128i a2 = _mm_hadd_epi32(a0, a1);
63 const __m128i b2 = _mm_hadd_epi32(b0, b1);
64 const __m128i mul = _mm_mullo_epi32(_mm_add_epi32(a2, b2), w);
65 cost = _mm_add_epi32(mul, cost);
66 }
67 return HorizontalSum_SSE41(cost);
68 }
69
70 //------------------------------------------------------------------------------
71 // Subtract-Green Transform
72
SubtractGreenFromBlueAndRed_SSE41(uint32_t * argb_data,int num_pixels)73 static void SubtractGreenFromBlueAndRed_SSE41(uint32_t* argb_data,
74 int num_pixels) {
75 int i;
76 const __m128i kCstShuffle = _mm_set_epi8(-1, 13, -1, 13, -1, 9, -1, 9,
77 -1, 5, -1, 5, -1, 1, -1, 1);
78 for (i = 0; i + 4 <= num_pixels; i += 4) {
79 const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]);
80 const __m128i in_0g0g = _mm_shuffle_epi8(in, kCstShuffle);
81 const __m128i out = _mm_sub_epi8(in, in_0g0g);
82 _mm_storeu_si128((__m128i*)&argb_data[i], out);
83 }
84 // fallthrough and finish off with plain-C
85 if (i != num_pixels) {
86 VP8LSubtractGreenFromBlueAndRed_C(argb_data + i, num_pixels - i);
87 }
88 }
89
90 //------------------------------------------------------------------------------
91 // Color Transform
92
93 // For sign-extended multiplying constants, pre-shifted by 5:
94 #define CST_5b(X) (((int16_t)((uint16_t)(X) << 8)) >> 5)
95
96 #define MK_CST_16(HI, LO) \
97 _mm_set1_epi32((int)(((uint32_t)(HI) << 16) | ((LO) & 0xffff)))
98
CollectColorBlueTransforms_SSE41(const uint32_t * WEBP_RESTRICT argb,int stride,int tile_width,int tile_height,int green_to_blue,int red_to_blue,uint32_t histo[])99 static void CollectColorBlueTransforms_SSE41(const uint32_t* WEBP_RESTRICT argb,
100 int stride,
101 int tile_width, int tile_height,
102 int green_to_blue, int red_to_blue,
103 uint32_t histo[]) {
104 const __m128i mult =
105 MK_CST_16(CST_5b(red_to_blue) + 256,CST_5b(green_to_blue));
106 const __m128i perm =
107 _mm_setr_epi8(-1, 1, -1, 2, -1, 5, -1, 6, -1, 9, -1, 10, -1, 13, -1, 14);
108 if (tile_width >= 4) {
109 int y;
110 for (y = 0; y < tile_height; ++y) {
111 const uint32_t* const src = argb + y * stride;
112 const __m128i A1 = _mm_loadu_si128((const __m128i*)src);
113 const __m128i B1 = _mm_shuffle_epi8(A1, perm);
114 const __m128i C1 = _mm_mulhi_epi16(B1, mult);
115 const __m128i D1 = _mm_sub_epi16(A1, C1);
116 __m128i E = _mm_add_epi16(_mm_srli_epi32(D1, 16), D1);
117 int x;
118 for (x = 4; x + 4 <= tile_width; x += 4) {
119 const __m128i A2 = _mm_loadu_si128((const __m128i*)(src + x));
120 __m128i B2, C2, D2;
121 ++histo[_mm_extract_epi8(E, 0)];
122 B2 = _mm_shuffle_epi8(A2, perm);
123 ++histo[_mm_extract_epi8(E, 4)];
124 C2 = _mm_mulhi_epi16(B2, mult);
125 ++histo[_mm_extract_epi8(E, 8)];
126 D2 = _mm_sub_epi16(A2, C2);
127 ++histo[_mm_extract_epi8(E, 12)];
128 E = _mm_add_epi16(_mm_srli_epi32(D2, 16), D2);
129 }
130 ++histo[_mm_extract_epi8(E, 0)];
131 ++histo[_mm_extract_epi8(E, 4)];
132 ++histo[_mm_extract_epi8(E, 8)];
133 ++histo[_mm_extract_epi8(E, 12)];
134 }
135 }
136 {
137 const int left_over = tile_width & 3;
138 if (left_over > 0) {
139 VP8LCollectColorBlueTransforms_C(argb + tile_width - left_over, stride,
140 left_over, tile_height,
141 green_to_blue, red_to_blue, histo);
142 }
143 }
144 }
145
CollectColorRedTransforms_SSE41(const uint32_t * WEBP_RESTRICT argb,int stride,int tile_width,int tile_height,int green_to_red,uint32_t histo[])146 static void CollectColorRedTransforms_SSE41(const uint32_t* WEBP_RESTRICT argb,
147 int stride,
148 int tile_width, int tile_height,
149 int green_to_red,
150 uint32_t histo[]) {
151 const __m128i mult = MK_CST_16(0, CST_5b(green_to_red));
152 const __m128i mask_g = _mm_set1_epi32(0x0000ff00);
153 if (tile_width >= 4) {
154 int y;
155 for (y = 0; y < tile_height; ++y) {
156 const uint32_t* const src = argb + y * stride;
157 const __m128i A1 = _mm_loadu_si128((const __m128i*)src);
158 const __m128i B1 = _mm_and_si128(A1, mask_g);
159 const __m128i C1 = _mm_madd_epi16(B1, mult);
160 __m128i D = _mm_sub_epi16(A1, C1);
161 int x;
162 for (x = 4; x + 4 <= tile_width; x += 4) {
163 const __m128i A2 = _mm_loadu_si128((const __m128i*)(src + x));
164 __m128i B2, C2;
165 ++histo[_mm_extract_epi8(D, 2)];
166 B2 = _mm_and_si128(A2, mask_g);
167 ++histo[_mm_extract_epi8(D, 6)];
168 C2 = _mm_madd_epi16(B2, mult);
169 ++histo[_mm_extract_epi8(D, 10)];
170 ++histo[_mm_extract_epi8(D, 14)];
171 D = _mm_sub_epi16(A2, C2);
172 }
173 ++histo[_mm_extract_epi8(D, 2)];
174 ++histo[_mm_extract_epi8(D, 6)];
175 ++histo[_mm_extract_epi8(D, 10)];
176 ++histo[_mm_extract_epi8(D, 14)];
177 }
178 }
179 {
180 const int left_over = tile_width & 3;
181 if (left_over > 0) {
182 VP8LCollectColorRedTransforms_C(argb + tile_width - left_over, stride,
183 left_over, tile_height, green_to_red,
184 histo);
185 }
186 }
187 }
188
189 #undef MK_CST_16
190
191 //------------------------------------------------------------------------------
192 // Entry point
193
194 extern void VP8LEncDspInitSSE41(void);
195
VP8LEncDspInitSSE41(void)196 WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitSSE41(void) {
197 VP8LExtraCost = ExtraCost_SSE41;
198 VP8LExtraCostCombined = ExtraCostCombined_SSE41;
199 VP8LSubtractGreenFromBlueAndRed = SubtractGreenFromBlueAndRed_SSE41;
200 VP8LCollectColorBlueTransforms = CollectColorBlueTransforms_SSE41;
201 VP8LCollectColorRedTransforms = CollectColorRedTransforms_SSE41;
202 }
203
204 #else // !WEBP_USE_SSE41
205
206 WEBP_DSP_INIT_STUB(VP8LEncDspInitSSE41)
207
208 #endif // WEBP_USE_SSE41
209