1 // Copyright 2015 Google Inc. All Rights Reserved.
2 //
3 // Use of this source code is governed by a BSD-style license
4 // that can be found in the COPYING file in the root of the source
5 // tree. An additional intellectual property rights grant can be found
6 // in the file PATENTS. All contributing project authors may
7 // be found in the AUTHORS file in the root of the source tree.
8 // -----------------------------------------------------------------------------
9 //
10 // SSE2 variant of methods for lossless encoder
11 //
12 // Author: Skal (pascal.massimino@gmail.com)
13
14 #include "./dsp.h"
15
16 #if defined(WEBP_USE_SSE2)
17 #include <assert.h>
18 #include <emmintrin.h>
19 #include "./lossless.h"
20 #include "./common_sse2.h"
21 #include "./lossless_common.h"
22
23 // For sign-extended multiplying constants, pre-shifted by 5:
24 #define CST_5b(X) (((int16_t)((uint16_t)X << 8)) >> 5)
25
26 //------------------------------------------------------------------------------
27 // Subtract-Green Transform
28
SubtractGreenFromBlueAndRed(uint32_t * argb_data,int num_pixels)29 static void SubtractGreenFromBlueAndRed(uint32_t* argb_data, int num_pixels) {
30 int i;
31 for (i = 0; i + 4 <= num_pixels; i += 4) {
32 const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); // argb
33 const __m128i A = _mm_srli_epi16(in, 8); // 0 a 0 g
34 const __m128i B = _mm_shufflelo_epi16(A, _MM_SHUFFLE(2, 2, 0, 0));
35 const __m128i C = _mm_shufflehi_epi16(B, _MM_SHUFFLE(2, 2, 0, 0)); // 0g0g
36 const __m128i out = _mm_sub_epi8(in, C);
37 _mm_storeu_si128((__m128i*)&argb_data[i], out);
38 }
39 // fallthrough and finish off with plain-C
40 if (i != num_pixels) {
41 VP8LSubtractGreenFromBlueAndRed_C(argb_data + i, num_pixels - i);
42 }
43 }
44
45 //------------------------------------------------------------------------------
46 // Color Transform
47
TransformColor(const VP8LMultipliers * const m,uint32_t * argb_data,int num_pixels)48 static void TransformColor(const VP8LMultipliers* const m,
49 uint32_t* argb_data, int num_pixels) {
50 const __m128i mults_rb = _mm_set_epi16(
51 CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_),
52 CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_),
53 CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_),
54 CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_));
55 const __m128i mults_b2 = _mm_set_epi16(
56 CST_5b(m->red_to_blue_), 0, CST_5b(m->red_to_blue_), 0,
57 CST_5b(m->red_to_blue_), 0, CST_5b(m->red_to_blue_), 0);
58 const __m128i mask_ag = _mm_set1_epi32(0xff00ff00); // alpha-green masks
59 const __m128i mask_rb = _mm_set1_epi32(0x00ff00ff); // red-blue masks
60 int i;
61 for (i = 0; i + 4 <= num_pixels; i += 4) {
62 const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); // argb
63 const __m128i A = _mm_and_si128(in, mask_ag); // a 0 g 0
64 const __m128i B = _mm_shufflelo_epi16(A, _MM_SHUFFLE(2, 2, 0, 0));
65 const __m128i C = _mm_shufflehi_epi16(B, _MM_SHUFFLE(2, 2, 0, 0)); // g0g0
66 const __m128i D = _mm_mulhi_epi16(C, mults_rb); // x dr x db1
67 const __m128i E = _mm_slli_epi16(in, 8); // r 0 b 0
68 const __m128i F = _mm_mulhi_epi16(E, mults_b2); // x db2 0 0
69 const __m128i G = _mm_srli_epi32(F, 16); // 0 0 x db2
70 const __m128i H = _mm_add_epi8(G, D); // x dr x db
71 const __m128i I = _mm_and_si128(H, mask_rb); // 0 dr 0 db
72 const __m128i out = _mm_sub_epi8(in, I);
73 _mm_storeu_si128((__m128i*)&argb_data[i], out);
74 }
75 // fallthrough and finish off with plain-C
76 if (i != num_pixels) {
77 VP8LTransformColor_C(m, argb_data + i, num_pixels - i);
78 }
79 }
80
81 //------------------------------------------------------------------------------
82 #define SPAN 8
CollectColorBlueTransforms(const uint32_t * argb,int stride,int tile_width,int tile_height,int green_to_blue,int red_to_blue,int histo[])83 static void CollectColorBlueTransforms(const uint32_t* argb, int stride,
84 int tile_width, int tile_height,
85 int green_to_blue, int red_to_blue,
86 int histo[]) {
87 const __m128i mults_r = _mm_set_epi16(
88 CST_5b(red_to_blue), 0, CST_5b(red_to_blue), 0,
89 CST_5b(red_to_blue), 0, CST_5b(red_to_blue), 0);
90 const __m128i mults_g = _mm_set_epi16(
91 0, CST_5b(green_to_blue), 0, CST_5b(green_to_blue),
92 0, CST_5b(green_to_blue), 0, CST_5b(green_to_blue));
93 const __m128i mask_g = _mm_set1_epi32(0x00ff00); // green mask
94 const __m128i mask_b = _mm_set1_epi32(0x0000ff); // blue mask
95 int y;
96 for (y = 0; y < tile_height; ++y) {
97 const uint32_t* const src = argb + y * stride;
98 int i, x;
99 for (x = 0; x + SPAN <= tile_width; x += SPAN) {
100 uint16_t values[SPAN];
101 const __m128i in0 = _mm_loadu_si128((__m128i*)&src[x + 0]);
102 const __m128i in1 = _mm_loadu_si128((__m128i*)&src[x + SPAN / 2]);
103 const __m128i A0 = _mm_slli_epi16(in0, 8); // r 0 | b 0
104 const __m128i A1 = _mm_slli_epi16(in1, 8);
105 const __m128i B0 = _mm_and_si128(in0, mask_g); // 0 0 | g 0
106 const __m128i B1 = _mm_and_si128(in1, mask_g);
107 const __m128i C0 = _mm_mulhi_epi16(A0, mults_r); // x db | 0 0
108 const __m128i C1 = _mm_mulhi_epi16(A1, mults_r);
109 const __m128i D0 = _mm_mulhi_epi16(B0, mults_g); // 0 0 | x db
110 const __m128i D1 = _mm_mulhi_epi16(B1, mults_g);
111 const __m128i E0 = _mm_sub_epi8(in0, D0); // x x | x b'
112 const __m128i E1 = _mm_sub_epi8(in1, D1);
113 const __m128i F0 = _mm_srli_epi32(C0, 16); // 0 0 | x db
114 const __m128i F1 = _mm_srli_epi32(C1, 16);
115 const __m128i G0 = _mm_sub_epi8(E0, F0); // 0 0 | x b'
116 const __m128i G1 = _mm_sub_epi8(E1, F1);
117 const __m128i H0 = _mm_and_si128(G0, mask_b); // 0 0 | 0 b
118 const __m128i H1 = _mm_and_si128(G1, mask_b);
119 const __m128i I = _mm_packs_epi32(H0, H1); // 0 b' | 0 b'
120 _mm_storeu_si128((__m128i*)values, I);
121 for (i = 0; i < SPAN; ++i) ++histo[values[i]];
122 }
123 }
124 {
125 const int left_over = tile_width & (SPAN - 1);
126 if (left_over > 0) {
127 VP8LCollectColorBlueTransforms_C(argb + tile_width - left_over, stride,
128 left_over, tile_height,
129 green_to_blue, red_to_blue, histo);
130 }
131 }
132 }
133
CollectColorRedTransforms(const uint32_t * argb,int stride,int tile_width,int tile_height,int green_to_red,int histo[])134 static void CollectColorRedTransforms(const uint32_t* argb, int stride,
135 int tile_width, int tile_height,
136 int green_to_red, int histo[]) {
137 const __m128i mults_g = _mm_set_epi16(
138 0, CST_5b(green_to_red), 0, CST_5b(green_to_red),
139 0, CST_5b(green_to_red), 0, CST_5b(green_to_red));
140 const __m128i mask_g = _mm_set1_epi32(0x00ff00); // green mask
141 const __m128i mask = _mm_set1_epi32(0xff);
142
143 int y;
144 for (y = 0; y < tile_height; ++y) {
145 const uint32_t* const src = argb + y * stride;
146 int i, x;
147 for (x = 0; x + SPAN <= tile_width; x += SPAN) {
148 uint16_t values[SPAN];
149 const __m128i in0 = _mm_loadu_si128((__m128i*)&src[x + 0]);
150 const __m128i in1 = _mm_loadu_si128((__m128i*)&src[x + SPAN / 2]);
151 const __m128i A0 = _mm_and_si128(in0, mask_g); // 0 0 | g 0
152 const __m128i A1 = _mm_and_si128(in1, mask_g);
153 const __m128i B0 = _mm_srli_epi32(in0, 16); // 0 0 | x r
154 const __m128i B1 = _mm_srli_epi32(in1, 16);
155 const __m128i C0 = _mm_mulhi_epi16(A0, mults_g); // 0 0 | x dr
156 const __m128i C1 = _mm_mulhi_epi16(A1, mults_g);
157 const __m128i E0 = _mm_sub_epi8(B0, C0); // x x | x r'
158 const __m128i E1 = _mm_sub_epi8(B1, C1);
159 const __m128i F0 = _mm_and_si128(E0, mask); // 0 0 | 0 r'
160 const __m128i F1 = _mm_and_si128(E1, mask);
161 const __m128i I = _mm_packs_epi32(F0, F1);
162 _mm_storeu_si128((__m128i*)values, I);
163 for (i = 0; i < SPAN; ++i) ++histo[values[i]];
164 }
165 }
166 {
167 const int left_over = tile_width & (SPAN - 1);
168 if (left_over > 0) {
169 VP8LCollectColorRedTransforms_C(argb + tile_width - left_over, stride,
170 left_over, tile_height,
171 green_to_red, histo);
172 }
173 }
174 }
175 #undef SPAN
176
177 //------------------------------------------------------------------------------
178
179 #define LINE_SIZE 16 // 8 or 16
AddVector(const uint32_t * a,const uint32_t * b,uint32_t * out,int size)180 static void AddVector(const uint32_t* a, const uint32_t* b, uint32_t* out,
181 int size) {
182 int i;
183 assert(size % LINE_SIZE == 0);
184 for (i = 0; i < size; i += LINE_SIZE) {
185 const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i + 0]);
186 const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]);
187 #if (LINE_SIZE == 16)
188 const __m128i a2 = _mm_loadu_si128((const __m128i*)&a[i + 8]);
189 const __m128i a3 = _mm_loadu_si128((const __m128i*)&a[i + 12]);
190 #endif
191 const __m128i b0 = _mm_loadu_si128((const __m128i*)&b[i + 0]);
192 const __m128i b1 = _mm_loadu_si128((const __m128i*)&b[i + 4]);
193 #if (LINE_SIZE == 16)
194 const __m128i b2 = _mm_loadu_si128((const __m128i*)&b[i + 8]);
195 const __m128i b3 = _mm_loadu_si128((const __m128i*)&b[i + 12]);
196 #endif
197 _mm_storeu_si128((__m128i*)&out[i + 0], _mm_add_epi32(a0, b0));
198 _mm_storeu_si128((__m128i*)&out[i + 4], _mm_add_epi32(a1, b1));
199 #if (LINE_SIZE == 16)
200 _mm_storeu_si128((__m128i*)&out[i + 8], _mm_add_epi32(a2, b2));
201 _mm_storeu_si128((__m128i*)&out[i + 12], _mm_add_epi32(a3, b3));
202 #endif
203 }
204 }
205
AddVectorEq(const uint32_t * a,uint32_t * out,int size)206 static void AddVectorEq(const uint32_t* a, uint32_t* out, int size) {
207 int i;
208 assert(size % LINE_SIZE == 0);
209 for (i = 0; i < size; i += LINE_SIZE) {
210 const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i + 0]);
211 const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]);
212 #if (LINE_SIZE == 16)
213 const __m128i a2 = _mm_loadu_si128((const __m128i*)&a[i + 8]);
214 const __m128i a3 = _mm_loadu_si128((const __m128i*)&a[i + 12]);
215 #endif
216 const __m128i b0 = _mm_loadu_si128((const __m128i*)&out[i + 0]);
217 const __m128i b1 = _mm_loadu_si128((const __m128i*)&out[i + 4]);
218 #if (LINE_SIZE == 16)
219 const __m128i b2 = _mm_loadu_si128((const __m128i*)&out[i + 8]);
220 const __m128i b3 = _mm_loadu_si128((const __m128i*)&out[i + 12]);
221 #endif
222 _mm_storeu_si128((__m128i*)&out[i + 0], _mm_add_epi32(a0, b0));
223 _mm_storeu_si128((__m128i*)&out[i + 4], _mm_add_epi32(a1, b1));
224 #if (LINE_SIZE == 16)
225 _mm_storeu_si128((__m128i*)&out[i + 8], _mm_add_epi32(a2, b2));
226 _mm_storeu_si128((__m128i*)&out[i + 12], _mm_add_epi32(a3, b3));
227 #endif
228 }
229 }
230 #undef LINE_SIZE
231
232 // Note we are adding uint32_t's as *signed* int32's (using _mm_add_epi32). But
233 // that's ok since the histogram values are less than 1<<28 (max picture size).
HistogramAdd(const VP8LHistogram * const a,const VP8LHistogram * const b,VP8LHistogram * const out)234 static void HistogramAdd(const VP8LHistogram* const a,
235 const VP8LHistogram* const b,
236 VP8LHistogram* const out) {
237 int i;
238 const int literal_size = VP8LHistogramNumCodes(a->palette_code_bits_);
239 assert(a->palette_code_bits_ == b->palette_code_bits_);
240 if (b != out) {
241 AddVector(a->literal_, b->literal_, out->literal_, NUM_LITERAL_CODES);
242 AddVector(a->red_, b->red_, out->red_, NUM_LITERAL_CODES);
243 AddVector(a->blue_, b->blue_, out->blue_, NUM_LITERAL_CODES);
244 AddVector(a->alpha_, b->alpha_, out->alpha_, NUM_LITERAL_CODES);
245 } else {
246 AddVectorEq(a->literal_, out->literal_, NUM_LITERAL_CODES);
247 AddVectorEq(a->red_, out->red_, NUM_LITERAL_CODES);
248 AddVectorEq(a->blue_, out->blue_, NUM_LITERAL_CODES);
249 AddVectorEq(a->alpha_, out->alpha_, NUM_LITERAL_CODES);
250 }
251 for (i = NUM_LITERAL_CODES; i < literal_size; ++i) {
252 out->literal_[i] = a->literal_[i] + b->literal_[i];
253 }
254 for (i = 0; i < NUM_DISTANCE_CODES; ++i) {
255 out->distance_[i] = a->distance_[i] + b->distance_[i];
256 }
257 }
258
259 //------------------------------------------------------------------------------
260 // Entropy
261
262 // Checks whether the X or Y contribution is worth computing and adding.
263 // Used in loop unrolling.
264 #define ANALYZE_X_OR_Y(x_or_y, j) \
265 do { \
266 if (x_or_y[i + j] != 0) retval -= VP8LFastSLog2(x_or_y[i + j]); \
267 } while (0)
268
269 // Checks whether the X + Y contribution is worth computing and adding.
270 // Used in loop unrolling.
271 #define ANALYZE_XY(j) \
272 do { \
273 if (tmp[j] != 0) { \
274 retval -= VP8LFastSLog2(tmp[j]); \
275 ANALYZE_X_OR_Y(X, j); \
276 } \
277 } while (0)
278
CombinedShannonEntropy(const int X[256],const int Y[256])279 static float CombinedShannonEntropy(const int X[256], const int Y[256]) {
280 int i;
281 double retval = 0.;
282 int sumX, sumXY;
283 int32_t tmp[4];
284 __m128i zero = _mm_setzero_si128();
285 // Sums up X + Y, 4 ints at a time (and will merge it at the end for sumXY).
286 __m128i sumXY_128 = zero;
287 __m128i sumX_128 = zero;
288
289 for (i = 0; i < 256; i += 4) {
290 const __m128i x = _mm_loadu_si128((const __m128i*)(X + i));
291 const __m128i y = _mm_loadu_si128((const __m128i*)(Y + i));
292
293 // Check if any X is non-zero: this actually provides a speedup as X is
294 // usually sparse.
295 if (_mm_movemask_epi8(_mm_cmpeq_epi32(x, zero)) != 0xFFFF) {
296 const __m128i xy_128 = _mm_add_epi32(x, y);
297 sumXY_128 = _mm_add_epi32(sumXY_128, xy_128);
298
299 sumX_128 = _mm_add_epi32(sumX_128, x);
300
301 // Analyze the different X + Y.
302 _mm_storeu_si128((__m128i*)tmp, xy_128);
303
304 ANALYZE_XY(0);
305 ANALYZE_XY(1);
306 ANALYZE_XY(2);
307 ANALYZE_XY(3);
308 } else {
309 // X is fully 0, so only deal with Y.
310 sumXY_128 = _mm_add_epi32(sumXY_128, y);
311
312 ANALYZE_X_OR_Y(Y, 0);
313 ANALYZE_X_OR_Y(Y, 1);
314 ANALYZE_X_OR_Y(Y, 2);
315 ANALYZE_X_OR_Y(Y, 3);
316 }
317 }
318
319 // Sum up sumX_128 to get sumX.
320 _mm_storeu_si128((__m128i*)tmp, sumX_128);
321 sumX = tmp[3] + tmp[2] + tmp[1] + tmp[0];
322
323 // Sum up sumXY_128 to get sumXY.
324 _mm_storeu_si128((__m128i*)tmp, sumXY_128);
325 sumXY = tmp[3] + tmp[2] + tmp[1] + tmp[0];
326
327 retval += VP8LFastSLog2(sumX) + VP8LFastSLog2(sumXY);
328 return (float)retval;
329 }
330 #undef ANALYZE_X_OR_Y
331 #undef ANALYZE_XY
332
333 //------------------------------------------------------------------------------
334
VectorMismatch(const uint32_t * const array1,const uint32_t * const array2,int length)335 static int VectorMismatch(const uint32_t* const array1,
336 const uint32_t* const array2, int length) {
337 int match_len;
338
339 if (length >= 12) {
340 __m128i A0 = _mm_loadu_si128((const __m128i*)&array1[0]);
341 __m128i A1 = _mm_loadu_si128((const __m128i*)&array2[0]);
342 match_len = 0;
343 do {
344 // Loop unrolling and early load both provide a speedup of 10% for the
345 // current function. Also, max_limit can be MAX_LENGTH=4096 at most.
346 const __m128i cmpA = _mm_cmpeq_epi32(A0, A1);
347 const __m128i B0 =
348 _mm_loadu_si128((const __m128i*)&array1[match_len + 4]);
349 const __m128i B1 =
350 _mm_loadu_si128((const __m128i*)&array2[match_len + 4]);
351 if (_mm_movemask_epi8(cmpA) != 0xffff) break;
352 match_len += 4;
353
354 {
355 const __m128i cmpB = _mm_cmpeq_epi32(B0, B1);
356 A0 = _mm_loadu_si128((const __m128i*)&array1[match_len + 4]);
357 A1 = _mm_loadu_si128((const __m128i*)&array2[match_len + 4]);
358 if (_mm_movemask_epi8(cmpB) != 0xffff) break;
359 match_len += 4;
360 }
361 } while (match_len + 12 < length);
362 } else {
363 match_len = 0;
364 // Unroll the potential first two loops.
365 if (length >= 4 &&
366 _mm_movemask_epi8(_mm_cmpeq_epi32(
367 _mm_loadu_si128((const __m128i*)&array1[0]),
368 _mm_loadu_si128((const __m128i*)&array2[0]))) == 0xffff) {
369 match_len = 4;
370 if (length >= 8 &&
371 _mm_movemask_epi8(_mm_cmpeq_epi32(
372 _mm_loadu_si128((const __m128i*)&array1[4]),
373 _mm_loadu_si128((const __m128i*)&array2[4]))) == 0xffff) {
374 match_len = 8;
375 }
376 }
377 }
378
379 while (match_len < length && array1[match_len] == array2[match_len]) {
380 ++match_len;
381 }
382 return match_len;
383 }
384
385 // Bundles multiple (1, 2, 4 or 8) pixels into a single pixel.
BundleColorMap_SSE2(const uint8_t * const row,int width,int xbits,uint32_t * dst)386 static void BundleColorMap_SSE2(const uint8_t* const row, int width, int xbits,
387 uint32_t* dst) {
388 int x;
389 assert(xbits >= 0);
390 assert(xbits <= 3);
391 switch (xbits) {
392 case 0: {
393 const __m128i ff = _mm_set1_epi16(0xff00);
394 const __m128i zero = _mm_setzero_si128();
395 // Store 0xff000000 | (row[x] << 8).
396 for (x = 0; x + 16 <= width; x += 16, dst += 16) {
397 const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]);
398 const __m128i in_lo = _mm_unpacklo_epi8(zero, in);
399 const __m128i dst0 = _mm_unpacklo_epi16(in_lo, ff);
400 const __m128i dst1 = _mm_unpackhi_epi16(in_lo, ff);
401 const __m128i in_hi = _mm_unpackhi_epi8(zero, in);
402 const __m128i dst2 = _mm_unpacklo_epi16(in_hi, ff);
403 const __m128i dst3 = _mm_unpackhi_epi16(in_hi, ff);
404 _mm_storeu_si128((__m128i*)&dst[0], dst0);
405 _mm_storeu_si128((__m128i*)&dst[4], dst1);
406 _mm_storeu_si128((__m128i*)&dst[8], dst2);
407 _mm_storeu_si128((__m128i*)&dst[12], dst3);
408 }
409 break;
410 }
411 case 1: {
412 const __m128i ff = _mm_set1_epi16(0xff00);
413 const __m128i mul = _mm_set1_epi16(0x110);
414 for (x = 0; x + 16 <= width; x += 16, dst += 8) {
415 // 0a0b | (where a/b are 4 bits).
416 const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]);
417 const __m128i tmp = _mm_mullo_epi16(in, mul); // aba0
418 const __m128i pack = _mm_and_si128(tmp, ff); // ab00
419 const __m128i dst0 = _mm_unpacklo_epi16(pack, ff);
420 const __m128i dst1 = _mm_unpackhi_epi16(pack, ff);
421 _mm_storeu_si128((__m128i*)&dst[0], dst0);
422 _mm_storeu_si128((__m128i*)&dst[4], dst1);
423 }
424 break;
425 }
426 case 2: {
427 const __m128i mask_or = _mm_set1_epi32(0xff000000);
428 const __m128i mul_cst = _mm_set1_epi16(0x0104);
429 const __m128i mask_mul = _mm_set1_epi16(0x0f00);
430 for (x = 0; x + 16 <= width; x += 16, dst += 4) {
431 // 000a000b000c000d | (where a/b/c/d are 2 bits).
432 const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]);
433 const __m128i mul = _mm_mullo_epi16(in, mul_cst); // 00ab00b000cd00d0
434 const __m128i tmp = _mm_and_si128(mul, mask_mul); // 00ab000000cd0000
435 const __m128i shift = _mm_srli_epi32(tmp, 12); // 00000000ab000000
436 const __m128i pack = _mm_or_si128(shift, tmp); // 00000000abcd0000
437 // Convert to 0xff00**00.
438 const __m128i res = _mm_or_si128(pack, mask_or);
439 _mm_storeu_si128((__m128i*)dst, res);
440 }
441 break;
442 }
443 default: {
444 assert(xbits == 3);
445 for (x = 0; x + 16 <= width; x += 16, dst += 2) {
446 // 0000000a00000000b... | (where a/b are 1 bit).
447 const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]);
448 const __m128i shift = _mm_slli_epi64(in, 7);
449 const uint32_t move = _mm_movemask_epi8(shift);
450 dst[0] = 0xff000000 | ((move & 0xff) << 8);
451 dst[1] = 0xff000000 | (move & 0xff00);
452 }
453 break;
454 }
455 }
456 if (x != width) {
457 VP8LBundleColorMap_C(row + x, width - x, xbits, dst);
458 }
459 }
460
461 //------------------------------------------------------------------------------
462 // Batch version of Predictor Transform subtraction
463
Average2_m128i(const __m128i * const a0,const __m128i * const a1,__m128i * const avg)464 static WEBP_INLINE void Average2_m128i(const __m128i* const a0,
465 const __m128i* const a1,
466 __m128i* const avg) {
467 // (a + b) >> 1 = ((a + b + 1) >> 1) - ((a ^ b) & 1)
468 const __m128i ones = _mm_set1_epi8(1);
469 const __m128i avg1 = _mm_avg_epu8(*a0, *a1);
470 const __m128i one = _mm_and_si128(_mm_xor_si128(*a0, *a1), ones);
471 *avg = _mm_sub_epi8(avg1, one);
472 }
473
474 // Predictor0: ARGB_BLACK.
PredictorSub0_SSE2(const uint32_t * in,const uint32_t * upper,int num_pixels,uint32_t * out)475 static void PredictorSub0_SSE2(const uint32_t* in, const uint32_t* upper,
476 int num_pixels, uint32_t* out) {
477 int i;
478 const __m128i black = _mm_set1_epi32(ARGB_BLACK);
479 for (i = 0; i + 4 <= num_pixels; i += 4) {
480 const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
481 const __m128i res = _mm_sub_epi8(src, black);
482 _mm_storeu_si128((__m128i*)&out[i], res);
483 }
484 if (i != num_pixels) {
485 VP8LPredictorsSub_C[0](in + i, upper + i, num_pixels - i, out + i);
486 }
487 }
488
489 #define GENERATE_PREDICTOR_1(X, IN) \
490 static void PredictorSub##X##_SSE2(const uint32_t* in, const uint32_t* upper, \
491 int num_pixels, uint32_t* out) { \
492 int i; \
493 for (i = 0; i + 4 <= num_pixels; i += 4) { \
494 const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); \
495 const __m128i pred = _mm_loadu_si128((const __m128i*)&(IN)); \
496 const __m128i res = _mm_sub_epi8(src, pred); \
497 _mm_storeu_si128((__m128i*)&out[i], res); \
498 } \
499 if (i != num_pixels) { \
500 VP8LPredictorsSub_C[(X)](in + i, upper + i, num_pixels - i, out + i); \
501 } \
502 }
503
504 GENERATE_PREDICTOR_1(1, in[i - 1]) // Predictor1: L
505 GENERATE_PREDICTOR_1(2, upper[i]) // Predictor2: T
506 GENERATE_PREDICTOR_1(3, upper[i + 1]) // Predictor3: TR
507 GENERATE_PREDICTOR_1(4, upper[i - 1]) // Predictor4: TL
508 #undef GENERATE_PREDICTOR_1
509
510 // Predictor5: avg2(avg2(L, TR), T)
PredictorSub5_SSE2(const uint32_t * in,const uint32_t * upper,int num_pixels,uint32_t * out)511 static void PredictorSub5_SSE2(const uint32_t* in, const uint32_t* upper,
512 int num_pixels, uint32_t* out) {
513 int i;
514 for (i = 0; i + 4 <= num_pixels; i += 4) {
515 const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]);
516 const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]);
517 const __m128i TR = _mm_loadu_si128((const __m128i*)&upper[i + 1]);
518 const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
519 __m128i avg, pred, res;
520 Average2_m128i(&L, &TR, &avg);
521 Average2_m128i(&avg, &T, &pred);
522 res = _mm_sub_epi8(src, pred);
523 _mm_storeu_si128((__m128i*)&out[i], res);
524 }
525 if (i != num_pixels) {
526 VP8LPredictorsSub_C[5](in + i, upper + i, num_pixels - i, out + i);
527 }
528 }
529
530 #define GENERATE_PREDICTOR_2(X, A, B) \
531 static void PredictorSub##X##_SSE2(const uint32_t* in, const uint32_t* upper, \
532 int num_pixels, uint32_t* out) { \
533 int i; \
534 for (i = 0; i + 4 <= num_pixels; i += 4) { \
535 const __m128i tA = _mm_loadu_si128((const __m128i*)&(A)); \
536 const __m128i tB = _mm_loadu_si128((const __m128i*)&(B)); \
537 const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); \
538 __m128i pred, res; \
539 Average2_m128i(&tA, &tB, &pred); \
540 res = _mm_sub_epi8(src, pred); \
541 _mm_storeu_si128((__m128i*)&out[i], res); \
542 } \
543 if (i != num_pixels) { \
544 VP8LPredictorsSub_C[(X)](in + i, upper + i, num_pixels - i, out + i); \
545 } \
546 }
547
548 GENERATE_PREDICTOR_2(6, in[i - 1], upper[i - 1]) // Predictor6: avg(L, TL)
549 GENERATE_PREDICTOR_2(7, in[i - 1], upper[i]) // Predictor7: avg(L, T)
550 GENERATE_PREDICTOR_2(8, upper[i - 1], upper[i]) // Predictor8: avg(TL, T)
551 GENERATE_PREDICTOR_2(9, upper[i], upper[i + 1]) // Predictor9: average(T, TR)
552 #undef GENERATE_PREDICTOR_2
553
554 // Predictor10: avg(avg(L,TL), avg(T, TR)).
PredictorSub10_SSE2(const uint32_t * in,const uint32_t * upper,int num_pixels,uint32_t * out)555 static void PredictorSub10_SSE2(const uint32_t* in, const uint32_t* upper,
556 int num_pixels, uint32_t* out) {
557 int i;
558 for (i = 0; i + 4 <= num_pixels; i += 4) {
559 const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]);
560 const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
561 const __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]);
562 const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]);
563 const __m128i TR = _mm_loadu_si128((const __m128i*)&upper[i + 1]);
564 __m128i avgTTR, avgLTL, avg, res;
565 Average2_m128i(&T, &TR, &avgTTR);
566 Average2_m128i(&L, &TL, &avgLTL);
567 Average2_m128i(&avgTTR, &avgLTL, &avg);
568 res = _mm_sub_epi8(src, avg);
569 _mm_storeu_si128((__m128i*)&out[i], res);
570 }
571 if (i != num_pixels) {
572 VP8LPredictorsSub_C[10](in + i, upper + i, num_pixels - i, out + i);
573 }
574 }
575
576 // Predictor11: select.
GetSumAbsDiff32(const __m128i * const A,const __m128i * const B,__m128i * const out)577 static void GetSumAbsDiff32(const __m128i* const A, const __m128i* const B,
578 __m128i* const out) {
579 // We can unpack with any value on the upper 32 bits, provided it's the same
580 // on both operands (to that their sum of abs diff is zero). Here we use *A.
581 const __m128i A_lo = _mm_unpacklo_epi32(*A, *A);
582 const __m128i B_lo = _mm_unpacklo_epi32(*B, *A);
583 const __m128i A_hi = _mm_unpackhi_epi32(*A, *A);
584 const __m128i B_hi = _mm_unpackhi_epi32(*B, *A);
585 const __m128i s_lo = _mm_sad_epu8(A_lo, B_lo);
586 const __m128i s_hi = _mm_sad_epu8(A_hi, B_hi);
587 *out = _mm_packs_epi32(s_lo, s_hi);
588 }
589
PredictorSub11_SSE2(const uint32_t * in,const uint32_t * upper,int num_pixels,uint32_t * out)590 static void PredictorSub11_SSE2(const uint32_t* in, const uint32_t* upper,
591 int num_pixels, uint32_t* out) {
592 int i;
593 for (i = 0; i + 4 <= num_pixels; i += 4) {
594 const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]);
595 const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]);
596 const __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]);
597 const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
598 __m128i pa, pb;
599 GetSumAbsDiff32(&T, &TL, &pa); // pa = sum |T-TL|
600 GetSumAbsDiff32(&L, &TL, &pb); // pb = sum |L-TL|
601 {
602 const __m128i mask = _mm_cmpgt_epi32(pb, pa);
603 const __m128i A = _mm_and_si128(mask, L);
604 const __m128i B = _mm_andnot_si128(mask, T);
605 const __m128i pred = _mm_or_si128(A, B); // pred = (L > T)? L : T
606 const __m128i res = _mm_sub_epi8(src, pred);
607 _mm_storeu_si128((__m128i*)&out[i], res);
608 }
609 }
610 if (i != num_pixels) {
611 VP8LPredictorsSub_C[11](in + i, upper + i, num_pixels - i, out + i);
612 }
613 }
614
615 // Predictor12: ClampedSubSubtractFull.
PredictorSub12_SSE2(const uint32_t * in,const uint32_t * upper,int num_pixels,uint32_t * out)616 static void PredictorSub12_SSE2(const uint32_t* in, const uint32_t* upper,
617 int num_pixels, uint32_t* out) {
618 int i;
619 const __m128i zero = _mm_setzero_si128();
620 for (i = 0; i + 4 <= num_pixels; i += 4) {
621 const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
622 const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]);
623 const __m128i L_lo = _mm_unpacklo_epi8(L, zero);
624 const __m128i L_hi = _mm_unpackhi_epi8(L, zero);
625 const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]);
626 const __m128i T_lo = _mm_unpacklo_epi8(T, zero);
627 const __m128i T_hi = _mm_unpackhi_epi8(T, zero);
628 const __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]);
629 const __m128i TL_lo = _mm_unpacklo_epi8(TL, zero);
630 const __m128i TL_hi = _mm_unpackhi_epi8(TL, zero);
631 const __m128i diff_lo = _mm_sub_epi16(T_lo, TL_lo);
632 const __m128i diff_hi = _mm_sub_epi16(T_hi, TL_hi);
633 const __m128i pred_lo = _mm_add_epi16(L_lo, diff_lo);
634 const __m128i pred_hi = _mm_add_epi16(L_hi, diff_hi);
635 const __m128i pred = _mm_packus_epi16(pred_lo, pred_hi);
636 const __m128i res = _mm_sub_epi8(src, pred);
637 _mm_storeu_si128((__m128i*)&out[i], res);
638 }
639 if (i != num_pixels) {
640 VP8LPredictorsSub_C[12](in + i, upper + i, num_pixels - i, out + i);
641 }
642 }
643
644 // Predictors13: ClampedAddSubtractHalf
PredictorSub13_SSE2(const uint32_t * in,const uint32_t * upper,int num_pixels,uint32_t * out)645 static void PredictorSub13_SSE2(const uint32_t* in, const uint32_t* upper,
646 int num_pixels, uint32_t* out) {
647 int i;
648 const __m128i zero = _mm_setzero_si128();
649 for (i = 0; i + 2 <= num_pixels; i += 2) {
650 // we can only process two pixels at a time
651 const __m128i L = _mm_loadl_epi64((const __m128i*)&in[i - 1]);
652 const __m128i src = _mm_loadl_epi64((const __m128i*)&in[i]);
653 const __m128i T = _mm_loadl_epi64((const __m128i*)&upper[i]);
654 const __m128i TL = _mm_loadl_epi64((const __m128i*)&upper[i - 1]);
655 const __m128i L_lo = _mm_unpacklo_epi8(L, zero);
656 const __m128i T_lo = _mm_unpacklo_epi8(T, zero);
657 const __m128i TL_lo = _mm_unpacklo_epi8(TL, zero);
658 const __m128i sum = _mm_add_epi16(T_lo, L_lo);
659 const __m128i avg = _mm_srli_epi16(sum, 1);
660 const __m128i A1 = _mm_sub_epi16(avg, TL_lo);
661 const __m128i bit_fix = _mm_cmpgt_epi16(TL_lo, avg);
662 const __m128i A2 = _mm_sub_epi16(A1, bit_fix);
663 const __m128i A3 = _mm_srai_epi16(A2, 1);
664 const __m128i A4 = _mm_add_epi16(avg, A3);
665 const __m128i pred = _mm_packus_epi16(A4, A4);
666 const __m128i res = _mm_sub_epi8(src, pred);
667 _mm_storel_epi64((__m128i*)&out[i], res);
668 }
669 if (i != num_pixels) {
670 VP8LPredictorsSub_C[13](in + i, upper + i, num_pixels - i, out + i);
671 }
672 }
673
674 //------------------------------------------------------------------------------
675 // Entry point
676
677 extern void VP8LEncDspInitSSE2(void);
678
VP8LEncDspInitSSE2(void)679 WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitSSE2(void) {
680 VP8LSubtractGreenFromBlueAndRed = SubtractGreenFromBlueAndRed;
681 VP8LTransformColor = TransformColor;
682 VP8LCollectColorBlueTransforms = CollectColorBlueTransforms;
683 VP8LCollectColorRedTransforms = CollectColorRedTransforms;
684 VP8LHistogramAdd = HistogramAdd;
685 VP8LCombinedShannonEntropy = CombinedShannonEntropy;
686 VP8LVectorMismatch = VectorMismatch;
687 VP8LBundleColorMap = BundleColorMap_SSE2;
688
689 VP8LPredictorsSub[0] = PredictorSub0_SSE2;
690 VP8LPredictorsSub[1] = PredictorSub1_SSE2;
691 VP8LPredictorsSub[2] = PredictorSub2_SSE2;
692 VP8LPredictorsSub[3] = PredictorSub3_SSE2;
693 VP8LPredictorsSub[4] = PredictorSub4_SSE2;
694 VP8LPredictorsSub[5] = PredictorSub5_SSE2;
695 VP8LPredictorsSub[6] = PredictorSub6_SSE2;
696 VP8LPredictorsSub[7] = PredictorSub7_SSE2;
697 VP8LPredictorsSub[8] = PredictorSub8_SSE2;
698 VP8LPredictorsSub[9] = PredictorSub9_SSE2;
699 VP8LPredictorsSub[10] = PredictorSub10_SSE2;
700 VP8LPredictorsSub[11] = PredictorSub11_SSE2;
701 VP8LPredictorsSub[12] = PredictorSub12_SSE2;
702 VP8LPredictorsSub[13] = PredictorSub13_SSE2;
703 VP8LPredictorsSub[14] = PredictorSub0_SSE2; // <- padding security sentinels
704 VP8LPredictorsSub[15] = PredictorSub0_SSE2;
705 }
706
707 #else // !WEBP_USE_SSE2
708
709 WEBP_DSP_INIT_STUB(VP8LEncDspInitSSE2)
710
711 #endif // WEBP_USE_SSE2
712