1 /*
2 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #include <assert.h>
13 #include <emmintrin.h> // SSE2
14
15 #include "config/aom_config.h"
16 #include "config/aom_dsp_rtcd.h"
17
18 #include "aom_dsp/blend.h"
19 #include "aom_dsp/x86/mem_sse2.h"
20 #include "aom_dsp/x86/synonyms.h"
21 #include "aom_ports/mem.h"
22
aom_get_mb_ss_sse2(const int16_t * src)23 unsigned int aom_get_mb_ss_sse2(const int16_t *src) {
24 __m128i vsum = _mm_setzero_si128();
25 int i;
26
27 for (i = 0; i < 32; ++i) {
28 const __m128i v = xx_loadu_128(src);
29 vsum = _mm_add_epi32(vsum, _mm_madd_epi16(v, v));
30 src += 8;
31 }
32
33 vsum = _mm_add_epi32(vsum, _mm_srli_si128(vsum, 8));
34 vsum = _mm_add_epi32(vsum, _mm_srli_si128(vsum, 4));
35 return (unsigned int)_mm_cvtsi128_si32(vsum);
36 }
37
load4x2_sse2(const uint8_t * const p,const int stride)38 static INLINE __m128i load4x2_sse2(const uint8_t *const p, const int stride) {
39 const __m128i p0 = _mm_cvtsi32_si128(loadu_int32(p + 0 * stride));
40 const __m128i p1 = _mm_cvtsi32_si128(loadu_int32(p + 1 * stride));
41 return _mm_unpacklo_epi8(_mm_unpacklo_epi32(p0, p1), _mm_setzero_si128());
42 }
43
load8_8to16_sse2(const uint8_t * const p)44 static INLINE __m128i load8_8to16_sse2(const uint8_t *const p) {
45 const __m128i p0 = _mm_loadl_epi64((const __m128i *)p);
46 return _mm_unpacklo_epi8(p0, _mm_setzero_si128());
47 }
48
load16_8to16_sse2(const uint8_t * const p,__m128i * out)49 static INLINE void load16_8to16_sse2(const uint8_t *const p, __m128i *out) {
50 const __m128i p0 = _mm_loadu_si128((const __m128i *)p);
51 out[0] = _mm_unpacklo_epi8(p0, _mm_setzero_si128()); // lower 8 values
52 out[1] = _mm_unpackhi_epi8(p0, _mm_setzero_si128()); // upper 8 values
53 }
54
55 // Accumulate 4 32bit numbers in val to 1 32bit number
add32x4_sse2(__m128i val)56 static INLINE unsigned int add32x4_sse2(__m128i val) {
57 val = _mm_add_epi32(val, _mm_srli_si128(val, 8));
58 val = _mm_add_epi32(val, _mm_srli_si128(val, 4));
59 return (unsigned int)_mm_cvtsi128_si32(val);
60 }
61
62 // Accumulate 8 16bit in sum to 4 32bit number
sum_to_32bit_sse2(const __m128i sum)63 static INLINE __m128i sum_to_32bit_sse2(const __m128i sum) {
64 const __m128i sum_lo = _mm_srai_epi32(_mm_unpacklo_epi16(sum, sum), 16);
65 const __m128i sum_hi = _mm_srai_epi32(_mm_unpackhi_epi16(sum, sum), 16);
66 return _mm_add_epi32(sum_lo, sum_hi);
67 }
68
variance_kernel_sse2(const __m128i src,const __m128i ref,__m128i * const sse,__m128i * const sum)69 static INLINE void variance_kernel_sse2(const __m128i src, const __m128i ref,
70 __m128i *const sse,
71 __m128i *const sum) {
72 const __m128i diff = _mm_sub_epi16(src, ref);
73 *sse = _mm_add_epi32(*sse, _mm_madd_epi16(diff, diff));
74 *sum = _mm_add_epi16(*sum, diff);
75 }
76
77 // Can handle 128 pixels' diff sum (such as 8x16 or 16x8)
78 // Slightly faster than variance_final_256_pel_sse2()
79 // diff sum of 128 pixels can still fit in 16bit integer
variance_final_128_pel_sse2(__m128i vsse,__m128i vsum,unsigned int * const sse,int * const sum)80 static INLINE void variance_final_128_pel_sse2(__m128i vsse, __m128i vsum,
81 unsigned int *const sse,
82 int *const sum) {
83 *sse = add32x4_sse2(vsse);
84
85 vsum = _mm_add_epi16(vsum, _mm_srli_si128(vsum, 8));
86 vsum = _mm_add_epi16(vsum, _mm_srli_si128(vsum, 4));
87 vsum = _mm_add_epi16(vsum, _mm_srli_si128(vsum, 2));
88 *sum = (int16_t)_mm_extract_epi16(vsum, 0);
89 }
90
91 // Can handle 256 pixels' diff sum (such as 16x16)
variance_final_256_pel_sse2(__m128i vsse,__m128i vsum,unsigned int * const sse,int * const sum)92 static INLINE void variance_final_256_pel_sse2(__m128i vsse, __m128i vsum,
93 unsigned int *const sse,
94 int *const sum) {
95 *sse = add32x4_sse2(vsse);
96
97 vsum = _mm_add_epi16(vsum, _mm_srli_si128(vsum, 8));
98 vsum = _mm_add_epi16(vsum, _mm_srli_si128(vsum, 4));
99 *sum = (int16_t)_mm_extract_epi16(vsum, 0);
100 *sum += (int16_t)_mm_extract_epi16(vsum, 1);
101 }
102
103 // Can handle 512 pixels' diff sum (such as 16x32 or 32x16)
variance_final_512_pel_sse2(__m128i vsse,__m128i vsum,unsigned int * const sse,int * const sum)104 static INLINE void variance_final_512_pel_sse2(__m128i vsse, __m128i vsum,
105 unsigned int *const sse,
106 int *const sum) {
107 *sse = add32x4_sse2(vsse);
108
109 vsum = _mm_add_epi16(vsum, _mm_srli_si128(vsum, 8));
110 vsum = _mm_unpacklo_epi16(vsum, vsum);
111 vsum = _mm_srai_epi32(vsum, 16);
112 *sum = (int)add32x4_sse2(vsum);
113 }
114
115 // Can handle 1024 pixels' diff sum (such as 32x32)
variance_final_1024_pel_sse2(__m128i vsse,__m128i vsum,unsigned int * const sse,int * const sum)116 static INLINE void variance_final_1024_pel_sse2(__m128i vsse, __m128i vsum,
117 unsigned int *const sse,
118 int *const sum) {
119 *sse = add32x4_sse2(vsse);
120
121 vsum = sum_to_32bit_sse2(vsum);
122 *sum = (int)add32x4_sse2(vsum);
123 }
124
variance4_sse2(const uint8_t * src,const int src_stride,const uint8_t * ref,const int ref_stride,const int h,__m128i * const sse,__m128i * const sum)125 static INLINE void variance4_sse2(const uint8_t *src, const int src_stride,
126 const uint8_t *ref, const int ref_stride,
127 const int h, __m128i *const sse,
128 __m128i *const sum) {
129 assert(h <= 256); // May overflow for larger height.
130 *sum = _mm_setzero_si128();
131
132 for (int i = 0; i < h; i += 2) {
133 const __m128i s = load4x2_sse2(src, src_stride);
134 const __m128i r = load4x2_sse2(ref, ref_stride);
135
136 variance_kernel_sse2(s, r, sse, sum);
137 src += 2 * src_stride;
138 ref += 2 * ref_stride;
139 }
140 }
141
variance8_sse2(const uint8_t * src,const int src_stride,const uint8_t * ref,const int ref_stride,const int h,__m128i * const sse,__m128i * const sum)142 static INLINE void variance8_sse2(const uint8_t *src, const int src_stride,
143 const uint8_t *ref, const int ref_stride,
144 const int h, __m128i *const sse,
145 __m128i *const sum) {
146 assert(h <= 128); // May overflow for larger height.
147 *sum = _mm_setzero_si128();
148 *sse = _mm_setzero_si128();
149 for (int i = 0; i < h; i++) {
150 const __m128i s = load8_8to16_sse2(src);
151 const __m128i r = load8_8to16_sse2(ref);
152
153 variance_kernel_sse2(s, r, sse, sum);
154 src += src_stride;
155 ref += ref_stride;
156 }
157 }
158
variance16_kernel_sse2(const uint8_t * const src,const uint8_t * const ref,__m128i * const sse,__m128i * const sum)159 static INLINE void variance16_kernel_sse2(const uint8_t *const src,
160 const uint8_t *const ref,
161 __m128i *const sse,
162 __m128i *const sum) {
163 const __m128i zero = _mm_setzero_si128();
164 const __m128i s = _mm_loadu_si128((const __m128i *)src);
165 const __m128i r = _mm_loadu_si128((const __m128i *)ref);
166 const __m128i src0 = _mm_unpacklo_epi8(s, zero);
167 const __m128i ref0 = _mm_unpacklo_epi8(r, zero);
168 const __m128i src1 = _mm_unpackhi_epi8(s, zero);
169 const __m128i ref1 = _mm_unpackhi_epi8(r, zero);
170
171 variance_kernel_sse2(src0, ref0, sse, sum);
172 variance_kernel_sse2(src1, ref1, sse, sum);
173 }
174
variance16_sse2(const uint8_t * src,const int src_stride,const uint8_t * ref,const int ref_stride,const int h,__m128i * const sse,__m128i * const sum)175 static INLINE void variance16_sse2(const uint8_t *src, const int src_stride,
176 const uint8_t *ref, const int ref_stride,
177 const int h, __m128i *const sse,
178 __m128i *const sum) {
179 assert(h <= 64); // May overflow for larger height.
180 *sum = _mm_setzero_si128();
181
182 for (int i = 0; i < h; ++i) {
183 variance16_kernel_sse2(src, ref, sse, sum);
184 src += src_stride;
185 ref += ref_stride;
186 }
187 }
188
variance32_sse2(const uint8_t * src,const int src_stride,const uint8_t * ref,const int ref_stride,const int h,__m128i * const sse,__m128i * const sum)189 static INLINE void variance32_sse2(const uint8_t *src, const int src_stride,
190 const uint8_t *ref, const int ref_stride,
191 const int h, __m128i *const sse,
192 __m128i *const sum) {
193 assert(h <= 32); // May overflow for larger height.
194 // Don't initialize sse here since it's an accumulation.
195 *sum = _mm_setzero_si128();
196
197 for (int i = 0; i < h; ++i) {
198 variance16_kernel_sse2(src + 0, ref + 0, sse, sum);
199 variance16_kernel_sse2(src + 16, ref + 16, sse, sum);
200 src += src_stride;
201 ref += ref_stride;
202 }
203 }
204
variance64_sse2(const uint8_t * src,const int src_stride,const uint8_t * ref,const int ref_stride,const int h,__m128i * const sse,__m128i * const sum)205 static INLINE void variance64_sse2(const uint8_t *src, const int src_stride,
206 const uint8_t *ref, const int ref_stride,
207 const int h, __m128i *const sse,
208 __m128i *const sum) {
209 assert(h <= 16); // May overflow for larger height.
210 *sum = _mm_setzero_si128();
211
212 for (int i = 0; i < h; ++i) {
213 variance16_kernel_sse2(src + 0, ref + 0, sse, sum);
214 variance16_kernel_sse2(src + 16, ref + 16, sse, sum);
215 variance16_kernel_sse2(src + 32, ref + 32, sse, sum);
216 variance16_kernel_sse2(src + 48, ref + 48, sse, sum);
217 src += src_stride;
218 ref += ref_stride;
219 }
220 }
221
variance128_sse2(const uint8_t * src,const int src_stride,const uint8_t * ref,const int ref_stride,const int h,__m128i * const sse,__m128i * const sum)222 static INLINE void variance128_sse2(const uint8_t *src, const int src_stride,
223 const uint8_t *ref, const int ref_stride,
224 const int h, __m128i *const sse,
225 __m128i *const sum) {
226 assert(h <= 8); // May overflow for larger height.
227 *sum = _mm_setzero_si128();
228
229 for (int i = 0; i < h; ++i) {
230 for (int j = 0; j < 4; ++j) {
231 const int offset0 = j << 5;
232 const int offset1 = offset0 + 16;
233 variance16_kernel_sse2(src + offset0, ref + offset0, sse, sum);
234 variance16_kernel_sse2(src + offset1, ref + offset1, sse, sum);
235 }
236 src += src_stride;
237 ref += ref_stride;
238 }
239 }
240
aom_get_var_sse_sum_8x8_quad_sse2(const uint8_t * src_ptr,int src_stride,const uint8_t * ref_ptr,int ref_stride,uint32_t * sse8x8,int * sum8x8,unsigned int * tot_sse,int * tot_sum,uint32_t * var8x8)241 void aom_get_var_sse_sum_8x8_quad_sse2(const uint8_t *src_ptr, int src_stride,
242 const uint8_t *ref_ptr, int ref_stride,
243 uint32_t *sse8x8, int *sum8x8,
244 unsigned int *tot_sse, int *tot_sum,
245 uint32_t *var8x8) {
246 // Loop over 4 8x8 blocks. Process one 8x32 block.
247 for (int k = 0; k < 4; k++) {
248 const uint8_t *src = src_ptr;
249 const uint8_t *ref = ref_ptr;
250 __m128i vsum = _mm_setzero_si128();
251 __m128i vsse = _mm_setzero_si128();
252 for (int i = 0; i < 8; i++) {
253 const __m128i s = load8_8to16_sse2(src + (k * 8));
254 const __m128i r = load8_8to16_sse2(ref + (k * 8));
255 const __m128i diff = _mm_sub_epi16(s, r);
256 vsse = _mm_add_epi32(vsse, _mm_madd_epi16(diff, diff));
257 vsum = _mm_add_epi16(vsum, diff);
258
259 src += src_stride;
260 ref += ref_stride;
261 }
262 variance_final_128_pel_sse2(vsse, vsum, &sse8x8[k], &sum8x8[k]);
263 }
264
265 // Calculate variance at 8x8 level and total sse, sum of 8x32 block.
266 *tot_sse += sse8x8[0] + sse8x8[1] + sse8x8[2] + sse8x8[3];
267 *tot_sum += sum8x8[0] + sum8x8[1] + sum8x8[2] + sum8x8[3];
268 for (int i = 0; i < 4; i++)
269 var8x8[i] = sse8x8[i] - (uint32_t)(((int64_t)sum8x8[i] * sum8x8[i]) >> 6);
270 }
271
aom_get_var_sse_sum_16x16_dual_sse2(const uint8_t * src_ptr,int src_stride,const uint8_t * ref_ptr,int ref_stride,uint32_t * sse16x16,unsigned int * tot_sse,int * tot_sum,uint32_t * var16x16)272 void aom_get_var_sse_sum_16x16_dual_sse2(const uint8_t *src_ptr, int src_stride,
273 const uint8_t *ref_ptr, int ref_stride,
274 uint32_t *sse16x16,
275 unsigned int *tot_sse, int *tot_sum,
276 uint32_t *var16x16) {
277 int sum16x16[2] = { 0 };
278 // Loop over 2 16x16 blocks. Process one 16x32 block.
279 for (int k = 0; k < 2; k++) {
280 const uint8_t *src = src_ptr;
281 const uint8_t *ref = ref_ptr;
282 __m128i vsum = _mm_setzero_si128();
283 __m128i vsse = _mm_setzero_si128();
284 for (int i = 0; i < 16; i++) {
285 __m128i s[2];
286 __m128i r[2];
287 load16_8to16_sse2(src + (k * 16), s);
288 load16_8to16_sse2(ref + (k * 16), r);
289 const __m128i diff0 = _mm_sub_epi16(s[0], r[0]);
290 const __m128i diff1 = _mm_sub_epi16(s[1], r[1]);
291 vsse = _mm_add_epi32(vsse, _mm_madd_epi16(diff0, diff0));
292 vsse = _mm_add_epi32(vsse, _mm_madd_epi16(diff1, diff1));
293 vsum = _mm_add_epi16(vsum, _mm_add_epi16(diff0, diff1));
294 src += src_stride;
295 ref += ref_stride;
296 }
297 variance_final_256_pel_sse2(vsse, vsum, &sse16x16[k], &sum16x16[k]);
298 }
299
300 // Calculate variance at 16x16 level and total sse, sum of 16x32 block.
301 *tot_sse += sse16x16[0] + sse16x16[1];
302 *tot_sum += sum16x16[0] + sum16x16[1];
303 for (int i = 0; i < 2; i++)
304 var16x16[i] =
305 sse16x16[i] - (uint32_t)(((int64_t)sum16x16[i] * sum16x16[i]) >> 8);
306 }
307
308 #define AOM_VAR_NO_LOOP_SSE2(bw, bh, bits, max_pixels) \
309 unsigned int aom_variance##bw##x##bh##_sse2( \
310 const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
311 unsigned int *sse) { \
312 __m128i vsse = _mm_setzero_si128(); \
313 __m128i vsum; \
314 int sum = 0; \
315 variance##bw##_sse2(src, src_stride, ref, ref_stride, bh, &vsse, &vsum); \
316 variance_final_##max_pixels##_pel_sse2(vsse, vsum, sse, &sum); \
317 assert(sum <= 255 * bw * bh); \
318 assert(sum >= -255 * bw * bh); \
319 return *sse - (uint32_t)(((int64_t)sum * sum) >> bits); \
320 }
321
322 AOM_VAR_NO_LOOP_SSE2(4, 4, 4, 128)
323 AOM_VAR_NO_LOOP_SSE2(4, 8, 5, 128)
324 AOM_VAR_NO_LOOP_SSE2(4, 16, 6, 128)
325
326 AOM_VAR_NO_LOOP_SSE2(8, 4, 5, 128)
327 AOM_VAR_NO_LOOP_SSE2(8, 8, 6, 128)
328 AOM_VAR_NO_LOOP_SSE2(8, 16, 7, 128)
329
330 AOM_VAR_NO_LOOP_SSE2(16, 8, 7, 128)
331 AOM_VAR_NO_LOOP_SSE2(16, 16, 8, 256)
332 AOM_VAR_NO_LOOP_SSE2(16, 32, 9, 512)
333
334 AOM_VAR_NO_LOOP_SSE2(32, 8, 8, 256)
335 AOM_VAR_NO_LOOP_SSE2(32, 16, 9, 512)
336 AOM_VAR_NO_LOOP_SSE2(32, 32, 10, 1024)
337
338 #if !CONFIG_REALTIME_ONLY
339 AOM_VAR_NO_LOOP_SSE2(16, 4, 6, 128)
340 AOM_VAR_NO_LOOP_SSE2(8, 32, 8, 256)
341 AOM_VAR_NO_LOOP_SSE2(16, 64, 10, 1024)
342 #endif
343
344 #define AOM_VAR_LOOP_SSE2(bw, bh, bits, uh) \
345 unsigned int aom_variance##bw##x##bh##_sse2( \
346 const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
347 unsigned int *sse) { \
348 __m128i vsse = _mm_setzero_si128(); \
349 __m128i vsum = _mm_setzero_si128(); \
350 for (int i = 0; i < (bh / uh); ++i) { \
351 __m128i vsum16; \
352 variance##bw##_sse2(src, src_stride, ref, ref_stride, uh, &vsse, \
353 &vsum16); \
354 vsum = _mm_add_epi32(vsum, sum_to_32bit_sse2(vsum16)); \
355 src += (src_stride * uh); \
356 ref += (ref_stride * uh); \
357 } \
358 *sse = add32x4_sse2(vsse); \
359 int sum = (int)add32x4_sse2(vsum); \
360 assert(sum <= 255 * bw * bh); \
361 assert(sum >= -255 * bw * bh); \
362 return *sse - (uint32_t)(((int64_t)sum * sum) >> bits); \
363 }
364
365 AOM_VAR_LOOP_SSE2(32, 64, 11, 32) // 32x32 * ( 64/32 )
366
367 AOM_VAR_LOOP_SSE2(64, 32, 11, 16) // 64x16 * ( 32/16 )
368 AOM_VAR_LOOP_SSE2(64, 64, 12, 16) // 64x16 * ( 64/16 )
369 AOM_VAR_LOOP_SSE2(64, 128, 13, 16) // 64x16 * ( 128/16 )
370
371 AOM_VAR_LOOP_SSE2(128, 64, 13, 8) // 128x8 * ( 64/8 )
372 AOM_VAR_LOOP_SSE2(128, 128, 14, 8) // 128x8 * ( 128/8 )
373
374 #if !CONFIG_REALTIME_ONLY
375 AOM_VAR_NO_LOOP_SSE2(64, 16, 10, 1024)
376 #endif
377
aom_mse8x8_sse2(const uint8_t * src,int src_stride,const uint8_t * ref,int ref_stride,unsigned int * sse)378 unsigned int aom_mse8x8_sse2(const uint8_t *src, int src_stride,
379 const uint8_t *ref, int ref_stride,
380 unsigned int *sse) {
381 aom_variance8x8_sse2(src, src_stride, ref, ref_stride, sse);
382 return *sse;
383 }
384
aom_mse8x16_sse2(const uint8_t * src,int src_stride,const uint8_t * ref,int ref_stride,unsigned int * sse)385 unsigned int aom_mse8x16_sse2(const uint8_t *src, int src_stride,
386 const uint8_t *ref, int ref_stride,
387 unsigned int *sse) {
388 aom_variance8x16_sse2(src, src_stride, ref, ref_stride, sse);
389 return *sse;
390 }
391
aom_mse16x8_sse2(const uint8_t * src,int src_stride,const uint8_t * ref,int ref_stride,unsigned int * sse)392 unsigned int aom_mse16x8_sse2(const uint8_t *src, int src_stride,
393 const uint8_t *ref, int ref_stride,
394 unsigned int *sse) {
395 aom_variance16x8_sse2(src, src_stride, ref, ref_stride, sse);
396 return *sse;
397 }
398
aom_mse16x16_sse2(const uint8_t * src,int src_stride,const uint8_t * ref,int ref_stride,unsigned int * sse)399 unsigned int aom_mse16x16_sse2(const uint8_t *src, int src_stride,
400 const uint8_t *ref, int ref_stride,
401 unsigned int *sse) {
402 aom_variance16x16_sse2(src, src_stride, ref, ref_stride, sse);
403 return *sse;
404 }
405
406 // The 2 unused parameters are place holders for PIC enabled build.
407 // These definitions are for functions defined in subpel_variance.asm
408 #define DECL(w, opt) \
409 int aom_sub_pixel_variance##w##xh_##opt( \
410 const uint8_t *src, ptrdiff_t src_stride, int x_offset, int y_offset, \
411 const uint8_t *dst, ptrdiff_t dst_stride, int height, unsigned int *sse, \
412 void *unused0, void *unused)
413 #define DECLS(opt) \
414 DECL(4, opt); \
415 DECL(8, opt); \
416 DECL(16, opt)
417
418 DECLS(ssse3);
419 #undef DECLS
420 #undef DECL
421
422 #define FN(w, h, wf, wlog2, hlog2, opt, cast_prod, cast) \
423 unsigned int aom_sub_pixel_variance##w##x##h##_##opt( \
424 const uint8_t *src, int src_stride, int x_offset, int y_offset, \
425 const uint8_t *dst, int dst_stride, unsigned int *sse_ptr) { \
426 /*Avoid overflow in helper by capping height.*/ \
427 const int hf = AOMMIN(h, 64); \
428 unsigned int sse = 0; \
429 int se = 0; \
430 for (int i = 0; i < (w / wf); ++i) { \
431 const uint8_t *src_ptr = src; \
432 const uint8_t *dst_ptr = dst; \
433 for (int j = 0; j < (h / hf); ++j) { \
434 unsigned int sse2; \
435 const int se2 = aom_sub_pixel_variance##wf##xh_##opt( \
436 src_ptr, src_stride, x_offset, y_offset, dst_ptr, dst_stride, hf, \
437 &sse2, NULL, NULL); \
438 dst_ptr += hf * dst_stride; \
439 src_ptr += hf * src_stride; \
440 se += se2; \
441 sse += sse2; \
442 } \
443 src += wf; \
444 dst += wf; \
445 } \
446 *sse_ptr = sse; \
447 return sse - (unsigned int)(cast_prod(cast se * se) >> (wlog2 + hlog2)); \
448 }
449
450 #if !CONFIG_REALTIME_ONLY
451 #define FNS(opt) \
452 FN(128, 128, 16, 7, 7, opt, (int64_t), (int64_t)) \
453 FN(128, 64, 16, 7, 6, opt, (int64_t), (int64_t)) \
454 FN(64, 128, 16, 6, 7, opt, (int64_t), (int64_t)) \
455 FN(64, 64, 16, 6, 6, opt, (int64_t), (int64_t)) \
456 FN(64, 32, 16, 6, 5, opt, (int64_t), (int64_t)) \
457 FN(32, 64, 16, 5, 6, opt, (int64_t), (int64_t)) \
458 FN(32, 32, 16, 5, 5, opt, (int64_t), (int64_t)) \
459 FN(32, 16, 16, 5, 4, opt, (int64_t), (int64_t)) \
460 FN(16, 32, 16, 4, 5, opt, (int64_t), (int64_t)) \
461 FN(16, 16, 16, 4, 4, opt, (uint32_t), (int64_t)) \
462 FN(16, 8, 16, 4, 3, opt, (int32_t), (int32_t)) \
463 FN(8, 16, 8, 3, 4, opt, (int32_t), (int32_t)) \
464 FN(8, 8, 8, 3, 3, opt, (int32_t), (int32_t)) \
465 FN(8, 4, 8, 3, 2, opt, (int32_t), (int32_t)) \
466 FN(4, 8, 4, 2, 3, opt, (int32_t), (int32_t)) \
467 FN(4, 4, 4, 2, 2, opt, (int32_t), (int32_t)) \
468 FN(4, 16, 4, 2, 4, opt, (int32_t), (int32_t)) \
469 FN(16, 4, 16, 4, 2, opt, (int32_t), (int32_t)) \
470 FN(8, 32, 8, 3, 5, opt, (uint32_t), (int64_t)) \
471 FN(32, 8, 16, 5, 3, opt, (uint32_t), (int64_t)) \
472 FN(16, 64, 16, 4, 6, opt, (int64_t), (int64_t)) \
473 FN(64, 16, 16, 6, 4, opt, (int64_t), (int64_t))
474 #else
475 #define FNS(opt) \
476 FN(128, 128, 16, 7, 7, opt, (int64_t), (int64_t)) \
477 FN(128, 64, 16, 7, 6, opt, (int64_t), (int64_t)) \
478 FN(64, 128, 16, 6, 7, opt, (int64_t), (int64_t)) \
479 FN(64, 64, 16, 6, 6, opt, (int64_t), (int64_t)) \
480 FN(64, 32, 16, 6, 5, opt, (int64_t), (int64_t)) \
481 FN(32, 64, 16, 5, 6, opt, (int64_t), (int64_t)) \
482 FN(32, 32, 16, 5, 5, opt, (int64_t), (int64_t)) \
483 FN(32, 16, 16, 5, 4, opt, (int64_t), (int64_t)) \
484 FN(16, 32, 16, 4, 5, opt, (int64_t), (int64_t)) \
485 FN(16, 16, 16, 4, 4, opt, (uint32_t), (int64_t)) \
486 FN(16, 8, 16, 4, 3, opt, (int32_t), (int32_t)) \
487 FN(8, 16, 8, 3, 4, opt, (int32_t), (int32_t)) \
488 FN(8, 8, 8, 3, 3, opt, (int32_t), (int32_t)) \
489 FN(8, 4, 8, 3, 2, opt, (int32_t), (int32_t)) \
490 FN(4, 8, 4, 2, 3, opt, (int32_t), (int32_t)) \
491 FN(4, 4, 4, 2, 2, opt, (int32_t), (int32_t))
492 #endif
493
494 FNS(ssse3)
495
496 #undef FNS
497 #undef FN
498
499 // The 2 unused parameters are place holders for PIC enabled build.
500 #define DECL(w, opt) \
501 int aom_sub_pixel_avg_variance##w##xh_##opt( \
502 const uint8_t *src, ptrdiff_t src_stride, int x_offset, int y_offset, \
503 const uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *sec, \
504 ptrdiff_t sec_stride, int height, unsigned int *sse, void *unused0, \
505 void *unused)
506 #define DECLS(opt) \
507 DECL(4, opt); \
508 DECL(8, opt); \
509 DECL(16, opt)
510
511 DECLS(ssse3);
512 #undef DECL
513 #undef DECLS
514
515 #define FN(w, h, wf, wlog2, hlog2, opt, cast_prod, cast) \
516 unsigned int aom_sub_pixel_avg_variance##w##x##h##_##opt( \
517 const uint8_t *src, int src_stride, int x_offset, int y_offset, \
518 const uint8_t *dst, int dst_stride, unsigned int *sse_ptr, \
519 const uint8_t *sec) { \
520 /*Avoid overflow in helper by capping height.*/ \
521 const int hf = AOMMIN(h, 64); \
522 unsigned int sse = 0; \
523 int se = 0; \
524 for (int i = 0; i < (w / wf); ++i) { \
525 const uint8_t *src_ptr = src; \
526 const uint8_t *dst_ptr = dst; \
527 const uint8_t *sec_ptr = sec; \
528 for (int j = 0; j < (h / hf); ++j) { \
529 unsigned int sse2; \
530 const int se2 = aom_sub_pixel_avg_variance##wf##xh_##opt( \
531 src_ptr, src_stride, x_offset, y_offset, dst_ptr, dst_stride, \
532 sec_ptr, w, hf, &sse2, NULL, NULL); \
533 dst_ptr += hf * dst_stride; \
534 src_ptr += hf * src_stride; \
535 sec_ptr += hf * w; \
536 se += se2; \
537 sse += sse2; \
538 } \
539 src += wf; \
540 dst += wf; \
541 sec += wf; \
542 } \
543 *sse_ptr = sse; \
544 return sse - (unsigned int)(cast_prod(cast se * se) >> (wlog2 + hlog2)); \
545 }
546
547 #if !CONFIG_REALTIME_ONLY
548 #define FNS(opt) \
549 FN(128, 128, 16, 7, 7, opt, (int64_t), (int64_t)) \
550 FN(128, 64, 16, 7, 6, opt, (int64_t), (int64_t)) \
551 FN(64, 128, 16, 6, 7, opt, (int64_t), (int64_t)) \
552 FN(64, 64, 16, 6, 6, opt, (int64_t), (int64_t)) \
553 FN(64, 32, 16, 6, 5, opt, (int64_t), (int64_t)) \
554 FN(32, 64, 16, 5, 6, opt, (int64_t), (int64_t)) \
555 FN(32, 32, 16, 5, 5, opt, (int64_t), (int64_t)) \
556 FN(32, 16, 16, 5, 4, opt, (int64_t), (int64_t)) \
557 FN(16, 32, 16, 4, 5, opt, (int64_t), (int64_t)) \
558 FN(16, 16, 16, 4, 4, opt, (uint32_t), (int64_t)) \
559 FN(16, 8, 16, 4, 3, opt, (uint32_t), (int32_t)) \
560 FN(8, 16, 8, 3, 4, opt, (uint32_t), (int32_t)) \
561 FN(8, 8, 8, 3, 3, opt, (uint32_t), (int32_t)) \
562 FN(8, 4, 8, 3, 2, opt, (uint32_t), (int32_t)) \
563 FN(4, 8, 4, 2, 3, opt, (uint32_t), (int32_t)) \
564 FN(4, 4, 4, 2, 2, opt, (uint32_t), (int32_t)) \
565 FN(4, 16, 4, 2, 4, opt, (int32_t), (int32_t)) \
566 FN(16, 4, 16, 4, 2, opt, (int32_t), (int32_t)) \
567 FN(8, 32, 8, 3, 5, opt, (uint32_t), (int64_t)) \
568 FN(32, 8, 16, 5, 3, opt, (uint32_t), (int64_t)) \
569 FN(16, 64, 16, 4, 6, opt, (int64_t), (int64_t)) \
570 FN(64, 16, 16, 6, 4, opt, (int64_t), (int64_t))
571 #else
572 #define FNS(opt) \
573 FN(128, 128, 16, 7, 7, opt, (int64_t), (int64_t)) \
574 FN(128, 64, 16, 7, 6, opt, (int64_t), (int64_t)) \
575 FN(64, 128, 16, 6, 7, opt, (int64_t), (int64_t)) \
576 FN(64, 64, 16, 6, 6, opt, (int64_t), (int64_t)) \
577 FN(64, 32, 16, 6, 5, opt, (int64_t), (int64_t)) \
578 FN(32, 64, 16, 5, 6, opt, (int64_t), (int64_t)) \
579 FN(32, 32, 16, 5, 5, opt, (int64_t), (int64_t)) \
580 FN(32, 16, 16, 5, 4, opt, (int64_t), (int64_t)) \
581 FN(16, 32, 16, 4, 5, opt, (int64_t), (int64_t)) \
582 FN(16, 16, 16, 4, 4, opt, (uint32_t), (int64_t)) \
583 FN(16, 8, 16, 4, 3, opt, (uint32_t), (int32_t)) \
584 FN(8, 16, 8, 3, 4, opt, (uint32_t), (int32_t)) \
585 FN(8, 8, 8, 3, 3, opt, (uint32_t), (int32_t)) \
586 FN(8, 4, 8, 3, 2, opt, (uint32_t), (int32_t)) \
587 FN(4, 8, 4, 2, 3, opt, (uint32_t), (int32_t)) \
588 FN(4, 4, 4, 2, 2, opt, (uint32_t), (int32_t))
589 #endif
590
FNS(ssse3)591 FNS(ssse3)
592
593 #undef FNS
594 #undef FN
595
596 static INLINE __m128i highbd_comp_mask_pred_line_sse2(const __m128i s0,
597 const __m128i s1,
598 const __m128i a) {
599 const __m128i alpha_max = _mm_set1_epi16((1 << AOM_BLEND_A64_ROUND_BITS));
600 const __m128i round_const =
601 _mm_set1_epi32((1 << AOM_BLEND_A64_ROUND_BITS) >> 1);
602 const __m128i a_inv = _mm_sub_epi16(alpha_max, a);
603
604 const __m128i s_lo = _mm_unpacklo_epi16(s0, s1);
605 const __m128i a_lo = _mm_unpacklo_epi16(a, a_inv);
606 const __m128i pred_lo = _mm_madd_epi16(s_lo, a_lo);
607 const __m128i pred_l = _mm_srai_epi32(_mm_add_epi32(pred_lo, round_const),
608 AOM_BLEND_A64_ROUND_BITS);
609
610 const __m128i s_hi = _mm_unpackhi_epi16(s0, s1);
611 const __m128i a_hi = _mm_unpackhi_epi16(a, a_inv);
612 const __m128i pred_hi = _mm_madd_epi16(s_hi, a_hi);
613 const __m128i pred_h = _mm_srai_epi32(_mm_add_epi32(pred_hi, round_const),
614 AOM_BLEND_A64_ROUND_BITS);
615
616 const __m128i comp = _mm_packs_epi32(pred_l, pred_h);
617
618 return comp;
619 }
620
aom_highbd_comp_mask_pred_sse2(uint8_t * comp_pred8,const uint8_t * pred8,int width,int height,const uint8_t * ref8,int ref_stride,const uint8_t * mask,int mask_stride,int invert_mask)621 void aom_highbd_comp_mask_pred_sse2(uint8_t *comp_pred8, const uint8_t *pred8,
622 int width, int height, const uint8_t *ref8,
623 int ref_stride, const uint8_t *mask,
624 int mask_stride, int invert_mask) {
625 int i = 0;
626 uint16_t *comp_pred = CONVERT_TO_SHORTPTR(comp_pred8);
627 uint16_t *pred = CONVERT_TO_SHORTPTR(pred8);
628 uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
629 const uint16_t *src0 = invert_mask ? pred : ref;
630 const uint16_t *src1 = invert_mask ? ref : pred;
631 const int stride0 = invert_mask ? width : ref_stride;
632 const int stride1 = invert_mask ? ref_stride : width;
633 const __m128i zero = _mm_setzero_si128();
634
635 if (width == 8) {
636 do {
637 const __m128i s0 = _mm_loadu_si128((const __m128i *)(src0));
638 const __m128i s1 = _mm_loadu_si128((const __m128i *)(src1));
639 const __m128i m_8 = _mm_loadl_epi64((const __m128i *)mask);
640 const __m128i m_16 = _mm_unpacklo_epi8(m_8, zero);
641
642 const __m128i comp = highbd_comp_mask_pred_line_sse2(s0, s1, m_16);
643
644 _mm_storeu_si128((__m128i *)comp_pred, comp);
645
646 src0 += stride0;
647 src1 += stride1;
648 mask += mask_stride;
649 comp_pred += width;
650 i += 1;
651 } while (i < height);
652 } else if (width == 16) {
653 do {
654 const __m128i s0 = _mm_loadu_si128((const __m128i *)(src0));
655 const __m128i s2 = _mm_loadu_si128((const __m128i *)(src0 + 8));
656 const __m128i s1 = _mm_loadu_si128((const __m128i *)(src1));
657 const __m128i s3 = _mm_loadu_si128((const __m128i *)(src1 + 8));
658
659 const __m128i m_8 = _mm_loadu_si128((const __m128i *)mask);
660 const __m128i m01_16 = _mm_unpacklo_epi8(m_8, zero);
661 const __m128i m23_16 = _mm_unpackhi_epi8(m_8, zero);
662
663 const __m128i comp = highbd_comp_mask_pred_line_sse2(s0, s1, m01_16);
664 const __m128i comp1 = highbd_comp_mask_pred_line_sse2(s2, s3, m23_16);
665
666 _mm_storeu_si128((__m128i *)comp_pred, comp);
667 _mm_storeu_si128((__m128i *)(comp_pred + 8), comp1);
668
669 src0 += stride0;
670 src1 += stride1;
671 mask += mask_stride;
672 comp_pred += width;
673 i += 1;
674 } while (i < height);
675 } else {
676 do {
677 for (int x = 0; x < width; x += 32) {
678 for (int j = 0; j < 2; j++) {
679 const __m128i s0 =
680 _mm_loadu_si128((const __m128i *)(src0 + x + j * 16));
681 const __m128i s2 =
682 _mm_loadu_si128((const __m128i *)(src0 + x + 8 + j * 16));
683 const __m128i s1 =
684 _mm_loadu_si128((const __m128i *)(src1 + x + j * 16));
685 const __m128i s3 =
686 _mm_loadu_si128((const __m128i *)(src1 + x + 8 + j * 16));
687
688 const __m128i m_8 =
689 _mm_loadu_si128((const __m128i *)(mask + x + j * 16));
690 const __m128i m01_16 = _mm_unpacklo_epi8(m_8, zero);
691 const __m128i m23_16 = _mm_unpackhi_epi8(m_8, zero);
692
693 const __m128i comp = highbd_comp_mask_pred_line_sse2(s0, s1, m01_16);
694 const __m128i comp1 = highbd_comp_mask_pred_line_sse2(s2, s3, m23_16);
695
696 _mm_storeu_si128((__m128i *)(comp_pred + j * 16), comp);
697 _mm_storeu_si128((__m128i *)(comp_pred + 8 + j * 16), comp1);
698 }
699 comp_pred += 32;
700 }
701 src0 += stride0;
702 src1 += stride1;
703 mask += mask_stride;
704 i += 1;
705 } while (i < height);
706 }
707 }
708
mse_4xh_16bit_sse2(uint8_t * dst,int dstride,uint16_t * src,int sstride,int h)709 static uint64_t mse_4xh_16bit_sse2(uint8_t *dst, int dstride, uint16_t *src,
710 int sstride, int h) {
711 uint64_t sum = 0;
712 __m128i dst0_8x8, dst1_8x8, dst_16x8;
713 __m128i src0_16x4, src1_16x4, src_16x8;
714 __m128i res0_32x4, res0_64x2, res1_64x2;
715 __m128i sub_result_16x8;
716 const __m128i zeros = _mm_setzero_si128();
717 __m128i square_result = _mm_setzero_si128();
718 for (int i = 0; i < h; i += 2) {
719 dst0_8x8 = _mm_cvtsi32_si128(*(int const *)(&dst[(i + 0) * dstride]));
720 dst1_8x8 = _mm_cvtsi32_si128(*(int const *)(&dst[(i + 1) * dstride]));
721 dst_16x8 = _mm_unpacklo_epi8(_mm_unpacklo_epi32(dst0_8x8, dst1_8x8), zeros);
722
723 src0_16x4 = _mm_loadl_epi64((__m128i const *)(&src[(i + 0) * sstride]));
724 src1_16x4 = _mm_loadl_epi64((__m128i const *)(&src[(i + 1) * sstride]));
725 src_16x8 = _mm_unpacklo_epi64(src0_16x4, src1_16x4);
726
727 sub_result_16x8 = _mm_sub_epi16(src_16x8, dst_16x8);
728
729 res0_32x4 = _mm_madd_epi16(sub_result_16x8, sub_result_16x8);
730
731 res0_64x2 = _mm_unpacklo_epi32(res0_32x4, zeros);
732 res1_64x2 = _mm_unpackhi_epi32(res0_32x4, zeros);
733
734 square_result =
735 _mm_add_epi64(square_result, _mm_add_epi64(res0_64x2, res1_64x2));
736 }
737 const __m128i sum_64x1 =
738 _mm_add_epi64(square_result, _mm_srli_si128(square_result, 8));
739 xx_storel_64(&sum, sum_64x1);
740 return sum;
741 }
742
mse_8xh_16bit_sse2(uint8_t * dst,int dstride,uint16_t * src,int sstride,int h)743 static uint64_t mse_8xh_16bit_sse2(uint8_t *dst, int dstride, uint16_t *src,
744 int sstride, int h) {
745 uint64_t sum = 0;
746 __m128i dst_8x8, dst_16x8;
747 __m128i src_16x8;
748 __m128i res0_32x4, res0_64x2, res1_64x2;
749 __m128i sub_result_16x8;
750 const __m128i zeros = _mm_setzero_si128();
751 __m128i square_result = _mm_setzero_si128();
752
753 for (int i = 0; i < h; i++) {
754 dst_8x8 = _mm_loadl_epi64((__m128i const *)(&dst[(i + 0) * dstride]));
755 dst_16x8 = _mm_unpacklo_epi8(dst_8x8, zeros);
756
757 src_16x8 = _mm_loadu_si128((__m128i *)&src[i * sstride]);
758
759 sub_result_16x8 = _mm_sub_epi16(src_16x8, dst_16x8);
760
761 res0_32x4 = _mm_madd_epi16(sub_result_16x8, sub_result_16x8);
762
763 res0_64x2 = _mm_unpacklo_epi32(res0_32x4, zeros);
764 res1_64x2 = _mm_unpackhi_epi32(res0_32x4, zeros);
765
766 square_result =
767 _mm_add_epi64(square_result, _mm_add_epi64(res0_64x2, res1_64x2));
768 }
769 const __m128i sum_64x1 =
770 _mm_add_epi64(square_result, _mm_srli_si128(square_result, 8));
771 xx_storel_64(&sum, sum_64x1);
772 return sum;
773 }
774
aom_mse_wxh_16bit_sse2(uint8_t * dst,int dstride,uint16_t * src,int sstride,int w,int h)775 uint64_t aom_mse_wxh_16bit_sse2(uint8_t *dst, int dstride, uint16_t *src,
776 int sstride, int w, int h) {
777 assert((w == 8 || w == 4) && (h == 8 || h == 4) &&
778 "w=8/4 and h=8/4 must satisfy");
779 switch (w) {
780 case 4: return mse_4xh_16bit_sse2(dst, dstride, src, sstride, h);
781 case 8: return mse_8xh_16bit_sse2(dst, dstride, src, sstride, h);
782 default: assert(0 && "unsupported width"); return -1;
783 }
784 }
785
aom_mse_16xh_16bit_sse2(uint8_t * dst,int dstride,uint16_t * src,int w,int h)786 uint64_t aom_mse_16xh_16bit_sse2(uint8_t *dst, int dstride, uint16_t *src,
787 int w, int h) {
788 assert((w == 8 || w == 4) && (h == 8 || h == 4) &&
789 "w=8/4 and h=8/4 must be satisfied");
790 const int num_blks = 16 / w;
791 uint64_t sum = 0;
792 for (int i = 0; i < num_blks; i++) {
793 sum += aom_mse_wxh_16bit_sse2(dst, dstride, src, w, w, h);
794 dst += w;
795 src += (w * h);
796 }
797 return sum;
798 }
799