1 /*
2 * Copyright (c) 2018, Alliance for Open Media. All rights reserved
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #include <tmmintrin.h>
13
14 #include "config/aom_config.h"
15 #include "config/aom_dsp_rtcd.h"
16
17 #include "aom_dsp/x86/synonyms.h"
18
aom_var_filter_block2d_bil_first_pass_ssse3(const uint8_t * a,uint16_t * b,unsigned int src_pixels_per_line,unsigned int pixel_step,unsigned int output_height,unsigned int output_width,const uint8_t * filter)19 void aom_var_filter_block2d_bil_first_pass_ssse3(
20 const uint8_t *a, uint16_t *b, unsigned int src_pixels_per_line,
21 unsigned int pixel_step, unsigned int output_height,
22 unsigned int output_width, const uint8_t *filter) {
23 // Note: filter[0], filter[1] could be {128, 0}, where 128 will overflow
24 // in computation using _mm_maddubs_epi16.
25 // Change {128, 0} to {64, 0} and reduce FILTER_BITS by 1 to avoid overflow.
26 const int16_t round = (1 << (FILTER_BITS - 1)) >> 1;
27 const __m128i r = _mm_set1_epi16(round);
28 const int8_t f0 = (int8_t)(filter[0] >> 1);
29 const int8_t f1 = (int8_t)(filter[1] >> 1);
30 const __m128i filters = _mm_setr_epi8(f0, f1, f0, f1, f0, f1, f0, f1, f0, f1,
31 f0, f1, f0, f1, f0, f1);
32 unsigned int i, j;
33 (void)pixel_step;
34
35 if (output_width >= 8) {
36 for (i = 0; i < output_height; ++i) {
37 for (j = 0; j < output_width; j += 8) {
38 // load source
39 __m128i source_low = xx_loadl_64(a);
40 __m128i source_hi = xx_loadl_64(a + 1);
41
42 // unpack to:
43 // { a[0], a[1], a[1], a[2], a[2], a[3], a[3], a[4],
44 // a[4], a[5], a[5], a[6], a[6], a[7], a[7], a[8] }
45 __m128i source = _mm_unpacklo_epi8(source_low, source_hi);
46
47 // b[i] = a[i] * filter[0] + a[i + 1] * filter[1]
48 __m128i res = _mm_maddubs_epi16(source, filters);
49
50 // round
51 res = _mm_srai_epi16(_mm_add_epi16(res, r), FILTER_BITS - 1);
52
53 xx_storeu_128(b, res);
54
55 a += 8;
56 b += 8;
57 }
58
59 a += src_pixels_per_line - output_width;
60 }
61 } else {
62 const __m128i shuffle_mask =
63 _mm_setr_epi8(0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8);
64 for (i = 0; i < output_height; ++i) {
65 // load source, only first 5 values are meaningful:
66 // { a[0], a[1], a[2], a[3], a[4], xxxx }
67 __m128i source = xx_loadl_64(a);
68
69 // shuffle, up to the first 8 are useful
70 // { a[0], a[1], a[1], a[2], a[2], a[3], a[3], a[4],
71 // a[4], a[5], a[5], a[6], a[6], a[7], a[7], a[8] }
72 __m128i source_shuffle = _mm_shuffle_epi8(source, shuffle_mask);
73
74 __m128i res = _mm_maddubs_epi16(source_shuffle, filters);
75 res = _mm_srai_epi16(_mm_add_epi16(res, r), FILTER_BITS - 1);
76
77 xx_storel_64(b, res);
78
79 a += src_pixels_per_line;
80 b += output_width;
81 }
82 }
83 }
84
aom_var_filter_block2d_bil_second_pass_ssse3(const uint16_t * a,uint8_t * b,unsigned int src_pixels_per_line,unsigned int pixel_step,unsigned int output_height,unsigned int output_width,const uint8_t * filter)85 void aom_var_filter_block2d_bil_second_pass_ssse3(
86 const uint16_t *a, uint8_t *b, unsigned int src_pixels_per_line,
87 unsigned int pixel_step, unsigned int output_height,
88 unsigned int output_width, const uint8_t *filter) {
89 const int16_t round = (1 << FILTER_BITS) >> 1;
90 const __m128i r = _mm_set1_epi32(round);
91 const __m128i filters =
92 _mm_setr_epi16(filter[0], filter[1], filter[0], filter[1], filter[0],
93 filter[1], filter[0], filter[1]);
94 const __m128i shuffle_mask =
95 _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15);
96 const __m128i mask =
97 _mm_setr_epi8(0, 4, 8, 12, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1);
98 unsigned int i, j;
99
100 for (i = 0; i < output_height; ++i) {
101 for (j = 0; j < output_width; j += 4) {
102 // load source as:
103 // { a[0], a[1], a[2], a[3], a[w], a[w+1], a[w+2], a[w+3] }
104 __m128i source1 = xx_loadl_64(a);
105 __m128i source2 = xx_loadl_64(a + pixel_step);
106 __m128i source = _mm_unpacklo_epi64(source1, source2);
107
108 // shuffle source to:
109 // { a[0], a[w], a[1], a[w+1], a[2], a[w+2], a[3], a[w+3] }
110 __m128i source_shuffle = _mm_shuffle_epi8(source, shuffle_mask);
111
112 // b[i] = a[i] * filter[0] + a[w + i] * filter[1]
113 __m128i res = _mm_madd_epi16(source_shuffle, filters);
114
115 // round
116 res = _mm_srai_epi32(_mm_add_epi32(res, r), FILTER_BITS);
117
118 // shuffle to get each lower 8 bit of every 32 bit
119 res = _mm_shuffle_epi8(res, mask);
120
121 xx_storel_32(b, res);
122
123 a += 4;
124 b += 4;
125 }
126
127 a += src_pixels_per_line - output_width;
128 }
129 }
130