1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/modules/audio_processing/aecm/aecm_core.h"
12
13 #include <arm_neon.h>
14 #include <assert.h>
15
16 #include "webrtc/common_audio/signal_processing/include/real_fft.h"
17
18 // TODO(kma): Re-write the corresponding assembly file, the offset
19 // generating script and makefile, to replace these C functions.
20
21 // Square root of Hanning window in Q14.
22 const ALIGN8_BEG int16_t WebRtcAecm_kSqrtHanning[] ALIGN8_END = {
23 0,
24 399, 798, 1196, 1594, 1990, 2386, 2780, 3172,
25 3562, 3951, 4337, 4720, 5101, 5478, 5853, 6224,
26 6591, 6954, 7313, 7668, 8019, 8364, 8705, 9040,
27 9370, 9695, 10013, 10326, 10633, 10933, 11227, 11514,
28 11795, 12068, 12335, 12594, 12845, 13089, 13325, 13553,
29 13773, 13985, 14189, 14384, 14571, 14749, 14918, 15079,
30 15231, 15373, 15506, 15631, 15746, 15851, 15947, 16034,
31 16111, 16179, 16237, 16286, 16325, 16354, 16373, 16384
32 };
33
AddLanes(uint32_t * ptr,uint32x4_t v)34 static inline void AddLanes(uint32_t* ptr, uint32x4_t v) {
35 #if defined(WEBRTC_ARCH_ARM64)
36 *(ptr) = vaddvq_u32(v);
37 #else
38 uint32x2_t tmp_v;
39 tmp_v = vadd_u32(vget_low_u32(v), vget_high_u32(v));
40 tmp_v = vpadd_u32(tmp_v, tmp_v);
41 *(ptr) = vget_lane_u32(tmp_v, 0);
42 #endif
43 }
44
WebRtcAecm_CalcLinearEnergiesNeon(AecmCore * aecm,const uint16_t * far_spectrum,int32_t * echo_est,uint32_t * far_energy,uint32_t * echo_energy_adapt,uint32_t * echo_energy_stored)45 void WebRtcAecm_CalcLinearEnergiesNeon(AecmCore* aecm,
46 const uint16_t* far_spectrum,
47 int32_t* echo_est,
48 uint32_t* far_energy,
49 uint32_t* echo_energy_adapt,
50 uint32_t* echo_energy_stored) {
51 int16_t* start_stored_p = aecm->channelStored;
52 int16_t* start_adapt_p = aecm->channelAdapt16;
53 int32_t* echo_est_p = echo_est;
54 const int16_t* end_stored_p = aecm->channelStored + PART_LEN;
55 const uint16_t* far_spectrum_p = far_spectrum;
56 int16x8_t store_v, adapt_v;
57 uint16x8_t spectrum_v;
58 uint32x4_t echo_est_v_low, echo_est_v_high;
59 uint32x4_t far_energy_v, echo_stored_v, echo_adapt_v;
60
61 far_energy_v = vdupq_n_u32(0);
62 echo_adapt_v = vdupq_n_u32(0);
63 echo_stored_v = vdupq_n_u32(0);
64
65 // Get energy for the delayed far end signal and estimated
66 // echo using both stored and adapted channels.
67 // The C code:
68 // for (i = 0; i < PART_LEN1; i++) {
69 // echo_est[i] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i],
70 // far_spectrum[i]);
71 // (*far_energy) += (uint32_t)(far_spectrum[i]);
72 // *echo_energy_adapt += aecm->channelAdapt16[i] * far_spectrum[i];
73 // (*echo_energy_stored) += (uint32_t)echo_est[i];
74 // }
75 while (start_stored_p < end_stored_p) {
76 spectrum_v = vld1q_u16(far_spectrum_p);
77 adapt_v = vld1q_s16(start_adapt_p);
78 store_v = vld1q_s16(start_stored_p);
79
80 far_energy_v = vaddw_u16(far_energy_v, vget_low_u16(spectrum_v));
81 far_energy_v = vaddw_u16(far_energy_v, vget_high_u16(spectrum_v));
82
83 echo_est_v_low = vmull_u16(vreinterpret_u16_s16(vget_low_s16(store_v)),
84 vget_low_u16(spectrum_v));
85 echo_est_v_high = vmull_u16(vreinterpret_u16_s16(vget_high_s16(store_v)),
86 vget_high_u16(spectrum_v));
87 vst1q_s32(echo_est_p, vreinterpretq_s32_u32(echo_est_v_low));
88 vst1q_s32(echo_est_p + 4, vreinterpretq_s32_u32(echo_est_v_high));
89
90 echo_stored_v = vaddq_u32(echo_est_v_low, echo_stored_v);
91 echo_stored_v = vaddq_u32(echo_est_v_high, echo_stored_v);
92
93 echo_adapt_v = vmlal_u16(echo_adapt_v,
94 vreinterpret_u16_s16(vget_low_s16(adapt_v)),
95 vget_low_u16(spectrum_v));
96 echo_adapt_v = vmlal_u16(echo_adapt_v,
97 vreinterpret_u16_s16(vget_high_s16(adapt_v)),
98 vget_high_u16(spectrum_v));
99
100 start_stored_p += 8;
101 start_adapt_p += 8;
102 far_spectrum_p += 8;
103 echo_est_p += 8;
104 }
105
106 AddLanes(far_energy, far_energy_v);
107 AddLanes(echo_energy_stored, echo_stored_v);
108 AddLanes(echo_energy_adapt, echo_adapt_v);
109
110 echo_est[PART_LEN] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[PART_LEN],
111 far_spectrum[PART_LEN]);
112 *echo_energy_stored += (uint32_t)echo_est[PART_LEN];
113 *far_energy += (uint32_t)far_spectrum[PART_LEN];
114 *echo_energy_adapt += aecm->channelAdapt16[PART_LEN] * far_spectrum[PART_LEN];
115 }
116
WebRtcAecm_StoreAdaptiveChannelNeon(AecmCore * aecm,const uint16_t * far_spectrum,int32_t * echo_est)117 void WebRtcAecm_StoreAdaptiveChannelNeon(AecmCore* aecm,
118 const uint16_t* far_spectrum,
119 int32_t* echo_est) {
120 assert((uintptr_t)echo_est % 32 == 0);
121 assert((uintptr_t)(aecm->channelStored) % 16 == 0);
122 assert((uintptr_t)(aecm->channelAdapt16) % 16 == 0);
123
124 // This is C code of following optimized code.
125 // During startup we store the channel every block.
126 // memcpy(aecm->channelStored,
127 // aecm->channelAdapt16,
128 // sizeof(int16_t) * PART_LEN1);
129 // Recalculate echo estimate
130 // for (i = 0; i < PART_LEN; i += 4) {
131 // echo_est[i] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i],
132 // far_spectrum[i]);
133 // echo_est[i + 1] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i + 1],
134 // far_spectrum[i + 1]);
135 // echo_est[i + 2] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i + 2],
136 // far_spectrum[i + 2]);
137 // echo_est[i + 3] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i + 3],
138 // far_spectrum[i + 3]);
139 // }
140 // echo_est[i] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i],
141 // far_spectrum[i]);
142 const uint16_t* far_spectrum_p = far_spectrum;
143 int16_t* start_adapt_p = aecm->channelAdapt16;
144 int16_t* start_stored_p = aecm->channelStored;
145 const int16_t* end_stored_p = aecm->channelStored + PART_LEN;
146 int32_t* echo_est_p = echo_est;
147
148 uint16x8_t far_spectrum_v;
149 int16x8_t adapt_v;
150 uint32x4_t echo_est_v_low, echo_est_v_high;
151
152 while (start_stored_p < end_stored_p) {
153 far_spectrum_v = vld1q_u16(far_spectrum_p);
154 adapt_v = vld1q_s16(start_adapt_p);
155
156 vst1q_s16(start_stored_p, adapt_v);
157
158 echo_est_v_low = vmull_u16(vget_low_u16(far_spectrum_v),
159 vget_low_u16(vreinterpretq_u16_s16(adapt_v)));
160 echo_est_v_high = vmull_u16(vget_high_u16(far_spectrum_v),
161 vget_high_u16(vreinterpretq_u16_s16(adapt_v)));
162
163 vst1q_s32(echo_est_p, vreinterpretq_s32_u32(echo_est_v_low));
164 vst1q_s32(echo_est_p + 4, vreinterpretq_s32_u32(echo_est_v_high));
165
166 far_spectrum_p += 8;
167 start_adapt_p += 8;
168 start_stored_p += 8;
169 echo_est_p += 8;
170 }
171 aecm->channelStored[PART_LEN] = aecm->channelAdapt16[PART_LEN];
172 echo_est[PART_LEN] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[PART_LEN],
173 far_spectrum[PART_LEN]);
174 }
175
WebRtcAecm_ResetAdaptiveChannelNeon(AecmCore * aecm)176 void WebRtcAecm_ResetAdaptiveChannelNeon(AecmCore* aecm) {
177 assert((uintptr_t)(aecm->channelStored) % 16 == 0);
178 assert((uintptr_t)(aecm->channelAdapt16) % 16 == 0);
179 assert((uintptr_t)(aecm->channelAdapt32) % 32 == 0);
180
181 // The C code of following optimized code.
182 // for (i = 0; i < PART_LEN1; i++) {
183 // aecm->channelAdapt16[i] = aecm->channelStored[i];
184 // aecm->channelAdapt32[i] = WEBRTC_SPL_LSHIFT_W32(
185 // (int32_t)aecm->channelStored[i], 16);
186 // }
187
188 int16_t* start_stored_p = aecm->channelStored;
189 int16_t* start_adapt16_p = aecm->channelAdapt16;
190 int32_t* start_adapt32_p = aecm->channelAdapt32;
191 const int16_t* end_stored_p = start_stored_p + PART_LEN;
192
193 int16x8_t stored_v;
194 int32x4_t adapt32_v_low, adapt32_v_high;
195
196 while (start_stored_p < end_stored_p) {
197 stored_v = vld1q_s16(start_stored_p);
198 vst1q_s16(start_adapt16_p, stored_v);
199
200 adapt32_v_low = vshll_n_s16(vget_low_s16(stored_v), 16);
201 adapt32_v_high = vshll_n_s16(vget_high_s16(stored_v), 16);
202
203 vst1q_s32(start_adapt32_p, adapt32_v_low);
204 vst1q_s32(start_adapt32_p + 4, adapt32_v_high);
205
206 start_stored_p += 8;
207 start_adapt16_p += 8;
208 start_adapt32_p += 8;
209 }
210 aecm->channelAdapt16[PART_LEN] = aecm->channelStored[PART_LEN];
211 aecm->channelAdapt32[PART_LEN] = (int32_t)aecm->channelStored[PART_LEN] << 16;
212 }
213