1 /*
2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "modules/audio_processing/aec3/block_processor.h"
11
12 #include <stddef.h>
13
14 #include <memory>
15 #include <utility>
16 #include <vector>
17
18 #include "absl/types/optional.h"
19 #include "api/audio/echo_canceller3_config.h"
20 #include "api/audio/echo_control.h"
21 #include "modules/audio_processing/aec3/aec3_common.h"
22 #include "modules/audio_processing/aec3/block_processor_metrics.h"
23 #include "modules/audio_processing/aec3/delay_estimate.h"
24 #include "modules/audio_processing/aec3/echo_path_variability.h"
25 #include "modules/audio_processing/aec3/echo_remover.h"
26 #include "modules/audio_processing/aec3/render_delay_buffer.h"
27 #include "modules/audio_processing/aec3/render_delay_controller.h"
28 #include "modules/audio_processing/logging/apm_data_dumper.h"
29 #include "rtc_base/atomic_ops.h"
30 #include "rtc_base/checks.h"
31 #include "rtc_base/logging.h"
32
33 namespace webrtc {
34 namespace {
35
36 enum class BlockProcessorApiCall { kCapture, kRender };
37
38 class BlockProcessorImpl final : public BlockProcessor {
39 public:
40 BlockProcessorImpl(const EchoCanceller3Config& config,
41 int sample_rate_hz,
42 size_t num_render_channels,
43 size_t num_capture_channels,
44 std::unique_ptr<RenderDelayBuffer> render_buffer,
45 std::unique_ptr<RenderDelayController> delay_controller,
46 std::unique_ptr<EchoRemover> echo_remover);
47
48 BlockProcessorImpl() = delete;
49
50 ~BlockProcessorImpl() override;
51
52 void ProcessCapture(
53 bool echo_path_gain_change,
54 bool capture_signal_saturation,
55 std::vector<std::vector<std::vector<float>>>* linear_output,
56 std::vector<std::vector<std::vector<float>>>* capture_block) override;
57
58 void BufferRender(
59 const std::vector<std::vector<std::vector<float>>>& block) override;
60
61 void UpdateEchoLeakageStatus(bool leakage_detected) override;
62
63 void GetMetrics(EchoControl::Metrics* metrics) const override;
64
65 void SetAudioBufferDelay(int delay_ms) override;
66
67 private:
68 static int instance_count_;
69 std::unique_ptr<ApmDataDumper> data_dumper_;
70 const EchoCanceller3Config config_;
71 bool capture_properly_started_ = false;
72 bool render_properly_started_ = false;
73 const size_t sample_rate_hz_;
74 std::unique_ptr<RenderDelayBuffer> render_buffer_;
75 std::unique_ptr<RenderDelayController> delay_controller_;
76 std::unique_ptr<EchoRemover> echo_remover_;
77 BlockProcessorMetrics metrics_;
78 RenderDelayBuffer::BufferingEvent render_event_;
79 size_t capture_call_counter_ = 0;
80 absl::optional<DelayEstimate> estimated_delay_;
81 };
82
83 int BlockProcessorImpl::instance_count_ = 0;
84
BlockProcessorImpl(const EchoCanceller3Config & config,int sample_rate_hz,size_t num_render_channels,size_t num_capture_channels,std::unique_ptr<RenderDelayBuffer> render_buffer,std::unique_ptr<RenderDelayController> delay_controller,std::unique_ptr<EchoRemover> echo_remover)85 BlockProcessorImpl::BlockProcessorImpl(
86 const EchoCanceller3Config& config,
87 int sample_rate_hz,
88 size_t num_render_channels,
89 size_t num_capture_channels,
90 std::unique_ptr<RenderDelayBuffer> render_buffer,
91 std::unique_ptr<RenderDelayController> delay_controller,
92 std::unique_ptr<EchoRemover> echo_remover)
93 : data_dumper_(
94 new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
95 config_(config),
96 sample_rate_hz_(sample_rate_hz),
97 render_buffer_(std::move(render_buffer)),
98 delay_controller_(std::move(delay_controller)),
99 echo_remover_(std::move(echo_remover)),
100 render_event_(RenderDelayBuffer::BufferingEvent::kNone) {
101 RTC_DCHECK(ValidFullBandRate(sample_rate_hz_));
102 }
103
104 BlockProcessorImpl::~BlockProcessorImpl() = default;
105
ProcessCapture(bool echo_path_gain_change,bool capture_signal_saturation,std::vector<std::vector<std::vector<float>>> * linear_output,std::vector<std::vector<std::vector<float>>> * capture_block)106 void BlockProcessorImpl::ProcessCapture(
107 bool echo_path_gain_change,
108 bool capture_signal_saturation,
109 std::vector<std::vector<std::vector<float>>>* linear_output,
110 std::vector<std::vector<std::vector<float>>>* capture_block) {
111 RTC_DCHECK(capture_block);
112 RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), capture_block->size());
113 RTC_DCHECK_EQ(kBlockSize, (*capture_block)[0][0].size());
114
115 capture_call_counter_++;
116
117 data_dumper_->DumpRaw("aec3_processblock_call_order",
118 static_cast<int>(BlockProcessorApiCall::kCapture));
119 data_dumper_->DumpWav("aec3_processblock_capture_input", kBlockSize,
120 &(*capture_block)[0][0][0], 16000, 1);
121
122 if (render_properly_started_) {
123 if (!capture_properly_started_) {
124 capture_properly_started_ = true;
125 render_buffer_->Reset();
126 if (delay_controller_)
127 delay_controller_->Reset(true);
128 }
129 } else {
130 // If no render data has yet arrived, do not process the capture signal.
131 return;
132 }
133
134 EchoPathVariability echo_path_variability(
135 echo_path_gain_change, EchoPathVariability::DelayAdjustment::kNone,
136 false);
137
138 if (render_event_ == RenderDelayBuffer::BufferingEvent::kRenderOverrun &&
139 render_properly_started_) {
140 echo_path_variability.delay_change =
141 EchoPathVariability::DelayAdjustment::kBufferFlush;
142 if (delay_controller_)
143 delay_controller_->Reset(true);
144 RTC_LOG(LS_WARNING) << "Reset due to render buffer overrun at block "
145 << capture_call_counter_;
146 }
147 render_event_ = RenderDelayBuffer::BufferingEvent::kNone;
148
149 // Update the render buffers with any newly arrived render blocks and prepare
150 // the render buffers for reading the render data corresponding to the current
151 // capture block.
152 RenderDelayBuffer::BufferingEvent buffer_event =
153 render_buffer_->PrepareCaptureProcessing();
154 // Reset the delay controller at render buffer underrun.
155 if (buffer_event == RenderDelayBuffer::BufferingEvent::kRenderUnderrun) {
156 if (delay_controller_)
157 delay_controller_->Reset(false);
158 }
159
160 data_dumper_->DumpWav("aec3_processblock_capture_input2", kBlockSize,
161 &(*capture_block)[0][0][0], 16000, 1);
162
163 bool has_delay_estimator = !config_.delay.use_external_delay_estimator;
164 if (has_delay_estimator) {
165 RTC_DCHECK(delay_controller_);
166 // Compute and apply the render delay required to achieve proper signal
167 // alignment.
168 estimated_delay_ = delay_controller_->GetDelay(
169 render_buffer_->GetDownsampledRenderBuffer(), render_buffer_->Delay(),
170 (*capture_block)[0]);
171
172 if (estimated_delay_) {
173 bool delay_change =
174 render_buffer_->AlignFromDelay(estimated_delay_->delay);
175 if (delay_change) {
176 rtc::LoggingSeverity log_level =
177 config_.delay.log_warning_on_delay_changes ? rtc::LS_WARNING
178 : rtc::LS_INFO;
179 RTC_LOG_V(log_level) << "Delay changed to " << estimated_delay_->delay
180 << " at block " << capture_call_counter_;
181 echo_path_variability.delay_change =
182 EchoPathVariability::DelayAdjustment::kNewDetectedDelay;
183 }
184 }
185
186 echo_path_variability.clock_drift = delay_controller_->HasClockdrift();
187
188 } else {
189 render_buffer_->AlignFromExternalDelay();
190 }
191
192 // Remove the echo from the capture signal.
193 if (has_delay_estimator || render_buffer_->HasReceivedBufferDelay()) {
194 echo_remover_->ProcessCapture(
195 echo_path_variability, capture_signal_saturation, estimated_delay_,
196 render_buffer_->GetRenderBuffer(), linear_output, capture_block);
197 }
198
199 // Update the metrics.
200 metrics_.UpdateCapture(false);
201 }
202
BufferRender(const std::vector<std::vector<std::vector<float>>> & block)203 void BlockProcessorImpl::BufferRender(
204 const std::vector<std::vector<std::vector<float>>>& block) {
205 RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), block.size());
206 RTC_DCHECK_EQ(kBlockSize, block[0][0].size());
207 data_dumper_->DumpRaw("aec3_processblock_call_order",
208 static_cast<int>(BlockProcessorApiCall::kRender));
209 data_dumper_->DumpWav("aec3_processblock_render_input", kBlockSize,
210 &block[0][0][0], 16000, 1);
211 data_dumper_->DumpWav("aec3_processblock_render_input2", kBlockSize,
212 &block[0][0][0], 16000, 1);
213
214 render_event_ = render_buffer_->Insert(block);
215
216 metrics_.UpdateRender(render_event_ !=
217 RenderDelayBuffer::BufferingEvent::kNone);
218
219 render_properly_started_ = true;
220 if (delay_controller_)
221 delay_controller_->LogRenderCall();
222 }
223
UpdateEchoLeakageStatus(bool leakage_detected)224 void BlockProcessorImpl::UpdateEchoLeakageStatus(bool leakage_detected) {
225 echo_remover_->UpdateEchoLeakageStatus(leakage_detected);
226 }
227
GetMetrics(EchoControl::Metrics * metrics) const228 void BlockProcessorImpl::GetMetrics(EchoControl::Metrics* metrics) const {
229 echo_remover_->GetMetrics(metrics);
230 constexpr int block_size_ms = 4;
231 absl::optional<size_t> delay = render_buffer_->Delay();
232 metrics->delay_ms = delay ? static_cast<int>(*delay) * block_size_ms : 0;
233 }
234
SetAudioBufferDelay(int delay_ms)235 void BlockProcessorImpl::SetAudioBufferDelay(int delay_ms) {
236 render_buffer_->SetAudioBufferDelay(delay_ms);
237 }
238
239 } // namespace
240
Create(const EchoCanceller3Config & config,int sample_rate_hz,size_t num_render_channels,size_t num_capture_channels)241 BlockProcessor* BlockProcessor::Create(const EchoCanceller3Config& config,
242 int sample_rate_hz,
243 size_t num_render_channels,
244 size_t num_capture_channels) {
245 std::unique_ptr<RenderDelayBuffer> render_buffer(
246 RenderDelayBuffer::Create(config, sample_rate_hz, num_render_channels));
247 std::unique_ptr<RenderDelayController> delay_controller;
248 if (!config.delay.use_external_delay_estimator) {
249 delay_controller.reset(RenderDelayController::Create(config, sample_rate_hz,
250 num_capture_channels));
251 }
252 std::unique_ptr<EchoRemover> echo_remover(EchoRemover::Create(
253 config, sample_rate_hz, num_render_channels, num_capture_channels));
254 return Create(config, sample_rate_hz, num_render_channels,
255 num_capture_channels, std::move(render_buffer),
256 std::move(delay_controller), std::move(echo_remover));
257 }
258
Create(const EchoCanceller3Config & config,int sample_rate_hz,size_t num_render_channels,size_t num_capture_channels,std::unique_ptr<RenderDelayBuffer> render_buffer)259 BlockProcessor* BlockProcessor::Create(
260 const EchoCanceller3Config& config,
261 int sample_rate_hz,
262 size_t num_render_channels,
263 size_t num_capture_channels,
264 std::unique_ptr<RenderDelayBuffer> render_buffer) {
265 std::unique_ptr<RenderDelayController> delay_controller;
266 if (!config.delay.use_external_delay_estimator) {
267 delay_controller.reset(RenderDelayController::Create(config, sample_rate_hz,
268 num_capture_channels));
269 }
270 std::unique_ptr<EchoRemover> echo_remover(EchoRemover::Create(
271 config, sample_rate_hz, num_render_channels, num_capture_channels));
272 return Create(config, sample_rate_hz, num_render_channels,
273 num_capture_channels, std::move(render_buffer),
274 std::move(delay_controller), std::move(echo_remover));
275 }
276
Create(const EchoCanceller3Config & config,int sample_rate_hz,size_t num_render_channels,size_t num_capture_channels,std::unique_ptr<RenderDelayBuffer> render_buffer,std::unique_ptr<RenderDelayController> delay_controller,std::unique_ptr<EchoRemover> echo_remover)277 BlockProcessor* BlockProcessor::Create(
278 const EchoCanceller3Config& config,
279 int sample_rate_hz,
280 size_t num_render_channels,
281 size_t num_capture_channels,
282 std::unique_ptr<RenderDelayBuffer> render_buffer,
283 std::unique_ptr<RenderDelayController> delay_controller,
284 std::unique_ptr<EchoRemover> echo_remover) {
285 return new BlockProcessorImpl(config, sample_rate_hz, num_render_channels,
286 num_capture_channels, std::move(render_buffer),
287 std::move(delay_controller),
288 std::move(echo_remover));
289 }
290
291 } // namespace webrtc
292