1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/modules/audio_processing/audio_processing_impl.h"
12
13 #include <assert.h>
14
15 #include "webrtc/common_audio/include/audio_util.h"
16 #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
17 #include "webrtc/modules/audio_processing/audio_buffer.h"
18 #include "webrtc/modules/audio_processing/common.h"
19 #include "webrtc/modules/audio_processing/echo_cancellation_impl.h"
20 #include "webrtc/modules/audio_processing/echo_control_mobile_impl.h"
21 #include "webrtc/modules/audio_processing/gain_control_impl.h"
22 #include "webrtc/modules/audio_processing/high_pass_filter_impl.h"
23 #include "webrtc/modules/audio_processing/level_estimator_impl.h"
24 #include "webrtc/modules/audio_processing/noise_suppression_impl.h"
25 #include "webrtc/modules/audio_processing/processing_component.h"
26 #include "webrtc/modules/audio_processing/voice_detection_impl.h"
27 #include "webrtc/modules/interface/module_common_types.h"
28 #include "webrtc/system_wrappers/interface/compile_assert.h"
29 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
30 #include "webrtc/system_wrappers/interface/file_wrapper.h"
31 #include "webrtc/system_wrappers/interface/logging.h"
32
33 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
34 // Files generated at build-time by the protobuf compiler.
35 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD
36 #include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h"
37 #else
38 #include "webrtc/audio_processing/debug.pb.h"
39 #endif
40 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
41
42 #define RETURN_ON_ERR(expr) \
43 do { \
44 int err = expr; \
45 if (err != kNoError) { \
46 return err; \
47 } \
48 } while (0)
49
50 namespace webrtc {
51
52 // Throughout webrtc, it's assumed that success is represented by zero.
53 COMPILE_ASSERT(AudioProcessing::kNoError == 0, no_error_must_be_zero);
54
Create(int id)55 AudioProcessing* AudioProcessing::Create(int id) {
56 return Create();
57 }
58
Create()59 AudioProcessing* AudioProcessing::Create() {
60 Config config;
61 return Create(config);
62 }
63
Create(const Config & config)64 AudioProcessing* AudioProcessing::Create(const Config& config) {
65 AudioProcessingImpl* apm = new AudioProcessingImpl(config);
66 if (apm->Initialize() != kNoError) {
67 delete apm;
68 apm = NULL;
69 }
70
71 return apm;
72 }
73
AudioProcessingImpl(const Config & config)74 AudioProcessingImpl::AudioProcessingImpl(const Config& config)
75 : echo_cancellation_(NULL),
76 echo_control_mobile_(NULL),
77 gain_control_(NULL),
78 high_pass_filter_(NULL),
79 level_estimator_(NULL),
80 noise_suppression_(NULL),
81 voice_detection_(NULL),
82 crit_(CriticalSectionWrapper::CreateCriticalSection()),
83 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
84 debug_file_(FileWrapper::Create()),
85 event_msg_(new audioproc::Event()),
86 #endif
87 fwd_in_format_(kSampleRate16kHz, 1),
88 fwd_proc_format_(kSampleRate16kHz, 1),
89 fwd_out_format_(kSampleRate16kHz),
90 rev_in_format_(kSampleRate16kHz, 1),
91 rev_proc_format_(kSampleRate16kHz, 1),
92 split_rate_(kSampleRate16kHz),
93 stream_delay_ms_(0),
94 delay_offset_ms_(0),
95 was_stream_delay_set_(false),
96 output_will_be_muted_(false),
97 key_pressed_(false) {
98 echo_cancellation_ = new EchoCancellationImpl(this, crit_);
99 component_list_.push_back(echo_cancellation_);
100
101 echo_control_mobile_ = new EchoControlMobileImpl(this, crit_);
102 component_list_.push_back(echo_control_mobile_);
103
104 gain_control_ = new GainControlImpl(this, crit_);
105 component_list_.push_back(gain_control_);
106
107 high_pass_filter_ = new HighPassFilterImpl(this, crit_);
108 component_list_.push_back(high_pass_filter_);
109
110 level_estimator_ = new LevelEstimatorImpl(this, crit_);
111 component_list_.push_back(level_estimator_);
112
113 noise_suppression_ = new NoiseSuppressionImpl(this, crit_);
114 component_list_.push_back(noise_suppression_);
115
116 voice_detection_ = new VoiceDetectionImpl(this, crit_);
117 component_list_.push_back(voice_detection_);
118
119 SetExtraOptions(config);
120 }
121
~AudioProcessingImpl()122 AudioProcessingImpl::~AudioProcessingImpl() {
123 {
124 CriticalSectionScoped crit_scoped(crit_);
125 while (!component_list_.empty()) {
126 ProcessingComponent* component = component_list_.front();
127 component->Destroy();
128 delete component;
129 component_list_.pop_front();
130 }
131
132 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
133 if (debug_file_->Open()) {
134 debug_file_->CloseFile();
135 }
136 #endif
137 }
138 delete crit_;
139 crit_ = NULL;
140 }
141
Initialize()142 int AudioProcessingImpl::Initialize() {
143 CriticalSectionScoped crit_scoped(crit_);
144 return InitializeLocked();
145 }
146
set_sample_rate_hz(int rate)147 int AudioProcessingImpl::set_sample_rate_hz(int rate) {
148 CriticalSectionScoped crit_scoped(crit_);
149 return InitializeLocked(rate,
150 rate,
151 rev_in_format_.rate(),
152 fwd_in_format_.num_channels(),
153 fwd_proc_format_.num_channels(),
154 rev_in_format_.num_channels());
155 }
156
Initialize(int input_sample_rate_hz,int output_sample_rate_hz,int reverse_sample_rate_hz,ChannelLayout input_layout,ChannelLayout output_layout,ChannelLayout reverse_layout)157 int AudioProcessingImpl::Initialize(int input_sample_rate_hz,
158 int output_sample_rate_hz,
159 int reverse_sample_rate_hz,
160 ChannelLayout input_layout,
161 ChannelLayout output_layout,
162 ChannelLayout reverse_layout) {
163 CriticalSectionScoped crit_scoped(crit_);
164 return InitializeLocked(input_sample_rate_hz,
165 output_sample_rate_hz,
166 reverse_sample_rate_hz,
167 ChannelsFromLayout(input_layout),
168 ChannelsFromLayout(output_layout),
169 ChannelsFromLayout(reverse_layout));
170 }
171
InitializeLocked()172 int AudioProcessingImpl::InitializeLocked() {
173 render_audio_.reset(new AudioBuffer(rev_in_format_.samples_per_channel(),
174 rev_in_format_.num_channels(),
175 rev_proc_format_.samples_per_channel(),
176 rev_proc_format_.num_channels(),
177 rev_proc_format_.samples_per_channel()));
178 capture_audio_.reset(new AudioBuffer(fwd_in_format_.samples_per_channel(),
179 fwd_in_format_.num_channels(),
180 fwd_proc_format_.samples_per_channel(),
181 fwd_proc_format_.num_channels(),
182 fwd_out_format_.samples_per_channel()));
183
184 // Initialize all components.
185 std::list<ProcessingComponent*>::iterator it;
186 for (it = component_list_.begin(); it != component_list_.end(); ++it) {
187 int err = (*it)->Initialize();
188 if (err != kNoError) {
189 return err;
190 }
191 }
192
193 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
194 if (debug_file_->Open()) {
195 int err = WriteInitMessage();
196 if (err != kNoError) {
197 return err;
198 }
199 }
200 #endif
201
202 return kNoError;
203 }
204
InitializeLocked(int input_sample_rate_hz,int output_sample_rate_hz,int reverse_sample_rate_hz,int num_input_channels,int num_output_channels,int num_reverse_channels)205 int AudioProcessingImpl::InitializeLocked(int input_sample_rate_hz,
206 int output_sample_rate_hz,
207 int reverse_sample_rate_hz,
208 int num_input_channels,
209 int num_output_channels,
210 int num_reverse_channels) {
211 if (input_sample_rate_hz <= 0 ||
212 output_sample_rate_hz <= 0 ||
213 reverse_sample_rate_hz <= 0) {
214 return kBadSampleRateError;
215 }
216 if (num_output_channels > num_input_channels) {
217 return kBadNumberChannelsError;
218 }
219 // Only mono and stereo supported currently.
220 if (num_input_channels > 2 || num_input_channels < 1 ||
221 num_output_channels > 2 || num_output_channels < 1 ||
222 num_reverse_channels > 2 || num_reverse_channels < 1) {
223 return kBadNumberChannelsError;
224 }
225
226 fwd_in_format_.set(input_sample_rate_hz, num_input_channels);
227 fwd_out_format_.set(output_sample_rate_hz);
228 rev_in_format_.set(reverse_sample_rate_hz, num_reverse_channels);
229
230 // We process at the closest native rate >= min(input rate, output rate)...
231 int min_proc_rate = std::min(fwd_in_format_.rate(), fwd_out_format_.rate());
232 int fwd_proc_rate;
233 if (min_proc_rate > kSampleRate16kHz) {
234 fwd_proc_rate = kSampleRate32kHz;
235 } else if (min_proc_rate > kSampleRate8kHz) {
236 fwd_proc_rate = kSampleRate16kHz;
237 } else {
238 fwd_proc_rate = kSampleRate8kHz;
239 }
240 // ...with one exception.
241 if (echo_control_mobile_->is_enabled() && min_proc_rate > kSampleRate16kHz) {
242 fwd_proc_rate = kSampleRate16kHz;
243 }
244
245 fwd_proc_format_.set(fwd_proc_rate, num_output_channels);
246
247 // We normally process the reverse stream at 16 kHz. Unless...
248 int rev_proc_rate = kSampleRate16kHz;
249 if (fwd_proc_format_.rate() == kSampleRate8kHz) {
250 // ...the forward stream is at 8 kHz.
251 rev_proc_rate = kSampleRate8kHz;
252 } else {
253 if (rev_in_format_.rate() == kSampleRate32kHz) {
254 // ...or the input is at 32 kHz, in which case we use the splitting
255 // filter rather than the resampler.
256 rev_proc_rate = kSampleRate32kHz;
257 }
258 }
259
260 // TODO(ajm): Enable this.
261 // Always downmix the reverse stream to mono for analysis.
262 //rev_proc_format_.set(rev_proc_rate, 1);
263 rev_proc_format_.set(rev_proc_rate, rev_in_format_.num_channels());
264
265 if (fwd_proc_format_.rate() == kSampleRate32kHz) {
266 split_rate_ = kSampleRate16kHz;
267 } else {
268 split_rate_ = fwd_proc_format_.rate();
269 }
270
271 return InitializeLocked();
272 }
273
274 // Calls InitializeLocked() if any of the audio parameters have changed from
275 // their current values.
MaybeInitializeLocked(int input_sample_rate_hz,int output_sample_rate_hz,int reverse_sample_rate_hz,int num_input_channels,int num_output_channels,int num_reverse_channels)276 int AudioProcessingImpl::MaybeInitializeLocked(int input_sample_rate_hz,
277 int output_sample_rate_hz,
278 int reverse_sample_rate_hz,
279 int num_input_channels,
280 int num_output_channels,
281 int num_reverse_channels) {
282 if (input_sample_rate_hz == fwd_in_format_.rate() &&
283 output_sample_rate_hz == fwd_out_format_.rate() &&
284 reverse_sample_rate_hz == rev_in_format_.rate() &&
285 num_input_channels == fwd_in_format_.num_channels() &&
286 num_output_channels == fwd_proc_format_.num_channels() &&
287 num_reverse_channels == rev_in_format_.num_channels()) {
288 return kNoError;
289 }
290
291 return InitializeLocked(input_sample_rate_hz,
292 output_sample_rate_hz,
293 reverse_sample_rate_hz,
294 num_input_channels,
295 num_output_channels,
296 num_reverse_channels);
297 }
298
SetExtraOptions(const Config & config)299 void AudioProcessingImpl::SetExtraOptions(const Config& config) {
300 CriticalSectionScoped crit_scoped(crit_);
301 std::list<ProcessingComponent*>::iterator it;
302 for (it = component_list_.begin(); it != component_list_.end(); ++it)
303 (*it)->SetExtraOptions(config);
304 }
305
EnableExperimentalNs(bool enable)306 int AudioProcessingImpl::EnableExperimentalNs(bool enable) {
307 return kNoError;
308 }
309
input_sample_rate_hz() const310 int AudioProcessingImpl::input_sample_rate_hz() const {
311 CriticalSectionScoped crit_scoped(crit_);
312 return fwd_in_format_.rate();
313 }
314
sample_rate_hz() const315 int AudioProcessingImpl::sample_rate_hz() const {
316 CriticalSectionScoped crit_scoped(crit_);
317 return fwd_in_format_.rate();
318 }
319
proc_sample_rate_hz() const320 int AudioProcessingImpl::proc_sample_rate_hz() const {
321 return fwd_proc_format_.rate();
322 }
323
proc_split_sample_rate_hz() const324 int AudioProcessingImpl::proc_split_sample_rate_hz() const {
325 return split_rate_;
326 }
327
num_reverse_channels() const328 int AudioProcessingImpl::num_reverse_channels() const {
329 return rev_proc_format_.num_channels();
330 }
331
num_input_channels() const332 int AudioProcessingImpl::num_input_channels() const {
333 return fwd_in_format_.num_channels();
334 }
335
num_output_channels() const336 int AudioProcessingImpl::num_output_channels() const {
337 return fwd_proc_format_.num_channels();
338 }
339
set_output_will_be_muted(bool muted)340 void AudioProcessingImpl::set_output_will_be_muted(bool muted) {
341 output_will_be_muted_ = muted;
342 }
343
output_will_be_muted() const344 bool AudioProcessingImpl::output_will_be_muted() const {
345 return output_will_be_muted_;
346 }
347
ProcessStream(const float * const * src,int samples_per_channel,int input_sample_rate_hz,ChannelLayout input_layout,int output_sample_rate_hz,ChannelLayout output_layout,float * const * dest)348 int AudioProcessingImpl::ProcessStream(const float* const* src,
349 int samples_per_channel,
350 int input_sample_rate_hz,
351 ChannelLayout input_layout,
352 int output_sample_rate_hz,
353 ChannelLayout output_layout,
354 float* const* dest) {
355 CriticalSectionScoped crit_scoped(crit_);
356 if (!src || !dest) {
357 return kNullPointerError;
358 }
359
360 RETURN_ON_ERR(MaybeInitializeLocked(input_sample_rate_hz,
361 output_sample_rate_hz,
362 rev_in_format_.rate(),
363 ChannelsFromLayout(input_layout),
364 ChannelsFromLayout(output_layout),
365 rev_in_format_.num_channels()));
366 if (samples_per_channel != fwd_in_format_.samples_per_channel()) {
367 return kBadDataLengthError;
368 }
369
370 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
371 if (debug_file_->Open()) {
372 event_msg_->set_type(audioproc::Event::STREAM);
373 audioproc::Stream* msg = event_msg_->mutable_stream();
374 const size_t channel_size = sizeof(float) * samples_per_channel;
375 for (int i = 0; i < fwd_in_format_.num_channels(); ++i)
376 msg->add_input_channel(src[i], channel_size);
377 }
378 #endif
379
380 capture_audio_->CopyFrom(src, samples_per_channel, input_layout);
381 RETURN_ON_ERR(ProcessStreamLocked());
382 if (output_copy_needed(is_data_processed())) {
383 capture_audio_->CopyTo(fwd_out_format_.samples_per_channel(),
384 output_layout,
385 dest);
386 }
387
388 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
389 if (debug_file_->Open()) {
390 audioproc::Stream* msg = event_msg_->mutable_stream();
391 const size_t channel_size = sizeof(float) * samples_per_channel;
392 for (int i = 0; i < fwd_proc_format_.num_channels(); ++i)
393 msg->add_output_channel(dest[i], channel_size);
394 RETURN_ON_ERR(WriteMessageToDebugFile());
395 }
396 #endif
397
398 return kNoError;
399 }
400
ProcessStream(AudioFrame * frame)401 int AudioProcessingImpl::ProcessStream(AudioFrame* frame) {
402 CriticalSectionScoped crit_scoped(crit_);
403 if (!frame) {
404 return kNullPointerError;
405 }
406 // Must be a native rate.
407 if (frame->sample_rate_hz_ != kSampleRate8kHz &&
408 frame->sample_rate_hz_ != kSampleRate16kHz &&
409 frame->sample_rate_hz_ != kSampleRate32kHz) {
410 return kBadSampleRateError;
411 }
412 if (echo_control_mobile_->is_enabled() &&
413 frame->sample_rate_hz_ > kSampleRate16kHz) {
414 LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates";
415 return kUnsupportedComponentError;
416 }
417
418 // TODO(ajm): The input and output rates and channels are currently
419 // constrained to be identical in the int16 interface.
420 RETURN_ON_ERR(MaybeInitializeLocked(frame->sample_rate_hz_,
421 frame->sample_rate_hz_,
422 rev_in_format_.rate(),
423 frame->num_channels_,
424 frame->num_channels_,
425 rev_in_format_.num_channels()));
426 if (frame->samples_per_channel_ != fwd_in_format_.samples_per_channel()) {
427 return kBadDataLengthError;
428 }
429
430 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
431 if (debug_file_->Open()) {
432 event_msg_->set_type(audioproc::Event::STREAM);
433 audioproc::Stream* msg = event_msg_->mutable_stream();
434 const size_t data_size = sizeof(int16_t) *
435 frame->samples_per_channel_ *
436 frame->num_channels_;
437 msg->set_input_data(frame->data_, data_size);
438 }
439 #endif
440
441 capture_audio_->DeinterleaveFrom(frame);
442 RETURN_ON_ERR(ProcessStreamLocked());
443 capture_audio_->InterleaveTo(frame, output_copy_needed(is_data_processed()));
444
445 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
446 if (debug_file_->Open()) {
447 audioproc::Stream* msg = event_msg_->mutable_stream();
448 const size_t data_size = sizeof(int16_t) *
449 frame->samples_per_channel_ *
450 frame->num_channels_;
451 msg->set_output_data(frame->data_, data_size);
452 RETURN_ON_ERR(WriteMessageToDebugFile());
453 }
454 #endif
455
456 return kNoError;
457 }
458
459
ProcessStreamLocked()460 int AudioProcessingImpl::ProcessStreamLocked() {
461 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
462 if (debug_file_->Open()) {
463 audioproc::Stream* msg = event_msg_->mutable_stream();
464 msg->set_delay(stream_delay_ms_);
465 msg->set_drift(echo_cancellation_->stream_drift_samples());
466 msg->set_level(gain_control_->stream_analog_level());
467 msg->set_keypress(key_pressed_);
468 }
469 #endif
470
471 AudioBuffer* ca = capture_audio_.get(); // For brevity.
472 bool data_processed = is_data_processed();
473 if (analysis_needed(data_processed)) {
474 for (int i = 0; i < fwd_proc_format_.num_channels(); i++) {
475 // Split into a low and high band.
476 WebRtcSpl_AnalysisQMF(ca->data(i),
477 ca->samples_per_channel(),
478 ca->low_pass_split_data(i),
479 ca->high_pass_split_data(i),
480 ca->filter_states(i)->analysis_filter_state1,
481 ca->filter_states(i)->analysis_filter_state2);
482 }
483 }
484
485 RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(ca));
486 RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(ca));
487 RETURN_ON_ERR(echo_cancellation_->ProcessCaptureAudio(ca));
488
489 if (echo_control_mobile_->is_enabled() && noise_suppression_->is_enabled()) {
490 ca->CopyLowPassToReference();
491 }
492 RETURN_ON_ERR(noise_suppression_->ProcessCaptureAudio(ca));
493 RETURN_ON_ERR(echo_control_mobile_->ProcessCaptureAudio(ca));
494 RETURN_ON_ERR(voice_detection_->ProcessCaptureAudio(ca));
495 RETURN_ON_ERR(gain_control_->ProcessCaptureAudio(ca));
496
497 if (synthesis_needed(data_processed)) {
498 for (int i = 0; i < fwd_proc_format_.num_channels(); i++) {
499 // Recombine low and high bands.
500 WebRtcSpl_SynthesisQMF(ca->low_pass_split_data(i),
501 ca->high_pass_split_data(i),
502 ca->samples_per_split_channel(),
503 ca->data(i),
504 ca->filter_states(i)->synthesis_filter_state1,
505 ca->filter_states(i)->synthesis_filter_state2);
506 }
507 }
508
509 // The level estimator operates on the recombined data.
510 RETURN_ON_ERR(level_estimator_->ProcessStream(ca));
511
512 was_stream_delay_set_ = false;
513 return kNoError;
514 }
515
AnalyzeReverseStream(const float * const * data,int samples_per_channel,int sample_rate_hz,ChannelLayout layout)516 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data,
517 int samples_per_channel,
518 int sample_rate_hz,
519 ChannelLayout layout) {
520 CriticalSectionScoped crit_scoped(crit_);
521 if (data == NULL) {
522 return kNullPointerError;
523 }
524
525 const int num_channels = ChannelsFromLayout(layout);
526 RETURN_ON_ERR(MaybeInitializeLocked(fwd_in_format_.rate(),
527 fwd_out_format_.rate(),
528 sample_rate_hz,
529 fwd_in_format_.num_channels(),
530 fwd_proc_format_.num_channels(),
531 num_channels));
532 if (samples_per_channel != rev_in_format_.samples_per_channel()) {
533 return kBadDataLengthError;
534 }
535
536 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
537 if (debug_file_->Open()) {
538 event_msg_->set_type(audioproc::Event::REVERSE_STREAM);
539 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream();
540 const size_t channel_size = sizeof(float) * samples_per_channel;
541 for (int i = 0; i < num_channels; ++i)
542 msg->add_channel(data[i], channel_size);
543 RETURN_ON_ERR(WriteMessageToDebugFile());
544 }
545 #endif
546
547 render_audio_->CopyFrom(data, samples_per_channel, layout);
548 return AnalyzeReverseStreamLocked();
549 }
550
AnalyzeReverseStream(AudioFrame * frame)551 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) {
552 CriticalSectionScoped crit_scoped(crit_);
553 if (frame == NULL) {
554 return kNullPointerError;
555 }
556 // Must be a native rate.
557 if (frame->sample_rate_hz_ != kSampleRate8kHz &&
558 frame->sample_rate_hz_ != kSampleRate16kHz &&
559 frame->sample_rate_hz_ != kSampleRate32kHz) {
560 return kBadSampleRateError;
561 }
562 // This interface does not tolerate different forward and reverse rates.
563 if (frame->sample_rate_hz_ != fwd_in_format_.rate()) {
564 return kBadSampleRateError;
565 }
566
567 RETURN_ON_ERR(MaybeInitializeLocked(fwd_in_format_.rate(),
568 fwd_out_format_.rate(),
569 frame->sample_rate_hz_,
570 fwd_in_format_.num_channels(),
571 fwd_in_format_.num_channels(),
572 frame->num_channels_));
573 if (frame->samples_per_channel_ != rev_in_format_.samples_per_channel()) {
574 return kBadDataLengthError;
575 }
576
577 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
578 if (debug_file_->Open()) {
579 event_msg_->set_type(audioproc::Event::REVERSE_STREAM);
580 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream();
581 const size_t data_size = sizeof(int16_t) *
582 frame->samples_per_channel_ *
583 frame->num_channels_;
584 msg->set_data(frame->data_, data_size);
585 RETURN_ON_ERR(WriteMessageToDebugFile());
586 }
587 #endif
588
589 render_audio_->DeinterleaveFrom(frame);
590 return AnalyzeReverseStreamLocked();
591 }
592
AnalyzeReverseStreamLocked()593 int AudioProcessingImpl::AnalyzeReverseStreamLocked() {
594 AudioBuffer* ra = render_audio_.get(); // For brevity.
595 if (rev_proc_format_.rate() == kSampleRate32kHz) {
596 for (int i = 0; i < rev_proc_format_.num_channels(); i++) {
597 // Split into low and high band.
598 WebRtcSpl_AnalysisQMF(ra->data(i),
599 ra->samples_per_channel(),
600 ra->low_pass_split_data(i),
601 ra->high_pass_split_data(i),
602 ra->filter_states(i)->analysis_filter_state1,
603 ra->filter_states(i)->analysis_filter_state2);
604 }
605 }
606
607 RETURN_ON_ERR(echo_cancellation_->ProcessRenderAudio(ra));
608 RETURN_ON_ERR(echo_control_mobile_->ProcessRenderAudio(ra));
609 RETURN_ON_ERR(gain_control_->ProcessRenderAudio(ra));
610
611 return kNoError;
612 }
613
set_stream_delay_ms(int delay)614 int AudioProcessingImpl::set_stream_delay_ms(int delay) {
615 Error retval = kNoError;
616 was_stream_delay_set_ = true;
617 delay += delay_offset_ms_;
618
619 if (delay < 0) {
620 delay = 0;
621 retval = kBadStreamParameterWarning;
622 }
623
624 // TODO(ajm): the max is rather arbitrarily chosen; investigate.
625 if (delay > 500) {
626 delay = 500;
627 retval = kBadStreamParameterWarning;
628 }
629
630 stream_delay_ms_ = delay;
631 return retval;
632 }
633
stream_delay_ms() const634 int AudioProcessingImpl::stream_delay_ms() const {
635 return stream_delay_ms_;
636 }
637
was_stream_delay_set() const638 bool AudioProcessingImpl::was_stream_delay_set() const {
639 return was_stream_delay_set_;
640 }
641
set_stream_key_pressed(bool key_pressed)642 void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) {
643 key_pressed_ = key_pressed;
644 }
645
stream_key_pressed() const646 bool AudioProcessingImpl::stream_key_pressed() const {
647 return key_pressed_;
648 }
649
set_delay_offset_ms(int offset)650 void AudioProcessingImpl::set_delay_offset_ms(int offset) {
651 CriticalSectionScoped crit_scoped(crit_);
652 delay_offset_ms_ = offset;
653 }
654
delay_offset_ms() const655 int AudioProcessingImpl::delay_offset_ms() const {
656 return delay_offset_ms_;
657 }
658
StartDebugRecording(const char filename[AudioProcessing::kMaxFilenameSize])659 int AudioProcessingImpl::StartDebugRecording(
660 const char filename[AudioProcessing::kMaxFilenameSize]) {
661 CriticalSectionScoped crit_scoped(crit_);
662 assert(kMaxFilenameSize == FileWrapper::kMaxFileNameSize);
663
664 if (filename == NULL) {
665 return kNullPointerError;
666 }
667
668 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
669 // Stop any ongoing recording.
670 if (debug_file_->Open()) {
671 if (debug_file_->CloseFile() == -1) {
672 return kFileError;
673 }
674 }
675
676 if (debug_file_->OpenFile(filename, false) == -1) {
677 debug_file_->CloseFile();
678 return kFileError;
679 }
680
681 int err = WriteInitMessage();
682 if (err != kNoError) {
683 return err;
684 }
685 return kNoError;
686 #else
687 return kUnsupportedFunctionError;
688 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
689 }
690
StartDebugRecording(FILE * handle)691 int AudioProcessingImpl::StartDebugRecording(FILE* handle) {
692 CriticalSectionScoped crit_scoped(crit_);
693
694 if (handle == NULL) {
695 return kNullPointerError;
696 }
697
698 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
699 // Stop any ongoing recording.
700 if (debug_file_->Open()) {
701 if (debug_file_->CloseFile() == -1) {
702 return kFileError;
703 }
704 }
705
706 if (debug_file_->OpenFromFileHandle(handle, true, false) == -1) {
707 return kFileError;
708 }
709
710 int err = WriteInitMessage();
711 if (err != kNoError) {
712 return err;
713 }
714 return kNoError;
715 #else
716 return kUnsupportedFunctionError;
717 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
718 }
719
StopDebugRecording()720 int AudioProcessingImpl::StopDebugRecording() {
721 CriticalSectionScoped crit_scoped(crit_);
722
723 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
724 // We just return if recording hasn't started.
725 if (debug_file_->Open()) {
726 if (debug_file_->CloseFile() == -1) {
727 return kFileError;
728 }
729 }
730 return kNoError;
731 #else
732 return kUnsupportedFunctionError;
733 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
734 }
735
echo_cancellation() const736 EchoCancellation* AudioProcessingImpl::echo_cancellation() const {
737 return echo_cancellation_;
738 }
739
echo_control_mobile() const740 EchoControlMobile* AudioProcessingImpl::echo_control_mobile() const {
741 return echo_control_mobile_;
742 }
743
gain_control() const744 GainControl* AudioProcessingImpl::gain_control() const {
745 return gain_control_;
746 }
747
high_pass_filter() const748 HighPassFilter* AudioProcessingImpl::high_pass_filter() const {
749 return high_pass_filter_;
750 }
751
level_estimator() const752 LevelEstimator* AudioProcessingImpl::level_estimator() const {
753 return level_estimator_;
754 }
755
noise_suppression() const756 NoiseSuppression* AudioProcessingImpl::noise_suppression() const {
757 return noise_suppression_;
758 }
759
voice_detection() const760 VoiceDetection* AudioProcessingImpl::voice_detection() const {
761 return voice_detection_;
762 }
763
is_data_processed() const764 bool AudioProcessingImpl::is_data_processed() const {
765 int enabled_count = 0;
766 std::list<ProcessingComponent*>::const_iterator it;
767 for (it = component_list_.begin(); it != component_list_.end(); it++) {
768 if ((*it)->is_component_enabled()) {
769 enabled_count++;
770 }
771 }
772
773 // Data is unchanged if no components are enabled, or if only level_estimator_
774 // or voice_detection_ is enabled.
775 if (enabled_count == 0) {
776 return false;
777 } else if (enabled_count == 1) {
778 if (level_estimator_->is_enabled() || voice_detection_->is_enabled()) {
779 return false;
780 }
781 } else if (enabled_count == 2) {
782 if (level_estimator_->is_enabled() && voice_detection_->is_enabled()) {
783 return false;
784 }
785 }
786 return true;
787 }
788
output_copy_needed(bool is_data_processed) const789 bool AudioProcessingImpl::output_copy_needed(bool is_data_processed) const {
790 // Check if we've upmixed or downmixed the audio.
791 return ((fwd_proc_format_.num_channels() != fwd_in_format_.num_channels()) ||
792 is_data_processed);
793 }
794
synthesis_needed(bool is_data_processed) const795 bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const {
796 return (is_data_processed && fwd_proc_format_.rate() == kSampleRate32kHz);
797 }
798
analysis_needed(bool is_data_processed) const799 bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const {
800 if (!is_data_processed && !voice_detection_->is_enabled()) {
801 // Only level_estimator_ is enabled.
802 return false;
803 } else if (fwd_proc_format_.rate() == kSampleRate32kHz) {
804 // Something besides level_estimator_ is enabled, and we have super-wb.
805 return true;
806 }
807 return false;
808 }
809
810 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
WriteMessageToDebugFile()811 int AudioProcessingImpl::WriteMessageToDebugFile() {
812 int32_t size = event_msg_->ByteSize();
813 if (size <= 0) {
814 return kUnspecifiedError;
815 }
816 #if defined(WEBRTC_ARCH_BIG_ENDIAN)
817 // TODO(ajm): Use little-endian "on the wire". For the moment, we can be
818 // pretty safe in assuming little-endian.
819 #endif
820
821 if (!event_msg_->SerializeToString(&event_str_)) {
822 return kUnspecifiedError;
823 }
824
825 // Write message preceded by its size.
826 if (!debug_file_->Write(&size, sizeof(int32_t))) {
827 return kFileError;
828 }
829 if (!debug_file_->Write(event_str_.data(), event_str_.length())) {
830 return kFileError;
831 }
832
833 event_msg_->Clear();
834
835 return kNoError;
836 }
837
WriteInitMessage()838 int AudioProcessingImpl::WriteInitMessage() {
839 event_msg_->set_type(audioproc::Event::INIT);
840 audioproc::Init* msg = event_msg_->mutable_init();
841 msg->set_sample_rate(fwd_in_format_.rate());
842 msg->set_num_input_channels(fwd_in_format_.num_channels());
843 msg->set_num_output_channels(fwd_proc_format_.num_channels());
844 msg->set_num_reverse_channels(rev_in_format_.num_channels());
845 msg->set_reverse_sample_rate(rev_in_format_.rate());
846 msg->set_output_sample_rate(fwd_out_format_.rate());
847
848 int err = WriteMessageToDebugFile();
849 if (err != kNoError) {
850 return err;
851 }
852
853 return kNoError;
854 }
855 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
856
857 } // namespace webrtc
858