1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #ifndef MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
12 #define MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
13
14 #include <stdio.h>
15
16 #include <atomic>
17 #include <list>
18 #include <memory>
19 #include <string>
20 #include <vector>
21
22 #include "absl/strings/string_view.h"
23 #include "absl/types/optional.h"
24 #include "api/array_view.h"
25 #include "api/function_view.h"
26 #include "modules/audio_processing/aec3/echo_canceller3.h"
27 #include "modules/audio_processing/agc/agc_manager_direct.h"
28 #include "modules/audio_processing/agc/gain_control.h"
29 #include "modules/audio_processing/agc2/input_volume_stats_reporter.h"
30 #include "modules/audio_processing/audio_buffer.h"
31 #include "modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.h"
32 #include "modules/audio_processing/echo_control_mobile_impl.h"
33 #include "modules/audio_processing/gain_control_impl.h"
34 #include "modules/audio_processing/gain_controller2.h"
35 #include "modules/audio_processing/high_pass_filter.h"
36 #include "modules/audio_processing/include/aec_dump.h"
37 #include "modules/audio_processing/include/audio_frame_proxies.h"
38 #include "modules/audio_processing/include/audio_processing.h"
39 #include "modules/audio_processing/include/audio_processing_statistics.h"
40 #include "modules/audio_processing/ns/noise_suppressor.h"
41 #include "modules/audio_processing/optionally_built_submodule_creators.h"
42 #include "modules/audio_processing/render_queue_item_verifier.h"
43 #include "modules/audio_processing/rms_level.h"
44 #include "modules/audio_processing/transient/transient_suppressor.h"
45 #include "rtc_base/gtest_prod_util.h"
46 #include "rtc_base/ignore_wundef.h"
47 #include "rtc_base/swap_queue.h"
48 #include "rtc_base/synchronization/mutex.h"
49 #include "rtc_base/thread_annotations.h"
50
51 namespace webrtc {
52
53 class ApmDataDumper;
54 class AudioConverter;
55
RuntimeSettingQueueSize()56 constexpr int RuntimeSettingQueueSize() {
57 return 100;
58 }
59
60 class AudioProcessingImpl : public AudioProcessing {
61 public:
62 // Methods forcing APM to run in a single-threaded manner.
63 // Acquires both the render and capture locks.
64 AudioProcessingImpl();
65 AudioProcessingImpl(const AudioProcessing::Config& config,
66 std::unique_ptr<CustomProcessing> capture_post_processor,
67 std::unique_ptr<CustomProcessing> render_pre_processor,
68 std::unique_ptr<EchoControlFactory> echo_control_factory,
69 rtc::scoped_refptr<EchoDetector> echo_detector,
70 std::unique_ptr<CustomAudioAnalyzer> capture_analyzer);
71 ~AudioProcessingImpl() override;
72 int Initialize() override;
73 int Initialize(const ProcessingConfig& processing_config) override;
74 void ApplyConfig(const AudioProcessing::Config& config) override;
75 bool CreateAndAttachAecDump(absl::string_view file_name,
76 int64_t max_log_size_bytes,
77 rtc::TaskQueue* worker_queue) override;
78 bool CreateAndAttachAecDump(FILE* handle,
79 int64_t max_log_size_bytes,
80 rtc::TaskQueue* worker_queue) override;
81 // TODO(webrtc:5298) Deprecated variant.
82 void AttachAecDump(std::unique_ptr<AecDump> aec_dump) override;
83 void DetachAecDump() override;
84 void SetRuntimeSetting(RuntimeSetting setting) override;
85 bool PostRuntimeSetting(RuntimeSetting setting) override;
86
87 // Capture-side exclusive methods possibly running APM in a
88 // multi-threaded manner. Acquire the capture lock.
89 int ProcessStream(const int16_t* const src,
90 const StreamConfig& input_config,
91 const StreamConfig& output_config,
92 int16_t* const dest) override;
93 int ProcessStream(const float* const* src,
94 const StreamConfig& input_config,
95 const StreamConfig& output_config,
96 float* const* dest) override;
97 bool GetLinearAecOutput(
98 rtc::ArrayView<std::array<float, 160>> linear_output) const override;
99 void set_output_will_be_muted(bool muted) override;
100 void HandleCaptureOutputUsedSetting(bool capture_output_used)
101 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
102 int set_stream_delay_ms(int delay) override;
103 void set_stream_key_pressed(bool key_pressed) override;
104 void set_stream_analog_level(int level) override;
105 int recommended_stream_analog_level() const
106 RTC_LOCKS_EXCLUDED(mutex_capture_) override;
107
108 // Render-side exclusive methods possibly running APM in a
109 // multi-threaded manner. Acquire the render lock.
110 int ProcessReverseStream(const int16_t* const src,
111 const StreamConfig& input_config,
112 const StreamConfig& output_config,
113 int16_t* const dest) override;
114 int AnalyzeReverseStream(const float* const* data,
115 const StreamConfig& reverse_config) override;
116 int ProcessReverseStream(const float* const* src,
117 const StreamConfig& input_config,
118 const StreamConfig& output_config,
119 float* const* dest) override;
120
121 // Methods only accessed from APM submodules or
122 // from AudioProcessing tests in a single-threaded manner.
123 // Hence there is no need for locks in these.
124 int proc_sample_rate_hz() const override;
125 int proc_split_sample_rate_hz() const override;
126 size_t num_input_channels() const override;
127 size_t num_proc_channels() const override;
128 size_t num_output_channels() const override;
129 size_t num_reverse_channels() const override;
130 int stream_delay_ms() const override;
131
GetStatistics(bool has_remote_tracks)132 AudioProcessingStats GetStatistics(bool has_remote_tracks) override {
133 return GetStatistics();
134 }
GetStatistics()135 AudioProcessingStats GetStatistics() override {
136 return stats_reporter_.GetStatistics();
137 }
138
139 AudioProcessing::Config GetConfig() const override;
140
141 protected:
142 // Overridden in a mock.
143 virtual void InitializeLocked()
144 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_, mutex_capture_);
AssertLockedForTest()145 void AssertLockedForTest()
146 RTC_ASSERT_EXCLUSIVE_LOCK(mutex_render_, mutex_capture_) {
147 mutex_render_.AssertHeld();
148 mutex_capture_.AssertHeld();
149 }
150
151 private:
152 // TODO(peah): These friend classes should be removed as soon as the new
153 // parameter setting scheme allows.
154 FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, DefaultBehavior);
155 FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, ValidConfigBehavior);
156 FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, InValidConfigBehavior);
157 FRIEND_TEST_ALL_PREFIXES(ApmWithSubmodulesExcludedTest,
158 ToggleTransientSuppressor);
159 FRIEND_TEST_ALL_PREFIXES(ApmWithSubmodulesExcludedTest,
160 ReinitializeTransientSuppressor);
161 FRIEND_TEST_ALL_PREFIXES(ApmWithSubmodulesExcludedTest,
162 BitexactWithDisabledModules);
163 FRIEND_TEST_ALL_PREFIXES(
164 AudioProcessingImplInputVolumeControllerExperimentParametrizedTest,
165 ConfigAdjustedWhenExperimentEnabled);
166
167 void set_stream_analog_level_locked(int level)
168 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
169 void UpdateRecommendedInputVolumeLocked()
170 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
171
172 void OverrideSubmoduleCreationForTesting(
173 const ApmSubmoduleCreationOverrides& overrides);
174
175 // Class providing thread-safe message pipe functionality for
176 // `runtime_settings_`.
177 class RuntimeSettingEnqueuer {
178 public:
179 explicit RuntimeSettingEnqueuer(
180 SwapQueue<RuntimeSetting>* runtime_settings);
181 ~RuntimeSettingEnqueuer();
182
183 // Enqueue setting and return whether the setting was successfully enqueued.
184 bool Enqueue(RuntimeSetting setting);
185
186 private:
187 SwapQueue<RuntimeSetting>& runtime_settings_;
188 };
189
190 const std::unique_ptr<ApmDataDumper> data_dumper_;
191 static std::atomic<int> instance_count_;
192 const bool use_setup_specific_default_aec3_config_;
193
194 // TODO(bugs.webrtc.org/7494): Remove the the config when the field trial is
195 // removed. "WebRTC-Audio-InputVolumeControllerExperiment" field trial
196 // override for the input volume controller config.
197 const absl::optional<InputVolumeController::Config>
198 input_volume_controller_config_override_;
199
200 const bool use_denormal_disabler_;
201
202 const TransientSuppressor::VadMode transient_suppressor_vad_mode_;
203
204 SwapQueue<RuntimeSetting> capture_runtime_settings_;
205 SwapQueue<RuntimeSetting> render_runtime_settings_;
206
207 RuntimeSettingEnqueuer capture_runtime_settings_enqueuer_;
208 RuntimeSettingEnqueuer render_runtime_settings_enqueuer_;
209
210 // EchoControl factory.
211 const std::unique_ptr<EchoControlFactory> echo_control_factory_;
212
213 class SubmoduleStates {
214 public:
215 SubmoduleStates(bool capture_post_processor_enabled,
216 bool render_pre_processor_enabled,
217 bool capture_analyzer_enabled);
218 // Updates the submodule state and returns true if it has changed.
219 bool Update(bool high_pass_filter_enabled,
220 bool mobile_echo_controller_enabled,
221 bool noise_suppressor_enabled,
222 bool adaptive_gain_controller_enabled,
223 bool gain_controller2_enabled,
224 bool voice_activity_detector_enabled,
225 bool gain_adjustment_enabled,
226 bool echo_controller_enabled,
227 bool transient_suppressor_enabled);
228 bool CaptureMultiBandSubModulesActive() const;
229 bool CaptureMultiBandProcessingPresent() const;
230 bool CaptureMultiBandProcessingActive(bool ec_processing_active) const;
231 bool CaptureFullBandProcessingActive() const;
232 bool CaptureAnalyzerActive() const;
233 bool RenderMultiBandSubModulesActive() const;
234 bool RenderFullBandProcessingActive() const;
235 bool RenderMultiBandProcessingActive() const;
236 bool HighPassFilteringRequired() const;
237
238 private:
239 const bool capture_post_processor_enabled_ = false;
240 const bool render_pre_processor_enabled_ = false;
241 const bool capture_analyzer_enabled_ = false;
242 bool high_pass_filter_enabled_ = false;
243 bool mobile_echo_controller_enabled_ = false;
244 bool noise_suppressor_enabled_ = false;
245 bool adaptive_gain_controller_enabled_ = false;
246 bool voice_activity_detector_enabled_ = false;
247 bool gain_controller2_enabled_ = false;
248 bool gain_adjustment_enabled_ = false;
249 bool echo_controller_enabled_ = false;
250 bool transient_suppressor_enabled_ = false;
251 bool first_update_ = true;
252 };
253
254 // Methods for modifying the formats struct that is used by both
255 // the render and capture threads. The check for whether modifications are
256 // needed is done while holding a single lock only, thereby avoiding that the
257 // capture thread blocks the render thread.
258 // Called by render: Holds the render lock when reading the format struct and
259 // acquires both locks if reinitialization is required.
260 void MaybeInitializeRender(const StreamConfig& input_config,
261 const StreamConfig& output_config)
262 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_);
263 // Called by capture: Acquires and releases the capture lock to read the
264 // format struct and acquires both locks if reinitialization is needed.
265 void MaybeInitializeCapture(const StreamConfig& input_config,
266 const StreamConfig& output_config);
267
268 // Method for updating the state keeping track of the active submodules.
269 // Returns a bool indicating whether the state has changed.
270 bool UpdateActiveSubmoduleStates()
271 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
272
273 // Methods requiring APM running in a single-threaded manner, requiring both
274 // the render and capture lock to be acquired.
275 void InitializeLocked(const ProcessingConfig& config)
276 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_, mutex_capture_);
277 void InitializeResidualEchoDetector()
278 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_, mutex_capture_);
279 void InitializeEchoController()
280 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_, mutex_capture_);
281
282 // Initializations of capture-only sub-modules, requiring the capture lock
283 // already acquired.
284 void InitializeHighPassFilter(bool forced_reset)
285 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
286 void InitializeGainController1() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
287 void InitializeTransientSuppressor()
288 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
289 // Initializes the `GainController2` sub-module. If the sub-module is enabled
290 // and `config_has_changed` is true, recreates the sub-module.
291 void InitializeGainController2(bool config_has_changed)
292 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
293 // Initializes the `VoiceActivityDetectorWrapper` sub-module. If the
294 // sub-module is enabled and `config_has_changed` is true, recreates the
295 // sub-module.
296 void InitializeVoiceActivityDetector(bool config_has_changed)
297 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
298 void InitializeNoiseSuppressor() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
299 void InitializeCaptureLevelsAdjuster()
300 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
301 void InitializePostProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
302 void InitializeAnalyzer() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
303
304 // Initializations of render-only submodules, requiring the render lock
305 // already acquired.
306 void InitializePreProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_);
307
308 // Sample rate used for the fullband processing.
309 int proc_fullband_sample_rate_hz() const
310 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
311
312 // Empties and handles the respective RuntimeSetting queues.
313 void HandleCaptureRuntimeSettings()
314 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
315 void HandleRenderRuntimeSettings()
316 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_);
317
318 void EmptyQueuedRenderAudio() RTC_LOCKS_EXCLUDED(mutex_capture_);
319 void EmptyQueuedRenderAudioLocked()
320 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
321 void AllocateRenderQueue()
322 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_, mutex_capture_);
323 void QueueBandedRenderAudio(AudioBuffer* audio)
324 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_);
325 void QueueNonbandedRenderAudio(AudioBuffer* audio)
326 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_);
327
328 // Capture-side exclusive methods possibly running APM in a multi-threaded
329 // manner that are called with the render lock already acquired.
330 int ProcessCaptureStreamLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
331
332 // Render-side exclusive methods possibly running APM in a multi-threaded
333 // manner that are called with the render lock already acquired.
334 int AnalyzeReverseStreamLocked(const float* const* src,
335 const StreamConfig& input_config,
336 const StreamConfig& output_config)
337 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_);
338 int ProcessRenderStreamLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_);
339
340 // Collects configuration settings from public and private
341 // submodules to be saved as an audioproc::Config message on the
342 // AecDump if it is attached. If not `forced`, only writes the current
343 // config if it is different from the last saved one; if `forced`,
344 // writes the config regardless of the last saved.
345 void WriteAecDumpConfigMessage(bool forced)
346 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
347
348 // Notifies attached AecDump of current configuration and capture data.
349 void RecordUnprocessedCaptureStream(const float* const* capture_stream)
350 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
351
352 void RecordUnprocessedCaptureStream(const int16_t* const data,
353 const StreamConfig& config)
354 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
355
356 // Notifies attached AecDump of current configuration and
357 // processed capture data and issues a capture stream recording
358 // request.
359 void RecordProcessedCaptureStream(
360 const float* const* processed_capture_stream)
361 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
362
363 void RecordProcessedCaptureStream(const int16_t* const data,
364 const StreamConfig& config)
365 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
366
367 // Notifies attached AecDump about current state (delay, drift, etc).
368 void RecordAudioProcessingState()
369 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
370
371 // Ensures that overruns in the capture runtime settings queue is properly
372 // handled by the code, providing safe-fallbacks to mitigate the implications
373 // of any settings being missed.
374 void HandleOverrunInCaptureRuntimeSettingsQueue()
375 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
376
377 // AecDump instance used for optionally logging APM config, input
378 // and output to file in the AEC-dump format defined in debug.proto.
379 std::unique_ptr<AecDump> aec_dump_;
380
381 // Hold the last config written with AecDump for avoiding writing
382 // the same config twice.
383 InternalAPMConfig apm_config_for_aec_dump_ RTC_GUARDED_BY(mutex_capture_);
384
385 // Critical sections.
386 mutable Mutex mutex_render_ RTC_ACQUIRED_BEFORE(mutex_capture_);
387 mutable Mutex mutex_capture_;
388
389 // Struct containing the Config specifying the behavior of APM.
390 AudioProcessing::Config config_;
391
392 // Overrides for testing the exclusion of some submodules from the build.
393 ApmSubmoduleCreationOverrides submodule_creation_overrides_
394 RTC_GUARDED_BY(mutex_capture_);
395
396 // Class containing information about what submodules are active.
397 SubmoduleStates submodule_states_;
398
399 // Struct containing the pointers to the submodules.
400 struct Submodules {
SubmodulesSubmodules401 Submodules(std::unique_ptr<CustomProcessing> capture_post_processor,
402 std::unique_ptr<CustomProcessing> render_pre_processor,
403 rtc::scoped_refptr<EchoDetector> echo_detector,
404 std::unique_ptr<CustomAudioAnalyzer> capture_analyzer)
405 : echo_detector(std::move(echo_detector)),
406 capture_post_processor(std::move(capture_post_processor)),
407 render_pre_processor(std::move(render_pre_processor)),
408 capture_analyzer(std::move(capture_analyzer)) {}
409 // Accessed internally from capture or during initialization.
410 const rtc::scoped_refptr<EchoDetector> echo_detector;
411 const std::unique_ptr<CustomProcessing> capture_post_processor;
412 const std::unique_ptr<CustomProcessing> render_pre_processor;
413 const std::unique_ptr<CustomAudioAnalyzer> capture_analyzer;
414 std::unique_ptr<AgcManagerDirect> agc_manager;
415 std::unique_ptr<GainControlImpl> gain_control;
416 std::unique_ptr<GainController2> gain_controller2;
417 std::unique_ptr<VoiceActivityDetectorWrapper> voice_activity_detector;
418 std::unique_ptr<HighPassFilter> high_pass_filter;
419 std::unique_ptr<EchoControl> echo_controller;
420 std::unique_ptr<EchoControlMobileImpl> echo_control_mobile;
421 std::unique_ptr<NoiseSuppressor> noise_suppressor;
422 std::unique_ptr<TransientSuppressor> transient_suppressor;
423 std::unique_ptr<CaptureLevelsAdjuster> capture_levels_adjuster;
424 } submodules_;
425
426 // State that is written to while holding both the render and capture locks
427 // but can be read without any lock being held.
428 // As this is only accessed internally of APM, and all internal methods in APM
429 // either are holding the render or capture locks, this construct is safe as
430 // it is not possible to read the variables while writing them.
431 struct ApmFormatState {
ApmFormatStateApmFormatState432 ApmFormatState()
433 : // Format of processing streams at input/output call sites.
434 api_format({{{kSampleRate16kHz, 1},
435 {kSampleRate16kHz, 1},
436 {kSampleRate16kHz, 1},
437 {kSampleRate16kHz, 1}}}),
438 render_processing_format(kSampleRate16kHz, 1) {}
439 ProcessingConfig api_format;
440 StreamConfig render_processing_format;
441 } formats_;
442
443 // APM constants.
444 const struct ApmConstants {
ApmConstantsApmConstants445 ApmConstants(bool multi_channel_render_support,
446 bool multi_channel_capture_support,
447 bool enforce_split_band_hpf,
448 bool minimize_processing_for_unused_output,
449 bool transient_suppressor_forced_off)
450 : multi_channel_render_support(multi_channel_render_support),
451 multi_channel_capture_support(multi_channel_capture_support),
452 enforce_split_band_hpf(enforce_split_band_hpf),
453 minimize_processing_for_unused_output(
454 minimize_processing_for_unused_output),
455 transient_suppressor_forced_off(transient_suppressor_forced_off) {}
456 bool multi_channel_render_support;
457 bool multi_channel_capture_support;
458 bool enforce_split_band_hpf;
459 bool minimize_processing_for_unused_output;
460 bool transient_suppressor_forced_off;
461 } constants_;
462
463 struct ApmCaptureState {
464 ApmCaptureState();
465 ~ApmCaptureState();
466 bool was_stream_delay_set;
467 bool capture_output_used;
468 bool capture_output_used_last_frame;
469 bool key_pressed;
470 std::unique_ptr<AudioBuffer> capture_audio;
471 std::unique_ptr<AudioBuffer> capture_fullband_audio;
472 std::unique_ptr<AudioBuffer> linear_aec_output;
473 // Only the rate and samples fields of capture_processing_format_ are used
474 // because the capture processing number of channels is mutable and is
475 // tracked by the capture_audio_.
476 StreamConfig capture_processing_format;
477 int split_rate;
478 bool echo_path_gain_change;
479 float prev_pre_adjustment_gain;
480 int playout_volume;
481 int prev_playout_volume;
482 AudioProcessingStats stats;
483 // Input volume applied on the audio input device when the audio is
484 // acquired. Unspecified when unknown.
485 absl::optional<int> applied_input_volume;
486 bool applied_input_volume_changed;
487 // Recommended input volume to apply on the audio input device the next time
488 // that audio is acquired. Unspecified when no input volume can be
489 // recommended.
490 absl::optional<int> recommended_input_volume;
491 } capture_ RTC_GUARDED_BY(mutex_capture_);
492
493 struct ApmCaptureNonLockedState {
ApmCaptureNonLockedStateApmCaptureNonLockedState494 ApmCaptureNonLockedState()
495 : capture_processing_format(kSampleRate16kHz),
496 split_rate(kSampleRate16kHz),
497 stream_delay_ms(0) {}
498 // Only the rate and samples fields of capture_processing_format_ are used
499 // because the forward processing number of channels is mutable and is
500 // tracked by the capture_audio_.
501 StreamConfig capture_processing_format;
502 int split_rate;
503 int stream_delay_ms;
504 bool echo_controller_enabled = false;
505 } capture_nonlocked_;
506
507 struct ApmRenderState {
508 ApmRenderState();
509 ~ApmRenderState();
510 std::unique_ptr<AudioConverter> render_converter;
511 std::unique_ptr<AudioBuffer> render_audio;
512 } render_ RTC_GUARDED_BY(mutex_render_);
513
514 // Class for statistics reporting. The class is thread-safe and no lock is
515 // needed when accessing it.
516 class ApmStatsReporter {
517 public:
518 ApmStatsReporter();
519 ~ApmStatsReporter();
520
521 // Returns the most recently reported statistics.
522 AudioProcessingStats GetStatistics();
523
524 // Update the cached statistics.
525 void UpdateStatistics(const AudioProcessingStats& new_stats);
526
527 private:
528 Mutex mutex_stats_;
529 AudioProcessingStats cached_stats_ RTC_GUARDED_BY(mutex_stats_);
530 SwapQueue<AudioProcessingStats> stats_message_queue_;
531 } stats_reporter_;
532
533 std::vector<int16_t> aecm_render_queue_buffer_ RTC_GUARDED_BY(mutex_render_);
534 std::vector<int16_t> aecm_capture_queue_buffer_
535 RTC_GUARDED_BY(mutex_capture_);
536
537 size_t agc_render_queue_element_max_size_ RTC_GUARDED_BY(mutex_render_)
538 RTC_GUARDED_BY(mutex_capture_) = 0;
539 std::vector<int16_t> agc_render_queue_buffer_ RTC_GUARDED_BY(mutex_render_);
540 std::vector<int16_t> agc_capture_queue_buffer_ RTC_GUARDED_BY(mutex_capture_);
541
542 size_t red_render_queue_element_max_size_ RTC_GUARDED_BY(mutex_render_)
543 RTC_GUARDED_BY(mutex_capture_) = 0;
544 std::vector<float> red_render_queue_buffer_ RTC_GUARDED_BY(mutex_render_);
545 std::vector<float> red_capture_queue_buffer_ RTC_GUARDED_BY(mutex_capture_);
546
547 RmsLevel capture_input_rms_ RTC_GUARDED_BY(mutex_capture_);
548 RmsLevel capture_output_rms_ RTC_GUARDED_BY(mutex_capture_);
549 int capture_rms_interval_counter_ RTC_GUARDED_BY(mutex_capture_) = 0;
550
551 InputVolumeStatsReporter applied_input_volume_stats_reporter_
552 RTC_GUARDED_BY(mutex_capture_);
553 InputVolumeStatsReporter recommended_input_volume_stats_reporter_
554 RTC_GUARDED_BY(mutex_capture_);
555
556 // Lock protection not needed.
557 std::unique_ptr<
558 SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>>
559 aecm_render_signal_queue_;
560 std::unique_ptr<
561 SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>>
562 agc_render_signal_queue_;
563 std::unique_ptr<SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>>
564 red_render_signal_queue_;
565 };
566
567 } // namespace webrtc
568
569 #endif // MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
570