1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_ 12 #define WEBRTC_MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_ 13 14 #include <list> 15 #include <string> 16 #include <vector> 17 18 #include "webrtc/base/criticalsection.h" 19 #include "webrtc/base/scoped_ptr.h" 20 #include "webrtc/base/thread_annotations.h" 21 #include "webrtc/modules/audio_processing/audio_buffer.h" 22 #include "webrtc/modules/audio_processing/include/audio_processing.h" 23 #include "webrtc/system_wrappers/include/file_wrapper.h" 24 25 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 26 // Files generated at build-time by the protobuf compiler. 27 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD 28 #include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h" 29 #else 30 #include "webrtc/audio_processing/debug.pb.h" 31 #endif 32 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP 33 34 namespace webrtc { 35 36 class AgcManagerDirect; 37 class AudioConverter; 38 39 template<typename T> 40 class Beamformer; 41 42 class AudioProcessingImpl : public AudioProcessing { 43 public: 44 // Methods forcing APM to run in a single-threaded manner. 45 // Acquires both the render and capture locks. 46 explicit AudioProcessingImpl(const Config& config); 47 // AudioProcessingImpl takes ownership of beamformer. 48 AudioProcessingImpl(const Config& config, Beamformer<float>* beamformer); 49 virtual ~AudioProcessingImpl(); 50 int Initialize() override; 51 int Initialize(int input_sample_rate_hz, 52 int output_sample_rate_hz, 53 int reverse_sample_rate_hz, 54 ChannelLayout input_layout, 55 ChannelLayout output_layout, 56 ChannelLayout reverse_layout) override; 57 int Initialize(const ProcessingConfig& processing_config) override; 58 void SetExtraOptions(const Config& config) override; 59 void UpdateHistogramsOnCallEnd() override; 60 int StartDebugRecording(const char filename[kMaxFilenameSize]) override; 61 int StartDebugRecording(FILE* handle) override; 62 int StartDebugRecordingForPlatformFile(rtc::PlatformFile handle) override; 63 int StopDebugRecording() override; 64 65 // Capture-side exclusive methods possibly running APM in a 66 // multi-threaded manner. Acquire the capture lock. 67 int ProcessStream(AudioFrame* frame) override; 68 int ProcessStream(const float* const* src, 69 size_t samples_per_channel, 70 int input_sample_rate_hz, 71 ChannelLayout input_layout, 72 int output_sample_rate_hz, 73 ChannelLayout output_layout, 74 float* const* dest) override; 75 int ProcessStream(const float* const* src, 76 const StreamConfig& input_config, 77 const StreamConfig& output_config, 78 float* const* dest) override; 79 void set_output_will_be_muted(bool muted) override; 80 int set_stream_delay_ms(int delay) override; 81 void set_delay_offset_ms(int offset) override; 82 int delay_offset_ms() const override; 83 void set_stream_key_pressed(bool key_pressed) override; 84 int input_sample_rate_hz() const override; 85 86 // Render-side exclusive methods possibly running APM in a 87 // multi-threaded manner. Acquire the render lock. 88 int AnalyzeReverseStream(AudioFrame* frame) override; 89 int ProcessReverseStream(AudioFrame* frame) override; 90 int AnalyzeReverseStream(const float* const* data, 91 size_t samples_per_channel, 92 int sample_rate_hz, 93 ChannelLayout layout) override; 94 int ProcessReverseStream(const float* const* src, 95 const StreamConfig& reverse_input_config, 96 const StreamConfig& reverse_output_config, 97 float* const* dest) override; 98 99 // Methods only accessed from APM submodules or 100 // from AudioProcessing tests in a single-threaded manner. 101 // Hence there is no need for locks in these. 102 int proc_sample_rate_hz() const override; 103 int proc_split_sample_rate_hz() const override; 104 size_t num_input_channels() const override; 105 size_t num_proc_channels() const override; 106 size_t num_output_channels() const override; 107 size_t num_reverse_channels() const override; 108 int stream_delay_ms() const override; 109 bool was_stream_delay_set() const override 110 EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 111 112 // Methods returning pointers to APM submodules. 113 // No locks are aquired in those, as those locks 114 // would offer no protection (the submodules are 115 // created only once in a single-treaded manner 116 // during APM creation). 117 EchoCancellation* echo_cancellation() const override; 118 EchoControlMobile* echo_control_mobile() const override; 119 GainControl* gain_control() const override; 120 HighPassFilter* high_pass_filter() const override; 121 LevelEstimator* level_estimator() const override; 122 NoiseSuppression* noise_suppression() const override; 123 VoiceDetection* voice_detection() const override; 124 125 protected: 126 // Overridden in a mock. 127 virtual int InitializeLocked() 128 EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); 129 130 private: 131 struct ApmPublicSubmodules; 132 struct ApmPrivateSubmodules; 133 134 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 135 // State for the debug dump. 136 struct ApmDebugDumpThreadState { ApmDebugDumpThreadStateApmDebugDumpThreadState137 ApmDebugDumpThreadState() : event_msg(new audioproc::Event()) {} 138 rtc::scoped_ptr<audioproc::Event> event_msg; // Protobuf message. 139 std::string event_str; // Memory for protobuf serialization. 140 141 // Serialized string of last saved APM configuration. 142 std::string last_serialized_config; 143 }; 144 145 struct ApmDebugDumpState { ApmDebugDumpStateApmDebugDumpState146 ApmDebugDumpState() : debug_file(FileWrapper::Create()) {} 147 rtc::scoped_ptr<FileWrapper> debug_file; 148 ApmDebugDumpThreadState render; 149 ApmDebugDumpThreadState capture; 150 }; 151 #endif 152 153 // Method for modifying the formats struct that are called from both 154 // the render and capture threads. The check for whether modifications 155 // are needed is done while holding the render lock only, thereby avoiding 156 // that the capture thread blocks the render thread. 157 // The struct is modified in a single-threaded manner by holding both the 158 // render and capture locks. 159 int MaybeInitialize(const ProcessingConfig& config) 160 EXCLUSIVE_LOCKS_REQUIRED(crit_render_); 161 162 int MaybeInitializeRender(const ProcessingConfig& processing_config) 163 EXCLUSIVE_LOCKS_REQUIRED(crit_render_); 164 165 int MaybeInitializeCapture(const ProcessingConfig& processing_config) 166 EXCLUSIVE_LOCKS_REQUIRED(crit_render_); 167 168 // Method for checking for the need of conversion. Accesses the formats 169 // structs in a read manner but the requirement for the render lock to be held 170 // was added as it currently anyway is always called in that manner. 171 bool rev_conversion_needed() const EXCLUSIVE_LOCKS_REQUIRED(crit_render_); 172 bool render_check_rev_conversion_needed() const 173 EXCLUSIVE_LOCKS_REQUIRED(crit_render_); 174 175 // Methods requiring APM running in a single-threaded manner. 176 // Are called with both the render and capture locks already 177 // acquired. 178 void InitializeExperimentalAgc() 179 EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); 180 void InitializeTransient() 181 EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); 182 void InitializeBeamformer() 183 EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); 184 void InitializeIntelligibility() 185 EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); 186 void InitializeHighPassFilter() 187 EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 188 void InitializeNoiseSuppression() 189 EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 190 void InitializeLevelEstimator() 191 EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 192 void InitializeVoiceDetection() 193 EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 194 int InitializeLocked(const ProcessingConfig& config) 195 EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); 196 197 // Capture-side exclusive methods possibly running APM in a multi-threaded 198 // manner that are called with the render lock already acquired. 199 int ProcessStreamLocked() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 200 bool output_copy_needed(bool is_data_processed) const 201 EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 202 bool is_data_processed() const EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 203 bool synthesis_needed(bool is_data_processed) const 204 EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 205 bool analysis_needed(bool is_data_processed) const 206 EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 207 void MaybeUpdateHistograms() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 208 209 // Render-side exclusive methods possibly running APM in a multi-threaded 210 // manner that are called with the render lock already acquired. 211 // TODO(ekm): Remove once all clients updated to new interface. 212 int AnalyzeReverseStreamLocked(const float* const* src, 213 const StreamConfig& input_config, 214 const StreamConfig& output_config) 215 EXCLUSIVE_LOCKS_REQUIRED(crit_render_); 216 bool is_rev_processed() const EXCLUSIVE_LOCKS_REQUIRED(crit_render_); 217 int ProcessReverseStreamLocked() EXCLUSIVE_LOCKS_REQUIRED(crit_render_); 218 219 // Debug dump methods that are internal and called without locks. 220 // TODO(peah): Make thread safe. 221 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 222 // TODO(andrew): make this more graceful. Ideally we would split this stuff 223 // out into a separate class with an "enabled" and "disabled" implementation. 224 static int WriteMessageToDebugFile(FileWrapper* debug_file, 225 rtc::CriticalSection* crit_debug, 226 ApmDebugDumpThreadState* debug_state); 227 int WriteInitMessage() EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); 228 229 // Writes Config message. If not |forced|, only writes the current config if 230 // it is different from the last saved one; if |forced|, writes the config 231 // regardless of the last saved. 232 int WriteConfigMessage(bool forced) EXCLUSIVE_LOCKS_REQUIRED(crit_capture_) 233 EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 234 235 // Critical section. 236 mutable rtc::CriticalSection crit_debug_; 237 238 // Debug dump state. 239 ApmDebugDumpState debug_dump_; 240 #endif 241 242 // Critical sections. 243 mutable rtc::CriticalSection crit_render_ ACQUIRED_BEFORE(crit_capture_); 244 mutable rtc::CriticalSection crit_capture_; 245 246 // Structs containing the pointers to the submodules. 247 rtc::scoped_ptr<ApmPublicSubmodules> public_submodules_; 248 rtc::scoped_ptr<ApmPrivateSubmodules> private_submodules_ 249 GUARDED_BY(crit_capture_); 250 251 // State that is written to while holding both the render and capture locks 252 // but can be read without any lock being held. 253 // As this is only accessed internally of APM, and all internal methods in APM 254 // either are holding the render or capture locks, this construct is safe as 255 // it is not possible to read the variables while writing them. 256 struct ApmFormatState { ApmFormatStateApmFormatState257 ApmFormatState() 258 : // Format of processing streams at input/output call sites. 259 api_format({{{kSampleRate16kHz, 1, false}, 260 {kSampleRate16kHz, 1, false}, 261 {kSampleRate16kHz, 1, false}, 262 {kSampleRate16kHz, 1, false}}}), 263 rev_proc_format(kSampleRate16kHz, 1) {} 264 ProcessingConfig api_format; 265 StreamConfig rev_proc_format; 266 } formats_; 267 268 // APM constants. 269 const struct ApmConstants { ApmConstantsApmConstants270 ApmConstants(int agc_startup_min_volume, 271 bool use_new_agc, 272 bool intelligibility_enabled) 273 : // Format of processing streams at input/output call sites. 274 agc_startup_min_volume(agc_startup_min_volume), 275 use_new_agc(use_new_agc), 276 intelligibility_enabled(intelligibility_enabled) {} 277 int agc_startup_min_volume; 278 bool use_new_agc; 279 bool intelligibility_enabled; 280 } constants_; 281 282 struct ApmCaptureState { ApmCaptureStateApmCaptureState283 ApmCaptureState(bool transient_suppressor_enabled, 284 const std::vector<Point>& array_geometry, 285 SphericalPointf target_direction) 286 : aec_system_delay_jumps(-1), 287 delay_offset_ms(0), 288 was_stream_delay_set(false), 289 last_stream_delay_ms(0), 290 last_aec_system_delay_ms(0), 291 stream_delay_jumps(-1), 292 output_will_be_muted(false), 293 key_pressed(false), 294 transient_suppressor_enabled(transient_suppressor_enabled), 295 array_geometry(array_geometry), 296 target_direction(target_direction), 297 fwd_proc_format(kSampleRate16kHz), 298 split_rate(kSampleRate16kHz) {} 299 int aec_system_delay_jumps; 300 int delay_offset_ms; 301 bool was_stream_delay_set; 302 int last_stream_delay_ms; 303 int last_aec_system_delay_ms; 304 int stream_delay_jumps; 305 bool output_will_be_muted; 306 bool key_pressed; 307 bool transient_suppressor_enabled; 308 std::vector<Point> array_geometry; 309 SphericalPointf target_direction; 310 rtc::scoped_ptr<AudioBuffer> capture_audio; 311 // Only the rate and samples fields of fwd_proc_format_ are used because the 312 // forward processing number of channels is mutable and is tracked by the 313 // capture_audio_. 314 StreamConfig fwd_proc_format; 315 int split_rate; 316 } capture_ GUARDED_BY(crit_capture_); 317 318 struct ApmCaptureNonLockedState { ApmCaptureNonLockedStateApmCaptureNonLockedState319 ApmCaptureNonLockedState(bool beamformer_enabled) 320 : fwd_proc_format(kSampleRate16kHz), 321 split_rate(kSampleRate16kHz), 322 stream_delay_ms(0), 323 beamformer_enabled(beamformer_enabled) {} 324 // Only the rate and samples fields of fwd_proc_format_ are used because the 325 // forward processing number of channels is mutable and is tracked by the 326 // capture_audio_. 327 StreamConfig fwd_proc_format; 328 int split_rate; 329 int stream_delay_ms; 330 bool beamformer_enabled; 331 } capture_nonlocked_; 332 333 struct ApmRenderState { 334 rtc::scoped_ptr<AudioConverter> render_converter; 335 rtc::scoped_ptr<AudioBuffer> render_audio; 336 } render_ GUARDED_BY(crit_render_); 337 }; 338 339 } // namespace webrtc 340 341 #endif // WEBRTC_MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_ 342