1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef WEBRTC_MODULES_AUDIO_CODING_MAIN_ACM2_AUDIO_CODING_MODULE_IMPL_H_ 12 #define WEBRTC_MODULES_AUDIO_CODING_MAIN_ACM2_AUDIO_CODING_MODULE_IMPL_H_ 13 14 #include <vector> 15 16 #include "webrtc/common_types.h" 17 #include "webrtc/engine_configurations.h" 18 #include "webrtc/modules/audio_coding/main/acm2/acm_codec_database.h" 19 #include "webrtc/modules/audio_coding/main/acm2/acm_receiver.h" 20 #include "webrtc/modules/audio_coding/main/acm2/acm_resampler.h" 21 #include "webrtc/system_wrappers/interface/scoped_ptr.h" 22 #include "webrtc/system_wrappers/interface/thread_annotations.h" 23 24 namespace webrtc { 25 26 class CriticalSectionWrapper; 27 28 namespace acm2 { 29 30 class ACMDTMFDetection; 31 class ACMGenericCodec; 32 33 class AudioCodingModuleImpl : public AudioCodingModule { 34 public: 35 explicit AudioCodingModuleImpl(const AudioCodingModule::Config& config); 36 ~AudioCodingModuleImpl(); 37 38 // Change the unique identifier of this object. 39 virtual int32_t ChangeUniqueId(const int32_t id); 40 41 // Returns the number of milliseconds until the module want a worker thread 42 // to call Process. 43 int32_t TimeUntilNextProcess(); 44 45 // Process any pending tasks such as timeouts. 46 int32_t Process(); 47 48 ///////////////////////////////////////// 49 // Sender 50 // 51 52 // Initialize send codec. 53 int InitializeSender(); 54 55 // Reset send codec. 56 int ResetEncoder(); 57 58 // Can be called multiple times for Codec, CNG, RED. 59 int RegisterSendCodec(const CodecInst& send_codec); 60 61 // Register Secondary codec for dual-streaming. Dual-streaming is activated 62 // right after the secondary codec is registered. 63 int RegisterSecondarySendCodec(const CodecInst& send_codec); 64 65 // Unregister the secondary codec. Dual-streaming is deactivated right after 66 // deregistering secondary codec. 67 void UnregisterSecondarySendCodec(); 68 69 // Get the secondary codec. 70 int SecondarySendCodec(CodecInst* secondary_codec) const; 71 72 // Get current send codec. 73 int SendCodec(CodecInst* current_codec) const; 74 75 // Get current send frequency. 76 int SendFrequency() const; 77 78 // Get encode bit-rate. 79 // Adaptive rate codecs return their current encode target rate, while other 80 // codecs return there long-term average or their fixed rate. 81 int SendBitrate() const; 82 83 // Set available bandwidth, inform the encoder about the 84 // estimated bandwidth received from the remote party. 85 virtual int SetReceivedEstimatedBandwidth(int bw); 86 87 // Register a transport callback which will be 88 // called to deliver the encoded buffers. 89 int RegisterTransportCallback(AudioPacketizationCallback* transport); 90 91 // Add 10 ms of raw (PCM) audio data to the encoder. 92 int Add10MsData(const AudioFrame& audio_frame); 93 94 ///////////////////////////////////////// 95 // (RED) Redundant Coding 96 // 97 98 // Configure RED status i.e. on/off. 99 int SetREDStatus(bool enable_red); 100 101 // Get RED status. 102 bool REDStatus() const; 103 104 ///////////////////////////////////////// 105 // (FEC) Forward Error Correction (codec internal) 106 // 107 108 // Configure FEC status i.e. on/off. 109 int SetCodecFEC(bool enabled_codec_fec); 110 111 // Get FEC status. 112 bool CodecFEC() const; 113 114 // Set target packet loss rate 115 int SetPacketLossRate(int loss_rate); 116 117 ///////////////////////////////////////// 118 // (VAD) Voice Activity Detection 119 // and 120 // (CNG) Comfort Noise Generation 121 // 122 123 int SetVAD(bool enable_dtx = true, 124 bool enable_vad = false, 125 ACMVADMode mode = VADNormal); 126 127 int VAD(bool* dtx_enabled, bool* vad_enabled, ACMVADMode* mode) const; 128 129 int RegisterVADCallback(ACMVADCallback* vad_callback); 130 131 ///////////////////////////////////////// 132 // Receiver 133 // 134 135 // Initialize receiver, resets codec database etc. 136 int InitializeReceiver(); 137 138 // Reset the decoder state. 139 int ResetDecoder(); 140 141 // Get current receive frequency. 142 int ReceiveFrequency() const; 143 144 // Get current playout frequency. 145 int PlayoutFrequency() const; 146 147 // Register possible receive codecs, can be called multiple times, 148 // for codecs, CNG, DTMF, RED. 149 int RegisterReceiveCodec(const CodecInst& receive_codec); 150 151 // Get current received codec. 152 int ReceiveCodec(CodecInst* current_codec) const; 153 154 // Incoming packet from network parsed and ready for decode. 155 int IncomingPacket(const uint8_t* incoming_payload, 156 int payload_length, 157 const WebRtcRTPHeader& rtp_info); 158 159 // Incoming payloads, without rtp-info, the rtp-info will be created in ACM. 160 // One usage for this API is when pre-encoded files are pushed in ACM. 161 int IncomingPayload(const uint8_t* incoming_payload, 162 int payload_length, 163 uint8_t payload_type, 164 uint32_t timestamp); 165 166 // Minimum playout delay. 167 int SetMinimumPlayoutDelay(int time_ms); 168 169 // Maximum playout delay. 170 int SetMaximumPlayoutDelay(int time_ms); 171 172 // Smallest latency NetEq will maintain. 173 int LeastRequiredDelayMs() const; 174 175 // Impose an initial delay on playout. ACM plays silence until |delay_ms| 176 // audio is accumulated in NetEq buffer, then starts decoding payloads. 177 int SetInitialPlayoutDelay(int delay_ms); 178 179 // TODO(turajs): DTMF playout is always activated in NetEq these APIs should 180 // be removed, as well as all VoE related APIs and methods. 181 // 182 // Configure Dtmf playout status i.e on/off playout the incoming outband Dtmf 183 // tone. SetDtmfPlayoutStatus(bool enable)184 int SetDtmfPlayoutStatus(bool enable) { return 0; } 185 186 // Get Dtmf playout status. DtmfPlayoutStatus()187 bool DtmfPlayoutStatus() const { return true; } 188 189 // Estimate the Bandwidth based on the incoming stream, needed 190 // for one way audio where the RTCP send the BW estimate. 191 // This is also done in the RTP module . 192 int DecoderEstimatedBandwidth() const; 193 194 // Set playout mode voice, fax. 195 int SetPlayoutMode(AudioPlayoutMode mode); 196 197 // Get playout mode voice, fax. 198 AudioPlayoutMode PlayoutMode() const; 199 200 // Get playout timestamp. 201 int PlayoutTimestamp(uint32_t* timestamp); 202 203 // Get 10 milliseconds of raw audio data to play out, and 204 // automatic resample to the requested frequency if > 0. 205 int PlayoutData10Ms(int desired_freq_hz, AudioFrame* audio_frame); 206 207 ///////////////////////////////////////// 208 // Statistics 209 // 210 211 int NetworkStatistics(ACMNetworkStatistics* statistics); 212 213 void DestructEncoderInst(void* inst); 214 215 // GET RED payload for iSAC. The method id called when 'this' ACM is 216 // the default ACM. 217 int REDPayloadISAC(int isac_rate, 218 int isac_bw_estimate, 219 uint8_t* payload, 220 int16_t* length_bytes); 221 222 int ReplaceInternalDTXWithWebRtc(bool use_webrtc_dtx); 223 224 int IsInternalDTXReplacedWithWebRtc(bool* uses_webrtc_dtx); 225 226 int SetISACMaxRate(int max_bit_per_sec); 227 228 int SetISACMaxPayloadSize(int max_size_bytes); 229 230 int ConfigISACBandwidthEstimator(int frame_size_ms, 231 int rate_bit_per_sec, 232 bool enforce_frame_size = false); 233 234 int UnregisterReceiveCodec(uint8_t payload_type); 235 236 int EnableNack(size_t max_nack_list_size); 237 238 void DisableNack(); 239 240 std::vector<uint16_t> GetNackList(int round_trip_time_ms) const; 241 242 void GetDecodingCallStatistics(AudioDecodingCallStats* stats) const; 243 244 private: 245 int UnregisterReceiveCodecSafe(int payload_type); 246 247 ACMGenericCodec* CreateCodec(const CodecInst& codec); 248 249 int InitializeReceiverSafe() EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); 250 251 bool HaveValidEncoder(const char* caller_name) const; 252 253 // Set VAD/DTX status. This function does not acquire a lock, and it is 254 // created to be called only from inside a critical section. 255 int SetVADSafe(bool enable_dtx, bool enable_vad, ACMVADMode mode) 256 EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); 257 258 // Process buffered audio when dual-streaming is not enabled (When RED is 259 // enabled still this function is used.) 260 int ProcessSingleStream(); 261 262 // Process buffered audio when dual-streaming is enabled, i.e. secondary send 263 // codec is registered. 264 int ProcessDualStream(); 265 266 // Preprocessing of input audio, including resampling and down-mixing if 267 // required, before pushing audio into encoder's buffer. 268 // 269 // in_frame: input audio-frame 270 // ptr_out: pointer to output audio_frame. If no preprocessing is required 271 // |ptr_out| will be pointing to |in_frame|, otherwise pointing to 272 // |preprocess_frame_|. 273 // 274 // Return value: 275 // -1: if encountering an error. 276 // 0: otherwise. 277 int PreprocessToAddData(const AudioFrame& in_frame, 278 const AudioFrame** ptr_out) 279 EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); 280 281 // Change required states after starting to receive the codec corresponding 282 // to |index|. 283 int UpdateUponReceivingCodec(int index); 284 285 int EncodeFragmentation(int fragmentation_index, 286 int payload_type, 287 uint32_t current_timestamp, 288 ACMGenericCodec* encoder, 289 uint8_t* stream) 290 EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); 291 292 void ResetFragmentation(int vector_size) 293 EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); 294 295 // Get a pointer to AudioDecoder of the given codec. For some codecs, e.g. 296 // iSAC, encoding and decoding have to be performed on a shared 297 // codec-instance. By calling this method, we get the codec-instance that ACM 298 // owns, then pass that to NetEq. This way, we perform both encoding and 299 // decoding on the same codec-instance. Furthermore, ACM would have control 300 // over decoder functionality if required. If |codec| does not share an 301 // instance between encoder and decoder, the |*decoder| is set NULL. 302 // The field ACMCodecDB::CodecSettings.owns_decoder indicates that if a 303 // codec owns the decoder-instance. For such codecs |*decoder| should be a 304 // valid pointer, otherwise it will be NULL. 305 int GetAudioDecoder(const CodecInst& codec, int codec_id, 306 int mirror_id, AudioDecoder** decoder); 307 308 CriticalSectionWrapper* acm_crit_sect_; 309 int id_; // TODO(henrik.lundin) Make const. 310 uint32_t expected_codec_ts_ GUARDED_BY(acm_crit_sect_); 311 uint32_t expected_in_ts_ GUARDED_BY(acm_crit_sect_); 312 CodecInst send_codec_inst_ GUARDED_BY(acm_crit_sect_); 313 314 uint8_t cng_nb_pltype_ GUARDED_BY(acm_crit_sect_); 315 uint8_t cng_wb_pltype_ GUARDED_BY(acm_crit_sect_); 316 uint8_t cng_swb_pltype_ GUARDED_BY(acm_crit_sect_); 317 uint8_t cng_fb_pltype_ GUARDED_BY(acm_crit_sect_); 318 319 uint8_t red_pltype_ GUARDED_BY(acm_crit_sect_); 320 bool vad_enabled_ GUARDED_BY(acm_crit_sect_); 321 bool dtx_enabled_ GUARDED_BY(acm_crit_sect_); 322 ACMVADMode vad_mode_ GUARDED_BY(acm_crit_sect_); 323 ACMGenericCodec* codecs_[ACMCodecDB::kMaxNumCodecs]; 324 int mirror_codec_idx_[ACMCodecDB::kMaxNumCodecs]; 325 bool stereo_send_ GUARDED_BY(acm_crit_sect_); 326 int current_send_codec_idx_; 327 bool send_codec_registered_; 328 ACMResampler resampler_ GUARDED_BY(acm_crit_sect_); 329 AcmReceiver receiver_; 330 331 // RED. 332 bool is_first_red_ GUARDED_BY(acm_crit_sect_); 333 bool red_enabled_ GUARDED_BY(acm_crit_sect_); 334 335 // TODO(turajs): |red_buffer_| is allocated in constructor, why having them 336 // as pointers and not an array. If concerned about the memory, then make a 337 // set-up function to allocate them only when they are going to be used, i.e. 338 // RED or Dual-streaming is enabled. 339 uint8_t* red_buffer_ GUARDED_BY(acm_crit_sect_); 340 341 // TODO(turajs): we actually don't need |fragmentation_| as a member variable. 342 // It is sufficient to keep the length & payload type of previous payload in 343 // member variables. 344 RTPFragmentationHeader fragmentation_ GUARDED_BY(acm_crit_sect_); 345 uint32_t last_red_timestamp_ GUARDED_BY(acm_crit_sect_); 346 347 // Codec internal FEC 348 bool codec_fec_enabled_; 349 350 // This is to keep track of CN instances where we can send DTMFs. 351 uint8_t previous_pltype_ GUARDED_BY(acm_crit_sect_); 352 353 // Used when payloads are pushed into ACM without any RTP info 354 // One example is when pre-encoded bit-stream is pushed from 355 // a file. 356 // IMPORTANT: this variable is only used in IncomingPayload(), therefore, 357 // no lock acquired when interacting with this variable. If it is going to 358 // be used in other methods, locks need to be taken. 359 WebRtcRTPHeader* aux_rtp_header_; 360 361 bool receiver_initialized_ GUARDED_BY(acm_crit_sect_); 362 363 AudioFrame preprocess_frame_ GUARDED_BY(acm_crit_sect_); 364 CodecInst secondary_send_codec_inst_ GUARDED_BY(acm_crit_sect_); 365 scoped_ptr<ACMGenericCodec> secondary_encoder_ GUARDED_BY(acm_crit_sect_); 366 uint32_t codec_timestamp_ GUARDED_BY(acm_crit_sect_); 367 bool first_10ms_data_ GUARDED_BY(acm_crit_sect_); 368 369 CriticalSectionWrapper* callback_crit_sect_; 370 AudioPacketizationCallback* packetization_callback_ 371 GUARDED_BY(callback_crit_sect_); 372 ACMVADCallback* vad_callback_ GUARDED_BY(callback_crit_sect_); 373 }; 374 375 } // namespace acm2 376 377 } // namespace webrtc 378 379 #endif // WEBRTC_MODULES_AUDIO_CODING_MAIN_ACM2_AUDIO_CODING_MODULE_IMPL_H_ 380