1 /* 2 * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MEDIA_ENGINE_WEBRTC_VOICE_ENGINE_H_ 12 #define MEDIA_ENGINE_WEBRTC_VOICE_ENGINE_H_ 13 14 #include <map> 15 #include <memory> 16 #include <string> 17 #include <vector> 18 19 #include "api/audio_codecs/audio_encoder_factory.h" 20 #include "api/field_trials_view.h" 21 #include "api/scoped_refptr.h" 22 #include "api/sequence_checker.h" 23 #include "api/task_queue/pending_task_safety_flag.h" 24 #include "api/task_queue/task_queue_factory.h" 25 #include "api/transport/rtp/rtp_source.h" 26 #include "call/audio_state.h" 27 #include "call/call.h" 28 #include "media/base/media_engine.h" 29 #include "media/base/rtp_utils.h" 30 #include "modules/async_audio_processing/async_audio_processing.h" 31 #include "rtc_base/buffer.h" 32 #include "rtc_base/network_route.h" 33 #include "rtc_base/task_queue.h" 34 35 namespace webrtc { 36 class AudioFrameProcessor; 37 } 38 39 namespace cricket { 40 41 class AudioSource; 42 class WebRtcVoiceMediaChannel; 43 44 // WebRtcVoiceEngine is a class to be used with CompositeMediaEngine. 45 // It uses the WebRtc VoiceEngine library for audio handling. 46 class WebRtcVoiceEngine final : public VoiceEngineInterface { 47 friend class WebRtcVoiceMediaChannel; 48 49 public: 50 WebRtcVoiceEngine( 51 webrtc::TaskQueueFactory* task_queue_factory, 52 webrtc::AudioDeviceModule* adm, 53 const rtc::scoped_refptr<webrtc::AudioEncoderFactory>& encoder_factory, 54 const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory, 55 rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer, 56 rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing, 57 webrtc::AudioFrameProcessor* audio_frame_processor, 58 const webrtc::FieldTrialsView& trials); 59 60 WebRtcVoiceEngine() = delete; 61 WebRtcVoiceEngine(const WebRtcVoiceEngine&) = delete; 62 WebRtcVoiceEngine& operator=(const WebRtcVoiceEngine&) = delete; 63 64 ~WebRtcVoiceEngine() override; 65 66 // Does initialization that needs to occur on the worker thread. 67 void Init() override; 68 69 rtc::scoped_refptr<webrtc::AudioState> GetAudioState() const override; 70 VoiceMediaChannel* CreateMediaChannel( 71 webrtc::Call* call, 72 const MediaConfig& config, 73 const AudioOptions& options, 74 const webrtc::CryptoOptions& crypto_options) override; 75 76 const std::vector<AudioCodec>& send_codecs() const override; 77 const std::vector<AudioCodec>& recv_codecs() const override; 78 std::vector<webrtc::RtpHeaderExtensionCapability> GetRtpHeaderExtensions() 79 const override; 80 81 // Starts AEC dump using an existing file. A maximum file size in bytes can be 82 // specified. When the maximum file size is reached, logging is stopped and 83 // the file is closed. If max_size_bytes is set to <= 0, no limit will be 84 // used. 85 bool StartAecDump(webrtc::FileWrapper file, int64_t max_size_bytes) override; 86 87 // Stops AEC dump. 88 void StopAecDump() override; 89 90 private: 91 // Every option that is "set" will be applied. Every option not "set" will be 92 // ignored. This allows us to selectively turn on and off different options 93 // easily at any time. 94 void ApplyOptions(const AudioOptions& options); 95 96 int CreateVoEChannel(); 97 98 webrtc::TaskQueueFactory* const task_queue_factory_; 99 std::unique_ptr<rtc::TaskQueue> low_priority_worker_queue_; 100 101 webrtc::AudioDeviceModule* adm(); 102 webrtc::AudioProcessing* apm() const; 103 webrtc::AudioState* audio_state(); 104 105 std::vector<AudioCodec> CollectCodecs( 106 const std::vector<webrtc::AudioCodecSpec>& specs) const; 107 108 webrtc::SequenceChecker signal_thread_checker_; 109 webrtc::SequenceChecker worker_thread_checker_; 110 111 // The audio device module. 112 rtc::scoped_refptr<webrtc::AudioDeviceModule> adm_; 113 rtc::scoped_refptr<webrtc::AudioEncoderFactory> encoder_factory_; 114 rtc::scoped_refptr<webrtc::AudioDecoderFactory> decoder_factory_; 115 rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer_; 116 // The audio processing module. 117 rtc::scoped_refptr<webrtc::AudioProcessing> apm_; 118 // Asynchronous audio processing. 119 webrtc::AudioFrameProcessor* const audio_frame_processor_; 120 // The primary instance of WebRtc VoiceEngine. 121 rtc::scoped_refptr<webrtc::AudioState> audio_state_; 122 std::vector<AudioCodec> send_codecs_; 123 std::vector<AudioCodec> recv_codecs_; 124 bool is_dumping_aec_ = false; 125 bool initialized_ = false; 126 127 // Jitter buffer settings for new streams. 128 size_t audio_jitter_buffer_max_packets_ = 200; 129 bool audio_jitter_buffer_fast_accelerate_ = false; 130 int audio_jitter_buffer_min_delay_ms_ = 0; 131 132 const bool minimized_remsampling_on_mobile_trial_enabled_; 133 }; 134 135 // WebRtcVoiceMediaChannel is an implementation of VoiceMediaChannel that uses 136 // WebRtc Voice Engine. 137 class WebRtcVoiceMediaChannel final : public VoiceMediaChannel, 138 public webrtc::Transport { 139 public: 140 WebRtcVoiceMediaChannel(WebRtcVoiceEngine* engine, 141 const MediaConfig& config, 142 const AudioOptions& options, 143 const webrtc::CryptoOptions& crypto_options, 144 webrtc::Call* call); 145 146 WebRtcVoiceMediaChannel() = delete; 147 WebRtcVoiceMediaChannel(const WebRtcVoiceMediaChannel&) = delete; 148 WebRtcVoiceMediaChannel& operator=(const WebRtcVoiceMediaChannel&) = delete; 149 150 ~WebRtcVoiceMediaChannel() override; 151 options()152 const AudioOptions& options() const { return options_; } 153 154 bool SetSendParameters(const AudioSendParameters& params) override; 155 bool SetRecvParameters(const AudioRecvParameters& params) override; 156 webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const override; 157 webrtc::RTCError SetRtpSendParameters( 158 uint32_t ssrc, 159 const webrtc::RtpParameters& parameters, 160 webrtc::SetParametersCallback callback) override; 161 webrtc::RtpParameters GetRtpReceiveParameters(uint32_t ssrc) const override; 162 webrtc::RtpParameters GetDefaultRtpReceiveParameters() const override; 163 164 void SetPlayout(bool playout) override; 165 void SetSend(bool send) override; 166 bool SetAudioSend(uint32_t ssrc, 167 bool enable, 168 const AudioOptions* options, 169 AudioSource* source) override; 170 bool AddSendStream(const StreamParams& sp) override; 171 bool RemoveSendStream(uint32_t ssrc) override; 172 bool AddRecvStream(const StreamParams& sp) override; 173 bool RemoveRecvStream(uint32_t ssrc) override; 174 void ResetUnsignaledRecvStream() override; 175 void OnDemuxerCriteriaUpdatePending() override; 176 void OnDemuxerCriteriaUpdateComplete() override; 177 178 // E2EE Frame API 179 // Set a frame decryptor to a particular ssrc that will intercept all 180 // incoming audio payloads and attempt to decrypt them before forwarding the 181 // result. 182 void SetFrameDecryptor(uint32_t ssrc, 183 rtc::scoped_refptr<webrtc::FrameDecryptorInterface> 184 frame_decryptor) override; 185 // Set a frame encryptor to a particular ssrc that will intercept all 186 // outgoing audio payloads frames and attempt to encrypt them and forward the 187 // result to the packetizer. 188 void SetFrameEncryptor(uint32_t ssrc, 189 rtc::scoped_refptr<webrtc::FrameEncryptorInterface> 190 frame_encryptor) override; 191 192 bool SetOutputVolume(uint32_t ssrc, double volume) override; 193 // Applies the new volume to current and future unsignaled streams. 194 bool SetDefaultOutputVolume(double volume) override; 195 196 bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) override; 197 absl::optional<int> GetBaseMinimumPlayoutDelayMs( 198 uint32_t ssrc) const override; 199 200 bool CanInsertDtmf() override; 201 bool InsertDtmf(uint32_t ssrc, int event, int duration) override; 202 203 void OnPacketReceived(rtc::CopyOnWriteBuffer packet, 204 int64_t packet_time_us) override; 205 void OnPacketSent(const rtc::SentPacket& sent_packet) override; 206 void OnNetworkRouteChanged(absl::string_view transport_name, 207 const rtc::NetworkRoute& network_route) override; 208 void OnReadyToSend(bool ready) override; 209 bool GetStats(VoiceMediaInfo* info, bool get_and_clear_legacy_stats) override; 210 211 // Set the audio sink for an existing stream. 212 void SetRawAudioSink( 213 uint32_t ssrc, 214 std::unique_ptr<webrtc::AudioSinkInterface> sink) override; 215 // Will set the audio sink on the latest unsignaled stream, future or 216 // current. Only one stream at a time will use the sink. 217 void SetDefaultRawAudioSink( 218 std::unique_ptr<webrtc::AudioSinkInterface> sink) override; 219 220 std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const override; 221 222 // Sets a frame transformer between encoder and packetizer, to transform 223 // encoded frames before sending them out the network. 224 void SetEncoderToPacketizerFrameTransformer( 225 uint32_t ssrc, 226 rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) 227 override; 228 void SetDepacketizerToDecoderFrameTransformer( 229 uint32_t ssrc, 230 rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) 231 override; 232 233 // implements Transport interface 234 bool SendRtp(const uint8_t* data, 235 size_t len, 236 const webrtc::PacketOptions& options) override; 237 238 bool SendRtcp(const uint8_t* data, size_t len) override; 239 240 private: 241 bool SetOptions(const AudioOptions& options); 242 bool SetRecvCodecs(const std::vector<AudioCodec>& codecs); 243 bool SetSendCodecs(const std::vector<AudioCodec>& codecs); 244 bool SetLocalSource(uint32_t ssrc, AudioSource* source); 245 bool MuteStream(uint32_t ssrc, bool mute); 246 engine()247 WebRtcVoiceEngine* engine() { return engine_; } 248 int CreateVoEChannel(); 249 bool DeleteVoEChannel(int channel); 250 bool SetMaxSendBitrate(int bps); 251 void SetupRecording(); 252 // Check if 'ssrc' is an unsignaled stream, and if so mark it as not being 253 // unsignaled anymore (i.e. it is now removed, or signaled), and return true. 254 bool MaybeDeregisterUnsignaledRecvStream(uint32_t ssrc); 255 256 webrtc::TaskQueueBase* const worker_thread_; 257 webrtc::ScopedTaskSafety task_safety_; 258 webrtc::SequenceChecker network_thread_checker_; 259 260 WebRtcVoiceEngine* const engine_ = nullptr; 261 std::vector<AudioCodec> send_codecs_; 262 263 // TODO(kwiberg): decoder_map_ and recv_codecs_ store the exact same 264 // information, in slightly different formats. Eliminate recv_codecs_. 265 std::map<int, webrtc::SdpAudioFormat> decoder_map_; 266 std::vector<AudioCodec> recv_codecs_; 267 268 int max_send_bitrate_bps_ = 0; 269 AudioOptions options_; 270 absl::optional<int> dtmf_payload_type_; 271 int dtmf_payload_freq_ = -1; 272 bool recv_transport_cc_enabled_ = false; 273 bool recv_nack_enabled_ = false; 274 bool enable_non_sender_rtt_ = false; 275 bool playout_ = false; 276 bool send_ = false; 277 webrtc::Call* const call_ = nullptr; 278 279 const MediaConfig::Audio audio_config_; 280 281 // Queue of unsignaled SSRCs; oldest at the beginning. 282 std::vector<uint32_t> unsignaled_recv_ssrcs_; 283 284 // This is a stream param that comes from the remote description, but wasn't 285 // signaled with any a=ssrc lines. It holds the information that was signaled 286 // before the unsignaled receive stream is created when the first packet is 287 // received. 288 StreamParams unsignaled_stream_params_; 289 290 // Volume for unsignaled streams, which may be set before the stream exists. 291 double default_recv_volume_ = 1.0; 292 293 // Delay for unsignaled streams, which may be set before the stream exists. 294 int default_recv_base_minimum_delay_ms_ = 0; 295 296 // Sink for latest unsignaled stream - may be set before the stream exists. 297 std::unique_ptr<webrtc::AudioSinkInterface> default_sink_; 298 // Default SSRC to use for RTCP receiver reports in case of no signaled 299 // send streams. See: https://code.google.com/p/webrtc/issues/detail?id=4740 300 // and https://code.google.com/p/chromium/issues/detail?id=547661 301 uint32_t receiver_reports_ssrc_ = 0xFA17FA17u; 302 303 class WebRtcAudioSendStream; 304 std::map<uint32_t, WebRtcAudioSendStream*> send_streams_; 305 std::vector<webrtc::RtpExtension> send_rtp_extensions_; 306 std::string mid_; 307 308 class WebRtcAudioReceiveStream; 309 std::map<uint32_t, WebRtcAudioReceiveStream*> recv_streams_; 310 std::vector<webrtc::RtpExtension> recv_rtp_extensions_; 311 312 absl::optional<webrtc::AudioSendStream::Config::SendCodecSpec> 313 send_codec_spec_; 314 315 // TODO(kwiberg): Per-SSRC codec pair IDs? 316 const webrtc::AudioCodecPairId codec_pair_id_ = 317 webrtc::AudioCodecPairId::Create(); 318 319 // Per peer connection crypto options that last for the lifetime of the peer 320 // connection. 321 const webrtc::CryptoOptions crypto_options_; 322 // Unsignaled streams have an option to have a frame decryptor set on them. 323 rtc::scoped_refptr<webrtc::FrameDecryptorInterface> 324 unsignaled_frame_decryptor_; 325 rtc::scoped_refptr<webrtc::FrameTransformerInterface> 326 unsignaled_frame_transformer_; 327 }; 328 } // namespace cricket 329 330 #endif // MEDIA_ENGINE_WEBRTC_VOICE_ENGINE_H_ 331