1 /* 2 * libjingle 3 * Copyright 2004 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #ifndef TALK_MEDIA_WEBRTCVOICEENGINE_H_ 29 #define TALK_MEDIA_WEBRTCVOICEENGINE_H_ 30 31 #include <map> 32 #include <set> 33 #include <string> 34 #include <vector> 35 36 #include "talk/base/buffer.h" 37 #include "talk/base/byteorder.h" 38 #include "talk/base/logging.h" 39 #include "talk/base/scoped_ptr.h" 40 #include "talk/base/stream.h" 41 #include "talk/media/base/rtputils.h" 42 #include "talk/media/webrtc/webrtccommon.h" 43 #include "talk/media/webrtc/webrtcexport.h" 44 #include "talk/media/webrtc/webrtcvoe.h" 45 #include "talk/session/media/channel.h" 46 #include "webrtc/common.h" 47 48 #if !defined(LIBPEERCONNECTION_LIB) && \ 49 !defined(LIBPEERCONNECTION_IMPLEMENTATION) 50 #error "Bogus include." 51 #endif 52 53 namespace cricket { 54 55 // WebRtcSoundclipStream is an adapter object that allows a memory stream to be 56 // passed into WebRtc, and support looping. 57 class WebRtcSoundclipStream : public webrtc::InStream { 58 public: WebRtcSoundclipStream(const char * buf,size_t len)59 WebRtcSoundclipStream(const char* buf, size_t len) 60 : mem_(buf, len), loop_(true) { 61 } set_loop(bool loop)62 void set_loop(bool loop) { loop_ = loop; } 63 virtual int Read(void* buf, int len); 64 virtual int Rewind(); 65 66 private: 67 talk_base::MemoryStream mem_; 68 bool loop_; 69 }; 70 71 // WebRtcMonitorStream is used to monitor a stream coming from WebRtc. 72 // For now we just dump the data. 73 class WebRtcMonitorStream : public webrtc::OutStream { Write(const void * buf,int len)74 virtual bool Write(const void *buf, int len) { 75 return true; 76 } 77 }; 78 79 class AudioDeviceModule; 80 class AudioRenderer; 81 class VoETraceWrapper; 82 class VoEWrapper; 83 class VoiceProcessor; 84 class WebRtcSoundclipMedia; 85 class WebRtcVoiceMediaChannel; 86 87 // WebRtcVoiceEngine is a class to be used with CompositeMediaEngine. 88 // It uses the WebRtc VoiceEngine library for audio handling. 89 class WebRtcVoiceEngine 90 : public webrtc::VoiceEngineObserver, 91 public webrtc::TraceCallback, 92 public webrtc::VoEMediaProcess { 93 public: 94 WebRtcVoiceEngine(); 95 // Dependency injection for testing. 96 WebRtcVoiceEngine(VoEWrapper* voe_wrapper, 97 VoEWrapper* voe_wrapper_sc, 98 VoETraceWrapper* tracing); 99 ~WebRtcVoiceEngine(); 100 bool Init(talk_base::Thread* worker_thread); 101 void Terminate(); 102 103 int GetCapabilities(); 104 VoiceMediaChannel* CreateChannel(); 105 106 SoundclipMedia* CreateSoundclip(); 107 GetOptions()108 AudioOptions GetOptions() const { return options_; } 109 bool SetOptions(const AudioOptions& options); 110 // Overrides, when set, take precedence over the options on a 111 // per-option basis. For example, if AGC is set in options and AEC 112 // is set in overrides, AGC and AEC will be both be set. Overrides 113 // can also turn off options. For example, if AGC is set to "on" in 114 // options and AGC is set to "off" in overrides, the result is that 115 // AGC will be off until different overrides are applied or until 116 // the overrides are cleared. Only one set of overrides is present 117 // at a time (they do not "stack"). And when the overrides are 118 // cleared, the media engine's state reverts back to the options set 119 // via SetOptions. This allows us to have both "persistent options" 120 // (the normal options) and "temporary options" (overrides). 121 bool SetOptionOverrides(const AudioOptions& options); 122 bool ClearOptionOverrides(); 123 bool SetDelayOffset(int offset); 124 bool SetDevices(const Device* in_device, const Device* out_device); 125 bool GetOutputVolume(int* level); 126 bool SetOutputVolume(int level); 127 int GetInputLevel(); 128 bool SetLocalMonitor(bool enable); 129 130 const std::vector<AudioCodec>& codecs(); 131 bool FindCodec(const AudioCodec& codec); 132 bool FindWebRtcCodec(const AudioCodec& codec, webrtc::CodecInst* gcodec); 133 134 const std::vector<RtpHeaderExtension>& rtp_header_extensions() const; 135 136 void SetLogging(int min_sev, const char* filter); 137 138 bool RegisterProcessor(uint32 ssrc, 139 VoiceProcessor* voice_processor, 140 MediaProcessorDirection direction); 141 bool UnregisterProcessor(uint32 ssrc, 142 VoiceProcessor* voice_processor, 143 MediaProcessorDirection direction); 144 145 // Method from webrtc::VoEMediaProcess 146 virtual void Process(int channel, 147 webrtc::ProcessingTypes type, 148 int16_t audio10ms[], 149 int length, 150 int sampling_freq, 151 bool is_stereo); 152 153 // For tracking WebRtc channels. Needed because we have to pause them 154 // all when switching devices. 155 // May only be called by WebRtcVoiceMediaChannel. 156 void RegisterChannel(WebRtcVoiceMediaChannel *channel); 157 void UnregisterChannel(WebRtcVoiceMediaChannel *channel); 158 159 // May only be called by WebRtcSoundclipMedia. 160 void RegisterSoundclip(WebRtcSoundclipMedia *channel); 161 void UnregisterSoundclip(WebRtcSoundclipMedia *channel); 162 163 // Called by WebRtcVoiceMediaChannel to set a gain offset from 164 // the default AGC target level. 165 bool AdjustAgcLevel(int delta); 166 voe()167 VoEWrapper* voe() { return voe_wrapper_.get(); } voe_sc()168 VoEWrapper* voe_sc() { return voe_wrapper_sc_.get(); } 169 int GetLastEngineError(); 170 171 // Set the external ADMs. This can only be called before Init. 172 bool SetAudioDeviceModule(webrtc::AudioDeviceModule* adm, 173 webrtc::AudioDeviceModule* adm_sc); 174 175 // Starts AEC dump using existing file. 176 bool StartAecDump(talk_base::PlatformFile file); 177 178 // Check whether the supplied trace should be ignored. 179 bool ShouldIgnoreTrace(const std::string& trace); 180 181 // Create a VoiceEngine Channel. 182 int CreateMediaVoiceChannel(); 183 int CreateSoundclipVoiceChannel(); 184 185 private: 186 typedef std::vector<WebRtcSoundclipMedia *> SoundclipList; 187 typedef std::vector<WebRtcVoiceMediaChannel *> ChannelList; 188 typedef sigslot:: 189 signal3<uint32, MediaProcessorDirection, AudioFrame*> FrameSignal; 190 191 void Construct(); 192 void ConstructCodecs(); 193 bool InitInternal(); 194 bool EnsureSoundclipEngineInit(); 195 void SetTraceFilter(int filter); 196 void SetTraceOptions(const std::string& options); 197 // Applies either options or overrides. Every option that is "set" 198 // will be applied. Every option not "set" will be ignored. This 199 // allows us to selectively turn on and off different options easily 200 // at any time. 201 bool ApplyOptions(const AudioOptions& options); 202 virtual void Print(webrtc::TraceLevel level, const char* trace, int length); 203 virtual void CallbackOnError(int channel, int errCode); 204 // Given the device type, name, and id, find device id. Return true and 205 // set the output parameter rtc_id if successful. 206 bool FindWebRtcAudioDeviceId( 207 bool is_input, const std::string& dev_name, int dev_id, int* rtc_id); 208 bool FindChannelAndSsrc(int channel_num, 209 WebRtcVoiceMediaChannel** channel, 210 uint32* ssrc) const; 211 bool FindChannelNumFromSsrc(uint32 ssrc, 212 MediaProcessorDirection direction, 213 int* channel_num); 214 bool ChangeLocalMonitor(bool enable); 215 bool PauseLocalMonitor(); 216 bool ResumeLocalMonitor(); 217 218 bool UnregisterProcessorChannel(MediaProcessorDirection channel_direction, 219 uint32 ssrc, 220 VoiceProcessor* voice_processor, 221 MediaProcessorDirection processor_direction); 222 223 void StartAecDump(const std::string& filename); 224 void StopAecDump(); 225 int CreateVoiceChannel(VoEWrapper* voe); 226 227 // When a voice processor registers with the engine, it is connected 228 // to either the Rx or Tx signals, based on the direction parameter. 229 // SignalXXMediaFrame will be invoked for every audio packet. 230 FrameSignal SignalRxMediaFrame; 231 FrameSignal SignalTxMediaFrame; 232 233 static const int kDefaultLogSeverity = talk_base::LS_WARNING; 234 235 // The primary instance of WebRtc VoiceEngine. 236 talk_base::scoped_ptr<VoEWrapper> voe_wrapper_; 237 // A secondary instance, for playing out soundclips (on the 'ring' device). 238 talk_base::scoped_ptr<VoEWrapper> voe_wrapper_sc_; 239 bool voe_wrapper_sc_initialized_; 240 talk_base::scoped_ptr<VoETraceWrapper> tracing_; 241 // The external audio device manager 242 webrtc::AudioDeviceModule* adm_; 243 webrtc::AudioDeviceModule* adm_sc_; 244 int log_filter_; 245 std::string log_options_; 246 bool is_dumping_aec_; 247 std::vector<AudioCodec> codecs_; 248 std::vector<RtpHeaderExtension> rtp_header_extensions_; 249 bool desired_local_monitor_enable_; 250 talk_base::scoped_ptr<WebRtcMonitorStream> monitor_; 251 SoundclipList soundclips_; 252 ChannelList channels_; 253 // channels_ can be read from WebRtc callback thread. We need a lock on that 254 // callback as well as the RegisterChannel/UnregisterChannel. 255 talk_base::CriticalSection channels_cs_; 256 webrtc::AgcConfig default_agc_config_; 257 258 webrtc::Config voe_config_; 259 260 bool initialized_; 261 // See SetOptions and SetOptionOverrides for a description of the 262 // difference between options and overrides. 263 // options_ are the base options, which combined with the 264 // option_overrides_, create the current options being used. 265 // options_ is stored so that when option_overrides_ is cleared, we 266 // can restore the options_ without the option_overrides. 267 AudioOptions options_; 268 AudioOptions option_overrides_; 269 270 // When the media processor registers with the engine, the ssrc is cached 271 // here so that a look up need not be made when the callback is invoked. 272 // This is necessary because the lookup results in mux_channels_cs lock being 273 // held and if a remote participant leaves the hangout at the same time 274 // we hit a deadlock. 275 uint32 tx_processor_ssrc_; 276 uint32 rx_processor_ssrc_; 277 278 talk_base::CriticalSection signal_media_critical_; 279 }; 280 281 // WebRtcMediaChannel is a class that implements the common WebRtc channel 282 // functionality. 283 template <class T, class E> 284 class WebRtcMediaChannel : public T, public webrtc::Transport { 285 public: WebRtcMediaChannel(E * engine,int channel)286 WebRtcMediaChannel(E *engine, int channel) 287 : engine_(engine), voe_channel_(channel) {} engine()288 E *engine() { return engine_; } voe_channel()289 int voe_channel() const { return voe_channel_; } valid()290 bool valid() const { return voe_channel_ != -1; } 291 292 protected: 293 // implements Transport interface SendPacket(int channel,const void * data,int len)294 virtual int SendPacket(int channel, const void *data, int len) { 295 talk_base::Buffer packet(data, len, kMaxRtpPacketLen); 296 if (!T::SendPacket(&packet)) { 297 return -1; 298 } 299 return len; 300 } 301 SendRTCPPacket(int channel,const void * data,int len)302 virtual int SendRTCPPacket(int channel, const void *data, int len) { 303 talk_base::Buffer packet(data, len, kMaxRtpPacketLen); 304 return T::SendRtcp(&packet) ? len : -1; 305 } 306 307 private: 308 E *engine_; 309 int voe_channel_; 310 }; 311 312 // WebRtcVoiceMediaChannel is an implementation of VoiceMediaChannel that uses 313 // WebRtc Voice Engine. 314 class WebRtcVoiceMediaChannel 315 : public WebRtcMediaChannel<VoiceMediaChannel, WebRtcVoiceEngine> { 316 public: 317 explicit WebRtcVoiceMediaChannel(WebRtcVoiceEngine *engine); 318 virtual ~WebRtcVoiceMediaChannel(); 319 virtual bool SetOptions(const AudioOptions& options); GetOptions(AudioOptions * options)320 virtual bool GetOptions(AudioOptions* options) const { 321 *options = options_; 322 return true; 323 } 324 virtual bool SetRecvCodecs(const std::vector<AudioCodec> &codecs); 325 virtual bool SetSendCodecs(const std::vector<AudioCodec> &codecs); 326 virtual bool SetRecvRtpHeaderExtensions( 327 const std::vector<RtpHeaderExtension>& extensions); 328 virtual bool SetSendRtpHeaderExtensions( 329 const std::vector<RtpHeaderExtension>& extensions); 330 virtual bool SetPlayout(bool playout); 331 bool PausePlayout(); 332 bool ResumePlayout(); 333 virtual bool SetSend(SendFlags send); 334 bool PauseSend(); 335 bool ResumeSend(); 336 virtual bool AddSendStream(const StreamParams& sp); 337 virtual bool RemoveSendStream(uint32 ssrc); 338 virtual bool AddRecvStream(const StreamParams& sp); 339 virtual bool RemoveRecvStream(uint32 ssrc); 340 virtual bool SetRemoteRenderer(uint32 ssrc, AudioRenderer* renderer); 341 virtual bool SetLocalRenderer(uint32 ssrc, AudioRenderer* renderer); 342 virtual bool GetActiveStreams(AudioInfo::StreamList* actives); 343 virtual int GetOutputLevel(); 344 virtual int GetTimeSinceLastTyping(); 345 virtual void SetTypingDetectionParameters(int time_window, 346 int cost_per_typing, int reporting_threshold, int penalty_decay, 347 int type_event_delay); 348 virtual bool SetOutputScaling(uint32 ssrc, double left, double right); 349 virtual bool GetOutputScaling(uint32 ssrc, double* left, double* right); 350 351 virtual bool SetRingbackTone(const char *buf, int len); 352 virtual bool PlayRingbackTone(uint32 ssrc, bool play, bool loop); 353 virtual bool CanInsertDtmf(); 354 virtual bool InsertDtmf(uint32 ssrc, int event, int duration, int flags); 355 356 virtual void OnPacketReceived(talk_base::Buffer* packet, 357 const talk_base::PacketTime& packet_time); 358 virtual void OnRtcpReceived(talk_base::Buffer* packet, 359 const talk_base::PacketTime& packet_time); OnReadyToSend(bool ready)360 virtual void OnReadyToSend(bool ready) {} 361 virtual bool MuteStream(uint32 ssrc, bool on); 362 virtual bool SetStartSendBandwidth(int bps); 363 virtual bool SetMaxSendBandwidth(int bps); 364 virtual bool GetStats(VoiceMediaInfo* info); 365 // Gets last reported error from WebRtc voice engine. This should be only 366 // called in response a failure. 367 virtual void GetLastMediaError(uint32* ssrc, 368 VoiceMediaChannel::Error* error); 369 bool FindSsrc(int channel_num, uint32* ssrc); 370 void OnError(uint32 ssrc, int error); 371 sending()372 bool sending() const { return send_ != SEND_NOTHING; } 373 int GetReceiveChannelNum(uint32 ssrc); 374 int GetSendChannelNum(uint32 ssrc); 375 376 protected: GetLastEngineError()377 int GetLastEngineError() { return engine()->GetLastEngineError(); } 378 int GetOutputLevel(int channel); 379 bool GetRedSendCodec(const AudioCodec& red_codec, 380 const std::vector<AudioCodec>& all_codecs, 381 webrtc::CodecInst* send_codec); 382 bool EnableRtcp(int channel); 383 bool ResetRecvCodecs(int channel); 384 bool SetPlayout(int channel, bool playout); 385 static uint32 ParseSsrc(const void* data, size_t len, bool rtcp); 386 static Error WebRtcErrorToChannelError(int err_code); 387 388 private: 389 class WebRtcVoiceChannelRenderer; 390 // Map of ssrc to WebRtcVoiceChannelRenderer object. A new object of 391 // WebRtcVoiceChannelRenderer will be created for every new stream and 392 // will be destroyed when the stream goes away. 393 typedef std::map<uint32, WebRtcVoiceChannelRenderer*> ChannelMap; 394 typedef int (webrtc::VoERTP_RTCP::* ExtensionSetterFunction)(int, bool, 395 unsigned char); 396 397 void SetNack(int channel, bool nack_enabled); 398 void SetNack(const ChannelMap& channels, bool nack_enabled); 399 bool SetSendCodec(const webrtc::CodecInst& send_codec); 400 bool SetSendCodec(int channel, const webrtc::CodecInst& send_codec); 401 bool ChangePlayout(bool playout); 402 bool ChangeSend(SendFlags send); 403 bool ChangeSend(int channel, SendFlags send); 404 void ConfigureSendChannel(int channel); 405 bool ConfigureRecvChannel(int channel); 406 bool DeleteChannel(int channel); InConferenceMode()407 bool InConferenceMode() const { 408 return options_.conference_mode.GetWithDefaultIfUnset(false); 409 } IsDefaultChannel(int channel_id)410 bool IsDefaultChannel(int channel_id) const { 411 return channel_id == voe_channel(); 412 } 413 bool SetSendCodecs(int channel, const std::vector<AudioCodec>& codecs); 414 bool SetSendBandwidthInternal(int bps); 415 416 bool SetHeaderExtension(ExtensionSetterFunction setter, int channel_id, 417 const RtpHeaderExtension* extension); 418 419 bool SetChannelRecvRtpHeaderExtensions( 420 int channel_id, 421 const std::vector<RtpHeaderExtension>& extensions); 422 bool SetChannelSendRtpHeaderExtensions( 423 int channel_id, 424 const std::vector<RtpHeaderExtension>& extensions); 425 426 talk_base::scoped_ptr<WebRtcSoundclipStream> ringback_tone_; 427 std::set<int> ringback_channels_; // channels playing ringback 428 std::vector<AudioCodec> recv_codecs_; 429 std::vector<AudioCodec> send_codecs_; 430 talk_base::scoped_ptr<webrtc::CodecInst> send_codec_; 431 bool send_bw_setting_; 432 int send_bw_bps_; 433 AudioOptions options_; 434 bool dtmf_allowed_; 435 bool desired_playout_; 436 bool nack_enabled_; 437 bool playout_; 438 bool typing_noise_detected_; 439 SendFlags desired_send_; 440 SendFlags send_; 441 442 // send_channels_ contains the channels which are being used for sending. 443 // When the default channel (voe_channel) is used for sending, it is 444 // contained in send_channels_, otherwise not. 445 ChannelMap send_channels_; 446 std::vector<RtpHeaderExtension> send_extensions_; 447 uint32 default_receive_ssrc_; 448 // Note the default channel (voe_channel()) can reside in both 449 // receive_channels_ and send_channels_ in non-conference mode and in that 450 // case it will only be there if a non-zero default_receive_ssrc_ is set. 451 ChannelMap receive_channels_; // for multiple sources 452 // receive_channels_ can be read from WebRtc callback thread. Access from 453 // the WebRtc thread must be synchronized with edits on the worker thread. 454 // Reads on the worker thread are ok. 455 // 456 std::vector<RtpHeaderExtension> receive_extensions_; 457 // Do not lock this on the VoE media processor thread; potential for deadlock 458 // exists. 459 mutable talk_base::CriticalSection receive_channels_cs_; 460 }; 461 462 } // namespace cricket 463 464 #endif // TALK_MEDIA_WEBRTCVOICEENGINE_H_ 465