1 /* 2 * libjingle 3 * Copyright 2004 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #ifndef TALK_MEDIA_WEBRTCVIDEOENGINE_H_ 29 #define TALK_MEDIA_WEBRTCVIDEOENGINE_H_ 30 31 #include <map> 32 #include <vector> 33 34 #include "talk/media/base/codec.h" 35 #include "talk/media/base/videocommon.h" 36 #include "talk/media/webrtc/webrtccommon.h" 37 #include "talk/media/webrtc/webrtcexport.h" 38 #include "talk/media/webrtc/webrtcvideoencoderfactory.h" 39 #include "talk/session/media/channel.h" 40 #include "webrtc/base/scoped_ptr.h" 41 #include "webrtc/video_engine/include/vie_base.h" 42 43 #if !defined(LIBPEERCONNECTION_LIB) && \ 44 !defined(LIBPEERCONNECTION_IMPLEMENTATION) 45 // If you hit this, then you've tried to include this header from outside 46 // a shared library. An instance of this class must only be created from 47 // within the library that actually implements it. 48 #error "Bogus include." 49 #endif 50 51 namespace webrtc { 52 class VideoCaptureModule; 53 class VideoDecoder; 54 class VideoEncoder; 55 class VideoRender; 56 class ViEExternalCapture; 57 class ViERTP_RTCP; 58 } 59 60 namespace rtc { 61 class CpuMonitor; 62 } // namespace rtc 63 64 namespace cricket { 65 66 class CoordinatedVideoAdapter; 67 class ViETraceWrapper; 68 class ViEWrapper; 69 class VideoCapturer; 70 class VideoFrame; 71 class VideoProcessor; 72 class VideoRenderer; 73 class VoiceMediaChannel; 74 class WebRtcDecoderObserver; 75 class WebRtcEncoderObserver; 76 class WebRtcLocalStreamInfo; 77 class WebRtcRenderAdapter; 78 class WebRtcVideoChannelRecvInfo; 79 class WebRtcVideoChannelSendInfo; 80 class WebRtcVideoDecoderFactory; 81 class WebRtcVideoEncoderFactory; 82 class WebRtcVideoMediaChannel; 83 class WebRtcVoiceEngine; 84 85 struct CapturedFrame; 86 struct Device; 87 88 class WebRtcVideoEngine : public sigslot::has_slots<>, 89 public webrtc::TraceCallback, 90 public WebRtcVideoEncoderFactory::Observer { 91 public: 92 // Creates the WebRtcVideoEngine with internal VideoCaptureModule. 93 WebRtcVideoEngine(); 94 // For testing purposes. Allows the WebRtcVoiceEngine, 95 // ViEWrapper and CpuMonitor to be mocks. 96 // TODO(juberti): Remove the 3-arg ctor once fake tracing is implemented. 97 WebRtcVideoEngine(WebRtcVoiceEngine* voice_engine, 98 ViEWrapper* vie_wrapper, 99 rtc::CpuMonitor* cpu_monitor); 100 WebRtcVideoEngine(WebRtcVoiceEngine* voice_engine, 101 ViEWrapper* vie_wrapper, 102 ViETraceWrapper* tracing, 103 rtc::CpuMonitor* cpu_monitor); 104 virtual ~WebRtcVideoEngine(); 105 106 // Basic video engine implementation. 107 bool Init(rtc::Thread* worker_thread); 108 void Terminate(); 109 110 int GetCapabilities(); 111 bool SetDefaultEncoderConfig(const VideoEncoderConfig& config); 112 VideoEncoderConfig GetDefaultEncoderConfig() const; 113 114 virtual WebRtcVideoMediaChannel* CreateChannel( 115 VoiceMediaChannel* voice_channel); 116 117 const std::vector<VideoCodec>& codecs() const; 118 const std::vector<RtpHeaderExtension>& rtp_header_extensions() const; 119 void SetLogging(int min_sev, const char* filter); 120 121 sigslot::repeater2<VideoCapturer*, CaptureState> SignalCaptureStateChange; 122 123 // Set the VoiceEngine for A/V sync. This can only be called before Init. 124 bool SetVoiceEngine(WebRtcVoiceEngine* voice_engine); 125 // Set a WebRtcVideoDecoderFactory for external decoding. Video engine does 126 // not take the ownership of |decoder_factory|. The caller needs to make sure 127 // that |decoder_factory| outlives the video engine. 128 void SetExternalDecoderFactory(WebRtcVideoDecoderFactory* decoder_factory); 129 // Set a WebRtcVideoEncoderFactory for external encoding. Video engine does 130 // not take the ownership of |encoder_factory|. The caller needs to make sure 131 // that |encoder_factory| outlives the video engine. 132 virtual void SetExternalEncoderFactory( 133 WebRtcVideoEncoderFactory* encoder_factory); 134 // Enable the render module with timing control. 135 bool EnableTimedRender(); 136 137 // Returns an external decoder for the given codec type. The return value 138 // can be NULL if decoder factory is not given or it does not support the 139 // codec type. The caller takes the ownership of the returned object. 140 webrtc::VideoDecoder* CreateExternalDecoder(webrtc::VideoCodecType type); 141 // Releases the decoder instance created by CreateExternalDecoder(). 142 void DestroyExternalDecoder(webrtc::VideoDecoder* decoder); 143 144 // Returns an external encoder for the given codec type. The return value 145 // can be NULL if encoder factory is not given or it does not support the 146 // codec type. The caller takes the ownership of the returned object. 147 webrtc::VideoEncoder* CreateExternalEncoder(webrtc::VideoCodecType type); 148 // Releases the encoder instance created by CreateExternalEncoder(). 149 void DestroyExternalEncoder(webrtc::VideoEncoder* encoder); 150 151 // Returns true if the codec type is supported by the external encoder. 152 bool IsExternalEncoderCodecType(webrtc::VideoCodecType type) const; 153 154 // Functions called by WebRtcVideoMediaChannel. worker_thread()155 rtc::Thread* worker_thread() { return worker_thread_; } vie()156 ViEWrapper* vie() { return vie_wrapper_.get(); } default_codec_format()157 const VideoFormat& default_codec_format() const { 158 return default_codec_format_; 159 } 160 int GetLastEngineError(); 161 bool FindCodec(const VideoCodec& in); 162 bool CanSendCodec(const VideoCodec& in, const VideoCodec& current, 163 VideoCodec* out); 164 void RegisterChannel(WebRtcVideoMediaChannel* channel); 165 void UnregisterChannel(WebRtcVideoMediaChannel* channel); 166 bool ConvertFromCricketVideoCodec(const VideoCodec& in_codec, 167 webrtc::VideoCodec* out_codec); 168 // Check whether the supplied trace should be ignored. 169 bool ShouldIgnoreTrace(const std::string& trace); 170 int GetNumOfChannels(); 171 GetStartCaptureFormat()172 VideoFormat GetStartCaptureFormat() const { return default_codec_format_; } 173 cpu_monitor()174 rtc::CpuMonitor* cpu_monitor() { return cpu_monitor_.get(); } 175 176 protected: 177 // When a video processor registers with the engine. 178 // SignalMediaFrame will be invoked for every video frame. 179 // See videoprocessor.h for param reference. 180 sigslot::signal3<uint32, VideoFrame*, bool*> SignalMediaFrame; 181 182 private: 183 typedef std::vector<WebRtcVideoMediaChannel*> VideoChannels; 184 struct VideoCodecPref { 185 const char* name; 186 int payload_type; 187 // For RTX, this field is the payload-type that RTX applies to. 188 // For other codecs, it should be set to -1. 189 int associated_payload_type; 190 int pref; 191 }; 192 193 static const VideoCodecPref kVideoCodecPrefs[]; 194 static const VideoFormatPod kVideoFormats[]; 195 static const VideoFormatPod kDefaultMaxVideoFormat; 196 197 void Construct(ViEWrapper* vie_wrapper, 198 ViETraceWrapper* tracing, 199 WebRtcVoiceEngine* voice_engine, 200 rtc::CpuMonitor* cpu_monitor); 201 bool SetDefaultCodec(const VideoCodec& codec); 202 bool RebuildCodecList(const VideoCodec& max_codec); 203 void SetTraceFilter(int filter); 204 void SetTraceOptions(const std::string& options); 205 bool InitVideoEngine(); 206 bool VerifyApt(const VideoCodec& in, int expected_apt) const; 207 208 // webrtc::TraceCallback implementation. 209 virtual void Print(webrtc::TraceLevel level, const char* trace, int length); 210 211 // WebRtcVideoEncoderFactory::Observer implementation. 212 virtual void OnCodecsAvailable(); 213 214 rtc::Thread* worker_thread_; 215 rtc::scoped_ptr<ViEWrapper> vie_wrapper_; 216 bool vie_wrapper_base_initialized_; 217 rtc::scoped_ptr<ViETraceWrapper> tracing_; 218 WebRtcVoiceEngine* voice_engine_; 219 rtc::scoped_ptr<webrtc::VideoRender> render_module_; 220 WebRtcVideoEncoderFactory* encoder_factory_; 221 WebRtcVideoDecoderFactory* decoder_factory_; 222 std::vector<VideoCodec> video_codecs_; 223 std::vector<RtpHeaderExtension> rtp_header_extensions_; 224 VideoFormat default_codec_format_; 225 226 bool initialized_; 227 rtc::CriticalSection channels_crit_; 228 VideoChannels channels_; 229 230 bool capture_started_; 231 232 rtc::scoped_ptr<rtc::CpuMonitor> cpu_monitor_; 233 }; 234 235 class WebRtcVideoMediaChannel : public rtc::MessageHandler, 236 public VideoMediaChannel, 237 public webrtc::Transport { 238 public: 239 WebRtcVideoMediaChannel(WebRtcVideoEngine* engine, 240 VoiceMediaChannel* voice_channel); 241 virtual ~WebRtcVideoMediaChannel(); 242 bool Init(); 243 engine()244 WebRtcVideoEngine* engine() { return engine_; } voice_channel()245 VoiceMediaChannel* voice_channel() { return voice_channel_; } sending()246 bool sending() const { return sending_; } 247 248 // Public for testing purpose. 249 uint32 GetDefaultSendChannelSsrc(); GetDefaultChannelId()250 int GetDefaultChannelId() const { return default_channel_id_; } 251 252 // VideoMediaChannel implementation 253 virtual bool SetRecvCodecs(const std::vector<VideoCodec> &codecs); 254 virtual bool SetSendCodecs(const std::vector<VideoCodec> &codecs); 255 virtual bool GetSendCodec(VideoCodec* send_codec); 256 virtual bool SetSendStreamFormat(uint32 ssrc, const VideoFormat& format); 257 virtual bool SetRender(bool render); 258 virtual bool SetSend(bool send); 259 260 virtual bool AddSendStream(const StreamParams& sp); 261 virtual bool RemoveSendStream(uint32 ssrc); 262 virtual bool AddRecvStream(const StreamParams& sp); 263 virtual bool RemoveRecvStream(uint32 ssrc); 264 virtual bool SetRenderer(uint32 ssrc, VideoRenderer* renderer); 265 virtual bool GetStats(const StatsOptions& options, VideoMediaInfo* info); 266 virtual bool SetCapturer(uint32 ssrc, VideoCapturer* capturer); 267 virtual bool SendIntraFrame(); 268 virtual bool RequestIntraFrame(); 269 270 virtual void OnPacketReceived(rtc::Buffer* packet, 271 const rtc::PacketTime& packet_time); 272 virtual void OnRtcpReceived(rtc::Buffer* packet, 273 const rtc::PacketTime& packet_time); 274 virtual void OnReadyToSend(bool ready); 275 virtual bool MuteStream(uint32 ssrc, bool on); 276 virtual bool SetRecvRtpHeaderExtensions( 277 const std::vector<RtpHeaderExtension>& extensions); 278 virtual bool SetSendRtpHeaderExtensions( 279 const std::vector<RtpHeaderExtension>& extensions); 280 virtual int GetRtpSendTimeExtnId() const; 281 virtual bool SetStartSendBandwidth(int bps); 282 virtual bool SetMaxSendBandwidth(int bps); 283 virtual bool SetOptions(const VideoOptions &options); GetOptions(VideoOptions * options)284 virtual bool GetOptions(VideoOptions *options) const { 285 *options = options_; 286 return true; 287 } 288 virtual void SetInterface(NetworkInterface* iface); 289 virtual void UpdateAspectRatio(int ratio_w, int ratio_h); 290 291 // Public functions for use by tests and other specialized code. send_ssrc()292 uint32 send_ssrc() const { return 0; } 293 bool GetRenderer(uint32 ssrc, VideoRenderer** renderer); 294 bool GetVideoAdapter(uint32 ssrc, CoordinatedVideoAdapter** video_adapter); 295 void SendFrame(VideoCapturer* capturer, const VideoFrame* frame); 296 bool SendFrame(WebRtcVideoChannelSendInfo* channel_info, 297 const VideoFrame* frame, bool is_screencast); 298 299 // Thunk functions for use with HybridVideoEngine OnLocalFrame(VideoCapturer * capturer,const VideoFrame * frame)300 void OnLocalFrame(VideoCapturer* capturer, const VideoFrame* frame) { 301 SendFrame(0u, frame, capturer->IsScreencast()); 302 } OnLocalFrameFormat(VideoCapturer * capturer,const VideoFormat * format)303 void OnLocalFrameFormat(VideoCapturer* capturer, const VideoFormat* format) { 304 } 305 306 virtual void OnMessage(rtc::Message* msg); 307 308 protected: GetLastEngineError()309 int GetLastEngineError() { return engine()->GetLastEngineError(); } 310 virtual int SendPacket(int channel, const void* data, int len); 311 virtual int SendRTCPPacket(int channel, const void* data, int len); 312 313 // Checks the current bitrate estimate and modifies the bitrates 314 // accordingly, including converting kAutoBandwidth to the correct defaults. 315 virtual void SanitizeBitrates( 316 int channel_id, webrtc::VideoCodec* video_codec); 317 virtual void LogSendCodecChange(const std::string& reason); 318 bool SetPrimaryAndRtxSsrcs( 319 int channel_id, int idx, uint32 primary_ssrc, 320 const StreamParams& send_params); 321 bool SetLimitedNumberOfSendSsrcs( 322 int channel_id, const StreamParams& send_params, size_t limit); 323 virtual bool SetSendSsrcs( 324 int channel_id, const StreamParams& send_params, 325 const webrtc::VideoCodec& codec); 326 327 private: 328 typedef std::map<uint32, WebRtcVideoChannelRecvInfo*> RecvChannelMap; 329 typedef std::map<uint32, WebRtcVideoChannelSendInfo*> SendChannelMap; 330 typedef std::map<uint32, uint32> SsrcMap; 331 typedef int (webrtc::ViERTP_RTCP::* ExtensionSetterFunction)(int, bool, int); 332 333 enum MediaDirection { MD_RECV, MD_SEND, MD_SENDRECV }; 334 335 // Creates and initializes a ViE channel. When successful 336 // |channel_id| will contain the new channel's ID. If |receiving| is 337 // true |ssrc| is the remote ssrc. If |sending| is true the ssrc is 338 // local ssrc. If both |receiving| and |sending| is true the ssrc 339 // must be kDefaultChannelSsrcKey and the channel will be created as 340 // a default channel. The ssrc must be different for receive 341 // channels and it must be different for send channels. If the same 342 // SSRC is being used for creating channel more than once, this 343 // function will fail returning false. 344 bool CreateChannel(uint32 ssrc_key, MediaDirection direction, 345 int* channel_id); 346 bool CreateUnsignalledRecvChannel(uint32 ssrc_key, int* channel_id); 347 bool ConfigureChannel(int channel_id, MediaDirection direction, 348 uint32 ssrc_key); 349 bool ConfigureReceiving(int channel_id, uint32 remote_ssrc_key); 350 bool ConfigureSending(int channel_id, uint32 local_ssrc_key); 351 bool SetNackFec(int channel_id, int red_payload_type, int fec_payload_type, 352 bool nack_enabled); 353 bool SetSendCodec(const webrtc::VideoCodec& codec); 354 bool SetSendCodec(WebRtcVideoChannelSendInfo* send_channel, 355 const webrtc::VideoCodec& codec); 356 // Prepares the channel with channel id |info->channel_id()| to receive all 357 // codecs in |receive_codecs_| and start receive packets. 358 bool SetReceiveCodecs(WebRtcVideoChannelRecvInfo* info); 359 // Returns the channel ID that receives the stream with SSRC |ssrc|. 360 int GetRecvChannelId(uint32 ssrc); 361 bool MaybeSetRtxSsrc(const StreamParams& sp, int channel_id); 362 // Create and register an external endcoder if it's possible to do 363 // so and one isn't already registered. 364 bool MaybeRegisterExternalEncoder( 365 WebRtcVideoChannelSendInfo* send_channel, 366 const webrtc::VideoCodec& codec); 367 // Given captured video frame size, checks if we need to reset vie send codec. 368 // |reset| is set to whether resetting has happened on vie or not. 369 // Returns false on error. 370 bool MaybeResetVieSendCodec(WebRtcVideoChannelSendInfo* send_channel, 371 int new_width, int new_height, bool is_screencast, 372 bool* reset); 373 // Helper function for starting the sending of media on all channels or 374 // |channel_id|. Note that these two function do not change |sending_|. 375 bool StartSend(); 376 bool StartSend(WebRtcVideoChannelSendInfo* send_channel); 377 // Helper function for stop the sending of media on all channels or 378 // |channel_id|. Note that these two function do not change |sending_|. 379 bool StopSend(); 380 bool StopSend(WebRtcVideoChannelSendInfo* send_channel); 381 bool SendIntraFrame(int channel_id); 382 383 bool HasReadySendChannels(); 384 bool DefaultSendChannelInUse(); 385 386 // Returns the ssrc key corresponding to the provided local SSRC in 387 // |ssrc_key|. The return value is true upon success. If the local 388 // ssrc correspond to that of the default channel the key is 389 // kDefaultChannelSsrcKey. For all other channels the returned ssrc 390 // key will be the same as the local ssrc. If a stream has more 391 // than one ssrc, the first (corresponding to 392 // StreamParams::first_ssrc()) is used as the key. 393 bool GetSendChannelSsrcKey(uint32 local_ssrc, uint32* ssrc_key); 394 WebRtcVideoChannelSendInfo* GetDefaultSendChannel(); 395 WebRtcVideoChannelSendInfo* GetSendChannelBySsrcKey(uint32 ssrc_key); 396 WebRtcVideoChannelSendInfo* GetSendChannelBySsrc(uint32 local_ssrc); 397 // Creates a new unique ssrc key that can be used for inserting a 398 // new send channel into |send_channels_| 399 bool CreateSendChannelSsrcKey(uint32 local_ssrc, uint32* ssrc_key); 400 // Get the number of the send channels |capturer| registered with. 401 int GetSendChannelNum(VideoCapturer* capturer); 402 IsDefaultChannelId(int channel_id)403 bool IsDefaultChannelId(int channel_id) const { 404 return channel_id == default_channel_id_; 405 } 406 bool DeleteSendChannel(uint32 ssrc_key); 407 408 WebRtcVideoChannelRecvInfo* GetDefaultRecvChannel(); 409 WebRtcVideoChannelRecvInfo* GetRecvChannelBySsrc(uint32 ssrc); 410 InConferenceMode()411 bool InConferenceMode() const { 412 return options_.conference_mode.GetWithDefaultIfUnset(false); 413 } 414 bool RemoveCapturer(uint32 ssrc); 415 worker_thread()416 rtc::MessageQueue* worker_thread() { return engine_->worker_thread(); } 417 void QueueBlackFrame(uint32 ssrc, int64 timestamp, int framerate); 418 void FlushBlackFrame(uint32 ssrc, int64 timestamp); 419 420 void SetNetworkTransmissionState(bool is_transmitting); 421 422 bool SetHeaderExtension(ExtensionSetterFunction setter, int channel_id, 423 const RtpHeaderExtension* extension); 424 bool SetHeaderExtension(ExtensionSetterFunction setter, int channel_id, 425 const std::vector<RtpHeaderExtension>& extensions, 426 const char header_extension_uri[]); 427 428 // Signal when cpu adaptation has no further scope to adapt. 429 void OnCpuAdaptationUnable(); 430 431 // Connect |capturer| to WebRtcVideoMediaChannel if it is only registered 432 // to one send channel, i.e. the first send channel. 433 void MaybeConnectCapturer(VideoCapturer* capturer); 434 // Disconnect |capturer| from WebRtcVideoMediaChannel if it is only registered 435 // to one send channel, i.e. the last send channel. 436 void MaybeDisconnectCapturer(VideoCapturer* capturer); 437 438 bool RemoveRecvStreamInternal(uint32 ssrc); 439 440 // Set the ssrc to use for RTCP receiver reports. 441 void SetReceiverReportSsrc(uint32 ssrc); 442 443 // Global state. 444 WebRtcVideoEngine* engine_; 445 VoiceMediaChannel* voice_channel_; 446 int default_channel_id_; 447 bool nack_enabled_; 448 // Receiver Estimated Max Bitrate 449 bool remb_enabled_; 450 VideoOptions options_; 451 452 // Global recv side state. 453 // Note the default channel (default_channel_id_), i.e. the send channel 454 // corresponding to all the receive channels (this must be done for REMB to 455 // work properly), resides in both recv_channels_ and send_channels_ with the 456 // ssrc key kDefaultChannelSsrcKey. 457 RecvChannelMap recv_channels_; // Contains all receive channels. 458 // A map from the SSRCs on which RTX packets are received to the media SSRCs 459 // the RTX packets are associated with. RTX packets will be delivered to the 460 // streams matching the primary SSRC. 461 SsrcMap rtx_to_primary_ssrc_; 462 std::vector<webrtc::VideoCodec> receive_codecs_; 463 // A map from codec payload types to their associated payload types, if any. 464 // TODO(holmer): This is a temporary solution until webrtc::VideoCodec has 465 // an associated payload type member, when it does we can rely on 466 // receive_codecs_. 467 std::map<int, int> associated_payload_types_; 468 bool render_started_; 469 uint32 first_receive_ssrc_; 470 uint32 receiver_report_ssrc_; 471 std::vector<RtpHeaderExtension> receive_extensions_; 472 int num_unsignalled_recv_channels_; 473 474 // Global send side state. 475 SendChannelMap send_channels_; 476 rtc::scoped_ptr<webrtc::VideoCodec> send_codec_; 477 int send_rtx_type_; 478 int send_red_type_; 479 int send_fec_type_; 480 bool sending_; 481 std::vector<RtpHeaderExtension> send_extensions_; 482 483 // The aspect ratio that the channel desires. 0 means there is no desired 484 // aspect ratio 485 int ratio_w_; 486 int ratio_h_; 487 }; 488 489 } // namespace cricket 490 491 #endif // TALK_MEDIA_WEBRTCVIDEOENGINE_H_ 492