1 /* 2 * libjingle 3 * Copyright 2004 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #ifndef TALK_MEDIA_WEBRTCVIDEOENGINE_H_ 29 #define TALK_MEDIA_WEBRTCVIDEOENGINE_H_ 30 31 #include <map> 32 #include <vector> 33 34 #include "talk/base/scoped_ptr.h" 35 #include "talk/media/base/codec.h" 36 #include "talk/media/base/videocommon.h" 37 #include "talk/media/webrtc/webrtccommon.h" 38 #include "talk/media/webrtc/webrtcexport.h" 39 #include "talk/media/webrtc/webrtcvideoencoderfactory.h" 40 #include "talk/session/media/channel.h" 41 #include "webrtc/video_engine/include/vie_base.h" 42 43 #if !defined(LIBPEERCONNECTION_LIB) && \ 44 !defined(LIBPEERCONNECTION_IMPLEMENTATION) 45 #error "Bogus include." 46 #endif 47 48 49 namespace webrtc { 50 class VideoCaptureModule; 51 class VideoDecoder; 52 class VideoEncoder; 53 class VideoRender; 54 class ViEExternalCapture; 55 class ViERTP_RTCP; 56 } 57 58 namespace talk_base { 59 class CpuMonitor; 60 } // namespace talk_base 61 62 namespace cricket { 63 64 class CoordinatedVideoAdapter; 65 class ViETraceWrapper; 66 class ViEWrapper; 67 class VideoCapturer; 68 class VideoFrame; 69 class VideoProcessor; 70 class VideoRenderer; 71 class VoiceMediaChannel; 72 class WebRtcDecoderObserver; 73 class WebRtcEncoderObserver; 74 class WebRtcLocalStreamInfo; 75 class WebRtcRenderAdapter; 76 class WebRtcVideoChannelRecvInfo; 77 class WebRtcVideoChannelSendInfo; 78 class WebRtcVideoDecoderFactory; 79 class WebRtcVideoEncoderFactory; 80 class WebRtcVideoMediaChannel; 81 class WebRtcVoiceEngine; 82 83 struct CapturedFrame; 84 struct Device; 85 86 class WebRtcVideoEngine : public sigslot::has_slots<>, 87 public webrtc::TraceCallback, 88 public WebRtcVideoEncoderFactory::Observer { 89 public: 90 // Creates the WebRtcVideoEngine with internal VideoCaptureModule. 91 WebRtcVideoEngine(); 92 // For testing purposes. Allows the WebRtcVoiceEngine, 93 // ViEWrapper and CpuMonitor to be mocks. 94 // TODO(juberti): Remove the 3-arg ctor once fake tracing is implemented. 95 WebRtcVideoEngine(WebRtcVoiceEngine* voice_engine, 96 ViEWrapper* vie_wrapper, 97 talk_base::CpuMonitor* cpu_monitor); 98 WebRtcVideoEngine(WebRtcVoiceEngine* voice_engine, 99 ViEWrapper* vie_wrapper, 100 ViETraceWrapper* tracing, 101 talk_base::CpuMonitor* cpu_monitor); 102 ~WebRtcVideoEngine(); 103 104 // Basic video engine implementation. 105 bool Init(talk_base::Thread* worker_thread); 106 void Terminate(); 107 108 int GetCapabilities(); 109 bool SetOptions(const VideoOptions &options); 110 bool SetDefaultEncoderConfig(const VideoEncoderConfig& config); 111 VideoEncoderConfig GetDefaultEncoderConfig() const; 112 113 WebRtcVideoMediaChannel* CreateChannel(VoiceMediaChannel* voice_channel); 114 115 const std::vector<VideoCodec>& codecs() const; 116 const std::vector<RtpHeaderExtension>& rtp_header_extensions() const; 117 void SetLogging(int min_sev, const char* filter); 118 119 bool SetLocalRenderer(VideoRenderer* renderer); 120 sigslot::repeater2<VideoCapturer*, CaptureState> SignalCaptureStateChange; 121 122 // Set the VoiceEngine for A/V sync. This can only be called before Init. 123 bool SetVoiceEngine(WebRtcVoiceEngine* voice_engine); 124 // Set a WebRtcVideoDecoderFactory for external decoding. Video engine does 125 // not take the ownership of |decoder_factory|. The caller needs to make sure 126 // that |decoder_factory| outlives the video engine. 127 void SetExternalDecoderFactory(WebRtcVideoDecoderFactory* decoder_factory); 128 // Set a WebRtcVideoEncoderFactory for external encoding. Video engine does 129 // not take the ownership of |encoder_factory|. The caller needs to make sure 130 // that |encoder_factory| outlives the video engine. 131 void SetExternalEncoderFactory(WebRtcVideoEncoderFactory* encoder_factory); 132 // Enable the render module with timing control. 133 bool EnableTimedRender(); 134 135 // Returns an external decoder for the given codec type. The return value 136 // can be NULL if decoder factory is not given or it does not support the 137 // codec type. The caller takes the ownership of the returned object. 138 webrtc::VideoDecoder* CreateExternalDecoder(webrtc::VideoCodecType type); 139 // Releases the decoder instance created by CreateExternalDecoder(). 140 void DestroyExternalDecoder(webrtc::VideoDecoder* decoder); 141 142 // Returns an external encoder for the given codec type. The return value 143 // can be NULL if encoder factory is not given or it does not support the 144 // codec type. The caller takes the ownership of the returned object. 145 webrtc::VideoEncoder* CreateExternalEncoder(webrtc::VideoCodecType type); 146 // Releases the encoder instance created by CreateExternalEncoder(). 147 void DestroyExternalEncoder(webrtc::VideoEncoder* encoder); 148 149 // Returns true if the codec type is supported by the external encoder. 150 bool IsExternalEncoderCodecType(webrtc::VideoCodecType type) const; 151 152 // Functions called by WebRtcVideoMediaChannel. worker_thread()153 talk_base::Thread* worker_thread() { return worker_thread_; } vie()154 ViEWrapper* vie() { return vie_wrapper_.get(); } default_codec_format()155 const VideoFormat& default_codec_format() const { 156 return default_codec_format_; 157 } 158 int GetLastEngineError(); 159 bool FindCodec(const VideoCodec& in); 160 bool CanSendCodec(const VideoCodec& in, const VideoCodec& current, 161 VideoCodec* out); 162 void RegisterChannel(WebRtcVideoMediaChannel* channel); 163 void UnregisterChannel(WebRtcVideoMediaChannel* channel); 164 bool ConvertFromCricketVideoCodec(const VideoCodec& in_codec, 165 webrtc::VideoCodec* out_codec); 166 // Check whether the supplied trace should be ignored. 167 bool ShouldIgnoreTrace(const std::string& trace); 168 int GetNumOfChannels(); 169 GetStartCaptureFormat()170 VideoFormat GetStartCaptureFormat() const { return default_codec_format_; } 171 cpu_monitor()172 talk_base::CpuMonitor* cpu_monitor() { return cpu_monitor_.get(); } 173 174 protected: 175 // When a video processor registers with the engine. 176 // SignalMediaFrame will be invoked for every video frame. 177 // See videoprocessor.h for param reference. 178 sigslot::signal3<uint32, VideoFrame*, bool*> SignalMediaFrame; 179 180 private: 181 typedef std::vector<WebRtcVideoMediaChannel*> VideoChannels; 182 struct VideoCodecPref { 183 const char* name; 184 int payload_type; 185 // For RTX, this field is the payload-type that RTX applies to. 186 // For other codecs, it should be set to -1. 187 int associated_payload_type; 188 int pref; 189 }; 190 191 static const VideoCodecPref kVideoCodecPrefs[]; 192 static const VideoFormatPod kVideoFormats[]; 193 static const VideoFormatPod kDefaultVideoFormat; 194 195 void Construct(ViEWrapper* vie_wrapper, 196 ViETraceWrapper* tracing, 197 WebRtcVoiceEngine* voice_engine, 198 talk_base::CpuMonitor* cpu_monitor); 199 bool SetDefaultCodec(const VideoCodec& codec); 200 bool RebuildCodecList(const VideoCodec& max_codec); 201 void SetTraceFilter(int filter); 202 void SetTraceOptions(const std::string& options); 203 bool InitVideoEngine(); 204 bool VerifyApt(const VideoCodec& in, int expected_apt) const; 205 206 // webrtc::TraceCallback implementation. 207 virtual void Print(webrtc::TraceLevel level, const char* trace, int length); 208 209 // WebRtcVideoEncoderFactory::Observer implementation. 210 virtual void OnCodecsAvailable(); 211 212 talk_base::Thread* worker_thread_; 213 talk_base::scoped_ptr<ViEWrapper> vie_wrapper_; 214 bool vie_wrapper_base_initialized_; 215 talk_base::scoped_ptr<ViETraceWrapper> tracing_; 216 WebRtcVoiceEngine* voice_engine_; 217 talk_base::scoped_ptr<webrtc::VideoRender> render_module_; 218 WebRtcVideoEncoderFactory* encoder_factory_; 219 WebRtcVideoDecoderFactory* decoder_factory_; 220 std::vector<VideoCodec> video_codecs_; 221 std::vector<RtpHeaderExtension> rtp_header_extensions_; 222 VideoFormat default_codec_format_; 223 224 bool initialized_; 225 talk_base::CriticalSection channels_crit_; 226 VideoChannels channels_; 227 228 bool capture_started_; 229 int local_renderer_w_; 230 int local_renderer_h_; 231 VideoRenderer* local_renderer_; 232 233 talk_base::scoped_ptr<talk_base::CpuMonitor> cpu_monitor_; 234 }; 235 236 class WebRtcVideoMediaChannel : public talk_base::MessageHandler, 237 public VideoMediaChannel, 238 public webrtc::Transport { 239 public: 240 WebRtcVideoMediaChannel(WebRtcVideoEngine* engine, 241 VoiceMediaChannel* voice_channel); 242 ~WebRtcVideoMediaChannel(); 243 bool Init(); 244 engine()245 WebRtcVideoEngine* engine() { return engine_; } voice_channel()246 VoiceMediaChannel* voice_channel() { return voice_channel_; } video_channel()247 int video_channel() const { return vie_channel_; } sending()248 bool sending() const { return sending_; } 249 250 // Public for testing purpose. 251 uint32 GetDefaultChannelSsrc(); 252 253 // VideoMediaChannel implementation 254 virtual bool SetRecvCodecs(const std::vector<VideoCodec> &codecs); 255 virtual bool SetSendCodecs(const std::vector<VideoCodec> &codecs); 256 virtual bool GetSendCodec(VideoCodec* send_codec); 257 virtual bool SetSendStreamFormat(uint32 ssrc, const VideoFormat& format); 258 virtual bool SetRender(bool render); 259 virtual bool SetSend(bool send); 260 261 virtual bool AddSendStream(const StreamParams& sp); 262 virtual bool RemoveSendStream(uint32 ssrc); 263 virtual bool AddRecvStream(const StreamParams& sp); 264 virtual bool RemoveRecvStream(uint32 ssrc); 265 virtual bool SetRenderer(uint32 ssrc, VideoRenderer* renderer); 266 virtual bool GetStats(const StatsOptions& options, VideoMediaInfo* info); 267 virtual bool SetCapturer(uint32 ssrc, VideoCapturer* capturer); 268 virtual bool SendIntraFrame(); 269 virtual bool RequestIntraFrame(); 270 271 virtual void OnPacketReceived(talk_base::Buffer* packet, 272 const talk_base::PacketTime& packet_time); 273 virtual void OnRtcpReceived(talk_base::Buffer* packet, 274 const talk_base::PacketTime& packet_time); 275 virtual void OnReadyToSend(bool ready); 276 virtual bool MuteStream(uint32 ssrc, bool on); 277 virtual bool SetRecvRtpHeaderExtensions( 278 const std::vector<RtpHeaderExtension>& extensions); 279 virtual bool SetSendRtpHeaderExtensions( 280 const std::vector<RtpHeaderExtension>& extensions); 281 virtual int GetRtpSendTimeExtnId() const; 282 virtual bool SetStartSendBandwidth(int bps); 283 virtual bool SetMaxSendBandwidth(int bps); 284 virtual bool SetOptions(const VideoOptions &options); GetOptions(VideoOptions * options)285 virtual bool GetOptions(VideoOptions *options) const { 286 *options = options_; 287 return true; 288 } 289 virtual void SetInterface(NetworkInterface* iface); 290 virtual void UpdateAspectRatio(int ratio_w, int ratio_h); 291 292 // Public functions for use by tests and other specialized code. send_ssrc()293 uint32 send_ssrc() const { return 0; } 294 bool GetRenderer(uint32 ssrc, VideoRenderer** renderer); 295 bool GetVideoAdapter(uint32 ssrc, CoordinatedVideoAdapter** video_adapter); 296 void SendFrame(VideoCapturer* capturer, const VideoFrame* frame); 297 bool SendFrame(WebRtcVideoChannelSendInfo* channel_info, 298 const VideoFrame* frame, bool is_screencast); 299 300 // Thunk functions for use with HybridVideoEngine OnLocalFrame(VideoCapturer * capturer,const VideoFrame * frame)301 void OnLocalFrame(VideoCapturer* capturer, const VideoFrame* frame) { 302 SendFrame(0u, frame, capturer->IsScreencast()); 303 } OnLocalFrameFormat(VideoCapturer * capturer,const VideoFormat * format)304 void OnLocalFrameFormat(VideoCapturer* capturer, const VideoFormat* format) { 305 } 306 307 virtual void OnMessage(talk_base::Message* msg); 308 309 protected: GetLastEngineError()310 int GetLastEngineError() { return engine()->GetLastEngineError(); } 311 virtual int SendPacket(int channel, const void* data, int len); 312 virtual int SendRTCPPacket(int channel, const void* data, int len); 313 314 private: 315 typedef std::map<uint32, WebRtcVideoChannelRecvInfo*> RecvChannelMap; 316 typedef std::map<uint32, WebRtcVideoChannelSendInfo*> SendChannelMap; 317 typedef std::map<uint32, uint32> SsrcMap; 318 typedef int (webrtc::ViERTP_RTCP::* ExtensionSetterFunction)(int, bool, int); 319 320 enum MediaDirection { MD_RECV, MD_SEND, MD_SENDRECV }; 321 322 // Creates and initializes a ViE channel. When successful |channel_id| will 323 // contain the new channel's ID. If |receiving| is true |ssrc| is the 324 // remote ssrc. If |sending| is true the ssrc is local ssrc. If both 325 // |receiving| and |sending| is true the ssrc must be 0 and the channel will 326 // be created as a default channel. The ssrc must be different for receive 327 // channels and it must be different for send channels. If the same SSRC is 328 // being used for creating channel more than once, this function will fail 329 // returning false. 330 bool CreateChannel(uint32 ssrc_key, MediaDirection direction, 331 int* channel_id); 332 bool CreateUnsignalledRecvChannel(uint32 ssrc_key, int* channel_id); 333 bool ConfigureChannel(int channel_id, MediaDirection direction, 334 uint32 ssrc_key); 335 bool ConfigureReceiving(int channel_id, uint32 remote_ssrc_key); 336 bool ConfigureSending(int channel_id, uint32 local_ssrc_key); 337 bool SetNackFec(int channel_id, int red_payload_type, int fec_payload_type, 338 bool nack_enabled); 339 bool SetSendCodec(const webrtc::VideoCodec& codec); 340 bool SetSendCodec(WebRtcVideoChannelSendInfo* send_channel, 341 const webrtc::VideoCodec& codec); 342 void LogSendCodecChange(const std::string& reason); 343 // Prepares the channel with channel id |info->channel_id()| to receive all 344 // codecs in |receive_codecs_| and start receive packets. 345 bool SetReceiveCodecs(WebRtcVideoChannelRecvInfo* info); 346 // Returns the channel number that receives the stream with SSRC |ssrc|. 347 int GetRecvChannelNum(uint32 ssrc); 348 bool MaybeSetRtxSsrc(const StreamParams& sp, int channel_id); 349 // Given captured video frame size, checks if we need to reset vie send codec. 350 // |reset| is set to whether resetting has happened on vie or not. 351 // Returns false on error. 352 bool MaybeResetVieSendCodec(WebRtcVideoChannelSendInfo* send_channel, 353 int new_width, int new_height, bool is_screencast, 354 bool* reset); 355 // Checks the current bitrate estimate and modifies the bitrates 356 // accordingly, including converting kAutoBandwidth to the correct defaults. 357 void MaybeChangeBitrates(int channel_id, webrtc::VideoCodec* video_codec); 358 // Helper function for starting the sending of media on all channels or 359 // |channel_id|. Note that these two function do not change |sending_|. 360 bool StartSend(); 361 bool StartSend(WebRtcVideoChannelSendInfo* send_channel); 362 // Helper function for stop the sending of media on all channels or 363 // |channel_id|. Note that these two function do not change |sending_|. 364 bool StopSend(); 365 bool StopSend(WebRtcVideoChannelSendInfo* send_channel); 366 bool SendIntraFrame(int channel_id); 367 368 bool HasReadySendChannels(); 369 370 // Send channel key returns the key corresponding to the provided local SSRC 371 // in |key|. The return value is true upon success. 372 // If the local ssrc correspond to that of the default channel the key is 0. 373 // For all other channels the returned key will be the same as the local ssrc. 374 bool GetSendChannelKey(uint32 local_ssrc, uint32* key); 375 WebRtcVideoChannelSendInfo* GetSendChannel(uint32 local_ssrc); 376 // Creates a new unique key that can be used for inserting a new send channel 377 // into |send_channels_| 378 bool CreateSendChannelKey(uint32 local_ssrc, uint32* key); 379 // Get the number of the send channels |capturer| registered with. 380 int GetSendChannelNum(VideoCapturer* capturer); 381 IsDefaultChannel(int channel_id)382 bool IsDefaultChannel(int channel_id) const { 383 return channel_id == vie_channel_; 384 } 385 386 bool DeleteSendChannel(uint32 ssrc_key); 387 InConferenceMode()388 bool InConferenceMode() const { 389 return options_.conference_mode.GetWithDefaultIfUnset(false); 390 } 391 bool RemoveCapturer(uint32 ssrc); 392 393 worker_thread()394 talk_base::MessageQueue* worker_thread() { return engine_->worker_thread(); } 395 void QueueBlackFrame(uint32 ssrc, int64 timestamp, int framerate); 396 void FlushBlackFrame(uint32 ssrc, int64 timestamp); 397 398 void SetNetworkTransmissionState(bool is_transmitting); 399 400 bool SetHeaderExtension(ExtensionSetterFunction setter, int channel_id, 401 const RtpHeaderExtension* extension); 402 bool SetHeaderExtension(ExtensionSetterFunction setter, int channel_id, 403 const std::vector<RtpHeaderExtension>& extensions, 404 const char header_extension_uri[]); 405 406 // Signal when cpu adaptation has no further scope to adapt. 407 void OnCpuAdaptationUnable(); 408 409 // Set the local (send-side) RTX SSRC corresponding to primary_ssrc. 410 bool SetLocalRtxSsrc(int channel_id, const StreamParams& send_params, 411 uint32 primary_ssrc, int stream_idx); 412 413 // Connect |capturer| to WebRtcVideoMediaChannel if it is only registered 414 // to one send channel, i.e. the first send channel. 415 void MaybeConnectCapturer(VideoCapturer* capturer); 416 // Disconnect |capturer| from WebRtcVideoMediaChannel if it is only registered 417 // to one send channel, i.e. the last send channel. 418 void MaybeDisconnectCapturer(VideoCapturer* capturer); 419 420 bool RemoveRecvStreamInternal(uint32 ssrc); 421 422 // Global state. 423 WebRtcVideoEngine* engine_; 424 VoiceMediaChannel* voice_channel_; 425 int vie_channel_; 426 bool nack_enabled_; 427 // Receiver Estimated Max Bitrate 428 bool remb_enabled_; 429 VideoOptions options_; 430 431 // Global recv side state. 432 // Note the default channel (vie_channel_), i.e. the send channel 433 // corresponding to all the receive channels (this must be done for REMB to 434 // work properly), resides in both recv_channels_ and send_channels_ with the 435 // ssrc key 0. 436 RecvChannelMap recv_channels_; // Contains all receive channels. 437 // A map from the SSRCs on which RTX packets are received to the media SSRCs 438 // the RTX packets are associated with. RTX packets will be delivered to the 439 // streams matching the primary SSRC. 440 SsrcMap rtx_to_primary_ssrc_; 441 std::vector<webrtc::VideoCodec> receive_codecs_; 442 // A map from codec payload types to their associated payload types, if any. 443 // TODO(holmer): This is a temporary solution until webrtc::VideoCodec has 444 // an associated payload type member, when it does we can rely on 445 // receive_codecs_. 446 std::map<int, int> associated_payload_types_; 447 bool render_started_; 448 uint32 first_receive_ssrc_; 449 std::vector<RtpHeaderExtension> receive_extensions_; 450 int num_unsignalled_recv_channels_; 451 452 // Global send side state. 453 SendChannelMap send_channels_; 454 talk_base::scoped_ptr<webrtc::VideoCodec> send_codec_; 455 int send_rtx_type_; 456 int send_red_type_; 457 int send_fec_type_; 458 bool sending_; 459 std::vector<RtpHeaderExtension> send_extensions_; 460 461 // The aspect ratio that the channel desires. 0 means there is no desired 462 // aspect ratio 463 int ratio_w_; 464 int ratio_h_; 465 }; 466 467 } // namespace cricket 468 469 #endif // TALK_MEDIA_WEBRTCVIDEOENGINE_H_ 470