1 /* 2 * libjingle 3 * Copyright 2014 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #ifndef TALK_MEDIA_WEBRTC_WEBRTCVIDEOENGINE2_H_ 29 #define TALK_MEDIA_WEBRTC_WEBRTCVIDEOENGINE2_H_ 30 31 #include <map> 32 #include <string> 33 #include <vector> 34 35 #include "talk/media/base/mediaengine.h" 36 #include "talk/media/webrtc/webrtcvideochannelfactory.h" 37 #include "talk/media/webrtc/webrtcvideodecoderfactory.h" 38 #include "talk/media/webrtc/webrtcvideoencoderfactory.h" 39 #include "webrtc/base/cpumonitor.h" 40 #include "webrtc/base/scoped_ptr.h" 41 #include "webrtc/base/thread_annotations.h" 42 #include "webrtc/common_video/interface/i420_video_frame.h" 43 #include "webrtc/transport.h" 44 #include "webrtc/video_receive_stream.h" 45 #include "webrtc/video_renderer.h" 46 #include "webrtc/video_send_stream.h" 47 48 namespace webrtc { 49 class Call; 50 class VideoCaptureModule; 51 class VideoDecoder; 52 class VideoEncoder; 53 class VideoRender; 54 class VideoSendStreamInput; 55 class VideoReceiveStream; 56 } 57 58 namespace rtc { 59 class CpuMonitor; 60 class Thread; 61 } // namespace rtc 62 63 namespace cricket { 64 65 class VideoCapturer; 66 class VideoFrame; 67 class VideoProcessor; 68 class VideoRenderer; 69 class VoiceMediaChannel; 70 class WebRtcDecoderObserver; 71 class WebRtcEncoderObserver; 72 class WebRtcLocalStreamInfo; 73 class WebRtcRenderAdapter; 74 class WebRtcVideoChannel2; 75 class WebRtcVideoChannelRecvInfo; 76 class WebRtcVideoChannelSendInfo; 77 class WebRtcVoiceEngine; 78 79 struct CapturedFrame; 80 struct Device; 81 82 class WebRtcVideoEngine2; 83 class WebRtcVideoChannel2; 84 class WebRtcVideoRenderer; 85 86 class UnsignalledSsrcHandler { 87 public: 88 enum Action { 89 kDropPacket, 90 kDeliverPacket, 91 }; 92 virtual Action OnUnsignalledSsrc(VideoMediaChannel* engine, 93 uint32_t ssrc) = 0; 94 }; 95 96 // TODO(pbos): Remove, use external handlers only. 97 class DefaultUnsignalledSsrcHandler : public UnsignalledSsrcHandler { 98 public: 99 DefaultUnsignalledSsrcHandler(); 100 virtual Action OnUnsignalledSsrc(VideoMediaChannel* engine, 101 uint32_t ssrc) OVERRIDE; 102 103 VideoRenderer* GetDefaultRenderer() const; 104 void SetDefaultRenderer(VideoMediaChannel* channel, VideoRenderer* renderer); 105 106 private: 107 uint32_t default_recv_ssrc_; 108 VideoRenderer* default_renderer_; 109 }; 110 111 class WebRtcVideoEncoderFactory2 { 112 public: 113 virtual ~WebRtcVideoEncoderFactory2(); 114 virtual std::vector<webrtc::VideoStream> CreateVideoStreams( 115 const VideoCodec& codec, 116 const VideoOptions& options, 117 size_t num_streams); 118 119 virtual webrtc::VideoEncoder* CreateVideoEncoder( 120 const VideoCodec& codec, 121 const VideoOptions& options); 122 123 virtual void* CreateVideoEncoderSettings(const VideoCodec& codec, 124 const VideoOptions& options); 125 126 virtual void DestroyVideoEncoderSettings(const VideoCodec& codec, 127 void* encoder_settings); 128 129 virtual bool SupportsCodec(const cricket::VideoCodec& codec); 130 }; 131 132 // WebRtcVideoEngine2 is used for the new native WebRTC Video API (webrtc:1667). 133 class WebRtcVideoEngine2 : public sigslot::has_slots<>, 134 public WebRtcVideoEncoderFactory::Observer { 135 public: 136 // Creates the WebRtcVideoEngine2 with internal VideoCaptureModule. 137 WebRtcVideoEngine2(); 138 virtual ~WebRtcVideoEngine2(); 139 140 // Use a custom WebRtcVideoChannelFactory (for testing purposes). 141 void SetChannelFactory(WebRtcVideoChannelFactory* channel_factory); 142 143 // Basic video engine implementation. 144 bool Init(rtc::Thread* worker_thread); 145 void Terminate(); 146 147 int GetCapabilities(); 148 bool SetDefaultEncoderConfig(const VideoEncoderConfig& config); 149 VideoEncoderConfig GetDefaultEncoderConfig() const; 150 151 WebRtcVideoChannel2* CreateChannel(VoiceMediaChannel* voice_channel); 152 153 const std::vector<VideoCodec>& codecs() const; 154 const std::vector<RtpHeaderExtension>& rtp_header_extensions() const; 155 void SetLogging(int min_sev, const char* filter); 156 157 // Set a WebRtcVideoDecoderFactory for external decoding. Video engine does 158 // not take the ownership of |decoder_factory|. The caller needs to make sure 159 // that |decoder_factory| outlives the video engine. 160 void SetExternalDecoderFactory(WebRtcVideoDecoderFactory* decoder_factory); 161 // Set a WebRtcVideoEncoderFactory for external encoding. Video engine does 162 // not take the ownership of |encoder_factory|. The caller needs to make sure 163 // that |encoder_factory| outlives the video engine. 164 virtual void SetExternalEncoderFactory( 165 WebRtcVideoEncoderFactory* encoder_factory); 166 167 bool EnableTimedRender(); 168 // This is currently ignored. 169 sigslot::repeater2<VideoCapturer*, CaptureState> SignalCaptureStateChange; 170 171 // Set the VoiceEngine for A/V sync. This can only be called before Init. 172 bool SetVoiceEngine(WebRtcVoiceEngine* voice_engine); 173 174 bool FindCodec(const VideoCodec& in); 175 bool CanSendCodec(const VideoCodec& in, 176 const VideoCodec& current, 177 VideoCodec* out); 178 // Check whether the supplied trace should be ignored. 179 bool ShouldIgnoreTrace(const std::string& trace); 180 GetStartCaptureFormat()181 VideoFormat GetStartCaptureFormat() const { return default_codec_format_; } 182 cpu_monitor()183 rtc::CpuMonitor* cpu_monitor() { return cpu_monitor_.get(); } 184 185 virtual WebRtcVideoEncoderFactory2* GetVideoEncoderFactory(); 186 187 private: 188 virtual void OnCodecsAvailable() OVERRIDE; 189 190 rtc::Thread* worker_thread_; 191 WebRtcVoiceEngine* voice_engine_; 192 std::vector<VideoCodec> video_codecs_; 193 std::vector<RtpHeaderExtension> rtp_header_extensions_; 194 VideoFormat default_codec_format_; 195 196 bool initialized_; 197 198 // Critical section to protect the media processor register/unregister 199 // while processing a frame 200 rtc::CriticalSection signal_media_critical_; 201 202 rtc::scoped_ptr<rtc::CpuMonitor> cpu_monitor_; 203 WebRtcVideoChannelFactory* channel_factory_; 204 WebRtcVideoEncoderFactory2 default_video_encoder_factory_; 205 206 WebRtcVideoDecoderFactory* external_decoder_factory_; 207 WebRtcVideoEncoderFactory* external_encoder_factory_; 208 }; 209 210 class WebRtcVideoChannel2 : public rtc::MessageHandler, 211 public VideoMediaChannel, 212 public webrtc::newapi::Transport { 213 public: 214 WebRtcVideoChannel2(WebRtcVideoEngine2* engine, 215 VoiceMediaChannel* voice_channel, 216 WebRtcVideoEncoderFactory2* encoder_factory); 217 // For testing purposes insert a pre-constructed call to verify that 218 // WebRtcVideoChannel2 calls the correct corresponding methods. 219 WebRtcVideoChannel2(webrtc::Call* call, 220 WebRtcVideoEngine2* engine, 221 WebRtcVideoEncoderFactory2* encoder_factory); 222 ~WebRtcVideoChannel2(); 223 bool Init(); 224 225 // VideoMediaChannel implementation 226 virtual bool SetRecvCodecs(const std::vector<VideoCodec>& codecs) OVERRIDE; 227 virtual bool SetSendCodecs(const std::vector<VideoCodec>& codecs) OVERRIDE; 228 virtual bool GetSendCodec(VideoCodec* send_codec) OVERRIDE; 229 virtual bool SetSendStreamFormat(uint32 ssrc, 230 const VideoFormat& format) OVERRIDE; 231 virtual bool SetRender(bool render) OVERRIDE; 232 virtual bool SetSend(bool send) OVERRIDE; 233 234 virtual bool AddSendStream(const StreamParams& sp) OVERRIDE; 235 virtual bool RemoveSendStream(uint32 ssrc) OVERRIDE; 236 virtual bool AddRecvStream(const StreamParams& sp) OVERRIDE; 237 virtual bool RemoveRecvStream(uint32 ssrc) OVERRIDE; 238 virtual bool SetRenderer(uint32 ssrc, VideoRenderer* renderer) OVERRIDE; 239 virtual bool GetStats(const StatsOptions& options, 240 VideoMediaInfo* info) OVERRIDE; 241 virtual bool SetCapturer(uint32 ssrc, VideoCapturer* capturer) OVERRIDE; 242 virtual bool SendIntraFrame() OVERRIDE; 243 virtual bool RequestIntraFrame() OVERRIDE; 244 245 virtual void OnPacketReceived(rtc::Buffer* packet, 246 const rtc::PacketTime& packet_time) 247 OVERRIDE; 248 virtual void OnRtcpReceived(rtc::Buffer* packet, 249 const rtc::PacketTime& packet_time) 250 OVERRIDE; 251 virtual void OnReadyToSend(bool ready) OVERRIDE; 252 virtual bool MuteStream(uint32 ssrc, bool mute) OVERRIDE; 253 254 // Set send/receive RTP header extensions. This must be done before creating 255 // streams as it only has effect on future streams. 256 virtual bool SetRecvRtpHeaderExtensions( 257 const std::vector<RtpHeaderExtension>& extensions) OVERRIDE; 258 virtual bool SetSendRtpHeaderExtensions( 259 const std::vector<RtpHeaderExtension>& extensions) OVERRIDE; 260 virtual bool SetStartSendBandwidth(int bps) OVERRIDE; 261 virtual bool SetMaxSendBandwidth(int bps) OVERRIDE; 262 virtual bool SetOptions(const VideoOptions& options) OVERRIDE; GetOptions(VideoOptions * options)263 virtual bool GetOptions(VideoOptions* options) const OVERRIDE { 264 *options = options_; 265 return true; 266 } 267 virtual void SetInterface(NetworkInterface* iface) OVERRIDE; 268 virtual void UpdateAspectRatio(int ratio_w, int ratio_h) OVERRIDE; 269 270 virtual void OnMessage(rtc::Message* msg) OVERRIDE; 271 272 // Implemented for VideoMediaChannelTest. sending()273 bool sending() const { return sending_; } GetDefaultSendChannelSsrc()274 uint32 GetDefaultSendChannelSsrc() { return default_send_ssrc_; } 275 bool GetRenderer(uint32 ssrc, VideoRenderer** renderer); 276 277 private: 278 void ConfigureReceiverRtp(webrtc::VideoReceiveStream::Config* config, 279 const StreamParams& sp) const; 280 281 struct VideoCodecSettings { 282 VideoCodecSettings(); 283 284 VideoCodec codec; 285 webrtc::FecConfig fec; 286 int rtx_payload_type; 287 }; 288 289 // Wrapper for the sender part, this is where the capturer is connected and 290 // frames are then converted from cricket frames to webrtc frames. 291 class WebRtcVideoSendStream : public sigslot::has_slots<> { 292 public: 293 WebRtcVideoSendStream( 294 webrtc::Call* call, 295 WebRtcVideoEncoderFactory2* encoder_factory, 296 const VideoOptions& options, 297 const Settable<VideoCodecSettings>& codec_settings, 298 const StreamParams& sp, 299 const std::vector<webrtc::RtpExtension>& rtp_extensions); 300 301 ~WebRtcVideoSendStream(); 302 void SetOptions(const VideoOptions& options); 303 void SetCodec(const VideoCodecSettings& codec); 304 void SetRtpExtensions( 305 const std::vector<webrtc::RtpExtension>& rtp_extensions); 306 307 void InputFrame(VideoCapturer* capturer, const VideoFrame* frame); 308 bool SetCapturer(VideoCapturer* capturer); 309 bool SetVideoFormat(const VideoFormat& format); 310 void MuteStream(bool mute); 311 bool DisconnectCapturer(); 312 313 void Start(); 314 void Stop(); 315 316 VideoSenderInfo GetVideoSenderInfo(); 317 318 private: 319 // Parameters needed to reconstruct the underlying stream. 320 // webrtc::VideoSendStream doesn't support setting a lot of options on the 321 // fly, so when those need to be changed we tear down and reconstruct with 322 // similar parameters depending on which options changed etc. 323 struct VideoSendStreamParameters { 324 VideoSendStreamParameters( 325 const webrtc::VideoSendStream::Config& config, 326 const VideoOptions& options, 327 const Settable<VideoCodecSettings>& codec_settings); 328 webrtc::VideoSendStream::Config config; 329 VideoOptions options; 330 Settable<VideoCodecSettings> codec_settings; 331 // Sent resolutions + bitrates etc. by the underlying VideoSendStream, 332 // typically changes when setting a new resolution or reconfiguring 333 // bitrates. 334 webrtc::VideoEncoderConfig encoder_config; 335 }; 336 337 void SetCodecAndOptions(const VideoCodecSettings& codec, 338 const VideoOptions& options) 339 EXCLUSIVE_LOCKS_REQUIRED(lock_); 340 void RecreateWebRtcStream() EXCLUSIVE_LOCKS_REQUIRED(lock_); 341 // When |override_max| is false constrain width/height to codec dimensions. 342 void SetDimensions(int width, int height, bool override_max) 343 EXCLUSIVE_LOCKS_REQUIRED(lock_); 344 345 webrtc::Call* const call_; 346 WebRtcVideoEncoderFactory2* const encoder_factory_; 347 348 rtc::CriticalSection lock_; 349 webrtc::VideoSendStream* stream_ GUARDED_BY(lock_); 350 VideoSendStreamParameters parameters_ GUARDED_BY(lock_); 351 352 VideoCapturer* capturer_ GUARDED_BY(lock_); 353 bool sending_ GUARDED_BY(lock_); 354 bool muted_ GUARDED_BY(lock_); 355 VideoFormat format_ GUARDED_BY(lock_); 356 357 rtc::CriticalSection frame_lock_; 358 webrtc::I420VideoFrame video_frame_ GUARDED_BY(frame_lock_); 359 }; 360 361 // Wrapper for the receiver part, contains configs etc. that are needed to 362 // reconstruct the underlying VideoReceiveStream. Also serves as a wrapper 363 // between webrtc::VideoRenderer and cricket::VideoRenderer. 364 class WebRtcVideoReceiveStream : public webrtc::VideoRenderer { 365 public: 366 WebRtcVideoReceiveStream( 367 webrtc::Call*, 368 const webrtc::VideoReceiveStream::Config& config, 369 const std::vector<VideoCodecSettings>& recv_codecs); 370 ~WebRtcVideoReceiveStream(); 371 372 void SetRecvCodecs(const std::vector<VideoCodecSettings>& recv_codecs); 373 void SetRtpExtensions(const std::vector<webrtc::RtpExtension>& extensions); 374 375 virtual void RenderFrame(const webrtc::I420VideoFrame& frame, 376 int time_to_render_ms) OVERRIDE; 377 378 void SetRenderer(cricket::VideoRenderer* renderer); 379 cricket::VideoRenderer* GetRenderer(); 380 381 VideoReceiverInfo GetVideoReceiverInfo(); 382 383 private: 384 void SetSize(int width, int height); 385 void RecreateWebRtcStream(); 386 387 webrtc::Call* const call_; 388 389 webrtc::VideoReceiveStream* stream_; 390 webrtc::VideoReceiveStream::Config config_; 391 392 rtc::CriticalSection renderer_lock_; 393 cricket::VideoRenderer* renderer_ GUARDED_BY(renderer_lock_); 394 int last_width_ GUARDED_BY(renderer_lock_); 395 int last_height_ GUARDED_BY(renderer_lock_); 396 }; 397 398 void Construct(webrtc::Call* call, WebRtcVideoEngine2* engine); 399 void SetDefaultOptions(); 400 401 virtual bool SendRtp(const uint8_t* data, size_t len) OVERRIDE; 402 virtual bool SendRtcp(const uint8_t* data, size_t len) OVERRIDE; 403 404 void StartAllSendStreams(); 405 void StopAllSendStreams(); 406 407 static std::vector<VideoCodecSettings> MapCodecs( 408 const std::vector<VideoCodec>& codecs); 409 std::vector<VideoCodecSettings> FilterSupportedCodecs( 410 const std::vector<VideoCodecSettings>& mapped_codecs); 411 412 void FillSenderStats(VideoMediaInfo* info); 413 void FillReceiverStats(VideoMediaInfo* info); 414 void FillBandwidthEstimationStats(VideoMediaInfo* info); 415 416 uint32_t rtcp_receiver_report_ssrc_; 417 bool sending_; 418 rtc::scoped_ptr<webrtc::Call> call_; 419 uint32_t default_send_ssrc_; 420 421 DefaultUnsignalledSsrcHandler default_unsignalled_ssrc_handler_; 422 UnsignalledSsrcHandler* const unsignalled_ssrc_handler_; 423 424 // Using primary-ssrc (first ssrc) as key. 425 std::map<uint32, WebRtcVideoSendStream*> send_streams_; 426 std::map<uint32, WebRtcVideoReceiveStream*> receive_streams_; 427 428 Settable<VideoCodecSettings> send_codec_; 429 std::vector<webrtc::RtpExtension> send_rtp_extensions_; 430 431 WebRtcVideoEncoderFactory2* const encoder_factory_; 432 std::vector<VideoCodecSettings> recv_codecs_; 433 std::vector<webrtc::RtpExtension> recv_rtp_extensions_; 434 VideoOptions options_; 435 }; 436 437 } // namespace cricket 438 439 #endif // TALK_MEDIA_WEBRTC_WEBRTCVIDEOENGINE2_H_ 440