1 /* 2 * libjingle 3 * Copyright 2014 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #ifndef TALK_MEDIA_WEBRTC_WEBRTCVIDEOENGINE2_H_ 29 #define TALK_MEDIA_WEBRTC_WEBRTCVIDEOENGINE2_H_ 30 31 #include <map> 32 #include <string> 33 #include <vector> 34 35 #include "talk/media/base/mediaengine.h" 36 #include "talk/media/webrtc/webrtcvideochannelfactory.h" 37 #include "talk/media/webrtc/webrtcvideodecoderfactory.h" 38 #include "talk/media/webrtc/webrtcvideoencoderfactory.h" 39 #include "webrtc/base/criticalsection.h" 40 #include "webrtc/base/scoped_ptr.h" 41 #include "webrtc/base/thread_annotations.h" 42 #include "webrtc/base/thread_checker.h" 43 #include "webrtc/call.h" 44 #include "webrtc/transport.h" 45 #include "webrtc/video_frame.h" 46 #include "webrtc/video_receive_stream.h" 47 #include "webrtc/video_renderer.h" 48 #include "webrtc/video_send_stream.h" 49 50 namespace webrtc { 51 class VideoDecoder; 52 class VideoEncoder; 53 } 54 55 namespace rtc { 56 class Thread; 57 } // namespace rtc 58 59 namespace cricket { 60 61 class VideoCapturer; 62 class VideoFrame; 63 class VideoProcessor; 64 class VideoRenderer; 65 class VoiceMediaChannel; 66 class WebRtcDecoderObserver; 67 class WebRtcEncoderObserver; 68 class WebRtcLocalStreamInfo; 69 class WebRtcRenderAdapter; 70 class WebRtcVideoChannelRecvInfo; 71 class WebRtcVideoChannelSendInfo; 72 class WebRtcVoiceEngine; 73 class WebRtcVoiceMediaChannel; 74 75 struct CapturedFrame; 76 struct Device; 77 78 // Exposed here for unittests. 79 std::vector<VideoCodec> DefaultVideoCodecList(); 80 81 class UnsignalledSsrcHandler { 82 public: 83 enum Action { 84 kDropPacket, 85 kDeliverPacket, 86 }; 87 virtual Action OnUnsignalledSsrc(WebRtcVideoChannel2* channel, 88 uint32_t ssrc) = 0; 89 }; 90 91 // TODO(pbos): Remove, use external handlers only. 92 class DefaultUnsignalledSsrcHandler : public UnsignalledSsrcHandler { 93 public: 94 DefaultUnsignalledSsrcHandler(); 95 Action OnUnsignalledSsrc(WebRtcVideoChannel2* channel, 96 uint32_t ssrc) override; 97 98 VideoRenderer* GetDefaultRenderer() const; 99 void SetDefaultRenderer(VideoMediaChannel* channel, VideoRenderer* renderer); 100 101 private: 102 uint32_t default_recv_ssrc_; 103 VideoRenderer* default_renderer_; 104 }; 105 106 // WebRtcVideoEngine2 is used for the new native WebRTC Video API (webrtc:1667). 107 class WebRtcVideoEngine2 { 108 public: 109 WebRtcVideoEngine2(); 110 ~WebRtcVideoEngine2(); 111 112 // Basic video engine implementation. 113 void Init(); 114 115 WebRtcVideoChannel2* CreateChannel(webrtc::Call* call, 116 const VideoOptions& options); 117 118 const std::vector<VideoCodec>& codecs() const; 119 RtpCapabilities GetCapabilities() const; 120 121 // Set a WebRtcVideoDecoderFactory for external decoding. Video engine does 122 // not take the ownership of |decoder_factory|. The caller needs to make sure 123 // that |decoder_factory| outlives the video engine. 124 void SetExternalDecoderFactory(WebRtcVideoDecoderFactory* decoder_factory); 125 // Set a WebRtcVideoEncoderFactory for external encoding. Video engine does 126 // not take the ownership of |encoder_factory|. The caller needs to make sure 127 // that |encoder_factory| outlives the video engine. 128 virtual void SetExternalEncoderFactory( 129 WebRtcVideoEncoderFactory* encoder_factory); 130 131 bool EnableTimedRender(); 132 133 bool FindCodec(const VideoCodec& in); 134 // Check whether the supplied trace should be ignored. 135 bool ShouldIgnoreTrace(const std::string& trace); 136 137 private: 138 std::vector<VideoCodec> GetSupportedCodecs() const; 139 140 std::vector<VideoCodec> video_codecs_; 141 142 bool initialized_; 143 144 WebRtcVideoDecoderFactory* external_decoder_factory_; 145 WebRtcVideoEncoderFactory* external_encoder_factory_; 146 rtc::scoped_ptr<WebRtcVideoEncoderFactory> simulcast_encoder_factory_; 147 }; 148 149 class WebRtcVideoChannel2 : public rtc::MessageHandler, 150 public VideoMediaChannel, 151 public webrtc::Transport, 152 public webrtc::LoadObserver { 153 public: 154 WebRtcVideoChannel2(webrtc::Call* call, 155 const VideoOptions& options, 156 const std::vector<VideoCodec>& recv_codecs, 157 WebRtcVideoEncoderFactory* external_encoder_factory, 158 WebRtcVideoDecoderFactory* external_decoder_factory); 159 ~WebRtcVideoChannel2() override; 160 161 // VideoMediaChannel implementation 162 bool SetSendParameters(const VideoSendParameters& params) override; 163 bool SetRecvParameters(const VideoRecvParameters& params) override; 164 bool GetSendCodec(VideoCodec* send_codec) override; 165 bool SetSendStreamFormat(uint32_t ssrc, const VideoFormat& format) override; 166 bool SetSend(bool send) override; 167 bool SetVideoSend(uint32_t ssrc, 168 bool mute, 169 const VideoOptions* options) override; 170 bool AddSendStream(const StreamParams& sp) override; 171 bool RemoveSendStream(uint32_t ssrc) override; 172 bool AddRecvStream(const StreamParams& sp) override; 173 bool AddRecvStream(const StreamParams& sp, bool default_stream); 174 bool RemoveRecvStream(uint32_t ssrc) override; 175 bool SetRenderer(uint32_t ssrc, VideoRenderer* renderer) override; 176 bool GetStats(VideoMediaInfo* info) override; 177 bool SetCapturer(uint32_t ssrc, VideoCapturer* capturer) override; 178 bool SendIntraFrame() override; 179 bool RequestIntraFrame() override; 180 181 void OnPacketReceived(rtc::Buffer* packet, 182 const rtc::PacketTime& packet_time) override; 183 void OnRtcpReceived(rtc::Buffer* packet, 184 const rtc::PacketTime& packet_time) override; 185 void OnReadyToSend(bool ready) override; 186 void SetInterface(NetworkInterface* iface) override; 187 void UpdateAspectRatio(int ratio_w, int ratio_h) override; 188 189 void OnMessage(rtc::Message* msg) override; 190 191 void OnLoadUpdate(Load load) override; 192 193 // Implemented for VideoMediaChannelTest. sending()194 bool sending() const { return sending_; } GetDefaultSendChannelSsrc()195 uint32_t GetDefaultSendChannelSsrc() { return default_send_ssrc_; } 196 bool GetRenderer(uint32_t ssrc, VideoRenderer** renderer); 197 198 private: 199 bool MuteStream(uint32_t ssrc, bool mute); 200 class WebRtcVideoReceiveStream; 201 202 bool SetSendCodecs(const std::vector<VideoCodec>& codecs); 203 bool SetSendRtpHeaderExtensions( 204 const std::vector<RtpHeaderExtension>& extensions); 205 bool SetMaxSendBandwidth(int bps); 206 bool SetOptions(const VideoOptions& options); 207 bool SetRecvCodecs(const std::vector<VideoCodec>& codecs); 208 bool SetRecvRtpHeaderExtensions( 209 const std::vector<RtpHeaderExtension>& extensions); 210 211 void ConfigureReceiverRtp(webrtc::VideoReceiveStream::Config* config, 212 const StreamParams& sp) const; 213 bool CodecIsExternallySupported(const std::string& name) const; 214 bool ValidateSendSsrcAvailability(const StreamParams& sp) const 215 EXCLUSIVE_LOCKS_REQUIRED(stream_crit_); 216 bool ValidateReceiveSsrcAvailability(const StreamParams& sp) const 217 EXCLUSIVE_LOCKS_REQUIRED(stream_crit_); 218 void DeleteReceiveStream(WebRtcVideoReceiveStream* stream) 219 EXCLUSIVE_LOCKS_REQUIRED(stream_crit_); 220 221 struct VideoCodecSettings { 222 VideoCodecSettings(); 223 224 bool operator==(const VideoCodecSettings& other) const; 225 bool operator!=(const VideoCodecSettings& other) const; 226 227 VideoCodec codec; 228 webrtc::FecConfig fec; 229 int rtx_payload_type; 230 }; 231 232 static std::string CodecSettingsVectorToString( 233 const std::vector<VideoCodecSettings>& codecs); 234 235 // Wrapper for the sender part, this is where the capturer is connected and 236 // frames are then converted from cricket frames to webrtc frames. 237 class WebRtcVideoSendStream : public sigslot::has_slots<> { 238 public: 239 WebRtcVideoSendStream( 240 webrtc::Call* call, 241 const StreamParams& sp, 242 const webrtc::VideoSendStream::Config& config, 243 WebRtcVideoEncoderFactory* external_encoder_factory, 244 const VideoOptions& options, 245 int max_bitrate_bps, 246 const rtc::Optional<VideoCodecSettings>& codec_settings, 247 const std::vector<webrtc::RtpExtension>& rtp_extensions, 248 const VideoSendParameters& send_params); 249 ~WebRtcVideoSendStream(); 250 251 void SetOptions(const VideoOptions& options); 252 void SetCodec(const VideoCodecSettings& codec); 253 void SetRtpExtensions( 254 const std::vector<webrtc::RtpExtension>& rtp_extensions); 255 // TODO(deadbeef): Move logic from SetCodec/SetRtpExtensions/etc. 256 // into this method. Currently this method only sets the RTCP mode. 257 void SetSendParameters(const VideoSendParameters& send_params); 258 259 void InputFrame(VideoCapturer* capturer, const VideoFrame* frame); 260 bool SetCapturer(VideoCapturer* capturer); 261 bool SetVideoFormat(const VideoFormat& format); 262 void MuteStream(bool mute); 263 bool DisconnectCapturer(); 264 265 void SetApplyRotation(bool apply_rotation); 266 267 void Start(); 268 void Stop(); 269 270 const std::vector<uint32_t>& GetSsrcs() const; 271 VideoSenderInfo GetVideoSenderInfo(); 272 void FillBandwidthEstimationInfo(BandwidthEstimationInfo* bwe_info); 273 274 void SetMaxBitrateBps(int max_bitrate_bps); 275 276 private: 277 // Parameters needed to reconstruct the underlying stream. 278 // webrtc::VideoSendStream doesn't support setting a lot of options on the 279 // fly, so when those need to be changed we tear down and reconstruct with 280 // similar parameters depending on which options changed etc. 281 struct VideoSendStreamParameters { 282 VideoSendStreamParameters( 283 const webrtc::VideoSendStream::Config& config, 284 const VideoOptions& options, 285 int max_bitrate_bps, 286 const rtc::Optional<VideoCodecSettings>& codec_settings); 287 webrtc::VideoSendStream::Config config; 288 VideoOptions options; 289 int max_bitrate_bps; 290 rtc::Optional<VideoCodecSettings> codec_settings; 291 // Sent resolutions + bitrates etc. by the underlying VideoSendStream, 292 // typically changes when setting a new resolution or reconfiguring 293 // bitrates. 294 webrtc::VideoEncoderConfig encoder_config; 295 }; 296 297 struct AllocatedEncoder { 298 AllocatedEncoder(webrtc::VideoEncoder* encoder, 299 webrtc::VideoCodecType type, 300 bool external); 301 webrtc::VideoEncoder* encoder; 302 webrtc::VideoEncoder* external_encoder; 303 webrtc::VideoCodecType type; 304 bool external; 305 }; 306 307 struct Dimensions { 308 // Initial encoder configuration (QCIF, 176x144) frame (to ensure that 309 // hardware encoders can be initialized). This gives us low memory usage 310 // but also makes it so configuration errors are discovered at the time we 311 // apply the settings rather than when we get the first frame (waiting for 312 // the first frame to know that you gave a bad codec parameter could make 313 // debugging hard). 314 // TODO(pbos): Consider setting up encoders lazily. DimensionsDimensions315 Dimensions() : width(176), height(144), is_screencast(false) {} 316 int width; 317 int height; 318 bool is_screencast; 319 }; 320 321 union VideoEncoderSettings { 322 webrtc::VideoCodecVP8 vp8; 323 webrtc::VideoCodecVP9 vp9; 324 }; 325 326 static std::vector<webrtc::VideoStream> CreateVideoStreams( 327 const VideoCodec& codec, 328 const VideoOptions& options, 329 int max_bitrate_bps, 330 size_t num_streams); 331 static std::vector<webrtc::VideoStream> CreateSimulcastVideoStreams( 332 const VideoCodec& codec, 333 const VideoOptions& options, 334 int max_bitrate_bps, 335 size_t num_streams); 336 337 void* ConfigureVideoEncoderSettings(const VideoCodec& codec, 338 const VideoOptions& options, 339 bool is_screencast) 340 EXCLUSIVE_LOCKS_REQUIRED(lock_); 341 342 AllocatedEncoder CreateVideoEncoder(const VideoCodec& codec) 343 EXCLUSIVE_LOCKS_REQUIRED(lock_); 344 void DestroyVideoEncoder(AllocatedEncoder* encoder) 345 EXCLUSIVE_LOCKS_REQUIRED(lock_); 346 void SetCodecAndOptions(const VideoCodecSettings& codec, 347 const VideoOptions& options) 348 EXCLUSIVE_LOCKS_REQUIRED(lock_); 349 void RecreateWebRtcStream() EXCLUSIVE_LOCKS_REQUIRED(lock_); 350 webrtc::VideoEncoderConfig CreateVideoEncoderConfig( 351 const Dimensions& dimensions, 352 const VideoCodec& codec) const EXCLUSIVE_LOCKS_REQUIRED(lock_); 353 void SetDimensions(int width, int height, bool is_screencast) 354 EXCLUSIVE_LOCKS_REQUIRED(lock_); 355 356 const std::vector<uint32_t> ssrcs_; 357 const std::vector<SsrcGroup> ssrc_groups_; 358 webrtc::Call* const call_; 359 WebRtcVideoEncoderFactory* const external_encoder_factory_ 360 GUARDED_BY(lock_); 361 362 rtc::CriticalSection lock_; 363 webrtc::VideoSendStream* stream_ GUARDED_BY(lock_); 364 VideoSendStreamParameters parameters_ GUARDED_BY(lock_); 365 VideoEncoderSettings encoder_settings_ GUARDED_BY(lock_); 366 AllocatedEncoder allocated_encoder_ GUARDED_BY(lock_); 367 Dimensions last_dimensions_ GUARDED_BY(lock_); 368 369 VideoCapturer* capturer_ GUARDED_BY(lock_); 370 bool sending_ GUARDED_BY(lock_); 371 bool muted_ GUARDED_BY(lock_); 372 VideoFormat format_ GUARDED_BY(lock_); 373 int old_adapt_changes_ GUARDED_BY(lock_); 374 375 // The timestamp of the first frame received 376 // Used to generate the timestamps of subsequent frames 377 int64_t first_frame_timestamp_ms_ GUARDED_BY(lock_); 378 379 // The timestamp of the last frame received 380 // Used to generate timestamp for the black frame when capturer is removed 381 int64_t last_frame_timestamp_ms_ GUARDED_BY(lock_); 382 }; 383 384 // Wrapper for the receiver part, contains configs etc. that are needed to 385 // reconstruct the underlying VideoReceiveStream. Also serves as a wrapper 386 // between webrtc::VideoRenderer and cricket::VideoRenderer. 387 class WebRtcVideoReceiveStream : public webrtc::VideoRenderer { 388 public: 389 WebRtcVideoReceiveStream( 390 webrtc::Call* call, 391 const StreamParams& sp, 392 const webrtc::VideoReceiveStream::Config& config, 393 WebRtcVideoDecoderFactory* external_decoder_factory, 394 bool default_stream, 395 const std::vector<VideoCodecSettings>& recv_codecs, 396 bool disable_prerenderer_smoothing); 397 ~WebRtcVideoReceiveStream(); 398 399 const std::vector<uint32_t>& GetSsrcs() const; 400 401 void SetLocalSsrc(uint32_t local_ssrc); 402 void SetFeedbackParameters(bool nack_enabled, 403 bool remb_enabled, 404 bool transport_cc_enabled); 405 void SetRecvCodecs(const std::vector<VideoCodecSettings>& recv_codecs); 406 void SetRtpExtensions(const std::vector<webrtc::RtpExtension>& extensions); 407 // TODO(deadbeef): Move logic from SetRecvCodecs/SetRtpExtensions/etc. 408 // into this method. Currently this method only sets the RTCP mode. 409 void SetRecvParameters(const VideoRecvParameters& recv_params); 410 411 void RenderFrame(const webrtc::VideoFrame& frame, 412 int time_to_render_ms) override; 413 bool IsTextureSupported() const override; 414 bool SmoothsRenderedFrames() const override; 415 bool IsDefaultStream() const; 416 417 void SetRenderer(cricket::VideoRenderer* renderer); 418 cricket::VideoRenderer* GetRenderer(); 419 420 VideoReceiverInfo GetVideoReceiverInfo(); 421 422 private: 423 struct AllocatedDecoder { 424 AllocatedDecoder(webrtc::VideoDecoder* decoder, 425 webrtc::VideoCodecType type, 426 bool external); 427 webrtc::VideoDecoder* decoder; 428 // Decoder wrapped into a fallback decoder to permit software fallback. 429 webrtc::VideoDecoder* external_decoder; 430 webrtc::VideoCodecType type; 431 bool external; 432 }; 433 434 void SetSize(int width, int height); 435 void RecreateWebRtcStream(); 436 437 AllocatedDecoder CreateOrReuseVideoDecoder( 438 std::vector<AllocatedDecoder>* old_decoder, 439 const VideoCodec& codec); 440 void ClearDecoders(std::vector<AllocatedDecoder>* allocated_decoders); 441 442 std::string GetCodecNameFromPayloadType(int payload_type); 443 444 webrtc::Call* const call_; 445 const std::vector<uint32_t> ssrcs_; 446 const std::vector<SsrcGroup> ssrc_groups_; 447 448 webrtc::VideoReceiveStream* stream_; 449 const bool default_stream_; 450 webrtc::VideoReceiveStream::Config config_; 451 452 WebRtcVideoDecoderFactory* const external_decoder_factory_; 453 std::vector<AllocatedDecoder> allocated_decoders_; 454 455 const bool disable_prerenderer_smoothing_; 456 457 rtc::CriticalSection renderer_lock_; 458 cricket::VideoRenderer* renderer_ GUARDED_BY(renderer_lock_); 459 int last_width_ GUARDED_BY(renderer_lock_); 460 int last_height_ GUARDED_BY(renderer_lock_); 461 // Expands remote RTP timestamps to int64_t to be able to estimate how long 462 // the stream has been running. 463 rtc::TimestampWrapAroundHandler timestamp_wraparound_handler_ 464 GUARDED_BY(renderer_lock_); 465 int64_t first_frame_timestamp_ GUARDED_BY(renderer_lock_); 466 // Start NTP time is estimated as current remote NTP time (estimated from 467 // RTCP) minus the elapsed time, as soon as remote NTP time is available. 468 int64_t estimated_remote_start_ntp_time_ms_ GUARDED_BY(renderer_lock_); 469 }; 470 471 void Construct(webrtc::Call* call, WebRtcVideoEngine2* engine); 472 void SetDefaultOptions(); 473 474 bool SendRtp(const uint8_t* data, 475 size_t len, 476 const webrtc::PacketOptions& options) override; 477 bool SendRtcp(const uint8_t* data, size_t len) override; 478 479 void StartAllSendStreams(); 480 void StopAllSendStreams(); 481 482 static std::vector<VideoCodecSettings> MapCodecs( 483 const std::vector<VideoCodec>& codecs); 484 std::vector<VideoCodecSettings> FilterSupportedCodecs( 485 const std::vector<VideoCodecSettings>& mapped_codecs) const; 486 static bool ReceiveCodecsHaveChanged(std::vector<VideoCodecSettings> before, 487 std::vector<VideoCodecSettings> after); 488 489 void FillSenderStats(VideoMediaInfo* info); 490 void FillReceiverStats(VideoMediaInfo* info); 491 void FillBandwidthEstimationStats(const webrtc::Call::Stats& stats, 492 VideoMediaInfo* info); 493 494 rtc::ThreadChecker thread_checker_; 495 496 uint32_t rtcp_receiver_report_ssrc_; 497 bool sending_; 498 webrtc::Call* const call_; 499 500 uint32_t default_send_ssrc_; 501 502 DefaultUnsignalledSsrcHandler default_unsignalled_ssrc_handler_; 503 UnsignalledSsrcHandler* const unsignalled_ssrc_handler_; 504 505 // Separate list of set capturers used to signal CPU adaptation. These should 506 // not be locked while calling methods that take other locks to prevent 507 // lock-order inversions. 508 rtc::CriticalSection capturer_crit_; 509 bool signal_cpu_adaptation_ GUARDED_BY(capturer_crit_); 510 std::map<uint32_t, VideoCapturer*> capturers_ GUARDED_BY(capturer_crit_); 511 512 rtc::CriticalSection stream_crit_; 513 // Using primary-ssrc (first ssrc) as key. 514 std::map<uint32_t, WebRtcVideoSendStream*> send_streams_ 515 GUARDED_BY(stream_crit_); 516 std::map<uint32_t, WebRtcVideoReceiveStream*> receive_streams_ 517 GUARDED_BY(stream_crit_); 518 std::set<uint32_t> send_ssrcs_ GUARDED_BY(stream_crit_); 519 std::set<uint32_t> receive_ssrcs_ GUARDED_BY(stream_crit_); 520 521 rtc::Optional<VideoCodecSettings> send_codec_; 522 std::vector<webrtc::RtpExtension> send_rtp_extensions_; 523 524 WebRtcVideoEncoderFactory* const external_encoder_factory_; 525 WebRtcVideoDecoderFactory* const external_decoder_factory_; 526 std::vector<VideoCodecSettings> recv_codecs_; 527 std::vector<webrtc::RtpExtension> recv_rtp_extensions_; 528 webrtc::Call::Config::BitrateConfig bitrate_config_; 529 VideoOptions options_; 530 // TODO(deadbeef): Don't duplicate information between 531 // send_params/recv_params, rtp_extensions, options, etc. 532 VideoSendParameters send_params_; 533 VideoRecvParameters recv_params_; 534 }; 535 536 } // namespace cricket 537 538 #endif // TALK_MEDIA_WEBRTC_WEBRTCVIDEOENGINE2_H_ 539