1 /* 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_ 12 #define MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_ 13 14 #include <map> 15 #include <memory> 16 #include <set> 17 #include <string> 18 #include <vector> 19 20 #include "absl/types/optional.h" 21 #include "api/call/transport.h" 22 #include "api/video/video_bitrate_allocator_factory.h" 23 #include "api/video/video_frame.h" 24 #include "api/video/video_sink_interface.h" 25 #include "api/video/video_source_interface.h" 26 #include "api/video_codecs/sdp_video_format.h" 27 #include "call/call.h" 28 #include "call/flexfec_receive_stream.h" 29 #include "call/video_receive_stream.h" 30 #include "call/video_send_stream.h" 31 #include "media/base/media_engine.h" 32 #include "media/engine/constants.h" 33 #include "media/engine/unhandled_packets_buffer.h" 34 #include "rtc_base/async_invoker.h" 35 #include "rtc_base/network_route.h" 36 #include "rtc_base/synchronization/mutex.h" 37 #include "rtc_base/thread_annotations.h" 38 #include "rtc_base/thread_checker.h" 39 40 namespace webrtc { 41 class VideoDecoderFactory; 42 class VideoEncoderFactory; 43 struct MediaConfig; 44 } // namespace webrtc 45 46 namespace rtc { 47 class Thread; 48 } // namespace rtc 49 50 namespace cricket { 51 52 class WebRtcVideoChannel; 53 54 // Public for testing. 55 // Inputs StreamStats for all types of substreams (kMedia, kRtx, kFlexfec) and 56 // merges any non-kMedia substream stats object into its referenced kMedia-type 57 // substream. The resulting substreams are all kMedia. This means, for example, 58 // that packet and byte counters of RTX and FlexFEC streams are accounted for in 59 // the relevant RTP media stream's stats. This makes the resulting StreamStats 60 // objects ready to be turned into "outbound-rtp" stats objects for GetStats() 61 // which does not create separate stream stats objects for complementary 62 // streams. 63 std::map<uint32_t, webrtc::VideoSendStream::StreamStats> 64 MergeInfoAboutOutboundRtpSubstreamsForTesting( 65 const std::map<uint32_t, webrtc::VideoSendStream::StreamStats>& substreams); 66 67 class UnsignalledSsrcHandler { 68 public: 69 enum Action { 70 kDropPacket, 71 kDeliverPacket, 72 }; 73 virtual Action OnUnsignalledSsrc(WebRtcVideoChannel* channel, 74 uint32_t ssrc) = 0; 75 virtual ~UnsignalledSsrcHandler() = default; 76 }; 77 78 // TODO(pbos): Remove, use external handlers only. 79 class DefaultUnsignalledSsrcHandler : public UnsignalledSsrcHandler { 80 public: 81 DefaultUnsignalledSsrcHandler(); 82 Action OnUnsignalledSsrc(WebRtcVideoChannel* channel, uint32_t ssrc) override; 83 84 rtc::VideoSinkInterface<webrtc::VideoFrame>* GetDefaultSink() const; 85 void SetDefaultSink(WebRtcVideoChannel* channel, 86 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink); 87 88 virtual ~DefaultUnsignalledSsrcHandler() = default; 89 90 private: 91 rtc::VideoSinkInterface<webrtc::VideoFrame>* default_sink_; 92 }; 93 94 // WebRtcVideoEngine is used for the new native WebRTC Video API (webrtc:1667). 95 class WebRtcVideoEngine : public VideoEngineInterface { 96 public: 97 // These video codec factories represents all video codecs, i.e. both software 98 // and external hardware codecs. 99 WebRtcVideoEngine( 100 std::unique_ptr<webrtc::VideoEncoderFactory> video_encoder_factory, 101 std::unique_ptr<webrtc::VideoDecoderFactory> video_decoder_factory); 102 103 ~WebRtcVideoEngine() override; 104 105 VideoMediaChannel* CreateMediaChannel( 106 webrtc::Call* call, 107 const MediaConfig& config, 108 const VideoOptions& options, 109 const webrtc::CryptoOptions& crypto_options, 110 webrtc::VideoBitrateAllocatorFactory* video_bitrate_allocator_factory) 111 override; 112 113 std::vector<VideoCodec> send_codecs() const override; 114 std::vector<VideoCodec> recv_codecs() const override; 115 std::vector<webrtc::RtpHeaderExtensionCapability> GetRtpHeaderExtensions() 116 const override; 117 118 private: 119 const std::unique_ptr<webrtc::VideoDecoderFactory> decoder_factory_; 120 const std::unique_ptr<webrtc::VideoEncoderFactory> encoder_factory_; 121 const std::unique_ptr<webrtc::VideoBitrateAllocatorFactory> 122 bitrate_allocator_factory_; 123 }; 124 125 class WebRtcVideoChannel : public VideoMediaChannel, 126 public webrtc::Transport, 127 public webrtc::EncoderSwitchRequestCallback { 128 public: 129 WebRtcVideoChannel( 130 webrtc::Call* call, 131 const MediaConfig& config, 132 const VideoOptions& options, 133 const webrtc::CryptoOptions& crypto_options, 134 webrtc::VideoEncoderFactory* encoder_factory, 135 webrtc::VideoDecoderFactory* decoder_factory, 136 webrtc::VideoBitrateAllocatorFactory* bitrate_allocator_factory); 137 ~WebRtcVideoChannel() override; 138 139 // VideoMediaChannel implementation 140 bool SetSendParameters(const VideoSendParameters& params) override; 141 bool SetRecvParameters(const VideoRecvParameters& params) override; 142 webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const override; 143 webrtc::RTCError SetRtpSendParameters( 144 uint32_t ssrc, 145 const webrtc::RtpParameters& parameters) override; 146 webrtc::RtpParameters GetRtpReceiveParameters(uint32_t ssrc) const override; 147 webrtc::RtpParameters GetDefaultRtpReceiveParameters() const override; 148 bool GetSendCodec(VideoCodec* send_codec) override; 149 bool SetSend(bool send) override; 150 bool SetVideoSend( 151 uint32_t ssrc, 152 const VideoOptions* options, 153 rtc::VideoSourceInterface<webrtc::VideoFrame>* source) override; 154 bool AddSendStream(const StreamParams& sp) override; 155 bool RemoveSendStream(uint32_t ssrc) override; 156 bool AddRecvStream(const StreamParams& sp) override; 157 bool AddRecvStream(const StreamParams& sp, bool default_stream); 158 bool RemoveRecvStream(uint32_t ssrc) override; 159 void ResetUnsignaledRecvStream() override; 160 bool SetSink(uint32_t ssrc, 161 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override; 162 void SetDefaultSink( 163 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override; 164 void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) override; 165 bool GetStats(VideoMediaInfo* info) override; 166 167 void OnPacketReceived(rtc::CopyOnWriteBuffer packet, 168 int64_t packet_time_us) override; 169 void OnReadyToSend(bool ready) override; 170 void OnNetworkRouteChanged(const std::string& transport_name, 171 const rtc::NetworkRoute& network_route) override; 172 void SetInterface(NetworkInterface* iface) override; 173 174 // E2E Encrypted Video Frame API 175 // Set a frame decryptor to a particular ssrc that will intercept all 176 // incoming video frames and attempt to decrypt them before forwarding the 177 // result. 178 void SetFrameDecryptor(uint32_t ssrc, 179 rtc::scoped_refptr<webrtc::FrameDecryptorInterface> 180 frame_decryptor) override; 181 // Set a frame encryptor to a particular ssrc that will intercept all 182 // outgoing video frames and attempt to encrypt them and forward the result 183 // to the packetizer. 184 void SetFrameEncryptor(uint32_t ssrc, 185 rtc::scoped_refptr<webrtc::FrameEncryptorInterface> 186 frame_encryptor) override; 187 188 void SetVideoCodecSwitchingEnabled(bool enabled) override; 189 190 bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) override; 191 192 absl::optional<int> GetBaseMinimumPlayoutDelayMs( 193 uint32_t ssrc) const override; 194 195 // Implemented for VideoMediaChannelTest. sending()196 bool sending() const { 197 RTC_DCHECK_RUN_ON(&thread_checker_); 198 return sending_; 199 } 200 201 absl::optional<uint32_t> GetDefaultReceiveStreamSsrc(); 202 unsignaled_stream_params()203 StreamParams unsignaled_stream_params() { 204 RTC_DCHECK_RUN_ON(&thread_checker_); 205 return unsignaled_stream_params_; 206 } 207 208 // AdaptReason is used for expressing why a WebRtcVideoSendStream request 209 // a lower input frame size than the currently configured camera input frame 210 // size. There can be more than one reason OR:ed together. 211 enum AdaptReason { 212 ADAPTREASON_NONE = 0, 213 ADAPTREASON_CPU = 1, 214 ADAPTREASON_BANDWIDTH = 2, 215 }; 216 217 static constexpr int kDefaultQpMax = 56; 218 219 std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const override; 220 221 // Take the buffered packets for |ssrcs| and feed them into DeliverPacket. 222 // This method does nothing unless unknown_ssrc_packet_buffer_ is configured. 223 void BackfillBufferedPackets(rtc::ArrayView<const uint32_t> ssrcs); 224 225 // Implements webrtc::EncoderSwitchRequestCallback. 226 void RequestEncoderFallback() override; 227 228 // TODO(bugs.webrtc.org/11341) : Remove this version of RequestEncoderSwitch. 229 void RequestEncoderSwitch( 230 const EncoderSwitchRequestCallback::Config& conf) override; 231 void RequestEncoderSwitch(const webrtc::SdpVideoFormat& format) override; 232 233 void SetRecordableEncodedFrameCallback( 234 uint32_t ssrc, 235 std::function<void(const webrtc::RecordableEncodedFrame&)> callback) 236 override; 237 void ClearRecordableEncodedFrameCallback(uint32_t ssrc) override; 238 void GenerateKeyFrame(uint32_t ssrc) override; 239 240 void SetEncoderToPacketizerFrameTransformer( 241 uint32_t ssrc, 242 rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) 243 override; 244 void SetDepacketizerToDecoderFrameTransformer( 245 uint32_t ssrc, 246 rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) 247 override; 248 249 private: 250 class WebRtcVideoReceiveStream; 251 252 // Finds VideoReceiveStream corresponding to ssrc. Aware of unsignalled ssrc 253 // handling. 254 WebRtcVideoReceiveStream* FindReceiveStream(uint32_t ssrc) 255 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 256 257 struct VideoCodecSettings { 258 VideoCodecSettings(); 259 260 // Checks if all members of |*this| are equal to the corresponding members 261 // of |other|. 262 bool operator==(const VideoCodecSettings& other) const; 263 bool operator!=(const VideoCodecSettings& other) const; 264 265 // Checks if all members of |a|, except |flexfec_payload_type|, are equal 266 // to the corresponding members of |b|. 267 static bool EqualsDisregardingFlexfec(const VideoCodecSettings& a, 268 const VideoCodecSettings& b); 269 270 VideoCodec codec; 271 webrtc::UlpfecConfig ulpfec; 272 int flexfec_payload_type; // -1 if absent. 273 int rtx_payload_type; // -1 if absent. 274 }; 275 276 struct ChangedSendParameters { 277 // These optionals are unset if not changed. 278 absl::optional<VideoCodecSettings> send_codec; 279 absl::optional<std::vector<VideoCodecSettings>> negotiated_codecs; 280 absl::optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions; 281 absl::optional<std::string> mid; 282 absl::optional<bool> extmap_allow_mixed; 283 absl::optional<int> max_bandwidth_bps; 284 absl::optional<bool> conference_mode; 285 absl::optional<webrtc::RtcpMode> rtcp_mode; 286 }; 287 288 struct ChangedRecvParameters { 289 // These optionals are unset if not changed. 290 absl::optional<std::vector<VideoCodecSettings>> codec_settings; 291 absl::optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions; 292 // Keep track of the FlexFEC payload type separately from |codec_settings|. 293 // This allows us to recreate the FlexfecReceiveStream separately from the 294 // VideoReceiveStream when the FlexFEC payload type is changed. 295 absl::optional<int> flexfec_payload_type; 296 }; 297 298 bool GetChangedSendParameters(const VideoSendParameters& params, 299 ChangedSendParameters* changed_params) const 300 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 301 bool ApplyChangedParams(const ChangedSendParameters& changed_params); 302 bool GetChangedRecvParameters(const VideoRecvParameters& params, 303 ChangedRecvParameters* changed_params) const 304 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 305 306 void ConfigureReceiverRtp( 307 webrtc::VideoReceiveStream::Config* config, 308 webrtc::FlexfecReceiveStream::Config* flexfec_config, 309 const StreamParams& sp) const 310 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 311 bool ValidateSendSsrcAvailability(const StreamParams& sp) const 312 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 313 bool ValidateReceiveSsrcAvailability(const StreamParams& sp) const 314 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 315 void DeleteReceiveStream(WebRtcVideoReceiveStream* stream) 316 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 317 318 static std::string CodecSettingsVectorToString( 319 const std::vector<VideoCodecSettings>& codecs); 320 321 // Wrapper for the sender part. 322 class WebRtcVideoSendStream 323 : public rtc::VideoSourceInterface<webrtc::VideoFrame> { 324 public: 325 WebRtcVideoSendStream( 326 webrtc::Call* call, 327 const StreamParams& sp, 328 webrtc::VideoSendStream::Config config, 329 const VideoOptions& options, 330 bool enable_cpu_overuse_detection, 331 int max_bitrate_bps, 332 const absl::optional<VideoCodecSettings>& codec_settings, 333 const absl::optional<std::vector<webrtc::RtpExtension>>& rtp_extensions, 334 const VideoSendParameters& send_params); 335 virtual ~WebRtcVideoSendStream(); 336 337 void SetSendParameters(const ChangedSendParameters& send_params); 338 webrtc::RTCError SetRtpParameters(const webrtc::RtpParameters& parameters); 339 webrtc::RtpParameters GetRtpParameters() const; 340 341 void SetFrameEncryptor( 342 rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor); 343 344 // Implements rtc::VideoSourceInterface<webrtc::VideoFrame>. 345 // WebRtcVideoSendStream acts as a source to the webrtc::VideoSendStream 346 // in |stream_|. This is done to proxy VideoSinkWants from the encoder to 347 // the worker thread. 348 void AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink, 349 const rtc::VideoSinkWants& wants) override; 350 void RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override; 351 352 bool SetVideoSend(const VideoOptions* options, 353 rtc::VideoSourceInterface<webrtc::VideoFrame>* source); 354 355 void SetSend(bool send); 356 357 const std::vector<uint32_t>& GetSsrcs() const; 358 // Returns per ssrc VideoSenderInfos. Useful for simulcast scenario. 359 std::vector<VideoSenderInfo> GetPerLayerVideoSenderInfos(bool log_stats); 360 // Aggregates per ssrc VideoSenderInfos to single VideoSenderInfo for 361 // legacy reasons. Used in old GetStats API and track stats. 362 VideoSenderInfo GetAggregatedVideoSenderInfo( 363 const std::vector<VideoSenderInfo>& infos) const; 364 void FillBitrateInfo(BandwidthEstimationInfo* bwe_info); 365 366 void SetEncoderToPacketizerFrameTransformer( 367 rtc::scoped_refptr<webrtc::FrameTransformerInterface> 368 frame_transformer); 369 370 private: 371 // Parameters needed to reconstruct the underlying stream. 372 // webrtc::VideoSendStream doesn't support setting a lot of options on the 373 // fly, so when those need to be changed we tear down and reconstruct with 374 // similar parameters depending on which options changed etc. 375 struct VideoSendStreamParameters { 376 VideoSendStreamParameters( 377 webrtc::VideoSendStream::Config config, 378 const VideoOptions& options, 379 int max_bitrate_bps, 380 const absl::optional<VideoCodecSettings>& codec_settings); 381 webrtc::VideoSendStream::Config config; 382 VideoOptions options; 383 int max_bitrate_bps; 384 bool conference_mode; 385 absl::optional<VideoCodecSettings> codec_settings; 386 // Sent resolutions + bitrates etc. by the underlying VideoSendStream, 387 // typically changes when setting a new resolution or reconfiguring 388 // bitrates. 389 webrtc::VideoEncoderConfig encoder_config; 390 }; 391 392 rtc::scoped_refptr<webrtc::VideoEncoderConfig::EncoderSpecificSettings> 393 ConfigureVideoEncoderSettings(const VideoCodec& codec); 394 void SetCodec(const VideoCodecSettings& codec); 395 void RecreateWebRtcStream(); 396 webrtc::VideoEncoderConfig CreateVideoEncoderConfig( 397 const VideoCodec& codec) const; 398 void ReconfigureEncoder(); 399 400 // Calls Start or Stop according to whether or not |sending_| is true, 401 // and whether or not the encoding in |rtp_parameters_| is active. 402 void UpdateSendState(); 403 404 webrtc::DegradationPreference GetDegradationPreference() const 405 RTC_EXCLUSIVE_LOCKS_REQUIRED(&thread_checker_); 406 407 rtc::ThreadChecker thread_checker_; 408 rtc::Thread* worker_thread_; 409 const std::vector<uint32_t> ssrcs_ RTC_GUARDED_BY(&thread_checker_); 410 const std::vector<SsrcGroup> ssrc_groups_ RTC_GUARDED_BY(&thread_checker_); 411 webrtc::Call* const call_; 412 const bool enable_cpu_overuse_detection_; 413 rtc::VideoSourceInterface<webrtc::VideoFrame>* source_ 414 RTC_GUARDED_BY(&thread_checker_); 415 416 webrtc::VideoSendStream* stream_ RTC_GUARDED_BY(&thread_checker_); 417 rtc::VideoSinkInterface<webrtc::VideoFrame>* encoder_sink_ 418 RTC_GUARDED_BY(&thread_checker_); 419 // Contains settings that are the same for all streams in the MediaChannel, 420 // such as codecs, header extensions, and the global bitrate limit for the 421 // entire channel. 422 VideoSendStreamParameters parameters_ RTC_GUARDED_BY(&thread_checker_); 423 // Contains settings that are unique for each stream, such as max_bitrate. 424 // Does *not* contain codecs, however. 425 // TODO(skvlad): Move ssrcs_ and ssrc_groups_ into rtp_parameters_. 426 // TODO(skvlad): Combine parameters_ and rtp_parameters_ once we have only 427 // one stream per MediaChannel. 428 webrtc::RtpParameters rtp_parameters_ RTC_GUARDED_BY(&thread_checker_); 429 430 bool sending_ RTC_GUARDED_BY(&thread_checker_); 431 432 // In order for the |invoker_| to protect other members from being 433 // destructed as they are used in asynchronous tasks it has to be destructed 434 // first. 435 rtc::AsyncInvoker invoker_; 436 }; 437 438 // Wrapper for the receiver part, contains configs etc. that are needed to 439 // reconstruct the underlying VideoReceiveStream. 440 class WebRtcVideoReceiveStream 441 : public rtc::VideoSinkInterface<webrtc::VideoFrame> { 442 public: 443 WebRtcVideoReceiveStream( 444 WebRtcVideoChannel* channel, 445 webrtc::Call* call, 446 const StreamParams& sp, 447 webrtc::VideoReceiveStream::Config config, 448 webrtc::VideoDecoderFactory* decoder_factory, 449 bool default_stream, 450 const std::vector<VideoCodecSettings>& recv_codecs, 451 const webrtc::FlexfecReceiveStream::Config& flexfec_config); 452 ~WebRtcVideoReceiveStream(); 453 454 const std::vector<uint32_t>& GetSsrcs() const; 455 456 std::vector<webrtc::RtpSource> GetSources(); 457 458 // Does not return codecs, they are filled by the owning WebRtcVideoChannel. 459 webrtc::RtpParameters GetRtpParameters() const; 460 461 void SetLocalSsrc(uint32_t local_ssrc); 462 // TODO(deadbeef): Move these feedback parameters into the recv parameters. 463 void SetFeedbackParameters(bool lntf_enabled, 464 bool nack_enabled, 465 bool transport_cc_enabled, 466 webrtc::RtcpMode rtcp_mode); 467 void SetRecvParameters(const ChangedRecvParameters& recv_params); 468 469 void OnFrame(const webrtc::VideoFrame& frame) override; 470 bool IsDefaultStream() const; 471 472 void SetFrameDecryptor( 473 rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor); 474 475 bool SetBaseMinimumPlayoutDelayMs(int delay_ms); 476 477 int GetBaseMinimumPlayoutDelayMs() const; 478 479 void SetSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink); 480 481 VideoReceiverInfo GetVideoReceiverInfo(bool log_stats); 482 483 void SetRecordableEncodedFrameCallback( 484 std::function<void(const webrtc::RecordableEncodedFrame&)> callback); 485 void ClearRecordableEncodedFrameCallback(); 486 void GenerateKeyFrame(); 487 488 void SetDepacketizerToDecoderFrameTransformer( 489 rtc::scoped_refptr<webrtc::FrameTransformerInterface> 490 frame_transformer); 491 492 private: 493 void RecreateWebRtcVideoStream(); 494 void MaybeRecreateWebRtcFlexfecStream(); 495 496 void MaybeAssociateFlexfecWithVideo(); 497 void MaybeDissociateFlexfecFromVideo(); 498 499 void ConfigureCodecs(const std::vector<VideoCodecSettings>& recv_codecs); 500 void ConfigureFlexfecCodec(int flexfec_payload_type); 501 502 std::string GetCodecNameFromPayloadType(int payload_type); 503 504 WebRtcVideoChannel* const channel_; 505 webrtc::Call* const call_; 506 const StreamParams stream_params_; 507 508 // Both |stream_| and |flexfec_stream_| are managed by |this|. They are 509 // destroyed by calling call_->DestroyVideoReceiveStream and 510 // call_->DestroyFlexfecReceiveStream, respectively. 511 webrtc::VideoReceiveStream* stream_; 512 const bool default_stream_; 513 webrtc::VideoReceiveStream::Config config_; 514 webrtc::FlexfecReceiveStream::Config flexfec_config_; 515 webrtc::FlexfecReceiveStream* flexfec_stream_; 516 517 webrtc::VideoDecoderFactory* const decoder_factory_; 518 519 webrtc::Mutex sink_lock_; 520 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink_ 521 RTC_GUARDED_BY(sink_lock_); 522 // Expands remote RTP timestamps to int64_t to be able to estimate how long 523 // the stream has been running. 524 rtc::TimestampWrapAroundHandler timestamp_wraparound_handler_ 525 RTC_GUARDED_BY(sink_lock_); 526 int64_t first_frame_timestamp_ RTC_GUARDED_BY(sink_lock_); 527 // Start NTP time is estimated as current remote NTP time (estimated from 528 // RTCP) minus the elapsed time, as soon as remote NTP time is available. 529 int64_t estimated_remote_start_ntp_time_ms_ RTC_GUARDED_BY(sink_lock_); 530 }; 531 532 void Construct(webrtc::Call* call, WebRtcVideoEngine* engine); 533 534 bool SendRtp(const uint8_t* data, 535 size_t len, 536 const webrtc::PacketOptions& options) override; 537 bool SendRtcp(const uint8_t* data, size_t len) override; 538 539 // Generate the list of codec parameters to pass down based on the negotiated 540 // "codecs". Note that VideoCodecSettings correspond to concrete codecs like 541 // VP8, VP9, H264 while VideoCodecs correspond also to "virtual" codecs like 542 // RTX, ULPFEC, FLEXFEC. 543 static std::vector<VideoCodecSettings> MapCodecs( 544 const std::vector<VideoCodec>& codecs); 545 // Get all codecs that are compatible with the receiver. 546 std::vector<VideoCodecSettings> SelectSendVideoCodecs( 547 const std::vector<VideoCodecSettings>& remote_mapped_codecs) const 548 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 549 550 static bool NonFlexfecReceiveCodecsHaveChanged( 551 std::vector<VideoCodecSettings> before, 552 std::vector<VideoCodecSettings> after); 553 554 void FillSenderStats(VideoMediaInfo* info, bool log_stats) 555 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 556 void FillReceiverStats(VideoMediaInfo* info, bool log_stats) 557 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 558 void FillBandwidthEstimationStats(const webrtc::Call::Stats& stats, 559 VideoMediaInfo* info) 560 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 561 void FillSendAndReceiveCodecStats(VideoMediaInfo* video_media_info) 562 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 563 564 rtc::Thread* worker_thread_; 565 rtc::ThreadChecker thread_checker_; 566 567 uint32_t rtcp_receiver_report_ssrc_ RTC_GUARDED_BY(thread_checker_); 568 bool sending_ RTC_GUARDED_BY(thread_checker_); 569 webrtc::Call* const call_ RTC_GUARDED_BY(thread_checker_); 570 571 DefaultUnsignalledSsrcHandler default_unsignalled_ssrc_handler_ 572 RTC_GUARDED_BY(thread_checker_); 573 UnsignalledSsrcHandler* const unsignalled_ssrc_handler_ 574 RTC_GUARDED_BY(thread_checker_); 575 576 // Delay for unsignaled streams, which may be set before the stream exists. 577 int default_recv_base_minimum_delay_ms_ RTC_GUARDED_BY(thread_checker_) = 0; 578 579 const MediaConfig::Video video_config_ RTC_GUARDED_BY(thread_checker_); 580 581 // Using primary-ssrc (first ssrc) as key. 582 std::map<uint32_t, WebRtcVideoSendStream*> send_streams_ 583 RTC_GUARDED_BY(thread_checker_); 584 std::map<uint32_t, WebRtcVideoReceiveStream*> receive_streams_ 585 RTC_GUARDED_BY(thread_checker_); 586 std::set<uint32_t> send_ssrcs_ RTC_GUARDED_BY(thread_checker_); 587 std::set<uint32_t> receive_ssrcs_ RTC_GUARDED_BY(thread_checker_); 588 589 absl::optional<VideoCodecSettings> send_codec_ 590 RTC_GUARDED_BY(thread_checker_); 591 std::vector<VideoCodecSettings> negotiated_codecs_ 592 RTC_GUARDED_BY(thread_checker_); 593 594 absl::optional<std::vector<webrtc::RtpExtension>> send_rtp_extensions_ 595 RTC_GUARDED_BY(thread_checker_); 596 597 webrtc::VideoEncoderFactory* const encoder_factory_ 598 RTC_GUARDED_BY(thread_checker_); 599 webrtc::VideoDecoderFactory* const decoder_factory_ 600 RTC_GUARDED_BY(thread_checker_); 601 webrtc::VideoBitrateAllocatorFactory* const bitrate_allocator_factory_ 602 RTC_GUARDED_BY(thread_checker_); 603 std::vector<VideoCodecSettings> recv_codecs_ RTC_GUARDED_BY(thread_checker_); 604 std::vector<webrtc::RtpExtension> recv_rtp_extensions_ 605 RTC_GUARDED_BY(thread_checker_); 606 // See reason for keeping track of the FlexFEC payload type separately in 607 // comment in WebRtcVideoChannel::ChangedRecvParameters. 608 int recv_flexfec_payload_type_ RTC_GUARDED_BY(thread_checker_); 609 webrtc::BitrateConstraints bitrate_config_ RTC_GUARDED_BY(thread_checker_); 610 // TODO(deadbeef): Don't duplicate information between 611 // send_params/recv_params, rtp_extensions, options, etc. 612 VideoSendParameters send_params_ RTC_GUARDED_BY(thread_checker_); 613 VideoOptions default_send_options_ RTC_GUARDED_BY(thread_checker_); 614 VideoRecvParameters recv_params_ RTC_GUARDED_BY(thread_checker_); 615 int64_t last_stats_log_ms_ RTC_GUARDED_BY(thread_checker_); 616 const bool discard_unknown_ssrc_packets_ RTC_GUARDED_BY(thread_checker_); 617 // This is a stream param that comes from the remote description, but wasn't 618 // signaled with any a=ssrc lines. It holds information that was signaled 619 // before the unsignaled receive stream is created when the first packet is 620 // received. 621 StreamParams unsignaled_stream_params_ RTC_GUARDED_BY(thread_checker_); 622 // Per peer connection crypto options that last for the lifetime of the peer 623 // connection. 624 const webrtc::CryptoOptions crypto_options_ RTC_GUARDED_BY(thread_checker_); 625 626 // Optional frame transformer set on unsignaled streams. 627 rtc::scoped_refptr<webrtc::FrameTransformerInterface> 628 unsignaled_frame_transformer_ RTC_GUARDED_BY(thread_checker_); 629 630 // Buffer for unhandled packets. 631 std::unique_ptr<UnhandledPacketsBuffer> unknown_ssrc_packet_buffer_ 632 RTC_GUARDED_BY(thread_checker_); 633 634 bool allow_codec_switching_ = false; 635 absl::optional<EncoderSwitchRequestCallback::Config> 636 requested_encoder_switch_; 637 638 // In order for the |invoker_| to protect other members from being destructed 639 // as they are used in asynchronous tasks it has to be destructed first. 640 rtc::AsyncInvoker invoker_; 641 }; 642 643 class EncoderStreamFactory 644 : public webrtc::VideoEncoderConfig::VideoStreamFactoryInterface { 645 public: 646 EncoderStreamFactory(std::string codec_name, 647 int max_qp, 648 bool is_screenshare, 649 bool conference_mode); 650 651 private: 652 std::vector<webrtc::VideoStream> CreateEncoderStreams( 653 int width, 654 int height, 655 const webrtc::VideoEncoderConfig& encoder_config) override; 656 657 std::vector<webrtc::VideoStream> CreateDefaultVideoStreams( 658 int width, 659 int height, 660 const webrtc::VideoEncoderConfig& encoder_config, 661 const absl::optional<webrtc::DataRate>& experimental_min_bitrate) const; 662 663 std::vector<webrtc::VideoStream> 664 CreateSimulcastOrConfereceModeScreenshareStreams( 665 int width, 666 int height, 667 const webrtc::VideoEncoderConfig& encoder_config, 668 const absl::optional<webrtc::DataRate>& experimental_min_bitrate) const; 669 670 const std::string codec_name_; 671 const int max_qp_; 672 const bool is_screenshare_; 673 // Allows a screenshare specific configuration, which enables temporal 674 // layering and various settings. 675 const bool conference_mode_; 676 }; 677 678 } // namespace cricket 679 680 #endif // MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_ 681