1 /* 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_ 12 #define MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_ 13 14 #include <map> 15 #include <memory> 16 #include <set> 17 #include <string> 18 #include <vector> 19 20 #include "absl/types/optional.h" 21 #include "api/call/transport.h" 22 #include "api/sequence_checker.h" 23 #include "api/task_queue/pending_task_safety_flag.h" 24 #include "api/transport/field_trial_based_config.h" 25 #include "api/video/video_bitrate_allocator_factory.h" 26 #include "api/video/video_frame.h" 27 #include "api/video/video_sink_interface.h" 28 #include "api/video/video_source_interface.h" 29 #include "api/video_codecs/sdp_video_format.h" 30 #include "call/call.h" 31 #include "call/flexfec_receive_stream.h" 32 #include "call/video_receive_stream.h" 33 #include "call/video_send_stream.h" 34 #include "media/base/media_engine.h" 35 #include "media/engine/unhandled_packets_buffer.h" 36 #include "rtc_base/network_route.h" 37 #include "rtc_base/synchronization/mutex.h" 38 #include "rtc_base/system/no_unique_address.h" 39 #include "rtc_base/thread_annotations.h" 40 41 namespace webrtc { 42 class VideoDecoderFactory; 43 class VideoEncoderFactory; 44 } // namespace webrtc 45 46 namespace cricket { 47 48 class WebRtcVideoChannel; 49 50 // Public for testing. 51 // Inputs StreamStats for all types of substreams (kMedia, kRtx, kFlexfec) and 52 // merges any non-kMedia substream stats object into its referenced kMedia-type 53 // substream. The resulting substreams are all kMedia. This means, for example, 54 // that packet and byte counters of RTX and FlexFEC streams are accounted for in 55 // the relevant RTP media stream's stats. This makes the resulting StreamStats 56 // objects ready to be turned into "outbound-rtp" stats objects for GetStats() 57 // which does not create separate stream stats objects for complementary 58 // streams. 59 std::map<uint32_t, webrtc::VideoSendStream::StreamStats> 60 MergeInfoAboutOutboundRtpSubstreamsForTesting( 61 const std::map<uint32_t, webrtc::VideoSendStream::StreamStats>& substreams); 62 63 class UnsignalledSsrcHandler { 64 public: 65 enum Action { 66 kDropPacket, 67 kDeliverPacket, 68 }; 69 virtual Action OnUnsignalledSsrc(WebRtcVideoChannel* channel, 70 uint32_t ssrc, 71 absl::optional<uint32_t> rtx_ssrc) = 0; 72 virtual ~UnsignalledSsrcHandler() = default; 73 }; 74 75 // TODO(pbos): Remove, use external handlers only. 76 class DefaultUnsignalledSsrcHandler : public UnsignalledSsrcHandler { 77 public: 78 DefaultUnsignalledSsrcHandler(); 79 Action OnUnsignalledSsrc(WebRtcVideoChannel* channel, 80 uint32_t ssrc, 81 absl::optional<uint32_t> rtx_ssrc) override; 82 83 rtc::VideoSinkInterface<webrtc::VideoFrame>* GetDefaultSink() const; 84 void SetDefaultSink(WebRtcVideoChannel* channel, 85 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink); 86 87 virtual ~DefaultUnsignalledSsrcHandler() = default; 88 89 private: 90 rtc::VideoSinkInterface<webrtc::VideoFrame>* default_sink_; 91 }; 92 93 // WebRtcVideoEngine is used for the new native WebRTC Video API (webrtc:1667). 94 class WebRtcVideoEngine : public VideoEngineInterface { 95 public: 96 // These video codec factories represents all video codecs, i.e. both software 97 // and external hardware codecs. 98 WebRtcVideoEngine( 99 std::unique_ptr<webrtc::VideoEncoderFactory> video_encoder_factory, 100 std::unique_ptr<webrtc::VideoDecoderFactory> video_decoder_factory, 101 const webrtc::FieldTrialsView& trials); 102 103 ~WebRtcVideoEngine() override; 104 105 VideoMediaChannel* CreateMediaChannel( 106 webrtc::Call* call, 107 const MediaConfig& config, 108 const VideoOptions& options, 109 const webrtc::CryptoOptions& crypto_options, 110 webrtc::VideoBitrateAllocatorFactory* video_bitrate_allocator_factory) 111 override; 112 send_codecs()113 std::vector<VideoCodec> send_codecs() const override { 114 return send_codecs(true); 115 } recv_codecs()116 std::vector<VideoCodec> recv_codecs() const override { 117 return recv_codecs(true); 118 } 119 std::vector<VideoCodec> send_codecs(bool include_rtx) const override; 120 std::vector<VideoCodec> recv_codecs(bool include_rtx) const override; 121 std::vector<webrtc::RtpHeaderExtensionCapability> GetRtpHeaderExtensions() 122 const override; 123 124 private: 125 const std::unique_ptr<webrtc::VideoDecoderFactory> decoder_factory_; 126 const std::unique_ptr<webrtc::VideoEncoderFactory> encoder_factory_; 127 const std::unique_ptr<webrtc::VideoBitrateAllocatorFactory> 128 bitrate_allocator_factory_; 129 const webrtc::FieldTrialsView& trials_; 130 }; 131 132 class WebRtcVideoChannel : public VideoMediaChannel, 133 public webrtc::Transport, 134 public webrtc::EncoderSwitchRequestCallback { 135 public: 136 WebRtcVideoChannel( 137 webrtc::Call* call, 138 const MediaConfig& config, 139 const VideoOptions& options, 140 const webrtc::CryptoOptions& crypto_options, 141 webrtc::VideoEncoderFactory* encoder_factory, 142 webrtc::VideoDecoderFactory* decoder_factory, 143 webrtc::VideoBitrateAllocatorFactory* bitrate_allocator_factory); 144 ~WebRtcVideoChannel() override; 145 146 // VideoMediaChannel implementation 147 bool SetSendParameters(const VideoSendParameters& params) override; 148 bool SetRecvParameters(const VideoRecvParameters& params) override; 149 webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const override; 150 webrtc::RTCError SetRtpSendParameters( 151 uint32_t ssrc, 152 const webrtc::RtpParameters& parameters, 153 webrtc::SetParametersCallback callback) override; 154 webrtc::RtpParameters GetRtpReceiveParameters(uint32_t ssrc) const override; 155 webrtc::RtpParameters GetDefaultRtpReceiveParameters() const override; 156 bool GetSendCodec(VideoCodec* send_codec) override; 157 bool SetSend(bool send) override; 158 bool SetVideoSend( 159 uint32_t ssrc, 160 const VideoOptions* options, 161 rtc::VideoSourceInterface<webrtc::VideoFrame>* source) override; 162 bool AddSendStream(const StreamParams& sp) override; 163 bool RemoveSendStream(uint32_t ssrc) override; 164 bool AddRecvStream(const StreamParams& sp) override; 165 bool AddRecvStream(const StreamParams& sp, bool default_stream); 166 bool RemoveRecvStream(uint32_t ssrc) override; 167 void ResetUnsignaledRecvStream() override; 168 void OnDemuxerCriteriaUpdatePending() override; 169 void OnDemuxerCriteriaUpdateComplete() override; 170 bool SetSink(uint32_t ssrc, 171 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override; 172 void SetDefaultSink( 173 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override; 174 void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) override; 175 bool GetStats(VideoMediaInfo* info) override; 176 177 void OnPacketReceived(rtc::CopyOnWriteBuffer packet, 178 int64_t packet_time_us) override; 179 void OnPacketSent(const rtc::SentPacket& sent_packet) override; 180 void OnReadyToSend(bool ready) override; 181 void OnNetworkRouteChanged(absl::string_view transport_name, 182 const rtc::NetworkRoute& network_route) override; 183 void SetInterface(NetworkInterface* iface) override; 184 185 // E2E Encrypted Video Frame API 186 // Set a frame decryptor to a particular ssrc that will intercept all 187 // incoming video frames and attempt to decrypt them before forwarding the 188 // result. 189 void SetFrameDecryptor(uint32_t ssrc, 190 rtc::scoped_refptr<webrtc::FrameDecryptorInterface> 191 frame_decryptor) override; 192 // Set a frame encryptor to a particular ssrc that will intercept all 193 // outgoing video frames and attempt to encrypt them and forward the result 194 // to the packetizer. 195 void SetFrameEncryptor(uint32_t ssrc, 196 rtc::scoped_refptr<webrtc::FrameEncryptorInterface> 197 frame_encryptor) override; 198 199 // note: The encoder_selector object must remain valid for the lifetime of the 200 // MediaChannel, unless replaced. 201 void SetEncoderSelector(uint32_t ssrc, 202 webrtc::VideoEncoderFactory::EncoderSelectorInterface* 203 encoder_selector) override; 204 205 void SetVideoCodecSwitchingEnabled(bool enabled) override; 206 207 bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) override; 208 209 absl::optional<int> GetBaseMinimumPlayoutDelayMs( 210 uint32_t ssrc) const override; 211 212 // Implemented for VideoMediaChannelTest. sending()213 bool sending() const { 214 RTC_DCHECK_RUN_ON(&thread_checker_); 215 return sending_; 216 } 217 218 absl::optional<uint32_t> GetDefaultReceiveStreamSsrc(); 219 unsignaled_stream_params()220 StreamParams unsignaled_stream_params() { 221 RTC_DCHECK_RUN_ON(&thread_checker_); 222 return unsignaled_stream_params_; 223 } 224 225 // AdaptReason is used for expressing why a WebRtcVideoSendStream request 226 // a lower input frame size than the currently configured camera input frame 227 // size. There can be more than one reason OR:ed together. 228 enum AdaptReason { 229 ADAPTREASON_NONE = 0, 230 ADAPTREASON_CPU = 1, 231 ADAPTREASON_BANDWIDTH = 2, 232 }; 233 234 static constexpr int kDefaultQpMax = 56; 235 236 std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const override; 237 238 // Take the buffered packets for `ssrcs` and feed them into DeliverPacket. 239 // This method does nothing unless unknown_ssrc_packet_buffer_ is configured. 240 void BackfillBufferedPackets(rtc::ArrayView<const uint32_t> ssrcs); 241 242 // Implements webrtc::EncoderSwitchRequestCallback. 243 void RequestEncoderFallback() override; 244 void RequestEncoderSwitch(const webrtc::SdpVideoFormat& format, 245 bool allow_default_fallback) override; 246 247 void SetRecordableEncodedFrameCallback( 248 uint32_t ssrc, 249 std::function<void(const webrtc::RecordableEncodedFrame&)> callback) 250 override; 251 void ClearRecordableEncodedFrameCallback(uint32_t ssrc) override; 252 void RequestRecvKeyFrame(uint32_t ssrc) override; 253 void GenerateSendKeyFrame(uint32_t ssrc, 254 const std::vector<std::string>& rids) override; 255 256 void SetEncoderToPacketizerFrameTransformer( 257 uint32_t ssrc, 258 rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) 259 override; 260 void SetDepacketizerToDecoderFrameTransformer( 261 uint32_t ssrc, 262 rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) 263 override; 264 265 private: 266 class WebRtcVideoReceiveStream; 267 268 // Finds VideoReceiveStreamInterface corresponding to ssrc. Aware of 269 // unsignalled ssrc handling. 270 WebRtcVideoReceiveStream* FindReceiveStream(uint32_t ssrc) 271 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 272 273 struct VideoCodecSettings { 274 VideoCodecSettings(); 275 276 // Checks if all members of |*this| are equal to the corresponding members 277 // of `other`. 278 bool operator==(const VideoCodecSettings& other) const; 279 bool operator!=(const VideoCodecSettings& other) const; 280 281 // Checks if all members of `a`, except `flexfec_payload_type`, are equal 282 // to the corresponding members of `b`. 283 static bool EqualsDisregardingFlexfec(const VideoCodecSettings& a, 284 const VideoCodecSettings& b); 285 286 VideoCodec codec; 287 webrtc::UlpfecConfig ulpfec; 288 int flexfec_payload_type; // -1 if absent. 289 int rtx_payload_type; // -1 if absent. 290 int rtx_time; // -1 if absent. 291 }; 292 293 struct ChangedSendParameters { 294 // These optionals are unset if not changed. 295 absl::optional<VideoCodecSettings> send_codec; 296 absl::optional<std::vector<VideoCodecSettings>> negotiated_codecs; 297 absl::optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions; 298 absl::optional<std::string> mid; 299 absl::optional<bool> extmap_allow_mixed; 300 absl::optional<int> max_bandwidth_bps; 301 absl::optional<bool> conference_mode; 302 absl::optional<webrtc::RtcpMode> rtcp_mode; 303 }; 304 305 struct ChangedRecvParameters { 306 // These optionals are unset if not changed. 307 absl::optional<std::vector<VideoCodecSettings>> codec_settings; 308 absl::optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions; 309 // Keep track of the FlexFEC payload type separately from `codec_settings`. 310 // This allows us to recreate the FlexfecReceiveStream separately from the 311 // VideoReceiveStreamInterface when the FlexFEC payload type is changed. 312 absl::optional<int> flexfec_payload_type; 313 }; 314 315 bool GetChangedSendParameters(const VideoSendParameters& params, 316 ChangedSendParameters* changed_params) const 317 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 318 bool ApplyChangedParams(const ChangedSendParameters& changed_params); 319 bool GetChangedRecvParameters(const VideoRecvParameters& params, 320 ChangedRecvParameters* changed_params) const 321 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 322 323 void ConfigureReceiverRtp( 324 webrtc::VideoReceiveStreamInterface::Config* config, 325 webrtc::FlexfecReceiveStream::Config* flexfec_config, 326 const StreamParams& sp) const 327 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 328 bool ValidateSendSsrcAvailability(const StreamParams& sp) const 329 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 330 bool ValidateReceiveSsrcAvailability(const StreamParams& sp) const 331 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 332 void DeleteReceiveStream(WebRtcVideoReceiveStream* stream) 333 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 334 335 static std::string CodecSettingsVectorToString( 336 const std::vector<VideoCodecSettings>& codecs); 337 338 // Populates `rtx_associated_payload_types`, `raw_payload_types` and 339 // `decoders` based on codec settings provided by `recv_codecs`. 340 // `recv_codecs` must be non-empty and all other parameters must be empty. 341 static void ExtractCodecInformation( 342 rtc::ArrayView<const VideoCodecSettings> recv_codecs, 343 std::map<int, int>& rtx_associated_payload_types, 344 std::set<int>& raw_payload_types, 345 std::vector<webrtc::VideoReceiveStreamInterface::Decoder>& decoders); 346 347 // Called when the local ssrc changes. Sets `rtcp_receiver_report_ssrc_` and 348 // updates the receive streams. 349 void SetReceiverReportSsrc(uint32_t ssrc) RTC_RUN_ON(&thread_checker_); 350 351 // Wrapper for the sender part. 352 class WebRtcVideoSendStream { 353 public: 354 WebRtcVideoSendStream( 355 webrtc::Call* call, 356 const StreamParams& sp, 357 webrtc::VideoSendStream::Config config, 358 const VideoOptions& options, 359 bool enable_cpu_overuse_detection, 360 int max_bitrate_bps, 361 const absl::optional<VideoCodecSettings>& codec_settings, 362 const absl::optional<std::vector<webrtc::RtpExtension>>& rtp_extensions, 363 const VideoSendParameters& send_params); 364 ~WebRtcVideoSendStream(); 365 366 void SetSendParameters(const ChangedSendParameters& send_params); 367 webrtc::RTCError SetRtpParameters(const webrtc::RtpParameters& parameters, 368 webrtc::SetParametersCallback callback); 369 webrtc::RtpParameters GetRtpParameters() const; 370 371 void SetFrameEncryptor( 372 rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor); 373 374 bool SetVideoSend(const VideoOptions* options, 375 rtc::VideoSourceInterface<webrtc::VideoFrame>* source); 376 377 // note: The encoder_selector object must remain valid for the lifetime of 378 // the MediaChannel, unless replaced. 379 void SetEncoderSelector( 380 webrtc::VideoEncoderFactory::EncoderSelectorInterface* 381 encoder_selector); 382 383 void SetSend(bool send); 384 385 const std::vector<uint32_t>& GetSsrcs() const; 386 // Returns per ssrc VideoSenderInfos. Useful for simulcast scenario. 387 std::vector<VideoSenderInfo> GetPerLayerVideoSenderInfos(bool log_stats); 388 // Aggregates per ssrc VideoSenderInfos to single VideoSenderInfo for 389 // legacy reasons. Used in old GetStats API and track stats. 390 VideoSenderInfo GetAggregatedVideoSenderInfo( 391 const std::vector<VideoSenderInfo>& infos) const; 392 void FillBitrateInfo(BandwidthEstimationInfo* bwe_info); 393 394 void SetEncoderToPacketizerFrameTransformer( 395 rtc::scoped_refptr<webrtc::FrameTransformerInterface> 396 frame_transformer); 397 void GenerateKeyFrame(const std::vector<std::string>& rids); 398 399 private: 400 // Parameters needed to reconstruct the underlying stream. 401 // webrtc::VideoSendStream doesn't support setting a lot of options on the 402 // fly, so when those need to be changed we tear down and reconstruct with 403 // similar parameters depending on which options changed etc. 404 struct VideoSendStreamParameters { 405 VideoSendStreamParameters( 406 webrtc::VideoSendStream::Config config, 407 const VideoOptions& options, 408 int max_bitrate_bps, 409 const absl::optional<VideoCodecSettings>& codec_settings); 410 webrtc::VideoSendStream::Config config; 411 VideoOptions options; 412 int max_bitrate_bps; 413 bool conference_mode; 414 absl::optional<VideoCodecSettings> codec_settings; 415 // Sent resolutions + bitrates etc. by the underlying VideoSendStream, 416 // typically changes when setting a new resolution or reconfiguring 417 // bitrates. 418 webrtc::VideoEncoderConfig encoder_config; 419 }; 420 421 rtc::scoped_refptr<webrtc::VideoEncoderConfig::EncoderSpecificSettings> 422 ConfigureVideoEncoderSettings(const VideoCodec& codec); 423 void SetCodec(const VideoCodecSettings& codec); 424 void RecreateWebRtcStream(); 425 webrtc::VideoEncoderConfig CreateVideoEncoderConfig( 426 const VideoCodec& codec) const; 427 void ReconfigureEncoder(webrtc::SetParametersCallback callback); 428 429 // Calls Start or Stop according to whether or not `sending_` is true, 430 // and whether or not the encoding in `rtp_parameters_` is active. 431 void UpdateSendState(); 432 433 webrtc::DegradationPreference GetDegradationPreference() const 434 RTC_EXCLUSIVE_LOCKS_REQUIRED(&thread_checker_); 435 436 RTC_NO_UNIQUE_ADDRESS webrtc::SequenceChecker thread_checker_; 437 webrtc::TaskQueueBase* const worker_thread_; 438 const std::vector<uint32_t> ssrcs_ RTC_GUARDED_BY(&thread_checker_); 439 const std::vector<SsrcGroup> ssrc_groups_ RTC_GUARDED_BY(&thread_checker_); 440 webrtc::Call* const call_; 441 const bool enable_cpu_overuse_detection_; 442 rtc::VideoSourceInterface<webrtc::VideoFrame>* source_ 443 RTC_GUARDED_BY(&thread_checker_); 444 445 webrtc::VideoSendStream* stream_ RTC_GUARDED_BY(&thread_checker_); 446 447 // Contains settings that are the same for all streams in the MediaChannel, 448 // such as codecs, header extensions, and the global bitrate limit for the 449 // entire channel. 450 VideoSendStreamParameters parameters_ RTC_GUARDED_BY(&thread_checker_); 451 // Contains settings that are unique for each stream, such as max_bitrate. 452 // Does *not* contain codecs, however. 453 // TODO(skvlad): Move ssrcs_ and ssrc_groups_ into rtp_parameters_. 454 // TODO(skvlad): Combine parameters_ and rtp_parameters_ once we have only 455 // one stream per MediaChannel. 456 webrtc::RtpParameters rtp_parameters_ RTC_GUARDED_BY(&thread_checker_); 457 458 bool sending_ RTC_GUARDED_BY(&thread_checker_); 459 460 // TODO(asapersson): investigate why setting 461 // DegrationPreferences::MAINTAIN_RESOLUTION isn't sufficient to disable 462 // downscaling everywhere in the pipeline. 463 const bool disable_automatic_resize_; 464 }; 465 466 // Wrapper for the receiver part, contains configs etc. that are needed to 467 // reconstruct the underlying VideoReceiveStreamInterface. 468 class WebRtcVideoReceiveStream 469 : public rtc::VideoSinkInterface<webrtc::VideoFrame> { 470 public: 471 WebRtcVideoReceiveStream( 472 WebRtcVideoChannel* channel, 473 webrtc::Call* call, 474 const StreamParams& sp, 475 webrtc::VideoReceiveStreamInterface::Config config, 476 bool default_stream, 477 const std::vector<VideoCodecSettings>& recv_codecs, 478 const webrtc::FlexfecReceiveStream::Config& flexfec_config); 479 ~WebRtcVideoReceiveStream(); 480 481 webrtc::VideoReceiveStreamInterface& stream(); 482 // Return value may be nullptr. 483 webrtc::FlexfecReceiveStream* flexfec_stream(); 484 485 const std::vector<uint32_t>& GetSsrcs() const; 486 487 std::vector<webrtc::RtpSource> GetSources(); 488 489 // Does not return codecs, they are filled by the owning WebRtcVideoChannel. 490 webrtc::RtpParameters GetRtpParameters() const; 491 492 // TODO(deadbeef): Move these feedback parameters into the recv parameters. 493 void SetFeedbackParameters(bool lntf_enabled, 494 bool nack_enabled, 495 bool transport_cc_enabled, 496 webrtc::RtcpMode rtcp_mode, 497 int rtx_time); 498 void SetRecvParameters(const ChangedRecvParameters& recv_params); 499 500 void OnFrame(const webrtc::VideoFrame& frame) override; 501 bool IsDefaultStream() const; 502 503 void SetFrameDecryptor( 504 rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor); 505 506 bool SetBaseMinimumPlayoutDelayMs(int delay_ms); 507 508 int GetBaseMinimumPlayoutDelayMs() const; 509 510 void SetSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink); 511 512 VideoReceiverInfo GetVideoReceiverInfo(bool log_stats); 513 514 void SetRecordableEncodedFrameCallback( 515 std::function<void(const webrtc::RecordableEncodedFrame&)> callback); 516 void ClearRecordableEncodedFrameCallback(); 517 void GenerateKeyFrame(); 518 519 void SetDepacketizerToDecoderFrameTransformer( 520 rtc::scoped_refptr<webrtc::FrameTransformerInterface> 521 frame_transformer); 522 523 void SetLocalSsrc(uint32_t local_ssrc); 524 525 private: 526 // Attempts to reconfigure an already existing `flexfec_stream_`, create 527 // one if the configuration is now complete or remove a flexfec stream 528 // when disabled. 529 void SetFlexFecPayload(int payload_type); 530 531 void RecreateReceiveStream(); 532 void CreateReceiveStream(); 533 void StartReceiveStream(); 534 535 // Applies a new receive codecs configration to `config_`. Returns true 536 // if the internal stream needs to be reconstructed, or false if no changes 537 // were applied. 538 bool ReconfigureCodecs(const std::vector<VideoCodecSettings>& recv_codecs); 539 540 WebRtcVideoChannel* const channel_; 541 webrtc::Call* const call_; 542 const StreamParams stream_params_; 543 544 // Both `stream_` and `flexfec_stream_` are managed by `this`. They are 545 // destroyed by calling call_->DestroyVideoReceiveStream and 546 // call_->DestroyFlexfecReceiveStream, respectively. 547 webrtc::VideoReceiveStreamInterface* stream_; 548 const bool default_stream_; 549 webrtc::VideoReceiveStreamInterface::Config config_; 550 webrtc::FlexfecReceiveStream::Config flexfec_config_; 551 webrtc::FlexfecReceiveStream* flexfec_stream_; 552 553 webrtc::Mutex sink_lock_; 554 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink_ 555 RTC_GUARDED_BY(sink_lock_); 556 // Expands remote RTP timestamps to int64_t to be able to estimate how long 557 // the stream has been running. 558 rtc::TimestampWrapAroundHandler timestamp_wraparound_handler_ 559 RTC_GUARDED_BY(sink_lock_); 560 int64_t first_frame_timestamp_ RTC_GUARDED_BY(sink_lock_); 561 // Start NTP time is estimated as current remote NTP time (estimated from 562 // RTCP) minus the elapsed time, as soon as remote NTP time is available. 563 int64_t estimated_remote_start_ntp_time_ms_ RTC_GUARDED_BY(sink_lock_); 564 }; 565 566 void Construct(webrtc::Call* call, WebRtcVideoEngine* engine); 567 568 bool SendRtp(const uint8_t* data, 569 size_t len, 570 const webrtc::PacketOptions& options) override; 571 bool SendRtcp(const uint8_t* data, size_t len) override; 572 573 // Generate the list of codec parameters to pass down based on the negotiated 574 // "codecs". Note that VideoCodecSettings correspond to concrete codecs like 575 // VP8, VP9, H264 while VideoCodecs correspond also to "virtual" codecs like 576 // RTX, ULPFEC, FLEXFEC. 577 static std::vector<VideoCodecSettings> MapCodecs( 578 const std::vector<VideoCodec>& codecs); 579 // Get all codecs that are compatible with the receiver. 580 std::vector<VideoCodecSettings> SelectSendVideoCodecs( 581 const std::vector<VideoCodecSettings>& remote_mapped_codecs) const 582 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 583 584 static bool NonFlexfecReceiveCodecsHaveChanged( 585 std::vector<VideoCodecSettings> before, 586 std::vector<VideoCodecSettings> after); 587 588 void FillSenderStats(VideoMediaInfo* info, bool log_stats) 589 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 590 void FillReceiverStats(VideoMediaInfo* info, bool log_stats) 591 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 592 void FillBandwidthEstimationStats(const webrtc::Call::Stats& stats, 593 VideoMediaInfo* info) 594 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 595 void FillSendAndReceiveCodecStats(VideoMediaInfo* video_media_info) 596 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); 597 598 webrtc::TaskQueueBase* const worker_thread_; 599 webrtc::ScopedTaskSafety task_safety_; 600 RTC_NO_UNIQUE_ADDRESS webrtc::SequenceChecker network_thread_checker_; 601 RTC_NO_UNIQUE_ADDRESS webrtc::SequenceChecker thread_checker_; 602 603 uint32_t rtcp_receiver_report_ssrc_ RTC_GUARDED_BY(thread_checker_); 604 bool sending_ RTC_GUARDED_BY(thread_checker_); 605 webrtc::Call* const call_; 606 607 DefaultUnsignalledSsrcHandler default_unsignalled_ssrc_handler_ 608 RTC_GUARDED_BY(thread_checker_); 609 UnsignalledSsrcHandler* const unsignalled_ssrc_handler_ 610 RTC_GUARDED_BY(thread_checker_); 611 612 // Delay for unsignaled streams, which may be set before the stream exists. 613 int default_recv_base_minimum_delay_ms_ RTC_GUARDED_BY(thread_checker_) = 0; 614 615 const MediaConfig::Video video_config_ RTC_GUARDED_BY(thread_checker_); 616 617 // Using primary-ssrc (first ssrc) as key. 618 std::map<uint32_t, WebRtcVideoSendStream*> send_streams_ 619 RTC_GUARDED_BY(thread_checker_); 620 std::map<uint32_t, WebRtcVideoReceiveStream*> receive_streams_ 621 RTC_GUARDED_BY(thread_checker_); 622 // When the channel and demuxer get reconfigured, there is a window of time 623 // where we have to be prepared for packets arriving based on the old demuxer 624 // criteria because the streams live on the worker thread and the demuxer 625 // lives on the network thread. Because packets are posted from the network 626 // thread to the worker thread, they can still be in-flight when streams are 627 // reconfgured. This can happen when `demuxer_criteria_id_` and 628 // `demuxer_criteria_completed_id_` don't match. During this time, we do not 629 // want to create unsignalled receive streams and should instead drop the 630 // packets. E.g: 631 // * If RemoveRecvStream(old_ssrc) was recently called, there may be packets 632 // in-flight for that ssrc. This happens when a receiver becomes inactive. 633 // * If we go from one to many m= sections, the demuxer may change from 634 // forwarding all packets to only forwarding the configured ssrcs, so there 635 // is a risk of receiving ssrcs for other, recently added m= sections. 636 uint32_t demuxer_criteria_id_ RTC_GUARDED_BY(thread_checker_) = 0; 637 uint32_t demuxer_criteria_completed_id_ RTC_GUARDED_BY(thread_checker_) = 0; 638 absl::optional<int64_t> last_unsignalled_ssrc_creation_time_ms_ 639 RTC_GUARDED_BY(thread_checker_); 640 std::set<uint32_t> send_ssrcs_ RTC_GUARDED_BY(thread_checker_); 641 std::set<uint32_t> receive_ssrcs_ RTC_GUARDED_BY(thread_checker_); 642 643 absl::optional<VideoCodecSettings> send_codec_ 644 RTC_GUARDED_BY(thread_checker_); 645 std::vector<VideoCodecSettings> negotiated_codecs_ 646 RTC_GUARDED_BY(thread_checker_); 647 648 std::vector<webrtc::RtpExtension> send_rtp_extensions_ 649 RTC_GUARDED_BY(thread_checker_); 650 651 webrtc::VideoEncoderFactory* const encoder_factory_ 652 RTC_GUARDED_BY(thread_checker_); 653 webrtc::VideoDecoderFactory* const decoder_factory_ 654 RTC_GUARDED_BY(thread_checker_); 655 webrtc::VideoBitrateAllocatorFactory* const bitrate_allocator_factory_ 656 RTC_GUARDED_BY(thread_checker_); 657 std::vector<VideoCodecSettings> recv_codecs_ RTC_GUARDED_BY(thread_checker_); 658 std::vector<webrtc::RtpExtension> recv_rtp_extensions_ 659 RTC_GUARDED_BY(thread_checker_); 660 // See reason for keeping track of the FlexFEC payload type separately in 661 // comment in WebRtcVideoChannel::ChangedRecvParameters. 662 int recv_flexfec_payload_type_ RTC_GUARDED_BY(thread_checker_); 663 webrtc::BitrateConstraints bitrate_config_ RTC_GUARDED_BY(thread_checker_); 664 // TODO(deadbeef): Don't duplicate information between 665 // send_params/recv_params, rtp_extensions, options, etc. 666 VideoSendParameters send_params_ RTC_GUARDED_BY(thread_checker_); 667 VideoOptions default_send_options_ RTC_GUARDED_BY(thread_checker_); 668 VideoRecvParameters recv_params_ RTC_GUARDED_BY(thread_checker_); 669 int64_t last_stats_log_ms_ RTC_GUARDED_BY(thread_checker_); 670 const bool discard_unknown_ssrc_packets_ RTC_GUARDED_BY(thread_checker_); 671 // This is a stream param that comes from the remote description, but wasn't 672 // signaled with any a=ssrc lines. It holds information that was signaled 673 // before the unsignaled receive stream is created when the first packet is 674 // received. 675 StreamParams unsignaled_stream_params_ RTC_GUARDED_BY(thread_checker_); 676 // Per peer connection crypto options that last for the lifetime of the peer 677 // connection. 678 const webrtc::CryptoOptions crypto_options_ RTC_GUARDED_BY(thread_checker_); 679 680 // Optional frame transformer set on unsignaled streams. 681 rtc::scoped_refptr<webrtc::FrameTransformerInterface> 682 unsignaled_frame_transformer_ RTC_GUARDED_BY(thread_checker_); 683 684 // Buffer for unhandled packets. 685 std::unique_ptr<UnhandledPacketsBuffer> unknown_ssrc_packet_buffer_ 686 RTC_GUARDED_BY(thread_checker_); 687 688 // TODO(bugs.webrtc.org/11341): Remove this and relevant PC API. Presence 689 // of multiple negotiated codecs allows generic encoder fallback on failures. 690 // Presence of EncoderSelector allows switching to specific encoders. 691 bool allow_codec_switching_ = false; 692 }; 693 694 } // namespace cricket 695 696 #endif // MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_ 697