1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_H_ 12 #define MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_H_ 13 14 #include <map> 15 #include <memory> 16 #include <vector> 17 18 #include "absl/strings/string_view.h" 19 #include "absl/types/optional.h" 20 #include "api/array_view.h" 21 #include "api/frame_transformer_interface.h" 22 #include "api/scoped_refptr.h" 23 #include "api/task_queue/task_queue_base.h" 24 #include "api/transport/rtp/dependency_descriptor.h" 25 #include "api/video/video_codec_type.h" 26 #include "api/video/video_frame_type.h" 27 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" 28 #include "modules/rtp_rtcp/source/absolute_capture_time_sender.h" 29 #include "modules/rtp_rtcp/source/active_decode_targets_helper.h" 30 #include "modules/rtp_rtcp/source/rtp_rtcp_config.h" 31 #include "modules/rtp_rtcp/source/rtp_sender.h" 32 #include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h" 33 #include "modules/rtp_rtcp/source/rtp_video_header.h" 34 #include "modules/rtp_rtcp/source/video_fec_generator.h" 35 #include "rtc_base/deprecation.h" 36 #include "rtc_base/one_time_event.h" 37 #include "rtc_base/race_checker.h" 38 #include "rtc_base/rate_statistics.h" 39 #include "rtc_base/synchronization/mutex.h" 40 #include "rtc_base/synchronization/sequence_checker.h" 41 #include "rtc_base/thread_annotations.h" 42 43 namespace webrtc { 44 45 class RTPFragmentationHeader; 46 class FrameEncryptorInterface; 47 class RtpPacketizer; 48 class RtpPacketToSend; 49 50 // kConditionallyRetransmitHigherLayers allows retransmission of video frames 51 // in higher layers if either the last frame in that layer was too far back in 52 // time, or if we estimate that a new frame will be available in a lower layer 53 // in a shorter time than it would take to request and receive a retransmission. 54 enum RetransmissionMode : uint8_t { 55 kRetransmitOff = 0x0, 56 kRetransmitBaseLayer = 0x2, 57 kRetransmitHigherLayers = 0x4, 58 kRetransmitAllLayers = 0x6, 59 kConditionallyRetransmitHigherLayers = 0x8 60 }; 61 62 class RTPSenderVideo { 63 public: 64 static constexpr int64_t kTLRateWindowSizeMs = 2500; 65 66 struct Config { 67 Config() = default; 68 Config(const Config&) = delete; 69 Config(Config&&) = default; 70 71 // All members of this struct, with the exception of |field_trials|, are 72 // expected to outlive the RTPSenderVideo object they are passed to. 73 Clock* clock = nullptr; 74 RTPSender* rtp_sender = nullptr; 75 FlexfecSender* flexfec_sender = nullptr; 76 VideoFecGenerator* fec_generator = nullptr; 77 // Some FEC data is duplicated here in preparation of moving FEC to 78 // the egress stage. 79 absl::optional<VideoFecGenerator::FecType> fec_type; 80 size_t fec_overhead_bytes = 0; // Per packet max FEC overhead. 81 FrameEncryptorInterface* frame_encryptor = nullptr; 82 bool require_frame_encryption = false; 83 bool enable_retransmit_all_layers = false; 84 absl::optional<int> red_payload_type; 85 const WebRtcKeyValueConfig* field_trials = nullptr; 86 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer; 87 TaskQueueBase* send_transport_queue = nullptr; 88 }; 89 90 explicit RTPSenderVideo(const Config& config); 91 92 virtual ~RTPSenderVideo(); 93 94 RTC_DEPRECATED SendVideo(int payload_type,absl::optional<VideoCodecType> codec_type,uint32_t rtp_timestamp,int64_t capture_time_ms,rtc::ArrayView<const uint8_t> payload,const RTPFragmentationHeader *,RTPVideoHeader video_header,absl::optional<int64_t> expected_retransmission_time_ms)95 bool SendVideo(int payload_type, 96 absl::optional<VideoCodecType> codec_type, 97 uint32_t rtp_timestamp, 98 int64_t capture_time_ms, 99 rtc::ArrayView<const uint8_t> payload, 100 const RTPFragmentationHeader* /*fragmentation*/, 101 RTPVideoHeader video_header, 102 absl::optional<int64_t> expected_retransmission_time_ms) { 103 return SendVideo(payload_type, codec_type, rtp_timestamp, capture_time_ms, 104 payload, video_header, expected_retransmission_time_ms); 105 } 106 107 // expected_retransmission_time_ms.has_value() -> retransmission allowed. 108 // Calls to this method is assumed to be externally serialized. 109 bool SendVideo(int payload_type, 110 absl::optional<VideoCodecType> codec_type, 111 uint32_t rtp_timestamp, 112 int64_t capture_time_ms, 113 rtc::ArrayView<const uint8_t> payload, 114 RTPVideoHeader video_header, 115 absl::optional<int64_t> expected_retransmission_time_ms); 116 117 bool SendEncodedImage( 118 int payload_type, 119 absl::optional<VideoCodecType> codec_type, 120 uint32_t rtp_timestamp, 121 const EncodedImage& encoded_image, 122 RTPVideoHeader video_header, 123 absl::optional<int64_t> expected_retransmission_time_ms); 124 125 // Configures video structures produced by encoder to send using the 126 // dependency descriptor rtp header extension. Next call to SendVideo should 127 // have video_header.frame_type == kVideoFrameKey. 128 // All calls to SendVideo after this call must use video_header compatible 129 // with the video_structure. 130 void SetVideoStructure(const FrameDependencyStructure* video_structure); 131 void SetVideoStructureUnderLock( 132 const FrameDependencyStructure* video_structure); 133 134 uint32_t VideoBitrateSent() const; 135 136 // Returns the current packetization overhead rate, in bps. Note that this is 137 // the payload overhead, eg the VP8 payload headers, not the RTP headers 138 // or extension/ 139 uint32_t PacketizationOverheadBps() const; 140 141 protected: 142 static uint8_t GetTemporalId(const RTPVideoHeader& header); 143 bool AllowRetransmission(uint8_t temporal_id, 144 int32_t retransmission_settings, 145 int64_t expected_retransmission_time_ms); 146 147 private: 148 struct TemporalLayerStats { TemporalLayerStatsTemporalLayerStats149 TemporalLayerStats() 150 : frame_rate_fp1000s(kTLRateWindowSizeMs, 1000 * 1000), 151 last_frame_time_ms(0) {} 152 // Frame rate, in frames per 1000 seconds. This essentially turns the fps 153 // value into a fixed point value with three decimals. Improves precision at 154 // low frame rates. 155 RateStatistics frame_rate_fp1000s; 156 int64_t last_frame_time_ms; 157 }; 158 159 void AddRtpHeaderExtensions( 160 const RTPVideoHeader& video_header, 161 const absl::optional<AbsoluteCaptureTime>& absolute_capture_time, 162 bool first_packet, 163 bool last_packet, 164 RtpPacketToSend* packet) const 165 RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_); 166 167 size_t FecPacketOverhead() const RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_); 168 169 void LogAndSendToNetwork( 170 std::vector<std::unique_ptr<RtpPacketToSend>> packets, 171 size_t unpacketized_payload_size); 172 red_enabled()173 bool red_enabled() const { return red_payload_type_.has_value(); } 174 175 bool UpdateConditionalRetransmit(uint8_t temporal_id, 176 int64_t expected_retransmission_time_ms) 177 RTC_EXCLUSIVE_LOCKS_REQUIRED(stats_mutex_); 178 179 void MaybeUpdateCurrentPlayoutDelay(const RTPVideoHeader& header) 180 RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_); 181 182 RTPSender* const rtp_sender_; 183 Clock* const clock_; 184 185 const int32_t retransmission_settings_; 186 187 // These members should only be accessed from within SendVideo() to avoid 188 // potential race conditions. 189 rtc::RaceChecker send_checker_; 190 VideoRotation last_rotation_ RTC_GUARDED_BY(send_checker_); 191 absl::optional<ColorSpace> last_color_space_ RTC_GUARDED_BY(send_checker_); 192 bool transmit_color_space_next_frame_ RTC_GUARDED_BY(send_checker_); 193 std::unique_ptr<FrameDependencyStructure> video_structure_ 194 RTC_GUARDED_BY(send_checker_); 195 196 // Current target playout delay. 197 PlayoutDelay current_playout_delay_ RTC_GUARDED_BY(send_checker_); 198 // Flag indicating if we need to propagate |current_playout_delay_| in order 199 // to guarantee it gets delivered. 200 bool playout_delay_pending_; 201 202 // Should never be held when calling out of this class. 203 Mutex mutex_; 204 205 const absl::optional<int> red_payload_type_; 206 VideoFecGenerator* const fec_generator_; 207 absl::optional<VideoFecGenerator::FecType> fec_type_; 208 const size_t fec_overhead_bytes_; // Per packet max FEC overhead. 209 210 mutable Mutex stats_mutex_; 211 // Bitrate used for video payload and RTP headers. 212 RateStatistics video_bitrate_ RTC_GUARDED_BY(stats_mutex_); 213 RateStatistics packetization_overhead_bitrate_ RTC_GUARDED_BY(stats_mutex_); 214 215 std::map<int, TemporalLayerStats> frame_stats_by_temporal_layer_ 216 RTC_GUARDED_BY(stats_mutex_); 217 218 OneTimeEvent first_frame_sent_; 219 220 // E2EE Custom Video Frame Encryptor (optional) 221 FrameEncryptorInterface* const frame_encryptor_ = nullptr; 222 // If set to true will require all outgoing frames to pass through an 223 // initialized frame_encryptor_ before being sent out of the network. 224 // Otherwise these payloads will be dropped. 225 const bool require_frame_encryption_; 226 // Set to true if the generic descriptor should be authenticated. 227 const bool generic_descriptor_auth_experiment_; 228 229 AbsoluteCaptureTimeSender absolute_capture_time_sender_; 230 // Tracks updates to the active decode targets and decides when active decode 231 // targets bitmask should be attached to the dependency descriptor. 232 ActiveDecodeTargetsHelper active_decode_targets_tracker_; 233 234 const rtc::scoped_refptr<RTPSenderVideoFrameTransformerDelegate> 235 frame_transformer_delegate_; 236 }; 237 238 } // namespace webrtc 239 240 #endif // MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_H_ 241