1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_H_ 12 #define MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_H_ 13 14 #include <map> 15 #include <memory> 16 #include <vector> 17 18 #include "absl/strings/string_view.h" 19 #include "absl/types/optional.h" 20 #include "api/array_view.h" 21 #include "api/frame_transformer_interface.h" 22 #include "api/scoped_refptr.h" 23 #include "api/sequence_checker.h" 24 #include "api/task_queue/task_queue_base.h" 25 #include "api/task_queue/task_queue_factory.h" 26 #include "api/transport/rtp/dependency_descriptor.h" 27 #include "api/video/video_codec_type.h" 28 #include "api/video/video_frame_type.h" 29 #include "api/video/video_layers_allocation.h" 30 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" 31 #include "modules/rtp_rtcp/source/absolute_capture_time_sender.h" 32 #include "modules/rtp_rtcp/source/active_decode_targets_helper.h" 33 #include "modules/rtp_rtcp/source/rtp_rtcp_config.h" 34 #include "modules/rtp_rtcp/source/rtp_sender.h" 35 #include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h" 36 #include "modules/rtp_rtcp/source/rtp_video_header.h" 37 #include "modules/rtp_rtcp/source/video_fec_generator.h" 38 #include "rtc_base/one_time_event.h" 39 #include "rtc_base/race_checker.h" 40 #include "rtc_base/rate_statistics.h" 41 #include "rtc_base/synchronization/mutex.h" 42 #include "rtc_base/thread_annotations.h" 43 44 namespace webrtc { 45 46 class FrameEncryptorInterface; 47 class RtpPacketizer; 48 class RtpPacketToSend; 49 50 // kConditionallyRetransmitHigherLayers allows retransmission of video frames 51 // in higher layers if either the last frame in that layer was too far back in 52 // time, or if we estimate that a new frame will be available in a lower layer 53 // in a shorter time than it would take to request and receive a retransmission. 54 enum RetransmissionMode : uint8_t { 55 kRetransmitOff = 0x0, 56 kRetransmitBaseLayer = 0x2, 57 kRetransmitHigherLayers = 0x4, 58 kRetransmitAllLayers = 0x6, 59 kConditionallyRetransmitHigherLayers = 0x8 60 }; 61 62 class RTPSenderVideo { 63 public: 64 static constexpr int64_t kTLRateWindowSizeMs = 2500; 65 66 struct Config { 67 Config() = default; 68 Config(const Config&) = delete; 69 Config(Config&&) = default; 70 71 // All members of this struct, with the exception of `field_trials`, are 72 // expected to outlive the RTPSenderVideo object they are passed to. 73 Clock* clock = nullptr; 74 RTPSender* rtp_sender = nullptr; 75 // Some FEC data is duplicated here in preparation of moving FEC to 76 // the egress stage. 77 absl::optional<VideoFecGenerator::FecType> fec_type; 78 size_t fec_overhead_bytes = 0; // Per packet max FEC overhead. 79 FrameEncryptorInterface* frame_encryptor = nullptr; 80 bool require_frame_encryption = false; 81 bool enable_retransmit_all_layers = false; 82 absl::optional<int> red_payload_type; 83 const FieldTrialsView* field_trials = nullptr; 84 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer; 85 TaskQueueFactory* task_queue_factory = nullptr; 86 }; 87 88 explicit RTPSenderVideo(const Config& config); 89 90 virtual ~RTPSenderVideo(); 91 92 // expected_retransmission_time_ms.has_value() -> retransmission allowed. 93 // `capture_time_ms` and `clock::CurrentTime` should be using the same epoch. 94 // Calls to this method are assumed to be externally serialized. 95 bool SendVideo(int payload_type, 96 absl::optional<VideoCodecType> codec_type, 97 uint32_t rtp_timestamp, 98 int64_t capture_time_ms, 99 rtc::ArrayView<const uint8_t> payload, 100 RTPVideoHeader video_header, 101 absl::optional<int64_t> expected_retransmission_time_ms); 102 103 bool SendEncodedImage( 104 int payload_type, 105 absl::optional<VideoCodecType> codec_type, 106 uint32_t rtp_timestamp, 107 const EncodedImage& encoded_image, 108 RTPVideoHeader video_header, 109 absl::optional<int64_t> expected_retransmission_time_ms); 110 111 // Configures video structures produced by encoder to send using the 112 // dependency descriptor rtp header extension. Next call to SendVideo should 113 // have video_header.frame_type == kVideoFrameKey. 114 // All calls to SendVideo after this call must use video_header compatible 115 // with the video_structure. 116 void SetVideoStructure(const FrameDependencyStructure* video_structure); 117 // Should only be used by a RTPSenderVideoFrameTransformerDelegate and exists 118 // to ensure correct syncronization. 119 void SetVideoStructureAfterTransformation( 120 const FrameDependencyStructure* video_structure); 121 122 // Sets current active VideoLayersAllocation. The allocation will be sent 123 // using the rtp video layers allocation extension. The allocation will be 124 // sent in full on every key frame. The allocation will be sent once on a 125 // none discardable delta frame per call to this method and will not contain 126 // resolution and frame rate. 127 void SetVideoLayersAllocation(VideoLayersAllocation allocation); 128 // Should only be used by a RTPSenderVideoFrameTransformerDelegate and exists 129 // to ensure correct syncronization. 130 void SetVideoLayersAllocationAfterTransformation( 131 VideoLayersAllocation allocation); 132 133 // Returns the current packetization overhead rate, in bps. Note that this is 134 // the payload overhead, eg the VP8 payload headers, not the RTP headers 135 // or extension/ 136 // TODO(sprang): Consider moving this to RtpSenderEgress so it's in the same 137 // place as the other rate stats. 138 uint32_t PacketizationOverheadBps() const; 139 140 protected: 141 static uint8_t GetTemporalId(const RTPVideoHeader& header); 142 bool AllowRetransmission(uint8_t temporal_id, 143 int32_t retransmission_settings, 144 int64_t expected_retransmission_time_ms); 145 146 private: 147 struct TemporalLayerStats { TemporalLayerStatsTemporalLayerStats148 TemporalLayerStats() 149 : frame_rate_fp1000s(kTLRateWindowSizeMs, 1000 * 1000), 150 last_frame_time_ms(0) {} 151 // Frame rate, in frames per 1000 seconds. This essentially turns the fps 152 // value into a fixed point value with three decimals. Improves precision at 153 // low frame rates. 154 RateStatistics frame_rate_fp1000s; 155 int64_t last_frame_time_ms; 156 }; 157 158 enum class SendVideoLayersAllocation { 159 kSendWithResolution, 160 kSendWithoutResolution, 161 kDontSend 162 }; 163 164 void SetVideoStructureInternal( 165 const FrameDependencyStructure* video_structure); 166 void SetVideoLayersAllocationInternal(VideoLayersAllocation allocation); 167 168 void AddRtpHeaderExtensions(const RTPVideoHeader& video_header, 169 bool first_packet, 170 bool last_packet, 171 RtpPacketToSend* packet) const 172 RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_); 173 174 size_t FecPacketOverhead() const RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_); 175 176 void LogAndSendToNetwork( 177 std::vector<std::unique_ptr<RtpPacketToSend>> packets, 178 size_t unpacketized_payload_size); 179 red_enabled()180 bool red_enabled() const { return red_payload_type_.has_value(); } 181 182 bool UpdateConditionalRetransmit(uint8_t temporal_id, 183 int64_t expected_retransmission_time_ms) 184 RTC_EXCLUSIVE_LOCKS_REQUIRED(stats_mutex_); 185 186 void MaybeUpdateCurrentPlayoutDelay(const RTPVideoHeader& header) 187 RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_); 188 189 RTPSender* const rtp_sender_; 190 Clock* const clock_; 191 192 const int32_t retransmission_settings_; 193 194 // These members should only be accessed from within SendVideo() to avoid 195 // potential race conditions. 196 rtc::RaceChecker send_checker_; 197 VideoRotation last_rotation_ RTC_GUARDED_BY(send_checker_); 198 absl::optional<ColorSpace> last_color_space_ RTC_GUARDED_BY(send_checker_); 199 bool transmit_color_space_next_frame_ RTC_GUARDED_BY(send_checker_); 200 std::unique_ptr<FrameDependencyStructure> video_structure_ 201 RTC_GUARDED_BY(send_checker_); 202 absl::optional<VideoLayersAllocation> allocation_ 203 RTC_GUARDED_BY(send_checker_); 204 // Flag indicating if we should send `allocation_`. 205 SendVideoLayersAllocation send_allocation_ RTC_GUARDED_BY(send_checker_); 206 absl::optional<VideoLayersAllocation> last_full_sent_allocation_ 207 RTC_GUARDED_BY(send_checker_); 208 209 // Current target playout delay. 210 VideoPlayoutDelay current_playout_delay_ RTC_GUARDED_BY(send_checker_); 211 // Flag indicating if we need to send `current_playout_delay_` in order 212 // to guarantee it gets delivered. 213 bool playout_delay_pending_; 214 // Set by the field trial WebRTC-ForceSendPlayoutDelay to override the playout 215 // delay of outgoing video frames. 216 const absl::optional<VideoPlayoutDelay> forced_playout_delay_; 217 218 // Should never be held when calling out of this class. 219 Mutex mutex_; 220 221 const absl::optional<int> red_payload_type_; 222 absl::optional<VideoFecGenerator::FecType> fec_type_; 223 const size_t fec_overhead_bytes_; // Per packet max FEC overhead. 224 225 mutable Mutex stats_mutex_; 226 RateStatistics packetization_overhead_bitrate_ RTC_GUARDED_BY(stats_mutex_); 227 228 std::map<int, TemporalLayerStats> frame_stats_by_temporal_layer_ 229 RTC_GUARDED_BY(stats_mutex_); 230 231 OneTimeEvent first_frame_sent_; 232 233 // E2EE Custom Video Frame Encryptor (optional) 234 FrameEncryptorInterface* const frame_encryptor_ = nullptr; 235 // If set to true will require all outgoing frames to pass through an 236 // initialized frame_encryptor_ before being sent out of the network. 237 // Otherwise these payloads will be dropped. 238 const bool require_frame_encryption_; 239 // Set to true if the generic descriptor should be authenticated. 240 const bool generic_descriptor_auth_experiment_; 241 242 AbsoluteCaptureTimeSender absolute_capture_time_sender_; 243 // Tracks updates to the active decode targets and decides when active decode 244 // targets bitmask should be attached to the dependency descriptor. 245 ActiveDecodeTargetsHelper active_decode_targets_tracker_; 246 247 const rtc::scoped_refptr<RTPSenderVideoFrameTransformerDelegate> 248 frame_transformer_delegate_; 249 250 const bool include_capture_clock_offset_; 251 }; 252 253 } // namespace webrtc 254 255 #endif // MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_H_ 256