1 /* 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef CALL_RTP_VIDEO_SENDER_H_ 12 #define CALL_RTP_VIDEO_SENDER_H_ 13 14 #include <map> 15 #include <memory> 16 #include <unordered_set> 17 #include <vector> 18 19 #include "absl/types/optional.h" 20 #include "api/array_view.h" 21 #include "api/call/transport.h" 22 #include "api/fec_controller.h" 23 #include "api/fec_controller_override.h" 24 #include "api/field_trials_view.h" 25 #include "api/rtc_event_log/rtc_event_log.h" 26 #include "api/sequence_checker.h" 27 #include "api/task_queue/task_queue_base.h" 28 #include "api/task_queue/task_queue_factory.h" 29 #include "api/video_codecs/video_encoder.h" 30 #include "call/rtp_config.h" 31 #include "call/rtp_payload_params.h" 32 #include "call/rtp_transport_controller_send_interface.h" 33 #include "call/rtp_video_sender_interface.h" 34 #include "modules/rtp_rtcp/include/flexfec_sender.h" 35 #include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" 36 #include "modules/rtp_rtcp/source/rtp_sender.h" 37 #include "modules/rtp_rtcp/source/rtp_sender_video.h" 38 #include "modules/rtp_rtcp/source/rtp_sequence_number_map.h" 39 #include "modules/rtp_rtcp/source/rtp_video_header.h" 40 #include "rtc_base/rate_limiter.h" 41 #include "rtc_base/synchronization/mutex.h" 42 #include "rtc_base/thread_annotations.h" 43 44 namespace webrtc { 45 46 class FrameEncryptorInterface; 47 class RtpTransportControllerSendInterface; 48 49 namespace webrtc_internal_rtp_video_sender { 50 // RTP state for a single simulcast stream. Internal to the implementation of 51 // RtpVideoSender. 52 struct RtpStreamSender { 53 RtpStreamSender(std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp, 54 std::unique_ptr<RTPSenderVideo> sender_video, 55 std::unique_ptr<VideoFecGenerator> fec_generator); 56 ~RtpStreamSender(); 57 58 RtpStreamSender(RtpStreamSender&&) = default; 59 RtpStreamSender& operator=(RtpStreamSender&&) = default; 60 61 // Note: Needs pointer stability. 62 std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp; 63 std::unique_ptr<RTPSenderVideo> sender_video; 64 std::unique_ptr<VideoFecGenerator> fec_generator; 65 }; 66 67 } // namespace webrtc_internal_rtp_video_sender 68 69 // RtpVideoSender routes outgoing data to the correct sending RTP module, based 70 // on the simulcast layer in RTPVideoHeader. 71 class RtpVideoSender : public RtpVideoSenderInterface, 72 public VCMProtectionCallback, 73 public StreamFeedbackObserver { 74 public: 75 // Rtp modules are assumed to be sorted in simulcast index order. 76 RtpVideoSender( 77 Clock* clock, 78 const std::map<uint32_t, RtpState>& suspended_ssrcs, 79 const std::map<uint32_t, RtpPayloadState>& states, 80 const RtpConfig& rtp_config, 81 int rtcp_report_interval_ms, 82 Transport* send_transport, 83 const RtpSenderObservers& observers, 84 RtpTransportControllerSendInterface* transport, 85 RtcEventLog* event_log, 86 RateLimiter* retransmission_limiter, // move inside RtpTransport 87 std::unique_ptr<FecController> fec_controller, 88 FrameEncryptorInterface* frame_encryptor, 89 const CryptoOptions& crypto_options, // move inside RtpTransport 90 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, 91 const FieldTrialsView& field_trials, 92 TaskQueueFactory* task_queue_factory); 93 ~RtpVideoSender() override; 94 95 RtpVideoSender(const RtpVideoSender&) = delete; 96 RtpVideoSender& operator=(const RtpVideoSender&) = delete; 97 98 // Sets the sending status of the rtp modules and appropriately sets the 99 // payload router to active if any rtp modules are active. 100 void SetActiveModules(const std::vector<bool>& active_modules) 101 RTC_LOCKS_EXCLUDED(mutex_) override; 102 void Stop() RTC_LOCKS_EXCLUDED(mutex_) override; 103 bool IsActive() RTC_LOCKS_EXCLUDED(mutex_) override; 104 105 void OnNetworkAvailability(bool network_available) 106 RTC_LOCKS_EXCLUDED(mutex_) override; 107 std::map<uint32_t, RtpState> GetRtpStates() const 108 RTC_LOCKS_EXCLUDED(mutex_) override; 109 std::map<uint32_t, RtpPayloadState> GetRtpPayloadStates() const 110 RTC_LOCKS_EXCLUDED(mutex_) override; 111 112 void DeliverRtcp(const uint8_t* packet, size_t length) 113 RTC_LOCKS_EXCLUDED(mutex_) override; 114 115 // Implements webrtc::VCMProtectionCallback. 116 int ProtectionRequest(const FecProtectionParams* delta_params, 117 const FecProtectionParams* key_params, 118 uint32_t* sent_video_rate_bps, 119 uint32_t* sent_nack_rate_bps, 120 uint32_t* sent_fec_rate_bps) 121 RTC_LOCKS_EXCLUDED(mutex_) override; 122 123 // Implements FecControllerOverride. 124 void SetFecAllowed(bool fec_allowed) RTC_LOCKS_EXCLUDED(mutex_) override; 125 126 // Implements EncodedImageCallback. 127 // Returns 0 if the packet was routed / sent, -1 otherwise. 128 EncodedImageCallback::Result OnEncodedImage( 129 const EncodedImage& encoded_image, 130 const CodecSpecificInfo* codec_specific_info) 131 RTC_LOCKS_EXCLUDED(mutex_) override; 132 133 void OnBitrateAllocationUpdated(const VideoBitrateAllocation& bitrate) 134 RTC_LOCKS_EXCLUDED(mutex_) override; 135 void OnVideoLayersAllocationUpdated( 136 const VideoLayersAllocation& layers) override; 137 void OnTransportOverheadChanged(size_t transport_overhead_bytes_per_packet) 138 RTC_LOCKS_EXCLUDED(mutex_) override; 139 void OnBitrateUpdated(BitrateAllocationUpdate update, int framerate) 140 RTC_LOCKS_EXCLUDED(mutex_) override; 141 uint32_t GetPayloadBitrateBps() const RTC_LOCKS_EXCLUDED(mutex_) override; 142 uint32_t GetProtectionBitrateBps() const RTC_LOCKS_EXCLUDED(mutex_) override; 143 void SetEncodingData(size_t width, size_t height, size_t num_temporal_layers) 144 RTC_LOCKS_EXCLUDED(mutex_) override; 145 146 std::vector<RtpSequenceNumberMap::Info> GetSentRtpPacketInfos( 147 uint32_t ssrc, 148 rtc::ArrayView<const uint16_t> sequence_numbers) const 149 RTC_LOCKS_EXCLUDED(mutex_) override; 150 151 // From StreamFeedbackObserver. 152 void OnPacketFeedbackVector( 153 std::vector<StreamPacketInfo> packet_feedback_vector) 154 RTC_LOCKS_EXCLUDED(mutex_) override; 155 156 private: 157 bool IsActiveLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 158 void SetActiveModulesLocked(const std::vector<bool>& active_modules) 159 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 160 void UpdateModuleSendingState() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 161 void ConfigureProtection(); 162 void ConfigureSsrcs(const std::map<uint32_t, RtpState>& suspended_ssrcs); 163 bool NackEnabled() const; 164 uint32_t GetPacketizationOverheadRate() const; 165 DataRate CalculateOverheadRate(DataRate data_rate, 166 DataSize packet_size, 167 DataSize overhead_per_packet, 168 Frequency framerate) const; 169 170 const FieldTrialsView& field_trials_; 171 const bool use_frame_rate_for_overhead_; 172 const bool has_packet_feedback_; 173 174 // Semantically equivalent to checking for `transport_->GetWorkerQueue()` 175 // but some tests need to be updated to call from the correct context. 176 RTC_NO_UNIQUE_ADDRESS SequenceChecker transport_checker_; 177 178 // TODO(bugs.webrtc.org/13517): Remove mutex_ once RtpVideoSender runs on the 179 // transport task queue. 180 mutable Mutex mutex_; 181 bool active_ RTC_GUARDED_BY(mutex_); 182 bool registered_for_feedback_ RTC_GUARDED_BY(transport_checker_) = false; 183 184 const std::unique_ptr<FecController> fec_controller_; 185 bool fec_allowed_ RTC_GUARDED_BY(mutex_); 186 187 // Rtp modules are assumed to be sorted in simulcast index order. 188 const std::vector<webrtc_internal_rtp_video_sender::RtpStreamSender> 189 rtp_streams_; 190 const RtpConfig rtp_config_; 191 const absl::optional<VideoCodecType> codec_type_; 192 RtpTransportControllerSendInterface* const transport_; 193 194 // When using the generic descriptor we want all simulcast streams to share 195 // one frame id space (so that the SFU can switch stream without having to 196 // rewrite the frame id), therefore `shared_frame_id` has to live in a place 197 // where we are aware of all the different streams. 198 int64_t shared_frame_id_ = 0; 199 std::vector<RtpPayloadParams> params_ RTC_GUARDED_BY(mutex_); 200 201 size_t transport_overhead_bytes_per_packet_ RTC_GUARDED_BY(mutex_); 202 uint32_t protection_bitrate_bps_; 203 uint32_t encoder_target_rate_bps_; 204 205 std::vector<bool> loss_mask_vector_ RTC_GUARDED_BY(mutex_); 206 207 std::vector<FrameCounts> frame_counts_ RTC_GUARDED_BY(mutex_); 208 FrameCountObserver* const frame_count_observer_; 209 210 // Effectively const map from SSRC to RtpRtcp, for all media SSRCs. 211 // This map is set at construction time and never changed, but it's 212 // non-trivial to make it properly const. 213 std::map<uint32_t, RtpRtcpInterface*> ssrc_to_rtp_module_; 214 }; 215 216 } // namespace webrtc 217 218 #endif // CALL_RTP_VIDEO_SENDER_H_ 219