1 /* 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef CALL_RTP_VIDEO_SENDER_H_ 12 #define CALL_RTP_VIDEO_SENDER_H_ 13 14 #include <map> 15 #include <memory> 16 #include <unordered_set> 17 #include <vector> 18 19 #include "absl/types/optional.h" 20 #include "api/array_view.h" 21 #include "api/call/transport.h" 22 #include "api/fec_controller.h" 23 #include "api/fec_controller_override.h" 24 #include "api/rtc_event_log/rtc_event_log.h" 25 #include "api/transport/field_trial_based_config.h" 26 #include "api/video_codecs/video_encoder.h" 27 #include "call/rtp_config.h" 28 #include "call/rtp_payload_params.h" 29 #include "call/rtp_transport_controller_send_interface.h" 30 #include "call/rtp_video_sender_interface.h" 31 #include "modules/rtp_rtcp/include/flexfec_sender.h" 32 #include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" 33 #include "modules/rtp_rtcp/source/rtp_sender.h" 34 #include "modules/rtp_rtcp/source/rtp_sender_video.h" 35 #include "modules/rtp_rtcp/source/rtp_sequence_number_map.h" 36 #include "modules/rtp_rtcp/source/rtp_video_header.h" 37 #include "modules/utility/include/process_thread.h" 38 #include "rtc_base/constructor_magic.h" 39 #include "rtc_base/rate_limiter.h" 40 #include "rtc_base/synchronization/mutex.h" 41 #include "rtc_base/thread_annotations.h" 42 #include "rtc_base/thread_checker.h" 43 44 namespace webrtc { 45 46 class FrameEncryptorInterface; 47 class RTPFragmentationHeader; 48 class RtpTransportControllerSendInterface; 49 50 namespace webrtc_internal_rtp_video_sender { 51 // RTP state for a single simulcast stream. Internal to the implementation of 52 // RtpVideoSender. 53 struct RtpStreamSender { 54 RtpStreamSender(std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp, 55 std::unique_ptr<RTPSenderVideo> sender_video, 56 std::unique_ptr<VideoFecGenerator> fec_generator); 57 ~RtpStreamSender(); 58 59 RtpStreamSender(RtpStreamSender&&) = default; 60 RtpStreamSender& operator=(RtpStreamSender&&) = default; 61 62 // Note: Needs pointer stability. 63 std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp; 64 std::unique_ptr<RTPSenderVideo> sender_video; 65 std::unique_ptr<VideoFecGenerator> fec_generator; 66 }; 67 68 } // namespace webrtc_internal_rtp_video_sender 69 70 // RtpVideoSender routes outgoing data to the correct sending RTP module, based 71 // on the simulcast layer in RTPVideoHeader. 72 class RtpVideoSender : public RtpVideoSenderInterface, 73 public VCMProtectionCallback, 74 public StreamFeedbackObserver { 75 public: 76 // Rtp modules are assumed to be sorted in simulcast index order. 77 RtpVideoSender( 78 Clock* clock, 79 std::map<uint32_t, RtpState> suspended_ssrcs, 80 const std::map<uint32_t, RtpPayloadState>& states, 81 const RtpConfig& rtp_config, 82 int rtcp_report_interval_ms, 83 Transport* send_transport, 84 const RtpSenderObservers& observers, 85 RtpTransportControllerSendInterface* transport, 86 RtcEventLog* event_log, 87 RateLimiter* retransmission_limiter, // move inside RtpTransport 88 std::unique_ptr<FecController> fec_controller, 89 FrameEncryptorInterface* frame_encryptor, 90 const CryptoOptions& crypto_options, // move inside RtpTransport 91 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer); 92 ~RtpVideoSender() override; 93 94 // RegisterProcessThread register |module_process_thread| with those objects 95 // that use it. Registration has to happen on the thread were 96 // |module_process_thread| was created (libjingle's worker thread). 97 // TODO(perkj): Replace the use of |module_process_thread| with a TaskQueue, 98 // maybe |worker_queue|. 99 void RegisterProcessThread(ProcessThread* module_process_thread) 100 RTC_LOCKS_EXCLUDED(mutex_) override; 101 void DeRegisterProcessThread() RTC_LOCKS_EXCLUDED(mutex_) override; 102 103 // RtpVideoSender will only route packets if being active, all packets will be 104 // dropped otherwise. 105 void SetActive(bool active) RTC_LOCKS_EXCLUDED(mutex_) override; 106 // Sets the sending status of the rtp modules and appropriately sets the 107 // payload router to active if any rtp modules are active. 108 void SetActiveModules(const std::vector<bool> active_modules) 109 RTC_LOCKS_EXCLUDED(mutex_) override; 110 bool IsActive() RTC_LOCKS_EXCLUDED(mutex_) override; 111 112 void OnNetworkAvailability(bool network_available) 113 RTC_LOCKS_EXCLUDED(mutex_) override; 114 std::map<uint32_t, RtpState> GetRtpStates() const 115 RTC_LOCKS_EXCLUDED(mutex_) override; 116 std::map<uint32_t, RtpPayloadState> GetRtpPayloadStates() const 117 RTC_LOCKS_EXCLUDED(mutex_) override; 118 119 void DeliverRtcp(const uint8_t* packet, size_t length) 120 RTC_LOCKS_EXCLUDED(mutex_) override; 121 122 // Implements webrtc::VCMProtectionCallback. 123 int ProtectionRequest(const FecProtectionParams* delta_params, 124 const FecProtectionParams* key_params, 125 uint32_t* sent_video_rate_bps, 126 uint32_t* sent_nack_rate_bps, 127 uint32_t* sent_fec_rate_bps) 128 RTC_LOCKS_EXCLUDED(mutex_) override; 129 130 // Implements FecControllerOverride. 131 void SetFecAllowed(bool fec_allowed) RTC_LOCKS_EXCLUDED(mutex_) override; 132 133 // Implements EncodedImageCallback. 134 // Returns 0 if the packet was routed / sent, -1 otherwise. 135 EncodedImageCallback::Result OnEncodedImage( 136 const EncodedImage& encoded_image, 137 const CodecSpecificInfo* codec_specific_info, 138 const RTPFragmentationHeader* fragmentation) 139 RTC_LOCKS_EXCLUDED(mutex_) override; 140 141 void OnBitrateAllocationUpdated(const VideoBitrateAllocation& bitrate) 142 RTC_LOCKS_EXCLUDED(mutex_) override; 143 144 void OnTransportOverheadChanged(size_t transport_overhead_bytes_per_packet) 145 RTC_LOCKS_EXCLUDED(mutex_) override; 146 void OnBitrateUpdated(BitrateAllocationUpdate update, int framerate) 147 RTC_LOCKS_EXCLUDED(mutex_) override; 148 uint32_t GetPayloadBitrateBps() const RTC_LOCKS_EXCLUDED(mutex_) override; 149 uint32_t GetProtectionBitrateBps() const RTC_LOCKS_EXCLUDED(mutex_) override; 150 void SetEncodingData(size_t width, size_t height, size_t num_temporal_layers) 151 RTC_LOCKS_EXCLUDED(mutex_) override; 152 153 std::vector<RtpSequenceNumberMap::Info> GetSentRtpPacketInfos( 154 uint32_t ssrc, 155 rtc::ArrayView<const uint16_t> sequence_numbers) const 156 RTC_LOCKS_EXCLUDED(mutex_) override; 157 158 // From StreamFeedbackObserver. 159 void OnPacketFeedbackVector( 160 std::vector<StreamPacketInfo> packet_feedback_vector) 161 RTC_LOCKS_EXCLUDED(mutex_) override; 162 163 private: 164 bool IsActiveLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 165 void SetActiveModulesLocked(const std::vector<bool> active_modules) 166 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 167 void UpdateModuleSendingState() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 168 void ConfigureProtection(); 169 void ConfigureSsrcs(); 170 void ConfigureRids(); 171 bool NackEnabled() const; 172 uint32_t GetPacketizationOverheadRate() const; 173 174 const FieldTrialBasedConfig field_trials_; 175 const bool send_side_bwe_with_overhead_; 176 const bool account_for_packetization_overhead_; 177 const bool use_early_loss_detection_; 178 const bool has_packet_feedback_; 179 const bool use_deferred_fec_; 180 181 // TODO(holmer): Remove mutex_ once RtpVideoSender runs on the 182 // transport task queue. 183 mutable Mutex mutex_; 184 bool active_ RTC_GUARDED_BY(mutex_); 185 186 ProcessThread* module_process_thread_; 187 rtc::ThreadChecker module_process_thread_checker_; 188 std::map<uint32_t, RtpState> suspended_ssrcs_; 189 190 const std::unique_ptr<FecController> fec_controller_; 191 bool fec_allowed_ RTC_GUARDED_BY(mutex_); 192 193 // Rtp modules are assumed to be sorted in simulcast index order. 194 const std::vector<webrtc_internal_rtp_video_sender::RtpStreamSender> 195 rtp_streams_; 196 const RtpConfig rtp_config_; 197 const absl::optional<VideoCodecType> codec_type_; 198 RtpTransportControllerSendInterface* const transport_; 199 200 // When using the generic descriptor we want all simulcast streams to share 201 // one frame id space (so that the SFU can switch stream without having to 202 // rewrite the frame id), therefore |shared_frame_id| has to live in a place 203 // where we are aware of all the different streams. 204 int64_t shared_frame_id_ = 0; 205 std::vector<RtpPayloadParams> params_ RTC_GUARDED_BY(mutex_); 206 207 size_t transport_overhead_bytes_per_packet_ RTC_GUARDED_BY(mutex_); 208 uint32_t protection_bitrate_bps_; 209 uint32_t encoder_target_rate_bps_; 210 211 std::vector<bool> loss_mask_vector_ RTC_GUARDED_BY(mutex_); 212 213 std::vector<FrameCounts> frame_counts_ RTC_GUARDED_BY(mutex_); 214 FrameCountObserver* const frame_count_observer_; 215 216 // Effectively const map from SSRC to RtpRtcp, for all media SSRCs. 217 // This map is set at construction time and never changed, but it's 218 // non-trivial to make it properly const. 219 std::map<uint32_t, RtpRtcpInterface*> ssrc_to_rtp_module_; 220 221 RTC_DISALLOW_COPY_AND_ASSIGN(RtpVideoSender); 222 }; 223 224 } // namespace webrtc 225 226 #endif // CALL_RTP_VIDEO_SENDER_H_ 227