1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_ 12 #define VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_ 13 14 #include <map> 15 #include <memory> 16 #include <string> 17 #include <vector> 18 19 #include "absl/types/optional.h" 20 #include "api/crypto/frame_decryptor_interface.h" 21 #include "api/sequence_checker.h" 22 #include "api/units/timestamp.h" 23 #include "api/video/color_space.h" 24 #include "api/video/video_codec_type.h" 25 #include "call/rtp_packet_sink_interface.h" 26 #include "call/syncable.h" 27 #include "call/video_receive_stream.h" 28 #include "modules/rtp_rtcp/include/receive_statistics.h" 29 #include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" 30 #include "modules/rtp_rtcp/include/rtp_header_extension_map.h" 31 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" 32 #include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h" 33 #include "modules/rtp_rtcp/source/capture_clock_offset_updater.h" 34 #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" 35 #include "modules/rtp_rtcp/source/rtp_packet_received.h" 36 #include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" 37 #include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" 38 #include "modules/rtp_rtcp/source/rtp_video_header.h" 39 #include "modules/rtp_rtcp/source/video_rtp_depacketizer.h" 40 #include "modules/video_coding/h264_sps_pps_tracker.h" 41 #include "modules/video_coding/loss_notification_controller.h" 42 #include "modules/video_coding/nack_requester.h" 43 #include "modules/video_coding/packet_buffer.h" 44 #include "modules/video_coding/rtp_frame_reference_finder.h" 45 #include "rtc_base/experiments/field_trial_parser.h" 46 #include "rtc_base/numerics/sequence_number_util.h" 47 #include "rtc_base/system/no_unique_address.h" 48 #include "rtc_base/thread_annotations.h" 49 #include "video/buffered_frame_decryptor.h" 50 #include "video/rtp_video_stream_receiver_frame_transformer_delegate.h" 51 #include "video/unique_timestamp_counter.h" 52 53 namespace webrtc { 54 55 class NackRequester; 56 class PacketRouter; 57 class ReceiveStatistics; 58 class RtcpRttStats; 59 class RtpPacketReceived; 60 class Transport; 61 class UlpfecReceiver; 62 63 class RtpVideoStreamReceiver2 : public LossNotificationSender, 64 public RecoveredPacketReceiver, 65 public RtpPacketSinkInterface, 66 public KeyFrameRequestSender, 67 public NackSender, 68 public OnDecryptedFrameCallback, 69 public OnDecryptionStatusChangeCallback, 70 public RtpVideoFrameReceiver { 71 public: 72 // A complete frame is a frame which has received all its packets and all its 73 // references are known. 74 class OnCompleteFrameCallback { 75 public: ~OnCompleteFrameCallback()76 virtual ~OnCompleteFrameCallback() {} 77 virtual void OnCompleteFrame(std::unique_ptr<EncodedFrame> frame) = 0; 78 }; 79 80 RtpVideoStreamReceiver2( 81 TaskQueueBase* current_queue, 82 Clock* clock, 83 Transport* transport, 84 RtcpRttStats* rtt_stats, 85 // The packet router is optional; if provided, the RtpRtcp module for this 86 // stream is registered as a candidate for sending REMB and transport 87 // feedback. 88 PacketRouter* packet_router, 89 const VideoReceiveStreamInterface::Config* config, 90 ReceiveStatistics* rtp_receive_statistics, 91 RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, 92 RtcpCnameCallback* rtcp_cname_callback, 93 NackPeriodicProcessor* nack_periodic_processor, 94 // The KeyFrameRequestSender is optional; if not provided, key frame 95 // requests are sent via the internal RtpRtcp module. 96 OnCompleteFrameCallback* complete_frame_callback, 97 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor, 98 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, 99 const FieldTrialsView& field_trials, 100 RtcEventLog* event_log); 101 ~RtpVideoStreamReceiver2() override; 102 103 void AddReceiveCodec(uint8_t payload_type, 104 VideoCodecType video_codec, 105 const std::map<std::string, std::string>& codec_params, 106 bool raw_payload); 107 void RemoveReceiveCodec(uint8_t payload_type); 108 109 // Clears state for all receive codecs added via `AddReceiveCodec`. 110 void RemoveReceiveCodecs(); 111 112 void StartReceive(); 113 void StopReceive(); 114 115 // Produces the transport-related timestamps; current_delay_ms is left unset. 116 absl::optional<Syncable::Info> GetSyncInfo() const; 117 118 bool DeliverRtcp(const uint8_t* rtcp_packet, size_t rtcp_packet_length); 119 120 void FrameContinuous(int64_t seq_num); 121 122 void FrameDecoded(int64_t seq_num); 123 124 void SignalNetworkState(NetworkState state); 125 126 // Returns number of different frames seen. GetUniqueFramesSeen()127 int GetUniqueFramesSeen() const { 128 RTC_DCHECK_RUN_ON(&packet_sequence_checker_); 129 return frame_counter_.GetUniqueSeen(); 130 } 131 132 // Implements RtpPacketSinkInterface. 133 void OnRtpPacket(const RtpPacketReceived& packet) override; 134 135 // Public only for tests. 136 void OnReceivedPayloadData(rtc::CopyOnWriteBuffer codec_payload, 137 const RtpPacketReceived& rtp_packet, 138 const RTPVideoHeader& video); 139 140 // Implements RecoveredPacketReceiver. 141 void OnRecoveredPacket(const uint8_t* packet, size_t packet_length) override; 142 143 // Send an RTCP keyframe request. 144 void RequestKeyFrame() override; 145 146 // Implements NackSender. 147 void SendNack(const std::vector<uint16_t>& sequence_numbers, 148 bool buffering_allowed) override; 149 150 // Implements LossNotificationSender. 151 void SendLossNotification(uint16_t last_decoded_seq_num, 152 uint16_t last_received_seq_num, 153 bool decodability_flag, 154 bool buffering_allowed) override; 155 156 // Returns true if a decryptor is attached and frames can be decrypted. 157 // Updated by OnDecryptionStatusChangeCallback. Note this refers to Frame 158 // Decryption not SRTP. 159 bool IsDecryptable() const; 160 161 // Implements OnDecryptedFrameCallback. 162 void OnDecryptedFrame(std::unique_ptr<RtpFrameObject> frame) override; 163 164 // Implements OnDecryptionStatusChangeCallback. 165 void OnDecryptionStatusChange( 166 FrameDecryptorInterface::Status status) override; 167 168 // Optionally set a frame decryptor after a stream has started. This will not 169 // reset the decoder state. 170 void SetFrameDecryptor( 171 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor); 172 173 // Sets a frame transformer after a stream has started, if no transformer 174 // has previously been set. Does not reset the decoder state. 175 void SetDepacketizerToDecoderFrameTransformer( 176 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer); 177 178 // Updates the rtp header extensions at runtime. Must be called on the 179 // `packet_sequence_checker_` thread. 180 void SetRtpExtensions(const std::vector<RtpExtension>& extensions); 181 const RtpHeaderExtensionMap& GetRtpExtensions() const; 182 183 // Called by VideoReceiveStreamInterface when stats are updated. 184 void UpdateRtt(int64_t max_rtt_ms); 185 186 // Called when the local_ssrc is changed to match with a sender. 187 void OnLocalSsrcChange(uint32_t local_ssrc); 188 189 // Forwards the call to set rtcp_sender_ to the RTCP mode of the rtcp sender. 190 void SetRtcpMode(RtcpMode mode); 191 192 void SetReferenceTimeReport(bool enabled); 193 194 // Sets or clears the callback sink that gets called for RTP packets. Used for 195 // packet handlers such as FlexFec. Must be called on the packet delivery 196 // thread (same context as `OnRtpPacket` is called on). 197 // TODO(bugs.webrtc.org/11993): Packet delivery thread today means `worker 198 // thread` but will be `network thread`. 199 void SetPacketSink(RtpPacketSinkInterface* packet_sink); 200 201 // Turns on/off loss notifications. Must be called on the packet delivery 202 // thread. 203 void SetLossNotificationEnabled(bool enabled); 204 205 void SetNackHistory(TimeDelta history); 206 207 int ulpfec_payload_type() const; 208 int red_payload_type() const; 209 void SetProtectionPayloadTypes(int red_payload_type, int ulpfec_payload_type); 210 211 absl::optional<int64_t> LastReceivedPacketMs() const; 212 absl::optional<int64_t> LastReceivedKeyframePacketMs() const; 213 214 private: 215 // Implements RtpVideoFrameReceiver. 216 void ManageFrame(std::unique_ptr<RtpFrameObject> frame) override; 217 218 void OnCompleteFrames(RtpFrameReferenceFinder::ReturnVector frame) 219 RTC_RUN_ON(packet_sequence_checker_); 220 221 // Used for buffering RTCP feedback messages and sending them all together. 222 // Note: 223 // 1. Key frame requests and NACKs are mutually exclusive, with the 224 // former taking precedence over the latter. 225 // 2. Loss notifications are orthogonal to either. (That is, may be sent 226 // alongside either.) 227 class RtcpFeedbackBuffer : public KeyFrameRequestSender, 228 public NackSender, 229 public LossNotificationSender { 230 public: 231 RtcpFeedbackBuffer(KeyFrameRequestSender* key_frame_request_sender, 232 NackSender* nack_sender, 233 LossNotificationSender* loss_notification_sender); 234 235 ~RtcpFeedbackBuffer() override = default; 236 237 // KeyFrameRequestSender implementation. 238 void RequestKeyFrame() override; 239 240 // NackSender implementation. 241 void SendNack(const std::vector<uint16_t>& sequence_numbers, 242 bool buffering_allowed) override; 243 244 // LossNotificationSender implementation. 245 void SendLossNotification(uint16_t last_decoded_seq_num, 246 uint16_t last_received_seq_num, 247 bool decodability_flag, 248 bool buffering_allowed) override; 249 250 // Send all RTCP feedback messages buffered thus far. 251 void SendBufferedRtcpFeedback(); 252 253 void ClearLossNotificationState(); 254 255 private: 256 // LNTF-related state. 257 struct LossNotificationState { LossNotificationStateLossNotificationState258 LossNotificationState(uint16_t last_decoded_seq_num, 259 uint16_t last_received_seq_num, 260 bool decodability_flag) 261 : last_decoded_seq_num(last_decoded_seq_num), 262 last_received_seq_num(last_received_seq_num), 263 decodability_flag(decodability_flag) {} 264 265 uint16_t last_decoded_seq_num; 266 uint16_t last_received_seq_num; 267 bool decodability_flag; 268 }; 269 270 RTC_NO_UNIQUE_ADDRESS SequenceChecker packet_sequence_checker_; 271 KeyFrameRequestSender* const key_frame_request_sender_; 272 NackSender* const nack_sender_; 273 LossNotificationSender* const loss_notification_sender_; 274 275 // Key-frame-request-related state. 276 bool request_key_frame_ RTC_GUARDED_BY(packet_sequence_checker_); 277 278 // NACK-related state. 279 std::vector<uint16_t> nack_sequence_numbers_ 280 RTC_GUARDED_BY(packet_sequence_checker_); 281 282 absl::optional<LossNotificationState> lntf_state_ 283 RTC_GUARDED_BY(packet_sequence_checker_); 284 }; 285 enum ParseGenericDependenciesResult { 286 kDropPacket, 287 kHasGenericDescriptor, 288 kNoGenericDescriptor 289 }; 290 291 // Entry point doing non-stats work for a received packet. Called 292 // for the same packet both before and after RED decapsulation. 293 void ReceivePacket(const RtpPacketReceived& packet) 294 RTC_RUN_ON(packet_sequence_checker_); 295 296 // Parses and handles RED headers. 297 // This function assumes that it's being called from only one thread. 298 void ParseAndHandleEncapsulatingHeader(const RtpPacketReceived& packet) 299 RTC_RUN_ON(packet_sequence_checker_); 300 void NotifyReceiverOfEmptyPacket(uint16_t seq_num) 301 RTC_RUN_ON(packet_sequence_checker_); 302 bool IsRedEnabled() const; 303 void InsertSpsPpsIntoTracker(uint8_t payload_type) 304 RTC_RUN_ON(packet_sequence_checker_); 305 void OnInsertedPacket(video_coding::PacketBuffer::InsertResult result) 306 RTC_RUN_ON(packet_sequence_checker_); 307 ParseGenericDependenciesResult ParseGenericDependenciesExtension( 308 const RtpPacketReceived& rtp_packet, 309 RTPVideoHeader* video_header) RTC_RUN_ON(packet_sequence_checker_); 310 void OnAssembledFrame(std::unique_ptr<RtpFrameObject> frame) 311 RTC_RUN_ON(packet_sequence_checker_); 312 void UpdatePacketReceiveTimestamps(const RtpPacketReceived& packet, 313 bool is_keyframe) 314 RTC_RUN_ON(packet_sequence_checker_); 315 316 const FieldTrialsView& field_trials_; 317 TaskQueueBase* const worker_queue_; 318 Clock* const clock_; 319 // Ownership of this object lies with VideoReceiveStreamInterface, which owns 320 // `this`. 321 const VideoReceiveStreamInterface::Config& config_; 322 PacketRouter* const packet_router_; 323 324 RemoteNtpTimeEstimator ntp_estimator_; 325 326 RtpHeaderExtensionMap rtp_header_extensions_ 327 RTC_GUARDED_BY(packet_sequence_checker_); 328 // Set by the field trial WebRTC-ForcePlayoutDelay to override any playout 329 // delay that is specified in the received packets. 330 FieldTrialOptional<int> forced_playout_delay_max_ms_; 331 FieldTrialOptional<int> forced_playout_delay_min_ms_; 332 ReceiveStatistics* const rtp_receive_statistics_; 333 std::unique_ptr<UlpfecReceiver> ulpfec_receiver_ 334 RTC_GUARDED_BY(packet_sequence_checker_); 335 int red_payload_type_ RTC_GUARDED_BY(packet_sequence_checker_); 336 337 RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_task_checker_; 338 // TODO(bugs.webrtc.org/11993): This checker conceptually represents 339 // operations that belong to the network thread. The Call class is currently 340 // moving towards handling network packets on the network thread and while 341 // that work is ongoing, this checker may in practice represent the worker 342 // thread, but still serves as a mechanism of grouping together concepts 343 // that belong to the network thread. Once the packets are fully delivered 344 // on the network thread, this comment will be deleted. 345 RTC_NO_UNIQUE_ADDRESS SequenceChecker packet_sequence_checker_; 346 RtpPacketSinkInterface* packet_sink_ RTC_GUARDED_BY(packet_sequence_checker_); 347 bool receiving_ RTC_GUARDED_BY(packet_sequence_checker_); 348 int64_t last_packet_log_ms_ RTC_GUARDED_BY(packet_sequence_checker_); 349 350 const std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_; 351 352 NackPeriodicProcessor* const nack_periodic_processor_; 353 OnCompleteFrameCallback* complete_frame_callback_; 354 const KeyFrameReqMethod keyframe_request_method_; 355 356 RtcpFeedbackBuffer rtcp_feedback_buffer_; 357 // TODO(tommi): Consider absl::optional<NackRequester> instead of unique_ptr 358 // since nack is usually configured. 359 std::unique_ptr<NackRequester> nack_module_ 360 RTC_GUARDED_BY(packet_sequence_checker_); 361 std::unique_ptr<LossNotificationController> loss_notification_controller_ 362 RTC_GUARDED_BY(packet_sequence_checker_); 363 364 video_coding::PacketBuffer packet_buffer_ 365 RTC_GUARDED_BY(packet_sequence_checker_); 366 UniqueTimestampCounter frame_counter_ 367 RTC_GUARDED_BY(packet_sequence_checker_); 368 SeqNumUnwrapper<uint16_t> frame_id_unwrapper_ 369 RTC_GUARDED_BY(packet_sequence_checker_); 370 371 // Video structure provided in the dependency descriptor in a first packet 372 // of a key frame. It is required to parse dependency descriptor in the 373 // following delta packets. 374 std::unique_ptr<FrameDependencyStructure> video_structure_ 375 RTC_GUARDED_BY(packet_sequence_checker_); 376 // Frame id of the last frame with the attached video structure. 377 // absl::nullopt when `video_structure_ == nullptr`; 378 absl::optional<int64_t> video_structure_frame_id_ 379 RTC_GUARDED_BY(packet_sequence_checker_); 380 381 std::unique_ptr<RtpFrameReferenceFinder> reference_finder_ 382 RTC_GUARDED_BY(packet_sequence_checker_); 383 absl::optional<VideoCodecType> current_codec_ 384 RTC_GUARDED_BY(packet_sequence_checker_); 385 uint32_t last_assembled_frame_rtp_timestamp_ 386 RTC_GUARDED_BY(packet_sequence_checker_); 387 388 std::map<int64_t, uint16_t> last_seq_num_for_pic_id_ 389 RTC_GUARDED_BY(packet_sequence_checker_); 390 video_coding::H264SpsPpsTracker tracker_ 391 RTC_GUARDED_BY(packet_sequence_checker_); 392 393 // Maps payload id to the depacketizer. 394 std::map<uint8_t, std::unique_ptr<VideoRtpDepacketizer>> payload_type_map_ 395 RTC_GUARDED_BY(packet_sequence_checker_); 396 397 // TODO(johan): Remove pt_codec_params_ once 398 // https://bugs.chromium.org/p/webrtc/issues/detail?id=6883 is resolved. 399 // Maps a payload type to a map of out-of-band supplied codec parameters. 400 std::map<uint8_t, std::map<std::string, std::string>> pt_codec_params_ 401 RTC_GUARDED_BY(packet_sequence_checker_); 402 int16_t last_payload_type_ RTC_GUARDED_BY(packet_sequence_checker_) = -1; 403 404 bool has_received_frame_ RTC_GUARDED_BY(packet_sequence_checker_); 405 406 absl::optional<uint32_t> last_received_rtp_timestamp_ 407 RTC_GUARDED_BY(packet_sequence_checker_); 408 absl::optional<uint32_t> last_received_keyframe_rtp_timestamp_ 409 RTC_GUARDED_BY(packet_sequence_checker_); 410 absl::optional<Timestamp> last_received_rtp_system_time_ 411 RTC_GUARDED_BY(packet_sequence_checker_); 412 absl::optional<Timestamp> last_received_keyframe_rtp_system_time_ 413 RTC_GUARDED_BY(packet_sequence_checker_); 414 415 // Handles incoming encrypted frames and forwards them to the 416 // rtp_reference_finder if they are decryptable. 417 std::unique_ptr<BufferedFrameDecryptor> buffered_frame_decryptor_ 418 RTC_PT_GUARDED_BY(packet_sequence_checker_); 419 bool frames_decryptable_ RTC_GUARDED_BY(worker_task_checker_); 420 absl::optional<ColorSpace> last_color_space_; 421 422 AbsoluteCaptureTimeInterpolator absolute_capture_time_interpolator_ 423 RTC_GUARDED_BY(packet_sequence_checker_); 424 425 CaptureClockOffsetUpdater capture_clock_offset_updater_ 426 RTC_GUARDED_BY(packet_sequence_checker_); 427 428 int64_t last_completed_picture_id_ = 0; 429 430 rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate> 431 frame_transformer_delegate_; 432 433 SeqNumUnwrapper<uint16_t> rtp_seq_num_unwrapper_ 434 RTC_GUARDED_BY(packet_sequence_checker_); 435 std::map<int64_t, RtpPacketInfo> packet_infos_ 436 RTC_GUARDED_BY(packet_sequence_checker_); 437 438 Timestamp next_keyframe_request_for_missing_video_structure_ = 439 Timestamp::MinusInfinity(); 440 }; 441 442 } // namespace webrtc 443 444 #endif // VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_ 445