1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "video/rtp_video_stream_receiver2.h"
12
13 #include <algorithm>
14 #include <limits>
15 #include <memory>
16 #include <utility>
17 #include <vector>
18
19 #include "absl/algorithm/container.h"
20 #include "absl/memory/memory.h"
21 #include "absl/types/optional.h"
22 #include "api/video/video_codec_type.h"
23 #include "media/base/media_constants.h"
24 #include "modules/pacing/packet_router.h"
25 #include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h"
26 #include "modules/rtp_rtcp/include/receive_statistics.h"
27 #include "modules/rtp_rtcp/include/rtp_cvo.h"
28 #include "modules/rtp_rtcp/source/create_video_rtp_depacketizer.h"
29 #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
30 #include "modules/rtp_rtcp/source/rtp_format.h"
31 #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor.h"
32 #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h"
33 #include "modules/rtp_rtcp/source/rtp_header_extensions.h"
34 #include "modules/rtp_rtcp/source/rtp_packet_received.h"
35 #include "modules/rtp_rtcp/source/rtp_rtcp_config.h"
36 #include "modules/rtp_rtcp/source/ulpfec_receiver.h"
37 #include "modules/rtp_rtcp/source/video_rtp_depacketizer.h"
38 #include "modules/rtp_rtcp/source/video_rtp_depacketizer_raw.h"
39 #include "modules/video_coding/frame_object.h"
40 #include "modules/video_coding/h264_sprop_parameter_sets.h"
41 #include "modules/video_coding/h264_sps_pps_tracker.h"
42 #include "modules/video_coding/nack_requester.h"
43 #include "modules/video_coding/packet_buffer.h"
44 #include "rtc_base/checks.h"
45 #include "rtc_base/logging.h"
46 #include "rtc_base/strings/string_builder.h"
47 #include "system_wrappers/include/metrics.h"
48 #include "system_wrappers/include/ntp_time.h"
49
50 namespace webrtc {
51
52 namespace {
53 // TODO(philipel): Change kPacketBufferStartSize back to 32 in M63 see:
54 // crbug.com/752886
55 constexpr int kPacketBufferStartSize = 512;
56 constexpr int kPacketBufferMaxSize = 2048;
57
58 constexpr int kMaxPacketAgeToNack = 450;
59
PacketBufferMaxSize(const FieldTrialsView & field_trials)60 int PacketBufferMaxSize(const FieldTrialsView& field_trials) {
61 // The group here must be a positive power of 2, in which case that is used as
62 // size. All other values shall result in the default value being used.
63 const std::string group_name =
64 field_trials.Lookup("WebRTC-PacketBufferMaxSize");
65 int packet_buffer_max_size = kPacketBufferMaxSize;
66 if (!group_name.empty() &&
67 (sscanf(group_name.c_str(), "%d", &packet_buffer_max_size) != 1 ||
68 packet_buffer_max_size <= 0 ||
69 // Verify that the number is a positive power of 2.
70 (packet_buffer_max_size & (packet_buffer_max_size - 1)) != 0)) {
71 RTC_LOG(LS_WARNING) << "Invalid packet buffer max size: " << group_name;
72 packet_buffer_max_size = kPacketBufferMaxSize;
73 }
74 return packet_buffer_max_size;
75 }
76
CreateRtpRtcpModule(Clock * clock,ReceiveStatistics * receive_statistics,Transport * outgoing_transport,RtcpRttStats * rtt_stats,RtcpPacketTypeCounterObserver * rtcp_packet_type_counter_observer,RtcpCnameCallback * rtcp_cname_callback,bool non_sender_rtt_measurement,uint32_t local_ssrc,RtcEventLog * rtc_event_log)77 std::unique_ptr<ModuleRtpRtcpImpl2> CreateRtpRtcpModule(
78 Clock* clock,
79 ReceiveStatistics* receive_statistics,
80 Transport* outgoing_transport,
81 RtcpRttStats* rtt_stats,
82 RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer,
83 RtcpCnameCallback* rtcp_cname_callback,
84 bool non_sender_rtt_measurement,
85 uint32_t local_ssrc,
86 RtcEventLog* rtc_event_log) {
87 RtpRtcpInterface::Configuration configuration;
88 configuration.clock = clock;
89 configuration.audio = false;
90 configuration.receiver_only = true;
91 configuration.receive_statistics = receive_statistics;
92 configuration.outgoing_transport = outgoing_transport;
93 configuration.rtt_stats = rtt_stats;
94 configuration.rtcp_packet_type_counter_observer =
95 rtcp_packet_type_counter_observer;
96 configuration.rtcp_cname_callback = rtcp_cname_callback;
97 configuration.local_media_ssrc = local_ssrc;
98 configuration.non_sender_rtt_measurement = non_sender_rtt_measurement;
99 configuration.event_log = rtc_event_log;
100
101 std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp =
102 ModuleRtpRtcpImpl2::Create(configuration);
103 rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound);
104
105 return rtp_rtcp;
106 }
107
MaybeConstructNackModule(TaskQueueBase * current_queue,NackPeriodicProcessor * nack_periodic_processor,const NackConfig & nack,Clock * clock,NackSender * nack_sender,KeyFrameRequestSender * keyframe_request_sender,const FieldTrialsView & field_trials)108 std::unique_ptr<NackRequester> MaybeConstructNackModule(
109 TaskQueueBase* current_queue,
110 NackPeriodicProcessor* nack_periodic_processor,
111 const NackConfig& nack,
112 Clock* clock,
113 NackSender* nack_sender,
114 KeyFrameRequestSender* keyframe_request_sender,
115 const FieldTrialsView& field_trials) {
116 if (nack.rtp_history_ms == 0)
117 return nullptr;
118
119 // TODO(bugs.webrtc.org/12420): pass rtp_history_ms to the nack module.
120 return std::make_unique<NackRequester>(current_queue, nack_periodic_processor,
121 clock, nack_sender,
122 keyframe_request_sender, field_trials);
123 }
124
MaybeConstructUlpfecReceiver(uint32_t remote_ssrc,int red_payload_type,int ulpfec_payload_type,rtc::ArrayView<const RtpExtension> extensions,RecoveredPacketReceiver * callback,Clock * clock)125 std::unique_ptr<UlpfecReceiver> MaybeConstructUlpfecReceiver(
126 uint32_t remote_ssrc,
127 int red_payload_type,
128 int ulpfec_payload_type,
129 rtc::ArrayView<const RtpExtension> extensions,
130 RecoveredPacketReceiver* callback,
131 Clock* clock) {
132 RTC_DCHECK_GE(red_payload_type, -1);
133 RTC_DCHECK_GE(ulpfec_payload_type, -1);
134 if (red_payload_type == -1)
135 return nullptr;
136
137 // TODO(tommi, brandtr): Consider including this check too once
138 // `UlpfecReceiver` has been updated to not consider both red and ulpfec
139 // payload ids.
140 // if (ulpfec_payload_type == -1)
141 // return nullptr;
142
143 return std::make_unique<UlpfecReceiver>(remote_ssrc, ulpfec_payload_type,
144 callback, extensions, clock);
145 }
146
147 static const int kPacketLogIntervalMs = 10000;
148
149 } // namespace
150
RtcpFeedbackBuffer(KeyFrameRequestSender * key_frame_request_sender,NackSender * nack_sender,LossNotificationSender * loss_notification_sender)151 RtpVideoStreamReceiver2::RtcpFeedbackBuffer::RtcpFeedbackBuffer(
152 KeyFrameRequestSender* key_frame_request_sender,
153 NackSender* nack_sender,
154 LossNotificationSender* loss_notification_sender)
155 : key_frame_request_sender_(key_frame_request_sender),
156 nack_sender_(nack_sender),
157 loss_notification_sender_(loss_notification_sender),
158 request_key_frame_(false) {
159 RTC_DCHECK(key_frame_request_sender_);
160 RTC_DCHECK(nack_sender_);
161 RTC_DCHECK(loss_notification_sender_);
162 packet_sequence_checker_.Detach();
163 }
164
RequestKeyFrame()165 void RtpVideoStreamReceiver2::RtcpFeedbackBuffer::RequestKeyFrame() {
166 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
167 request_key_frame_ = true;
168 }
169
SendNack(const std::vector<uint16_t> & sequence_numbers,bool buffering_allowed)170 void RtpVideoStreamReceiver2::RtcpFeedbackBuffer::SendNack(
171 const std::vector<uint16_t>& sequence_numbers,
172 bool buffering_allowed) {
173 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
174 RTC_DCHECK(!sequence_numbers.empty());
175 nack_sequence_numbers_.insert(nack_sequence_numbers_.end(),
176 sequence_numbers.cbegin(),
177 sequence_numbers.cend());
178 if (!buffering_allowed) {
179 // Note that while *buffering* is not allowed, *batching* is, meaning that
180 // previously buffered messages may be sent along with the current message.
181 SendBufferedRtcpFeedback();
182 }
183 }
184
SendLossNotification(uint16_t last_decoded_seq_num,uint16_t last_received_seq_num,bool decodability_flag,bool buffering_allowed)185 void RtpVideoStreamReceiver2::RtcpFeedbackBuffer::SendLossNotification(
186 uint16_t last_decoded_seq_num,
187 uint16_t last_received_seq_num,
188 bool decodability_flag,
189 bool buffering_allowed) {
190 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
191 RTC_DCHECK(buffering_allowed);
192 RTC_DCHECK(!lntf_state_)
193 << "SendLossNotification() called twice in a row with no call to "
194 "SendBufferedRtcpFeedback() in between.";
195 lntf_state_ = absl::make_optional<LossNotificationState>(
196 last_decoded_seq_num, last_received_seq_num, decodability_flag);
197 }
198
SendBufferedRtcpFeedback()199 void RtpVideoStreamReceiver2::RtcpFeedbackBuffer::SendBufferedRtcpFeedback() {
200 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
201
202 bool request_key_frame = false;
203 std::vector<uint16_t> nack_sequence_numbers;
204 absl::optional<LossNotificationState> lntf_state;
205
206 std::swap(request_key_frame, request_key_frame_);
207 std::swap(nack_sequence_numbers, nack_sequence_numbers_);
208 std::swap(lntf_state, lntf_state_);
209
210 if (lntf_state) {
211 // If either a NACK or a key frame request is sent, we should buffer
212 // the LNTF and wait for them (NACK or key frame request) to trigger
213 // the compound feedback message.
214 // Otherwise, the LNTF should be sent out immediately.
215 const bool buffering_allowed =
216 request_key_frame || !nack_sequence_numbers.empty();
217
218 loss_notification_sender_->SendLossNotification(
219 lntf_state->last_decoded_seq_num, lntf_state->last_received_seq_num,
220 lntf_state->decodability_flag, buffering_allowed);
221 }
222
223 if (request_key_frame) {
224 key_frame_request_sender_->RequestKeyFrame();
225 } else if (!nack_sequence_numbers.empty()) {
226 nack_sender_->SendNack(nack_sequence_numbers, true);
227 }
228 }
229
ClearLossNotificationState()230 void RtpVideoStreamReceiver2::RtcpFeedbackBuffer::ClearLossNotificationState() {
231 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
232 lntf_state_.reset();
233 }
234
RtpVideoStreamReceiver2(TaskQueueBase * current_queue,Clock * clock,Transport * transport,RtcpRttStats * rtt_stats,PacketRouter * packet_router,const VideoReceiveStreamInterface::Config * config,ReceiveStatistics * rtp_receive_statistics,RtcpPacketTypeCounterObserver * rtcp_packet_type_counter_observer,RtcpCnameCallback * rtcp_cname_callback,NackPeriodicProcessor * nack_periodic_processor,OnCompleteFrameCallback * complete_frame_callback,rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,const FieldTrialsView & field_trials,RtcEventLog * event_log)235 RtpVideoStreamReceiver2::RtpVideoStreamReceiver2(
236 TaskQueueBase* current_queue,
237 Clock* clock,
238 Transport* transport,
239 RtcpRttStats* rtt_stats,
240 PacketRouter* packet_router,
241 const VideoReceiveStreamInterface::Config* config,
242 ReceiveStatistics* rtp_receive_statistics,
243 RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer,
244 RtcpCnameCallback* rtcp_cname_callback,
245 NackPeriodicProcessor* nack_periodic_processor,
246 OnCompleteFrameCallback* complete_frame_callback,
247 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
248 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
249 const FieldTrialsView& field_trials,
250 RtcEventLog* event_log)
251 : field_trials_(field_trials),
252 worker_queue_(current_queue),
253 clock_(clock),
254 config_(*config),
255 packet_router_(packet_router),
256 ntp_estimator_(clock),
257 rtp_header_extensions_(config_.rtp.extensions),
258 forced_playout_delay_max_ms_("max_ms", absl::nullopt),
259 forced_playout_delay_min_ms_("min_ms", absl::nullopt),
260 rtp_receive_statistics_(rtp_receive_statistics),
261 ulpfec_receiver_(
262 MaybeConstructUlpfecReceiver(config->rtp.remote_ssrc,
263 config->rtp.red_payload_type,
264 config->rtp.ulpfec_payload_type,
265 config->rtp.extensions,
266 this,
267 clock_)),
268 red_payload_type_(config_.rtp.red_payload_type),
269 packet_sink_(config->rtp.packet_sink_),
270 receiving_(false),
271 last_packet_log_ms_(-1),
272 rtp_rtcp_(CreateRtpRtcpModule(
273 clock,
274 rtp_receive_statistics_,
275 transport,
276 rtt_stats,
277 rtcp_packet_type_counter_observer,
278 rtcp_cname_callback,
279 config_.rtp.rtcp_xr.receiver_reference_time_report,
280 config_.rtp.local_ssrc,
281 event_log)),
282 nack_periodic_processor_(nack_periodic_processor),
283 complete_frame_callback_(complete_frame_callback),
284 keyframe_request_method_(config_.rtp.keyframe_method),
285 // TODO(bugs.webrtc.org/10336): Let `rtcp_feedback_buffer_` communicate
286 // directly with `rtp_rtcp_`.
287 rtcp_feedback_buffer_(this, this, this),
288 nack_module_(MaybeConstructNackModule(current_queue,
289 nack_periodic_processor,
290 config_.rtp.nack,
291 clock_,
292 &rtcp_feedback_buffer_,
293 &rtcp_feedback_buffer_,
294 field_trials_)),
295 packet_buffer_(kPacketBufferStartSize,
296 PacketBufferMaxSize(field_trials_)),
297 reference_finder_(std::make_unique<RtpFrameReferenceFinder>()),
298 has_received_frame_(false),
299 frames_decryptable_(false),
300 absolute_capture_time_interpolator_(clock) {
301 packet_sequence_checker_.Detach();
302 constexpr bool remb_candidate = true;
303 if (packet_router_)
304 packet_router_->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate);
305
306 RTC_DCHECK(config_.rtp.rtcp_mode != RtcpMode::kOff)
307 << "A stream should not be configured with RTCP disabled. This value is "
308 "reserved for internal usage.";
309 // TODO(pbos): What's an appropriate local_ssrc for receive-only streams?
310 RTC_DCHECK(config_.rtp.local_ssrc != 0);
311 RTC_DCHECK(config_.rtp.remote_ssrc != config_.rtp.local_ssrc);
312
313 rtp_rtcp_->SetRTCPStatus(config_.rtp.rtcp_mode);
314 rtp_rtcp_->SetRemoteSSRC(config_.rtp.remote_ssrc);
315
316 if (config_.rtp.nack.rtp_history_ms > 0) {
317 rtp_receive_statistics_->SetMaxReorderingThreshold(config_.rtp.remote_ssrc,
318 kMaxPacketAgeToNack);
319 }
320 ParseFieldTrial(
321 {&forced_playout_delay_max_ms_, &forced_playout_delay_min_ms_},
322 field_trials_.Lookup("WebRTC-ForcePlayoutDelay"));
323
324 if (config_.rtp.lntf.enabled) {
325 loss_notification_controller_ =
326 std::make_unique<LossNotificationController>(&rtcp_feedback_buffer_,
327 &rtcp_feedback_buffer_);
328 }
329
330 // Only construct the encrypted receiver if frame encryption is enabled.
331 if (config_.crypto_options.sframe.require_frame_encryption) {
332 buffered_frame_decryptor_ =
333 std::make_unique<BufferedFrameDecryptor>(this, this, field_trials_);
334 if (frame_decryptor != nullptr) {
335 buffered_frame_decryptor_->SetFrameDecryptor(std::move(frame_decryptor));
336 }
337 }
338
339 if (frame_transformer) {
340 frame_transformer_delegate_ =
341 rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
342 this, std::move(frame_transformer), rtc::Thread::Current(),
343 config_.rtp.remote_ssrc);
344 frame_transformer_delegate_->Init();
345 }
346 }
347
~RtpVideoStreamReceiver2()348 RtpVideoStreamReceiver2::~RtpVideoStreamReceiver2() {
349 if (packet_router_)
350 packet_router_->RemoveReceiveRtpModule(rtp_rtcp_.get());
351 ulpfec_receiver_.reset();
352 if (frame_transformer_delegate_)
353 frame_transformer_delegate_->Reset();
354 }
355
AddReceiveCodec(uint8_t payload_type,VideoCodecType video_codec,const std::map<std::string,std::string> & codec_params,bool raw_payload)356 void RtpVideoStreamReceiver2::AddReceiveCodec(
357 uint8_t payload_type,
358 VideoCodecType video_codec,
359 const std::map<std::string, std::string>& codec_params,
360 bool raw_payload) {
361 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
362 if (codec_params.count(cricket::kH264FmtpSpsPpsIdrInKeyframe) > 0 ||
363 field_trials_.IsEnabled("WebRTC-SpsPpsIdrIsH264Keyframe")) {
364 packet_buffer_.ForceSpsPpsIdrIsH264Keyframe();
365 }
366 payload_type_map_.emplace(
367 payload_type, raw_payload ? std::make_unique<VideoRtpDepacketizerRaw>()
368 : CreateVideoRtpDepacketizer(video_codec));
369 pt_codec_params_.emplace(payload_type, codec_params);
370 }
371
RemoveReceiveCodec(uint8_t payload_type)372 void RtpVideoStreamReceiver2::RemoveReceiveCodec(uint8_t payload_type) {
373 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
374 auto codec_params_it = pt_codec_params_.find(payload_type);
375 if (codec_params_it == pt_codec_params_.end())
376 return;
377
378 const bool sps_pps_idr_in_key_frame =
379 codec_params_it->second.count(cricket::kH264FmtpSpsPpsIdrInKeyframe) > 0;
380
381 pt_codec_params_.erase(codec_params_it);
382 payload_type_map_.erase(payload_type);
383
384 if (sps_pps_idr_in_key_frame) {
385 bool reset_setting = true;
386 for (auto& [unused, codec_params] : pt_codec_params_) {
387 if (codec_params.count(cricket::kH264FmtpSpsPpsIdrInKeyframe) > 0) {
388 reset_setting = false;
389 break;
390 }
391 }
392
393 if (reset_setting) {
394 packet_buffer_.ResetSpsPpsIdrIsH264Keyframe();
395 }
396 }
397 }
398
RemoveReceiveCodecs()399 void RtpVideoStreamReceiver2::RemoveReceiveCodecs() {
400 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
401
402 pt_codec_params_.clear();
403 payload_type_map_.clear();
404 packet_buffer_.ResetSpsPpsIdrIsH264Keyframe();
405 }
406
GetSyncInfo() const407 absl::optional<Syncable::Info> RtpVideoStreamReceiver2::GetSyncInfo() const {
408 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
409 Syncable::Info info;
410 if (rtp_rtcp_->RemoteNTP(&info.capture_time_ntp_secs,
411 &info.capture_time_ntp_frac,
412 /*rtcp_arrival_time_secs=*/nullptr,
413 /*rtcp_arrival_time_frac=*/nullptr,
414 &info.capture_time_source_clock) != 0) {
415 return absl::nullopt;
416 }
417
418 if (!last_received_rtp_timestamp_ || !last_received_rtp_system_time_) {
419 return absl::nullopt;
420 }
421 info.latest_received_capture_timestamp = *last_received_rtp_timestamp_;
422 info.latest_receive_time_ms = last_received_rtp_system_time_->ms();
423
424 // Leaves info.current_delay_ms uninitialized.
425 return info;
426 }
427
428 RtpVideoStreamReceiver2::ParseGenericDependenciesResult
ParseGenericDependenciesExtension(const RtpPacketReceived & rtp_packet,RTPVideoHeader * video_header)429 RtpVideoStreamReceiver2::ParseGenericDependenciesExtension(
430 const RtpPacketReceived& rtp_packet,
431 RTPVideoHeader* video_header) {
432 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
433 if (rtp_packet.HasExtension<RtpDependencyDescriptorExtension>()) {
434 webrtc::DependencyDescriptor dependency_descriptor;
435 if (!rtp_packet.GetExtension<RtpDependencyDescriptorExtension>(
436 video_structure_.get(), &dependency_descriptor)) {
437 // Descriptor is there, but failed to parse. Either it is invalid,
438 // or too old packet (after relevant video_structure_ changed),
439 // or too new packet (before relevant video_structure_ arrived).
440 // Drop such packet to be on the safe side.
441 // TODO(bugs.webrtc.org/10342): Stash too new packet.
442 RTC_LOG(LS_WARNING) << "ssrc: " << rtp_packet.Ssrc()
443 << " Failed to parse dependency descriptor.";
444 return kDropPacket;
445 }
446 if (dependency_descriptor.attached_structure != nullptr &&
447 !dependency_descriptor.first_packet_in_frame) {
448 RTC_LOG(LS_WARNING) << "ssrc: " << rtp_packet.Ssrc()
449 << "Invalid dependency descriptor: structure "
450 "attached to non first packet of a frame.";
451 return kDropPacket;
452 }
453 video_header->is_first_packet_in_frame =
454 dependency_descriptor.first_packet_in_frame;
455 video_header->is_last_packet_in_frame =
456 dependency_descriptor.last_packet_in_frame;
457
458 int64_t frame_id =
459 frame_id_unwrapper_.Unwrap(dependency_descriptor.frame_number);
460 auto& generic_descriptor_info = video_header->generic.emplace();
461 generic_descriptor_info.frame_id = frame_id;
462 generic_descriptor_info.spatial_index =
463 dependency_descriptor.frame_dependencies.spatial_id;
464 generic_descriptor_info.temporal_index =
465 dependency_descriptor.frame_dependencies.temporal_id;
466 for (int fdiff : dependency_descriptor.frame_dependencies.frame_diffs) {
467 generic_descriptor_info.dependencies.push_back(frame_id - fdiff);
468 }
469 generic_descriptor_info.decode_target_indications =
470 dependency_descriptor.frame_dependencies.decode_target_indications;
471 if (dependency_descriptor.resolution) {
472 video_header->width = dependency_descriptor.resolution->Width();
473 video_header->height = dependency_descriptor.resolution->Height();
474 }
475
476 // FrameDependencyStructure is sent in dependency descriptor of the first
477 // packet of a key frame and required for parsed dependency descriptor in
478 // all the following packets until next key frame.
479 // Save it if there is a (potentially) new structure.
480 if (dependency_descriptor.attached_structure) {
481 RTC_DCHECK(dependency_descriptor.first_packet_in_frame);
482 if (video_structure_frame_id_ > frame_id) {
483 RTC_LOG(LS_WARNING)
484 << "Arrived key frame with id " << frame_id << " and structure id "
485 << dependency_descriptor.attached_structure->structure_id
486 << " is older than the latest received key frame with id "
487 << *video_structure_frame_id_ << " and structure id "
488 << video_structure_->structure_id;
489 return kDropPacket;
490 }
491 video_structure_ = std::move(dependency_descriptor.attached_structure);
492 video_structure_frame_id_ = frame_id;
493 video_header->frame_type = VideoFrameType::kVideoFrameKey;
494 } else {
495 video_header->frame_type = VideoFrameType::kVideoFrameDelta;
496 }
497 return kHasGenericDescriptor;
498 }
499
500 RtpGenericFrameDescriptor generic_frame_descriptor;
501 if (!rtp_packet.GetExtension<RtpGenericFrameDescriptorExtension00>(
502 &generic_frame_descriptor)) {
503 return kNoGenericDescriptor;
504 }
505
506 video_header->is_first_packet_in_frame =
507 generic_frame_descriptor.FirstPacketInSubFrame();
508 video_header->is_last_packet_in_frame =
509 generic_frame_descriptor.LastPacketInSubFrame();
510
511 if (generic_frame_descriptor.FirstPacketInSubFrame()) {
512 video_header->frame_type =
513 generic_frame_descriptor.FrameDependenciesDiffs().empty()
514 ? VideoFrameType::kVideoFrameKey
515 : VideoFrameType::kVideoFrameDelta;
516
517 auto& generic_descriptor_info = video_header->generic.emplace();
518 int64_t frame_id =
519 frame_id_unwrapper_.Unwrap(generic_frame_descriptor.FrameId());
520 generic_descriptor_info.frame_id = frame_id;
521 generic_descriptor_info.spatial_index =
522 generic_frame_descriptor.SpatialLayer();
523 generic_descriptor_info.temporal_index =
524 generic_frame_descriptor.TemporalLayer();
525 for (uint16_t fdiff : generic_frame_descriptor.FrameDependenciesDiffs()) {
526 generic_descriptor_info.dependencies.push_back(frame_id - fdiff);
527 }
528 }
529 video_header->width = generic_frame_descriptor.Width();
530 video_header->height = generic_frame_descriptor.Height();
531 return kHasGenericDescriptor;
532 }
533
OnReceivedPayloadData(rtc::CopyOnWriteBuffer codec_payload,const RtpPacketReceived & rtp_packet,const RTPVideoHeader & video)534 void RtpVideoStreamReceiver2::OnReceivedPayloadData(
535 rtc::CopyOnWriteBuffer codec_payload,
536 const RtpPacketReceived& rtp_packet,
537 const RTPVideoHeader& video) {
538 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
539
540 auto packet =
541 std::make_unique<video_coding::PacketBuffer::Packet>(rtp_packet, video);
542
543 int64_t unwrapped_rtp_seq_num =
544 rtp_seq_num_unwrapper_.Unwrap(rtp_packet.SequenceNumber());
545
546 RtpPacketInfo& packet_info =
547 packet_infos_
548 .emplace(unwrapped_rtp_seq_num,
549 RtpPacketInfo(rtp_packet.Ssrc(), rtp_packet.Csrcs(),
550 rtp_packet.Timestamp(),
551 /*receive_time_ms=*/clock_->CurrentTime()))
552 .first->second;
553
554 // Try to extrapolate absolute capture time if it is missing.
555 packet_info.set_absolute_capture_time(
556 absolute_capture_time_interpolator_.OnReceivePacket(
557 AbsoluteCaptureTimeInterpolator::GetSource(packet_info.ssrc(),
558 packet_info.csrcs()),
559 packet_info.rtp_timestamp(),
560 // Assume frequency is the same one for all video frames.
561 kVideoPayloadTypeFrequency,
562 rtp_packet.GetExtension<AbsoluteCaptureTimeExtension>()));
563
564 RTPVideoHeader& video_header = packet->video_header;
565 video_header.rotation = kVideoRotation_0;
566 video_header.content_type = VideoContentType::UNSPECIFIED;
567 video_header.video_timing.flags = VideoSendTiming::kInvalid;
568 video_header.is_last_packet_in_frame |= rtp_packet.Marker();
569
570 rtp_packet.GetExtension<VideoOrientation>(&video_header.rotation);
571 rtp_packet.GetExtension<VideoContentTypeExtension>(
572 &video_header.content_type);
573 rtp_packet.GetExtension<VideoTimingExtension>(&video_header.video_timing);
574 if (forced_playout_delay_max_ms_ && forced_playout_delay_min_ms_) {
575 video_header.playout_delay.max_ms = *forced_playout_delay_max_ms_;
576 video_header.playout_delay.min_ms = *forced_playout_delay_min_ms_;
577 } else {
578 rtp_packet.GetExtension<PlayoutDelayLimits>(&video_header.playout_delay);
579 }
580
581 ParseGenericDependenciesResult generic_descriptor_state =
582 ParseGenericDependenciesExtension(rtp_packet, &video_header);
583
584 if (!rtp_packet.recovered()) {
585 UpdatePacketReceiveTimestamps(
586 rtp_packet, video_header.frame_type == VideoFrameType::kVideoFrameKey);
587 }
588
589 if (generic_descriptor_state == kDropPacket) {
590 Timestamp now = clock_->CurrentTime();
591 if (video_structure_ == nullptr &&
592 next_keyframe_request_for_missing_video_structure_ < now) {
593 // No video structure received yet, most likely part of the initial
594 // keyframe was lost.
595 RequestKeyFrame();
596 next_keyframe_request_for_missing_video_structure_ =
597 now + TimeDelta::Seconds(1);
598 }
599 return;
600 }
601
602 // Color space should only be transmitted in the last packet of a frame,
603 // therefore, neglect it otherwise so that last_color_space_ is not reset by
604 // mistake.
605 if (video_header.is_last_packet_in_frame) {
606 video_header.color_space = rtp_packet.GetExtension<ColorSpaceExtension>();
607 if (video_header.color_space ||
608 video_header.frame_type == VideoFrameType::kVideoFrameKey) {
609 // Store color space since it's only transmitted when changed or for key
610 // frames. Color space will be cleared if a key frame is transmitted
611 // without color space information.
612 last_color_space_ = video_header.color_space;
613 } else if (last_color_space_) {
614 video_header.color_space = last_color_space_;
615 }
616 }
617 video_header.video_frame_tracking_id =
618 rtp_packet.GetExtension<VideoFrameTrackingIdExtension>();
619
620 if (loss_notification_controller_) {
621 if (rtp_packet.recovered()) {
622 // TODO(bugs.webrtc.org/10336): Implement support for reordering.
623 RTC_LOG(LS_INFO)
624 << "LossNotificationController does not support reordering.";
625 } else if (generic_descriptor_state == kNoGenericDescriptor) {
626 RTC_LOG(LS_WARNING) << "LossNotificationController requires generic "
627 "frame descriptor, but it is missing.";
628 } else {
629 if (video_header.is_first_packet_in_frame) {
630 RTC_DCHECK(video_header.generic);
631 LossNotificationController::FrameDetails frame;
632 frame.is_keyframe =
633 video_header.frame_type == VideoFrameType::kVideoFrameKey;
634 frame.frame_id = video_header.generic->frame_id;
635 frame.frame_dependencies = video_header.generic->dependencies;
636 loss_notification_controller_->OnReceivedPacket(
637 rtp_packet.SequenceNumber(), &frame);
638 } else {
639 loss_notification_controller_->OnReceivedPacket(
640 rtp_packet.SequenceNumber(), nullptr);
641 }
642 }
643 }
644
645 if (nack_module_) {
646 const bool is_keyframe =
647 video_header.is_first_packet_in_frame &&
648 video_header.frame_type == VideoFrameType::kVideoFrameKey;
649
650 packet->times_nacked = nack_module_->OnReceivedPacket(
651 rtp_packet.SequenceNumber(), is_keyframe, rtp_packet.recovered());
652 } else {
653 packet->times_nacked = -1;
654 }
655
656 if (codec_payload.size() == 0) {
657 NotifyReceiverOfEmptyPacket(packet->seq_num);
658 rtcp_feedback_buffer_.SendBufferedRtcpFeedback();
659 return;
660 }
661
662 if (packet->codec() == kVideoCodecH264) {
663 // Only when we start to receive packets will we know what payload type
664 // that will be used. When we know the payload type insert the correct
665 // sps/pps into the tracker.
666 if (packet->payload_type != last_payload_type_) {
667 last_payload_type_ = packet->payload_type;
668 InsertSpsPpsIntoTracker(packet->payload_type);
669 }
670
671 video_coding::H264SpsPpsTracker::FixedBitstream fixed =
672 tracker_.CopyAndFixBitstream(
673 rtc::MakeArrayView(codec_payload.cdata(), codec_payload.size()),
674 &packet->video_header);
675
676 switch (fixed.action) {
677 case video_coding::H264SpsPpsTracker::kRequestKeyframe:
678 rtcp_feedback_buffer_.RequestKeyFrame();
679 rtcp_feedback_buffer_.SendBufferedRtcpFeedback();
680 [[fallthrough]];
681 case video_coding::H264SpsPpsTracker::kDrop:
682 return;
683 case video_coding::H264SpsPpsTracker::kInsert:
684 packet->video_payload = std::move(fixed.bitstream);
685 break;
686 }
687
688 } else {
689 packet->video_payload = std::move(codec_payload);
690 }
691
692 rtcp_feedback_buffer_.SendBufferedRtcpFeedback();
693 frame_counter_.Add(packet->timestamp);
694 OnInsertedPacket(packet_buffer_.InsertPacket(std::move(packet)));
695 }
696
OnRecoveredPacket(const uint8_t * rtp_packet,size_t rtp_packet_length)697 void RtpVideoStreamReceiver2::OnRecoveredPacket(const uint8_t* rtp_packet,
698 size_t rtp_packet_length) {
699 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
700
701 RtpPacketReceived packet;
702 if (!packet.Parse(rtp_packet, rtp_packet_length))
703 return;
704 if (packet.PayloadType() == red_payload_type_) {
705 RTC_LOG(LS_WARNING) << "Discarding recovered packet with RED encapsulation";
706 return;
707 }
708
709 packet.IdentifyExtensions(rtp_header_extensions_);
710 packet.set_payload_type_frequency(kVideoPayloadTypeFrequency);
711 // TODO(bugs.webrtc.org/7135): UlpfecReceiverImpl::ProcessReceivedFec passes
712 // both original (decapsulated) media packets and recovered packets to this
713 // callback. We need a way to distinguish, for setting packet.recovered()
714 // correctly. Ideally, move RED decapsulation out of the Ulpfec
715 // implementation.
716
717 ReceivePacket(packet);
718 }
719
720 // This method handles both regular RTP packets and packets recovered
721 // via FlexFEC.
OnRtpPacket(const RtpPacketReceived & packet)722 void RtpVideoStreamReceiver2::OnRtpPacket(const RtpPacketReceived& packet) {
723 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
724
725 if (!receiving_)
726 return;
727
728 ReceivePacket(packet);
729
730 // Update receive statistics after ReceivePacket.
731 // Receive statistics will be reset if the payload type changes (make sure
732 // that the first packet is included in the stats).
733 if (!packet.recovered()) {
734 rtp_receive_statistics_->OnRtpPacket(packet);
735 }
736
737 if (packet_sink_) {
738 packet_sink_->OnRtpPacket(packet);
739 }
740 }
741
RequestKeyFrame()742 void RtpVideoStreamReceiver2::RequestKeyFrame() {
743 RTC_DCHECK_RUN_ON(&worker_task_checker_);
744 // TODO(bugs.webrtc.org/10336): Allow the sender to ignore key frame requests
745 // issued by anything other than the LossNotificationController if it (the
746 // sender) is relying on LNTF alone.
747 if (keyframe_request_method_ == KeyFrameReqMethod::kPliRtcp) {
748 rtp_rtcp_->SendPictureLossIndication();
749 } else if (keyframe_request_method_ == KeyFrameReqMethod::kFirRtcp) {
750 rtp_rtcp_->SendFullIntraRequest();
751 }
752 }
753
SendNack(const std::vector<uint16_t> & sequence_numbers,bool)754 void RtpVideoStreamReceiver2::SendNack(
755 const std::vector<uint16_t>& sequence_numbers,
756 bool /*buffering_allowed*/) {
757 rtp_rtcp_->SendNack(sequence_numbers);
758 }
759
SendLossNotification(uint16_t last_decoded_seq_num,uint16_t last_received_seq_num,bool decodability_flag,bool buffering_allowed)760 void RtpVideoStreamReceiver2::SendLossNotification(
761 uint16_t last_decoded_seq_num,
762 uint16_t last_received_seq_num,
763 bool decodability_flag,
764 bool buffering_allowed) {
765 RTC_DCHECK(config_.rtp.lntf.enabled);
766 rtp_rtcp_->SendLossNotification(last_decoded_seq_num, last_received_seq_num,
767 decodability_flag, buffering_allowed);
768 }
769
IsDecryptable() const770 bool RtpVideoStreamReceiver2::IsDecryptable() const {
771 RTC_DCHECK_RUN_ON(&worker_task_checker_);
772 return frames_decryptable_;
773 }
774
OnInsertedPacket(video_coding::PacketBuffer::InsertResult result)775 void RtpVideoStreamReceiver2::OnInsertedPacket(
776 video_coding::PacketBuffer::InsertResult result) {
777 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
778 RTC_DCHECK_RUN_ON(&worker_task_checker_);
779 video_coding::PacketBuffer::Packet* first_packet = nullptr;
780 int max_nack_count;
781 int64_t min_recv_time;
782 int64_t max_recv_time;
783 std::vector<rtc::ArrayView<const uint8_t>> payloads;
784 RtpPacketInfos::vector_type packet_infos;
785
786 bool frame_boundary = true;
787 for (auto& packet : result.packets) {
788 // PacketBuffer promisses frame boundaries are correctly set on each
789 // packet. Document that assumption with the DCHECKs.
790 RTC_DCHECK_EQ(frame_boundary, packet->is_first_packet_in_frame());
791 int64_t unwrapped_rtp_seq_num =
792 rtp_seq_num_unwrapper_.Unwrap(packet->seq_num);
793 RTC_DCHECK_GT(packet_infos_.count(unwrapped_rtp_seq_num), 0);
794 RtpPacketInfo& packet_info = packet_infos_[unwrapped_rtp_seq_num];
795 if (packet->is_first_packet_in_frame()) {
796 first_packet = packet.get();
797 max_nack_count = packet->times_nacked;
798 min_recv_time = packet_info.receive_time().ms();
799 max_recv_time = packet_info.receive_time().ms();
800 } else {
801 max_nack_count = std::max(max_nack_count, packet->times_nacked);
802 min_recv_time = std::min(min_recv_time, packet_info.receive_time().ms());
803 max_recv_time = std::max(max_recv_time, packet_info.receive_time().ms());
804 }
805 payloads.emplace_back(packet->video_payload);
806 packet_infos.push_back(packet_info);
807
808 frame_boundary = packet->is_last_packet_in_frame();
809 if (packet->is_last_packet_in_frame()) {
810 auto depacketizer_it = payload_type_map_.find(first_packet->payload_type);
811 RTC_CHECK(depacketizer_it != payload_type_map_.end());
812
813 rtc::scoped_refptr<EncodedImageBuffer> bitstream =
814 depacketizer_it->second->AssembleFrame(payloads);
815 if (!bitstream) {
816 // Failed to assemble a frame. Discard and continue.
817 continue;
818 }
819
820 const video_coding::PacketBuffer::Packet& last_packet = *packet;
821 OnAssembledFrame(std::make_unique<RtpFrameObject>(
822 first_packet->seq_num, //
823 last_packet.seq_num, //
824 last_packet.marker_bit, //
825 max_nack_count, //
826 min_recv_time, //
827 max_recv_time, //
828 first_packet->timestamp, //
829 ntp_estimator_.Estimate(first_packet->timestamp), //
830 last_packet.video_header.video_timing, //
831 first_packet->payload_type, //
832 first_packet->codec(), //
833 last_packet.video_header.rotation, //
834 last_packet.video_header.content_type, //
835 first_packet->video_header, //
836 last_packet.video_header.color_space, //
837 RtpPacketInfos(std::move(packet_infos)), //
838 std::move(bitstream)));
839 payloads.clear();
840 packet_infos.clear();
841 }
842 }
843 RTC_DCHECK(frame_boundary);
844 if (result.buffer_cleared) {
845 last_received_rtp_system_time_.reset();
846 last_received_keyframe_rtp_system_time_.reset();
847 last_received_keyframe_rtp_timestamp_.reset();
848 packet_infos_.clear();
849 RequestKeyFrame();
850 }
851 }
852
OnAssembledFrame(std::unique_ptr<RtpFrameObject> frame)853 void RtpVideoStreamReceiver2::OnAssembledFrame(
854 std::unique_ptr<RtpFrameObject> frame) {
855 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
856 RTC_DCHECK(frame);
857
858 const absl::optional<RTPVideoHeader::GenericDescriptorInfo>& descriptor =
859 frame->GetRtpVideoHeader().generic;
860
861 if (loss_notification_controller_ && descriptor) {
862 loss_notification_controller_->OnAssembledFrame(
863 frame->first_seq_num(), descriptor->frame_id,
864 absl::c_linear_search(descriptor->decode_target_indications,
865 DecodeTargetIndication::kDiscardable),
866 descriptor->dependencies);
867 }
868
869 // If frames arrive before a key frame, they would not be decodable.
870 // In that case, request a key frame ASAP.
871 if (!has_received_frame_) {
872 if (frame->FrameType() != VideoFrameType::kVideoFrameKey) {
873 // `loss_notification_controller_`, if present, would have already
874 // requested a key frame when the first packet for the non-key frame
875 // had arrived, so no need to replicate the request.
876 if (!loss_notification_controller_) {
877 RequestKeyFrame();
878 }
879 }
880 has_received_frame_ = true;
881 }
882
883 // Reset `reference_finder_` if `frame` is new and the codec have changed.
884 if (current_codec_) {
885 bool frame_is_newer =
886 AheadOf(frame->Timestamp(), last_assembled_frame_rtp_timestamp_);
887
888 if (frame->codec_type() != current_codec_) {
889 if (frame_is_newer) {
890 // When we reset the `reference_finder_` we don't want new picture ids
891 // to overlap with old picture ids. To ensure that doesn't happen we
892 // start from the `last_completed_picture_id_` and add an offset in case
893 // of reordering.
894 reference_finder_ = std::make_unique<RtpFrameReferenceFinder>(
895 last_completed_picture_id_ + std::numeric_limits<uint16_t>::max());
896 current_codec_ = frame->codec_type();
897 } else {
898 // Old frame from before the codec switch, discard it.
899 return;
900 }
901 }
902
903 if (frame_is_newer) {
904 last_assembled_frame_rtp_timestamp_ = frame->Timestamp();
905 }
906 } else {
907 current_codec_ = frame->codec_type();
908 last_assembled_frame_rtp_timestamp_ = frame->Timestamp();
909 }
910
911 if (buffered_frame_decryptor_ != nullptr) {
912 buffered_frame_decryptor_->ManageEncryptedFrame(std::move(frame));
913 } else if (frame_transformer_delegate_) {
914 frame_transformer_delegate_->TransformFrame(std::move(frame));
915 } else {
916 OnCompleteFrames(reference_finder_->ManageFrame(std::move(frame)));
917 }
918 }
919
OnCompleteFrames(RtpFrameReferenceFinder::ReturnVector frames)920 void RtpVideoStreamReceiver2::OnCompleteFrames(
921 RtpFrameReferenceFinder::ReturnVector frames) {
922 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
923 for (auto& frame : frames) {
924 last_seq_num_for_pic_id_[frame->Id()] = frame->last_seq_num();
925
926 last_completed_picture_id_ =
927 std::max(last_completed_picture_id_, frame->Id());
928 complete_frame_callback_->OnCompleteFrame(std::move(frame));
929 }
930 }
931
OnDecryptedFrame(std::unique_ptr<RtpFrameObject> frame)932 void RtpVideoStreamReceiver2::OnDecryptedFrame(
933 std::unique_ptr<RtpFrameObject> frame) {
934 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
935 OnCompleteFrames(reference_finder_->ManageFrame(std::move(frame)));
936 }
937
OnDecryptionStatusChange(FrameDecryptorInterface::Status status)938 void RtpVideoStreamReceiver2::OnDecryptionStatusChange(
939 FrameDecryptorInterface::Status status) {
940 RTC_DCHECK_RUN_ON(&worker_task_checker_);
941 // Called from BufferedFrameDecryptor::DecryptFrame.
942 frames_decryptable_ =
943 (status == FrameDecryptorInterface::Status::kOk) ||
944 (status == FrameDecryptorInterface::Status::kRecoverable);
945 }
946
SetFrameDecryptor(rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor)947 void RtpVideoStreamReceiver2::SetFrameDecryptor(
948 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor) {
949 // TODO(bugs.webrtc.org/11993): Update callers or post the operation over to
950 // the network thread.
951 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
952 if (buffered_frame_decryptor_ == nullptr) {
953 buffered_frame_decryptor_ =
954 std::make_unique<BufferedFrameDecryptor>(this, this, field_trials_);
955 }
956 buffered_frame_decryptor_->SetFrameDecryptor(std::move(frame_decryptor));
957 }
958
SetDepacketizerToDecoderFrameTransformer(rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)959 void RtpVideoStreamReceiver2::SetDepacketizerToDecoderFrameTransformer(
960 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) {
961 RTC_DCHECK_RUN_ON(&worker_task_checker_);
962 frame_transformer_delegate_ =
963 rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
964 this, std::move(frame_transformer), rtc::Thread::Current(),
965 config_.rtp.remote_ssrc);
966 frame_transformer_delegate_->Init();
967 }
968
SetRtpExtensions(const std::vector<RtpExtension> & extensions)969 void RtpVideoStreamReceiver2::SetRtpExtensions(
970 const std::vector<RtpExtension>& extensions) {
971 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
972 rtp_header_extensions_.Reset(extensions);
973 }
974
GetRtpExtensions() const975 const RtpHeaderExtensionMap& RtpVideoStreamReceiver2::GetRtpExtensions() const {
976 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
977 return rtp_header_extensions_;
978 }
979
UpdateRtt(int64_t max_rtt_ms)980 void RtpVideoStreamReceiver2::UpdateRtt(int64_t max_rtt_ms) {
981 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
982 if (nack_module_)
983 nack_module_->UpdateRtt(max_rtt_ms);
984 }
985
OnLocalSsrcChange(uint32_t local_ssrc)986 void RtpVideoStreamReceiver2::OnLocalSsrcChange(uint32_t local_ssrc) {
987 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
988 rtp_rtcp_->SetLocalSsrc(local_ssrc);
989 }
990
SetRtcpMode(RtcpMode mode)991 void RtpVideoStreamReceiver2::SetRtcpMode(RtcpMode mode) {
992 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
993 rtp_rtcp_->SetRTCPStatus(mode);
994 }
995
SetReferenceTimeReport(bool enabled)996 void RtpVideoStreamReceiver2::SetReferenceTimeReport(bool enabled) {
997 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
998 rtp_rtcp_->SetNonSenderRttMeasurement(enabled);
999 }
1000
SetPacketSink(RtpPacketSinkInterface * packet_sink)1001 void RtpVideoStreamReceiver2::SetPacketSink(
1002 RtpPacketSinkInterface* packet_sink) {
1003 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
1004 packet_sink_ = packet_sink;
1005 }
1006
SetLossNotificationEnabled(bool enabled)1007 void RtpVideoStreamReceiver2::SetLossNotificationEnabled(bool enabled) {
1008 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
1009 if (enabled && !loss_notification_controller_) {
1010 loss_notification_controller_ =
1011 std::make_unique<LossNotificationController>(&rtcp_feedback_buffer_,
1012 &rtcp_feedback_buffer_);
1013 } else if (!enabled && loss_notification_controller_) {
1014 loss_notification_controller_.reset();
1015 rtcp_feedback_buffer_.ClearLossNotificationState();
1016 }
1017 }
1018
SetNackHistory(TimeDelta history)1019 void RtpVideoStreamReceiver2::SetNackHistory(TimeDelta history) {
1020 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
1021 if (history.ms() == 0) {
1022 nack_module_.reset();
1023 } else if (!nack_module_) {
1024 nack_module_ = std::make_unique<NackRequester>(
1025 worker_queue_, nack_periodic_processor_, clock_, &rtcp_feedback_buffer_,
1026 &rtcp_feedback_buffer_, field_trials_);
1027 }
1028
1029 rtp_receive_statistics_->SetMaxReorderingThreshold(
1030 config_.rtp.remote_ssrc,
1031 history.ms() > 0 ? kMaxPacketAgeToNack : kDefaultMaxReorderingThreshold);
1032 }
1033
ulpfec_payload_type() const1034 int RtpVideoStreamReceiver2::ulpfec_payload_type() const {
1035 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
1036 return ulpfec_receiver_ ? ulpfec_receiver_->ulpfec_payload_type() : -1;
1037 }
1038
red_payload_type() const1039 int RtpVideoStreamReceiver2::red_payload_type() const {
1040 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
1041 return red_payload_type_;
1042 }
1043
SetProtectionPayloadTypes(int red_payload_type,int ulpfec_payload_type)1044 void RtpVideoStreamReceiver2::SetProtectionPayloadTypes(
1045 int red_payload_type,
1046 int ulpfec_payload_type) {
1047 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
1048 RTC_DCHECK(red_payload_type >= -1 && red_payload_type < 0x80);
1049 RTC_DCHECK(ulpfec_payload_type >= -1 && ulpfec_payload_type < 0x80);
1050 red_payload_type_ = red_payload_type;
1051 ulpfec_receiver_ = MaybeConstructUlpfecReceiver(
1052 config_.rtp.remote_ssrc, red_payload_type, ulpfec_payload_type,
1053 config_.rtp.extensions, this, clock_);
1054 }
1055
LastReceivedPacketMs() const1056 absl::optional<int64_t> RtpVideoStreamReceiver2::LastReceivedPacketMs() const {
1057 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
1058 if (last_received_rtp_system_time_) {
1059 return absl::optional<int64_t>(last_received_rtp_system_time_->ms());
1060 }
1061 return absl::nullopt;
1062 }
1063
LastReceivedKeyframePacketMs() const1064 absl::optional<int64_t> RtpVideoStreamReceiver2::LastReceivedKeyframePacketMs()
1065 const {
1066 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
1067 if (last_received_keyframe_rtp_system_time_) {
1068 return absl::optional<int64_t>(
1069 last_received_keyframe_rtp_system_time_->ms());
1070 }
1071 return absl::nullopt;
1072 }
1073
ManageFrame(std::unique_ptr<RtpFrameObject> frame)1074 void RtpVideoStreamReceiver2::ManageFrame(
1075 std::unique_ptr<RtpFrameObject> frame) {
1076 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
1077 OnCompleteFrames(reference_finder_->ManageFrame(std::move(frame)));
1078 }
1079
ReceivePacket(const RtpPacketReceived & packet)1080 void RtpVideoStreamReceiver2::ReceivePacket(const RtpPacketReceived& packet) {
1081 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
1082
1083 if (packet.payload_size() == 0) {
1084 // Padding or keep-alive packet.
1085 // TODO(nisse): Could drop empty packets earlier, but need to figure out how
1086 // they should be counted in stats.
1087 NotifyReceiverOfEmptyPacket(packet.SequenceNumber());
1088 return;
1089 }
1090 if (packet.PayloadType() == red_payload_type_) {
1091 ParseAndHandleEncapsulatingHeader(packet);
1092 return;
1093 }
1094
1095 const auto type_it = payload_type_map_.find(packet.PayloadType());
1096 if (type_it == payload_type_map_.end()) {
1097 return;
1098 }
1099 absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed_payload =
1100 type_it->second->Parse(packet.PayloadBuffer());
1101 if (parsed_payload == absl::nullopt) {
1102 RTC_LOG(LS_WARNING) << "Failed parsing payload.";
1103 return;
1104 }
1105
1106 OnReceivedPayloadData(std::move(parsed_payload->video_payload), packet,
1107 parsed_payload->video_header);
1108 }
1109
ParseAndHandleEncapsulatingHeader(const RtpPacketReceived & packet)1110 void RtpVideoStreamReceiver2::ParseAndHandleEncapsulatingHeader(
1111 const RtpPacketReceived& packet) {
1112 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
1113 RTC_DCHECK_EQ(packet.PayloadType(), red_payload_type_);
1114
1115 if (!ulpfec_receiver_ || packet.payload_size() == 0U)
1116 return;
1117
1118 if (packet.payload()[0] == ulpfec_receiver_->ulpfec_payload_type()) {
1119 // Notify video_receiver about received FEC packets to avoid NACKing these
1120 // packets.
1121 NotifyReceiverOfEmptyPacket(packet.SequenceNumber());
1122 }
1123 if (ulpfec_receiver_->AddReceivedRedPacket(packet)) {
1124 ulpfec_receiver_->ProcessReceivedFec();
1125 }
1126 }
1127
1128 // In the case of a video stream without picture ids and no rtx the
1129 // RtpFrameReferenceFinder will need to know about padding to
1130 // correctly calculate frame references.
NotifyReceiverOfEmptyPacket(uint16_t seq_num)1131 void RtpVideoStreamReceiver2::NotifyReceiverOfEmptyPacket(uint16_t seq_num) {
1132 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
1133 RTC_DCHECK_RUN_ON(&worker_task_checker_);
1134
1135 OnCompleteFrames(reference_finder_->PaddingReceived(seq_num));
1136
1137 OnInsertedPacket(packet_buffer_.InsertPadding(seq_num));
1138 if (nack_module_) {
1139 nack_module_->OnReceivedPacket(seq_num, /* is_keyframe = */ false,
1140 /* is _recovered = */ false);
1141 }
1142 if (loss_notification_controller_) {
1143 // TODO(bugs.webrtc.org/10336): Handle empty packets.
1144 RTC_LOG(LS_WARNING)
1145 << "LossNotificationController does not expect empty packets.";
1146 }
1147 }
1148
DeliverRtcp(const uint8_t * rtcp_packet,size_t rtcp_packet_length)1149 bool RtpVideoStreamReceiver2::DeliverRtcp(const uint8_t* rtcp_packet,
1150 size_t rtcp_packet_length) {
1151 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
1152
1153 if (!receiving_) {
1154 return false;
1155 }
1156
1157 rtp_rtcp_->IncomingRtcpPacket(rtcp_packet, rtcp_packet_length);
1158
1159 int64_t rtt = 0;
1160 rtp_rtcp_->RTT(config_.rtp.remote_ssrc, &rtt, nullptr, nullptr, nullptr);
1161 if (rtt == 0) {
1162 // Waiting for valid rtt.
1163 return true;
1164 }
1165 uint32_t ntp_secs = 0;
1166 uint32_t ntp_frac = 0;
1167 uint32_t rtp_timestamp = 0;
1168 uint32_t received_ntp_secs = 0;
1169 uint32_t received_ntp_frac = 0;
1170 if (rtp_rtcp_->RemoteNTP(&ntp_secs, &ntp_frac, &received_ntp_secs,
1171 &received_ntp_frac, &rtp_timestamp) != 0) {
1172 // Waiting for RTCP.
1173 return true;
1174 }
1175 NtpTime received_ntp(received_ntp_secs, received_ntp_frac);
1176 int64_t time_since_received =
1177 clock_->CurrentNtpInMilliseconds() - received_ntp.ToMs();
1178 // Don't use old SRs to estimate time.
1179 if (time_since_received <= 1) {
1180 ntp_estimator_.UpdateRtcpTimestamp(
1181 TimeDelta::Millis(rtt), NtpTime(ntp_secs, ntp_frac), rtp_timestamp);
1182 absl::optional<int64_t> remote_to_local_clock_offset =
1183 ntp_estimator_.EstimateRemoteToLocalClockOffset();
1184 if (remote_to_local_clock_offset.has_value()) {
1185 capture_clock_offset_updater_.SetRemoteToLocalClockOffset(
1186 *remote_to_local_clock_offset);
1187 }
1188 }
1189
1190 return true;
1191 }
1192
FrameContinuous(int64_t picture_id)1193 void RtpVideoStreamReceiver2::FrameContinuous(int64_t picture_id) {
1194 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
1195 if (!nack_module_)
1196 return;
1197
1198 int seq_num = -1;
1199 auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id);
1200 if (seq_num_it != last_seq_num_for_pic_id_.end())
1201 seq_num = seq_num_it->second;
1202 if (seq_num != -1)
1203 nack_module_->ClearUpTo(seq_num);
1204 }
1205
FrameDecoded(int64_t picture_id)1206 void RtpVideoStreamReceiver2::FrameDecoded(int64_t picture_id) {
1207 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
1208 int seq_num = -1;
1209 auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id);
1210 if (seq_num_it != last_seq_num_for_pic_id_.end()) {
1211 seq_num = seq_num_it->second;
1212 last_seq_num_for_pic_id_.erase(last_seq_num_for_pic_id_.begin(),
1213 ++seq_num_it);
1214 }
1215
1216 if (seq_num != -1) {
1217 int64_t unwrapped_rtp_seq_num = rtp_seq_num_unwrapper_.Unwrap(seq_num);
1218 packet_infos_.erase(packet_infos_.begin(),
1219 packet_infos_.upper_bound(unwrapped_rtp_seq_num));
1220 packet_buffer_.ClearTo(seq_num);
1221 reference_finder_->ClearTo(seq_num);
1222 }
1223 }
1224
SignalNetworkState(NetworkState state)1225 void RtpVideoStreamReceiver2::SignalNetworkState(NetworkState state) {
1226 RTC_DCHECK_RUN_ON(&worker_task_checker_);
1227 rtp_rtcp_->SetRTCPStatus(state == kNetworkUp ? config_.rtp.rtcp_mode
1228 : RtcpMode::kOff);
1229 }
1230
StartReceive()1231 void RtpVideoStreamReceiver2::StartReceive() {
1232 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
1233 receiving_ = true;
1234 }
1235
StopReceive()1236 void RtpVideoStreamReceiver2::StopReceive() {
1237 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
1238 receiving_ = false;
1239 }
1240
InsertSpsPpsIntoTracker(uint8_t payload_type)1241 void RtpVideoStreamReceiver2::InsertSpsPpsIntoTracker(uint8_t payload_type) {
1242 RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
1243 RTC_DCHECK_RUN_ON(&worker_task_checker_);
1244
1245 auto codec_params_it = pt_codec_params_.find(payload_type);
1246 if (codec_params_it == pt_codec_params_.end())
1247 return;
1248
1249 RTC_LOG(LS_INFO) << "Found out of band supplied codec parameters for"
1250 " payload type: "
1251 << static_cast<int>(payload_type);
1252
1253 H264SpropParameterSets sprop_decoder;
1254 auto sprop_base64_it =
1255 codec_params_it->second.find(cricket::kH264FmtpSpropParameterSets);
1256
1257 if (sprop_base64_it == codec_params_it->second.end())
1258 return;
1259
1260 if (!sprop_decoder.DecodeSprop(sprop_base64_it->second.c_str()))
1261 return;
1262
1263 tracker_.InsertSpsPpsNalus(sprop_decoder.sps_nalu(),
1264 sprop_decoder.pps_nalu());
1265 }
1266
UpdatePacketReceiveTimestamps(const RtpPacketReceived & packet,bool is_keyframe)1267 void RtpVideoStreamReceiver2::UpdatePacketReceiveTimestamps(
1268 const RtpPacketReceived& packet,
1269 bool is_keyframe) {
1270 Timestamp now = clock_->CurrentTime();
1271 if (is_keyframe ||
1272 last_received_keyframe_rtp_timestamp_ == packet.Timestamp()) {
1273 last_received_keyframe_rtp_timestamp_ = packet.Timestamp();
1274 last_received_keyframe_rtp_system_time_ = now;
1275 }
1276 last_received_rtp_system_time_ = now;
1277 last_received_rtp_timestamp_ = packet.Timestamp();
1278
1279 // Periodically log the RTP header of incoming packets.
1280 if (now.ms() - last_packet_log_ms_ > kPacketLogIntervalMs) {
1281 rtc::StringBuilder ss;
1282 ss << "Packet received on SSRC: " << packet.Ssrc()
1283 << " with payload type: " << static_cast<int>(packet.PayloadType())
1284 << ", timestamp: " << packet.Timestamp()
1285 << ", sequence number: " << packet.SequenceNumber()
1286 << ", arrival time: " << ToString(packet.arrival_time());
1287 int32_t time_offset;
1288 if (packet.GetExtension<TransmissionOffset>(&time_offset)) {
1289 ss << ", toffset: " << time_offset;
1290 }
1291 uint32_t send_time;
1292 if (packet.GetExtension<AbsoluteSendTime>(&send_time)) {
1293 ss << ", abs send time: " << send_time;
1294 }
1295 RTC_LOG(LS_INFO) << ss.str();
1296 last_packet_log_ms_ = now.ms();
1297 }
1298 }
1299
1300 } // namespace webrtc
1301