1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/video/vie_receiver.h"
12
13 #include <vector>
14
15 #include "webrtc/base/logging.h"
16 #include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h"
17 #include "webrtc/modules/rtp_rtcp/include/fec_receiver.h"
18 #include "webrtc/modules/rtp_rtcp/include/receive_statistics.h"
19 #include "webrtc/modules/rtp_rtcp/include/remote_ntp_time_estimator.h"
20 #include "webrtc/modules/rtp_rtcp/include/rtp_cvo.h"
21 #include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h"
22 #include "webrtc/modules/rtp_rtcp/include/rtp_payload_registry.h"
23 #include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h"
24 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
25 #include "webrtc/modules/video_coding/include/video_coding.h"
26 #include "webrtc/system_wrappers/include/critical_section_wrapper.h"
27 #include "webrtc/system_wrappers/include/metrics.h"
28 #include "webrtc/system_wrappers/include/tick_util.h"
29 #include "webrtc/system_wrappers/include/timestamp_extrapolator.h"
30 #include "webrtc/system_wrappers/include/trace.h"
31
32 namespace webrtc {
33
34 static const int kPacketLogIntervalMs = 10000;
35
ViEReceiver(VideoCodingModule * module_vcm,RemoteBitrateEstimator * remote_bitrate_estimator,RtpFeedback * rtp_feedback)36 ViEReceiver::ViEReceiver(VideoCodingModule* module_vcm,
37 RemoteBitrateEstimator* remote_bitrate_estimator,
38 RtpFeedback* rtp_feedback)
39 : receive_cs_(CriticalSectionWrapper::CreateCriticalSection()),
40 clock_(Clock::GetRealTimeClock()),
41 rtp_header_parser_(RtpHeaderParser::Create()),
42 rtp_payload_registry_(
43 new RTPPayloadRegistry(RTPPayloadStrategy::CreateStrategy(false))),
44 rtp_receiver_(
45 RtpReceiver::CreateVideoReceiver(clock_,
46 this,
47 rtp_feedback,
48 rtp_payload_registry_.get())),
49 rtp_receive_statistics_(ReceiveStatistics::Create(clock_)),
50 fec_receiver_(FecReceiver::Create(this)),
51 rtp_rtcp_(NULL),
52 vcm_(module_vcm),
53 remote_bitrate_estimator_(remote_bitrate_estimator),
54 ntp_estimator_(new RemoteNtpTimeEstimator(clock_)),
55 receiving_(false),
56 restored_packet_in_use_(false),
57 receiving_ast_enabled_(false),
58 receiving_cvo_enabled_(false),
59 receiving_tsn_enabled_(false),
60 last_packet_log_ms_(-1) {
61 assert(remote_bitrate_estimator);
62 }
63
~ViEReceiver()64 ViEReceiver::~ViEReceiver() {
65 UpdateHistograms();
66 }
67
UpdateHistograms()68 void ViEReceiver::UpdateHistograms() {
69 FecPacketCounter counter = fec_receiver_->GetPacketCounter();
70 if (counter.num_packets > 0) {
71 RTC_HISTOGRAM_PERCENTAGE_SPARSE(
72 "WebRTC.Video.ReceivedFecPacketsInPercent",
73 static_cast<int>(counter.num_fec_packets * 100 / counter.num_packets));
74 }
75 if (counter.num_fec_packets > 0) {
76 RTC_HISTOGRAM_PERCENTAGE_SPARSE(
77 "WebRTC.Video.RecoveredMediaPacketsInPercentOfFec",
78 static_cast<int>(counter.num_recovered_packets * 100 /
79 counter.num_fec_packets));
80 }
81 }
82
SetReceiveCodec(const VideoCodec & video_codec)83 bool ViEReceiver::SetReceiveCodec(const VideoCodec& video_codec) {
84 int8_t old_pltype = -1;
85 if (rtp_payload_registry_->ReceivePayloadType(video_codec.plName,
86 kVideoPayloadTypeFrequency,
87 0,
88 video_codec.maxBitrate,
89 &old_pltype) != -1) {
90 rtp_payload_registry_->DeRegisterReceivePayload(old_pltype);
91 }
92
93 return RegisterPayload(video_codec);
94 }
95
RegisterPayload(const VideoCodec & video_codec)96 bool ViEReceiver::RegisterPayload(const VideoCodec& video_codec) {
97 return rtp_receiver_->RegisterReceivePayload(video_codec.plName,
98 video_codec.plType,
99 kVideoPayloadTypeFrequency,
100 0,
101 video_codec.maxBitrate) == 0;
102 }
103
SetNackStatus(bool enable,int max_nack_reordering_threshold)104 void ViEReceiver::SetNackStatus(bool enable,
105 int max_nack_reordering_threshold) {
106 if (!enable) {
107 // Reset the threshold back to the lower default threshold when NACK is
108 // disabled since we no longer will be receiving retransmissions.
109 max_nack_reordering_threshold = kDefaultMaxReorderingThreshold;
110 }
111 rtp_receive_statistics_->SetMaxReorderingThreshold(
112 max_nack_reordering_threshold);
113 rtp_receiver_->SetNACKStatus(enable ? kNackRtcp : kNackOff);
114 }
115
SetRtxPayloadType(int payload_type,int associated_payload_type)116 void ViEReceiver::SetRtxPayloadType(int payload_type,
117 int associated_payload_type) {
118 rtp_payload_registry_->SetRtxPayloadType(payload_type,
119 associated_payload_type);
120 }
121
SetUseRtxPayloadMappingOnRestore(bool val)122 void ViEReceiver::SetUseRtxPayloadMappingOnRestore(bool val) {
123 rtp_payload_registry_->set_use_rtx_payload_mapping_on_restore(val);
124 }
125
SetRtxSsrc(uint32_t ssrc)126 void ViEReceiver::SetRtxSsrc(uint32_t ssrc) {
127 rtp_payload_registry_->SetRtxSsrc(ssrc);
128 }
129
GetRtxSsrc(uint32_t * ssrc) const130 bool ViEReceiver::GetRtxSsrc(uint32_t* ssrc) const {
131 return rtp_payload_registry_->GetRtxSsrc(ssrc);
132 }
133
IsFecEnabled() const134 bool ViEReceiver::IsFecEnabled() const {
135 return rtp_payload_registry_->ulpfec_payload_type() > -1;
136 }
137
GetRemoteSsrc() const138 uint32_t ViEReceiver::GetRemoteSsrc() const {
139 return rtp_receiver_->SSRC();
140 }
141
GetCsrcs(uint32_t * csrcs) const142 int ViEReceiver::GetCsrcs(uint32_t* csrcs) const {
143 return rtp_receiver_->CSRCs(csrcs);
144 }
145
SetRtpRtcpModule(RtpRtcp * module)146 void ViEReceiver::SetRtpRtcpModule(RtpRtcp* module) {
147 rtp_rtcp_ = module;
148 }
149
GetRtpReceiver() const150 RtpReceiver* ViEReceiver::GetRtpReceiver() const {
151 return rtp_receiver_.get();
152 }
153
RegisterRtpRtcpModules(const std::vector<RtpRtcp * > & rtp_modules)154 void ViEReceiver::RegisterRtpRtcpModules(
155 const std::vector<RtpRtcp*>& rtp_modules) {
156 CriticalSectionScoped cs(receive_cs_.get());
157 // Only change the "simulcast" modules, the base module can be accessed
158 // without a lock whereas the simulcast modules require locking as they can be
159 // changed in runtime.
160 rtp_rtcp_simulcast_ =
161 std::vector<RtpRtcp*>(rtp_modules.begin() + 1, rtp_modules.end());
162 }
163
SetReceiveTimestampOffsetStatus(bool enable,int id)164 bool ViEReceiver::SetReceiveTimestampOffsetStatus(bool enable, int id) {
165 if (enable) {
166 return rtp_header_parser_->RegisterRtpHeaderExtension(
167 kRtpExtensionTransmissionTimeOffset, id);
168 } else {
169 return rtp_header_parser_->DeregisterRtpHeaderExtension(
170 kRtpExtensionTransmissionTimeOffset);
171 }
172 }
173
SetReceiveAbsoluteSendTimeStatus(bool enable,int id)174 bool ViEReceiver::SetReceiveAbsoluteSendTimeStatus(bool enable, int id) {
175 if (enable) {
176 if (rtp_header_parser_->RegisterRtpHeaderExtension(
177 kRtpExtensionAbsoluteSendTime, id)) {
178 receiving_ast_enabled_ = true;
179 return true;
180 } else {
181 return false;
182 }
183 } else {
184 receiving_ast_enabled_ = false;
185 return rtp_header_parser_->DeregisterRtpHeaderExtension(
186 kRtpExtensionAbsoluteSendTime);
187 }
188 }
189
SetReceiveVideoRotationStatus(bool enable,int id)190 bool ViEReceiver::SetReceiveVideoRotationStatus(bool enable, int id) {
191 if (enable) {
192 if (rtp_header_parser_->RegisterRtpHeaderExtension(
193 kRtpExtensionVideoRotation, id)) {
194 receiving_cvo_enabled_ = true;
195 return true;
196 } else {
197 return false;
198 }
199 } else {
200 receiving_cvo_enabled_ = false;
201 return rtp_header_parser_->DeregisterRtpHeaderExtension(
202 kRtpExtensionVideoRotation);
203 }
204 }
205
SetReceiveTransportSequenceNumber(bool enable,int id)206 bool ViEReceiver::SetReceiveTransportSequenceNumber(bool enable, int id) {
207 if (enable) {
208 if (rtp_header_parser_->RegisterRtpHeaderExtension(
209 kRtpExtensionTransportSequenceNumber, id)) {
210 receiving_tsn_enabled_ = true;
211 return true;
212 } else {
213 return false;
214 }
215 } else {
216 receiving_tsn_enabled_ = false;
217 return rtp_header_parser_->DeregisterRtpHeaderExtension(
218 kRtpExtensionTransportSequenceNumber);
219 }
220 }
221
ReceivedRTPPacket(const void * rtp_packet,size_t rtp_packet_length,const PacketTime & packet_time)222 int ViEReceiver::ReceivedRTPPacket(const void* rtp_packet,
223 size_t rtp_packet_length,
224 const PacketTime& packet_time) {
225 return InsertRTPPacket(static_cast<const uint8_t*>(rtp_packet),
226 rtp_packet_length, packet_time);
227 }
228
ReceivedRTCPPacket(const void * rtcp_packet,size_t rtcp_packet_length)229 int ViEReceiver::ReceivedRTCPPacket(const void* rtcp_packet,
230 size_t rtcp_packet_length) {
231 return InsertRTCPPacket(static_cast<const uint8_t*>(rtcp_packet),
232 rtcp_packet_length);
233 }
234
OnReceivedPayloadData(const uint8_t * payload_data,const size_t payload_size,const WebRtcRTPHeader * rtp_header)235 int32_t ViEReceiver::OnReceivedPayloadData(const uint8_t* payload_data,
236 const size_t payload_size,
237 const WebRtcRTPHeader* rtp_header) {
238 WebRtcRTPHeader rtp_header_with_ntp = *rtp_header;
239 rtp_header_with_ntp.ntp_time_ms =
240 ntp_estimator_->Estimate(rtp_header->header.timestamp);
241 if (vcm_->IncomingPacket(payload_data,
242 payload_size,
243 rtp_header_with_ntp) != 0) {
244 // Check this...
245 return -1;
246 }
247 return 0;
248 }
249
OnRecoveredPacket(const uint8_t * rtp_packet,size_t rtp_packet_length)250 bool ViEReceiver::OnRecoveredPacket(const uint8_t* rtp_packet,
251 size_t rtp_packet_length) {
252 RTPHeader header;
253 if (!rtp_header_parser_->Parse(rtp_packet, rtp_packet_length, &header)) {
254 return false;
255 }
256 header.payload_type_frequency = kVideoPayloadTypeFrequency;
257 bool in_order = IsPacketInOrder(header);
258 return ReceivePacket(rtp_packet, rtp_packet_length, header, in_order);
259 }
260
InsertRTPPacket(const uint8_t * rtp_packet,size_t rtp_packet_length,const PacketTime & packet_time)261 int ViEReceiver::InsertRTPPacket(const uint8_t* rtp_packet,
262 size_t rtp_packet_length,
263 const PacketTime& packet_time) {
264 {
265 CriticalSectionScoped cs(receive_cs_.get());
266 if (!receiving_) {
267 return -1;
268 }
269 }
270
271 RTPHeader header;
272 if (!rtp_header_parser_->Parse(rtp_packet, rtp_packet_length,
273 &header)) {
274 return -1;
275 }
276 size_t payload_length = rtp_packet_length - header.headerLength;
277 int64_t arrival_time_ms;
278 int64_t now_ms = clock_->TimeInMilliseconds();
279 if (packet_time.timestamp != -1)
280 arrival_time_ms = (packet_time.timestamp + 500) / 1000;
281 else
282 arrival_time_ms = now_ms;
283
284 {
285 // Periodically log the RTP header of incoming packets.
286 CriticalSectionScoped cs(receive_cs_.get());
287 if (now_ms - last_packet_log_ms_ > kPacketLogIntervalMs) {
288 std::stringstream ss;
289 ss << "Packet received on SSRC: " << header.ssrc << " with payload type: "
290 << static_cast<int>(header.payloadType) << ", timestamp: "
291 << header.timestamp << ", sequence number: " << header.sequenceNumber
292 << ", arrival time: " << arrival_time_ms;
293 if (header.extension.hasTransmissionTimeOffset)
294 ss << ", toffset: " << header.extension.transmissionTimeOffset;
295 if (header.extension.hasAbsoluteSendTime)
296 ss << ", abs send time: " << header.extension.absoluteSendTime;
297 LOG(LS_INFO) << ss.str();
298 last_packet_log_ms_ = now_ms;
299 }
300 }
301
302 remote_bitrate_estimator_->IncomingPacket(arrival_time_ms, payload_length,
303 header, true);
304 header.payload_type_frequency = kVideoPayloadTypeFrequency;
305
306 bool in_order = IsPacketInOrder(header);
307 rtp_payload_registry_->SetIncomingPayloadType(header);
308 int ret = ReceivePacket(rtp_packet, rtp_packet_length, header, in_order)
309 ? 0
310 : -1;
311 // Update receive statistics after ReceivePacket.
312 // Receive statistics will be reset if the payload type changes (make sure
313 // that the first packet is included in the stats).
314 rtp_receive_statistics_->IncomingPacket(
315 header, rtp_packet_length, IsPacketRetransmitted(header, in_order));
316 return ret;
317 }
318
ReceivePacket(const uint8_t * packet,size_t packet_length,const RTPHeader & header,bool in_order)319 bool ViEReceiver::ReceivePacket(const uint8_t* packet,
320 size_t packet_length,
321 const RTPHeader& header,
322 bool in_order) {
323 if (rtp_payload_registry_->IsEncapsulated(header)) {
324 return ParseAndHandleEncapsulatingHeader(packet, packet_length, header);
325 }
326 const uint8_t* payload = packet + header.headerLength;
327 assert(packet_length >= header.headerLength);
328 size_t payload_length = packet_length - header.headerLength;
329 PayloadUnion payload_specific;
330 if (!rtp_payload_registry_->GetPayloadSpecifics(header.payloadType,
331 &payload_specific)) {
332 return false;
333 }
334 return rtp_receiver_->IncomingRtpPacket(header, payload, payload_length,
335 payload_specific, in_order);
336 }
337
ParseAndHandleEncapsulatingHeader(const uint8_t * packet,size_t packet_length,const RTPHeader & header)338 bool ViEReceiver::ParseAndHandleEncapsulatingHeader(const uint8_t* packet,
339 size_t packet_length,
340 const RTPHeader& header) {
341 if (rtp_payload_registry_->IsRed(header)) {
342 int8_t ulpfec_pt = rtp_payload_registry_->ulpfec_payload_type();
343 if (packet[header.headerLength] == ulpfec_pt) {
344 rtp_receive_statistics_->FecPacketReceived(header, packet_length);
345 // Notify vcm about received FEC packets to avoid NACKing these packets.
346 NotifyReceiverOfFecPacket(header);
347 }
348 if (fec_receiver_->AddReceivedRedPacket(
349 header, packet, packet_length, ulpfec_pt) != 0) {
350 return false;
351 }
352 return fec_receiver_->ProcessReceivedFec() == 0;
353 } else if (rtp_payload_registry_->IsRtx(header)) {
354 if (header.headerLength + header.paddingLength == packet_length) {
355 // This is an empty packet and should be silently dropped before trying to
356 // parse the RTX header.
357 return true;
358 }
359 // Remove the RTX header and parse the original RTP header.
360 if (packet_length < header.headerLength)
361 return false;
362 if (packet_length > sizeof(restored_packet_))
363 return false;
364 CriticalSectionScoped cs(receive_cs_.get());
365 if (restored_packet_in_use_) {
366 LOG(LS_WARNING) << "Multiple RTX headers detected, dropping packet.";
367 return false;
368 }
369 if (!rtp_payload_registry_->RestoreOriginalPacket(
370 restored_packet_, packet, &packet_length, rtp_receiver_->SSRC(),
371 header)) {
372 LOG(LS_WARNING) << "Incoming RTX packet: Invalid RTP header";
373 return false;
374 }
375 restored_packet_in_use_ = true;
376 bool ret = OnRecoveredPacket(restored_packet_, packet_length);
377 restored_packet_in_use_ = false;
378 return ret;
379 }
380 return false;
381 }
382
NotifyReceiverOfFecPacket(const RTPHeader & header)383 void ViEReceiver::NotifyReceiverOfFecPacket(const RTPHeader& header) {
384 int8_t last_media_payload_type =
385 rtp_payload_registry_->last_received_media_payload_type();
386 if (last_media_payload_type < 0) {
387 LOG(LS_WARNING) << "Failed to get last media payload type.";
388 return;
389 }
390 // Fake an empty media packet.
391 WebRtcRTPHeader rtp_header = {};
392 rtp_header.header = header;
393 rtp_header.header.payloadType = last_media_payload_type;
394 rtp_header.header.paddingLength = 0;
395 PayloadUnion payload_specific;
396 if (!rtp_payload_registry_->GetPayloadSpecifics(last_media_payload_type,
397 &payload_specific)) {
398 LOG(LS_WARNING) << "Failed to get payload specifics.";
399 return;
400 }
401 rtp_header.type.Video.codec = payload_specific.Video.videoCodecType;
402 rtp_header.type.Video.rotation = kVideoRotation_0;
403 if (header.extension.hasVideoRotation) {
404 rtp_header.type.Video.rotation =
405 ConvertCVOByteToVideoRotation(header.extension.videoRotation);
406 }
407 OnReceivedPayloadData(NULL, 0, &rtp_header);
408 }
409
InsertRTCPPacket(const uint8_t * rtcp_packet,size_t rtcp_packet_length)410 int ViEReceiver::InsertRTCPPacket(const uint8_t* rtcp_packet,
411 size_t rtcp_packet_length) {
412 {
413 CriticalSectionScoped cs(receive_cs_.get());
414 if (!receiving_) {
415 return -1;
416 }
417
418 for (RtpRtcp* rtp_rtcp : rtp_rtcp_simulcast_)
419 rtp_rtcp->IncomingRtcpPacket(rtcp_packet, rtcp_packet_length);
420 }
421 assert(rtp_rtcp_); // Should be set by owner at construction time.
422 int ret = rtp_rtcp_->IncomingRtcpPacket(rtcp_packet, rtcp_packet_length);
423 if (ret != 0) {
424 return ret;
425 }
426
427 int64_t rtt = 0;
428 rtp_rtcp_->RTT(rtp_receiver_->SSRC(), &rtt, NULL, NULL, NULL);
429 if (rtt == 0) {
430 // Waiting for valid rtt.
431 return 0;
432 }
433 uint32_t ntp_secs = 0;
434 uint32_t ntp_frac = 0;
435 uint32_t rtp_timestamp = 0;
436 if (0 != rtp_rtcp_->RemoteNTP(&ntp_secs, &ntp_frac, NULL, NULL,
437 &rtp_timestamp)) {
438 // Waiting for RTCP.
439 return 0;
440 }
441 ntp_estimator_->UpdateRtcpTimestamp(rtt, ntp_secs, ntp_frac, rtp_timestamp);
442
443 return 0;
444 }
445
StartReceive()446 void ViEReceiver::StartReceive() {
447 CriticalSectionScoped cs(receive_cs_.get());
448 receiving_ = true;
449 }
450
StopReceive()451 void ViEReceiver::StopReceive() {
452 CriticalSectionScoped cs(receive_cs_.get());
453 receiving_ = false;
454 }
455
GetReceiveStatistics() const456 ReceiveStatistics* ViEReceiver::GetReceiveStatistics() const {
457 return rtp_receive_statistics_.get();
458 }
459
IsPacketInOrder(const RTPHeader & header) const460 bool ViEReceiver::IsPacketInOrder(const RTPHeader& header) const {
461 StreamStatistician* statistician =
462 rtp_receive_statistics_->GetStatistician(header.ssrc);
463 if (!statistician)
464 return false;
465 return statistician->IsPacketInOrder(header.sequenceNumber);
466 }
467
IsPacketRetransmitted(const RTPHeader & header,bool in_order) const468 bool ViEReceiver::IsPacketRetransmitted(const RTPHeader& header,
469 bool in_order) const {
470 // Retransmissions are handled separately if RTX is enabled.
471 if (rtp_payload_registry_->RtxEnabled())
472 return false;
473 StreamStatistician* statistician =
474 rtp_receive_statistics_->GetStatistician(header.ssrc);
475 if (!statistician)
476 return false;
477 // Check if this is a retransmission.
478 int64_t min_rtt = 0;
479 rtp_rtcp_->RTT(rtp_receiver_->SSRC(), NULL, NULL, &min_rtt, NULL);
480 return !in_order &&
481 statistician->IsRetransmitOfOldPacket(header, min_rtt);
482 }
483 } // namespace webrtc
484