1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/rtp_rtcp/source/rtp_sender_video.h"
12
13 #include <stdlib.h>
14 #include <string.h>
15
16 #include <algorithm>
17 #include <limits>
18 #include <memory>
19 #include <string>
20 #include <utility>
21
22 #include "absl/algorithm/container.h"
23 #include "absl/memory/memory.h"
24 #include "absl/strings/match.h"
25 #include "api/crypto/frame_encryptor_interface.h"
26 #include "api/transport/rtp/dependency_descriptor.h"
27 #include "modules/remote_bitrate_estimator/test/bwe_test_logging.h"
28 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
29 #include "modules/rtp_rtcp/source/absolute_capture_time_sender.h"
30 #include "modules/rtp_rtcp/source/byte_io.h"
31 #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
32 #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
33 #include "modules/rtp_rtcp/source/rtp_format.h"
34 #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h"
35 #include "modules/rtp_rtcp/source/rtp_header_extensions.h"
36 #include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
37 #include "modules/rtp_rtcp/source/rtp_video_layers_allocation_extension.h"
38 #include "modules/rtp_rtcp/source/time_util.h"
39 #include "rtc_base/checks.h"
40 #include "rtc_base/experiments/field_trial_parser.h"
41 #include "rtc_base/logging.h"
42 #include "rtc_base/trace_event.h"
43
44 namespace webrtc {
45
46 namespace {
47 constexpr size_t kRedForFecHeaderLength = 1;
48 constexpr int64_t kMaxUnretransmittableFrameIntervalMs = 33 * 4;
49 constexpr char kIncludeCaptureClockOffset[] =
50 "WebRTC-IncludeCaptureClockOffset";
51
BuildRedPayload(const RtpPacketToSend & media_packet,RtpPacketToSend * red_packet)52 void BuildRedPayload(const RtpPacketToSend& media_packet,
53 RtpPacketToSend* red_packet) {
54 uint8_t* red_payload = red_packet->AllocatePayload(
55 kRedForFecHeaderLength + media_packet.payload_size());
56 RTC_DCHECK(red_payload);
57 red_payload[0] = media_packet.PayloadType();
58
59 auto media_payload = media_packet.payload();
60 memcpy(&red_payload[kRedForFecHeaderLength], media_payload.data(),
61 media_payload.size());
62 }
63
MinimizeDescriptor(RTPVideoHeader * video_header)64 bool MinimizeDescriptor(RTPVideoHeader* video_header) {
65 if (auto* vp8 =
66 absl::get_if<RTPVideoHeaderVP8>(&video_header->video_type_header)) {
67 // Set minimum fields the RtpPacketizer is using to create vp8 packets.
68 // nonReference is the only field that doesn't require extra space.
69 bool non_reference = vp8->nonReference;
70 vp8->InitRTPVideoHeaderVP8();
71 vp8->nonReference = non_reference;
72 return true;
73 }
74 // TODO(danilchap): Reduce vp9 codec specific descriptor too.
75 return false;
76 }
77
IsBaseLayer(const RTPVideoHeader & video_header)78 bool IsBaseLayer(const RTPVideoHeader& video_header) {
79 switch (video_header.codec) {
80 case kVideoCodecVP8: {
81 const auto& vp8 =
82 absl::get<RTPVideoHeaderVP8>(video_header.video_type_header);
83 return (vp8.temporalIdx == 0 || vp8.temporalIdx == kNoTemporalIdx);
84 }
85 case kVideoCodecVP9: {
86 const auto& vp9 =
87 absl::get<RTPVideoHeaderVP9>(video_header.video_type_header);
88 return (vp9.temporal_idx == 0 || vp9.temporal_idx == kNoTemporalIdx);
89 }
90 case kVideoCodecH264:
91 // TODO(kron): Implement logic for H264 once WebRTC supports temporal
92 // layers for H264.
93 break;
94 default:
95 break;
96 }
97 return true;
98 }
99
FrameTypeToString(VideoFrameType frame_type)100 [[maybe_unused]] const char* FrameTypeToString(VideoFrameType frame_type) {
101 switch (frame_type) {
102 case VideoFrameType::kEmptyFrame:
103 return "empty";
104 case VideoFrameType::kVideoFrameKey:
105 return "video_key";
106 case VideoFrameType::kVideoFrameDelta:
107 return "video_delta";
108 default:
109 RTC_DCHECK_NOTREACHED();
110 return "";
111 }
112 }
113
IsNoopDelay(const VideoPlayoutDelay & delay)114 bool IsNoopDelay(const VideoPlayoutDelay& delay) {
115 return delay.min_ms == -1 && delay.max_ms == -1;
116 }
117
LoadVideoPlayoutDelayOverride(const FieldTrialsView * key_value_config)118 absl::optional<VideoPlayoutDelay> LoadVideoPlayoutDelayOverride(
119 const FieldTrialsView* key_value_config) {
120 RTC_DCHECK(key_value_config);
121 FieldTrialOptional<int> playout_delay_min_ms("min_ms", absl::nullopt);
122 FieldTrialOptional<int> playout_delay_max_ms("max_ms", absl::nullopt);
123 ParseFieldTrial({&playout_delay_max_ms, &playout_delay_min_ms},
124 key_value_config->Lookup("WebRTC-ForceSendPlayoutDelay"));
125 return playout_delay_max_ms && playout_delay_min_ms
126 ? absl::make_optional<VideoPlayoutDelay>(*playout_delay_min_ms,
127 *playout_delay_max_ms)
128 : absl::nullopt;
129 }
130
131 // Some packets can be skipped and the stream can still be decoded. Those
132 // packets are less likely to be retransmitted if they are lost.
PacketWillLikelyBeRequestedForRestransmitionIfLost(const RTPVideoHeader & video_header)133 bool PacketWillLikelyBeRequestedForRestransmitionIfLost(
134 const RTPVideoHeader& video_header) {
135 return IsBaseLayer(video_header) &&
136 !(video_header.generic.has_value()
137 ? absl::c_linear_search(
138 video_header.generic->decode_target_indications,
139 DecodeTargetIndication::kDiscardable)
140 : false);
141 }
142
143 } // namespace
144
RTPSenderVideo(const Config & config)145 RTPSenderVideo::RTPSenderVideo(const Config& config)
146 : rtp_sender_(config.rtp_sender),
147 clock_(config.clock),
148 retransmission_settings_(
149 config.enable_retransmit_all_layers
150 ? kRetransmitAllLayers
151 : (kRetransmitBaseLayer | kConditionallyRetransmitHigherLayers)),
152 last_rotation_(kVideoRotation_0),
153 transmit_color_space_next_frame_(false),
154 send_allocation_(SendVideoLayersAllocation::kDontSend),
155 current_playout_delay_{-1, -1},
156 playout_delay_pending_(false),
157 forced_playout_delay_(LoadVideoPlayoutDelayOverride(config.field_trials)),
158 red_payload_type_(config.red_payload_type),
159 fec_type_(config.fec_type),
160 fec_overhead_bytes_(config.fec_overhead_bytes),
161 packetization_overhead_bitrate_(1000, RateStatistics::kBpsScale),
162 frame_encryptor_(config.frame_encryptor),
163 require_frame_encryption_(config.require_frame_encryption),
164 generic_descriptor_auth_experiment_(!absl::StartsWith(
165 config.field_trials->Lookup("WebRTC-GenericDescriptorAuth"),
166 "Disabled")),
167 absolute_capture_time_sender_(config.clock),
168 frame_transformer_delegate_(
169 config.frame_transformer
170 ? rtc::make_ref_counted<RTPSenderVideoFrameTransformerDelegate>(
171 this,
172 config.frame_transformer,
173 rtp_sender_->SSRC(),
174 config.task_queue_factory)
175 : nullptr),
176 include_capture_clock_offset_(!absl::StartsWith(
177 config.field_trials->Lookup(kIncludeCaptureClockOffset),
178 "Disabled")) {
179 if (frame_transformer_delegate_)
180 frame_transformer_delegate_->Init();
181 }
182
~RTPSenderVideo()183 RTPSenderVideo::~RTPSenderVideo() {
184 if (frame_transformer_delegate_)
185 frame_transformer_delegate_->Reset();
186 }
187
LogAndSendToNetwork(std::vector<std::unique_ptr<RtpPacketToSend>> packets,size_t unpacketized_payload_size)188 void RTPSenderVideo::LogAndSendToNetwork(
189 std::vector<std::unique_ptr<RtpPacketToSend>> packets,
190 size_t unpacketized_payload_size) {
191 {
192 MutexLock lock(&stats_mutex_);
193 size_t packetized_payload_size = 0;
194 for (const auto& packet : packets) {
195 if (*packet->packet_type() == RtpPacketMediaType::kVideo) {
196 packetized_payload_size += packet->payload_size();
197 }
198 }
199 // AV1 and H264 packetizers may produce less packetized bytes than
200 // unpacketized.
201 if (packetized_payload_size >= unpacketized_payload_size) {
202 packetization_overhead_bitrate_.Update(
203 packetized_payload_size - unpacketized_payload_size,
204 clock_->TimeInMilliseconds());
205 }
206 }
207
208 rtp_sender_->EnqueuePackets(std::move(packets));
209 }
210
FecPacketOverhead() const211 size_t RTPSenderVideo::FecPacketOverhead() const {
212 size_t overhead = fec_overhead_bytes_;
213 if (red_enabled()) {
214 // The RED overhead is due to a small header.
215 overhead += kRedForFecHeaderLength;
216
217 if (fec_type_ == VideoFecGenerator::FecType::kUlpFec) {
218 // For ULPFEC, the overhead is the FEC headers plus RED for FEC header
219 // (see above) plus anything in RTP header beyond the 12 bytes base header
220 // (CSRC list, extensions...)
221 // This reason for the header extensions to be included here is that
222 // from an FEC viewpoint, they are part of the payload to be protected.
223 // (The base RTP header is already protected by the FEC header.)
224 overhead +=
225 rtp_sender_->FecOrPaddingPacketMaxRtpHeaderLength() - kRtpHeaderSize;
226 }
227 }
228 return overhead;
229 }
230
SetVideoStructure(const FrameDependencyStructure * video_structure)231 void RTPSenderVideo::SetVideoStructure(
232 const FrameDependencyStructure* video_structure) {
233 if (frame_transformer_delegate_) {
234 frame_transformer_delegate_->SetVideoStructureUnderLock(video_structure);
235 return;
236 }
237 SetVideoStructureInternal(video_structure);
238 }
239
SetVideoStructureAfterTransformation(const FrameDependencyStructure * video_structure)240 void RTPSenderVideo::SetVideoStructureAfterTransformation(
241 const FrameDependencyStructure* video_structure) {
242 SetVideoStructureInternal(video_structure);
243 }
244
SetVideoStructureInternal(const FrameDependencyStructure * video_structure)245 void RTPSenderVideo::SetVideoStructureInternal(
246 const FrameDependencyStructure* video_structure) {
247 RTC_DCHECK_RUNS_SERIALIZED(&send_checker_);
248 if (video_structure == nullptr) {
249 video_structure_ = nullptr;
250 return;
251 }
252 // Simple sanity checks video structure is set up.
253 RTC_DCHECK_GT(video_structure->num_decode_targets, 0);
254 RTC_DCHECK_GT(video_structure->templates.size(), 0);
255
256 int structure_id = 0;
257 if (video_structure_) {
258 if (*video_structure_ == *video_structure) {
259 // Same structure (just a new key frame), no update required.
260 return;
261 }
262 // When setting different video structure make sure structure_id is updated
263 // so that templates from different structures do not collide.
264 static constexpr int kMaxTemplates = 64;
265 structure_id =
266 (video_structure_->structure_id + video_structure_->templates.size()) %
267 kMaxTemplates;
268 }
269
270 video_structure_ =
271 std::make_unique<FrameDependencyStructure>(*video_structure);
272 video_structure_->structure_id = structure_id;
273 }
274
SetVideoLayersAllocation(VideoLayersAllocation allocation)275 void RTPSenderVideo::SetVideoLayersAllocation(
276 VideoLayersAllocation allocation) {
277 if (frame_transformer_delegate_) {
278 frame_transformer_delegate_->SetVideoLayersAllocationUnderLock(
279 std::move(allocation));
280 return;
281 }
282 SetVideoLayersAllocationInternal(std::move(allocation));
283 }
284
SetVideoLayersAllocationAfterTransformation(VideoLayersAllocation allocation)285 void RTPSenderVideo::SetVideoLayersAllocationAfterTransformation(
286 VideoLayersAllocation allocation) {
287 SetVideoLayersAllocationInternal(std::move(allocation));
288 }
289
SetVideoLayersAllocationInternal(VideoLayersAllocation allocation)290 void RTPSenderVideo::SetVideoLayersAllocationInternal(
291 VideoLayersAllocation allocation) {
292 RTC_DCHECK_RUNS_SERIALIZED(&send_checker_);
293 if (!allocation_ || allocation.active_spatial_layers.size() !=
294 allocation_->active_spatial_layers.size()) {
295 send_allocation_ = SendVideoLayersAllocation::kSendWithResolution;
296 } else if (send_allocation_ == SendVideoLayersAllocation::kDontSend) {
297 send_allocation_ = SendVideoLayersAllocation::kSendWithoutResolution;
298 }
299 if (send_allocation_ == SendVideoLayersAllocation::kSendWithoutResolution) {
300 // Check if frame rate changed more than 5fps since the last time the
301 // extension was sent with frame rate and resolution.
302 for (size_t i = 0; i < allocation.active_spatial_layers.size(); ++i) {
303 if (abs(static_cast<int>(
304 allocation.active_spatial_layers[i].frame_rate_fps) -
305 static_cast<int>(
306 last_full_sent_allocation_->active_spatial_layers[i]
307 .frame_rate_fps)) > 5) {
308 send_allocation_ = SendVideoLayersAllocation::kSendWithResolution;
309 break;
310 }
311 }
312 }
313 allocation_ = std::move(allocation);
314 }
315
AddRtpHeaderExtensions(const RTPVideoHeader & video_header,bool first_packet,bool last_packet,RtpPacketToSend * packet) const316 void RTPSenderVideo::AddRtpHeaderExtensions(const RTPVideoHeader& video_header,
317 bool first_packet,
318 bool last_packet,
319 RtpPacketToSend* packet) const {
320 // Send color space when changed or if the frame is a key frame. Keep
321 // sending color space information until the first base layer frame to
322 // guarantee that the information is retrieved by the receiver.
323 bool set_color_space =
324 video_header.color_space != last_color_space_ ||
325 video_header.frame_type == VideoFrameType::kVideoFrameKey ||
326 transmit_color_space_next_frame_;
327 // Color space requires two-byte header extensions if HDR metadata is
328 // included. Therefore, it's best to add this extension first so that the
329 // other extensions in the same packet are written as two-byte headers at
330 // once.
331 if (last_packet && set_color_space && video_header.color_space)
332 packet->SetExtension<ColorSpaceExtension>(video_header.color_space.value());
333
334 // According to
335 // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/
336 // ts_126114v120700p.pdf Section 7.4.5:
337 // The MTSI client shall add the payload bytes as defined in this clause
338 // onto the last RTP packet in each group of packets which make up a key
339 // frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265
340 // (HEVC)). The MTSI client may also add the payload bytes onto the last RTP
341 // packet in each group of packets which make up another type of frame
342 // (e.g. a P-Frame) only if the current value is different from the previous
343 // value sent.
344 // Set rotation when key frame or when changed (to follow standard).
345 // Or when different from 0 (to follow current receiver implementation).
346 bool set_video_rotation =
347 video_header.frame_type == VideoFrameType::kVideoFrameKey ||
348 video_header.rotation != last_rotation_ ||
349 video_header.rotation != kVideoRotation_0;
350 if (last_packet && set_video_rotation)
351 packet->SetExtension<VideoOrientation>(video_header.rotation);
352
353 // Report content type only for key frames.
354 if (last_packet &&
355 video_header.frame_type == VideoFrameType::kVideoFrameKey &&
356 video_header.content_type != VideoContentType::UNSPECIFIED)
357 packet->SetExtension<VideoContentTypeExtension>(video_header.content_type);
358
359 if (last_packet &&
360 video_header.video_timing.flags != VideoSendTiming::kInvalid)
361 packet->SetExtension<VideoTimingExtension>(video_header.video_timing);
362
363 // If transmitted, add to all packets; ack logic depends on this.
364 if (playout_delay_pending_) {
365 packet->SetExtension<PlayoutDelayLimits>(current_playout_delay_);
366 }
367
368 if (first_packet && video_header.absolute_capture_time.has_value()) {
369 packet->SetExtension<AbsoluteCaptureTimeExtension>(
370 *video_header.absolute_capture_time);
371 }
372
373 if (video_header.generic) {
374 bool extension_is_set = false;
375 if (packet->IsRegistered<RtpDependencyDescriptorExtension>() &&
376 video_structure_ != nullptr) {
377 DependencyDescriptor descriptor;
378 descriptor.first_packet_in_frame = first_packet;
379 descriptor.last_packet_in_frame = last_packet;
380 descriptor.frame_number = video_header.generic->frame_id & 0xFFFF;
381 descriptor.frame_dependencies.spatial_id =
382 video_header.generic->spatial_index;
383 descriptor.frame_dependencies.temporal_id =
384 video_header.generic->temporal_index;
385 for (int64_t dep : video_header.generic->dependencies) {
386 descriptor.frame_dependencies.frame_diffs.push_back(
387 video_header.generic->frame_id - dep);
388 }
389 descriptor.frame_dependencies.chain_diffs =
390 video_header.generic->chain_diffs;
391 descriptor.frame_dependencies.decode_target_indications =
392 video_header.generic->decode_target_indications;
393 RTC_DCHECK_EQ(
394 descriptor.frame_dependencies.decode_target_indications.size(),
395 video_structure_->num_decode_targets);
396
397 if (first_packet) {
398 descriptor.active_decode_targets_bitmask =
399 active_decode_targets_tracker_.ActiveDecodeTargetsBitmask();
400 }
401 // VP9 mark all layer frames of the first picture as kVideoFrameKey,
402 // Structure should be attached to the descriptor to lowest spatial layer
403 // when inter layer dependency is used, i.e. L structures; or to all
404 // layers when inter layer dependency is not used, i.e. S structures.
405 // Distinguish these two cases by checking if there are any dependencies.
406 if (video_header.frame_type == VideoFrameType::kVideoFrameKey &&
407 video_header.generic->dependencies.empty() && first_packet) {
408 // To avoid extra structure copy, temporary share ownership of the
409 // video_structure with the dependency descriptor.
410 descriptor.attached_structure =
411 absl::WrapUnique(video_structure_.get());
412 }
413 extension_is_set = packet->SetExtension<RtpDependencyDescriptorExtension>(
414 *video_structure_,
415 active_decode_targets_tracker_.ActiveChainsBitmask(), descriptor);
416
417 // Remove the temporary shared ownership.
418 descriptor.attached_structure.release();
419 }
420
421 // Do not use generic frame descriptor when dependency descriptor is stored.
422 if (packet->IsRegistered<RtpGenericFrameDescriptorExtension00>() &&
423 !extension_is_set) {
424 RtpGenericFrameDescriptor generic_descriptor;
425 generic_descriptor.SetFirstPacketInSubFrame(first_packet);
426 generic_descriptor.SetLastPacketInSubFrame(last_packet);
427
428 if (first_packet) {
429 generic_descriptor.SetFrameId(
430 static_cast<uint16_t>(video_header.generic->frame_id));
431 for (int64_t dep : video_header.generic->dependencies) {
432 generic_descriptor.AddFrameDependencyDiff(
433 video_header.generic->frame_id - dep);
434 }
435
436 uint8_t spatial_bimask = 1 << video_header.generic->spatial_index;
437 generic_descriptor.SetSpatialLayersBitmask(spatial_bimask);
438
439 generic_descriptor.SetTemporalLayer(
440 video_header.generic->temporal_index);
441
442 if (video_header.frame_type == VideoFrameType::kVideoFrameKey) {
443 generic_descriptor.SetResolution(video_header.width,
444 video_header.height);
445 }
446 }
447
448 packet->SetExtension<RtpGenericFrameDescriptorExtension00>(
449 generic_descriptor);
450 }
451 }
452
453 if (packet->IsRegistered<RtpVideoLayersAllocationExtension>() &&
454 first_packet &&
455 send_allocation_ != SendVideoLayersAllocation::kDontSend &&
456 (video_header.frame_type == VideoFrameType::kVideoFrameKey ||
457 PacketWillLikelyBeRequestedForRestransmitionIfLost(video_header))) {
458 VideoLayersAllocation allocation = allocation_.value();
459 allocation.resolution_and_frame_rate_is_valid =
460 send_allocation_ == SendVideoLayersAllocation::kSendWithResolution;
461 packet->SetExtension<RtpVideoLayersAllocationExtension>(allocation);
462 }
463
464 if (first_packet && video_header.video_frame_tracking_id) {
465 packet->SetExtension<VideoFrameTrackingIdExtension>(
466 *video_header.video_frame_tracking_id);
467 }
468 }
469
SendVideo(int payload_type,absl::optional<VideoCodecType> codec_type,uint32_t rtp_timestamp,int64_t capture_time_ms,rtc::ArrayView<const uint8_t> payload,RTPVideoHeader video_header,absl::optional<int64_t> expected_retransmission_time_ms)470 bool RTPSenderVideo::SendVideo(
471 int payload_type,
472 absl::optional<VideoCodecType> codec_type,
473 uint32_t rtp_timestamp,
474 int64_t capture_time_ms,
475 rtc::ArrayView<const uint8_t> payload,
476 RTPVideoHeader video_header,
477 absl::optional<int64_t> expected_retransmission_time_ms) {
478 TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", capture_time_ms, "Send", "type",
479 FrameTypeToString(video_header.frame_type));
480 RTC_CHECK_RUNS_SERIALIZED(&send_checker_);
481
482 if (video_header.frame_type == VideoFrameType::kEmptyFrame)
483 return true;
484
485 if (payload.empty())
486 return false;
487 if (!rtp_sender_->SendingMedia()) {
488 return false;
489 }
490
491 int32_t retransmission_settings = retransmission_settings_;
492 if (codec_type == VideoCodecType::kVideoCodecH264) {
493 // Backward compatibility for older receivers without temporal layer logic.
494 retransmission_settings = kRetransmitBaseLayer | kRetransmitHigherLayers;
495 }
496
497 MaybeUpdateCurrentPlayoutDelay(video_header);
498 if (video_header.frame_type == VideoFrameType::kVideoFrameKey) {
499 if (!IsNoopDelay(current_playout_delay_)) {
500 // Force playout delay on key-frames, if set.
501 playout_delay_pending_ = true;
502 }
503 if (allocation_) {
504 // Send the bitrate allocation on every key frame.
505 send_allocation_ = SendVideoLayersAllocation::kSendWithResolution;
506 }
507 }
508
509 if (video_structure_ != nullptr && video_header.generic) {
510 active_decode_targets_tracker_.OnFrame(
511 video_structure_->decode_target_protected_by_chain,
512 video_header.generic->active_decode_targets,
513 video_header.frame_type == VideoFrameType::kVideoFrameKey,
514 video_header.generic->frame_id, video_header.generic->chain_diffs);
515 }
516
517 const uint8_t temporal_id = GetTemporalId(video_header);
518 // No FEC protection for upper temporal layers, if used.
519 const bool use_fec = fec_type_.has_value() &&
520 (temporal_id == 0 || temporal_id == kNoTemporalIdx);
521
522 // Maximum size of packet including rtp headers.
523 // Extra space left in case packet will be resent using fec or rtx.
524 int packet_capacity = rtp_sender_->MaxRtpPacketSize() -
525 (use_fec ? FecPacketOverhead() : 0) -
526 (rtp_sender_->RtxStatus() ? kRtxHeaderSize : 0);
527
528 std::unique_ptr<RtpPacketToSend> single_packet =
529 rtp_sender_->AllocatePacket();
530 RTC_DCHECK_LE(packet_capacity, single_packet->capacity());
531 single_packet->SetPayloadType(payload_type);
532 single_packet->SetTimestamp(rtp_timestamp);
533 single_packet->set_capture_time(Timestamp::Millis(capture_time_ms));
534
535 // Construct the absolute capture time extension if not provided.
536 if (!video_header.absolute_capture_time.has_value()) {
537 video_header.absolute_capture_time.emplace();
538 video_header.absolute_capture_time->absolute_capture_timestamp =
539 Int64MsToUQ32x32(
540 clock_->ConvertTimestampToNtpTimeInMilliseconds(capture_time_ms));
541 if (include_capture_clock_offset_) {
542 video_header.absolute_capture_time->estimated_capture_clock_offset = 0;
543 }
544 }
545
546 // Let `absolute_capture_time_sender_` decide if the extension should be sent.
547 video_header.absolute_capture_time =
548 absolute_capture_time_sender_.OnSendPacket(
549 AbsoluteCaptureTimeSender::GetSource(single_packet->Ssrc(),
550 single_packet->Csrcs()),
551 single_packet->Timestamp(), kVideoPayloadTypeFrequency,
552 video_header.absolute_capture_time->absolute_capture_timestamp,
553 video_header.absolute_capture_time->estimated_capture_clock_offset);
554
555 auto first_packet = std::make_unique<RtpPacketToSend>(*single_packet);
556 auto middle_packet = std::make_unique<RtpPacketToSend>(*single_packet);
557 auto last_packet = std::make_unique<RtpPacketToSend>(*single_packet);
558 // Simplest way to estimate how much extensions would occupy is to set them.
559 AddRtpHeaderExtensions(video_header,
560 /*first_packet=*/true, /*last_packet=*/true,
561 single_packet.get());
562 if (video_structure_ != nullptr &&
563 single_packet->IsRegistered<RtpDependencyDescriptorExtension>() &&
564 !single_packet->HasExtension<RtpDependencyDescriptorExtension>()) {
565 RTC_DCHECK_EQ(video_header.frame_type, VideoFrameType::kVideoFrameKey);
566 // Disable attaching dependency descriptor to delta packets (including
567 // non-first packet of a key frame) when it wasn't attached to a key frame,
568 // as dependency descriptor can't be usable in such case.
569 RTC_LOG(LS_WARNING) << "Disable dependency descriptor because failed to "
570 "attach it to a key frame.";
571 video_structure_ = nullptr;
572 }
573
574 AddRtpHeaderExtensions(video_header,
575 /*first_packet=*/true, /*last_packet=*/false,
576 first_packet.get());
577 AddRtpHeaderExtensions(video_header,
578 /*first_packet=*/false, /*last_packet=*/false,
579 middle_packet.get());
580 AddRtpHeaderExtensions(video_header,
581 /*first_packet=*/false, /*last_packet=*/true,
582 last_packet.get());
583
584 RTC_DCHECK_GT(packet_capacity, single_packet->headers_size());
585 RTC_DCHECK_GT(packet_capacity, first_packet->headers_size());
586 RTC_DCHECK_GT(packet_capacity, middle_packet->headers_size());
587 RTC_DCHECK_GT(packet_capacity, last_packet->headers_size());
588 RtpPacketizer::PayloadSizeLimits limits;
589 limits.max_payload_len = packet_capacity - middle_packet->headers_size();
590
591 RTC_DCHECK_GE(single_packet->headers_size(), middle_packet->headers_size());
592 limits.single_packet_reduction_len =
593 single_packet->headers_size() - middle_packet->headers_size();
594
595 RTC_DCHECK_GE(first_packet->headers_size(), middle_packet->headers_size());
596 limits.first_packet_reduction_len =
597 first_packet->headers_size() - middle_packet->headers_size();
598
599 RTC_DCHECK_GE(last_packet->headers_size(), middle_packet->headers_size());
600 limits.last_packet_reduction_len =
601 last_packet->headers_size() - middle_packet->headers_size();
602
603 bool has_generic_descriptor =
604 first_packet->HasExtension<RtpGenericFrameDescriptorExtension00>() ||
605 first_packet->HasExtension<RtpDependencyDescriptorExtension>();
606
607 // Minimization of the vp8 descriptor may erase temporal_id, so use
608 // `temporal_id` rather than reference `video_header` beyond this point.
609 if (has_generic_descriptor) {
610 MinimizeDescriptor(&video_header);
611 }
612
613 // TODO(benwright@webrtc.org) - Allocate enough to always encrypt inline.
614 rtc::Buffer encrypted_video_payload;
615 if (frame_encryptor_ != nullptr) {
616 const size_t max_ciphertext_size =
617 frame_encryptor_->GetMaxCiphertextByteSize(cricket::MEDIA_TYPE_VIDEO,
618 payload.size());
619 encrypted_video_payload.SetSize(max_ciphertext_size);
620
621 size_t bytes_written = 0;
622
623 // Enable header authentication if the field trial isn't disabled.
624 std::vector<uint8_t> additional_data;
625 if (generic_descriptor_auth_experiment_) {
626 additional_data = RtpDescriptorAuthentication(video_header);
627 }
628
629 if (frame_encryptor_->Encrypt(
630 cricket::MEDIA_TYPE_VIDEO, first_packet->Ssrc(), additional_data,
631 payload, encrypted_video_payload, &bytes_written) != 0) {
632 return false;
633 }
634
635 encrypted_video_payload.SetSize(bytes_written);
636 payload = encrypted_video_payload;
637 } else if (require_frame_encryption_) {
638 RTC_LOG(LS_WARNING)
639 << "No FrameEncryptor is attached to this video sending stream but "
640 "one is required since require_frame_encryptor is set";
641 }
642
643 std::unique_ptr<RtpPacketizer> packetizer =
644 RtpPacketizer::Create(codec_type, payload, limits, video_header);
645
646 // TODO(bugs.webrtc.org/10714): retransmission_settings_ should generally be
647 // replaced by expected_retransmission_time_ms.has_value(). For now, though,
648 // only VP8 with an injected frame buffer controller actually controls it.
649 const bool allow_retransmission =
650 expected_retransmission_time_ms.has_value()
651 ? AllowRetransmission(temporal_id, retransmission_settings,
652 expected_retransmission_time_ms.value())
653 : false;
654 const size_t num_packets = packetizer->NumPackets();
655
656 if (num_packets == 0)
657 return false;
658
659 bool first_frame = first_frame_sent_();
660 std::vector<std::unique_ptr<RtpPacketToSend>> rtp_packets;
661 for (size_t i = 0; i < num_packets; ++i) {
662 std::unique_ptr<RtpPacketToSend> packet;
663 int expected_payload_capacity;
664 // Choose right packet template:
665 if (num_packets == 1) {
666 packet = std::move(single_packet);
667 expected_payload_capacity =
668 limits.max_payload_len - limits.single_packet_reduction_len;
669 } else if (i == 0) {
670 packet = std::move(first_packet);
671 expected_payload_capacity =
672 limits.max_payload_len - limits.first_packet_reduction_len;
673 } else if (i == num_packets - 1) {
674 packet = std::move(last_packet);
675 expected_payload_capacity =
676 limits.max_payload_len - limits.last_packet_reduction_len;
677 } else {
678 packet = std::make_unique<RtpPacketToSend>(*middle_packet);
679 expected_payload_capacity = limits.max_payload_len;
680 }
681
682 packet->set_first_packet_of_frame(i == 0);
683
684 if (!packetizer->NextPacket(packet.get()))
685 return false;
686 RTC_DCHECK_LE(packet->payload_size(), expected_payload_capacity);
687
688 packet->set_allow_retransmission(allow_retransmission);
689 packet->set_is_key_frame(video_header.frame_type ==
690 VideoFrameType::kVideoFrameKey);
691
692 // Put packetization finish timestamp into extension.
693 if (packet->HasExtension<VideoTimingExtension>()) {
694 packet->set_packetization_finish_time(clock_->CurrentTime());
695 }
696
697 packet->set_fec_protect_packet(use_fec);
698
699 if (red_enabled()) {
700 // TODO(sprang): Consider packetizing directly into packets with the RED
701 // header already in place, to avoid this copy.
702 std::unique_ptr<RtpPacketToSend> red_packet(new RtpPacketToSend(*packet));
703 BuildRedPayload(*packet, red_packet.get());
704 red_packet->SetPayloadType(*red_payload_type_);
705 red_packet->set_is_red(true);
706
707 // Append `red_packet` instead of `packet` to output.
708 red_packet->set_packet_type(RtpPacketMediaType::kVideo);
709 red_packet->set_allow_retransmission(packet->allow_retransmission());
710 rtp_packets.emplace_back(std::move(red_packet));
711 } else {
712 packet->set_packet_type(RtpPacketMediaType::kVideo);
713 rtp_packets.emplace_back(std::move(packet));
714 }
715
716 if (first_frame) {
717 if (i == 0) {
718 RTC_LOG(LS_INFO)
719 << "Sent first RTP packet of the first video frame (pre-pacer)";
720 }
721 if (i == num_packets - 1) {
722 RTC_LOG(LS_INFO)
723 << "Sent last RTP packet of the first video frame (pre-pacer)";
724 }
725 }
726 }
727
728 LogAndSendToNetwork(std::move(rtp_packets), payload.size());
729
730 // Update details about the last sent frame.
731 last_rotation_ = video_header.rotation;
732
733 if (video_header.color_space != last_color_space_) {
734 last_color_space_ = video_header.color_space;
735 transmit_color_space_next_frame_ = !IsBaseLayer(video_header);
736 } else {
737 transmit_color_space_next_frame_ =
738 transmit_color_space_next_frame_ ? !IsBaseLayer(video_header) : false;
739 }
740
741 if (video_header.frame_type == VideoFrameType::kVideoFrameKey ||
742 PacketWillLikelyBeRequestedForRestransmitionIfLost(video_header)) {
743 // This frame will likely be delivered, no need to populate playout
744 // delay extensions until it changes again.
745 playout_delay_pending_ = false;
746 if (send_allocation_ == SendVideoLayersAllocation::kSendWithResolution) {
747 last_full_sent_allocation_ = allocation_;
748 }
749 send_allocation_ = SendVideoLayersAllocation::kDontSend;
750 }
751
752 TRACE_EVENT_ASYNC_END1("webrtc", "Video", capture_time_ms, "timestamp",
753 rtp_timestamp);
754 return true;
755 }
756
SendEncodedImage(int payload_type,absl::optional<VideoCodecType> codec_type,uint32_t rtp_timestamp,const EncodedImage & encoded_image,RTPVideoHeader video_header,absl::optional<int64_t> expected_retransmission_time_ms)757 bool RTPSenderVideo::SendEncodedImage(
758 int payload_type,
759 absl::optional<VideoCodecType> codec_type,
760 uint32_t rtp_timestamp,
761 const EncodedImage& encoded_image,
762 RTPVideoHeader video_header,
763 absl::optional<int64_t> expected_retransmission_time_ms) {
764 if (frame_transformer_delegate_) {
765 // The frame will be sent async once transformed.
766 return frame_transformer_delegate_->TransformFrame(
767 payload_type, codec_type, rtp_timestamp, encoded_image, video_header,
768 expected_retransmission_time_ms);
769 }
770 return SendVideo(payload_type, codec_type, rtp_timestamp,
771 encoded_image.capture_time_ms_, encoded_image, video_header,
772 expected_retransmission_time_ms);
773 }
774
PacketizationOverheadBps() const775 uint32_t RTPSenderVideo::PacketizationOverheadBps() const {
776 MutexLock lock(&stats_mutex_);
777 return packetization_overhead_bitrate_.Rate(clock_->TimeInMilliseconds())
778 .value_or(0);
779 }
780
AllowRetransmission(uint8_t temporal_id,int32_t retransmission_settings,int64_t expected_retransmission_time_ms)781 bool RTPSenderVideo::AllowRetransmission(
782 uint8_t temporal_id,
783 int32_t retransmission_settings,
784 int64_t expected_retransmission_time_ms) {
785 if (retransmission_settings == kRetransmitOff)
786 return false;
787
788 MutexLock lock(&stats_mutex_);
789 // Media packet storage.
790 if ((retransmission_settings & kConditionallyRetransmitHigherLayers) &&
791 UpdateConditionalRetransmit(temporal_id,
792 expected_retransmission_time_ms)) {
793 retransmission_settings |= kRetransmitHigherLayers;
794 }
795
796 if (temporal_id == kNoTemporalIdx)
797 return true;
798
799 if ((retransmission_settings & kRetransmitBaseLayer) && temporal_id == 0)
800 return true;
801
802 if ((retransmission_settings & kRetransmitHigherLayers) && temporal_id > 0)
803 return true;
804
805 return false;
806 }
807
GetTemporalId(const RTPVideoHeader & header)808 uint8_t RTPSenderVideo::GetTemporalId(const RTPVideoHeader& header) {
809 struct TemporalIdGetter {
810 uint8_t operator()(const RTPVideoHeaderVP8& vp8) { return vp8.temporalIdx; }
811 uint8_t operator()(const RTPVideoHeaderVP9& vp9) {
812 return vp9.temporal_idx;
813 }
814 uint8_t operator()(const RTPVideoHeaderH264&) { return kNoTemporalIdx; }
815 uint8_t operator()(const RTPVideoHeaderLegacyGeneric&) {
816 return kNoTemporalIdx;
817 }
818 uint8_t operator()(const absl::monostate&) { return kNoTemporalIdx; }
819 };
820 return absl::visit(TemporalIdGetter(), header.video_type_header);
821 }
822
UpdateConditionalRetransmit(uint8_t temporal_id,int64_t expected_retransmission_time_ms)823 bool RTPSenderVideo::UpdateConditionalRetransmit(
824 uint8_t temporal_id,
825 int64_t expected_retransmission_time_ms) {
826 int64_t now_ms = clock_->TimeInMilliseconds();
827 // Update stats for any temporal layer.
828 TemporalLayerStats* current_layer_stats =
829 &frame_stats_by_temporal_layer_[temporal_id];
830 current_layer_stats->frame_rate_fp1000s.Update(1, now_ms);
831 int64_t tl_frame_interval = now_ms - current_layer_stats->last_frame_time_ms;
832 current_layer_stats->last_frame_time_ms = now_ms;
833
834 // Conditional retransmit only applies to upper layers.
835 if (temporal_id != kNoTemporalIdx && temporal_id > 0) {
836 if (tl_frame_interval >= kMaxUnretransmittableFrameIntervalMs) {
837 // Too long since a retransmittable frame in this layer, enable NACK
838 // protection.
839 return true;
840 } else {
841 // Estimate when the next frame of any lower layer will be sent.
842 const int64_t kUndefined = std::numeric_limits<int64_t>::max();
843 int64_t expected_next_frame_time = kUndefined;
844 for (int i = temporal_id - 1; i >= 0; --i) {
845 TemporalLayerStats* stats = &frame_stats_by_temporal_layer_[i];
846 absl::optional<uint32_t> rate = stats->frame_rate_fp1000s.Rate(now_ms);
847 if (rate) {
848 int64_t tl_next = stats->last_frame_time_ms + 1000000 / *rate;
849 if (tl_next - now_ms > -expected_retransmission_time_ms &&
850 tl_next < expected_next_frame_time) {
851 expected_next_frame_time = tl_next;
852 }
853 }
854 }
855
856 if (expected_next_frame_time == kUndefined ||
857 expected_next_frame_time - now_ms > expected_retransmission_time_ms) {
858 // The next frame in a lower layer is expected at a later time (or
859 // unable to tell due to lack of data) than a retransmission is
860 // estimated to be able to arrive, so allow this packet to be nacked.
861 return true;
862 }
863 }
864 }
865
866 return false;
867 }
868
MaybeUpdateCurrentPlayoutDelay(const RTPVideoHeader & header)869 void RTPSenderVideo::MaybeUpdateCurrentPlayoutDelay(
870 const RTPVideoHeader& header) {
871 VideoPlayoutDelay requested_delay =
872 forced_playout_delay_.value_or(header.playout_delay);
873
874 if (IsNoopDelay(requested_delay)) {
875 return;
876 }
877
878 if (requested_delay.min_ms > PlayoutDelayLimits::kMaxMs ||
879 requested_delay.max_ms > PlayoutDelayLimits::kMaxMs) {
880 RTC_DLOG(LS_ERROR)
881 << "Requested playout delay values out of range, ignored";
882 return;
883 }
884 if (requested_delay.max_ms != -1 &&
885 requested_delay.min_ms > requested_delay.max_ms) {
886 RTC_DLOG(LS_ERROR) << "Requested playout delay values out of order";
887 return;
888 }
889
890 if (!playout_delay_pending_) {
891 current_playout_delay_ = requested_delay;
892 playout_delay_pending_ = true;
893 return;
894 }
895
896 if ((requested_delay.min_ms == -1 ||
897 requested_delay.min_ms == current_playout_delay_.min_ms) &&
898 (requested_delay.max_ms == -1 ||
899 requested_delay.max_ms == current_playout_delay_.max_ms)) {
900 // No change, ignore.
901 return;
902 }
903
904 if (requested_delay.min_ms == -1) {
905 RTC_DCHECK_GE(requested_delay.max_ms, 0);
906 requested_delay.min_ms =
907 std::min(current_playout_delay_.min_ms, requested_delay.max_ms);
908 }
909 if (requested_delay.max_ms == -1) {
910 requested_delay.max_ms =
911 std::max(current_playout_delay_.max_ms, requested_delay.min_ms);
912 }
913
914 current_playout_delay_ = requested_delay;
915 playout_delay_pending_ = true;
916 }
917
918 } // namespace webrtc
919