1 /*
2 * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "video/video_stream_buffer_controller.h"
12
13 #include <algorithm>
14 #include <memory>
15 #include <utility>
16
17 #include "absl/base/attributes.h"
18 #include "absl/functional/bind_front.h"
19 #include "api/sequence_checker.h"
20 #include "api/task_queue/task_queue_base.h"
21 #include "api/units/data_size.h"
22 #include "api/video/encoded_frame.h"
23 #include "api/video/frame_buffer.h"
24 #include "api/video/video_content_type.h"
25 #include "modules/video_coding/frame_helpers.h"
26 #include "modules/video_coding/timing/inter_frame_delay.h"
27 #include "modules/video_coding/timing/jitter_estimator.h"
28 #include "rtc_base/checks.h"
29 #include "rtc_base/logging.h"
30 #include "rtc_base/thread_annotations.h"
31 #include "video/frame_decode_scheduler.h"
32 #include "video/frame_decode_timing.h"
33 #include "video/task_queue_frame_decode_scheduler.h"
34 #include "video/video_receive_stream_timeout_tracker.h"
35
36 namespace webrtc {
37
38 namespace {
39
40 // Max number of frames the buffer will hold.
41 static constexpr size_t kMaxFramesBuffered = 800;
42 // Max number of decoded frame info that will be saved.
43 static constexpr int kMaxFramesHistory = 1 << 13;
44
45 // Default value for the maximum decode queue size that is used when the
46 // low-latency renderer is used.
47 static constexpr size_t kZeroPlayoutDelayDefaultMaxDecodeQueueSize = 8;
48
49 struct FrameMetadata {
FrameMetadatawebrtc::__anon1a5f74440111::FrameMetadata50 explicit FrameMetadata(const EncodedFrame& frame)
51 : is_last_spatial_layer(frame.is_last_spatial_layer),
52 is_keyframe(frame.is_keyframe()),
53 size(frame.size()),
54 contentType(frame.contentType()),
55 delayed_by_retransmission(frame.delayed_by_retransmission()),
56 rtp_timestamp(frame.Timestamp()),
57 receive_time(frame.ReceivedTimestamp()) {}
58
59 const bool is_last_spatial_layer;
60 const bool is_keyframe;
61 const size_t size;
62 const VideoContentType contentType;
63 const bool delayed_by_retransmission;
64 const uint32_t rtp_timestamp;
65 const absl::optional<Timestamp> receive_time;
66 };
67
ReceiveTime(const EncodedFrame & frame)68 Timestamp ReceiveTime(const EncodedFrame& frame) {
69 absl::optional<Timestamp> ts = frame.ReceivedTimestamp();
70 RTC_DCHECK(ts.has_value()) << "Received frame must have a timestamp set!";
71 return *ts;
72 }
73
74 } // namespace
75
VideoStreamBufferController(Clock * clock,TaskQueueBase * worker_queue,VCMTiming * timing,VCMReceiveStatisticsCallback * stats_proxy,FrameSchedulingReceiver * receiver,TimeDelta max_wait_for_keyframe,TimeDelta max_wait_for_frame,std::unique_ptr<FrameDecodeScheduler> frame_decode_scheduler,const FieldTrialsView & field_trials)76 VideoStreamBufferController::VideoStreamBufferController(
77 Clock* clock,
78 TaskQueueBase* worker_queue,
79 VCMTiming* timing,
80 VCMReceiveStatisticsCallback* stats_proxy,
81 FrameSchedulingReceiver* receiver,
82 TimeDelta max_wait_for_keyframe,
83 TimeDelta max_wait_for_frame,
84 std::unique_ptr<FrameDecodeScheduler> frame_decode_scheduler,
85 const FieldTrialsView& field_trials)
86 : field_trials_(field_trials),
87 clock_(clock),
88 stats_proxy_(stats_proxy),
89 receiver_(receiver),
90 timing_(timing),
91 frame_decode_scheduler_(std::move(frame_decode_scheduler)),
92 jitter_estimator_(clock_, field_trials),
93 buffer_(std::make_unique<FrameBuffer>(kMaxFramesBuffered,
94 kMaxFramesHistory,
95 field_trials)),
96 decode_timing_(clock_, timing_),
97 timeout_tracker_(
98 clock_,
99 worker_queue,
100 VideoReceiveStreamTimeoutTracker::Timeouts{
101 .max_wait_for_keyframe = max_wait_for_keyframe,
102 .max_wait_for_frame = max_wait_for_frame},
103 absl::bind_front(&VideoStreamBufferController::OnTimeout, this)),
104 zero_playout_delay_max_decode_queue_size_(
105 "max_decode_queue_size",
106 kZeroPlayoutDelayDefaultMaxDecodeQueueSize) {
107 RTC_DCHECK(stats_proxy_);
108 RTC_DCHECK(receiver_);
109 RTC_DCHECK(timing_);
110 RTC_DCHECK(clock_);
111 RTC_DCHECK(frame_decode_scheduler_);
112
113 ParseFieldTrial({&zero_playout_delay_max_decode_queue_size_},
114 field_trials.Lookup("WebRTC-ZeroPlayoutDelay"));
115 }
116
Stop()117 void VideoStreamBufferController::Stop() {
118 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
119 frame_decode_scheduler_->Stop();
120 timeout_tracker_.Stop();
121 decoder_ready_for_new_frame_ = false;
122 }
123
SetProtectionMode(VCMVideoProtection protection_mode)124 void VideoStreamBufferController::SetProtectionMode(
125 VCMVideoProtection protection_mode) {
126 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
127 protection_mode_ = protection_mode;
128 }
129
Clear()130 void VideoStreamBufferController::Clear() {
131 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
132 stats_proxy_->OnDroppedFrames(buffer_->CurrentSize());
133 buffer_ = std::make_unique<FrameBuffer>(kMaxFramesBuffered, kMaxFramesHistory,
134 field_trials_);
135 frame_decode_scheduler_->CancelOutstanding();
136 }
137
InsertFrame(std::unique_ptr<EncodedFrame> frame)138 absl::optional<int64_t> VideoStreamBufferController::InsertFrame(
139 std::unique_ptr<EncodedFrame> frame) {
140 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
141 FrameMetadata metadata(*frame);
142 int complete_units = buffer_->GetTotalNumberOfContinuousTemporalUnits();
143 if (buffer_->InsertFrame(std::move(frame))) {
144 RTC_DCHECK(metadata.receive_time) << "Frame receive time must be set!";
145 if (!metadata.delayed_by_retransmission && metadata.receive_time &&
146 (field_trials_.IsDisabled("WebRTC-IncomingTimestampOnMarkerBitOnly") ||
147 metadata.is_last_spatial_layer)) {
148 timing_->IncomingTimestamp(metadata.rtp_timestamp,
149 *metadata.receive_time);
150 }
151 if (complete_units < buffer_->GetTotalNumberOfContinuousTemporalUnits()) {
152 stats_proxy_->OnCompleteFrame(metadata.is_keyframe, metadata.size,
153 metadata.contentType);
154 MaybeScheduleFrameForRelease();
155 }
156 }
157
158 return buffer_->LastContinuousFrameId();
159 }
160
UpdateRtt(int64_t max_rtt_ms)161 void VideoStreamBufferController::UpdateRtt(int64_t max_rtt_ms) {
162 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
163 jitter_estimator_.UpdateRtt(TimeDelta::Millis(max_rtt_ms));
164 }
165
SetMaxWaits(TimeDelta max_wait_for_keyframe,TimeDelta max_wait_for_frame)166 void VideoStreamBufferController::SetMaxWaits(TimeDelta max_wait_for_keyframe,
167 TimeDelta max_wait_for_frame) {
168 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
169 timeout_tracker_.SetTimeouts({.max_wait_for_keyframe = max_wait_for_keyframe,
170 .max_wait_for_frame = max_wait_for_frame});
171 }
172
StartNextDecode(bool keyframe_required)173 void VideoStreamBufferController::StartNextDecode(bool keyframe_required) {
174 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
175 if (!timeout_tracker_.Running())
176 timeout_tracker_.Start(keyframe_required);
177 keyframe_required_ = keyframe_required;
178 if (keyframe_required_) {
179 timeout_tracker_.SetWaitingForKeyframe();
180 }
181 decoder_ready_for_new_frame_ = true;
182 MaybeScheduleFrameForRelease();
183 }
184
Size()185 int VideoStreamBufferController::Size() {
186 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
187 return buffer_->CurrentSize();
188 }
189
OnFrameReady(absl::InlinedVector<std::unique_ptr<EncodedFrame>,4> frames,Timestamp render_time)190 void VideoStreamBufferController::OnFrameReady(
191 absl::InlinedVector<std::unique_ptr<EncodedFrame>, 4> frames,
192 Timestamp render_time) {
193 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
194 RTC_CHECK(!frames.empty())
195 << "Callers must ensure there is at least one frame to decode.";
196
197 timeout_tracker_.OnEncodedFrameReleased();
198
199 Timestamp now = clock_->CurrentTime();
200 bool superframe_delayed_by_retransmission = false;
201 DataSize superframe_size = DataSize::Zero();
202 const EncodedFrame& first_frame = *frames.front();
203 Timestamp receive_time = ReceiveTime(first_frame);
204
205 if (first_frame.is_keyframe())
206 keyframe_required_ = false;
207
208 // Gracefully handle bad RTP timestamps and render time issues.
209 if (FrameHasBadRenderTiming(render_time, now) ||
210 TargetVideoDelayIsTooLarge(timing_->TargetVideoDelay())) {
211 RTC_LOG(LS_WARNING) << "Resetting jitter estimator and timing module due "
212 "to bad render timing for rtp_timestamp="
213 << first_frame.Timestamp();
214 jitter_estimator_.Reset();
215 timing_->Reset();
216 render_time = timing_->RenderTime(first_frame.Timestamp(), now);
217 }
218
219 for (std::unique_ptr<EncodedFrame>& frame : frames) {
220 frame->SetRenderTime(render_time.ms());
221
222 superframe_delayed_by_retransmission |= frame->delayed_by_retransmission();
223 receive_time = std::max(receive_time, ReceiveTime(*frame));
224 superframe_size += DataSize::Bytes(frame->size());
225 }
226
227 if (!superframe_delayed_by_retransmission) {
228 auto frame_delay = inter_frame_delay_.CalculateDelay(
229 first_frame.Timestamp(), receive_time);
230 if (frame_delay) {
231 jitter_estimator_.UpdateEstimate(*frame_delay, superframe_size);
232 }
233
234 float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0;
235 absl::optional<TimeDelta> rtt_mult_add_cap_ms = absl::nullopt;
236 if (rtt_mult_settings_.has_value()) {
237 rtt_mult = rtt_mult_settings_->rtt_mult_setting;
238 rtt_mult_add_cap_ms =
239 TimeDelta::Millis(rtt_mult_settings_->rtt_mult_add_cap_ms);
240 }
241 timing_->SetJitterDelay(
242 jitter_estimator_.GetJitterEstimate(rtt_mult, rtt_mult_add_cap_ms));
243 timing_->UpdateCurrentDelay(render_time, now);
244 } else if (RttMultExperiment::RttMultEnabled()) {
245 jitter_estimator_.FrameNacked();
246 }
247
248 // Update stats.
249 UpdateDroppedFrames();
250 UpdateJitterDelay();
251 UpdateTimingFrameInfo();
252
253 std::unique_ptr<EncodedFrame> frame =
254 CombineAndDeleteFrames(std::move(frames));
255
256 timing_->SetLastDecodeScheduledTimestamp(now);
257
258 decoder_ready_for_new_frame_ = false;
259 receiver_->OnEncodedFrame(std::move(frame));
260 }
261
OnTimeout(TimeDelta delay)262 void VideoStreamBufferController::OnTimeout(TimeDelta delay) {
263 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
264
265 // Stop sending timeouts until receiver starts waiting for a new frame.
266 timeout_tracker_.Stop();
267
268 // If the stream is paused then ignore the timeout.
269 if (!decoder_ready_for_new_frame_) {
270 return;
271 }
272 decoder_ready_for_new_frame_ = false;
273 receiver_->OnDecodableFrameTimeout(delay);
274 }
275
FrameReadyForDecode(uint32_t rtp_timestamp,Timestamp render_time)276 void VideoStreamBufferController::FrameReadyForDecode(uint32_t rtp_timestamp,
277 Timestamp render_time) {
278 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
279 // Check that the frame to decode is still valid before passing the frame for
280 // decoding.
281 auto decodable_tu_info = buffer_->DecodableTemporalUnitsInfo();
282 if (!decodable_tu_info) {
283 RTC_LOG(LS_ERROR)
284 << "The frame buffer became undecodable during the wait "
285 "to decode frame with rtp-timestamp "
286 << rtp_timestamp
287 << ". Cancelling the decode of this frame, decoding "
288 "will resume when the frame buffers become decodable again.";
289 return;
290 }
291 RTC_DCHECK_EQ(rtp_timestamp, decodable_tu_info->next_rtp_timestamp)
292 << "Frame buffer's next decodable frame was not the one sent for "
293 "extraction.";
294 auto frames = buffer_->ExtractNextDecodableTemporalUnit();
295 if (frames.empty()) {
296 RTC_LOG(LS_ERROR)
297 << "The frame buffer should never return an empty temporal until list "
298 "when there is a decodable temporal unit.";
299 RTC_DCHECK_NOTREACHED();
300 return;
301 }
302 OnFrameReady(std::move(frames), render_time);
303 }
304
UpdateDroppedFrames()305 void VideoStreamBufferController::UpdateDroppedFrames()
306 RTC_RUN_ON(&worker_sequence_checker_) {
307 const int dropped_frames = buffer_->GetTotalNumberOfDroppedFrames() -
308 frames_dropped_before_last_new_frame_;
309 if (dropped_frames > 0)
310 stats_proxy_->OnDroppedFrames(dropped_frames);
311 frames_dropped_before_last_new_frame_ =
312 buffer_->GetTotalNumberOfDroppedFrames();
313 }
314
UpdateJitterDelay()315 void VideoStreamBufferController::UpdateJitterDelay() {
316 auto timings = timing_->GetTimings();
317 if (timings.num_decoded_frames) {
318 stats_proxy_->OnFrameBufferTimingsUpdated(
319 timings.max_decode_duration.ms(), timings.current_delay.ms(),
320 timings.target_delay.ms(), timings.jitter_buffer_delay.ms(),
321 timings.min_playout_delay.ms(), timings.render_delay.ms());
322 }
323 }
324
UpdateTimingFrameInfo()325 void VideoStreamBufferController::UpdateTimingFrameInfo() {
326 absl::optional<TimingFrameInfo> info = timing_->GetTimingFrameInfo();
327 if (info)
328 stats_proxy_->OnTimingFrameInfoUpdated(*info);
329 }
330
IsTooManyFramesQueued() const331 bool VideoStreamBufferController::IsTooManyFramesQueued() const
332 RTC_RUN_ON(&worker_sequence_checker_) {
333 return buffer_->CurrentSize() > zero_playout_delay_max_decode_queue_size_;
334 }
335
ForceKeyFrameReleaseImmediately()336 void VideoStreamBufferController::ForceKeyFrameReleaseImmediately()
337 RTC_RUN_ON(&worker_sequence_checker_) {
338 RTC_DCHECK(keyframe_required_);
339 // Iterate through the frame buffer until there is a complete keyframe and
340 // release this right away.
341 while (buffer_->DecodableTemporalUnitsInfo()) {
342 auto next_frame = buffer_->ExtractNextDecodableTemporalUnit();
343 if (next_frame.empty()) {
344 RTC_DCHECK_NOTREACHED()
345 << "Frame buffer should always return at least 1 frame.";
346 continue;
347 }
348 // Found keyframe - decode right away.
349 if (next_frame.front()->is_keyframe()) {
350 auto render_time = timing_->RenderTime(next_frame.front()->Timestamp(),
351 clock_->CurrentTime());
352 OnFrameReady(std::move(next_frame), render_time);
353 return;
354 }
355 }
356 }
357
MaybeScheduleFrameForRelease()358 void VideoStreamBufferController::MaybeScheduleFrameForRelease()
359 RTC_RUN_ON(&worker_sequence_checker_) {
360 auto decodable_tu_info = buffer_->DecodableTemporalUnitsInfo();
361 if (!decoder_ready_for_new_frame_ || !decodable_tu_info) {
362 return;
363 }
364
365 if (keyframe_required_) {
366 return ForceKeyFrameReleaseImmediately();
367 }
368
369 // If already scheduled then abort.
370 if (frame_decode_scheduler_->ScheduledRtpTimestamp() ==
371 decodable_tu_info->next_rtp_timestamp) {
372 return;
373 }
374
375 TimeDelta max_wait = timeout_tracker_.TimeUntilTimeout();
376 // Ensures the frame is scheduled for decode before the stream times out.
377 // This is otherwise a race condition.
378 max_wait = std::max(max_wait - TimeDelta::Millis(1), TimeDelta::Zero());
379 absl::optional<FrameDecodeTiming::FrameSchedule> schedule;
380 while (decodable_tu_info) {
381 schedule = decode_timing_.OnFrameBufferUpdated(
382 decodable_tu_info->next_rtp_timestamp,
383 decodable_tu_info->last_rtp_timestamp, max_wait,
384 IsTooManyFramesQueued());
385 if (schedule) {
386 // Don't schedule if already waiting for the same frame.
387 if (frame_decode_scheduler_->ScheduledRtpTimestamp() !=
388 decodable_tu_info->next_rtp_timestamp) {
389 frame_decode_scheduler_->CancelOutstanding();
390 frame_decode_scheduler_->ScheduleFrame(
391 decodable_tu_info->next_rtp_timestamp, *schedule,
392 absl::bind_front(&VideoStreamBufferController::FrameReadyForDecode,
393 this));
394 }
395 return;
396 }
397 // If no schedule for current rtp, drop and try again.
398 buffer_->DropNextDecodableTemporalUnit();
399 decodable_tu_info = buffer_->DecodableTemporalUnitsInfo();
400 }
401 }
402
403 } // namespace webrtc
404