1 /*
2 * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "api/video/frame_buffer.h"
12
13 #include <algorithm>
14
15 #include "absl/algorithm/container.h"
16 #include "absl/container/inlined_vector.h"
17 #include "rtc_base/logging.h"
18 #include "rtc_base/numerics/sequence_number_util.h"
19
20 namespace webrtc {
21 namespace {
ValidReferences(const EncodedFrame & frame)22 bool ValidReferences(const EncodedFrame& frame) {
23 // All references must point backwards, and duplicates are not allowed.
24 for (size_t i = 0; i < frame.num_references; ++i) {
25 if (frame.references[i] >= frame.Id())
26 return false;
27
28 for (size_t j = i + 1; j < frame.num_references; ++j) {
29 if (frame.references[i] == frame.references[j])
30 return false;
31 }
32 }
33
34 return true;
35 }
36
37 // Since FrameBuffer::FrameInfo is private it can't be used in the function
38 // signature, hence the FrameIteratorT type.
39 template <typename FrameIteratorT>
GetReferences(const FrameIteratorT & it)40 rtc::ArrayView<const int64_t> GetReferences(const FrameIteratorT& it) {
41 return {it->second.encoded_frame->references,
42 std::min<size_t>(it->second.encoded_frame->num_references,
43 EncodedFrame::kMaxFrameReferences)};
44 }
45
46 template <typename FrameIteratorT>
GetFrameId(const FrameIteratorT & it)47 int64_t GetFrameId(const FrameIteratorT& it) {
48 return it->first;
49 }
50
51 template <typename FrameIteratorT>
GetTimestamp(const FrameIteratorT & it)52 uint32_t GetTimestamp(const FrameIteratorT& it) {
53 return it->second.encoded_frame->Timestamp();
54 }
55
56 template <typename FrameIteratorT>
IsLastFrameInTemporalUnit(const FrameIteratorT & it)57 bool IsLastFrameInTemporalUnit(const FrameIteratorT& it) {
58 return it->second.encoded_frame->is_last_spatial_layer;
59 }
60 } // namespace
61
FrameBuffer(int max_size,int max_decode_history,const FieldTrialsView & field_trials)62 FrameBuffer::FrameBuffer(int max_size,
63 int max_decode_history,
64 const FieldTrialsView& field_trials)
65 : legacy_frame_id_jump_behavior_(
66 !field_trials.IsDisabled("WebRTC-LegacyFrameIdJumpBehavior")),
67 max_size_(max_size),
68 decoded_frame_history_(max_decode_history) {}
69
InsertFrame(std::unique_ptr<EncodedFrame> frame)70 bool FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) {
71 if (!ValidReferences(*frame)) {
72 RTC_DLOG(LS_WARNING) << "Frame " << frame->Id()
73 << " has invalid references, dropping frame.";
74 return false;
75 }
76
77 if (frame->Id() <= decoded_frame_history_.GetLastDecodedFrameId()) {
78 if (legacy_frame_id_jump_behavior_ && frame->is_keyframe() &&
79 AheadOf(frame->Timestamp(),
80 *decoded_frame_history_.GetLastDecodedFrameTimestamp())) {
81 RTC_DLOG(LS_WARNING)
82 << "Keyframe " << frame->Id()
83 << " has newer timestamp but older picture id, clearing buffer.";
84 Clear();
85 } else {
86 // Already decoded past this frame.
87 return false;
88 }
89 }
90
91 if (frames_.size() == max_size_) {
92 if (frame->is_keyframe()) {
93 RTC_DLOG(LS_WARNING) << "Keyframe " << frame->Id()
94 << " inserted into full buffer, clearing buffer.";
95 Clear();
96 } else {
97 // No space for this frame.
98 return false;
99 }
100 }
101
102 const int64_t frame_id = frame->Id();
103 auto insert_res = frames_.emplace(frame_id, FrameInfo{std::move(frame)});
104 if (!insert_res.second) {
105 // Frame has already been inserted.
106 return false;
107 }
108
109 if (frames_.size() == max_size_) {
110 RTC_DLOG(LS_WARNING) << "Frame " << frame_id
111 << " inserted, buffer is now full.";
112 }
113
114 PropagateContinuity(insert_res.first);
115 FindNextAndLastDecodableTemporalUnit();
116 return true;
117 }
118
119 absl::InlinedVector<std::unique_ptr<EncodedFrame>, 4>
ExtractNextDecodableTemporalUnit()120 FrameBuffer::ExtractNextDecodableTemporalUnit() {
121 absl::InlinedVector<std::unique_ptr<EncodedFrame>, 4> res;
122 if (!next_decodable_temporal_unit_) {
123 return res;
124 }
125
126 auto end_it = std::next(next_decodable_temporal_unit_->last_frame);
127 for (auto it = next_decodable_temporal_unit_->first_frame; it != end_it;
128 ++it) {
129 decoded_frame_history_.InsertDecoded(GetFrameId(it), GetTimestamp(it));
130 res.push_back(std::move(it->second.encoded_frame));
131 }
132
133 DropNextDecodableTemporalUnit();
134 return res;
135 }
136
DropNextDecodableTemporalUnit()137 void FrameBuffer::DropNextDecodableTemporalUnit() {
138 if (!next_decodable_temporal_unit_) {
139 return;
140 }
141
142 auto end_it = std::next(next_decodable_temporal_unit_->last_frame);
143 num_dropped_frames_ += std::count_if(
144 frames_.begin(), end_it,
145 [](const auto& f) { return f.second.encoded_frame != nullptr; });
146
147 frames_.erase(frames_.begin(), end_it);
148 FindNextAndLastDecodableTemporalUnit();
149 }
150
LastContinuousFrameId() const151 absl::optional<int64_t> FrameBuffer::LastContinuousFrameId() const {
152 return last_continuous_frame_id_;
153 }
154
LastContinuousTemporalUnitFrameId() const155 absl::optional<int64_t> FrameBuffer::LastContinuousTemporalUnitFrameId() const {
156 return last_continuous_temporal_unit_frame_id_;
157 }
158
159 absl::optional<FrameBuffer::DecodabilityInfo>
DecodableTemporalUnitsInfo() const160 FrameBuffer::DecodableTemporalUnitsInfo() const {
161 return decodable_temporal_units_info_;
162 }
163
GetTotalNumberOfContinuousTemporalUnits() const164 int FrameBuffer::GetTotalNumberOfContinuousTemporalUnits() const {
165 return num_continuous_temporal_units_;
166 }
GetTotalNumberOfDroppedFrames() const167 int FrameBuffer::GetTotalNumberOfDroppedFrames() const {
168 return num_dropped_frames_;
169 }
170
CurrentSize() const171 size_t FrameBuffer::CurrentSize() const {
172 return frames_.size();
173 }
174
IsContinuous(const FrameIterator & it) const175 bool FrameBuffer::IsContinuous(const FrameIterator& it) const {
176 for (int64_t reference : GetReferences(it)) {
177 if (decoded_frame_history_.WasDecoded(reference)) {
178 continue;
179 }
180
181 auto reference_frame_it = frames_.find(reference);
182 if (reference_frame_it != frames_.end() &&
183 reference_frame_it->second.continuous) {
184 continue;
185 }
186
187 return false;
188 }
189
190 return true;
191 }
192
PropagateContinuity(const FrameIterator & frame_it)193 void FrameBuffer::PropagateContinuity(const FrameIterator& frame_it) {
194 for (auto it = frame_it; it != frames_.end(); ++it) {
195 if (!it->second.continuous) {
196 if (IsContinuous(it)) {
197 it->second.continuous = true;
198 if (last_continuous_frame_id_ < GetFrameId(it)) {
199 last_continuous_frame_id_ = GetFrameId(it);
200 }
201 if (IsLastFrameInTemporalUnit(it)) {
202 num_continuous_temporal_units_++;
203 if (last_continuous_temporal_unit_frame_id_ < GetFrameId(it)) {
204 last_continuous_temporal_unit_frame_id_ = GetFrameId(it);
205 }
206 }
207 }
208 }
209 }
210 }
211
FindNextAndLastDecodableTemporalUnit()212 void FrameBuffer::FindNextAndLastDecodableTemporalUnit() {
213 next_decodable_temporal_unit_.reset();
214 decodable_temporal_units_info_.reset();
215
216 if (!last_continuous_temporal_unit_frame_id_) {
217 return;
218 }
219
220 FrameIterator first_frame_it = frames_.begin();
221 FrameIterator last_frame_it = frames_.begin();
222 absl::InlinedVector<int64_t, 4> frames_in_temporal_unit;
223 uint32_t last_decodable_temporal_unit_timestamp;
224 for (auto frame_it = frames_.begin(); frame_it != frames_.end();) {
225 if (GetFrameId(frame_it) > *last_continuous_temporal_unit_frame_id_) {
226 break;
227 }
228
229 if (GetTimestamp(frame_it) != GetTimestamp(first_frame_it)) {
230 frames_in_temporal_unit.clear();
231 first_frame_it = frame_it;
232 }
233
234 frames_in_temporal_unit.push_back(GetFrameId(frame_it));
235
236 last_frame_it = frame_it++;
237
238 if (IsLastFrameInTemporalUnit(last_frame_it)) {
239 bool temporal_unit_decodable = true;
240 for (auto it = first_frame_it; it != frame_it && temporal_unit_decodable;
241 ++it) {
242 for (int64_t reference : GetReferences(it)) {
243 if (!decoded_frame_history_.WasDecoded(reference) &&
244 !absl::c_linear_search(frames_in_temporal_unit, reference)) {
245 // A frame in the temporal unit has a non-decoded reference outside
246 // the temporal unit, so it's not yet ready to be decoded.
247 temporal_unit_decodable = false;
248 break;
249 }
250 }
251 }
252
253 if (temporal_unit_decodable) {
254 if (!next_decodable_temporal_unit_) {
255 next_decodable_temporal_unit_ = {first_frame_it, last_frame_it};
256 }
257
258 last_decodable_temporal_unit_timestamp = GetTimestamp(first_frame_it);
259 }
260 }
261 }
262
263 if (next_decodable_temporal_unit_) {
264 decodable_temporal_units_info_ = {
265 .next_rtp_timestamp =
266 GetTimestamp(next_decodable_temporal_unit_->first_frame),
267 .last_rtp_timestamp = last_decodable_temporal_unit_timestamp};
268 }
269 }
270
Clear()271 void FrameBuffer::Clear() {
272 frames_.clear();
273 next_decodable_temporal_unit_.reset();
274 decodable_temporal_units_info_.reset();
275 last_continuous_frame_id_.reset();
276 last_continuous_temporal_unit_frame_id_.reset();
277 decoded_frame_history_.Clear();
278 }
279
280 } // namespace webrtc
281