1 /*
2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/rtp_vp8_ref_finder.h"
12
13 #include <utility>
14
15 #include "rtc_base/logging.h"
16
17 namespace webrtc {
18
ManageFrame(std::unique_ptr<RtpFrameObject> frame)19 RtpFrameReferenceFinder::ReturnVector RtpVp8RefFinder::ManageFrame(
20 std::unique_ptr<RtpFrameObject> frame) {
21 const RTPVideoHeaderVP8& codec_header = absl::get<RTPVideoHeaderVP8>(
22 frame->GetRtpVideoHeader().video_type_header);
23
24 if (codec_header.temporalIdx != kNoTemporalIdx)
25 frame->SetTemporalIndex(codec_header.temporalIdx);
26
27 int64_t unwrapped_tl0 = tl0_unwrapper_.Unwrap(codec_header.tl0PicIdx & 0xFF);
28 FrameDecision decision =
29 ManageFrameInternal(frame.get(), codec_header, unwrapped_tl0);
30
31 RtpFrameReferenceFinder::ReturnVector res;
32 switch (decision) {
33 case kStash:
34 if (stashed_frames_.size() > kMaxStashedFrames) {
35 stashed_frames_.pop_back();
36 }
37 stashed_frames_.push_front(
38 {.unwrapped_tl0 = unwrapped_tl0, .frame = std::move(frame)});
39 return res;
40 case kHandOff:
41 res.push_back(std::move(frame));
42 RetryStashedFrames(res);
43 return res;
44 case kDrop:
45 return res;
46 }
47
48 return res;
49 }
50
ManageFrameInternal(RtpFrameObject * frame,const RTPVideoHeaderVP8 & codec_header,int64_t unwrapped_tl0)51 RtpVp8RefFinder::FrameDecision RtpVp8RefFinder::ManageFrameInternal(
52 RtpFrameObject* frame,
53 const RTPVideoHeaderVP8& codec_header,
54 int64_t unwrapped_tl0) {
55 // Protect against corrupted packets with arbitrary large temporal idx.
56 if (codec_header.temporalIdx >= kMaxTemporalLayers)
57 return kDrop;
58
59 frame->SetSpatialIndex(0);
60 frame->SetId(codec_header.pictureId & 0x7FFF);
61
62 if (last_picture_id_ == -1)
63 last_picture_id_ = frame->Id();
64
65 // Clean up info about not yet received frames that are too old.
66 uint16_t old_picture_id =
67 Subtract<kFrameIdLength>(frame->Id(), kMaxNotYetReceivedFrames);
68 auto clean_frames_to = not_yet_received_frames_.lower_bound(old_picture_id);
69 not_yet_received_frames_.erase(not_yet_received_frames_.begin(),
70 clean_frames_to);
71 // Avoid re-adding picture ids that were just erased.
72 if (AheadOf<uint16_t, kFrameIdLength>(old_picture_id, last_picture_id_)) {
73 last_picture_id_ = old_picture_id;
74 }
75 // Find if there has been a gap in fully received frames and save the picture
76 // id of those frames in `not_yet_received_frames_`.
77 if (AheadOf<uint16_t, kFrameIdLength>(frame->Id(), last_picture_id_)) {
78 do {
79 last_picture_id_ = Add<kFrameIdLength>(last_picture_id_, 1);
80 not_yet_received_frames_.insert(last_picture_id_);
81 } while (last_picture_id_ != frame->Id());
82 }
83
84 // Clean up info for base layers that are too old.
85 int64_t old_tl0_pic_idx = unwrapped_tl0 - kMaxLayerInfo;
86 auto clean_layer_info_to = layer_info_.lower_bound(old_tl0_pic_idx);
87 layer_info_.erase(layer_info_.begin(), clean_layer_info_to);
88
89 if (frame->frame_type() == VideoFrameType::kVideoFrameKey) {
90 if (codec_header.temporalIdx != 0) {
91 return kDrop;
92 }
93 frame->num_references = 0;
94 layer_info_[unwrapped_tl0].fill(-1);
95 UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx);
96 return kHandOff;
97 }
98
99 auto layer_info_it = layer_info_.find(
100 codec_header.temporalIdx == 0 ? unwrapped_tl0 - 1 : unwrapped_tl0);
101
102 // If we don't have the base layer frame yet, stash this frame.
103 if (layer_info_it == layer_info_.end())
104 return kStash;
105
106 // A non keyframe base layer frame has been received, copy the layer info
107 // from the previous base layer frame and set a reference to the previous
108 // base layer frame.
109 if (codec_header.temporalIdx == 0) {
110 layer_info_it =
111 layer_info_.emplace(unwrapped_tl0, layer_info_it->second).first;
112 frame->num_references = 1;
113 int64_t last_pid_on_layer = layer_info_it->second[0];
114
115 // Is this an old frame that has already been used to update the state? If
116 // so, drop it.
117 if (AheadOrAt<uint16_t, kFrameIdLength>(last_pid_on_layer, frame->Id())) {
118 return kDrop;
119 }
120
121 frame->references[0] = last_pid_on_layer;
122 UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx);
123 return kHandOff;
124 }
125
126 // Layer sync frame, this frame only references its base layer frame.
127 if (codec_header.layerSync) {
128 frame->num_references = 1;
129 int64_t last_pid_on_layer = layer_info_it->second[codec_header.temporalIdx];
130
131 // Is this an old frame that has already been used to update the state? If
132 // so, drop it.
133 if (last_pid_on_layer != -1 &&
134 AheadOrAt<uint16_t, kFrameIdLength>(last_pid_on_layer, frame->Id())) {
135 return kDrop;
136 }
137
138 frame->references[0] = layer_info_it->second[0];
139 UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx);
140 return kHandOff;
141 }
142
143 // Find all references for this frame.
144 frame->num_references = 0;
145 for (uint8_t layer = 0; layer <= codec_header.temporalIdx; ++layer) {
146 // If we have not yet received a previous frame on this temporal layer,
147 // stash this frame.
148 if (layer_info_it->second[layer] == -1)
149 return kStash;
150
151 // If the last frame on this layer is ahead of this frame it means that
152 // a layer sync frame has been received after this frame for the same
153 // base layer frame, drop this frame.
154 if (AheadOf<uint16_t, kFrameIdLength>(layer_info_it->second[layer],
155 frame->Id())) {
156 return kDrop;
157 }
158
159 // If we have not yet received a frame between this frame and the referenced
160 // frame then we have to wait for that frame to be completed first.
161 auto not_received_frame_it =
162 not_yet_received_frames_.upper_bound(layer_info_it->second[layer]);
163 if (not_received_frame_it != not_yet_received_frames_.end() &&
164 AheadOf<uint16_t, kFrameIdLength>(frame->Id(),
165 *not_received_frame_it)) {
166 return kStash;
167 }
168
169 if (!(AheadOf<uint16_t, kFrameIdLength>(frame->Id(),
170 layer_info_it->second[layer]))) {
171 RTC_LOG(LS_WARNING) << "Frame with picture id " << frame->Id()
172 << " and packet range [" << frame->first_seq_num()
173 << ", " << frame->last_seq_num()
174 << "] already received, "
175 " dropping frame.";
176 return kDrop;
177 }
178
179 ++frame->num_references;
180 frame->references[layer] = layer_info_it->second[layer];
181 }
182
183 UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx);
184 return kHandOff;
185 }
186
UpdateLayerInfoVp8(RtpFrameObject * frame,int64_t unwrapped_tl0,uint8_t temporal_idx)187 void RtpVp8RefFinder::UpdateLayerInfoVp8(RtpFrameObject* frame,
188 int64_t unwrapped_tl0,
189 uint8_t temporal_idx) {
190 auto layer_info_it = layer_info_.find(unwrapped_tl0);
191
192 // Update this layer info and newer.
193 while (layer_info_it != layer_info_.end()) {
194 if (layer_info_it->second[temporal_idx] != -1 &&
195 AheadOf<uint16_t, kFrameIdLength>(layer_info_it->second[temporal_idx],
196 frame->Id())) {
197 // The frame was not newer, then no subsequent layer info have to be
198 // update.
199 break;
200 }
201
202 layer_info_it->second[temporal_idx] = frame->Id();
203 ++unwrapped_tl0;
204 layer_info_it = layer_info_.find(unwrapped_tl0);
205 }
206 not_yet_received_frames_.erase(frame->Id());
207
208 UnwrapPictureIds(frame);
209 }
210
RetryStashedFrames(RtpFrameReferenceFinder::ReturnVector & res)211 void RtpVp8RefFinder::RetryStashedFrames(
212 RtpFrameReferenceFinder::ReturnVector& res) {
213 bool complete_frame = false;
214 do {
215 complete_frame = false;
216 for (auto it = stashed_frames_.begin(); it != stashed_frames_.end();) {
217 const RTPVideoHeaderVP8& codec_header = absl::get<RTPVideoHeaderVP8>(
218 it->frame->GetRtpVideoHeader().video_type_header);
219 FrameDecision decision =
220 ManageFrameInternal(it->frame.get(), codec_header, it->unwrapped_tl0);
221
222 switch (decision) {
223 case kStash:
224 ++it;
225 break;
226 case kHandOff:
227 complete_frame = true;
228 res.push_back(std::move(it->frame));
229 [[fallthrough]];
230 case kDrop:
231 it = stashed_frames_.erase(it);
232 }
233 }
234 } while (complete_frame);
235 }
236
UnwrapPictureIds(RtpFrameObject * frame)237 void RtpVp8RefFinder::UnwrapPictureIds(RtpFrameObject* frame) {
238 for (size_t i = 0; i < frame->num_references; ++i)
239 frame->references[i] = unwrapper_.Unwrap(frame->references[i]);
240 frame->SetId(unwrapper_.Unwrap(frame->Id()));
241 }
242
ClearTo(uint16_t seq_num)243 void RtpVp8RefFinder::ClearTo(uint16_t seq_num) {
244 auto it = stashed_frames_.begin();
245 while (it != stashed_frames_.end()) {
246 if (AheadOf<uint16_t>(seq_num, it->frame->first_seq_num())) {
247 it = stashed_frames_.erase(it);
248 } else {
249 ++it;
250 }
251 }
252 }
253
254 } // namespace webrtc
255