1 /*
2 * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/codecs/vp8/default_temporal_layers.h"
12
13 #include <cstdint>
14 #include <memory>
15
16 #include "api/video/video_bitrate_allocation.h"
17 #include "api/video_codecs/video_codec.h"
18 #include "api/video_codecs/vp8_frame_config.h"
19 #include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h"
20 #include "modules/video_coding/include/video_codec_interface.h"
21 #include "modules/video_coding/utility/simulcast_rate_allocator.h"
22 #include "test/field_trial.h"
23 #include "test/gmock.h"
24 #include "test/gtest.h"
25 #include "vpx/vp8cx.h"
26
27 // TODO(bugs.webrtc.org/10582): Test the behavior of UpdateConfiguration().
28
29 namespace webrtc {
30 namespace test {
31 namespace {
32
33 using ::testing::Each;
34
35 enum {
36 kTemporalUpdateLast = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
37 VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF,
38 kTemporalUpdateGoldenWithoutDependency =
39 VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF |
40 VP8_EFLAG_NO_UPD_LAST,
41 kTemporalUpdateGolden =
42 VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
43 kTemporalUpdateAltrefWithoutDependency =
44 VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF |
45 VP8_EFLAG_NO_UPD_LAST,
46 kTemporalUpdateAltref = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_LAST,
47 kTemporalUpdateNone = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
48 VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
49 kTemporalUpdateNoneNoRefAltRef =
50 VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
51 VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
52 kTemporalUpdateNoneNoRefGolden =
53 VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
54 VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
55 kTemporalUpdateNoneNoRefGoldenAltRef =
56 VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_REF_ARF |
57 VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
58 kTemporalUpdateGoldenWithoutDependencyRefAltRef =
59 VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
60 kTemporalUpdateGoldenRefAltRef = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
61 kTemporalUpdateLastRefAltRef =
62 VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF,
63 kTemporalUpdateLastAndGoldenRefAltRef =
64 VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF,
65 };
66
67 using BufferFlags = Vp8FrameConfig::BufferFlags;
68 using Vp8BufferReference = Vp8FrameConfig::Vp8BufferReference;
69
70 constexpr uint8_t kNone = static_cast<uint8_t>(Vp8BufferReference::kNone);
71 constexpr uint8_t kLast = static_cast<uint8_t>(Vp8BufferReference::kLast);
72 constexpr uint8_t kGolden = static_cast<uint8_t>(Vp8BufferReference::kGolden);
73 constexpr uint8_t kAltref = static_cast<uint8_t>(Vp8BufferReference::kAltref);
74 constexpr uint8_t kAll = kLast | kGolden | kAltref;
75
ToVp8CodecFlags(uint8_t referenced_buffers,uint8_t updated_buffers,bool update_entropy)76 constexpr int ToVp8CodecFlags(uint8_t referenced_buffers,
77 uint8_t updated_buffers,
78 bool update_entropy) {
79 return (((referenced_buffers & kLast) == 0) ? VP8_EFLAG_NO_REF_LAST : 0) |
80 (((referenced_buffers & kGolden) == 0) ? VP8_EFLAG_NO_REF_GF : 0) |
81 (((referenced_buffers & kAltref) == 0) ? VP8_EFLAG_NO_REF_ARF : 0) |
82 (((updated_buffers & kLast) == 0) ? VP8_EFLAG_NO_UPD_LAST : 0) |
83 (((updated_buffers & kGolden) == 0) ? VP8_EFLAG_NO_UPD_GF : 0) |
84 (((updated_buffers & kAltref) == 0) ? VP8_EFLAG_NO_UPD_ARF : 0) |
85 (update_entropy ? 0 : VP8_EFLAG_NO_UPD_ENTROPY);
86 }
87
88 constexpr int kKeyFrameFlags = ToVp8CodecFlags(kNone, kAll, true);
89
GetTemporalLayerRates(int target_bitrate_kbps,int framerate_fps,int num_temporal_layers)90 std::vector<uint32_t> GetTemporalLayerRates(int target_bitrate_kbps,
91 int framerate_fps,
92 int num_temporal_layers) {
93 VideoCodec codec;
94 codec.codecType = VideoCodecType::kVideoCodecVP8;
95 codec.numberOfSimulcastStreams = 1;
96 codec.maxBitrate = target_bitrate_kbps;
97 codec.maxFramerate = framerate_fps;
98 codec.simulcastStream[0].targetBitrate = target_bitrate_kbps;
99 codec.simulcastStream[0].maxBitrate = target_bitrate_kbps;
100 codec.simulcastStream[0].numberOfTemporalLayers = num_temporal_layers;
101 codec.simulcastStream[0].active = true;
102 SimulcastRateAllocator allocator(codec);
103 return allocator
104 .Allocate(
105 VideoBitrateAllocationParameters(target_bitrate_kbps, framerate_fps))
106 .GetTemporalLayerAllocation(0);
107 }
108
109 constexpr int kDefaultBitrateBps = 500;
110 constexpr int kDefaultFramerate = 30;
111 constexpr int kDefaultBytesPerFrame =
112 (kDefaultBitrateBps / 8) / kDefaultFramerate;
113 constexpr int kDefaultQp = 2;
114 } // namespace
115
116 class TemporalLayersTest : public ::testing::Test {
117 public:
118 ~TemporalLayersTest() override = default;
119
IgnoredCodecSpecificInfo()120 CodecSpecificInfo* IgnoredCodecSpecificInfo() {
121 codec_specific_info_ = std::make_unique<CodecSpecificInfo>();
122 return codec_specific_info_.get();
123 }
124
125 private:
126 std::unique_ptr<CodecSpecificInfo> codec_specific_info_;
127 };
128
129 TEST_F(TemporalLayersTest, 2Layers) {
130 constexpr int kNumLayers = 2;
131 DefaultTemporalLayers tl(kNumLayers);
132 DefaultTemporalLayersChecker checker(kNumLayers);
133 tl.OnRatesUpdated(0,
134 GetTemporalLayerRates(kDefaultBytesPerFrame,
135 kDefaultFramerate, kNumLayers),
136 kDefaultFramerate);
137 tl.UpdateConfiguration(0);
138
139 constexpr size_t kPatternSize = 4;
140 constexpr size_t kRepetitions = 4;
141
142 const int expected_flags[kPatternSize] = {
143 ToVp8CodecFlags(kLast, kLast, true),
144 ToVp8CodecFlags(kLast, kGolden, true),
145 ToVp8CodecFlags(kLast, kLast, true),
146 ToVp8CodecFlags(kLast | kGolden, kNone, false),
147 };
148 const int expected_temporal_idx[kPatternSize] = {0, 1, 0, 1};
149 const bool expected_layer_sync[kPatternSize] = {false, true, false, false};
150
151 uint32_t timestamp = 0;
152 for (size_t i = 0; i < kPatternSize * kRepetitions; ++i) {
153 const size_t ind = i % kPatternSize;
154 const bool is_keyframe = (i == 0);
155 CodecSpecificInfo info;
156 Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
157 EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[ind],
158 LibvpxVp8Encoder::EncodeFlags(tl_config))
159 << i;
160 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe,
161 kDefaultQp, &info);
162 EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config));
163 EXPECT_EQ(expected_temporal_idx[ind], info.codecSpecific.VP8.temporalIdx);
164 EXPECT_EQ(expected_temporal_idx[ind], tl_config.packetizer_temporal_idx);
165 EXPECT_EQ(expected_temporal_idx[ind], tl_config.encoder_layer_id);
166 EXPECT_EQ(is_keyframe || expected_layer_sync[ind],
167 info.codecSpecific.VP8.layerSync);
168 EXPECT_EQ(expected_layer_sync[ind], tl_config.layer_sync);
169 timestamp += 3000;
170 }
171 }
172
173 TEST_F(TemporalLayersTest, 3Layers) {
174 constexpr int kNumLayers = 3;
175 DefaultTemporalLayers tl(kNumLayers);
176 DefaultTemporalLayersChecker checker(kNumLayers);
177 tl.OnRatesUpdated(0,
178 GetTemporalLayerRates(kDefaultBytesPerFrame,
179 kDefaultFramerate, kNumLayers),
180 kDefaultFramerate);
181 tl.UpdateConfiguration(0);
182
183 int expected_flags[16] = {
184 kTemporalUpdateLast,
185 kTemporalUpdateNoneNoRefGoldenAltRef,
186 kTemporalUpdateGoldenWithoutDependency,
187 kTemporalUpdateNoneNoRefAltRef,
188 kTemporalUpdateLast,
189 kTemporalUpdateNoneNoRefAltRef,
190 kTemporalUpdateGolden,
191 kTemporalUpdateNoneNoRefAltRef,
192 kTemporalUpdateLast,
193 kTemporalUpdateNoneNoRefGoldenAltRef,
194 kTemporalUpdateGoldenWithoutDependency,
195 kTemporalUpdateNoneNoRefAltRef,
196 kTemporalUpdateLast,
197 kTemporalUpdateNoneNoRefAltRef,
198 kTemporalUpdateGolden,
199 kTemporalUpdateNoneNoRefAltRef,
200 };
201 int expected_temporal_idx[16] = {0, 2, 1, 2, 0, 2, 1, 2,
202 0, 2, 1, 2, 0, 2, 1, 2};
203
204 bool expected_layer_sync[16] = {false, true, true, false, false, false,
205 false, false, false, true, true, false,
206 false, false, false, false};
207
208 unsigned int timestamp = 0;
209 for (int i = 0; i < 16; ++i) {
210 const bool is_keyframe = (i == 0);
211 CodecSpecificInfo info;
212 Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
213 EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[i],
214 LibvpxVp8Encoder::EncodeFlags(tl_config))
215 << i;
216 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe,
217 kDefaultQp, &info);
218 EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config));
219 EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx);
220 EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
221 EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
222 EXPECT_EQ(is_keyframe || expected_layer_sync[i],
223 info.codecSpecific.VP8.layerSync);
224 EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
225 timestamp += 3000;
226 }
227 }
228
TEST_F(TemporalLayersTest,Alternative3Layers)229 TEST_F(TemporalLayersTest, Alternative3Layers) {
230 constexpr int kNumLayers = 3;
231 ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
232 DefaultTemporalLayers tl(kNumLayers);
233 DefaultTemporalLayersChecker checker(kNumLayers);
234 tl.OnRatesUpdated(0,
235 GetTemporalLayerRates(kDefaultBytesPerFrame,
236 kDefaultFramerate, kNumLayers),
237 kDefaultFramerate);
238 tl.UpdateConfiguration(0);
239
240 int expected_flags[8] = {kTemporalUpdateLast,
241 kTemporalUpdateAltrefWithoutDependency,
242 kTemporalUpdateGoldenWithoutDependency,
243 kTemporalUpdateNone,
244 kTemporalUpdateLast,
245 kTemporalUpdateAltrefWithoutDependency,
246 kTemporalUpdateGoldenWithoutDependency,
247 kTemporalUpdateNone};
248 int expected_temporal_idx[8] = {0, 2, 1, 2, 0, 2, 1, 2};
249
250 bool expected_layer_sync[8] = {false, true, true, false,
251 false, true, true, false};
252
253 unsigned int timestamp = 0;
254 for (int i = 0; i < 8; ++i) {
255 const bool is_keyframe = (i == 0);
256 CodecSpecificInfo info;
257 Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
258 EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[i],
259 LibvpxVp8Encoder::EncodeFlags(tl_config))
260 << i;
261 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe,
262 kDefaultQp, &info);
263 EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config));
264 EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx);
265 EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
266 EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
267 EXPECT_EQ(is_keyframe || expected_layer_sync[i],
268 info.codecSpecific.VP8.layerSync);
269 EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
270 timestamp += 3000;
271 }
272 }
273
TEST_F(TemporalLayersTest,SearchOrder)274 TEST_F(TemporalLayersTest, SearchOrder) {
275 constexpr int kNumLayers = 3;
276 ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
277 DefaultTemporalLayers tl(kNumLayers);
278 DefaultTemporalLayersChecker checker(kNumLayers);
279 tl.OnRatesUpdated(0,
280 GetTemporalLayerRates(kDefaultBytesPerFrame,
281 kDefaultFramerate, kNumLayers),
282 kDefaultFramerate);
283 tl.UpdateConfiguration(0);
284
285 // Use a repeating pattern of tl 0, 2, 1, 2.
286 // Tl 0, 1, 2 update last, golden, altref respectively.
287
288 // Start with a key-frame. tl_config flags can be ignored.
289 uint32_t timestamp = 0;
290 Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
291 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
292 IgnoredCodecSpecificInfo());
293
294 // TL2 frame. First one only references TL0. Updates altref.
295 tl_config = tl.NextFrameConfig(0, ++timestamp);
296 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
297 IgnoredCodecSpecificInfo());
298 EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
299 EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
300
301 // TL1 frame. Can only reference TL0. Updated golden.
302 tl_config = tl.NextFrameConfig(0, ++timestamp);
303 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
304 IgnoredCodecSpecificInfo());
305 EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
306 EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
307
308 // TL2 frame. Can reference all three buffers. Golden was the last to be
309 // updated, the next to last was altref.
310 tl_config = tl.NextFrameConfig(0, ++timestamp);
311 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
312 IgnoredCodecSpecificInfo());
313 EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kGolden);
314 EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kAltref);
315 }
316
TEST_F(TemporalLayersTest,SearchOrderWithDrop)317 TEST_F(TemporalLayersTest, SearchOrderWithDrop) {
318 constexpr int kNumLayers = 3;
319 ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
320 DefaultTemporalLayers tl(kNumLayers);
321 DefaultTemporalLayersChecker checker(kNumLayers);
322 tl.OnRatesUpdated(0,
323 GetTemporalLayerRates(kDefaultBytesPerFrame,
324 kDefaultFramerate, kNumLayers),
325 kDefaultFramerate);
326 tl.UpdateConfiguration(0);
327
328 // Use a repeating pattern of tl 0, 2, 1, 2.
329 // Tl 0, 1, 2 update last, golden, altref respectively.
330
331 // Start with a key-frame. tl_config flags can be ignored.
332 uint32_t timestamp = 0;
333 Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
334 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
335 IgnoredCodecSpecificInfo());
336
337 // TL2 frame. First one only references TL0. Updates altref.
338 tl_config = tl.NextFrameConfig(0, ++timestamp);
339 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
340 IgnoredCodecSpecificInfo());
341 EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
342 EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
343
344 // Dropped TL1 frame. Can only reference TL0. Should have updated golden.
345 tl_config = tl.NextFrameConfig(0, ++timestamp);
346 tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
347
348 // TL2 frame. Can normally reference all three buffers, but golden has not
349 // been populated this cycle. Altref was last to be updated, before that last.
350 tl_config = tl.NextFrameConfig(0, ++timestamp);
351 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
352 IgnoredCodecSpecificInfo());
353 EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kAltref);
354 EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kLast);
355 }
356
357 TEST_F(TemporalLayersTest, 4Layers) {
358 constexpr int kNumLayers = 4;
359 DefaultTemporalLayers tl(kNumLayers);
360 DefaultTemporalLayersChecker checker(kNumLayers);
361 tl.OnRatesUpdated(0,
362 GetTemporalLayerRates(kDefaultBytesPerFrame,
363 kDefaultFramerate, kNumLayers),
364 kDefaultFramerate);
365 tl.UpdateConfiguration(0);
366 int expected_flags[16] = {
367 kTemporalUpdateLast,
368 kTemporalUpdateNoneNoRefGoldenAltRef,
369 kTemporalUpdateAltrefWithoutDependency,
370 kTemporalUpdateNoneNoRefGolden,
371 kTemporalUpdateGoldenWithoutDependency,
372 kTemporalUpdateNone,
373 kTemporalUpdateAltref,
374 kTemporalUpdateNone,
375 kTemporalUpdateLast,
376 kTemporalUpdateNone,
377 kTemporalUpdateAltref,
378 kTemporalUpdateNone,
379 kTemporalUpdateGolden,
380 kTemporalUpdateNone,
381 kTemporalUpdateAltref,
382 kTemporalUpdateNone,
383 };
384 int expected_temporal_idx[16] = {0, 3, 2, 3, 1, 3, 2, 3,
385 0, 3, 2, 3, 1, 3, 2, 3};
386
387 bool expected_layer_sync[16] = {false, true, true, false, true, false,
388 false, false, false, false, false, false,
389 false, false, false, false};
390
391 uint32_t timestamp = 0;
392 for (int i = 0; i < 16; ++i) {
393 const bool is_keyframe = (i == 0);
394 CodecSpecificInfo info;
395 Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
396 EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[i],
397 LibvpxVp8Encoder::EncodeFlags(tl_config))
398 << i;
399 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe,
400 kDefaultQp, &info);
401 EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config));
402 EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx);
403 EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
404 EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
405 EXPECT_EQ(is_keyframe || expected_layer_sync[i],
406 info.codecSpecific.VP8.layerSync);
407 EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
408 timestamp += 3000;
409 }
410 }
411
TEST_F(TemporalLayersTest,DoesNotReferenceDroppedFrames)412 TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) {
413 constexpr int kNumLayers = 3;
414 // Use a repeating pattern of tl 0, 2, 1, 2.
415 // Tl 0, 1, 2 update last, golden, altref respectively.
416 ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
417 DefaultTemporalLayers tl(kNumLayers);
418 DefaultTemporalLayersChecker checker(kNumLayers);
419 tl.OnRatesUpdated(0,
420 GetTemporalLayerRates(kDefaultBytesPerFrame,
421 kDefaultFramerate, kNumLayers),
422 kDefaultFramerate);
423 tl.UpdateConfiguration(0);
424
425 // Start with a keyframe.
426 uint32_t timestamp = 0;
427 Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
428 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
429 IgnoredCodecSpecificInfo());
430
431 // Dropped TL2 frame.
432 tl_config = tl.NextFrameConfig(0, ++timestamp);
433 tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
434
435 // Dropped TL1 frame.
436 tl_config = tl.NextFrameConfig(0, ++timestamp);
437 tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
438
439 // TL2 frame. Can reference all three buffers, valid since golden and altref
440 // both contain the last keyframe.
441 tl_config = tl.NextFrameConfig(0, ++timestamp);
442 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
443 IgnoredCodecSpecificInfo());
444 EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
445 EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference);
446 EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference);
447
448 // Restart of cycle!
449
450 // TL0 base layer frame, updating and referencing last.
451 tl_config = tl.NextFrameConfig(0, ++timestamp);
452 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
453 IgnoredCodecSpecificInfo());
454
455 // TL2 frame, updating altref.
456 tl_config = tl.NextFrameConfig(0, ++timestamp);
457 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
458 IgnoredCodecSpecificInfo());
459
460 // TL1 frame, updating golden.
461 tl_config = tl.NextFrameConfig(0, ++timestamp);
462 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
463 IgnoredCodecSpecificInfo());
464
465 // TL2 frame. Can still reference all buffer since they have been update this
466 // cycle.
467 tl_config = tl.NextFrameConfig(0, ++timestamp);
468 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
469 IgnoredCodecSpecificInfo());
470 EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
471 EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference);
472 EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference);
473
474 // Restart of cycle!
475
476 // TL0 base layer frame, updating and referencing last.
477 tl_config = tl.NextFrameConfig(0, ++timestamp);
478 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
479 IgnoredCodecSpecificInfo());
480
481 // Dropped TL2 frame.
482 tl_config = tl.NextFrameConfig(0, ++timestamp);
483 tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
484
485 // Dropped TL1 frame.
486 tl_config = tl.NextFrameConfig(0, ++timestamp);
487 tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
488
489 // TL2 frame. This time golden and altref contain data from the previous cycle
490 // and cannot be referenced.
491 tl_config = tl.NextFrameConfig(0, ++timestamp);
492 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
493 IgnoredCodecSpecificInfo());
494 EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
495 EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference);
496 EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
497 }
498
TEST_F(TemporalLayersTest,DoesNotReferenceUnlessGuaranteedToExist)499 TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExist) {
500 constexpr int kNumLayers = 3;
501 // Use a repeating pattern of tl 0, 2, 1, 2.
502 // Tl 0, 1 updates last, golden respectively. Altref is always last keyframe.
503 DefaultTemporalLayers tl(kNumLayers);
504 DefaultTemporalLayersChecker checker(kNumLayers);
505 tl.OnRatesUpdated(0,
506 GetTemporalLayerRates(kDefaultBytesPerFrame,
507 kDefaultFramerate, kNumLayers),
508 kDefaultFramerate);
509 tl.UpdateConfiguration(0);
510
511 // Start with a keyframe.
512 uint32_t timestamp = 0;
513 Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
514 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
515 IgnoredCodecSpecificInfo());
516
517 // Do a full cycle of the pattern.
518 for (int i = 0; i < 7; ++i) {
519 tl_config = tl.NextFrameConfig(0, ++timestamp);
520 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
521 IgnoredCodecSpecificInfo());
522 }
523
524 // TL0 base layer frame, starting the cycle over.
525 tl_config = tl.NextFrameConfig(0, ++timestamp);
526 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
527 IgnoredCodecSpecificInfo());
528
529 // TL2 frame.
530 tl_config = tl.NextFrameConfig(0, ++timestamp);
531 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
532 IgnoredCodecSpecificInfo());
533
534 // Encoder has a hiccup and builds a queue, so frame encoding is delayed.
535 // TL1 frame, updating golden.
536 tl_config = tl.NextFrameConfig(0, ++timestamp);
537
538 // TL2 frame, that should be referencing golden, but we can't be certain it's
539 // not going to be dropped, so that is not allowed.
540 tl_config = tl.NextFrameConfig(0, timestamp + 1);
541 EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
542 EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference);
543 EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
544
545 // TL0 base layer frame.
546 tl_config = tl.NextFrameConfig(0, timestamp + 2);
547
548 // The previous four enqueued frames finally get encoded, and the updated
549 // buffers are now OK to reference.
550 // Enqueued TL1 frame ready.
551 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
552 IgnoredCodecSpecificInfo());
553 // Enqueued TL2 frame.
554 tl.OnEncodeDone(0, ++timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
555 IgnoredCodecSpecificInfo());
556 // Enqueued TL0 frame.
557 tl.OnEncodeDone(0, ++timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
558 IgnoredCodecSpecificInfo());
559
560 // TL2 frame, all buffers are now in a known good state, OK to reference.
561 tl_config = tl.NextFrameConfig(0, ++timestamp + 1);
562 EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
563 EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference);
564 EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
565 }
566
TEST_F(TemporalLayersTest,DoesNotReferenceUnlessGuaranteedToExistLongDelay)567 TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExistLongDelay) {
568 constexpr int kNumLayers = 3;
569 // Use a repeating pattern of tl 0, 2, 1, 2.
570 // Tl 0, 1 updates last, golden, altref respectively.
571 ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
572 DefaultTemporalLayers tl(kNumLayers);
573 DefaultTemporalLayersChecker checker(kNumLayers);
574 tl.OnRatesUpdated(0,
575 GetTemporalLayerRates(kDefaultBytesPerFrame,
576 kDefaultFramerate, kNumLayers),
577 kDefaultFramerate);
578 tl.UpdateConfiguration(0);
579
580 // Start with a keyframe.
581 uint32_t timestamp = 0;
582 Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
583 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
584 IgnoredCodecSpecificInfo());
585
586 // Do a full cycle of the pattern.
587 for (int i = 0; i < 3; ++i) {
588 tl_config = tl.NextFrameConfig(0, ++timestamp);
589 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
590 IgnoredCodecSpecificInfo());
591 }
592
593 // TL0 base layer frame, starting the cycle over.
594 tl_config = tl.NextFrameConfig(0, ++timestamp);
595 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
596 IgnoredCodecSpecificInfo());
597
598 // TL2 frame.
599 tl_config = tl.NextFrameConfig(0, ++timestamp);
600 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
601 IgnoredCodecSpecificInfo());
602
603 // Encoder has a hiccup and builds a queue, so frame encoding is delayed.
604 // Encoded, but delayed frames in TL 1, 2.
605 tl_config = tl.NextFrameConfig(0, timestamp + 1);
606 tl_config = tl.NextFrameConfig(0, timestamp + 2);
607
608 // Restart of the pattern!
609
610 // Encoded, but delayed frames in TL 2, 1.
611 tl_config = tl.NextFrameConfig(0, timestamp + 3);
612 tl_config = tl.NextFrameConfig(0, timestamp + 4);
613
614 // TL1 frame from last cycle is ready.
615 tl.OnEncodeDone(0, timestamp + 1, kDefaultBytesPerFrame, false, kDefaultQp,
616 IgnoredCodecSpecificInfo());
617 // TL2 frame from last cycle is ready.
618 tl.OnEncodeDone(0, timestamp + 2, kDefaultBytesPerFrame, false, kDefaultQp,
619 IgnoredCodecSpecificInfo());
620
621 // TL2 frame, that should be referencing all buffers, but altref and golden
622 // haven not been updated this cycle. (Don't be fooled by the late frames from
623 // the last cycle!)
624 tl_config = tl.NextFrameConfig(0, timestamp + 5);
625 EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
626 EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference);
627 EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
628 }
629
TEST_F(TemporalLayersTest,KeyFrame)630 TEST_F(TemporalLayersTest, KeyFrame) {
631 constexpr int kNumLayers = 3;
632 DefaultTemporalLayers tl(kNumLayers);
633 DefaultTemporalLayersChecker checker(kNumLayers);
634 tl.OnRatesUpdated(0,
635 GetTemporalLayerRates(kDefaultBytesPerFrame,
636 kDefaultFramerate, kNumLayers),
637 kDefaultFramerate);
638 tl.UpdateConfiguration(0);
639
640 int expected_flags[8] = {
641 kTemporalUpdateLastRefAltRef,
642 kTemporalUpdateNoneNoRefGoldenAltRef,
643 kTemporalUpdateGoldenWithoutDependency,
644 kTemporalUpdateNoneNoRefAltRef,
645 kTemporalUpdateLast,
646 kTemporalUpdateNoneNoRefAltRef,
647 kTemporalUpdateGolden,
648 kTemporalUpdateNone,
649 };
650 int expected_temporal_idx[8] = {0, 2, 1, 2, 0, 2, 1, 2};
651 bool expected_layer_sync[8] = {true, true, true, false,
652 false, false, false, false};
653
654 uint32_t timestamp = 0;
655 for (int i = 0; i < 7; ++i) {
656 // Temporal pattern starts from 0 after key frame. Let the first |i| - 1
657 // frames be delta frames, and the |i|th one key frame.
658 for (int j = 1; j <= i; ++j) {
659 // Since last frame was always a keyframe and thus index 0 in the pattern,
660 // this loop starts at index 1.
661 Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
662 EXPECT_EQ(expected_flags[j], LibvpxVp8Encoder::EncodeFlags(tl_config))
663 << j;
664 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
665 IgnoredCodecSpecificInfo());
666 EXPECT_TRUE(checker.CheckTemporalConfig(false, tl_config));
667 EXPECT_EQ(expected_temporal_idx[j], tl_config.packetizer_temporal_idx);
668 EXPECT_EQ(expected_temporal_idx[j], tl_config.encoder_layer_id);
669 EXPECT_EQ(expected_layer_sync[j], tl_config.layer_sync);
670 timestamp += 3000;
671 }
672
673 CodecSpecificInfo info;
674 Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
675 tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
676 &info);
677 EXPECT_TRUE(info.codecSpecific.VP8.layerSync)
678 << "Key frame should be marked layer sync.";
679 EXPECT_EQ(0, info.codecSpecific.VP8.temporalIdx)
680 << "Key frame should always be packetized as layer 0";
681 EXPECT_EQ(0, info.generic_frame_info->temporal_id)
682 << "Key frame should always be packetized as layer 0";
683 EXPECT_THAT(info.generic_frame_info->decode_target_indications,
684 Each(DecodeTargetIndication::kSwitch))
685 << "Key frame is universal switch";
686 EXPECT_TRUE(checker.CheckTemporalConfig(true, tl_config));
687 }
688 }
689
690 class TemporalLayersReferenceTest : public TemporalLayersTest,
691 public ::testing::WithParamInterface<int> {
692 public:
TemporalLayersReferenceTest()693 TemporalLayersReferenceTest()
694 : timestamp_(1),
695 last_sync_timestamp_(timestamp_),
696 tl0_reference_(nullptr) {}
~TemporalLayersReferenceTest()697 virtual ~TemporalLayersReferenceTest() {}
698
699 protected:
700 static const int kMaxPatternLength = 32;
701
702 struct BufferState {
BufferStatewebrtc::test::TemporalLayersReferenceTest::BufferState703 BufferState() : BufferState(-1, 0, false) {}
BufferStatewebrtc::test::TemporalLayersReferenceTest::BufferState704 BufferState(int temporal_idx, uint32_t timestamp, bool sync)
705 : temporal_idx(temporal_idx), timestamp(timestamp), sync(sync) {}
706 int temporal_idx;
707 uint32_t timestamp;
708 bool sync;
709 };
710
UpdateSyncRefState(const BufferFlags & flags,BufferState * buffer_state)711 bool UpdateSyncRefState(const BufferFlags& flags, BufferState* buffer_state) {
712 if (flags & BufferFlags::kReference) {
713 if (buffer_state->temporal_idx == -1)
714 return true; // References key-frame.
715 if (buffer_state->temporal_idx == 0) {
716 // No more than one reference to TL0 frame.
717 EXPECT_EQ(nullptr, tl0_reference_);
718 tl0_reference_ = buffer_state;
719 return true;
720 }
721 return false; // References higher layer.
722 }
723 return true; // No reference, does not affect sync frame status.
724 }
725
ValidateReference(const BufferFlags & flags,const BufferState & buffer_state,int temporal_layer)726 void ValidateReference(const BufferFlags& flags,
727 const BufferState& buffer_state,
728 int temporal_layer) {
729 if (flags & BufferFlags::kReference) {
730 if (temporal_layer > 0 && buffer_state.timestamp > 0) {
731 // Check that high layer reference does not go past last sync frame.
732 EXPECT_GE(buffer_state.timestamp, last_sync_timestamp_);
733 }
734 // No reference to buffer in higher layer.
735 EXPECT_LE(buffer_state.temporal_idx, temporal_layer);
736 }
737 }
738
739 uint32_t timestamp_ = 1;
740 uint32_t last_sync_timestamp_ = timestamp_;
741 BufferState* tl0_reference_;
742
743 BufferState last_state;
744 BufferState golden_state;
745 BufferState altref_state;
746 };
747
748 INSTANTIATE_TEST_SUITE_P(DefaultTemporalLayersTest,
749 TemporalLayersReferenceTest,
750 ::testing::Range(1, kMaxTemporalStreams + 1));
751
TEST_P(TemporalLayersReferenceTest,ValidFrameConfigs)752 TEST_P(TemporalLayersReferenceTest, ValidFrameConfigs) {
753 const int num_layers = GetParam();
754 DefaultTemporalLayers tl(num_layers);
755 tl.OnRatesUpdated(
756 0, GetTemporalLayerRates(kDefaultBytesPerFrame, kDefaultFramerate, 1),
757 kDefaultFramerate);
758 tl.UpdateConfiguration(0);
759
760 // Run through the pattern and store the frame dependencies, plus keep track
761 // of the buffer state; which buffers references which temporal layers (if
762 // (any). If a given buffer is never updated, it is legal to reference it
763 // even for sync frames. In order to be general, don't assume TL0 always
764 // updates |last|.
765 std::vector<Vp8FrameConfig> tl_configs(kMaxPatternLength);
766 for (int i = 0; i < kMaxPatternLength; ++i) {
767 Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp_);
768 tl.OnEncodeDone(0, timestamp_, kDefaultBytesPerFrame, i == 0, kDefaultQp,
769 IgnoredCodecSpecificInfo());
770 ++timestamp_;
771 EXPECT_FALSE(tl_config.drop_frame);
772 tl_configs.push_back(tl_config);
773 int temporal_idx = tl_config.encoder_layer_id;
774 // For the default layers, always keep encoder and rtp layers in sync.
775 EXPECT_EQ(tl_config.packetizer_temporal_idx, temporal_idx);
776
777 // Determine if this frame is in a higher layer but references only TL0
778 // or untouched buffers, if so verify it is marked as a layer sync.
779 bool is_sync_frame = true;
780 tl0_reference_ = nullptr;
781 if (temporal_idx <= 0) {
782 is_sync_frame = false; // TL0 by definition not a sync frame.
783 } else if (!UpdateSyncRefState(tl_config.last_buffer_flags, &last_state)) {
784 is_sync_frame = false;
785 } else if (!UpdateSyncRefState(tl_config.golden_buffer_flags,
786 &golden_state)) {
787 is_sync_frame = false;
788 } else if (!UpdateSyncRefState(tl_config.arf_buffer_flags, &altref_state)) {
789 is_sync_frame = false;
790 }
791 if (is_sync_frame) {
792 // Cache timestamp for last found sync frame, so that we can verify no
793 // references back past this frame.
794 ASSERT_TRUE(tl0_reference_);
795 last_sync_timestamp_ = tl0_reference_->timestamp;
796 }
797 EXPECT_EQ(tl_config.layer_sync, is_sync_frame);
798
799 // Validate no reference from lower to high temporal layer, or backwards
800 // past last reference frame.
801 ValidateReference(tl_config.last_buffer_flags, last_state, temporal_idx);
802 ValidateReference(tl_config.golden_buffer_flags, golden_state,
803 temporal_idx);
804 ValidateReference(tl_config.arf_buffer_flags, altref_state, temporal_idx);
805
806 // Update the current layer state.
807 BufferState state = {temporal_idx, timestamp_, is_sync_frame};
808 if (tl_config.last_buffer_flags & BufferFlags::kUpdate)
809 last_state = state;
810 if (tl_config.golden_buffer_flags & BufferFlags::kUpdate)
811 golden_state = state;
812 if (tl_config.arf_buffer_flags & BufferFlags::kUpdate)
813 altref_state = state;
814 }
815 }
816 } // namespace test
817 } // namespace webrtc
818