1 /*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/modules/video_coding/utility/quality_scaler.h"
12
13 #include "testing/gtest/include/gtest/gtest.h"
14
15 namespace webrtc {
16 namespace {
17 static const int kNumSeconds = 10;
18 static const int kWidth = 1920;
19 static const int kHalfWidth = kWidth / 2;
20 static const int kHeight = 1080;
21 static const int kFramerate = 30;
22 static const int kLowQp = 15;
23 static const int kNormalQp = 30;
24 static const int kHighQp = 40;
25 static const int kMaxQp = 56;
26 } // namespace
27
28 class QualityScalerTest : public ::testing::Test {
29 public:
30 // Temporal and spatial resolution.
31 struct Resolution {
32 int framerate;
33 int width;
34 int height;
35 };
36
37 protected:
38 enum ScaleDirection {
39 kKeepScaleAtHighQp,
40 kScaleDown,
41 kScaleDownAboveHighQp,
42 kScaleUp
43 };
44 enum BadQualityMetric { kDropFrame, kReportLowQP };
45
QualityScalerTest()46 QualityScalerTest() {
47 input_frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, kHalfWidth,
48 kHalfWidth);
49 qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false);
50 qs_.ReportFramerate(kFramerate);
51 qs_.OnEncodeFrame(input_frame_);
52 }
53
TriggerScale(ScaleDirection scale_direction)54 bool TriggerScale(ScaleDirection scale_direction) {
55 qs_.OnEncodeFrame(input_frame_);
56 int initial_width = qs_.GetScaledResolution().width;
57 for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
58 switch (scale_direction) {
59 case kScaleUp:
60 qs_.ReportQP(kLowQp);
61 break;
62 case kScaleDown:
63 qs_.ReportDroppedFrame();
64 break;
65 case kKeepScaleAtHighQp:
66 qs_.ReportQP(kHighQp);
67 break;
68 case kScaleDownAboveHighQp:
69 qs_.ReportQP(kHighQp + 1);
70 break;
71 }
72 qs_.OnEncodeFrame(input_frame_);
73 if (qs_.GetScaledResolution().width != initial_width)
74 return true;
75 }
76
77 return false;
78 }
79
ExpectOriginalFrame()80 void ExpectOriginalFrame() {
81 EXPECT_EQ(&input_frame_, &qs_.GetScaledFrame(input_frame_))
82 << "Using scaled frame instead of original input.";
83 }
84
ExpectScaleUsingReportedResolution()85 void ExpectScaleUsingReportedResolution() {
86 qs_.OnEncodeFrame(input_frame_);
87 QualityScaler::Resolution res = qs_.GetScaledResolution();
88 const VideoFrame& scaled_frame = qs_.GetScaledFrame(input_frame_);
89 EXPECT_EQ(res.width, scaled_frame.width());
90 EXPECT_EQ(res.height, scaled_frame.height());
91 }
92
93 void ContinuouslyDownscalesByHalfDimensionsAndBackUp();
94
95 void DoesNotDownscaleFrameDimensions(int width, int height);
96
97 Resolution TriggerResolutionChange(BadQualityMetric dropframe_lowqp,
98 int num_second,
99 int initial_framerate);
100
101 void VerifyQualityAdaptation(int initial_framerate,
102 int seconds,
103 bool expect_spatial_resize,
104 bool expect_framerate_reduction);
105
106 void DownscaleEndsAt(int input_width,
107 int input_height,
108 int end_width,
109 int end_height);
110
111 QualityScaler qs_;
112 VideoFrame input_frame_;
113 };
114
TEST_F(QualityScalerTest,UsesOriginalFrameInitially)115 TEST_F(QualityScalerTest, UsesOriginalFrameInitially) {
116 ExpectOriginalFrame();
117 }
118
TEST_F(QualityScalerTest,ReportsOriginalResolutionInitially)119 TEST_F(QualityScalerTest, ReportsOriginalResolutionInitially) {
120 qs_.OnEncodeFrame(input_frame_);
121 QualityScaler::Resolution res = qs_.GetScaledResolution();
122 EXPECT_EQ(input_frame_.width(), res.width);
123 EXPECT_EQ(input_frame_.height(), res.height);
124 }
125
TEST_F(QualityScalerTest,DownscalesAfterContinuousFramedrop)126 TEST_F(QualityScalerTest, DownscalesAfterContinuousFramedrop) {
127 EXPECT_TRUE(TriggerScale(kScaleDown)) << "No downscale within " << kNumSeconds
128 << " seconds.";
129 QualityScaler::Resolution res = qs_.GetScaledResolution();
130 EXPECT_LT(res.width, input_frame_.width());
131 EXPECT_LT(res.height, input_frame_.height());
132 }
133
TEST_F(QualityScalerTest,KeepsScaleAtHighQp)134 TEST_F(QualityScalerTest, KeepsScaleAtHighQp) {
135 EXPECT_FALSE(TriggerScale(kKeepScaleAtHighQp))
136 << "Downscale at high threshold which should keep scale.";
137 QualityScaler::Resolution res = qs_.GetScaledResolution();
138 EXPECT_EQ(res.width, input_frame_.width());
139 EXPECT_EQ(res.height, input_frame_.height());
140 }
141
TEST_F(QualityScalerTest,DownscalesAboveHighQp)142 TEST_F(QualityScalerTest, DownscalesAboveHighQp) {
143 EXPECT_TRUE(TriggerScale(kScaleDownAboveHighQp))
144 << "No downscale within " << kNumSeconds << " seconds.";
145 QualityScaler::Resolution res = qs_.GetScaledResolution();
146 EXPECT_LT(res.width, input_frame_.width());
147 EXPECT_LT(res.height, input_frame_.height());
148 }
149
TEST_F(QualityScalerTest,DownscalesAfterTwoThirdsFramedrop)150 TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) {
151 for (int i = 0; i < kFramerate * kNumSeconds / 3; ++i) {
152 qs_.ReportQP(kNormalQp);
153 qs_.ReportDroppedFrame();
154 qs_.ReportDroppedFrame();
155 qs_.OnEncodeFrame(input_frame_);
156 if (qs_.GetScaledResolution().width < input_frame_.width())
157 return;
158 }
159
160 FAIL() << "No downscale within " << kNumSeconds << " seconds.";
161 }
162
TEST_F(QualityScalerTest,DoesNotDownscaleOnNormalQp)163 TEST_F(QualityScalerTest, DoesNotDownscaleOnNormalQp) {
164 for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
165 qs_.ReportQP(kNormalQp);
166 qs_.OnEncodeFrame(input_frame_);
167 ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
168 << "Unexpected scale on half framedrop.";
169 }
170 }
171
TEST_F(QualityScalerTest,DoesNotDownscaleAfterHalfFramedrop)172 TEST_F(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) {
173 for (int i = 0; i < kFramerate * kNumSeconds / 2; ++i) {
174 qs_.ReportQP(kNormalQp);
175 qs_.OnEncodeFrame(input_frame_);
176 ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
177 << "Unexpected scale on half framedrop.";
178
179 qs_.ReportDroppedFrame();
180 qs_.OnEncodeFrame(input_frame_);
181 ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
182 << "Unexpected scale on half framedrop.";
183 }
184 }
185
ContinuouslyDownscalesByHalfDimensionsAndBackUp()186 void QualityScalerTest::ContinuouslyDownscalesByHalfDimensionsAndBackUp() {
187 const int initial_min_dimension = input_frame_.width() < input_frame_.height()
188 ? input_frame_.width()
189 : input_frame_.height();
190 int min_dimension = initial_min_dimension;
191 int current_shift = 0;
192 // Drop all frames to force-trigger downscaling.
193 while (min_dimension >= 2 * QualityScaler::kDefaultMinDownscaleDimension) {
194 EXPECT_TRUE(TriggerScale(kScaleDown)) << "No downscale within "
195 << kNumSeconds << " seconds.";
196 qs_.OnEncodeFrame(input_frame_);
197 QualityScaler::Resolution res = qs_.GetScaledResolution();
198 min_dimension = res.width < res.height ? res.width : res.height;
199 ++current_shift;
200 ASSERT_EQ(input_frame_.width() >> current_shift, res.width);
201 ASSERT_EQ(input_frame_.height() >> current_shift, res.height);
202 ExpectScaleUsingReportedResolution();
203 }
204
205 // Make sure we can scale back with good-quality frames.
206 while (min_dimension < initial_min_dimension) {
207 EXPECT_TRUE(TriggerScale(kScaleUp)) << "No upscale within " << kNumSeconds
208 << " seconds.";
209 qs_.OnEncodeFrame(input_frame_);
210 QualityScaler::Resolution res = qs_.GetScaledResolution();
211 min_dimension = res.width < res.height ? res.width : res.height;
212 --current_shift;
213 ASSERT_EQ(input_frame_.width() >> current_shift, res.width);
214 ASSERT_EQ(input_frame_.height() >> current_shift, res.height);
215 ExpectScaleUsingReportedResolution();
216 }
217
218 // Verify we don't start upscaling after further low use.
219 for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
220 qs_.ReportQP(kLowQp);
221 ExpectOriginalFrame();
222 }
223 }
224
TEST_F(QualityScalerTest,ContinuouslyDownscalesByHalfDimensionsAndBackUp)225 TEST_F(QualityScalerTest, ContinuouslyDownscalesByHalfDimensionsAndBackUp) {
226 ContinuouslyDownscalesByHalfDimensionsAndBackUp();
227 }
228
TEST_F(QualityScalerTest,ContinuouslyDownscalesOddResolutionsByHalfDimensionsAndBackUp)229 TEST_F(QualityScalerTest,
230 ContinuouslyDownscalesOddResolutionsByHalfDimensionsAndBackUp) {
231 const int kOddWidth = 517;
232 const int kHalfOddWidth = (kOddWidth + 1) / 2;
233 const int kOddHeight = 1239;
234 input_frame_.CreateEmptyFrame(kOddWidth, kOddHeight, kOddWidth, kHalfOddWidth,
235 kHalfOddWidth);
236 ContinuouslyDownscalesByHalfDimensionsAndBackUp();
237 }
238
DoesNotDownscaleFrameDimensions(int width,int height)239 void QualityScalerTest::DoesNotDownscaleFrameDimensions(int width, int height) {
240 input_frame_.CreateEmptyFrame(width, height, width, (width + 1) / 2,
241 (width + 1) / 2);
242
243 for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
244 qs_.ReportDroppedFrame();
245 qs_.OnEncodeFrame(input_frame_);
246 ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
247 << "Unexpected scale of minimal-size frame.";
248 }
249 }
250
TEST_F(QualityScalerTest,DoesNotDownscaleFrom1PxWidth)251 TEST_F(QualityScalerTest, DoesNotDownscaleFrom1PxWidth) {
252 DoesNotDownscaleFrameDimensions(1, kHeight);
253 }
254
TEST_F(QualityScalerTest,DoesNotDownscaleFrom1PxHeight)255 TEST_F(QualityScalerTest, DoesNotDownscaleFrom1PxHeight) {
256 DoesNotDownscaleFrameDimensions(kWidth, 1);
257 }
258
TEST_F(QualityScalerTest,DoesNotDownscaleFrom1Px)259 TEST_F(QualityScalerTest, DoesNotDownscaleFrom1Px) {
260 DoesNotDownscaleFrameDimensions(1, 1);
261 }
262
TriggerResolutionChange(BadQualityMetric dropframe_lowqp,int num_second,int initial_framerate)263 QualityScalerTest::Resolution QualityScalerTest::TriggerResolutionChange(
264 BadQualityMetric dropframe_lowqp,
265 int num_second,
266 int initial_framerate) {
267 QualityScalerTest::Resolution res;
268 res.framerate = initial_framerate;
269 qs_.OnEncodeFrame(input_frame_);
270 res.width = qs_.GetScaledResolution().width;
271 res.height = qs_.GetScaledResolution().height;
272 for (int i = 0; i < kFramerate * num_second; ++i) {
273 switch (dropframe_lowqp) {
274 case kReportLowQP:
275 qs_.ReportQP(kLowQp);
276 break;
277 case kDropFrame:
278 qs_.ReportDroppedFrame();
279 break;
280 }
281 qs_.OnEncodeFrame(input_frame_);
282 // Simulate the case when SetRates is called right after reducing
283 // framerate.
284 qs_.ReportFramerate(initial_framerate);
285 res.framerate = qs_.GetTargetFramerate();
286 if (res.framerate != -1)
287 qs_.ReportFramerate(res.framerate);
288 res.width = qs_.GetScaledResolution().width;
289 res.height = qs_.GetScaledResolution().height;
290 }
291 return res;
292 }
293
VerifyQualityAdaptation(int initial_framerate,int seconds,bool expect_spatial_resize,bool expect_framerate_reduction)294 void QualityScalerTest::VerifyQualityAdaptation(
295 int initial_framerate,
296 int seconds,
297 bool expect_spatial_resize,
298 bool expect_framerate_reduction) {
299 const int kDisabledBadQpThreshold = kMaxQp + 1;
300 qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator,
301 kDisabledBadQpThreshold, true);
302 qs_.OnEncodeFrame(input_frame_);
303 int init_width = qs_.GetScaledResolution().width;
304 int init_height = qs_.GetScaledResolution().height;
305
306 // Test reducing framerate by dropping frame continuously.
307 QualityScalerTest::Resolution res =
308 TriggerResolutionChange(kDropFrame, seconds, initial_framerate);
309
310 if (expect_framerate_reduction) {
311 EXPECT_LT(res.framerate, initial_framerate);
312 } else {
313 // No framerate reduction, video decimator should be disabled.
314 EXPECT_EQ(-1, res.framerate);
315 }
316
317 if (expect_spatial_resize) {
318 EXPECT_LT(res.width, init_width);
319 EXPECT_LT(res.height, init_height);
320 } else {
321 EXPECT_EQ(init_width, res.width);
322 EXPECT_EQ(init_height, res.height);
323 }
324
325 // The "seconds * 1.5" is to ensure spatial resolution to recover.
326 // For example, in 10 seconds test, framerate reduction happens in the first
327 // 5 seconds from 30fps to 15fps and causes the buffer size to be half of the
328 // original one. Then it will take only 75 samples to downscale (twice in 150
329 // samples). So to recover the resolution changes, we need more than 10
330 // seconds (i.e, seconds * 1.5). This is because the framerate increases
331 // before spatial size recovers, so it will take 150 samples to recover
332 // spatial size (300 for twice).
333 res = TriggerResolutionChange(kReportLowQP, seconds * 1.5, initial_framerate);
334 EXPECT_EQ(-1, res.framerate);
335 EXPECT_EQ(init_width, res.width);
336 EXPECT_EQ(init_height, res.height);
337 }
338
339 // In 5 seconds test, only framerate adjusting should happen.
TEST_F(QualityScalerTest,ChangeFramerateOnly)340 TEST_F(QualityScalerTest, ChangeFramerateOnly) {
341 VerifyQualityAdaptation(kFramerate, 5, false, true);
342 }
343
344 // In 10 seconds test, framerate adjusting and scaling are both
345 // triggered, it shows that scaling would happen after framerate
346 // adjusting.
TEST_F(QualityScalerTest,ChangeFramerateAndSpatialSize)347 TEST_F(QualityScalerTest, ChangeFramerateAndSpatialSize) {
348 VerifyQualityAdaptation(kFramerate, 10, true, true);
349 }
350
351 // When starting from a low framerate, only spatial size will be changed.
TEST_F(QualityScalerTest,ChangeSpatialSizeOnly)352 TEST_F(QualityScalerTest, ChangeSpatialSizeOnly) {
353 qs_.ReportFramerate(kFramerate >> 1);
354 VerifyQualityAdaptation(kFramerate >> 1, 10, true, false);
355 }
356
TEST_F(QualityScalerTest,DoesNotDownscaleBelow2xDefaultMinDimensionsWidth)357 TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsWidth) {
358 DoesNotDownscaleFrameDimensions(
359 2 * QualityScaler::kDefaultMinDownscaleDimension - 1, 1000);
360 }
361
TEST_F(QualityScalerTest,DoesNotDownscaleBelow2xDefaultMinDimensionsHeight)362 TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsHeight) {
363 DoesNotDownscaleFrameDimensions(
364 1000, 2 * QualityScaler::kDefaultMinDownscaleDimension - 1);
365 }
366
DownscaleEndsAt(int input_width,int input_height,int end_width,int end_height)367 void QualityScalerTest::DownscaleEndsAt(int input_width,
368 int input_height,
369 int end_width,
370 int end_height) {
371 // Create a frame with 2x expected end width/height to verify that we can
372 // scale down to expected end width/height.
373 input_frame_.CreateEmptyFrame(input_width, input_height, input_width,
374 (input_width + 1) / 2, (input_width + 1) / 2);
375
376 int last_width = input_width;
377 int last_height = input_height;
378 // Drop all frames to force-trigger downscaling.
379 while (true) {
380 TriggerScale(kScaleDown);
381 QualityScaler::Resolution res = qs_.GetScaledResolution();
382 if (last_width == res.width) {
383 EXPECT_EQ(last_height, res.height);
384 EXPECT_EQ(end_width, res.width);
385 EXPECT_EQ(end_height, res.height);
386 break;
387 }
388 last_width = res.width;
389 last_height = res.height;
390 }
391 }
392
TEST_F(QualityScalerTest,DefaultDownscalesTo160x90)393 TEST_F(QualityScalerTest, DefaultDownscalesTo160x90) {
394 DownscaleEndsAt(320, 180, 160, 90);
395 }
396
TEST_F(QualityScalerTest,DefaultDownscalesTo90x160)397 TEST_F(QualityScalerTest, DefaultDownscalesTo90x160) {
398 DownscaleEndsAt(180, 320, 90, 160);
399 }
400
TEST_F(QualityScalerTest,DefaultDownscalesFrom1280x720To160x90)401 TEST_F(QualityScalerTest, DefaultDownscalesFrom1280x720To160x90) {
402 DownscaleEndsAt(1280, 720, 160, 90);
403 }
404
TEST_F(QualityScalerTest,DefaultDoesntDownscaleBelow160x90)405 TEST_F(QualityScalerTest, DefaultDoesntDownscaleBelow160x90) {
406 DownscaleEndsAt(320 - 1, 180 - 1, 320 - 1, 180 - 1);
407 }
408
TEST_F(QualityScalerTest,DefaultDoesntDownscaleBelow90x160)409 TEST_F(QualityScalerTest, DefaultDoesntDownscaleBelow90x160) {
410 DownscaleEndsAt(180 - 1, 320 - 1, 180 - 1, 320 - 1);
411 }
412
TEST_F(QualityScalerTest,RespectsMinResolutionWidth)413 TEST_F(QualityScalerTest, RespectsMinResolutionWidth) {
414 // Should end at 200x100, as width can't go lower.
415 qs_.SetMinResolution(200, 10);
416 DownscaleEndsAt(1600, 800, 200, 100);
417 }
418
TEST_F(QualityScalerTest,RespectsMinResolutionHeight)419 TEST_F(QualityScalerTest, RespectsMinResolutionHeight) {
420 // Should end at 100x200, as height can't go lower.
421 qs_.SetMinResolution(10, 200);
422 DownscaleEndsAt(800, 1600, 100, 200);
423 }
424
425 } // namespace webrtc
426