1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/modules/video_processing/test/video_processing_unittest.h"
12
13 #include <gflags/gflags.h>
14
15 #include <string>
16
17 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
18 #include "webrtc/system_wrappers/include/tick_util.h"
19 #include "webrtc/test/testsupport/fileutils.h"
20
21 namespace webrtc {
22
23 namespace {
24
25 // Define command line flag 'gen_files' (default value: false).
26 DEFINE_bool(gen_files, false, "Output files for visual inspection.");
27
28 } // namespace
29
30 static void PreprocessFrameAndVerify(const VideoFrame& source,
31 int target_width,
32 int target_height,
33 VideoProcessing* vpm,
34 const VideoFrame* out_frame);
35 static void CropFrame(const uint8_t* source_data,
36 int source_width,
37 int source_height,
38 int offset_x,
39 int offset_y,
40 int cropped_width,
41 int cropped_height,
42 VideoFrame* cropped_frame);
43 // The |source_data| is cropped and scaled to |target_width| x |target_height|,
44 // and then scaled back to the expected cropped size. |expected_psnr| is used to
45 // verify basic quality, and is set to be ~0.1/0.05dB lower than actual PSNR
46 // verified under the same conditions.
47 static void TestSize(const VideoFrame& source_frame,
48 const VideoFrame& cropped_source_frame,
49 int target_width,
50 int target_height,
51 double expected_psnr,
52 VideoProcessing* vpm);
53 static bool CompareFrames(const webrtc::VideoFrame& frame1,
54 const webrtc::VideoFrame& frame2);
55 static void WriteProcessedFrameForVisualInspection(const VideoFrame& source,
56 const VideoFrame& processed);
57
VideoProcessingTest()58 VideoProcessingTest::VideoProcessingTest()
59 : vp_(NULL),
60 source_file_(NULL),
61 width_(352),
62 half_width_((width_ + 1) / 2),
63 height_(288),
64 size_y_(width_ * height_),
65 size_uv_(half_width_ * ((height_ + 1) / 2)),
66 frame_length_(CalcBufferSize(kI420, width_, height_)) {}
67
SetUp()68 void VideoProcessingTest::SetUp() {
69 vp_ = VideoProcessing::Create();
70 ASSERT_TRUE(vp_ != NULL);
71
72 ASSERT_EQ(0, video_frame_.CreateEmptyFrame(width_, height_, width_,
73 half_width_, half_width_));
74 // Clear video frame so DrMemory/Valgrind will allow reads of the buffer.
75 memset(video_frame_.buffer(kYPlane), 0, video_frame_.allocated_size(kYPlane));
76 memset(video_frame_.buffer(kUPlane), 0, video_frame_.allocated_size(kUPlane));
77 memset(video_frame_.buffer(kVPlane), 0, video_frame_.allocated_size(kVPlane));
78 const std::string video_file =
79 webrtc::test::ResourcePath("foreman_cif", "yuv");
80 source_file_ = fopen(video_file.c_str(), "rb");
81 ASSERT_TRUE(source_file_ != NULL)
82 << "Cannot read source file: " + video_file + "\n";
83 }
84
TearDown()85 void VideoProcessingTest::TearDown() {
86 if (source_file_ != NULL) {
87 ASSERT_EQ(0, fclose(source_file_));
88 }
89 source_file_ = NULL;
90 delete vp_;
91 vp_ = NULL;
92 }
93
94 #if defined(WEBRTC_IOS)
TEST_F(VideoProcessingTest,DISABLED_HandleNullBuffer)95 TEST_F(VideoProcessingTest, DISABLED_HandleNullBuffer) {
96 #else
97 TEST_F(VideoProcessingTest, HandleNullBuffer) {
98 #endif
99 // TODO(mikhal/stefan): Do we need this one?
100 VideoProcessing::FrameStats stats;
101 // Video frame with unallocated buffer.
102 VideoFrame videoFrame;
103
104 vp_->GetFrameStats(videoFrame, &stats);
105 EXPECT_EQ(stats.num_pixels, 0u);
106
107 EXPECT_EQ(-1, vp_->Deflickering(&videoFrame, &stats));
108
109 EXPECT_EQ(-3, vp_->BrightnessDetection(videoFrame, stats));
110 }
111
112 #if defined(WEBRTC_IOS)
113 TEST_F(VideoProcessingTest, DISABLED_HandleBadStats) {
114 #else
115 TEST_F(VideoProcessingTest, HandleBadStats) {
116 #endif
117 VideoProcessing::FrameStats stats;
118 vp_->ClearFrameStats(&stats);
119 rtc::scoped_ptr<uint8_t[]> video_buffer(new uint8_t[frame_length_]);
120 ASSERT_EQ(frame_length_,
121 fread(video_buffer.get(), 1, frame_length_, source_file_));
122 EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, width_, height_,
123 0, kVideoRotation_0, &video_frame_));
124
125 EXPECT_EQ(-1, vp_->Deflickering(&video_frame_, &stats));
126
127 EXPECT_EQ(-3, vp_->BrightnessDetection(video_frame_, stats));
128 }
129
130 #if defined(WEBRTC_IOS)
131 TEST_F(VideoProcessingTest, DISABLED_IdenticalResultsAfterReset) {
132 #else
133 TEST_F(VideoProcessingTest, IdenticalResultsAfterReset) {
134 #endif
135 VideoFrame video_frame2;
136 VideoProcessing::FrameStats stats;
137 // Only testing non-static functions here.
138 rtc::scoped_ptr<uint8_t[]> video_buffer(new uint8_t[frame_length_]);
139 ASSERT_EQ(frame_length_,
140 fread(video_buffer.get(), 1, frame_length_, source_file_));
141 EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, width_, height_,
142 0, kVideoRotation_0, &video_frame_));
143 vp_->GetFrameStats(video_frame_, &stats);
144 EXPECT_GT(stats.num_pixels, 0u);
145 ASSERT_EQ(0, video_frame2.CopyFrame(video_frame_));
146 ASSERT_EQ(0, vp_->Deflickering(&video_frame_, &stats));
147
148 // Retrieve frame stats again in case Deflickering() has zeroed them.
149 vp_->GetFrameStats(video_frame2, &stats);
150 EXPECT_GT(stats.num_pixels, 0u);
151 ASSERT_EQ(0, vp_->Deflickering(&video_frame2, &stats));
152 EXPECT_TRUE(CompareFrames(video_frame_, video_frame2));
153
154 ASSERT_EQ(frame_length_,
155 fread(video_buffer.get(), 1, frame_length_, source_file_));
156 EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, width_, height_,
157 0, kVideoRotation_0, &video_frame_));
158 vp_->GetFrameStats(video_frame_, &stats);
159 EXPECT_GT(stats.num_pixels, 0u);
160 video_frame2.CopyFrame(video_frame_);
161 ASSERT_EQ(0, vp_->BrightnessDetection(video_frame_, stats));
162
163 ASSERT_EQ(0, vp_->BrightnessDetection(video_frame2, stats));
164 EXPECT_TRUE(CompareFrames(video_frame_, video_frame2));
165 }
166
167 #if defined(WEBRTC_IOS)
168 TEST_F(VideoProcessingTest, DISABLED_FrameStats) {
169 #else
170 TEST_F(VideoProcessingTest, FrameStats) {
171 #endif
172 VideoProcessing::FrameStats stats;
173 vp_->ClearFrameStats(&stats);
174 rtc::scoped_ptr<uint8_t[]> video_buffer(new uint8_t[frame_length_]);
175 ASSERT_EQ(frame_length_,
176 fread(video_buffer.get(), 1, frame_length_, source_file_));
177 EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, width_, height_,
178 0, kVideoRotation_0, &video_frame_));
179
180 EXPECT_FALSE(vp_->ValidFrameStats(stats));
181 vp_->GetFrameStats(video_frame_, &stats);
182 EXPECT_GT(stats.num_pixels, 0u);
183 EXPECT_TRUE(vp_->ValidFrameStats(stats));
184
185 printf("\nFrameStats\n");
186 printf("mean: %u\nnum_pixels: %u\nsubSamplFactor: %u\nsum: %u\n\n",
187 static_cast<unsigned int>(stats.mean),
188 static_cast<unsigned int>(stats.num_pixels),
189 static_cast<unsigned int>(stats.sub_sampling_factor),
190 static_cast<unsigned int>(stats.sum));
191
192 vp_->ClearFrameStats(&stats);
193 EXPECT_FALSE(vp_->ValidFrameStats(stats));
194 }
195
196 #if defined(WEBRTC_IOS)
197 TEST_F(VideoProcessingTest, DISABLED_PreprocessorLogic) {
198 #else
199 TEST_F(VideoProcessingTest, PreprocessorLogic) {
200 #endif
201 // Disable temporal sampling (frame dropping).
202 vp_->EnableTemporalDecimation(false);
203 int resolution = 100;
204 EXPECT_EQ(VPM_OK, vp_->SetTargetResolution(resolution, resolution, 15));
205 EXPECT_EQ(VPM_OK, vp_->SetTargetResolution(resolution, resolution, 30));
206 // Disable spatial sampling.
207 vp_->SetInputFrameResampleMode(kNoRescaling);
208 EXPECT_EQ(VPM_OK, vp_->SetTargetResolution(resolution, resolution, 30));
209 VideoFrame* out_frame = NULL;
210 // Set rescaling => output frame != NULL.
211 vp_->SetInputFrameResampleMode(kFastRescaling);
212 PreprocessFrameAndVerify(video_frame_, resolution, resolution, vp_,
213 out_frame);
214 // No rescaling=> output frame = NULL.
215 vp_->SetInputFrameResampleMode(kNoRescaling);
216 EXPECT_TRUE(vp_->PreprocessFrame(video_frame_) != nullptr);
217 }
218
219 #if defined(WEBRTC_IOS)
220 TEST_F(VideoProcessingTest, DISABLED_Resampler) {
221 #else
222 TEST_F(VideoProcessingTest, Resampler) {
223 #endif
224 enum { NumRuns = 1 };
225
226 int64_t min_runtime = 0;
227 int64_t total_runtime = 0;
228
229 rewind(source_file_);
230 ASSERT_TRUE(source_file_ != NULL) << "Cannot read input file \n";
231
232 // CA not needed here
233 vp_->EnableContentAnalysis(false);
234 // no temporal decimation
235 vp_->EnableTemporalDecimation(false);
236
237 // Reading test frame
238 rtc::scoped_ptr<uint8_t[]> video_buffer(new uint8_t[frame_length_]);
239 ASSERT_EQ(frame_length_,
240 fread(video_buffer.get(), 1, frame_length_, source_file_));
241 // Using ConvertToI420 to add stride to the image.
242 EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, width_, height_,
243 0, kVideoRotation_0, &video_frame_));
244 // Cropped source frame that will contain the expected visible region.
245 VideoFrame cropped_source_frame;
246 cropped_source_frame.CopyFrame(video_frame_);
247
248 for (uint32_t run_idx = 0; run_idx < NumRuns; run_idx++) {
249 // Initiate test timer.
250 const TickTime time_start = TickTime::Now();
251
252 // Init the sourceFrame with a timestamp.
253 video_frame_.set_render_time_ms(time_start.MillisecondTimestamp());
254 video_frame_.set_timestamp(time_start.MillisecondTimestamp() * 90);
255
256 // Test scaling to different sizes: source is of |width|/|height| = 352/288.
257 // Pure scaling:
258 TestSize(video_frame_, video_frame_, width_ / 4, height_ / 4, 25.2, vp_);
259 TestSize(video_frame_, video_frame_, width_ / 2, height_ / 2, 28.1, vp_);
260 // No resampling:
261 TestSize(video_frame_, video_frame_, width_, height_, -1, vp_);
262 TestSize(video_frame_, video_frame_, 2 * width_, 2 * height_, 32.2, vp_);
263
264 // Scaling and cropping. The cropped source frame is the largest center
265 // aligned region that can be used from the source while preserving aspect
266 // ratio.
267 CropFrame(video_buffer.get(), width_, height_, 0, 56, 352, 176,
268 &cropped_source_frame);
269 TestSize(video_frame_, cropped_source_frame, 100, 50, 24.0, vp_);
270
271 CropFrame(video_buffer.get(), width_, height_, 0, 30, 352, 225,
272 &cropped_source_frame);
273 TestSize(video_frame_, cropped_source_frame, 400, 256, 31.3, vp_);
274
275 CropFrame(video_buffer.get(), width_, height_, 68, 0, 216, 288,
276 &cropped_source_frame);
277 TestSize(video_frame_, cropped_source_frame, 480, 640, 32.15, vp_);
278
279 CropFrame(video_buffer.get(), width_, height_, 0, 12, 352, 264,
280 &cropped_source_frame);
281 TestSize(video_frame_, cropped_source_frame, 960, 720, 32.2, vp_);
282
283 CropFrame(video_buffer.get(), width_, height_, 0, 44, 352, 198,
284 &cropped_source_frame);
285 TestSize(video_frame_, cropped_source_frame, 1280, 720, 32.15, vp_);
286
287 // Upsampling to odd size.
288 CropFrame(video_buffer.get(), width_, height_, 0, 26, 352, 233,
289 &cropped_source_frame);
290 TestSize(video_frame_, cropped_source_frame, 501, 333, 32.05, vp_);
291 // Downsample to odd size.
292 CropFrame(video_buffer.get(), width_, height_, 0, 34, 352, 219,
293 &cropped_source_frame);
294 TestSize(video_frame_, cropped_source_frame, 281, 175, 29.3, vp_);
295
296 // Stop timer.
297 const int64_t runtime = (TickTime::Now() - time_start).Microseconds();
298 if (runtime < min_runtime || run_idx == 0) {
299 min_runtime = runtime;
300 }
301 total_runtime += runtime;
302 }
303
304 printf("\nAverage run time = %d us / frame\n",
305 static_cast<int>(total_runtime));
306 printf("Min run time = %d us / frame\n\n", static_cast<int>(min_runtime));
307 }
308
309 void PreprocessFrameAndVerify(const VideoFrame& source,
310 int target_width,
311 int target_height,
312 VideoProcessing* vpm,
313 const VideoFrame* out_frame) {
314 ASSERT_EQ(VPM_OK, vpm->SetTargetResolution(target_width, target_height, 30));
315 out_frame = vpm->PreprocessFrame(source);
316 EXPECT_TRUE(out_frame != nullptr);
317
318 // If no resizing is needed, expect the original frame.
319 if (target_width == source.width() && target_height == source.height()) {
320 EXPECT_EQ(&source, out_frame);
321 return;
322 }
323
324 // Verify the resampled frame.
325 EXPECT_TRUE(out_frame != NULL);
326 EXPECT_EQ(source.render_time_ms(), (out_frame)->render_time_ms());
327 EXPECT_EQ(source.timestamp(), (out_frame)->timestamp());
328 EXPECT_EQ(target_width, (out_frame)->width());
329 EXPECT_EQ(target_height, (out_frame)->height());
330 }
331
332 void CropFrame(const uint8_t* source_data,
333 int source_width,
334 int source_height,
335 int offset_x,
336 int offset_y,
337 int cropped_width,
338 int cropped_height,
339 VideoFrame* cropped_frame) {
340 cropped_frame->CreateEmptyFrame(cropped_width, cropped_height, cropped_width,
341 (cropped_width + 1) / 2,
342 (cropped_width + 1) / 2);
343 EXPECT_EQ(0,
344 ConvertToI420(kI420, source_data, offset_x, offset_y, source_width,
345 source_height, 0, kVideoRotation_0, cropped_frame));
346 }
347
348 void TestSize(const VideoFrame& source_frame,
349 const VideoFrame& cropped_source_frame,
350 int target_width,
351 int target_height,
352 double expected_psnr,
353 VideoProcessing* vpm) {
354 // Resample source_frame to out_frame.
355 VideoFrame* out_frame = NULL;
356 vpm->SetInputFrameResampleMode(kBox);
357 PreprocessFrameAndVerify(source_frame, target_width, target_height, vpm,
358 out_frame);
359 if (out_frame == NULL)
360 return;
361 WriteProcessedFrameForVisualInspection(source_frame, *out_frame);
362
363 // Scale |resampled_source_frame| back to the source scale.
364 VideoFrame resampled_source_frame;
365 resampled_source_frame.CopyFrame(*out_frame);
366 PreprocessFrameAndVerify(resampled_source_frame, cropped_source_frame.width(),
367 cropped_source_frame.height(), vpm, out_frame);
368 WriteProcessedFrameForVisualInspection(resampled_source_frame, *out_frame);
369
370 // Compute PSNR against the cropped source frame and check expectation.
371 double psnr = I420PSNR(&cropped_source_frame, out_frame);
372 EXPECT_GT(psnr, expected_psnr);
373 printf(
374 "PSNR: %f. PSNR is between source of size %d %d, and a modified "
375 "source which is scaled down/up to: %d %d, and back to source size \n",
376 psnr, source_frame.width(), source_frame.height(), target_width,
377 target_height);
378 }
379
380 bool CompareFrames(const webrtc::VideoFrame& frame1,
381 const webrtc::VideoFrame& frame2) {
382 for (int plane = 0; plane < webrtc::kNumOfPlanes; plane++) {
383 webrtc::PlaneType plane_type = static_cast<webrtc::PlaneType>(plane);
384 int allocated_size1 = frame1.allocated_size(plane_type);
385 int allocated_size2 = frame2.allocated_size(plane_type);
386 if (allocated_size1 != allocated_size2)
387 return false;
388 const uint8_t* plane_buffer1 = frame1.buffer(plane_type);
389 const uint8_t* plane_buffer2 = frame2.buffer(plane_type);
390 if (memcmp(plane_buffer1, plane_buffer2, allocated_size1))
391 return false;
392 }
393 return true;
394 }
395
396 void WriteProcessedFrameForVisualInspection(const VideoFrame& source,
397 const VideoFrame& processed) {
398 // Skip if writing to files is not enabled.
399 if (!FLAGS_gen_files)
400 return;
401 // Write the processed frame to file for visual inspection.
402 std::ostringstream filename;
403 filename << webrtc::test::OutputPath() << "Resampler_from_" << source.width()
404 << "x" << source.height() << "_to_" << processed.width() << "x"
405 << processed.height() << "_30Hz_P420.yuv";
406 std::cout << "Watch " << filename.str() << " and verify that it is okay."
407 << std::endl;
408 FILE* stand_alone_file = fopen(filename.str().c_str(), "wb");
409 if (PrintVideoFrame(processed, stand_alone_file) < 0)
410 std::cerr << "Failed to write: " << filename.str() << std::endl;
411 if (stand_alone_file)
412 fclose(stand_alone_file);
413 }
414
415 } // namespace webrtc
416