1 /*
2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "rtc_tools/frame_analyzer/video_geometry_aligner.h"
12
13 #include <map>
14
15 #include "api/video/i420_buffer.h"
16 #include "rtc_base/checks.h"
17 #include "rtc_base/ref_counted_object.h"
18 #include "rtc_tools/frame_analyzer/video_quality_analysis.h"
19 #include "third_party/libyuv/include/libyuv/scale.h"
20
21 namespace webrtc {
22 namespace test {
23
24 namespace {
25
IsValidRegion(const CropRegion & region,const rtc::scoped_refptr<I420BufferInterface> & frame)26 bool IsValidRegion(const CropRegion& region,
27 const rtc::scoped_refptr<I420BufferInterface>& frame) {
28 return region.left >= 0 && region.right >= 0 && region.top >= 0 &&
29 region.bottom >= 0 && region.left + region.right < frame->width() &&
30 region.top + region.bottom < frame->height();
31 }
32
33 } // namespace
34
CropAndZoom(const CropRegion & crop_region,const rtc::scoped_refptr<I420BufferInterface> & frame)35 rtc::scoped_refptr<I420BufferInterface> CropAndZoom(
36 const CropRegion& crop_region,
37 const rtc::scoped_refptr<I420BufferInterface>& frame) {
38 RTC_CHECK(IsValidRegion(crop_region, frame));
39
40 const int uv_crop_left = crop_region.left / 2;
41 const int uv_crop_top = crop_region.top / 2;
42
43 const int cropped_width =
44 frame->width() - crop_region.left - crop_region.right;
45 const int cropped_height =
46 frame->height() - crop_region.top - crop_region.bottom;
47
48 // Crop by only adjusting pointers.
49 const uint8_t* y_plane =
50 frame->DataY() + frame->StrideY() * crop_region.top + crop_region.left;
51 const uint8_t* u_plane =
52 frame->DataU() + frame->StrideU() * uv_crop_top + uv_crop_left;
53 const uint8_t* v_plane =
54 frame->DataV() + frame->StrideV() * uv_crop_top + uv_crop_left;
55
56 // Stretch the cropped frame to the original size using libyuv.
57 rtc::scoped_refptr<I420Buffer> adjusted_frame =
58 I420Buffer::Create(frame->width(), frame->height());
59 libyuv::I420Scale(y_plane, frame->StrideY(), u_plane, frame->StrideU(),
60 v_plane, frame->StrideV(), cropped_width, cropped_height,
61 adjusted_frame->MutableDataY(), adjusted_frame->StrideY(),
62 adjusted_frame->MutableDataU(), adjusted_frame->StrideU(),
63 adjusted_frame->MutableDataV(), adjusted_frame->StrideV(),
64 frame->width(), frame->height(), libyuv::kFilterBilinear);
65
66 return adjusted_frame;
67 }
68
CalculateCropRegion(const rtc::scoped_refptr<I420BufferInterface> & reference_frame,const rtc::scoped_refptr<I420BufferInterface> & test_frame)69 CropRegion CalculateCropRegion(
70 const rtc::scoped_refptr<I420BufferInterface>& reference_frame,
71 const rtc::scoped_refptr<I420BufferInterface>& test_frame) {
72 RTC_CHECK_EQ(reference_frame->width(), test_frame->width());
73 RTC_CHECK_EQ(reference_frame->height(), test_frame->height());
74
75 CropRegion best_region;
76 double best_ssim = Ssim(reference_frame, test_frame);
77
78 typedef int CropRegion::*CropParameter;
79 CropParameter crop_parameters[4] = {&CropRegion::left, &CropRegion::top,
80 &CropRegion::right, &CropRegion::bottom};
81
82 while (true) {
83 // Find the parameter in which direction SSIM improves the most.
84 CropParameter best_parameter = nullptr;
85 const CropRegion prev_best_region = best_region;
86
87 for (CropParameter crop_parameter : crop_parameters) {
88 CropRegion test_region = prev_best_region;
89 ++(test_region.*crop_parameter);
90
91 if (!IsValidRegion(test_region, reference_frame))
92 continue;
93
94 const double ssim =
95 Ssim(CropAndZoom(test_region, reference_frame), test_frame);
96
97 if (ssim > best_ssim) {
98 best_ssim = ssim;
99 best_parameter = crop_parameter;
100 best_region = test_region;
101 }
102 }
103
104 // No improvement among any direction, stop iteration.
105 if (best_parameter == nullptr)
106 break;
107
108 // Iterate in the best direction as long as it improves SSIM.
109 for (CropRegion test_region = best_region;
110 IsValidRegion(test_region, reference_frame);
111 ++(test_region.*best_parameter)) {
112 const double ssim =
113 Ssim(CropAndZoom(test_region, reference_frame), test_frame);
114 if (ssim <= best_ssim)
115 break;
116
117 best_ssim = ssim;
118 best_region = test_region;
119 }
120 }
121
122 return best_region;
123 }
124
AdjustCropping(const rtc::scoped_refptr<I420BufferInterface> & reference_frame,const rtc::scoped_refptr<I420BufferInterface> & test_frame)125 rtc::scoped_refptr<I420BufferInterface> AdjustCropping(
126 const rtc::scoped_refptr<I420BufferInterface>& reference_frame,
127 const rtc::scoped_refptr<I420BufferInterface>& test_frame) {
128 return CropAndZoom(CalculateCropRegion(reference_frame, test_frame),
129 reference_frame);
130 }
131
AdjustCropping(const rtc::scoped_refptr<Video> & reference_video,const rtc::scoped_refptr<Video> & test_video)132 rtc::scoped_refptr<Video> AdjustCropping(
133 const rtc::scoped_refptr<Video>& reference_video,
134 const rtc::scoped_refptr<Video>& test_video) {
135 class CroppedVideo : public rtc::RefCountedObject<Video> {
136 public:
137 CroppedVideo(const rtc::scoped_refptr<Video>& reference_video,
138 const rtc::scoped_refptr<Video>& test_video)
139 : reference_video_(reference_video), test_video_(test_video) {
140 RTC_CHECK_EQ(reference_video->number_of_frames(),
141 test_video->number_of_frames());
142 RTC_CHECK_EQ(reference_video->width(), test_video->width());
143 RTC_CHECK_EQ(reference_video->height(), test_video->height());
144 }
145
146 int width() const override { return test_video_->width(); }
147 int height() const override { return test_video_->height(); }
148 size_t number_of_frames() const override {
149 return test_video_->number_of_frames();
150 }
151
152 rtc::scoped_refptr<I420BufferInterface> GetFrame(
153 size_t index) const override {
154 const rtc::scoped_refptr<I420BufferInterface> reference_frame =
155 reference_video_->GetFrame(index);
156
157 // Only calculate cropping region once per frame since it's expensive.
158 if (!crop_regions_.count(index)) {
159 crop_regions_[index] =
160 CalculateCropRegion(reference_frame, test_video_->GetFrame(index));
161 }
162
163 return CropAndZoom(crop_regions_[index], reference_frame);
164 }
165
166 private:
167 const rtc::scoped_refptr<Video> reference_video_;
168 const rtc::scoped_refptr<Video> test_video_;
169 // Mutable since this is a cache that affects performance and not logical
170 // behavior.
171 mutable std::map<size_t, CropRegion> crop_regions_;
172 };
173
174 return new CroppedVideo(reference_video, test_video);
175 }
176
177 } // namespace test
178 } // namespace webrtc
179