1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "webrtc/modules/video_processing/main/source/content_analysis.h"
11
12 #include <math.h>
13 #include <stdlib.h>
14
15 #include "webrtc/system_wrappers/interface/cpu_features_wrapper.h"
16 #include "webrtc/system_wrappers/interface/tick_util.h"
17
18 namespace webrtc {
19
VPMContentAnalysis(bool runtime_cpu_detection)20 VPMContentAnalysis::VPMContentAnalysis(bool runtime_cpu_detection)
21 : orig_frame_(NULL),
22 prev_frame_(NULL),
23 width_(0),
24 height_(0),
25 skip_num_(1),
26 border_(8),
27 motion_magnitude_(0.0f),
28 spatial_pred_err_(0.0f),
29 spatial_pred_err_h_(0.0f),
30 spatial_pred_err_v_(0.0f),
31 first_frame_(true),
32 ca_Init_(false),
33 content_metrics_(NULL) {
34 ComputeSpatialMetrics = &VPMContentAnalysis::ComputeSpatialMetrics_C;
35 TemporalDiffMetric = &VPMContentAnalysis::TemporalDiffMetric_C;
36
37 if (runtime_cpu_detection) {
38 #if defined(WEBRTC_ARCH_X86_FAMILY)
39 if (WebRtc_GetCPUInfo(kSSE2)) {
40 ComputeSpatialMetrics = &VPMContentAnalysis::ComputeSpatialMetrics_SSE2;
41 TemporalDiffMetric = &VPMContentAnalysis::TemporalDiffMetric_SSE2;
42 }
43 #endif
44 }
45 Release();
46 }
47
~VPMContentAnalysis()48 VPMContentAnalysis::~VPMContentAnalysis() {
49 Release();
50 }
51
52
ComputeContentMetrics(const I420VideoFrame & inputFrame)53 VideoContentMetrics* VPMContentAnalysis::ComputeContentMetrics(
54 const I420VideoFrame& inputFrame) {
55 if (inputFrame.IsZeroSize())
56 return NULL;
57
58 // Init if needed (native dimension change).
59 if (width_ != inputFrame.width() || height_ != inputFrame.height()) {
60 if (VPM_OK != Initialize(inputFrame.width(), inputFrame.height()))
61 return NULL;
62 }
63 // Only interested in the Y plane.
64 orig_frame_ = inputFrame.buffer(kYPlane);
65
66 // Compute spatial metrics: 3 spatial prediction errors.
67 (this->*ComputeSpatialMetrics)();
68
69 // Compute motion metrics
70 if (first_frame_ == false)
71 ComputeMotionMetrics();
72
73 // Saving current frame as previous one: Y only.
74 memcpy(prev_frame_, orig_frame_, width_ * height_);
75
76 first_frame_ = false;
77 ca_Init_ = true;
78
79 return ContentMetrics();
80 }
81
Release()82 int32_t VPMContentAnalysis::Release() {
83 if (content_metrics_ != NULL) {
84 delete content_metrics_;
85 content_metrics_ = NULL;
86 }
87
88 if (prev_frame_ != NULL) {
89 delete [] prev_frame_;
90 prev_frame_ = NULL;
91 }
92
93 width_ = 0;
94 height_ = 0;
95 first_frame_ = true;
96
97 return VPM_OK;
98 }
99
Initialize(int width,int height)100 int32_t VPMContentAnalysis::Initialize(int width, int height) {
101 width_ = width;
102 height_ = height;
103 first_frame_ = true;
104
105 // skip parameter: # of skipped rows: for complexity reduction
106 // temporal also currently uses it for column reduction.
107 skip_num_ = 1;
108
109 // use skipNum = 2 for 4CIF, WHD
110 if ( (height_ >= 576) && (width_ >= 704) ) {
111 skip_num_ = 2;
112 }
113 // use skipNum = 4 for FULLL_HD images
114 if ( (height_ >= 1080) && (width_ >= 1920) ) {
115 skip_num_ = 4;
116 }
117
118 if (content_metrics_ != NULL) {
119 delete content_metrics_;
120 }
121
122 if (prev_frame_ != NULL) {
123 delete [] prev_frame_;
124 }
125
126 // Spatial Metrics don't work on a border of 8. Minimum processing
127 // block size is 16 pixels. So make sure the width and height support this.
128 if (width_ <= 32 || height_ <= 32) {
129 ca_Init_ = false;
130 return VPM_PARAMETER_ERROR;
131 }
132
133 content_metrics_ = new VideoContentMetrics();
134 if (content_metrics_ == NULL) {
135 return VPM_MEMORY;
136 }
137
138 prev_frame_ = new uint8_t[width_ * height_]; // Y only.
139 if (prev_frame_ == NULL) return VPM_MEMORY;
140
141 return VPM_OK;
142 }
143
144
145 // Compute motion metrics: magnitude over non-zero motion vectors,
146 // and size of zero cluster
ComputeMotionMetrics()147 int32_t VPMContentAnalysis::ComputeMotionMetrics() {
148 // Motion metrics: only one is derived from normalized
149 // (MAD) temporal difference
150 (this->*TemporalDiffMetric)();
151 return VPM_OK;
152 }
153
154 // Normalized temporal difference (MAD): used as a motion level metric
155 // Normalize MAD by spatial contrast: images with more contrast
156 // (pixel variance) likely have larger temporal difference
157 // To reduce complexity, we compute the metric for a reduced set of points.
TemporalDiffMetric_C()158 int32_t VPMContentAnalysis::TemporalDiffMetric_C() {
159 // size of original frame
160 int sizei = height_;
161 int sizej = width_;
162 uint32_t tempDiffSum = 0;
163 uint32_t pixelSum = 0;
164 uint64_t pixelSqSum = 0;
165
166 uint32_t num_pixels = 0; // Counter for # of pixels.
167 const int width_end = ((width_ - 2*border_) & -16) + border_;
168
169 for (int i = border_; i < sizei - border_; i += skip_num_) {
170 for (int j = border_; j < width_end; j++) {
171 num_pixels += 1;
172 int ssn = i * sizej + j;
173
174 uint8_t currPixel = orig_frame_[ssn];
175 uint8_t prevPixel = prev_frame_[ssn];
176
177 tempDiffSum += (uint32_t)abs((int16_t)(currPixel - prevPixel));
178 pixelSum += (uint32_t) currPixel;
179 pixelSqSum += (uint64_t) (currPixel * currPixel);
180 }
181 }
182
183 // Default.
184 motion_magnitude_ = 0.0f;
185
186 if (tempDiffSum == 0) return VPM_OK;
187
188 // Normalize over all pixels.
189 float const tempDiffAvg = (float)tempDiffSum / (float)(num_pixels);
190 float const pixelSumAvg = (float)pixelSum / (float)(num_pixels);
191 float const pixelSqSumAvg = (float)pixelSqSum / (float)(num_pixels);
192 float contrast = pixelSqSumAvg - (pixelSumAvg * pixelSumAvg);
193
194 if (contrast > 0.0) {
195 contrast = sqrt(contrast);
196 motion_magnitude_ = tempDiffAvg/contrast;
197 }
198 return VPM_OK;
199 }
200
201 // Compute spatial metrics:
202 // To reduce complexity, we compute the metric for a reduced set of points.
203 // The spatial metrics are rough estimates of the prediction error cost for
204 // each QM spatial mode: 2x2,1x2,2x1
205 // The metrics are a simple estimate of the up-sampling prediction error,
206 // estimated assuming sub-sampling for decimation (no filtering),
207 // and up-sampling back up with simple bilinear interpolation.
ComputeSpatialMetrics_C()208 int32_t VPMContentAnalysis::ComputeSpatialMetrics_C() {
209 const int sizei = height_;
210 const int sizej = width_;
211
212 // Pixel mean square average: used to normalize the spatial metrics.
213 uint32_t pixelMSA = 0;
214
215 uint32_t spatialErrSum = 0;
216 uint32_t spatialErrVSum = 0;
217 uint32_t spatialErrHSum = 0;
218
219 // make sure work section is a multiple of 16
220 const int width_end = ((sizej - 2*border_) & -16) + border_;
221
222 for (int i = border_; i < sizei - border_; i += skip_num_) {
223 for (int j = border_; j < width_end; j++) {
224 int ssn1= i * sizej + j;
225 int ssn2 = (i + 1) * sizej + j; // bottom
226 int ssn3 = (i - 1) * sizej + j; // top
227 int ssn4 = i * sizej + j + 1; // right
228 int ssn5 = i * sizej + j - 1; // left
229
230 uint16_t refPixel1 = orig_frame_[ssn1] << 1;
231 uint16_t refPixel2 = orig_frame_[ssn1] << 2;
232
233 uint8_t bottPixel = orig_frame_[ssn2];
234 uint8_t topPixel = orig_frame_[ssn3];
235 uint8_t rightPixel = orig_frame_[ssn4];
236 uint8_t leftPixel = orig_frame_[ssn5];
237
238 spatialErrSum += (uint32_t) abs((int16_t)(refPixel2
239 - (uint16_t)(bottPixel + topPixel + leftPixel + rightPixel)));
240 spatialErrVSum += (uint32_t) abs((int16_t)(refPixel1
241 - (uint16_t)(bottPixel + topPixel)));
242 spatialErrHSum += (uint32_t) abs((int16_t)(refPixel1
243 - (uint16_t)(leftPixel + rightPixel)));
244 pixelMSA += orig_frame_[ssn1];
245 }
246 }
247
248 // Normalize over all pixels.
249 const float spatialErr = (float)(spatialErrSum >> 2);
250 const float spatialErrH = (float)(spatialErrHSum >> 1);
251 const float spatialErrV = (float)(spatialErrVSum >> 1);
252 const float norm = (float)pixelMSA;
253
254 // 2X2:
255 spatial_pred_err_ = spatialErr / norm;
256 // 1X2:
257 spatial_pred_err_h_ = spatialErrH / norm;
258 // 2X1:
259 spatial_pred_err_v_ = spatialErrV / norm;
260 return VPM_OK;
261 }
262
ContentMetrics()263 VideoContentMetrics* VPMContentAnalysis::ContentMetrics() {
264 if (ca_Init_ == false) return NULL;
265
266 content_metrics_->spatial_pred_err = spatial_pred_err_;
267 content_metrics_->spatial_pred_err_h = spatial_pred_err_h_;
268 content_metrics_->spatial_pred_err_v = spatial_pred_err_v_;
269 // Motion metric: normalized temporal difference (MAD).
270 content_metrics_->motion_magnitude = motion_magnitude_;
271
272 return content_metrics_;
273 }
274
275 } // namespace webrtc
276