1 /*
2 * stablizer.cpp - abstract for DVS (Digital Video Stabilizer)
3 *
4 * Copyright (c) 2014-2016 Intel Corporation
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * Author: Zong Wei <wei.zong@intel.com>
19 */
20
21 #include "stabilizer.h"
22
23 using namespace cv;
24 using namespace cv::videostab;
25 using namespace std;
26
27 Mat
nextStabilizedMotion(DvsData * frame,int & stablizedPos)28 OnePassVideoStabilizer::nextStabilizedMotion(DvsData* frame, int& stablizedPos)
29 {
30 if (!(frame->data.empty()))
31 {
32 Mat image;
33 frame->data.getMat(ACCESS_READ).copyTo(image);
34
35 curPos_++;
36
37 if (curPos_ > 0)
38 {
39 at(curPos_, frames_) = image;
40
41 if (doDeblurring_)
42 at(curPos_, blurrinessRates_) = calcBlurriness(image);
43
44 at(curPos_ - 1, motions_) = estimateMotion();
45
46 if (curPos_ >= radius_)
47 {
48 curStabilizedPos_ = curPos_ - radius_;
49 Mat stabilizationMotion = estimateStabilizationMotion();
50 if (doCorrectionForInclusion_)
51 stabilizationMotion = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_);
52
53 at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion;
54 stablizedPos = curStabilizedPos_;
55
56 return stabilizationMotion;
57 }
58 }
59 else
60 setUpFrame(image);
61
62 log_->print(".");
63 return Mat();
64 }
65
66 if (curStabilizedPos_ < curPos_)
67 {
68 curStabilizedPos_++;
69 stablizedPos = curStabilizedPos_;
70 at(curStabilizedPos_ + radius_, frames_) = at(curPos_, frames_);
71 at(curStabilizedPos_ + radius_ - 1, motions_) = Mat::eye(3, 3, CV_32F);
72
73 Mat stabilizationMotion = estimateStabilizationMotion();
74 if (doCorrectionForInclusion_)
75 stabilizationMotion = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_);
76
77 at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion;
78
79 log_->print(".");
80
81 return stabilizationMotion;
82 }
83
84 return Mat();
85 }
86
87
88 Mat
estimateMotion()89 OnePassVideoStabilizer::estimateMotion()
90 {
91 #if ENABLE_DVS_CL_PATH
92 cv::UMat frame0 = at(curPos_ - 1, frames_).getUMat(ACCESS_READ);
93 cv::UMat frame1 = at(curPos_, frames_).getUMat(ACCESS_READ);
94
95 cv::UMat ugrayImage0;
96 cv::UMat ugrayImage1;
97 if ( frame0.type() != CV_8U )
98 {
99 cvtColor( frame0, ugrayImage0, COLOR_BGR2GRAY );
100 }
101 else
102 {
103 ugrayImage0 = frame0;
104 }
105
106 if ( frame1.type() != CV_8U )
107 {
108 cvtColor( frame1, ugrayImage1, COLOR_BGR2GRAY );
109 }
110 else
111 {
112 ugrayImage1 = frame1;
113 }
114
115 return motionEstimator_.dynamicCast<KeypointBasedMotionEstimator>()->estimate(ugrayImage0, ugrayImage1);
116 #else
117 return motionEstimator_.dynamicCast<KeypointBasedMotionEstimator>()->estimate(at(curPos_ - 1, frames_), at(curPos_, frames_));
118 #endif
119 }
120
121 void
setUpFrame(const Mat & firstFrame)122 OnePassVideoStabilizer::setUpFrame(const Mat &firstFrame)
123 {
124 frameSize_ = firstFrame.size();
125 frameMask_.create(frameSize_, CV_8U);
126 frameMask_.setTo(255);
127
128 int cacheSize = 2 * radius_ + 1;
129 frames_.resize(2);
130 motions_.resize(cacheSize);
131 stabilizationMotions_.resize(cacheSize);
132
133 for (int i = -radius_; i < 0; ++i)
134 {
135 at(i, motions_) = Mat::eye(3, 3, CV_32F);
136 at(i, frames_) = firstFrame;
137 }
138
139 at(0, frames_) = firstFrame;
140
141 StabilizerBase::setUp(firstFrame);
142 }
143
144
145 Mat
nextStabilizedMotion(DvsData * frame,int & stablizedPos)146 TwoPassVideoStabilizer::nextStabilizedMotion(DvsData* frame, int& stablizedPos)
147 {
148 if (!(frame->data.empty()))
149 {
150 Mat image;
151 frame->data.getMat(ACCESS_READ).copyTo(image);
152
153 curPos_++;
154
155 if (curPos_ > 0)
156 {
157 at(curPos_, frames_) = image;
158
159 if (doDeblurring_)
160 at(curPos_, blurrinessRates_) = calcBlurriness(image);
161
162 at(curPos_ - 1, motions_) = estimateMotion();
163
164 if (curPos_ >= radius_)
165 {
166 curStabilizedPos_ = curPos_ - radius_;
167 Mat stabilizationMotion = estimateStabilizationMotion();
168 if (doCorrectionForInclusion_)
169 stabilizationMotion = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_);
170
171 at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion;
172 stablizedPos = curStabilizedPos_;
173
174 return stabilizationMotion;
175 }
176 }
177 else
178 setUpFrame(image);
179
180 log_->print(".");
181 return Mat();
182 }
183
184 if (curStabilizedPos_ < curPos_)
185 {
186 curStabilizedPos_++;
187 stablizedPos = curStabilizedPos_;
188 at(curStabilizedPos_ + radius_, frames_) = at(curPos_, frames_);
189 at(curStabilizedPos_ + radius_ - 1, motions_) = Mat::eye(3, 3, CV_32F);
190
191 Mat stabilizationMotion = estimateStabilizationMotion();
192 if (doCorrectionForInclusion_)
193 stabilizationMotion = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_);
194
195 at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion;
196
197 log_->print(".");
198
199 return stabilizationMotion;
200 }
201
202 return Mat();
203 }
204
205
206 Mat
estimateMotion()207 TwoPassVideoStabilizer::estimateMotion()
208 {
209 #if ENABLE_DVS_CL_PATH
210 cv::UMat frame0 = at(curPos_ - 1, frames_).getUMat(ACCESS_READ);
211 cv::UMat frame1 = at(curPos_, frames_).getUMat(ACCESS_READ);
212
213 cv::UMat ugrayImage0;
214 cv::UMat ugrayImage1;
215 if ( frame0.type() != CV_8U )
216 {
217 cvtColor( frame0, ugrayImage0, COLOR_BGR2GRAY );
218 }
219 else
220 {
221 ugrayImage0 = frame0;
222 }
223
224 if ( frame1.type() != CV_8U )
225 {
226 cvtColor( frame1, ugrayImage1, COLOR_BGR2GRAY );
227 }
228 else
229 {
230 ugrayImage1 = frame1;
231 }
232
233 return motionEstimator_.dynamicCast<KeypointBasedMotionEstimator>()->estimate(ugrayImage0, ugrayImage1);
234 #else
235 return motionEstimator_.dynamicCast<KeypointBasedMotionEstimator>()->estimate(at(curPos_ - 1, frames_), at(curPos_, frames_));
236 #endif
237 }
238
239 void
setUpFrame(const Mat & firstFrame)240 TwoPassVideoStabilizer::setUpFrame(const Mat &firstFrame)
241 {
242 //int cacheSize = 2*radius_ + 1;
243 frames_.resize(2);
244 stabilizedFrames_.resize(2);
245 stabilizedMasks_.resize(2);
246
247 for (int i = -1; i <= 0; ++i)
248 at(i, frames_) = firstFrame;
249
250 StabilizerBase::setUp(firstFrame);
251 }
252
VideoStabilizer(bool isTwoPass,bool wobbleSuppress,bool deblur,bool inpainter)253 VideoStabilizer::VideoStabilizer(
254 bool isTwoPass,
255 bool wobbleSuppress,
256 bool deblur,
257 bool inpainter)
258 : isTwoPass_ (isTwoPass)
259 , trimRatio_ (0.05f)
260 {
261 Ptr<MotionEstimatorRansacL2> est = makePtr<MotionEstimatorRansacL2>(MM_HOMOGRAPHY);
262 Ptr<IOutlierRejector> outlierRejector = makePtr<TranslationBasedLocalOutlierRejector>();
263 Ptr<KeypointBasedMotionEstimator> kbest = makePtr<KeypointBasedMotionEstimator>(est);
264 kbest->setDetector(GFTTDetector::create(1000, 0.01, 15));
265 kbest->setOutlierRejector(outlierRejector);
266
267 if (isTwoPass)
268 {
269 Ptr<TwoPassVideoStabilizer> twoPassStabilizer = makePtr<TwoPassVideoStabilizer>();
270 stabilizer_ = twoPassStabilizer;
271 twoPassStabilizer->setEstimateTrimRatio(false);
272 twoPassStabilizer->setMotionStabilizer(makePtr<GaussianMotionFilter>(15, 10));
273
274 if (wobbleSuppress) {
275 Ptr<MoreAccurateMotionWobbleSuppressorBase> ws = makePtr<MoreAccurateMotionWobbleSuppressor>();
276
277 ws->setMotionEstimator(kbest);
278 ws->setPeriod(30);
279 twoPassStabilizer->setWobbleSuppressor(ws);
280 }
281 } else {
282 Ptr<OnePassVideoStabilizer> onePassStabilizer = makePtr<OnePassVideoStabilizer>();
283 stabilizer_ = onePassStabilizer;
284 onePassStabilizer->setMotionFilter(makePtr<GaussianMotionFilter>(15, 10));
285 }
286
287 stabilizer_->setMotionEstimator(kbest);
288
289 stabilizer_->setRadius(15);
290
291 if (deblur)
292 {
293 Ptr<WeightingDeblurer> deblurrer = makePtr<WeightingDeblurer>();
294 deblurrer->setRadius(3);
295 deblurrer->setSensitivity(0.001f);
296 stabilizer_->setDeblurer(deblurrer);
297 }
298
299 if (inpainter)
300 {
301 bool inpaintMosaic = true;
302 bool inpaintColorAverage = true;
303 bool inpaintColorNs = false;
304 bool inpaintColorTelea = false;
305
306 // init inpainter
307 InpaintingPipeline *inpainters = new InpaintingPipeline();
308 Ptr<InpainterBase> inpainters_(inpainters);
309 if (true == inpaintMosaic)
310 {
311 Ptr<ConsistentMosaicInpainter> inp = makePtr<ConsistentMosaicInpainter>();
312 inp->setStdevThresh(10.0f);
313 inpainters->pushBack(inp);
314 }
315 if (true == inpaintColorAverage)
316 inpainters->pushBack(makePtr<ColorAverageInpainter>());
317 else if (true == inpaintColorNs)
318 inpainters->pushBack(makePtr<ColorInpainter>(0, 2));
319 else if (true == inpaintColorTelea)
320 inpainters->pushBack(makePtr<ColorInpainter>(1, 2));
321 if (!inpainters->empty())
322 {
323 inpainters->setRadius(2);
324 stabilizer_->setInpainter(inpainters_);
325 }
326 }
327 }
328
~VideoStabilizer()329 VideoStabilizer::~VideoStabilizer() {}
330
331 void
configFeatureDetector(int features,double minDistance)332 VideoStabilizer::configFeatureDetector(int features, double minDistance)
333 {
334 Ptr<ImageMotionEstimatorBase> estimator = stabilizer_->motionEstimator();
335 Ptr<FeatureDetector> detector = estimator.dynamicCast<KeypointBasedMotionEstimator>()->detector();
336 if (NULL == detector) {
337 return;
338 }
339
340 detector.dynamicCast<GFTTDetector>()->setMaxFeatures(features);
341 detector.dynamicCast<GFTTDetector>()->setMinDistance(minDistance);
342 }
343
344 void
configMotionFilter(int radius,float stdev)345 VideoStabilizer::configMotionFilter(int radius, float stdev)
346 {
347 if (isTwoPass_) {
348 Ptr<TwoPassVideoStabilizer> stab = stabilizer_.dynamicCast<TwoPassVideoStabilizer>();
349 Ptr<IMotionStabilizer> motionStabilizer = stab->motionStabilizer();
350 motionStabilizer.dynamicCast<GaussianMotionFilter>()->setParams(radius, stdev);
351 } else {
352 Ptr<OnePassVideoStabilizer> stab = stabilizer_.dynamicCast<OnePassVideoStabilizer>();
353 Ptr<MotionFilterBase> motionFilter = stab->motionFilter();
354 motionFilter.dynamicCast<GaussianMotionFilter>()->setParams(radius, stdev);
355 }
356 stabilizer_->setRadius(radius);
357 }
358
359 void
configDeblurrer(int radius,double sensitivity)360 VideoStabilizer::configDeblurrer(int radius, double sensitivity)
361 {
362 Ptr<DeblurerBase> deblurrer = stabilizer_->deblurrer();
363 if (NULL == deblurrer) {
364 return;
365 }
366
367 deblurrer->setRadius(radius);
368 deblurrer.dynamicCast<WeightingDeblurer>()->setSensitivity(sensitivity);
369 }
370
371 void
setFrameSize(Size frameSize)372 VideoStabilizer::setFrameSize(Size frameSize)
373 {
374 frameSize_ = frameSize;
375 }
376
377 Mat
nextFrame()378 VideoStabilizer::nextFrame()
379 {
380 if (isTwoPass_) {
381 Ptr<TwoPassVideoStabilizer> stab = stabilizer_.dynamicCast<TwoPassVideoStabilizer>();
382 return stab->nextFrame();
383 } else {
384 Ptr<OnePassVideoStabilizer> stab = stabilizer_.dynamicCast<OnePassVideoStabilizer>();
385 return stab->nextFrame();
386 }
387 }
388
389 Mat
nextStabilizedMotion(DvsData * frame,int & stablizedPos)390 VideoStabilizer::nextStabilizedMotion(DvsData* frame, int& stablizedPos)
391 {
392 Mat HMatrix;
393
394 if (isTwoPass_) {
395 Ptr<TwoPassVideoStabilizer> stab = stabilizer_.dynamicCast<TwoPassVideoStabilizer>();
396 HMatrix = stab->nextStabilizedMotion(frame, stablizedPos);
397 } else {
398 Ptr<OnePassVideoStabilizer> stab = stabilizer_.dynamicCast<OnePassVideoStabilizer>();
399 HMatrix = stab->nextStabilizedMotion(frame, stablizedPos);
400 }
401
402 return HMatrix;
403 }
404
405 Size
trimedVideoSize(Size frameSize)406 VideoStabilizer::trimedVideoSize(Size frameSize)
407 {
408 Size outputFrameSize;
409 outputFrameSize.width = ((int)((float)frameSize.width * (1 - 2 * trimRatio_)) >> 3) << 3;
410 outputFrameSize.height = ((int)((float)frameSize.height * (1 - 2 * trimRatio_)) >> 3) << 3;
411
412 return (outputFrameSize);
413 }
414
415 Mat
cropVideoFrame(Mat & frame)416 VideoStabilizer::cropVideoFrame(Mat& frame)
417 {
418 Rect cropROI;
419 Size inputFrameSize = frame.size();
420 Size outputFrameSize = trimedVideoSize(inputFrameSize);
421
422 cropROI.x = (inputFrameSize.width - outputFrameSize.width) >> 1;
423 cropROI.y = (inputFrameSize.height - outputFrameSize.height) >> 1;
424 cropROI.width = outputFrameSize.width;
425 cropROI.height = outputFrameSize.height;
426
427 Mat croppedFrame = frame(cropROI).clone();
428
429 return croppedFrame;
430 }
431
432