1 /*
2 * test-video-stabilization.cpp - test video stabilization using Gyroscope
3 *
4 * Copyright (c) 2017 Intel Corporation
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * Author: Zong Wei <wei.zong@intel.com>
19 */
20
21 #include "test_common.h"
22 #include "test_inline.h"
23 #include <unistd.h>
24 #include <getopt.h>
25 #include <ocl/cl_utils.h>
26 #include <ocl/cl_device.h>
27 #include <ocl/cl_context.h>
28 #include <ocl/cl_blender.h>
29 #include <image_file_handle.h>
30 #include <ocl/cl_video_stabilizer.h>
31 #include <dma_video_buffer.h>
32
33 #if HAVE_OPENCV
34 #include <opencv2/opencv.hpp>
35 #include <opencv2/core/ocl.hpp>
36 #include <ocl/cv_base_class.h>
37 #endif
38
39 using namespace XCam;
40
41 static int read_device_pose (const char *file, DevicePoseList &pose, uint32_t pose_size);
42
43 static void
usage(const char * arg0)44 usage(const char* arg0)
45 {
46 printf ("Usage:\n"
47 "%s --input file --output file"
48 " [--input-w width] [--input-h height] \n"
49 "\t--input, input image(NV12)\n"
50 "\t--gyro, input gyro pose data;\n"
51 "\t--output, output image(NV12) PREFIX\n"
52 "\t--input-w, optional, input width; default:1920\n"
53 "\t--input-h, optional, input height; default:1080\n"
54 "\t--save, optional, save file or not, default true; select from [true/false]\n"
55 "\t--loop optional, how many loops need to run for performance test, default: 1\n"
56 "\t--help, usage\n",
57 arg0);
58 }
59
main(int argc,char * argv[])60 int main (int argc, char *argv[])
61 {
62 XCamReturn ret = XCAM_RETURN_NO_ERROR;
63
64 SmartPtr<CLVideoStabilizer> video_stab;
65
66 SmartPtr<CLContext> context;
67 SmartPtr<BufferPool> buf_pool;
68
69 VideoBufferInfo input_buf_info;
70 VideoBufferInfo output_buf_info;
71 SmartPtr<VideoBuffer> input_buf;
72 SmartPtr<VideoBuffer> output_buf;
73
74 uint32_t input_format = V4L2_PIX_FMT_NV12;
75 uint32_t input_width = 1920;
76 uint32_t input_height = 1080;
77 uint32_t output_width = 1920;
78 uint32_t output_height = 1080;
79
80 ImageFileHandle file_in, file_out;
81 const char *file_in_name = NULL;
82 const char *file_out_name = NULL;
83
84 const char *gyro_data = "gyro_data.csv";
85
86 bool need_save_output = true;
87 double framerate = 30.0;
88 int loop = 1;
89
90 const struct option long_opts[] = {
91 {"input", required_argument, NULL, 'i'},
92 {"gyro", required_argument, NULL, 'g'},
93 {"output", required_argument, NULL, 'o'},
94 {"input-w", required_argument, NULL, 'w'},
95 {"input-h", required_argument, NULL, 'h'},
96 {"save", required_argument, NULL, 's'},
97 {"loop", required_argument, NULL, 'l'},
98 {"help", no_argument, NULL, 'H'},
99 {0, 0, 0, 0},
100 };
101
102 int opt = -1;
103 while ((opt = getopt_long(argc, argv, "", long_opts, NULL)) != -1) {
104 switch (opt) {
105 case 'i':
106 file_in_name = optarg;
107 break;
108 case 'g':
109 gyro_data = optarg;
110 break;
111 case 'o':
112 file_out_name = optarg;
113 break;
114 case 'w':
115 input_width = atoi(optarg);
116 output_width = input_width;
117 break;
118 case 'h':
119 input_height = atoi(optarg);
120 output_height = input_height;
121 break;
122 case 's':
123 need_save_output = (strcasecmp (optarg, "false") == 0 ? false : true);
124 break;
125 case 'l':
126 loop = atoi(optarg);
127 break;
128 case 'H':
129 usage (argv[0]);
130 return -1;
131 default:
132 printf ("getopt_long return unknown value:%c\n", opt);
133 usage (argv[0]);
134 return -1;
135 }
136 }
137
138 if (optind < argc || argc < 2) {
139 printf("unknown option %s\n", argv[optind]);
140 usage (argv[0]);
141 return -1;
142 }
143
144 if (!file_in_name || !file_out_name) {
145 XCAM_LOG_ERROR ("input/output path is NULL");
146 return -1;
147 }
148
149 printf ("Description-----------\n");
150 printf ("input video file:%s\n", file_in_name);
151 printf ("gyro pose file:%s\n", gyro_data);
152 printf ("output file PREFIX:%s\n", file_out_name);
153 printf ("input width:%d\n", input_width);
154 printf ("input height:%d\n", input_height);
155 printf ("need save file:%s\n", need_save_output ? "true" : "false");
156 printf ("loop count:\t\t%d\n", loop);
157 printf ("----------------------\n");
158
159 DevicePoseList device_pose;
160
161 const int pose_size = sizeof(DevicePose::orientation) / sizeof(double) +
162 sizeof(DevicePose::translation) / sizeof(double) +
163 sizeof(DevicePose::timestamp) / sizeof(int64_t);
164
165 const int count = read_device_pose (gyro_data, device_pose, pose_size);
166 if (count <= 0 || device_pose.size () <= 0) {
167 XCAM_LOG_ERROR ("read gyro file(%s) failed.", gyro_data);
168 return -1;
169 }
170
171 context = CLDevice::instance ()->get_context ();
172 video_stab = create_cl_video_stab_handler (context).dynamic_cast_ptr<CLVideoStabilizer> ();
173 XCAM_ASSERT (video_stab.ptr ());
174 video_stab->set_pool_type (CLImageHandler::CLVideoPoolType);
175
176 /*
177 Color CameraIntrinsics:
178 image_width: 1920, image_height :1080,
179 fx: 1707.799171, fy: 1710.337510,
180 cx: 940.413257, cy: 540.198348,
181 image_plane_distance: 1.778957.
182
183 Color Camera Frame with respect to IMU Frame:
184 Position: 0.045699, -0.008592, -0.006434
185 Orientation: -0.013859, -0.999889, 0.002361, 0.005021
186 */
187 double focal_x = 1707.799171;
188 double focal_y = 1710.337510;
189 double offset_x = 940.413257;
190 double offset_y = 540.198348;
191 double skew = 0;
192 video_stab->set_camera_intrinsics (focal_x, focal_y, offset_x, offset_y, skew);
193
194 CoordinateSystemConv world_to_device (AXIS_X, AXIS_MINUS_Z, AXIS_NONE);
195 CoordinateSystemConv device_to_image (AXIS_X, AXIS_Y, AXIS_Y);
196 video_stab->align_coordinate_system (world_to_device, device_to_image);
197
198 uint32_t radius = 15;
199 float stdev = 10;
200 video_stab->set_motion_filter (radius, stdev);
201
202 input_buf_info.init (input_format, input_width, input_height);
203 output_buf_info.init (input_format, output_width, output_height);
204 buf_pool = new CLVideoBufferPool ();
205 XCAM_ASSERT (buf_pool.ptr ());
206 buf_pool->set_video_info (input_buf_info);
207 if (!buf_pool->reserve (36)) {
208 XCAM_LOG_ERROR ("init buffer pool failed");
209 return -1;
210 }
211
212 ret = file_in.open (file_in_name, "rb");
213 CHECK (ret, "open %s failed", file_in_name);
214
215 #if HAVE_OPENCV
216 cv::VideoWriter writer;
217 if (need_save_output) {
218 cv::Size dst_size = cv::Size (output_width, output_height);
219 if (!writer.open (file_out_name, CV_FOURCC('X', '2', '6', '4'), framerate, dst_size)) {
220 XCAM_LOG_ERROR ("open file %s failed", file_out_name);
221 return -1;
222 }
223 }
224 #endif
225
226 int i = 0;
227 while (loop--) {
228 ret = file_in.rewind ();
229 CHECK (ret, "video stabilization stitch rewind file(%s) failed", file_in_name);
230
231 video_stab->reset_counter ();
232
233 DevicePoseList::iterator pose_iterator = device_pose.begin ();
234 do {
235 input_buf = buf_pool->get_buffer (buf_pool);
236 XCAM_ASSERT (input_buf.ptr ());
237 ret = file_in.read_buf (input_buf);
238 if (ret == XCAM_RETURN_BYPASS)
239 break;
240 if (ret == XCAM_RETURN_ERROR_FILE) {
241 XCAM_LOG_ERROR ("read buffer from %s failed", file_in_name);
242 return -1;
243 }
244
245 SmartPtr<MetaData> pose_data = *(pose_iterator);
246 SmartPtr<DevicePose> data = *(pose_iterator);
247 input_buf->add_metadata (pose_data);
248 input_buf->set_timestamp (pose_data->timestamp);
249
250 ret = video_stab->execute (input_buf, output_buf);
251 if (++pose_iterator == device_pose.end ()) {
252 break;
253 }
254 if (ret == XCAM_RETURN_BYPASS) {
255 continue;
256 }
257
258 #if HAVE_OPENCV
259 if (need_save_output) {
260 cv::Mat out_mat;
261 convert_to_mat (output_buf, out_mat);
262 writer.write (out_mat);
263 } else
264 #endif
265 ensure_gpu_buffer_done (output_buf);
266
267 FPS_CALCULATION (video_stabilizer, XCAM_OBJ_DUR_FRAME_NUM);
268 ++i;
269
270 } while (true);
271 }
272
273 return ret;
274 }
275
276 //return count
277
278 #define RELEASE_FILE_MEM { \
279 xcam_free (ptr); \
280 if (p_f) fclose (p_f); \
281 return -1; \
282 }
283
read_device_pose(const char * file,DevicePoseList & pose_list,uint32_t members)284 int read_device_pose (const char* file, DevicePoseList &pose_list, uint32_t members)
285 {
286 char *ptr = NULL;
287 SmartPtr<DevicePose> data;
288
289 FILE *p_f = fopen (file, "rb");
290 CHECK_EXP (p_f, "open gyro pos data(%s) failed", file);
291
292 CHECK_DECLARE (
293 ERROR,
294 !fseek (p_f, 0L, SEEK_END),
295 RELEASE_FILE_MEM, "seek to file(%s) end failed", file);
296
297 size_t size = ftell(p_f);
298 int entries = size / members;
299
300 fseek (p_f, 0L, SEEK_SET);
301
302 ptr = (char*) xcam_malloc0 (size + 1);
303 CHECK_DECLARE (ERROR, ptr, RELEASE_FILE_MEM, "malloc file buffer failed");
304
305 CHECK_DECLARE (
306 ERROR,
307 fread (ptr, 1, size, p_f) == size,
308 RELEASE_FILE_MEM, "read pose file(%s)failed", file);
309 ptr[size] = 0;
310 fclose (p_f);
311 p_f = NULL;
312
313 char *str_num = NULL;
314 char tokens[] = "\t ,\r\n";
315 str_num = strtok (ptr, tokens);
316 int count = 0;
317 int x = 0, y = 0;
318 const int orient_size = sizeof(DevicePose::orientation) / sizeof(double);
319 const int trans_size = sizeof(DevicePose::translation) / sizeof(double);
320
321 while (str_num != NULL) {
322 float num = strtof (str_num, NULL);
323
324 x = count % members;
325 y = count / members;
326 if (y >= entries) {
327 break;
328 }
329 if (x == 0) {
330 data = new DevicePose ();
331 }
332
333 CHECK_DECLARE (ERROR, data.ptr (), RELEASE_FILE_MEM, "invalid buffer pointer(device pose is null)");
334 if (x < orient_size) {
335 data->orientation[x] = num;
336 } else if (x < orient_size + trans_size) {
337 data->translation[x - orient_size] = num;
338 } else if (x == orient_size + trans_size) {
339 data->timestamp = num * 1000000;
340 pose_list.push_back (data);
341 } else {
342 CHECK_DECLARE (ERROR, false, RELEASE_FILE_MEM, "unknow branch");
343 }
344
345 ++count;
346 str_num = strtok (NULL, tokens);
347 }
348 free (ptr);
349 ptr = NULL;
350
351 return count / members;
352 }
353
354