/* * soft_stitcher.cpp - soft stitcher implementation * * Copyright (c) 2017 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * * Author: Wind Yuan */ #include "soft_stitcher.h" #include "soft_blender.h" #include "soft_geo_mapper.h" #include "soft_video_buf_allocator.h" #include "interface/feature_match.h" #include "surview_fisheye_dewarp.h" #include "soft_copy_task.h" #include "xcam_utils.h" #include #define ENABLE_FEATURE_MATCH HAVE_OPENCV #if ENABLE_FEATURE_MATCH #include "cv_capi_feature_match.h" #ifndef ANDROID #include #endif #endif #define SOFT_STITCHER_ALIGNMENT_X 8 #define SOFT_STITCHER_ALIGNMENT_Y 4 #define MAP_FACTOR_X 16 #define MAP_FACTOR_Y 16 #define DUMP_STITCHER 0 namespace XCam { #if DUMP_STITCHER static void stitcher_dump_buf (const SmartPtr buf, uint32_t idx, const char *prefix) { XCAM_ASSERT (prefix); char name[256]; snprintf (name, 256, "%s-%d", prefix, idx); dump_buf_perfix_path (buf, name); } #else static void stitcher_dump_buf (const SmartPtr buf, ...) { XCAM_UNUSED (buf); } #endif namespace SoftSitcherPriv { DECLARE_HANDLER_CALLBACK (CbGeoMap, SoftStitcher, dewarp_done); DECLARE_HANDLER_CALLBACK (CbBlender, SoftStitcher, blender_done); DECLARE_WORK_CALLBACK (CbCopyTask, SoftStitcher, copy_task_done); struct BlenderParam : SoftBlender::BlenderParam { SmartPtr stitch_param; uint32_t idx; BlenderParam ( uint32_t i, const SmartPtr &in0, const SmartPtr &in1, const SmartPtr &out) : SoftBlender::BlenderParam (in0, in1, out) , idx (i) {} }; typedef std::map> BlenderParams; typedef std::map BlendCopyTaskNums; struct HandlerParam : ImageHandler::Parameters { SmartPtr stitch_param; uint32_t idx; HandlerParam (uint32_t i) : idx (i) {} }; struct StitcherCopyArgs : XCamSoftTasks::CopyTask::Args { uint32_t idx; StitcherCopyArgs ( uint32_t i, const SmartPtr ¶m) : XCamSoftTasks::CopyTask::Args (param) , idx (i) {} }; struct Factor { float x, y; Factor () : x (1.0f), y (1.0f) {} void reset () { x = 1.0f; y = 1.0f; } }; struct Overlap { SmartPtr matcher; SmartPtr blender; BlenderParams param_map; SmartPtr find_blender_param_in_map ( const SmartPtr &key, const uint32_t idx); }; struct FisheyeDewarp { SmartPtr dewarp; SmartPtr buf_pool; Factor left_match_factor, right_match_factor; bool set_dewarp_factor (); XCamReturn set_dewarp_geo_table ( SmartPtr mapper, const CameraInfo &cam_info, const Stitcher::RoundViewSlice &view_slice, const BowlDataConfig &bowl); }; struct Copier { SmartPtr copy_task; Stitcher::CopyArea copy_area; XCamReturn start_copy_task ( const SmartPtr ¶m, const uint32_t idx, const SmartPtr &buf); }; typedef std::vector Copiers; class StitcherImpl { friend class XCam::SoftStitcher; public: StitcherImpl (SoftStitcher *handler) : _stitcher (handler) {} XCamReturn init_config (uint32_t count); bool remove_task_count (const SmartPtr ¶m); int32_t dec_task_count (const SmartPtr ¶m); XCamReturn start_dewarp_works (const SmartPtr ¶m); XCamReturn start_task_count (const SmartPtr ¶m); XCamReturn start_overlap_tasks ( const SmartPtr ¶m, const uint32_t idx, const SmartPtr &buf); XCamReturn start_copy_tasks ( const SmartPtr ¶m, const uint32_t idx, const SmartPtr &buf); XCamReturn start_single_blender (const uint32_t idx, const SmartPtr ¶m); XCamReturn stop (); XCamReturn fisheye_dewarp_to_table (); XCamReturn feature_match ( const SmartPtr &left_buf, const SmartPtr &right_buf, const uint32_t idx); bool get_and_reset_feature_match_factors (uint32_t idx, Factor &left, Factor &right); private: XCamReturn init_fisheye (uint32_t idx); bool init_dewarp_factors (uint32_t idx); XCamReturn create_copier (Stitcher::CopyArea area); private: FisheyeDewarp _fisheye [XCAM_STITCH_MAX_CAMERAS]; Overlap _overlaps [XCAM_STITCH_MAX_CAMERAS]; Copiers _copiers; SmartPtr _dewarp_pool; Mutex _map_mutex; BlendCopyTaskNums _task_counts; SoftStitcher *_stitcher; }; bool StitcherImpl::init_dewarp_factors (uint32_t idx) { XCAM_FAIL_RETURN ( ERROR, _fisheye[idx].dewarp.ptr (), false, "FisheyeDewarp dewarp handler empty"); Factor match_left_factor, match_right_factor; get_and_reset_feature_match_factors (idx, match_left_factor, match_right_factor); Factor unify_factor, last_left_factor, last_right_factor; _fisheye[idx].dewarp->get_factors (unify_factor.x, unify_factor.y); last_left_factor = last_right_factor = unify_factor; if (XCAM_DOUBLE_EQUAL_AROUND (unify_factor.x, 0.0f) || XCAM_DOUBLE_EQUAL_AROUND (unify_factor.y, 0.0f)) { // not started. return true; } Factor cur_left, cur_right; cur_left.x = last_left_factor.x * match_left_factor.x; cur_left.y = last_left_factor.y * match_left_factor.y; cur_right.x = last_right_factor.x * match_right_factor.x; cur_right.y = last_right_factor.y * match_right_factor.y; unify_factor.x = (cur_left.x + cur_right.x) / 2.0f; unify_factor.y = (cur_left.y + cur_right.y) / 2.0f; _fisheye[idx].dewarp->set_factors (unify_factor.x, unify_factor.y); return true; } XCamReturn FisheyeDewarp::set_dewarp_geo_table ( SmartPtr mapper, const CameraInfo &cam_info, const Stitcher::RoundViewSlice &view_slice, const BowlDataConfig &bowl) { PolyFisheyeDewarp fd; fd.set_intrinsic_param (cam_info.calibration.intrinsic); fd.set_extrinsic_param (cam_info.calibration.extrinsic); uint32_t table_width, table_height; table_width = view_slice.width / MAP_FACTOR_X; table_width = XCAM_ALIGN_UP (table_width, 4); table_height = view_slice.height / MAP_FACTOR_Y; table_height = XCAM_ALIGN_UP (table_height, 2); SurViewFisheyeDewarp::MapTable map_table(table_width * table_height); fd.fisheye_dewarp ( map_table, table_width, table_height, view_slice.width, view_slice.height, bowl); XCAM_FAIL_RETURN ( ERROR, mapper->set_lookup_table (map_table.data (), table_width, table_height), XCAM_RETURN_ERROR_UNKNOWN, "set fisheye dewarp lookup table failed"); return XCAM_RETURN_NO_ERROR; } bool StitcherImpl::get_and_reset_feature_match_factors (uint32_t idx, Factor &left, Factor &right) { uint32_t cam_num = _stitcher->get_camera_num (); XCAM_FAIL_RETURN ( ERROR, idx < cam_num, false, "get dewarp factor failed, idx(%d) > camera_num(%d)", idx, cam_num); SmartLock locker (_map_mutex); left = _fisheye[idx].left_match_factor; right = _fisheye[idx].right_match_factor; _fisheye[idx].left_match_factor.reset (); _fisheye[idx].right_match_factor.reset (); return true; } XCamReturn StitcherImpl::init_fisheye (uint32_t idx) { FisheyeDewarp &fisheye = _fisheye[idx]; SmartPtr dewarp_cb = new CbGeoMap (_stitcher); fisheye.dewarp = new SoftGeoMapper ("sitcher_remapper"); XCAM_ASSERT (fisheye.dewarp.ptr ()); fisheye.dewarp->set_callback (dewarp_cb); Stitcher::RoundViewSlice view_slice = _stitcher->get_round_view_slice (idx); VideoBufferInfo buf_info; buf_info.init ( V4L2_PIX_FMT_NV12, view_slice.width, view_slice.height, XCAM_ALIGN_UP (view_slice.width, SOFT_STITCHER_ALIGNMENT_X), XCAM_ALIGN_UP (view_slice.height, SOFT_STITCHER_ALIGNMENT_Y)); fisheye.buf_pool = new SoftVideoBufAllocator (buf_info); XCAM_ASSERT (fisheye.buf_pool.ptr ()); XCAM_FAIL_RETURN ( ERROR, fisheye.buf_pool->reserve (2), XCAM_RETURN_ERROR_MEM, "stitcher:%s reserve dewarp buffer pool(w:%d,h:%d) failed", XCAM_STR (_stitcher->get_name ()), buf_info.width, buf_info.height); return XCAM_RETURN_NO_ERROR; } XCamReturn StitcherImpl::create_copier (Stitcher::CopyArea area) { XCAM_FAIL_RETURN ( ERROR, area.in_idx != INVALID_INDEX && area.in_area.width == area.out_area.width && area.in_area.height == area.out_area.height, XCAM_RETURN_ERROR_PARAM, "stitcher: copy area (idx:%d) is invalid", area.in_idx); SmartPtr copy_cb = new CbCopyTask (_stitcher); XCAM_ASSERT (copy_cb.ptr ()); Copier copier; copier.copy_task = new XCamSoftTasks::CopyTask (copy_cb); XCAM_ASSERT (copier.copy_task.ptr ()); copier.copy_area = area; _copiers.push_back (copier); return XCAM_RETURN_NO_ERROR; } XCamReturn StitcherImpl::init_config (uint32_t count) { XCamReturn ret = XCAM_RETURN_NO_ERROR; SmartPtr blender_cb = new CbBlender (_stitcher); for (uint32_t i = 0; i < count; ++i) { ret = init_fisheye (i); XCAM_FAIL_RETURN ( ERROR, xcam_ret_is_ok (ret), ret, "stitcher:%s init fisheye failed, idx:%d.", XCAM_STR (_stitcher->get_name ()), i); #if ENABLE_FEATURE_MATCH _overlaps[i].matcher = new CVCapiFeatureMatch; CVFMConfig config; config.sitch_min_width = 136; config.min_corners = 4; config.offset_factor = 0.8f; config.delta_mean_offset = 120.0f; config.recur_offset_error = 8.0f; config.max_adjusted_offset = 24.0f; config.max_valid_offset_y = 20.0f; #ifndef ANDROID config.max_track_error = 28.0f; #else config.max_track_error = 3600.0f; #endif _overlaps[i].matcher->set_config (config); _overlaps[i].matcher->set_fm_index (i); #endif _overlaps[i].blender = create_soft_blender ().dynamic_cast_ptr(); XCAM_ASSERT (_overlaps[i].blender.ptr ()); _overlaps[i].blender->set_callback (blender_cb); _overlaps[i].param_map.clear (); } Stitcher::CopyAreaArray areas = _stitcher->get_copy_area (); uint32_t size = areas.size (); for (uint32_t i = 0; i < size; ++i) { XCAM_LOG_DEBUG ("soft-stitcher:copy area (idx:%d) input area(%d, %d, %d, %d) output area(%d, %d, %d, %d)", areas[i].in_idx, areas[i].in_area.pos_x, areas[i].in_area.pos_y, areas[i].in_area.width, areas[i].in_area.height, areas[i].out_area.pos_x, areas[i].out_area.pos_y, areas[i].out_area.width, areas[i].out_area.height); XCAM_ASSERT (areas[i].in_idx < size); ret = create_copier (areas[i]); XCAM_FAIL_RETURN ( ERROR, xcam_ret_is_ok (ret), ret, "soft-stitcher::%s init copier failed, idx:%d.", XCAM_STR (_stitcher->get_name ()), i); } return XCAM_RETURN_NO_ERROR; } bool StitcherImpl::remove_task_count (const SmartPtr ¶m) { XCAM_ASSERT (param.ptr ()); SmartLock locker (_map_mutex); BlendCopyTaskNums::iterator i = _task_counts.find (param.ptr ()); if (i == _task_counts.end ()) return false; _task_counts.erase (i); return true; } int32_t StitcherImpl::dec_task_count (const SmartPtr ¶m) { XCAM_ASSERT (param.ptr ()); SmartLock locker (_map_mutex); BlendCopyTaskNums::iterator i = _task_counts.find (param.ptr ()); if (i == _task_counts.end ()) return -1; int32_t &count = i->second; --count; if (count > 0) return count; XCAM_ASSERT (count == 0); _task_counts.erase (i); return 0; } XCamReturn StitcherImpl::fisheye_dewarp_to_table () { uint32_t camera_num = _stitcher->get_camera_num (); for (uint32_t i = 0; i < camera_num; ++i) { CameraInfo cam_info; _stitcher->get_camera_info (i, cam_info); Stitcher::RoundViewSlice view_slice = _stitcher->get_round_view_slice (i); BowlDataConfig bowl = _stitcher->get_bowl_config (); bowl.angle_start = view_slice.hori_angle_start; bowl.angle_end = format_angle (view_slice.hori_angle_start + view_slice.hori_angle_range); uint32_t out_width, out_height; _stitcher->get_output_size (out_width, out_height); _fisheye[i].dewarp->set_output_size (view_slice.width, view_slice.height); if (bowl.angle_end < bowl.angle_start) bowl.angle_start -= 360.0f; XCAM_LOG_INFO ( "soft-stitcher:%s camera(idx:%d) info (angle start:%.2f, range:%.2f), bowl info (angle start%.2f, end:%.2f)", XCAM_STR (_stitcher->get_name ()), i, view_slice.hori_angle_start, view_slice.hori_angle_range, bowl.angle_start, bowl.angle_end); XCamReturn ret = _fisheye[i].set_dewarp_geo_table (_fisheye[i].dewarp, cam_info, view_slice, bowl); XCAM_FAIL_RETURN ( ERROR, xcam_ret_is_ok (ret), ret, "stitcher:%s set dewarp geo table failed, idx:%d.", XCAM_STR (_stitcher->get_name ()), i); } return XCAM_RETURN_NO_ERROR; } XCamReturn StitcherImpl::start_dewarp_works (const SmartPtr ¶m) { uint32_t camera_num = _stitcher->get_camera_num (); Factor cur_left, cur_right; for (uint32_t i = 0; i < camera_num; ++i) { SmartPtr out_buf = _fisheye[i].buf_pool->get_buffer (); SmartPtr dewarp_params = new HandlerParam (i); dewarp_params->in_buf = param->in_bufs[i]; dewarp_params->out_buf = out_buf; dewarp_params->stitch_param = param; init_dewarp_factors (i); XCamReturn ret = _fisheye[i].dewarp->execute_buffer (dewarp_params, false); XCAM_FAIL_RETURN ( ERROR, xcam_ret_is_ok (ret), ret, "soft-stitcher:%s fisheye dewarp buffer failed", XCAM_STR (_stitcher->get_name ())); } return XCAM_RETURN_NO_ERROR; } SmartPtr Overlap::find_blender_param_in_map ( const SmartPtr &key, const uint32_t idx) { SmartPtr param; BlenderParams::iterator i = param_map.find (key.ptr ()); if (i == param_map.end ()) { param = new BlenderParam (idx, NULL, NULL, NULL); XCAM_ASSERT (param.ptr ()); param->stitch_param = key; param_map.insert (std::make_pair ((void*)key.ptr (), param)); } else { param = (*i).second; } return param; } XCamReturn StitcherImpl::feature_match ( const SmartPtr &left_buf, const SmartPtr &right_buf, const uint32_t idx) { const Stitcher::ImageOverlapInfo overlap_info = _stitcher->get_overlap (idx); Rect left_ovlap = overlap_info.left; Rect right_ovlap = overlap_info.right; const VideoBufferInfo left_buf_info = left_buf->get_video_info (); left_ovlap.pos_y = left_ovlap.height / 5; left_ovlap.height = left_ovlap.height / 2; right_ovlap.pos_y = right_ovlap.height / 5; right_ovlap.height = right_ovlap.height / 2; _overlaps[idx].matcher->reset_offsets (); _overlaps[idx].matcher->optical_flow_feature_match ( left_buf, right_buf, left_ovlap, right_ovlap, left_buf_info.width); float left_offsetx = _overlaps[idx].matcher->get_current_left_offset_x (); Factor left_factor, right_factor; uint32_t left_idx = idx; float center_x = (float) _stitcher->get_center (left_idx).slice_center_x; float feature_center_x = (float)left_ovlap.pos_x + (left_ovlap.width / 2.0f); float range = feature_center_x - center_x; XCAM_ASSERT (range > 1.0f); right_factor.x = (range + left_offsetx / 2.0f) / range; right_factor.y = 1.0; XCAM_ASSERT (right_factor.x > 0.0f && right_factor.x < 2.0f); uint32_t right_idx = (idx + 1) % _stitcher->get_camera_num (); center_x = (float) _stitcher->get_center (right_idx).slice_center_x; feature_center_x = (float)right_ovlap.pos_x + (right_ovlap.width / 2.0f); range = center_x - feature_center_x; XCAM_ASSERT (range > 1.0f); left_factor.x = (range + left_offsetx / 2.0f) / range; left_factor.y = 1.0; XCAM_ASSERT (left_factor.x > 0.0f && left_factor.x < 2.0f); { SmartLock locker (_map_mutex); _fisheye[left_idx].right_match_factor = right_factor; _fisheye[right_idx].left_match_factor = left_factor; } return XCAM_RETURN_NO_ERROR; } XCamReturn StitcherImpl::start_single_blender ( const uint32_t idx, const SmartPtr ¶m) { SmartPtr blender = _overlaps[idx].blender; const Stitcher::ImageOverlapInfo &overlap_info = _stitcher->get_overlap (idx); uint32_t out_width, out_height; _stitcher->get_output_size (out_width, out_height); blender->set_output_size (out_width, out_height); blender->set_merge_window (overlap_info.out_area); blender->set_input_valid_area (overlap_info.left, 0); blender->set_input_valid_area (overlap_info.right, 1); blender->set_input_merge_area (overlap_info.left, 0); blender->set_input_merge_area (overlap_info.right, 1); return blender->execute_buffer (param, false); } XCamReturn StitcherImpl::start_overlap_tasks ( const SmartPtr ¶m, const uint32_t idx, const SmartPtr &buf) { SmartPtr cur_param, prev_param; const uint32_t camera_num = _stitcher->get_camera_num (); uint32_t pre_idx = (idx + camera_num - 1) % camera_num; XCamReturn ret = XCAM_RETURN_NO_ERROR; { SmartPtr param_b; SmartLock locker (_map_mutex); param_b = _overlaps[idx].find_blender_param_in_map (param, idx); param_b->in_buf = buf; if (param_b->in_buf.ptr () && param_b->in1_buf.ptr ()) { cur_param = param_b; _overlaps[idx].param_map.erase (param.ptr ()); } param_b = _overlaps[pre_idx].find_blender_param_in_map (param, pre_idx); param_b->in1_buf = buf; if (param_b->in_buf.ptr () && param_b->in1_buf.ptr ()) { prev_param = param_b; _overlaps[pre_idx].param_map.erase (param.ptr ()); } } if (cur_param.ptr ()) { cur_param->out_buf = param->out_buf; ret = start_single_blender (idx, cur_param); XCAM_FAIL_RETURN ( ERROR, xcam_ret_is_ok (ret), ret, "soft-stitcher:%s blend overlap idx:%d failed", XCAM_STR (_stitcher->get_name ()), idx); } if (prev_param.ptr ()) { prev_param->out_buf = param->out_buf; ret = start_single_blender (pre_idx, prev_param); XCAM_FAIL_RETURN ( ERROR, xcam_ret_is_ok (ret), ret, "soft-stitcher:%s blend overlap idx:%d failed", XCAM_STR (_stitcher->get_name ()), pre_idx); } #if ENABLE_FEATURE_MATCH //start feature match if (cur_param.ptr ()) { ret = feature_match (cur_param->in_buf, cur_param->in1_buf, idx); XCAM_FAIL_RETURN ( ERROR, xcam_ret_is_ok (ret), ret, "soft-stitcher:%s feature-match overlap idx:%d failed", XCAM_STR (_stitcher->get_name ()), idx); } if (prev_param.ptr ()) { ret = feature_match (prev_param->in_buf, prev_param->in1_buf, pre_idx); XCAM_FAIL_RETURN ( ERROR, xcam_ret_is_ok (ret), ret, "soft-stitcher:%s feature-match overlap idx:%d failed", XCAM_STR (_stitcher->get_name ()), pre_idx); } #endif return XCAM_RETURN_NO_ERROR; } XCamReturn Copier::start_copy_task ( const SmartPtr ¶m, const uint32_t idx, const SmartPtr &buf) { XCAM_ASSERT (copy_task.ptr ()); SmartPtr in_buf = buf, out_buf = param->out_buf; const VideoBufferInfo &in_info = in_buf->get_video_info (); const VideoBufferInfo &out_info = out_buf->get_video_info (); SmartPtr args = new StitcherCopyArgs (idx, param); args->in_luma = new UcharImage ( in_buf, copy_area.in_area.width, copy_area.in_area.height, in_info.strides[0], in_info.offsets[0] + copy_area.in_area.pos_x + copy_area.in_area.pos_y * in_info.strides[0]); args->in_uv = new Uchar2Image ( in_buf, copy_area.in_area.width / 2, copy_area.in_area.height / 2, in_info.strides[0], in_info.offsets[1] + copy_area.in_area.pos_x + copy_area.in_area.pos_y / 2 * in_info.strides[1]); args->out_luma = new UcharImage ( out_buf, copy_area.out_area.width, copy_area.out_area.height, out_info.strides[0], out_info.offsets[0] + copy_area.out_area.pos_x + copy_area.out_area.pos_y * out_info.strides[0]); args->out_uv = new Uchar2Image ( out_buf, copy_area.out_area.width / 2, copy_area.out_area.height / 2, out_info.strides[0], out_info.offsets[1] + copy_area.out_area.pos_x + copy_area.out_area.pos_y / 2 * out_info.strides[1]); uint32_t thread_x = 1, thread_y = 4; WorkSize global_size (1, xcam_ceil (copy_area.in_area.height, 2) / 2); WorkSize local_size ( xcam_ceil (global_size.value[0], thread_x) / thread_x, xcam_ceil (global_size.value[1], thread_y) / thread_y); copy_task->set_local_size (local_size); copy_task->set_global_size (global_size); return copy_task->work (args); } XCamReturn StitcherImpl::start_copy_tasks ( const SmartPtr ¶m, const uint32_t idx, const SmartPtr &buf) { uint32_t size = _stitcher->get_copy_area ().size (); for (uint32_t i = 0; i < size; ++i) { if(_copiers[i].copy_area.in_idx == idx) { XCamReturn ret = _copiers[i].start_copy_task (param, idx, buf); XCAM_FAIL_RETURN ( ERROR, xcam_ret_is_ok (ret), ret, "soft-stitcher:%s start copy task failed, idx:%d", XCAM_STR (_stitcher->get_name ()), idx); } } return XCAM_RETURN_NO_ERROR; } XCamReturn StitcherImpl::stop () { uint32_t cam_num = _stitcher->get_camera_num (); for (uint32_t i = 0; i < cam_num; ++i) { if (_fisheye[i].dewarp.ptr ()) { _fisheye[i].dewarp->terminate (); _fisheye[i].dewarp.release (); } if (_fisheye[i].buf_pool.ptr ()) { _fisheye[i].buf_pool->stop (); } if (_overlaps[i].blender.ptr ()) { _overlaps[i].blender->terminate (); _overlaps[i].blender.release (); } } for (Copiers::iterator i_copy = _copiers.begin (); i_copy != _copiers.end (); ++i_copy) { Copier © = *i_copy; if (copy.copy_task.ptr ()) { copy.copy_task->stop (); copy.copy_task.release (); } } if (_dewarp_pool.ptr ()) { _dewarp_pool->stop (); } return XCAM_RETURN_NO_ERROR; } }; SoftStitcher::SoftStitcher (const char *name) : SoftHandler (name) , Stitcher (SOFT_STITCHER_ALIGNMENT_X, SOFT_STITCHER_ALIGNMENT_Y) { _impl = new SoftSitcherPriv::StitcherImpl (this); XCAM_ASSERT (_impl.ptr ()); #if ENABLE_FEATURE_MATCH #ifndef ANDROID cv::ocl::setUseOpenCL (false); #endif #endif } SoftStitcher::~SoftStitcher () { } XCamReturn SoftStitcher::stitch_buffers (const VideoBufferList &in_bufs, SmartPtr &out_buf) { XCAM_FAIL_RETURN ( ERROR, !in_bufs.empty (), XCAM_RETURN_ERROR_PARAM, "soft-stitcher:%s stitch buffer failed, in_bufs is empty", XCAM_STR (get_name ())); SmartPtr param = new StitcherParam; param->out_buf = out_buf; uint32_t count = 0; for (VideoBufferList::const_iterator i = in_bufs.begin(); i != in_bufs.end (); ++i) { SmartPtr buf = *i; XCAM_ASSERT (buf.ptr ()); param->in_bufs[count++] = buf; } param->in_buf_num = count; XCamReturn ret = execute_buffer (param, true); if (!out_buf.ptr () && xcam_ret_is_ok (ret)) { out_buf = param->out_buf; } return ret; } XCamReturn SoftStitcher::terminate () { _impl->stop (); return SoftHandler::terminate (); } XCamReturn SoftStitcher::start_task_count (const SmartPtr ¶m) { XCAM_ASSERT (param.ptr ()); XCAM_ASSERT (_impl.ptr ()); SmartLock locker (_impl->_map_mutex); XCAM_FAIL_RETURN ( ERROR, check_work_continue (param, XCAM_RETURN_NO_ERROR), XCAM_RETURN_ERROR_PARAM, "soft-stitcher:%s start task count failed in work check", XCAM_STR (get_name ())); if (_impl->_task_counts.find (param.ptr ()) != _impl->_task_counts.end ()) { XCAM_LOG_ERROR ("tasks already started, this should never happen."); return XCAM_RETURN_ERROR_UNKNOWN; } int32_t count = get_camera_num (); count += get_copy_area ().size (); XCAM_LOG_DEBUG ("stitcher :%s start task count :%d", XCAM_STR(get_name ()), count); _impl->_task_counts.insert (std::make_pair((void*)param.ptr(), count)); return XCAM_RETURN_NO_ERROR; } void SoftStitcher::dewarp_done ( const SmartPtr &handler, const SmartPtr &base, const XCamReturn error) { SmartPtr dewarp_param = base.dynamic_cast_ptr (); XCAM_ASSERT (dewarp_param.ptr ()); SmartPtr param = dewarp_param->stitch_param; XCAM_ASSERT (param.ptr ()); XCAM_UNUSED (handler); if (!check_work_continue (param, error)) return; XCAM_LOG_INFO ("soft-stitcher:%s camera(idx:%d) dewarp done", XCAM_STR (get_name ()), dewarp_param->idx); stitcher_dump_buf (dewarp_param->out_buf, dewarp_param->idx, "stitcher-dewarp"); //start both blender and feature match XCamReturn ret = _impl->start_overlap_tasks (param, dewarp_param->idx, dewarp_param->out_buf); if (!xcam_ret_is_ok (ret)) { work_broken (param, ret); } ret = _impl->start_copy_tasks (param, dewarp_param->idx, dewarp_param->out_buf); if (!xcam_ret_is_ok (ret)) { work_broken (param, ret); } } void SoftStitcher::blender_done ( const SmartPtr &handler, const SmartPtr &base, const XCamReturn error) { SmartPtr blender_param = base.dynamic_cast_ptr (); XCAM_ASSERT (blender_param.ptr ()); SmartPtr param = blender_param->stitch_param; XCAM_ASSERT (param.ptr ()); XCAM_UNUSED (handler); if (!check_work_continue (param, error)) { _impl->remove_task_count (param); return; } stitcher_dump_buf (blender_param->out_buf, blender_param->idx, "stitcher-blend"); XCAM_LOG_INFO ("blender:(%s) overlap:%d done", XCAM_STR (handler->get_name ()), blender_param->idx); if (_impl->dec_task_count (param) == 0) { work_well_done (param, error); } } void SoftStitcher::copy_task_done ( const SmartPtr &worker, const SmartPtr &base, const XCamReturn error) { XCAM_UNUSED (worker); XCAM_ASSERT (worker.ptr ()); SmartPtr args = base.dynamic_cast_ptr (); XCAM_ASSERT (args.ptr ()); const SmartPtr param = args->get_param ().dynamic_cast_ptr (); XCAM_ASSERT (param.ptr ()); if (!check_work_continue (param, error)) { _impl->remove_task_count (param); return; } XCAM_LOG_INFO ("soft-stitcher:%s camera(idx:%d) copy done", XCAM_STR (get_name ()), args->idx); if (_impl->dec_task_count (param) == 0) { work_well_done (param, error); } } XCamReturn SoftStitcher::configure_resource (const SmartPtr ¶m) { XCAM_UNUSED (param); XCAM_ASSERT (_impl.ptr ()); XCamReturn ret = estimate_round_slices (); XCAM_FAIL_RETURN ( ERROR, xcam_ret_is_ok (ret), ret, "soft-stitcher:%s estimate round view slices failed", XCAM_STR (get_name ())); ret = estimate_coarse_crops (); XCAM_FAIL_RETURN ( ERROR, xcam_ret_is_ok (ret), ret, "soft-stitcher:%s estimate coarse crops failed", XCAM_STR (get_name ())); ret = mark_centers (); XCAM_FAIL_RETURN ( ERROR, xcam_ret_is_ok (ret), ret, "soft-stitcher:%s mark centers failed", XCAM_STR (get_name ())); ret = estimate_overlap (); XCAM_FAIL_RETURN ( ERROR, xcam_ret_is_ok (ret), ret, "soft-stitcher:%s estimake coarse overlap failed", XCAM_STR (get_name ())); ret = update_copy_areas (); XCAM_FAIL_RETURN ( ERROR, xcam_ret_is_ok (ret), ret, "soft-stitcher:%s update copy areas failed", XCAM_STR (get_name ())); uint32_t camera_count = get_camera_num (); ret = _impl->init_config (camera_count); XCAM_FAIL_RETURN ( ERROR, xcam_ret_is_ok (ret), ret, "soft-stitcher:%s initialize private config failed", XCAM_STR (get_name ())); ret = _impl->fisheye_dewarp_to_table (); XCAM_FAIL_RETURN ( ERROR, xcam_ret_is_ok (ret), ret, "soft-stitcher:%s fisheye_dewarp_to_table failed", XCAM_STR (get_name ())); VideoBufferInfo out_info; uint32_t out_width, out_height; get_output_size (out_width, out_height); XCAM_FAIL_RETURN ( ERROR, out_width && out_height, XCAM_RETURN_ERROR_PARAM, "soft-stitcher:%s output size was not set", XCAM_STR(get_name ())); out_info.init ( V4L2_PIX_FMT_NV12, out_width, out_height, XCAM_ALIGN_UP (out_width, SOFT_STITCHER_ALIGNMENT_X), XCAM_ALIGN_UP (out_height, SOFT_STITCHER_ALIGNMENT_Y)); set_out_video_info (out_info); return ret; } XCamReturn SoftStitcher::start_work (const SmartPtr &base) { SmartPtr param = base.dynamic_cast_ptr (); XCAM_FAIL_RETURN ( ERROR, param.ptr () && param->in_buf_num > 0 && param->in_bufs[0].ptr (), XCAM_RETURN_ERROR_PARAM, "soft_stitcher:%s start_work failed, params(in_buf_num) in_bufs are set", XCAM_STR (get_name ())); XCamReturn ret = start_task_count (param); XCAM_FAIL_RETURN ( ERROR, xcam_ret_is_ok (ret), XCAM_RETURN_ERROR_PARAM, "soft_stitcher:%s start blender count failed", XCAM_STR (get_name ())); ret = _impl->start_dewarp_works (param); XCAM_FAIL_RETURN ( ERROR, xcam_ret_is_ok (ret), XCAM_RETURN_ERROR_PARAM, "soft_stitcher:%s start dewarp works failed", XCAM_STR (get_name ())); //for (uint32_t i = 0; i < param->in_buf_num; ++i) { // param->in_bufs[i].release (); //} return ret; } SmartPtr Stitcher::create_soft_stitcher () { return new SoftStitcher; } }