1 /*
2 * Copyright (c) 2017-2020 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 #include "arm_compute/runtime/CL/functions/CLOpticalFlow.h"
25
26 #include "arm_compute/core/CL/ICLTensor.h"
27 #include "arm_compute/core/Error.h"
28 #include "arm_compute/core/TensorInfo.h"
29 #include "arm_compute/core/Window.h"
30 #include "arm_compute/runtime/CL/CLPyramid.h"
31 #include "arm_compute/runtime/CL/CLScheduler.h"
32 #include "arm_compute/runtime/CL/CLTensor.h"
33 #include "arm_compute/runtime/CL/CLTensorAllocator.h"
34 #include "arm_compute/runtime/CL/functions/CLScharr3x3.h"
35 #include "src/core/CL/kernels/CLFillBorderKernel.h"
36 #include "src/core/CL/kernels/CLLKTrackerKernel.h"
37 #include "support/MemorySupport.h"
38
39 using namespace arm_compute;
40
CLOpticalFlow(std::shared_ptr<IMemoryManager> memory_manager)41 CLOpticalFlow::CLOpticalFlow(std::shared_ptr<IMemoryManager> memory_manager) // NOLINT
42 : _memory_group(std::move(memory_manager)),
43 _tracker_init_kernel(),
44 _tracker_stage0_kernel(),
45 _tracker_stage1_kernel(),
46 _tracker_finalize_kernel(support::cpp14::make_unique<CLLKTrackerFinalizeKernel>()),
47 _func_scharr(),
48 _scharr_gx(),
49 _scharr_gy(),
50 _old_points(nullptr),
51 _new_points_estimates(nullptr),
52 _new_points(nullptr),
53 _old_points_internal(),
54 _new_points_internal(),
55 _coefficient_table(),
56 _old_values(),
57 _num_levels(0)
58 {
59 }
60
61 CLOpticalFlow::~CLOpticalFlow() = default;
62
configure(const CLPyramid * old_pyramid,const CLPyramid * new_pyramid,const ICLKeyPointArray * old_points,const ICLKeyPointArray * new_points_estimates,ICLKeyPointArray * new_points,Termination termination,float epsilon,size_t num_iterations,size_t window_dimension,bool use_initial_estimate,BorderMode border_mode,uint8_t constant_border_value)63 void CLOpticalFlow::configure(const CLPyramid *old_pyramid, const CLPyramid *new_pyramid,
64 const ICLKeyPointArray *old_points, const ICLKeyPointArray *new_points_estimates, ICLKeyPointArray *new_points,
65 Termination termination, float epsilon, size_t num_iterations, size_t window_dimension, bool use_initial_estimate,
66 BorderMode border_mode, uint8_t constant_border_value)
67 {
68 configure(CLKernelLibrary::get().get_compile_context(), old_pyramid, new_pyramid, old_points, new_points_estimates, new_points, termination, epsilon, num_iterations, window_dimension,
69 use_initial_estimate, border_mode, constant_border_value);
70 }
71
configure(const CLCompileContext & compile_context,const CLPyramid * old_pyramid,const CLPyramid * new_pyramid,const ICLKeyPointArray * old_points,const ICLKeyPointArray * new_points_estimates,ICLKeyPointArray * new_points,Termination termination,float epsilon,size_t num_iterations,size_t window_dimension,bool use_initial_estimate,BorderMode border_mode,uint8_t constant_border_value)72 void CLOpticalFlow::configure(const CLCompileContext &compile_context, const CLPyramid *old_pyramid, const CLPyramid *new_pyramid,
73 const ICLKeyPointArray *old_points, const ICLKeyPointArray *new_points_estimates, ICLKeyPointArray *new_points,
74 Termination termination, float epsilon, size_t num_iterations, size_t window_dimension, bool use_initial_estimate,
75 BorderMode border_mode, uint8_t constant_border_value)
76 {
77 ARM_COMPUTE_ERROR_ON(nullptr == old_pyramid);
78 ARM_COMPUTE_ERROR_ON(nullptr == new_pyramid);
79 ARM_COMPUTE_ERROR_ON(nullptr == old_points);
80 ARM_COMPUTE_ERROR_ON(nullptr == new_points_estimates);
81 ARM_COMPUTE_ERROR_ON(nullptr == new_points);
82 ARM_COMPUTE_ERROR_ON(old_pyramid->info()->num_levels() != new_pyramid->info()->num_levels());
83 ARM_COMPUTE_ERROR_ON(0 == old_pyramid->info()->num_levels());
84 ARM_COMPUTE_ERROR_ON(old_pyramid->info()->width() != new_pyramid->info()->width());
85 ARM_COMPUTE_ERROR_ON(old_pyramid->info()->height() != new_pyramid->info()->height());
86 ARM_COMPUTE_ERROR_ON(use_initial_estimate && old_points->num_values() != new_points_estimates->num_values());
87
88 // Set member variables
89 _old_points = old_points;
90 _new_points_estimates = new_points_estimates;
91 _new_points = new_points;
92 _num_levels = old_pyramid->info()->num_levels();
93
94 const float pyr_scale = old_pyramid->info()->scale();
95 const int list_length = old_points->num_values();
96 const int old_values_list_length = list_length * window_dimension * window_dimension;
97
98 // Create kernels and tensors
99 _tracker_init_kernel.reserve(_num_levels);
100 _tracker_stage0_kernel.reserve(_num_levels);
101 _tracker_stage1_kernel.reserve(_num_levels);
102 _func_scharr.resize(_num_levels);
103 _scharr_gx.resize(_num_levels);
104 _scharr_gy.resize(_num_levels);
105
106 // Create internal keypoint arrays
107 _old_points_internal = arm_compute::support::cpp14::make_unique<CLLKInternalKeypointArray>(list_length);
108 _old_points_internal->resize(list_length);
109 _new_points_internal = arm_compute::support::cpp14::make_unique<CLLKInternalKeypointArray>(list_length);
110 _new_points_internal->resize(list_length);
111 _coefficient_table = arm_compute::support::cpp14::make_unique<CLCoefficientTableArray>(list_length);
112 _coefficient_table->resize(list_length);
113 _old_values = arm_compute::support::cpp14::make_unique<CLOldValueArray>(old_values_list_length);
114 _old_values->resize(old_values_list_length);
115 _new_points->resize(list_length);
116
117 for(size_t i = 0; i < _num_levels; ++i)
118 {
119 // Get images from the ith level of old and right pyramid
120 ICLImage *old_ith_input = old_pyramid->get_pyramid_level(i);
121 ICLImage *new_ith_input = new_pyramid->get_pyramid_level(i);
122
123 // Get width and height of images
124 const unsigned int width_ith = old_ith_input->info()->dimension(0);
125 const unsigned int height_ith = new_ith_input->info()->dimension(1);
126
127 // Initialize Scharr tensors
128 TensorInfo tensor_info(TensorShape(width_ith, height_ith), 1, DataType::S16);
129 _scharr_gx[i].allocator()->init(tensor_info);
130 _scharr_gy[i].allocator()->init(tensor_info);
131
132 // Manage intermediate buffers
133 _memory_group.manage(&_scharr_gx[i]);
134 _memory_group.manage(&_scharr_gy[i]);
135
136 // Init Scharr kernel
137 _func_scharr[i].configure(compile_context, old_ith_input, &_scharr_gx[i], &_scharr_gy[i], border_mode, constant_border_value);
138
139 // Init Lucas-Kanade init kernel
140 _tracker_init_kernel.emplace_back(support::cpp14::make_unique<CLLKTrackerInitKernel>());
141 _tracker_init_kernel.back()->configure(compile_context, old_points, new_points_estimates, _old_points_internal.get(), _new_points_internal.get(), use_initial_estimate, i, _num_levels, pyr_scale);
142
143 // Init Lucas-Kanade stage0 kernel
144 _tracker_stage0_kernel.emplace_back(support::cpp14::make_unique<CLLKTrackerStage0Kernel>());
145 _tracker_stage0_kernel.back()->configure(compile_context, old_ith_input, &_scharr_gx[i], &_scharr_gy[i],
146 _old_points_internal.get(), _new_points_internal.get(), _coefficient_table.get(), _old_values.get(),
147 window_dimension, i);
148
149 // Init Lucas-Kanade stage1 kernel
150 _tracker_stage1_kernel.emplace_back(support::cpp14::make_unique<CLLKTrackerStage1Kernel>());
151 _tracker_stage1_kernel.back()->configure(compile_context, new_ith_input, _new_points_internal.get(), _coefficient_table.get(), _old_values.get(),
152 termination, epsilon, num_iterations, window_dimension, i);
153
154 // Allocate intermediate buffers
155 _scharr_gx[i].allocator()->allocate();
156 _scharr_gy[i].allocator()->allocate();
157 }
158
159 // Finalize Lucas-Kanade
160 _tracker_finalize_kernel->configure(compile_context, _new_points_internal.get(), new_points);
161 }
162
run()163 void CLOpticalFlow::run()
164 {
165 ARM_COMPUTE_ERROR_ON_MSG(_num_levels == 0, "Unconfigured function");
166
167 MemoryGroupResourceScope scope_mg(_memory_group);
168
169 for(unsigned int level = _num_levels; level > 0; --level)
170 {
171 // Run Scharr kernel
172 _func_scharr[level - 1].run();
173
174 // Run Lucas-Kanade init kernel
175 CLScheduler::get().enqueue(*_tracker_init_kernel[level - 1]);
176
177 // Run Lucas-Kanade stage0 kernel
178 CLScheduler::get().enqueue(*_tracker_stage0_kernel[level - 1]);
179
180 // Run Lucas-Kanade stage1 kernel
181 CLScheduler::get().enqueue(*_tracker_stage1_kernel[level - 1]);
182 }
183
184 CLScheduler::get().enqueue(*_tracker_finalize_kernel, true);
185 }
186