1 /*
2 * Copyright (c) 2016-2020 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 #include "arm_compute/runtime/NEON/functions/NEOpticalFlow.h"
25
26 #include "arm_compute/core/Error.h"
27 #include "arm_compute/core/ITensor.h"
28 #include "arm_compute/core/TensorInfo.h"
29 #include "arm_compute/core/Window.h"
30 #include "arm_compute/runtime/NEON/NEScheduler.h"
31 #include "arm_compute/runtime/NEON/functions/NEScharr3x3.h"
32 #include "arm_compute/runtime/Pyramid.h"
33 #include "arm_compute/runtime/Tensor.h"
34 #include "arm_compute/runtime/TensorAllocator.h"
35 #include "src/core/NEON/kernels/NEFillBorderKernel.h"
36 #include "src/core/NEON/kernels/NELKTrackerKernel.h"
37 #include "support/MemorySupport.h"
38
39 namespace arm_compute
40 {
41 NEOpticalFlow::~NEOpticalFlow() = default;
42
NEOpticalFlow(std::shared_ptr<IMemoryManager> memory_manager)43 NEOpticalFlow::NEOpticalFlow(std::shared_ptr<IMemoryManager> memory_manager) // NOLINT
44 : _memory_group(std::move(memory_manager)),
45 _func_scharr(),
46 _kernel_tracker(),
47 _scharr_gx(),
48 _scharr_gy(),
49 _new_points(nullptr),
50 _new_points_estimates(nullptr),
51 _old_points(nullptr),
52 _new_points_internal(),
53 _old_points_internal(),
54 _num_levels(0)
55 {
56 }
57
configure(const Pyramid * old_pyramid,const Pyramid * new_pyramid,const IKeyPointArray * old_points,const IKeyPointArray * new_points_estimates,IKeyPointArray * new_points,Termination termination,float epsilon,unsigned int num_iterations,size_t window_dimension,bool use_initial_estimate,BorderMode border_mode,uint8_t constant_border_value)58 void NEOpticalFlow::configure(const Pyramid *old_pyramid, const Pyramid *new_pyramid, const IKeyPointArray *old_points, const IKeyPointArray *new_points_estimates,
59 IKeyPointArray *new_points, Termination termination, float epsilon, unsigned int num_iterations, size_t window_dimension,
60 bool use_initial_estimate, BorderMode border_mode, uint8_t constant_border_value)
61 {
62 ARM_COMPUTE_ERROR_ON(nullptr == old_pyramid);
63 ARM_COMPUTE_ERROR_ON(nullptr == new_pyramid);
64 ARM_COMPUTE_ERROR_ON(nullptr == old_points);
65 ARM_COMPUTE_ERROR_ON(nullptr == new_points_estimates);
66 ARM_COMPUTE_ERROR_ON(nullptr == new_points);
67 ARM_COMPUTE_ERROR_ON(old_pyramid->info()->num_levels() != new_pyramid->info()->num_levels());
68 ARM_COMPUTE_ERROR_ON(0 == old_pyramid->info()->num_levels());
69 ARM_COMPUTE_ERROR_ON(old_pyramid->info()->width() != new_pyramid->info()->width());
70 ARM_COMPUTE_ERROR_ON(old_pyramid->info()->height() != new_pyramid->info()->height());
71 ARM_COMPUTE_ERROR_ON(use_initial_estimate && old_points->num_values() != new_points_estimates->num_values());
72
73 _num_levels = old_pyramid->info()->num_levels();
74 _old_points = old_points;
75 _new_points = new_points;
76 _new_points_estimates = new_points_estimates;
77
78 const float pyr_scale = old_pyramid->info()->scale();
79
80 _func_scharr.clear();
81 _kernel_tracker.clear();
82 _scharr_gx.clear();
83 _scharr_gy.clear();
84
85 _func_scharr.resize(_num_levels);
86 _kernel_tracker.resize(_num_levels);
87 _scharr_gx.resize(_num_levels);
88 _scharr_gy.resize(_num_levels);
89
90 _old_points_internal = LKInternalKeypointArray(old_points->num_values());
91 _new_points_internal = LKInternalKeypointArray(old_points->num_values());
92 _new_points->resize(old_points->num_values());
93
94 for(unsigned int i = 0; i < _num_levels; ++i)
95 {
96 // Get images from the ith level of old and right pyramid
97 IImage *old_ith_input = old_pyramid->get_pyramid_level(i);
98 IImage *new_ith_input = new_pyramid->get_pyramid_level(i);
99
100 // Get width and height of images
101 const unsigned int width_ith = old_ith_input->info()->dimension(0);
102 const unsigned int height_ith = new_ith_input->info()->dimension(1);
103
104 TensorInfo tensor_info(TensorShape(width_ith, height_ith), Format::S16);
105
106 _scharr_gx[i].allocator()->init(tensor_info);
107 _scharr_gy[i].allocator()->init(tensor_info);
108
109 // Manage intermediate buffers
110 _memory_group.manage(&_scharr_gx[i]);
111 _memory_group.manage(&_scharr_gy[i]);
112
113 // Init Scharr kernel
114 _func_scharr[i].configure(old_ith_input, &_scharr_gx[i], &_scharr_gy[i], border_mode, constant_border_value);
115
116 // Init Lucas-Kanade kernel
117 _kernel_tracker[i] = arm_compute::support::cpp14::make_unique<NELKTrackerKernel>();
118 _kernel_tracker[i]->configure(old_ith_input, new_ith_input, &_scharr_gx[i], &_scharr_gy[i],
119 old_points, new_points_estimates, new_points,
120 &_old_points_internal, &_new_points_internal,
121 termination, use_initial_estimate, epsilon, num_iterations, window_dimension,
122 i, _num_levels, pyr_scale);
123
124 _scharr_gx[i].allocator()->allocate();
125 _scharr_gy[i].allocator()->allocate();
126 }
127 }
128
run()129 void NEOpticalFlow::run()
130 {
131 ARM_COMPUTE_ERROR_ON_MSG(_num_levels == 0, "Unconfigured function");
132
133 MemoryGroupResourceScope scope_mg(_memory_group);
134
135 for(unsigned int level = _num_levels; level > 0; --level)
136 {
137 // Run Scharr kernel
138 _func_scharr[level - 1].run();
139
140 // Run Lucas-Kanade kernel
141 NEScheduler::get().schedule(_kernel_tracker[level - 1].get(), Window::DimX);
142 }
143 }
144 } // namespace arm_compute
145