1 /*
2 * Copyright (c) 2017-2020 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 #include "src/core/NEON/kernels/NEWinogradConvolutionLayerKernel.h"
25
26 #include "arm_compute/core/Error.h"
27 #include "arm_compute/core/Helpers.h"
28 #include "arm_compute/core/IAccessWindow.h"
29 #include "arm_compute/core/ITensor.h"
30 #include "arm_compute/core/TensorInfo.h"
31 #include "arm_compute/core/Validate.h"
32 #include "arm_compute/core/Window.h"
33 #include "arm_compute/core/utils/misc/ShapeCalculator.h"
34 #include "src/core/AccessWindowStatic.h"
35 #include "src/core/NEON/kernels/convolution/common/utils.hpp"
36 #include "src/core/helpers/AutoConfiguration.h"
37 #include "src/core/helpers/WindowHelpers.h"
38 #include "support/MemorySupport.h"
39
40 #include "src/core/NEON/kernels/convolution/winograd/winograd_layer.hpp"
41
42 namespace arm_compute
43 {
44 //Batched Gemms
45
46 namespace
47 {
is_kernel_size_supported(DataType data_type,Size2D size)48 inline bool is_kernel_size_supported(DataType data_type, Size2D size)
49 {
50 const std::array<Size2D, 8> f32_support = { { Size2D(1, 3), Size2D(3, 1), Size2D(5, 5), Size2D(3, 3), Size2D(1, 5), Size2D(5, 1), Size2D(7, 1), Size2D(1, 7) } };
51 const std::array<Size2D, 8> f16_support = { { Size2D(3, 3) } };
52
53 switch(data_type)
54 {
55 case DataType::F16:
56 return std::end(f16_support) != std::find(std::begin(f16_support), std::end(f16_support), size);
57 case DataType::F32:
58 return std::end(f32_support) != std::find(std::begin(f32_support), std::end(f32_support), size);
59 default:
60 return false;
61 }
62 }
63
validate_arguments_winograd_weight_trans(const ITensorInfo * input,const ITensorInfo * output,const WinogradInfo & winograd_info)64 Status validate_arguments_winograd_weight_trans(const ITensorInfo *input, const ITensorInfo *output, const WinogradInfo &winograd_info)
65 {
66 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input);
67 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(output);
68 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F16, DataType::F32);
69
70 const size_t idx_width = get_data_layout_dimension_index(input->data_layout(), DataLayoutDimension::WIDTH);
71 const size_t idx_height = get_data_layout_dimension_index(input->data_layout(), DataLayoutDimension::HEIGHT);
72 const auto input_width = input->dimension(idx_width);
73 const auto input_height = input->dimension(idx_height);
74 ARM_COMPUTE_RETURN_ERROR_ON_MSG(!is_kernel_size_supported(input->data_type(), Size2D(input_width, input_height)),
75 "Only 1x3, 3x1, 1x5, 5x1, 7x1, 1x7, 3x3 and 5x5 kernels are supported");
76 ARM_COMPUTE_RETURN_ERROR_ON(input->num_dimensions() > 4);
77 const Size2D &output_tile = winograd_info.output_tile_size;
78 const std::array<Size2D, 8> supported_tile_sizes = { { Size2D(2U, 2U), Size2D(4U, 4U), Size2D(1U, 6U), Size2D(6U, 1U), Size2D(4, 1), Size2D(1, 4), Size2D(2, 1), Size2D(1, 2) } };
79 ARM_COMPUTE_RETURN_ERROR_ON(std::end(supported_tile_sizes) == std::find(std::begin(supported_tile_sizes), std::end(supported_tile_sizes), output_tile));
80
81 // Checks performed when output is configured
82 if(output->total_size() != 0)
83 {
84 const TensorInfo tensor_info_output = input->clone()->set_tensor_shape(arm_compute::misc::shape_calculator::compute_winograd_filter_transform_shape(*input, winograd_info));
85
86 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(output, &tensor_info_output);
87 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output);
88 }
89
90 return Status{};
91 }
92
validate_and_configure_window_winograd_weight_trans(ITensorInfo * input,ITensorInfo * output,const WinogradInfo & winograd_info)93 std::pair<Status, Window> validate_and_configure_window_winograd_weight_trans(ITensorInfo *input, ITensorInfo *output, const WinogradInfo &winograd_info)
94 {
95 // Output tensor auto inizialitation if not yet initialized
96 auto_init_if_empty(*output, input->clone()->set_tensor_shape(arm_compute::misc::shape_calculator::compute_winograd_filter_transform_shape(*input, winograd_info)));
97 const Window win = calculate_max_window(*input, Steps(), true /* skip border*/);
98 return std::make_pair(Status{}, win);
99 }
100
validate_arguments_winograd_input_trans(const ITensorInfo * input,const ITensorInfo * output,const WinogradInfo & winograd_info)101 Status validate_arguments_winograd_input_trans(const ITensorInfo *input, const ITensorInfo *output, const WinogradInfo &winograd_info)
102 {
103 const Size2D &kernel_dims = winograd_info.kernel_size;
104 const PadStrideInfo &conv_info = winograd_info.convolution_info;
105 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input);
106 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(output);
107 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F16, DataType::F32);
108 ARM_COMPUTE_RETURN_ERROR_ON_MSG(conv_info.stride().first != 1 || conv_info.stride().second != 1, "Winograd input transform only supports unit strides");
109 ARM_COMPUTE_RETURN_ERROR_ON_MSG(!is_kernel_size_supported(input->data_type(), Size2D(kernel_dims.width, kernel_dims.height)),
110 "Only 1x3, 3x1, 3x3 and 5x5 kernels are supported");
111
112 // Validate configured output
113 if(output->total_size() != 0)
114 {
115 const TensorShape output_shape = misc::shape_calculator::compute_winograd_input_transform_shape(*input, winograd_info);
116
117 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DIMENSIONS(output->tensor_shape(), output_shape);
118 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output);
119 }
120
121 return Status{};
122 }
123
validate_and_configure_window_winograd_input_trans(ITensorInfo * input,ITensorInfo * output,const WinogradInfo & winograd_info)124 std::pair<Status, Window> validate_and_configure_window_winograd_input_trans(ITensorInfo *input, ITensorInfo *output, const WinogradInfo &winograd_info)
125 {
126 const TensorShape output_shape = misc::shape_calculator::compute_winograd_input_transform_shape(*input, winograd_info);
127 // Output auto inizialitation if not yet initialized
128 auto_init_if_empty(*output, input->clone()->set_tensor_shape(output_shape));
129 return std::make_pair(Status{}, calculate_max_window(*input, Steps(), true));
130 }
131
validate_arguments_winograd_output_trans(const ITensorInfo * input,const ITensorInfo * bias,const ITensorInfo * output,const WinogradInfo & winograd_info)132 Status validate_arguments_winograd_output_trans(const ITensorInfo *input, const ITensorInfo *bias, const ITensorInfo *output, const WinogradInfo &winograd_info)
133 {
134 const PadStrideInfo &conv_info = winograd_info.convolution_info;
135 const Size2D kernel_dims = winograd_info.kernel_size;
136
137 // Number of tiles along the X and Y direction
138 const unsigned int num_tiles_x = std::ceil((winograd_info.input_dimensions.x() - (kernel_dims.width - 1) + conv_info.pad_left() + conv_info.pad_right()) / static_cast<float>
139 (winograd_info.output_tile_size.width));
140 const unsigned int num_tiles_y = std::ceil((winograd_info.input_dimensions.y() - (kernel_dims.height - 1) + conv_info.pad_top() + conv_info.pad_bottom()) / static_cast<float>
141 (winograd_info.output_tile_size.height));
142 const Size2D num_tiles = Size2D(num_tiles_x, num_tiles_y);
143
144 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input);
145 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(output);
146 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F16, DataType::F32);
147 ARM_COMPUTE_RETURN_ERROR_ON(input->dimension(1) != num_tiles.area());
148 ARM_COMPUTE_RETURN_ERROR_ON_MSG(!is_kernel_size_supported(input->data_type(), Size2D(kernel_dims.width, kernel_dims.height)),
149 "Only 1x3, 3x1, 3x3 and 5x5 kernels are supported");
150
151 const std::array<unsigned int, 3> supported_gemm_sizes = { { 8U, 16U, 36U } };
152 ARM_COMPUTE_RETURN_ERROR_ON(std::end(supported_gemm_sizes) == std::find(std::begin(supported_gemm_sizes), std::end(supported_gemm_sizes), input->dimension(2)));
153 ARM_COMPUTE_UNUSED(kernel_dims);
154 if(bias != nullptr)
155 {
156 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, bias);
157 ARM_COMPUTE_RETURN_ERROR_ON(input->dimension(0) != bias->dimension(0));
158 ARM_COMPUTE_RETURN_ERROR_ON(bias->num_dimensions() != size_t(1));
159 }
160
161 // Checks performed when output is configured
162 if(output->total_size() != 0)
163 {
164 const TensorInfo tensor_info_output = input->clone()->set_tensor_shape(arm_compute::misc::shape_calculator::compute_winograd_output_transform_shape(*input, winograd_info));
165 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(output, &tensor_info_output);
166 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output);
167 }
168 return Status{};
169 }
170
validate_and_configure_window_winograd_output_trans(ITensorInfo * input,ITensorInfo * output,const WinogradInfo & winograd_info)171 std::pair<Status, Window> validate_and_configure_window_winograd_output_trans(ITensorInfo *input, ITensorInfo *output, const WinogradInfo &winograd_info)
172 {
173 // Output tensor auto initialization if not yet initialized
174 auto_init_if_empty(*output, input->clone()->set_tensor_shape(arm_compute::misc::shape_calculator::compute_winograd_output_transform_shape(*input, winograd_info)));
175
176 return std::make_pair(Status{}, calculate_max_window(*input, Steps(), true));
177 }
178 } // namespace
179
validate(const ITensorInfo * input,const ITensorInfo * weights)180 Status INEWinogradLayerTransformWeightsKernel::validate(const ITensorInfo *input, const ITensorInfo *weights)
181 {
182 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F16, DataType::F32);
183 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, weights);
184 const DataLayout data_layout = input->data_layout();
185 const unsigned int width_idx = get_data_layout_dimension_index(data_layout, DataLayoutDimension::WIDTH);
186 const unsigned int height_idx = get_data_layout_dimension_index(data_layout, DataLayoutDimension::HEIGHT);
187 ARM_COMPUTE_RETURN_ERROR_ON_MSG(!is_kernel_size_supported(input->data_type(), Size2D(weights->dimension(width_idx), weights->dimension(height_idx))),
188 "Only 1x3, 3x1, 3x3 and 5x5 kernels are supported");
189 ARM_COMPUTE_RETURN_ERROR_ON(weights->num_dimensions() > 4);
190 return Status{};
191 }
192
193 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
get_weight_storage_size(int num_output_channels,int num_input_channels) const194 unsigned int NEWinogradLayerTransformWeightsKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::get_weight_storage_size(int num_output_channels, int num_input_channels) const
195 {
196 const KernelShape shape(num_output_channels, KernelRows, KernelCols, num_input_channels);
197 return static_cast<unsigned int>(
198 // WinogradConv returns the size in bytes, we divide by `sizeof(T)` to express that in units of T
199 WinogradConv::get_kernel_storage_size(num_input_channels, num_output_channels) / sizeof(T));
200 }
201
202 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
NEWinogradLayerTransformWeightsKernel()203 NEWinogradLayerTransformWeightsKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::NEWinogradLayerTransformWeightsKernel()
204 : _transform(nullptr), _weights_hwio(nullptr), _output(nullptr), _matrix_stride(0), _num_output_channels(0), _num_input_channels(0)
205 {
206 }
207
208 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
get_matrix_stride(int num_output_channels,int num_input_channels) const209 int NEWinogradLayerTransformWeightsKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::get_matrix_stride(int num_output_channels, int num_input_channels) const
210 {
211 return WinogradConv::get_kernel_matrix_stride(num_input_channels, num_output_channels);
212 }
213
214 #ifndef DOXYGEN_SKIP_THIS
215 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
configure(const ITensor * weights_hwio,ITensor * output,const int matrix_stride,const int num_output_channels,const int num_input_channels)216 void NEWinogradLayerTransformWeightsKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::configure(
217 const ITensor *weights_hwio,
218 ITensor *output,
219 const int matrix_stride, /** Stride across matrices in the output. */
220 const int num_output_channels, /** Number of filters. */
221 const int num_input_channels) /** Number of channels in each filter. */
222 {
223 _weights_hwio = weights_hwio;
224 _output = output;
225 _matrix_stride = matrix_stride;
226 _num_output_channels = num_output_channels;
227 _num_input_channels = num_input_channels;
228 _transform = arm_compute::support::cpp14::make_unique<WeightsTransform>(num_output_channels, num_input_channels);
229
230 Window win;
231 auto win_last = _transform->get_window();
232 win.set(Window::DimX, Window::Dimension(0, win_last, 1));
233 INEKernel::configure(win);
234 }
235 #endif /* DOXYGEN_SKIP_THIS */
236
237 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
run(const Window & window,const ThreadInfo & info)238 void NEWinogradLayerTransformWeightsKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::run(const Window &window, const ThreadInfo &info)
239 {
240 ARM_COMPUTE_UNUSED(info);
241 ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
242 const size_t fst = window.x().start();
243 const size_t lst = window.x().end();
244 _transform->set_weight_tensor(_weights_hwio->buffer());
245 const int matrix_row_stride = roundup(_num_output_channels, WinogradConv::N_BLOCK);
246 _transform->set_output_matrices(_output->buffer(), _matrix_stride, matrix_row_stride);
247 _transform->set_working_space(_output->buffer());
248
249 _transform->run(fst, lst);
250 }
251
252 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
is_parallelisable() const253 bool NEWinogradLayerTransformWeightsKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::is_parallelisable() const
254 {
255 return false;
256 }
257
258 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
validate(const ITensorInfo * input,const ITensorInfo * output,const WinogradInfo & winograd_info)259 Status NEWinogradLayerTransformWeightsKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::validate(const ITensorInfo *input, const ITensorInfo *output,
260 const WinogradInfo &winograd_info)
261 {
262 ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments_winograd_weight_trans(input, output, winograd_info));
263 ARM_COMPUTE_RETURN_ON_ERROR(validate_and_configure_window_winograd_weight_trans(input->clone().get(), output->clone().get(), winograd_info).first);
264 return Status{};
265 }
266
267 template class NEWinogradLayerTransformWeightsKernel<float, 2, 2, 3, 3>;
268 template class NEWinogradLayerTransformWeightsKernel<float, 4, 4, 3, 3>;
269 template class NEWinogradLayerTransformWeightsKernel<float, 2, 2, 5, 5>;
270 template class NEWinogradLayerTransformWeightsKernel<float, 1, 6, 1, 3>;
271 template class NEWinogradLayerTransformWeightsKernel<float, 6, 1, 3, 1>;
272
273 template class NEWinogradLayerTransformWeightsKernel<float, 1, 4, 1, 5>;
274 template class NEWinogradLayerTransformWeightsKernel<float, 4, 1, 5, 1>;
275 template class NEWinogradLayerTransformWeightsKernel<float, 1, 2, 1, 7>;
276 template class NEWinogradLayerTransformWeightsKernel<float, 2, 1, 7, 1>;
277
278 #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
279 template class NEWinogradLayerTransformWeightsKernel<__fp16, 4, 4, 3, 3>;
280 #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
281
282 // Input transform
283
284 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
get_input_storage_size(int num_batches,int num_channels,int num_rows,int num_cols,bool same_padding) const285 unsigned int NEWinogradLayerTransformInputKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::get_input_storage_size(
286 int num_batches, /* Number of batches in the input tensor. */
287 int num_channels, /* Number of feature maps in the input tensor. */
288 int num_rows, /* Number of rows in each feature map. */
289 int num_cols, /* Number of columns in each feature map. */
290 bool same_padding /* Use "SAME" padding, otherwise use "VALID". */
291 ) const
292 {
293 // Construct shapes for the input and kernel tensors.
294 const Tensor4DShape input_shape(num_batches, num_rows, num_cols, num_channels);
295 const KernelShape kern_shape(1, KernelRows, KernelCols, num_channels);
296 // Return the size, converted into units of TIn
297 return static_cast<unsigned int>(WinogradConv::get_input_storage_size(num_batches, num_rows, num_cols, num_channels, same_padding) / sizeof(T));
298 }
299
300 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
get_working_space_size(unsigned int num_threads) const301 unsigned int NEWinogradLayerTransformInputKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::get_working_space_size(unsigned int num_threads) const
302 {
303 return _transform->get_working_space_size(num_threads) / sizeof(T);
304 }
305
306 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
get_matrix_stride(int num_batches,int num_channels,int num_rows,int num_cols,bool same_padding) const307 int NEWinogradLayerTransformInputKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::get_matrix_stride(
308 int num_batches, /* Number of batches in the input tensor. */
309 int num_channels, /* Number of feature maps in the input tensor. */
310 int num_rows, /* Number of rows in each feature map. */
311 int num_cols, /* Number of columns in each feature map. */
312 bool same_padding /* Use "SAME" padding, otherwise use "VALID". */) const
313 {
314 return WinogradConv::get_input_matrix_stride(num_batches, num_rows, num_cols, num_channels, same_padding);
315 }
316
317 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
NEWinogradLayerTransformInputKernel()318 NEWinogradLayerTransformInputKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::NEWinogradLayerTransformInputKernel()
319 : _transform(nullptr), _input_nhwc(nullptr), _num_batches(0), _num_rows(0), _num_cols(0), _num_channels(0), _padding(), _output(nullptr), _matrix_stride(0), _padding_top(), _padding_left(),
320 _padding_right(), _padding_bottom(), _workspace(nullptr)
321 {
322 }
323
324 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
configure(const ITensor * input_nhwc,const int num_batches,const int num_rows,const int num_cols,const int num_channels,const PaddingType padding,ITensor * output,const int matrix_stride,ITensor * workspace)325 void NEWinogradLayerTransformInputKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::configure(
326 const ITensor *input_nhwc,
327 const int num_batches, /* Number of batches in input tensor. */
328 const int num_rows, /* Number of rows in input tensor. */
329 const int num_cols, /* Number of columns in input tensor. */
330 const int num_channels, /* Number of channels in input tensor. */
331 const PaddingType padding, /* Padding type. */
332 ITensor *output, /* Base of output matrices. */
333 const int matrix_stride, /* Stride between output matrices. */
334 ITensor *workspace)
335 {
336 _input_nhwc = input_nhwc;
337 _num_batches = num_batches;
338 _num_rows = num_rows;
339 _num_cols = num_cols;
340 _num_channels = num_channels;
341 _padding = padding;
342 _output = output;
343 _matrix_stride = matrix_stride;
344 _workspace = workspace;
345
346 _padding_top = (padding == PADDING_SAME) ? (KernelRows - 1) / 2 : 0;
347 _padding_left = (padding == PADDING_SAME) ? (KernelCols - 1) / 2 : 0;
348 _padding_bottom = (padding == PADDING_SAME) ? iceildiv(KernelRows - 1, 2) : 0;
349 _padding_right = (padding == PADDING_SAME) ? iceildiv(KernelCols - 1, 2) : 0;
350
351 _transform = arm_compute::support::cpp14::make_unique<InputTransform>(
352 KernelRows,
353 KernelCols,
354 num_batches,
355 num_rows,
356 num_cols,
357 num_channels,
358 _padding_top, /**< Padding to apply to the top of the image. */
359 _padding_left, /**< Padding to apply to the left of the image. */
360 _padding_bottom, /**< Padding to apply to the bottom of the image. */
361 _padding_right /**< Padding to apply to the right of the image. */
362 );
363
364 Window win;
365 auto win_last = _transform->get_window();
366 win.set(Window::DimX, Window::Dimension(0, win_last, 1));
367 INEKernel::configure(win);
368 }
369
370 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
run(const Window & window,const ThreadInfo & info)371 void NEWinogradLayerTransformInputKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::run(const Window &window, const ThreadInfo &info)
372 {
373 ARM_COMPUTE_UNUSED(info);
374 ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
375 ARM_COMPUTE_ERROR_ON_NULLPTR(_workspace);
376
377 const int element_size_in_bytes = _input_nhwc->info()->element_size();
378 const int input_col_stride = _input_nhwc->info()->strides_in_bytes().y() / element_size_in_bytes;
379 const int input_row_stride = _input_nhwc->info()->strides_in_bytes().z() / element_size_in_bytes;
380 const int input_batch_stride = _input_nhwc->info()->strides_in_bytes()[3] / element_size_in_bytes;
381 const auto input_nhwc_ptr = reinterpret_cast<const T *>(_input_nhwc->buffer() + _input_nhwc->info()->offset_first_element_in_bytes());
382 auto output_ptr = reinterpret_cast<T *>(_output->buffer() + _output->info()->offset_first_element_in_bytes());
383 ARM_COMPUTE_ERROR_ON_NULLPTR(output_ptr);
384
385 _transform->set_input_tensor(input_nhwc_ptr, input_batch_stride, input_row_stride, input_col_stride);
386 _transform->set_output_matrices(output_ptr, _matrix_stride, _num_channels);
387
388 _transform->set_working_space(_workspace->buffer());
389
390 // The code below cannot be moved to configure because biases hasn't been allocated at that point
391 const size_t fst = window.x().start();
392 const size_t lst = window.x().end();
393 _transform->run(fst, lst, info.thread_id);
394 }
395
396 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
validate(const ITensorInfo * input,const ITensorInfo * output,const WinogradInfo & winograd_info)397 Status NEWinogradLayerTransformInputKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::validate(const ITensorInfo *input, const ITensorInfo *output, const WinogradInfo &winograd_info)
398 {
399 ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments_winograd_input_trans(input, output, winograd_info));
400 ARM_COMPUTE_RETURN_ON_ERROR(validate_and_configure_window_winograd_input_trans(input->clone().get(), output->clone().get(), winograd_info).first);
401
402 return Status{};
403 }
404
405 template class NEWinogradLayerTransformInputKernel<float, 2, 2, 3, 3>;
406 template class NEWinogradLayerTransformInputKernel<float, 4, 4, 3, 3>;
407 template class NEWinogradLayerTransformInputKernel<float, 2, 2, 5, 5>;
408 template class NEWinogradLayerTransformInputKernel<float, 1, 6, 1, 3>;
409 template class NEWinogradLayerTransformInputKernel<float, 6, 1, 3, 1>;
410
411 template class NEWinogradLayerTransformInputKernel<float, 1, 4, 1, 5>;
412 template class NEWinogradLayerTransformInputKernel<float, 4, 1, 5, 1>;
413 template class NEWinogradLayerTransformInputKernel<float, 1, 2, 1, 7>;
414 template class NEWinogradLayerTransformInputKernel<float, 2, 1, 7, 1>;
415
416 #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
417 template class NEWinogradLayerTransformInputKernel<__fp16, 4, 4, 3, 3>;
418 #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
419
420 // Output transform
421
422 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
get_output_storage_size(int num_batches,int num_rows,int num_cols,int num_output_channels) const423 unsigned int NEWinogradLayerTransformOutputKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::get_output_storage_size(
424 int num_batches, /* Number of batches in the output tensor. */
425 int num_rows, /* Number of rows in each feature map of the input tensor. */
426 int num_cols, /* Number of columns in each feature map of the input tensor. */
427 int num_output_channels /* Number of feature maps in the output tensor. */
428 ) const
429 {
430 // Construct shapes for the input and kernel tensors.
431 const Tensor4DShape input_shape(num_batches, num_rows, num_cols, 1);
432 const KernelShape kern_shape(num_output_channels, KernelRows, KernelCols, 1);
433 // Return the size, converted into units of TOut
434 return static_cast<unsigned int>(
435 WinogradConv::get_output_storage_size(num_batches, num_rows, num_cols, num_output_channels) / sizeof(T));
436 }
437
438 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
NEWinogradLayerTransformOutputKernel()439 NEWinogradLayerTransformOutputKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::NEWinogradLayerTransformOutputKernel()
440 : _transform(nullptr), _biases(nullptr), _transformed_output(nullptr), _workspace(nullptr), _matrix_stride(0), _matrix_row_stride(0), _output_nhwc(nullptr), _num_batches(0), _num_rows(0),
441 _num_cols(0), _num_channels(0)
442 {
443 }
444
445 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
get_working_space_size(unsigned int num_threads) const446 unsigned int NEWinogradLayerTransformOutputKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::get_working_space_size(unsigned int num_threads) const
447 {
448 return _transform->get_working_space_size(num_threads) / sizeof(T);
449 }
450
451 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
get_matrix_stride(int num_batches,int num_rows,int num_cols,int num_output_channels) const452 int NEWinogradLayerTransformOutputKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::get_matrix_stride(
453 int num_batches, /* Number of batches in the output tensor. */
454 int num_rows, /* Number of rows in each feature map of the input tensor. */
455 int num_cols, /* Number of columns in each feature map of the input tensor. */
456 int num_output_channels /* Number of feature maps in the output tensor. */
457 ) const
458 {
459 return WinogradConv::get_output_matrix_stride(num_batches, num_rows, num_cols, num_output_channels);
460 }
461
462 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
get_output_shape(int num_rows,int num_cols,bool padding_same) const463 std::pair<unsigned int, unsigned int> NEWinogradLayerTransformOutputKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::get_output_shape(
464 int num_rows, /* Number of rows in each feature map of the input tensor. */
465 int num_cols, /* Number of columns in each feature map of the input tensor. */
466 bool padding_same) const
467 {
468 return WinogradConv::get_output_shape(std::make_pair<unsigned int, unsigned int>(num_rows, num_cols), padding_same);
469 }
470
471 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
configure(const ITensor * biases,const ITensor * transformed_output,const int matrix_stride,ITensor * output_nhwc,const int num_batches,const int num_rows,const int num_cols,const int num_channels,ITensor * workspace,const arm_gemm::Activation & activation)472 void NEWinogradLayerTransformOutputKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::configure(
473 const ITensor *biases,
474 const ITensor *transformed_output,
475 const int matrix_stride,
476 ITensor *output_nhwc,
477 const int num_batches,
478 const int num_rows,
479 const int num_cols,
480 const int num_channels,
481 ITensor *workspace,
482 const arm_gemm::Activation &activation)
483 {
484 _biases = biases;
485 _workspace = workspace;
486 _transformed_output = transformed_output;
487 _matrix_stride = matrix_stride;
488 _matrix_row_stride = roundup(num_channels, WinogradConv::N_BLOCK);
489 _output_nhwc = output_nhwc;
490 _num_batches = num_batches;
491 _num_rows = num_rows;
492 _num_cols = num_cols;
493 _num_channels = num_channels;
494 // We don't have the biases buffer at this stage as it hasn't been allocated, we pass in nullptr OutputTransform is only used here to compute the window
495 _transform = arm_compute::support::cpp14::make_unique<OutputTransform>(num_batches, num_rows, num_cols, num_channels, activation);
496 Window win;
497 auto win_last = _transform->get_window();
498 win.set(Window::DimX, Window::Dimension(0, win_last, 1));
499 _output_nhwc->info()->set_valid_region(ValidRegion(Coordinates(), _output_nhwc->info()->tensor_shape()));
500
501 INEKernel::configure(win);
502 }
503
504 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
run(const Window & window,const ThreadInfo & info)505 void NEWinogradLayerTransformOutputKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::run(const Window &window, const ThreadInfo &info)
506 {
507 ARM_COMPUTE_UNUSED(info);
508 ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
509 ARM_COMPUTE_ERROR_ON_NULLPTR(_workspace);
510 ARM_COMPUTE_ERROR_ON_NULLPTR(_transformed_output);
511 ARM_COMPUTE_ERROR_ON_NULLPTR(_output_nhwc);
512
513 const int out_batch_stride = _output_nhwc->info()->strides_in_bytes()[3] / sizeof(T);
514 const int out_row_stride = _output_nhwc->info()->strides_in_bytes()[2] / sizeof(T);
515 const int out_col_stride = _output_nhwc->info()->strides_in_bytes()[1] / sizeof(T);
516
517 _transform->set_input_matrices(_transformed_output->buffer(), _matrix_stride, _matrix_row_stride);
518 _transform->set_bias((_biases ? reinterpret_cast<T *>(_biases->buffer() + _biases->info()->offset_first_element_in_bytes()) : nullptr));
519 _transform->set_output_tensor(_output_nhwc->buffer() + _output_nhwc->info()->offset_first_element_in_bytes(), out_batch_stride, out_row_stride, out_col_stride);
520 _transform->set_working_space(_workspace->buffer());
521 // The code below cannot be moved to configure because biases hasn't been allocated at that point
522 const size_t fst = window.x().start();
523 const size_t lst = window.x().end();
524 _transform->run(fst, lst, info.thread_id);
525 }
526
527 template <typename T, int OutputTileRows, int OutputTileCols, int KernelRows, int KernelCols>
validate(const ITensorInfo * input,const ITensorInfo * bias,const ITensorInfo * output,const WinogradInfo & winograd_info)528 Status NEWinogradLayerTransformOutputKernel<T, OutputTileRows, OutputTileCols, KernelRows, KernelCols>::validate(const ITensorInfo *input, const ITensorInfo *bias, const ITensorInfo *output,
529 const WinogradInfo &winograd_info)
530 {
531 ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments_winograd_output_trans(input, (bias != nullptr ? bias->clone().get() : nullptr), output, winograd_info));
532 ARM_COMPUTE_RETURN_ON_ERROR(validate_and_configure_window_winograd_output_trans(input->clone().get(), output->clone().get(), winograd_info).first);
533
534 return Status{};
535 }
536
537 template class NEWinogradLayerTransformOutputKernel<float, 2, 2, 3, 3>;
538 template class NEWinogradLayerTransformOutputKernel<float, 4, 4, 3, 3>;
539 template class NEWinogradLayerTransformOutputKernel<float, 2, 2, 5, 5>;
540 template class NEWinogradLayerTransformOutputKernel<float, 1, 6, 1, 3>;
541 template class NEWinogradLayerTransformOutputKernel<float, 6, 1, 3, 1>;
542
543 template class NEWinogradLayerTransformOutputKernel<float, 1, 4, 1, 5>;
544 template class NEWinogradLayerTransformOutputKernel<float, 4, 1, 5, 1>;
545 template class NEWinogradLayerTransformOutputKernel<float, 1, 2, 1, 7>;
546 template class NEWinogradLayerTransformOutputKernel<float, 2, 1, 7, 1>;
547
548 #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
549 template class NEWinogradLayerTransformOutputKernel<__fp16, 4, 4, 3, 3>;
550 #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
551 } // namespace arm_compute
552