1 /*
2 * Copyright (c) 2019 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all
15 * copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 * SOFTWARE.
24 */
25
26 #include "depthwise_dilated.hpp"
27 #include "utils.hpp"
28
29 #define MEMBERFN(TOUT) \
30 template <unsigned int OutputTileRows, unsigned int OutputTileColumns, \
31 unsigned int KernelRows, unsigned int KernelColumns, \
32 unsigned int StrideRows, unsigned int StrideColumns, typename TIn, \
33 typename TBias, typename TOut> \
34 TOUT DilatedDepthwiseConvolution<OutputTileRows, OutputTileColumns, \
35 KernelRows, KernelColumns, StrideRows, \
36 StrideColumns, TIn, TBias, TOut>
37
38 namespace depthwise {
39
MEMBERFN()40 MEMBERFN()
41 ::DilatedDepthwiseConvolution(const int n_batches, const int n_input_rows,
42 const int n_input_cols, const int n_channels,
43 const int dilation_factor,
44 nck::ActivationFunction activation,
45 const unsigned int padding_top,
46 const unsigned int padding_left,
47 const unsigned int padding_bottom,
48 const unsigned int padding_right)
49 : DilatedDepthwiseConvolution(
50 n_batches, n_input_rows, n_input_cols, n_channels, dilation_factor,
51 DilatedDepthwiseConvolution::get_output_size(
52 n_input_rows, padding_top, padding_bottom, dilation_factor),
53 DilatedDepthwiseConvolution::get_output_size(
54 n_input_cols, padding_left, padding_right, dilation_factor),
55 activation, padding_top, padding_left, padding_bottom,
56 padding_right) {}
57
MEMBERFN()58 MEMBERFN()
59 ::DilatedDepthwiseConvolution(const int n_batches, const int n_input_rows,
60 const int n_input_cols, const int n_channels,
61 const int dilation_factor,
62 const int n_output_rows, const int n_output_cols,
63 nck::ActivationFunction activation,
64 const unsigned int padding_top,
65 const unsigned int padding_left,
66 const unsigned int, // padding_bottom
67 const unsigned int // padding_right
68 )
69 : DilatedDepthwiseConvolution(
70 n_batches, n_input_rows, n_input_cols, n_channels, dilation_factor,
71 n_output_rows, n_output_cols, activation, padding_top, padding_left,
72 0, 0,
73 // Function which creates a new (standard) depthwise convolution
74 [](const int n_batches, const int n_input_rows,
75 const int n_input_cols, const int n_channels,
76 const int n_output_rows, const int n_output_cols,
77 const nck::ActivationFunction activation,
78 const unsigned int padding_top, const unsigned int padding_left,
79 const unsigned int padding_bottom,
80 const unsigned int padding_right) -> IDepthwiseConvolution * {
81 return new DepthwiseConvolution<
82 OutputTileRows, OutputTileColumns, KernelRows, KernelColumns,
83 StrideRows, StrideColumns, TIn, TBias, TOut>(
84 n_batches, n_input_rows, n_input_cols, n_channels,
85 n_output_rows, n_output_cols, activation, padding_top,
86 padding_left, padding_bottom, padding_right);
87 }) {}
88
MEMBERFN()89 MEMBERFN()
90 ::DilatedDepthwiseConvolution(
91 const int n_batches, const int n_input_rows, const int n_input_cols,
92 const int n_channels, const int dilation_factor, const int n_output_rows,
93 const int n_output_cols, nck::ActivationFunction activation,
94 const unsigned int padding_top, const unsigned int padding_left,
95 const unsigned int, // padding_bottom
96 const unsigned int, // padding_right
97 std::function<IDepthwiseConvolution *(
98 int, int, int, int, int, int, nck::ActivationFunction, unsigned int,
99 unsigned int, unsigned int, unsigned int)>
100 subconvfn // Function to create a new convolution
101 )
102 : _dilation_factor(dilation_factor), _n_input_rows(n_input_rows),
103 _n_input_cols(n_input_cols), _n_channels(n_channels),
104 _padding_top(static_cast<int>(padding_top)),
105 _padding_left(static_cast<int>(padding_left)),
106 _n_output_rows(n_output_rows), _n_output_cols(n_output_cols),
107 _convs(_dilation_factor) {
108 // Instantiate the base convolutions
109 for (uint32_t i = 0; i < static_cast<uint32_t>(_dilation_factor); i++) {
110 // Compute properties of this row of base convolutions
111 const int row_top =
112 i * StrideRows - _padding_top; // -ve values are in the padding
113 const int row_pad_top =
114 row_top < 0 ? iceildiv(-row_top, dilation_factor) : 0;
115
116 const int _n_input_rows = iceildiv(n_input_rows - i, dilation_factor);
117 const int _n_output_rows = iceildiv(n_output_rows - i, dilation_factor);
118
119 for (uint32_t j = 0; j < static_cast<uint32_t>(_dilation_factor); j++) {
120 // Compute properties of the base convolution
121 const int col_left =
122 j * StrideColumns - padding_left; // -ve values are in the padding
123 const int col_pad_left =
124 col_left < 0 ? iceildiv(-col_left, dilation_factor) : 0;
125
126 const int _n_input_cols = iceildiv(n_input_cols - j, dilation_factor);
127 const int _n_output_cols = iceildiv(n_output_cols - j, dilation_factor);
128
129 // Create new depthwise convolution engine and include it in the vector
130 // of engines. The new depthwise convolution engine is created by calling
131 // the delegate function we received as an argument.
132 _convs[i].emplace_back(subconvfn(
133 n_batches, _n_input_rows, _n_input_cols, n_channels, _n_output_rows,
134 _n_output_cols, activation,
135 // Note: since we have computed the output tensor size we don't need
136 // to explicitly provide bottom and right padding values to the
137 // depthwise convolution.
138 row_pad_top, col_pad_left, 0, 0));
139 }
140 }
141 }
142
MEMBERFN(void)143 MEMBERFN(void)::set_input(const void *const inptr) {
144 set_input(inptr, _n_channels);
145 }
146
MEMBERFN(void)147 MEMBERFN(void)::set_input(const void *const inptr, const int ldcol) {
148 set_input(inptr, _n_input_cols * ldcol, ldcol);
149 }
150
MEMBERFN(void)151 MEMBERFN(void)
152 ::set_input(const void *const inptr, const int ldrow, const int ldcol) {
153 set_input(inptr, _n_input_rows * ldrow, ldrow, ldcol);
154 }
155
MEMBERFN(void)156 MEMBERFN(void)
157 ::set_input(const void *const inptr, const int ldbatch, const int ldrow,
158 const int ldcol) {
159 // Compute dilated strides
160 const int ldrow_dilated = ldrow * _dilation_factor;
161 const int ldcol_dilated = ldcol * _dilation_factor;
162
163 // Pass input parameters on to base convolutions
164 for (uint32_t i = 0; i < static_cast<uint32_t>(_dilation_factor); i++) {
165 const int top_pos =
166 i * StrideRows - _padding_top +
167 ((static_cast<int>(i * StrideRows) < _padding_top)
168 ? iceildiv(_padding_top - i * StrideRows, _dilation_factor) *
169 _dilation_factor
170 : 0);
171 const TIn *const inptr_i =
172 static_cast<const TIn *>(inptr) + top_pos * ldrow;
173
174 for (uint32_t j = 0; j < static_cast<uint32_t>(_dilation_factor); j++) {
175 int left_pos = j * StrideColumns - _padding_left;
176 while (left_pos < 0)
177 left_pos += _dilation_factor;
178
179 // Modify the pointer to point to the first element of the dilated input
180 // tensor, then set the input for this convolution engine.
181 const void *const inptr_ij = inptr_i + left_pos * ldcol;
182 _convs[i][j]->set_input(inptr_ij, ldbatch, ldrow_dilated, ldcol_dilated);
183 }
184 }
185 }
186
MEMBERFN(void)187 MEMBERFN(void)::set_output(void *const outptr) {
188 set_output(outptr, _n_channels);
189 }
190
MEMBERFN(void)191 MEMBERFN(void)::set_output(void *const outptr, const int ldcol) {
192 set_output(outptr, _n_output_cols * ldcol, ldcol);
193 }
194
MEMBERFN(void)195 MEMBERFN(void)
196 ::set_output(void *const outptr, const int ldrow, const int ldcol) {
197 set_output(outptr, _n_output_rows * ldrow, ldrow, ldcol);
198 }
199
MEMBERFN(void)200 MEMBERFN(void)
201 ::set_output(void *const outptr, const int ldbatch, const int ldrow,
202 const int ldcol) {
203 // Compute dilated strides
204 const int ldrow_dilated = ldrow * _dilation_factor;
205 const int ldcol_dilated = ldcol * _dilation_factor;
206
207 // Pass input parameters on to base convolutions
208 for (uint32_t i = 0; i < static_cast<uint32_t>(_dilation_factor); i++) {
209 for (uint32_t j = 0; j < static_cast<uint32_t>(_dilation_factor); j++) {
210 // Modify the pointer to point to the first element of the dilated input
211 // tensor, then set the input for this convolution engine.
212 void *const outptr_ij =
213 static_cast<TOut *>(outptr) + i * ldrow + j * ldcol;
214 _convs[i][j]->set_output(outptr_ij, ldbatch, ldrow_dilated,
215 ldcol_dilated);
216 }
217 }
218 }
219
MEMBERFN(int)220 MEMBERFN(int)
221 ::get_output_size(const int dim_size, const unsigned int padding_before,
222 const unsigned int padding_after, const int dilation_factor) {
223 const int input_size =
224 dim_size + static_cast<int>(padding_before + padding_after);
225 const int window_size = (KernelRows - 1) * dilation_factor + 1;
226 return iceildiv(input_size - window_size + 1, StrideRows);
227 }
228
MEMBERFN(int)229 MEMBERFN(int)
230 ::output_size(const int dim_size, const unsigned int padding_before,
231 const unsigned int padding_after) const {
232 return get_output_size(dim_size, padding_before, padding_after,
233 _dilation_factor);
234 }
235
MEMBERFN(size_t)236 MEMBERFN(size_t)::get_packed_params_size(void) const {
237 return _convs[0][0]->get_packed_params_size();
238 }
239
MEMBERFN(void)240 MEMBERFN(void)::set_packed_params_buffer(void *buffer) {
241 // Set the buffer for all convolution engines
242 for (auto &&row : _convs) {
243 for (auto &&conv : row) {
244 conv->set_packed_params_buffer(buffer);
245 }
246 }
247 }
248
MEMBERFN(void)249 MEMBERFN(void)
250 ::pack_params(const void *const weights, const void *const biases) const {
251 _convs[0][0]->pack_params(weights, biases);
252 }
253
MEMBERFN(void)254 MEMBERFN(void)
255 ::pack_params(void *const buffer, const void *const weights,
256 const void *const biases) const {
257 _convs[0][0]->pack_params(buffer, weights, biases);
258 }
259
MEMBERFN(void)260 MEMBERFN(void)
261 ::pack_params(void *const buffer, const void *const weights,
262 const unsigned int ldrow, const unsigned int ldcol,
263 const void *const biases) const {
264 _convs[0][0]->pack_params(buffer, weights, ldrow, ldcol, biases);
265 }
266
MEMBERFN(size_t)267 MEMBERFN(size_t)::get_working_space_size(unsigned int nthreads) const {
268 return _convs[0][0]->get_working_space_size(nthreads);
269 }
270
MEMBERFN(void)271 MEMBERFN(void)::set_working_space(void *const ws) {
272 // Use the same working space set for all contained depthwise engines.
273 for (auto &&row : _convs) {
274 for (auto &&conv : row) {
275 conv->set_working_space(ws);
276 }
277 }
278 }
279
MEMBERFN(unsigned int)280 MEMBERFN(unsigned int)::get_window(void) const {
281 return _convs[0][0]->get_window();
282 }
283
MEMBERFN(void)284 MEMBERFN(void)
285 ::run(const unsigned int start, const unsigned int stop,
286 const unsigned int threadid) {
287 // Run each contained convolution in turn
288 for (auto &&row : _convs) {
289 for (auto &&conv : row) {
290 conv->run(start, stop, threadid);
291 }
292 }
293 }
294
295 } // namespace depthwise
296