1 /*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "Operations"
18
19 #include "Conv2D.h"
20
21 #include <algorithm>
22 #include <iterator>
23 #include <memory>
24 #include <vector>
25
26 #include "LegacyUtils.h"
27 #include "OperationResolver.h"
28 #include "Operations.h"
29 #include "OperationsExecutionUtils.h"
30 #include "Tracing.h"
31
32 #ifdef NN_INCLUDE_CPU_IMPLEMENTATION
33 #pragma clang diagnostic push
34 #pragma clang diagnostic ignored "-Wunused-parameter"
35 #pragma clang diagnostic ignored "-Wsign-compare"
36 #pragma clang diagnostic ignored "-Winvalid-partial-specialization"
37 #include <tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h>
38 #include <tensorflow/lite/kernels/internal/reference/integer_ops/conv.h>
39 #include <tensorflow/lite/kernels/internal/types.h>
40 #pragma clang diagnostic pop
41
42 #include "CpuOperationUtils.h"
43 #endif // NN_INCLUDE_CPU_IMPLEMENTATION
44
45 namespace android {
46 namespace nn {
47 namespace conv_2d {
48
49 #ifdef NN_INCLUDE_CPU_IMPLEMENTATION
50 namespace {
51
52 // If possible we will use this static buffer for the tensor.
53 constexpr size_t kStaticBufferSize = 1605632;
54 [[maybe_unused]] char static_scratch_buffer[kStaticBufferSize];
55
56 // executionMutex is used to protect concurrent access of the static_scratch_buffer
57 // and other non-threadsafe resources like gemmlowp::GemmContext.
58 // std::mutex is safe for pthreads on Android.
59 std::mutex executionMutex;
60
61 struct Conv2dParam {
62 int32_t padding_left, padding_right;
63 int32_t padding_top, padding_bottom;
64 int32_t stride_width, stride_height;
65 int32_t dilation_width_factor = 1, dilation_height_factor = 1;
66 int32_t activation;
67 bool useNchw = false;
68
initializeandroid::nn::conv_2d::__anon72a8f1430111::Conv2dParam69 bool initialize(const IOperationExecutionContext* context) {
70 uint32_t inCount = context->getNumInputs();
71 int32_t padding_implicit = 0;
72 bool useImplicitPadding = false;
73 if ((inCount >= 8 && context->getInputType(7) == OperandType::BOOL) || inCount == 7) {
74 padding_implicit = context->getInputValue<int32_t>(3);
75 stride_width = context->getInputValue<int32_t>(4);
76 stride_height = context->getInputValue<int32_t>(5);
77 activation = context->getInputValue<int32_t>(6);
78 if (inCount >= 8) {
79 useNchw = context->getInputValue<bool>(7);
80 }
81 if (inCount == 10) {
82 dilation_width_factor = context->getInputValue<int32_t>(8);
83 dilation_height_factor = context->getInputValue<int32_t>(9);
84 }
85 useImplicitPadding = true;
86 } else if (inCount >= 10 && context->getInputType(7) == OperandType::INT32) {
87 padding_left = context->getInputValue<int32_t>(3);
88 padding_right = context->getInputValue<int32_t>(4);
89 padding_top = context->getInputValue<int32_t>(5);
90 padding_bottom = context->getInputValue<int32_t>(6);
91 stride_width = context->getInputValue<int32_t>(7);
92 stride_height = context->getInputValue<int32_t>(8);
93 activation = context->getInputValue<int32_t>(9);
94 if (inCount >= 11) {
95 useNchw = context->getInputValue<bool>(10);
96 }
97 if (inCount == 13) {
98 dilation_width_factor = context->getInputValue<int32_t>(11);
99 dilation_height_factor = context->getInputValue<int32_t>(12);
100 }
101 } else {
102 NN_RET_CHECK_FAIL() << "Unsupported input spec for operation " << kOperationName;
103 }
104 if (useImplicitPadding) {
105 Shape inputShape = context->getInputShape(kInputTensor);
106 Shape filterShape = context->getInputShape(kFilterTensor);
107 int32_t input_width = getSizeOfDimension(inputShape, useNchw ? 3 : 2);
108 int32_t input_height = getSizeOfDimension(inputShape, useNchw ? 2 : 1);
109 int32_t filter_width = getSizeOfDimension(filterShape, 2);
110 int32_t filter_height = getSizeOfDimension(filterShape, 1);
111 calculateExplicitPadding(input_width, stride_width, dilation_width_factor, filter_width,
112 padding_implicit, &padding_left, &padding_right);
113 calculateExplicitPadding(input_height, stride_height, dilation_height_factor,
114 filter_height, padding_implicit, &padding_top,
115 &padding_bottom);
116 }
117 NN_RET_CHECK_GE(padding_left, 0);
118 NN_RET_CHECK_GE(padding_right, 0);
119 NN_RET_CHECK_GE(padding_top, 0);
120 NN_RET_CHECK_GE(padding_bottom, 0);
121 NN_RET_CHECK_GT(stride_width, 0);
122 NN_RET_CHECK_GT(stride_height, 0);
123 NN_RET_CHECK_GT(dilation_width_factor, 0);
124 NN_RET_CHECK_GT(dilation_height_factor, 0);
125 NN_RET_CHECK_GE(activation, 0);
126 return true;
127 }
128 };
129
130 #define ANDROID_NN_CONV_PARAMETERS(Type) \
131 [[maybe_unused]] uint32_t height = getSizeOfDimension(inputShape, 1); \
132 [[maybe_unused]] uint32_t width = getSizeOfDimension(inputShape, 2); \
133 uint32_t filterHeight = getSizeOfDimension(filterShape, 1); \
134 uint32_t filterWidth = getSizeOfDimension(filterShape, 2); \
135 [[maybe_unused]] uint32_t outHeight = getSizeOfDimension(outputShape, 1); \
136 [[maybe_unused]] uint32_t outWidth = getSizeOfDimension(outputShape, 2); \
137 uint32_t inDepth = getSizeOfDimension(inputShape, 3); \
138 \
139 uint32_t paddingHeight = (uint32_t)padding_top; \
140 uint32_t paddingWidth = (uint32_t)padding_left; \
141 \
142 tflite::Dims<4> im2colDim; \
143 im2colDim.sizes[3] = (int)getSizeOfDimension(outputShape, 0); \
144 im2colDim.sizes[2] = (int)getSizeOfDimension(outputShape, 1); \
145 im2colDim.sizes[1] = (int)getSizeOfDimension(outputShape, 2); \
146 im2colDim.sizes[0] = (int)inDepth * filterHeight * filterWidth; \
147 \
148 im2colDim.strides[0] = 1; \
149 for (int i = 1; i < 4; i++) { \
150 im2colDim.strides[i] = im2colDim.strides[i - 1] * im2colDim.sizes[i - 1]; \
151 } \
152 \
153 Type* im2colData = nullptr; \
154 uint64_t im2colByteSize = sizeof(Type); \
155 std::unique_ptr<Type[]> im2colGuard; \
156 for (int i = 0; i < 4; i++) { \
157 im2colByteSize *= im2colDim.sizes[i]; \
158 } \
159 /* http://b/77982879, tflite::optimized_ops::Conv uses int for offsets */ \
160 if (im2colByteSize >= 0x7fffffff) { \
161 LOG(ERROR) << "Conv size is too large, not enough memory"; \
162 return false; \
163 } \
164 if (im2colByteSize <= kStaticBufferSize) { \
165 im2colData = reinterpret_cast<Type*>(static_scratch_buffer); \
166 } else { \
167 im2colData = new (std::nothrow) Type[im2colByteSize / sizeof(Type)]; \
168 if (im2colData == nullptr) { \
169 LOG(ERROR) << "Conv size is too large, not enough memory"; \
170 return false; \
171 } \
172 im2colGuard.reset(im2colData); \
173 }
174
needim2colData(const Shape & filterShape,int32_t stride_width,int32_t stride_height,int32_t dilation_width_factor,int32_t dilation_height_factor)175 bool needim2colData(const Shape& filterShape, int32_t stride_width, int32_t stride_height,
176 int32_t dilation_width_factor, int32_t dilation_height_factor) {
177 // Within tflite::optimized_ops::Conv, the following tests are performed,
178 // and in the case (!need_dilated_im2col && !need_im2col), then the
179 // method doesn't expect to receive outputData. In debug mode this is
180 // asserted and fails tests, so we need to perform this check as the caller
181 // also. See:
182 // tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h:2655
183 const int filter_width = getSizeOfDimension(filterShape, 2);
184 const int filter_height = getSizeOfDimension(filterShape, 1);
185 const bool need_dilated_im2col = dilation_width_factor != 1 || dilation_height_factor != 1;
186 const bool need_im2col =
187 stride_width != 1 || stride_height != 1 || filter_width != 1 || filter_height != 1;
188 return need_dilated_im2col || need_im2col;
189 }
190
convNhwc(const float * inputData,const Shape & inputShape,const float * filterData,const Shape & filterShape,const float * biasData,const Shape & biasShape,int32_t padding_left,int32_t,int32_t padding_top,int32_t,int32_t stride_width,int32_t stride_height,int32_t dilation_width_factor,int32_t dilation_height_factor,int32_t activation,float * outputData,const Shape & outputShape)191 bool convNhwc(const float* inputData, const Shape& inputShape, const float* filterData,
192 const Shape& filterShape, const float* biasData, const Shape& biasShape,
193 int32_t padding_left, int32_t /*padding_right*/, int32_t padding_top,
194 int32_t /*padding_bottom*/, int32_t stride_width, int32_t stride_height,
195 int32_t dilation_width_factor, int32_t dilation_height_factor, int32_t activation,
196 float* outputData, const Shape& outputShape) {
197 NNTRACE_TRANS("convFloat32");
198
199 ANDROID_NN_CONV_PARAMETERS(float)
200
201 float output_activation_min, output_activation_max;
202 CalculateActivationRangeFloat(activation, &output_activation_min, &output_activation_max);
203
204 // Prevent concurrent executions that may access the scratch buffer.
205 std::unique_lock<std::mutex> lock(executionMutex);
206 NNTRACE_COMP_SWITCH("optimized_ops::Conv");
207
208 const bool need_im2colData = needim2colData(filterShape, stride_width, stride_height,
209 dilation_width_factor, dilation_height_factor);
210
211 tflite::optimized_ops::Conv(
212 inputData, convertShapeToDims(inputShape), filterData, convertShapeToDims(filterShape),
213 biasData, convertShapeToDims(biasShape), stride_width, stride_height,
214 dilation_width_factor, dilation_height_factor, paddingWidth, paddingHeight,
215 output_activation_min, output_activation_max, outputData,
216 convertShapeToDims(outputShape), need_im2colData ? im2colData : nullptr, im2colDim);
217 return true;
218 }
219
convNhwc(const uint8_t * inputData,const Shape & inputShape,const uint8_t * filterData,const Shape & filterShape,const int32_t * biasData,const Shape & biasShape,int32_t padding_left,int32_t,int32_t padding_top,int32_t,int32_t stride_width,int32_t stride_height,int32_t dilation_width_factor,int32_t dilation_height_factor,int32_t activation,uint8_t * outputData,const Shape & outputShape)220 bool convNhwc(const uint8_t* inputData, const Shape& inputShape, const uint8_t* filterData,
221 const Shape& filterShape, const int32_t* biasData, const Shape& biasShape,
222 int32_t padding_left, int32_t /*padding_right*/, int32_t padding_top,
223 int32_t /*padding_bottom*/, int32_t stride_width, int32_t stride_height,
224 int32_t dilation_width_factor, int32_t dilation_height_factor, int32_t activation,
225 uint8_t* outputData, const Shape& outputShape) {
226 NNTRACE_TRANS("convQuant8");
227
228 ANDROID_NN_CONV_PARAMETERS(uint8_t)
229
230 int32_t inputOffset = -inputShape.offset;
231 int32_t filterOffset = -filterShape.offset;
232 int32_t outputOffset = outputShape.offset;
233
234 double real_multiplier = 0.0;
235 int32_t output_multiplier = 0;
236 int32_t output_shift = 0;
237 int32_t output_activation_min = 0;
238 int32_t output_activation_max = 0;
239
240 NN_RET_CHECK(GetQuantizedConvolutionMultiplier(inputShape, filterShape, biasShape, outputShape,
241 &real_multiplier));
242 int exponent;
243 NN_RET_CHECK(QuantizeMultiplier(real_multiplier, &output_multiplier, &exponent));
244 output_shift = -exponent;
245 CalculateActivationRangeUint8(activation, outputShape, &output_activation_min,
246 &output_activation_max);
247
248 static gemmlowp::GemmContext gemm_context;
249
250 // Prevent concurrent executions that may access the scratch buffer and
251 // gemm_context.
252 std::unique_lock<std::mutex> lock(executionMutex);
253 // Alow gemmlowp automatically decide how many threads to use.
254 gemm_context.set_max_num_threads(0);
255
256 NNTRACE_COMP_SWITCH("optimized_ops::Conv");
257
258 const bool need_im2colData = needim2colData(filterShape, stride_width, stride_height,
259 dilation_width_factor, dilation_height_factor);
260
261 tflite::optimized_ops::Conv(inputData, convertShapeToDims(inputShape), inputOffset, filterData,
262 convertShapeToDims(filterShape), filterOffset, biasData,
263 convertShapeToDims(biasShape), stride_width, stride_height,
264 dilation_width_factor, dilation_height_factor, paddingWidth,
265 paddingHeight, outputOffset, output_multiplier, output_shift,
266 output_activation_min, output_activation_max, outputData,
267 convertShapeToDims(outputShape),
268 need_im2colData ? im2colData : nullptr, im2colDim, &gemm_context);
269 return true;
270 }
271
272 // Passing input, filter and output shapes by value, so that we can change the
273 // offsets without modifying the actual shapes.
convNhwc(const int8_t * inputData,Shape inputShape,const int8_t * filterData,Shape filterShape,const int32_t * biasData,const Shape & biasShape,int32_t padding_left,int32_t padding_right,int32_t padding_top,int32_t padding_bottom,int32_t stride_width,int32_t stride_height,int32_t dilation_width_factor,int32_t dilation_height_factor,int32_t activation,int8_t * outputData,Shape outputShape)274 bool convNhwc(const int8_t* inputData, Shape inputShape, const int8_t* filterData,
275 Shape filterShape, const int32_t* biasData, const Shape& biasShape,
276 int32_t padding_left, int32_t padding_right, int32_t padding_top,
277 int32_t padding_bottom, int32_t stride_width, int32_t stride_height,
278 int32_t dilation_width_factor, int32_t dilation_height_factor, int32_t activation,
279 int8_t* outputData, Shape outputShape) {
280 NNTRACE_TRANS("convQuant8");
281
282 std::vector<uint8_t> unsignedInput(getNumberOfElements(inputShape));
283 convertInt8ToUInt8(inputData, &unsignedInput);
284 inputShape.offset += 128;
285
286 std::vector<uint8_t> unsignedFilter(getNumberOfElements(filterShape));
287 convertInt8ToUInt8(filterData, &unsignedFilter);
288 filterShape.offset += 128;
289
290 std::vector<uint8_t> unsignedOutput(getNumberOfElements(outputShape));
291 outputShape.offset += 128;
292
293 NN_RET_CHECK(convNhwc(unsignedInput.data(), inputShape, unsignedFilter.data(), filterShape,
294 biasData, biasShape, padding_left, padding_right, padding_top,
295 padding_bottom, stride_width, stride_height, dilation_width_factor,
296 dilation_height_factor, activation, unsignedOutput.data(), outputShape));
297
298 convertUInt8ToInt8(unsignedOutput, outputData);
299
300 return true;
301 }
302
convNhwc(const _Float16 * inputData,const Shape & inputShape,const _Float16 * filterData,const Shape & filterShape,const _Float16 * biasData,const Shape & biasShape,int32_t padding_left,int32_t padding_right,int32_t padding_top,int32_t padding_bottom,int32_t stride_width,int32_t stride_height,int32_t dilation_width_factor,int32_t dilation_height_factor,int32_t activation,_Float16 * outputData,const Shape & outputShape)303 bool convNhwc(const _Float16* inputData, const Shape& inputShape, const _Float16* filterData,
304 const Shape& filterShape, const _Float16* biasData, const Shape& biasShape,
305 int32_t padding_left, int32_t padding_right, int32_t padding_top,
306 int32_t padding_bottom, int32_t stride_width, int32_t stride_height,
307 int32_t dilation_width_factor, int32_t dilation_height_factor, int32_t activation,
308 _Float16* outputData, const Shape& outputShape) {
309 NNTRACE_TRANS("convFloat16");
310
311 std::vector<float> inputData_float32(getNumberOfElements(inputShape));
312 std::vector<float> filterData_float32(getNumberOfElements(filterShape));
313 std::vector<float> biasData_float32(getNumberOfElements(biasShape));
314 std::vector<float> outputData_float32(getNumberOfElements(outputShape));
315
316 convertFloat16ToFloat32(inputData, &inputData_float32);
317 convertFloat16ToFloat32(filterData, &filterData_float32);
318 convertFloat16ToFloat32(biasData, &biasData_float32);
319
320 convNhwc(inputData_float32.data(), inputShape, filterData_float32.data(), filterShape,
321 biasData_float32.data(), biasShape, padding_left, padding_right, padding_top,
322 padding_bottom, stride_width, stride_height, dilation_width_factor,
323 dilation_height_factor, activation, outputData_float32.data(), outputShape);
324 convertFloat32ToFloat16(outputData_float32, outputData);
325
326 return true;
327 }
328
329 template <typename T_Input, typename T_Filter, typename T_Bias>
conv(const T_Input * inputData,const Shape & inputShape,const T_Filter * filterData,const Shape & filterShape,const T_Bias * biasData,const Shape & biasShape,int32_t padding_left,int32_t padding_right,int32_t padding_top,int32_t padding_bottom,int32_t stride_width,int32_t stride_height,int32_t dilation_width_factor,int32_t dilation_height_factor,int32_t activation,bool useNchw,T_Input * outputData,const Shape & outputShape)330 bool conv(const T_Input* inputData, const Shape& inputShape, const T_Filter* filterData,
331 const Shape& filterShape, const T_Bias* biasData, const Shape& biasShape,
332 int32_t padding_left, int32_t padding_right, int32_t padding_top, int32_t padding_bottom,
333 int32_t stride_width, int32_t stride_height, int32_t dilation_width_factor,
334 int32_t dilation_height_factor, int32_t activation, bool useNchw, T_Input* outputData,
335 const Shape& outputShape) {
336 InputWithLayout<T_Input> input(useNchw);
337 OutputWithLayout<T_Input> output(useNchw);
338 NN_RET_CHECK(input.initialize(inputData, inputShape));
339 NN_RET_CHECK(output.initialize(outputData, outputShape));
340 NN_RET_CHECK(convNhwc(input.getNhwcBuffer(), input.getNhwcShape(), filterData, filterShape,
341 biasData, biasShape, padding_left, padding_right, padding_top,
342 padding_bottom, stride_width, stride_height, dilation_width_factor,
343 dilation_height_factor, activation, output.getNhwcBuffer(),
344 output.getNhwcShape()));
345 NN_RET_CHECK(output.commit());
346 return true;
347 }
348
convQuant8PerChannelNhwc(const uint8_t * inputData,const Shape & inputShape,const int8_t * filterData,const Shape & filterShape,const float * filterScales,const int32_t * biasData,const Shape & biasShape,int32_t paddingLeft,int32_t,int32_t paddingTop,int32_t,int32_t strideWidth,int32_t strideHeight,int32_t dilationWidthFactor,int32_t dilationHeightFactor,int32_t activation,uint8_t * outputData,const Shape & outputShape)349 bool convQuant8PerChannelNhwc(const uint8_t* inputData, const Shape& inputShape,
350 const int8_t* filterData, const Shape& filterShape,
351 const float* filterScales, const int32_t* biasData,
352 const Shape& biasShape, int32_t paddingLeft, int32_t /*paddingRight*/,
353 int32_t paddingTop, int32_t /*paddingBottom*/, int32_t strideWidth,
354 int32_t strideHeight, int32_t dilationWidthFactor,
355 int32_t dilationHeightFactor, int32_t activation, uint8_t* outputData,
356 const Shape& outputShape) {
357 NNTRACE_TRANS("convQuant8PerChannel");
358
359 uint32_t numBatches = getSizeOfDimension(inputShape, 0);
360 uint32_t inputHeight = getSizeOfDimension(inputShape, 1);
361 uint32_t inputWidth = getSizeOfDimension(inputShape, 2);
362 uint32_t inputDepth = getSizeOfDimension(inputShape, 3);
363 uint32_t filterHeight = getSizeOfDimension(filterShape, 1);
364 uint32_t filterWidth = getSizeOfDimension(filterShape, 2);
365 uint32_t filterDepth = getSizeOfDimension(filterShape, 3);
366 uint32_t outputHeight = getSizeOfDimension(outputShape, 1);
367 uint32_t outputWidth = getSizeOfDimension(outputShape, 2);
368 uint32_t outputDepth = getSizeOfDimension(outputShape, 3);
369
370 int32_t inputOffset = -inputShape.offset;
371 int32_t outputOffset = outputShape.offset;
372
373 auto realMultiplier = std::vector<double>(outputDepth, .0f);
374 auto outputMultiplier = std::vector<int32_t>(outputDepth, 0);
375 auto outputShift = std::vector<int32_t>(outputDepth, .0f);
376
377 for (uint32_t i = 0; i < outputDepth; ++i) {
378 Shape filterChannelShape = filterShape;
379 filterChannelShape.scale = filterScales[i];
380 Shape biasChannelShape = biasShape;
381 biasChannelShape.scale = filterScales[i] * inputShape.scale;
382 NN_RET_CHECK(GetQuantizedConvolutionMultiplier(
383 inputShape, filterChannelShape, biasChannelShape, outputShape, &realMultiplier[i]));
384 int exponent;
385 NN_RET_CHECK(QuantizeMultiplier(realMultiplier[i], &outputMultiplier[i], &exponent));
386 outputShift[i] = -exponent;
387 }
388
389 int32_t output_activation_min = 0, output_activation_max = 0;
390 CalculateActivationRangeUint8(activation, outputShape, &output_activation_min,
391 &output_activation_max);
392 const uint8_t* inputBase = inputData;
393 uint8_t* outPtr = outputData;
394 for (uint32_t b = 0; b < numBatches; b++) {
395 for (uint32_t h = 0; h < outputHeight; h++) {
396 for (uint32_t w = 0; w < outputWidth; w++) {
397 const int8_t* filterBase = filterData;
398
399 for (uint32_t d = 0; d < outputDepth; d++) {
400 int32_t wInputOrigin = static_cast<int32_t>(w) * strideWidth - paddingLeft;
401 int32_t hInputOrigin = static_cast<int32_t>(h) * strideHeight - paddingTop;
402 int32_t sum = 0.0f;
403
404 for (uint32_t i = 0; i < filterHeight; i++) {
405 for (uint32_t j = 0; j < filterWidth; j++) {
406 for (uint32_t k = 0; k < filterDepth; k++) {
407 int32_t hInput = hInputOrigin +
408 dilationHeightFactor * static_cast<int32_t>(i);
409 int32_t wInput = wInputOrigin +
410 dilationWidthFactor * static_cast<int32_t>(j);
411 uint32_t dInput = k;
412 if (hInput >= 0 && hInput < static_cast<int32_t>(inputHeight) &&
413 wInput >= 0 && wInput < static_cast<int32_t>(inputWidth)) {
414 uint32_t filterIndex =
415 i * filterWidth * filterDepth + j * filterDepth + k;
416 uint32_t inputIndex = hInput * inputWidth * inputDepth +
417 wInput * inputDepth + dInput;
418 sum += (static_cast<int32_t>(filterBase[filterIndex])) *
419 (static_cast<int32_t>(inputBase[inputIndex]) +
420 inputOffset);
421 }
422 }
423 }
424 }
425 sum += biasData[d];
426 sum = tflite::MultiplyByQuantizedMultiplier(sum, outputMultiplier[d],
427 -outputShift[d]);
428 sum += outputOffset;
429 sum = std::max(std::min(sum, output_activation_max), output_activation_min);
430 outPtr[d] = static_cast<uint8_t>(sum);
431 filterBase += filterHeight * filterWidth * filterDepth;
432 }
433 outPtr += outputDepth;
434 }
435 }
436 inputBase += inputHeight * inputWidth * inputDepth;
437 }
438
439 return true;
440 }
441
convQuant8PerChannelNhwc(const int8_t * inputData,const Shape & inputShape,const int8_t * filterData,const Shape & filterShape,const float * filterScales,const int32_t * biasData,const Shape & biasShape,int32_t paddingLeft,int32_t,int32_t paddingTop,int32_t,int32_t strideWidth,int32_t strideHeight,int32_t dilationWidthFactor,int32_t dilationHeightFactor,int32_t activation,int8_t * outputData,const Shape & outputShape)442 bool convQuant8PerChannelNhwc(const int8_t* inputData, const Shape& inputShape,
443 const int8_t* filterData, const Shape& filterShape,
444 const float* filterScales, const int32_t* biasData,
445 const Shape& biasShape, int32_t paddingLeft, int32_t /*paddingRight*/,
446 int32_t paddingTop, int32_t /*paddingBottom*/, int32_t strideWidth,
447 int32_t strideHeight, int32_t dilationWidthFactor,
448 int32_t dilationHeightFactor, int32_t activation, int8_t* outputData,
449 const Shape& outputShape) {
450 NNTRACE_TRANS("convQuant8SignedPerChannel");
451
452 [[maybe_unused]] uint32_t numBatches = getSizeOfDimension(inputShape, 0);
453 [[maybe_unused]] uint32_t inputHeight = getSizeOfDimension(inputShape, 1);
454 [[maybe_unused]] uint32_t inputWidth = getSizeOfDimension(inputShape, 2);
455 [[maybe_unused]] uint32_t inputDepth = getSizeOfDimension(inputShape, 3);
456 [[maybe_unused]] uint32_t filterHeight = getSizeOfDimension(filterShape, 1);
457 [[maybe_unused]] uint32_t filterWidth = getSizeOfDimension(filterShape, 2);
458 [[maybe_unused]] uint32_t filterDepth = getSizeOfDimension(filterShape, 3);
459 [[maybe_unused]] uint32_t outputHeight = getSizeOfDimension(outputShape, 1);
460 [[maybe_unused]] uint32_t outputWidth = getSizeOfDimension(outputShape, 2);
461 uint32_t outputDepth = getSizeOfDimension(outputShape, 3);
462
463 [[maybe_unused]] int32_t inputOffset = -inputShape.offset;
464 [[maybe_unused]] int32_t outputOffset = outputShape.offset;
465
466 auto realMultiplier = std::vector<double>(outputDepth, .0f);
467 auto outputMultiplier = std::vector<int32_t>(outputDepth, 0);
468 auto outputShift = std::vector<int32_t>(outputDepth, .0f);
469
470 for (uint32_t i = 0; i < outputDepth; ++i) {
471 Shape filterChannelShape = filterShape;
472 filterChannelShape.scale = filterScales[i];
473 Shape biasChannelShape = biasShape;
474 biasChannelShape.scale = filterScales[i] * inputShape.scale;
475 NN_RET_CHECK(GetQuantizedConvolutionMultiplier(
476 inputShape, filterChannelShape, biasChannelShape, outputShape, &realMultiplier[i]));
477 NN_RET_CHECK(QuantizeMultiplier(realMultiplier[i], &outputMultiplier[i], &outputShift[i]));
478 }
479
480 int32_t output_activation_min = 0, output_activation_max = 0;
481 CalculateActivationRangeInt8(activation, outputShape, &output_activation_min,
482 &output_activation_max);
483
484 tflite::ConvParams convParams;
485 convParams.input_offset = -inputShape.offset;
486 convParams.output_offset = outputShape.offset;
487 convParams.stride_height = strideHeight;
488 convParams.stride_width = strideWidth;
489 convParams.dilation_height_factor = dilationHeightFactor;
490 convParams.dilation_width_factor = dilationWidthFactor;
491 convParams.padding_values.height = paddingTop;
492 convParams.padding_values.width = paddingLeft;
493 convParams.quantized_activation_min = output_activation_min;
494 convParams.quantized_activation_max = output_activation_max;
495
496 NNTRACE_COMP_SWITCH("reference_integer_ops::ConvPerChannel");
497 tflite::reference_integer_ops::ConvPerChannel(
498 convParams, outputMultiplier.data(), outputShift.data(),
499 convertShapeToTflshape(inputShape), inputData, convertShapeToTflshape(filterShape),
500 filterData, convertShapeToTflshape(biasShape), biasData,
501 convertShapeToTflshape(outputShape), outputData);
502 return true;
503 }
504
505 template <typename T>
convQuant8PerChannel(const T * inputData,const Shape & inputShape,const int8_t * filterData,const Shape & filterShape,const float * filterScales,const int32_t * biasData,const Shape & biasShape,int32_t paddingLeft,int32_t paddingRight,int32_t paddingTop,int32_t paddingBottom,int32_t strideWidth,int32_t strideHeight,int32_t dilationWidthFactor,int32_t dilationHeightFactor,int32_t activation,bool useNchw,T * outputData,const Shape & outputShape)506 bool convQuant8PerChannel(const T* inputData, const Shape& inputShape, const int8_t* filterData,
507 const Shape& filterShape, const float* filterScales,
508 const int32_t* biasData, const Shape& biasShape, int32_t paddingLeft,
509 int32_t paddingRight, int32_t paddingTop, int32_t paddingBottom,
510 int32_t strideWidth, int32_t strideHeight, int32_t dilationWidthFactor,
511 int32_t dilationHeightFactor, int32_t activation, bool useNchw,
512 T* outputData, const Shape& outputShape) {
513 InputWithLayout<T> input(useNchw);
514 OutputWithLayout<T> output(useNchw);
515 NN_RET_CHECK(input.initialize(inputData, inputShape));
516 NN_RET_CHECK(output.initialize(outputData, outputShape));
517 NN_RET_CHECK(convQuant8PerChannelNhwc(
518 input.getNhwcBuffer(), input.getNhwcShape(), filterData, filterShape, filterScales,
519 biasData, biasShape, paddingLeft, paddingRight, paddingTop, paddingBottom, strideWidth,
520 strideHeight, dilationWidthFactor, dilationHeightFactor, activation,
521 output.getNhwcBuffer(), output.getNhwcShape()));
522 NN_RET_CHECK(output.commit());
523 return true;
524 }
525
526 #undef ANDROID_NN_CONV_PARAMETERS
527
528 } // namespace
529
prepare(IOperationExecutionContext * context)530 bool prepare(IOperationExecutionContext* context) {
531 Shape input = context->getInputShape(kInputTensor);
532 Shape filter = context->getInputShape(kFilterTensor);
533 Shape bias = context->getInputShape(kBiasTensor);
534
535 if (filter.type == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
536 NN_RET_CHECK(input.type == OperandType::TENSOR_QUANT8_ASYMM ||
537 input.type == OperandType::TENSOR_QUANT8_ASYMM_SIGNED);
538 } else {
539 NN_RET_CHECK(input.type == filter.type);
540 }
541 if (input.type == OperandType::TENSOR_QUANT8_ASYMM ||
542 input.type == OperandType::TENSOR_QUANT8_ASYMM_SIGNED) {
543 NN_RET_CHECK(bias.type == OperandType::TENSOR_INT32);
544 } else {
545 NN_RET_CHECK(input.type == bias.type);
546 }
547 NN_RET_CHECK_EQ(getNumberOfDimensions(input), 4u);
548 NN_RET_CHECK_EQ(getNumberOfDimensions(filter), 4u);
549 NN_RET_CHECK_EQ(getNumberOfDimensions(bias), 1u);
550
551 Conv2dParam param;
552 NN_RET_CHECK(param.initialize(context));
553
554 uint32_t batches = getSizeOfDimension(input, 0);
555 uint32_t height = getSizeOfDimension(input, param.useNchw ? 2 : 1);
556 uint32_t width = getSizeOfDimension(input, param.useNchw ? 3 : 2);
557 uint32_t channels_in = getSizeOfDimension(input, param.useNchw ? 1 : 3);
558 uint32_t channels_out = getSizeOfDimension(filter, 0);
559 uint32_t filterHeight = getSizeOfDimension(filter, 1);
560 uint32_t filterWidth = getSizeOfDimension(filter, 2);
561 // Only batches can be zero.
562 NN_RET_CHECK_EQ(channels_in, getSizeOfDimension(filter, 3));
563 NN_RET_CHECK_EQ(channels_out, getSizeOfDimension(bias, 0));
564 NN_RET_CHECK_GT(height, 0u);
565 NN_RET_CHECK_GT(width, 0u);
566 NN_RET_CHECK_GT(channels_in, 0u);
567 NN_RET_CHECK_GT(channels_out, 0u);
568
569 int32_t effectiveFilterWidth = (filterWidth - 1) * param.dilation_width_factor + 1;
570 int32_t effectiveFilterHeight = (filterHeight - 1) * param.dilation_height_factor + 1;
571 NN_RET_CHECK_GT(effectiveFilterWidth, param.padding_left);
572 NN_RET_CHECK_GT(effectiveFilterWidth, param.padding_right);
573 NN_RET_CHECK_GT(effectiveFilterHeight, param.padding_top);
574 NN_RET_CHECK_GT(effectiveFilterHeight, param.padding_bottom);
575
576 uint32_t outWidth =
577 computeOutSize(width, filterWidth, param.stride_width, param.dilation_width_factor,
578 param.padding_left, param.padding_right);
579 uint32_t outHeight =
580 computeOutSize(height, filterHeight, param.stride_height, param.dilation_height_factor,
581 param.padding_top, param.padding_bottom);
582
583 Shape output = context->getOutputShape(kOutputTensor);
584 output.type = input.type;
585 if (param.useNchw) {
586 output.dimensions = {batches, channels_out, outHeight, outWidth};
587 } else {
588 output.dimensions = {batches, outHeight, outWidth, channels_out};
589 }
590 return context->setOutputShape(kOutputTensor, output);
591 }
592
execute(IOperationExecutionContext * context)593 bool execute(IOperationExecutionContext* context) {
594 // Bypass execution in the case of zero-sized input.
595 if (getNumberOfElements(context->getOutputShape(kOutputTensor)) == 0) return true;
596 Conv2dParam param;
597 NN_RET_CHECK(param.initialize(context));
598 switch (context->getInputType(kInputTensor)) {
599 case OperandType::TENSOR_FLOAT32:
600 return conv(context->getInputBuffer<float>(kInputTensor),
601 context->getInputShape(kInputTensor),
602 context->getInputBuffer<float>(kFilterTensor),
603 context->getInputShape(kFilterTensor),
604 context->getInputBuffer<float>(kBiasTensor),
605 context->getInputShape(kBiasTensor), param.padding_left,
606 param.padding_right, param.padding_top, param.padding_bottom,
607 param.stride_width, param.stride_height, param.dilation_width_factor,
608 param.dilation_height_factor, param.activation, param.useNchw,
609 context->getOutputBuffer<float>(kOutputTensor),
610 context->getOutputShape(kOutputTensor));
611 case OperandType::TENSOR_FLOAT16:
612 return conv(context->getInputBuffer<_Float16>(kInputTensor),
613 context->getInputShape(kInputTensor),
614 context->getInputBuffer<_Float16>(kFilterTensor),
615 context->getInputShape(kFilterTensor),
616 context->getInputBuffer<_Float16>(kBiasTensor),
617 context->getInputShape(kBiasTensor), param.padding_left,
618 param.padding_right, param.padding_top, param.padding_bottom,
619 param.stride_width, param.stride_height, param.dilation_width_factor,
620 param.dilation_height_factor, param.activation, param.useNchw,
621 context->getOutputBuffer<_Float16>(kOutputTensor),
622 context->getOutputShape(kOutputTensor));
623 case OperandType::TENSOR_QUANT8_ASYMM:
624 if (context->getInputType(kFilterTensor) ==
625 OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
626 return convQuant8PerChannel(
627 context->getInputBuffer<uint8_t>(kInputTensor),
628 context->getInputShape(kInputTensor),
629 context->getInputBuffer<int8_t>(kFilterTensor),
630 context->getInputShape(kFilterTensor),
631 std::get<Operand::SymmPerChannelQuantParams>(
632 context->getInputExtraParams(kFilterTensor))
633 .scales.data(),
634 context->getInputBuffer<int32_t>(kBiasTensor),
635 context->getInputShape(kBiasTensor), param.padding_left,
636 param.padding_right, param.padding_top, param.padding_bottom,
637 param.stride_width, param.stride_height, param.dilation_width_factor,
638 param.dilation_height_factor, param.activation, param.useNchw,
639 context->getOutputBuffer<uint8_t>(kOutputTensor),
640 context->getOutputShape(kOutputTensor));
641 } else if (context->getInputType(kFilterTensor) == OperandType::TENSOR_QUANT8_ASYMM) {
642 return conv(context->getInputBuffer<uint8_t>(kInputTensor),
643 context->getInputShape(kInputTensor),
644 context->getInputBuffer<uint8_t>(kFilterTensor),
645 context->getInputShape(kFilterTensor),
646 context->getInputBuffer<int32_t>(kBiasTensor),
647 context->getInputShape(kBiasTensor), param.padding_left,
648 param.padding_right, param.padding_top, param.padding_bottom,
649 param.stride_width, param.stride_height, param.dilation_width_factor,
650 param.dilation_height_factor, param.activation, param.useNchw,
651 context->getOutputBuffer<uint8_t>(kOutputTensor),
652 context->getOutputShape(kOutputTensor));
653 } else {
654 NN_RET_CHECK_FAIL() << "Unsupported filter type for operation " << kOperationName;
655 }
656 case OperandType::TENSOR_QUANT8_ASYMM_SIGNED:
657 if (context->getInputType(kFilterTensor) ==
658 OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
659 return convQuant8PerChannel(
660 context->getInputBuffer<int8_t>(kInputTensor),
661 context->getInputShape(kInputTensor),
662 context->getInputBuffer<int8_t>(kFilterTensor),
663 context->getInputShape(kFilterTensor),
664 std::get<Operand::SymmPerChannelQuantParams>(
665 context->getInputExtraParams(kFilterTensor))
666 .scales.data(),
667 context->getInputBuffer<int32_t>(kBiasTensor),
668 context->getInputShape(kBiasTensor), param.padding_left,
669 param.padding_right, param.padding_top, param.padding_bottom,
670 param.stride_width, param.stride_height, param.dilation_width_factor,
671 param.dilation_height_factor, param.activation, param.useNchw,
672 context->getOutputBuffer<int8_t>(kOutputTensor),
673 context->getOutputShape(kOutputTensor));
674 } else if (context->getInputType(kFilterTensor) ==
675 OperandType::TENSOR_QUANT8_ASYMM_SIGNED) {
676 return conv(context->getInputBuffer<int8_t>(kInputTensor),
677 context->getInputShape(kInputTensor),
678 context->getInputBuffer<int8_t>(kFilterTensor),
679 context->getInputShape(kFilterTensor),
680 context->getInputBuffer<int32_t>(kBiasTensor),
681 context->getInputShape(kBiasTensor), param.padding_left,
682 param.padding_right, param.padding_top, param.padding_bottom,
683 param.stride_width, param.stride_height, param.dilation_width_factor,
684 param.dilation_height_factor, param.activation, param.useNchw,
685 context->getOutputBuffer<int8_t>(kOutputTensor),
686 context->getOutputShape(kOutputTensor));
687 } else {
688 NN_RET_CHECK_FAIL() << "Unsupported filter type for operation " << kOperationName;
689 }
690 default:
691 NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation " << kOperationName;
692 }
693 }
694 #endif // NN_INCLUDE_CPU_IMPLEMENTATION
695
696 } // namespace conv_2d
697
698 NN_REGISTER_OPERATION_DEFAULT_VALIDATION(CONV_2D, conv_2d::prepare, conv_2d::execute,
699 .allowZeroSizedInput = true);
700
701 } // namespace nn
702 } // namespace android
703