1 /*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "Operations"
18
19 #include <tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h>
20 #include <tensorflow/lite/kernels/internal/reference/integer_ops/conv.h>
21 #include <tensorflow/lite/kernels/internal/types.h>
22
23 #include <algorithm>
24 #include <iterator>
25 #include <memory>
26 #include <vector>
27
28 #include "CpuOperationUtils.h"
29 #include "HalInterfaces.h"
30 #include "OperationResolver.h"
31 #include "Operations.h"
32 #include "OperationsUtils.h"
33 #include "Tracing.h"
34 #include "Utils.h"
35
36 namespace android {
37 namespace nn {
38 namespace conv_2d {
39
40 constexpr char kOperationName[] = "CONV_2D";
41
42 constexpr uint32_t kNumInputsArray[] = {7, 8, 10, 11, 13};
43 constexpr uint32_t kInputTensor = 0;
44 constexpr uint32_t kFilterTensor = 1;
45 constexpr uint32_t kBiasTensor = 2;
46
47 constexpr uint32_t kNumOutputs = 1;
48 constexpr uint32_t kOutputTensor = 0;
49
50 namespace {
51
52 using namespace hal;
53
54 // If possible we will use this static buffer for the tensor.
55 constexpr size_t kStaticBufferSize = 1605632;
56 char static_scratch_buffer[kStaticBufferSize];
57
58 // executionMutex is used to protect concurrent access of the static_scratch_buffer
59 // and other non-threadsafe resources like gemmlowp::GemmContext.
60 // std::mutex is safe for pthreads on Android.
61 std::mutex executionMutex;
62
63 struct Conv2dParam {
64 int32_t padding_left, padding_right;
65 int32_t padding_top, padding_bottom;
66 int32_t stride_width, stride_height;
67 int32_t dilation_width_factor = 1, dilation_height_factor = 1;
68 int32_t activation;
69 bool useNchw = false;
70
initializeandroid::nn::conv_2d::__anon4d5fbfb50111::Conv2dParam71 bool initialize(const IOperationExecutionContext* context) {
72 uint32_t inCount = context->getNumInputs();
73 int32_t padding_implicit = 0;
74 bool useImplicitPadding = false;
75 if ((inCount >= 8 && context->getInputType(7) == OperandType::BOOL) || inCount == 7) {
76 padding_implicit = context->getInputValue<int32_t>(3);
77 stride_width = context->getInputValue<int32_t>(4);
78 stride_height = context->getInputValue<int32_t>(5);
79 activation = context->getInputValue<int32_t>(6);
80 if (inCount >= 8) {
81 useNchw = context->getInputValue<bool>(7);
82 }
83 if (inCount == 10) {
84 dilation_width_factor = context->getInputValue<int32_t>(8);
85 dilation_height_factor = context->getInputValue<int32_t>(9);
86 }
87 useImplicitPadding = true;
88 } else if (inCount >= 10 && context->getInputType(7) == OperandType::INT32) {
89 padding_left = context->getInputValue<int32_t>(3);
90 padding_right = context->getInputValue<int32_t>(4);
91 padding_top = context->getInputValue<int32_t>(5);
92 padding_bottom = context->getInputValue<int32_t>(6);
93 stride_width = context->getInputValue<int32_t>(7);
94 stride_height = context->getInputValue<int32_t>(8);
95 activation = context->getInputValue<int32_t>(9);
96 if (inCount >= 11) {
97 useNchw = context->getInputValue<bool>(10);
98 }
99 if (inCount == 13) {
100 dilation_width_factor = context->getInputValue<int32_t>(11);
101 dilation_height_factor = context->getInputValue<int32_t>(12);
102 }
103 } else {
104 NN_RET_CHECK_FAIL() << "Unsupported input spec for operation " << kOperationName;
105 }
106 if (useImplicitPadding) {
107 Shape inputShape = context->getInputShape(kInputTensor);
108 Shape filterShape = context->getInputShape(kFilterTensor);
109 int32_t input_width = getSizeOfDimension(inputShape, useNchw ? 3 : 2);
110 int32_t input_height = getSizeOfDimension(inputShape, useNchw ? 2 : 1);
111 int32_t filter_width = getSizeOfDimension(filterShape, 2);
112 int32_t filter_height = getSizeOfDimension(filterShape, 1);
113 calculateExplicitPadding(input_width, stride_width, dilation_width_factor, filter_width,
114 padding_implicit, &padding_left, &padding_right);
115 calculateExplicitPadding(input_height, stride_height, dilation_height_factor,
116 filter_height, padding_implicit, &padding_top,
117 &padding_bottom);
118 }
119 NN_RET_CHECK_GE(padding_left, 0);
120 NN_RET_CHECK_GE(padding_right, 0);
121 NN_RET_CHECK_GE(padding_top, 0);
122 NN_RET_CHECK_GE(padding_bottom, 0);
123 NN_RET_CHECK_GT(stride_width, 0);
124 NN_RET_CHECK_GT(stride_height, 0);
125 NN_RET_CHECK_GT(dilation_width_factor, 0);
126 NN_RET_CHECK_GT(dilation_height_factor, 0);
127 NN_RET_CHECK_GE(activation, 0);
128 return true;
129 }
130 };
131
132 #define ANDROID_NN_CONV_PARAMETERS(Type) \
133 uint32_t height = getSizeOfDimension(inputShape, 1); \
134 uint32_t width = getSizeOfDimension(inputShape, 2); \
135 uint32_t filterHeight = getSizeOfDimension(filterShape, 1); \
136 uint32_t filterWidth = getSizeOfDimension(filterShape, 2); \
137 uint32_t outHeight = getSizeOfDimension(outputShape, 1); \
138 uint32_t outWidth = getSizeOfDimension(outputShape, 2); \
139 uint32_t inDepth = getSizeOfDimension(inputShape, 3); \
140 \
141 uint32_t paddingHeight = (uint32_t)padding_top; \
142 uint32_t paddingWidth = (uint32_t)padding_left; \
143 \
144 tflite::Dims<4> im2colDim; \
145 im2colDim.sizes[3] = (int)getSizeOfDimension(outputShape, 0); \
146 im2colDim.sizes[2] = (int)getSizeOfDimension(outputShape, 1); \
147 im2colDim.sizes[1] = (int)getSizeOfDimension(outputShape, 2); \
148 im2colDim.sizes[0] = (int)inDepth * filterHeight * filterWidth; \
149 \
150 im2colDim.strides[0] = 1; \
151 for (int i = 1; i < 4; i++) { \
152 im2colDim.strides[i] = im2colDim.strides[i - 1] * im2colDim.sizes[i - 1]; \
153 } \
154 \
155 Type* im2colData = nullptr; \
156 uint64_t im2colByteSize = sizeof(Type); \
157 std::unique_ptr<Type[]> im2colGuard; \
158 for (int i = 0; i < 4; i++) { \
159 im2colByteSize *= im2colDim.sizes[i]; \
160 } \
161 /* http://b/77982879, tflite::optimized_ops::Conv uses int for offsets */ \
162 if (im2colByteSize >= 0x7fffffff) { \
163 LOG(ERROR) << "Conv size is too large, not enough memory"; \
164 return false; \
165 } \
166 if (im2colByteSize <= kStaticBufferSize) { \
167 im2colData = reinterpret_cast<Type*>(static_scratch_buffer); \
168 } else { \
169 im2colData = new (std::nothrow) Type[im2colByteSize / sizeof(Type)]; \
170 if (im2colData == nullptr) { \
171 LOG(ERROR) << "Conv size is too large, not enough memory"; \
172 return false; \
173 } \
174 im2colGuard.reset(im2colData); \
175 }
176
convNhwc(const float * inputData,const Shape & inputShape,const float * filterData,const Shape & filterShape,const float * biasData,const Shape & biasShape,int32_t padding_left,int32_t padding_right,int32_t padding_top,int32_t padding_bottom,int32_t stride_width,int32_t stride_height,int32_t dilation_width_factor,int32_t dilation_height_factor,int32_t activation,float * outputData,const Shape & outputShape)177 bool convNhwc(const float* inputData, const Shape& inputShape, const float* filterData,
178 const Shape& filterShape, const float* biasData, const Shape& biasShape,
179 int32_t padding_left, int32_t padding_right, int32_t padding_top,
180 int32_t padding_bottom, int32_t stride_width, int32_t stride_height,
181 int32_t dilation_width_factor, int32_t dilation_height_factor, int32_t activation,
182 float* outputData, const Shape& outputShape) {
183 NNTRACE_TRANS("convFloat32");
184
185 ANDROID_NN_CONV_PARAMETERS(float)
186
187 float output_activation_min, output_activation_max;
188 CalculateActivationRangeFloat(activation, &output_activation_min, &output_activation_max);
189
190 // Prevent concurrent executions that may access the scratch buffer.
191 std::unique_lock<std::mutex> lock(executionMutex);
192 NNTRACE_COMP_SWITCH("optimized_ops::Conv");
193 tflite::optimized_ops::Conv(inputData, convertShapeToDims(inputShape), filterData,
194 convertShapeToDims(filterShape), biasData,
195 convertShapeToDims(biasShape), stride_width, stride_height,
196 dilation_width_factor, dilation_height_factor, paddingWidth,
197 paddingHeight, output_activation_min, output_activation_max,
198 outputData, convertShapeToDims(outputShape), im2colData, im2colDim);
199 return true;
200 }
201
convNhwc(const uint8_t * inputData,const Shape & inputShape,const uint8_t * filterData,const Shape & filterShape,const int32_t * biasData,const Shape & biasShape,int32_t padding_left,int32_t padding_right,int32_t padding_top,int32_t padding_bottom,int32_t stride_width,int32_t stride_height,int32_t dilation_width_factor,int32_t dilation_height_factor,int32_t activation,uint8_t * outputData,const Shape & outputShape)202 bool convNhwc(const uint8_t* inputData, const Shape& inputShape, const uint8_t* filterData,
203 const Shape& filterShape, const int32_t* biasData, const Shape& biasShape,
204 int32_t padding_left, int32_t padding_right, int32_t padding_top,
205 int32_t padding_bottom, int32_t stride_width, int32_t stride_height,
206 int32_t dilation_width_factor, int32_t dilation_height_factor, int32_t activation,
207 uint8_t* outputData, const Shape& outputShape) {
208 NNTRACE_TRANS("convQuant8");
209
210 ANDROID_NN_CONV_PARAMETERS(uint8_t)
211
212 int32_t inputOffset = -inputShape.offset;
213 int32_t filterOffset = -filterShape.offset;
214 int32_t outputOffset = outputShape.offset;
215
216 double real_multiplier = 0.0;
217 int32_t output_multiplier = 0;
218 int32_t output_shift = 0;
219 int32_t output_activation_min = 0;
220 int32_t output_activation_max = 0;
221
222 NN_RET_CHECK(GetQuantizedConvolutionMultipler(inputShape, filterShape, biasShape, outputShape,
223 &real_multiplier));
224 int exponent;
225 NN_RET_CHECK(QuantizeMultiplier(real_multiplier, &output_multiplier, &exponent));
226 output_shift = -exponent;
227 CalculateActivationRangeUint8(activation, outputShape, &output_activation_min,
228 &output_activation_max);
229
230 static gemmlowp::GemmContext gemm_context;
231
232 // Prevent concurrent executions that may access the scratch buffer and
233 // gemm_context.
234 std::unique_lock<std::mutex> lock(executionMutex);
235 // Alow gemmlowp automatically decide how many threads to use.
236 gemm_context.set_max_num_threads(0);
237
238 NNTRACE_COMP_SWITCH("optimized_ops::Conv");
239 tflite::optimized_ops::Conv(
240 inputData, convertShapeToDims(inputShape), inputOffset, filterData,
241 convertShapeToDims(filterShape), filterOffset, biasData, convertShapeToDims(biasShape),
242 stride_width, stride_height, dilation_width_factor, dilation_height_factor,
243 paddingWidth, paddingHeight, outputOffset, output_multiplier, output_shift,
244 output_activation_min, output_activation_max, outputData,
245 convertShapeToDims(outputShape), im2colData, im2colDim, &gemm_context);
246 return true;
247 }
248
249 // Passing input, filter and output shapes by value, so that we can change the
250 // offsets without modifying the actual shapes.
convNhwc(const int8_t * inputData,Shape inputShape,const int8_t * filterData,Shape filterShape,const int32_t * biasData,const Shape & biasShape,int32_t padding_left,int32_t padding_right,int32_t padding_top,int32_t padding_bottom,int32_t stride_width,int32_t stride_height,int32_t dilation_width_factor,int32_t dilation_height_factor,int32_t activation,int8_t * outputData,Shape outputShape)251 bool convNhwc(const int8_t* inputData, Shape inputShape, const int8_t* filterData,
252 Shape filterShape, const int32_t* biasData, const Shape& biasShape,
253 int32_t padding_left, int32_t padding_right, int32_t padding_top,
254 int32_t padding_bottom, int32_t stride_width, int32_t stride_height,
255 int32_t dilation_width_factor, int32_t dilation_height_factor, int32_t activation,
256 int8_t* outputData, Shape outputShape) {
257 NNTRACE_TRANS("convQuant8");
258
259 std::vector<uint8_t> unsignedInput(getNumberOfElements(inputShape));
260 convertInt8ToUInt8(inputData, &unsignedInput);
261 inputShape.offset += 128;
262
263 std::vector<uint8_t> unsignedFilter(getNumberOfElements(filterShape));
264 convertInt8ToUInt8(filterData, &unsignedFilter);
265 filterShape.offset += 128;
266
267 std::vector<uint8_t> unsignedOutput(getNumberOfElements(outputShape));
268 outputShape.offset += 128;
269
270 NN_RET_CHECK(convNhwc(unsignedInput.data(), inputShape, unsignedFilter.data(), filterShape,
271 biasData, biasShape, padding_left, padding_right, padding_top,
272 padding_bottom, stride_width, stride_height, dilation_width_factor,
273 dilation_height_factor, activation, unsignedOutput.data(), outputShape));
274
275 convertUInt8ToInt8(unsignedOutput, outputData);
276
277 return true;
278 }
279
convNhwc(const _Float16 * inputData,const Shape & inputShape,const _Float16 * filterData,const Shape & filterShape,const _Float16 * biasData,const Shape & biasShape,int32_t padding_left,int32_t padding_right,int32_t padding_top,int32_t padding_bottom,int32_t stride_width,int32_t stride_height,int32_t dilation_width_factor,int32_t dilation_height_factor,int32_t activation,_Float16 * outputData,const Shape & outputShape)280 bool convNhwc(const _Float16* inputData, const Shape& inputShape, const _Float16* filterData,
281 const Shape& filterShape, const _Float16* biasData, const Shape& biasShape,
282 int32_t padding_left, int32_t padding_right, int32_t padding_top,
283 int32_t padding_bottom, int32_t stride_width, int32_t stride_height,
284 int32_t dilation_width_factor, int32_t dilation_height_factor, int32_t activation,
285 _Float16* outputData, const Shape& outputShape) {
286 NNTRACE_TRANS("convFloat16");
287
288 std::vector<float> inputData_float32(getNumberOfElements(inputShape));
289 std::vector<float> filterData_float32(getNumberOfElements(filterShape));
290 std::vector<float> biasData_float32(getNumberOfElements(biasShape));
291 std::vector<float> outputData_float32(getNumberOfElements(outputShape));
292
293 convertFloat16ToFloat32(inputData, &inputData_float32);
294 convertFloat16ToFloat32(filterData, &filterData_float32);
295 convertFloat16ToFloat32(biasData, &biasData_float32);
296
297 convNhwc(inputData_float32.data(), inputShape, filterData_float32.data(), filterShape,
298 biasData_float32.data(), biasShape, padding_left, padding_right, padding_top,
299 padding_bottom, stride_width, stride_height, dilation_width_factor,
300 dilation_height_factor, activation, outputData_float32.data(), outputShape);
301 convertFloat32ToFloat16(outputData_float32, outputData);
302
303 return true;
304 }
305
306 template <typename T_Input, typename T_Filter, typename T_Bias>
conv(const T_Input * inputData,const Shape & inputShape,const T_Filter * filterData,const Shape & filterShape,const T_Bias * biasData,const Shape & biasShape,int32_t padding_left,int32_t padding_right,int32_t padding_top,int32_t padding_bottom,int32_t stride_width,int32_t stride_height,int32_t dilation_width_factor,int32_t dilation_height_factor,int32_t activation,bool useNchw,T_Input * outputData,const Shape & outputShape)307 bool conv(const T_Input* inputData, const Shape& inputShape, const T_Filter* filterData,
308 const Shape& filterShape, const T_Bias* biasData, const Shape& biasShape,
309 int32_t padding_left, int32_t padding_right, int32_t padding_top, int32_t padding_bottom,
310 int32_t stride_width, int32_t stride_height, int32_t dilation_width_factor,
311 int32_t dilation_height_factor, int32_t activation, bool useNchw, T_Input* outputData,
312 const Shape& outputShape) {
313 InputWithLayout<T_Input> input(useNchw);
314 OutputWithLayout<T_Input> output(useNchw);
315 NN_RET_CHECK(input.initialize(inputData, inputShape));
316 NN_RET_CHECK(output.initialize(outputData, outputShape));
317 NN_RET_CHECK(convNhwc(input.getNhwcBuffer(), input.getNhwcShape(), filterData, filterShape,
318 biasData, biasShape, padding_left, padding_right, padding_top,
319 padding_bottom, stride_width, stride_height, dilation_width_factor,
320 dilation_height_factor, activation, output.getNhwcBuffer(),
321 output.getNhwcShape()));
322 NN_RET_CHECK(output.commit());
323 return true;
324 }
325
convQuant8PerChannelNhwc(const uint8_t * inputData,const Shape & inputShape,const int8_t * filterData,const Shape & filterShape,const float * filterScales,const int32_t * biasData,const Shape & biasShape,int32_t paddingLeft,int32_t paddingRight,int32_t paddingTop,int32_t paddingBottom,int32_t strideWidth,int32_t strideHeight,int32_t dilationWidthFactor,int32_t dilationHeightFactor,int32_t activation,uint8_t * outputData,const Shape & outputShape)326 bool convQuant8PerChannelNhwc(const uint8_t* inputData, const Shape& inputShape,
327 const int8_t* filterData, const Shape& filterShape,
328 const float* filterScales, const int32_t* biasData,
329 const Shape& biasShape, int32_t paddingLeft, int32_t paddingRight,
330 int32_t paddingTop, int32_t paddingBottom, int32_t strideWidth,
331 int32_t strideHeight, int32_t dilationWidthFactor,
332 int32_t dilationHeightFactor, int32_t activation, uint8_t* outputData,
333 const Shape& outputShape) {
334 NNTRACE_TRANS("convQuant8PerChannel");
335
336 uint32_t numBatches = getSizeOfDimension(inputShape, 0);
337 uint32_t inputHeight = getSizeOfDimension(inputShape, 1);
338 uint32_t inputWidth = getSizeOfDimension(inputShape, 2);
339 uint32_t inputDepth = getSizeOfDimension(inputShape, 3);
340 uint32_t filterHeight = getSizeOfDimension(filterShape, 1);
341 uint32_t filterWidth = getSizeOfDimension(filterShape, 2);
342 uint32_t filterDepth = getSizeOfDimension(filterShape, 3);
343 uint32_t outputHeight = getSizeOfDimension(outputShape, 1);
344 uint32_t outputWidth = getSizeOfDimension(outputShape, 2);
345 uint32_t outputDepth = getSizeOfDimension(outputShape, 3);
346
347 int32_t inputOffset = -inputShape.offset;
348 int32_t outputOffset = outputShape.offset;
349
350 auto realMultiplier = std::vector<double>(outputDepth, .0f);
351 auto outputMultiplier = std::vector<int32_t>(outputDepth, 0);
352 auto outputShift = std::vector<int32_t>(outputDepth, .0f);
353
354 for (int i = 0; i < outputDepth; ++i) {
355 Shape filterChannelShape = filterShape;
356 filterChannelShape.scale = filterScales[i];
357 Shape biasChannelShape = biasShape;
358 biasChannelShape.scale = filterScales[i] * inputShape.scale;
359 NN_RET_CHECK(GetQuantizedConvolutionMultipler(
360 inputShape, filterChannelShape, biasChannelShape, outputShape, &realMultiplier[i]));
361 int exponent;
362 NN_RET_CHECK(QuantizeMultiplier(realMultiplier[i], &outputMultiplier[i], &exponent));
363 outputShift[i] = -exponent;
364 }
365
366 int32_t output_activation_min = 0, output_activation_max = 0;
367 CalculateActivationRangeUint8(activation, outputShape, &output_activation_min,
368 &output_activation_max);
369 const uint8_t* inputBase = inputData;
370 uint8_t* outPtr = outputData;
371 for (uint32_t b = 0; b < numBatches; b++) {
372 for (uint32_t h = 0; h < outputHeight; h++) {
373 for (uint32_t w = 0; w < outputWidth; w++) {
374 const int8_t* filterBase = filterData;
375
376 for (uint32_t d = 0; d < outputDepth; d++) {
377 int32_t wInputOrigin = static_cast<int32_t>(w) * strideWidth - paddingLeft;
378 int32_t hInputOrigin = static_cast<int32_t>(h) * strideHeight - paddingTop;
379 int32_t sum = 0.0f;
380
381 for (uint32_t i = 0; i < filterHeight; i++) {
382 for (uint32_t j = 0; j < filterWidth; j++) {
383 for (uint32_t k = 0; k < filterDepth; k++) {
384 int32_t hInput = hInputOrigin +
385 dilationHeightFactor * static_cast<int32_t>(i);
386 int32_t wInput = wInputOrigin +
387 dilationWidthFactor * static_cast<int32_t>(j);
388 uint32_t dInput = k;
389 if (hInput >= 0 && hInput < static_cast<int32_t>(inputHeight) &&
390 wInput >= 0 && wInput < static_cast<int32_t>(inputWidth)) {
391 uint32_t filterIndex =
392 i * filterWidth * filterDepth + j * filterDepth + k;
393 uint32_t inputIndex = hInput * inputWidth * inputDepth +
394 wInput * inputDepth + dInput;
395 sum += (static_cast<int32_t>(filterBase[filterIndex])) *
396 (static_cast<int32_t>(inputBase[inputIndex]) +
397 inputOffset);
398 }
399 }
400 }
401 }
402 sum += biasData[d];
403 sum = tflite::MultiplyByQuantizedMultiplier(sum, outputMultiplier[d],
404 -outputShift[d]);
405 sum += outputOffset;
406 sum = std::max(std::min(sum, output_activation_max), output_activation_min);
407 outPtr[d] = static_cast<uint8_t>(sum);
408 filterBase += filterHeight * filterWidth * filterDepth;
409 }
410 outPtr += outputDepth;
411 }
412 }
413 inputBase += inputHeight * inputWidth * inputDepth;
414 }
415
416 return true;
417 }
418
convQuant8PerChannelNhwc(const int8_t * inputData,const Shape & inputShape,const int8_t * filterData,const Shape & filterShape,const float * filterScales,const int32_t * biasData,const Shape & biasShape,int32_t paddingLeft,int32_t paddingRight,int32_t paddingTop,int32_t paddingBottom,int32_t strideWidth,int32_t strideHeight,int32_t dilationWidthFactor,int32_t dilationHeightFactor,int32_t activation,int8_t * outputData,const Shape & outputShape)419 bool convQuant8PerChannelNhwc(const int8_t* inputData, const Shape& inputShape,
420 const int8_t* filterData, const Shape& filterShape,
421 const float* filterScales, const int32_t* biasData,
422 const Shape& biasShape, int32_t paddingLeft, int32_t paddingRight,
423 int32_t paddingTop, int32_t paddingBottom, int32_t strideWidth,
424 int32_t strideHeight, int32_t dilationWidthFactor,
425 int32_t dilationHeightFactor, int32_t activation, int8_t* outputData,
426 const Shape& outputShape) {
427 NNTRACE_TRANS("convQuant8SignedPerChannel");
428
429 uint32_t numBatches = getSizeOfDimension(inputShape, 0);
430 uint32_t inputHeight = getSizeOfDimension(inputShape, 1);
431 uint32_t inputWidth = getSizeOfDimension(inputShape, 2);
432 uint32_t inputDepth = getSizeOfDimension(inputShape, 3);
433 uint32_t filterHeight = getSizeOfDimension(filterShape, 1);
434 uint32_t filterWidth = getSizeOfDimension(filterShape, 2);
435 uint32_t filterDepth = getSizeOfDimension(filterShape, 3);
436 uint32_t outputHeight = getSizeOfDimension(outputShape, 1);
437 uint32_t outputWidth = getSizeOfDimension(outputShape, 2);
438 uint32_t outputDepth = getSizeOfDimension(outputShape, 3);
439
440 int32_t inputOffset = -inputShape.offset;
441 int32_t outputOffset = outputShape.offset;
442
443 auto realMultiplier = std::vector<double>(outputDepth, .0f);
444 auto outputMultiplier = std::vector<int32_t>(outputDepth, 0);
445 auto outputShift = std::vector<int32_t>(outputDepth, .0f);
446
447 for (int i = 0; i < outputDepth; ++i) {
448 Shape filterChannelShape = filterShape;
449 filterChannelShape.scale = filterScales[i];
450 Shape biasChannelShape = biasShape;
451 biasChannelShape.scale = filterScales[i] * inputShape.scale;
452 NN_RET_CHECK(GetQuantizedConvolutionMultipler(
453 inputShape, filterChannelShape, biasChannelShape, outputShape, &realMultiplier[i]));
454 NN_RET_CHECK(QuantizeMultiplier(realMultiplier[i], &outputMultiplier[i], &outputShift[i]));
455 }
456
457 int32_t output_activation_min = 0, output_activation_max = 0;
458 CalculateActivationRangeInt8(activation, outputShape, &output_activation_min,
459 &output_activation_max);
460
461 tflite::ConvParams convParams;
462 convParams.input_offset = -inputShape.offset;
463 convParams.output_offset = outputShape.offset;
464 convParams.stride_height = strideHeight;
465 convParams.stride_width = strideWidth;
466 convParams.dilation_height_factor = dilationHeightFactor;
467 convParams.dilation_width_factor = dilationWidthFactor;
468 convParams.padding_values.height = paddingTop;
469 convParams.padding_values.width = paddingLeft;
470 convParams.quantized_activation_min = output_activation_min;
471 convParams.quantized_activation_max = output_activation_max;
472
473 NNTRACE_COMP_SWITCH("reference_integer_ops::ConvPerChannel");
474 tflite::reference_integer_ops::ConvPerChannel(
475 convParams, outputMultiplier.data(), outputShift.data(),
476 convertShapeToTflshape(inputShape), inputData, convertShapeToTflshape(filterShape),
477 filterData, convertShapeToTflshape(biasShape), biasData,
478 convertShapeToTflshape(outputShape), outputData);
479 return true;
480 }
481
482 template <typename T>
convQuant8PerChannel(const T * inputData,const Shape & inputShape,const int8_t * filterData,const Shape & filterShape,const float * filterScales,const int32_t * biasData,const Shape & biasShape,int32_t paddingLeft,int32_t paddingRight,int32_t paddingTop,int32_t paddingBottom,int32_t strideWidth,int32_t strideHeight,int32_t dilationWidthFactor,int32_t dilationHeightFactor,int32_t activation,bool useNchw,T * outputData,const Shape & outputShape)483 bool convQuant8PerChannel(const T* inputData, const Shape& inputShape, const int8_t* filterData,
484 const Shape& filterShape, const float* filterScales,
485 const int32_t* biasData, const Shape& biasShape, int32_t paddingLeft,
486 int32_t paddingRight, int32_t paddingTop, int32_t paddingBottom,
487 int32_t strideWidth, int32_t strideHeight, int32_t dilationWidthFactor,
488 int32_t dilationHeightFactor, int32_t activation, bool useNchw,
489 T* outputData, const Shape& outputShape) {
490 InputWithLayout<T> input(useNchw);
491 OutputWithLayout<T> output(useNchw);
492 NN_RET_CHECK(input.initialize(inputData, inputShape));
493 NN_RET_CHECK(output.initialize(outputData, outputShape));
494 NN_RET_CHECK(convQuant8PerChannelNhwc(
495 input.getNhwcBuffer(), input.getNhwcShape(), filterData, filterShape, filterScales,
496 biasData, biasShape, paddingLeft, paddingRight, paddingTop, paddingBottom, strideWidth,
497 strideHeight, dilationWidthFactor, dilationHeightFactor, activation,
498 output.getNhwcBuffer(), output.getNhwcShape()));
499 NN_RET_CHECK(output.commit());
500 return true;
501 }
502
503 #undef ANDROID_NN_CONV_PARAMETERS
504
505 } // namespace
506
validate(const IOperationValidationContext * context)507 bool validate(const IOperationValidationContext* context) {
508 const uint32_t numInputs = context->getNumInputs();
509 NN_RET_CHECK(
510 std::binary_search(std::begin(kNumInputsArray), std::end(kNumInputsArray), numInputs));
511 NN_RET_CHECK_EQ(context->getNumOutputs(), kNumOutputs);
512 const auto inputRank = getNumberOfDimensions(context->getInputShape(kInputTensor));
513 const auto filterRank = getNumberOfDimensions(context->getInputShape(kFilterTensor));
514 if (inputRank != 0) {
515 NN_RET_CHECK_EQ(inputRank, 4);
516 }
517 if (filterRank != 0) {
518 NN_RET_CHECK_EQ(filterRank, 4);
519 }
520 auto inputCount = context->getNumInputs();
521 auto inputType = context->getInputType(kInputTensor);
522 auto filterType = context->getInputType(kFilterTensor);
523 std::vector<OperandType> inExpectedTypes;
524 if (inputType == OperandType::TENSOR_FLOAT32) {
525 inExpectedTypes = {OperandType::TENSOR_FLOAT32, OperandType::TENSOR_FLOAT32,
526 OperandType::TENSOR_FLOAT32, OperandType::INT32,
527 OperandType::INT32, OperandType::INT32,
528 OperandType::INT32};
529 } else if (inputType == OperandType::TENSOR_FLOAT16) {
530 inExpectedTypes = {OperandType::TENSOR_FLOAT16, OperandType::TENSOR_FLOAT16,
531 OperandType::TENSOR_FLOAT16, OperandType::INT32,
532 OperandType::INT32, OperandType::INT32,
533 OperandType::INT32};
534 } else if (inputType == OperandType::TENSOR_QUANT8_ASYMM ||
535 inputType == OperandType::TENSOR_QUANT8_ASYMM_SIGNED) {
536 NN_RET_CHECK(filterType == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL ||
537 filterType == inputType)
538 << "Unsupported filter tensor type for operation " << kOperationName;
539 inExpectedTypes = {inputType, filterType, OperandType::TENSOR_INT32,
540 OperandType::INT32, OperandType::INT32, OperandType::INT32,
541 OperandType::INT32};
542
543 if (filterType == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
544 NN_RET_CHECK_EQ(context->getInputExtraParams(kFilterTensor).channelQuant().channelDim,
545 0)
546 << "Unsupported filter tensor channel dimension for operation "
547 << kOperationName;
548 }
549 } else {
550 NN_RET_CHECK_FAIL() << "Unsupported input tensor type for operation " << kOperationName;
551 }
552
553 // NeuralNetworks.h specifies that ANEURALNETWORKS_CONV_2D's output must
554 // meet "outputScale > inputScale * filterScale" for the operand type
555 // ANEURALNETWORKS_TENSOR_QUANT8_ASYMM before API level 29. For other
556 // operand types (e.g., ANEURALNETWORKS_TENSOR_FLOAT32), this constraint
557 // does not apply, so by default the constraint is met.
558 bool meetsQuantizedScaleConstraintBeforeV1_2 = true;
559 if (inputType == OperandType::TENSOR_QUANT8_ASYMM) {
560 const float inputScale = context->getInputShape(kInputTensor).scale;
561 const float filterScale = context->getInputShape(kFilterTensor).scale;
562 const float outputScale = context->getInputShape(kOutputTensor).scale;
563 meetsQuantizedScaleConstraintBeforeV1_2 = (outputScale > inputScale * filterScale);
564 }
565
566 bool withExplicitPadding = false;
567 bool withLayout = false;
568 bool withDilation = false;
569 if (inputCount >= 8) {
570 if (context->getInputType(7) == OperandType::INT32 && inputCount >= 10) {
571 std::vector<OperandType> explicitScalarTypes(3, OperandType::INT32);
572 inExpectedTypes.insert(inExpectedTypes.end(), explicitScalarTypes.begin(),
573 explicitScalarTypes.end());
574 withExplicitPadding = true;
575 }
576 int inputOffset = withExplicitPadding ? 3 : 0;
577 if (inputCount >= 8 + inputOffset) {
578 inExpectedTypes.push_back(OperandType::BOOL);
579 withLayout = true;
580 }
581 NN_RET_CHECK_NE(inputCount, 9 + inputOffset)
582 << "Provided only one dilation factor value, two values are requred for operation "
583 << kOperationName;
584 if (inputCount == 10 + inputOffset) {
585 inExpectedTypes.push_back(OperandType::INT32);
586 inExpectedTypes.push_back(OperandType::INT32);
587 withDilation = true;
588 }
589 }
590
591 if (inputType == OperandType::TENSOR_QUANT8_ASYMM_SIGNED) {
592 NN_RET_CHECK(validateHalVersion(context, HalVersion::V1_3));
593 } else if (inputType == OperandType::TENSOR_FLOAT16 ||
594 filterType == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL || withLayout ||
595 withDilation || !meetsQuantizedScaleConstraintBeforeV1_2) {
596 NN_RET_CHECK(validateHalVersion(context, HalVersion::V1_2));
597 } else {
598 NN_RET_CHECK(validateHalVersion(context, HalVersion::V1_0));
599 }
600 return validateInputTypes(context, inExpectedTypes) &&
601 validateOutputTypes(context, {inputType});
602 }
603
prepare(IOperationExecutionContext * context)604 bool prepare(IOperationExecutionContext* context) {
605 Shape input = context->getInputShape(kInputTensor);
606 Shape filter = context->getInputShape(kFilterTensor);
607 Shape bias = context->getInputShape(kBiasTensor);
608
609 if (filter.type == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
610 NN_RET_CHECK(input.type == OperandType::TENSOR_QUANT8_ASYMM ||
611 input.type == OperandType::TENSOR_QUANT8_ASYMM_SIGNED);
612 } else {
613 NN_RET_CHECK(input.type == filter.type);
614 }
615 if (input.type == OperandType::TENSOR_QUANT8_ASYMM ||
616 input.type == OperandType::TENSOR_QUANT8_ASYMM_SIGNED) {
617 NN_RET_CHECK(bias.type == OperandType::TENSOR_INT32);
618 } else {
619 NN_RET_CHECK(input.type == bias.type);
620 }
621 NN_RET_CHECK_EQ(getNumberOfDimensions(input), 4);
622 NN_RET_CHECK_EQ(getNumberOfDimensions(filter), 4);
623 NN_RET_CHECK_EQ(getNumberOfDimensions(bias), 1);
624
625 Conv2dParam param;
626 NN_RET_CHECK(param.initialize(context));
627
628 uint32_t batches = getSizeOfDimension(input, 0);
629 uint32_t height = getSizeOfDimension(input, param.useNchw ? 2 : 1);
630 uint32_t width = getSizeOfDimension(input, param.useNchw ? 3 : 2);
631 uint32_t channels_in = getSizeOfDimension(input, param.useNchw ? 1 : 3);
632 uint32_t channels_out = getSizeOfDimension(filter, 0);
633 uint32_t filterHeight = getSizeOfDimension(filter, 1);
634 uint32_t filterWidth = getSizeOfDimension(filter, 2);
635 // Only batches can be zero.
636 NN_RET_CHECK_EQ(channels_in, getSizeOfDimension(filter, 3));
637 NN_RET_CHECK_EQ(channels_out, getSizeOfDimension(bias, 0));
638 NN_RET_CHECK_GT(height, 0);
639 NN_RET_CHECK_GT(width, 0);
640 NN_RET_CHECK_GT(channels_in, 0);
641 NN_RET_CHECK_GT(channels_out, 0);
642
643 int32_t effectiveFilterWidth = (filterWidth - 1) * param.dilation_width_factor + 1;
644 int32_t effectiveFilterHeight = (filterHeight - 1) * param.dilation_height_factor + 1;
645 NN_RET_CHECK_GT(effectiveFilterWidth, param.padding_left);
646 NN_RET_CHECK_GT(effectiveFilterWidth, param.padding_right);
647 NN_RET_CHECK_GT(effectiveFilterHeight, param.padding_top);
648 NN_RET_CHECK_GT(effectiveFilterHeight, param.padding_bottom);
649
650 uint32_t outWidth =
651 computeOutSize(width, filterWidth, param.stride_width, param.dilation_width_factor,
652 param.padding_left, param.padding_right);
653 uint32_t outHeight =
654 computeOutSize(height, filterHeight, param.stride_height, param.dilation_height_factor,
655 param.padding_top, param.padding_bottom);
656
657 Shape output = context->getOutputShape(kOutputTensor);
658 output.type = input.type;
659 if (param.useNchw) {
660 output.dimensions = {batches, channels_out, outHeight, outWidth};
661 } else {
662 output.dimensions = {batches, outHeight, outWidth, channels_out};
663 }
664 return context->setOutputShape(kOutputTensor, output);
665 }
666
execute(IOperationExecutionContext * context)667 bool execute(IOperationExecutionContext* context) {
668 // Bypass execution in the case of zero-sized input.
669 if (getNumberOfElements(context->getOutputShape(kOutputTensor)) == 0) return true;
670 Conv2dParam param;
671 NN_RET_CHECK(param.initialize(context));
672 switch (context->getInputType(kInputTensor)) {
673 case OperandType::TENSOR_FLOAT32:
674 return conv(context->getInputBuffer<float>(kInputTensor),
675 context->getInputShape(kInputTensor),
676 context->getInputBuffer<float>(kFilterTensor),
677 context->getInputShape(kFilterTensor),
678 context->getInputBuffer<float>(kBiasTensor),
679 context->getInputShape(kBiasTensor), param.padding_left,
680 param.padding_right, param.padding_top, param.padding_bottom,
681 param.stride_width, param.stride_height, param.dilation_width_factor,
682 param.dilation_height_factor, param.activation, param.useNchw,
683 context->getOutputBuffer<float>(kOutputTensor),
684 context->getOutputShape(kOutputTensor));
685 case OperandType::TENSOR_FLOAT16:
686 return conv(context->getInputBuffer<_Float16>(kInputTensor),
687 context->getInputShape(kInputTensor),
688 context->getInputBuffer<_Float16>(kFilterTensor),
689 context->getInputShape(kFilterTensor),
690 context->getInputBuffer<_Float16>(kBiasTensor),
691 context->getInputShape(kBiasTensor), param.padding_left,
692 param.padding_right, param.padding_top, param.padding_bottom,
693 param.stride_width, param.stride_height, param.dilation_width_factor,
694 param.dilation_height_factor, param.activation, param.useNchw,
695 context->getOutputBuffer<_Float16>(kOutputTensor),
696 context->getOutputShape(kOutputTensor));
697 case OperandType::TENSOR_QUANT8_ASYMM:
698 if (context->getInputType(kFilterTensor) ==
699 OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
700 return convQuant8PerChannel(
701 context->getInputBuffer<uint8_t>(kInputTensor),
702 context->getInputShape(kInputTensor),
703 context->getInputBuffer<int8_t>(kFilterTensor),
704 context->getInputShape(kFilterTensor),
705 context->getInputExtraParams(kFilterTensor).channelQuant().scales.data(),
706 context->getInputBuffer<int32_t>(kBiasTensor),
707 context->getInputShape(kBiasTensor), param.padding_left,
708 param.padding_right, param.padding_top, param.padding_bottom,
709 param.stride_width, param.stride_height, param.dilation_width_factor,
710 param.dilation_height_factor, param.activation, param.useNchw,
711 context->getOutputBuffer<uint8_t>(kOutputTensor),
712 context->getOutputShape(kOutputTensor));
713 } else if (context->getInputType(kFilterTensor) == OperandType::TENSOR_QUANT8_ASYMM) {
714 return conv(context->getInputBuffer<uint8_t>(kInputTensor),
715 context->getInputShape(kInputTensor),
716 context->getInputBuffer<uint8_t>(kFilterTensor),
717 context->getInputShape(kFilterTensor),
718 context->getInputBuffer<int32_t>(kBiasTensor),
719 context->getInputShape(kBiasTensor), param.padding_left,
720 param.padding_right, param.padding_top, param.padding_bottom,
721 param.stride_width, param.stride_height, param.dilation_width_factor,
722 param.dilation_height_factor, param.activation, param.useNchw,
723 context->getOutputBuffer<uint8_t>(kOutputTensor),
724 context->getOutputShape(kOutputTensor));
725 } else {
726 NN_RET_CHECK_FAIL() << "Unsupported filter type for operation " << kOperationName;
727 }
728 case OperandType::TENSOR_QUANT8_ASYMM_SIGNED:
729 if (context->getInputType(kFilterTensor) ==
730 OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
731 return convQuant8PerChannel(
732 context->getInputBuffer<int8_t>(kInputTensor),
733 context->getInputShape(kInputTensor),
734 context->getInputBuffer<int8_t>(kFilterTensor),
735 context->getInputShape(kFilterTensor),
736 context->getInputExtraParams(kFilterTensor).channelQuant().scales.data(),
737 context->getInputBuffer<int32_t>(kBiasTensor),
738 context->getInputShape(kBiasTensor), param.padding_left,
739 param.padding_right, param.padding_top, param.padding_bottom,
740 param.stride_width, param.stride_height, param.dilation_width_factor,
741 param.dilation_height_factor, param.activation, param.useNchw,
742 context->getOutputBuffer<int8_t>(kOutputTensor),
743 context->getOutputShape(kOutputTensor));
744 } else if (context->getInputType(kFilterTensor) ==
745 OperandType::TENSOR_QUANT8_ASYMM_SIGNED) {
746 return conv(context->getInputBuffer<int8_t>(kInputTensor),
747 context->getInputShape(kInputTensor),
748 context->getInputBuffer<int8_t>(kFilterTensor),
749 context->getInputShape(kFilterTensor),
750 context->getInputBuffer<int32_t>(kBiasTensor),
751 context->getInputShape(kBiasTensor), param.padding_left,
752 param.padding_right, param.padding_top, param.padding_bottom,
753 param.stride_width, param.stride_height, param.dilation_width_factor,
754 param.dilation_height_factor, param.activation, param.useNchw,
755 context->getOutputBuffer<int8_t>(kOutputTensor),
756 context->getOutputShape(kOutputTensor));
757 } else {
758 NN_RET_CHECK_FAIL() << "Unsupported filter type for operation " << kOperationName;
759 }
760 default:
761 NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation " << kOperationName;
762 }
763 }
764
765 } // namespace conv_2d
766
767 NN_REGISTER_OPERATION(CONV_2D, conv_2d::kOperationName, conv_2d::validate, conv_2d::prepare,
768 conv_2d::execute, .allowZeroSizedInput = true);
769
770 } // namespace nn
771 } // namespace android
772