1 /*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "Operations"
18
19 #include <tensorflow/lite/kernels/internal/reference/reference_ops.h>
20
21 #include <algorithm>
22 #include <functional>
23 #include <vector>
24
25 #include "CpuOperationUtils.h"
26 #include "HalInterfaces.h"
27 #include "OperationResolver.h"
28 #include "Tracing.h"
29
30 namespace android {
31 namespace nn {
32
33 using namespace hal;
34
35 namespace resize_image {
36
37 constexpr uint32_t kNumInputs = 4;
38 constexpr uint32_t kInputTensor = 0;
39 // The following two scalars represent output shape if INT32, scale if floating point.
40 constexpr uint32_t kOutputWidthParamScalar = 1;
41 constexpr uint32_t kOutputHeightParamScalar = 2;
42 constexpr uint32_t kLayoutScalar = 3;
43 constexpr uint32_t kNumOptionalInputs = 2;
44 constexpr uint32_t kAlignCornersScalar = 4;
45 constexpr uint32_t kHalfPixelCentersScalar = 5;
46
47 constexpr uint32_t kNumOutputs = 1;
48 constexpr uint32_t kOutputTensor = 0;
49
50 namespace {
51
scaleHalfPixel(const int x,const float scale)52 inline float scaleHalfPixel(const int x, const float scale) {
53 return (static_cast<float>(x) + 0.5f) * scale;
54 }
55
scaleLegacy(const int x,const float scale)56 inline float scaleLegacy(const int x, const float scale) {
57 return static_cast<float>(x) * scale;
58 }
59
calculateResizeScale(int32_t inSize,int32_t outSize,bool alignCorners)60 inline float calculateResizeScale(int32_t inSize, int32_t outSize, bool alignCorners) {
61 return (alignCorners && outSize > 1) ? (inSize - 1) / static_cast<float>(outSize - 1)
62 : inSize / static_cast<float>(outSize);
63 }
64
65 template <typename T>
resizeNearestNeighbor(const T * inputData,const Shape & inputShape,bool alignCorners,bool halfPixelCenters,T * outputData,const Shape & outputShape)66 bool resizeNearestNeighbor(const T* inputData, const Shape& inputShape, bool alignCorners,
67 bool halfPixelCenters, T* outputData, const Shape& outputShape) {
68 const int batchSize = getSizeOfDimension(inputShape, 0);
69 const int inHeight = getSizeOfDimension(inputShape, 1);
70 const int inWidth = getSizeOfDimension(inputShape, 2);
71 const int channels = getSizeOfDimension(inputShape, 3);
72 const int outHeight = getSizeOfDimension(outputShape, 1);
73 const int outWidth = getSizeOfDimension(outputShape, 2);
74
75 const float heightScale = calculateResizeScale(inHeight, outHeight, alignCorners);
76 const float widthScale = calculateResizeScale(inWidth, outWidth, alignCorners);
77
78 const std::function<float(const int, const float)> scaler =
79 halfPixelCenters ? scaleHalfPixel : scaleLegacy;
80
81 for (int b = 0; b < batchSize; ++b) {
82 for (int y = 0; y < outHeight; ++y) {
83 int inY = std::min((alignCorners) ? static_cast<int>(roundf(scaler(y, heightScale)))
84 : static_cast<int>(floorf(scaler(y, heightScale))),
85 inHeight - 1);
86 if (halfPixelCenters) {
87 inY = std::max(static_cast<int>(0), inY);
88 }
89 for (int x = 0; x < outWidth; ++x) {
90 int inX = std::min((alignCorners) ? static_cast<int>(roundf(scaler(x, widthScale)))
91 : static_cast<int>(floorf(scaler(x, widthScale))),
92 inWidth - 1);
93 if (halfPixelCenters) {
94 inX = std::max(static_cast<int>(0), inX);
95 }
96 std::copy_n(inputData + b * inHeight * inWidth * channels +
97 inY * inWidth * channels + inX * channels,
98 channels,
99 outputData + b * outHeight * outWidth * channels +
100 y * outWidth * channels + x * channels);
101 }
102 }
103 }
104
105 return true;
106 }
107
108 template <typename T>
resizeImageOpNhwc(OperationType opType,const T * inputData,const Shape & inputShape,bool alignCorners,bool halfPixelCenters,T * outputData,const Shape & outputShape)109 bool resizeImageOpNhwc(OperationType opType, const T* inputData, const Shape& inputShape,
110 bool alignCorners, bool halfPixelCenters, T* outputData,
111 const Shape& outputShape) {
112 NNTRACE_TRANS("resizeImageOpNhwc");
113 int32_t height = static_cast<int32_t>(getSizeOfDimension(outputShape, 1));
114 int32_t width = static_cast<int32_t>(getSizeOfDimension(outputShape, 2));
115 // We have to fake a tensor here, to satisfy tflite implementation.
116 int32_t outDimData[2] = {height, width};
117 Shape outDimShape;
118 outDimShape.dimensions = {2};
119
120 if (opType == OperationType::RESIZE_BILINEAR) {
121 NNTRACE_COMP_SWITCH("optimized_ops::ResizeBilinear");
122 tflite::reference_ops::ResizeBilinear(
123 {.align_corners = alignCorners, .half_pixel_centers = halfPixelCenters},
124 convertShapeToTflshape(inputShape), inputData, convertShapeToTflshape(outDimShape),
125 outDimData, convertShapeToTflshape(outputShape), outputData);
126 } else if (opType == OperationType::RESIZE_NEAREST_NEIGHBOR) {
127 // Align corners = true is not supported.
128 NNTRACE_COMP_SWITCH("ResizeNearestNeighbor");
129 resizeNearestNeighbor(inputData, inputShape, alignCorners, halfPixelCenters, outputData,
130 outputShape);
131 }
132 return true;
133 }
134
135 template <>
resizeImageOpNhwc(OperationType opType,const _Float16 * inputData,const Shape & inputShape,bool alignCorners,bool halfPixelCenters,_Float16 * outputData,const Shape & outputShape)136 bool resizeImageOpNhwc<_Float16>(OperationType opType, const _Float16* inputData,
137 const Shape& inputShape, bool alignCorners, bool halfPixelCenters,
138 _Float16* outputData, const Shape& outputShape) {
139 NNTRACE_TRANS("resizeImageOpNhwcFloat16");
140 std::vector<float> inputData_float32(getNumberOfElements(inputShape));
141 convertFloat16ToFloat32(inputData, &inputData_float32);
142 std::vector<float> outputData_float32(getNumberOfElements(outputShape));
143 NN_RET_CHECK(resizeImageOpNhwc(opType, inputData_float32.data(), inputShape, alignCorners,
144 halfPixelCenters, outputData_float32.data(), outputShape));
145 convertFloat32ToFloat16(outputData_float32, outputData);
146 return true;
147 }
148
149 template <typename T>
resizeImageOp(OperationType opType,const T * inputData,const Shape & inputShape,bool useNchw,bool alignCorners,bool halfPixelCenters,T * outputData,const Shape & outputShape)150 bool resizeImageOp(OperationType opType, const T* inputData, const Shape& inputShape, bool useNchw,
151 bool alignCorners, bool halfPixelCenters, T* outputData,
152 const Shape& outputShape) {
153 InputWithLayout<T> input(useNchw);
154 OutputWithLayout<T> output(useNchw);
155 NN_RET_CHECK(input.initialize(inputData, inputShape));
156 NN_RET_CHECK(output.initialize(outputData, outputShape));
157 NN_RET_CHECK(resizeImageOpNhwc(opType, input.getNhwcBuffer(), input.getNhwcShape(),
158 alignCorners, halfPixelCenters, output.getNhwcBuffer(),
159 output.getNhwcShape()));
160 NN_RET_CHECK(output.commit());
161 return true;
162 }
163
getOptionalScalar(const IOperationExecutionContext * context,uint32_t scalarIndex)164 inline bool getOptionalScalar(const IOperationExecutionContext* context, uint32_t scalarIndex) {
165 bool scalarValue = false;
166 if (context->getNumInputs() > scalarIndex) {
167 scalarValue = context->getInputValue<bool>(scalarIndex);
168 }
169 return scalarValue;
170 }
171
172 } // namespace
173
validate(OperationType opType,const IOperationValidationContext * context)174 bool validate(OperationType opType, const IOperationValidationContext* context) {
175 const auto numInputs = context->getNumInputs();
176 if (opType == OperationType::RESIZE_BILINEAR) {
177 NN_RET_CHECK(numInputs >= kNumInputs - 1 && numInputs <= kNumInputs + kNumOptionalInputs);
178 } else if (opType == OperationType::RESIZE_NEAREST_NEIGHBOR) {
179 NN_RET_CHECK(numInputs >= kNumInputs && numInputs <= kNumInputs + kNumOptionalInputs);
180 } else {
181 NN_RET_CHECK_FAIL() << "Unsupported operation " << getOperationName(opType);
182 }
183 NN_RET_CHECK_EQ(context->getNumOutputs(), kNumOutputs);
184 auto inputType = context->getInputType(kInputTensor);
185 auto scalarType = context->getInputType(kOutputHeightParamScalar);
186 std::vector<OperandType> inExpectedTypes = {inputType, scalarType, scalarType};
187 NN_RET_CHECK(inputType == OperandType::TENSOR_FLOAT16 ||
188 inputType == OperandType::TENSOR_FLOAT32 ||
189 inputType == OperandType::TENSOR_QUANT8_ASYMM ||
190 inputType == OperandType::TENSOR_QUANT8_ASYMM_SIGNED)
191 << "Unsupported tensor type for operation " << getOperationName(opType);
192 if (inputType == OperandType::TENSOR_FLOAT16 || inputType == OperandType::TENSOR_QUANT8_ASYMM) {
193 NN_RET_CHECK(validateHalVersion(context, HalVersion::V1_2));
194 }
195 if (inputType == OperandType::TENSOR_QUANT8_ASYMM_SIGNED) {
196 NN_RET_CHECK(validateHalVersion(context, HalVersion::V1_3));
197 }
198 if (scalarType != OperandType::INT32) {
199 NN_RET_CHECK(validateHalVersion(context, HalVersion::V1_2));
200 if (inputType == OperandType::TENSOR_FLOAT32) {
201 NN_RET_CHECK(scalarType == OperandType::FLOAT32);
202 } else if (inputType == OperandType::TENSOR_FLOAT16) {
203 NN_RET_CHECK(scalarType == OperandType::FLOAT16);
204 } else if (inputType == OperandType::TENSOR_QUANT8_ASYMM ||
205 inputType == OperandType::TENSOR_QUANT8_ASYMM_SIGNED) {
206 NN_RET_CHECK(scalarType == OperandType::FLOAT32);
207 }
208 }
209 if (numInputs < kNumInputs) {
210 NN_RET_CHECK(validateHalVersion(context, HalVersion::V1_0));
211 } else if (numInputs == kNumInputs) {
212 inExpectedTypes.push_back(OperandType::BOOL);
213 NN_RET_CHECK(validateHalVersion(context, HalVersion::V1_2));
214 } else {
215 while (inExpectedTypes.size() < numInputs) {
216 inExpectedTypes.push_back(OperandType::BOOL);
217 }
218 NN_RET_CHECK(validateHalVersion(context, HalVersion::V1_3));
219 }
220 return validateInputTypes(context, inExpectedTypes) &&
221 validateOutputTypes(context, {inputType});
222 }
223
prepare(OperationType opType,IOperationExecutionContext * context)224 bool prepare(OperationType opType, IOperationExecutionContext* context) {
225 Shape input = context->getInputShape(kInputTensor);
226 NN_RET_CHECK_EQ(getNumberOfDimensions(input), 4);
227 const auto numInputs = context->getNumInputs();
228 const bool useNchw = getOptionalScalar(context, kLayoutScalar);
229 const bool alignCorners = getOptionalScalar(context, kAlignCornersScalar);
230 const bool halfPixelCenters = getOptionalScalar(context, kHalfPixelCentersScalar);
231
232 NN_RET_CHECK(!halfPixelCenters || (halfPixelCenters && !alignCorners));
233
234 // Only batches can be zero.
235 uint32_t batches = getSizeOfDimension(input, 0);
236 uint32_t inHeight = getSizeOfDimension(input, useNchw ? 2 : 1);
237 uint32_t inWidth = getSizeOfDimension(input, useNchw ? 3 : 2);
238 uint32_t channels = getSizeOfDimension(input, useNchw ? 1 : 3);
239 NN_RET_CHECK_GT(inHeight, 0);
240 NN_RET_CHECK_GT(inWidth, 0);
241 NN_RET_CHECK_GT(channels, 0);
242
243 int32_t height, width;
244 auto scalarType = context->getInputType(kOutputHeightParamScalar);
245 if (scalarType == OperandType::INT32) {
246 height = context->getInputValue<int32_t>(kOutputHeightParamScalar);
247 width = context->getInputValue<int32_t>(kOutputWidthParamScalar);
248 } else if (scalarType == OperandType::FLOAT32) {
249 height = std::floor(static_cast<float>(inHeight) *
250 context->getInputValue<float>(kOutputHeightParamScalar));
251 width = std::floor(static_cast<float>(inWidth) *
252 context->getInputValue<float>(kOutputWidthParamScalar));
253 } else if (scalarType == OperandType::FLOAT16) {
254 height = std::floor(
255 static_cast<float>(inHeight) *
256 static_cast<float>(context->getInputValue<_Float16>(kOutputHeightParamScalar)));
257 width = std::floor(
258 static_cast<float>(inWidth) *
259 static_cast<float>(context->getInputValue<_Float16>(kOutputWidthParamScalar)));
260 } else {
261 NN_RET_CHECK_FAIL() << "Unsupported scalar type for operation " << getOperationName(opType);
262 }
263 NN_RET_CHECK_GT(height, 0);
264 NN_RET_CHECK_GT(width, 0);
265
266 Shape output = input;
267 if (useNchw) {
268 output.dimensions = {batches, channels, (uint32_t)height, (uint32_t)width};
269 } else {
270 output.dimensions = {batches, (uint32_t)height, (uint32_t)width, channels};
271 }
272 return context->setOutputShape(kOutputTensor, output);
273 }
274
execute(OperationType opType,IOperationExecutionContext * context)275 bool execute(OperationType opType, IOperationExecutionContext* context) {
276 // Bypass execution in the case of zero-sized input.
277 if (getNumberOfElements(context->getOutputShape(kOutputTensor)) == 0) return true;
278
279 const bool useNchw = getOptionalScalar(context, kLayoutScalar);
280 const bool alignCorners = getOptionalScalar(context, kAlignCornersScalar);
281 const bool halfPixelCenters = getOptionalScalar(context, kHalfPixelCentersScalar);
282
283 switch (context->getInputType(kInputTensor)) {
284 case OperandType::TENSOR_FLOAT16:
285 return resizeImageOp(opType, context->getInputBuffer<_Float16>(kInputTensor),
286 context->getInputShape(kInputTensor), useNchw, alignCorners,
287 halfPixelCenters,
288 context->getOutputBuffer<_Float16>(kOutputTensor),
289 context->getOutputShape(kOutputTensor));
290 case OperandType::TENSOR_FLOAT32:
291 return resizeImageOp(opType, context->getInputBuffer<float>(kInputTensor),
292 context->getInputShape(kInputTensor), useNchw, alignCorners,
293 halfPixelCenters, context->getOutputBuffer<float>(kOutputTensor),
294 context->getOutputShape(kOutputTensor));
295 case OperandType::TENSOR_QUANT8_ASYMM:
296 return resizeImageOp(opType, context->getInputBuffer<uint8_t>(kInputTensor),
297 context->getInputShape(kInputTensor), useNchw, alignCorners,
298 halfPixelCenters, context->getOutputBuffer<uint8_t>(kOutputTensor),
299 context->getOutputShape(kOutputTensor));
300 case OperandType::TENSOR_QUANT8_ASYMM_SIGNED:
301 return resizeImageOp(opType, context->getInputBuffer<int8_t>(kInputTensor),
302 context->getInputShape(kInputTensor), useNchw, alignCorners,
303 halfPixelCenters, context->getOutputBuffer<int8_t>(kOutputTensor),
304 context->getOutputShape(kOutputTensor));
305
306 default:
307 NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation "
308 << getOperationName(opType);
309 }
310 }
311
312 } // namespace resize_image
313
314 using std::placeholders::_1;
315
316 NN_REGISTER_OPERATION(RESIZE_BILINEAR, "RESIZE_BILINEAR",
317 std::bind(resize_image::validate, OperationType::RESIZE_BILINEAR, _1),
318 std::bind(resize_image::prepare, OperationType::RESIZE_BILINEAR, _1),
319 std::bind(resize_image::execute, OperationType::RESIZE_BILINEAR, _1),
320 .allowZeroSizedInput = true);
321
322 NN_REGISTER_OPERATION(RESIZE_NEAREST_NEIGHBOR, "RESIZE_NEAREST_NEIGHBOR",
323 std::bind(resize_image::validate, OperationType::RESIZE_NEAREST_NEIGHBOR, _1),
324 std::bind(resize_image::prepare, OperationType::RESIZE_NEAREST_NEIGHBOR, _1),
325 std::bind(resize_image::execute, OperationType::RESIZE_NEAREST_NEIGHBOR, _1),
326 .allowZeroSizedInput = true);
327
328 } // namespace nn
329 } // namespace android
330