/* * Copyright (C) 2017 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // Contains the implementation of the operations. #define LOG_TAG "Operations" #include "CpuOperationUtils.h" #include "OperationResolver.h" #include "tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h" #include "tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h" #include "Tracing.h" #include namespace android { namespace nn { namespace broadcast { constexpr uint32_t kNumInputs = 3; constexpr uint32_t kInputTensor1 = 0; constexpr uint32_t kInputTensor2 = 1; constexpr uint32_t kActivationScalar = 2; constexpr uint32_t kNumOutputs = 1; constexpr uint32_t kOutputTensor = 0; namespace { #define ANDROID_NN_MACRO_DISPATCH(macro) \ switch (activation) { \ case (int32_t)FusedActivationFunc::NONE: \ macro(kNone); \ break; \ case (int32_t)FusedActivationFunc::RELU: \ macro(kRelu); \ break; \ case (int32_t)FusedActivationFunc::RELU1: \ macro(kRelu1); \ break; \ case (int32_t)FusedActivationFunc::RELU6: \ macro(kRelu6); \ break; \ default: \ LOG(ERROR) << "Unsupported fused activation function type"; \ return false; \ } using binaryFunctionFloat32 = std::function; bool binaryOperationFloat16(const _Float16* in1, const Shape& shape1, const _Float16* in2, const Shape& shape2, int32_t activation, _Float16* out, const Shape& shapeOut, binaryFunctionFloat32 operationFloat32) { std::vector in1_float32(getNumberOfElements(shape1)); convertFloat16ToFloat32(in1, &in1_float32); std::vector in2_float32(getNumberOfElements(shape2)); convertFloat16ToFloat32(in2, &in2_float32); std::vector out_float32(getNumberOfElements(shapeOut)); operationFloat32(in1_float32.data(), shape1, in2_float32.data(), shape2, activation, out_float32.data(), shapeOut); convertFloat32ToFloat16(out_float32, out); return true; } bool addFloat32(const float* in1, const Shape& shape1, const float* in2, const Shape& shape2, int32_t activation, float* out, const Shape& shapeOut) { NNTRACE_TRANS("addFloat32"); bool needBroadcast = !SameShape(shape1, shape2); if (needBroadcast) { NNTRACE_COMP_SWITCH("optimized_ops::BroadcastAdd"); #define ANDROID_NN_BROADCAST_ADD(activation) \ tflite::optimized_ops::BroadcastAdd( \ in1, convertShapeToDims(shape1), in2, convertShapeToDims(shape2), out, \ convertShapeToDims(shapeOut)) ANDROID_NN_MACRO_DISPATCH(ANDROID_NN_BROADCAST_ADD) #undef ANDROID_NN_BROADCAST_ADD } else { NNTRACE_COMP_SWITCH("optimized_ops::Add"); #define ANDROID_NN_ADD(activation) \ tflite::optimized_ops::Add( \ in1, convertShapeToDims(shape1), in2, convertShapeToDims(shape2), out, \ convertShapeToDims(shapeOut)) ANDROID_NN_MACRO_DISPATCH(ANDROID_NN_ADD) #undef ANDROID_NN_ADD } return true; } bool addFloat16(const _Float16* in1, const Shape& shape1, const _Float16* in2, const Shape& shape2, int32_t activation, _Float16* out, const Shape& shapeOut) { NNTRACE_TRANS("addFloat16"); return binaryOperationFloat16(in1, shape1, in2, shape2, activation, out, shapeOut, &addFloat32); } bool addQuant8(const uint8_t* in1, const Shape& shape1, const uint8_t* in2, const Shape& shape2, int32_t activation, uint8_t* out, const Shape& shapeOut) { NNTRACE_TRANS("addQuant8"); bool needBroadcast = !SameShape(shape1, shape2); const int32_t input1_offset = -shape1.offset; const int32_t input2_offset = -shape2.offset; const int32_t output_offset = shapeOut.offset; const int left_shift = 20; const double twice_max_input_scale = 2 * std::max(shape1.scale, shape2.scale); const double real_input1_multiplier = shape1.scale / twice_max_input_scale; const double real_input2_multiplier = shape2.scale / twice_max_input_scale; const double real_output_multiplier = twice_max_input_scale / ((1 << left_shift) * shapeOut.scale); int32_t input1_multiplier; int32_t input1_shift; if (!QuantizeMultiplierSmallerThanOne(real_input1_multiplier, &input1_multiplier, &input1_shift)) { return false; } int32_t input2_multiplier; int32_t input2_shift; if (!QuantizeMultiplierSmallerThanOne(real_input2_multiplier, &input2_multiplier, &input2_shift)) { return false; } int32_t output_multiplier; int32_t output_shift; if (!QuantizeMultiplierSmallerThanOne(real_output_multiplier, &output_multiplier, &output_shift)) { return false; } int32_t output_activation_min; int32_t output_activation_max; CalculateActivationRangeUint8(activation, shapeOut, &output_activation_min, &output_activation_max); if (needBroadcast) { NNTRACE_COMP_SWITCH("optimized_ops::BroadcastAdd"); #define ANDROID_NN_BROADCAST_ADD(activation) \ tflite::optimized_ops::BroadcastAdd( \ left_shift, in1, convertShapeToDims(shape1), input1_offset, input1_multiplier, \ input1_shift, in2, convertShapeToDims(shape2), input2_offset, input2_multiplier, \ input2_shift, output_offset, output_multiplier, output_shift, output_activation_min, \ output_activation_max, out, convertShapeToDims(shapeOut)) ANDROID_NN_MACRO_DISPATCH(ANDROID_NN_BROADCAST_ADD) #undef ANDROID_NN_BROADCAST_ADD } else { NNTRACE_COMP_SWITCH("optimized_ops::Add"); #define ANDROID_NN_NORMAL_ADD(activation) \ tflite::optimized_ops::Add( \ left_shift, in1, convertShapeToDims(shape1), input1_offset, input1_multiplier, \ input1_shift, in2, convertShapeToDims(shape2), input2_offset, input2_multiplier, \ input2_shift, output_offset, output_multiplier, output_shift, output_activation_min, \ output_activation_max, out, convertShapeToDims(shapeOut)) ANDROID_NN_MACRO_DISPATCH(ANDROID_NN_NORMAL_ADD) #undef ANDROID_NN_NORMAL_ADD } return true; } bool mulFloat32(const float* in1, const Shape& shape1, const float* in2, const Shape& shape2, int32_t activation, float* out, const Shape& shapeOut) { NNTRACE_TRANS("mulFloat32"); bool needBroadcast = !SameShape(shape1, shape2); if (needBroadcast) { NNTRACE_COMP_SWITCH("optimized_ops::BroadcastMul"); #define ANDROID_NN_BROADCAST_MUL(activation) \ tflite::optimized_ops::BroadcastMul( \ in1, convertShapeToDims(shape1), in2, convertShapeToDims(shape2), out, \ convertShapeToDims(shapeOut)) ANDROID_NN_MACRO_DISPATCH(ANDROID_NN_BROADCAST_MUL) #undef ANDROID_NN_BROADCAST_MUL } else { float output_activation_min, output_activation_max; CalculateActivationRangeFloat(activation, &output_activation_min, &output_activation_max); NNTRACE_COMP_SWITCH("optimized_ops::Mul"); tflite::optimized_ops::Mul(in1, convertShapeToDims(shape1), in2, convertShapeToDims(shape2), output_activation_min, output_activation_max, out, convertShapeToDims(shapeOut)); } return true; } bool mulFloat16(const _Float16* in1, const Shape& shape1, const _Float16* in2, const Shape& shape2, int32_t activation, _Float16* out, const Shape& shapeOut) { NNTRACE_TRANS("mulFloat16"); return binaryOperationFloat16(in1, shape1, in2, shape2, activation, out, shapeOut, &mulFloat32); } bool mulQuant8(const uint8_t* in1, const Shape& shape1, const uint8_t* in2, const Shape& shape2, int32_t activation, uint8_t* out, const Shape& shapeOut) { NNTRACE_TRANS("mulQuant8"); const int32_t input1_offset = -shape1.offset; const int32_t input2_offset = -shape2.offset; const int32_t output_offset = shapeOut.offset; const double input_product_scale = shape1.scale * shape2.scale; const double real_multiplier = input_product_scale / shapeOut.scale; int32 output_multiplier; int output_shift; if (!QuantizeMultiplierSmallerThanOne(real_multiplier, &output_multiplier, &output_shift)) { return false; } int32_t output_activation_min; int32_t output_activation_max; CalculateActivationRangeUint8(activation, shapeOut, &output_activation_min, &output_activation_max); // Use BROADCAST version to handle the normal case. NNTRACE_COMP_SWITCH("optimized_ops::BroadcastMul"); tflite::optimized_ops::BroadcastMul(in1, convertShapeToDims(shape1), input1_offset, in2, convertShapeToDims(shape2), input2_offset, output_offset, output_multiplier, output_shift, output_activation_min, output_activation_max, out, convertShapeToDims(shapeOut)); return true; } bool subFloat32(const float* in1, const Shape& shape1, const float* in2, const Shape& shape2, int32_t activation, float* out, const Shape& shapeOut) { NNTRACE_TRANS("subFloat32"); NNTRACE_COMP_SWITCH("optimized_ops::Sub"); tflite::optimized_ops::Sub(in1, convertShapeToDims(shape1), in2, convertShapeToDims(shape2), out, convertShapeToDims(shapeOut)); // TFLite does not apply activation to broadcast sub. float output_activation_min, output_activation_max; CalculateActivationRangeFloat(activation, &output_activation_min, &output_activation_max); uint32_t numOutputElements = getNumberOfElements(shapeOut); for (uint32_t i = 0; i < numOutputElements; i++) { out[i] = std::min(std::max(out[i], output_activation_min), output_activation_max); } return true; } bool subFloat16(const _Float16* in1, const Shape& shape1, const _Float16* in2, const Shape& shape2, int32_t activation, _Float16* out, const Shape& shapeOut) { NNTRACE_TRANS("subFloat16"); return binaryOperationFloat16(in1, shape1, in2, shape2, activation, out, shapeOut, &subFloat32); } bool subQuant8(const uint8_t* in1, const Shape& shape1, const uint8_t* in2, const Shape& shape2, int32_t activation, uint8_t* out, const Shape& shapeOut) { NNTRACE_TRANS("subQuant8"); const int32_t input1_offset = -shape1.offset; const int32_t input2_offset = -shape2.offset; const int32_t output_offset = shapeOut.offset; const int left_shift = 20; const double twice_max_input_scale = 2 * std::max(shape1.scale, shape2.scale); const double real_input1_multiplier = shape1.scale / twice_max_input_scale; const double real_input2_multiplier = shape2.scale / twice_max_input_scale; const double real_output_multiplier = twice_max_input_scale / ((1 << left_shift) * shapeOut.scale); int32_t input1_multiplier; int32_t input1_shift; if (!QuantizeMultiplierSmallerThanOne(real_input1_multiplier, &input1_multiplier, &input1_shift)) { return false; } int32_t input2_multiplier; int32_t input2_shift; if (!QuantizeMultiplierSmallerThanOne(real_input2_multiplier, &input2_multiplier, &input2_shift)) { return false; } input2_multiplier *= -1; int32_t output_multiplier; int32_t output_shift; if (!QuantizeMultiplierSmallerThanOne(real_output_multiplier, &output_multiplier, &output_shift)) { return false; } int32_t output_activation_min; int32_t output_activation_max; CalculateActivationRangeUint8(activation, shapeOut, &output_activation_min, &output_activation_max); // We are using tflite::optimized_ops::BroadcastAdd unconditionally here // because tflite::optimized_ops::Add fails to pass some of the // sub_quantized_different_scales tests. NNTRACE_COMP_SWITCH("optimized_ops::BroadcastAdd"); #define ANDROID_NN_BROADCAST_ADD(activation) \ tflite::optimized_ops::BroadcastAdd( \ left_shift, in1, convertShapeToDims(shape1), input1_offset, input1_multiplier, \ input1_shift, in2, convertShapeToDims(shape2), input2_offset, input2_multiplier, \ input2_shift, output_offset, output_multiplier, output_shift, output_activation_min, \ output_activation_max, out, convertShapeToDims(shapeOut)) ANDROID_NN_MACRO_DISPATCH(ANDROID_NN_BROADCAST_ADD) #undef ANDROID_NN_BROADCAST_ADD return true; } bool divFloat32(const float* in1, const Shape& shape1, const float* in2, const Shape& shape2, int32_t activation, float* out, const Shape& shapeOut) { NNTRACE_TRANS("divFloat32"); float output_activation_min, output_activation_max; CalculateActivationRangeFloat(activation, &output_activation_min, &output_activation_max); bool needBroadcast = !SameShape(shape1, shape2); if (needBroadcast) { NNTRACE_COMP_SWITCH("optimized_ops::BroadcastDiv"); tflite::optimized_ops::BroadcastDiv( in1, convertShapeToDims(shape1), in2, convertShapeToDims(shape2), output_activation_min, output_activation_max, out, convertShapeToDims(shapeOut)); } else { NNTRACE_COMP_SWITCH("optimized_ops::Div"); tflite::optimized_ops::Div(in1, convertShapeToDims(shape1), in2, convertShapeToDims(shape2), output_activation_min, output_activation_max, out, convertShapeToDims(shapeOut)); } return true; } bool divFloat16(const _Float16* in1, const Shape& shape1, const _Float16* in2, const Shape& shape2, int32_t activation, _Float16* out, const Shape& shapeOut) { NNTRACE_TRANS("divFloat16"); return binaryOperationFloat16(in1, shape1, in2, shape2, activation, out, shapeOut, &divFloat32); } } // namespace bool validate(OperationType opType, const IOperationValidationContext* context) { const HalVersion opIntroducedAt = (opType == OperationType::DIV || opType == OperationType::SUB) ? HalVersion::V1_1 : HalVersion::V1_0; NN_RET_CHECK_EQ(context->getNumInputs(), kNumInputs); NN_RET_CHECK_EQ(context->getNumOutputs(), kNumOutputs); auto inputType = context->getInputType(kInputTensor1); if (inputType == OperandType::TENSOR_FLOAT32) { NN_RET_CHECK(validateHalVersion(context, std::max(HalVersion::V1_0, opIntroducedAt))); } else if (inputType == OperandType::TENSOR_FLOAT16) { NN_RET_CHECK(validateHalVersion(context, std::max(HalVersion::V1_2, opIntroducedAt))); } else if (inputType == OperandType::TENSOR_QUANT8_ASYMM) { if (opType == OperationType::SUB) { NN_RET_CHECK(validateHalVersion(context, std::max(HalVersion::V1_2, opIntroducedAt))); } else if (opType == OperationType::DIV) { NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation DIV"; } else if (opType == OperationType::MUL) { Shape output = context->getOutputShape(kOutputTensor); Shape input1 = context->getInputShape(kInputTensor1); Shape input2 = context->getInputShape(kInputTensor2); NN_RET_CHECK_GT(output.scale, input1.scale * input2.scale); NN_RET_CHECK(validateHalVersion(context, std::max(HalVersion::V1_0, opIntroducedAt))); } else { NN_RET_CHECK(validateHalVersion(context, std::max(HalVersion::V1_0, opIntroducedAt))); } } else { NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation " << getOperationName(opType); } return validateInputTypes(context, {inputType, inputType, OperandType::INT32}) && validateOutputTypes(context, {inputType}); } bool prepare(IOperationExecutionContext* context) { Shape input1 = context->getInputShape(kInputTensor1); Shape input2 = context->getInputShape(kInputTensor2); Shape output = context->getOutputShape(kOutputTensor); NN_RET_CHECK_LE(getNumberOfDimensions(input1), 4); NN_RET_CHECK_LE(getNumberOfDimensions(input2), 4); NN_RET_CHECK(calculateBroadcastedShape(input1, input2, &output)); return context->setOutputShape(kOutputTensor, output); } bool executeAdd(IOperationExecutionContext* context) { // Bypass execution in the case of zero-sized input. if (getNumberOfElements(context->getOutputShape(kOutputTensor)) == 0) return true; switch (context->getInputType(kInputTensor1)) { case OperandType::TENSOR_FLOAT16: return addFloat16(context->getInputBuffer<_Float16>(kInputTensor1), context->getInputShape(kInputTensor1), context->getInputBuffer<_Float16>(kInputTensor2), context->getInputShape(kInputTensor2), context->getInputValue(kActivationScalar), context->getOutputBuffer<_Float16>(kOutputTensor), context->getOutputShape(kOutputTensor)); case OperandType::TENSOR_FLOAT32: return addFloat32(context->getInputBuffer(kInputTensor1), context->getInputShape(kInputTensor1), context->getInputBuffer(kInputTensor2), context->getInputShape(kInputTensor2), context->getInputValue(kActivationScalar), context->getOutputBuffer(kOutputTensor), context->getOutputShape(kOutputTensor)); case OperandType::TENSOR_QUANT8_ASYMM: return addQuant8(context->getInputBuffer(kInputTensor1), context->getInputShape(kInputTensor1), context->getInputBuffer(kInputTensor2), context->getInputShape(kInputTensor2), context->getInputValue(kActivationScalar), context->getOutputBuffer(kOutputTensor), context->getOutputShape(kOutputTensor)); default: NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation ADD"; } } bool executeMul(IOperationExecutionContext* context) { // Bypass execution in the case of zero-sized input. if (getNumberOfElements(context->getOutputShape(kOutputTensor)) == 0) return true; switch (context->getInputType(kInputTensor1)) { case OperandType::TENSOR_FLOAT16: return mulFloat16(context->getInputBuffer<_Float16>(kInputTensor1), context->getInputShape(kInputTensor1), context->getInputBuffer<_Float16>(kInputTensor2), context->getInputShape(kInputTensor2), context->getInputValue(kActivationScalar), context->getOutputBuffer<_Float16>(kOutputTensor), context->getOutputShape(kOutputTensor)); case OperandType::TENSOR_FLOAT32: return mulFloat32(context->getInputBuffer(kInputTensor1), context->getInputShape(kInputTensor1), context->getInputBuffer(kInputTensor2), context->getInputShape(kInputTensor2), context->getInputValue(kActivationScalar), context->getOutputBuffer(kOutputTensor), context->getOutputShape(kOutputTensor)); case OperandType::TENSOR_QUANT8_ASYMM: return mulQuant8(context->getInputBuffer(kInputTensor1), context->getInputShape(kInputTensor1), context->getInputBuffer(kInputTensor2), context->getInputShape(kInputTensor2), context->getInputValue(kActivationScalar), context->getOutputBuffer(kOutputTensor), context->getOutputShape(kOutputTensor)); default: NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation MUL"; } } bool executeSub(IOperationExecutionContext* context) { // Bypass execution in the case of zero-sized input. if (getNumberOfElements(context->getOutputShape(kOutputTensor)) == 0) return true; switch (context->getInputType(kInputTensor1)) { case OperandType::TENSOR_FLOAT16: return subFloat16(context->getInputBuffer<_Float16>(kInputTensor1), context->getInputShape(kInputTensor1), context->getInputBuffer<_Float16>(kInputTensor2), context->getInputShape(kInputTensor2), context->getInputValue(kActivationScalar), context->getOutputBuffer<_Float16>(kOutputTensor), context->getOutputShape(kOutputTensor)); case OperandType::TENSOR_FLOAT32: return subFloat32(context->getInputBuffer(kInputTensor1), context->getInputShape(kInputTensor1), context->getInputBuffer(kInputTensor2), context->getInputShape(kInputTensor2), context->getInputValue(kActivationScalar), context->getOutputBuffer(kOutputTensor), context->getOutputShape(kOutputTensor)); case OperandType::TENSOR_QUANT8_ASYMM: return subQuant8(context->getInputBuffer(kInputTensor1), context->getInputShape(kInputTensor1), context->getInputBuffer(kInputTensor2), context->getInputShape(kInputTensor2), context->getInputValue(kActivationScalar), context->getOutputBuffer(kOutputTensor), context->getOutputShape(kOutputTensor)); default: NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation SUB"; } } bool executeDiv(IOperationExecutionContext* context) { // Bypass execution in the case of zero-sized input. if (getNumberOfElements(context->getOutputShape(kOutputTensor)) == 0) return true; switch (context->getInputType(kInputTensor1)) { case OperandType::TENSOR_FLOAT16: return divFloat16(context->getInputBuffer<_Float16>(kInputTensor1), context->getInputShape(kInputTensor1), context->getInputBuffer<_Float16>(kInputTensor2), context->getInputShape(kInputTensor2), context->getInputValue(kActivationScalar), context->getOutputBuffer<_Float16>(kOutputTensor), context->getOutputShape(kOutputTensor)); case OperandType::TENSOR_FLOAT32: return divFloat32(context->getInputBuffer(kInputTensor1), context->getInputShape(kInputTensor1), context->getInputBuffer(kInputTensor2), context->getInputShape(kInputTensor2), context->getInputValue(kActivationScalar), context->getOutputBuffer(kOutputTensor), context->getOutputShape(kOutputTensor)); default: NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation DIV"; } } } // namespace broadcast using std::placeholders::_1; NN_REGISTER_OPERATION(ADD, "ADD", std::bind(broadcast::validate, OperationType::ADD, _1), broadcast::prepare, broadcast::executeAdd, .allowZeroSizedInput = true); NN_REGISTER_OPERATION(MUL, "MUL", std::bind(broadcast::validate, OperationType::MUL, _1), broadcast::prepare, broadcast::executeMul, .allowZeroSizedInput = true); NN_REGISTER_OPERATION(SUB, "SUB", std::bind(broadcast::validate, OperationType::SUB, _1), broadcast::prepare, broadcast::executeSub, .allowZeroSizedInput = true); NN_REGISTER_OPERATION(DIV, "DIV", std::bind(broadcast::validate, OperationType::DIV, _1), broadcast::prepare, broadcast::executeDiv, .allowZeroSizedInput = true); } // namespace nn } // namespace android