1 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 #include "tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.h"
16
17 #include <stdint.h>
18
19 #include <limits>
20
21 #include "tensorflow/lite/c/builtin_op_data.h"
22 #include "tensorflow/lite/delegates/hexagon/hexagon_nn/hexagon_nn.h"
23 #include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h"
24 #include "tensorflow/lite/kernels/kernel_util.h"
25 #include "tensorflow/lite/kernels/padding.h"
26
27 namespace tflite {
28 namespace delegates {
29 namespace hexagon {
30 namespace {
31
32 constexpr uint8_t k8BitSignFlipConstant = 0x80;
33 // 1/1024 ~ 0.0009766 is a restriction set by Hexagon's kernels.
34 // TODO(b/151103818): Figure out a way to retrieve this constant reliably.
35 constexpr float kHexagonMinRelativeScale = 0.0009766f;
36
37 } // namespace
38
PopulateSubGraph(const TfLiteIntArray * inputs,const TfLiteIntArray * outputs,TfLiteContext * context)39 TfLiteStatus TransposeConv2dOpBuilder::PopulateSubGraph(
40 const TfLiteIntArray* inputs, const TfLiteIntArray* outputs,
41 TfLiteContext* context) {
42 // DATA TENSOR.
43 int tensor_id = inputs->data[2];
44 const auto& data_tensor = context->tensors[tensor_id];
45 AddInput(graph_builder_->GetHexagonTensorId(tensor_id));
46
47 // WEIGHTS.
48 tensor_id = inputs->data[1];
49 const auto& weights_tensor = context->tensors[tensor_id];
50 if (weights_tensor.allocation_type != kTfLiteMmapRo) {
51 context->ReportError(
52 context, "Weights tensor doesn't have correct allocation type: %s",
53 weights_tensor.name);
54 return kTfLiteError;
55 }
56 int filter_batch_size, filter_height_size, filter_width_size,
57 filter_depth_size;
58 GetDims(&filter_batch_size, &filter_height_size, &filter_width_size,
59 &filter_depth_size, weights_tensor.dims);
60 // Weights tensor could be int8 even for per-tensor quantization.
61 // Therefore, we look at the number of scale values to check if it is
62 // per-channel quantized.
63 TfLiteAffineQuantization* weights_quant_params =
64 reinterpret_cast<TfLiteAffineQuantization*>(
65 weights_tensor.quantization.params);
66 const bool is_per_channel_quant = weights_quant_params->scale->size > 1;
67 AddInput(graph_builder_->GetHexagonTensorId(tensor_id));
68
69 // Handle weights quantization.
70 float weights_min = 0;
71 float weights_max = 0;
72 if (is_per_channel_quant) {
73 ProcessPerChannelQuantizedWeights(inputs, outputs, context, &weights_min,
74 &weights_max, graph_builder_,
75 &per_channel_quant_);
76 } else {
77 TF_LITE_ENSURE_STATUS(ComputeMinAndMaxQuantValues(
78 weights_tensor, &weights_min, &weights_max));
79 }
80 auto* weights_min_const = graph_builder_->AddConstNodeWithData(
81 kScalarShape, reinterpret_cast<char*>(&weights_min), sizeof(weights_min));
82 auto* weights_max_const = graph_builder_->AddConstNodeWithData(
83 kScalarShape, reinterpret_cast<char*>(&weights_max), sizeof(weights_max));
84
85 // Min/max inputs for data & weights tensors.
86 TF_LITE_ENSURE_STATUS(ComputeAndAddMinAndMax(context, data_tensor));
87 AddInput(TensorID(weights_min_const->GetID(), 0));
88 AddInput(TensorID(weights_max_const->GetID(), 0));
89
90 // Output dims are required to compute padding.
91 int output_batch_size, output_height_size, output_width_size,
92 output_depth_size;
93 GetDims(&output_batch_size, &output_height_size, &output_width_size,
94 &output_depth_size, context->tensors[outputs->data[0]].dims);
95
96 // PADDING & STRIDE.
97 // Hexagon TransposeConv requires an explicit padding tensor. So we compute
98 // the same using stride, input & output info.
99 const TfLiteTransposeConvParams* params =
100 reinterpret_cast<const TfLiteTransposeConvParams*>(builtin_data_);
101 int unused_output_height, unused_output_width;
102 TfLitePaddingValues padding = ComputePaddingHeightWidth(
103 params->stride_height, params->stride_width, 1, 1, output_height_size,
104 output_width_size, filter_height_size, filter_width_size, params->padding,
105 &unused_output_height, &unused_output_width);
106 std::vector<int> padding_tensor = {padding.height, padding.height,
107 padding.width, padding.width};
108 std::vector<int> padding_tensor_shape = {1, 1, 2, 2};
109 auto* padding_const = graph_builder_->AddConstNodeWithData(
110 padding_tensor_shape.data(),
111 reinterpret_cast<char*>(padding_tensor.data()), (sizeof(int) * 4));
112 AddInput(TensorID(padding_const->GetID(), 0));
113
114 // Stride shape.
115 int stride_height = params->stride_height;
116 int stride_width = params->stride_width;
117 static int dummy = 0;
118 stride_shape_ = {1, stride_height, stride_width, 1};
119 auto* stride_node = graph_builder_->AddConstNodeWithData(
120 stride_shape_.data(), reinterpret_cast<char*>(&dummy), sizeof(dummy));
121 AddInput(TensorID(stride_node->GetID(), 0));
122
123 // BIAS.
124 const bool has_bias = inputs->size == 4;
125 OpBuilder* bias_const = nullptr;
126 OpBuilder* bias_min_const = nullptr;
127 OpBuilder* bias_max_const = nullptr;
128 if (!has_bias) {
129 // If the TFLite node does not have a bias, we simply feed in 0s.
130 std::vector<int> bias_data(output_depth_size, 0);
131 bias_shape_ = {1, 1, 1, output_depth_size};
132 bias_const = graph_builder_->AddConstNodeWithData(
133 bias_shape_.data(), reinterpret_cast<char*>(bias_data.data()),
134 sizeof(bias_data[0]) * bias_data.size());
135 float zero_bound = 0;
136 bias_min_const = graph_builder_->AddConstNodeWithData(
137 kScalarShape, reinterpret_cast<char*>(&zero_bound), sizeof(zero_bound));
138 bias_max_const = graph_builder_->AddConstNodeWithData(
139 kScalarShape, reinterpret_cast<char*>(&zero_bound), sizeof(zero_bound));
140 } else {
141 const auto& bias_tensor = context->tensors[inputs->data[3]];
142 if (bias_tensor.allocation_type != kTfLiteMmapRo) {
143 TF_LITE_KERNEL_LOG(context,
144 "Bias tensor doesn't have correct allocation type: %s",
145 bias_tensor.name);
146 return kTfLiteError;
147 }
148 float bias_min = 0;
149 float bias_max = 0;
150 if (per_channel_quant_.channel_scales_node != nullptr) {
151 ProcessPerChannelQuantizedBias(inputs, outputs, context, &bias_min,
152 &bias_max, graph_builder_,
153 &per_channel_quant_, &bias_const);
154 } else {
155 bias_const =
156 graph_builder_->AddConstNodeWithData(inputs->data[3], bias_tensor);
157 TF_LITE_ENSURE_STATUS(
158 ComputeMinAndMaxQuantValues(bias_tensor, &bias_min, &bias_max));
159 }
160
161 bias_min_const = graph_builder_->AddConstNodeWithData(
162 kScalarShape, reinterpret_cast<char*>(&bias_min), sizeof(bias_min));
163 bias_max_const = graph_builder_->AddConstNodeWithData(
164 kScalarShape, reinterpret_cast<char*>(&bias_max), sizeof(bias_max));
165 }
166 AddInput(TensorID(bias_const->GetID(), 0));
167 AddInput(TensorID(bias_min_const->GetID(), 0));
168 AddInput(TensorID(bias_max_const->GetID(), 0));
169
170 // Output quantization.
171 TF_LITE_ENSURE_STATUS(
172 ComputeAndAddMinAndMax(context, context->tensors[outputs->data[0]]));
173
174 // Channel scales, if this op is per-channel quantized.
175 if (per_channel_quant_.channel_scales_node != nullptr) {
176 AddInput(TensorID(per_channel_quant_.channel_scales_node->GetID(), 0));
177 }
178
179 // Hexagon outputs for this node.
180 node_output_ = AddOutput(sizeof(uint8_t), 4,
181 {output_batch_size, output_height_size,
182 output_width_size, output_depth_size});
183 AddOutput(sizeof(float), 4, kScalarShape);
184 AddOutput(sizeof(float), 4, kScalarShape);
185
186 return kTfLiteOk;
187 }
188
RegisterOutputs(const TfLiteIntArray * outputs,TfLiteContext * context)189 TfLiteStatus TransposeConv2dOpBuilder::RegisterOutputs(
190 const TfLiteIntArray* outputs, TfLiteContext* context) {
191 // Should be only 1 output.
192 graph_builder_->AddTensorWithID(outputs->data[0], node_output_.first,
193 node_output_.second);
194 return kTfLiteOk;
195 }
196
~TransposeConv2dOpBuilder()197 TransposeConv2dOpBuilder::~TransposeConv2dOpBuilder() {}
198
CreateTransposeConv2DBuilder(GraphBuilder * graph_builder,int op_type)199 OpBuilder* CreateTransposeConv2DBuilder(GraphBuilder* graph_builder,
200 int op_type) {
201 return new TransposeConv2dOpBuilder(graph_builder, op_type);
202 }
203
204 } // namespace hexagon
205 } // namespace delegates
206 } // namespace tflite
207